diff --git a/SILICONFOREST-SOURCES.json b/SILICONFOREST-SOURCES.json
index d8deab631e51..cada149652cf 100644
--- a/SILICONFOREST-SOURCES.json
+++ b/SILICONFOREST-SOURCES.json
@@ -2862,7 +2862,7 @@
     "aurora_merge": "5323fb87815718ca294ebe3c4f92b56cc18c577e",
     "scott_merge": "44fe9cab321168c782babaeb0f035088b574d823",
     "kernel_repository": "https://github.com/omacom/linux-aurora",
-    "kernel_commit": "7b969982306e1498a8cbe6e5da9bde743acd8fc5",
+    "kernel_commit": "6c66b41af3e77c0509ce24f3615c1bc2016cea14",
     "kernel_header": "include/uapi/drm/asahi_drm.h",
     "hardware_qualified": false,
     "system_install_performed": false
diff --git a/include/drm-uapi/asahi_drm.h b/include/drm-uapi/asahi_drm.h
index 3260e2f7ba28..a22a2ca73c9a 100644
--- a/include/drm-uapi/asahi_drm.h
+++ b/include/drm-uapi/asahi_drm.h
@@ -211,6 +211,20 @@ struct drm_asahi_params_global {
 	 * DRM_ASAHI_GPU_HAL_300 for G17G/G17S/G17C.
 	 */
 	__u32 gpu_hal_generation;
+
+	/**
+	 * @memory_limit: Device-wide limit on user committed plus kernel
+	 * resident bytes. Zero means no driver limit. This is a capacity,
+	 * not current free memory; other clients share it.
+	 */
+	__u64 memory_limit;
+
+	/**
+	 * @client_memory_limit: Per-DRM-file limit on user committed bytes.
+	 * Zero means no driver limit. Userspace must zero-initialize the reply
+	 * buffer: older kernels leave these optional tail fields untouched.
+	 */
+	__u64 client_memory_limit;
 };
 
 /**
@@ -304,6 +318,47 @@ enum drm_asahi_feature {
 	 * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance.
 	 */
 	DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16,
+
+	/*
+	 * Bits 17 to 28 are reported only by the M5 (G17G/G17S) runtime. They
+	 * follow the numbering above, so one userspace reads every kernel's
+	 * features the same way. Bits 25 and 27 are reserved for that runtime.
+	 */
+
+	/** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: Per-program render storage declarations. */
+	DRM_ASAHI_FEATURE_RENDER_RESOURCES = (1UL) << 17,
+	/** @DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2: Versioned SCS and mesh storage. */
+	DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2 = (1UL) << 18,
+	/** @DRM_ASAHI_FEATURE_SPARSE_TILES: DRM_ASAHI_BIND_SPARSE_TILE accepted. */
+	DRM_ASAHI_FEATURE_SPARSE_TILES = (1UL) << 19,
+	/** @DRM_ASAHI_FEATURE_COMPUTE_MATRIX: Native compute matrix instructions. */
+	DRM_ASAHI_FEATURE_COMPUTE_MATRIX = (1UL) << 20,
+	/*
+	 * Render programs accept DRM_ASAHI_RENDER_PROGRAM_TASK; task kicks get the
+	 * TA mesh rendering buffer that backs the object payload address.
+	 */
+	DRM_ASAHI_FEATURE_TASK_MESH_STORAGE = (1UL) << 21,
+	/** @DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: Per-render external shader writes. */
+	DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES = (1UL) << 22,
+	/** @DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: D32 2x/4x compressed depth and copy admission. */
+	DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION = (1UL) << 23,
+	/**
+	 * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: A render may carry a
+	 * separate compute prefix for its tiling half
+	 * (DRM_ASAHI_RENDER_COMPUTE_STAGE_BARRIERS,
+	 * struct drm_asahi_cmd_render_with_stage_prior).
+	 */
+	DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24,
+	/** @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: single-layer 2x/4x compressed stencil in the native ZLS form. */
+	DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26,
+	/** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: Per-dispatch storage declarations. */
+	DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28,
+	/**
+	 * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one
+	 * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs
+	 * include the earlier points of that timeline without adding GPU waits.
+	 */
+	DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29,
 };
 
 #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1
@@ -546,6 +601,15 @@ enum drm_asahi_bind_flags {
 	 * It is intended as a mechanism to accelerate sparse.
 	 */
 	DRM_ASAHI_BIND_SINGLE_PAGE = (1L << 3),
+
+	/**
+	 * @DRM_ASAHI_BIND_SPARSE_TILE: With DRM_ASAHI_BIND_SINGLE_PAGE, fill
+	 * the range from kernel-owned, physically contiguous tiles instead of
+	 * repeating the BO's first page: read-only binds read zeroes, writable
+	 * binds target a sink; both tile pools are private to the VM. The BO
+	 * only anchors the mapping. Requires DRM_ASAHI_FEATURE_SPARSE_TILES.
+	 */
+	DRM_ASAHI_BIND_SPARSE_TILE = (1L << 4),
 };
 
 /**
@@ -707,6 +771,18 @@ enum drm_asahi_cmd_type {
 	 * Followed by (possibly multiple) @drm_asahi_attachment payloads.
 	 */
 	DRM_ASAHI_SET_COMPUTE_ATTACHMENTS = 4,
+
+	/**
+	 * @DRM_ASAHI_SET_RENDER_RESOURCES: Replace linked render program storage.
+	 * Followed by zero or more drm_asahi_render_program entries. This software
+	 * command has no barriers and does not advance either command index.
+	 */
+	DRM_ASAHI_SET_RENDER_RESOURCES = 5,
+
+	/** @DRM_ASAHI_SET_COMPUTE_RESOURCES: Replace compute storage declarations. */
+	DRM_ASAHI_SET_COMPUTE_RESOURCES = 6,
+	/** Replace the render list with drm_asahi_render_program_v2 entries. */
+	DRM_ASAHI_SET_RENDER_RESOURCES_V2 = 7,
 };
 
 /**
@@ -794,6 +870,16 @@ enum drm_asahi_sync_type {
 
 	/** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */
 	DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1,
+
+	/**
+	 * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue
+	 * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point.
+	 * At most one is allowed. All outputs receive completion of this submit
+	 * and the earlier anchor points.
+	 * Other outputs must not alias the anchor. No execution dependency is
+	 * added. Calls updating this private anchor must be externally serialized.
+	 */
+	DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2,
 };
 
 /**
@@ -819,6 +905,87 @@ struct drm_asahi_sync {
  */
 #define DRM_ASAHI_BARRIER_NONE (0xFFFFu)
 
+/**
+ * struct drm_asahi_compute_program - Storage of one linked compute dispatch.
+ *
+ * SET_COMPUTE_RESOURCES replaces the list for subsequent compute commands.
+ * Include each distinct dispatch profile in the command stream. The kernel
+ * combines their requirements before publishing work. All sizes are bytes.
+ */
+struct drm_asahi_compute_program {
+	/** @primary_bytes: Per-thread primary storage. */
+	__u32 primary_bytes;
+	/** @secondary_bytes: Per-thread secondary storage. */
+	__u32 secondary_bytes;
+	/** @linked_bytes: Additional primary storage after linking. */
+	__u32 linked_bytes;
+	/** @minimum_primary_bytes: Minimum primary storage across variants. */
+	__u32 minimum_primary_bytes;
+	/** @threadgroup_bytes: Storage shared by a workgroup. */
+	__u32 threadgroup_bytes;
+	/** @threads: Number of invocations in a workgroup. */
+	__u32 threads;
+	/** @flags: Reserved; must be zero. */
+	__u32 flags;
+};
+
+/**
+ * struct drm_asahi_render_program - Storage of one linked render program.
+ *
+ * SET_RENDER_RESOURCES replaces the list for subsequent renders in this submit.
+ * Include every vertex and fragment program, including background and end-of-tile
+ * programs. Sizes describe the final linked program, in bytes. Each entry is
+ * evaluated separately before taking the maximum demand across the pass.
+ */
+struct drm_asahi_render_program {
+	/** @stage: 0 for vertex, 1 for fragment (including background/end-of-tile). */
+	__u32 stage;
+	/** @primary_bytes: Primary per-thread storage after linking. */
+	__u32 primary_bytes;
+	/** @secondary_bytes: Secondary per-thread storage after linking. */
+	__u32 secondary_bytes;
+	/** @linked_bytes: Additional primary storage from linked programs. */
+	__u32 linked_bytes;
+	/** @minimum_primary_bytes: Minimum primary storage across linked variants. */
+	__u32 minimum_primary_bytes;
+	/** @auxiliary_count: Fragment auxiliary entries; zero for vertex programs. */
+	__u32 auxiliary_count;
+	/** @threadgroup_bytes: Fragment threadgroup storage; zero for vertex. */
+	__u32 threadgroup_bytes;
+	/** @imageblock_bytes: Fragment imageblock storage; zero for vertex. */
+	__u32 imageblock_bytes;
+	/** @multisample_bytes: Fragment multisample storage; zero for vertex. */
+	__u32 multisample_bytes;
+	/** @flags: MBZ. */
+	__u32 flags;
+};
+/** Version 2 linked render storage. Version 1 remains exactly 40 bytes. */
+#define DRM_ASAHI_RENDER_PROGRAM_SCS (1U << 0)
+#define DRM_ASAHI_RENDER_PROGRAM_MESH (1U << 1)
+#define DRM_ASAHI_RENDER_PROGRAM_OBJECT (1U << 2)
+/* With OBJECT: the object program is an API task stage. The kernel binds the
+ * TA mesh rendering buffer (the backing of the object payload address) to
+ * such kicks only. Requires DRM_ASAHI_FEATURE_TASK_MESH_STORAGE. */
+#define DRM_ASAHI_RENDER_PROGRAM_TASK (1U << 3)
+struct drm_asahi_render_program_v2 {
+	struct drm_asahi_render_program legacy;
+	/** @threads: Mesh/object workgroup invocation count; zero otherwise. */
+	__u32 threads;
+	/** @usage_flags: DRM_ASAHI_RENDER_PROGRAM_* bits. */
+	__u32 usage_flags;
+	/** @scs_layout: Packed depth/levels/cliques/user/driver SCS layout. */
+	__u64 scs_layout;
+	/** @auxiliary_allocation: Stage native auxiliary allocation word. */
+	__u32 auxiliary_allocation;
+	__u32 reserved;
+	__u64 intersection_pool;
+	__u64 intersection_pool_bytes;
+	__u32 intersection_first;
+	__u32 intersection_count;
+	__u64 intersection_default_program;
+	__u64 intersection_default_control;
+};
+
 /**
  * struct drm_asahi_cmd_header - Top level command structure
  *
@@ -1032,6 +1199,24 @@ enum drm_asahi_render_flags {
 	 * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS.
 	 */
 	DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20),
+
+	/**
+	 * @DRM_ASAHI_RENDER_SHADER_WRITES: A graphics program in this command
+	 * writes external memory, including buffer/image stores or atomic RMW.
+	 * Ordinary raster outputs and compiler-private transport do not count.
+	 * Requires DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES. Omission retains the
+	 * existing encoding, allowing older userspace to submit unchanged.
+	 */
+	DRM_ASAHI_RENDER_SHADER_WRITES = (1U << 21),
+
+	/**
+	 * @DRM_ASAHI_RENDER_COMPUTE_STAGE_BARRIERS: The command is a struct
+	 * drm_asahi_cmd_render_with_stage_prior. Its tiling half waits only on
+	 * the compute prefix named by ta_cdm_barrier and ta_prior_cdm, which must
+	 * not exceed the command's own compute prefix; its fragment half waits on
+	 * the whole one. Requires DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS.
+	 */
+	DRM_ASAHI_RENDER_COMPUTE_STAGE_BARRIERS = (1U << 22),
 };
 
 /**
@@ -1370,6 +1555,19 @@ struct drm_asahi_cmd_render_with_prior {
 	__u64 prior_cdm;
 };
 
+/* DRM_ASAHI_RENDER_COMPUTE_STAGE_BARRIERS: the tiling half's compute prefix,
+ * a command index like drm_asahi_cmd_header::cdm_barrier and an exact prior
+ * ordinal like prior_cdm. Its tail takes the place of the ray-query stage
+ * state of struct drm_asahi_cmd_render_with_ray_query, so a command carries
+ * one of the two. */
+struct drm_asahi_cmd_render_with_stage_prior {
+	struct drm_asahi_cmd_render_with_prior legacy;
+	__u16 ta_cdm_barrier;
+	__u16 reserved0;
+	__u32 reserved1;
+	__u64 ta_prior_cdm;
+};
+
 struct drm_asahi_render_ray_query_stage {
 	__u64 scs_layout;
 	__u32 intersection_aux;
@@ -1444,13 +1642,30 @@ struct drm_asahi_cmd_compute_with_prior {
 	__u64 scs_layout;
 	__u64 prior_vdm;
 	__u64 prior_cdm;
+	/* Native M5 compute allocation word, independent of scs_layout.
+	 * Zero preserves the legacy path. Older kernels reject a nonzero tail. */
 	__u32 cdm_auxiliary_allocation;
 	__u32 reserved;
+	/** @intersection_pool: M5 intersection-table pool, 16 KiB aligned. */
 	__u64 intersection_pool;
+	/** @intersection_pool_bytes: Mapped size, a multiple of 16 KiB. */
 	__u64 intersection_pool_bytes;
+	/** @intersection_first: First 128-byte driver table block. */
 	__u32 intersection_first;
+	/** @intersection_count: Number of driver table blocks (1..262143). */
 	__u32 intersection_count;
+	/**
+	 * @intersection_default_program: Native default program descriptor.
+	 * Low 48 bits are the 64-byte aligned program address; upper 16 bits
+	 * contain the low resource-specification bits.
+	 */
 	__u64 intersection_default_program;
+	/**
+	 * @intersection_default_control: Native default program control word.
+	 * Bits 13:0 and 46:32 are defined by the M5 program descriptor. All
+	 * other bits must be zero. A nonzero default program requires bit 40.
+	 * An all-zero intersection extension preserves the ordinary compute path.
+	 */
 	__u64 intersection_default_control;
 };
 
@@ -1473,6 +1688,16 @@ enum drm_asahi_compute_flags {
 	 * ordered helpers cannot time out while waiting for the build.
 	 */
 	DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2),
+
+	/*
+	 * Bits 16 and up are accepted only by the M5 (G17G/G17S) runtime.
+	 */
+
+	/**
+	 * @DRM_ASAHI_COMPUTE_USES_MATRIX: The command's dispatches use native
+	 * matrix instructions. Requires DRM_ASAHI_FEATURE_COMPUTE_MATRIX.
+	 */
+	DRM_ASAHI_COMPUTE_USES_MATRIX = (1L << 16),
 };
 
 /**
diff --git a/src/asahi/clc/asahi_clc.c b/src/asahi/clc/asahi_clc.c
index a1fe94414749..b99a754a56ce 100644
--- a/src/asahi/clc/asahi_clc.c
+++ b/src/asahi/clc/asahi_clc.c
@@ -25,7 +25,7 @@
 #include "util/macros.h"
 #include <sys/mman.h>
 
-const char *targets[] = {"g13g", "g13x", "g17p", "g15s", "g15g", "g16g"};
+const char *targets[] = {"g13g", "g13x", "g17p", "g15s", "g15g", "g16g", "g17g"};
 
 #define foreach_target(target)                                                 \
    for (const char **target = &targets[0];                                     \
@@ -36,6 +36,8 @@ target_chip(const char *target)
 {
    if (!strcmp(target, "g17p"))
       return AGX_CHIP_G17P;
+   else if (!strcmp(target, "g17g"))
+      return AGX_CHIP_G17G;
    else if (!strcmp(target, "g15s"))
       return AGX_CHIP_G15S;
    else if (!strcmp(target, "g15g"))
@@ -55,6 +57,15 @@ target_is_native_agx3(const char *target)
    return agx_chip_native_agx3(target_chip(target));
 }
 
+/* Whether a target executes AGX3 code: the native AGX3B path or the M5 AGX3C
+ * one. */
+static bool
+target_is_agx3(const char *target)
+{
+   const enum agx_chip chip = target_chip(target);
+   return agx_chip_native_agx3(chip) || agx_chip_g17_m5(chip);
+}
+
 static const struct spirv_to_nir_options spirv_options = {
    .environment = NIR_SPIRV_OPENCL,
    .shared_addr_format = nir_address_format_62bit_generic,
@@ -285,6 +296,12 @@ remap_variant(nir_function *func, unsigned variant, const char *target)
 static bool
 target_supports_program(const char *target, const nir_function *func)
 {
+   if (!strcmp(func->name, "libagx_m5_decompress_rgba16f"))
+      return !strcmp(target, "g17g");
+
+   if (!strcmp(target, "g17g") &&
+       !strncmp(func->name, "libagx_d3d12_", strlen("libagx_d3d12_")))
+      return false;
    /* The range copy uses the native query command's producer ordering and
     * volatile WAIT loop. Never select this program for the AGX2/G13 path. */
    if (!strcmp(func->name, "libagx_copy_query_g17p"))
@@ -294,10 +311,20 @@ target_supports_program(const char *target, const nir_function *func)
     * Ioto/CDM launch contract. Keep unqualified slots NULL and fail closed.
     * Fast clear is required by the qualified G17P compressed color shapes;
     * compressed render-pass clears dispatch this program for metadata/body. */
-   return !target_is_native_agx3(target) ||
+   return !target_is_agx3(target) ||
+          /* M5 native acceleration-structure copy, compaction,
+           * serialization and property kernels (copy.cl, C87 GPU-side AS
+           * operations). Native AS images exist only on G17S. */
+          (!strcmp(target, "g17g") &&
+           !strncmp(func->name, "libagx_agx3_as_", strlen("libagx_agx3_as_"))) ||
           !strcmp(func->name, "libagx_fill_uint4") ||
           !strcmp(func->name, "libagx_gen1_as_property") ||
           !strcmp(func->name, "libagx_gen1_as_copy") ||
+          /* Native G17P AS copies relocate child slots and then restore the
+           * instance-record slots which can resemble node markers. */
+          (!strcmp(target, "g17p") &&
+           (!strcmp(func->name, "libagx_gen1_as_rebase") ||
+            !strcmp(func->name, "libagx_gen1_as_restore_records"))) ||
           !strcmp(func->name, "libagx_gen1_as_serialize") ||
           !strcmp(func->name, "libagx_gen1_as_deserialize") ||
           !strcmp(func->name, "libagx_gen1_as_deserialize_references") ||
@@ -318,8 +345,24 @@ target_supports_program(const char *target, const nir_function *func)
            * is behind HK_G17P_UNVERIFIED_FEATURES.  Do not read this entry as
            * a claim that it runs. */
           !strcmp(func->name, "libagx_trace_rays_indirect_setup") ||
+          /* Its per-raygen-kernel sibling: the same one-invocation shape
+           * (constant loads of the record, the raygen handle and the launch
+           * dimensions; one VkDispatchIndirectCommand store per kernel; the
+           * same shift/compare arithmetic, with hk_rt_launch_group_count as
+           * the host oracle).  No atomics, barriers, printf or scratch.
+           * Qualifying its compilation here is not an execution claim; the
+           * hardware test of the per-raygen dispatch covers it. */
+          !strcmp(func->name, "libagx_trace_rays_select_setup") ||
+          /* Its single-kick sibling (opt-in): same reads, plus 11 global
+           * stores of a launch template into a stream slot. */
+          !strcmp(func->name, "libagx_trace_rays_select_launch") ||
           !strcmp(func->name, "libagx_draw_indirect_byte_count") ||
           !strcmp(func->name, "libagx_conditional_render") ||
+          /* M5 task draws with a GPU-side grid, count or predicate: one
+           * invocation of constant loads, compares and four global stores
+           * into the mesh dispatch record and the draw's execute word. */
+          (!strcmp(target, "g17g") &&
+           !strcmp(func->name, "libagx_m5_task_dispatch")) ||
           !strcmp(func->name, "libagx_draw_robust_index") ||
           !strcmp(func->name, "libagx_draw_robust_index_batch") ||
           !strcmp(func->name, "libagx_unroll_restart") ||
@@ -327,6 +370,7 @@ target_supports_program(const char *target, const nir_function *func)
           !strcmp(func->name, "libagx_prefix_sum_geom") ||
           /* Query batches use the same native pointer loads/global stores as
            * the existing copy-query helpers; qualify their batch launch too. */
+          !strcmp(func->name, "libagx_write_u32") ||
           !strcmp(func->name, "libagx_write_u32s") ||
           !strcmp(func->name, "libagx_reset_query") ||
           !strcmp(func->name, "libagx_copy_uchar") ||
@@ -435,6 +479,19 @@ rebase_preamble_for_agx3(nir_builder *b, nir_intrinsic_instr *intr,
    return true;
 }
 
+/* This helper consumes native heap indices, not descriptor byte offsets. */
+static bool
+lower_m5_decompress_load(nir_builder *b, nir_intrinsic_instr *intr,
+                          UNUSED void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+   if (intr->intrinsic == nir_intrinsic_bindless_image_agx) {
+      nir_src_rewrite(&intr->src[0], nir_u2u64(b, intr->src[0].ssa));
+      return true;
+   }
+   return false;
+}
+
 static nir_def *
 load_kernel_input(nir_builder *b, unsigned num_components, unsigned bit_size,
                   unsigned offset_B)
@@ -565,15 +622,17 @@ main(int argc, char **argv)
 
                .reserved_preamble = layout.size_B / 2,
                .is_helper = is_helper,
-               .no_preamble = target_is_native_agx3(*target),
+               .no_preamble = target_is_agx3(*target),
 
                /* AGX3 has a smaller register file than AGX2, so the largest
                 * kernels spill to thread-private storage like API shaders.
                 */
-               .has_scratch = target_is_native_agx3(*target),
+               .has_scratch = target_is_agx3(*target),
             };
 
-            key.dev.encoding = target_is_native_agx3(*target)
+            key.dev.encoding = !strcmp(*target, "g17g")
+                                  ? AGX_ISA_ENCODING_AGX3C
+                                  : target_is_native_agx3(*target)
                                   ? AGX_ISA_ENCODING_AGX3B
                                   : AGX_ISA_ENCODING_AGX2;
             key.dev.chip = target_chip(*target);
@@ -594,8 +653,14 @@ main(int argc, char **argv)
                key.reserved_preamble += AGX_ABI_AGX3_PRECOMP_ARG_HALFS;
             }
 
+            if (!strcmp(libfunc->name, "libagx_m5_decompress_rgba16f")) {
+               key.g17p_texture_uniform_base = AGX_ABI_AGX3_PRECOMP_ARG_HALFS;
+               NIR_PASS(_, clone, nir_shader_intrinsics_pass,
+                        lower_m5_decompress_load, nir_metadata_none, NULL);
+            }
+
             uint16_t printf_uniform = 0;
-            if (target_is_native_agx3(*target)) {
+            if (target_is_agx3(*target)) {
                /* Both frontends publish the allocator-owned printf buffer.
                 * Keep that runtime ABI instead of dropping kernel assertions
                 * as upstream did before this branch supplied the address. */
@@ -618,8 +683,9 @@ main(int argc, char **argv)
                }
             }
             if (!agx_compile_shader_nir(clone, &key, &compiled)) {
-               fprintf(stderr, "Failed to compile %s for %s\n",
-                       libfunc->name, *target);
+               extern const char *agx_last_compile_refusal(void);
+               fprintf(stderr, "Failed to compile %s for %s: %s\n",
+                       libfunc->name, *target, agx_last_compile_refusal());
                ralloc_free(clone);
                return 1;
             }
@@ -629,7 +695,7 @@ main(int argc, char **argv)
             ralloc_free(clone);
 
             assert((compiled.info.scratch_size == 0 ||
-                    target_is_native_agx3(*target)) &&
+                    target_is_agx3(*target)) &&
                    "only AGX3 internal shaders spill");
 
             assert(compiled.info.preamble_scratch_size == 0 &&
diff --git a/src/asahi/compiler/agx3_alu_core.c b/src/asahi/compiler/agx3_alu_core.c
index 2f95b4b73a4a..eac65ec06475 100644
--- a/src/asahi/compiler/agx3_alu_core.c
+++ b/src/asahi/compiler/agx3_alu_core.c
@@ -8,15 +8,43 @@
 
 #include <string.h>
 
+bool
+agx3_alu_bitrev_m5(uint16_t dest, uint16_t src, bool src_half,
+                   enum agx3_alu_hint hint, bool dest_cache, uint16_t wait_mask,
+                   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 || (dest & 1) ||
+       (!src_half && (src & 1)) || hint > AGX3_ALU_HINT_CACHE ||
+       wait_mask > 255)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 2}, {23, 2}, {24, src_half ? 1 : 2}, {29, 3}, {30, 143},
+   };
+   struct agx3_field_kv fields[13] = {
+      {2, dest},
+      {11, dest_cache},
+      {16, src},
+      {25, hint == AGX3_ALU_HINT_CACHE},
+      {27, hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 5;
+   for (unsigned i = 0; i < 8; ++i)
+      if (wait_mask & (1u << i))
+         fields[n++] = (struct agx3_field_kv){31 + i, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "rev", 0, base, 6, fields,
+                            n, out, len);
+}
+
 bool
 agx3_alu_popcount(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                   bool half, enum agx3_alu_hint hint, bool dest_cache,
                   uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
                   uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len || dest >= 256 ||
        src >= 256 || (!half && ((dest | src) & 1)) ||
-       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
 
    /* applegpu_g17p `popcount` is the eight-byte cbs point. f10/f24 are the
@@ -33,21 +61,25 @@ agx3_alu_popcount(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * and, with enough 64-bit values live that the source lands high:
     *   2705548003005d04  $r64 <- popcount(^r64)
     *   2705549803305d04  $r76 <- popcount(^r76)
-    *   2705549003005c04  $r72 <- popcount(^r0) */
+    *   2705549003005c04  $r72 <- popcount(^r0)
+    * native G17G (port/cwit3 k_popc_u16, k_popc_u16w) counts a ushort the
+    * same way with half widths and C's form constants f29 = 3, f30 = 23; the
+    * uint result of the second clears the high half after it:
+    *   3f0200003220aa020400  h2 <- popcount(^h2), wait slot 8 (f38) */
    const struct agx3_field_kv base[] = {
-      {9, 2}, {10, half ? 1 : 2}, {23, 2}, {24, half ? 1 : 2},
+      {9, 2}, {10, half ? 1 : 2}, {23, 2}, {24, half ? 1 : 2}, {29,3}, {30,23},
    };
-   struct agx3_field_kv fields[11] = {
+   struct agx3_field_kv fields[13] = {
       {2, dest}, {11, dest_cache}, {16, src},
       {25, hint == AGX3_ALU_HINT_CACHE},
       {27, hint == AGX3_ALU_HINT_DISCARD},
    };
    unsigned n = 5;
-   for (unsigned slot = 0; slot < 6; ++slot) {
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot) {
       if (wait_mask & (1u << slot))
          fields[n++] = (struct agx3_field_kv){31 + slot, 1};
    }
-   return agx3_bind_compose(enc, "cbs", 0, base, 4, fields, n, out, len);
+   return agx3_bind_compose(enc, "cbs", 0, base, m5 ? 6 : 4, fields, n, out, len);
 }
 
 bool
@@ -56,6 +88,22 @@ agx3_alu_ft1_u32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                  uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
                  uint8_t *len)
 {
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      if (!out || !len || dest >= 256 || src >= 256 || ((dest | src) & 1) ||
+          (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0xffu))
+         return false;
+      const struct agx3_field_kv base[] = {
+         {9,2},{10,2},{23,2},{24,2},{29,3},{30,79},
+      };
+      struct agx3_field_kv f[13] = {
+         {2,dest},{11,dest_cache},{16,src},
+         {25,hint == AGX3_ALU_HINT_CACHE},{27,hint == AGX3_ALU_HINT_DISCARD},
+      };
+      for (unsigned i=0;i<8;i++)
+         f[5+i]=(struct agx3_field_kv){31+i,!!(wait_mask & (1u<<i))};
+      return agx3_bind_compose(enc,"ft1",0,base,6,f,13,out,len);
+   }
+
    if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
        src >= 256 || ((dest | src) & 1) ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE ||
@@ -99,7 +147,8 @@ agx3_alu_sqrt_step_source_modifiers(enum agx_isa_encoding enc, uint16_t dest, ui
                        uint16_t wait_mask, bool src_abs, bool src_neg,
                        uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || (m5 && wait_mask) || !out || !len || dest >= 256 ||
        src >= 256 || ((dest | src) & 1) ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~UINT16_C(0x3f)))
       return false;
@@ -110,6 +159,9 @@ agx3_alu_sqrt_step_source_modifiers(enum agx_isa_encoding enc, uint16_t dest, ui
     * separate move. Witnessed as a single-field difference against the plain
     * control; see agx3_alu_core.h for the byte pair. */
    const struct agx3_field_kv base[] = {{9,2},{10,2},{25,2},{26,2}};
+   const struct agx3_field_kv base_c[] = {
+      {9,2},{10,2},{25,2},{26,2},{33,3},{34,68},
+   };
    struct agx3_field_kv fields[13] = {
       {2,dest},{18,src},{11,dest_cache},
       {27,hint == AGX3_ALU_HINT_CACHE},{29,hint == AGX3_ALU_HINT_DISCARD},
@@ -117,7 +169,7 @@ agx3_alu_sqrt_step_source_modifiers(enum agx_isa_encoding enc, uint16_t dest, ui
    };
    for (unsigned i = 0; i < 6; ++i)
       fields[7+i] = (struct agx3_field_kv){35+i,!!(wait_mask & (1u<<i))};
-   return agx3_bind_compose(enc,"f32sqrtstep",0,base,4,fields,13,out,len);
+   return agx3_bind_compose(enc,"f32sqrtstep",0,m5 ? base_c : base,m5 ? 6 : 4,fields,13,out,len);
 }
 
 bool
@@ -129,20 +181,30 @@ agx3_alu_sqrt_step_half(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    /* Apple emits the half-input square-root step with its float32 temporary
     * cached and its half retained for the following multiply. A six-load
     * kernel witnesses the complete wait bank f35..f40 on this width pair. */
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len || dest >= 256 ||
        src >= 256 || (dest & 1) || hint != AGX3_ALU_HINT_CACHE ||
-       !dest_cache || (wait_mask & ~UINT16_C(0x3f)))
+       (!m5 && !dest_cache) || (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
 
    const struct agx3_field_kv base[] = {{9,2},{10,2},{25,2},{26,1}};
-   struct agx3_field_kv fields[10] = {
-      {2,dest}, {18,src}, {11,1}, {27,1},
+   struct agx3_field_kv fields[14] = {
+      {2, dest},
+      {18, src},
+      {11, dest_cache},
+      {27, 1},
    };
    for (unsigned i = 0; i < 6; ++i)
       fields[4 + i] =
          (struct agx3_field_kv){35 + i, !!(wait_mask & (1u << i))};
-   return agx3_bind_compose(enc, "f32sqrtstep", 0, base, 4, fields, 10,
-                            out, len);
+   if (m5) {
+      fields[10] = (struct agx3_field_kv){33, 3};
+      fields[11] = (struct agx3_field_kv){34, 68};
+      fields[12] = (struct agx3_field_kv){41, !!(wait_mask & 64)};
+      fields[13] = (struct agx3_field_kv){42, !!(wait_mask & 128)};
+   }
+   return agx3_bind_compose(enc, "f32sqrtstep", 0, base, 4, fields,
+                            m5 ? 14 : 10, out, len);
 }
 
 bool
@@ -153,7 +215,7 @@ agx3_alu_fmul_minifloat_mod(
    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
    return agx3_alu_fmul_minifloat_narrow(enc, dest, src, immediate,
-                                         dest_cache, src_abs, false, hint,
+                                         dest_cache, src_abs, false, false, hint,
                                          wait_mask, out, len);
 }
 
@@ -161,14 +223,18 @@ bool
 agx3_alu_fmul_minifloat_narrow(
    enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    uint16_t immediate, bool dest_cache, bool src_abs, bool dest_half,
+   bool saturate,
    enum agx3_alu_hint hint, uint16_t wait_mask,
    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (!out || !len || enc != AGX_ISA_ENCODING_AGX3B || dest >= 256 ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   if (!out || !len || (!gpu1 && enc != AGX_ISA_ENCODING_AGX3B) ||
+        dest >= 256 ||
        src >= 256 || (src & 1) || (!dest_half && (dest & 1)) ||
        immediate > 0xff ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~UINT16_C(0x3f)) || src_abs)
+       (wait_mask & ~(gpu1 ? UINT16_C(0xff) : UINT16_C(0x3f))) ||
+       ((src_abs || saturate) && !gpu1))
       return false;
 
    /* f10 is the DESTINATION width and f26 the source's; they are independent,
@@ -179,16 +245,20 @@ agx3_alu_fmul_minifloat_narrow(
     *
     * Witnessed with `metal-nt -arch applegpu_g17p`, a single-field difference
     * from the all-32-bit control, and re-composed byte-exactly through this
-    * encoder:
+    * encoder. G17G also independently witnesses narrowing to even and odd
+    * half destinations, including the slot-eight dependency form:
+    *   half4(input * 0.5f): 290005182280ac020a0020000210,
+    *                       01a115038400, 11a115058000, 11a115078400.
+    * G17P controls:
     *
     *   o[i] = a[i] * 0.5f        (float o)  19 a1 15 03 80 c0   f10 = 2
     *   o[i] = half(a[i] * 0.5f)  (half  o)  11 a1 15 03 80 c0   f10 = 1
     *   o[i] = half(a[i] * 2.0f)             11 c1 15 03 80 c0   f17 = 64
     *
     * The recovered six-byte endpoint fixes bits 32:33. There is no grounded
-    * fused ABS form: hardware testing showed that mutating bit 33 executes
-    * incorrectly. Callers must lower ABS through fadd_abs_negzero followed by
-    * this plain multiply. */
+    * fused ABS form on G17P: hardware testing showed that mutating bit 33
+    * executes incorrectly. G17G instead carries ABS in field47 of the longer
+    * form, retaining its source lifetime and dependency wait. */
 
    /* Fresh Apple mul_scale: 0983150180c0 is loaded f32 * minifloat0x02
     * (1/32), wait6. mul_reuse: 0983050380c0 retains its half2 source.
@@ -198,19 +268,35 @@ agx3_alu_fmul_minifloat_narrow(
    const struct agx3_field_kv base[] = {
       {9, 2}, {10, dest_half ? 1 : 2}, {25, 1}, {26, 2},
       {42, 2}, {43, 2}, {60, 2}, {74, 2},
+      {78, 3}, {79, 62},
    };
-   struct agx3_field_kv fields[12] = {
+   struct agx3_field_kv fields[16] = {
       {2, dest}, {35, src}, {17, immediate}, {46, hint == AGX3_ALU_HINT_DISCARD},
       {44, hint == AGX3_ALU_HINT_CACHE}, {11, dest_cache},
    };
    unsigned n = 6;
-   for (unsigned slot = 0; slot < 6; ++slot)
+   if (gpu1)
+      fields[n++] = (struct agx3_field_kv){47, src_abs};
+   /* G17G saturate(x * 1/16) differs from the plain scale only in f14;
+    * ABS uses f47 and narrowing independently selects f10=1. */
+   if (saturate)
+      fields[n++] = (struct agx3_field_kv){14, 1};
+   for (unsigned slot = 0; slot < (gpu1 ? 8 : 6); ++slot)
       if (wait_mask & (1u << slot))
          fields[n++] = (struct agx3_field_kv){80 + slot, 1};
 
    uint8_t encoded[AGX3_ALU_MAX_LENGTH], encoded_len = 0;
-   if (!agx3_bind_compose(enc, "f32mul", 1, base, 8, fields, n, encoded,
-                          &encoded_len))
+   bool composed = false;
+   /* Native G17G scale kernels use ep1 for ready inputs and ep3 when
+    * waiting on slot8. Preserve the G17P ep1-only admission. */
+   for (unsigned ep = 1; ep <= (gpu1 ? 3 : 1); ++ep) {
+      if (agx3_bind_compose(enc, "f32mul", ep, base, gpu1 ? 10 : 8,
+                             fields, n, encoded, &encoded_len)) {
+         composed = true;
+         break;
+      }
+   }
+   if (!composed)
       return false;
 
    memcpy(out, encoded, encoded_len);
@@ -244,6 +330,44 @@ agx3_alu_fmul_half_minifloat_to_f32(
                             len);
 }
 
+bool
+agx3_alu_fmul_half_minifloat_m5(
+   uint16_t dest, uint16_t src, uint16_t immediate, bool dest_cache,
+   bool src_abs, bool saturate, enum agx3_alu_hint hint, uint16_t wait_mask,
+   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 || (dest & 1) ||
+       immediate > UINT8_MAX || (unsigned)hint > AGX3_ALU_HINT_CACHE ||
+       (wait_mask & ~UINT16_C(0xff)))
+      return false;
+
+   /* G17G float(h) * k (port/hmulwit): the float-source minifloat multiply
+    * with f43=1 for the half register in f35, e.g.
+    *   k_hmul_half  float(h) * 0.5f      290005102280ac02028020000210
+    *   k_hmul_sat   saturate(...)        ...028020020210        (f14)
+    *   k_hmul_abs   float(fabs(h)) * .5  ...028020080210        (f47)
+    *   k_hmul_reuse3 cached source, no wait: 19c105868000      (f44, ep1)
+    * Slot-eight dependencies select ep3 with f87, as for the float form. */
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 2}, {25, 1}, {26, 2}, {42, 2}, {43, 1}, {60, 2}, {74, 2},
+      {78, 3}, {79, 62},
+   };
+   struct agx3_field_kv fields[16] = {
+      {2, dest}, {35, src}, {17, immediate},
+      {46, hint == AGX3_ALU_HINT_DISCARD}, {44, hint == AGX3_ALU_HINT_CACHE},
+      {11, dest_cache}, {47, src_abs}, {14, saturate},
+   };
+   unsigned n = 8;
+   for (unsigned slot = 0; slot < 8; ++slot)
+      fields[n++] = (struct agx3_field_kv){80 + slot, !!(wait_mask & (1u << slot))};
+
+   for (unsigned ep = 1; ep <= 3; ++ep)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "f32mul", ep, base, 10,
+                            fields, n, out, len))
+         return true;
+   return false;
+}
+
 bool
 agx3_alu_fadd_minifloat(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                        uint16_t immediate, bool dest_cache,
@@ -437,9 +561,12 @@ agx3_alu_float_reg(enum agx_isa_encoding enc, enum agx3_alu_op op,
                      const enum agx3_alu_hint hints[3], uint16_t wait_mask,
                      uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
+   const unsigned column = enc == AGX_ISA_ENCODING_AGX3C ? 1 : 0;
+   const uint16_t slots = enc == AGX_ISA_ENCODING_AGX3C ? 0xff : 0x3f;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       !src || !hints || !out || !len ||
        (op != AGX3_ALU_FADD && op != AGX3_ALU_FMUL && op != AGX3_ALU_FMA) ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~slots))
       return false;
    const unsigned count = op == AGX3_ALU_FMA ? 3 : 2;
    /* Native FADD/FMUL/FMA all encode eight-bit half-register operands.
@@ -457,8 +584,8 @@ agx3_alu_float_reg(enum agx_isa_encoding enc, enum agx3_alu_op op,
     * Fresh add_live/mul_live/fma_live witnesses retain all their operands;
     * all_ops additionally witnesses independently cached/discarded sources.
     */
-   const struct agx3_alu_form *f = &agx3_alu_forms[op][0];
-   struct agx3_field_kv base[20], ov[16];
+   const struct agx3_alu_form *f = &agx3_alu_forms[op][column];
+   struct agx3_field_kv base[20], ov[1 + 3 * 3 + 8];
    unsigned nb = 0, n = 0;
    for (unsigned i = 0; i < f->nbase; ++i) {
       unsigned field = f->base[i].f;
@@ -481,6 +608,21 @@ agx3_alu_float_reg(enum agx_isa_encoding enc, enum agx3_alu_op op,
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv){f->wait[i], 1};
 
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      static const uint8_t add_eps[] = {0, 1, 2, 3};
+      static const uint8_t mul_eps[] = {0, 1, 2, 4, 5, 3};
+      static const uint8_t fma_eps[] = {0, 1, 2, 3, 4, 5, 6, 7};
+      const uint8_t *eps = op == AGX3_ALU_FMA ? fma_eps
+                           : op == AGX3_ALU_FMUL ? mul_eps : add_eps;
+      const unsigned count_eps = op == AGX3_ALU_FMA ? 8
+                                 : op == AGX3_ALU_FMUL ? 6 : 4;
+      for (unsigned i = 0; i < count_eps; ++i)
+         if (agx3_bind_compose(enc, f->mnemonic, eps[i], base, nb, ov, n,
+                               out, len))
+            return true;
+      return false;
+   }
+
    const unsigned first = op == AGX3_ALU_FMA ? 2 : 0;
    const unsigned last = op == AGX3_ALU_FMA ? 3 : 1;
    for (unsigned ep = first; ep <= last; ++ep)
@@ -628,10 +770,11 @@ agx3_alu_float_to_int32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                         bool dest_cache, uint16_t wait_mask,
                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (!out || !len || enc != AGX_ISA_ENCODING_AGX3B || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if (!out || !len || (enc != AGX_ISA_ENCODING_AGX3B && !m5) || dest >= 256 ||
        src >= 256 || (dest & 1) || (src & 1) ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
       return false;
 
    /* Apple's argument-evaluation witness keeps the converted float live:
@@ -649,10 +792,10 @@ agx3_alu_float_to_int32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * shaders whose float-to-integer result landed at half 128.
     */
    const struct agx3_field_kv base[] = {
-      {9, 2}, {10, 2}, {23, 2}, {24, 2},
-      {41, is_signed ? 5 : 4}, {42, 1},
+      {9, 2},  {10, 2}, {23, 2},  {24, 2}, {41, is_signed ? 5 : 4},
+      {42, 1}, {31, 3}, {32, 71},
    };
-   struct agx3_field_kv fields[11] = {
+   struct agx3_field_kv fields[13] = {
       {2, dest},
       {11, dest_cache},
       {16, src},
@@ -660,11 +803,12 @@ agx3_alu_float_to_int32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
       {27, hint == AGX3_ALU_HINT_DISCARD},
    };
    unsigned n = 5;
-   for (unsigned slot = 0; slot < 6; ++slot)
-      fields[n++] = (struct agx3_field_kv){33 + slot,
-                                           !!(wait_mask & (1u << slot))};
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot)
+      fields[n++] =
+         (struct agx3_field_kv){33 + slot, !!(wait_mask & (1u << slot))};
 
-   return agx3_bind_compose(enc, "flt2int", 0, base, 6, fields, n, out, len);
+   return agx3_bind_compose(enc, "flt2int", 0, base, m5 ? 8 : 6, fields, n, out,
+                            len);
 }
 
 bool
@@ -673,10 +817,11 @@ agx3_alu_float_to_int16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                         bool dest_cache, uint16_t wait_mask,
                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (!out || !len || enc != AGX_ISA_ENCODING_AGX3B || dest >= 256 ||
-       src >= 128 || (!src_16 && (src & 1)) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if (!out || !len || (enc != AGX_ISA_ENCODING_AGX3B && !m5) || dest >= 256 ||
+       src >= (m5 ? 256 : 128) || (!src_16 && (src & 1)) ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
       return false;
 
    /*
@@ -706,18 +851,18 @@ agx3_alu_float_to_int16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     */
    const struct agx3_field_kv base[] = {
       {9, 2}, {10, 1}, {23, 2}, {24, src_16 ? 1 : 2},
-      {41, is_signed ? 3 : 2}, {42, 1},
+      {41, is_signed ? 3 : 2}, {42, 1}, {31,3}, {32,71},
    };
-   struct agx3_field_kv fields[10] = {
+   struct agx3_field_kv fields[12] = {
       {2, dest}, {11, dest_cache}, {16, src},
       {27, hint == AGX3_ALU_HINT_DISCARD},
    };
    unsigned n = 4;
-   for (unsigned slot = 0; slot < 6; ++slot)
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot)
       fields[n++] = (struct agx3_field_kv){33 + slot,
                                            !!(wait_mask & (1u << slot))};
 
-   return agx3_bind_compose(enc, "flt2int", 0, base, 6, fields, n, out, len);
+   return agx3_bind_compose(enc, "flt2int", 0, base, m5 ? 8 : 6, fields, n, out, len);
 }
 
 bool
@@ -864,6 +1009,39 @@ agx3_alu_fadd_mixed_half(
    enum agx3_alu_hint hint32, enum agx3_alu_hint hint16, bool dest_cache,
    uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      if (!out || !len || dest >= 256 || f32 >= 256 || f16 >= 256 ||
+          ((dest | f32) & 1) ||
+          (unsigned)hint32 > AGX3_ALU_HINT_CACHE ||
+          (unsigned)hint16 > AGX3_ALU_HINT_CACHE || (wait_mask & ~0xffu))
+         return false;
+
+      /* G17G texture-backed half values add directly to full pixel
+       * coordinates. 098524820020 retains both sources and waits slot 1;
+       * 29053c021000 ends both sources. C keeps B's register routing and
+       * width fields, with its own required f50/f51 endpoint selectors. */
+      const struct agx3_field_kv base[] = {
+         {9, 2}, {10, 2}, {25, 2}, {26, 2}, {42, 2}, {43, 1},
+         {50, 3}, {51, 48},
+      };
+      struct agx3_field_kv fields[16] = {
+         {2, dest}, {11, dest_cache}, {18, f32}, {35, f16},
+         {27, hint32 == AGX3_ALU_HINT_CACHE},
+         {29, hint32 == AGX3_ALU_HINT_DISCARD},
+         {44, hint16 == AGX3_ALU_HINT_CACHE},
+         {46, hint16 == AGX3_ALU_HINT_DISCARD},
+      };
+      unsigned n = 8;
+      for (unsigned slot = 0; slot < 8; ++slot)
+         if (wait_mask & (1u << slot))
+            fields[n++] = (struct agx3_field_kv){52 + slot, 1};
+
+      const unsigned ep = (wait_mask & 0xc0) ? 3 :
+                          ((wait_mask & (wait_mask - 1)) ? 2 : 1);
+      return agx3_bind_compose(enc, "f32add", ep, base, 8, fields, n,
+                               out, len);
+   }
+
    if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
        f32 >= 256 || f16 >= 256 || ((dest | f32) & 1) ||
        (unsigned)hint32 > AGX3_ALU_HINT_CACHE ||
@@ -934,39 +1112,47 @@ agx3_alu_fadd_abs_mixed_half(
 }
 
 bool
-agx3_alu_fmul_mixed_half(enum agx_isa_encoding enc, uint16_t dest,
-                         uint16_t f32, uint16_t f16,
-                         enum agx3_alu_hint hint32,
+agx3_alu_fmul_mixed_half(enum agx_isa_encoding enc, uint16_t dest, uint16_t f32,
+                         uint16_t f16, enum agx3_alu_hint hint32,
                          enum agx3_alu_hint hint16, bool dest_cache,
-                         uint16_t wait_mask,
-                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+                         uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                         uint8_t *len)
 {
    /* This is the exact second half of Apple's scalar half sqrt sequence.
     * Field10 selects the half destination, including odd half registers.
     * Fresh retained-source probes establish f27/f29 and f44/f46, while the
     * ordinary long f32mul family supplies the same six wait fields f80..f85. */
-   if (!out || !len || enc != AGX_ISA_ENCODING_AGX3B || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if (!out || !len || (enc != AGX_ISA_ENCODING_AGX3B && !m5) || dest >= 256 ||
        f32 >= 256 || f16 >= 256 || (f32 & 1) ||
        (unsigned)hint32 > AGX3_ALU_HINT_CACHE ||
        (unsigned)hint16 > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
       return false;
 
    const struct agx3_field_kv base[] = {
-      {9,2},{10,1},{25,2},{26,2},{42,2},{43,1},{60,2},{74,2},
+      {9, 2}, {10, 1}, {25, 2}, {26, 2}, {42, 2}, {43, 1}, {60, 2}, {74, 2},
    };
-   struct agx3_field_kv fields[14] = {
-      {2,dest}, {18,f32}, {35,f16}, {11,dest_cache},
-      {27,hint32 == AGX3_ALU_HINT_CACHE},
-      {29,hint32 == AGX3_ALU_HINT_DISCARD},
-      {44,hint16 == AGX3_ALU_HINT_CACHE},
-      {46,hint16 == AGX3_ALU_HINT_DISCARD},
+   struct agx3_field_kv fields[18] = {
+      {2, dest},
+      {18, f32},
+      {35, f16},
+      {11, dest_cache},
+      {27, hint32 == AGX3_ALU_HINT_CACHE},
+      {29, hint32 == AGX3_ALU_HINT_DISCARD},
+      {44, hint16 == AGX3_ALU_HINT_CACHE},
+      {46, hint16 == AGX3_ALU_HINT_DISCARD},
    };
    for (unsigned i = 0; i < 6; ++i)
-      fields[8 + i] =
-         (struct agx3_field_kv){80 + i, !!(wait_mask & (1u << i))};
-   for (unsigned ep = 0; ep < 2; ++ep)
-      if (agx3_bind_compose(enc, "f32mul", ep, base, 8, fields, 14,
+      fields[8 + i] = (struct agx3_field_kv){80 + i, !!(wait_mask & (1u << i))};
+   if (m5) {
+      fields[14] = (struct agx3_field_kv){78, 3};
+      fields[15] = (struct agx3_field_kv){79, 62};
+      fields[16] = (struct agx3_field_kv){86, !!(wait_mask & 64)};
+      fields[17] = (struct agx3_field_kv){87, !!(wait_mask & 128)};
+   }
+   for (unsigned ep = 0; ep < (m5 ? 4 : 2); ++ep)
+      if (agx3_bind_compose(enc, "f32mul", ep, base, 8, fields, m5 ? 18 : 14,
                             out, len))
          return true;
    return false;
@@ -1143,15 +1329,16 @@ agx3_alu_bit_width(enum agx_isa_encoding enc, enum agx3_alu_op op, bool narrow,
                  const enum agx3_alu_hint hints[2], uint16_t wait_mask,
                  uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if(enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
+   if((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+      !src || !hints || !out || !len ||
       (op != AGX3_ALU_BAND && op != AGX3_ALU_BOR && op != AGX3_ALU_BXOR) ||
-      dest >= 256 || (!narrow && (dest & 1)) || (wait_mask & ~UINT16_C(0x3f)))
+      dest >= 256 || (!narrow && (dest & 1)) || (wait_mask & ~(enc == AGX_ISA_ENCODING_AGX3C ? 0xffu : 0x3fu)))
       return false;
    for(unsigned i=0;i<2;i++)
       if(src[i]>=256 || (!narrow && (src[i]&1)) || (unsigned)hints[i]>AGX3_ALU_HINT_CACHE)
          return false;
-   const struct agx3_alu_form *f=&agx3_alu_forms[op][0];
-   struct agx3_field_kv base[20],ov[14];unsigned nb=0,n=0;
+   const struct agx3_alu_form *f=&agx3_alu_forms[op][enc == AGX_ISA_ENCODING_AGX3C ? 1 : 0];
+   struct agx3_field_kv base[20],ov[15];unsigned nb=0,n=0;
    for(unsigned i=0;i<f->nbase;i++){
       unsigned field=f->base[i].f;bool wait=false;
       for(unsigned j=0;j<f->nwait;j++)wait|=field==f->wait[j];
@@ -1168,6 +1355,9 @@ agx3_alu_bit_width(enum agx_isa_encoding enc, enum agx3_alu_op op, bool narrow,
    }
    for(unsigned i=0;i<f->nwait;i++)
       if(wait_mask&(1u<<i))ov[n++]=(struct agx3_field_kv){f->wait[i],1};
+   if (enc == AGX_ISA_ENCODING_AGX3C &&
+       agx3_bind_compose(enc,f->mnemonic,0,base,nb,ov,n,out,len))
+      return true;
    return agx3_bind_compose(enc,f->mnemonic,f->ep,base,nb,ov,n,out,len);
 }
 
@@ -1274,15 +1464,18 @@ agx3_alu_band_identity_wait(enum agx_isa_encoding enc, uint16_t reg,
                             uint16_t wait_mask,
                             uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B)
-      return agx3_alu_compose(enc, AGX3_ALU_BAND, reg, reg, reg, 0, 0,
-                              wait_mask, out, len);
+   if (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C)
+      return false;
 
    const uint16_t src[] = {reg, reg};
    const enum agx3_alu_hint hints[] = {AGX3_ALU_HINT_NONE, AGX3_ALU_HINT_NONE};
 
-   /* Every retire carrier also waits slot 6, as the witnessed specimen does. */
-   const uint16_t extra = UINT16_C(1) << 5;
+   /* The witnessed carriers wait slot6 on G17P and slot8 on G17S. Neither
+    * operand is dead: r0 may hold an independent live half-register value.
+    * The raw form table includes its specimen's f27 discard and must not
+    * be used for this identity operation. */
+   const uint16_t extra = UINT16_C(1) <<
+      (enc == AGX_ISA_ENCODING_AGX3C ? 7 : 5);
    return agx3_alu_bit_reg(enc, AGX3_ALU_BAND, reg, src, hints,
                            wait_mask | extra, out, len);
 }
@@ -1293,14 +1486,26 @@ agx3_alu_bit_imm(enum agx_isa_encoding enc, enum agx3_alu_op op, bool narrow,
                  enum agx3_alu_hint hint, uint16_t wait_mask,
                  uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len ||
+   /* AGX3C uses its own table entry for the same fields. Native G17G
+    * witnesses (the native G17G compiler, port/cwit k_or1/k_and255/
+    * k_xor1/k_or1h) carry f37 = 1 and the constant in f29 on that point:
+    *
+    *   a | 1             2b0007803238a3024480   bit_or  f29=1   f43=16
+    *   a & 0xff          2b0007803030a302c4bf   bit_and f29=255 f43=10
+    *   a ^ 1             2b0006803238a3024480   bit_xor f29=1   f43=19
+    *   (ushort) a | 1    3b0207003228a3024480   bit_or  f10=1 f24=1, f38=2
+    *
+    * each after a load whose slot-7 wait is f51. */
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const uint16_t waits = m5 ? 0xff : 0x3f; /* eight slots on AGX3C, six on AGX3B */
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len ||
        (op != AGX3_ALU_BAND && op != AGX3_ALU_BOR && op != AGX3_ALU_BXOR) ||
        dest >= 256 || (!narrow && (dest & 1)) || src >= 256 ||
        (!narrow && (src & 1)) || (unsigned)hint > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~waits))
       return false;
 
-   const struct agx3_alu_form *f = &agx3_alu_forms[op][0];
+   const struct agx3_alu_form *f = &agx3_alu_forms[op][m5 ? 1 : 0];
    struct agx3_field_kv base[20], ov[12];
    unsigned nb = 0, n = 0;
    for (unsigned i = 0; i < f->nbase; ++i) {
@@ -1338,6 +1543,114 @@ agx3_alu_bit_reg16(enum agx_isa_encoding enc, enum agx3_alu_op op,
    return agx3_alu_bit_width(enc, op, true, dest, src, hints, wait_mask, out, len);
 }
 
+/* Native G17G multisample DEPTHF payloads select a low coverage bit with
+ * AND and retain SR39's API byte with ANDC(0xff). Both are half operations;
+ * their destination/source lifetime and eight wait fields are independent. */
+bool
+agx3_alu_and_imm16_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+                     bool complement, bool immediate_first,
+                     enum agx3_alu_hint hint,
+                     bool dest_cache, uint16_t wait_mask,
+                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 || wait_mask > 255 ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE ||
+       (immediate_first && !complement))
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {23, immediate_first ? 1 : 2},
+      {24, immediate_first ? 2 : 1}, {37, immediate_first ? 2 : 1},
+      {38, immediate_first ? 1 : 2},
+      {42, 3}, {43, complement ? 11 : 10},
+   };
+   struct agx3_field_kv fields[15] = {
+      {2, dest}, {11, dest_cache},
+      {immediate_first ? 30 : 16, src},
+      {immediate_first ? 15 : 29, immediate},
+      {immediate_first ? 39 : 25, hint == AGX3_ALU_HINT_CACHE},
+      {immediate_first ? 41 : 27, hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 6;
+   for (unsigned slot = 0; slot < 8; ++slot)
+      if (wait_mask & (1u << slot))
+         fields[n++] = (struct agx3_field_kv){44 + slot, 1};
+   const char *name = complement ? "bit_andc" : "bit_and";
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, name, 0, base, 8,
+                             fields, n, out, len) ||
+          agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, name, 1, base, 8,
+                             fields, n, out, len);
+}
+
+bool
+agx3_alu_complement_reg16_m5(uint16_t dest, const uint16_t src[2],
+                            const enum agx3_alu_hint hints[2],
+                            bool disjunction, bool dest_cache,
+                            uint16_t wait_mask,
+                            uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!src || !hints || !out || !len || dest >= 256 || src[0] >= 256 ||
+       src[1] >= 256 || wait_mask > 255 ||
+       (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
+       (unsigned)hints[1] > AGX3_ALU_HINT_CACHE)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {23, 2}, {24, 1}, {37, 2}, {38, 1},
+      {42, 3}, {43, disjunction ? 17 : 11},
+   };
+   struct agx3_field_kv fields[16] = {
+      {2, dest}, {11, dest_cache}, {16, src[0]}, {30, src[1]},
+      {25, hints[0] == AGX3_ALU_HINT_CACHE},
+      {27, hints[0] == AGX3_ALU_HINT_DISCARD},
+      {39, hints[1] == AGX3_ALU_HINT_CACHE},
+      {41, hints[1] == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 8;
+   for (unsigned slot = 0; slot < 8; ++slot)
+      if (wait_mask & (1u << slot))
+         fields[n++] = (struct agx3_field_kv){44 + slot, 1};
+   const char *name = disjunction ? "bit_orc" : "bit_andc";
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, name, 0, base, 8,
+                             fields, n, out, len) ||
+          agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, name, 1, base, 8,
+                             fields, n, out, len);
+}
+
+bool
+agx3_alu_bit_not_m5(uint16_t dest, uint16_t src, bool word,
+                    enum agx3_alu_hint hint, bool dest_cache,
+                    uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                    uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 ||
+       (word && ((dest | src) & 1)) ||
+       (hint != AGX3_ALU_HINT_NONE && hint != AGX3_ALU_HINT_CACHE &&
+        hint != AGX3_ALU_HINT_DISCARD) || (wait_mask & ~UINT16_C(0xff)))
+      return false;
+
+   /* Native G17G `~x` on ushort and uint (port/cwit3 k_not_u16/u32): the
+    * eight-byte `inv` with C's form constants f28 = 3, f29 = 93, result and
+    * source widths f10/f24, source cache/discard f25/f27 and the eight
+    * scoreboard slots f30..f37 (bit-flip map of the witness):
+    *   3b0226182108a002  h2 <- ~^h2, wait slot 8
+    *   2b00261a2108a802  r0 <- ~^r0, wait slot 8 */
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, word ? 2 : 1}, {23, 2}, {24, word ? 2 : 1},
+      {28, 3}, {29, 93},
+   };
+   struct agx3_field_kv fields[13] = {
+      {2, dest}, {11, dest_cache}, {16, src},
+      {25, hint == AGX3_ALU_HINT_CACHE},
+      {27, hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 5;
+   for (unsigned slot = 0; slot < 8; ++slot)
+      if (wait_mask & (1u << slot))
+         fields[n++] = (struct agx3_field_kv){30 + slot, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "inv", 0, base,
+                            sizeof(base) / sizeof(base[0]), fields, n, out,
+                            len);
+}
+
 bool
 agx3_alu_bit_not_reg16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                        enum agx3_alu_hint hint, bool dest_cache,
@@ -1370,24 +1683,31 @@ agx3_alu_bit_not_reg16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
  * ordinary bitops' operand, hint and wait topology; only the mnemonic
  * carries the truth table. */
 static bool
-agx3_alu_bit_named(enum agx_isa_encoding enc, const char *mnemonic,
+agx3_alu_bit_named_width(enum agx_isa_encoding enc, const char *mnemonic,
                    uint16_t dest, const uint16_t src[2],
-                   const enum agx3_alu_hint hints[2], bool dest_cache,
+                   const enum agx3_alu_hint hints[2], const bool half[2], bool dest_cache,
                    uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
                    uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !src || !hints || !out || !len ||
        dest >= 256 || src[0] >= 256 || src[1] >= 256 ||
-       ((dest | src[0] | src[1]) & 1) || (wait_mask & ~UINT16_C(0x3f)) ||
+       ((dest & 1) || (!half[0] && (src[0] & 1)) ||
+        (!half[1] && (src[1] & 1))) || (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[1] > AGX3_ALU_HINT_CACHE)
       return false;
 
+   const unsigned table = !strcmp(mnemonic, "bit_andc") ? 11 :
+      !strcmp(mnemonic, "bit_nor") ? 14 : !strcmp(mnemonic, "bit_nand") ? 12 :
+      !strcmp(mnemonic, "bit_xnor") ? 18 : !strcmp(mnemonic, "bit_orc") ? 17 : 0;
+   if (m5 && !table)
+      return false;
    const struct agx3_field_kv base[] = {
-      {9, 2}, {10, 2}, {23, 2}, {24, 2},
-      {37, 2}, {38, 2},
+      {9, 2}, {10, 2}, {23, 2}, {24, half[0] ? 1 : 2},
+      {37, 2}, {38, half[1] ? 1 : 2}, {42, 3}, {43, table},
    };
-   struct agx3_field_kv fields[14] = {
+   struct agx3_field_kv fields[16] = {
       {2, dest}, {16, src[0]}, {30, src[1]}, {11, dest_cache},
       {25, hints[0] == AGX3_ALU_HINT_CACHE},
       {27, hints[0] == AGX3_ALU_HINT_DISCARD},
@@ -1395,10 +1715,34 @@ agx3_alu_bit_named(enum agx_isa_encoding enc, const char *mnemonic,
       {41, hints[1] == AGX3_ALU_HINT_DISCARD},
    };
    unsigned n = 8;
-   for (unsigned slot = 0; slot < 6; ++slot)
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot)
       fields[n++] = (struct agx3_field_kv){44 + slot,
                                           !!(wait_mask & (1u << slot))};
-   return agx3_bind_compose(enc, mnemonic, 0, base, 6, fields, n, out, len);
+   return agx3_bind_compose(enc, mnemonic, 0, base, m5 ? 8 : 6, fields, n, out, len) ||
+          (m5 && agx3_bind_compose(enc, mnemonic, 1, base, 8, fields, n, out, len));
+}
+
+static bool
+agx3_alu_bit_named(enum agx_isa_encoding enc, const char *mnemonic,
+                   uint16_t dest, const uint16_t src[2],
+                   const enum agx3_alu_hint hints[2], bool dest_cache,
+                   uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                   uint8_t *len)
+{
+   const bool half[] = {false, false};
+   return agx3_alu_bit_named_width(enc, mnemonic, dest, src, hints, half,
+                                  dest_cache, wait_mask, out, len);
+}
+
+bool
+agx3_alu_bit_andc_width_m5(uint16_t dest, const uint16_t src[2],
+                           const bool half[2], const enum agx3_alu_hint hints[2],
+                           bool dest_cache, uint16_t wait_mask,
+                           uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_alu_bit_named_width(AGX_ISA_ENCODING_AGX3C, "bit_andc", dest,
+                                  src, hints, half, dest_cache, wait_mask,
+                                  out, len);
 }
 
 bool
@@ -2324,25 +2668,27 @@ agx3_alu_fcmp_select_zero(enum agx_isa_encoding enc, uint16_t dest,
  * address later stores. Keep the native instruction and actual RA hints. */
 bool
 agx3_alu_int_to_float(enum agx_isa_encoding enc, uint16_t dest,
-                      uint16_t src, bool is_signed, enum agx3_alu_hint hint,
+                      uint16_t src, bool is_signed, enum agx3_alu_hint hint, bool dest_cache,
                       uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
                       uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len ||
        dest >= 256 || src >= 256 || ((dest | src) & 1) ||
-       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
    const struct agx3_field_kv base[] = {
-      {9, 2}, {10, 2}, {25, 2}, {26, 2},
+      {9, 2}, {10, 2}, {25, 2}, {26, 2}, {31, 3}, {32, 91},
    };
-   struct agx3_field_kv fields[12] = {
+   struct agx3_field_kv fields[15] = {
       {2, dest}, {18, src}, {27, hint == AGX3_ALU_HINT_CACHE},
       {29, hint == AGX3_ALU_HINT_DISCARD}, {30, is_signed},
+      {11, m5 && dest_cache},
    };
-   unsigned n = 5;
-   for (unsigned i = 0; i < 6; ++i)
+   unsigned n = 6;
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       fields[n++] = (struct agx3_field_kv){33 + i, !!(wait_mask & (1u << i))};
-   return agx3_bind_compose(enc, "int2flt", 0, base, 4, fields, n, out, len);
+   return agx3_bind_compose(enc, "int2flt", 0, base, m5 ? 6 : 4, fields, n, out, len);
 }
 
 bool
@@ -2352,10 +2698,12 @@ agx3_alu_narrow_int_to_float(enum agx_isa_encoding enc, uint16_t dest,
                              bool dest_cache, uint16_t wait_mask,
                              uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 192 ||
-       (!dest_16 && (dest & 1)) || src >= 192 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) ||
+       !out || !len || dest >= (m5 ? 256 : 192) ||
+       (!dest_16 && (dest & 1)) || src >= (m5 ? 256 : 192) ||
        (src_8 && !is_signed && dest_16) ||
-       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
 
    /* applegpu_g17p `half(char)`, `float(char)`, `half(short)` and
@@ -2372,18 +2720,18 @@ agx3_alu_narrow_int_to_float(enum agx_isa_encoding enc, uint16_t dest,
     *   a72754020204b070  h2 <- char(^h2), wait slot 2
     *   a707560202068220  h2 <- ushort($h3), wait slot 6
     *   a707540202089430  r1 <- uchar(h4) */
-   const struct agx3_field_kv base[] = {{9, 2}, {10, dest_16 ? 1 : 2}, {25, 2}};
-   struct agx3_field_kv fields[14] = {
+   const struct agx3_field_kv base[] = {{9, 2}, {10, dest_16 ? 1 : 2}, {25, 2}, {31,3}, {32,91}};
+   struct agx3_field_kv fields[16] = {
       {2, dest}, {11, dest_cache}, {18, src},
       {15, src_8}, {26, src_8 ? 0 : 1},
       {27, hint == AGX3_ALU_HINT_CACHE},
       {29, hint == AGX3_ALU_HINT_DISCARD}, {30, is_signed},
    };
    unsigned n = 8;
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       if (wait_mask & (1u << i))
          fields[n++] = (struct agx3_field_kv){33 + i, 1};
-   return agx3_bind_compose(enc, "int2flt", 0, base, 3, fields, n, out, len);
+   return agx3_bind_compose(enc, "int2flt", 0, base, m5 ? 5 : 3, fields, n, out, len);
 }
 
 bool
@@ -2392,9 +2740,10 @@ agx3_alu_i32_to_f16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                     bool dest_cache, uint16_t wait_mask,
                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len || dest >= 256 ||
        src >= 256 || (src & 1) || (unsigned)hint > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~0x3fu))
+       (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
 
    /* applegpu-nt 32023.917 emits int2flt with f10=1 for a half result and
@@ -2402,7 +2751,7 @@ agx3_alu_i32_to_f16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * f27/f29 carry the post-RA source hint and f33..f38 are dependency waits.
     * Exact witnesses cover signed/unsigned, live/dead, no-wait and wait0. */
    const struct agx3_field_kv base[] = {
-      {9, 2}, {10, 1}, {25, 2}, {26, 2},
+      {9, 2}, {10, 1}, {25, 2}, {26, 2}, {31, 3}, {32, 91},
    };
    /* f27 is the CACHE half of the post-RA source hint and f29 the DISCARD
     * half; the comment above named both but only f29 was ever set, so a
@@ -2424,15 +2773,15 @@ agx3_alu_i32_to_f16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * reverse -- which is the one-hot {none, discard, cache} the enum already
     * describes. The typical user is an i32-to-f16 convert whose source stays
     * live. */
-   struct agx3_field_kv fields[12] = {
+   struct agx3_field_kv fields[14] = {
       {2, dest}, {11, dest_cache}, {18, src},
       {27, hint == AGX3_ALU_HINT_CACHE},
       {29, hint == AGX3_ALU_HINT_DISCARD}, {30, is_signed},
    };
    unsigned n = 6;
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       fields[n++] = (struct agx3_field_kv){33 + i, !!(wait_mask & (1u << i))};
-   return agx3_bind_compose(enc, "int2flt", 0, base, 4, fields, n, out, len);
+   return agx3_bind_compose(enc, "int2flt", 0, base, m5 ? 6 : 4, fields, n, out, len);
 }
 
 bool
@@ -2441,10 +2790,11 @@ agx3_alu_f16_to_int32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                       bool dest_cache, uint16_t wait_mask,
                       uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 128 ||
-       src >= 128 || (dest & 1) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len || dest >= (m5 ? 256 : 128) ||
+       src >= (m5 ? 256 : 128) || (dest & 1) ||
        (hint != AGX3_ALU_HINT_NONE && hint != AGX3_ALU_HINT_DISCARD) ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
       return false;
 
    /* Fresh applegpu_g17p kernels establish the flt2int width/sign axes:
@@ -2463,38 +2813,42 @@ agx3_alu_f16_to_int32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * stay outside this helper.
    */
    const struct agx3_field_kv base[] = {
-      {9, 2}, {10, 2}, {23, 2}, {24, 1}, {41, is_signed ? 5 : 4}, {42, 1},
+      {9, 2}, {10, 2}, {23, 2}, {24, 1}, {41, is_signed ? 5 : 4}, {42, 1}, {31,3}, {32,71},
    };
-   struct agx3_field_kv fields[10] = {
+   struct agx3_field_kv fields[12] = {
       {2, dest}, {11, dest_cache}, {16, src},
       {27, hint == AGX3_ALU_HINT_DISCARD},
    };
    unsigned n = 4;
-   for (unsigned slot = 0; slot < 6; ++slot)
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot)
       fields[n++] = (struct agx3_field_kv){33 + slot,
                                            !!(wait_mask & (1u << slot))};
 
-   return agx3_bind_compose(enc, "flt2int", 0, base, 6, fields, n, out, len);
+   return agx3_bind_compose(enc, "flt2int", 0, base, m5 ? 8 : 6, fields, n, out, len);
 }
 
-bool
-agx3_alu_bfeil(enum agx_isa_encoding enc, uint16_t dest, uint16_t source,
+static bool
+agx3_alu_bfeil_width(enum agx_isa_encoding enc, uint16_t dest, uint16_t source,
                uint16_t shift, bool shift_is_register, uint8_t width,
+               bool source_16, bool shift_16,
                const enum agx3_alu_hint hints[2], uint16_t wait_mask,
                uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *length)
 {
-   if (!out || !length || !hints || enc != AGX_ISA_ENCODING_AGX3B ||
-       dest >= 256 || source >= 256 || ((dest | source) & 1) ||
-       (shift_is_register ? (shift >= 256 || (shift & 1)) : shift >= 32) ||
-       width < 1 || width > 31 || (wait_mask & ~0x3f) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   if (!out || !length || !hints || (!gpu1 && enc != AGX_ISA_ENCODING_AGX3B) ||
+       dest >= 256 || source >= 256 || ((dest & 1) || (!source_16 && (source & 1))) ||
+       (shift_is_register ? (shift >= 256 || (!shift_16 && (shift & 1))) : shift >= 32) ||
+       width < 1 || width > 31 || (wait_mask & ~(gpu1 ? 0xffu : 0x3fu)) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[1] > AGX3_ALU_HINT_CACHE ||
        (!shift_is_register && hints[1] != AGX3_ALU_HINT_NONE))
       return false;
    const struct agx3_field_kv base[] = {
-      {9,2},{10,2},{23,2},{24,2},{39,2},{52,1},{53,2},
+      {9,2},{10,2},{23,2},{24,source_16 ? 1 : 2},{39,shift_16 ? 1 : 2},{52,1},{53,2},
+      /* M5 SRL retains the extract width in f57 and uses its own dispatch. */
+      {58,3},{59,183},
    };
-   struct agx3_field_kv fields[16] = {
+   struct agx3_field_kv fields[19] = {
       {2,dest},{16,source},{25,hints[0] == AGX3_ALU_HINT_CACHE},
       {27,hints[0] == AGX3_ALU_HINT_DISCARD},
       {38,shift_is_register ? 2 : 1},{shift_is_register ? 31 : 30,shift},
@@ -2505,10 +2859,32 @@ agx3_alu_bfeil(enum agx_isa_encoding enc, uint16_t dest, uint16_t source,
       fields[count++] = (struct agx3_field_kv){40,hints[1] == AGX3_ALU_HINT_CACHE};
       fields[count++] = (struct agx3_field_kv){42,hints[1] == AGX3_ALU_HINT_DISCARD};
    }
-   for (unsigned i=0;i<6;++i)
+   for (unsigned i=0;i<(gpu1 ? 8 : 6);++i)
       if (wait_mask & (1u<<i))
          fields[count++] = (struct agx3_field_kv){60+i,1};
-   return agx3_bind_compose(enc,"srl",0,base,7,fields,count,out,length);
+   return agx3_bind_compose(enc,"srl",0,base,gpu1 ? 9 : 7,fields,count,out,length);
+}
+
+bool
+agx3_alu_bfeil(enum agx_isa_encoding enc, uint16_t dest, uint16_t source,
+               uint16_t shift, bool shift_is_register, uint8_t width,
+               const enum agx3_alu_hint hints[2], uint16_t wait_mask,
+               uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *length)
+{
+   return agx3_alu_bfeil_width(enc,dest,source,shift,shift_is_register,width,
+                              false,false,hints,wait_mask,out,length);
+}
+
+bool
+agx3_alu_bfeil_m5(uint16_t dest, uint16_t source, bool source_16,
+                  uint16_t shift, bool shift_is_register, bool shift_16,
+                  uint8_t width, const enum agx3_alu_hint hints[2],
+                  uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                  uint8_t *length)
+{
+   return agx3_alu_bfeil_width(AGX_ISA_ENCODING_AGX3C,dest,source,shift,
+                              shift_is_register,width,source_16,shift_16,
+                              hints,wait_mask,out,length);
 }
 
 bool
@@ -2577,6 +2953,47 @@ agx3_alu_add_sat16(enum agx_isa_encoding enc, uint16_t dest,
    return agx3_bind_compose(enc, "iadd", 0, base, 7, fields, n, out, len);
 }
 
+bool
+agx3_alu_addsub_sat16_m5(uint16_t dest, const uint16_t src[2],
+                         const enum agx3_alu_hint hints[2], bool is_signed,
+                         bool subtract, bool dest_cache, uint16_t wait_mask,
+                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!src || !hints || !out || !len || dest >= 256 || src[0] >= 256 ||
+       src[1] >= 256 || (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
+       (unsigned)hints[1] > AGX3_ALU_HINT_CACHE || (wait_mask & ~0xffu))
+      return false;
+
+   /* Native G17G (port/cwit3) `addsat`/`subsat` on ushort/short: C's
+    * iadd32 (f45 = 80) or isub32 (f45 = 100) register point with half
+    * widths for the result and both operands (f10/f24/f39 = 1), saturation
+    * f54, and the signed form marking both halves signed (f28/f43):
+    *   3f0004382100a302b0002000  h2 <- ^h4 +sat ^h2, unsigned
+    *   3f0004382100a302b000e000  h2 <- ^h4 +sat ^h2, signed
+    *   3f0004382508a20290006000  h2 <- ^h4 -sat ^h2, signed
+    * Waits are C's iadd32 f46..f53. */
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {23, 2}, {24, 1}, {38, 2}, {39, 1},
+      {44, 3}, {45, subtract ? 100 : 80}, {54, 1},
+   };
+   struct agx3_field_kv fields[18] = {
+      {2, dest}, {11, dest_cache}, {16, src[0]}, {31, src[1]},
+      {25, hints[0] == AGX3_ALU_HINT_CACHE},
+      {27, hints[0] == AGX3_ALU_HINT_DISCARD},
+      {40, hints[1] == AGX3_ALU_HINT_CACHE},
+      {42, hints[1] == AGX3_ALU_HINT_DISCARD},
+      {28, is_signed}, {43, is_signed},
+   };
+   unsigned n = 10;
+   for (unsigned i = 0; i < 8; ++i)
+      if (wait_mask & (1u << i))
+         fields[n++] = (struct agx3_field_kv){46 + i, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,
+                            subtract ? "isub32" : "iadd32", 0, base,
+                            sizeof(base) / sizeof(base[0]), fields, n, out,
+                            len);
+}
+
 /* The scalar-half isub point is the iadd layout above: slot A at f16 with
  * cache/discard f25/f27, slot B at f31 with f40/f42, destination cache f11
  * and waits f45..f50. Both register slots end below half 192. */
@@ -2586,16 +3003,19 @@ agx3_alu_isub16(enum agx_isa_encoding enc, uint16_t dest, bool zero_minuend,
                 bool dest_cache, uint16_t wait_mask,
                 uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 192 ||
-       (!zero_minuend && src[0] >= 192) || src[1] >= 192 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned limit = m5 ? 256 : 192;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len || dest >= limit ||
+       (!zero_minuend && src[0] >= limit) || src[1] >= limit ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
-       (unsigned)hints[1] > AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hints[1] > AGX3_ALU_HINT_CACHE || (wait_mask >> (m5 ? 8 : 6)))
       return false;
 
    const struct agx3_field_kv base[] = {
       {9, 2}, {10, 1}, {23, zero_minuend ? 1 : 2}, {24, 1}, {38, 2}, {39, 1},
+      {44, 3}, {45, 100},
    };
-   struct agx3_field_kv fields[14] = {
+   struct agx3_field_kv fields[16] = {
       {2, dest}, {11, dest_cache}, {31, src[1]},
       {40, hints[1] == AGX3_ALU_HINT_CACHE},
       {42, hints[1] == AGX3_ALU_HINT_DISCARD},
@@ -2607,10 +3027,11 @@ agx3_alu_isub16(enum agx_isa_encoding enc, uint16_t dest, bool zero_minuend,
       fields[n++] =
          (struct agx3_field_kv){27, hints[0] == AGX3_ALU_HINT_DISCARD};
    }
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       if (wait_mask & (1u << i))
-         fields[n++] = (struct agx3_field_kv){45 + i, 1};
-   return agx3_bind_compose(enc, "isub", 0, base, 6, fields, n, out, len);
+         fields[n++] = (struct agx3_field_kv){(m5 ? 46 : 45) + i, 1};
+   return agx3_bind_compose(enc, m5 ? "isub32" : "isub", 0, base,
+                            m5 ? 8 : 6, fields, n, out, len);
 }
 
 bool
@@ -2654,33 +3075,40 @@ agx3_alu_shift_reg(const char *mnemonic, enum agx_isa_encoding enc,
                    const enum agx3_alu_hint hints[2], uint16_t wait_mask,
                    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !src_16 || !hints || !out ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) ||
+       (m5 && strcmp(mnemonic, "sll") && strcmp(mnemonic, "srl")) || !src || !src_16 || !hints || !out ||
        !len || (dest_16 ? dest >= 256 : (dest >= 256 || (dest & 1))) ||
        (src_16[0] ? src[0] >= 256 : (src[0] >= 256 || (src[0] & 1))) ||
        (src_16[1] ? src[1] >= 256 : (src[1] >= 256 || (src[1] & 1))) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
-       (unsigned)hints[1] > AGX3_ALU_HINT_DISCARD || (wait_mask & ~0x3fu))
+       (unsigned)hints[1] > AGX3_ALU_HINT_DISCARD || (wait_mask >> (m5 ? 8 : 6)))
       return false;
 
    /* Fresh applegpu_g17p Metal vectors pin the independent result/source
     * widths, value cache/discard (f25/f27), shift discard (f42), and slot-2
     * wait. In particular, a uint shift count remains width 2 even when the
     * value is ushort. */
-   const struct agx3_field_kv base[] = {
+   struct agx3_field_kv base[10] = {
       {9, 2}, {10, dest_16 ? 1 : 2}, {23, 2}, {24, src_16[0] ? 1 : 2},
       {38, 2}, {39, src_16[1] ? 1 : 2}, {52, 1}, {53, 2},
    };
-   struct agx3_field_kv fields[15] = {
+   unsigned nb = 8;
+   if (m5) {
+      base[nb++] = (struct agx3_field_kv){58, 3};
+      base[nb++] = (struct agx3_field_kv){59, !strcmp(mnemonic, "srl") ? 183 : 162};
+   }
+   struct agx3_field_kv fields[16] = {
       {2, dest}, {11, dest_cache}, {16, src[0]},
       {25, hints[0] == AGX3_ALU_HINT_CACHE},
       {27, hints[0] == AGX3_ALU_HINT_DISCARD}, {31, src[1]},
       {40, 0}, {42, hints[1] == AGX3_ALU_HINT_DISCARD},
    };
    unsigned n = 8;
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       if (wait_mask & (1u << i))
          fields[n++] = (struct agx3_field_kv){60 + i, 1};
-   return agx3_bind_compose(enc, mnemonic, 0, base, 8, fields, n, out, len);
+   return agx3_bind_compose(enc, mnemonic, 0, base, nb, fields, n, out, len);
 }
 
 bool
@@ -2734,13 +3162,52 @@ agx3_alu_sll_one_reg32(enum agx_isa_encoding enc, uint16_t dest,
 }
 
 bool
-agx3_alu_srl_reg(enum agx_isa_encoding enc, uint16_t dest, bool dest_16,
-                 bool dest_cache, const uint16_t src[2],
-                 const bool src_16[2], const enum agx3_alu_hint hints[2],
-                 uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
-                 uint8_t *len)
+agx3_alu_sll_one_m5(uint16_t dest, bool dest_16, uint16_t shift,
+                    enum agx3_alu_hint shift_hint, bool dest_cache,
+                    uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                    uint8_t *len)
 {
-   return agx3_alu_shift_reg("srl", enc, dest, dest_16, dest_cache, src,
+   if (!out || !len || dest >= 256 || (!dest_16 && (dest & 1)) ||
+       shift >= 256 || (shift & 1) ||
+       (unsigned)shift_hint > AGX3_ALU_HINT_DISCARD || (wait_mask & ~0xffu))
+      return false;
+
+   /* Native G17G `ushort(1) << uint(n)` (port/cwit3 k_shl1_u16, also
+    * k_shl1_u16b/k_shl1_u8) computes in 32 bits: one sll with the immediate
+    * one in slot A (f15 = 1, f23 = 1, f24 = 1), the full uint count in slot B
+    * (f31, f39 = 2, last use f42) and C's form constants f58 = 3, f59 = 162:
+    *   2701201a2600a002008020340000  r0 <- 1 << ^r0
+    * The result width f10 is the same independent field as in
+    * agx3_alu_sll_reg(); a half destination sets f10 = 1. */
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, dest_16 ? 1 : 2}, {15, 1}, {23, 1}, {24, 1},
+      {38, 2}, {39, 2}, {52, 1}, {53, 2}, {58, 3}, {59, 162},
+   };
+   struct agx3_field_kv fields[12] = {
+      {2, dest},
+      {11, dest_cache},
+      {31, shift},
+      {42, shift_hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned count = 4;
+   for (unsigned slot = 0; slot < 8; ++slot) {
+      if (wait_mask & (1u << slot))
+         fields[count++] = (struct agx3_field_kv){60 + slot, 1};
+   }
+
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "sll", 0, base,
+                            sizeof(base) / sizeof(base[0]), fields, count, out,
+                            len);
+}
+
+bool
+agx3_alu_srl_reg(enum agx_isa_encoding enc, uint16_t dest, bool dest_16,
+                 bool dest_cache, const uint16_t src[2],
+                 const bool src_16[2], const enum agx3_alu_hint hints[2],
+                 uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                 uint8_t *len)
+{
+   return agx3_alu_shift_reg("srl", enc, dest, dest_16, dest_cache, src,
                              src_16, hints, wait_mask, out, len);
 }
 
@@ -2795,10 +3262,11 @@ agx3_alu_shift_imm32(enum agx_isa_encoding enc, bool left,
                       uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
                       uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len ||
-       dest >= 192 || src >= 192 || ((dest | src) & 1) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !out || !len ||
+       dest >= (m5 ? 256 : 192) || src >= (m5 ? 256 : 192) || ((dest | src) & 1) ||
        shift < 1 || shift > 31 ||
-       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
 
    /* Native runtime archives for uint shifts pin the literal count at f30,
@@ -2807,21 +3275,36 @@ agx3_alu_shift_imm32(enum agx_isa_encoding enc, bool left,
     * srl_17:a700560002004400f0110100
     * Both use a discarded r0 value and wait slot 6. No count register or
     * arithmetic boundary conversion is part of this immediate form. */
-   const struct agx3_field_kv base[] = {
+   /* G17G benchmark archive, the same immediate count at f30:
+    * sll #13:0782001a0a00800a788300340000
+    * srl #17:3782001a2a00a10a788400340000
+    * M5 carries the implicit boundary at f58/f59; SLL additionally
+    * selects the full-width source lanes at f71..f73. */
+   struct agx3_field_kv base[13] = {
       {9, 2}, {10, 2}, {23, 2}, {24, 2},
       {38, 1}, {39, 2}, {52, 1}, {53, 2},
    };
-   struct agx3_field_kv fields[12] = {
+   unsigned nb = 8;
+   if (m5) {
+      base[nb++] = (struct agx3_field_kv){58, 3};
+      base[nb++] = (struct agx3_field_kv){59, left ? 162 : 183};
+      if (left) {
+         base[nb++] = (struct agx3_field_kv){71, 2};
+         base[nb++] = (struct agx3_field_kv){72, 2};
+         base[nb++] = (struct agx3_field_kv){73, 2};
+      }
+   }
+   struct agx3_field_kv fields[14] = {
       {2, dest}, {11, dest_cache}, {16, src},
       {25, hint == AGX3_ALU_HINT_CACHE},
       {27, hint == AGX3_ALU_HINT_DISCARD}, {30, shift},
    };
    unsigned n = 6;
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       if (wait_mask & (1u << i))
          fields[n++] = (struct agx3_field_kv){60 + i, 1};
    return agx3_bind_compose(enc, left ? "sll" : "srl", 0,
-                            base, 8, fields, n, out, len);
+                            base, nb, fields, n, out, len);
 }
 
 bool
@@ -2830,9 +3313,10 @@ agx3_alu_srl_imm16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                    uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
                    uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || out == NULL || len == NULL ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || out == NULL || len == NULL ||
        dest >= 256 || src >= 256 || shift < 1 || shift > 31 ||
-       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~(m5 ? 0xffu : 0x3fu)))
       return false;
 
    /* applegpu_g17p emits this form when extracting [[sample_mask]] from
@@ -2840,7 +3324,7 @@ agx3_alu_srl_imm16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * implicit full-width boundary operand rather than a second GPR. */
    const struct agx3_field_kv base[] = {
       {9, 2}, {10, 1}, {23, 2}, {24, 1},
-      {38, 1}, {39, 2}, {52, 1}, {53, 2},
+      {38, 1}, {39, 2}, {52, 1}, {53, 2}, {58, 3}, {59, 183},
    };
    struct agx3_field_kv fields[14] = {
       {2, dest},
@@ -2851,12 +3335,12 @@ agx3_alu_srl_imm16(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
       {30, shift},
    };
    unsigned n = 6;
-   for (unsigned i = 0; i < 6; ++i) {
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i) {
       if (wait_mask & (1u << i))
          fields[n++] = (struct agx3_field_kv){60 + i, 1};
    }
 
-   return agx3_bind_compose(enc, "srl", 0, base, 8, fields, n, out, len);
+   return agx3_bind_compose(enc, "srl", 0, base, m5 ? 10 : 8, fields, n, out, len);
 }
 
 bool
@@ -2866,24 +3350,92 @@ agx3_alu_extract_imm32(enum agx_isa_encoding enc, uint16_t dest,
                        uint16_t wait_mask,
                        uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *length)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !length || !low_high ||
-       !hints || dest>=192 || low_high[0]>=192 || low_high[1]>=192 ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!gpu1 && enc != AGX_ISA_ENCODING_AGX3B) || !out || !length || !low_high ||
+       !hints || dest >= (gpu1 ? 256 : 192) ||
+       low_high[0] >= (gpu1 ? 256 : 192) ||
+       low_high[1] >= (gpu1 ? 256 : 192) ||
        ((dest | low_high[0] | low_high[1]) & 1) || shift<1 || shift>31 ||
        (unsigned)hints[0]>AGX3_ALU_HINT_CACHE ||
-       (unsigned)hints[1]>AGX3_ALU_HINT_CACHE || (wait_mask & ~0x3fu))
+       (unsigned)hints[1]>AGX3_ALU_HINT_CACHE ||
+       (wait_mask & ~(gpu1 ? 0xffu : 0x3fu)))
       return false;
+   /* AGX3c retains the high/shift/low operand field mapping but requires
+    * GPR dispatch f58=3, opcode f59=40 and has eight independent wait bits.
+    * Descriptor address extraction consumes a load in slot7 on M5. */
    const struct agx3_field_kv base[] = {
       {9,2},{10,2},{23,2},{24,2},{38,1},{39,2},{52,2},{53,2},
+      {58,3},{59,40},
    };
-   struct agx3_field_kv fields[16] = {
+   struct agx3_field_kv fields[19] = {
       {2,dest},{11,dest_cache},{16,low_high[1]},{30,shift},{45,low_high[0]},
       {25,hints[1]==AGX3_ALU_HINT_CACHE},{27,hints[1]==AGX3_ALU_HINT_DISCARD},
       {54,hints[0]==AGX3_ALU_HINT_CACHE},{56,hints[0]==AGX3_ALU_HINT_DISCARD},
       {57,0},
    };
-   for(unsigned i=0;i<6;i++)
+   for(unsigned i=0;i<(gpu1 ? 8 : 6);i++)
       fields[10+i]=(struct agx3_field_kv){60+i,!!(wait_mask & (1u<<i))};
-   return agx3_bind_compose(enc,"extract",0,base,8,fields,16,out,length);
+   return agx3_bind_compose(enc,"extract",0,base,gpu1 ? 10 : 8,
+                           fields,gpu1 ? 18 : 16,out,length);
+}
+
+/* Metal532: runtime int and short shifts. The full-word value uses a
+ * half count (only its low five bits matter); the short value uses a word
+ * count. f28 selects sign fill and f45..52 carry independent waits. */
+bool
+agx3_alu_asr_reg_m5(uint16_t dest, uint16_t source, uint16_t shift,
+                     bool dest_16, bool source_16,
+                     const enum agx3_alu_hint hints[2], bool dest_cache,
+                     uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                     uint8_t *length)
+{
+   if (!out || !length || !hints || dest >= 256 || source >= 256 ||
+       shift >= 256 || (shift & 1) || (dest_16 && !source_16) ||
+       (!dest_16 && (dest & 1)) || (!source_16 && (source & 1)) ||
+       (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
+       (unsigned)hints[1] > AGX3_ALU_HINT_CACHE || (wait_mask & ~0xffu))
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9,2},{10,dest_16 ? 1 : 2},{23,2},{24,source_16 ? 1 : 2},
+      {28,1},{38,2},{39,dest_16 ? 2 : 1},{43,3},{44,167},
+   };
+   struct agx3_field_kv fields[16] = {
+      {2,dest},{11,dest_cache},{16,source},{31,shift},
+      {25,hints[0] == AGX3_ALU_HINT_CACHE},
+      {27,hints[0] == AGX3_ALU_HINT_DISCARD},
+      {40,hints[1] == AGX3_ALU_HINT_CACHE},
+      {42,hints[1] == AGX3_ALU_HINT_DISCARD},
+   };
+   for (unsigned i=0;i<8;i++)
+      fields[8+i]=(struct agx3_field_kv){45+i,!!(wait_mask & (1u<<i))};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,"sra",0,base,9,
+                           fields,16,out,length);
+}
+
+bool
+agx3_alu_asr_imm_m5(uint16_t dest, uint16_t source, bool dest_16,
+                     bool source_16, unsigned shift, enum agx3_alu_hint hint,
+                     bool dest_cache, uint16_t wait_mask,
+                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *length)
+{
+   if (!out || !length || dest >= 256 || source >= 256 ||
+       (!dest_16 && (dest & 1)) || (!source_16 && (source & 1)) ||
+       shift < 1 || shift > (source_16 ? 15 : 31) ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0xffu))
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9,2},{10,dest_16 ? 1 : 2},{23,2},{24,source_16 ? 1 : 2},
+      {38,1},{39,2},{43,3},{44,167},
+   };
+   struct agx3_field_kv fields[15] = {
+      {2,dest},{11,dest_cache},{16,source},{30,shift},
+      {25,hint == AGX3_ALU_HINT_CACHE},{27,hint == AGX3_ALU_HINT_DISCARD},
+      {28,1},
+   };
+   for (unsigned i=0;i<8;i++)
+      fields[7+i]=(struct agx3_field_kv){45+i,!!(wait_mask & (1u<<i))};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,"sra",0,base,8,
+                           fields,15,out,length);
 }
 
 /* Native arithmetic right shift keeps signed fill without a second operation. */
@@ -3701,9 +4253,13 @@ agx3_alu_rcp(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * hk_shader.c turns a packer refusal into VK_ERROR_FEATURE_NOT_PRESENT for
     * the whole pipeline. Three of the ten distinct programs three.js compiles
     * for its first few examples died on `rcp [d32:reg140 s0=32:reg86.abs]`. */
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 192 ||
-       src >= 192 || (!dest_half && (dest & 1)) || (!src_half && (src & 1)) ||
-       (wait_mask & ~UINT16_C(0x3f)) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned regs = m5 ? 256 : 192;
+   const unsigned slots = m5 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !out || !len ||
+       dest >= regs || src >= regs ||
+       (!dest_half && (dest & 1)) || (!src_half && (src & 1)) ||
+       (wait_mask & ~((1u << slots) - 1)) ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE)
       return false;
 
@@ -3715,7 +4271,7 @@ agx3_alu_rcp(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
       {9, 2}, {10, dest_half ? 1 : 2}, {25, 2},
       {26, src_half ? 1 : 2}, {43, 2},
    };
-   struct agx3_field_kv fields[13] = {
+   struct agx3_field_kv fields[17] = {
       {2, dest},
       {11, dest_cache},
       {18, src},
@@ -3724,11 +4280,16 @@ agx3_alu_rcp(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
       {30, src_abs},
       {31, src_neg},
    };
-   for (unsigned slot = 0; slot < 6; ++slot)
-      fields[7 + slot] = (struct agx3_field_kv){49 + slot,
+   unsigned n = 7;
+   if (m5) {
+      fields[n++] = (struct agx3_field_kv){47, 3};
+      fields[n++] = (struct agx3_field_kv){48, 64};
+   }
+   for (unsigned slot = 0; slot < slots; ++slot)
+      fields[n++] = (struct agx3_field_kv){49 + slot,
                                                !!(wait_mask & (1u << slot))};
 
-   return agx3_bind_compose(enc, "f32rcp", 0, base, 5, fields, 13, out, len);
+   return agx3_bind_compose(enc, "f32rcp", 0, base, 5, fields, n, out, len);
 }
 
 bool
@@ -3740,6 +4301,13 @@ agx3_alu_f32rcp(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                        dest_cache, wait_mask, out, len);
 }
 
+static bool agx3_alu_unop(enum agx_isa_encoding enc, const char *mnemonic,
+                          uint16_t dest, uint16_t src, bool dest_half,
+                          bool src_half, enum agx3_alu_hint hint,
+                          bool dest_cache, uint16_t wait_mask, bool src_neg,
+                          bool src_abs, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                          uint8_t *len);
+
 bool
 agx3_alu_f32rsqrt_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                       enum agx3_alu_hint hint, bool dest_cache,
@@ -3750,6 +4318,9 @@ agx3_alu_f32rsqrt_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
     * sets both to 1, the 32-bit form to 2. f30 and f31 fold the source's
     * absolute value and negation in, exactly as on f32log2/f32exp2/f32rcp;
     * see agx3_alu_core.h for the four witnessed bytes. */
+   if (enc == AGX_ISA_ENCODING_AGX3C)
+      return agx3_alu_unop(enc, "f32rsqrt", dest, src, false, false, hint,
+                           dest_cache, wait_mask, src_neg, src_abs, out, len);
    if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
        src >= 256 || (dest & 1) || (src & 1) ||
        (wait_mask & ~UINT16_C(0x3f)) ||
@@ -3770,6 +4341,88 @@ agx3_alu_f32rsqrt_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    return agx3_bind_compose(enc, "f32rsqrt", 0, base, 4, fields, 13, out, len);
 }
 
+bool
+agx3_alu_unary_m5(enum agx3_funop op, uint16_t dest, uint16_t src,
+                   bool dest_half, bool src_half, bool src_abs,
+                   bool src_neg, bool saturate, enum agx3_alu_hint hint,
+                   bool dest_cache, uint16_t wait_mask,
+                   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || (unsigned)op > AGX3_FUNOP_EXP2 || dest >= 256 || src >= 256 ||
+       (!dest_half && (dest & 1)) || (!src_half && (src & 1)) ||
+       /* No direct narrowing witness; captured probes convert separately. */
+       (dest_half && !src_half) || (wait_mask & ~0xffu) ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE)
+      return false;
+
+   const bool rounding = op <= AGX3_FUNOP_RINT;
+   const char *mnemonic = rounding ? "f32rint" :
+                          op == AGX3_FUNOP_LOG2 ? "f32log2" : "f32exp2";
+   const unsigned opcode = rounding ? 65 : op == AGX3_FUNOP_LOG2 ? 59 : 52;
+   /* Fresh G17G profiles prove odd half registers, half-to-full inputs,
+    * f14 saturation, f30/f31 modifiers, actual lifetimes and zero/slot8
+    * waits on this same native instruction family. */
+   struct agx3_field_kv base[7] = {
+      {9, 2}, {10, dest_half ? 1 : 2}, {25, 2},
+      {26, src_half ? 1 : 2}, {33, 3}, {34, opcode},
+   };
+   unsigned nbase = 6;
+   if (rounding)
+      base[nbase++] = (struct agx3_field_kv){43, 3 - op};
+   struct agx3_field_kv fields[16] = {
+      {2, dest}, {18, src}, {11, dest_cache}, {14, saturate},
+      {27, hint == AGX3_ALU_HINT_CACHE},
+      {29, hint == AGX3_ALU_HINT_DISCARD}, {30, src_abs}, {31, src_neg},
+   };
+   unsigned n = 8;
+   for (unsigned slot = 0; slot < 8; ++slot)
+      if (wait_mask & (1u << slot))
+         fields[n++] = (struct agx3_field_kv){35 + slot, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, mnemonic, 0, base, nbase,
+                            fields, n, out, len);
+}
+
+/*
+ * The gradients are the same f33=3 class as rint/log2/exp2 above (f34 49 and
+ * 50): the destination and source registers, widths, modifiers, saturation
+ * and wait slots sit in the same fields. Native G17G fragment programs use
+ * 16-bit gradients with odd half registers and sources above h127, and wait
+ * on slots 0, 1, 2 and 7 directly. The destination and source share one width
+ * there; no mixed-width gradient was observed, so none is encoded.
+ *
+ * The cache hints keep the canonical gradient values (f11 set, and f27 set
+ * on dfdx only), as the 32-bit low-register encoding always has.
+ */
+bool
+agx3_alu_derivative_m5(bool dfdy, uint16_t dest, uint16_t src, bool half,
+                       bool src_abs, bool src_neg, bool saturate,
+                       uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                       uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 ||
+       (!half && ((dest & 1) || (src & 1))) || (wait_mask & ~0xffu))
+      return false;
+
+   struct agx3_field_kv base[8] = {
+      {9, 2}, {10, half ? 1 : 2}, {11, 1}, {25, 2}, {26, half ? 1 : 2},
+      {33, 3}, {34, dfdy ? 50 : 49},
+   };
+   unsigned nbase = 7;
+   if (!dfdy)
+      base[nbase++] = (struct agx3_field_kv){27, 1};
+
+   struct agx3_field_kv fields[13] = {
+      {2, dest}, {18, src}, {14, saturate}, {30, src_abs}, {31, src_neg},
+   };
+   unsigned n = 5;
+   for (unsigned slot = 0; slot < 8; ++slot)
+      if (wait_mask & (1u << slot))
+         fields[n++] = (struct agx3_field_kv){35 + slot, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,
+                            dfdy ? "f32dfdy" : "f32dfdx", 0, base, nbase,
+                            fields, n, out, len);
+}
+
 /* log2/exp2 share independently selected f10 destination and f26 source
  * widths. Full-width forms have all six recovered wait bits; any half-width
  * log2 is currently grounded for no wait and slot 6. */
@@ -3780,6 +4433,40 @@ agx3_alu_unop(enum agx_isa_encoding enc, const char *mnemonic,
               bool src_neg, bool src_abs,
               uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      if (!strcmp(mnemonic, "f32rsqrt")) {
+         if (!out || !len || dest >= 256 || src >= 256 ||
+             (!dest_half && (dest & 1)) || (!src_half && (src & 1)) ||
+             (wait_mask & ~0xffu) || (unsigned)hint > AGX3_ALU_HINT_CACHE)
+            return false;
+         const struct agx3_field_kv base[] = {
+            {9, 2},  {10, dest_half ? 1 : 2},
+            {25, 2}, {26, src_half ? 1 : 2},
+            {33, 3}, {34, 66},
+         };
+         struct agx3_field_kv f[15] = {
+            {2, dest},
+            {18, src},
+            {11, dest_cache},
+            {27, hint == AGX3_ALU_HINT_CACHE},
+            {29, hint == AGX3_ALU_HINT_DISCARD},
+            {30, src_abs},
+            {31, src_neg},
+         };
+         for (unsigned i = 0; i < 8; i++)
+            f[7 + i] =
+               (struct agx3_field_kv){35 + i, !!(wait_mask & (1u << i))};
+         return agx3_bind_compose(enc, mnemonic, 0, base, 6, f, 15, out, len);
+      }
+      if (strcmp(mnemonic, "f32log2") && strcmp(mnemonic, "f32exp2"))
+         return false;
+      return agx3_alu_unary_m5(!strcmp(mnemonic, "f32log2")
+                                 ? AGX3_FUNOP_LOG2 : AGX3_FUNOP_EXP2,
+                                dest, src, dest_half, src_half, src_abs,
+                                src_neg, false, hint, dest_cache, wait_mask,
+                                out, len);
+   }
+
    if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
        src >= 256 || (!dest_half && (dest & 1)) ||
        (!src_half && (src & 1)) ||
@@ -3811,7 +4498,8 @@ agx3_alu_log2(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
 {
    /* applegpu_g17p's f32log2 operand grammar names halves 0..191. Full
     * scalars additionally require their ordinary even-pair alignment. */
-   if (dest >= 192 || src >= 192)
+   const unsigned registers = enc == AGX_ISA_ENCODING_AGX3C ? 256 : 192;
+   if (dest >= registers || src >= registers)
       return false;
 
    return agx3_alu_unop(enc, "f32log2", dest, src, dest_half, src_half, hint,
@@ -3840,6 +4528,14 @@ agx3_alu_f32_rint(enum agx_isa_encoding enc, uint64_t mode, uint16_t dest,
                   uint16_t wait_mask, bool src_neg, bool src_abs,
                   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      if (mode > 3)
+         return false;
+      return agx3_alu_unary_m5(AGX3_FUNOP_RINT - mode, dest, src, false,
+                                false, src_abs, src_neg, false, hint,
+                                dest_cache, wait_mask, out, len);
+   }
+
    if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
        src >= 256 || (dest & 1) || (src & 1) ||
        (wait_mask & ~UINT16_C(0x3f)) ||
@@ -3891,6 +4587,16 @@ agx3_alu_f32roundeven(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                             src_neg, src_abs, out, len);
 }
 
+bool
+agx3_alu_f32trunc(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+                  enum agx3_alu_hint hint, bool dest_cache, uint16_t wait_mask,
+                  bool src_neg, bool src_abs,
+                  uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_alu_f32_rint(enc, 1, dest, src, hint, dest_cache, wait_mask,
+                            src_neg, src_abs, out, len);
+}
+
 bool
 agx3_alu_hrsqrt(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                 enum agx3_alu_hint hint, bool dest_cache,
@@ -3985,24 +4691,37 @@ agx3_alu_fmul_narrow(enum agx_isa_encoding enc, uint16_t dest,
                      bool dest_cache, uint16_t wait_mask,
                      uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
-       dest >= 256 || src[0] >= 256 || src[1] >= 256 ||
-       ((src[0] | src[1]) & 1) || (wait_mask & ~UINT16_C(0x3f)) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !src || !hints || !out ||
+       !len || dest >= 256 || src[0] >= 256 || src[1] >= 256 ||
+       ((src[0] | src[1]) & 1) || (wait_mask & ~(m5 ? 0xffu : 0x3fu)) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[1] > AGX3_ALU_HINT_CACHE)
       return false;
    const struct agx3_field_kv base[] = {
       {9,2},{10,1},{25,2},{26,2},{42,2},{43,2},{60,2},{74,2},
    };
-   struct agx3_field_kv fields[14] = {
-      {2,dest},{18,src[0]},{35,src[1]},{11,dest_cache},
-      {27,hints[0] == AGX3_ALU_HINT_CACHE},{29,hints[0] == AGX3_ALU_HINT_DISCARD},
-      {44,hints[1] == AGX3_ALU_HINT_CACHE},{46,hints[1] == AGX3_ALU_HINT_DISCARD},
+   struct agx3_field_kv fields[18] = {
+      {2, dest},
+      {18, src[0]},
+      {35, src[1]},
+      {11, dest_cache},
+      {27, hints[0] == AGX3_ALU_HINT_CACHE},
+      {29, hints[0] == AGX3_ALU_HINT_DISCARD},
+      {44, hints[1] == AGX3_ALU_HINT_CACHE},
+      {46, hints[1] == AGX3_ALU_HINT_DISCARD},
    };
    for (unsigned i=0;i<6;i++)
       fields[8+i] = (struct agx3_field_kv){80+i,!!(wait_mask & (1u<<i))};
-   for (unsigned ep=0;ep<2;ep++)
-      if (agx3_bind_compose(enc,"f32mul",ep,base,8,fields,14,out,len))
+   if (m5) {
+      fields[14] = (struct agx3_field_kv){78, 3};
+      fields[15] = (struct agx3_field_kv){79, 62};
+      fields[16] = (struct agx3_field_kv){86, !!(wait_mask & 64)};
+      fields[17] = (struct agx3_field_kv){87, !!(wait_mask & 128)};
+   }
+   for (unsigned ep = 0; ep < (m5 ? 4 : 2); ep++)
+      if (agx3_bind_compose(enc, "f32mul", ep, base, 8, fields, m5 ? 18 : 14,
+                            out, len))
          return true;
    return false;
 }
@@ -4032,13 +4751,14 @@ static bool
 agx3_alu_h_binop(enum agx_isa_encoding enc, const char *mnemonic,
                  unsigned max_ep, uint16_t dest, const uint16_t src[2],
                  const enum agx3_alu_hint hints[2], bool dest_cache,
-                 bool saturate, const bool src_abs[2], uint16_t wait_mask,
+                 bool saturate, const bool src_abs[2], const bool src_neg[2], uint16_t wait_mask,
                  uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !src_abs || !out ||
-       !len ||
-       dest >= 256 || src[0] >= 256 || src[1] >= 256 ||
-       (wait_mask & ~UINT16_C(0x3f)) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) ||
+       !src || !hints || !src_abs || !out || !len || dest >= 256 ||
+       src[0] >= 256 || src[1] >= 256 ||
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[1] > AGX3_ALU_HINT_CACHE)
       return false;
@@ -4046,7 +4766,7 @@ agx3_alu_h_binop(enum agx_isa_encoding enc, const char *mnemonic,
    const struct agx3_field_kv base[] = {
       {9, 2}, {10, 1}, {24, 2}, {25, 1}, {41, 2}, {42, 1},
    };
-   struct agx3_field_kv fields[17] = {
+   struct agx3_field_kv fields[24] = {
       {2, dest},
       {17, src[0]},
       {34, src[1]},
@@ -4062,11 +4782,19 @@ agx3_alu_h_binop(enum agx_isa_encoding enc, const char *mnemonic,
       fields[n++] = (struct agx3_field_kv){29, 1};
    if (src_abs[1])
       fields[n++] = (struct agx3_field_kv){46, 1};
-   for (unsigned i = 0; i < 6; ++i)
+   if (src_neg && src_neg[0])
+      fields[n++] = (struct agx3_field_kv){30, 1};
+   if (src_neg && src_neg[1])
+      fields[n++] = (struct agx3_field_kv){47, 1};
+   if (m5) {
+      fields[n++] = (struct agx3_field_kv){49, 3};
+      fields[n++] = (struct agx3_field_kv){50, !strcmp(mnemonic, "f16mul") ? 47 : 41};
+   }
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       fields[n++] =
          (struct agx3_field_kv){51 + i, !!(wait_mask & (1u << i))};
 
-   for (unsigned ep = 0; ep < max_ep; ++ep) {
+   for (unsigned ep = 0; ep < (m5 ? 4 : max_ep); ++ep) {
       if (agx3_bind_compose(enc, mnemonic, ep, base, 6, fields, n, out, len))
          return true;
    }
@@ -4082,7 +4810,7 @@ agx3_alu_hmul(enum agx_isa_encoding enc, uint16_t dest,
 {
    const bool src_abs[2] = {false, false};
    return agx3_alu_h_binop(enc, "f16mul", 2, dest, src, hints, dest_cache,
-                           false, src_abs, wait_mask, out, len);
+                           false, src_abs, NULL, wait_mask, out, len);
 }
 
 bool
@@ -4094,7 +4822,7 @@ agx3_alu_hmul_abs(enum agx_isa_encoding enc, uint16_t dest,
                   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
    return agx3_alu_h_binop(enc, "f16mul", 4, dest, src, hints, dest_cache,
-                           false, src_abs, wait_mask, out, len);
+                           false, src_abs, NULL, wait_mask, out, len);
 }
 
 bool
@@ -4105,7 +4833,7 @@ agx3_alu_hmul_saturate(enum agx_isa_encoding enc, uint16_t dest,
 {
    const bool src_abs[2] = {false, false};
    return agx3_alu_h_binop(enc, "f16mul", 3, dest, src, hints, dest_cache,
-                           true, src_abs, wait_mask, out, len);
+                           true, src_abs, NULL, wait_mask, out, len);
 }
 
 bool
@@ -4115,18 +4843,21 @@ agx3_alu_hmul_minifloat(enum agx_isa_encoding enc, uint16_t dest,
                         uint16_t wait_mask,
                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !out || !len || dest >= 256 ||
        src >= 256 || immediate > UINT8_MAX ||
        (unsigned)hint > AGX3_ALU_HINT_CACHE ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
       return false;
 
    /* Fresh G17P Metal witnesses place the minifloat in field16, retain the
-    * ordinary half register/lifetime fields, and carry all six wait bits. */
+    * ordinary half register/lifetime fields, and carry all six wait bits.
+    * G17G uses the same compact operands and its f16mul ep3 selectors;
+    * native constant sweeps and live/dead half4 vectors pin slot 8. */
    const struct agx3_field_kv base[] = {
       {9, 2}, {10, 1}, {24, 1}, {25, 1}, {41, 2}, {42, 1},
    };
-   struct agx3_field_kv fields[12] = {
+   struct agx3_field_kv fields[16] = {
       {2, dest},
       {16, immediate},
       {34, src},
@@ -4134,12 +4865,17 @@ agx3_alu_hmul_minifloat(enum agx_isa_encoding enc, uint16_t dest,
       {43, hint == AGX3_ALU_HINT_CACHE},
       {45, hint == AGX3_ALU_HINT_DISCARD},
    };
-   for (unsigned i = 0; i < 6; ++i)
-      fields[6 + i] = (struct agx3_field_kv){51 + i,
+   unsigned n = 6;
+   if (m5) {
+      fields[n++] = (struct agx3_field_kv){49, 3};
+      fields[n++] = (struct agx3_field_kv){50, 47};
+   }
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
+      fields[n++] = (struct agx3_field_kv){51 + i,
                                             !!(wait_mask & (1u << i))};
 
-   for (unsigned ep = 0; ep < 2; ++ep)
-      if (agx3_bind_compose(enc, "f16mul", ep, base, 6, fields, 12, out,
+   for (unsigned ep = 0; ep < (m5 ? 4 : 2); ++ep)
+      if (agx3_bind_compose(enc, "f16mul", ep, base, 6, fields, n, out,
                             len))
          return true;
 
@@ -4156,7 +4892,63 @@ agx3_alu_hadd(enum agx_isa_encoding enc, uint16_t dest,
 {
    const bool src_abs[2] = {false, false};
    return agx3_alu_h_binop(enc, "f16add", 4, dest, src, hints, dest_cache,
-                           false, src_abs, wait_mask, out, len);
+                           false, src_abs, NULL, wait_mask, out, len);
+}
+
+bool
+agx3_alu_hadd_mod_m5(uint16_t dest, const uint16_t src[2],
+                     const enum agx3_alu_hint hints[2], const bool src_abs[2],
+                     const bool src_neg[2], bool dest_cache, bool saturate,
+                     uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                     uint8_t *len)
+{
+   return agx3_alu_h_binop(AGX_ISA_ENCODING_AGX3C, "f16add", 4, dest, src,
+                           hints, dest_cache, saturate, src_abs, src_neg,
+                           wait_mask, out, len);
+}
+
+bool
+agx3_alu_half_add_imm_to_f32_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+                                bool src_abs, bool src_neg,
+                                enum agx3_alu_hint hint, bool dest_cache,
+                                uint16_t wait_mask,
+                                uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 || (dest & 1) ||
+       (wait_mask & ~0xffu) || (unsigned)hint > AGX3_ALU_HINT_CACHE)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 2}, {25, 1}, {26, 2}, {42, 2}, {43, 1}, {50, 3}, {51, 48},
+   };
+   struct agx3_field_kv f[16] = {
+      {2, dest},
+      {11, dest_cache},
+      {17, immediate},
+      {35, src},
+      {44, hint == AGX3_ALU_HINT_CACHE},
+      {46, hint == AGX3_ALU_HINT_DISCARD},
+      {47, src_abs},
+      {48, src_neg},
+   };
+   for (unsigned i = 0; i < 8; i++)
+      f[8 + i] = (struct agx3_field_kv){52 + i, !!(wait_mask & (1u << i))};
+   for (unsigned ep = 0; ep < 4; ep++)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "f32add", ep, base, 8, f,
+                            16, out, len))
+         return true;
+   return false;
+}
+
+bool
+agx3_alu_hadd_signed(enum agx_isa_encoding enc, uint16_t dest,
+                      const uint16_t src[2], const enum agx3_alu_hint hints[2],
+                      const bool src_neg[2], bool dest_cache,
+                      uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                      uint8_t *len)
+{
+   const bool src_abs[] = {false, false};
+   return agx3_alu_h_binop(enc, "f16add", 4, dest, src, hints, dest_cache,
+                           false, src_abs, src_neg, wait_mask, out, len);
 }
 
 bool
@@ -4167,7 +4959,44 @@ agx3_alu_hadd_saturate(enum agx_isa_encoding enc, uint16_t dest,
 {
    const bool src_abs[2] = {false, false};
    return agx3_alu_h_binop(enc, "f16add", 4, dest, src, hints, dest_cache,
-                           true, src_abs, wait_mask, out, len);
+                           true, src_abs, NULL, wait_mask, out, len);
+}
+
+bool
+agx3_alu_half_add_negzero_m5(uint16_t dest, uint16_t src, bool src_abs,
+                             bool src_neg, bool saturate, enum agx3_alu_hint hint,
+                             bool dest_cache, uint16_t wait_mask,
+                             uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || (wait_mask & ~0xffu))
+      return false;
+
+   /* G17G device-half witnesses: abs, negate and negative-abs keep the
+    * source in a scalar half and use the native modifier fields.
+    * saturate(half), with or without either modifier, is this same form
+    * with f14 set (port/hsatwit k_hsat, k_hsat_neg, k_hsat_abs). */
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {24, 2}, {25, 1}, {41, 1}, {42, 1}, {49, 3}, {50, 41},
+   };
+   struct agx3_field_kv fields[17] = {
+      {2, dest},
+      {17, src},
+      {33, 0x80},
+      {11, dest_cache},
+      {26, hint == AGX3_ALU_HINT_CACHE},
+      {28, hint == AGX3_ALU_HINT_DISCARD},
+      {29, src_abs},
+      {30, src_neg},
+   };
+   for (unsigned i = 0; i < 8; ++i)
+      fields[8 + i] = (struct agx3_field_kv){51 + i, !!(wait_mask & (1u << i))};
+   fields[16] = (struct agx3_field_kv){14, saturate};
+   for (unsigned ep = 0; ep < 4; ++ep)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "f16add", ep, base, 8,
+                            fields, 17, out, len))
+         return true;
+   return false;
 }
 
 bool
@@ -4270,16 +5099,17 @@ agx3_alu_hadd_one_neg(enum agx_isa_encoding enc, uint16_t dest,
 }
 
 /* Native scalar half FMA preserves the single-rounding f16 semantics. */
-bool
-agx3_alu_hfma(enum agx_isa_encoding enc, uint16_t dest,
+static bool
+hfma(enum agx_isa_encoding enc, uint16_t dest,
               const uint16_t src[3], const enum agx3_alu_hint hints[3],
-              bool product_neg, bool src2_neg, bool dest_cache,
+              bool product_neg, bool src2_neg, bool dest_cache, bool saturate,
               uint16_t wait_mask,
               uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
-       dest >= 256 || src[0] >= 256 || src[1] >= 256 || src[2] >= 256 ||
-       (wait_mask & ~UINT16_C(0x3f)) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !src || !hints || !out || !len ||
+       (saturate && !m5) || dest >= 256 || src[0] >= 256 || src[1] >= 256 || src[2] >= 256 ||
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[1] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[2] > AGX3_ALU_HINT_CACHE)
@@ -4289,7 +5119,7 @@ agx3_alu_hfma(enum agx_isa_encoding enc, uint16_t dest,
       {9, 2}, {10, 1}, {24, 2}, {25, 1},
       {41, 2}, {42, 1}, {58, 2}, {59, 1},
    };
-   struct agx3_field_kv fields[19] = {
+   struct agx3_field_kv fields[24] = {
       {2, dest},
       {17, src[0]},
       {34, src[1]},
@@ -4303,18 +5133,25 @@ agx3_alu_hfma(enum agx_isa_encoding enc, uint16_t dest,
       {62, hints[2] == AGX3_ALU_HINT_DISCARD},
       {47, product_neg},
       {64, src2_neg},
+      {14, saturate},
    };
-   for (unsigned i = 0; i < 6; ++i)
-      fields[13 + i] =
+   unsigned n = 14;
+   if (m5) {
+      fields[n++] = (struct agx3_field_kv){66, 3};
+      fields[n++] = (struct agx3_field_kv){67, 42};
+   }
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
+      fields[n++] =
          (struct agx3_field_kv){68 + i, !!(wait_mask & (1u << i))};
 
-   /* The compact ep3 tail is a one-of-six wait selector. ep4 and ep5 carry
+   /* G17G has native ep2 witnesses and an ep5 slot-8 dependency witness.
+    * G17P's compact ep3 tail is a one-of-six wait selector. ep4 and ep5 carry
     * the six fields independently, so dependency sets with several producers
     * must begin there instead of aliasing to a different compact selector. */
    const unsigned first_ep =
-      wait_mask && (wait_mask & (wait_mask - 1)) ? 4 : 3;
+      m5 ? 2 : (wait_mask && (wait_mask & (wait_mask - 1)) ? 4 : 3);
    for (unsigned ep = first_ep; ep < 6; ++ep) {
-      if (agx3_bind_compose(enc, "f16fma", ep, base, 8, fields, 19, out,
+      if (agx3_bind_compose(enc, "f16fma", ep, base, 8, fields, n, out,
                             len))
          return true;
    }
@@ -4322,6 +5159,26 @@ agx3_alu_hfma(enum agx_isa_encoding enc, uint16_t dest,
    return false;
 }
 
+bool
+agx3_alu_hfma(enum agx_isa_encoding enc, uint16_t dest,
+              const uint16_t src[3], const enum agx3_alu_hint hints[3],
+              bool product_neg, bool src2_neg, bool dest_cache,
+              uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return hfma(enc, dest, src, hints, product_neg, src2_neg, dest_cache,
+               false, wait_mask, out, len);
+}
+
+bool
+agx3_alu_hfma_saturate_m5(uint16_t dest, const uint16_t src[3],
+              const enum agx3_alu_hint hints[3], bool product_neg,
+              bool src2_neg, bool dest_cache, uint16_t wait_mask,
+              uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return hfma(AGX_ISA_ENCODING_AGX3C, dest, src, hints, product_neg,
+               src2_neg, dest_cache, true, wait_mask, out, len);
+}
+
 /* Native scalar half FMA with an 8-bit minifloat addend. Fresh G17P Metal
  * witnesses place the addend in field50 and select the immediate third-source
  * form with fields58..59 = 1,1. This is distinct from both the three-register
@@ -4332,9 +5189,10 @@ agx3_alu_hfma_addend_minifloat(
    uint16_t immediate, const enum agx3_alu_hint hints[2], bool dest_cache,
    uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
-       dest >= 256 || src[0] >= 256 || src[1] >= 256 ||
-       immediate > UINT8_MAX || (wait_mask & ~UINT16_C(0x3f)) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !src || !hints || !out ||
+       !len || dest >= 256 || src[0] >= 256 || src[1] >= 256 ||
+       immediate > UINT8_MAX || (wait_mask & ~(m5 ? 0xffu : 0x3fu)) ||
        (unsigned)hints[0] > AGX3_ALU_HINT_CACHE ||
        (unsigned)hints[1] > AGX3_ALU_HINT_CACHE)
       return false;
@@ -4343,7 +5201,7 @@ agx3_alu_hfma_addend_minifloat(
       {9, 2}, {10, 1}, {24, 2}, {25, 1},
       {41, 2}, {42, 1}, {58, 1}, {59, 1},
    };
-   struct agx3_field_kv fields[15] = {
+   struct agx3_field_kv fields[19] = {
       {2, dest},
       {17, src[0]},
       {34, src[1]},
@@ -4358,12 +5216,19 @@ agx3_alu_hfma_addend_minifloat(
       fields[9 + i] =
          (struct agx3_field_kv){68 + i, !!(wait_mask & (1u << i))};
 
+   if (m5) {
+      fields[15] = (struct agx3_field_kv){66, 3};
+      fields[16] = (struct agx3_field_kv){67, 42};
+      fields[17] = (struct agx3_field_kv){74, !!(wait_mask & 64)};
+      fields[18] = (struct agx3_field_kv){75, !!(wait_mask & 128)};
+   }
+
    /* ep3 carries one selected wait; the extended endpoints carry a mask. */
    const unsigned first_ep =
       wait_mask && (wait_mask & (wait_mask - 1)) ? 4 : 3;
    for (unsigned ep = first_ep; ep < 6; ++ep) {
-      if (agx3_bind_compose(enc, "f16fma", ep, base, 8, fields, 15, out,
-                            len))
+      if (agx3_bind_compose(enc, "f16fma", ep, base, 8, fields, m5 ? 19 : 15,
+                            out, len))
          return true;
    }
 
@@ -4377,28 +5242,436 @@ agx3_alu_widen_half(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                     bool dest_cache, uint16_t wait_mask,
                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !out || !len || dest >= 256 ||
-       src >= 256 || (dest & 1) || (wait_mask & ~UINT16_C(0x3f)) ||
-       (unsigned)hint > AGX3_ALU_HINT_CACHE || (src_abs && src_neg))
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !out || !len ||
+       dest >= 256 || src >= 256 || (dest & 1) ||
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))) ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE ||
+       (!m5 && src_abs && src_neg))
       return false;
+   /* G17G Metal half4 -> float4 witnesses retain the half operand and add
+    * negative zero. M5 also requires its floating-point pipeline selectors;
+    * these are not the G17P long-form tail bits. Native modifier and slot-8
+    * wait variants round-trip through the AGX3C composer without patching.
+    */
    const struct agx3_field_kv base[] = {
       {9,2},{10,2},{25,2},{26,1},{42,1},{43,2},
+      {50,3},{51,48},
    };
-   struct agx3_field_kv fields[14] = {
+   struct agx3_field_kv fields[16] = {
       {2,dest},{18,src},{34,0x80},{11,dest_cache},
       {27,hint == AGX3_ALU_HINT_CACHE},{29,hint == AGX3_ALU_HINT_DISCARD},
       {30,src_abs},{31,src_neg},
    };
-   for (unsigned i=0;i<6;i++)
+   const unsigned slots = m5 ? 8 : 6;
+   for (unsigned i=0;i<slots;i++)
       fields[8+i] = (struct agx3_field_kv){52+i,!!(wait_mask & (1u<<i))};
    for (unsigned ep=0;ep<4;ep++)
-      if (agx3_bind_compose(enc,"f32add",ep,base,6,fields,14,out,len)) {
+      if (agx3_bind_compose(enc,"f32add",ep,base,m5 ? 8 : 6,
+                           fields,8+slots,out,len)) {
          /* metal-nt canonicalizes modifier-specific tail bits. */
-         if (src_abs && *len == 10)
+         if (!m5 && src_abs && *len == 10)
             out[7] |= 0x80;
-         if (src_neg && *len == 8)
+         if (!m5 && src_neg && *len == 8)
             out[7] |= 0x80;
          return true;
       }
    return false;
 }
+
+/* G17G no-fast-math fmin/fmax emit condition 3/7 with selected values
+ * tied to their comparands. Do not admit those codes for an arbitrary
+ * compare/select: only this native minNum/maxNum domain is witnessed. */
+static bool
+agx3_gpu1_tied_minmax(const struct agx3_fsel32_gpu1_desc *d, bool half)
+{
+   return (d->condition == 3 || d->condition == 7) &&
+          !d->cmp1_immediate && !d->sel_immediate[0] &&
+          !d->sel_immediate[1] && d->cmp0_16 == half && d->cmp1_16 == half &&
+          d->cmp0 == d->sel[0] && d->cmp1 == d->sel[1];
+}
+
+bool
+agx3_alu_fcmp_select32_gpu1(const struct agx3_fsel32_gpu1_desc *d,
+                            uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!d || !out || !len || d->dest >= 256 || (d->dest & 1) ||
+       (d->wait_mask & ~0xffu) || (d->cmp1_16 && d->cmp1_immediate) ||
+       (d->condition != 0 && d->condition != 1 && d->condition != 2 &&
+        d->condition != 5 && d->condition != 6 &&
+        !agx3_gpu1_tied_minmax(d, false)))
+      return false;
+
+
+   if (d->sel_immediate[0] && d->sel_immediate[1] && !d->sel_integer)
+      return false;
+
+   /* Native G17G fragments compare a uniform against a register
+    * and select a minifloat or a register: fself32 with f19 = the uniform
+    * half and f25 = 3, conditions 5 and 6, no lifetime hints on the
+    * uniform (16 Minecraft terrain/entity witnesses, asserted in the M5
+    * instruction test). Only that shape is admitted. */
+   if (d->cmp0_uniform &&
+       (d->cmp0_16 || (d->cmp0 & 1) || d->cmp0 >= 256 ||
+        d->cmp0_hint != AGX3_ALU_HINT_NONE || d->cmp1_immediate ||
+        d->cmp1_16 || d->sel_integer || !d->sel_immediate[0] ||
+        d->sel_immediate[1] || (d->condition != 5 && d->condition != 6)))
+      return false;
+
+   const uint16_t src[4] = {d->cmp0, d->cmp1, d->sel[0], d->sel[1]};
+   const bool imm[4] = {false, d->cmp1_immediate, d->sel_immediate[0],
+                        d->sel_immediate[1]};
+   const enum agx3_alu_hint hints[4] = {d->cmp0_hint, d->cmp1_hint,
+                                        d->sel_hint[0], d->sel_hint[1]};
+   const bool half[4] = {d->cmp0_16, d->cmp1_16, false, false};
+   for (unsigned i = 0; i < 4; ++i) {
+      if (src[i] >= 256 || (!imm[i] && !half[i] && (src[i] & 1)) ||
+          (unsigned)hints[i] > AGX3_ALU_HINT_CACHE ||
+          (imm[i] && hints[i] != AGX3_ALU_HINT_NONE))
+         return false;
+   }
+
+
+   const bool fself = !d->sel_integer && (imm[2] || imm[3]);
+   const unsigned o2 = fself ? 2 : 0;
+   const unsigned o5 = fself ? 5 : 0;
+   const unsigned o8 = fself ? 8 : 0;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 2}, {23 + o2, d->cmp0_uniform ? 3 : 2},
+      {24 + o2, d->cmp0_16 ? 1 : 2},
+      {40 + o2, imm[1] ? 1 : 2}, {41 + o2, d->cmp1_16 ? 1 : 2},
+      {57 + o2, imm[2] ? 1 : 2}, {58 + o2, 2},
+      {71 + o5, imm[3] ? 1 : 2}, {72 + o5, 2},
+      {76 + o8, 3}, {77 + o8, fself ? 74 : 76},
+   };
+   struct agx3_field_kv fields[23] = {
+      {2, d->dest}, {11, d->dest_cache},
+      {(d->cmp0_uniform ? 17 : 16) + o2, src[0]},
+      {(imm[1] ? 32 : 33) + o2, src[1]},
+      {(imm[2] ? 49 : 50) + o2, src[2]},
+      {(imm[3] ? 63 : 64) + o5, src[3]},
+      {86 + o8, d->condition},
+   };
+   const uint16_t cache[] = {25 + o2, 42 + o2, 59 + o2, 73 + o5};
+   const uint16_t discard[] = {27 + o2, 44 + o2, 61 + o2, 75 + o5};
+   unsigned n = 7;
+   for (unsigned i = 0; i < 4; ++i) {
+      fields[n++] = (struct agx3_field_kv){
+         cache[i], hints[i] == AGX3_ALU_HINT_CACHE};
+      fields[n++] = (struct agx3_field_kv){
+         discard[i], hints[i] == AGX3_ALU_HINT_DISCARD};
+   }
+   for (unsigned i = 0; i < 8; ++i) {
+      fields[n++] = (struct agx3_field_kv){
+         78 + o8 + i, !!(d->wait_mask & (1u << i))};
+   }
+   /* G17G's second NClamp operation is the compact six-byte tied fmin,
+    * with the slot-one wait in ep0. Other general selects retain ep4. */
+   if (agx3_gpu1_tied_minmax(d, false) &&
+       agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "fseli32", 0, base, 12,
+                        fields, n, out, len))
+      return true;
+   /* The uniform-comparand witnesses use the compact fself32 endpoints
+    * (8 and 10 bytes): take the shortest that holds the fields. */
+   if (d->cmp0_uniform) {
+      uint8_t best[AGX3_ALU_MAX_LENGTH], best_len = 0;
+      for (unsigned ep = 0; ep < 8; ++ep) {
+         uint8_t bytes[AGX3_ALU_MAX_LENGTH], size;
+         if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "fself32", ep, base,
+                               12, fields, n, bytes, &size) &&
+             (!best_len || size < best_len)) {
+            memcpy(best, bytes, size);
+            best_len = size;
+         }
+      }
+      if (!best_len)
+         return false;
+      memcpy(out, best, best_len);
+      *len = best_len;
+      return true;
+   }
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,
+                            fself ? "fself32" : "fseli32", 4, base, 12,
+                            fields, n, out, len);
+}
+
+
+bool
+agx3_alu_hadd_minifloat(enum agx_isa_encoding enc, uint16_t dest,
+                      uint16_t src, uint16_t immediate, bool src_neg,
+                      enum agx3_alu_hint hint,
+                      bool dest_cache, uint16_t wait_mask,
+                      uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !out || !len || dest >= 256 ||
+       src >= 256 || immediate > UINT8_MAX ||
+       (!m5 && (immediate != UINT8_C(0x30) || !src_neg)) ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE ||
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
+      return false;
+
+
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {24, 1}, {25, 1}, {41, 2}, {42, 1},
+   };
+   struct agx3_field_kv fields[17] = {
+      {2, dest},
+      {16, immediate},
+      {34, src},
+      {47, src_neg},
+      {11, dest_cache},
+      {43, hint == AGX3_ALU_HINT_CACHE},
+      {45, hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 7;
+   if (m5) {
+      fields[n++] = (struct agx3_field_kv){49, 3};
+      fields[n++] = (struct agx3_field_kv){50, 41};
+   }
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot)
+      fields[n++] = (struct agx3_field_kv){51 + slot,
+                                           !!(wait_mask & (1u << slot))};
+
+   for (unsigned ep = 1; ep <= (m5 ? 3 : 1); ++ep) {
+      if (agx3_bind_compose(enc, "f16add", ep, base, 6, fields, n, out, len))
+         return true;
+   }
+   return false;
+}
+
+bool
+agx3_alu_m5_ldimm(uint16_t dest, bool half, bool dest_cache, uint32_t bits,
+                  uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                  uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || (!half && (dest & 1)) ||
+       (half && bits > UINT16_MAX) || wait_mask > 0xff)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, half ? 1 : 2}, {15, 3}, {16, 105},
+   };
+   struct agx3_field_kv fields[11] = {
+      {2, dest}, {11, dest_cache}, {14, bits},
+   };
+   for (unsigned slot = 0; slot < 8; ++slot)
+      fields[3 + slot] = (struct agx3_field_kv){17 + slot,
+                                               !!(wait_mask & (1u << slot))};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "ldimm", 2, base, 4,
+                            fields, 11, out, len);
+}
+
+bool
+agx3_alu_fcmp_select16_gpu1(const struct agx3_fsel32_gpu1_desc *d,
+                            uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!d || !out || !len || d->dest >= 256 || (d->wait_mask & ~0xffu) ||
+       d->cmp0_uniform ||
+       (d->condition != 0 && d->condition != 1 && d->condition != 2 &&
+        d->condition != 5 && d->condition != 6 &&
+        !agx3_gpu1_tied_minmax(d, true)))
+      return false;
+
+   const uint16_t src[4] = {d->cmp0, d->cmp1, d->sel[0], d->sel[1]};
+   const bool imm[4] = {false, d->cmp1_immediate, d->sel_immediate[0],
+                        d->sel_immediate[1]};
+   const enum agx3_alu_hint hints[4] = {d->cmp0_hint, d->cmp1_hint,
+                                        d->sel_hint[0], d->sel_hint[1]};
+
+   if (d->cmp1_16 && d->cmp1_immediate)
+      return false;
+   const bool half[4] = {d->cmp0_16, d->cmp1_16, true, true};
+   for (unsigned i = 0; i < 4; ++i) {
+
+      if (src[i] >= 256 || (!imm[i] && !half[i] && (src[i] & 1)) ||
+          (unsigned)hints[i] > AGX3_ALU_HINT_CACHE ||
+          (imm[i] && hints[i] != AGX3_ALU_HINT_NONE))
+         return false;
+   }
+
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {23, 2}, {24, d->cmp0_16 ? 1 : 2},
+      {40, imm[1] ? 1 : 2}, {41, d->cmp1_16 ? 1 : 2},
+      {57, imm[2] ? 1 : 2}, {58, 1},
+      {71, imm[3] ? 1 : 2}, {72, 1},
+      {76, 3}, {77, 75},
+   };
+   struct agx3_field_kv fields[23] = {
+      {2, d->dest}, {11, d->dest_cache}, {16, src[0]},
+      {imm[1] ? 32 : 33, src[1]},
+      {imm[2] ? 49 : 50, src[2]},
+      {imm[3] ? 63 : 64, src[3]},
+      {86, d->condition},
+   };
+   static const uint16_t cache[] = {25, 42, 59, 73};
+   static const uint16_t discard[] = {27, 44, 61, 75};
+   unsigned n = 7;
+   for (unsigned i = 0; i < 4; ++i) {
+      fields[n++] = (struct agx3_field_kv){
+         cache[i], hints[i] == AGX3_ALU_HINT_CACHE};
+      fields[n++] = (struct agx3_field_kv){
+         discard[i], hints[i] == AGX3_ALU_HINT_DISCARD};
+   }
+   for (unsigned i = 0; i < 8; ++i)
+      fields[n++] = (struct agx3_field_kv){78 + i, !!(d->wait_mask & (1u << i))};
+   for (unsigned ep = 0; ep < 16; ++ep)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "fseli16", ep, base,
+                             sizeof(base) / sizeof(base[0]), fields, n, out, len))
+         return true;
+   return false;
+}
+
+bool
+agx3_alu_iadd16_gpu1(uint16_t dest, bool dest_cache, bool src0_immediate,
+                     uint16_t src0, enum agx3_alu_hint src0_hint,
+                     uint16_t src1, enum agx3_alu_hint src1_hint,
+                     unsigned shift, uint16_t wait_mask,
+                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+
+   if (!out || !len || dest >= 256 || src1 >= 256 || shift > 4 ||
+       (src0_immediate ? src0 > 255 : src0 >= 256) ||
+       (unsigned)src0_hint > AGX3_ALU_HINT_CACHE ||
+       (unsigned)src1_hint > AGX3_ALU_HINT_CACHE || wait_mask > 0xff)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {24, 1}, {38, 2}, {39, 1}, {44, 3}, {45, 80},
+   };
+   struct agx3_field_kv ov[20];
+   unsigned n = 0;
+   ov[n++] = (struct agx3_field_kv){2, dest};
+   ov[n++] = (struct agx3_field_kv){11, dest_cache ? 1 : 0};
+   if (src0_immediate) {
+      ov[n++] = (struct agx3_field_kv){23, 1};
+      ov[n++] = (struct agx3_field_kv){15, src0};
+   } else {
+      ov[n++] = (struct agx3_field_kv){23, 2};
+      ov[n++] = (struct agx3_field_kv){16, src0};
+      ov[n++] = (struct agx3_field_kv){25, src0_hint == AGX3_ALU_HINT_CACHE};
+      ov[n++] = (struct agx3_field_kv){27, src0_hint == AGX3_ALU_HINT_DISCARD};
+   }
+   ov[n++] = (struct agx3_field_kv){31, src1};
+   ov[n++] = (struct agx3_field_kv){40, src1_hint == AGX3_ALU_HINT_CACHE};
+   ov[n++] = (struct agx3_field_kv){42, src1_hint == AGX3_ALU_HINT_DISCARD};
+   ov[n++] = (struct agx3_field_kv){55, shift};
+   for (unsigned i = 0; i < 8; ++i)
+      if (wait_mask & (1u << i))
+         ov[n++] = (struct agx3_field_kv){46 + i, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "iadd32", 0, base, 7, ov,
+                            n, out, len);
+}
+
+
+bool
+agx3_alu_isub_imm16_m5(uint16_t dest, uint16_t src, uint16_t immediate,
+                      enum agx3_alu_hint hint, bool dest_cache,
+                      uint16_t wait_mask,
+                      uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len || dest >= 256 || src >= 256 || immediate > 255 ||
+       (unsigned)hint > AGX3_ALU_HINT_CACHE || wait_mask > 255)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9,2}, {10,1}, {23,2}, {24,1}, {38,1}, {39,1}, {44,3}, {45,100},
+   };
+   struct agx3_field_kv fields[14] = {
+      {2,dest}, {11,dest_cache}, {16,src}, {30,immediate},
+      {25,hint == AGX3_ALU_HINT_CACHE}, {27,hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 6;
+   for (unsigned i=0; i<8; ++i)
+      if (wait_mask & (1u << i))
+         fields[n++] = (struct agx3_field_kv){46+i,1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "isub32", 0,
+                            base, 8, fields, n, out, len);
+}
+
+static bool
+agx3_alu_bit_widen_m5(enum agx3_alu_op op, bool first_half, uint16_t dest, uint16_t word, uint16_t half,
+                     enum agx3_alu_hint word_hint,
+                     enum agx3_alu_hint half_hint, bool dest_cache,
+                     uint16_t wait_mask,
+                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   if ((op != AGX3_ALU_BAND && op != AGX3_ALU_BOR && op != AGX3_ALU_BXOR) || !out || !len || dest >= 256 || word >= 256 || half >= 256 ||
+       ((dest & 1) || (!first_half && (word & 1))) || (unsigned)word_hint > AGX3_ALU_HINT_CACHE ||
+       (unsigned)half_hint > AGX3_ALU_HINT_CACHE || wait_mask > 255)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9,2}, {10,2}, {23,2}, {24,first_half ? 1 : 2}, {37,2}, {38,1}, {42,3}, {43,op == AGX3_ALU_BAND ? 10 : op == AGX3_ALU_BOR ? 16 : 19},
+   };
+   struct agx3_field_kv fields[16] = {
+      {2,dest}, {11,dest_cache}, {16,word}, {30,half},
+      {25,word_hint == AGX3_ALU_HINT_CACHE}, {27,word_hint == AGX3_ALU_HINT_DISCARD},
+      {39,half_hint == AGX3_ALU_HINT_CACHE}, {41,half_hint == AGX3_ALU_HINT_DISCARD},
+   };
+   unsigned n = 8;
+   for (unsigned i=0; i<8; ++i)
+      if (wait_mask & (1u << i))
+         fields[n++] = (struct agx3_field_kv){44+i,1};
+   for (unsigned ep=0; ep<2; ++ep)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, op == AGX3_ALU_BAND ? "bit_and" : op == AGX3_ALU_BOR ? "bit_or" : "bit_xor", ep,
+                             base, 8, fields, n, out, len))
+         return true;
+   return false;
+}
+
+/* Metal's ushort2 -> uint3 AND/OR/XOR witness uses a word destination
+ * (field 10 = 2) and two half sources (fields 24 and 38 = 1). */
+bool
+agx3_alu_bit_half_to_word_m5(enum agx3_alu_op op, uint16_t dest,
+                            const uint16_t src[2],
+                            const enum agx3_alu_hint hints[2], bool dest_cache,
+                            uint16_t wait_mask,
+                            uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return src && hints && agx3_alu_bit_widen_m5(
+      op, true, dest, src[0], src[1], hints[0], hints[1], dest_cache,
+      wait_mask, out, len);
+}
+
+bool
+agx3_alu_bit_mixed_m5(enum agx3_alu_op op, uint16_t dest, uint16_t word,
+                     uint16_t half, enum agx3_alu_hint word_hint,
+                     enum agx3_alu_hint half_hint, bool dest_cache,
+                     uint16_t wait_mask,
+                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_alu_bit_widen_m5(op, false, dest, word, half, word_hint,
+                               half_hint, dest_cache, wait_mask, out, len);
+}
+
+bool
+agx3_alu_and_mixed_m5(uint16_t dest, uint16_t word, uint16_t half,
+                     enum agx3_alu_hint word_hint,
+                     enum agx3_alu_hint half_hint, bool dest_cache,
+                     uint16_t wait_mask,
+                     uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_alu_bit_mixed_m5(AGX3_ALU_BAND, dest, word, half, word_hint,
+                               half_hint, dest_cache, wait_mask, out, len);
+}
+
+bool
+agx3_alu_hfma_immediates_m5(uint16_t dest, uint16_t src,
+   uint8_t multiplier, uint8_t addend, enum agx3_alu_hint hint,
+   bool dest_cache, uint8_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+   uint8_t *len)
+{
+   if (dest >= 256 || src >= 256 || hint > AGX3_ALU_HINT_CACHE || !out || !len)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9,2},{10,1},{24,2},{25,1},{41,1},{42,1},{58,1},{59,1},{66,3},{67,42},
+   };
+   struct agx3_field_kv fields[15] = {
+      {2,dest},{17,src},{33,multiplier},{50,addend},{11,dest_cache},
+      {26,hint == AGX3_ALU_HINT_CACHE},{28,hint == AGX3_ALU_HINT_DISCARD},
+   };
+   for (unsigned i=0;i<8;++i)
+      fields[7+i]=(struct agx3_field_kv){68+i,!!(wait_mask & (1u<<i))};
+   for (unsigned ep=2;ep<6;++ep)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,"f16fma",ep,
+                           base,10,fields,15,out,len))
+         return true;
+   return false;
+}
diff --git a/src/asahi/compiler/agx3_alu_core.h b/src/asahi/compiler/agx3_alu_core.h
index a16c9c9112e8..01af3aa4f840 100644
--- a/src/asahi/compiler/agx3_alu_core.h
+++ b/src/asahi/compiler/agx3_alu_core.h
@@ -16,6 +16,7 @@
 #include <stdint.h>
 #include "asahi/lib/agx_chip.h"
 #include "agx3_alu_table.h"
+#include "agx3_funop_core.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -32,8 +33,30 @@ enum agx3_alu_hint {
    AGX3_ALU_HINT_CACHE = 2,
 };
 
+/* Native C scalar rounding/log/exp with independent 16/32-bit widths,
+ * source lifetime, modifiers, saturation and producer waits. */
+bool agx3_alu_unary_m5(enum agx3_funop op, uint16_t dest, uint16_t src,
+                       bool dest_half, bool src_half, bool src_abs,
+                       bool src_neg, bool saturate, enum agx3_alu_hint hint,
+                       bool dest_cache, uint16_t wait_mask,
+                       uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+/* Native C fragment gradients (f32dfdx/f32dfdy) over the whole register
+ * file, at 32 or 16 bits. */
+bool agx3_alu_derivative_m5(bool dfdy, uint16_t dest, uint16_t src, bool half,
+                            bool src_abs, bool src_neg, bool saturate,
+                            uint16_t wait_mask,
+                            uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /* Native G17P population count of a scalar ushort or uint, with post-RA
  * source lifetime and producer waits. */
+/* Native M5 bit reversal. A half source is zero-extended before reversing
+ * all 32 bits, so its reversed bits occupy the high half of the result. */
+bool agx3_alu_bitrev_m5(uint16_t dest, uint16_t src, bool src_half,
+                        enum agx3_alu_hint hint, bool dest_cache,
+                        uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                        uint8_t *len);
+
 bool agx3_alu_popcount(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                        bool half, enum agx3_alu_hint hint, bool dest_cache,
                        uint16_t wait_mask,
@@ -69,6 +92,18 @@ bool agx3_alu_bit_reg16(enum agx_isa_encoding enc, enum agx3_alu_op op,
                           const enum agx3_alu_hint hints[2], uint16_t wait_mask,
                           uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+bool agx3_alu_and_imm16_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+                         bool complement, bool immediate_first,
+                         enum agx3_alu_hint hint,
+                         bool dest_cache, uint16_t wait_mask,
+                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+bool agx3_alu_complement_reg16_m5(uint16_t dest, const uint16_t src[2],
+                                const enum agx3_alu_hint hints[2],
+                                bool disjunction, bool dest_cache,
+                                uint16_t wait_mask,
+                                uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /*
  * The same AND/OR/XOR with the second operand an IMMEDIATE. The operand kind
  * is a field, f37, on the very encoding point the register form already uses,
@@ -114,6 +149,12 @@ bool agx3_alu_bit_not_reg16(enum agx_isa_encoding enc, uint16_t dest,
                             bool dest_cache, uint16_t wait_mask,
                             uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+/* AGX3C `inv` on a half or a 32-bit register. */
+bool agx3_alu_bit_not_m5(uint16_t dest, uint16_t src, bool word,
+                         enum agx3_alu_hint hint, bool dest_cache,
+                         uint16_t wait_mask,
+                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /* G17P 32-bit a & ~b.  Apple exposes this truth-table shape as bit_andc;
  * unlike AND it is not commutative, so the source order is preserved. */
 bool agx3_alu_bit_andc(enum agx_isa_encoding enc, uint16_t dest,
@@ -160,6 +201,13 @@ bool agx3_alu_add_sat16(enum agx_isa_encoding enc, uint16_t dest,
                         bool dest_cache, uint16_t wait_mask,
                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+/* AGX3C saturating 16-bit add/subtract (iadd32/isub32 with a half result). */
+bool agx3_alu_addsub_sat16_m5(uint16_t dest, const uint16_t src[2],
+                              const enum agx3_alu_hint hints[2],
+                              bool is_signed, bool subtract, bool dest_cache,
+                              uint16_t wait_mask,
+                              uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /* Native scalar-half register subtraction, src[0] - src[1]. */
 bool agx3_alu_sub16(enum agx_isa_encoding enc, uint16_t dest,
                     const uint16_t src[2], const enum agx3_alu_hint hints[2],
@@ -185,6 +233,13 @@ bool agx3_alu_sll_one_reg32(enum agx_isa_encoding enc, uint16_t dest,
                             bool dest_cache, uint16_t wait_mask,
                             uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+/* AGX3C `1 << uint_register`: sll with immediate one and a 32-bit count, to
+ * a half or 32-bit destination. */
+bool agx3_alu_sll_one_m5(uint16_t dest, bool dest_16, uint16_t shift,
+                         enum agx3_alu_hint shift_hint, bool dest_cache,
+                         uint16_t wait_mask,
+                         uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /* G17P logical shift-right with register value and signed boundary. */
 bool agx3_alu_srl_reg(enum agx_isa_encoding enc, uint16_t dest, bool dest_16,
                       bool dest_cache, const uint16_t src[2],
@@ -227,7 +282,7 @@ bool agx3_alu_integer_reg_shift_cache(enum agx_isa_encoding enc,
 /* Native signed/unsigned32 to float32 with post-RA source hints. */
 bool agx3_alu_int_to_float(enum agx_isa_encoding enc, uint16_t dest,
                             uint16_t src, bool is_signed,
-                            enum agx3_alu_hint hint, uint16_t wait_mask,
+                            enum agx3_alu_hint hint, bool dest_cache, uint16_t wait_mask,
                             uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
 /* Native signed/unsigned f32-to-i32 with post-RA source and destination cache
@@ -565,12 +620,13 @@ bool agx3_alu_fmul_minifloat_mod(
  * optimizer folds that f2f16 into the multiply, so this mixed-width form is
  * what actually reaches the packer. See the witnesses in agx3_alu_core.c.
  *
- * An absolute source is refused for either destination width. The six-byte
- * endpoint has no grounded fused ABS form; use explicit FABS plus plain FMUL.
+ * G17P refuses ABS and saturation. G17G carries them directly in its longer
+ * endpoint, independently of the result width and producer dependency waits.
  */
 bool agx3_alu_fmul_minifloat_narrow(
    enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    uint16_t immediate, bool dest_cache, bool src_abs, bool dest_half,
+   bool saturate,
    enum agx3_alu_hint hint, uint16_t wait_mask,
    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
@@ -579,9 +635,22 @@ bool agx3_alu_fmul_half_minifloat_to_f32(
    enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    uint16_t immediate, enum agx3_alu_hint hint, uint16_t wait_mask,
    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+/* G17G scalar-half times minifloat with a float32 result: the f32mul
+ * minifloat form with a half source (f43=1), carrying ABS, saturation,
+ * operand-cache hints and eight wait slots like the float-source form. */
+bool agx3_alu_fmul_half_minifloat_m5(
+   uint16_t dest, uint16_t src, uint16_t immediate, bool dest_cache,
+   bool src_abs, bool saturate, enum agx3_alu_hint hint, uint16_t wait_mask,
+   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 /* Unsigned G17P extract into a zero base, with a fixed1..31-bit width.
  * Native srl performs shift+mask in one instruction. Dynamic shifts and
  * immediate14, dead/live sources and slot6 waits are native witnesses. */
+bool agx3_alu_bfeil_m5(uint16_t dest, uint16_t source, bool source_16,
+                       uint16_t shift, bool shift_is_register, bool shift_16,
+                       uint8_t width, const enum agx3_alu_hint hints[2],
+                       uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                       uint8_t *length);
 bool agx3_alu_bfeil(enum agx_isa_encoding enc, uint16_t dest, uint16_t source,
                     uint16_t shift, bool shift_is_register, uint8_t width,
                     const enum agx3_alu_hint hints[2], uint16_t wait_mask,
@@ -596,6 +665,17 @@ bool agx3_alu_extract_imm32(enum agx_isa_encoding enc, uint16_t dest,
                             uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *length);
 
 /* Signed right shifts by an immediate, with native source lifetime/waits. */
+bool agx3_alu_asr_reg_m5(uint16_t dest, uint16_t source, uint16_t shift,
+                       bool dest_16, bool source_16,
+                       const enum agx3_alu_hint hints[2], bool dest_cache,
+                       uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                       uint8_t *length);
+
+bool agx3_alu_asr_imm_m5(uint16_t dest, uint16_t source, bool dest_16,
+                          bool source_16, unsigned shift,
+                          enum agx3_alu_hint hint, bool dest_cache,
+                          uint16_t wait_mask,
+                          uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *length);
 bool agx3_alu_asr_imm32(enum agx_isa_encoding enc, uint16_t dest, uint16_t source,
                         unsigned shift, enum agx3_alu_hint hint, bool dest_cache,
                         uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
@@ -804,6 +884,11 @@ bool agx3_alu_f32ceil(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                       uint16_t wait_mask, bool src_neg, bool src_abs,
                       uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+bool agx3_alu_f32trunc(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+                       enum agx3_alu_hint hint, bool dest_cache,
+                       uint16_t wait_mask, bool src_neg, bool src_abs,
+                       uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /*
  * Round-to-nearest-even is the same form with f43 absent, the first line of
  * the witness table above.
@@ -938,7 +1023,13 @@ bool agx3_alu_hadd_saturate_negzero(
    enum agx3_alu_hint hint, bool dest_cache, uint16_t wait_mask,
    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
-/* Native -src + -0.0h used by half negate and matrix inverse lowering. */
+/* Native -src + -0.0h used by half negate and matrix inverse lowering, and
+ * the same form with saturation for saturate(half). */
+bool agx3_alu_half_add_negzero_m5(
+   uint16_t dest, uint16_t src, bool src_abs, bool src_neg, bool saturate,
+   enum agx3_alu_hint hint, bool dest_cache, uint16_t wait_mask,
+   uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 bool agx3_alu_hadd_neg_negzero(
    enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    enum agx3_alu_hint hint, bool dest_cache, uint16_t wait_mask,
@@ -956,6 +1047,11 @@ bool agx3_alu_hfma(enum agx_isa_encoding enc, uint16_t dest,
                    uint16_t wait_mask,
                    uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+bool agx3_alu_hfma_saturate_m5(uint16_t dest, const uint16_t src[3],
+              const enum agx3_alu_hint hints[3], bool product_neg,
+              bool src2_neg, bool dest_cache, uint16_t wait_mask,
+              uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
 /* Native scalar half FMA with an 8-bit minifloat addend. */
 bool agx3_alu_hfma_addend_minifloat(
    enum agx_isa_encoding enc, uint16_t dest, const uint16_t src[2],
@@ -967,6 +1063,107 @@ bool agx3_alu_widen_half(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                          bool dest_cache, uint16_t wait_mask,
                          uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
 
+struct agx3_fsel32_gpu1_desc {
+   uint16_t dest;
+   bool dest_cache;
+
+   uint16_t cmp0;
+   bool cmp0_16;
+   enum agx3_alu_hint cmp0_hint;
+   /* cmp0 names an aligned 32-bit uniform (fself32 only, see the encoder). */
+   bool cmp0_uniform;
+
+   bool cmp1_immediate;
+   uint16_t cmp1;
+   enum agx3_alu_hint cmp1_hint;
+
+   bool cmp1_16;
+
+
+   bool sel_immediate[2];
+   uint16_t sel[2];
+   enum agx3_alu_hint sel_hint[2];
+   bool sel_integer;
+
+   unsigned condition;
+   uint16_t wait_mask;
+};
+
+bool agx3_alu_fcmp_select32_gpu1(const struct agx3_fsel32_gpu1_desc *d,
+                                 uint8_t out[AGX3_ALU_MAX_LENGTH],
+                                 uint8_t *len);
+
+bool agx3_alu_hadd_minifloat(enum agx_isa_encoding enc, uint16_t dest,
+                            uint16_t src, uint16_t immediate, bool src_neg,
+                            enum agx3_alu_hint hint, bool dest_cache,
+                            uint16_t wait_mask,
+                            uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+bool agx3_alu_m5_ldimm(uint16_t dest, bool half, bool dest_cache, uint32_t bits,
+                       uint16_t wait_mask,
+                       uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+bool agx3_alu_fcmp_select16_gpu1(const struct agx3_fsel32_gpu1_desc *d,
+                                 uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+bool agx3_alu_iadd16_gpu1(uint16_t dest, bool dest_cache, bool src0_immediate,
+                          uint16_t src0, enum agx3_alu_hint src0_hint,
+                          uint16_t src1, enum agx3_alu_hint src1_hint,
+                          unsigned shift, uint16_t wait_mask,
+                          uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+bool agx3_alu_isub_imm16_m5(uint16_t dest, uint16_t src, uint16_t immediate,
+                           enum agx3_alu_hint hint, bool dest_cache,
+                           uint16_t wait_mask,
+                           uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+bool agx3_alu_bit_half_to_word_m5(enum agx3_alu_op op, uint16_t dest,
+                                  const uint16_t src[2],
+                                  const enum agx3_alu_hint hints[2], bool dest_cache,
+                                  uint16_t wait_mask,
+                                  uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+bool agx3_alu_bit_mixed_m5(enum agx3_alu_op op, uint16_t dest, uint16_t word, uint16_t half,
+                          enum agx3_alu_hint word_hint,
+                          enum agx3_alu_hint half_hint, bool dest_cache,
+                          uint16_t wait_mask,
+                          uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+bool agx3_alu_and_mixed_m5(uint16_t dest, uint16_t word, uint16_t half,
+                          enum agx3_alu_hint word_hint,
+                          enum agx3_alu_hint half_hint, bool dest_cache,
+                          uint16_t wait_mask,
+                          uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+
+bool agx3_alu_bit_andc_width_m5(uint16_t dest, const uint16_t src[2],
+                                const bool half[2],
+                                const enum agx3_alu_hint hints[2],
+                                bool dest_cache, uint16_t wait_mask,
+                                uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+
+bool agx3_alu_hadd_signed(enum agx_isa_encoding enc, uint16_t dest,
+                            const uint16_t src[2], const enum agx3_alu_hint hints[2],
+                            const bool src_neg[2], bool dest_cache,
+                            uint16_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+                            uint8_t *len);
+
+/* Scalar half FMA with a register and two minifloat operands. */
+bool agx3_alu_hfma_immediates_m5(uint16_t dest, uint16_t src,
+   uint8_t multiplier, uint8_t addend, enum agx3_alu_hint hint,
+   bool dest_cache, uint8_t wait_mask, uint8_t out[AGX3_ALU_MAX_LENGTH],
+   uint8_t *len);
+
+/* M5 scalar-half native modifier forms. */
+bool agx3_alu_hadd_mod_m5(uint16_t dest, const uint16_t src[2],
+                          const enum agx3_alu_hint hints[2],
+                          const bool src_abs[2], const bool src_neg[2],
+                          bool dest_cache, bool saturate, uint16_t wait_mask,
+                          uint8_t out[AGX3_ALU_MAX_LENGTH], uint8_t *len);
+bool agx3_alu_half_add_imm_to_f32_m5(uint16_t dest, uint16_t src,
+                                     uint8_t immediate, bool src_abs,
+                                     bool src_neg, enum agx3_alu_hint hint,
+                                     bool dest_cache, uint16_t wait_mask,
+                                     uint8_t out[AGX3_ALU_MAX_LENGTH],
+                                     uint8_t *len);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/asahi/compiler/agx3_atomic_core.c b/src/asahi/compiler/agx3_atomic_core.c
index 31910ced64b9..617a3f962f44 100644
--- a/src/asahi/compiler/agx3_atomic_core.c
+++ b/src/asahi/compiler/agx3_atomic_core.c
@@ -206,13 +206,12 @@ agx3_atomic_encode_local(enum agx_isa_encoding enc,
            d->address_kind != AGX3_ATOMIC_INDEX_REGISTER) ||
           (d->address_kind == AGX3_ATOMIC_INDEX_IMMEDIATE &&
            (d->address & 3) != 0) ||
-          (d->address_kind == AGX3_ATOMIC_INDEX_REGISTER &&
-           (enc != AGX_ISA_ENCODING_AGX3B || d->address > 191)) ||
+          (d->address_kind == AGX3_ATOMIC_INDEX_REGISTER && d->address > 191) ||
           d->address_kill)
          return false;
    } else {
-      /* The no-result and addressed forms are isolated on AGX3b. */
-      if (enc != AGX_ISA_ENCODING_AGX3B || d->slot != 0)
+      /* M5 also has a no-result form at an immediate byte address. */
+      if (d->slot != 0)
          return false;
 
       if (d->address_kind == AGX3_ATOMIC_INDEX_IMMEDIATE) {
@@ -262,13 +261,15 @@ agx3_atomic_encode_local(enum agx_isa_encoding enc,
          enc == AGX_ISA_ENCODING_AGX3C ? 83 : 81, 1);
 
    if (enc == AGX_ISA_ENCODING_AGX3C) {
-      f[n++] = (struct agx3_field_kv)KV(11, 1);
+      if (d->returns_value)
+         f[n++] = (struct agx3_field_kv)KV(11, 1);
       f[n++] = (struct agx3_field_kv)KV(59, 15);
       f[n++] = (struct agx3_field_kv)KV(67, 1);
       f[n++] = (struct agx3_field_kv)KV(73, 7);
       f[n++] = (struct agx3_field_kv)KV(
          74, d->op == AGX3_ATOMIC_CMPXCHG ? 3 : 1);
-      f[n++] = (struct agx3_field_kv)KV(75, 1);
+      if (d->returns_value)
+         f[n++] = (struct agx3_field_kv)KV(75, 1);
       f[n++] = (struct agx3_field_kv)KV(95, 2);
       f[n++] = (struct agx3_field_kv)KV(98, 1);
    }
diff --git a/src/asahi/compiler/agx3_cf_core.c b/src/asahi/compiler/agx3_cf_core.c
index ab1e02fd059a..477a9e75a837 100644
--- a/src/asahi/compiler/agx3_cf_core.c
+++ b/src/asahi/compiler/agx3_cf_core.c
@@ -72,6 +72,34 @@ agx3_cf_icmp_reg16(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
    return false;
 }
 
+bool
+agx3_cf_icmp_width_m5(enum agx3_cf_icond cond, unsigned slot,
+                      const uint16_t src[2], const bool half[2],
+                      const struct agx3_cf_source_hints hints[2],
+                      uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
+{
+   if (!icond_ok(cond) || !slot_ok(slot) || !src || !half || !hints || !out || !len)
+      return false;
+   for (unsigned i = 0; i < 2; ++i)
+      if (src[i] >= 256 || (!half[i] && (src[i] & 1)) ||
+          (hints[i].cache && hints[i].discard))
+         return false;
+   const bool sign = cond == AGX3_CF_ICOND_SLT || cond == AGX3_CF_ICOND_SGT;
+   const struct agx3_field_kv base[] = {
+      KV(9,2), KV(10,half[0]?1:2), KV(24,2), KV(25,half[1]?1:2),
+      KV(32,3), KV(33,82),
+   };
+   const struct agx3_field_kv fields[] = {
+      KV(2,src[0]), KV(17,src[1]), KV(31,slot), KV(42,cond),
+      KV(11,hints[0].cache), KV(13,hints[0].discard), KV(14,sign && half[0]),
+      KV(26,hints[1].cache), KV(28,hints[1].discard), KV(29,sign && half[1]),
+   };
+   for (unsigned ep = 0; ep < 3; ++ep)
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,"icmp",ep,base,6,fields,10,out,len))
+         return true;
+   return false;
+}
+
 /*
  * The source fields of every comparison form are HALF-register indices with
  * eight bits, so halves 0..255 -- all 128 architectural 32-bit registers --
@@ -122,24 +150,26 @@ agx3_cf_icmp_reg(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
  * point as the register/register compare:
  *   fs-apple.bin +0xbc  0a832307060007020000  i == u6, cached i, slot 0
  *   cs-apple.bin +0x9e  0a03230b060007020000  i == u10, slot 0
- * f13 is the register's discard hint, as on the register forms. */
+ * f13 is the register's discard hint, as on the register forms. The fresh
+ * G17G native ALU probe at main+0x9a uses this shape with f32=3/f33=82. */
 bool
 agx3_cf_icmp_reg_uniform32(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
                            unsigned slot, uint16_t src0, uint16_t uniform,
                            struct agx3_cf_source_hints hint,
                            uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
 {
-   if (!agx3b(enc) || !icond_ok(cond) || !slot_ok(slot) || src0 >= 256 ||
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       !icond_ok(cond) || !slot_ok(slot) || src0 >= 256 ||
        (src0 & 1) || uniform >= 256 || (uniform & 1) ||
        (hint.cache && hint.discard) || !out || !len)
       return false;
    const struct agx3_field_kv base[] = {KV(9, 2), KV(10, 2), KV(24, 3),
-                                        KV(25, 2)};
+                                        KV(25, 2), KV(32, 3), KV(33, 82)};
    const struct agx3_field_kv ov[] = {KV(2, src0), KV(18, uniform),
                                       KV(31, slot), KV(42, cond),
                                       KV(11, hint.cache),
                                       KV(13, hint.discard)};
-   return agx3_bind_compose(enc, "icmp", 1, base, 4, ov, 6, out, len);
+   return agx3_bind_compose(enc, "icmp", 1, base, agx3b(enc) ? 4 : 6, ov, 6, out, len);
 }
 
 /* An aligned 32-bit uniform against an 8-bit immediate: Apple's G17P loop
@@ -148,20 +178,22 @@ agx3_cf_icmp_reg_uniform32(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
  * the first operand's class f9 = 3 selects the uniform, whose half index is
  * f3 (the register form's f2 is absent):
  *   fs-apple.bin +0x0c  0a072380060087020000  u6 == 0, slot 0
- * AGX3B only; no AGX3c witness. */
+ * G17G native ALU main+0x18 witnesses the same operands with its M5
+ * f32=3/f33=82 selectors. */
 bool
 agx3_cf_icmp_uniform_imm(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
                          unsigned slot, uint16_t uniform, uint8_t imm,
                          uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
 {
-   if (!agx3b(enc) || !icond_ok(cond) || !slot_ok(slot) || uniform >= 256 ||
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       !icond_ok(cond) || !slot_ok(slot) || uniform >= 256 ||
        (uniform & 1) || !out || !len)
       return false;
    const struct agx3_field_kv base[] = {KV(9, 3), KV(10, 2), KV(24, 1),
-                                        KV(25, 1)};
+                                        KV(25, 1), KV(32, 3), KV(33, 82)};
    const struct agx3_field_kv ov[] = {KV(3, uniform), KV(16, imm),
                                       KV(42, cond), KV(31, slot)};
-   return agx3_bind_compose(enc, "icmp", 1, base, 4, ov, 4, out, len);
+   return agx3_bind_compose(enc, "icmp", 1, base, agx3b(enc) ? 4 : 6, ov, 4, out, len);
 }
 
 bool
@@ -187,13 +219,28 @@ agx3_cf_icmp_imm16_runtime(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
                            struct agx3_cf_source_hints hint, uint16_t wait_mask,
                            uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
 {
-   if (!agx3b(enc) || !icond_ok(cond) || !slot_ok(slot) || src >= 256 ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!agx3b(enc) && !gpu1) || !icond_ok(cond) || !slot_ok(slot) ||
+       src >= 256 ||
        !out || !len || (hint.cache && hint.discard) ||
-       (wait_mask != 0 && wait_mask != 32))
+       (gpu1 ? wait_mask > 255 : (wait_mask != 0 && wait_mask != 32)))
+      return false;
+   if (gpu1) {
+      const bool sign = cond == AGX3_CF_ICOND_SLT || cond == AGX3_CF_ICOND_SGT;
+      const struct agx3_field_kv base[] = {
+         KV(9,2), KV(10,1), KV(14,sign), KV(24,1), KV(25,1), KV(32,3), KV(33,82),
+      };
+      struct agx3_field_kv fields[14] = {
+         KV(2,src), KV(16,imm), KV(42,cond), KV(31,slot),
+         KV(11,hint.cache), KV(13,hint.discard),
+      };
+      for (unsigned i = 0; i < 8; ++i)
+         fields[6+i] = (struct agx3_field_kv)KV(34+i, !!(wait_mask & (1u << i)));
+      for (unsigned ep = 0; ep < 3; ++ep)
+         if (agx3_bind_compose(enc,"icmp",ep,base,7,fields,14,out,len))
+            return true;
       return false;
-   /* Unsigned ordered comparisons against nonzero constants use the compact
-    * form. Unlike the ten-byte signed/equality form, it has no source
-    * lifetime fields. Fresh dead/live witnesses produce identical bytes. */
+   }
    if (imm != 0 &&
        (cond == AGX3_CF_ICOND_ULT || cond == AGX3_CF_ICOND_UGT)) {
       const struct agx3_field_kv base[] = {
@@ -206,17 +253,14 @@ agx3_cf_icmp_imm16_runtime(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
       return agx3_bind_compose(enc, "icmp", 0, base, 4, fields, 5, out,
                                len);
    }
-   /* Native signed and unsigned zero tests both sign-extend the half.
-    * This preserves equality and every relation to zero. Live witnesses
-    * retain the source for the taken branch instead of discarding it. */
    const struct agx3_field_kv base[] = {
-      KV(9,2), KV(10,1), KV(14,1), KV(24,1), KV(25,1),
+      KV(9,2), KV(10,1), KV(14,1), KV(24,1), KV(25,1), KV(32,3), KV(33,82),
    };
    const struct agx3_field_kv fields[] = {
       KV(2,src), KV(16,imm), KV(42,cond), KV(31,slot), KV(11,hint.cache),
       KV(13,hint.discard), KV(39,wait_mask == 32),
    };
-   return agx3_bind_compose(enc,"icmp",1,base,5,fields,7,out,len);
+   return agx3_bind_compose(enc,"icmp",1,base,gpu1 ? 7 : 5,fields,7,out,len);
 }
 
 /*
@@ -271,9 +315,11 @@ agx3_cf_fcmp_reg_neg(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
                      const struct agx3_cf_source_hints hints_in[2],
                      uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src_in || !neg_in || !hints_in ||
-       !out || !len || !slot_ok(slot) || or_nan ||
-       (cond != AGX3_CF_FCOND_LT && cond != AGX3_CF_FCOND_GT) ||
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) || !src_in || !neg_in || !hints_in ||
+       !out || !len || !slot_ok(slot) ||
+       (or_nan && enc != AGX_ISA_ENCODING_AGX3C) ||
+       (cond != AGX3_CF_FCOND_LT && cond != AGX3_CF_FCOND_GT &&
+        !(enc == AGX_ISA_ENCODING_AGX3C && cond == AGX3_CF_FCOND_EQ)) ||
        (neg_in[0] == neg_in[1]) || src_in[0] >= 256 || src_in[1] >= 256 ||
        (src_in[0] & 1) || (src_in[1] & 1) ||
        (hints_in[0].cache && hints_in[0].discard) ||
@@ -289,16 +335,31 @@ agx3_cf_fcmp_reg_neg(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
       const struct agx3_cf_source_hints hint = hints[0];
       hints[0] = hints[1];
       hints[1] = hint;
-      cond = cond == AGX3_CF_FCOND_LT ? AGX3_CF_FCOND_GT
-                                      : AGX3_CF_FCOND_LT;
+      if (cond != AGX3_CF_FCOND_EQ)
+         cond = cond == AGX3_CF_FCOND_LT ? AGX3_CF_FCOND_GT
+                                       : AGX3_CF_FCOND_LT;
    }
 
    const struct agx3_field_kv base[] = {
       KV(9, 2), KV(10, 2), KV(26, 2), KV(27, 2), KV(38, 1)};
-   const struct agx3_field_kv ov[] = {
+   struct agx3_field_kv ov[] = {
       KV(2, src[0]), KV(11, hints[0].cache), KV(13, hints[0].discard),
       KV(19, src[1]), KV(28, hints[1].cache), KV(30, hints[1].discard),
-      KV(32, 1), KV(35, slot), KV(46, cond)};
+      KV(32, 1), KV(35, slot), KV(46, cond), KV(34, 0)};
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      /* Metal537/540: ordered and unordered comparisons with a negated
+       * f32 source use the M5 selector plus independent negate
+       * and NaN fields. CF boundaries have already retired producer waits. */
+      ov[8].value = (unsigned)cond + (or_nan && cond != AGX3_CF_FCOND_EQ ? 4 : 0);
+      ov[9].value = or_nan;
+      const struct agx3_field_kv m5_base[] = {
+         KV(9,2), KV(10,2), KV(26,2), KV(27,2), KV(36,3), KV(37,69),
+      };
+      for (unsigned ep=0; ep<4; ep++)
+         if (agx3_bind_compose(enc, "fcmp", ep, m5_base, 6, ov, 10, out, len))
+            return true;
+      return false;
+   }
    return agx3_bind_compose(enc, "fcmp", 0, base, 5, ov, 9, out, len);
 }
 
@@ -317,11 +378,12 @@ agx3_cf_fcmp_reg16(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
                    uint16_t wait_mask, uint8_t out[AGX3_CF_MAX_LENGTH],
                    uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !hints || !out || !len ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !src || !hints || !out || !len ||
        !slot_ok(slot) || !fcond_ok(cond) || src[0] >= 256 || src[1] >= 256 ||
        (hints[0].cache && hints[0].discard) ||
        (hints[1].cache && hints[1].discard) ||
-       (wait_mask & ~UINT16_C(0x3f)))
+       (wait_mask & ~(m5 ? UINT16_C(0xff) : UINT16_C(0x3f))))
       return false;
 
    /* Same condition folding as the f32 form: lt/gt or unordered are codes
@@ -336,13 +398,17 @@ agx3_cf_fcmp_reg16(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
 
    const struct agx3_field_kv base[] = {KV(9, 2), KV(10, 1), KV(26, 2),
                                         KV(27, 1)};
-   struct agx3_field_kv ov[16] = {
+   struct agx3_field_kv ov[19] = {
       KV(2, src[0]), KV(11, hints[0].cache), KV(13, hints[0].discard),
       KV(19, src[1]), KV(28, hints[1].cache), KV(30, hints[1].discard),
       KV(34, or_nan), KV(35, slot), KV(46, fcond),
    };
    unsigned n = 9;
-   for (unsigned wait_slot = 0; wait_slot < 6; ++wait_slot)
+   if (m5) {
+      ov[n++] = (struct agx3_field_kv){36, 3};
+      ov[n++] = (struct agx3_field_kv){37, 69};
+   }
+   for (unsigned wait_slot = 0; wait_slot < (m5 ? 8 : 6); ++wait_slot)
       ov[n++] = (struct agx3_field_kv){38 + wait_slot,
                                        !!(wait_mask & (1u << wait_slot))};
 
@@ -377,6 +443,50 @@ agx3_cf_fcmp_reg16(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
  * f38..f43 are the six producer wait slots on this form; Apple's kernels
  * above wait on slot 6 because their operands come from a load.
  */
+bool
+agx3_cf_fcmp_width_m5(enum agx3_cf_fcond cond, bool or_nan, unsigned slot,
+                     const uint16_t src[2], const bool half[2],
+                     const bool absolute[2],
+                     const struct agx3_cf_source_hints hints[2],
+                     uint16_t wait_mask,
+                     uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
+{
+   if (!src || !half || !absolute || !hints || !out || !len ||
+       !slot_ok(slot) || !fcond_ok(cond) || wait_mask > 0xff)
+      return false;
+   for (unsigned i = 0; i < 2; ++i) {
+      if (src[i] >= 256 || (!half[i] && (src[i] & 1)) ||
+          (hints[i].cache && hints[i].discard))
+         return false;
+   }
+   unsigned condition = cond;
+   if (or_nan && cond == AGX3_CF_FCOND_LT)
+      condition = 5;
+   else if (or_nan && cond == AGX3_CF_FCOND_GT)
+      condition = 6;
+   else if (or_nan)
+      return false;
+   const struct agx3_field_kv base[] = {
+      KV(9, 2), KV(26, 2), KV(36, 3), KV(37, 69),
+   };
+   struct agx3_field_kv fields[22] = {
+      KV(2, src[0]), KV(10, half[0] ? 1 : 2),
+      KV(11, hints[0].cache), KV(13, hints[0].discard), KV(14, absolute[0]),
+      KV(19, src[1]), KV(27, half[1] ? 1 : 2),
+      KV(28, hints[1].cache), KV(30, hints[1].discard), KV(31, absolute[1]),
+      KV(34, or_nan), KV(35, slot), KV(46, condition),
+   };
+   unsigned count = 13;
+   for (unsigned i = 0; i < 8; ++i)
+      fields[count++] = (struct agx3_field_kv)KV(38 + i, (wait_mask >> i) & 1);
+   for (unsigned ep = 0; ep < 4; ++ep) {
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "fcmp", ep, base, 4,
+                           fields, count, out, len))
+         return true;
+   }
+   return false;
+}
+
 bool
 agx3_cf_fcmp_reg_abs(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
                      bool or_nan, unsigned slot, const uint16_t src[2],
@@ -562,18 +672,19 @@ bool agx3_cf_fcmp_zero_imm(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
                         uint16_t wait_mask,
                         uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len)
 {
-   if (!agx3b(enc) || !out || !len || !fcond_ok(cond) || !slot_ok(slot) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!agx3b(enc) && !m5) || !out || !len || !fcond_ok(cond) || !slot_ok(slot) ||
        source >= 256 || (!source_16 && (source & 1)) || (hint.cache && hint.discard) ||
-       (wait_mask & ~0x3fu) || (or_nan && cond == AGX3_CF_FCOND_EQ))
+       (wait_mask >> (m5 ? 8 : 6)) || (or_nan && cond == AGX3_CF_FCOND_EQ))
       return false;
-   const struct agx3_field_kv base[]={KV(9,2),KV(10,source_16?1:2),KV(26,1),KV(27,2)};
-   struct agx3_field_kv fields[13]={
+   const struct agx3_field_kv base[]={KV(9,2),KV(10,source_16?1:2),KV(26,1),KV(27,2),KV(36,3),KV(37,69)};
+   struct agx3_field_kv fields[15]={
       KV(2,source),KV(18,minifloat),KV(11,hint.cache),KV(13,hint.discard),
       KV(34,or_nan),KV(35,slot),KV(46,(unsigned)cond+(or_nan?4:0)),
    };
-   for(unsigned i=0;i<6;i++)fields[7+i]=(struct agx3_field_kv)KV(38+i,!!(wait_mask&(1u<<i)));
-   for(unsigned ep=0;ep<=1;ep++)
-      if(agx3_bind_compose(enc,"fcmp",ep,base,4,fields,13,out,len))return true;
+   for(unsigned i=0;i<(m5?8:6);i++)fields[7+i]=(struct agx3_field_kv)KV(38+i,!!(wait_mask&(1u<<i)));
+   for(unsigned ep=0;ep<=(m5?3:1);ep++)
+      if(agx3_bind_compose(enc,"fcmp",ep,base,m5?6:4,fields,m5?15:13,out,len))return true;
    return false;
 }
 
diff --git a/src/asahi/compiler/agx3_cf_core.h b/src/asahi/compiler/agx3_cf_core.h
index 7e46b9b6fbc6..f531a6258e68 100644
--- a/src/asahi/compiler/agx3_cf_core.h
+++ b/src/asahi/compiler/agx3_cf_core.h
@@ -65,6 +65,11 @@ struct agx3_cf_source_hints {
    bool discard;
 };
 
+bool agx3_cf_icmp_width_m5(enum agx3_cf_icond cond, unsigned slot,
+                            const uint16_t src[2], const bool half[2],
+                            const struct agx3_cf_source_hints hints[2],
+                            uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len);
+
 bool agx3_cf_icmp_reg16(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
                         unsigned slot, const uint16_t src[2],
                         const struct agx3_cf_source_hints hints[2], uint16_t wait_mask,
@@ -75,14 +80,14 @@ bool agx3_cf_icmp_reg(enum agx_isa_encoding enc, enum agx3_cf_icond cond,
                       unsigned slot, uint16_t src0, uint16_t src1,
                       uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len);
 /* flag[slot] = (32-bit register src0 cond 32-bit uniform), the form Apple's
- * G17P loop test uses. `uniform` is an even half index below 256. */
+ * G17P and G17G loop tests use. `uniform` is an even half index below 256. */
 bool agx3_cf_icmp_reg_uniform32(enum agx_isa_encoding enc,
                                 enum agx3_cf_icond cond, unsigned slot,
                                 uint16_t src0, uint16_t uniform,
                                 struct agx3_cf_source_hints hint,
                                 uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len);
 /* flag[slot] = (32-bit uniform cond imm); `uniform` is an even half index
- * below 256. Apple's G17P loop-guard form; AGX3B only. */
+ * below 256. Apple's G17P/G17G loop-guard form. */
 bool agx3_cf_icmp_uniform_imm(enum agx_isa_encoding enc,
                               enum agx3_cf_icond cond, unsigned slot,
                               uint16_t uniform, uint8_t imm,
@@ -133,6 +138,13 @@ bool agx3_cf_fcmp_reg16(enum agx_isa_encoding enc, enum agx3_cf_fcond cond,
                         uint16_t wait_mask, uint8_t out[AGX3_CF_MAX_LENGTH],
                         uint8_t *len);
 
+bool agx3_cf_fcmp_width_m5(enum agx3_cf_fcond cond, bool or_nan, unsigned slot,
+                          const uint16_t src[2], const bool half[2],
+                          const bool absolute[2],
+                          const struct agx3_cf_source_hints hints[2],
+                          uint16_t wait_mask,
+                          uint8_t out[AGX3_CF_MAX_LENGTH], uint8_t *len);
+
 /* G17P register comparison with exactly one negated f32 source. */
 bool agx3_cf_fcmp_reg_neg(enum agx_isa_encoding enc,
                           enum agx3_cf_fcond cond, bool or_nan, unsigned slot,
diff --git a/src/asahi/compiler/agx3_copy_core.c b/src/asahi/compiler/agx3_copy_core.c
index 5ebe28c7cc0c..5e5b0b5bf955 100644
--- a/src/asahi/compiler/agx3_copy_core.c
+++ b/src/asahi/compiler/agx3_copy_core.c
@@ -44,26 +44,39 @@ agx3_copy_encode_reg16(enum agx_isa_encoding encoding, uint16_t dest_half,
                        bool src_discard, uint16_t wait_mask,
                        uint8_t out[AGX3_COPY_MAX_LENGTH], uint8_t *len)
 {
-   if (out == NULL || len == NULL || encoding != AGX_ISA_ENCODING_AGX3B ||
-       dest_half >= AGX3_COPY_REG16_DEST_HALVES || src_half >= 256 ||
-       wait_mask >= (1u << 6) || (src_cache && src_discard))
+   const bool agx3c = encoding == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = agx3c ? 8 : 6;
+   if (out == NULL || len == NULL ||
+       (encoding != AGX_ISA_ENCODING_AGX3B && !agx3c) ||
+       dest_half >= (agx3c ? 256 : AGX3_COPY_REG16_DEST_HALVES) || src_half >= 256 ||
+       wait_mask >= (1u << slots) || (src_cache && src_discard))
       return false;
 
-   const struct agx3_field_kv baseline[] = {
+   const struct agx3_field_kv baseline_b[] = {
       KV(9, 2), KV(10, 1), KV(23, 2), KV(24, 1),
    };
-   struct agx3_field_kv overrides[11] = {
-      KV(2, dest_half),  KV(16, src_half),    KV(11, dest_cache),
+   const struct agx3_field_kv baseline_c[] = {
+      KV(9, 2), KV(10, 1), KV(23, 2), KV(24, 1), KV(28, 3), KV(29, 35),
+   };
+   struct agx3_field_kv overrides[13] = {
+      KV(2, dest_half), KV(16, src_half), KV(11, dest_cache),
       KV(25, src_cache), KV(27, src_discard),
    };
    unsigned n = 5;
-   for (unsigned i = 0; i < 6; ++i) {
+   for (unsigned i = 0; i < slots; ++i) {
       if (wait_mask & (1u << i))
          overrides[n++] = (struct agx3_field_kv)KV(30 + i, 1);
    }
-   return agx3_bind_compose(encoding, "copy", 0, baseline,
-                            AGX3_COPY_ARRAY_SIZE(baseline), overrides, n, out,
-                            len);
+   for (int ep = dest_half >= AGX3_COPY_REG16_DEST_HALVES ? 1 : 0;
+        ep <= (agx3c ? 1 : 0); ++ep) {
+      if (agx3_bind_compose(encoding, "copy", ep,
+                            agx3c ? baseline_c : baseline_b,
+                            agx3c ? AGX3_COPY_ARRAY_SIZE(baseline_c)
+                                  : AGX3_COPY_ARRAY_SIZE(baseline_b),
+                            overrides, n, out, len))
+         return true;
+   }
+   return false;
 }
 
 bool
@@ -107,7 +120,23 @@ agx3_copy_encode_reg_to_uniform(enum agx_isa_encoding encoding,
    if (out == NULL || len == NULL)
       return false;
 
-   /* Only AGX3b is witnessed. */
+   if (encoding == AGX_ISA_ENCODING_AGX3C) {
+      if ((uniform_half & 1) || uniform_half + 2 > 512 || (src_half & 1) ||
+          src_half + 2 > 256)
+         return false;
+      /* Same M5 COPY uniform destination as the captured IOTO publisher.
+       * Retain its exact domain tags and caller-owned last-use hint. */
+      const struct agx3_field_kv fields[] = {
+         KV(3, uniform_half), KV(9, 3), KV(10, 2), KV(16, src_half),
+         KV(23, 2), KV(24, 2), KV(27, kill), KV(28, 1), KV(29, 35),
+      };
+      for (unsigned ep = 0; ep <= 1; ++ep) {
+         if (agx3_bind_compose(encoding, "copy", ep, NULL, 0, fields,
+                              AGX3_COPY_ARRAY_SIZE(fields), out, len))
+            return true;
+      }
+      return false;
+   }
    if (encoding != AGX_ISA_ENCODING_AGX3B)
       return false;
 
@@ -161,9 +190,10 @@ agx3_copy_encode_uniform_to_reg16(enum agx_isa_encoding encoding,
                                   uint8_t out[AGX3_COPY_MAX_LENGTH],
                                   uint8_t *len)
 {
-   /* Only AGX3b is witnessed; the AGX3c corpus has no 16-bit uniform copy. */
-   if (out == NULL || len == NULL || encoding != AGX_ISA_ENCODING_AGX3B ||
-       dest_half >= 128 || uniform_half >= 512)
+   const bool m5 = encoding == AGX_ISA_ENCODING_AGX3C;
+   if (out == NULL || len == NULL ||
+       (encoding != AGX_ISA_ENCODING_AGX3B && !m5) ||
+       dest_half >= (m5 ? 256 : 128) || uniform_half >= 512)
       return false;
 
    /* The 16-bit widths of agx3_copy_encode_reg16() with the uniform source
@@ -175,9 +205,12 @@ agx3_copy_encode_uniform_to_reg16(enum agx_isa_encoding encoding,
    };
    const struct agx3_field_kv overrides[] = {
       KV(2, dest_half), KV(17, uniform_half), KV(11, dest_cache),
+      KV(28, m5 ? 3 : 0), KV(29, m5 ? 35 : 0),
    };
 
-   return agx3_bind_compose(encoding, "copy", 0, baseline,
+   return agx3_bind_compose(encoding, "copy",
+                            dest_half >= AGX3_COPY_REG16_DEST_HALVES ? 1 : 0,
+                            baseline,
                             AGX3_COPY_ARRAY_SIZE(baseline), overrides,
                             AGX3_COPY_ARRAY_SIZE(overrides), out, len);
 }
@@ -216,8 +249,7 @@ agx3_copy_encode_uniform_to_reg32(enum agx_isa_encoding encoding,
       KV(17, uniform_half),
    };
 
-   if (dest_half < AGX3_COPY_REG16_DEST_HALVES ||
-       encoding != AGX_ISA_ENCODING_AGX3B)
+   if (dest_half < AGX3_COPY_REG16_DEST_HALVES)
       return agx3_bind_compose(encoding, "copy", 0, baseline, nbaseline,
                                overrides,
                                AGX3_COPY_ARRAY_SIZE(overrides), out, len);
@@ -248,9 +280,9 @@ agx3_copy_encode_uniform_to_reg32(enum agx_isa_encoding encoding,
     * pressure register allocation placed those destinations at halves
     * 134-154, refusing the whole shader.
     *
-    * AGX3c has a copy.ep1 too (eight bytes) and its tables accept the same
-    * wide destinations, but no G17S witness establishes it, so that encoding
-    * still fails closed rather than being assumed from the AGX3b result.
+    * AGX3c carries the same high destination bit in its eight-byte ep1.
+    * The source kind and index fields are independent of that destination
+    * extension, as with the corresponding extended half-register copy.
     */
    return agx3_bind_compose(encoding, "copy", 1, baseline, nbaseline,
                             overrides,
diff --git a/src/asahi/compiler/agx3_copy_core.h b/src/asahi/compiler/agx3_copy_core.h
index 0949c3b6c4e7..62ff99d5f347 100644
--- a/src/asahi/compiler/agx3_copy_core.h
+++ b/src/asahi/compiler/agx3_copy_core.h
@@ -34,14 +34,14 @@
 extern "C" {
 #endif
 
-/* Apple's ordinary `copy` is four bytes; its extended ep1 record is ten. */
+/* Ordinary COPY is four bytes. Extended forms need up to ten bytes. */
 #define AGX3_COPY_MAX_LENGTH 10u
 
 /* The compact COPY's destination field (f2) is seven bits wide at EVERY
  * width, so no four-byte copy can name a destination at or above this half
- * index -- the 16-bit forms refuse there, and the 32-bit uniform form moves
- * to the ten-byte ep1.  Register allocation needs the same number to know
- * when a half victim could become unencodable.  The SOURCE fields are not
+ * index. G17P half copies refuse there; G17S uses its eight-byte ep1 half
+ * copy, and the 32-bit uniform form uses ten-byte ep1. Register allocation
+ * uses this boundary for G17P half-copy restrictions. The SOURCE fields are not
  * narrow this way: Apple's own ep0 copies carry sources well past it. */
 #define AGX3_COPY_REG16_DEST_HALVES 128u
 
@@ -63,7 +63,8 @@ extern "C" {
  * 32-bit `iadd d, s, #0` trick the select core uses is grounded only at 32
  * bits, and booleans live in 16-bit halves. Apple's own corpus carries this
  * form (copy.ep0 with a register source and f24 = 1), including odd
- * destination halves, which need no separate encoding point here.
+ * destination halves. G17S destinations from half 128 use ep1 to preserve
+ * the high destination bit rather than truncating it in compact ep0.
  *
  * Scoreboard waits ride on f30 + slot - 1, the same convention
  * agx3_tex_stage_copy() documents; a corpus copy with f30 set reproduces
diff --git a/src/asahi/compiler/agx3_cprog_input.c b/src/asahi/compiler/agx3_cprog_input.c
index 9d8058a41a92..2f253caa12bc 100644
--- a/src/asahi/compiler/agx3_cprog_input.c
+++ b/src/asahi/compiler/agx3_cprog_input.c
@@ -75,9 +75,9 @@ agx3_cprog_input_publish(const struct agx3_cprog_inputs *inputs,
 }
 
 bool
-agx3_cprog_input_publish_at(unsigned mask, unsigned tps_half,
-                            unsigned source_half, void *output,
-                            size_t capacity, size_t *written)
+agx3_cprog_input_publish_at_encoding(
+   enum agx_isa_encoding encoding, unsigned mask, unsigned tps_half,
+   unsigned source_half, void *output, size_t capacity, size_t *written)
 {
    if (!output || !written || mask > 3 || source_half >= 255 ||
        (source_half & 1) || tps_half < 8 || tps_half >= 520 ||
@@ -100,7 +100,7 @@ agx3_cprog_input_publish_at(unsigned mask, unsigned tps_half,
       /* The native loader uses extended COPY for both widths, including
        * the otherwise compact-encodable 16-bit form. */
       if (!agx3_static_spill_encode_store_form(
-             AGX_ISA_ENCODING_AGX3B, &desc, 1, bytes + size, &length))
+             encoding, &desc, 1, bytes + size, &length))
          return false;
       size += length;
       if (pair)
@@ -114,9 +114,10 @@ agx3_cprog_input_publish_at(unsigned mask, unsigned tps_half,
 }
 
 bool
-agx3_cprog_input_import(const struct agx3_cprog_inputs *inputs,
-                        const struct agx3_cprog_import *desc,
-                        void *output, size_t capacity, size_t *written)
+agx3_cprog_input_import_encoding(
+   enum agx_isa_encoding encoding, const struct agx3_cprog_inputs *inputs,
+   const struct agx3_cprog_import *desc, void *output, size_t capacity,
+   size_t *written)
 {
    if (!valid_inputs(inputs) || !desc || !output || !written ||
        (desc->width != AGX3_MEM_W16 && desc->width != AGX3_MEM_W32))
@@ -137,10 +138,28 @@ agx3_cprog_input_import(const struct agx3_cprog_inputs *inputs,
       .structural_flag = desc->structural_flag,
    };
    uint8_t bytes[AGX3_STATIC_SPILL_MAX_LENGTH], length;
-   if (!agx3_static_spill_encode_load(AGX_ISA_ENCODING_AGX3B, &load, bytes, &length) ||
+   if (!agx3_static_spill_encode_load(encoding, &load, bytes, &length) ||
        length > capacity)
       return false;
    memcpy(output, bytes, length);
    *written = length;
    return true;
 }
+
+bool
+agx3_cprog_input_publish_at(unsigned mask, unsigned tps_half,
+                            unsigned source_half, void *output,
+                            size_t capacity, size_t *written)
+{
+   return agx3_cprog_input_publish_at_encoding(AGX_ISA_ENCODING_AGX3B,
+      mask, tps_half, source_half, output, capacity, written);
+}
+
+bool
+agx3_cprog_input_import(const struct agx3_cprog_inputs *inputs,
+                        const struct agx3_cprog_import *desc,
+                        void *output, size_t capacity, size_t *written)
+{
+   return agx3_cprog_input_import_encoding(AGX_ISA_ENCODING_AGX3B, inputs,
+      desc, output, capacity, written);
+}
diff --git a/src/asahi/compiler/agx3_cprog_input.h b/src/asahi/compiler/agx3_cprog_input.h
index 96feaf4748a5..6986540307ed 100644
--- a/src/asahi/compiler/agx3_cprog_input.h
+++ b/src/asahi/compiler/agx3_cprog_input.h
@@ -50,6 +50,12 @@ bool agx3_cprog_input_publish_at(unsigned mask, unsigned tps_half,
                                 unsigned source_half, void *output,
                                 size_t capacity, size_t *written);
 
+/* Chip-aware forms preserve the classic ABI and encode M5's native TPS
+ * COPY/RLD forms. M5 capture uses slot6 and compact8B extended COPY. */
+bool agx3_cprog_input_publish_at_encoding(
+   enum agx_isa_encoding encoding, unsigned mask, unsigned tps_half,
+   unsigned source_half, void *output, size_t capacity, size_t *written);
+
 struct agx3_cprog_import {
    uint16_t uniform_half;
    uint16_t dest_half;
@@ -71,6 +77,12 @@ bool agx3_cprog_input_import(const struct agx3_cprog_inputs *inputs,
                              const struct agx3_cprog_import *desc,
                              void *output, size_t capacity, size_t *written);
 
+
+bool agx3_cprog_input_import_encoding(
+   enum agx_isa_encoding encoding, const struct agx3_cprog_inputs *inputs,
+   const struct agx3_cprog_import *desc, void *output, size_t capacity,
+   size_t *written);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/asahi/compiler/agx3_falu_mod_core.c b/src/asahi/compiler/agx3_falu_mod_core.c
index e872a0ae8694..1e0a89f54d5c 100644
--- a/src/asahi/compiler/agx3_falu_mod_core.c
+++ b/src/asahi/compiler/agx3_falu_mod_core.c
@@ -88,17 +88,18 @@ agx3_falu_add_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src0,
 /* Keep subtraction and other modified adds fused while preserving both live
  * operands and their producer slots. The native forms grow from4 to10bytes
  * only when modifiers or wait combinations require it. */
-bool
-agx3_falu_add_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
+static bool
+add_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
                          uint16_t src0, uint16_t src1,
-                         const struct agx3_falu_mods *m,
+                         bool dest_half, const struct agx3_falu_mods *m,
                          const struct agx3_falu_runtime *runtime,
                          uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !m || !runtime || !out || !len ||
-       (dest & 1) || dest >= 256 || !slot_ok(src0, m->src0_16) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !m || !runtime || !out || !len ||
+       (!dest_half && (dest & 1)) || (dest_half && !m5) || dest >= 256 || !slot_ok(src0, m->src0_16) ||
        !slot_ok(src1, m->src1_16) || m->addend_abs || m->addend_neg ||
-       runtime->wait_mask > 0x3f)
+       runtime->wait_mask > (m5 ? 0xff : 0x3f))
       return false;
    for (unsigned i = 0; i < 2; ++i)
       if (runtime->cache[i] && runtime->discard[i])
@@ -121,17 +122,18 @@ agx3_falu_add_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
       struct agx3_field_kv ov[32];
       unsigned n = common_mods(dest, swap ? src1 : src0, swap ? src0 : src1,
                                &mods, ov);
+      ov[n++] = (struct agx3_field_kv)KV(10, dest_half ? 1 : 2);
       ov[n++] = (struct agx3_field_kv)KV(11, runtime->dest_cache);
       ov[n++] = (struct agx3_field_kv)KV(44, runtime->cache[swap]);
       ov[n++] = (struct agx3_field_kv)KV(46, runtime->discard[swap]);
       ov[n++] = (struct agx3_field_kv)KV(27, runtime->cache[1 - swap]);
       ov[n++] = (struct agx3_field_kv)KV(29, runtime->discard[1 - swap]);
-      for (unsigned i = 0; i < 6; ++i)
+      for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
          ov[n++] = (struct agx3_field_kv)KV(52 + i,
                                           (runtime->wait_mask >> i) & 1);
       for (unsigned ep = 0; ep <= 3; ++ep) {
          uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], size;
-         if (agx3_bind_compose(enc, "f32add", ep, add_b, 9, ov, n, bytes, &size)) {
+         if (agx3_bind_compose(enc, "f32add", ep, m5 ? add_c : add_b, m5 ? 11 : 9, ov, n, bytes, &size)) {
             if (!best_size || size < best_size) {
                for (unsigned i = 0; i < size; ++i)
                   best[i] = bytes[i];
@@ -149,32 +151,67 @@ agx3_falu_add_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
    return true;
 }
 
+bool
+agx3_falu_add_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
+                          uint16_t src0, uint16_t src1,
+                          const struct agx3_falu_mods *m,
+                          const struct agx3_falu_runtime *runtime,
+                          uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return add_mod_runtime(enc, dest, src0, src1, false, m, runtime, out, len);
+}
+
+bool
+agx3_falu_add_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+                         const struct agx3_falu_mods *m,
+                         const struct agx3_falu_runtime *runtime,
+                         uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return add_mod_runtime(AGX_ISA_ENCODING_AGX3C, dest, src0, src1, true,
+                           m, runtime, out, len);
+}
+
 /* Apple lowers float-to-half conversion to f32add with a16-bit destination
  * and the exact negative-zero minifloat in the other input. This preserves
  * signed zero and rounds only once; half4 witnesses include odd halves. */
-bool
-agx3_falu_narrow_f32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+static bool
+add_immediate(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+                     bool half, uint8_t immediate, bool src_abs, bool src_neg,
                      bool saturate, const struct agx3_falu_runtime *runtime,
                      uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !runtime || !out || !len ||
-       dest >= 256 || src >= 256 || (src & 1) || runtime->wait_mask > 0x3f ||
+   const bool agx3c = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = agx3c ? 8 : 6;
+   if ((!b(enc) && !agx3c) || !runtime || !out || !len ||
+       dest >= 256 || (!half && (dest & 1)) || src >= 256 || (src & 1) ||
+       runtime->wait_mask >= (1u << slots) ||
        (runtime->cache[0] && runtime->discard[0]))
       return false;
    const struct agx3_field_kv base[] = {
-      KV(9, 2), KV(10, 1), KV(25, 2), KV(26, 2), KV(42, 1), KV(43, 2),
+      KV(9, 2), KV(10, half ? 1 : 2), KV(25, 2), KV(26, 2), KV(42, 1), KV(43, 2),
+   };
+   /* G17S cvt_f2h, r_mrt and tx_cube_sample witness this narrowing
+    * operation, with f50=3/f51=48 and an eight-slot wait bank. The native
+    * r_fragcoord variants set f15=1; its meaning is not established here
+    * and that variant is not modeled by this API.
+    * Saturating narrowing remains separate from the full-width add. */
+   const struct agx3_field_kv base_c[] = {
+      KV(9, 2), KV(10, half ? 1 : 2), KV(25, 2), KV(26, 2), KV(42, 1), KV(43, 2),
+      KV(50, 3), KV(51, 48),
    };
    struct agx3_field_kv ov[32] = {
-      KV(2, dest), KV(18, src), KV(34, 0x80), KV(11, runtime->dest_cache),
+      KV(2, dest), KV(18, src), KV(34, immediate), KV(11, runtime->dest_cache),
       KV(27, runtime->cache[0]), KV(29, runtime->discard[0]), KV(14, saturate),
+      KV(30, src_abs), KV(31, src_neg),
    };
-   unsigned n = 7;
-   for (unsigned i = 0; i < 6; ++i)
+   unsigned n = 9;
+   for (unsigned i = 0; i < slots; ++i)
       ov[n++] = (struct agx3_field_kv)KV(52 + i,
                                        (runtime->wait_mask >> i) & 1);
    for (unsigned ep = 0; ep <= 3; ++ep) {
       uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], size;
-      if (agx3_bind_compose(enc, "f32add", ep, base, 6, ov, n, bytes, &size)) {
+      if (agx3_bind_compose(enc, "f32add", ep, agx3c ? base_c : base,
+                           agx3c ? 8 : 6, ov, n, bytes, &size)) {
          for (unsigned i = 0; i < size; ++i)
             out[i] = bytes[i];
          *len = size;
@@ -184,6 +221,25 @@ agx3_falu_narrow_f32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
    return false;
 }
 
+bool
+agx3_falu_narrow_f32(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+                     bool saturate, const struct agx3_falu_runtime *runtime,
+                     uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return add_immediate(enc, dest, src, true, 0x80, false, false, saturate,
+                         runtime, out, len);
+}
+
+bool
+agx3_falu_add_imm_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+                     bool src_abs, bool src_neg, bool saturate,
+                     const struct agx3_falu_runtime *runtime,
+                     uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return add_immediate(AGX_ISA_ENCODING_AGX3C, dest, src, false, immediate,
+                         src_abs, src_neg, saturate, runtime, out, len);
+}
+
 /* Half of a minifloat minus a register: f32add with a 16-bit destination,
  * the register negated in the first operand and the minifloat in the other.
  * Apple's half4(1.0 - x) and half4(-x - 1.0) (blend/native/negpack3.py
@@ -238,17 +294,18 @@ agx3_falu_mul_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src0,
 /* Saturation and source modifiers stay fused with the multiply. Native
  * live/dead witnesses carry independent hints and the same six dependency
  * slots as plain FMUL; select the shortest encoding that fits them. */
-bool
-agx3_falu_mul_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
+static bool
+mul_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
                           uint16_t src0, uint16_t src1,
-                          const struct agx3_falu_mods *m,
+                          bool dest_half, const struct agx3_falu_mods *m,
                           const struct agx3_falu_runtime *runtime,
                           uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !m || !runtime || !out || !len ||
-       (dest & 1) || dest >= 256 || !slot_ok(src0, m->src0_16) ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !m || !runtime || !out || !len ||
+       (!dest_half && (dest & 1)) || (dest_half && !m5) || dest >= 256 || !slot_ok(src0, m->src0_16) ||
        !slot_ok(src1, m->src1_16) || m->addend_abs || m->addend_neg ||
-       runtime->wait_mask > 0x3f)
+       runtime->wait_mask > (m5 ? 0xff : 0x3f))
       return false;
    for (unsigned i = 0; i < 2; ++i)
       if (runtime->cache[i] && runtime->discard[i])
@@ -256,17 +313,18 @@ agx3_falu_mul_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
 
    struct agx3_field_kv ov[32];
    unsigned n = common_mods(dest, src0, src1, m, ov);
+   ov[n++] = (struct agx3_field_kv)KV(10, dest_half ? 1 : 2);
    ov[n++] = (struct agx3_field_kv)KV(11, runtime->dest_cache);
    ov[n++] = (struct agx3_field_kv)KV(44, runtime->cache[0]);
    ov[n++] = (struct agx3_field_kv)KV(46, runtime->discard[0]);
    ov[n++] = (struct agx3_field_kv)KV(27, runtime->cache[1]);
    ov[n++] = (struct agx3_field_kv)KV(29, runtime->discard[1]);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       ov[n++] = (struct agx3_field_kv)KV(80 + i,
                                        (runtime->wait_mask >> i) & 1);
    for (unsigned ep = 0; ep <= 3; ++ep) {
       uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], size;
-      if (agx3_bind_compose(enc, "f32mul", ep, mul_b, 11, ov, n, bytes, &size)) {
+      if (agx3_bind_compose(enc, "f32mul", ep, m5 ? mul_c : mul_b, m5 ? 13 : 11, ov, n, bytes, &size)) {
          for (unsigned i = 0; i < size; ++i)
             out[i] = bytes[i];
          *len = size;
@@ -276,6 +334,26 @@ agx3_falu_mul_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
    return false;
 }
 
+bool
+agx3_falu_mul_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
+                          uint16_t src0, uint16_t src1,
+                          const struct agx3_falu_mods *m,
+                          const struct agx3_falu_runtime *runtime,
+                          uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return mul_mod_runtime(enc, dest, src0, src1, false, m, runtime, out, len);
+}
+
+bool
+agx3_falu_mul_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+                         const struct agx3_falu_mods *m,
+                         const struct agx3_falu_runtime *runtime,
+                         uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return mul_mod_runtime(AGX_ISA_ENCODING_AGX3C, dest, src0, src1, true,
+                           m, runtime, out, len);
+}
+
 bool
 agx3_falu_fma_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src0,
                   uint16_t src1, uint16_t addend,
@@ -301,17 +379,19 @@ agx3_falu_fma_mod(enum agx_isa_encoding enc, uint16_t dest, uint16_t src0,
  * modifiers and dependency waits; choose the shortest exact representation.
  * This keeps fog subtraction fused while waiting only on its dependencies.
  */
-bool
-agx3_falu_fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
-                          uint16_t src0, uint16_t src1, uint16_t addend,
+static bool
+fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
+                          uint16_t src0, uint16_t src1, uint16_t addend, bool dest_half,
                           const struct agx3_falu_mods *m,
                           const struct agx3_falu_runtime *runtime,
                           uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || m == NULL || runtime == NULL ||
-       out == NULL || len == NULL || (dest & 1) || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || m == NULL || runtime == NULL ||
+       out == NULL || len == NULL || (!dest_half && (dest & 1)) ||
+       (dest_half && !m5) || dest >= 256 ||
        !slot_ok(src0, m->src0_16) || !slot_ok(src1, m->src1_16) ||
-       !slot_ok(addend, m->addend_16) || runtime->wait_mask > 0x3f)
+       !slot_ok(addend, m->addend_16) || runtime->wait_mask > (m5 ? 0xff : 0x3f))
       return false;
    for (unsigned i = 0; i < 3; ++i)
       if (runtime->cache[i] && runtime->discard[i])
@@ -319,6 +399,7 @@ agx3_falu_fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
 
    struct agx3_field_kv ov[32];
    unsigned n = common_mods(dest, src0, src1, m, ov);
+   ov[n++] = (struct agx3_field_kv)KV(10, dest_half ? 1 : 2);
    ov[n++] = (struct agx3_field_kv)KV(52, addend);
    ov[n++] = (struct agx3_field_kv)KV(64, m->addend_abs);
    ov[n++] = (struct agx3_field_kv)KV(65, m->addend_neg);
@@ -330,12 +411,12 @@ agx3_falu_fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
    ov[n++] = (struct agx3_field_kv)KV(29, runtime->discard[1]);
    ov[n++] = (struct agx3_field_kv)KV(61, runtime->cache[2]);
    ov[n++] = (struct agx3_field_kv)KV(63, runtime->discard[2]);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       ov[n++] = (struct agx3_field_kv)KV(97 + i,
                                        (runtime->wait_mask >> i) & 1);
-   for (unsigned ep = 2; ep <= 5; ++ep) {
+   for (unsigned ep = m5 ? 0 : 2; ep <= (m5 ? 7 : 5); ++ep) {
       uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], size;
-      if (agx3_bind_compose(enc, "f32fma", ep, fma_b, 14, ov, n,
+      if (agx3_bind_compose(enc, "f32fma", ep, m5 ? fma_c : fma_b, m5 ? 16 : 14, ov, n,
                            bytes, &size)) {
          for (unsigned i = 0; i < size; ++i)
             out[i] = bytes[i];
@@ -346,19 +427,43 @@ agx3_falu_fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
    return false;
 }
 
+bool
+agx3_falu_fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
+                          uint16_t src0, uint16_t src1, uint16_t addend,
+                          const struct agx3_falu_mods *m,
+                          const struct agx3_falu_runtime *runtime,
+                          uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return fma_mod_runtime(enc, dest, src0, src1, addend, false, m, runtime,
+                          out, len);
+}
+
+bool
+agx3_falu_fma_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+                        uint16_t addend, const struct agx3_falu_mods *m,
+                        const struct agx3_falu_runtime *runtime,
+                        uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return fma_mod_runtime(AGX_ISA_ENCODING_AGX3C, dest, src0, src1, addend,
+                          true, m, runtime, out, len);
+}
+
 /* Native FMA has a minifloat addend form: keep a*b+c fused without a
  * temporary constant register, including absolute/negated multipliers, and
  * preserve each multiplier's lifetime. */
-bool
-agx3_falu_fma_addend_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
+static bool
+fma_addend_imm_width_mod(enum agx_isa_encoding enc, uint16_t dest,
                             uint16_t src0, uint16_t src1, uint8_t immediate,
-                            bool src0_abs, bool src0_neg, bool src1_abs,
-                            bool src1_neg,
+                            bool dest_half, const struct agx3_falu_mods *m,
                             const struct agx3_falu_runtime *runtime,
                             uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !runtime || !out || !len ||
-       !regs_ok(dest, src0, src1) || runtime->wait_mask > 0x3f)
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !m || !runtime || !out || !len ||
+       (!dest_half && (dest & 1)) || (dest_half && !m5) || dest >= 256 || !slot_ok(src0, m->src0_16) ||
+       !slot_ok(src1, m->src1_16) || m->addend_abs || m->addend_neg ||
+       m->addend_16 || (!m5 && (m->src0_16 || m->src1_16 || m->saturate)) ||
+       runtime->wait_mask > (m5 ? 0xff : 0x3f))
       return false;
    for (unsigned i = 0; i < 2; ++i)
       if (runtime->cache[i] && runtime->discard[i])
@@ -366,73 +471,148 @@ agx3_falu_fma_addend_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
    const struct agx3_field_kv base[] = {
       KV(9,2), KV(10,2), KV(25,2), KV(26,2), KV(42,2), KV(43,2),
       KV(59,1), KV(60,2), KV(77,2), KV(91,2), KV(102,1),
+      KV(95,3), KV(96,55),
    };
    struct agx3_field_kv ov[32] = {
-      KV(2,dest), KV(35,src0), KV(18,src1), KV(51,immediate),
+      KV(10,dest_half ? 1 : 2), KV(2,dest), KV(35,src0), KV(18,src1), KV(51,immediate),
       KV(11,runtime->dest_cache), KV(44,runtime->cache[0]),
       KV(46,runtime->discard[0]), KV(27,runtime->cache[1]),
-      KV(29,runtime->discard[1]), KV(47,src0_abs), KV(48,src0_neg),
-      KV(30,src1_abs), KV(31,src1_neg),
+      KV(29,runtime->discard[1]), KV(47,m->src0_abs), KV(48,m->src0_neg),
+      KV(30,m->src1_abs), KV(31,m->src1_neg),
+      KV(43,m->src0_16 ? 1 : 2), KV(26,m->src1_16 ? 1 : 2),
+      KV(14,m->saturate),
    };
-   unsigned n = 13;
-   for (unsigned i = 0; i < 6; ++i)
-      if (runtime->wait_mask & (1u << i))
-         ov[n++] = (struct agx3_field_kv)KV(97+i,1);
-   for (unsigned ep = 0; ep <= 5; ++ep) {
-      if (agx3_bind_compose(enc,"f32fma",ep,base,11,ov,n,out,len))
+   unsigned n = 17;
+   /* G17G preserves the operand/modifier fields, but its long endpoint
+    * carries eight dependency slots. Override the G17P template's f102
+    * too, so a specimen's slot-6 wait cannot leak into the instruction. */
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
+      if (m5 || (runtime->wait_mask & (1u << i)))
+         ov[n++] = (struct agx3_field_kv)KV(97+i, !!(runtime->wait_mask & (1u << i)));
+   for (unsigned ep = 0; ep <= (m5 ? 7 : 5); ++ep) {
+      if (agx3_bind_compose(enc,"f32fma",ep,base,m5 ? 13 : 11,ov,n,out,len))
          return true;
    }
    return false;
 }
 
-/* The immediate multiplier and addend occupy independent native fields.
- * This is still one fused operation with one rounding, e.g. depth*2-1. */
 bool
-agx3_falu_fma_two_imm(enum agx_isa_encoding enc, uint16_t dest,
-                     uint16_t src, uint8_t multiplier, uint8_t addend,
-                     const struct agx3_falu_runtime *runtime,
-                     uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+agx3_falu_fma_addend_imm_width_mod(enum agx_isa_encoding enc, uint16_t dest,
+   uint16_t src0, uint16_t src1, uint8_t immediate,
+   const struct agx3_falu_mods *m, const struct agx3_falu_runtime *runtime,
+   uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !runtime || !out || !len ||
-       !regs_ok(dest, src, 0) || runtime->wait_mask > 0x3f ||
+   return fma_addend_imm_width_mod(enc, dest, src0, src1, immediate, false,
+                                  m, runtime, out, len);
+}
+
+bool
+agx3_falu_fma_addend_imm_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+   uint8_t immediate, const struct agx3_falu_mods *m,
+   const struct agx3_falu_runtime *runtime,
+   uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return fma_addend_imm_width_mod(AGX_ISA_ENCODING_AGX3C, dest, src0, src1,
+                                  immediate, true, m, runtime, out, len);
+}
+
+bool
+agx3_falu_fma_addend_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
+                            uint16_t src0, uint16_t src1, uint8_t immediate,
+                            bool src0_abs, bool src0_neg, bool src1_abs,
+                            bool src1_neg,
+                            const struct agx3_falu_runtime *runtime,
+                            uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   const struct agx3_falu_mods m = {
+      .src0_abs = src0_abs, .src0_neg = src0_neg,
+      .src1_abs = src1_abs, .src1_neg = src1_neg,
+   };
+   return agx3_falu_fma_addend_imm_width_mod(
+      enc, dest, src0, src1, immediate, &m, runtime, out, len);
+}
+
+
+static bool
+fma_two_imm(enum agx_isa_encoding enc, uint16_t dest,
+            uint16_t src, uint8_t multiplier, uint8_t addend,
+            bool dest_half, bool src_abs, bool saturate,
+            const struct agx3_falu_runtime *runtime,
+            uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || !runtime || !out || !len ||
+       dest >= 256 || src >= 256 || (src & 1) || (!dest_half && (dest & 1)) ||
+       (!m5 && (dest_half || src_abs || saturate)) ||
+       runtime->wait_mask > (m5 ? 0xff : 0x3f) ||
        (runtime->cache[0] && runtime->discard[0]))
       return false;
    const struct agx3_field_kv base[] = {
-      KV(9,2), KV(10,2), KV(25,2), KV(26,2), KV(42,1), KV(43,2),
-      KV(59,1), KV(60,2), KV(77,2), KV(91,2),
+      KV(9,2), KV(10,dest_half ? 1 : 2), KV(25,2), KV(26,2), KV(42,1), KV(43,2),
+      KV(59,1), KV(60,2), KV(77,2), KV(91,2), KV(95,3), KV(96,55), KV(104,1),
    };
    struct agx3_field_kv ov[32] = {
       KV(2,dest), KV(18,src), KV(34,multiplier), KV(51,addend),
       KV(11,runtime->dest_cache), KV(27,runtime->cache[0]),
-      KV(29,runtime->discard[0]),
+      KV(29,runtime->discard[0]), KV(30,src_abs), KV(14,saturate),
    };
-   unsigned n = 7;
-   for (unsigned i = 0; i < 6; ++i)
-      if (runtime->wait_mask & (1u << i))
-         ov[n++] = (struct agx3_field_kv)KV(97+i,1);
-   for (unsigned ep = 0; ep <= 5; ++ep)
-      if (agx3_bind_compose(enc,"f32fma",ep,base,10,ov,n,out,len))
+   unsigned n = 9;
+   /* The specimen's f104 is a slot-8 dependency, not an opcode bit.
+    * Override every M5 wait field, including zero, before selecting an
+    * encoding point. Otherwise every full-width FMA waits on slot 8 and
+    * expands to the 16-byte endpoint even without a dependency. */
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
+      if (m5 || (runtime->wait_mask & (1u << i)))
+         ov[n++] = (struct agx3_field_kv)KV(97+i,!!(runtime->wait_mask & (1u << i)));
+   for (unsigned ep = 0; ep <= (m5 ? 7 : 5); ++ep)
+      if (agx3_bind_compose(enc,"f32fma",ep,base,m5 ? 13 : 10,ov,n,out,len))
          return true;
    return false;
 }
 
+bool
+agx3_falu_fma_two_imm(enum agx_isa_encoding enc, uint16_t dest,
+                     uint16_t src, uint8_t multiplier, uint8_t addend,
+                     const struct agx3_falu_runtime *runtime,
+                     uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return fma_two_imm(enc, dest, src, multiplier, addend, false, false, false,
+                       runtime, out, len);
+}
+
+/* G17G narrows a full-input FMA directly through f32fma's result width f10.
+ * A forced device-half store proves this form, distinct from f16fma's half
+ * arithmetic and operand widths. Keep its single F32 rounding to F16. */
+bool
+agx3_falu_fma_two_imm_narrow_m5(uint16_t dest, uint16_t src,
+                                uint8_t multiplier, uint8_t addend,
+                                bool src_abs, bool saturate,
+                                const struct agx3_falu_runtime *runtime,
+                                uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return fma_two_imm(AGX_ISA_ENCODING_AGX3C, dest, src, multiplier, addend,
+                       true, src_abs, saturate, runtime, out, len);
+}
+
 /* Immediate multiplier (f34, selected by f42=1) with a modifier-bearing
  * register multiplicand (f18 + f26/f30/f31) and addend (f52 + f60/f64/f65).
  * Baseline is the constant part of Apple's witnessed fma(0.5, b, c); every
  * operand, modifier, lifetime hint and wait bit is an override, so a field
  * the selected encoding point does not consume makes the composer refuse
  * rather than silently drop it. */
-bool
-agx3_falu_fma_mul_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
-                          uint8_t multiplier, uint16_t src1, uint16_t addend,
-                          const struct agx3_falu_mods *m,
-                          const struct agx3_falu_runtime *runtime,
-                          uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+static bool
+fma_mul_imm(enum agx_isa_encoding enc, uint16_t dest, bool dest_half,
+            uint8_t multiplier, uint16_t src1, uint16_t addend,
+            const struct agx3_falu_mods *m,
+            const struct agx3_falu_runtime *runtime,
+            uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || m == NULL || runtime == NULL ||
-       out == NULL || len == NULL || (dest & 1) || dest >= 256 ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || m == NULL || runtime == NULL ||
+       out == NULL || len == NULL || (dest_half && !m5) ||
+       (!dest_half && (dest & 1)) || dest >= 256 ||
        !slot_ok(src1, m->src1_16) || !slot_ok(addend, m->addend_16) ||
-       runtime->wait_mask > 0x3f)
+       runtime->wait_mask > (m5 ? 0xff : 0x3f))
       return false;
 
    /* There is no f35 register slot in this form, so nothing can carry a
@@ -452,9 +632,13 @@ agx3_falu_fma_mul_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
       if (runtime->cache[i] && runtime->discard[i])
          return false;
 
+   /* f10 is the destination width: G17G narrows the fused result straight
+    * into a half (port/hfmawit, half(fma(a, 0.5f, b))). */
    const struct agx3_field_kv base[] = {
-      KV(9, 2), KV(10, 2), KV(25, 2), KV(26, 2), KV(42, 1),
+      KV(9, 2), KV(10, dest_half ? 1 : 2), KV(25, 2), KV(26, 2), KV(42, 1),
       KV(43, 2), KV(59, 2), KV(60, 2), KV(77, 2), KV(91, 2),
+      /* Native G17G retains operand fields and its own long endpoint. */
+      KV(95, 3), KV(96, 55),
    };
    /* Appended rather than counted: a designated-initialiser list with a
     * hand-written length silently grows a zeroed {field 0, value 0} entry the
@@ -476,12 +660,45 @@ agx3_falu_fma_mul_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
    ov[n++] = (struct agx3_field_kv)KV(29, runtime->discard[1]);
    ov[n++] = (struct agx3_field_kv)KV(61, runtime->cache[2]);
    ov[n++] = (struct agx3_field_kv)KV(63, runtime->discard[2]);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
       ov[n++] = (struct agx3_field_kv)KV(97 + i, (runtime->wait_mask >> i) & 1);
 
-   for (unsigned ep = 0; ep <= 5; ++ep) {
-      if (agx3_bind_compose(enc, "f32fma", ep, base, 10, ov, n, out, len))
+   for (unsigned ep = 0; ep <= (m5 ? 7 : 5); ++ep) {
+      if (agx3_bind_compose(enc, "f32fma", ep, base, m5 ? 12 : 10, ov, n, out, len))
          return true;
    }
    return false;
 }
+
+bool
+agx3_falu_fma_mul_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
+                          uint8_t multiplier, uint16_t src1, uint16_t addend,
+                          const struct agx3_falu_mods *m,
+                          const struct agx3_falu_runtime *runtime,
+                          uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return fma_mul_imm(enc, dest, false, multiplier, src1, addend, m, runtime,
+                      out, len);
+}
+
+bool
+agx3_falu_fma_mul_imm_narrow_m5(uint16_t dest, uint8_t multiplier,
+                                uint16_t src1, uint16_t addend,
+                                const struct agx3_falu_mods *m,
+                                const struct agx3_falu_runtime *runtime,
+                                uint8_t out[AGX3_FALU_MOD_MAX_LENGTH],
+                                uint8_t *len)
+{
+   return fma_mul_imm(AGX_ISA_ENCODING_AGX3C, dest, true, multiplier, src1,
+                      addend, m, runtime, out, len);
+}
+
+bool
+agx3_falu_add_imm_narrow_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+                         bool src_abs, bool src_neg, bool saturate,
+                         const struct agx3_falu_runtime *runtime,
+                         uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len)
+{
+   return add_immediate(AGX_ISA_ENCODING_AGX3C, dest, src, true, immediate,
+                        src_abs, src_neg, saturate, runtime, out, len);
+}
diff --git a/src/asahi/compiler/agx3_falu_mod_core.h b/src/asahi/compiler/agx3_falu_mod_core.h
index b85e486df51e..2d98df13c482 100644
--- a/src/asahi/compiler/agx3_falu_mod_core.h
+++ b/src/asahi/compiler/agx3_falu_mod_core.h
@@ -121,6 +121,30 @@ bool agx3_falu_fma_mod_runtime(enum agx_isa_encoding enc, uint16_t dest,
                               uint8_t out[AGX3_FALU_MOD_MAX_LENGTH],
                               uint8_t *len);
 
+/* Native add/multiply with a single rounding to a scalar half result. */
+bool agx3_falu_add_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+                            const struct agx3_falu_mods *mods,
+                            const struct agx3_falu_runtime *runtime,
+                            uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+bool agx3_falu_mul_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+                            const struct agx3_falu_mods *mods,
+                            const struct agx3_falu_runtime *runtime,
+                            uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+
+/* Native F32 arithmetic with one final rounding directly to a half result. */
+bool agx3_falu_fma_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+                            uint16_t addend, const struct agx3_falu_mods *m,
+                            const struct agx3_falu_runtime *runtime,
+                            uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+
+/* C FMA32 also consumes half GPRs with an immediate addend. Preserve the
+ * independent widths, modifiers, lifetime and saturation in one instruction. */
+bool agx3_falu_fma_addend_imm_width_mod(
+   enum agx_isa_encoding enc, uint16_t dest, uint16_t src0, uint16_t src1,
+   uint8_t immediate, const struct agx3_falu_mods *m,
+   const struct agx3_falu_runtime *runtime,
+   uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+
 bool agx3_falu_fma_addend_imm_mod(
    enum agx_isa_encoding enc, uint16_t dest, uint16_t src0, uint16_t src1,
    uint8_t immediate, bool src0_abs, bool src0_neg, bool src1_abs,
@@ -133,6 +157,11 @@ bool agx3_falu_fma_two_imm(enum agx_isa_encoding enc, uint16_t dest,
                           const struct agx3_falu_runtime *runtime,
                           uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
 
+bool agx3_falu_fma_two_imm_narrow_m5(
+   uint16_t dest, uint16_t src, uint8_t multiplier, uint8_t addend,
+   bool src_abs, bool saturate, const struct agx3_falu_runtime *runtime,
+   uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+
 /*
  * FMA whose MULTIPLIER is a minifloat immediate while the multiplicand and the
  * addend stay registers that carry their own modifiers and widths.
@@ -168,6 +197,27 @@ bool agx3_falu_fma_mul_imm_mod(enum agx_isa_encoding enc, uint16_t dest,
                               const struct agx3_falu_runtime *runtime,
                               uint8_t out[AGX3_FALU_MOD_MAX_LENGTH],
                               uint8_t *len);
+/* The same G17G form narrowing its result into a scalar half (f10=1). */
+bool agx3_falu_fma_mul_imm_narrow_m5(uint16_t dest, uint8_t multiplier,
+                                     uint16_t src1, uint16_t addend,
+                                     const struct agx3_falu_mods *m,
+                                     const struct agx3_falu_runtime *runtime,
+                                     uint8_t out[AGX3_FALU_MOD_MAX_LENGTH],
+                                     uint8_t *len);
+
+bool agx3_falu_add_imm_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+                         bool src_abs, bool src_neg, bool saturate,
+                         const struct agx3_falu_runtime *runtime,
+                         uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+
+bool agx3_falu_add_imm_narrow_m5(uint16_t dest, uint16_t src, uint8_t immediate,
+   bool src_abs, bool src_neg, bool saturate,
+   const struct agx3_falu_runtime *runtime,
+   uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
+bool agx3_falu_fma_addend_imm_narrow_m5(uint16_t dest, uint16_t src0, uint16_t src1,
+   uint8_t immediate, const struct agx3_falu_mods *mods,
+   const struct agx3_falu_runtime *runtime,
+   uint8_t out[AGX3_FALU_MOD_MAX_LENGTH], uint8_t *len);
 
 #ifdef __cplusplus
 }
diff --git a/src/asahi/compiler/agx3_fltnorm_core.c b/src/asahi/compiler/agx3_fltnorm_core.c
index 199cb95780fe..5227e473dfce 100644
--- a/src/asahi/compiler/agx3_fltnorm_core.c
+++ b/src/asahi/compiler/agx3_fltnorm_core.c
@@ -140,8 +140,6 @@ agx3_fltnorm_encode(enum agx_isa_encoding enc,
       return false;
    if (d->two_lanes != (d->src[1].kind != AGX3_FLTNORM_SRC_NONE))
       return false;
-   if (d->is_16bit && enc != AGX_ISA_ENCODING_AGX3B)
-      return false;
 
    static const struct src_fields s0 = {
       .f_kind = 23, .f_imm = 15, .f_reg = 16, .f_uniform = 17,
@@ -308,9 +306,9 @@ agx3_persp_rcp_encode(enum agx_isa_encoding enc,
                        const struct agx3_persp_rcp_desc *d,
                        uint8_t out[AGX3_FLTNORM_MAX_LENGTH], uint8_t *len)
 {
-   if (!out || !len || !d || !b_only(enc) || d->dest >= 192 ||
-       d->rhw >= 192 || ((d->dest | d->rhw) & 1) ||
-       d->coef_index > 0x7f || (d->wait_mask & ~UINT16_C(0x3f)))
+   if (!out || !len || !d || !enc_ok(enc) || d->dest >= (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 192) ||
+       d->rhw >= (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 192) || ((d->dest | d->rhw) & 1) ||
+       d->coef_index > 0x7f || (d->wait_mask & ~wait_limit(enc)))
       return false;
 
    /* Fresh native smooth_scalar: af005400030000482001 (r0 -> r0).
@@ -321,12 +319,16 @@ agx3_persp_rcp_encode(enum agx_isa_encoding enc,
       KV(9, 2), KV(10, 2), KV(25, 2), KV(26, 2),
       KV(42, 6), KV(43, 2), KV(57, 1),
    };
-   struct agx3_field_kv ov[11] = {
+   struct agx3_field_kv ov[15] = {
       KV(2, d->dest), KV(11, 1), KV(18, d->rhw),
       KV(27, d->rhw_cache), KV(39, d->coef_index),
    };
    unsigned n = 5;
-   for (unsigned i = 0; i < 6; ++i)
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      ov[n++] = (struct agx3_field_kv)KV(47, 3);
+      ov[n++] = (struct agx3_field_kv)KV(48, 64);
+   }
+   for (unsigned i = 0; i < nslots(enc); ++i)
       if (d->wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(49 + i, 1);
    return agx3_bind_compose(enc, "f32rcp", 0, base, 7, ov, n, out, len);
@@ -342,12 +344,12 @@ agx3_persp_mul_encode(enum agx_isa_encoding enc,
    memset(out, 0, AGX3_FLTNORM_MAX_LENGTH);
    *len = 0;
 
-   if (!b_only(enc))
+   if (!enc_ok(enc))
       return false;
    if ((d->dest & 1) || d->dest >= 256 || (d->reciprocal & 1) ||
        d->reciprocal >= 256 || (d->raw & 1) || d->raw >= 256)
       return false;
-   if ((d->wait_mask & ~(uint16_t)0x3f) != 0)
+   if ((d->wait_mask & ~wait_limit(enc)) != 0)
       return false;
    /* f70 is seven bits wide. */
    if (d->vary_index > 0x7f)
@@ -364,7 +366,7 @@ agx3_persp_mul_encode(enum agx_isa_encoding enc,
       KV(59, 6), KV(60, 2), KV(73, 6), KV(74, 2), KV(88, 1),
    };
 
-   struct agx3_field_kv ov[16];
+   struct agx3_field_kv ov[20];
    unsigned n = 0;
    if (d->stage_dest) {
       ov[n++] = (struct agx3_field_kv)KV(9, 5);
@@ -387,7 +389,11 @@ agx3_persp_mul_encode(enum agx_isa_encoding enc,
    }
    ov[n++] = (struct agx3_field_kv)KV(70, d->vary_index);
 
-   for (unsigned i = 0; i < 6; ++i)
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      ov[n++] = (struct agx3_field_kv)KV(78, 3);
+      ov[n++] = (struct agx3_field_kv)KV(79, 62);
+   }
+   for (unsigned i = 0; i < nslots(enc); ++i)
       if (d->wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(80 + i, 1);
 
@@ -417,8 +423,6 @@ agx3_normflt_encode(enum agx_isa_encoding enc,
 
    if (!enc_ok(enc) || (d->dest & 1) || d->dest >= 256 || d->src >= 256)
       return false;
-   if (d->is_16bit && enc != AGX_ISA_ENCODING_AGX3B)
-      return false;
    if ((d->wait_mask & ~wait_limit(enc)) != 0)
       return false;
 
@@ -468,11 +472,12 @@ agx3_normflt_encode(enum agx_isa_encoding enc,
  * One native conversion replaces per-channel multiply/convert/shift/or work.
  */
 bool
-agx3_rgb10a2_encode(const struct agx3_rgb10a2_desc *d,
-                    uint8_t out[14], uint8_t *length)
+agx3_rgb10a2_encode_for_encoding(enum agx_isa_encoding enc,
+                    const struct agx3_rgb10a2_desc *d,
+                    uint8_t out[16], uint8_t *length)
 {
-   if (!d || !out || !length || d->dest >= 256 || (d->dest & 1) ||
-       (d->wait_mask & ~0x3f) || d->layout > AGX3_PACKED_RG11B10)
+   if (!enc_ok(enc) || !d || !out || !length || d->dest >= 256 || (d->dest & 1) ||
+       (d->wait_mask & ~wait_limit(enc)) || d->layout > AGX3_PACKED_RG11B10)
       return false;
    const struct agx3_field_kv base[] = {KV(9, 2), KV(10, 2)};
    struct agx3_field_kv fields[48];
@@ -505,39 +510,63 @@ agx3_rgb10a2_encode(const struct agx3_rgb10a2_desc *d,
       }
    }
    /* Fresh texture-to-RGB10 native main waits for smp slot1 at field84. */
-   for (unsigned i = 0; i < 6; ++i)
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      fields[n++] = (struct agx3_field_kv)KV(82, 3);
+      fields[n++] = (struct agx3_field_kv)KV(83, 116);
+   }
+   for (unsigned i = 0; i < nslots(enc); ++i)
       if (d->wait_mask & (1u << i))
          fields[n++] = (struct agx3_field_kv)KV(84 + i, 1);
    fields[n++] = (struct agx3_field_kv)KV(102, d->first_instruction);
    fields[n++] = (struct agx3_field_kv)KV(94, 0);
    fields[n++] = (struct agx3_field_kv)KV(95, 0);
    fields[n++] = (struct agx3_field_kv)KV(96, 0);
-   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3B, "pack", 0,
+   return agx3_bind_compose(enc, "pack", 0,
                            base, 2, fields, n, out, length);
 }
 
+bool
+agx3_rgb10a2_encode(const struct agx3_rgb10a2_desc *d,
+                    uint8_t out[14], uint8_t *length)
+{
+   uint8_t bytes[16];
+   if (!agx3_rgb10a2_encode_for_encoding(AGX_ISA_ENCODING_AGX3B,
+                                        d, bytes, length))
+      return false;
+   memcpy(out, bytes, *length);
+   return true;
+}
+
 /* Fresh single- and two-attachment fetch programs identify the source register
  * and slot1/slot6 wait fields independently. The result is four contiguous f32s. */
 bool
-agx3_rgb10a2_unpack_encode(const struct agx3_rgb10a2_unpack_desc *d,
-                         uint8_t out[8], uint8_t *length)
+agx3_rgb10a2_unpack_encode(
+   enum agx_isa_encoding enc, const struct agx3_rgb10a2_unpack_desc *d,
+   uint8_t out[16], uint8_t *length)
 {
-   if (!d || !out || !length || d->dest > 248 || d->source > 254 ||
-       ((d->dest | d->source) & 1) || (d->wait_mask & ~0x3f) ||
+   if (!enc_ok(enc) || !d || !out || !length || d->dest > 248 ||
+       d->source > 254 || ((d->dest | d->source) & 1) ||
+       (d->wait_mask & ~wait_limit(enc)) ||
        d->layout > AGX3_PACKED_RG11B10)
       return false;
-   const struct agx3_field_kv base[] = {
+   struct agx3_field_kv base[7] = {
       KV(9, 2), KV(10, 2), KV(25, 2), KV(26, 2),
    };
-   /* Seven fixed fields plus all six scoreboard wait bits. */
-   struct agx3_field_kv fields[13] = {
+   unsigned nbase = 4;
+   /* G17G UInt1010102Normalized and FloatRG11B10 vertex witnesses. */
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      base[nbase++] = (struct agx3_field_kv)KV(11, d->dest_hint);
+      base[nbase++] = (struct agx3_field_kv)KV(30, 3);
+      base[nbase++] = (struct agx3_field_kv)KV(31, 191);
+   }
+   struct agx3_field_kv fields[15] = {
       KV(2, d->dest), KV(18, d->source), KV(29, d->discard),
       KV(40, d->layout), KV(41, 0), KV(42, 0), KV(43, 0),
    };
    unsigned n = 7;
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < nslots(enc); ++i)
       if (d->wait_mask & (1u << i))
          fields[n++] = (struct agx3_field_kv)KV(32 + i, 1);
-   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3B, "unpack", 0,
-                           base, 4, fields, n, out, length);
+   return agx3_bind_compose(enc, "unpack", 0,
+                           base, nbase, fields, n, out, length);
 }
diff --git a/src/asahi/compiler/agx3_fltnorm_core.h b/src/asahi/compiler/agx3_fltnorm_core.h
index e8479505297f..7a5ddfb268a1 100644
--- a/src/asahi/compiler/agx3_fltnorm_core.h
+++ b/src/asahi/compiler/agx3_fltnorm_core.h
@@ -197,6 +197,9 @@ struct agx3_rgb10a2_desc {
    bool neg[4];
    uint8_t layout; /* enum agx3_packed_layout */
 };
+bool agx3_rgb10a2_encode_for_encoding(enum agx_isa_encoding enc,
+                        const struct agx3_rgb10a2_desc *desc,
+                        uint8_t out[16], uint8_t *length);
 bool agx3_rgb10a2_encode(const struct agx3_rgb10a2_desc *desc,
                         uint8_t out[14], uint8_t *length);
 
@@ -204,10 +207,12 @@ struct agx3_rgb10a2_unpack_desc {
    uint16_t dest, source; /* half indices: four f32 outputs, one u32 input */
    uint16_t wait_mask;
    bool discard;
+   bool dest_hint; /* AGX3C f11 */
    uint8_t layout; /* enum agx3_packed_layout */
 };
-bool agx3_rgb10a2_unpack_encode(const struct agx3_rgb10a2_unpack_desc *desc,
-                               uint8_t out[8], uint8_t *length);
+bool agx3_rgb10a2_unpack_encode(
+   enum agx_isa_encoding enc, const struct agx3_rgb10a2_unpack_desc *desc,
+   uint8_t out[16], uint8_t *length);
 
 struct agx3_fltnorm_desc {
    uint16_t dest;                 /* 16-bit (half) register index */
@@ -215,7 +220,7 @@ struct agx3_fltnorm_desc {
    struct agx3_fltnorm_src src[2];
    bool two_lanes;                /* pack both sources; false = one lane */
    bool snorm;                    /* signed-normalized target */
-   bool is_16bit;                 /* 16-bit normalized lane, G17P only */
+   bool is_16bit;                 /* 16-bit normalized lane */
    uint8_t srgb;                  /* enum agx3_fltnorm_srgb */
    uint16_t wait_mask;            /* bit i waits scoreboard slot i + 1 */
    bool first_instruction;       /* G17P f68, first main instruction only */
@@ -236,8 +241,8 @@ bool agx3_fltnorm_relocate_start(uint8_t *code, size_t size,
 struct agx3_colour_link_info {
    /* Producer-recorded native pixel-entry provenance for part linking.
     * straight_line admits a derivative-consumer entry gate in this part;
-    * branch_free says no branch crosses any byte of the part, so the
-    * linker may remove this part's entry gate. */
+    * entry_unspanned proves that deleting the entry leaves every relative
+    * branch displacement unchanged. */
    bool rt_query_write_acquired;
    uint8_t rt_query_wait_count;
    bool rt_query_wait_unsupported;
@@ -245,8 +250,14 @@ struct agx3_colour_link_info {
    uint32_t rt_query_wait_offset;
    bool entry_emitted;
    bool straight_line;
-   bool branch_free;
+   bool entry_unspanned;
    uint32_t entry_offset;
+   /* M5: this part ends a covered-sample depth/stencil loop with a plain
+    * class-1 release at zs_loop_release_offset. When this part's entry is
+    * the linked program's, agx_fast_link drops that release and marks the
+    * entry with f34, the native form. */
+   bool zs_loop_release;
+   uint32_t zs_loop_release_offset;
 
    /* Colour pixel classes consumed by this part: bit n is the class of the
     * tile partitions n, n + 3, ...  The linked fragment state loader must keep
diff --git a/src/asahi/compiler/agx3_funop_core.c b/src/asahi/compiler/agx3_funop_core.c
index 876793c7d908..d076d9ebd6ca 100644
--- a/src/asahi/compiler/agx3_funop_core.c
+++ b/src/asahi/compiler/agx3_funop_core.c
@@ -99,6 +99,13 @@ is_b(enum agx_isa_encoding e)
    return e == AGX_ISA_ENCODING_AGX3B;
 }
 
+bool
+agx3_funop_carries_wait(enum agx_isa_encoding enc, enum agx3_funop op)
+{
+   return (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+          op < AGX3_FUNOP_COUNT && forms[op].wait_ok;
+}
+
 bool
 agx3_funop_encode_source_lifetime(
    enum agx_isa_encoding enc, enum agx3_funop op, uint16_t dest, uint16_t src,
diff --git a/src/asahi/compiler/agx3_funop_core.h b/src/asahi/compiler/agx3_funop_core.h
index bb69728a9ae2..86e56078de81 100644
--- a/src/asahi/compiler/agx3_funop_core.h
+++ b/src/asahi/compiler/agx3_funop_core.h
@@ -56,6 +56,11 @@ bool agx3_funop_encode_source_lifetime(
    bool src_abs, bool src_neg, bool saturate, uint16_t wait_mask, bool src_live,
    uint8_t out[AGX3_FUNOP_MAX_LENGTH], uint8_t *len);
 
+/* Whether the witnessed form of `op` has scoreboard wait fields. A caller
+ * with an outstanding dependency on a form without them must retire it
+ * first (as for the gradients). */
+bool agx3_funop_carries_wait(enum agx_isa_encoding enc, enum agx3_funop op);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/asahi/compiler/agx3_graphics_program.h b/src/asahi/compiler/agx3_graphics_program.h
index f4b2410a8fa9..ccd1609b99ab 100644
--- a/src/asahi/compiler/agx3_graphics_program.h
+++ b/src/asahi/compiler/agx3_graphics_program.h
@@ -52,7 +52,6 @@ struct agx3_g17p_graphics_program_desc {
    uint16_t nr_gprs;
    bool compiler_d_valid;
    uint32_t compiler_d;
-   uint32_t threadgroup_memory_size_B;
    bool fragment;
    bool writes_sample_mask;
    uint8_t colour_classes;
@@ -65,6 +64,9 @@ struct agx3_g17p_graphics_program_desc {
     * materialize every absolute range themselves. */
    bool preloaded_root;
 
+   /* Shared memory in bytes for a compute, mesh or object launch. */
+   uint32_t threadgroup_memory_size_B;
+
    /* Address-independent shape (agx3_ioto_uniform_program_desc::patchable):
     * every range materializes its own base with fixed-size LDIMMs, so the
     * recorded relocations can be patched for other addresses. */
@@ -84,6 +86,24 @@ struct agx3_g17p_graphics_program {
    uint16_t length;
 };
 
+bool agx3_g17g_graphics_program_bound(
+   enum agx_chip chip, const struct agx3_ioto_uniform_range *ranges,
+   size_t range_count, size_t *size);
+bool agx3_g17g_graphics_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program *program);
+/* G17S-only native TA mesh/object program composition. Shader entry ABI and
+ * resources must already be supplied; this does not advertise mesh support. */
+bool agx3_g17g_mesh_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   bool object, struct agx3_g17p_graphics_program *program);
+bool agx3_g17g_compute_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program *program);
+bool agx3_g17g_tile_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   bool store, struct agx3_g17p_graphics_program *program);
+
 struct agx3_g17p_graphics_program_template {
    struct agx3_g17p_graphics_program program;
    struct agx_ioto_load_relocation
@@ -92,6 +112,22 @@ struct agx3_g17p_graphics_program_template {
    uint16_t constant_suffix_offset;
 };
 
+
+
+bool agx3_g17g_graphics_program_template_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program_template *program);
+
+/* Capture the exact address materializations of the normal M5 compute
+ * program. Relocations refer to bytes of the final composed program. */
+bool agx3_g17g_compute_program_template_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program_template *program);
+
+bool agx3_g17g_tile_program_template_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   bool store, struct agx3_g17p_graphics_program_template *program);
+
 enum agx3_g17p_rt0_bg_op {
    AGX3_G17P_RT0_BG_CLEAR,
    AGX3_G17P_RT0_BG_RELOAD,
diff --git a/src/asahi/compiler/agx3_iadd_core.c b/src/asahi/compiler/agx3_iadd_core.c
index 5c56130904f4..0541de377162 100644
--- a/src/asahi/compiler/agx3_iadd_core.c
+++ b/src/asahi/compiler/agx3_iadd_core.c
@@ -35,12 +35,24 @@ agx3_iadd_encode(enum agx_isa_encoding encoding,
     * shifted adds witness 1..4; shift 0 is the unshifted form. */
    if (d->shift > 4 || d->wait_mask >= (1u << slots))
       return false;
-   if ((d->subtract || d->saturate || d->src_a_cache || d->src_cache) &&
-       !agx3b)
+   const bool m5_imm_sub = !agx3b && d->subtract &&
+      !d->src_a_is_register && !d->saturate && !d->shift;
+   const bool m5_reg_sub = !agx3b && d->subtract &&
+      d->src_a_is_register && !d->saturate;
+   /* Native G17G (port/cwit3) addsat/subsat on uint and int: the iadd32 or
+    * isub32 register point with f54; the signed form also sets f28/f43:
+    *   2f00041a2100a30238812000  r0 <- ^r0 +sat ^r2, unsigned
+    *   2f00041a2100a3023881e000  r0 <- ^r0 +sat ^r2, signed
+    *   2f00041a2500a20218812000  r0 <- ^r0 -sat ^r2, unsigned */
+   const bool m5_sat = !agx3b && d->src_a_is_register &&
+      !d->src_a_16 && !d->src_16 && !d->shift && d->saturate &&
+      d->src_a_sign_extend == d->sign_extend;
+   if (!agx3b && (d->subtract || d->saturate) &&
+       !m5_imm_sub && !m5_reg_sub && !m5_sat)
       return false;
    if ((d->saturate || d->src_a_cache) && !d->src_a_is_register)
       return false;
-   if (d->src_cache && !d->src_a_is_register)
+   if (d->src_cache && !d->src_a_is_register && agx3b)
       return false;
    if ((d->src_a_cache && d->src_a_kill) || (d->src_cache && d->src_kill))
       return false;
@@ -82,11 +94,11 @@ agx3_iadd_encode(enum agx_isa_encoding encoding,
     */
    const struct agx3_field_kv baseline_c_reg[] = {
       KV(9, 2),  KV(10, 2), KV(23, 2), KV(24, d->src_a_16 ? 1 : 2),
-      KV(38, 2), KV(39, d->src_16 ? 1 : 2), KV(44, 3), KV(45, 80),
+      KV(38, 2), KV(39, d->src_16 ? 1 : 2), KV(44, 3), KV(45, d->subtract ? 100 : 80),
    };
    const struct agx3_field_kv baseline_c[] = {
       KV(9, 2),  KV(10, 2), KV(23, 1), KV(24, 1),
-      KV(38, 2), KV(42, 1), KV(44, 3),  KV(45, 80),
+      KV(38, 2), KV(42, 1), KV(44, 3),  KV(45, d->subtract ? 100 : 80),
    };
    const struct agx3_field_kv *baseline =
       d->src_a_is_register ? (agx3b ? baseline_b_reg : baseline_c_reg)
@@ -113,16 +125,16 @@ agx3_iadd_encode(enum agx_isa_encoding encoding,
          ov[n++] = (struct agx3_field_kv)KV(25, 1);
       if (d->src_cache)
          ov[n++] = (struct agx3_field_kv)KV(40, 1);
-      if (d->saturate)
+      if (d->saturate && agx3b)
          ov[n++] = (struct agx3_field_kv)KV(53, 1);
    }
    else
       ov[n++] = (struct agx3_field_kv)KV(15, d->immediate);
-   /* f42 is source liveness for both immediate and register slot A.
-    * In particular, an address add must preserve an ID used by later stores.
-    * Keep the separately witnessed AGX3c baseline unchanged here. */
-   if (agx3b && !d->src_a_is_register)
+   /* The immediate form must preserve live inputs, like the register form. */
+   if (!d->src_a_is_register)
       ov[n++] = (struct agx3_field_kv)KV(42, d->src_kill ? 1 : 0);
+   if (!d->src_a_is_register && d->src_cache)
+      ov[n++] = (struct agx3_field_kv)KV(40, 1);
    ov[n++] = (struct agx3_field_kv)KV(31, d->src);
    if (!d->src_a_is_register)
       ov[n++] = (struct agx3_field_kv)KV(39, d->src_16 ? 1 : 2);
@@ -130,26 +142,32 @@ agx3_iadd_encode(enum agx_isa_encoding encoding,
       ov[n++] = (struct agx3_field_kv)KV(43, 1);
    if (d->shift)
       ov[n++] = (struct agx3_field_kv)KV(agx3b ? 54 : 55, d->shift);
+   if (m5_sat)
+      ov[n++] = (struct agx3_field_kv)KV(54, 1);
    for (unsigned i = 0; i < slots; ++i) {
       if (d->wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV((agx3b ? 45 : 46) + i, 1);
    }
 
    return agx3_bind_compose(encoding,
-                            agx3b ? (d->subtract ? "isub" : "iadd") : "iadd32",
+                            agx3b ? (d->subtract ? "isub" : "iadd")
+                                  : (d->subtract ? "isub32" : "iadd32"),
                             0, baseline, nbaseline, ov, n, out, len);
 }
 
-bool
-agx3_isub_immediate_encode(enum agx_isa_encoding encoding,
+static bool
+agx3_add_sub_immediate_encode(enum agx_isa_encoding encoding, bool subtract,
                             const struct agx3_isub_imm_desc *d,
                             uint8_t out[AGX3_IADD_MAX_LENGTH], uint8_t *len)
 {
-   if (!d || !out || !len || encoding != AGX_ISA_ENCODING_AGX3B ||
+   const bool gpu1 = encoding == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if (!d || !out || !len ||
+       (encoding != AGX_ISA_ENCODING_AGX3B && !gpu1) || (!subtract && !gpu1) ||
        d->dest > 254 || (d->dest & 1) || d->src >= 256 ||
        (!d->src_16 && ((d->src & 1) || d->sign_extend)) ||
        d->immediate > 255 || (d->src_cache && d->src_kill) ||
-       (d->wait_mask & ~UINT16_C(0x3f)))
+       (d->wait_mask & ~((1u << slots) - 1)))
       return false;
 
    /* Native ushort/short/uint minus1/7/28/255, with paired live/dead
@@ -159,17 +177,37 @@ agx3_isub_immediate_encode(enum agx_isa_encoding encoding,
     */
    const struct agx3_field_kv base[] = {
       KV(9,2), KV(10,2), KV(23,2), KV(24,d->src_16 ? 1 : 2),
-      KV(38,1), KV(39,1),
+      KV(38,1), KV(39,1), KV(44,3), KV(45,subtract ? 100 : 80),
    };
-   struct agx3_field_kv fields[13] = {
+   struct agx3_field_kv fields[15] = {
       KV(2,d->dest), KV(16,d->src), KV(30,d->immediate),
       KV(25,d->src_cache), KV(27,d->src_kill), KV(28,d->sign_extend),
+      KV(11,d->dest_cache),
    };
-   unsigned n=6;
-   for (unsigned slot=0;slot<6;slot++)
+   unsigned n=7;
+   /* HAL300 AGX3c isub32 retains the operand fields, with explicit GPR
+    * dispatch/opcode and eight dependencies in fields 46..53. */
+   for (unsigned slot=0;slot<slots;slot++)
       if(d->wait_mask & (1u<<slot))
-         fields[n++]=(struct agx3_field_kv)KV(45+slot,1);
-   return agx3_bind_compose(encoding,"isub",0,base,6,fields,n,out,len);
+         fields[n++]=(struct agx3_field_kv)KV((gpu1 ? 46 : 45)+slot,1);
+   return agx3_bind_compose(encoding,gpu1 ? (subtract ? "isub32" : "iadd32") : "isub",0,
+                            base,gpu1 ? 8 : 6,fields,n,out,len);
+}
+
+bool
+agx3_isub_immediate_encode(enum agx_isa_encoding encoding,
+                           const struct agx3_isub_imm_desc *d,
+                           uint8_t out[AGX3_IADD_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_add_sub_immediate_encode(encoding, true, d, out, len);
+}
+
+bool
+agx3_iadd_immediate_encode(enum agx_isa_encoding encoding,
+                           const struct agx3_isub_imm_desc *d,
+                           uint8_t out[AGX3_IADD_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_add_sub_immediate_encode(encoding, false, d, out, len);
 }
 
 bool
@@ -180,8 +218,14 @@ agx3_imad_encode(enum agx_isa_encoding encoding,
    if (d == NULL || out == NULL || len == NULL)
       return false;
 
-   /* Only the AGX3b form is witnessed. */
-   if (encoding != AGX_ISA_ENCODING_AGX3B)
+   const bool m5 = encoding == AGX_ISA_ENCODING_AGX3C;
+   if (encoding != AGX_ISA_ENCODING_AGX3B && !m5)
+      return false;
+
+   if (m5 && ((d->src_c_16 && d->dest_64) || (d->src_c_64 && !d->dest_64) ||
+              (d->dest_64 && d->no_addend) ||
+              (!d->dest_64 && d->src_a_sign_extend && !d->src_a_16) ||
+              d->src_c_sign_extend))
       return false;
 
    if (d->dest_16 && d->dest_64)
@@ -217,8 +261,8 @@ agx3_imad_encode(enum agx_isa_encoding encoding,
    if (d->dest_64) {
       /* A 32->64 product widens operands that are already 32-bit, so a narrow
        * operand carrying the bit is unwitnessed here. */
-      if ((d->src_a_sign_extend && d->src_a_16) ||
-          (d->multiplier_sign_extend && d->multiplier_16))
+      if (!m5 && ((d->src_a_sign_extend && d->src_a_16) ||
+                  (d->multiplier_sign_extend && d->multiplier_16)))
          return false;
    } else if (!d->dest_16) {
       /* A 32-bit product widens each 16-bit operand by exactly one step, by
@@ -247,7 +291,7 @@ agx3_imad_encode(enum agx_isa_encoding encoding,
          return false;
       if ((d->src_a_16 || d->multiplier_16) && d->addend_is_immediate)
          return false;
-      if (d->multiplier_16 && !d->no_addend &&
+      if (!m5 && d->multiplier_16 && !d->no_addend &&
           (d->src_c_16 || d->src_c_64))
          return false;
    }
@@ -271,7 +315,7 @@ agx3_imad_encode(enum agx_isa_encoding encoding,
         d->src_a_sign_extend || d->multiplier_sign_extend ||
         d->src_c_sign_extend))
       return false;
-   if (d->wait_mask >= (1u << 6))
+   if (d->wait_mask >= (1u << (m5 ? 8 : 6)))
       return false;
 
    if (d->multiplier_is_register &&
@@ -321,7 +365,7 @@ agx3_imad_encode(enum agx_isa_encoding encoding,
          ? (sizeof(baseline_reg) / sizeof(baseline_reg[0]))
          : (sizeof(baseline_imm) / sizeof(baseline_imm[0]));
 
-   struct agx3_field_kv ov[24];
+   struct agx3_field_kv ov[28];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(2, d->dest);
    ov[n++] = (struct agx3_field_kv)KV(11, d->dest_cache);
@@ -353,17 +397,23 @@ agx3_imad_encode(enum agx_isa_encoding encoding,
    }
    if (d->src_c_sign_extend)
       ov[n++] = (struct agx3_field_kv)KV(58, 1);
-   for (unsigned i = 0; i < 6; ++i) {
+   if (m5) {
+      ov[n++] = (struct agx3_field_kv)KV(59, 3);
+      ov[n++] = (struct agx3_field_kv)KV(60, d->dest_64 ? 85 : 84);
+   }
+   for (unsigned i = 0; i < (m5 ? 8 : 6); ++i) {
       if (d->wait_mask & (1u << i))
-         ov[n++] = (struct agx3_field_kv)KV(60 + i, 1);
+         ov[n++] = (struct agx3_field_kv)KV((m5 ? 61 : 60) + i, 1);
    }
 
    /* A negated addend is Apple's `imsub`: same fields, one opcode bit. A
     * pure multiply has no addend to negate, so refuse that rather than emit a
     * form nothing witnesses. */
-   if (d->negate_addend && d->no_addend)
+   if (d->negate_addend && (d->no_addend || m5))
       return false;
 
-   return agx3_bind_compose(encoding, d->negate_addend ? "imsub" : "imadd", 0,
+   return agx3_bind_compose(encoding,
+                            m5 ? (d->dest_64 ? "imadd64" : "imadd32")
+                               : (d->negate_addend ? "imsub" : "imadd"), 0,
                             baseline, nbaseline, ov, n, out, len);
 }
diff --git a/src/asahi/compiler/agx3_iadd_core.h b/src/asahi/compiler/agx3_iadd_core.h
index 845480960bc4..cf0cae06407d 100644
--- a/src/asahi/compiler/agx3_iadd_core.h
+++ b/src/asahi/compiler/agx3_iadd_core.h
@@ -42,7 +42,7 @@
 extern "C" {
 #endif
 
-#define AGX3_IADD_MAX_LENGTH 12u
+#define AGX3_IADD_MAX_LENGTH 14u
 
 struct agx3_iadd_desc {
    /*
@@ -124,6 +124,7 @@ bool agx3_iadd_encode(enum agx_isa_encoding encoding,
  * not commuted: the register occupies slot A, immediate slot B. */
 struct agx3_isub_imm_desc {
    uint16_t dest, src, immediate;
+   bool dest_cache;
    bool src_16, sign_extend;
    bool src_cache, src_kill;
    uint16_t wait_mask;
@@ -131,6 +132,9 @@ struct agx3_isub_imm_desc {
 bool agx3_isub_immediate_encode(enum agx_isa_encoding encoding,
                                  const struct agx3_isub_imm_desc *desc,
                                  uint8_t out[AGX3_IADD_MAX_LENGTH], uint8_t *len);
+bool agx3_iadd_immediate_encode(enum agx_isa_encoding encoding,
+                                 const struct agx3_isub_imm_desc *d,
+                                 uint8_t out[AGX3_IADD_MAX_LENGTH], uint8_t *len);
 
 /*
  * `imad dst32, srcA, #imm, srcC` -- multiply-accumulate with a constant
diff --git a/src/asahi/compiler/agx3_ioto_load_packer.c b/src/asahi/compiler/agx3_ioto_load_packer.c
index 7f3b05c35644..4b168609cd33 100644
--- a/src/asahi/compiler/agx3_ioto_load_packer.c
+++ b/src/asahi/compiler/agx3_ioto_load_packer.c
@@ -226,9 +226,7 @@ pack_absolute_copy(void *context, enum agx_chip chip,
       return false;
 
    uint16_t destination_half, source_half;
-   const uint16_t destination_stride =
-      packer->encoding == AGX_IOTO_LOAD_ENCODING_GPU0 ? 2 : 4;
-   if (!add_half(load->native_slot_half, (uint16_t)(destination_stride * dword),
+   if (!add_half(load->native_slot_half, (uint16_t)(2u * dword),
                  &destination_half) ||
        !add_half(load->data.half, (uint16_t)(2u * dword), &source_half))
       return false;
@@ -243,7 +241,8 @@ pack_absolute_copy(void *context, enum agx_chip chip,
       if (!packer->copy_wait_emitted) {
          if (load->signal_slot == 0 || load->signal_slot > 8)
             return false;
-         wait_slot = load->signal_slot;
+         /* LDDEV slots are one-based; COPY wait fields are zero-based. */
+         wait_slot = load->signal_slot - 1;
       }
    } else {
       return false;
@@ -266,7 +265,8 @@ pack_cprog_copy(void *context, enum agx_chip chip,
                 struct agx_ioto_load_raw_record records[2], uint8_t *count)
 {
    struct agx3_ioto_load_packer *packer = context;
-   if (!packer || packer->chip != chip || !agx_chip_native_preamble(chip) ||
+   if (!packer || packer->chip != chip ||
+       (!agx_chip_native_preamble(chip) && !agx_chip_g17_m5(chip)) ||
        !load || !records || !count || dword >= load->count_dwords ||
        dword >= 4 || !direct_load_registers_ok(load))
       return false;
@@ -276,15 +276,18 @@ pack_cprog_copy(void *context, enum agx_chip chip,
    unsigned mask = (load->cprog_input_mask >> (2 * dword)) & 3;
    uint8_t bytes[AGX3_CPROG_INPUT_COPY_MAX_SIZE];
    size_t length;
-   if (!agx3_cprog_input_publish_at(mask, tps, load->data.half + 2u * dword,
-                                    bytes, sizeof(bytes), &length) ||
-       length > 20 || length % 10)
+   const bool m5 = agx_chip_g17_m5(chip);
+   const unsigned copy_size = m5 ? 8 : 10;
+   if (!agx3_cprog_input_publish_at_encoding(
+          m5 ? AGX_ISA_ENCODING_AGX3C : AGX_ISA_ENCODING_AGX3B,
+          mask, tps, load->data.half + 2u * dword,
+          bytes, sizeof(bytes), &length) || length > 20 || length % copy_size)
       return false;
-   *count = length / 10;
+   *count = length / copy_size;
    for (unsigned i = 0; i < *count; ++i) {
       records[i].kind = AGX_IOTO_LOAD_RAW_COPY_EP1;
-      records[i].length = 10;
-      memcpy(records[i].bytes, bytes + 10 * i, 10);
+      records[i].length = copy_size;
+      memcpy(records[i].bytes, bytes + copy_size * i, copy_size);
    }
    return true;
 }
diff --git a/src/asahi/compiler/agx3_ioto_uniform_program.c b/src/asahi/compiler/agx3_ioto_uniform_program.c
index 2994c186e870..9b0b18f4b04f 100644
--- a/src/asahi/compiler/agx3_ioto_uniform_program.c
+++ b/src/asahi/compiler/agx3_ioto_uniform_program.c
@@ -89,27 +89,48 @@ agx3_ioto_uniform_program_desc_body_bound(
           checked.count != inputs->count ||
           checked.storage_bytes != inputs->storage_bytes ||
           (inputs->count &&
-           (!agx_chip_native_preamble(chip) || desc->fast_path_enabled)))
+           ((!agx_chip_native_preamble(chip) && !agx_chip_g17_m5(chip)) ||
+            desc->fast_path_enabled)))
          return false;
       uint64_t covered[AGX3_CPROG_INPUT_WORDS] = {0};
+      size_t copies = 0;
       for (size_t i = 0; i < desc->range_count; ++i) {
          const struct agx3_ioto_uniform_range *range = &desc->ranges[i];
+         /* Ranges are dword aligned (uniform_range_valid) and the loader
+          * publishes per loaded dword (pack_cprog_copy): one 32-bit COPY for
+          * a selected pair at an even compact position, else one 16-bit
+          * COPY per selected half. */
          for (unsigned h = range->start_halfs;
-              h < range->start_halfs + range->size_halfs; ++h) {
-            const uint64_t bit = UINT64_C(1) << (h % 64);
-            if (!(inputs->halves[h / 64] & bit))
+              h < range->start_halfs + range->size_halfs; h += 2) {
+            unsigned mask = 0;
+            for (unsigned j = 0; j < 2; ++j) {
+               const unsigned half = h + j;
+               const uint64_t bit = UINT64_C(1) << (half % 64);
+               if (!(inputs->halves[half / 64] & bit))
+                  continue;
+               if (covered[half / 64] & bit)
+                  return false;
+               covered[half / 64] |= bit;
+               mask |= 1u << j;
+            }
+            if (!mask)
                continue;
-            if (covered[h / 64] & bit)
+            uint16_t tps;
+            if (!agx3_cprog_input_tps_half(&checked, (mask & 1) ? h : h + 1,
+                                           &tps))
                return false;
-            covered[h / 64] |= bit;
+            copies += (mask == 3 && !(tps & 1)) ? 1 : __builtin_popcount(mask);
          }
       }
       for (unsigned w = 0; w < AGX3_CPROG_INPUT_WORDS; ++w) {
          if (covered[w] != inputs->halves[w])
             return false;
       }
-      /* At worst each selected half uses its own extended COPY. */
-      if (!add_size(&bound, 10u * inputs->count))
+      /* An extended COPY is at most 10 bytes (8 on M5). Bounding each
+       * selected half by its own COPY instead roughly doubled this term and
+       * refused M5 compute loaders that read a few hundred input halves
+       * (e.g. ubo.*_compute) although they compose to about 4 KiB. */
+      if (!add_size(&bound, 10u * copies))
          return false;
    }
    *bytes = bound;
diff --git a/src/asahi/compiler/agx3_isel_core.c b/src/asahi/compiler/agx3_isel_core.c
index c09d89449ebc..6f4e633b7b00 100644
--- a/src/asahi/compiler/agx3_isel_core.c
+++ b/src/asahi/compiler/agx3_isel_core.c
@@ -302,6 +302,78 @@ encode_explicit(const struct agx3_isel_desc *d,
                            base, sizeof(base) / sizeof(base[0]), ov, n, out, len);
 }
 
+static bool
+encode_agx3c(const struct agx3_isel_desc *d,
+             uint8_t out[AGX3_ISEL_MAX_LENGTH], uint8_t *len)
+{
+   const bool cmp1_16 = d->cmp_16 != d->cmp_mixed_width;
+   if (!(d->cmp_16 ? reg16_ok(d->cmp0) : reg32_ok(d->cmp0)) ||
+       (d->cmp1_uniform ? ((d->cmp1 & 1) || d->cmp1 >= 512) :
+        d->cmp1_immediate ? d->cmp1 > 255 :
+        !(cmp1_16 ? reg16_ok(d->cmp1) : reg32_ok(d->cmp1))))
+      return false;
+   if (d->wait_mask >= (1u << 8))
+      return false;
+   for (unsigned i = 0; i < 4; ++i) {
+      if (d->cache[i] && d->discard[i])
+         return false;
+   }
+
+   const bool ti = d->true_kind == AGX3_ISEL_IMMEDIATE;
+   const bool fi = d->false_kind == AGX3_ISEL_IMMEDIATE;
+   if ((d->cmp1_immediate && (d->cache[1] || d->discard[1])) ||
+       (ti && (d->cache[2] || d->discard[2])) ||
+       (fi && (d->cache[3] || d->discard[3])))
+      return false;
+
+   const unsigned width = d->dest_16 ? 1 : 2;
+   const struct agx3_field_kv base[] = {
+      KV(9, 2), KV(10, width), KV(11, d->dest_cache),
+      KV(23, 2), KV(24, d->cmp_16 ? 1 : 2),
+      KV(38, d->cmp1_uniform ? 3 : d->cmp1_immediate ? 1 : 2),
+      KV(39, (d->cmp1_immediate || cmp1_16) ? 1 : 2),
+      KV(53, d->true_kind == AGX3_ISEL_UNIFORM ? 3 : ti ? 1 : 2), KV(54, width),
+      KV(67, d->false_kind == AGX3_ISEL_UNIFORM ? 3 : fi ? 1 : 2), KV(68, width),
+      KV(72, 3), KV(73, d->dest_16 ? 96 : 97),
+   };
+   struct agx3_field_kv ov[24];
+   unsigned n = 0;
+   ov[n++] = (struct agx3_field_kv)KV(2, d->dest);
+   ov[n++] = (struct agx3_field_kv)KV(16, d->cmp0);
+   ov[n++] = (struct agx3_field_kv)KV(d->cmp1_uniform ? 32 : d->cmp1_immediate ? 30 : 31, d->cmp1);
+   ov[n++] = (struct agx3_field_kv)KV(82, d->cond);
+   ov[n++] = (struct agx3_field_kv)KV(d->true_kind == AGX3_ISEL_UNIFORM ? 47 : ti ? 45 : 46, d->sel_true);
+   ov[n++] = (struct agx3_field_kv)KV(d->false_kind == AGX3_ISEL_UNIFORM ? 61 : fi ? 59 : 60, d->sel_false);
+   ov[n++] = (struct agx3_field_kv)KV(25, d->cache[0]);
+   ov[n++] = (struct agx3_field_kv)KV(27, d->discard[0]);
+   if (d->cmp_16 && d->cond >= AGX3_ISEL_ICOND_EQ)
+      ov[n++] = (struct agx3_field_kv)KV(28, 1);
+   if (!d->cmp1_immediate && !d->cmp1_uniform) {
+      ov[n++] = (struct agx3_field_kv)KV(40, d->cache[1]);
+      ov[n++] = (struct agx3_field_kv)KV(42, d->discard[1]);
+      if (cmp1_16 && d->cond >= AGX3_ISEL_ICOND_EQ)
+         ov[n++] = (struct agx3_field_kv)KV(43, 1);
+   }
+   if (!ti && d->true_kind != AGX3_ISEL_UNIFORM) {
+      ov[n++] = (struct agx3_field_kv)KV(55, d->cache[2]);
+      ov[n++] = (struct agx3_field_kv)KV(57, d->discard[2]);
+   }
+   if (!fi && d->false_kind != AGX3_ISEL_UNIFORM) {
+      ov[n++] = (struct agx3_field_kv)KV(69, d->cache[3]);
+      ov[n++] = (struct agx3_field_kv)KV(71, d->discard[3]);
+   }
+   for (unsigned i = 0; i < 8; ++i)
+      ov[n++] = (struct agx3_field_kv)KV(74 + i, (d->wait_mask >> i) & 1);
+
+   const char *mnemonic = d->dest_16 ? "iseli16" : "iseli32";
+   for (unsigned ep = 0; ep < 16; ++ep) {
+      if (agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, mnemonic, ep, base,
+                            sizeof(base) / sizeof(base[0]), ov, n, out, len))
+         return true;
+   }
+   return false;
+}
+
 bool
 agx3_isel_encode(enum agx_isa_encoding encoding,
                  const struct agx3_isel_desc *d,
@@ -310,9 +382,28 @@ agx3_isel_encode(enum agx_isa_encoding encoding,
    if (d == NULL || out == NULL || len == NULL)
       return false;
 
-   /* The probes are AGX3b. AGX3c spells this `iseli32`/`iseli16` with its own
-    * forms, which no probe here covers. */
-   if (encoding != AGX_ISA_ENCODING_AGX3B)
+   if (encoding == AGX_ISA_ENCODING_AGX3C) {
+      const bool tu = d->true_kind == AGX3_ISEL_UNIFORM;
+      const bool fu = d->false_kind == AGX3_ISEL_UNIFORM;
+      const bool any_uniform = d->cmp1_uniform || tu || fu;
+      /* Native M5 ray/SVGF shaders use an aligned full32 uniform as the
+       * right comparand and/or either selected value. A left uniform and
+       * narrow/mixed-width uniform forms are not part of this admission. */
+      if (!cond_ok(d->cond) || d->cmp0_uniform ||
+          (any_uniform && (d->dest_16 || d->cmp_16 || d->cmp_mixed_width)) ||
+          (d->cmp1_uniform && (d->cmp1_immediate || (d->cmp1 & 1) ||
+                              d->cmp1 >= 512 || d->cache[1] || d->discard[1])) ||
+          !(d->dest_16 ? reg16_ok(d->dest) : reg32_ok(d->dest)) ||
+          (tu ? ((d->sel_true & 1) || d->sel_true >= 512 ||
+                 d->cache[2] || d->discard[2]) :
+                !operand_ok(d->true_kind, d->sel_true, d->dest_16)) ||
+          (fu ? ((d->sel_false & 1) || d->sel_false >= 512 ||
+                 d->cache[3] || d->discard[3]) :
+                !operand_ok(d->false_kind, d->sel_false, d->dest_16)))
+         return false;
+      return encode_agx3c(d, out, len);
+   }
+   if (encoding != AGX_ISA_ENCODING_AGX3B || d->cmp_mixed_width)
       return false;
 
    /* The register comparands share a width; immediates carry no register. */
diff --git a/src/asahi/compiler/agx3_isel_core.h b/src/asahi/compiler/agx3_isel_core.h
index bec81223e178..1412f223c32a 100644
--- a/src/asahi/compiler/agx3_isel_core.h
+++ b/src/asahi/compiler/agx3_isel_core.h
@@ -70,7 +70,7 @@
 extern "C" {
 #endif
 
-#define AGX3_ISEL_MAX_LENGTH 10u
+#define AGX3_ISEL_MAX_LENGTH 14u
 
 /* == enum agx_icond. Only the five Apple emits are encodable. */
 enum agx3_isel_icond {
@@ -107,6 +107,7 @@ struct agx3_isel_desc {
    bool cmp0_uniform;
    bool cmp1_uniform;
    bool cmp_16;
+   bool cmp_mixed_width; /* M5: the second register has the opposite width. */
    bool explicit_hints;
    bool dest_cache; /* f12: result is consumed through the register cache */
    bool cache[4];
diff --git a/src/asahi/compiler/agx3_local_tps_core.c b/src/asahi/compiler/agx3_local_tps_core.c
index a3c67fae2729..a29724f284cb 100644
--- a/src/asahi/compiler/agx3_local_tps_core.c
+++ b/src/asahi/compiler/agx3_local_tps_core.c
@@ -158,6 +158,8 @@ tps_ok(enum agx_isa_encoding enc, const struct agx3_tps_desc *d, bool is_load)
    const unsigned slots = slot_count(enc);
    return d != NULL && slots != 0 && d->data < 256 && width_ok(d->width) &&
           d->mask >= 1 && d->mask <= 15 && d->shift <= 3 && d->index < 256 &&
+          (!d->index_immediate || (enc == AGX_ISA_ENCODING_AGX3C &&
+           d->index == 0 && !d->index_kill && !d->index_signed)) &&
           d->frame_region >= 1 && d->frame_region <= 6 &&
           d->wait_mask < (1u << slots) &&
           (!is_load || (d->slot >= 1 && d->slot <= slots)) &&
@@ -173,8 +175,8 @@ tps_overrides(const struct agx3_tps_desc *d, bool is_load,
    ov[n++] = (struct agx3_field_kv)KV(10, d->width);
    ov[n++] = (struct agx3_field_kv)KV(11, d->narrow_extend);
    ov[n++] = (struct agx3_field_kv)KV(16, d->index);
-   ov[n++] = (struct agx3_field_kv)KV(23, 2);
-   ov[n++] = (struct agx3_field_kv)KV(24, 2);
+   ov[n++] = (struct agx3_field_kv)KV(23, d->index_immediate ? 1 : 2);
+   ov[n++] = (struct agx3_field_kv)KV(24, d->index_immediate ? 1 : 2);
    ov[n++] = (struct agx3_field_kv)KV(27, d->index_kill);
    ov[n++] = (struct agx3_field_kv)KV(28, d->index_signed);
    ov[n++] = (struct agx3_field_kv)KV(41, d->frame_region);
diff --git a/src/asahi/compiler/agx3_local_tps_core.h b/src/asahi/compiler/agx3_local_tps_core.h
index 6f46b08f776e..7a05ad1f37da 100644
--- a/src/asahi/compiler/agx3_local_tps_core.h
+++ b/src/asahi/compiler/agx3_local_tps_core.h
@@ -42,6 +42,7 @@ struct agx3_local_desc {
 
    bool index_is_register;
    uint16_t index;
+   bool index_immediate;
    bool index_kill;
    uint16_t byte_offset;
 
@@ -72,9 +73,10 @@ struct agx3_tps_desc {
    uint8_t shift;
    uint16_t byte_offset;
 
-   /* All ldtps/sttps witnesses use the grounded dynamic 32-bit index path.
-    * Static spills use the separate copy/rld remote-file forms. */
+   /* Byte accesses also support zero plus an immediate byte offset.
+    * Static whole-word spills use the separate copy/rld forms. */
    uint16_t index;
+   bool index_immediate;
    bool index_kill;
    bool index_signed;
 
diff --git a/src/asahi/compiler/agx3_lower_cprog_inputs.c b/src/asahi/compiler/agx3_lower_cprog_inputs.c
index 843e82fb251f..25fcec9f7106 100644
--- a/src/asahi/compiler/agx3_lower_cprog_inputs.c
+++ b/src/asahi/compiler/agx3_lower_cprog_inputs.c
@@ -36,7 +36,8 @@ bool
 agx3_lower_cprog_inputs(agx_context *ctx)
 {
    if (!ctx->is_preamble || !ctx->out || ctx->agx3_cprog_inputs_lowered ||
-       ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+       (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C))
       return false;
 
    uint64_t reads[8] = {0}, writes[8] = {0};
@@ -189,6 +190,8 @@ cprog_other_storage(const agx_instr *I)
    case AGX_OPCODE_SCS_STORE_REGISTER_INDEXED_AGX3:
    case AGX_OPCODE_SCS_LOAD_REGISTER_BASE_AGX3:
    case AGX_OPCODE_SCS_STORE_REGISTER_BASE_AGX3:
+   case AGX_OPCODE_INTERSECT_RAY_M5:
+   case AGX_OPCODE_CONTINUE_RAY_M5:
       return true;
    default:
       return false;
diff --git a/src/asahi/compiler/agx3_lower_texture_publications.c b/src/asahi/compiler/agx3_lower_texture_publications.c
index 2577001c8631..ba0821b5c0ff 100644
--- a/src/asahi/compiler/agx3_lower_texture_publications.c
+++ b/src/asahi/compiler/agx3_lower_texture_publications.c
@@ -6,6 +6,11 @@
 #include "agx3_resource_ir.h"
 #include "agx3_staging.h"
 #include <string.h>
+#include "util/u_debug.h"
+
+DEBUG_GET_ONCE_BOOL_OPTION(m5_scalar_3d_coords, "ASAHI_M5_SCALAR_3D_COORDS", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_wide_scalar_sources,
+                           "ASAHI_M5_WIDE_SCALAR_SOURCES", true)
 
 struct publication {
    agx_index coordinates, reference;
@@ -269,6 +274,126 @@ agx3_lower_scalar_cube_coordinates(agx_context *ctx)
    return progress;
 }
 
+/* An M5 texel fetch stages its coordinates with one COPY per component, and a
+ * multisampled or array read publishes its sample index or layer with
+ * PACK_SMPIR, so the read never needs its coordinates as a contiguous vector.
+ * A filtered 2D sample stages x and y with one COPY each as well; its level,
+ * bias, gradient, offset and comparison operands stage on their own. A
+ * collected vector per read costs a GPR move for every component that already
+ * lives elsewhere: a depth pyramid that reads four samples at each of four
+ * texels moves three words before each of its sixteen reads, and a filtered
+ * shadow lookup moves both coordinates of every tap. Hand the read the
+ * components themselves. */
+static bool
+split_coords_candidate(const agx_instr *I)
+{
+   if (I->nr_srcs != 6 || I->nr_dests != 1 || I->agx3_tex_coords_staged ||
+       I->agx3_split_coords || I->agx3_cube_scalar_coords || I->query_lod ||
+       I->sparse || I->src[0].type != AGX_INDEX_NORMAL || I->src[0].memory ||
+       I->src[0].abs || I->src[0].neg || I->src[0].size != AGX_SIZE_32)
+      return false;
+
+   if (I->op == AGX_OPCODE_TEXTURE_SAMPLE)
+      return I->dim == AGX_DIM_2D && agx_channels(I->src[0]) == 2;
+
+   return I->op == AGX_OPCODE_TEXTURE_LOAD && !I->offset && !I->shadow &&
+          I->gather == AGX_GATHER_NONE &&
+          (I->dim == AGX_DIM_2D || I->dim == AGX_DIM_2D_MS ||
+           I->dim == AGX_DIM_2D_ARRAY) &&
+          agx_channels(I->src[0]) == (I->dim == AGX_DIM_2D ? 2 : 3) &&
+          agx_is_null(I->src[5]);
+}
+
+bool
+agx3_split_texture_coordinates(agx_context *ctx)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C || ctx->is_preamble)
+      return false;
+
+   /* Every M5 stage stages its fetch and sample coordinates
+    * (agx3_pack_binary.c). */
+   switch (ctx->stage) {
+   case MESA_SHADER_VERTEX:
+   case MESA_SHADER_FRAGMENT:
+   case MESA_SHADER_COMPUTE:
+   case MESA_SHADER_TESS_CTRL:
+   case MESA_SHADER_GEOMETRY:
+   case MESA_SHADER_MESH:
+   case MESA_SHADER_TASK:
+      break;
+   default:
+      return false;
+   }
+
+   agx_instr **defs = rzalloc_array(ctx, agx_instr *, ctx->alloc);
+   if (!defs)
+      return false;
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_dest(I, d)
+         defs[I->dest[d].value] = I;
+   }
+
+   bool progress = false;
+   agx_foreach_instr_global_safe(ctx, I) {
+      if (!split_coords_candidate(I))
+         continue;
+      const unsigned n = agx_channels(I->src[0]);
+      agx_instr *collect = defs[I->src[0].value];
+      if (!collect || collect->op != AGX_OPCODE_COLLECT ||
+          collect->nr_srcs != n)
+         continue;
+      bool eligible = true;
+      for (unsigned c = 0; c < n; ++c) {
+         eligible &= collect->src[c].type == AGX_INDEX_NORMAL &&
+                     scalar_publication_source(collect->src[c]);
+         /* A component that is also another operand (a level, a reference
+          * or an offset word) would name that operand's register: its own
+          * staging may then discard the register before or after the
+          * coordinate COPY reads it. Keep the vector. */
+         for (unsigned s = 1; s < 6; ++s) {
+            eligible &= I->src[s].type != AGX_INDEX_NORMAL ||
+                        I->src[s].value != collect->src[c].value;
+         }
+      }
+      if (!eligible)
+         continue;
+
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+      agx_instr *split = agx_alloc_instr(&b, I->op, 1, 8);
+      memcpy(&split->op, &I->op, sizeof(*I) - offsetof(agx_instr, op));
+      split->nr_srcs = 8;
+      split->dest[0] = I->dest[0];
+      memcpy(split->src, I->src, 6 * sizeof(agx_index));
+      for (unsigned c = 0; c < 3; ++c) {
+         agx_index component = c < n ? collect->src[c] : agx_null();
+         /* Liveness is recomputed by RA; drop the collect's annotations. */
+         component.kill = component.cache = component.discard = false;
+         split->src[c ? c + 5 : 0] = component;
+      }
+      split->agx3_split_coords = true;
+
+      /* A fetch keeps the vector's placement as an RA affinity hint: where
+       * the components can sit together, allocation matches the vector's. A
+       * sample does not: filter taps pair each coordinate with several
+       * others (a PCF footprint reads x0 with y0 and y1, x1 with both), so a
+       * pair hint per tap only constrains where the coordinates may live. */
+      if (I->op == AGX_OPCODE_TEXTURE_LOAD) {
+         if (!ctx->collect_split_hints.data)
+            util_dynarray_init(&ctx->collect_split_hints, ctx);
+         agx_instr *hint = agx_alloc_instr(&b, AGX_OPCODE_COLLECT, 1, n);
+         hint->dest[0] = collect->dest[0];
+         for (unsigned c = 0; c < n; ++c)
+            hint->src[c] = collect->src[c];
+         util_dynarray_append(&ctx->collect_split_hints, hint);
+      }
+
+      agx_builder_insert(&b.cursor, split);
+      agx_remove_instruction(I);
+      progress = true;
+   }
+   return progress;
+}
+
 static bool
 ssa_preserves_publications(const agx_instr *I)
 {
@@ -304,7 +429,9 @@ ssa_preserves_publications(const agx_instr *I)
 bool
 agx3_lower_scalar_texture_publications(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   const bool m5 = ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+      debug_get_option_m5_scalar_3d_coords();
+   if ((!m5 && ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble)
       return false;
 
@@ -330,24 +457,30 @@ agx3_lower_scalar_texture_publications(agx_context *ctx)
       agx_foreach_instr_in_block_safe(block, I) {
          bool eligible =
             I->op == AGX_OPCODE_TEXTURE_SAMPLE && !agx_texture_resource(I) &&
-            !I->agx3_tex_coords_staged && I->dim == AGX_DIM_2D && !I->query_lod &&
+            !I->agx3_tex_coords_staged && I->dim == (m5 ? AGX_DIM_3D : AGX_DIM_2D) && !I->query_lod &&
             (!I->offset || I->offset_xyz) && I->nr_srcs == 6 &&
             I->src[0].type == AGX_INDEX_NORMAL && !I->src[0].memory &&
             !I->src[0].abs && !I->src[0].neg && I->src[0].size == AGX_SIZE_32 &&
-            agx_channels(I->src[0]) == 2 &&
+            agx_channels(I->src[0]) == (m5 ? 3 : 2) &&
+            (!m5 || (I->nr_dests == 1 && I->dest[0].size == AGX_SIZE_32 &&
+                     !I->shadow && !I->offset && !I->sparse &&
+                     I->gather == AGX_GATHER_NONE &&
+                     I->lod_mode == AGX_LOD_MODE_AUTO_LOD)) &&
             (I->lod_mode == AGX_LOD_MODE_AUTO_LOD ||
              ((I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS ||
                I->lod_mode == AGX_LOD_MODE_LOD_MIN) &&
               I->src[1].type == AGX_INDEX_IMMEDIATE));
          agx_instr *collect = eligible ? defs[I->src[0].value] : NULL;
          eligible &= collect && collect->op == AGX_OPCODE_COLLECT &&
-                     collect->nr_srcs == 2 &&
+                     collect->nr_srcs == (m5 ? 3 : 2) &&
+                     (!m5 || scalar_publication_source(collect->src[2])) &&
                      scalar_publication_source(collect->src[0]) &&
                      scalar_publication_source(collect->src[1]) &&
                      (!I->shadow || scalar_publication_source(I->src[5]));
          if (eligible) {
             agx_index x = collect->src[0], y = collect->src[1];
-            agx_index reference = I->shadow ? I->src[5] : agx_null();
+            agx_index reference = m5 ? collect->src[2] :
+                                  I->shadow ? I->src[5] : agx_null();
             unsigned match = count;
             for (unsigned j = 0; j < count; ++j) {
                if (same_register_value(available[j].x, x) &&
@@ -365,20 +498,31 @@ agx3_lower_scalar_texture_publications(agx_context *ctx)
                        (count - match - 1) * sizeof(*available));
                available[count - 1] = hit;
             } else {
-               const unsigned cost = I->shadow ? 8 : 4;
+               const unsigned cost = (m5 || I->shadow) ? 8 : 4;
                while (reserved + cost > AGX3_STAGING_HALFS) {
                   reserved -= available[0].cost;
                   memmove(available, available + 1, --count * sizeof(*available));
                }
                agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
-               agx_publish_texture_scalar_agx3(&b, x, y, reference, ++next);
+               agx_index px = x, py = y, pz = reference;
+               /* The M5 publication stages x, y and z with the wide scalar
+                * COPY, which reads the whole register file, so it reads
+                * them where they live. Only the seven-bit form needs its
+                * operands in the low source bank; copies keep long-lived
+                * values out of that constraint. */
+               if (m5 && !debug_get_option_m5_wide_scalar_sources()) {
+                  px = agx_mov(&b, x);
+                  py = agx_mov(&b, y);
+                  pz = agx_mov(&b, reference);
+               }
+               agx_publish_texture_scalar_agx3(&b, px, py, pz, ++next);
                value = next;
                available[count++] = (struct scalar_publication){
                   x, y, reference, value, cost,
                };
                reserved += cost;
             }
-            I->src[0] = agx_get_vec_index(value, AGX_SIZE_32, I->shadow ? 3 : 2);
+            I->src[0] = agx_get_vec_index(value, AGX_SIZE_32, (m5 || I->shadow) ? 3 : 2);
             I->src[0].type = AGX_INDEX_RESOURCE;
             if (I->shadow)
                I->src[5] = agx_null();
diff --git a/src/asahi/compiler/agx3_lower_uniform_program.c b/src/asahi/compiler/agx3_lower_uniform_program.c
index c9c92c9442d1..3a859c13f843 100644
--- a/src/asahi/compiler/agx3_lower_uniform_program.c
+++ b/src/asahi/compiler/agx3_lower_uniform_program.c
@@ -11,7 +11,8 @@ bool
 agx3_analyze_preamble_uniform_inputs(agx_context *ctx, bool *has_inputs)
 {
    if (!has_inputs || !ctx->is_preamble || !ctx->num_blocks ||
-       ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+       (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C))
       return false;
 
    struct uniform_block {
@@ -125,7 +126,8 @@ agx3_analyze_preamble_uniform_inputs(agx_context *ctx, bool *has_inputs)
 bool
 agx3_lower_uniform_half_exports(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B || !ctx->is_preamble)
+   if ((ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C) || !ctx->is_preamble)
       return false;
 
    bool occupied[512] = {false};
@@ -228,12 +230,13 @@ agx3_lower_uniform_half_exports(agx_context *ctx)
 
       agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
       /* COPY writes an aligned word. Keep the live half in its original
-       * lane and clear only the unoccupied neighbour. A late DCE can leave
+       * lane; the unoccupied neighbour is padding that nothing reads, so
+       * leave it undefined rather than clearing it. A late DCE can leave
        * just the odd half of an allocated pair live. */
       agx_index word = agx_temp(ctx, AGX_SIZE_32);
       agx_instr *collect = agx_collect_to(&b, word, 2);
       collect->src[base & 1] = value;
-      collect->src[(base & 1) ^ 1] = agx_immediate(0);
+      collect->src[(base & 1) ^ 1] = agx_undef(AGX_SIZE_16);
       I->src[0] = word;
       I->src[1] = agx_immediate(base & ~1u);
       if (ctx->out)
@@ -247,11 +250,17 @@ agx3_lower_uniform_half_exports(agx_context *ctx)
  * and RA. Only fold a same-block, single-use scalar value into its uniform
  * export. Native AIR-NT add/mul/fma witnesses ground plain f32 inputs and
  * low-file aligned outputs. Texture/sampler state stores are unrelated.
+ *
+ * M5 (G17G) constant programs witness two shapes only: an
+ * all-register FMA (the vertex matrix products) and an immediate*register
+ * MUL (agx3_uniform_float_store_encode). The other values keep their
+ * uniform store.
  */
 bool
 agx3_lower_uniform_program(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   const bool m5 = ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B) ||
        !ctx->is_preamble || !ctx->alloc)
       return false;
 
@@ -320,6 +329,7 @@ agx3_lower_uniform_program(agx_context *ctx)
             admitted &= !alu->saturate && alu->nr_dests == 1 &&
                alu->nr_srcs == (alu->op == AGX_OPCODE_FFMA ? 3 : 2);
             bool has_register = false;
+            unsigned registers = 0, immediates = 0;
             agx_foreach_src(alu, s) {
                const agx_index src = alu->src[s];
                const bool reg = src.type == AGX_INDEX_NORMAL ||
@@ -329,8 +339,16 @@ agx3_lower_uniform_program(agx_context *ctx)
                   ((reg && src.size == AGX_SIZE_32 && agx_channels(src) == 1) ||
                    (immediate && src.value <= 255));
                has_register |= reg;
+               registers += reg;
+               immediates += immediate;
             }
             admitted &= has_register;
+            if (m5) {
+               admitted &= alu->op == AGX_OPCODE_FFMA ? registers == 3
+                           : alu->op == AGX_OPCODE_FMUL
+                              ? registers == 1 && immediates == 1
+                              : false;
+            }
          }
          if (!admitted)
             continue;
diff --git a/src/asahi/compiler/agx3_m5_graphics_program.c b/src/asahi/compiler/agx3_m5_graphics_program.c
new file mode 100644
index 000000000000..85775bc5985e
--- /dev/null
+++ b/src/asahi/compiler/agx3_m5_graphics_program.c
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: MIT */
+/* M5 program composition over the shared uniform-load scheduler. */
+#include "agx3_graphics_program.h"
+#include "agx3_ioto_load_packer.h"
+#include "asahi/lib/agx_ioto_state_core.h"
+#include "asahi/lib/agx_ioto_preshader_core.h"
+#include "util/macros.h"
+#include <string.h>
+
+bool
+agx3_g17g_graphics_program_bound(
+   enum agx_chip chip, const struct agx3_ioto_uniform_range *ranges,
+   size_t range_count, size_t *size)
+{
+   size_t body, chunks;
+   if (!size || !agx_chip_g17_m5(chip) ||
+       range_count > AGX3_G17P_GRAPHICS_MAX_RANGES ||
+       !agx3_ioto_uniform_program_body_bound(chip, ranges, range_count,
+                                             &body, &chunks) ||
+       chunks > AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS ||
+       body > AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE -
+                 AGX_IOTO_G17G_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE)
+      return false;
+
+   *size = body + AGX_IOTO_G17G_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE;
+   return true;
+}
+
+static bool
+g17g_graphics_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   enum agx_ioto_g17g_program_stage stage,
+   struct agx3_g17p_graphics_program *program,
+   struct agx3_g17p_graphics_program_template *template)
+{
+   size_t bound;
+   const bool mesh = stage == AGX_IOTO_G17G_PROGRAM_MESH ||
+                     stage == AGX_IOTO_G17G_PROGRAM_OBJECT;
+   if (!agx_chip_g17_m5(chip) || (mesh && chip != AGX_CHIP_G17S) ||
+       !desc || !program || (mesh && desc->fragment) ||
+       !desc->shader_binary_size ||
+       desc->preloaded_root ||
+       (desc->cprog_inputs && desc->cprog_inputs->count &&
+        !desc->constant_program.code_size) ||
+       (desc->constant_program.code_size &&
+        stage != AGX_IOTO_G17G_PROGRAM_COMPUTE &&
+        stage != AGX_IOTO_G17G_PROGRAM_VERTEX &&
+        stage != AGX_IOTO_G17G_PROGRAM_FRAGMENT &&
+        stage != AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT) ||
+       !desc->nr_gprs || desc->nr_gprs > 256 ||
+       desc->scratch_size > UINT16_MAX * 4u ||
+       !desc->compiler_d_valid ||
+
+       desc->compiler_d >
+          ((desc->fragment || stage == AGX_IOTO_G17G_PROGRAM_COMPUTE ||
+            stage == AGX_IOTO_G17G_PROGRAM_VERTEX) ? 512 : 256) ||
+       (desc->writes_sample_mask && !desc->fragment) ||
+       (desc->threadgroup_memory_size_B &&
+        ((stage != AGX_IOTO_G17G_PROGRAM_COMPUTE && !mesh) ||
+         desc->threadgroup_memory_size_B < (mesh ? 64 : 256) ||
+         desc->threadgroup_memory_size_B > 32768 ||
+         (desc->threadgroup_memory_size_B & 63))) ||
+       (desc->colour_classes & ~7u) ||
+       (!desc->fragment && desc->colour_classes) ||
+
+       (desc->fragment &&
+        ((desc->imageblock.sample_size_B > 248 ||
+          (desc->imageblock.sample_size_B & 7)) ||
+         (desc->imageblock.nr_samples != 1 &&
+          desc->imageblock.nr_samples != 2 &&
+          desc->imageblock.nr_samples != 4 &&
+          desc->imageblock.nr_samples != 8) ||
+         (desc->imageblock.tile_size_px != 32 * 32 &&
+          desc->imageblock.tile_size_px != 32 * 16 &&
+          desc->imageblock.tile_size_px != 16 * 16) ||
+         desc->imageblock.nr_samples * desc->imageblock.tile_size_px > 2048 ||
+         desc->imageblock.sample_size_B * desc->imageblock.nr_samples *
+            desc->imageblock.tile_size_px > 65536)) ||
+       !agx3_g17g_graphics_program_bound(chip, desc->uniform_ranges,
+                                          desc->uniform_range_count, &bound))
+      return false;
+
+
+   static const uint8_t speclm[2][AGX_IOTO_SPECLM_SIZE] = {
+      {0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00},
+      {0x17,0x00,0x00,0x2c,0x00,0x00,0x18,0x08},
+   };
+   struct agx_ioto_state ldshdr = {0}, spectpr = {0};
+   ldshdr.f[0] = desc->shader_va;
+
+   const uint64_t span =
+      (((uint64_t)desc->shader_binary_size + 63) & ~UINT64_C(63)) >> 7;
+   ldshdr.f[1] = MIN2(span, 3);
+   ldshdr.f[5] = 3;
+   ldshdr.f[6] = 107;
+   ldshdr.f[23] = 1;
+   spectpr.f[0] = 3;
+   spectpr.f[1] = 166;
+   spectpr.f[20] = DIV_ROUND_UP(desc->scratch_size, 4);
+   spectpr.f[21] = spectpr.f[20];
+   spectpr.f[22] = (((desc->compiler_d + 3) >> 2) + 1) & ~1u;
+
+   struct agx_ioto_state_tail_output tail = {
+      .length = AGX_IOTO_STATE_TAIL_TYPE0_SIZE,
+   };
+   /* Original LdShdrTransformFragment skips encodeImageBlockData when the
+    * allocation is zero and emits the no-imageblock form, including for
+    * fragment tile programs. Nonzero strides are rounded to eight bytes;
+    * they need not be powers of two (native stride24 emits byte7=0x18). */
+   memcpy(tail.bytes,
+          speclm[desc->fragment && desc->imageblock.sample_size_B != 0],
+          AGX_IOTO_SPECLM_SIZE);
+   if (desc->threadgroup_memory_size_B) {
+
+      struct agx_ioto_state shared = {0};
+      shared.f[0] = 3;
+      shared.f[1] = 165;
+      shared.f[18] = desc->threadgroup_memory_size_B;
+      shared.f[24] = 1;
+      if (!agx_ioto_state_pack_speclm(chip, &shared, tail.bytes,
+                                       AGX_IOTO_SPECLM_SIZE))
+         return false;
+   }
+   if (desc->fragment && desc->imageblock.sample_size_B) {
+
+      tail.bytes[7] = (uint8_t)desc->imageblock.sample_size_B;
+      tail.bytes[3] = desc->imageblock.tile_size_px == 16 * 16 ? 0xac :
+                      desc->imageblock.tile_size_px == 32 * 16 ? 0x1c : 0x2c;
+      tail.bytes[6] = desc->imageblock.tile_size_px == 16 * 16 ? 0x10 : 0x18;
+   }
+   if (desc->fragment) {
+      /* Sample count and tile geometry occupy independent SpecLM fields. */
+      const unsigned samples = desc->imageblock.nr_samples;
+      tail.bytes[6] |= (samples == 8 ? 3u : samples == 4 ? 2u :
+                           samples == 2 ? 1u : 0u) << 6;
+   }
+   if (!agx_ioto_state_pack_ldshdr(
+          chip, &ldshdr, tail.bytes + AGX_IOTO_SPECLM_SIZE,
+          AGX_IOTO_LDSHDR_SIZE) ||
+       !agx_ioto_state_pack_spectpr(
+          chip, &spectpr, tail.bytes + AGX_IOTO_SPECLM_SIZE +
+                            AGX_IOTO_LDSHDR_SIZE, AGX_IOTO_SPECTPR_SIZE))
+      return false;
+
+   uint8_t body[AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE];
+   struct agx_ioto_inflight_load loads[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+   struct agx_ioto_inflight_load sort[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+   struct agx_ioto_load_cache_entry cache[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+   size_t relocation_count = 0;
+   const struct agx3_ioto_uniform_program_desc load_desc = {
+      .program_va = desc->program_va,
+      .ranges = desc->uniform_ranges,
+      .range_count = desc->uniform_range_count,
+      .patchable = desc->patchable,
+      .cprog_inputs = desc->cprog_inputs,
+   };
+   const struct agx3_ioto_uniform_program_workspace workspace = {
+      .body = body, .body_capacity = sizeof(body),
+      .loads = loads, .load_capacity = ARRAY_SIZE(loads),
+      .sort = sort, .sort_capacity = ARRAY_SIZE(sort),
+      .cache = cache, .cache_capacity = ARRAY_SIZE(cache),
+      .relocations = template ? template->relocations : NULL,
+      .relocation_capacity = template ? ARRAY_SIZE(template->relocations) : 0,
+      .relocation_count = &relocation_count,
+   };
+   size_t input_bound, input_chunks;
+   if (!agx3_ioto_uniform_program_desc_body_bound(
+          chip, &load_desc, &input_bound, &input_chunks) ||
+       input_chunks > ARRAY_SIZE(loads) || input_bound > sizeof(body))
+      return false;
+   /* The existing range bound excludes import stores and CProg suffix.
+    * The public program buffer is the transactional final bound here. */
+   if (desc->constant_program.code_size)
+      bound = AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE;
+   size_t body_size, copy_offset, written;
+   struct agx3_g17p_graphics_program staged = {0};
+   if (!agx3_ioto_uniform_program_body_generate(
+          chip, &load_desc, workspace, &body_size, &copy_offset) ||
+       !agx_ioto_m5_resource_program_compose_classes(
+          chip, desc->program_va, body, body_size, &tail,
+          stage, MAX2(desc->colour_classes, 1),
+          staged.bytes, sizeof(staged.bytes), &written) ||
+       written > bound || written > UINT16_MAX)
+      return false;
+   /* The retained native compute loader executes the profile callback before
+    * publishing CProg input TPS. The callback context effect is
+    * unknown; preserve that observed order in this probe. Put the prefix
+    * before all loads:
+    * placing it merely before COPY can clobber pending GPR0/2 carriers.
+    * Graphics stages keep their composed order: the constant program only
+    * replaces the final END, after PIXREL, as HAL300
+    * native state-load encoder's finish step appends it (PIXREL, then
+    * the inline code or the external branch). */
+   const bool compute_cprog = desc->constant_program.code_size &&
+                              stage == AGX_IOTO_G17G_PROGRAM_COMPUTE;
+   if (compute_cprog) {
+      const size_t state_size = AGX_IOTO_STATE_TAIL_TYPE0_SIZE;
+      const size_t helper_size = AGX_IOTO_G17S_MINIMAL_PROFILE_LOADER_SIZE;
+      const size_t prefix_size = state_size + helper_size;
+      const size_t end_size = AGX_IOTO_PROGRAM_TERMINATE_SIZE;
+      uint8_t helper[AGX_IOTO_G17S_MINIMAL_PROFILE_LOADER_SIZE];
+      if (written != body_size + prefix_size + end_size)
+         return false;
+      memcpy(helper, staged.bytes + body_size + state_size, helper_size);
+      memcpy(staged.bytes, tail.bytes, state_size);
+      memcpy(staged.bytes + state_size, helper, helper_size);
+      memcpy(staged.bytes + prefix_size, body, body_size);
+   }
+   const size_t constant_suffix_offset = written - AGX_IOTO_PROGRAM_TERMINATE_SIZE;
+   if (desc->constant_program.code_size) {
+      static const uint8_t absent[] = {0x0e, 0, 0, 0};
+      if (written < sizeof(absent) ||
+          memcmp(staged.bytes + written - sizeof(absent), absent, sizeof(absent)))
+         return false;
+      const size_t suffix_offset = written - sizeof(absent);
+      struct agx_ioto_preshader_output suffix;
+      if (!agx_ioto_constant_program_emit(chip, &desc->constant_program,
+             desc->program_va + suffix_offset, &suffix) ||
+          suffix.length > sizeof(staged.bytes) - suffix_offset)
+         return false;
+      memcpy(staged.bytes + suffix_offset, suffix.bytes, suffix.length);
+      written = suffix_offset + suffix.length;
+   }
+   if (template) {
+      /* A large uniform pool can flush an earlier allocation round.
+       * Initialize shader state before its copies too, then relocate every
+       * later address materialization past the inserted state prefix. */
+      size_t first_copy;
+      if (!agx_ioto_program_first_copy_offset(chip, body, body_size, &first_copy))
+         return false;
+      const size_t prefix = AGX_IOTO_STATE_TAIL_TYPE0_SIZE +
+         (stage == AGX_IOTO_G17G_PROGRAM_COMPUTE ? 0 :
+          AGX_IOTO_G17S_MINIMAL_PROFILE_LOADER_SIZE);
+      for (size_t i = 0; i < relocation_count; ++i) {
+         struct agx_ioto_load_relocation *r = &template->relocations[i];
+         if (compute_cprog) {
+            r->offset += AGX_IOTO_STATE_TAIL_TYPE0_SIZE +
+                         AGX_IOTO_G17S_MINIMAL_PROFILE_LOADER_SIZE;
+         } else if (r->offset >= first_copy) {
+            r->offset += prefix;
+         } else if (r->offset + r->low_length + r->high_length > first_copy) {
+            return false;
+         }
+      }
+      template->relocation_count = relocation_count;
+      template->constant_suffix_offset = constant_suffix_offset;
+   }
+   staged.length = written;
+   *program = staged;
+   return true;
+}
+
+bool
+agx3_g17g_graphics_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program *program)
+{
+
+   const enum agx_ioto_g17g_program_stage stage =
+      !desc                      ? AGX_IOTO_G17G_PROGRAM_VERTEX
+      : !desc->fragment          ? AGX_IOTO_G17G_PROGRAM_VERTEX
+      : desc->writes_sample_mask ? AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT
+                                 : AGX_IOTO_G17G_PROGRAM_FRAGMENT;
+   return desc && g17g_graphics_program_generate(chip, desc, stage, program, NULL);
+}
+
+bool
+agx3_g17g_mesh_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   bool object, struct agx3_g17p_graphics_program *program)
+{
+   /* The native mesh/object tails use the ordinary TA order: shader state,
+    * profile helper, then uniform COPYs. TGmem64/192 are captured leaves;
+    * full dispatch ownership and resource budgets belong to the caller. */
+   return desc && g17g_graphics_program_generate(chip, desc,
+      object ? AGX_IOTO_G17G_PROGRAM_OBJECT : AGX_IOTO_G17G_PROGRAM_MESH,
+      program, NULL);
+}
+
+bool
+agx3_g17g_compute_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program *program)
+{
+   return desc && !desc->fragment && g17g_graphics_program_generate(
+      chip, desc, AGX_IOTO_G17G_PROGRAM_COMPUTE, program, NULL);
+}
+
+bool
+agx3_g17g_tile_program_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   bool store, struct agx3_g17p_graphics_program *program)
+{
+   return desc && desc->fragment && g17g_graphics_program_generate(
+      chip, desc, store ? AGX_IOTO_G17G_PROGRAM_EOT
+                       : AGX_IOTO_G17G_PROGRAM_BACKGROUND, program, NULL);
+}
+
+bool
+agx3_g17g_graphics_program_template_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program_template *program)
+{
+   if (!desc || !program)
+      return false;
+   const enum agx_ioto_g17g_program_stage stage =
+      !desc->fragment ? AGX_IOTO_G17G_PROGRAM_VERTEX
+      : desc->writes_sample_mask ? AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT
+                                 : AGX_IOTO_G17G_PROGRAM_FRAGMENT;
+   return g17g_graphics_program_generate(chip, desc, stage,
+                                         &program->program, program);
+}
+
+
+bool
+agx3_g17g_compute_program_template_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   struct agx3_g17p_graphics_program_template *program)
+{
+   return desc && program && !desc->fragment &&
+          g17g_graphics_program_generate(chip, desc,
+             AGX_IOTO_G17G_PROGRAM_COMPUTE, &program->program, program);
+}
+
+bool
+agx3_g17g_tile_program_template_generate(
+   enum agx_chip chip, const struct agx3_g17p_graphics_program_desc *desc,
+   bool store, struct agx3_g17p_graphics_program_template *program)
+{
+   return desc && program && desc->fragment &&
+          g17g_graphics_program_generate(chip, desc,
+             store ? AGX_IOTO_G17G_PROGRAM_EOT : AGX_IOTO_G17G_PROGRAM_BACKGROUND,
+             &program->program, program);
+}
diff --git a/src/asahi/compiler/agx3_m5_mrt_eot_data.h b/src/asahi/compiler/agx3_m5_mrt_eot_data.h
new file mode 100644
index 000000000000..d473a80efc3a
--- /dev/null
+++ b/src/asahi/compiler/agx3_m5_mrt_eot_data.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: MIT */
+/* Original native G17G builtin origin arithmetic; full text replayed in host checks. */
+
+static const uint8_t m5_mrt_prefix[48] = {
+   0x1c,0x9c,0x10,0x06,0x24,0xa8,0x14,0x06,0x0c,0x9d,0x10,0x26,
+   0x24,0x81,0x01,0x40,0x28,0x01,0xa0,0x0a,0x37,0x06,0x04,0x1a,
+   0x29,0x00,0xa1,0x02,0x68,0x01,0x20,0x00,0x01,0x02,0x27,0x08,
+   0x04,0x1a,0x29,0x00,0xa1,0x02,0x28,0x01,0x20,0x00,0x01,0x02,
+};
+
+static const uint8_t m5_rt0_prefix[48] = {
+   0x1c,0x9c,0x10,0x06,0x24,0xa8,0x14,0x06,0x0c,0x9d,0x10,0x26,
+   0x24,0x81,0x01,0x40,0x28,0x01,0xa0,0x0a,0x27,0x06,0x04,0x0a,
+   0x21,0x00,0xa1,0x02,0x68,0x01,0x20,0x00,0x01,0x02,0x37,0x08,
+   0x04,0x0a,0x21,0x00,0xa1,0x02,0x28,0x01,0x20,0x00,0x01,0x02,
+};
diff --git a/src/asahi/compiler/agx3_matrix_core.c b/src/asahi/compiler/agx3_matrix_core.c
new file mode 100644
index 000000000000..7df8f7cfefe0
--- /dev/null
+++ b/src/asahi/compiler/agx3_matrix_core.c
@@ -0,0 +1,100 @@
+#include "agx3_matrix_core.h"
+#include "agx3_binder.h"
+
+bool
+agx3_matrix_encode_mma8(enum agx_isa_encoding encoding,
+                        const struct agx3_matrix_mma8_desc *d,
+                        uint8_t out[14], uint8_t *length)
+{
+   if (!d)
+      return false;
+   unsigned type = d->input_type ? d->input_type : AGX3_MATRIX_INPUT_F16;
+   unsigned align_mask = type == AGX3_MATRIX_INPUT_F16 ? 3 : 1;
+   unsigned source_max = type == AGX3_MATRIX_INPUT_F32 ? 252 : 254;
+   if (encoding != AGX_ISA_ENCODING_AGX3C || !out || !length ||
+       (type != AGX3_MATRIX_INPUT_F16 && type != AGX3_MATRIX_INPUT_F32 &&
+        type != AGX3_MATRIX_INPUT_BF16) ||
+       (d->dest & align_mask) || d->dest > 252 ||
+       (d->a & 1) || d->a > source_max || (d->b & 1) || d->b > source_max ||
+       (!d->c_zero && ((d->c & align_mask) || d->c > 252)) ||
+       (d->c_zero && (d->c || d->cache[2] || d->discard[2])))
+      return false;
+
+   for (unsigned i = 0; i < 3; ++i) {
+      if (d->cache[i] && d->discard[i])
+         return false;
+   }
+
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 2}, {25, 2}, {26, 1}, {42, 2}, {43, 1},
+      {60, 2}, {67, 3}, {68, 61},
+   };
+   struct agx3_field_kv fields[23];
+   unsigned count = 0;
+   fields[count++] = (struct agx3_field_kv){26, type};
+   fields[count++] = (struct agx3_field_kv){43, type};
+   fields[count++] = (struct agx3_field_kv){2, d->dest};
+   fields[count++] = (struct agx3_field_kv){11, d->dest_cache};
+   fields[count++] = (struct agx3_field_kv){18, d->a};
+   fields[count++] = (struct agx3_field_kv){35, d->b};
+   fields[count++] = (struct agx3_field_kv){52, d->c};
+   fields[count++] = (struct agx3_field_kv){59, d->c_zero ? 1 : 2};
+   fields[count++] = (struct agx3_field_kv){27, d->cache[0]};
+   fields[count++] = (struct agx3_field_kv){29, d->discard[0]};
+   fields[count++] = (struct agx3_field_kv){44, d->cache[1]};
+   fields[count++] = (struct agx3_field_kv){46, d->discard[1]};
+   fields[count++] = (struct agx3_field_kv){61, d->cache[2]};
+   fields[count++] = (struct agx3_field_kv){63, d->discard[2]};
+   for (unsigned i = 0; i < 8; ++i)
+      fields[count++] = (struct agx3_field_kv){69 + i,
+                                             (d->wait_mask >> i) & 1};
+
+   return agx3_bind_compose(encoding, "f32matmad8x8", 0, base,
+                            sizeof(base) / sizeof(base[0]), fields, count,
+                            out, length);
+}
+
+bool
+agx3_matrix_encode_gemm16(enum agx_isa_encoding encoding,
+                         const struct agx3_matrix_gemm16_desc *d,
+                         uint8_t out[10], uint8_t *length)
+{
+   if (!d)
+      return false;
+   enum agx3_matrix_input_type b_type = d->b_input_type ? d->b_input_type : d->input_type;
+   if (encoding != AGX_ISA_ENCODING_AGX3C || !out || !length ||
+       (b_type != AGX3_MATRIX_INPUT_F16 && b_type != AGX3_MATRIX_INPUT_BF16) ||
+       (d->dest & 15) || d->dest > 240 ||
+       (d->a & 7) || d->a > 248 || (d->b & 7) || d->b > 248 ||
+       (d->input_type != AGX3_MATRIX_INPUT_F16 &&
+        d->input_type != AGX3_MATRIX_INPUT_BF16))
+      return false;
+
+   const struct agx3_field_kv base[] = {
+      {1, 2}, {2, 2}, {9, 2}, {14, 1}, {20, 2}, {25, 1},
+      {30, 1}, {31, 3}, {32, 70},
+   };
+   struct agx3_field_kv fields[29];
+   unsigned count = 0;
+   fields[count++] = (struct agx3_field_kv){0, d->dest};
+   fields[count++] = (struct agx3_field_kv){3, d->dest_hint};
+   fields[count++] = (struct agx3_field_kv){8, d->a};
+   fields[count++] = (struct agx3_field_kv){10, d->input_type};
+   fields[count++] = (struct agx3_field_kv){19, d->b};
+   fields[count++] = (struct agx3_field_kv){21, b_type};
+   fields[count++] = (struct agx3_field_kv){7, d->zero_accumulator};
+   fields[count++] = (struct agx3_field_kv){15, d->transpose_a};
+   fields[count++] = (struct agx3_field_kv){14, !d->preserve_a};
+   fields[count++] = (struct agx3_field_kv){25, !d->preserve_b};
+   fields[count++] = (struct agx3_field_kv){26, d->transpose_b};
+   fields[count++] = (struct agx3_field_kv){33, d->field33};
+   for (unsigned i = 0; i < 8; ++i)
+      fields[count++] = (struct agx3_field_kv){40 + i,
+                                             (d->field40_mask >> i) & 1};
+   for (unsigned i = 0; i < 8; ++i)
+      if (d->wait_mask & (1u << i))
+         fields[count++] = (struct agx3_field_kv){33 + i, 1};
+   return agx3_bind_compose(encoding, "fGeMM16x16x16", 0, base,
+                            sizeof(base) / sizeof(base[0]), fields, count,
+                            out, length);
+}
diff --git a/src/asahi/compiler/agx3_matrix_core.h b/src/asahi/compiler/agx3_matrix_core.h
new file mode 100644
index 000000000000..e521a200852c
--- /dev/null
+++ b/src/asahi/compiler/agx3_matrix_core.h
@@ -0,0 +1,57 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "asahi/lib/agx_chip.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum agx3_matrix_input_type {
+   AGX3_MATRIX_INPUT_F16 = 1,
+   AGX3_MATRIX_INPUT_F32 = 2,
+   AGX3_MATRIX_INPUT_BF16 = 4,
+};
+
+struct agx3_matrix_gemm16_desc {
+   uint16_t dest;
+   uint16_t a;
+   uint16_t b;
+   enum agx3_matrix_input_type input_type;
+   bool dest_hint;
+   bool transpose_b;
+   bool field33;
+   uint8_t field40_mask;
+   bool zero_accumulator;
+   bool transpose_a;
+   bool preserve_a;
+   bool preserve_b;
+   uint8_t wait_mask;
+   enum agx3_matrix_input_type b_input_type;
+};
+
+bool agx3_matrix_encode_gemm16(enum agx_isa_encoding encoding,
+                              const struct agx3_matrix_gemm16_desc *desc,
+                              uint8_t out[10], uint8_t *length);
+
+struct agx3_matrix_mma8_desc {
+   uint16_t dest;
+   uint16_t a;
+   uint16_t b;
+   uint16_t c;
+   bool c_zero;
+   bool dest_cache;
+   bool cache[3];
+   bool discard[3];
+   uint8_t wait_mask;
+   enum agx3_matrix_input_type input_type;
+};
+
+bool agx3_matrix_encode_mma8(enum agx_isa_encoding encoding,
+                            const struct agx3_matrix_mma8_desc *desc,
+                            uint8_t out[14], uint8_t *length);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/src/asahi/compiler/agx3_mem_core.c b/src/asahi/compiler/agx3_mem_core.c
index a4b7d4bbeea5..0f40f5e69963 100644
--- a/src/asahi/compiler/agx3_mem_core.c
+++ b/src/asahi/compiler/agx3_mem_core.c
@@ -10,7 +10,7 @@
 #define KV(f, v) {(uint16_t)(f), (uint64_t)(v)}
 
 static bool
-common_ok(const struct agx3_mem_desc *d)
+common_ok(enum agx_isa_encoding enc, const struct agx3_mem_desc *d)
 {
    if (d == NULL || d->data >= 256 || d->mask < 1 || d->mask > 15 ||
        d->shift > 4)
@@ -21,11 +21,11 @@ common_ok(const struct agx3_mem_desc *d)
    if (d->base_is_uniform) {
       if ((d->base & 3) != 0 || d->base > 508)
          return false;
-   } else if ((d->base & 1) != 0 || d->base > 188) {
+   } else if ((d->base & 1) != 0 || d->base > (enc == AGX_ISA_ENCODING_AGX3C ? 252 : 188)) {
       return false;
    }
    if (d->idx_kind == AGX3_MEM_IDX_REG &&
-       ((d->idx_reg & 1) != 0 || d->idx_reg > 191))
+       ((d->idx_reg & 1) != 0 || d->idx_reg > (enc == AGX_ISA_ENCODING_AGX3C ? 254 : 191)))
       return false;
    if (d->idx_kind == AGX3_MEM_IDX_IMM) {
       if (d->idx_scaled) {
@@ -111,7 +111,7 @@ agx3_mem_encode_load_form(enum agx_isa_encoding enc,
                           const struct agx3_mem_desc *d, uint8_t ep,
                           uint8_t out[AGX3_MEM_MAX_LENGTH], uint8_t *len)
 {
-   if (!common_ok(d) || d->slot < 1 ||
+   if (!common_ok(enc, d) || d->slot < 1 ||
        d->slot > (enc == AGX_ISA_ENCODING_AGX3C ? 8 : 6) ||
        (enc == AGX_ISA_ENCODING_AGX3B ? ep != 0 : ep > 5))
       return false;
@@ -137,6 +137,14 @@ bool
 agx3_mem_encode_load(enum agx_isa_encoding enc, const struct agx3_mem_desc *d,
                      uint8_t out[AGX3_MEM_MAX_LENGTH], uint8_t *len)
 {
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      static const uint8_t forms[] = {0, 1, 3, 4, 2, 5};
+      for (unsigned i = 0; i < sizeof(forms); ++i) {
+         if (agx3_mem_encode_load_form(enc, d, forms[i], out, len))
+            return true;
+      }
+      return false;
+   }
    return agx3_mem_encode_load_form(enc, d, 0, out, len);
 }
 
@@ -144,7 +152,7 @@ bool
 agx3_mem_encode_store(enum agx_isa_encoding enc, const struct agx3_mem_desc *d,
                       uint8_t out[AGX3_MEM_MAX_LENGTH], uint8_t *len)
 {
-   if (!common_ok(d) || d->agx3b_last_load)
+   if (!common_ok(enc, d) || d->agx3b_last_load)
       return false;
 
    struct agx3_field_kv base[6];
@@ -155,5 +163,14 @@ agx3_mem_encode_store(enum agx_isa_encoding enc, const struct agx3_mem_desc *d,
    unsigned n = common_overrides(d, /*off*/ 74, /*shift*/ 73, /*mask*/ 71, ov);
    ov[n++] = (struct agx3_field_kv)KV(75, d->coherent ? 1 : 0);
 
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      static const uint8_t forms[] = {0, 1, 3, 4, 2, 5};
+      for (unsigned i = 0; i < sizeof(forms); ++i) {
+         if (agx3_bind_compose(enc, "stdev", forms[i], base, nbase,
+                               ov, n, out, len))
+            return true;
+      }
+      return false;
+   }
    return agx3_bind_compose(enc, "stdev", 0, base, nbase, ov, n, out, len);
 }
diff --git a/src/asahi/compiler/agx3_pack.c b/src/asahi/compiler/agx3_pack.c
index f60d7134625f..c064e073117c 100644
--- a/src/asahi/compiler/agx3_pack.c
+++ b/src/asahi/compiler/agx3_pack.c
@@ -4,6 +4,7 @@
  */
 
 #include "agx3_pack.h"
+#include "agx3_binder.h"
 
 #include <string.h>
 
@@ -2252,6 +2253,26 @@ agx3_pack_scoped_barrier(enum agx_isa_encoding encoding,
       memset(out, 0, sizeof(*out));
       return true;
    }
+   if (scope == AGX3_BARRIER_SCOPE_MESH_PAYLOAD_ENTRY ||
+       scope == AGX3_BARRIER_SCOPE_OBJECT_PAYLOAD_EXIT) {
+      if (encoding != AGX_ISA_ENCODING_AGX3C || out == NULL) return false;
+      /* mesh_amplified +0xa, before reading SR62/63 and LDDEV payload.
+       * Retain the capture's anonymous selector fields exactly. */
+      const struct agx3_field_kv mesh_fields[] = {
+         {14,3},{15,9},{24,1},{30,3},{40,1},
+      };
+      const struct agx3_field_kv object_fields[] = {
+         {14,3},{15,9},{24,1},{30,4},{39,1},{40,1},
+      };
+      bool object=scope==AGX3_BARRIER_SCOPE_OBJECT_PAYLOAD_EXIT;
+      const struct agx3_field_kv *fields=object ? object_fields : mesh_fields;
+      unsigned count=object ? sizeof(object_fields)/sizeof(object_fields[0])
+                            : sizeof(mesh_fields)/sizeof(mesh_fields[0]);
+      memset(out,0,sizeof(*out));
+      return agx3_bind_compose(encoding,"barrier",0,fields,
+         count,NULL,0,out->bytes,&out->length);
+   }
+
    static const uint8_t canonical_b[][AGX3_SCOPED_BARRIER_MAX_LENGTH] = {
       [AGX3_BARRIER_SCOPE_EXECUTION] =
          {0x07, 0x04, 0x54, 0x41, 0x09, 0x00},
@@ -2279,12 +2300,27 @@ agx3_pack_scoped_barrier(enum agx_isa_encoding encoding,
          {0x07, 0x51, 0x00, 0x06, 0x00, 0x00},
       [AGX3_BARRIER_SCOPE_THREADGROUP] =
          {0x27, 0x51, 0x00, 0x06, 0x00, 0x00},
+      /* G17G atomic_thread_fence(mem_device, seq_cst, thread_scope_device).
+       * Unlike the workgroup barrier, this is valid in one active lane. */
+      [AGX3_BARRIER_SCOPE_DEVICE_FENCE] =
+         {0x0f, 0x2b, 0x00, 0x06, 0x00, 0x00},
       [AGX3_BARRIER_SCOPE_DEVICE] =
          {0x0f, 0x69, 0x00, 0x06, 0x00, 0x00},
       [AGX3_BARRIER_SCOPE_TEXTURE] = {
          0x87, 0x67, 0x00, 0x06, 0x00, 0x00,
          0x87, 0x5f, 0x00, 0x06, 0x00, 0x00,
       },
+      /* G17G texture.fence() and atomic_thread_fence(mem_texture) at device
+       * or simdgroup scope (port/fencewit k_fence, k_atf_dev, k_atf_simd). */
+      [AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT] = {
+         0x87, 0x26, 0x00, 0x06, 0x00, 0x00,
+         0x87, 0x1e, 0x00, 0x06, 0x00, 0x00,
+      },
+      /* atomic_thread_fence(mem_texture, thread_scope_threadgroup). */
+      [AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP] = {
+         0x87, 0x27, 0x00, 0x06, 0x00, 0x00,
+         0x87, 0x1f, 0x00, 0x06, 0x00, 0x00,
+      },
    };
    const uint8_t(*canonical)[AGX3_SCOPED_BARRIER_MAX_LENGTH] =
       encoding == AGX_ISA_ENCODING_AGX3B ? canonical_b :
@@ -2292,15 +2328,18 @@ agx3_pack_scoped_barrier(enum agx_isa_encoding encoding,
    const uint8_t length =
       scope == AGX3_BARRIER_SCOPE_TEXTURE ||
             scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE ||
-            scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT
+            scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT ||
+            scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP
          ? 12
          : 6;
 
    if (canonical == NULL || out == NULL ||
        scope < AGX3_BARRIER_SCOPE_EXECUTION ||
-       scope > AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT ||
+       scope > AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP ||
+       (encoding == AGX_ISA_ENCODING_AGX3B &&
+        scope > AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT) ||
        (encoding == AGX_ISA_ENCODING_AGX3C &&
-        scope > AGX3_BARRIER_SCOPE_TEXTURE))
+        scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE))
       return false;
 
    memset(out, 0, sizeof(*out));
@@ -2309,6 +2348,53 @@ agx3_pack_scoped_barrier(enum agx_isa_encoding encoding,
    return true;
 }
 
+bool
+agx3_pack_scoped_barrier_wait(enum agx_isa_encoding encoding,
+                              enum agx3_barrier_scope scope,
+                              uint16_t wait_mask,
+                              struct agx3_encoded_scoped_barrier *out)
+{
+   if (!agx3_pack_scoped_barrier(encoding, scope, out))
+      return false;
+   if (!wait_mask)
+      return true;
+
+   /* Field 29 is the threadgroup rendezvous and field 39 the threadgroup
+    * memory scope; the first word is scope 4, the second scope 3. */
+   const bool rendezvous = scope == AGX3_BARRIER_SCOPE_TEXTURE;
+   const bool threadgroup =
+      rendezvous || scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP;
+   if (encoding != AGX_ISA_ENCODING_AGX3C || (wait_mask & ~UINT16_C(0xff)) ||
+       (scope != AGX3_BARRIER_SCOPE_TEXTURE &&
+        scope != AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT &&
+        scope != AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP)) {
+      memset(out, 0, sizeof(*out));
+      return false;
+   }
+
+   const struct agx3_field_kv base[] = {
+      {14, 3}, {15, 9}, {28, 1}, {29, rendezvous}, {30, 4},
+      {39, threadgroup}, {40, 1}, {41, 1},
+   };
+   struct agx3_field_kv wait[8];
+   unsigned n = 0;
+   for (unsigned i = 0; i < 8; ++i) {
+      if (wait_mask & (1u << i))
+         wait[n++] = (struct agx3_field_kv){16 + i, 1};
+   }
+
+   uint8_t first[AGX3_SCOPED_BARRIER_MAX_LENGTH], len = 0;
+   if (!agx3_bind_compose(encoding, "barrier", 0, base,
+                          sizeof(base) / sizeof(base[0]), wait, n, first,
+                          &len) ||
+       len != 6) {
+      memset(out, 0, sizeof(*out));
+      return false;
+   }
+   memcpy(out->bytes, first, 6);
+   return true;
+}
+
 bool
 agx3_pack_threadgroup_barrier(
    enum agx_isa_encoding encoding,
diff --git a/src/asahi/compiler/agx3_pack.h b/src/asahi/compiler/agx3_pack.h
index 9f6ce2fa4d11..5a6ffd2bfa0b 100644
--- a/src/asahi/compiler/agx3_pack.h
+++ b/src/asahi/compiler/agx3_pack.h
@@ -212,7 +212,14 @@ enum agx3_barrier_scope {
    AGX3_BARRIER_SCOPE_DEVICE_FENCE,
    AGX3_BARRIER_SCOPE_TEXTURE_FENCE,
    AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT,
+   /* AGX3C only: a memory-only texture fence at threadgroup scope. */
+   AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP,
+   /* Compiler ordering only: M5 subgroup shared memory needs no rendezvous. */
    AGX3_BARRIER_SCOPE_SUBGROUP_SHARED,
+   /* Captured native mesh_amplified entry before the payload device load. */
+   AGX3_BARRIER_SCOPE_MESH_PAYLOAD_ENTRY,
+   /* Captured object.write_childcount after the API workgroup barrier. */
+   AGX3_BARRIER_SCOPE_OBJECT_PAYLOAD_EXIT,
 };
 
 struct agx3_encoded_scoped_barrier {
@@ -870,6 +877,19 @@ bool agx3_pack_scoped_barrier(enum agx_isa_encoding encoding,
                               enum agx3_barrier_scope scope,
                               struct agx3_encoded_scoped_barrier *out);
 
+/*
+ * AGX3C texture barriers carry their scoreboard wait in the first barrier
+ * word itself (fields 16..23 name slots 1..8). G17G waits there on the result
+ * slots of the image writes still in flight: Metal texture.fence(),
+ * atomic_thread_fence(mem_texture) at every scope and
+ * threadgroup_barrier(mem_texture) all do. Only the AGX3C texture scopes
+ * accept a wait; a zero wait gives agx3_pack_scoped_barrier()'s bytes.
+ */
+bool agx3_pack_scoped_barrier_wait(enum agx_isa_encoding encoding,
+                                   enum agx3_barrier_scope scope,
+                                   uint16_t wait_mask,
+                                   struct agx3_encoded_scoped_barrier *out);
+
 /*
  * Encode/decode the canonical unconditional shader terminator.  The exact
  * bytes 0e000000 terminate every decoded code region in the hash-pinned
diff --git a/src/asahi/compiler/agx3_pack_binary.c b/src/asahi/compiler/agx3_pack_binary.c
index 2eec0c4893fd..d53ab77bd109 100644
--- a/src/asahi/compiler/agx3_pack_binary.c
+++ b/src/asahi/compiler/agx3_pack_binary.c
@@ -32,6 +32,7 @@
 #include "agx3_isel_core.h"
 #include "agx3_local_tps_core.h"
 #include "agx3_mem_core.h"
+#include "agx3_matrix_core.h"
 #include "agx3_pack.h"
 #include "agx3_staging.h"
 #include "agx3_resource_ir.h"
@@ -39,6 +40,7 @@
 #include "agx3_scs_core.h"
 #include "agx3_rt_gen1_traversal.h"
 #include "agx3_rt_core.h"
+#include "agx3_rt_m5_core.h"
 #include "agx3_select_core.h"
 #include "agx3_slot_core.h"
 #include "agx3_sr_core.h"
@@ -63,6 +65,14 @@
 #include <stdio.h>
 #include <string.h>
 
+DEBUG_GET_ONCE_BOOL_OPTION(m5_begin_cf_live, "ASAHI_M5_BEGIN_CF_LIVE", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_scalar_3d_coords, "ASAHI_M5_SCALAR_3D_COORDS", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_wide_scalar_sources,
+                           "ASAHI_M5_WIDE_SCALAR_SOURCES", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_fs_3d_banks, "ASAHI_M5_FS_3D_BANKS", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_texture_result_tokens,
+                           "ASAHI_M5_TEXTURE_RESULT_TOKENS", true)
+
 struct agx3_branch_fixup {
    agx_block *block;
    off_t offset;
@@ -118,9 +128,24 @@ agx3_index_kind(const agx_index x)
  * only the mnemonic costs a round trip to hardware every time -- "no grounded
  * AGX3 encoding for fcmpsel" is true of the shapes that encode as well.
  */
+/* The first refusal of the current pack, kept without AGX_MESA_DEBUG so that a
+ * compile failure (and the driver's fallback warning) names the instruction
+ * and gate. Reset by agx3_pack_binary(). */
+static __thread char agx3_first_refusal[96];
+
+const char *
+agx3_last_pack_refusal(void)
+{
+   return agx3_first_refusal;
+}
+
 static void
 agx3_report_refusal_why(const agx_instr *I, unsigned line, const char *why)
 {
+   if (!agx3_first_refusal[0])
+      snprintf(agx3_first_refusal, sizeof(agx3_first_refusal), "%s at line %u%s%s",
+               agx_opcodes_info[I->op].name, line, why ? ", " : "", why ? why : "");
+
    if (!(agx_compiler_debug & AGX_DBG_SHADERS))
       return;
 
@@ -457,6 +482,20 @@ agx3_emit(struct util_dynarray *emission, enum agx_isa_encoding enc,
  * 32-bit compares), so the caller must skip its handler rather than refuse the
  * shader.
  */
+/* Native G17G fmin/fmax (compiled without fast math) retain condition 3/7
+ * and tie each selected operand to its comparand. They are minNum/maxNum,
+ * not general ordered comparisons: GE/LE plus a selection swap returns a
+ * NaN when just one input is NaN. Keep the existing unordered-comparison
+ * mapping for untied selects. */
+static bool
+agx3_fcmpsel_tied_minmax(const agx_instr *I)
+{
+   return !I->invert_cond && !I->saturate &&
+          (I->fcond == AGX_FCOND_LTN || I->fcond == AGX_FCOND_GTN) &&
+          agx_is_equiv(I->src[0], I->src[2]) &&
+          agx_is_equiv(I->src[1], I->src[3]);
+}
+
 static bool
 agx3_fcmpsel_all_half_regs(const agx_instr *I)
 {
@@ -575,10 +614,9 @@ agx3_try_uniform_integer(struct util_dynarray *emission,
                          unsigned wait)
 {
    struct agx3_uniform_integer_desc d;
-   if (enc != AGX_ISA_ENCODING_AGX3B ||
-       !agx3_uniform_integer_instruction(I, true, &d))
+   if (!agx3_uniform_integer_for_encoding(enc, I, true, &d))
       return 0;
-   if (wait >= 64)
+   if (wait >= (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 64))
       return -1;
    d.wait_mask = wait;
    uint8_t bytes[16], length;
@@ -594,7 +632,8 @@ agx3_try_uniform_float(struct util_dynarray *emission,
 {
    const bool half = I->op == AGX_OPCODE_HADD || I->op == AGX_OPCODE_HMUL ||
                      I->op == AGX_OPCODE_HFMA;
-   if (enc != AGX_ISA_ENCODING_AGX3B ||
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || (m5 && half) ||
        (!half && I->op != AGX_OPCODE_FADD && I->op != AGX_OPCODE_FMUL &&
         I->op != AGX_OPCODE_FFMA))
       return 0;
@@ -661,6 +700,16 @@ agx3_try_float_modifier(struct util_dynarray *emission,
    bool has_mod = I->saturate;
    for (unsigned s = 0; s < I->nr_srcs; ++s)
       has_mod |= I->src[s].abs || I->src[s].neg;
+   /* Native C FMA consumes full and half registers in independently typed
+    * operand slots, including unmodified inputs. Route mixed operands through
+    * the same lifetime-aware modifier encoder. Plain F32 stays compact. */
+   if (enc == AGX_ISA_ENCODING_AGX3C && (is_fma || is_mul))
+      for (unsigned s = 0; s < I->nr_srcs; ++s)
+         has_mod |= agx3_src_is_register(I->src[s]) &&
+                    I->src[s].size == AGX_SIZE_16;
+   const bool narrow = enc == AGX_ISA_ENCODING_AGX3C &&
+                       I->dest[0].size == AGX_SIZE_16;
+   has_mod |= narrow;
    if (!has_mod)
       return 0; /* the compact no-modifier packer handles it */
 
@@ -670,7 +719,8 @@ agx3_try_float_modifier(struct util_dynarray *emission,
    if (I->nr_dests < 1 || I->dest[0].type == AGX_INDEX_NULL ||
        !agx3_src_is_register(I->dest[0]) || I->dest[0].memory ||
        agx_channels(I->dest[0]) != 1 ||
-       I->dest[0].size != AGX_SIZE_32 || (I->dest[0].value & 1) != 0 ||
+       (!narrow && (I->dest[0].size != AGX_SIZE_32 ||
+                        (I->dest[0].value & 1) != 0)) ||
        I->dest[0].value >= 256 || I->nr_srcs != want)
       return -1;
    for (unsigned s = 0; s < want; ++s)
@@ -692,7 +742,8 @@ agx3_try_float_modifier(struct util_dynarray *emission,
    bool ok;
    const uint16_t d = I->dest[0].value, s0 = I->src[0].value,
                   s1 = I->src[1].value;
-   if ((is_add || is_mul) && enc == AGX_ISA_ENCODING_AGX3B) {
+   if ((is_add || is_mul) &&
+       (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C)) {
       struct agx3_falu_runtime runtime = {
          .dest_cache = I->dest[0].cache,
          .wait_mask = wait,
@@ -703,7 +754,10 @@ agx3_try_float_modifier(struct util_dynarray *emission,
          runtime.cache[i] = I->src[i].cache;
          runtime.discard[i] = I->src[i].discard;
       }
-      ok = is_add
+      ok = narrow
+         ? (is_add ? agx3_falu_add_narrow_m5(d, s0, s1, &m, &runtime, bytes, &len)
+                   : agx3_falu_mul_narrow_m5(d, s0, s1, &m, &runtime, bytes, &len))
+         : is_add
          ? agx3_falu_add_mod_runtime(enc, d, s0, s1, &m, &runtime, bytes, &len)
          : agx3_falu_mul_mod_runtime(enc, d, s0, s1, &m, &runtime, bytes, &len);
    } else if (is_add)
@@ -714,7 +768,7 @@ agx3_try_float_modifier(struct util_dynarray *emission,
       m.addend_abs = I->src[2].abs;
       m.addend_neg = I->src[2].neg;
       m.addend_16 = I->src[2].size == AGX_SIZE_16;
-      if (enc == AGX_ISA_ENCODING_AGX3B) {
+      if (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) {
          struct agx3_falu_runtime runtime = {
             .dest_cache = I->dest[0].cache,
             .wait_mask = wait,
@@ -725,14 +779,18 @@ agx3_try_float_modifier(struct util_dynarray *emission,
             runtime.cache[i] = I->src[i].cache;
             runtime.discard[i] = I->src[i].discard;
          }
-         ok = agx3_falu_fma_mod_runtime(enc, d, s0, s1, I->src[2].value,
-                                       &m, &runtime, bytes, &len);
+         ok = narrow
+            ? agx3_falu_fma_narrow_m5(d, s0, s1, I->src[2].value,
+                                      &m, &runtime, bytes, &len)
+            : agx3_falu_fma_mod_runtime(enc, d, s0, s1, I->src[2].value,
+                                        &m, &runtime, bytes, &len);
       } else {
          ok = agx3_falu_fma_mod(enc, d, s0, s1, I->src[2].value,
                                &m, bytes, &len);
       }
    }
-   if (!ok || (wait && enc != AGX_ISA_ENCODING_AGX3B))
+   if (!ok || (wait && enc != AGX_ISA_ENCODING_AGX3B && !is_add &&
+               !((is_fma || is_mul) && enc == AGX_ISA_ENCODING_AGX3C)))
       return -1;
 
    memcpy(util_dynarray_grow_bytes(emission, 1, len), bytes, len);
@@ -774,6 +832,51 @@ static bool
 agx3_emit_funop(struct util_dynarray *emission, enum agx_isa_encoding enc,
                 const agx_instr *I, enum agx3_funop op, uint16_t wait_mask)
 {
+   if (enc == AGX_ISA_ENCODING_AGX3C && op <= AGX3_FUNOP_EXP2) {
+      if (I->nr_dests != 1 || I->nr_srcs != 1)
+         return false;
+
+      const agx_index dest = I->dest[0], src = I->src[0];
+      if (!agx3_src_is_register(dest) || dest.memory ||
+          (dest.size != AGX_SIZE_16 && dest.size != AGX_SIZE_32) ||
+          agx_channels(dest) != 1 || !agx3_src_is_register(src) || src.memory ||
+          (src.size != AGX_SIZE_16 && src.size != AGX_SIZE_32) ||
+          agx_channels(src) != 1 || src.kill || (src.cache && src.discard))
+         return false;
+
+      const enum agx3_alu_hint hint = src.cache ? AGX3_ALU_HINT_CACHE :
+                                    src.discard ? AGX3_ALU_HINT_DISCARD :
+                                                  AGX3_ALU_HINT_NONE;
+      uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+      return agx3_alu_unary_m5(op, dest.value, src.value,
+                               dest.size == AGX_SIZE_16,
+                               src.size == AGX_SIZE_16, src.abs, src.neg,
+                               I->saturate, hint, dest.cache, wait_mask,
+                               bytes, &len) &&
+             agx3_append(emission, bytes, len);
+   }
+
+   if (enc == AGX_ISA_ENCODING_AGX3C &&
+       (op == AGX3_FUNOP_DFDX || op == AGX3_FUNOP_DFDY)) {
+      if (I->nr_dests != 1 || I->nr_srcs != 1)
+         return false;
+
+      const agx_index dest = I->dest[0], src = I->src[0];
+      if (!agx3_src_is_register(dest) || dest.memory ||
+          !agx3_src_is_register(src) || src.memory ||
+          agx_channels(dest) != 1 || agx_channels(src) != 1 ||
+          dest.size != src.size ||
+          (dest.size != AGX_SIZE_16 && dest.size != AGX_SIZE_32))
+         return false;
+
+      uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+      return agx3_alu_derivative_m5(op == AGX3_FUNOP_DFDY, dest.value,
+                                    src.value, dest.size == AGX_SIZE_16,
+                                    src.abs, src.neg, I->saturate, wait_mask,
+                                    bytes, &len) &&
+             agx3_append(emission, bytes, len);
+   }
+
    if (I->nr_dests != 1 || I->nr_srcs != 1 ||
        I->dest[0].type == AGX_INDEX_NULL ||
        !agx3_src_is_register(I->dest[0]) || I->dest[0].memory ||
@@ -884,7 +987,7 @@ agx3_map_sr(enum agx_isa_encoding enc, enum agx_sr sr, uint8_t *apple,
       *width = sr == AGX_SR_MESH_PRIMITIVE_STORAGE_LO ? AGX3_SR_W32 : AGX3_SR_W16;
       return true;
    case AGX_SR_INPUT_SAMPLE_MASK:
-      if (enc != AGX_ISA_ENCODING_AGX3B)
+      if (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C)
          return false;
       *apple = 39;
       *width = AGX3_SR_W16;
@@ -892,13 +995,15 @@ agx3_map_sr(enum agx_isa_encoding enc, enum agx_sr sr, uint8_t *apple,
       SR(AGX_SR_COVERAGE_MASK, 39, AGX3_SR_W16);
       SR(AGX_SR_IS_ACTIVE_THREAD, 2, AGX3_SR_W16);
    case AGX_SR_BACKFACING:
-      if (enc != AGX_ISA_ENCODING_AGX3B)
+      if (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C)
          return false;
       *apple = 41;
       *width = AGX3_SR_W16;
       return true;
    case AGX_SR_BACKGROUND_LAYER_ID:
-      if (enc != AGX_ISA_ENCODING_AGX3B)
+      /* Native G17G llvm.agx2.get.layerid reads the same SR12/W16
+       * tile-layer input as G17P (background-load internal witness). */
+      if (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C)
          return false;
       *apple = 12;
       *width = AGX3_SR_W16;
@@ -1111,7 +1216,7 @@ agx3_interp_uses_coords(enum agx_isa_encoding enc, const agx_instr *I)
 {
    return I->interpolation == AGX_INTERPOLATION_SAMPLE ||
           (I->interpolation == AGX_INTERPOLATION_CENTROID &&
-           enc == AGX_ISA_ENCODING_AGX3B);
+           (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C));
 }
 
 static bool
@@ -1123,13 +1228,13 @@ agx3_emit_interp(struct util_dynarray *emission, enum agx_isa_encoding enc,
        !agx3_vary_reg32(I->dest[0]) || I->src[0].type != AGX_INDEX_IMMEDIATE)
       return false;
 
-   /* Sample and G17P centroid interpolation consume an ITRCOORDS token
-    * through the same f32interp leaves. */
+   /* Sample and centroid interpolation consume an ITRCOORDS token through
+    * the same f32interp leaves on G17P and G17G (lane-mesh/cwit-interp). */
    const bool token = agx3_interp_uses_coords(enc, I);
    enum agx3_vary_location location;
    if (token) {
-      if (enc != AGX_ISA_ENCODING_AGX3B || first ||
-          !agx3_vary_reg32(I->src[1]) || I->src[1].kill ||
+      if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+          first || !agx3_vary_reg32(I->src[1]) || I->src[1].kill ||
           (I->src[1].cache && I->src[1].discard))
          return false;
    } else {
@@ -1296,6 +1401,10 @@ static bool
 agx3_retire_slots_noop(struct util_dynarray *emission,
                        enum agx_isa_encoding enc, uint16_t mask);
 
+static bool
+agx3_retire_wait_on(struct util_dynarray *emission, enum agx_isa_encoding enc,
+                    agx_index reg, uint16_t mask);
+
 static bool
 agx3_retire_result_wait(struct util_dynarray *emission,
                         enum agx_isa_encoding enc, const agx_instr *I,
@@ -1324,6 +1433,20 @@ agx3_tex_plain_uniform64(agx_index index)
           !index.kill && !index.cache && !index.discard;
 }
 
+/* Whether agx3_emit_texture_sample() publishes this sample's CEM face
+ * coordinates with the sample's result slot rather than a staging slot: a cube
+ * sample whose texture and sampler operands are both non-register. */
+static bool
+agx3_tex_cube_groups_result_slot(const agx_instr *I)
+{
+   if (I->op != AGX_OPCODE_TEXTURE_SAMPLE ||
+       (I->dim != AGX_DIM_CUBE && I->dim != AGX_DIM_CUBE_ARRAY) ||
+       I->nr_srcs < 5)
+      return false;
+   return I->src[3].type != AGX_INDEX_REGISTER &&
+          I->src[4].type != AGX_INDEX_REGISTER;
+}
+
 /* Translate the corpus-grounded texel fetch. Dynamic integer LOD is
  * published into operand bank 6. Native G17P array fetches publish their
  * combined layer/LOD word and return the texture result on slot 1; plain 2D
@@ -1334,6 +1457,118 @@ agx3_tex_plain_uniform64(agx_index index)
  * staged half within the COPY destination field. */
 #define AGX3_STAGED_WRITE_LIMIT 128u
 
+struct agx3_fetch_group {
+   bool reuse;
+   bool keep;
+   uint16_t coord;
+};
+
+/* Coordinate word c of a texel fetch or sample: that word of its coordinate
+ * vector, or its own scalar source when it takes split coordinates. */
+static agx_index
+agx3_tex_coord(const agx_instr *I, unsigned c)
+{
+   if (I->agx3_split_coords)
+      return c ? I->src[c + 5] : I->src[0];
+   agx_index coord = I->src[0];
+   coord.value += 2 * c;
+   coord.channels_m1 = 0;
+   return coord;
+}
+
+/* Whether staging coordinate word c is the last read of its register by this
+ * fetch or sample. Split components may name one register twice; only the
+ * final read may discard it. Words are read in order, x, y, then the third
+ * word. */
+static bool
+agx3_tex_coord_discard(const agx_instr *I, unsigned c, unsigned count)
+{
+   const agx_index coord = agx3_tex_coord(I, c);
+   if (!coord.discard)
+      return false;
+   for (unsigned later = c + 1; later < count; ++later) {
+      if (agx3_tex_coord(I, later).value == coord.value)
+         return false;
+   }
+   return true;
+}
+
+static unsigned
+agx3_fetch_nr_srcs(const agx_instr *I)
+{
+   return I->agx3_split_coords ? 8 : 6;
+}
+
+/* Consecutive raw 2D reads can consume the same immutable coordinate pair.
+ * Do not cross even a single intervening instruction: it may overwrite the
+ * GPR pair or publish another sampler-private operand. */
+static bool
+agx3_fetch_coordinates_match(const agx_instr *a, const agx_instr *b)
+{
+   if (!b || a->op != AGX_OPCODE_TEXTURE_LOAD || b->op != a->op ||
+       a->nr_srcs != agx3_fetch_nr_srcs(a) ||
+       b->nr_srcs != agx3_fetch_nr_srcs(b) || a->nr_dests != 1 ||
+       a->dim != AGX_DIM_2D || b->dim != AGX_DIM_2D ||
+       a->lod_mode != AGX_LOD_MODE_LOD_MIN || b->lod_mode != a->lod_mode ||
+       a->src[1].type != AGX_INDEX_IMMEDIATE ||
+       !agx3_tex_imm(b->src[1], a->src[1].value) ||
+       a->offset || b->offset || a->shadow || b->shadow ||
+       a->sparse || b->sparse || a->query_lod || b->query_lod ||
+       (!a->agx3_split_coords && agx_channels(a->src[0]) != 2) ||
+       (!b->agx3_split_coords && agx_channels(b->src[0]) != 2))
+      return false;
+
+   /* Matching physical register numbers do not imply matching values if the
+    * first fetch replaces any part of the second fetch's coordinate input. */
+   const unsigned dst = a->dest[0].value;
+   for (unsigned c = 0; c < 2; ++c) {
+      const agx_index x = agx3_tex_coord(a, c), y = agx3_tex_coord(b, c);
+      if (!agx3_tex_plain_reg(x) || !agx3_tex_plain_reg(y) ||
+          x.size != AGX_SIZE_32 || y.size != AGX_SIZE_32 ||
+          x.value != y.value)
+         return false;
+      if (agx3_tex_plain_reg(a->dest[0]) &&
+          dst + agx_index_size_16(a->dest[0]) > x.value && dst < x.value + 2)
+         return false;
+   }
+   return true;
+}
+
+/* Split coordinate word c: x and y use the wide scalar COPY, which reads the
+ * whole register file; a sample index or layer keeps PACK_SMPIR's seven-bit
+ * source. */
+static bool
+agx3_split_coord_ok(agx_index coord, unsigned c)
+{
+   return agx3_tex_plain_reg(coord) && coord.size == AGX_SIZE_32 &&
+          agx_channels(coord) == 1 && !(coord.value & 1) &&
+          coord.value + 2 <= (c == 2 ? 128 : 256);
+}
+
+/* The coordinate operands of a texel fetch: a vector of `count` words, or,
+ * for a staged M5 2D or multisampled read, `count` independent scalars. */
+static bool
+agx3_fetch_coords_ok(const agx_instr *I, bool m5, bool staged, unsigned count)
+{
+   if (I->agx3_split_coords) {
+      if (!m5 || !staged ||
+          (I->dim != AGX_DIM_2D && I->dim != AGX_DIM_2D_MS &&
+           I->dim != AGX_DIM_2D_ARRAY) ||
+          (count == 2) != agx_is_null(I->src[7]))
+         return false;
+      for (unsigned c = 0; c < count; ++c) {
+         if (!agx3_split_coord_ok(agx3_tex_coord(I, c), c))
+            return false;
+      }
+      return true;
+   }
+   return agx3_tex_plain_reg(I->src[0]) && I->src[0].size == AGX_SIZE_32 &&
+          agx_channels(I->src[0]) == count &&
+          (staged ? !(I->src[0].value & 1) &&
+                       I->src[0].value + 2 * count <= 128
+                  : !(I->src[0].value & 3) && I->src[0].value <= 60);
+}
+
 static bool
 agx3_emit_texture_fetch(struct util_dynarray *emission,
                         enum agx_isa_encoding enc, const agx_instr *I,
@@ -1342,18 +1577,21 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
                         uint16_t ms_array_scratch,
                         struct agx3_staging_banks *banks,
                         uint16_t outstanding,
-                        uint16_t *staged_end, bool pixel_entry,
-                        uint32_t *pixel_entry_offset,
-                        uint16_t heap_uniform_base)
+                        uint16_t *staged_end, bool fragment, bool pixel_entry,
+                        uint32_t *pixel_entry_offset, uint16_t heap_uniform_base,
+                        struct agx3_fetch_group *group)
 {
-   const bool immediate = I->nr_srcs == 6 &&
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const bool immediate = I->nr_srcs == agx3_fetch_nr_srcs(I) &&
       I->src[3].type == AGX_INDEX_IMMEDIATE;
    const bool immediate_lod = I->lod_mode == AGX_LOD_MODE_LOD_MIN &&
       I->src[1].type == AGX_INDEX_IMMEDIATE && !I->src[1].memory &&
       !I->src[1].abs && !I->src[1].neg && I->src[1].value <= 31;
    const bool register_lod =
       I->lod_mode == AGX_LOD_MODE_LOD_MIN &&
-      agx3_tex_plain_reg(I->src[1]) && I->src[1].size == AGX_SIZE_32 &&
+      agx3_tex_plain_reg(I->src[1]) &&
+      (I->src[1].size == AGX_SIZE_32 ||
+       (m5 && I->src[1].size == AGX_SIZE_16)) &&
       agx_channels(I->src[1]) == 1;
    const bool ms_2d = I->dim == AGX_DIM_2D_MS;
    const bool ms_array = I->dim == AGX_DIM_2D_MS_ARRAY;
@@ -1370,27 +1608,23 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
     * staging slot 2, then issues the texture result on slot 1. Keep those two
     * independent scoreboard domains; reusing slot 1 for both is not a native
     * dependency shape. */
-   const uint8_t staging_slot = ms_array || combined_array_lod
+   const uint8_t staging_slot = m5 ? slot : ms_array || combined_array_lod
       ? 1 : register_lod ? 2 : slot;
    /* A sparse read writes one extra register after its masked channels, so
     * agx_emit_texture reserves an eight-channel destination for it (see
     * agx_validate.c's sparse size rule). The encoded result mask stays the
     * four-bit channel mask; the residency component is implicit. */
-   if (I->nr_dests != 1 || I->nr_srcs != 6 ||
+   const unsigned coord_count =
+      (I->dim == AGX_DIM_2D_ARRAY || ms_array || ms_2d || three_d) ? 3 : 2;
+   if (I->nr_dests != 1 || I->nr_srcs != agx3_fetch_nr_srcs(I) ||
        !agx3_tex_plain_reg(I->dest[0]) ||
        (I->dest[0].size != AGX_SIZE_16 &&
         I->dest[0].size != AGX_SIZE_32) ||
        agx_channels(I->dest[0]) != (I->sparse ? 8u : 4u) ||
-       I->dest[0].value >= 128 ||
+       I->dest[0].value >= (m5 ? 256 : 128) ||
+       I->dest[0].value + agx_index_size_16(I->dest[0]) > 256 ||
        (I->dest[0].size == AGX_SIZE_32 && (I->dest[0].value & 1)) ||
-       !agx3_tex_plain_reg(I->src[0]) ||
-       I->src[0].size != AGX_SIZE_32 ||
-       agx_channels(I->src[0]) !=
-          ((I->dim == AGX_DIM_2D_ARRAY || ms_array || ms_2d || three_d)
-              ? 3 : 2) ||
-       (stage_coordinates ? ((I->src[0].value & 1) ||
-         I->src[0].value + 2 * agx_channels(I->src[0]) > 128) :
-         ((I->src[0].value & 3) || I->src[0].value > 60)) ||
+       !agx3_fetch_coords_ok(I, m5, stage_coordinates, coord_count) ||
        ((ms_array || ms_2d)
            ? !agx3_tex_imm(I->src[1], 0)
            : (!(I->lod_mode == AGX_LOD_MODE_AUTO_LOD &&
@@ -1402,7 +1636,7 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
          ((ms_array || ms_2d) && agx3_tex_plain_uniform64(I->src[2])) ||
          bindless_image) ||
        (immediate ? I->src[3].value > 15 :
-        (enc != AGX_ISA_ENCODING_AGX3B ||
+        ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
          (I->src[3].type != AGX_INDEX_REGISTER && I->src[3].type != AGX_INDEX_UNIFORM) ||
          (I->src[3].size != AGX_SIZE_64 &&
           !((ms_array || ms_2d) && I->src[3].size == AGX_SIZE_32)) ||
@@ -1413,17 +1647,16 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
        !agx_is_null(I->src[5]) ||
        (I->dim != AGX_DIM_2D && I->dim != AGX_DIM_2D_ARRAY && !ms_2d &&
         !ms_array && !three_d) ||
-       (three_d && enc != AGX_ISA_ENCODING_AGX3B) ||
+
        /* agx_instr::mask is four bits wide, so the residency component of a
         * sparse result never appears in it; the hardware writes that
         * component implicitly, after the masked channels. */
-       (enc == AGX_ISA_ENCODING_AGX3B ? (I->mask == 0 || (I->mask & ~15u))
-                                    : (I->mask != 1 && I->mask != 15)) ||
+       (I->mask == 0 || (I->mask & ~15u)) ||
        I->offset || I->shadow ||
        I->query_lod || I->gather != AGX_GATHER_NONE ||
-       /* Only AGX3b witnesses the sparse read, and the multisample form has
-        * no witness at all. */
-       (I->sparse && (enc != AGX_ISA_ENCODING_AGX3B || ms_2d || ms_array)))
+       /* Both encodings support sparse reads; sparse multisample has no
+        * qualified form. */
+       (I->sparse && (ms_2d || ms_array)))
       return false;
 
    /* A read/write image stages its read coordinates at halves16/18, keeping
@@ -1438,7 +1671,7 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
       (I->op == AGX_OPCODE_IMAGE_LOAD && !bindless_image && !ms_array &&
        !ms_2d) ||
             separate_write_coords ? 16 : 0;
-   if (register_lod && slot != 1)
+   if (!m5 && register_lod && slot != 1)
       return false;
 
    /* A staged read consumes, and kills, its coordinate bank asynchronously,
@@ -1448,24 +1681,50 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
     * read with PACK_SMPIR operands keeps the base bank and occupies halves
     * 0..7. An outstanding read sharing the halves is retired first. */
    const bool plain_bank = !ms_2d && !ms_array && !array && !register_lod;
+   const bool m5_ms_banks = m5 && (ms_2d || ms_array);
+   const bool banked_read = plain_bank || m5_ms_banks;
+   /* Plain M5 fetch coordinates can name any aligned bank through half 60.
+    * Use independent banks for all eight scoreboard slots before recycling
+    * one. The diagnostic override only restricts this proven address range. */
+   const unsigned plain_banks = m5 ?
+      CLAMP(debug_get_num_option("ASAHI_M5_FETCH_BANKS", 8), 1,
+            MIN2(8, 1 + (60 - staging_base) / 8)) : 2;
+   /* Keep independent non-array MS publications in separate operand banks. */
+   const unsigned ms_banks = m5_ms_banks && !ms_array ?
+      CLAMP(debug_get_num_option("ASAHI_M5_MS_FETCH_BANKS", 8), 1, 16) : 4;
    const uint16_t bank_span =
-      plain_bank ? 2 * agx_channels(I->src[0]) : staging_base + 8;
+      m5_ms_banks ? 8 : plain_bank ? 2 * coord_count
+                                 : staging_base + 8;
    uint16_t staging_coord = staging_base;
    uint16_t bank_retire = 0;
-   if (stage_coordinates && banks) {
+   const bool reuse = group && group->reuse;
+   if (reuse) {
+      if (!m5 || !stage_coordinates || !banks || !plain_bank)
+         return false;
+      staging_coord = group->coord;
+      /* Other sources and destination hazards still belong to this fetch;
+       * only the already-completed coordinate publication is shared. The
+       * coordinates are ready once this fetch's wait retires, so they carry
+       * it instead of r0, whose producer may be outside the mask. */
+      if (wait_mask && !agx3_retire_wait_on(emission, enc, I->src[0],
+                                            wait_mask))
+         return false;
+      wait_mask = 0;
+   } else if (stage_coordinates && banks) {
       agx3_staging_release(banks, outstanding);
 
       /* A read with PACK_SMPIR operands occupies halves 0..bank_span. */
-      const uint16_t first = plain_bank ? staging_base : 0;
+      const uint16_t first = banked_read ? staging_base : 0;
       bool placed = false;
-      for (unsigned c = 0; c < (plain_bank ? 2u : 1u) && !placed; ++c) {
+      for (unsigned c = 0; c < (m5_ms_banks ? ms_banks : plain_bank ? plain_banks : 1u) &&
+                           !placed; ++c) {
          const uint16_t start = first + 8 * c;
          const uint16_t conflict = agx3_staging_conflicts(
-            banks, plain_bank ? start : 0, bank_span);
+            banks, banked_read ? start : 0, bank_span);
          if (c == 0)
             bank_retire = conflict;
          if (!conflict) {
-            staging_coord = plain_bank ? start : staging_base;
+            staging_coord = banked_read ? start : staging_base;
             placed = true;
          }
       }
@@ -1474,17 +1733,30 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
       else
          agx3_staging_forget(banks, bank_retire);
    }
-   if (bank_retire && !agx3_retire_slots_noop(emission, enc, bank_retire))
+   /* The r0 carrier reads and rewrites r0, so it is sound only while r0 is
+    * ready or its producer is in the mask. When M5 multisample reads fill
+    * all four operand banks, the next one retires the first bank's read,
+    * and r0 may be the destination of a later read still in flight: in the
+    * stencil resolve of dEQP-VK.renderpasses.renderpass2.
+    * depth_stencil_resolve.*.stencil_min, five texel fetches were in
+    * flight, the second writing r0..r3, and the carrier waited for the
+    * first alone. Carry the retirement on this fetch's coordinates with
+    * the fetch's own wait, which covers every pending write of them; the
+    * staging COPYs right after it wait for the same producers. */
+   if (bank_retire &&
+       !(m5 ? agx3_retire_wait_on(emission, enc, I->src[0],
+                                  bank_retire | wait_mask)
+            : agx3_retire_slots_noop(emission, enc, bank_retire)))
       return false;
    if (staged_end)
-      *staged_end = plain_bank ? staging_coord + bank_span : bank_span;
+      *staged_end = banked_read ? staging_coord + bank_span : bank_span;
 
    /* The plain 2D fetch publishes its per-lane level after the coordinates,
     * so no PACK_SMPIR leads the group to absorb a combined producer mask. */
    const bool plain_2d_lod =
       register_lod && !combined_array_lod && I->dim == AGX_DIM_2D;
 
-   if (stage_coordinates) {
+   if (stage_coordinates && !reuse) {
       uint16_t staging_wait = wait_mask & (uint16_t)~bank_retire;
       /* COPY carries one wait bit, while the image coordinates and heap
        * index may have several independent producers. A runtime LOD's
@@ -1500,12 +1772,31 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
             return false;
          staging_wait = 0;
       }
+      if (m5 && plain_2d_lod) {
+         uint8_t lod[AGX3_TEX_MAX_LENGTH], lod_len = 0;
+         if (!agx3_tex_stage_typed_lod_at(
+                enc, staging_coord, I->src[1].value, staging_slot,
+                staging_wait, false, I->src[1].discard,
+                AGX3_TEX_LEVEL_INTEGER, lod, &lod_len) ||
+             !agx3_append(emission, lod, lod_len))
+            return false;
+         staging_wait = 0;
+      }
       if (combined_array_lod) {
          uint8_t packed[AGX3_TEX_MAX_LENGTH], packed_len = 0;
+         /* The layer is read before the x/y COPYs: it may only discard a
+          * register neither of them still reads. */
+         const agx_index layer = agx3_tex_coord(I, 2);
+         const bool layer_discard = layer.discard &&
+            agx3_tex_coord(I, 0).value != layer.value &&
+            agx3_tex_coord(I, 1).value != layer.value;
          if (!agx3_tex_stage_array_integer_lod_at(
-                enc, staging_base, I->src[0].value + 4, I->src[1].value,
-                staging_slot, staging_wait, I->src[0].cache,
-                I->src[0].discard, I->src[1].cache, I->src[1].discard,
+                enc, staging_base, layer.value, I->src[1].value,
+                staging_slot, staging_wait,
+                m5 ? !layer_discard : layer.cache,
+                layer_discard,
+                m5 ? !I->src[1].discard : I->src[1].cache,
+                I->src[1].discard,
                 packed, &packed_len) ||
              !agx3_append(emission, packed, packed_len))
             return false;
@@ -1514,15 +1805,15 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
          uint8_t lod[AGX3_TEX_MAX_LENGTH], lod_len = 0;
          if (!agx3_tex_stage_typed_lod_at(
                 enc, staging_base, I->src[1].value, staging_slot,
-                staging_wait, I->src[1].cache, I->src[1].discard,
-                AGX3_TEX_LEVEL_INTEGER, lod, &lod_len) ||
+                staging_wait, m5 ? false : I->src[1].cache,
+                I->src[1].discard, AGX3_TEX_LEVEL_INTEGER, lod, &lod_len) ||
              !agx3_append(emission, lod, lod_len))
             return false;
          staging_wait = 0;
       }
 
       const unsigned plain_coords =
-         (ms_2d || ms_array || array) ? 2 : agx_channels(I->src[0]);
+         (ms_2d || ms_array || array) ? 2 : coord_count;
       /* The staging COPY carries one wait bit. A fetch whose operands come
        * from several scoreboard slots repeats the first coordinate copy, one
        * outstanding bit each; every repeat writes the same register from the
@@ -1539,13 +1830,20 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
             pending_wait &= (uint16_t)~wait_bit;
          }
          uint8_t copy[AGX3_TEX_MAX_LENGTH], copy_len = 0;
-         if (!agx3_tex_stage_copy_hints(enc, staging_coord + 2 * i,
-                                  I->src[0].value + 2 * i,
-                                  staging_slot, wait_bit,
-                                  enc == AGX_ISA_ENCODING_AGX3B && I->src[0].cache,
-                                  enc != AGX_ISA_ENCODING_AGX3B ||
-                                     (I->src[0].discard && rep + 1 == reps),
-                                  copy, &copy_len) ||
+         const agx_index coord = agx3_tex_coord(I, i);
+         const bool last =
+            agx3_tex_coord_discard(I, i, coord_count) && rep + 1 == reps;
+         /* Split coordinates may live anywhere in the register file: stage
+          * them with the wide scalar COPY, as scalar publications do. */
+         const bool copied = I->agx3_split_coords
+            ? agx3_tex_stage_copy_scalar_m5(staging_coord + 2 * i, coord.value,
+                                            staging_slot, wait_bit, !last, last,
+                                            copy, &copy_len)
+            : agx3_tex_stage_copy_hints(enc, staging_coord + 2 * i,
+                                        coord.value, staging_slot, wait_bit,
+                                        m5 ? !last : coord.cache, last,
+                                        copy, &copy_len);
+         if (!copied ||
              !agx3_append(emission, copy, copy_len))
             return false;
        }
@@ -1554,7 +1852,7 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
       /* A plain 2D fragment texel fetch stages its level after its
        * coordinates (coordinate copies, PACK_SMPIR, SMP) and into its own
        * operand bank; a level published elsewhere reads back as zero. */
-      if (plain_2d_lod) {
+      if (plain_2d_lod && !m5) {
          uint8_t lod[AGX3_TEX_MAX_LENGTH], lod_len = 0;
          if (!agx3_tex_stage_typed_lod_bank(
                 enc, staging_coord + 6, I->src[1].value, staging_slot,
@@ -1571,9 +1869,10 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
        * lowered layer is the low16 bits of the third coordinate word. */
       if (array && !combined_array_lod) {
          uint8_t packed[AGX3_TEX_MAX_LENGTH], packed_len = 0;
+         const agx_index layer = agx3_tex_coord(I, 2);
          if (!agx3_tex_stage_array_layer_at(
-                enc, staging_base, I->src[0].value + 4, staging_slot, 0,
-                I->src[0].cache, I->src[0].discard,
+                enc, staging_base, layer.value, staging_slot, 0,
+                layer.cache, layer.discard,
                 packed, &packed_len) ||
              !agx3_append(emission, packed, packed_len))
             return false;
@@ -1583,9 +1882,10 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
        * Publish it through native's dedicated integer PACK_SMPIR operand. */
       if (ms_2d) {
          uint8_t packed[AGX3_TEX_MAX_LENGTH], packed_len = 0;
-         if (!agx3_tex_stage_ms_sample(
-                enc, I->src[0].value + 4, 0,
-                I->src[0].cache, I->src[0].discard,
+         const agx_index sample = agx3_tex_coord(I, 2);
+         if (!agx3_tex_stage_ms_sample_at(
+                enc, staging_coord, sample.value, staging_slot, 0,
+                sample.cache, sample.discard,
                 packed, &packed_len) ||
              !agx3_append(emission, packed, packed_len))
             return false;
@@ -1599,8 +1899,9 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
                 enc, I->src[0].value + 5, ms_array_scratch,
                 I->src[0].cache, false, packed, &packed_len) ||
              !agx3_append(emission, packed, packed_len) ||
-             !agx3_tex_stage_ms_array_sample(
-                enc, I->src[0].value + 4, ms_array_scratch,
+             !agx3_tex_stage_ms_array_sample_at(
+                enc, staging_coord, I->src[0].value + 4, ms_array_scratch,
+                staging_slot,
                 I->src[0].cache, I->src[0].discard, packed, &packed_len) ||
              !agx3_append(emission, packed, packed_len))
             return false;
@@ -1608,12 +1909,13 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
    }
 
    if (stage_coordinates && banks &&
-       !agx3_staging_hold(banks, plain_bank ? staging_coord : 0, bank_span,
+       !agx3_staging_hold(banks, banked_read ? staging_coord : 0, bank_span,
                           slot))
       return false;
 
    const struct agx3_tex_fetch_desc d = {
       .heap_uniform_base = heap_uniform_base,
+      .dest_cache = m5 && I->dest[0].cache,
       .dest = I->dest[0].value,
       .coord = stage_coordinates ? staging_coord : I->src[0].value,
       .width = I->dest[0].size == AGX_SIZE_32 ? AGX3_TEX_W32 : AGX3_TEX_W16,
@@ -1624,16 +1926,22 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
          I->src[3].type == AGX_INDEX_UNIFORM ? AGX3_TEX_OPERAND_UNIFORM :
                                              AGX3_TEX_OPERAND_REGISTER,
       .texture_value = immediate ? 0 : I->src[3].value,
-      .texture_kill = !immediate && I->src[3].type == AGX_INDEX_REGISTER &&
+      .texture_kill = enc != AGX_ISA_ENCODING_AGX3C && !immediate &&
+         I->src[3].type == AGX_INDEX_REGISTER &&
          (I->src[3].discard || I->src[3].kill),
       .slot = slot,
-      .wait_mask = stage_coordinates ? (1u << (staging_slot - 1)) : wait_mask,
-      .coord_kill = stage_coordinates || I->src[0].kill,
+      .wait_mask = reuse ? 0 : stage_coordinates
+         ? (1u << (staging_slot - 1)) : wait_mask,
+      .coord_kill = stage_coordinates ? !(group && group->keep)
+                                       : I->src[0].kill,
       .final = final,
       .array = I->dim == AGX_DIM_2D_ARRAY || ms_array,
       .multisample = ms_2d || ms_array,
       .staged_sample = (ms_2d || ms_array) && stage_coordinates,
-      .sampler_value = (ms_2d || ms_array) && stage_coordinates ? 1 : 0,
+      /* M5 raw reads use sampler selector1 in fragment programs and0 in
+       * compute programs, for both ordinary and multisampled images. */
+      .sampler_value = m5 ? (fragment ? 1 : 0) :
+         ((ms_2d || ms_array) && stage_coordinates ? 1 : 0),
       .three_d = three_d,
       .lod_x64 = immediate_lod ? I->src[1].value * 64 : 0,
       .register_lod = register_lod,
@@ -1655,8 +1963,12 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
          return false;
    }
 
-   return agx3_tex_encode_fetch(enc, &d, bytes, &len) &&
-          agx3_append(emission, bytes, len);
+   if (!agx3_tex_encode_fetch(enc, &d, bytes, &len) ||
+       !agx3_append(emission, bytes, len))
+      return false;
+   if (group)
+      group->coord = staging_coord;
+   return true;
 }
 
 /* Two bounded public storage-image forms: integer 2D state0 fed by an image
@@ -1670,6 +1982,16 @@ agx3_emit_texture_fetch(struct util_dynarray *emission,
  * producer; either way native imgwci waits for the new staging-slot1 writes.
  * Validation happens before the first append, so a refusal leaves the output
  * transaction untouched. */
+static bool
+agx3_image_write_copy(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+                      bool kill, uint8_t slot, uint16_t wait,
+                      uint8_t out[AGX3_STAGE_MAX_LENGTH], uint8_t *len)
+{
+   return enc == AGX_ISA_ENCODING_AGX3C
+      ? agx3_tex_stage_copy_hints(enc, dest, src, slot, wait, !kill, kill, out, len)
+      : agx3_stage_copy_source(enc, dest, src, kill, slot, wait, out, len);
+}
+
 static bool
 agx3_emit_storage_image_write(struct util_dynarray *emission,
                               enum agx_isa_encoding enc, const agx_instr *I,
@@ -1679,6 +2001,9 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
                               uint8_t *write_slot,
                               uint16_t heap_uniform_base)
 {
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = m5 ? 8 : 6;
+   const unsigned bank_limit = m5 ? 256 : AGX3_STAGED_WRITE_LIMIT;
    const bool normalized = I->format == AGX_FORMAT_U8NORM ||
                            I->format == AGX_FORMAT_RGB10A2;
    const bool immediate_state = agx3_tex_imm(I->src[3], 0) &&
@@ -1707,15 +2032,19 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
    const bool array = I->dim == AGX_DIM_2D_ARRAY;
    const bool three_d = I->dim == AGX_DIM_3D;
    const unsigned coords = (array || three_d) ? 3 : 2;
-   if (enc != AGX_ISA_ENCODING_AGX3B || I->nr_dests != 0 ||
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || I->nr_dests != 0 ||
+       (m5 && ((I->dim != AGX_DIM_2D && !array && !three_d) || !heap_state ||
+               ((array || three_d) && I->format != AGX_FORMAT_I8) ||
+               (I->format != AGX_FORMAT_I8 && I->format != AGX_FORMAT_I32 &&
+                I->format != AGX_FORMAT_U8NORM))) ||
        I->nr_srcs != 5 ||
        (I->dim != AGX_DIM_2D && !array && !three_d) ||
-       (wait_mask & ~UINT16_C(0x3f)) != 0 ||
+       (wait_mask & ~((1u << slots) - 1)) != 0 ||
        !agx3_tex_plain_reg(I->src[0]) || I->src[0].size != AGX_SIZE_32 ||
        agx_channels(I->src[0]) != 4 || (I->src[0].value & 1) ||
-       I->src[0].value > 248 || !agx3_tex_plain_reg(I->src[1]) ||
+       I->src[0].value > (m5 ? 120 : 248) || !agx3_tex_plain_reg(I->src[1]) ||
        I->src[1].size != AGX_SIZE_32 || agx_channels(I->src[1]) != coords ||
-       (I->src[1].value & 1) || I->src[1].value + 2 * (coords - 1) > 254 ||
+       (I->src[1].value & 1) || I->src[1].value + 2 * (coords - 1) > (m5 ? 126 : 254) ||
        !agx3_tex_imm(I->src[2], 0) ||
        (!immediate_state && !heap_state)) {
       if (agx_compiler_debug & AGX_DBG_SHADERS)
@@ -1731,6 +2060,12 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
     * waited; consecutive writes (and a write after a texture read) stage in
     * distinct banks, as Apple's do. The resolved-PBE forms below keep bank 0
     * and slot 1. */
+   if (m5 && util_bitcount(wait_mask) > 1) {
+      if (!agx3_retire_slots_noop(emission, enc, wait_mask))
+         return false;
+      *outstanding &= (uint16_t)~wait_mask;
+      wait_mask = 0;
+   }
    const unsigned bank_span = 16;
    uint16_t bank = 0;
    uint8_t slot = 1;
@@ -1740,9 +2075,9 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
       uint16_t retire = 0;
       bool placed = false;
       for (unsigned start = heap_state ? bank_floor : 0;
-           start + bank_span <= AGX3_STAGED_WRITE_LIMIT && !placed &&
+           start + bank_span <= bank_limit && !placed &&
            (heap_state || start == 0);
-           start += 8) {
+           start += m5 ? 16 : 8) {
          if (!agx3_staging_conflicts(banks, start, bank_span)) {
             bank = start;
             placed = true;
@@ -1758,16 +2093,16 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
       const uint16_t busy = *outstanding & (uint16_t)~(wait_mask | retire);
       if (heap_state) {
          slot = 0;
-         for (unsigned i = 0; i < 6 && !slot; ++i) {
-            const uint8_t s = (uint8_t)((banks->next_write_slot + i) % 6 + 1);
+         for (unsigned i = 0; i < slots && !slot; ++i) {
+            const uint8_t s = (uint8_t)((banks->next_write_slot + i) % slots + 1);
             if (!(busy & (1u << (s - 1))))
                slot = s;
          }
          if (!slot) {
-            slot = (uint8_t)(banks->next_write_slot % 6 + 1);
+            slot = (uint8_t)(banks->next_write_slot % slots + 1);
             retire |= 1u << (slot - 1);
          }
-         banks->next_write_slot = slot % 6;
+         banks->next_write_slot = slot % slots;
       } else if (busy & 1u) {
          retire |= 1u;
       }
@@ -1801,14 +2136,14 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
     * The scoreboard wait goes on the FIRST staged COPY of the group (the
     * coordinates), as for every other staged producer: a coordinate from an
     * outstanding device load must retire before it is copied. */
-   if (!agx3_stage_copy_source(enc, bank, I->src[1].value, false, slot,
+   if (!agx3_image_write_copy(enc, bank, I->src[1].value, false, slot,
                                wait_mask, coord_bytes, &coord_len) ||
        !agx3_append(&staged, coord_bytes, coord_len) ||
-       !agx3_stage_copy_source(enc, bank + 2, I->src[1].value + 2, false,
+       !agx3_image_write_copy(enc, bank + 2, I->src[1].value + 2, false,
                                slot, 0, coord_bytes, &coord_len) ||
        !agx3_append(&staged, coord_bytes, coord_len) ||
        (three_d &&
-        (!agx3_stage_copy_source(enc, bank + 4, I->src[1].value + 4, false,
+        (!agx3_image_write_copy(enc, bank + 4, I->src[1].value + 4, false,
                                  slot, 0, coord_bytes, &coord_len) ||
          !agx3_append(&staged, coord_bytes, coord_len))) ||
        (array &&
@@ -1823,9 +2158,10 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
 
    for (unsigned c = 0; c < 4; ++c) {
       uint8_t bytes[AGX3_STAGE_MAX_LENGTH], len = 0;
-      if (!agx3_stage_copy_source(enc, data_dest[c],
-                                  I->src[0].value + 2 * c,
-                                  I->src[0].discard, slot, 0,
+      const unsigned source = I->src[0].value + 2 * c;
+      const bool index_alias = m5 && heap_state && source == I->src[4].value;
+      if (!agx3_image_write_copy(enc, data_dest[c], source,
+                                  I->src[0].discard && !index_alias, slot, 0,
                                   bytes, &len) ||
           !agx3_append(&staged, bytes, len)) {
          if (agx_compiler_debug & AGX_DBG_SHADERS)
@@ -1834,7 +2170,7 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
       }
    }
 
-   uint8_t imgwci[AGX3_IMAGE_WRITE_LENGTH], imgwci_len = 0;
+   uint8_t imgwci[AGX3_TEX_MAX_LENGTH], imgwci_len = 0;
    const struct agx3_image_write_desc d = {
       .wait_mask = wait_mask,
       .staging_bank = bank,
@@ -1849,7 +2185,10 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
                        : AGX3_IMAGE_WRITE_2D,
    };
    bool encoded;
-   if (heap_state) {
+   if (m5) {
+      encoded = agx3_tex_encode_image_write_agx3c(
+         &d, I->src[3].value, slot, imgwci, &imgwci_len);
+   } else if (heap_state) {
       /* Consume this write's COPYs, as the resolved PBE form below does. */
       encoded = agx3_tex_encode_image_write_2d(
          enc, &d, I->src[3].value, slot, imgwci, &imgwci_len);
@@ -1872,8 +2211,12 @@ agx3_emit_storage_image_write(struct util_dynarray *emission,
    }
    if (!encoded || !agx3_append(&staged, imgwci, imgwci_len) ||
        !agx3_append(emission, staged.data, staged.size)) {
-      if (agx_compiler_debug & AGX_DBG_SHADERS)
+      if (agx_compiler_debug & AGX_DBG_SHADERS) {
          fprintf(stderr, "G17P imgwci instruction compose refusal\n");
+         fprintf(stderr, "imgwci state: m5=%u bank=%u slot=%u base=%u index=%u kind=%u kill=%u normalized=%u wait=%x dim=%u encoded=%u length=%u\n",
+                 m5, bank, slot, I->src[3].value, d.image_value, d.image_kind,
+                 d.image_kill, d.normalized, d.wait_mask, d.dim, encoded, imgwci_len);
+      }
       goto fail;
    }
    *write_slot = slot;
@@ -1914,7 +2257,8 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
 {
    const bool published = agx_texture_resource(I) != 0;
    const bool scalar_cube = I->agx3_cube_scalar_coords;
-   const unsigned source_count = scalar_cube ? 8 : 6;
+   const bool split = I->agx3_split_coords;
+   const unsigned source_count = scalar_cube || split ? 8 : 6;
    const bool explicit_lod = I->lod_mode == AGX_LOD_MODE_LOD_MIN;
    const bool bias_min = I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS_MIN;
    const bool gradient_min = I->lod_mode == AGX_LOD_MODE_LOD_GRAD_MIN;
@@ -1938,8 +2282,7 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
    const bool register_bias = I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS &&
                               I->src[1].type != AGX_INDEX_IMMEDIATE;
    const bool implicit_lod = I->lod_mode == AGX_LOD_MODE_AUTO_LOD || zero_bias;
-   /* Explicit gradients. The four scalars are staged into their own banks
-    * and the sample names coordinate bank 0, as every witness does. */
+   /* Gradient operands share the sample coordinate base. */
    const bool gradient_lod = I->lod_mode == AGX_LOD_MODE_LOD_GRAD || gradient_min;
    /* Gather uses the implicit-LOD selector for 2D, array and cube forms,
     * including compute where it reads level zero. Map the IR component
@@ -1991,11 +2334,20 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
     * share one operand test: a live single-channel 16-bit register. */
    const bool scalar_lod_reg = agx3_tex_plain_reg(I->src[1]) &&
       I->src[1].size == AGX_SIZE_16 && agx_channels(I->src[1]) == 1;
+   const bool packed_lod_reg = enc == AGX_ISA_ENCODING_AGX3C &&
+      I->memory_is_packed && agx3_tex_plain_reg(I->src[1]) &&
+      I->src[1].size == AGX_SIZE_32 && agx_channels(I->src[1]) == 1 &&
+      !(I->src[1].value & 1) && I->src[1].value < 128;
+   /* The C scalar publisher consumes its half-float source; a reusable
+    * scalar is first materialized as a packed word. Composite layer/LOD
+    * and face/LOD publishers also encode a live half-float operand. */
+   const bool native_scalar_lod_reg = packed_lod_reg || (scalar_lod_reg &&
+      (enc == AGX_ISA_ENCODING_AGX3B ||
+       (enc == AGX_ISA_ENCODING_AGX3C && I->src[1].value < 128 &&
+        ((array || cube) || (!I->src[1].cache && I->src[1].discard)))));
    const bool bias_min_operand = agx3_tex_plain_reg(I->src[1]) &&
       I->src[1].size == AGX_SIZE_32 && agx_channels(I->src[1]) == 1 &&
       (I->src[1].value & 1) == 0 && I->src[1].value < 127;
-   /* A 2D gradient is four 32-bit scalars in one aligned run, which the four
-    * staging copies name two halves apart. */
    /* A 2D gradient is four 32-bit scalars in one aligned run; a 3D one is
     * six, and agx_emit_tex()'s collect already lays both out in the
     * interleaved order the staging banks want. */
@@ -2006,16 +2358,15 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       (I->src[1].value & 1) == 0 &&
       I->src[1].value + (2 * (gradient_channels + gradient_min)) - 2 < 128;
    const bool valid_lod = bias_min ?
-      (enc == AGX_ISA_ENCODING_AGX3B && bias_min_operand) : explicit_lod ?
-      (enc == AGX_ISA_ENCODING_AGX3B &&
-       (valid_immediate_lod || scalar_lod_reg)) :
+      bias_min_operand : explicit_lod ?
+      (valid_immediate_lod || native_scalar_lod_reg) :
       biased_lod
-         ? (enc == AGX_ISA_ENCODING_AGX3B && valid_immediate_bias &&
+         ? (valid_immediate_bias &&
             immediate_level_x64 != 0.0f)
          : register_bias
-         ? (enc == AGX_ISA_ENCODING_AGX3B && scalar_lod_reg)
+         ? native_scalar_lod_reg
          : gradient_lod
-         ? (enc == AGX_ISA_ENCODING_AGX3B && gradient_operand)
+         ? gradient_operand
          : (implicit_lod &&
             (zero_bias || agx3_tex_imm(I->src[1], 0)));
    const bool texture_immediate =
@@ -2032,7 +2383,12 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
                   I->src[5].value == I->offset_xyz &&
                   !I->src[5].memory && !I->src[5].abs && !I->src[5].neg &&
                   !I->src[5].cache && !I->src[5].discard && !I->src[5].kill));
+   /* AGX3C stages an array gather's runtime offset like an array sample's:
+    * one bank-6 word with the layer (native texwit6 g_arr/g_arr_y). */
+   const bool m5_array_gather = enc == AGX_ISA_ENCODING_AGX3C && gather &&
+      array;
    const bool register_offset = offset && !I->offset_xyz && gather &&
+      !m5_array_gather &&
       agx3_tex_plain_reg(I->src[5]) && I->src[5].size == AGX_SIZE_32 &&
       agx_channels(I->src[5]) == (shadow ? 2 : 1) &&
       !(I->src[5].value & 1) && I->src[5].value + (shadow ? 2 : 0) <= 126;
@@ -2050,7 +2406,8 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       (I->src[5].value & 1) == 0 &&
       I->src[5].value + 2 * staged_offset_components <= 256;
    const unsigned packed_offset_components = 1 + (shadow ? 1 : 0);
-   const bool staged_packed_offset = offset && !gather && I->offset_xyz == 0 &&
+   const bool staged_packed_offset = offset &&
+      (!gather || m5_array_gather) && I->offset_xyz == 0 &&
       (array || explicit_lod || register_bias || min_lod) &&
       agx3_tex_plain_reg(I->src[5]) && I->src[5].size == AGX_SIZE_32 &&
       agx_channels(I->src[5]) == packed_offset_components &&
@@ -2058,19 +2415,17 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       I->src[5].value + 2 * packed_offset_components <= 128;
    const bool staged_offset =
       staged_component_offset || staged_packed_offset;
-   /* AGX3B coordinates are copied to the fixed staging bank before SMP. COPY
-    * and CEM each name a seven-bit source half-register, so the entire vector
-    * must end below half 128. AGX3C retains its independently recovered direct
-    * coordinate bound. */
-   const unsigned max_coord = enc == AGX_ISA_ENCODING_AGX3B
-                                 ? 128 - (2 * agx_channels(I->src[0]))
-                                 : 60;
+   /* Both encodings' COPY sources name seven-bit GPR half-registers.
+    * SMP names a separate staging bank, not those source GPRs. */
+   const unsigned max_coord = 128 - (2 * agx_channels(I->src[0]));
    const bool published_shape =
-      enc == AGX_ISA_ENCODING_AGX3B && I->dim == AGX_DIM_2D && !query_lod &&
+      ((enc == AGX_ISA_ENCODING_AGX3B && I->dim == AGX_DIM_2D) ||
+       (enc == AGX_ISA_ENCODING_AGX3C && three_d && !shadow && !offset &&
+        debug_get_option_m5_scalar_3d_coords())) && !query_lod &&
       !I->src[0].memory && !I->src[0].abs && !I->src[0].neg &&
       !I->src[0].cache && !I->src[0].discard &&
       I->src[0].size == AGX_SIZE_32 &&
-      agx_channels(I->src[0]) == (shadow ? 3 : 2) &&
+      agx_channels(I->src[0]) == ((three_d || shadow) ? 3 : 2) &&
       (implicit_lod || immediate_lod) &&
       (agx_is_null(I->src[5]) || (!shadow && immediate_offset)) &&
       (!offset || immediate_offset);
@@ -2087,7 +2442,30 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
             return agx3_tex_decline(I, __LINE__);
       }
    }
-   const bool coordinates_valid = published ? published_shape : scalar_cube ||
+   if (split) {
+      /* Split x and y stage through the wide scalar COPY. No other operand
+       * names their registers (agx3_split_texture_coordinates() keeps such a
+       * component in the vector): a level, bias or offset staged before them
+       * could otherwise discard one first. */
+      if (enc != AGX_ISA_ENCODING_AGX3C || I->nr_srcs != 8 ||
+          I->dim != AGX_DIM_2D || published || query_lod ||
+          native_coord_slot || !agx_is_null(I->src[7]))
+         return agx3_tex_decline(I, __LINE__);
+      for (unsigned c = 0; c < 2; ++c) {
+         const agx_index coord = agx3_tex_coord(I, c);
+         if (!agx3_split_coord_ok(coord, c))
+            return agx3_tex_decline(I, __LINE__);
+         for (unsigned s = 1; s < 6; ++s) {
+            const agx_index other = I->src[s];
+            if (agx3_tex_plain_reg(other) &&
+                other.value < coord.value + 2 &&
+                coord.value < other.value + agx_index_size_16(other))
+               return agx3_tex_decline(I, __LINE__);
+         }
+      }
+   }
+   const bool coordinates_valid = published ? published_shape :
+      scalar_cube || split ||
       (agx3_tex_plain_reg(I->src[0]) && I->src[0].size == AGX_SIZE_32 &&
        agx_channels(I->src[0]) == (query_lod ? (three_d || cube ? 3 : 2) :
                                   cube_array ? 4 :
@@ -2100,7 +2478,8 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
        /* Sparse residency widens only the destination reservation: the extra
         * register the instruction writes after its masked channels. */
        agx_channels(I->dest[0]) != (I->sparse ? 8u : 4u) ||
-       I->dest[0].value >= 128 ||
+       I->dest[0].value >= (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 128) ||
+       I->dest[0].value + agx_index_size_16(I->dest[0]) > 256 ||
        (I->dest[0].size == AGX_SIZE_32 && (I->dest[0].value & 1)) ||
        !coordinates_valid ||
        !valid_lod || !agx3_tex_imm(I->src[2], 0) ||
@@ -2131,39 +2510,45 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
                      : !agx_is_null(I->src[5]))) ||
        (I->dim != AGX_DIM_2D && I->dim != AGX_DIM_2D_ARRAY && !three_d &&
         !cube) ||
-       (array && enc != AGX_ISA_ENCODING_AGX3B) ||
-       /* f76 = 3 is orthogonal to f139; only the gradient is held back,
-        * because gradient3d stages six scalars and that layout has no
-        * witness. See agx3_tex_core.c for the five-row table. */
-       (three_d &&
-        (enc != AGX_ISA_ENCODING_AGX3B ||
-         (!valid_immediate_lod && !implicit_lod && !biased_lod &&
-          !register_bias && !explicit_lod && !gradient_lod && !bias_min))) ||
-       (cube &&
-        (enc != AGX_ISA_ENCODING_AGX3B ||
-         (!implicit_lod && !explicit_lod && !biased_lod &&
-          !register_bias && !bias_min))) ||
+       /* The dimension selector is independent of the supported LOD mode;
+        * gradient and minimum operands retain their generation gates above. */
+       (three_d && !valid_immediate_lod && !implicit_lod && !biased_lod &&
+        !register_bias && !explicit_lod && !gradient_lod && !bias_min) ||
+       (cube && !implicit_lod && !explicit_lod && !biased_lod &&
+        !register_bias && !bias_min) ||
        (I->mask == 0 || (I->mask & ~15u)) ||
-       /* Sparse samples exist on AGX3b for the implicit, immediate level,
-        * immediate bias, array, 3D, cube and gather shapes; not for a depth
-        * comparison or a LOD query. */
-       (I->sparse && (enc != AGX_ISA_ENCODING_AGX3B || shadow || query_lod)) ||
+       /* Both encodings support sparse samples, excluding depth comparison
+        * and LOD query. */
+       (I->sparse && (shadow || query_lod)) ||
        /* A minimum LOD and a runtime offset share bank 6 only through the
         * packed word. */
        (min_lod && ((cube && !bias_min) || vertex_stage ||
                     (three_d && offset && !bias_min && !immediate_offset) ||
                     staged_component_offset)) ||
        (query_lod &&
-        (enc != AGX_ISA_ENCODING_AGX3B || !implicit_lod || shadow || offset ||
+        (!implicit_lod || shadow || offset ||
          gather || I->dest[0].size != AGX_SIZE_32 || (I->mask & ~3u))) ||
        /* Immediate 3D offsets carry their third signed nibble in f43.
         * Dynamic 3D offsets have already lowered to coordinate addition;
         * cubes have no offset form. */
-       (offset && (enc != AGX_ISA_ENCODING_AGX3B || cube)) ||
+       (offset && (cube ||
+                    (enc != AGX_ISA_ENCODING_AGX3B &&
+                     !(enc == AGX_ISA_ENCODING_AGX3C &&
+                       (immediate_offset ||
+                        (!array && !three_d &&
+                         (staged_component_offset || register_offset)) ||
+                        /* The complete bank-6 word of a 2D or 2D-array
+                         * sample with its layer and/or register LOD. */
+                        (staged_packed_offset &&
+                         (!three_d || bias_min))))))) ||
        /* A depth comparison is orthogonal to the LOD shape on a plain 2D
         * sample. The array and cube comparisons use their own packed layer /
         * CEM operands, with implicit LOD or the witnessed explicit zero. */
-       (shadow && (three_d || enc != AGX_ISA_ENCODING_AGX3B ||
+       (shadow && (three_d ||
+                   (enc == AGX_ISA_ENCODING_AGX3C &&
+                    offset && !immediate_offset &&
+                    !staged_component_offset &&
+                    !register_offset && !staged_packed_offset) ||
                    I->mask != (gather ? 15u : 1u) ||
                    ((cube || (array && !offset && !vertex_stage)) &&
                     !implicit_lod && !shadow_level_zero &&
@@ -2179,8 +2564,7 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
                     !((explicit_lod && !immediate_lod) || register_bias ||
                       (cube && bias_min) ||
                       (array && !cube && gradient_lod))))) ||
-       (gather && (I->lod_mode != AGX_LOD_MODE_AUTO_LOD || three_d ||
-                   enc != AGX_ISA_ENCODING_AGX3B)))
+       (gather && (I->lod_mode != AGX_LOD_MODE_AUTO_LOD || three_d)))
       return agx3_tex_decline(I, __LINE__);
 
    /* The caller lowers additional required dependencies before staging. */
@@ -2219,15 +2603,33 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
     * test_linked_staging_witnesses. Additional operands retain the existing
     * mode-specific ABI; only the relocatable profile can choose a new base.
     */
+   const bool m5_fragment_3d = enc == AGX_ISA_ENCODING_AGX3C && three_d &&
+      !vertex_stage && !compute_stage && implicit_lod &&
+      debug_get_option_m5_fs_3d_banks();
+   const bool m5_banked_3d = enc == AGX_ISA_ENCODING_AGX3C && three_d &&
+      !shadow && !offset && !query_lod && !min_lod &&
+      (((vertex_stage || compute_stage) && (immediate_lod || gradient_lod)) ||
+       m5_fragment_3d);
    if (bank_base != 0 &&
-       (enc != AGX_ISA_ENCODING_AGX3B || native_staged_coords ||
-        (array && !(vertex_stage && explicit_lod && immediate_lod && !shadow && !offset)) ||
-        (three_d && !vertex_stage) || cube || gradient_lod || bias_min || gradient_min ||
-        query_lod || (explicit_lod && !immediate_lod) || register_bias ||
+       ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+        native_staged_coords ||
+        (array && !(enc == AGX_ISA_ENCODING_AGX3B && vertex_stage &&
+                    explicit_lod && immediate_lod && !shadow && !offset)) ||
+        (three_d && !m5_banked_3d &&
+         !(enc == AGX_ISA_ENCODING_AGX3B && vertex_stage)) ||
+        cube || (gradient_lod && !m5_banked_3d) || bias_min || gradient_min || query_lod ||
+        (enc == AGX_ISA_ENCODING_AGX3B &&
+         ((explicit_lod && !immediate_lod) || register_bias)) ||
+        (enc == AGX_ISA_ENCODING_AGX3C &&
+         (((explicit_lod && !immediate_lod) || register_bias) &&
+          (bank_base & 7))) ||
         register_offset || (bank_base & 3) ||
-        bank_base > AGX3_STAGING_HALFS - ((array || three_d) ? 8 : 4)))
+        bank_base > AGX3_STAGING_HALFS -
+           (m5_banked_3d && gradient_lod
+               ? (vertex_stage ? 40 : 32) : (array || three_d) ? 8 : 4)))
       return agx3_tex_decline(I, __LINE__);
    const uint16_t coord = (enc == AGX_ISA_ENCODING_AGX3B ||
+                           enc == AGX_ISA_ENCODING_AGX3C ||
                            native_staged_coords || explicit_lod ||
                            register_bias || array || three_d || cube ||
                            gradient_lod || shadow)
@@ -2241,12 +2643,19 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
    const uint16_t offset_base = I->src[5].value + (shadow ? 2 : 0);
    /* The comparison 2D-array bias and the staged 3D bias read their minimum
     * from a packed register. */
-   const bool register_minimum = bias_min &&
+   const bool register_minimum = enc == AGX_ISA_ENCODING_AGX3B && bias_min &&
       ((array && shadow) || (three_d && staged_packed_offset));
+
+   const bool bank8_minimum = enc == AGX_ISA_ENCODING_AGX3C && bias_min &&
+      ((array && shadow && !cube) || (three_d && staged_packed_offset));
    /* A runtime offset arrives as one NIR-packed bank-6 word that already
     * carries the layer and the bias, or the minimum: a gradient's, or a
     * non-array bias's. Only the 2D-array bias still stages its minimum. */
    const bool packed_array_word = array && staged_packed_offset;
+   /* C's COPY names exactly one lifetime hint per source, as its other
+    * staging publishers do: cache unless this is the last use. */
+   const bool word_cache = enc == AGX_ISA_ENCODING_AGX3C
+      ? !I->src[5].discard : I->src[5].cache;
    if (bias_min && array && !cube) {
       /* The minimum moves out of bank 6 so the bias can join the layer
        * there: to half 4 as Apple publishes it, or for a comparison, whose
@@ -2265,7 +2674,7 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       const bool bank6 = packed_array_word
          ? agx3_tex_stage_copy_hints(
               enc, 6, offset_base, stage_slot, shadow ? wait_mask : 0,
-              I->src[5].cache, I->src[5].discard, bytes, &len)
+              word_cache, I->src[5].discard, bytes, &len)
          : agx3_tex_stage_array_lod(
               enc, I->src[0].value + 4, I->src[1].value, stage_slot,
               shadow ? wait_mask : 0, I->src[0].cache, I->src[0].discard,
@@ -2274,9 +2683,13 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       if (!bank6 || !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       if (shadow &&
-          (!agx3_tex_pack_minimum(enc, I->src[1].value + 1, array_scratch,
-                                  I->src[1].cache, I->src[1].discard, bytes,
-                                  &len) ||
+          (!(bank8_minimum
+                ? agx3_tex_stage_minimum_bank8_m5(
+                     I->src[1].value + 1, stage_slot, 0, !I->src[1].discard,
+                     I->src[1].discard, bytes, &len)
+                : agx3_tex_pack_minimum(enc, I->src[1].value + 1,
+                                        array_scratch, I->src[1].cache,
+                                        I->src[1].discard, bytes, &len)) ||
            !agx3_append(emission, bytes, len)))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
@@ -2284,12 +2697,17 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       if (!(packed_array_word
                ? agx3_tex_stage_copy_hints(
                     enc, 6, offset_base, stage_slot, wait_mask,
-                    I->src[5].cache, I->src[5].discard, bytes, &len)
+                    word_cache, I->src[5].discard, bytes, &len)
                : agx3_tex_stage_array_minimum_gradient(
                     enc, I->src[0].value + 4,
                     I->src[1].value + 2 * gradient_channels, stage_slot,
-                    wait_mask, I->src[0].cache, I->src[0].discard,
-                    I->src[1].cache, I->src[1].discard, bytes, &len)) ||
+                    wait_mask,
+                    enc == AGX_ISA_ENCODING_AGX3C ? !I->src[0].discard
+                                                  : I->src[0].cache,
+                    I->src[0].discard,
+                    enc == AGX_ISA_ENCODING_AGX3C ? !I->src[1].discard
+                                                  : I->src[1].cache,
+                    I->src[1].discard, bytes, &len)) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
@@ -2298,12 +2716,16 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
        * so the minimum goes to the packed register SMP names. The register
        * pack carries no wait, so it follows the staging that does. */
       if (!agx3_tex_stage_copy_hints(
-             enc, 6, offset_base, stage_slot, wait_mask, I->src[5].cache,
+             enc, 6, offset_base, stage_slot, wait_mask, word_cache,
              I->src[5].discard, bytes, &len) ||
           !agx3_append(emission, bytes, len) ||
-          !agx3_tex_pack_minimum(enc, I->src[1].value + 1, array_scratch,
-                                 I->src[1].cache, I->src[1].discard, bytes,
-                                 &len) ||
+          !(bank8_minimum
+               ? agx3_tex_stage_minimum_bank8_m5(
+                    I->src[1].value + 1, stage_slot, 0, !I->src[1].discard,
+                    I->src[1].discard, bytes, &len)
+               : agx3_tex_pack_minimum(enc, I->src[1].value + 1,
+                                       array_scratch, I->src[1].cache,
+                                       I->src[1].discard, bytes, &len)) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
@@ -2317,31 +2739,44 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
    } else if (gradient_min && !staged_packed_offset) {
       if (!agx3_tex_stage_typed_lod(
              enc, I->src[1].value + 2 * gradient_channels, stage_slot,
-             wait_mask, I->src[1].cache, I->src[1].discard,
-             AGX3_TEX_LEVEL_MINIMUM, bytes, &len) ||
+             wait_mask,
+             enc == AGX_ISA_ENCODING_AGX3C ? !I->src[1].discard
+                                           : I->src[1].cache,
+             I->src[1].discard, AGX3_TEX_LEVEL_MINIMUM, bytes, &len) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
    } else if (staged_packed_offset) {
       if (!agx3_tex_stage_copy_hints(
              enc, 6, offset_base, stage_slot, wait_mask,
-             I->src[5].cache, I->src[5].discard, bytes, &len) ||
+             word_cache, I->src[5].discard, bytes, &len) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
    } else if (combined_array_lod) {
       if (!agx3_tex_stage_array_lod(
              enc, I->src[0].value + 4, I->src[1].value, stage_slot, wait_mask,
-             I->src[0].cache, I->src[0].discard,
-             I->src[1].cache, I->src[1].discard, bytes, &len) ||
+             enc == AGX_ISA_ENCODING_AGX3C ? !I->src[0].discard
+                                           : I->src[0].cache,
+             I->src[0].discard,
+             enc == AGX_ISA_ENCODING_AGX3C ? !I->src[1].discard
+                                           : I->src[1].cache,
+             I->src[1].discard, bytes, &len) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
    } else if (((explicit_lod && !immediate_lod) || register_bias) &&
               !cube_register_lod) {
-      if (!agx3_tex_stage_lod(enc, I->src[1].value, stage_slot, wait_mask,
-                             I->src[1].cache, I->src[1].discard,
-                             bytes, &len) ||
+      const bool staged = packed_lod_reg
+         ? agx3_tex_stage_copy_hints(enc, bank_base + 6, I->src[1].value,
+                                     stage_slot, wait_mask,
+                                     !I->src[1].discard, I->src[1].discard,
+                                     bytes, &len)
+         : agx3_tex_stage_typed_lod_at(enc, bank_base, I->src[1].value,
+                                      stage_slot, wait_mask,
+                                      I->src[1].cache, I->src[1].discard,
+                                      AGX3_TEX_LEVEL_FLOAT, bytes, &len);
+      if (!staged ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       wait_mask = 0;
@@ -2352,9 +2787,11 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
     * be clobbered by the cube-coordinate calculation. Apple's cube and
     * cube-array comparison shaders likewise publish the reference before the
     * final CEM axes. */
-   if (shadow && cube &&
+   if (shadow && cube && enc == AGX_ISA_ENCODING_AGX3B &&
        (!agx3_tex_stage_copy_hints(
-           enc, 4, I->src[5].value, stage_slot, 0, I->src[5].cache,
+           enc, 4, I->src[5].value, stage_slot, 0,
+           enc == AGX_ISA_ENCODING_AGX3C ? !I->src[5].discard
+                                         : I->src[5].cache,
            I->src[5].discard, bytes, &len) ||
         !agx3_append(emission, bytes, len)))
       return agx3_tex_decline(I, __LINE__);
@@ -2375,11 +2812,18 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
             axis == 2 && (scalar_cube ? I->src[6].discard : I->src[0].discard),
             axis == 2 && (scalar_cube ? I->src[7].discard : I->src[0].discard),
          };
-         if (!agx3_tex_cube_cem_calc_hints_to(
-                enc, I->src[0].value,
-                scalar_cube ? I->src[6].value : I->src[0].value + 2,
-                scalar_cube ? I->src[7].value : I->src[0].value + 4,
-                scratch[axis], axis, discard, bytes, &len) ||
+         const uint16_t cy = scalar_cube ? I->src[6].value : I->src[0].value + 2;
+         const uint16_t cz = scalar_cube ? I->src[7].value : I->src[0].value + 4;
+         /* Scalar cube coordinates are AGX3B-only and exclude shadow, so
+          * the AGX3C reference form always sees uniform discard hints. */
+         const bool packed = enc == AGX_ISA_ENCODING_AGX3C && shadow
+            ? agx3_tex_cube_cem_calc_reference_to(
+                 enc, I->src[0].value, cy, cz, scratch[axis], axis,
+                 discard[0], I->src[5].value, bytes, &len)
+            : agx3_tex_cube_cem_calc_hints_to(
+                 enc, I->src[0].value, cy, cz, scratch[axis], axis, discard,
+                 bytes, &len);
+         if (!packed ||
              !agx3_append(emission, bytes, len))
             return agx3_tex_decline(I, __LINE__);
 
@@ -2396,31 +2840,47 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
              !agx3_append(emission, bytes, len))
             return agx3_tex_decline(I, __LINE__);
       }
+   } else if (split) {
+      /* Split coordinates may live anywhere in the register file: stage
+       * them with the wide scalar COPY, as split texel fetches do. Nothing
+       * after them reads their registers, so the later read of a shared
+       * register discards it. */
+      for (unsigned c = 0; c < 2; ++c) {
+         const bool last = agx3_tex_coord_discard(I, c, 2);
+         if (!agx3_tex_stage_copy_scalar_m5(coord + 2 * c,
+                                            agx3_tex_coord(I, c).value,
+                                            stage_slot, c ? 0 : wait_mask,
+                                            !last, last, bytes, &len) ||
+             !agx3_append(emission, bytes, len))
+            return agx3_tex_decline(I, __LINE__);
+      }
    } else if (!native_staged_coords && !published) {
       if (!agx3_tex_stage_copy_hints(enc, coord, I->src[0].value, stage_slot,
                                     wait_mask,
-                                    enc == AGX_ISA_ENCODING_AGX3B &&
-                                       I->src[0].cache,
-                                    enc != AGX_ISA_ENCODING_AGX3B ||
-                                       I->src[0].discard,
+                                    enc == AGX_ISA_ENCODING_AGX3C
+                                       ? !I->src[0].discard
+                                       : I->src[0].cache,
+                                    I->src[0].discard,
                                     bytes, &len) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
       if (!agx3_tex_stage_copy_hints(enc, coord + 2, I->src[0].value + 2,
                                     stage_slot, 0,
-                                    enc == AGX_ISA_ENCODING_AGX3B &&
-                                       I->src[0].cache,
-                                    enc != AGX_ISA_ENCODING_AGX3B ||
-                                       I->src[0].discard,
+                                    enc == AGX_ISA_ENCODING_AGX3C
+                                       ? !I->src[0].discard
+                                       : I->src[0].cache,
+                                    I->src[0].discard,
                                     bytes, &len) ||
           !agx3_append(emission, bytes, len))
          return agx3_tex_decline(I, __LINE__);
    }
 
-   if (three_d &&
+   if (three_d && !published &&
        (!agx3_tex_stage_copy_hints(
            enc, coord + 4, I->src[0].value + 4, stage_slot, 0,
-           I->src[0].cache, I->src[0].discard, bytes, &len) ||
+           enc == AGX_ISA_ENCODING_AGX3C
+              ? !I->src[0].discard : I->src[0].cache,
+           I->src[0].discard, bytes, &len) ||
         !agx3_append(emission, bytes, len)))
       return agx3_tex_decline(I, __LINE__);
 
@@ -2433,7 +2893,9 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
        !register_offset &&
        !staged_offset &&
        (!agx3_tex_stage_array_layer_at(
-           enc, bank_base, I->src[0].value + 4, stage_slot, 0, I->src[0].cache,
+           enc, bank_base, I->src[0].value + 4, stage_slot, 0,
+           enc == AGX_ISA_ENCODING_AGX3C ? !I->src[0].discard
+                                         : I->src[0].cache,
            I->src[0].discard, bytes, &len) ||
         !agx3_append(emission, bytes, len)))
       return agx3_tex_decline(I, __LINE__);
@@ -2457,7 +2919,9 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
                     agx3_append(emission, bytes, len))) &&
                   agx3_tex_stage_cube_array_lod(
                      enc, face, I->src[1].value, array_scratch, stage_slot, 0,
-                     I->src[1].cache, I->src[1].discard, bytes, &len);
+                     enc == AGX_ISA_ENCODING_AGX3C ? !I->src[1].discard
+                                                   : I->src[1].cache,
+                     I->src[1].discard, bytes, &len);
       } else if (bias_min) {
          packed = agx3_tex_pack_cube_face(enc, face, array_scratch, bytes,
                                           &len) &&
@@ -2466,13 +2930,17 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
                      enc, I->src[1].value, I->src[1].value + 1,
                      array_scratch, stage_slot, 0, I->src[1].cache,
                      I->src[1].discard, bytes, &len);
-      } else if (cube_array) {
+      } else if (cube_array && !(enc == AGX_ISA_ENCODING_AGX3C && query_lod)) {
          packed = agx3_tex_stage_cube_array_layer(
-            enc, face, layer, stage_slot, 0, I->src[0].cache,
+            enc, face, layer, stage_slot, 0,
+            enc == AGX_ISA_ENCODING_AGX3C ? !I->src[0].discard
+                                          : I->src[0].cache,
             I->src[0].discard, bytes, &len);
       } else if (cube_register_lod) {
          packed = agx3_tex_stage_cube_lod(
-            enc, face, I->src[1].value, stage_slot, 0, I->src[1].cache,
+            enc, face, I->src[1].value, stage_slot, 0,
+            enc == AGX_ISA_ENCODING_AGX3C ? !I->src[1].discard
+                                          : I->src[1].cache,
             I->src[1].discard, bytes, &len);
       } else {
          packed = agx3_tex_stage_cube_z(enc, face, stage_slot, 0, false, true,
@@ -2509,9 +2977,15 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
    /* The comparison reference belongs to this sample's own staging group,
     * at bank + 4 (see the sampler-private note above), not at a fixed half
     * 4 that later samples would overwrite. */
-   if (shadow && !cube && !published &&
+   if (shadow && !published &&
+       (!cube || enc == AGX_ISA_ENCODING_AGX3C) &&
        (!agx3_tex_stage_copy_hints(
-           enc, coord + 4, I->src[5].value, stage_slot, 0, I->src[5].cache,
+           enc, coord + 4, I->src[5].value, stage_slot, 0,
+           /* C names exactly one hint; with a staged offset after it the
+            * reference copy keeps the register (cache). */
+           enc == AGX_ISA_ENCODING_AGX3C
+              ? (staged_offset || !I->src[5].discard)
+              : I->src[5].cache,
            staged_offset ? false : I->src[5].discard, bytes, &len) ||
         !agx3_append(emission, bytes, len)))
       return agx3_tex_decline(I, __LINE__);
@@ -2536,10 +3010,12 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       };
 
       for (unsigned i = 0; i < gradient_channels; ++i) {
-         if (!agx3_tex_stage_copy_hints(enc, bank[i],
+         if (!agx3_tex_stage_copy_hints(enc, bank_base + bank[i],
                                         I->src[1].value + (2 * i), stage_slot,
-                                        0, I->src[1].cache, I->src[1].discard,
-                                        bytes, &len) ||
+                                        0, enc == AGX_ISA_ENCODING_AGX3C
+                                              ? !I->src[1].discard
+                                              : I->src[1].cache,
+                                        I->src[1].discard, bytes, &len) ||
              !agx3_append(emission, bytes, len))
             return agx3_tex_decline(I, __LINE__);
       }
@@ -2572,7 +3048,8 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
       .texture_value = texture_immediate ? 0 : I->src[3].value,
       .sampler_value = sampler_immediate ? 0 : I->src[4].value,
       .texture_kill = !texture_immediate && I->src[3].kill,
-      .sampler_kill = !sampler_immediate && I->src[4].kill,
+      .sampler_kill = enc != AGX_ISA_ENCODING_AGX3C &&
+                        !sampler_immediate && I->src[4].kill,
       .gather = gather_component,
       .shadow = shadow,
       .query_lod = query_lod,
@@ -2609,6 +3086,8 @@ agx3_emit_texture_sample(struct util_dynarray *emission,
                       : cube ? true
                          : native_staged_coords
                               ? true
+                              : enc == AGX_ISA_ENCODING_AGX3C
+                                   ? true
                               : I->src[0].kill,
       .final = final,
       .sample_group_nonterminal = sample_group_nonterminal,
@@ -2718,7 +3197,7 @@ agx3_tile_sample_operand(agx_index sample, struct agx3_tile_sample *out)
       return true;
    }
 
-   if (!agx3_src_is_register(sample) || sample.value >= 192)
+   if (!agx3_src_is_register(sample) || sample.value >= 256)
       return false;
 
    /* Register allocation rewrites `kill` into the post-RA `discard` hint on
@@ -2923,7 +3402,8 @@ agx3_emit_fltnorm(struct util_dynarray *emission, enum agx_isa_encoding enc,
 }
 
 static bool
-agx3_emit_rgb10a2(struct util_dynarray *emission, const agx_instr *I,
+agx3_emit_rgb10a2(struct util_dynarray *emission, enum agx_isa_encoding enc,
+                  const agx_instr *I,
                  uint16_t wait, bool first)
 {
    const bool unpack = I->op == AGX_OPCODE_UNPACK_RGB10A2_AGX3;
@@ -2937,7 +3417,7 @@ agx3_emit_rgb10a2(struct util_dynarray *emission, const agx_instr *I,
    if (!agx3_src_is_register(dest) || dest.memory || dest.size != AGX_SIZE_32 ||
        agx_channels(dest) != (unpack ? lanes : 1))
       return false;
-   uint8_t bytes[14], length = 0;
+   uint8_t bytes[16], length = 0;
    if (unpack) {
       const agx_index src = I->src[0];
       if (!agx3_src_is_register(src) || src.memory || src.abs || src.neg ||
@@ -2946,8 +3426,9 @@ agx3_emit_rgb10a2(struct util_dynarray *emission, const agx_instr *I,
       const struct agx3_rgb10a2_unpack_desc d = {
          .dest = dest.value, .source = src.value,
          .wait_mask = wait, .discard = src.discard, .layout = layout,
+         .dest_hint = enc == AGX_ISA_ENCODING_AGX3C,
       };
-      if (!agx3_rgb10a2_unpack_encode(&d, bytes, &length))
+      if (!agx3_rgb10a2_unpack_encode(enc, &d, bytes, &length))
          return false;
    } else {
       struct agx3_rgb10a2_desc d = {.dest = dest.value, .layout = layout,
@@ -2970,7 +3451,7 @@ agx3_emit_rgb10a2(struct util_dynarray *emission, const agx_instr *I,
          d.abs[i] = src.abs;
          d.neg[i] = src.neg;
       }
-      if (!agx3_rgb10a2_encode(&d, bytes, &length))
+      if (!agx3_rgb10a2_encode_for_encoding(enc, &d, bytes, &length))
          return false;
    }
    return agx3_append(emission, bytes, length);
@@ -3187,6 +3668,14 @@ agx3_translate_scs(enum agx_isa_encoding enc, const agx_instr *I, bool is_store,
       d->b_status_load = enc == AGX_ISA_ENCODING_AGX3B &&
                          !is_store && I->scs_c_status_load;
       d->c_load_marker = !is_store && I->scs_c_load_marker;
+      if (I->scs_rt_completion) {
+         if (is_store || width != AGX3_MEM_W8 ||
+             !(register_indexed ? I->scs_address_selector == 3 &&
+                                  I->scs_c_load_marker
+                                : register_base && I->scs_address_selector == 4))
+            return false;
+         d->wait_mask |= 0x80;
+      }
       return true;
    }
 
@@ -3226,8 +3715,8 @@ agx3_translate_scs(enum agx_isa_encoding enc, const agx_instr *I, bool is_store,
 /* Translate a device_load/device_store into the normalized memory descriptor.
  * Returns false for any operand shape outside the grounded domain. */
 static bool
-agx3_translate_mem(const agx_instr *I, bool is_store, uint8_t slot,
-                   uint16_t wait_mask, struct agx3_mem_desc *d)
+agx3_translate_mem(enum agx_isa_encoding enc, const agx_instr *I, bool is_store,
+                   uint8_t slot, uint16_t wait_mask, struct agx3_mem_desc *d)
 {
    memset(d, 0, sizeof(*d));
 
@@ -3239,8 +3728,8 @@ agx3_translate_mem(const agx_instr *I, bool is_store, uint8_t slot,
    d->shift = (uint8_t)I->shift;
 
    const agx_index data = is_store ? I->src[0] : I->dest[0];
-   const bool packed_byte_store =
-      is_store && w == AGX3_MEM_W8 && I->memory_is_packed;
+   const bool packed_bytes = w == AGX3_MEM_W8 && I->memory_is_packed;
+   const bool packed_byte_store = is_store && packed_bytes;
    /* Native scalar byte loads write an aligned low half. Stores can also
     * address individual halves: Apple's byte_halves_odd_zero stores
     * ushort2.y from half3 with W8/mask1. A packed vector byte store likewise
@@ -3250,7 +3739,7 @@ agx3_translate_mem(const agx_instr *I, bool is_store, uint8_t slot,
       (w == AGX3_MEM_W16 || w == AGX3_MEM_W8) &&
       I->mask == 1 && agx_channels(data) == 1;
    const bool byte_load_half = !is_store && w == AGX3_MEM_W8 && scalar_half;
-   const unsigned data_halfs = packed_byte_store
+   const unsigned data_halfs = packed_bytes
       ? DIV_ROUND_UP(util_bitcount(I->mask), 2)
       : agx_size_align_16(data.size) * util_bitcount(I->mask);
    if (!agx3_src_is_register(data) || data.memory ||
@@ -3309,7 +3798,9 @@ agx3_translate_mem(const agx_instr *I, bool is_store, uint8_t slot,
       }
    } else if (agx3_src_is_register(index) && !index.memory) {
       d->idx_kind = AGX3_MEM_IDX_REG;
-      if ((index.value & 1) != 0 || index.value > 191)
+      if ((index.value & 1) != 0 ||
+          index.value >= (enc == AGX_ISA_ENCODING_AGX3C
+                             ? AGX3C_NUM_REGS : AGX3B_NUM_REGS))
          return false;
       d->idx_reg = index.value;
       /* Keep the IR's own liveness: forcing this bit to 1 (so the store's
@@ -3363,8 +3854,7 @@ agx3_translate_local(enum agx_isa_encoding enc, const agx_instr *I,
    const agx_index data = is_store ? I->src[0] : I->dest[0];
    const enum agx_size expected =
       d->width == AGX3_MEM_W32 ? AGX_SIZE_32 : AGX_SIZE_16;
-   const bool packed_bytes = enc == AGX_ISA_ENCODING_AGX3B &&
-      d->width == AGX3_MEM_W8 && I->memory_is_packed;
+   const bool packed_bytes = d->width == AGX3_MEM_W8 && I->memory_is_packed;
    const unsigned halves = packed_bytes ? DIV_ROUND_UP(util_bitcount(I->mask), 2)
       : agx_size_align_16(expected) * util_bitcount(I->mask);
    if (!agx3_src_is_register(data) || data.memory || data.size != expected ||
@@ -3377,8 +3867,15 @@ agx3_translate_local(enum agx_isa_encoding enc, const agx_instr *I,
    const agx_index base = I->src[is_store ? 1 : 0];
    if (base.size != AGX_SIZE_16 || base.memory || base.abs || base.neg)
       return false;
-   if (base.type == AGX_INDEX_IMMEDIATE && base.value == 0) {
+   if (base.type == AGX_INDEX_IMMEDIATE && base.value <= 255) {
+      /* An immediate base is a constant byte offset folded off the address
+       * by agx_nir_lower_shared_element_index(): it travels in the
+       * instruction's immediate byte-offset field (f71, witnessed in Apple's
+       * CmAntiFirefly ldtg at 16/32/48/64 next to a register base) and the
+       * workgroup-relative base stays zero.
+       */
       d->base_kind = AGX3_LOCAL_BASE_IMMEDIATE;
+      d->byte_offset = base.value;
    } else if (base.type == AGX_INDEX_UNIFORM && base.value < 512) {
       d->base_kind = AGX3_LOCAL_BASE_UNIFORM;
       d->base = base.value;
@@ -3410,14 +3907,9 @@ agx3_translate_local(enum agx_isa_encoding enc, const agx_instr *I,
    }
 
 
-   /*
-    * f11, `narrow_extend`, exists on AGX3c only: the AGX3b `ldtg`/`sttg`
-    * forms cannot encode f11 = 1 at any width, so a G17P threadgroup access
-    * carries it clear -- as on AGX2, with an 8-bit value held in a 16-bit
-    * register.
-    */
-   d->narrow_extend =
-      d->width == AGX3_MEM_W8 && enc == AGX_ISA_ENCODING_AGX3C;
+   /* Native M5 byte vectors occupy packed byte lanes, just as on G17P.
+    * Metal535 uchar3 LDTG/STTG both leave f11 clear. */
+   d->narrow_extend = false;
    d->slot = slot;
    d->wait_mask = wait_mask;
    return true;
@@ -3796,13 +4288,46 @@ static bool
 agx3_dynamic_stack_load(const agx_instr *I)
 {
    return I->op == AGX_OPCODE_STACK_LOAD && I->nr_srcs == 1 &&
-          I->src[0].type == AGX_INDEX_REGISTER;
+          (I->src[0].type == AGX_INDEX_REGISTER ||
+           (I->format == AGX_FORMAT_I8 && I->memory_is_packed));
+}
+
+static bool
+agx3_translate_byte_stack(const agx_instr *I, bool store, uint8_t slot,
+                           uint16_t wait, struct agx3_tps_desc *d)
+{
+   const agx_index data = store ? I->src[0] : I->dest[0];
+   const agx_index index = I->src[store ? 1 : 0];
+   const bool immediate = index.type == AGX_INDEX_IMMEDIATE;
+   const unsigned halves = DIV_ROUND_UP(util_bitcount(I->mask), 2);
+   if (I->format != AGX_FORMAT_I8 || !I->memory_is_packed ||
+       !I->mask || (I->mask & ~15) || !agx3_tex_plain_reg(data) ||
+       data.size != AGX_SIZE_16 || agx_channels(data) != halves ||
+       data.value + halves > 256 || (store && data.cache && data.discard) ||
+       (immediate ? index.value > UINT16_MAX || index.neg || index.memory ||
+                      index.cache || index.discard || index.kill
+                  : !agx3_tex_plain_reg(index) || index.size != AGX_SIZE_32 ||
+                      agx_channels(index) != 1 || (index.value & 1) ||
+                      index.value > 254 || (index.cache && index.discard)))
+      return false;
+   *d = (struct agx3_tps_desc){
+      .data = data.value, .width = AGX3_MEM_W8, .mask = I->mask,
+      .index = immediate ? 0 : index.value, .index_immediate = immediate,
+      .index_kill = !immediate && index.discard,
+      .byte_offset = immediate ? index.value : 0,
+      .frame_region = 6, .value_kill = store && data.discard,
+      .retain_tps = !store, .slot = store ? 0 : slot, .wait_mask = wait,
+   };
+   return true;
 }
 
 static bool
 agx3_translate_dynamic_stack_load(const agx_instr *I, uint8_t slot,
                                   struct agx3_tps_desc *d)
 {
+   if (I->format == AGX_FORMAT_I8)
+      return agx3_translate_byte_stack(I, false, slot, 0, d);
+
    if (!agx3_dynamic_stack_load(I) || I->nr_dests != 1 ||
        I->format != AGX_FORMAT_I32 || !I->mask || (I->mask & ~15))
       return false;
@@ -3828,18 +4353,22 @@ static bool
 agx3_dynamic_stack_store(const agx_instr *I)
 {
    return I->op == AGX_OPCODE_STACK_STORE && I->nr_srcs == 2 &&
-          I->src[1].type == AGX_INDEX_REGISTER;
+          (I->src[1].type == AGX_INDEX_REGISTER ||
+           (I->format == AGX_FORMAT_I8 && I->memory_is_packed));
 }
 
-/* G16G private arrays: a dynamically indexed store is STTPS frame 1 with the
- * data mask and a register index, like the dynamic load; a float4 store uses
- * W32 mask 15 with a byte index. STTPS signals no scoreboard slot. Apple
- * issues static RLD reads and dynamic LDTPS reads of the same array right
- * after it without any wait, so the TPS pipeline orders them itself. */
+/* G16G private arrays use STTPS frame 1 with a byte index. On M5, STTPS
+ * completion is observed through slot 1: Apple's fixed-index array reads use
+ * RLD with wait 1 after STTPS, and a byte-exact hardware replay requires that
+ * wait to see the new value. Track this producer even though STTPS has no
+ * explicit slot operand. */
 static bool
 agx3_translate_dynamic_stack_store(const agx_instr *I, uint16_t wait_mask,
                                    struct agx3_tps_desc *d)
 {
+   if (I->format == AGX_FORMAT_I8)
+      return agx3_translate_byte_stack(I, true, 0, wait_mask, d);
+
    if (!agx3_dynamic_stack_store(I) || I->nr_dests != 0 ||
        I->format != AGX_FORMAT_I32 || !I->mask || (I->mask & ~15))
       return false;
@@ -3863,7 +4392,8 @@ agx3_translate_dynamic_stack_store(const agx_instr *I, uint16_t wait_mask,
 
 static bool
 agx3_append_static_spill_tokens(const agx_instr *I,
-                                struct agx3_slot_instr *slot_instr)
+                                struct agx3_slot_instr *slot_instr,
+                                enum agx_isa_encoding enc)
 {
    if (agx3_dynamic_stack_load(I)) {
       struct agx3_tps_desc desc;
@@ -3874,14 +4404,19 @@ agx3_append_static_spill_tokens(const agx_instr *I,
       return true;
    }
    if (agx3_dynamic_stack_store(I)) {
-      /* No slot is signalled; the alias token only records the write so the
-       * scoreboard treats the store like any other consumer of pending
-       * static TPS traffic. */
+      /* M5 STTPS produces an implicit slot-1 completion. Preserve the
+       * alias read as well, so earlier dynamic reads retire before a write. */
       struct agx3_tps_desc desc;
       if (!agx3_translate_dynamic_stack_store(I, 0, &desc) ||
           slot_instr->nr_srcs >= AGX3_SLOT_MAX_REGS)
          return false;
       slot_instr->src[slot_instr->nr_srcs++] = AGX3_SLOT_DYNAMIC_TPS_TOKEN;
+      if (enc == AGX_ISA_ENCODING_AGX3C) {
+         if (slot_instr->nr_dsts >= AGX3_SLOT_MAX_REGS)
+            return false;
+         slot_instr->dst[slot_instr->nr_dsts++] = AGX3_SLOT_DYNAMIC_TPS_TOKEN;
+         slot_instr->preferred_slot = 1;
+      }
       return true;
    }
    if (slot_instr->nr_srcs >= AGX3_SLOT_MAX_REGS)
@@ -4090,7 +4625,7 @@ agx3_subgroup_plain_half(agx_index index)
    return index.type == AGX_INDEX_REGISTER && !index.memory &&
           index.size == AGX_SIZE_16 && agx_channels(index) == 1 &&
           !index.abs && !index.neg && !index.kill &&
-          !(index.cache && index.discard) && index.value < 192;
+          !(index.cache && index.discard) && index.value < 256;
 }
 
 /* A plain immediate operand: an unmodified single-channel constant. */
@@ -4115,7 +4650,7 @@ agx3_emit_simd_broadcast(struct util_dynarray *emission,
    /* G17P puts the lane straight into f29 for every lane the field holds, so
     * route AGX3b through the bounded immediate encoder and let it carry the
     * producer's wait. See agx3_subgroup_core.c for Apple's per-lane sweep. */
-   if (enc == AGX_ISA_ENCODING_AGX3B) {
+   if (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) {
       const struct agx3_subgroup_shuffle_desc d = {
          .op = AGX3_SUBGROUP_SHUFFLE_BROADCAST,
          .lane_kind = AGX3_SUBGROUP_LANE_IMMEDIATE,
@@ -4205,8 +4740,10 @@ agx3_emit_subgroup_shuffle(struct util_dynarray *emission,
       lane.type == AGX_INDEX_REGISTER && !lane.memory &&
       (lane.size == AGX_SIZE_16 || lane.size == AGX_SIZE_32) &&
       agx_channels(lane) == 1 && !lane.abs && !lane.neg && !lane.kill &&
-      !(lane.cache && lane.discard) && lane.value < 192;
-   if (enc == AGX_ISA_ENCODING_AGX3B && !quad && I->nr_dests == 1 &&
+      !(lane.cache && lane.discard) &&
+      lane.value < (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 192);
+   if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+       !quad && I->nr_dests == 1 &&
        I->nr_srcs == 2 && !I->saturate &&
        (register_lane ||
         (I->op == AGX_OPCODE_SHUFFLE && I->dest[0].size == AGX_SIZE_16))) {
@@ -4251,7 +4788,7 @@ agx3_emit_subgroup_shuffle(struct util_dynarray *emission,
     * shapes down to the register-lane path, which cannot take a wait and so
     * had to spend a whole synchronous prelude retiring one. */
    if (I->nr_srcs == 2 && agx3_subgroup_imm_lane(I->src[1]) &&
-       enc == AGX_ISA_ENCODING_AGX3B) {
+       (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C)) {
       /* 16-bit data narrows this encoding point's width pair, a field
        * separate from the quad selector, so a half broadcast or butterfly
        * encodes for quad as for SIMD (agx3_subgroup_encode_shuffle()). The
@@ -4418,6 +4955,69 @@ agx3_emit_subgroup_ballot(struct util_dynarray *emission,
                          I->op == AGX_OPCODE_FCMP_QUAD_BALLOT;
    const bool quad = I->op == AGX_OPCODE_ICMP_QUAD_BALLOT ||
                      I->op == AGX_OPCODE_FCMP_QUAD_BALLOT;
+   if (enc == AGX_ISA_ENCODING_AGX3C && !is_float && !quad &&
+       I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate && wait_mask <= 255 &&
+       agx3_subgroup_plain(I->dest[0],AGX_INDEX_REGISTER) &&
+       agx3_subgroup_imm_lane(I->src[0]) && I->src[0].value == 1 &&
+       agx3_src_is_zero_imm(I->src[1]) && I->icond == AGX_ICOND_UEQ &&
+       I->invert_cond) {
+      uint8_t bytes[AGX3_SUBGROUP_MAX_LENGTH], length;
+      return agx3_subgroup_encode_active_mask(I->dest[0].value,
+                I->dest[0].cache,wait_mask,bytes,&length) &&
+             agx3_append(emission,bytes,length);
+   }
+   /* A 16-bit quad ballot of a boolean immediate (a quad vote whose operand
+    * NIR proved constant, e.g. after terminateInvocation). Native G17G keeps
+    * it as a ballot: `v == 0` is the inverted ballot of `!v`. */
+   if (enc == AGX_ISA_ENCODING_AGX3C && quad && !is_float &&
+       I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+       wait_mask <= 255 && agx3_subgroup_plain_half(I->dest[0]) &&
+       agx3_subgroup_imm_lane(I->src[0]) && I->src[0].value <= 1 &&
+       agx3_src_is_zero_imm(I->src[1]) &&
+       (I->icond == AGX_ICOND_UEQ || I->icond == AGX_ICOND_SEQ)) {
+      const uint16_t nonzero = I->invert_cond ? I->src[0].value
+                                              : (I->src[0].value == 0);
+      uint8_t bytes[AGX3_SUBGROUP_MAX_LENGTH], length;
+      return agx3_subgroup_encode_quad_ballot_imm(
+                I->dest[0].value, I->dest[0].cache, nonzero, wait_mask, bytes,
+                &length) &&
+             agx3_append(emission, bytes, length);
+   }
+   if (enc == AGX_ISA_ENCODING_AGX3C && !quad &&
+       I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+       agx3_subgroup_plain(I->dest[0], AGX_INDEX_REGISTER) &&
+       (agx3_subgroup_plain_half(I->src[0]) ||
+        agx3_subgroup_plain(I->src[0], AGX_INDEX_REGISTER)) &&
+       (agx3_subgroup_plain_half(I->src[1]) ||
+        agx3_subgroup_plain(I->src[1], AGX_INDEX_REGISTER)) &&
+       (is_float || I->icond == AGX_ICOND_UEQ || I->icond == AGX_ICOND_SEQ ||
+        I->icond == AGX_ICOND_ULT || I->icond == AGX_ICOND_UGT ||
+        I->icond == AGX_ICOND_SLT || I->icond == AGX_ICOND_SGT)) {
+      const struct agx3_subgroup_ballot_desc d = {
+         .type = is_float ? AGX3_SUBGROUP_BALLOT_FLOAT : AGX3_SUBGROUP_BALLOT_INTEGER,
+         .dest = I->dest[0].value,
+         .dest_cache = I->dest[0].cache,
+         .src = I->src[0].value,
+         .src_half = I->src[0].size == AGX_SIZE_16,
+         .src_live = !I->src[0].discard,
+         .src_cache = I->src[0].cache,
+         .compare_register = true,
+         .compare_value = I->src[1].value,
+         .compare_half = I->src[1].size == AGX_SIZE_16,
+         .compare_cache = I->src[1].cache,
+         .compare_discard = I->src[1].discard,
+         .condition = is_float ? I->fcond :
+            (I->icond == AGX_ICOND_UEQ || I->icond == AGX_ICOND_SEQ) ? 4 :
+            I->icond == AGX_ICOND_SLT ? 5 :
+            I->icond == AGX_ICOND_SGT ? 6 :
+            I->icond == AGX_ICOND_ULT ? 1 : 2,
+         .invert = I->invert_cond,
+         .wait_mask = wait_mask,
+      };
+      uint8_t bytes[AGX3_SUBGROUP_MAX_LENGTH], length;
+      return agx3_subgroup_encode_ballot(enc, &d, bytes, &length) &&
+             agx3_append(emission, bytes, length);
+   }
    /* A 16-bit source is grounded for the immediate integer form only; the
     * encoder refuses every other shape. */
    const agx_index src = I->src[0];
@@ -4478,6 +5078,11 @@ agx3_emit_subgroup_ballot(struct util_dynarray *emission,
           * the 16-bit quad ballot returned above. AGX3B only.
           */
          if (I->invert_cond) {
+            if (!quad && enc == AGX_ISA_ENCODING_AGX3C) {
+               compare_immediate = true;
+               condition = 4;
+               break;
+            }
             if (!quad && enc == AGX_ISA_ENCODING_AGX3B) {
                if (compare_value != 0)
                   return false;
@@ -4551,7 +5156,8 @@ agx3_emit_subgroup_ballot(struct util_dynarray *emission,
     * predicate may still come directly from an asynchronous SSBO/device
     * load, so retire those exact producer slots with the existing synchronous
     * identity prelude before issuing the unchanged no-wait ballot. */
-   if (wait_mask != 0 &&
+   const bool direct_wait = enc == AGX_ISA_ENCODING_AGX3C && !is_float && !quad;
+   if (!direct_wait && wait_mask != 0 &&
        !agx3_retire_result_wait(emission, enc, I, wait_mask))
       return false;
 
@@ -4567,14 +5173,23 @@ agx3_emit_subgroup_ballot(struct util_dynarray *emission,
       .dest_cache = I->dest[0].cache,
       .src_live = !I->src[0].discard,
       .condition = condition,
-      .invert = compare_immediate ? false : I->invert_cond,
-      .wait_mask = 0,
+      .invert = compare_immediate && !direct_wait ? false : I->invert_cond,
+      .wait_mask = direct_wait ? wait_mask : 0,
    };
    uint8_t bytes[AGX3_SUBGROUP_MAX_LENGTH], length = 0;
    return agx3_subgroup_encode_ballot(enc, &d, bytes, &length) &&
           agx3_append(emission, bytes, length);
 }
 
+static bool
+agx3_m5_texture_barrier(enum agx_isa_encoding enc, unsigned scope)
+{
+   return enc == AGX_ISA_ENCODING_AGX3C &&
+          (scope == AGX3_BARRIER_SCOPE_TEXTURE ||
+           scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT ||
+           scope == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP);
+}
+
 static bool
 agx3_emit_scoped_barrier(struct util_dynarray *emission,
                          enum agx_isa_encoding enc, const agx_instr *I,
@@ -4582,10 +5197,15 @@ agx3_emit_scoped_barrier(struct util_dynarray *emission,
 {
    if (I->op != AGX_OPCODE_SCOPED_BARRIER_AGX3 ||
        I->nr_dests != 0 || I->nr_srcs != 0 ||
-       I->imm > AGX3_BARRIER_SCOPE_SUBGROUP_SHARED)
+       I->imm > AGX3_BARRIER_SCOPE_OBJECT_PAYLOAD_EXIT)
       return false;
 
    struct agx3_encoded_scoped_barrier encoded;
+   /* AGX3C texture barriers wait in their own first word, as on G17G. */
+   if (agx3_m5_texture_barrier(enc, I->imm))
+      return agx3_pack_scoped_barrier_wait(enc, I->imm, wait_mask, &encoded) &&
+             agx3_append(emission, encoded.bytes, encoded.length);
+
    return (!wait_mask || agx3_retire_slots_noop(emission, enc, wait_mask)) &&
           agx3_pack_scoped_barrier(enc, I->imm, &encoded) &&
           agx3_append(emission, encoded.bytes, encoded.length);
@@ -4763,6 +5383,47 @@ agx3_emit_cf_compare(struct util_dynarray *emission, enum agx_isa_encoding enc,
    if (I->nr_srcs != 2)
       return false;
 
+   if (is_float && enc == AGX_ISA_ENCODING_AGX3C &&
+       agx3_src_is_register(I->src[0]) && agx3_src_is_register(I->src[1]) &&
+       !I->src[0].memory && !I->src[1].memory &&
+       !I->src[0].neg && !I->src[1].neg &&
+       !I->src[0].kill && !I->src[1].kill &&
+       agx_channels(I->src[0]) == 1 && agx_channels(I->src[1]) == 1 &&
+       (I->src[0].size == AGX_SIZE_16 || I->src[0].size == AGX_SIZE_32) &&
+       (I->src[1].size == AGX_SIZE_16 || I->src[1].size == AGX_SIZE_32)) {
+      enum agx3_cf_fcond cond;
+      bool or_nan;
+      const uint16_t src[] = {I->src[0].value, I->src[1].value};
+      const bool half[] = {I->src[0].size == AGX_SIZE_16,
+                           I->src[1].size == AGX_SIZE_16};
+      const bool absolute[] = {I->src[0].abs, I->src[1].abs};
+      const struct agx3_cf_source_hints hints[] = {
+         {I->src[0].cache, I->src[0].discard},
+         {I->src[1].cache, I->src[1].discard},
+      };
+      return agx3_cf_map_fcond(I->fcond, &cond, &or_nan, invert) &&
+             agx3_cf_fcmp_width_m5(cond, or_nan, flag_slot, src, half,
+                                    absolute, hints, 0, bytes, &len) &&
+             agx3_append(emission, bytes, len);
+   }
+
+   if (!is_float && enc == AGX_ISA_ENCODING_AGX3C &&
+       (I->src[0].size == AGX_SIZE_16 || I->src[1].size == AGX_SIZE_16) &&
+       (agx3_cf_reg16(I->src[0]) || agx3_cf_reg32(I->src[0])) &&
+       (agx3_cf_reg16(I->src[1]) || agx3_cf_reg32(I->src[1]))) {
+      enum agx3_cf_icond cond;
+      const uint16_t src[] = {I->src[0].value, I->src[1].value};
+      const bool half[] = {I->src[0].size == AGX_SIZE_16,
+                           I->src[1].size == AGX_SIZE_16};
+      const struct agx3_cf_source_hints hints[] = {
+         {I->src[0].cache, I->src[0].discard},
+         {I->src[1].cache, I->src[1].discard},
+      };
+      return agx3_cf_map_icond(I->icond, &cond) &&
+             agx3_cf_icmp_width_m5(cond, flag_slot, src, half, hints, bytes, &len) &&
+             agx3_append(emission, bytes, len);
+   }
+
    if (!is_float && agx3_cf_reg16(I->src[0]) && agx3_cf_reg16(I->src[1]) &&
        agx_channels(I->src[0]) == 1 && agx_channels(I->src[1]) == 1) {
       enum agx3_cf_icond condition;
@@ -4821,7 +5482,7 @@ agx3_emit_cf_compare(struct util_dynarray *emission, enum agx_isa_encoding enc,
 
    /* Branching on a float against a constant: f18, the compare's second
     * operand, holds the minifloat. */
-   if (is_float && enc == AGX_ISA_ENCODING_AGX3B &&
+   if (is_float && (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
        ((agx3_cf_fminifloat(I->src[1]) &&
          agx3_cf_immediate_freg(I->src[0])) ||
         (agx3_cf_fminifloat(I->src[0]) &&
@@ -4888,7 +5549,8 @@ agx3_emit_cf_compare(struct util_dynarray *emission, enum agx_isa_encoding enc,
       return agx3_append(emission,bytes,len);
    }
 
-   if (is_float && enc == AGX_ISA_ENCODING_AGX3B &&
+   if (is_float && (enc == AGX_ISA_ENCODING_AGX3B ||
+                    enc == AGX_ISA_ENCODING_AGX3C) &&
        agx3_src_is_register(I->src[0]) &&
        agx3_src_is_register(I->src[1]) &&
        I->src[0].size == AGX_SIZE_32 &&
@@ -4942,7 +5604,7 @@ agx3_emit_cf_compare(struct util_dynarray *emission, enum agx_isa_encoding enc,
 
    /* Two half-float registers compare natively, as Apple's G17P compiler
     * does for `if (h0 < h1)`, with both sources' actual lifetimes. */
-   if (is_float && enc == AGX_ISA_ENCODING_AGX3B &&
+   if (is_float && (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
        agx3_src_is_register(I->src[0]) && agx3_src_is_register(I->src[1]) &&
        I->src[0].size == AGX_SIZE_16 && I->src[1].size == AGX_SIZE_16 &&
        agx_channels(I->src[0]) == 1 && agx_channels(I->src[1]) == 1 &&
@@ -4967,7 +5629,8 @@ agx3_emit_cf_compare(struct util_dynarray *emission, enum agx_isa_encoding enc,
     * agx_lower_uniform_sources(). Canonicalize the uniform to the first
     * operand (swapping the relation, as the register/immediate path below
     * does) and emit Apple's loop-guard form (fs-apple.bin +0x0c). */
-   if (!is_float && enc == AGX_ISA_ENCODING_AGX3B) {
+   if (!is_float && (enc == AGX_ISA_ENCODING_AGX3B ||
+                     enc == AGX_ISA_ENCODING_AGX3C)) {
       const bool uniform_first = agx3_cf_uniform32(I->src[0]) &&
                                  agx3_cf_imm8(I->src[1]);
       const bool uniform_second = agx3_cf_imm8(I->src[0]) &&
@@ -4987,7 +5650,8 @@ agx3_emit_cf_compare(struct util_dynarray *emission, enum agx_isa_encoding enc,
 
    /* A register against an aligned 32-bit uniform, admitted by
     * agx_instr_accepts_uniform() as operand 1. */
-   if (!is_float && enc == AGX_ISA_ENCODING_AGX3B &&
+   if (!is_float && (enc == AGX_ISA_ENCODING_AGX3B ||
+                     enc == AGX_ISA_ENCODING_AGX3C) &&
        agx3_cf_reg32(I->src[0]) && agx_channels(I->src[0]) == 1 &&
        !I->src[0].kill && I->src[1].type == AGX_INDEX_UNIFORM &&
        I->src[1].size == AGX_SIZE_32 && agx_channels(I->src[1]) == 1 &&
@@ -5116,6 +5780,28 @@ agx3_retire_slots_noop(struct util_dynarray *emission, enum agx_isa_encoding enc
           agx3_append(emission, bytes, len);
 }
 
+/* Carry `mask` on an identity of `reg`, which must be ready once `mask` has
+ * retired: every pending producer of it is in the mask. Only touch one half
+ * for a half register: its adjacent half may still belong to an independent
+ * asynchronous load. Keep the existing slot-6 carrier contract; changing that
+ * protocol requires separate hardware qualification. */
+static bool
+agx3_retire_wait_on(struct util_dynarray *emission, enum agx_isa_encoding enc,
+                    agx_index reg, uint16_t mask)
+{
+   uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+   /* The immediate halfword form also reaches high registers unavailable to
+    * the word bitop form. The full register wait covers this subset too. */
+   const bool half = reg.size == AGX_SIZE_16 || reg.value >= 192;
+   const bool ok = half
+      ? agx3_alu_bit_imm(enc, AGX3_ALU_BOR, true, reg.value, reg.value, 0,
+                          AGX3_ALU_HINT_NONE,
+                          mask | (1u << (enc == AGX_ISA_ENCODING_AGX3C ? 7 : 5)),
+                          bytes, &len)
+      : agx3_alu_band_identity_wait(enc, reg.value, mask, bytes, &len);
+   return ok && agx3_append(emission, bytes, len);
+}
+
 /* Carry a result-producing instruction's complete logical wait on its own
  * destination. The slot pass already includes WAW producers and outstanding
  * payload readers of that footprint, in addition to the instruction's RAW
@@ -5125,10 +5811,6 @@ agx3_retire_slots_noop(struct util_dynarray *emission, enum agx_isa_encoding enc
  * full wait without issuing any new readers/writers of the destination. The
  * latter permits resource-only waits during operand staging, when the result
  * footprint is already ready. A partial mask alone is not sufficient.
- *
- * Only touch one half for a half result: its adjacent half may still belong
- * to an independent asynchronous load. Keep the existing slot-6 carrier
- * contract; changing that protocol requires separate hardware qualification.
  */
 static bool
 agx3_retire_result_wait(struct util_dynarray *emission,
@@ -5137,22 +5819,13 @@ agx3_retire_result_wait(struct util_dynarray *emission,
 {
    if (!mask)
       return true;
-   if (enc != AGX_ISA_ENCODING_AGX3B)
+   if (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C)
       return agx3_retire_slots_noop(emission, enc, mask);
    if (I->nr_dests != 1 || !agx3_src_is_register(I->dest[0]) ||
        I->dest[0].memory)
       return false;
 
-   const agx_index dest = I->dest[0];
-   uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
-   /* The immediate halfword form also reaches high registers unavailable to
-    * the word bitop form. The full destination wait covers this subset too. */
-   const bool half = dest.size == AGX_SIZE_16 || dest.value >= 192;
-   const bool ok = half
-      ? agx3_alu_bit_imm(enc, AGX3_ALU_BOR, true, dest.value, dest.value, 0,
-                          AGX3_ALU_HINT_NONE, mask | (1u << 5), bytes, &len)
-      : agx3_alu_band_identity_wait(enc, dest.value, mask, bytes, &len);
-   return ok && agx3_append(emission, bytes, len);
+   return agx3_retire_wait_on(emission, enc, I->dest[0], mask);
 }
 
 /* The compact register+minifloat ALU forms (agx3_alu_fadd_minifloat(),
@@ -5194,10 +5867,15 @@ agx3_allocate_staging(struct util_dynarray *emission,
                       enum agx_isa_encoding enc, const agx_instr *I,
                       struct agx3_staging_banks *banks,
                       uint16_t *outstanding, uint8_t slot,
-                      unsigned span, uint16_t *base)
+                      unsigned span, unsigned avoid_end, uint16_t *base)
 {
    uint16_t waits;
-   if (!agx3_staging_plan(banks, span, 4, AGX3_STAGING_HALFS, base, &waits))
+   /* C's packed per-lane LOD occupies bank+6 in an eight-half group.
+    * Ordinary coordinates keep their native four-half alignment. */
+   const unsigned alignment =
+      enc == AGX_ISA_ENCODING_AGX3C && span == 8 ? 8 : 4;
+   if (!agx3_staging_plan_above(banks, span, alignment, AGX3_STAGING_HALFS,
+                                avoid_end, base, &waits))
       return false;
 
    if (waits) {
@@ -5393,8 +6071,11 @@ agx3_try_control_flow(struct util_dynarray *emission,
     * would carry its slot across the branch (or a backedge), which needs a
     * predecessor-aware analysis this backend does not have, so it is refused.
     */
+   const bool live_begin = enc == AGX_ISA_ENCODING_AGX3C &&
+      I->op == AGX_OPCODE_BEGIN_CF &&
+      debug_get_option_m5_begin_cf_live();
    if (outstanding != 0 &&
-       (I->op == AGX_OPCODE_BEGIN_CF || is_if || is_while || is_break_if ||
+       ((!live_begin && I->op == AGX_OPCODE_BEGIN_CF) || is_if || is_while || is_break_if ||
         is_else || I->op == AGX_OPCODE_POP_EXEC || is_jump))
       return -1;
 
@@ -5939,8 +6620,9 @@ agx3_g17p_rt0_eot_store_kind(const struct agx_shader_key *key,
 static enum agx3_g17p_rt0_eot_kind
 agx3_g17p_rt0_eot_kind(const agx_context *ctx, enum agx_isa_encoding enc)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || ctx->stage != MESA_SHADER_COMPUTE ||
-       ctx->is_preamble || ctx->key->no_stop)
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C && ctx->key->m5_rt0_eot;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) ||
+       ctx->stage != MESA_SHADER_COMPUTE || ctx->is_preamble || ctx->key->no_stop)
       return AGX3_G17P_RT0_EOT_NONE;
 
    unsigned instructions = 0, get_x = 0, get_y = 0, offsets = 0, stores = 0;
@@ -6030,19 +6712,27 @@ agx3_g17p_rt0_eot_kind(const agx_context *ctx, enum agx_isa_encoding enc)
         ctx->key->g17p_rt0_eot_words != 4))
       return AGX3_G17P_RT0_EOT_NONE;
 
+   if (m5 && store_format != AGX_FORMAT_U8NORM &&
+       store_format != AGX_FORMAT_I8 && store_format != AGX_FORMAT_U16NORM &&
+       store_format != AGX_FORMAT_S16NORM && store_format != AGX_FORMAT_F16 &&
+       store_format != AGX_FORMAT_S8NORM && store_format != AGX_FORMAT_I16 &&
+       store_format != AGX_FORMAT_I32 && store_format != AGX_FORMAT_RGB10A2)
+      return AGX3_G17P_RT0_EOT_NONE;
+
    return agx3_g17p_rt0_eot_store_kind(ctx->key, store_format);
 }
 
-/* Apple's genuine builtin EOT factory, configured for a 2D-array attachment,
- * computes the tile origin from threadgroup position and dispatched group
- * size before publishing the layer and XY image wires. Match the complete
- * allocated Mesa EOT graph and its registers before substituting that exact
- * 84-byte program; no general array block-store shape is implied. */
+/* Apple's layered EOT computes the tile origin from threadgroup position
+ * and dispatched group size before publishing the layer and XY image wires.
+ * Match the complete allocated Mesa EOT graph before selecting the G17P
+ * builtin or the G17G array wire sequence with its uniform PBE/offset ABI;
+ * no general array block-store shape is implied. */
 static enum agx3_g17p_rt0_eot_kind
 agx3_g17p_layered_eot_kind(const agx_context *ctx, enum agx_isa_encoding enc)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || ctx->stage != MESA_SHADER_COMPUTE ||
-       ctx->is_preamble || ctx->key->no_stop)
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C && ctx->key->m5_rt0_eot;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) ||
+       ctx->stage != MESA_SHADER_COMPUTE || ctx->is_preamble || ctx->key->no_stop)
       return AGX3_G17P_RT0_EOT_NONE;
 
    enum agx_format store_format = AGX_FORMAT_U8NORM;
@@ -6124,7 +6814,12 @@ agx3_g17p_layered_eot_kind(const agx_context *ctx, enum agx_isa_encoding enc)
       }
    }
 
-   if (instruction != 8)
+   if (instruction != 8 ||
+       (m5 && store_format != AGX_FORMAT_U8NORM &&
+        store_format != AGX_FORMAT_I8 && store_format != AGX_FORMAT_U16NORM &&
+        store_format != AGX_FORMAT_S16NORM && store_format != AGX_FORMAT_F16 &&
+        store_format != AGX_FORMAT_S8NORM && store_format != AGX_FORMAT_I16 &&
+        store_format != AGX_FORMAT_I32 && store_format != AGX_FORMAT_RGB10A2))
       return AGX3_G17P_RT0_EOT_NONE;
 
    return agx3_g17p_rt0_eot_store_kind(ctx->key, store_format);
@@ -6507,24 +7202,23 @@ static unsigned
 agx3_g17p_sample_store_protocol(agx_context *ctx, bool colour)
 {
    agx_block *tail = colour ? agx_reconverged_mrt_tail(ctx) : NULL;
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if ((ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        ctx->key->fs.ignore_tib_dependencies == colour ||
        (list_length(&ctx->blocks) != 1 && !tail))
       return 0;
 
-   /* Post-RA, the definition that reaches each store is the most recent write
-    * of that half register, and the block is straight-line by the test above.
-    * 192 is the sample operand's own register ceiling in
-    * agx3_tile_sample_operand(). */
-   const agx_instr *reaching[192] = {NULL};
+   /* Track the definition reaching each mask over the C register file.
+    * The native tile leaf retains B's own 192-half operand ceiling. */
+   const agx_instr *reaching[256] = {NULL};
    struct {
       uint16_t offset;
       enum agx_format format;
       unsigned component_mask;
       unsigned stores;
-      uint16_t mask_regs[4];
-      const agx_instr *mask_defs[4];
+      uint16_t mask_regs[8];
+      const agx_instr *mask_defs[8];
    } targets[AGX_MAX_RTS];
    unsigned target_count = 0;
    uint64_t written_B = 0;
@@ -6646,7 +7340,9 @@ agx3_g17p_sample_store_protocol(agx_context *ctx, bool colour)
    }
 
    const bool complete = colour
-      ? (ctx->key->fs.nr_samples == 2 || ctx->key->fs.nr_samples == 4) &&
+      ? (ctx->key->fs.nr_samples == 2 || ctx->key->fs.nr_samples == 4 ||
+         (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          ctx->key->fs.nr_samples == 8)) &&
            samples > 0 && samples <= ctx->key->fs.nr_samples
       : samples == 2 || samples == 4;
    return stopped && waited == colour && complete &&
@@ -6713,8 +7409,12 @@ agx3_g17p_mrt_protocol(agx_context *ctx)
    const bool interleaved_zs = agx3_g17p_interleaved_zs(ctx);
    struct agx3_g17p_mrt_protocol p = {0};
    agx_block *tail = agx_reconverged_mrt_tail(ctx);
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if ((ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        !(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          ctx->key->fs.nr_samples <= 8)) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
+       (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+        ctx->key->fs.ignore_tib_dependencies) ||
        !ctx->agx3_mrt_tile_ops_ordered ||
        (!ctx->key->secondary && !ctx->key->fs.ignore_tib_dependencies &&
         (ctx->key->no_stop ||
@@ -6824,9 +7524,9 @@ agx3_g17p_mrt_protocol(agx_context *ctx)
    p.multisample = ctx->key->fs.nr_samples > 1 &&
                    !ctx->key->fs.ignore_tib_dependencies;
    unsigned previous_load = p.count;
-   uint8_t load_order[AGX_MAX_RTS * 4];
+   uint8_t load_order[AGX_MAX_RTS * 8];
    unsigned load_instr_count = 0;
-   uint8_t store_order[AGX_MAX_RTS * 4];
+   uint8_t store_order[AGX_MAX_RTS * 8];
    unsigned store_instr_count = 0;
    bool stores_started = false;
    bool stopped = false;
@@ -6992,7 +7692,9 @@ agx3_g17p_mrt_protocol(agx_context *ctx)
 static bool
 agx3_g17p_multi_background_protocol(const agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if ((ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        !(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          ctx->key->fs.nr_samples <= 8)) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        !ctx->key->fs.ignore_tib_dependencies ||
        list_length(&ctx->blocks) != 1)
@@ -7055,8 +7757,13 @@ agx3_g17p_multi_background_protocol(const agx_context *ctx)
          return false;
       }
    }
-   if (ctx->key->g17p_mrt_bg)
-      return stopped && stores >= 2;
+   if (ctx->key->g17p_mrt_bg) {
+      /* A sparse M5 attachment set can leave one live target at a nonzero
+       * logical slot. It uses the same transaction-free background ABI. */
+      const unsigned minimum =
+         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C ? 1 : 2;
+      return stopped && stores >= minimum;
+   }
    return stopped && ((stride == 16 && stores >= 2 && stores <= 4) ||
                       (stride == 4 && (stores == 4 ||
                        (stores == 3 && format == AGX_FORMAT_U8NORM))));
@@ -7826,9 +8533,17 @@ agx3_reusable_cube_scratch(agx_context *ctx, unsigned maximum,
    return true;
 }
 
+/* Keep refusal diagnostics optional and emission failure transactional. */
+#define AGX3_PACK_FAIL() do { \
+   if (agx_compiler_debug & AGX_DBG_SHADERS) \
+      fprintf(stderr, "AGX3 pack failure at line %u\n", __LINE__); \
+   goto fail; \
+} while (0)
+
 bool
 agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 {
+   agx3_first_refusal[0] = 0;
    const enum agx_isa_encoding enc = ctx->key->dev.encoding;
    const unsigned nslots = (enc == AGX_ISA_ENCODING_AGX3C) ? 8 : 6;
    const unsigned initial_size = emission->size;
@@ -8041,6 +8756,117 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
    const enum agx3_g17p_rt0_eot_kind layered_eot =
       agx3_g17p_layered_eot_kind(ctx, enc);
+   const enum agx3_g17p_rt0_eot_kind rt0_eot =
+      agx3_g17p_rt0_eot_kind(ctx, enc);
+   const enum agx3_g17p_rt0_eot_kind m5_eot =
+      layered_eot != AGX3_G17P_RT0_EOT_NONE ? layered_eot : rt0_eot;
+   if (enc == AGX_ISA_ENCODING_AGX3C && m5_eot != AGX3_G17P_RT0_EOT_NONE) {
+      static const uint8_t m5_rt0_eot[78] = {
+         0x1c,0x9c,0x10,0x06,0x24,0xa8,0x14,0x06,
+         0x0c,0x9d,0x10,0x26,0x24,0x81,0x01,0x40,
+         0x28,0x01,0xa0,0x0a,0x27,0x06,0x04,0x0a,
+         0x21,0x00,0xa1,0x02,0x68,0x01,0x20,0x00,
+         0x01,0x02,0x37,0x08,0x04,0x0a,0x21,0x00,
+         0xa1,0x02,0x28,0x01,0x20,0x00,0x01,0x02,
+         0x27,0x00,0x20,0x2a,0x21,0x00,0xab,0x0a,
+         0x00,0x00,0x4a,0x30,0x18,0x00,0x25,0x04,
+         0x15,0x00,0x24,0x00,0xa0,0x22,0x00,0xe0,
+         0x00,0x55,0x0e,0x00,0x00,0x00,
+      };
+      uint8_t program[sizeof(m5_rt0_eot)];
+      memcpy(program, m5_rt0_eot, sizeof(program));
+      switch (m5_eot) {
+      case AGX3_G17P_RT0_EOT_SELECTOR_15: program[71] = 0x20; break;
+      case AGX3_G17P_RT0_EOT_SELECTOR_35: program[71] = 0x60; break;
+      case AGX3_G17P_RT0_EOT_SELECTOR_55: program[71] = 0xa0; break;
+      case AGX3_G17P_RT0_EOT_SELECTOR_75: break;
+      case AGX3_G17P_RT0_EOT_NORM16:
+         program[69] = 0x2a;
+         program[71] = 0x20;
+         break;
+      default: return false;
+      }
+      if (ctx->key->g17p_rt0_eot_multisample)
+         program[73] = (program[73] & ~0xc0u) | 0x80u;
+      if (ctx->key->m5_rt0_eot_store_and_resolve) {
+         if (layered_eot != AGX3_G17P_RT0_EOT_NONE ||
+             m5_eot != AGX3_G17P_RT0_EOT_SELECTOR_75 ||
+             !ctx->key->g17p_rt0_eot_multisample)
+            return false;
+
+         /* Native G17G paired IMGWBLK retains the one coordinate wire for
+          * the MS store (slot 2), then consumes it in the 2D resolve (slot 1).
+          * Keep the qualified uniform-PBE ABI: U0 source, U4 tile offset,
+          * and U8 resolve PBE. Descriptor operands remain uniform/64-bit. */
+         static const struct agx3_field_kv img[] = {
+            {9, 5}, {10, 2}, {14, 2}, {15, 1}, {16, 86},
+            {29, 3}, {32, 1}, {45, 3}, {46, 3},
+            {59, 1}, {60, 1}, {73, 1},
+         };
+         static const struct agx3_field_kv sample_store[] = {
+            {12, 2}, {17, 1}, {30, 2},
+         };
+         static const struct agx3_field_kv resolve_store[] = {
+            {12, 1}, {13, 1}, {30, 1}, {39, 8},
+         };
+         uint8_t sample_bytes[16], resolve_bytes[16];
+         uint8_t sample_len = 0, resolve_len = 0;
+         if (!agx3_bind_compose(enc, "imgwblk", 0, img, ARRAY_SIZE(img),
+                                sample_store, ARRAY_SIZE(sample_store),
+                                sample_bytes, &sample_len) ||
+             !agx3_bind_compose(enc, "imgwblk", 0, img, ARRAY_SIZE(img),
+                                resolve_store, ARRAY_SIZE(resolve_store),
+                                resolve_bytes, &resolve_len) ||
+             sample_len != 12 || resolve_len != 12)
+            return false;
+         ctx->max_reg = MAX2(ctx->max_reg, 7);
+         ctx->agx3_staging_halfs = MAX2(ctx->agx3_staging_halfs, 8);
+         return agx3_append(emission, program, 62) &&
+                agx3_append(emission, sample_bytes, sample_len) &&
+                agx3_append(emission, resolve_bytes, resolve_len) &&
+                agx3_append(emission, program + 74, 4);
+      }
+      uint8_t layer_sr[AGX3_SR_MAX_LENGTH], layer_pack[16];
+      uint8_t layer_sr_len = 0, layer_pack_len = 0;
+      if (layered_eot != AGX3_G17P_RT0_EOT_NONE) {
+         /* G17G's array intrinsic publishes a 16-bit layer into a GPR
+          * wire before the XY wire. Keep the proven builtin's uniform
+          * PBE pointer (halves 0..3) and tile byte offset (halves 4..5).
+          * Native llvm.agx2.get.layerid reads SR12/W16, including internal
+          * tile programs; dispatched grid Z is not the tile's layer.
+          * XY origin arithmetic has consumed slot 2 and GPR half 4. */
+         const struct agx3_sr_desc layer = {
+            .dest = 6, .width = AGX3_SR_W16, .sr = 12, .slot = 2,
+         };
+         static const struct agx3_field_kv layer_fields[] = {
+            {2, 4}, {9, 2}, {10, 2}, {11, 1}, {16, 6},
+            {23, 2}, {24, 1}, {27, 1}, {38, 1}, {51, 1},
+            {52, 2}, {56, 3}, {57, 118}, {59, 1}, {74, 4},
+         };
+         if (!agx3_sr_encode(enc, &layer, layer_sr, &layer_sr_len) ||
+             !agx3_bind_compose(enc, "pack_imgwir", 0, layer_fields,
+                                ARRAY_SIZE(layer_fields), NULL, 0,
+                                layer_pack, &layer_pack_len))
+            return false;
+
+         /* Native 2D/array pairs isolate the layer operand: f44=4,
+          * f51=2, f55=1 on XY PACK_IMGWIR, and f31=2 on IMGWBLK.
+          * All descriptor, offset, staging and format fields stay intact. */
+         program[59] = 0x40;
+         program[60] = 0x1c;
+         program[73] &= ~0x10u;
+      }
+      ctx->max_reg = layered_eot != AGX3_G17P_RT0_EOT_NONE
+                        ? 7 : MAX2(ctx->max_reg, 7);
+      ctx->agx3_staging_halfs = MAX2(ctx->agx3_staging_halfs, 8);
+      if (layered_eot != AGX3_G17P_RT0_EOT_NONE) {
+         return agx3_append(emission, program, 48) &&
+                agx3_append(emission, layer_sr, layer_sr_len) &&
+                agx3_append(emission, layer_pack, layer_pack_len) &&
+                agx3_append(emission, program + 48, sizeof(program) - 48);
+      }
+      return agx3_append(emission, program, sizeof(program));
+   }
    if (layered_eot != AGX3_G17P_RT0_EOT_NONE) {
       /* The 2D-array end-of-tile builtin is one 84-byte program for every
        * format, differing only in the IMGWBLK lane-class selector at byte
@@ -8075,8 +8901,6 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
       return agx3_append(emission, program, sizeof(program));
    }
 
-   const enum agx3_g17p_rt0_eot_kind rt0_eot =
-      agx3_g17p_rt0_eot_kind(ctx, enc);
    if (rt0_eot != AGX3_G17P_RT0_EOT_NONE) {
       static const uint8_t g17p_rt0_eot[][68] = {{
          0x1c, 0x9c, 0x10, 0x06, 0x24, 0xa8, 0x14, 0x06,
@@ -8242,10 +9066,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    /* The last pixel release of a program-ending fragment part, when no
     * colour store, coverage update, acquisition or branch follows it. */
    const agx_instr *g17p_final_release = NULL;
-   bool has_zs_emit = false;
+   bool has_zs_emit = false, zs_sample_loop = false;
    unsigned zs_staging_halfs = 0;
    bool g17p_fragment_helper_protocol =
-      enc == AGX_ISA_ENCODING_AGX3B &&
+      (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
       ctx->stage == MESA_SHADER_FRAGMENT && ctx->nir &&
       (ctx->nir->info.fs.needs_coarse_quad_helper_invocations ||
        ctx->nir->info.fs.needs_full_quad_helper_invocations);
@@ -8265,7 +9089,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
    assert(enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C);
    util_dynarray_init(&fixups, ctx);
-   const bool early_compare = enc == AGX_ISA_ENCODING_AGX3B;
+   const bool early_compare = enc == AGX_ISA_ENCODING_AGX3B ||
+      (enc == AGX_ISA_ENCODING_AGX3C && !ctx->is_preamble &&
+       ctx->stage == MESA_SHADER_COMPUTE &&
+       debug_get_bool_option("ASAHI_M5_EARLY_COMPARE", true));
    struct util_dynarray spans, compares;
    util_dynarray_init(&spans, ctx);
    util_dynarray_init(&compares, ctx);
@@ -8321,12 +9148,16 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             has_zs_emit = true;
          if (scan_I->op == AGX_OPCODE_ZS_EMIT_AGX3)
             zs_staging_halfs += (scan_I->zs & 1) ? 4 : 2;
+         if (scan_I->op == AGX_OPCODE_ZS_EMIT_AGX3 && (scan_I->zs & 4))
+            zs_sample_loop = true;
          if (scan_I->op == AGX_OPCODE_ATOMIC ||
              scan_I->op == AGX_OPCODE_DEVICE_STORE ||
              scan_I->op == AGX_OPCODE_IMAGE_WRITE)
             has_memory_write = true;
          if (scan_I->op == AGX_OPCODE_INTERSECT_RAY_GEN1 ||
-             scan_I->op == AGX_OPCODE_CONTINUE_RAY_GEN1)
+             scan_I->op == AGX_OPCODE_CONTINUE_RAY_GEN1 ||
+             scan_I->op == AGX_OPCODE_INTERSECT_RAY_M5 ||
+             scan_I->op == AGX_OPCODE_CONTINUE_RAY_M5)
             has_ray_traversal = true;
          if (scan_I->op == AGX_OPCODE_DEVICE_LOAD)
             device_loads_left++;
@@ -8426,9 +9257,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    const uint16_t entry_wait = cprog_imports ? (1u << 5) : 0;
    if (cprog_imports) {
       if (!ctx->is_preamble || !ctx->agx3_cprog_inputs_lowered ||
-          enc != AGX_ISA_ENCODING_AGX3B || !ctx->out ||
-          !ctx->out->agx3_preamble_inputs.count)
-         goto fail;
+          (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+          !ctx->out || !ctx->out->agx3_preamble_inputs.count)
+         AGX3_PACK_FAIL();
       outstanding = entry_wait;
       for (unsigned h = 0; h < 512; ++h)
          uniform_half_used[h] |=
@@ -8436,22 +9267,33 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                (UINT64_C(1) << (h % 64)));
    }
 
-   /* Fresh G17P fragment witnesses with two/three ordinary SMPs clear f156 on
+   /* Native G17P and G17G fragment witnesses with ordinary SMPs clear f156 on
     * every earlier group member and set it only on the final member, including
     * an all-explicit-LEVEL group. Explicit gradients are outside the group.
-    * Other stages and AGX3C keep their existing per-op encodings. */
-   const bool g17p_sample_group = enc == AGX_ISA_ENCODING_AGX3B &&
+    * Other stages keep their existing per-op encodings. */
+   const bool g17p_sample_group =
+                                  (enc == AGX_ISA_ENCODING_AGX3B ||
+                                   enc == AGX_ISA_ENCODING_AGX3C) &&
                                   ctx->stage == MESA_SHADER_FRAGMENT &&
                                   sample_group_members_left > 1;
 
+   /* G17G sampled fragments use unmarked coefficient inputs, then acquire
+    * 512 after coordinate/LOD publication and immediately before SMP. */
+   if (enc == AGX_ISA_ENCODING_AGX3C &&
+       ctx->stage == MESA_SHADER_FRAGMENT && texture_samples_left)
+      g17p_fragment_helper_protocol = true;
+
    /* CEM axis 0 writes a magnitude/face pair, axes 1 and 2 one 32-bit
     * coordinate each; the implicit face word is reserved too, or axis 1
     * overwrites it. */
    uint16_t cem_scratch[3] = {0};
    if (has_cube_sample) {
-      const unsigned base = (observed_max_reg + 2) & ~1u;
+      /* M5's RA reserves a low bank so unrelated values may use the full
+       * register file; older callers retain the bank above their allocation. */
+      const unsigned base = ctx->agx3_cem_low_scratch
+                               ? 8 : (observed_max_reg + 2) & ~1u;
       if (base + 6 > 190)
-         goto fail;
+         AGX3_PACK_FAIL();
       cem_scratch[0] = base;
       cem_scratch[1] = base + 4;
       cem_scratch[2] = base + 6;
@@ -8462,7 +9304,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             MAX2(ctx->max_reg,
                  MAX3(reusable[0] + 3, reusable[1] + 1, reusable[2] + 1));
       } else {
-         ctx->max_reg = base + 7;
+         /* M5's low CEM bank may sit below the current allocation. */
+         ctx->max_reg = MAX2(ctx->max_reg, base + 7);
       }
    }
 
@@ -8471,11 +9314,15 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
     * live RA operand and CEM scratch; successive samples reuse it. */
    uint16_t ms_array_scratch = 0;
    if (has_packed_array_layer) {
-      const unsigned base = (MAX2(observed_max_reg, ctx->max_reg) + 2) & ~1u;
-      if (base > 190)
-         goto fail;
+      /* Array staging follows both CEM finalizers, so their first axis word
+       * is dead and can hold the packed layer/LOD on M5. */
+      const unsigned base = ctx->agx3_cem_low_scratch
+                               ? cem_scratch[1]
+                               : (MAX2(observed_max_reg, ctx->max_reg) + 2) & ~1u;
+      if (base > (enc == AGX_ISA_ENCODING_AGX3C ? 254 : 190))
+         AGX3_PACK_FAIL();
       ms_array_scratch = base;
-      ctx->max_reg = base + 1;
+      ctx->max_reg = MAX2(ctx->max_reg, base + 1);
    }
 
    /* Apple's sampled G17P fragment objects use a distinct constant program
@@ -8500,7 +9347,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    const bool g17p_sample_colour_stores =
       tile_stores_left > 1 && agx3_g17p_sample_store_protocol(ctx, true);
    const bool g17p_colour_protocol =
-      (enc == AGX_ISA_ENCODING_AGX3B &&
+      ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
        ctx->stage == MESA_SHADER_FRAGMENT &&
        !ctx->key->fs.ignore_tib_dependencies &&
        (tile_stores_left == 1 || g17p_sample_colour_stores) &&
@@ -8509,11 +9356,12 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    /* Multiple colour stores need an attachment-specific pixel transaction.
     * Emitting individual tile operations without a proven partition graph
     * produces a syntactically valid shader that writes only partition zero. */
-   if (enc == AGX_ISA_ENCODING_AGX3B &&
+   if ((enc == AGX_ISA_ENCODING_AGX3B ||
+        (enc == AGX_ISA_ENCODING_AGX3C && ctx->key->fs.nr_samples <= 1)) &&
        ctx->stage == MESA_SHADER_FRAGMENT &&
        tile_stores_left > 1 && !g17p_mrt.valid && !g17p_multi_background &&
        !g17p_msaa_background_stores && !g17p_sample_colour_stores)
-      goto fail;
+      AGX3_PACK_FAIL();
    /* Exact native RGBA8 alpha pipeline: one class-8 read wait/load followed
     * by one class-4 write wait/store, under the same class-12 entry/release.
     * Keep MRT, special coordinates, other formats and control flow closed.
@@ -8579,7 +9427,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    const bool g17p_pixel_protocol =
       g17p_colour_protocol || g17p_blend_protocol || has_coverage_write;
    const bool g17p_colourless_fragment =
-      enc == AGX_ISA_ENCODING_AGX3B && ctx->stage == MESA_SHADER_FRAGMENT &&
+      (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+      ctx->stage == MESA_SHADER_FRAGMENT &&
       !ctx->key->fs.ignore_tib_dependencies && tile_stores_left == 0 &&
       !(ctx->nir && (ctx->nir->info.outputs_written &
                      BITFIELD64_RANGE(FRAG_RESULT_DATA0, AGX_MAX_RTS)));
@@ -8590,10 +9439,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    /* The partition a non-MRT colour transaction belongs to selects its
     * classes, exactly as the MRT protocol selects a lone store's. */
    unsigned g17p_lone_pair = 0;
-   if (enc == AGX_ISA_ENCODING_AGX3B && ctx->stage == MESA_SHADER_FRAGMENT &&
+   if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+       ctx->stage == MESA_SHADER_FRAGMENT &&
        !g17p_mrt.valid && (g17p_colour_protocol || g17p_blend_protocol) &&
        !agx3_g17p_lone_colour_pair(ctx, &g17p_lone_pair))
-      goto fail;
+      AGX3_PACK_FAIL();
    /* The fused colour link template is the class-12 form only. */
    agx3_g17p_colour_link_info(ctx, g17p_colour_protocol &&
                                   g17p_deferred_write != NULL &&
@@ -8604,17 +9454,15 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                            ? BITFIELD_BIT(g17p_lone_pair)
                            : 0);
    ctx->agx3_colour_link.straight_line = list_length(&ctx->blocks) == 1;
-   ctx->agx3_colour_link.branch_free = list_length(&ctx->blocks) == 1;
    agx_foreach_instr_global(ctx, entry_I) {
-      if (agx3_is_cf_op(entry_I))
-         ctx->agx3_colour_link.branch_free = false;
       if (agx3_is_cf_op(entry_I) ||
           entry_I->op == AGX_OPCODE_SAMPLE_MASK_AGX3 ||
           entry_I->op == AGX_OPCODE_ZS_EMIT_AGX3)
          ctx->agx3_colour_link.straight_line = false;
    }
    const bool fragment_ray_query = has_ray_traversal &&
-      enc == AGX_ISA_ENCODING_AGX3B && ctx->stage == MESA_SHADER_FRAGMENT;
+      (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+      ctx->stage == MESA_SHADER_FRAGMENT;
    if (fragment_ray_query && (ctx->key->fs.nr_samples > 1 ||
        (ctx->agx3_colour_link.colour_classes & ~1u)))
       goto fail;
@@ -8733,7 +9581,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
        native_coord_slots == NULL || !native_coord_values ||
        !resources || !resource_uses ||
        !resource_bases || !publication_values || !resource_last_use)
-      goto fail;
+      AGX3_PACK_FAIL();
 
    unsigned resource_count = 0, resource_use_count = 0;
    unsigned active_coord_resource = UINT32_MAX;
@@ -8752,6 +9600,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
    unsigned k = 0;
    unsigned vertex_loads_scheduled = 0;
+   const unsigned device_load_batch =
+      enc == AGX_ISA_ENCODING_AGX3C && !ctx->is_preamble &&
+      ctx->stage == MESA_SHADER_COMPUTE && ctx->nir && !ctx->nir->info.internal
+         ? debug_get_num_option("ASAHI_M5_DEVICE_LOAD_BATCH", 1) : 0;
    agx_foreach_block(ctx, slot_block) {
       agx_foreach_instr_in_block(slot_block, I) {
          if (I->op == AGX_OPCODE_STOP)
@@ -8776,6 +9628,21 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
          e->is_async = agx3_is_async(I) || I->agx3_tex_coord_start;
+         /* Native G17G loads16/loads32 issue several independent 32-bit
+          * device loads on each queue before the first consumer wait.
+          * Keep coherent operations and other asynchronous engines separate. */
+         if ((device_load_batch == 1 || device_load_batch == 2) &&
+             I->op == AGX_OPCODE_DEVICE_LOAD &&
+             I->format == AGX_FORMAT_I32 && !I->coherent &&
+             !I->agx3_tex_coord_start) {
+            e->join_class = 1;
+            e->join_limit = 8;
+            e->prefer_join = device_load_batch == 2;
+         }
+         /* Native query-once +0x5fc reads the traversal completion byte on
+          * slot8, waiting for the intersect/continue engine on that slot. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->scs_rt_completion)
+            e->preferred_slot = 8;
          e->latches_srcs = agx_is_texture_publication(I);
          /* Release ordering includes reads whose results are used after the
           * fence. Register dependencies alone do not order those reads. */
@@ -8841,31 +9708,65 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             e->preferred_slot = 6;
          if (I->agx3_tex_coords_staged)
             e->preferred_slot = 1;
+         /* A cube sample whose descriptors are uniforms or immediates groups
+          * CEM's face finalizers with its own result slot (f12), and that
+          * field only carries slots 1..7: on AGX3C a round-robin slot 8 made
+          * the whole shader unencodable. Register descriptors stage on slot
+          * 6 instead and are unaffected. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && agx3_tex_cube_groups_result_slot(I))
+            e->max_slot = 7;
 
          unsigned nd = 0, ns = 0;
          if (!I->agx3_tex_coord_stage) {
-            for (unsigned d = 0; d < I->nr_dests; ++d)
-               nd = agx3_expand_regs(I->dest[d], e->dst, nd);
+            for (unsigned d = 0; d < I->nr_dests; ++d) {
+               agx_index dest = I->dest[d];
+               /* Masked samples write a dense result, not SSA padding. */
+               if (enc == AGX_ISA_ENCODING_AGX3C &&
+                   I->op == AGX_OPCODE_TEXTURE_SAMPLE && !I->sparse &&
+                   I->gather == AGX_GATHER_NONE && !I->query_lod &&
+                   d == 0 && dest.size == AGX_SIZE_32 &&
+                   I->mask &&
+                   !(I->mask & ~0xfu) &&
+                   debug_get_option_m5_texture_result_tokens()) {
+                  const unsigned channels = util_bitcount(I->mask);
+                  if (channels < agx_channels(dest))
+                     dest.channels_m1 = channels - 1;
+               }
+               nd = agx3_expand_regs(dest, e->dst, nd);
+            }
          }
          for (unsigned s = 0; s < I->nr_srcs; ++s) {
              if (I->agx3_tex_coords_staged && s == 0)
                continue;
             if (agx_texture_resource(I) && (s == 0 || (s == 5 && I->shadow)))
                continue;
+            const unsigned first = ns;
             ns = agx3_expand_regs(I->src[s], e->src, ns);
+            /* AGX3C SMP reads a register texture/sampler descriptor after
+             * issue: redefining that register while the sample is in flight
+             * gave wrong texels intermittently (vertex-stage comparisons
+             * with runtime offsets, 13 of 40 runs on M5 vmfix6; 0 of 40
+             * with the register treated as read late). Write-after-read
+             * then waits for the sample's slot before the redefinition. */
+            if (enc == AGX_ISA_ENCODING_AGX3C &&
+                I->op == AGX_OPCODE_TEXTURE_SAMPLE && (s == 3 || s == 4) &&
+                I->src[s].type == AGX_INDEX_REGISTER) {
+               for (unsigned t = first; t < ns && t < 32; ++t)
+                  e->late_src_mask |= UINT32_C(1) << t;
+            }
          }
          if (I->op == AGX_OPCODE_LOAD_CPROG_AGX3) {
             if (ns >= AGX3_SLOT_MAX_REGS)
-               goto fail;
+               AGX3_PACK_FAIL();
             e->src[ns++] = AGX3_SLOT_CPROG_INPUT_TOKEN;
          }
          if (nd > AGX3_SLOT_MAX_REGS || ns > AGX3_SLOT_MAX_REGS) {
             agx3_report_refusal(I, __LINE__);
-            goto fail;
+            AGX3_PACK_FAIL();
          }
          if (I->agx3_tex_coord_start) {
             if (active_coord_resource != UINT32_MAX)
-               goto fail;
+               AGX3_PACK_FAIL();
             active_coord_resource = resource_count++;
             /* Both scalar publications join one native completion event.
              * Represent their coordinate pair as a real resource value,
@@ -8883,7 +9784,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          }
          if (I->agx3_tex_coords_staged) {
             if (active_coord_resource == UINT32_MAX)
-               goto fail;
+               AGX3_PACK_FAIL();
             resource_uses[resource_use_count++] = (struct agx3_resource_use){
                .instruction = k - 1, .value = active_coord_resource,
                .first = 0, .count = 4, .through_completion = true,
@@ -8894,20 +9795,21 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const bool scalar = I->op == AGX_OPCODE_PUBLISH_TEXTURE_SCALAR_AGX3;
             if (!I->imm || I->imm > resource_capacity ||
                 publication_values[I->imm] || I->nr_srcs != (scalar ? 3 : 2))
-               goto fail;
+               AGX3_PACK_FAIL();
             const unsigned value = resource_count++;
             publication_values[I->imm] = value + 1;
             resources[value] = (struct agx3_resource_value){
                .producer = k - 1,
                .halves = agx_is_null(I->src[scalar ? 2 : 1]) ? 4 : 6,
-               .alignment = 4, .fixed_base = UINT16_MAX,
+               .alignment = enc == AGX_ISA_ENCODING_AGX3C ? 8 : 4,
+               .fixed_base = UINT16_MAX,
             };
          }
          if (agx_texture_resource(I)) {
             const unsigned id = agx_texture_resource(I);
             if (I->op != AGX_OPCODE_TEXTURE_SAMPLE || id > resource_capacity ||
                 !publication_values[id])
-               goto fail;
+               AGX3_PACK_FAIL();
             const unsigned value = publication_values[id] - 1;
             resource_uses[resource_use_count++] = (struct agx3_resource_use){
                .instruction = k - 1, .value = value,
@@ -8937,7 +9839,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           *
           * A four-channel f32 texture result expands to eight tokens, leaving
           * room for the four scratch words, including axis0's implicit face. */
-         if (enc == AGX_ISA_ENCODING_AGX3B &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B ||
+              enc == AGX_ISA_ENCODING_AGX3C) &&
              I->op == AGX_OPCODE_TEXTURE_SAMPLE &&
              (I->dim == AGX_DIM_CUBE ||
               I->dim == AGX_DIM_CUBE_ARRAY)) {
@@ -8973,13 +9876,13 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          e->nr_srcs = (uint8_t)ns;
          if ((I->op == AGX_OPCODE_STACK_LOAD ||
               I->op == AGX_OPCODE_STACK_STORE) &&
-             !agx3_append_static_spill_tokens(I, e)) {
+             !agx3_append_static_spill_tokens(I, e, enc)) {
             /* Report like every other refusal.  Without this the whole
              * shader fails with no diagnostic at all, which is how a
              * scratch-lowered constant array once looked like a mystery
              * VK_ERROR_FEATURE_NOT_PRESENT from vkCreateComputePipelines. */
             agx3_report_refusal(I, __LINE__);
-            goto fail;
+            AGX3_PACK_FAIL();
          }
       }
    }
@@ -9059,10 +9962,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
         !agx3_resource_lower(resources, resource_count, resource_uses,
                               resource_use_count, sl, count,
                               AGX3_SLOT_RESOURCE_HALVES, resource_bases, sl)))
-      goto fail;
+      AGX3_PACK_FAIL();
    if (!agx3_slot_assign_with_entry(sl, count, nslots, &input_entry,
                                     cprog_imports ? 1 : 0, res))
-      goto fail;
+      AGX3_PACK_FAIL();
 
    /* Emission drains every live scoreboard generation before control flow,
     * but the linear slot assignment above cannot see that architectural
@@ -9081,7 +9984,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          cf_live |= bit;
       }
 
-      if (agx3_is_cf_op(ordered[i])) {
+      const bool live_begin = enc == AGX_ISA_ENCODING_AGX3C &&
+         ordered[i]->op == AGX_OPCODE_BEGIN_CF &&
+         debug_get_option_m5_begin_cf_live();
+      if (agx3_is_cf_op(ordered[i]) && !live_begin) {
          cf_retired |= cf_live;
          cf_live = 0;
       }
@@ -9093,29 +9999,29 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
       const agx_instr *I = ordered[i];
       if (I->agx3_tex_coord_start) {
          if (active_coord_slot || !res[i].slot)
-            goto fail;
+            AGX3_PACK_FAIL();
          active_coord_slot = res[i].slot;
       }
       if (I->agx3_tex_coord_stage) {
          if (!active_coord_slot)
-            goto fail;
+            AGX3_PACK_FAIL();
          native_coord_slots[i] = active_coord_slot;
          staged_components |= BITFIELD_BIT(I->agx3_tex_coord_component);
       }
       if (I->agx3_tex_coords_staged) {
          if (!active_coord_slot || staged_components != 3)
-            goto fail;
+            AGX3_PACK_FAIL();
          native_coord_slots[i] = active_coord_slot;
          active_coord_slot = 0;
          staged_components = 0;
       }
    }
    if (active_coord_slot)
-      goto fail;
+      AGX3_PACK_FAIL();
 
    const bool *tps_rereads = agx3_static_reload_rereads(ctx, ordered, count);
    if (!tps_rereads)
-      goto fail;
+      AGX3_PACK_FAIL();
 
    k = 0;
    const bool interleaved_zs =
@@ -9143,7 +10049,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          (has_coverage_write || has_zs_emit)) ? 8
                                               : 0;
    const uint16_t image_write_bank_floor =
-      MAX2(fixed_staging_floor, ALIGN_POT(zs_staging_halfs, 8));
+      ALIGN_POT(MAX2(fixed_staging_floor, zs_staging_halfs),
+                enc == AGX_ISA_ENCODING_AGX3C ? 16 : 8);
    if ((has_coverage_write ||
         ((has_memory_write || has_ray_traversal) && !ctx->key->secondary)) &&
        enc == AGX_ISA_ENCODING_AGX3B &&
@@ -9152,12 +10059,13 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
       /* A discarding or side-effecting fragment enters the pixel
        * transaction before its first device load or memory side effect
        * (stores and atomics alike), then uses unmarked SR/RHW inputs; the
-       * input protocol is not restarted after MOVMSK. agx_fast_link removes
-       * the epilog's gate. */
+       * input protocol is not restarted after MOVMSK. This is also the
+       * native G17G atomic_color path: acquire 512 before the atomic and use
+       * an unmarked first RHW input. agx_fast_link removes the epilog's gate. */
       uint8_t entry[AGX3_PIXEL_MAX_LENGTH], entry_len;
       if (!agx3_pixel_colour_entry(enc, entry, &entry_len) ||
           !agx3_append(emission, entry, entry_len))
-         goto fail;
+         AGX3_PACK_FAIL();
       ctx->agx3_colour_link.entry_emitted = true;
       ctx->agx3_colour_link.entry_offset = emission->size - entry_len - initial_size;
       g17p_colour_open = true;
@@ -9173,11 +10081,76 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
       g17p_mrt_acquired = true;
    }
 
+   /* Native G17G fragments enter their pixel transaction once, in uniform
+    * control flow ahead of the first kill (native G17G compiler discard and
+    * alpha-to-coverage fragments: pixwait 512, then each kill in its own
+    * cndst). When the first coverage update is nested, open the entry before
+    * the top-level IF that leads to it, so no path skips it; agx_fast_link
+    * relies on this part's entry having run when it removes a later part's.
+    */
+   const agx_instr *m5_coverage_entry_before = NULL;
+   if (enc == AGX_ISA_ENCODING_AGX3C && ctx->stage == MESA_SHADER_FRAGMENT &&
+       has_coverage_write) {
+      int depth = 0;
+      const agx_instr *top_if = NULL;
+      bool done = false;
+      agx_foreach_block(ctx, scan_block) {
+         agx_foreach_instr_in_block(scan_block, scan_I) {
+            if (done)
+               break;
+            if (scan_I->op == AGX_OPCODE_IF_ICMP ||
+                scan_I->op == AGX_OPCODE_IF_FCMP) {
+               if (depth == 0)
+                  top_if = scan_I;
+               depth += scan_I->nest;
+            } else if (scan_I->op == AGX_OPCODE_POP_EXEC) {
+               depth = MAX2(depth - (int)scan_I->nest, 0);
+            } else if (scan_I->op == AGX_OPCODE_WHILE_ICMP ||
+                       scan_I->op == AGX_OPCODE_WHILE_FCMP ||
+                       scan_I->op == AGX_OPCODE_BREAK_IF_ICMP ||
+                       scan_I->op == AGX_OPCODE_BREAK_IF_FCMP) {
+               done = true; /* loops keep the lazy entry */
+            } else if (scan_I->op == AGX_OPCODE_SAMPLE_MASK_AGX3 ||
+                       scan_I->op == AGX_OPCODE_ZS_EMIT_AGX3) {
+               if (depth > 0)
+                  m5_coverage_entry_before = top_if;
+               done = true;
+            }
+         }
+      }
+   }
+
+   /* Native G17G side-effecting fragments read their initial coefficients
+    * without the implicit entry marker, then acquire before the conditional
+    * side effect. Preserve the coverage read preceding that acquisition. */
+   const bool m5_sidefx_entry = enc == AGX_ISA_ENCODING_AGX3C &&
+      ctx->stage == MESA_SHADER_FRAGMENT && has_memory_write &&
+      !ctx->key->secondary && !ctx->key->fs.ignore_tib_dependencies;
+   fragment_input_started |= m5_sidefx_entry;
+
    agx_foreach_block(ctx, block) {
       block->offset = emission->size;
       block->loop_exit_offset = -1;
       bool after_loop_backedge = false;
       bool after_loop_exit_test = false;
+      struct agx3_fetch_group fetch_group = {0};
+      /* An M5 fragment gradient sample stages at fixed banks from 0 (its
+       * coordinates, then the interleaved derivatives up to
+       * AGX3_TEX_GRADIENT_BANK_DDY_Y or _DDY_Z). Samples placed ahead of it
+       * in the same block take free banks above that range first, so it need
+       * not wait for them: Apple's RGSS terrain fragment puts its eight
+       * explicit-LOD samples at banks 16..79 and the gradient sample at 0. */
+      unsigned fixed_gradient_staging = 0;
+      if (enc == AGX_ISA_ENCODING_AGX3C && ctx->stage == MESA_SHADER_FRAGMENT) {
+         agx_foreach_instr_in_block(block, J) {
+            if (J->op == AGX_OPCODE_TEXTURE_SAMPLE && !agx_texture_resource(J) &&
+                J->lod_mode == AGX_LOD_MODE_LOD_GRAD) {
+               fixed_gradient_staging = MAX2(fixed_gradient_staging,
+                  (J->dim == AGX_DIM_3D ? AGX3_TEX_GRADIENT_BANK_DDY_Z
+                                        : AGX3_TEX_GRADIENT_BANK_DDY_Y) + 2);
+            }
+         }
+      }
       agx_foreach_instr_in_block(block, I) {
          block->last_offset = emission->size;
 
@@ -9210,11 +10183,84 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          if (slot)
             outstanding |= (uint16_t)(1u << (slot - 1));
 
+         if (I->op == AGX_OPCODE_MATMAD16_AGX3) {
+            if (enc != AGX_ISA_ENCODING_AGX3C || ctx->is_preamble ||
+                ctx->stage != MESA_SHADER_COMPUTE || I->nr_dests != 1 ||
+                I->nr_srcs != 3 || I->saturate || wait > 0xff ||
+                !agx3_src_is_register(I->dest[0]) || I->dest[0].memory ||
+                I->dest[0].size != AGX_SIZE_32 ||
+                agx_channels(I->dest[0]) != 8 ||
+                I->dest[0].value != I->src[2].value)
+               AGX3_REFUSE(I);
+            for (unsigned s = 0; s < 3; ++s) {
+               const agx_index src = I->src[s];
+               if (!agx3_src_is_register(src) || src.memory || src.abs ||
+                   src.neg || agx_channels(src) != 8 ||
+                   src.size != (s == 2 ? AGX_SIZE_32 : AGX_SIZE_16))
+                  AGX3_REFUSE(I);
+            }
+            struct agx3_matrix_gemm16_desc d = {
+               .dest = I->dest[0].value,
+               .a = I->src[0].value,
+               .b = I->src[1].value,
+               .input_type = I->matrix_type & 15,
+               .b_input_type = (I->matrix_type >> 4) & 15,
+               .wait_mask = wait,
+               .preserve_a = !I->src[0].discard,
+               .preserve_b = !I->src[1].discard,
+            };
+            uint8_t bytes[10], length;
+            if (!agx3_matrix_encode_gemm16(enc, &d, bytes, &length) ||
+                !agx3_append(emission, bytes, length))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         if (I->op == AGX_OPCODE_MATMAD8_AGX3) {
+            if ((I->matrix_type & 15) != ((I->matrix_type >> 4) & 15))
+               AGX3_REFUSE(I);
+            if (enc != AGX_ISA_ENCODING_AGX3C || ctx->is_preamble ||
+                ctx->stage != MESA_SHADER_COMPUTE || I->nr_dests != 1 ||
+                I->nr_srcs != 3 || I->saturate || wait > 0xff ||
+                !agx3_src_is_register(I->dest[0]) || I->dest[0].memory ||
+                I->dest[0].size != AGX_SIZE_32 ||
+                agx_channels(I->dest[0]) != 2)
+               AGX3_REFUSE(I);
+
+            struct agx3_matrix_mma8_desc d = {
+               .dest = I->dest[0].value,
+               .a = I->src[0].value,
+               .b = I->src[1].value,
+               .c = I->src[2].value,
+               .dest_cache = I->dest[0].cache,
+               .wait_mask = wait,
+               .input_type = I->matrix_type & 15,
+            };
+            for (unsigned s = 0; s < 3; ++s) {
+               const agx_index src = I->src[s];
+               if (!agx3_src_is_register(src) || src.memory || src.abs ||
+                   src.neg || agx_channels(src) != 2 ||
+                   src.size != (s == 2 || (I->matrix_type & 15) == AGX3_MATRIX_INPUT_F32
+                                  ? AGX_SIZE_32 : AGX_SIZE_16))
+                  AGX3_REFUSE(I);
+               d.cache[s] = src.cache;
+               d.discard[s] = src.discard;
+            }
+            uint8_t bytes[14], length;
+            if (!agx3_matrix_encode_mma8(enc, &d, bytes, &length) ||
+                !agx3_append(emission, bytes, length))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* Give every control-flow boundary an empty scoreboard, rather
           * than refusing the shader when something is still in flight. */
          uint16_t cf_wait = wait;
          uint16_t cf_outstanding = outstanding_here;
-         if (agx3_is_cf_op(I)) {
+         const bool live_begin = enc == AGX_ISA_ENCODING_AGX3C &&
+            I->op == AGX_OPCODE_BEGIN_CF &&
+            debug_get_option_m5_begin_cf_live();
+         if (agx3_is_cf_op(I) && !live_begin) {
             const uint16_t retire = (uint16_t)(cf_wait | cf_outstanding);
             if (retire != 0) {
                if (!agx3_retire_slots_noop(emission, enc, retire))
@@ -9225,6 +10271,28 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
+         if (m5_sidefx_entry && !g17p_colour_open &&
+             (I->op == AGX_OPCODE_IF_ICMP || I->op == AGX_OPCODE_IF_FCMP ||
+              I->op == AGX_OPCODE_ATOMIC || I->op == AGX_OPCODE_DEVICE_STORE ||
+              I->op == AGX_OPCODE_IMAGE_WRITE)) {
+            uint8_t entry[AGX3_PIXEL_MAX_LENGTH], entry_len;
+            if (!agx3_pixel_colour_entry(enc, entry, &entry_len) ||
+                !agx3_append(emission, entry, entry_len))
+               AGX3_REFUSE(I);
+            ctx->agx3_colour_link.entry_emitted = true;
+            ctx->agx3_colour_link.entry_offset = emission->size - entry_len - initial_size;
+            g17p_colour_open = true;
+         }
+         if (I == m5_coverage_entry_before && !g17p_colour_open) {
+            uint8_t entry[AGX3_PIXEL_MAX_LENGTH], entry_len;
+            if (!agx3_pixel_colour_entry(enc, entry, &entry_len) ||
+                !agx3_append(emission, entry, entry_len))
+               AGX3_REFUSE(I);
+            ctx->agx3_colour_link.entry_emitted = true;
+            ctx->agx3_colour_link.entry_offset = emission->size - entry_len - initial_size;
+            g17p_colour_open = true;
+         }
+
          const bool loop_entry_push = agx3_is_loop_entry_push(I);
          const bool loop_pop =
             I->op == AGX_OPCODE_POP_EXEC && after_loop_backedge;
@@ -9274,6 +10342,25 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          after_loop_backedge = false;
          after_loop_exit_test = false;
 
+         if (I->op == AGX_OPCODE_PACK_SAMPLER_LOD_AGX3) {
+            if (enc != AGX_ISA_ENCODING_AGX3C || I->nr_dests != 1 ||
+                I->nr_srcs != 1 || I->saturate ||
+                !agx3_tex_plain_reg(I->dest[0]) ||
+                I->dest[0].size != AGX_SIZE_32 ||
+                agx_channels(I->dest[0]) != 1 ||
+                !agx3_tex_plain_reg(I->src[0]) ||
+                I->src[0].size != AGX_SIZE_16 ||
+                agx_channels(I->src[0]) != 1)
+               AGX3_REFUSE(I);
+            uint8_t bytes[AGX3_TEX_MAX_LENGTH], len;
+            const bool encoded = agx3_tex_pack_float_lod_m5(
+               I->src[0].value, I->dest[0].value,
+               I->src[0].cache, I->src[0].discard, wait, bytes, &len);
+            if (!encoded || !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          if (I->op == AGX_OPCODE_UNIFORM_FADD_AGX3 ||
              I->op == AGX_OPCODE_UNIFORM_FMUL_AGX3 ||
              I->op == AGX_OPCODE_UNIFORM_FFMA_AGX3) {
@@ -9347,7 +10434,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                continue;
             }
             const bool scalar = I->op == AGX_OPCODE_PUBLISH_TEXTURE_SCALAR_AGX3;
-            if (enc != AGX_ISA_ENCODING_AGX3B || !slot ||
+            const bool m5_scalar = enc == AGX_ISA_ENCODING_AGX3C && scalar &&
+               I->nr_srcs == 3 && !agx_is_null(I->src[2]) &&
+               debug_get_option_m5_scalar_3d_coords();
+            if ((!m5_scalar && enc != AGX_ISA_ENCODING_AGX3B) || !slot ||
                 I->nr_srcs != (scalar ? 3 : 2) ||
                 !I->imm || I->imm > resource_capacity ||
                 !publication_values[I->imm])
@@ -9374,11 +10464,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             if (retire) {
                /* The mask covers this publication's source producers. An
                 * unrelated r0 generation can still be pending, so carry
-                * the wait on the first coordinate instead of modifying r0. */
-               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
-               if (!agx3_alu_band_identity_wait(enc, I->src[0].value, retire,
-                                               bytes, &len) ||
-                   !agx3_append(emission, bytes, len))
+                * the wait on the first coordinate instead of modifying r0.
+                * A wide M5 coordinate may live above half 192, which only
+                * the halfword identity reaches. */
+               if (!agx3_retire_wait_on(emission, enc, I->src[0], retire))
                   AGX3_REFUSE(I);
             }
             outstanding &= (uint16_t)~retire;
@@ -9400,10 +10489,17 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                      next_src.value + (!scalar && later == 1 ? 2 : 0);
                   last &= next_reg != reg;
                }
-               if (!agx3_tex_stage_copy_hints(enc, base + 2 * c, reg, slot, 0,
-                                             src.cache, src.discard && last,
-                                             bytes, &len) ||
-                   !agx3_append(emission, bytes, len))
+               const bool wide_scalar = m5_scalar &&
+                  debug_get_option_m5_wide_scalar_sources();
+               const bool copied = wide_scalar
+                  ? agx3_tex_stage_copy_scalar_m5(
+                       base + 2 * c, reg, slot, 0, !(src.discard && last),
+                       src.discard && last, bytes, &len)
+                  : agx3_tex_stage_copy_hints(
+                       enc, base + 2 * c, reg, slot, 0,
+                       m5_scalar ? !(src.discard && last) : src.cache,
+                       src.discard && last, bytes, &len);
+               if (!copied || !agx3_append(emission, bytes, len))
                   AGX3_REFUSE(I);
             }
             if (!agx3_staging_hold(&staging_banks, base, span, slot))
@@ -9532,8 +10628,14 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          if (I->op == AGX_OPCODE_ITRCOORDS_AGX3) {
             if (ctx->stage != MESA_SHADER_FRAGMENT ||
-                !agx3_emit_itrcoords(emission, enc, I, wait))
-               AGX3_REFUSE(I);
+                !agx3_emit_itrcoords(emission, enc, I, wait)) {
+               /* The operand list cannot show the mode or the scoreboard
+                * slots waited on, which are what the encoder gates on. */
+               char why[48];
+               snprintf(why, sizeof(why), "interpolation %u, wait 0x%x",
+                        (unsigned)I->interpolation, (unsigned)wait);
+               AGX3_REFUSE_WHY(I, why);
+            }
             continue;
          }
 
@@ -9543,7 +10645,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 !agx3_sample_coords_use(block, I))
                AGX3_REFUSE(I);
             if (I->op == AGX_OPCODE_RHW_INTERP_AGX3 &&
-                (enc != AGX_ISA_ENCODING_AGX3B ||
+                ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
                  ctx->stage != MESA_SHADER_FRAGMENT ||
                  !agx3_private_rhw_use(ctx, block, I)))
                AGX3_REFUSE(I);
@@ -9577,7 +10679,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                outstanding &= (uint16_t)~interp_wait;
                interp_wait = 0;
             }
-            const bool first = enc == AGX_ISA_ENCODING_AGX3B &&
+            const bool first = (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
                ctx->stage == MESA_SHADER_FRAGMENT && !fragment_input_started &&
                !g17p_fragment_helper_protocol;
             if (!agx3_emit_interp(emission, enc, I, interp_wait, first,
@@ -9636,8 +10738,18 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                .wait_mask = wait,
             };
             uint8_t bytes[AGX3_FLTNORM_MAX_LENGTH], len = 0;
-            if (!agx3_persp_mul_encode(enc, &desc, bytes, &len) ||
-                !agx3_append(emission, bytes, len))
+            bool ok = agx3_persp_mul_encode(enc, &desc, bytes, &len);
+            if (!ok && wait) {
+               /* The f32mul forms carry waits only up to slot 6, but AGX3C
+                * allocates slots 7 and 8 too. Carry such a wait on an
+                * identity of the destination first, as for other producers
+                * without a wait field for it, then multiply without one. */
+               struct agx3_persp_mul_desc unwaited = desc;
+               unwaited.wait_mask = 0;
+               ok = agx3_retire_result_wait(emission, enc, I, wait) &&
+                    agx3_persp_mul_encode(enc, &unwaited, bytes, &len);
+            }
+            if (!ok || !agx3_append(emission, bytes, len))
                AGX3_REFUSE(I);
             continue;
          }
@@ -9646,7 +10758,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const bool first = enc == AGX_ISA_ENCODING_AGX3B &&
                ctx->stage == MESA_SHADER_FRAGMENT && !fragment_input_started &&
                !g17p_fragment_helper_protocol;
-            if (!agx3_emit_ldcf(emission, enc, I, slot, wait, first))
+            if (wait && !agx3_retire_slots_noop(emission, enc, wait))
+               AGX3_REFUSE(I);
+            if (!agx3_emit_ldcf(emission, enc, I, slot, 0, first))
                AGX3_REFUSE(I);
             fragment_input_started = true;
             continue;
@@ -9677,6 +10791,45 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                stage_wait = wait | (g17p_user_output_protocol ? outstanding_here :
                                      outstanding_here & staging_bit);
             }
+            /* UVSW reuses the same staging file as asynchronous image
+             * operations. Retire consumers of these two halves before the
+             * coordinate/data COPY, even if they signal another slot. */
+            agx3_staging_release(&staging_banks, outstanding_here);
+            const uint16_t staging_conflicts =
+               agx3_staging_conflicts(&staging_banks, staging_half, 2);
+            if (staging_conflicts) {
+               /* The first COPY's own wait retires the conflicts it already
+                * waits on; only the others need a carrier before it. */
+               const uint16_t carried = staging_conflicts & (uint16_t)~stage_wait;
+               /* The r0 carrier reads and rewrites r0: sound only when r0
+                * is ready or its producer is in the mask. In M5 vertex
+                * shaders of texturelodoffset_pcoffset *shadow_vertex the
+                * position load into r0 was outstanding when the first
+                * output's COPY needed the sample's operand staging retired,
+                * and the carrier waited for the sample alone (misplaced
+                * vertices, intermittently, under concurrent load). Carry the
+                * conflict on the stored register with this store's producer
+                * wait, which covers every pending write of it (the COPY
+                * waits for those producers anyway). */
+               agx_index carrier = agx_null();
+               if (enc == AGX_ISA_ENCODING_AGX3C) {
+                  for (unsigned s = 1; s < I->nr_srcs; ++s) {
+                     if (agx3_src_is_register(I->src[s]) &&
+                         !I->src[s].memory && I->src[s].value < 256) {
+                        carrier = I->src[s];
+                        break;
+                     }
+                  }
+               }
+               if (carried &&
+                   !(carrier.type == AGX_INDEX_REGISTER
+                        ? agx3_retire_wait_on(emission, enc, carrier,
+                                              carried | wait)
+                        : agx3_retire_slots_noop(emission, enc, carried)))
+                  AGX3_REFUSE(I);
+               outstanding &= (uint16_t)~carried;
+               agx3_staging_forget(&staging_banks, staging_conflicts);
+            }
             if (!agx3_emit_uvsw(emission, enc, I, staging_half, stage_slot,
                                 stage_wait, g17p_position_protocol ||
                                             g17p_user_output_protocol))
@@ -9747,13 +10900,16 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             /* Apple's argument-buffer vertex read publishes uint2 coordinates
              * before SMP.EP4. G17P indexed imageLoad uses texelFetch's
              * read-only texture heap and coordinate ABI. Legacy immediate
-             * IMAGE_LOAD reads keep the unstaged form; a vertex texel fetch
-             * always stages (the unstaged AGX3B form returns only x and y). */
+             * IMAGE_LOAD reads keep the unstaged form on AGX3B. M5 vertex
+             * image reads also require coordinate publication; without it a
+             * storage-texel read followed by a write stores incorrect values. */
             const bool vertex_fetch_stages_coordinates =
-               enc == AGX_ISA_ENCODING_AGX3B &&
+               (enc == AGX_ISA_ENCODING_AGX3B ||
+                enc == AGX_ISA_ENCODING_AGX3C) &&
                ctx->stage == MESA_SHADER_VERTEX &&
                (I->op == AGX_OPCODE_TEXTURE_LOAD ||
-                I->op == AGX_OPCODE_IMAGE_LOAD);
+                I->op == AGX_OPCODE_IMAGE_LOAD ||
+                enc == AGX_ISA_ENCODING_AGX3C);
             const bool staged =
                (enc == AGX_ISA_ENCODING_AGX3B &&
                 I->dim == AGX_DIM_2D_MS_ARRAY) ||
@@ -9763,6 +10919,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 ctx->stage == MESA_SHADER_VERTEX &&
                 I->src[3].type != AGX_INDEX_IMMEDIATE) ||
                ctx->stage == MESA_SHADER_FRAGMENT ||
+               (enc == AGX_ISA_ENCODING_AGX3C &&
+                (ctx->stage == MESA_SHADER_MESH ||
+                 ctx->stage == MESA_SHADER_TASK)) ||
                agx3_stage_runs_as_kernel(ctx->stage);
             /* imgwci takes its coordinates from staging halves 0/2, so in a
              * shader that also writes images a read stages its coordinates at
@@ -9792,8 +10951,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             assert(texture_ops_left > 0);
             const bool final = --texture_ops_left == 0;
             const uint16_t fetch_wait =
-               wait | ((I->dim == AGX_DIM_2D_MS ||
-                         I->dim == AGX_DIM_2D_MS_ARRAY)
+               wait | ((enc == AGX_ISA_ENCODING_AGX3B &&
+                         (I->dim == AGX_DIM_2D_MS ||
+                          I->dim == AGX_DIM_2D_MS_ARRAY))
                           ? (outstanding_here & 1u)
                           : 0);
             /* A fragment texel fetch with a per-lane level stages that level
@@ -9807,15 +10967,26 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                ctx->agx3_colour_link.straight_line &&
                register_fetch_lod && I->dim == AGX_DIM_2D && !g17p_colour_open;
             uint16_t staged_end = 0;
+            const bool share_fetch_coords =
+               enc == AGX_ISA_ENCODING_AGX3C &&
+               ctx->stage == MESA_SHADER_COMPUTE && staged &&
+               debug_get_bool_option("ASAHI_M5_FETCH_REUSE", true);
+            const agx_instr *next = k < count ? ordered[k] : NULL;
+            fetch_group.reuse = share_fetch_coords && fetch_group.keep;
+            fetch_group.keep = share_fetch_coords && next &&
+               I->link.next == &next->link &&
+               agx3_fetch_coordinates_match(I, next);
             if (!agx3_emit_texture_fetch(emission, enc, I, slot, fetch_wait,
                                          final, staged, separate_write_coords,
                                          ms_array_scratch,
                                          staged_bank_reads ? &staging_banks
                                                            : NULL,
                                          outstanding_here, &staged_end,
+                                         ctx->stage == MESA_SHADER_FRAGMENT,
                                          open_pixel_fetch,
                                          &ctx->agx3_colour_link.entry_offset,
-                                         ctx->key->g17p_texture_uniform_base))
+                                         ctx->key->g17p_texture_uniform_base,
+                                         share_fetch_coords ? &fetch_group : NULL))
                AGX3_REFUSE(I);
             if (open_pixel_fetch) {
                ctx->agx3_colour_link.entry_emitted = true;
@@ -9907,7 +11078,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * transaction right before their first SMP, after coefficient
              * interpolation: entering later lets a scissor edge retire
              * uncovered lanes before SMP forms its derivative quad.
-             * Explicit-LOD and gradient samples are exempt. The entry is
+             * Neo explicit-LOD and gradient samples are exempt. G17G's
+             * explicit-level witnesses acquire at the same point too. The entry is
              * published in the link metadata so agx_fast_link drops the split
              * epilog's duplicate gate; it is never placed inside one
              * control-flow arm. */
@@ -9917,11 +11089,13 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS_MIN;
             const bool native_staged_coords = I->agx3_tex_coords_staged;
             const bool open_pixel =
-               enc == AGX_ISA_ENCODING_AGX3B &&
+               (enc == AGX_ISA_ENCODING_AGX3B ||
+                enc == AGX_ISA_ENCODING_AGX3C) &&
                ctx->stage == MESA_SHADER_FRAGMENT &&
                !ctx->key->fs.ignore_tib_dependencies &&
                ctx->agx3_colour_link.straight_line &&
-               implicit_derivatives && !g17p_colour_open;
+               (implicit_derivatives || enc == AGX_ISA_ENCODING_AGX3C) &&
+               !g17p_colour_open;
             /* Implicit LOD needs the 2x2 quad's derivatives, which exist
              * only in fragment shaders. Compute and vertex explicit-level
              * samples encode like fragment ones and stage a runtime level the
@@ -9948,21 +11122,37 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             /* Relocation is an operand-ABI property. Other forms retain
              * their fixed operand locations and wait only for overlapping
              * live ranges before publishing into those locations. */
+            const bool m5_fragment_3d =
+               ctx->stage == MESA_SHADER_FRAGMENT &&
+               I->lod_mode == AGX_LOD_MODE_AUTO_LOD &&
+               debug_get_option_m5_fs_3d_banks();
+            const bool m5_banked_3d =
+               enc == AGX_ISA_ENCODING_AGX3C && I->dim == AGX_DIM_3D &&
+               !I->shadow && !I->offset && !I->query_lod &&
+               (((ctx->stage == MESA_SHADER_VERTEX ||
+                  ctx->stage == MESA_SHADER_COMPUTE) &&
+                 ((I->lod_mode == AGX_LOD_MODE_LOD_MIN &&
+                   I->src[1].type == AGX_INDEX_IMMEDIATE) ||
+                  I->lod_mode == AGX_LOD_MODE_LOD_GRAD)) ||
+                m5_fragment_3d);
             const bool relocatable_coordinates =
-               enc == AGX_ISA_ENCODING_AGX3B && slot != 0 &&
+               (enc == AGX_ISA_ENCODING_AGX3B ||
+                enc == AGX_ISA_ENCODING_AGX3C) && slot != 0 &&
                !native_staged_coords &&
-               (I->dim == AGX_DIM_2D ||
-                ((I->dim == AGX_DIM_3D ||
+               (m5_banked_3d || I->dim == AGX_DIM_2D ||
+                (enc == AGX_ISA_ENCODING_AGX3B &&
+                 (I->dim == AGX_DIM_3D ||
                   (I->dim == AGX_DIM_2D_ARRAY && !I->shadow && !I->offset)) &&
                  ctx->stage == MESA_SHADER_VERTEX &&
                  ctx->key->g17p_uniform_dma &&
                  I->lod_mode == AGX_LOD_MODE_LOD_MIN &&
                  I->src[1].type == AGX_INDEX_IMMEDIATE)) &&
                !I->query_lod && !(I->offset && !I->offset_xyz) &&
-               (I->lod_mode == AGX_LOD_MODE_AUTO_LOD ||
+               (m5_banked_3d || I->lod_mode == AGX_LOD_MODE_AUTO_LOD ||
                 ((I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS ||
                   I->lod_mode == AGX_LOD_MODE_LOD_MIN) &&
-                 I->src[1].type == AGX_INDEX_IMMEDIATE));
+                 (I->src[1].type == AGX_INDEX_IMMEDIATE ||
+                  enc == AGX_ISA_ENCODING_AGX3C)));
             const bool staged_scalar =
                (I->lod_mode == AGX_LOD_MODE_LOD_MIN &&
                 I->src[1].type != AGX_INDEX_IMMEDIATE) ||
@@ -9978,6 +11168,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                published ? agx_index_size_16(I->src[0]) :
                     ((I->dim == AGX_DIM_CUBE ||
                      I->dim == AGX_DIM_CUBE_ARRAY) ? 8
+                    : enc == AGX_ISA_ENCODING_AGX3C &&
+                      I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS_MIN &&
+                      ((I->dim == AGX_DIM_2D_ARRAY && I->shadow) ||
+                       (I->dim == AGX_DIM_3D && staged_sample_offset))
+                         ? 10
                     : native_staged_coords ? 4
                     : vertex_explicit_lod &&
                        (I->lod_mode != AGX_LOD_MODE_LOD_MIN ||
@@ -9993,10 +11188,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                        I->dim == AGX_DIM_3D)
                        ? 8
                     : staged_compare ? 6
-                    : enc == AGX_ISA_ENCODING_AGX3B ? 4
-                                                   : I->src[0].value + 4);
+                    : 4);
             uint16_t bank_base = published ? resource_bases[published_resource] : 0;
-            if (enc == AGX_ISA_ENCODING_AGX3B && !published) {
+            if ((enc == AGX_ISA_ENCODING_AGX3B ||
+                 enc == AGX_ISA_ENCODING_AGX3C) && !published) {
                /* Texel fetches and filtered samples publish into the same
                 * sampler-private file. A sample must not overwrite a bank
                 * still consumed by a fetch (and vice versa). */
@@ -10005,6 +11200,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   assert(staged_sample);
                   if (!agx3_allocate_staging(emission, enc, I, &staging_banks,
                                              &outstanding, slot, sample_span,
+                                             fixed_gradient_staging,
                                              &bank_base))
                      AGX3_REFUSE(I);
                } else {
@@ -10057,7 +11253,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   I->src[3].type == AGX_INDEX_REGISTER ||
                   I->src[4].type == AGX_INDEX_REGISTER;
                const uint16_t native_wait =
-                  register_operands ? (sample_wait & (1u << 5)) : 0;
+                  register_operands && enc == AGX_ISA_ENCODING_AGX3B
+                     ? (sample_wait & (1u << 5)) : 0;
                const uint16_t retire = sample_wait & ~native_wait;
                if (retire && !agx3_retire_slots_noop(emission, enc, retire))
                   AGX3_REFUSE(I);
@@ -10089,7 +11286,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
             if ((ctx->stage != MESA_SHADER_FRAGMENT && !explicit_level &&
                  !compute_gradient && !vertex_explicit_lod &&
-                 !(enc == AGX_ISA_ENCODING_AGX3B &&
+                 !((enc == AGX_ISA_ENCODING_AGX3B ||
+                    enc == AGX_ISA_ENCODING_AGX3C) &&
                    I->gather != AGX_GATHER_NONE)) ||
                 /* Keep the pixel-entry gate after all native coordinate
                  * staging and immediately before SMP, matching Apple. */
@@ -10098,7 +11296,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                                           sample_group_end, open_pixel,
                                           &ctx->agx3_colour_link.entry_offset,
                                           native_coord_slot, cem_scratch,
-                                          ctx->stage == MESA_SHADER_VERTEX,
+                                          vertex_explicit_lod,
                                           ctx->stage == MESA_SHADER_COMPUTE,
                                           ms_array_scratch, bank_base,
                                           published && resource_last_use[
@@ -10123,7 +11321,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * placement and scoreboard slot. */
             ctx->agx3_staging_halfs =
                MAX2(ctx->agx3_staging_halfs, bank_base + sample_span);
-            if (enc == AGX_ISA_ENCODING_AGX3B && slot) {
+            if ((enc == AGX_ISA_ENCODING_AGX3B ||
+                 enc == AGX_ISA_ENCODING_AGX3C) && slot) {
                agx3_staging_release(&staging_banks, outstanding);
                if (!agx3_staging_hold(&staging_banks, bank_base,
                                       sample_span,
@@ -10144,7 +11343,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * pass-through Z value before acquiring class 1 in Apple's fresh
              * G17P atomic fragment. Hold this wait until ZS_EMIT can reproduce
              * that ordering; SIGNAL_PIX remains in its original position. */
-            if (has_zs_emit && I->op == AGX_OPCODE_WAIT_PIX &&
+            /* Native M5 covered-sample loops (zs_depth_4x and kin) take
+             * class 1 once, ahead of the loop and its staging, instead. */
+            if (has_zs_emit && !zs_sample_loop && I->op == AGX_OPCODE_WAIT_PIX &&
                 I->writeout == 1) {
                if (interleaved_zs && g17p_zs_writes > 0) {
                   if (!agx3_retire_slots_noop(emission, enc, wait))
@@ -10224,9 +11425,21 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                ctx->agx3_colour_link.entry_offset = emission->size - entry_len - initial_size;
                g17p_colour_open = true;
             }
+            /* The class-1 release closing a covered-sample loop, recorded
+             * for agx_fast_link when it is the plain form. */
+            const bool zs_loop_release =
+               zs_sample_loop && I->op == AGX_OPCODE_SIGNAL_PIX &&
+               I->writeout == 1 && wait == 0 && g17p_zs_writes > 0 &&
+               I != g17p_final_release && !ctx->agx3_colour_link.zs_loop_release;
+            const size_t release_at = emission->size;
             if (!agx3_emit_pixel(emission, enc, I, wait,
                                  I == g17p_final_release, g17p_lone_pair, &ctx->agx3_colour_link, initial_size))
                AGX3_REFUSE(I);
+            if (zs_loop_release && emission->size - release_at == AGX3_PIXEL_MAX_LENGTH) {
+               ctx->agx3_colour_link.zs_loop_release = true;
+               ctx->agx3_colour_link.zs_loop_release_offset =
+                  release_at - initial_size;
+            }
             continue;
          }
 
@@ -10249,8 +11462,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const bool first = ctx->stage == MESA_SHADER_FRAGMENT &&
                !ctx->key->fs.ignore_tib_dependencies &&
                emission->size == initial_size;
-            if (enc != AGX_ISA_ENCODING_AGX3B ||
-                !agx3_emit_rgb10a2(emission, I, wait, first))
+            if ((enc != AGX_ISA_ENCODING_AGX3B &&
+                 enc != AGX_ISA_ENCODING_AGX3C) ||
+                !agx3_emit_rgb10a2(emission, enc, I, wait, first))
                AGX3_REFUSE(I);
             ctx->agx3_fltnorm_start |= first &&
                I->op == AGX_OPCODE_PACK_RGB10A2_AGX3;
@@ -10319,7 +11533,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                       /* Native per-sample exports use halves
                        * 0..31 even for one/two targets. */
                       true,
-                      ctx->key->g17p_emrt && !load && I->src[0].value >= 16) ||
+                      (enc == AGX_ISA_ENCODING_AGX3C && !load &&
+                       (I->src[0].kill || I->src[0].discard)) ||
+                         (enc != AGX_ISA_ENCODING_AGX3C &&
+                          ctx->key->g17p_emrt && !load &&
+                          I->src[0].value >= 16)) ||
                    (!load && last &&
                     (!agx3_pixel_release(
                         enc, interleaved_zs ? 0 : g17p_mrt.combined_classes,
@@ -10371,7 +11589,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 * reuses (it reads back as zero on G17P); the hint stays on
                 * unique payloads. */
                const bool store_f13 =
-                  !load && g17p_mrt_stores_seen++ < g17p_mrt.f13_stores &&
+                  !load &&
+                  (enc == AGX_ISA_ENCODING_AGX3C ||
+                   g17p_mrt_stores_seen++ < g17p_mrt.f13_stores) &&
                   (I->src[0].kill || I->src[0].discard);
                /* A multisample store keeps its own coverage dependency. */
                const bool store_waits = !acquire_class || g17p_mrt.multisample;
@@ -10418,15 +11638,20 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                                     false, g17p_lone_pair, &ctx->agx3_colour_link, initial_size))
                   AGX3_REFUSE(I);
             }
-            /* Apple's per-sample RGBA32 fragments store from halves 24 and
-             * 16 with f13 clear, as the multi-target backgrounds do. */
+            /* Native C multisample stores retain a reused payload with
+             * f13=0 and release it only at its last use. B's raw32 stores
+             * retain their own policy. */
+            const bool store_f13 = enc == AGX_ISA_ENCODING_AGX3C &&
+               ((!g17p_multi_background &&
+                 (!g17p_sample_colour_stores || ctx->key->fs.nr_samples <= 1)) ||
+                I->src[0].kill || I->src[0].discard);
             if (!agx3_emit_tile(emission, enc, I, slot, wait, first, last,
                                 g17p_multi_background ||
                                    g17p_sample_colour_stores,
                                 (g17p_multi_background ||
                                  g17p_msaa_background_targets > 1) &&
                                    ctx->key->g17p_mrt_bg,
-                                false))
+                                store_f13))
                AGX3_REFUSE(I);
 
             if (g17p_pixel_protocol && I->op == AGX_OPCODE_ST_TILE && last) {
@@ -10490,14 +11715,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 * 24 and 26 -- consecutive 32-bit slots two halves apart.
                 */
                bool ok = true;
-               if (wait && enc != AGX_ISA_ENCODING_AGX3B) {
-                  ok = agx3_retire_slots_noop(emission, enc, wait);
-               } else if (wait) {
-                  /* This is a partial wait. r0 may still be the destination
-                   * of an unrelated load (for example a UBO sample index).
-                   * Carry the wait on the value being published, whose
-                   * producer is covered by the mask, so retiring another
-                   * uniform cannot overwrite that pending r0 generation. */
+               if (wait) {
+                  /* A different load may still be producing r0. Carry a
+                   * partial wait on the value being published, whose producer
+                   * is covered by the mask, on both AGX3 encodings. */
                   uint8_t carrier[AGX3_ALU_MAX_LENGTH], carrier_len = 0;
                   const bool retired = scalar16
                      ? agx3_alu_bit_imm(enc, AGX3_ALU_BOR, true, value.value,
@@ -10532,7 +11753,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             /* SR2 is the one-sample coverage word. A multisample depth
              * export writes each sample's depth separately in Apple's
              * fragments, which this single update does not do. */
-            if (enc != AGX_ISA_ENCODING_AGX3B || ctx->stage != MESA_SHADER_FRAGMENT ||
+            if ((enc != AGX_ISA_ENCODING_AGX3B &&
+                 enc != AGX_ISA_ENCODING_AGX3C) ||
+                ctx->stage != MESA_SHADER_FRAGMENT ||
                 ctx->key->fs.nr_samples > 1 ||
                 I->nr_dests != 1 || I->nr_srcs ||
                 !agx3_src_is_register(I->dest[0]) || I->dest[0].memory ||
@@ -10552,18 +11775,26 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          if (I->op == AGX_OPCODE_ZS_EMIT_AGX3) {
             const bool depth = I->zs & 1, stencil = I->zs & 2;
+            /* One word of a covered-sample loop, its class-1 wait already
+             * emitted before the loop. */
+            const bool sample_loop = I->zs & 4;
             /* Apple's 2x/4x per-sample depth/stencil writes stage each word,
              * and its depth, after the previous sample's, all under the one
              * class-1 wait taken before the first DEPTHF (msaa/native/zexp2
              * fs_id_{2,4}x, msaa/native/zmrt/zsten fs_id_s_4x). */
             const unsigned staging = g17p_zs_staging;
-            if (enc != AGX_ISA_ENCODING_AGX3B ||
+            if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
                 ctx->stage != MESA_SHADER_FRAGMENT || I->nr_dests ||
-                I->nr_srcs != 1 || I->zs < 1 || I->zs > 3 ||
+                I->nr_srcs != 1 || !(I->zs & 3) || I->zs > 7 ||
+                (sample_loop && ((I->zs & 3) == 3 || !zs_sample_loop ||
+                                 enc != AGX_ISA_ENCODING_AGX3C ||
+                                 g17p_deferred_zs_wait)) ||
                 !agx3_vary_reg32(I->src[0]) ||
                 agx_channels(I->src[0]) != (depth ? 2 : 1) ||
-                (g17p_zs_writes == 0) != (g17p_deferred_zs_wait != NULL) ||
-                g17p_zs_writes >= 4)
+                (!sample_loop &&
+                 (g17p_zs_writes == 0) != (g17p_deferred_zs_wait != NULL)) ||
+                g17p_zs_writes >=
+                   (enc == AGX_ISA_ENCODING_AGX3C ? 8 : 4))
                AGX3_REFUSE(I);
 
             uint8_t bytes[AGX3_STAGE_MAX_LENGTH];
@@ -10582,13 +11813,13 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * Apple's stencil-only [[stencil]] fragment publishes its single
              * coverage/stencil dword without it. */
             if (!agx3_stage_copy_liveness(enc, staging, I->src[0].value,
-                                        depth, I->src[0].discard, slot,
+                                        depth && enc == AGX_ISA_ENCODING_AGX3B, I->src[0].discard, slot,
                                         wait | outstanding_here, bytes, &len) ||
                 !agx3_append(emission, bytes, len))
                AGX3_REFUSE(I);
             if (depth &&
                 (!agx3_stage_copy_liveness(enc, staging + 2,
-                                           I->src[0].value + 2, true,
+                                           I->src[0].value + 2, enc == AGX_ISA_ENCODING_AGX3B,
                                            I->src[0].discard, slot, 0,
                                            bytes, &len) ||
                  !agx3_append(emission, bytes, len)))
@@ -10599,9 +11830,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                AGX3_REFUSE(I);
             g17p_deferred_zs_wait = NULL;
 
+            /* The loop restages the word each iteration; native keeps f13
+             * clear on that DEPTHF. */
             const struct agx3_zs_desc zs = {
                .value = staging,
-               .value_kill = true,
+               .value_kill = !sample_loop,
                .slot = slot,
                .wait_mask = (uint16_t)(1u << (slot - 1)),
             };
@@ -10628,24 +11861,36 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                I->sr == AGX_SR_IS_ACTIVE_THREAD &&
                ctx->stage == MESA_SHADER_FRAGMENT;
             const bool sample_coverage =
-               I->sr == AGX_SR_INPUT_SAMPLE_MASK &&
+               (I->sr == AGX_SR_INPUT_SAMPLE_MASK ||
+                (enc == AGX_ISA_ENCODING_AGX3C &&
+                 I->sr == AGX_SR_COVERAGE_MASK)) &&
                ctx->stage == MESA_SHADER_FRAGMENT;
             /* A G17P tessellation-control program is a compute dispatch
              * (hk_launch_tess(); hk_shader.c gives it the compute
              * texture/sampler heaps), so SR34/SR36 read the same subgroup
-             * state as in compute. */
+             * state as in compute. So is the main geometry kernel
+             * (hk_launch_gs_prerast()), which runs one GS invocation per
+             * thread. M5 object (task) and mesh programs read SR34 the same
+             * way: a native G17G first-thread election in an object
+             * shader reads SR34 into a half register. */
             const bool subgroup_stage =
                ctx->stage == MESA_SHADER_COMPUTE ||
                ctx->stage == MESA_SHADER_VERTEX ||
                ctx->stage == MESA_SHADER_TESS_CTRL ||
-               ctx->stage == MESA_SHADER_FRAGMENT;
+               ctx->stage == MESA_SHADER_GEOMETRY ||
+               ctx->stage == MESA_SHADER_FRAGMENT ||
+               (enc == AGX_ISA_ENCODING_AGX3C &&
+                (ctx->stage == MESA_SHADER_MESH ||
+                 ctx->stage == MESA_SHADER_TASK));
             const bool active_index =
                I->sr == AGX_SR_ACTIVE_THREAD_INDEX_IN_SUBGROUP &&
                subgroup_stage;
             const bool active_count =
                I->sr == AGX_SR_TOTAL_ACTIVE_THREADS_IN_SUBGROUP &&
                subgroup_stage;
-            if (enc != AGX_ISA_ENCODING_AGX3B ||
+            if ((enc != AGX_ISA_ENCODING_AGX3B &&
+                 !(enc == AGX_ISA_ENCODING_AGX3C &&
+                   (active_index || active_thread || sample_coverage))) ||
                 !(active_thread || sample_coverage || active_index || active_count) ||
                 I->nr_dests != 1 ||
                 I->nr_srcs != 0 || !agx3_src_is_register(I->dest[0]) ||
@@ -10657,11 +11902,17 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                .sr = sample_coverage ? 39 : active_index ? 34 : active_count ? 36 : 2,
                .slot = slot,
                .wait_mask = wait,
+               .first = enc == AGX_ISA_ENCODING_AGX3C && sample_coverage &&
+                  !ctx->key->fs.ignore_tib_dependencies &&
+                  emission->size == initial_size && !fragment_input_started &&
+                  !g17p_fragment_helper_protocol,
             };
             uint8_t bytes[AGX3_SR_MAX_LENGTH], len = 0;
             if (!agx3_sr_encode(enc, &d, bytes, &len) ||
                 !agx3_append(emission, bytes, len))
                AGX3_REFUSE(I);
+            fragment_input_started |= enc == AGX_ISA_ENCODING_AGX3C &&
+               sample_coverage && !ctx->key->fs.ignore_tib_dependencies;
             continue;
          }
 
@@ -10712,11 +11963,14 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          }
 
          if (I->op == AGX_OPCODE_SAMPLE_MASK_AGX3) {
-            if (enc != AGX_ISA_ENCODING_AGX3B || ctx->stage != MESA_SHADER_FRAGMENT ||
+            if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+                ctx->stage != MESA_SHADER_FRAGMENT ||
                 I->nr_dests || I->nr_srcs != 2 ||
                 !(agx3_src_is_register(I->src[0]) ||
                   (I->src[0].type == AGX_INDEX_IMMEDIATE &&
-                   I->src[0].value == 1)) || I->src[0].memory ||
+                   (I->src[0].value == 1 ||
+                    (enc == AGX_ISA_ENCODING_AGX3C &&
+                     I->src[0].value <= 255)))) || I->src[0].memory ||
                 I->src[0].size != AGX_SIZE_16 || agx_channels(I->src[0]) != 1 ||
                 !agx3_vary_reg32(I->src[1]) || agx_channels(I->src[1]) != 1)
                AGX3_REFUSE(I);
@@ -10775,9 +12029,13 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                AGX3_REFUSE(I);
             /* The coverage word is a fragment input. Apple's programs that
              * open with it put the entry marker on that read. */
-            const bool coverage_input = enc == AGX_ISA_ENCODING_AGX3B &&
-               ctx->stage == MESA_SHADER_FRAGMENT &&
-               I->sr == AGX_SR_INPUT_SAMPLE_MASK;
+            const bool coverage_input = ctx->stage == MESA_SHADER_FRAGMENT &&
+               ((enc == AGX_ISA_ENCODING_AGX3B &&
+                 I->sr == AGX_SR_INPUT_SAMPLE_MASK) ||
+                (enc == AGX_ISA_ENCODING_AGX3C &&
+                 !ctx->key->fs.ignore_tib_dependencies &&
+                 (I->sr == AGX_SR_INPUT_SAMPLE_MASK ||
+                  I->sr == AGX_SR_COVERAGE_MASK)));
             d.first = coverage_input &&
                       emission->size == initial_size &&
                       !fragment_input_started &&
@@ -10796,7 +12054,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             struct agx3_mem_desc d;
             uint8_t bytes[AGX3_MEM_MAX_LENGTH];
             uint8_t len = 0;
-            if (!agx3_translate_mem(I, is_store, slot, wait, &d))
+            if (!agx3_translate_mem(enc, I, is_store, slot, wait, &d))
                AGX3_REFUSE(I);
             if (g17p_vertex_load_policy && !is_store) {
                const bool first = vertex_loads_seen++ == 0;
@@ -10982,6 +12240,121 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (I->op == AGX_OPCODE_ENTER_QUERY_M5) {
+            /* The native entry writes a 32-bit query pointer to its f2
+             * register (r92/r94 in the native witnesses). It must be a real,
+             * allocated destination: the old fixed encoding wrote r0 behind
+             * the register allocator's back. */
+            uint8_t enter[16], length;
+            if (I->nr_dests != 1 || I->nr_srcs ||
+                !agx3_src_is_register(I->dest[0]) ||
+                I->dest[0].size != AGX_SIZE_32 || I->dest[0].memory ||
+                I->dest[0].abs || I->dest[0].neg || slot ||
+                !agx3_rt_m5_encode_enter_query(enc, I->dest[0].value, I->imm,
+                                               enter, &length) ||
+                (wait && !agx3_retire_slots_noop(emission, enc, wait)) ||
+                !agx3_append(emission, enter, length))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         if (I->op == AGX_OPCODE_INTERSECT_RAY_M5 ||
+             I->op == AGX_OPCODE_CONTINUE_RAY_M5) {
+            uint8_t bytes[16], len = 0;
+            if (enc != AGX_ISA_ENCODING_AGX3C || I->nr_dests || slot)
+               AGX3_REFUSE(I);
+            if (I->op == AGX_OPCODE_INTERSECT_RAY_M5) {
+               if (I->nr_srcs != 5)
+                  AGX3_REFUSE(I);
+               agx_index address = I->src[0], bvh = I->src[1];
+               const bool bvh_kill = bvh.kill || bvh.discard;
+               address.kill = address.cache = address.discard = false;
+               bvh.kill = bvh.cache = bvh.discard = false;
+               if (!agx3_src_is_register(address) ||
+                   address.size != AGX_SIZE_64 ||
+                   !agx3_scs_plain_index(address) ||
+                   (bvh.type != AGX_INDEX_UNIFORM &&
+                    !agx3_src_is_register(bvh)) ||
+                   bvh.size != AGX_SIZE_64 || !agx3_scs_plain_index(bvh))
+                  AGX3_REFUSE(I);
+               const struct agx3_rt_m5_intersect_desc d = {
+                  .address_register = address.value,
+                  .bvh_base = bvh.value,
+                  .bvh_uniform = bvh.type == AGX_INDEX_UNIFORM,
+                  .bvh_kill = bvh_kill,
+               };
+               if (!agx3_rt_m5_encode_intersect(enc, &d, bytes, &len))
+                  AGX3_REFUSE(I);
+               /* Native M5 main +0x4a8/+0x4c4/+0x4cc publishes three
+                * launch words before traversal. GPR values and the SCS packet
+                * alone do not provide these staging operands. Keep publication
+                * fused with its consumer so texture staging cannot intervene.
+                */
+               if (wait && !agx3_retire_slots_noop(emission, enc, wait))
+                  AGX3_REFUSE(I);
+               for (unsigned w = 0; w < 3; ++w) {
+                  agx_index word = I->src[2 + w];
+                  if (!agx3_src_is_register(word) || word.size != AGX_SIZE_32 ||
+                      word.memory || word.abs || word.neg || (word.value & 1))
+                     AGX3_REFUSE(I);
+                  uint8_t copy[AGX3_STAGE_MAX_LENGTH], copy_len;
+                  if (!agx3_stage_copy_liveness(enc, 2 * w, word.value,
+                         false, word.discard, 1, 0, copy, &copy_len) ||
+                      !agx3_append(emission, copy, copy_len))
+                     AGX3_REFUSE(I);
+               }
+               const uint8_t publish[] = {0x17, 0x11, 0, 6, 0, 0};
+               if (!agx3_append(emission, publish, sizeof(publish)))
+                  AGX3_REFUSE(I);
+               if (fragment_ray_query && !ray_query_acquired) {
+                  uint8_t pixel[AGX3_PIXEL_MAX_LENGTH], pixel_len;
+                  const bool ok = g17p_colourless_fragment
+                     ? agx3_pixel_atomic_acquire(enc, pixel, &pixel_len)
+                     : agx3_pixel_wait(enc, 4, 0, pixel, &pixel_len);
+                  if (!ok || !agx3_append(emission, pixel, pixel_len))
+                     AGX3_REFUSE(I);
+                  ray_query_acquired = true;
+                  ctx->agx3_colour_link.rt_query_write_acquired = !g17p_colourless_fragment;
+               }
+               ctx->agx3_staging_halfs = MAX2(ctx->agx3_staging_halfs, 6);
+            } else {
+               if (I->nr_srcs != 1 ||
+                   !agx3_rt_m5_encode_continue(enc, bytes, &len))
+                  AGX3_REFUSE(I);
+               agx_index resident_id = I->src[0];
+               if (!agx3_src_is_register(resident_id) ||
+                   resident_id.size != AGX_SIZE_32 || resident_id.memory ||
+                   resident_id.abs || resident_id.neg || (resident_id.value & 1))
+                  AGX3_REFUSE(I);
+               /* Native query main +0x658 stages SR59 at half8 before
+                * continue_ray consumes it. Staging0/2/4 belongs to the
+                * initial intersection and cannot identify a continuation. */
+               if (wait && !agx3_retire_slots_noop(emission, enc, wait))
+                  AGX3_REFUSE(I);
+               wait = 0;
+               uint8_t copy[AGX3_STAGE_MAX_LENGTH], copy_len;
+               if (!agx3_stage_copy_liveness(enc, 8, resident_id.value,
+                      false, resident_id.discard, 1, 0, copy, &copy_len) ||
+                   !agx3_append(emission, copy, copy_len))
+                  AGX3_REFUSE(I);
+               ctx->agx3_staging_halfs = MAX2(ctx->agx3_staging_halfs, 10);
+            }
+            if (I->op == AGX_OPCODE_CONTINUE_RAY_M5) {
+               /* Publish the resident ID and status before resuming. */
+               const uint8_t resume_barrier[] = {0x17, 0x09, 0, 6, 0, 0};
+               if (!agx3_append(emission, resume_barrier, sizeof(resume_barrier)))
+                  AGX3_REFUSE(I);
+            }
+            /* Native RT forms have no ordinary scoreboard wait field.
+             * Retire only their actual input dependencies. The following
+             * query-status loads and continuation remain explicit IR. */
+            if ((I->op == AGX_OPCODE_CONTINUE_RAY_M5 && wait &&
+                 !agx3_retire_slots_noop(emission, enc, wait)) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          if (I->op == AGX_OPCODE_ALLOCATE_UVB_M5 ||
              I->op == AGX_OPCODE_STORE_UVB_M5) {
             uint8_t bytes[8], length;
@@ -11015,9 +12388,24 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                                          I->src[0].kill || I->src[0].discard,
                                          bytes, &length);
             }
-            if (!encoded || (wait && !agx3_retire_slots_noop(emission, enc, wait)) ||
+            /* M5 allocate_uvb completes on scoreboard slot 1: every native
+             * G17G write_uvb helper waits slot 1 on the instruction right
+             * after it (cndst f4, e.g. 1e00002e). UVSTs issued before that
+             * completion lost the exports of alternate mesh workgroups on
+             * hardware (workgroups 2, 4, 6 of 8). Retire any older slot-1
+             * generation before the allocation claims the slot, then wait
+             * for the allocation itself. */
+            const bool alloc_slot = I->op == AGX_OPCODE_ALLOCATE_UVB_M5 &&
+                                    enc == AGX_ISA_ENCODING_AGX3C;
+            const uint16_t pre_wait = alloc_slot ? (wait | (outstanding_here & 1u)) : wait;
+            if (!encoded || (pre_wait && !agx3_retire_slots_noop(emission, enc, pre_wait)) ||
                 !agx3_append(emission, bytes, length))
                AGX3_REFUSE(I);
+            if (alloc_slot) {
+               if (!agx3_retire_slots_noop(emission, enc, 1u))
+                  AGX3_REFUSE(I);
+               outstanding &= (uint16_t)~1u;
+            }
             continue;
          }
 
@@ -11105,6 +12493,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 !agx3_append(emission, bytes, len))
                AGX3_REFUSE(I);
             outstanding &= ~retire;
+            if (enc == AGX_ISA_ENCODING_AGX3C) {
+               assert(slot == 1);
+               outstanding |= 1u;
+            }
             continue;
          }
 
@@ -11124,7 +12516,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             };
             uint8_t bytes[AGX3_STATIC_SPILL_MAX_LENGTH];
             size_t length;
-            if (!agx3_cprog_input_import(&ctx->out->agx3_preamble_inputs,
+            if (!agx3_cprog_input_import_encoding(enc, &ctx->out->agx3_preamble_inputs,
                                         &import, bytes, sizeof(bytes), &length) ||
                 !agx3_append(emission, bytes, length))
                AGX3_REFUSE(I);
@@ -11195,20 +12587,35 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * Unrelated memory/texture producers do not join this wait. */
             const bool texture_wait =
                I->imm == AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT;
-            const uint16_t image_wait = texture_wait
-               ? image_write_slots & outstanding_here & ~1u : 0;
+            /* AGX3C has no implied slot: every G17G texture barrier waits,
+             * in its first word, on the result slots of all image writes
+             * still in flight (port/fencewit k_fence, k_fence2,
+             * k_fence_twice, k_tgbar_tex, k_atf_tg, k_atf_dev), and on no
+             * unrelated producer (k_fence_ld). */
+            const bool m5_texture = agx3_m5_texture_barrier(enc, I->imm);
+            const uint16_t image_wait =
+               m5_texture ? image_write_slots & outstanding_here :
+               texture_wait ? image_write_slots & outstanding_here & ~1u : 0;
             const uint16_t retire =
                wait | image_wait | (device ? outstanding_here : 0);
             if (!agx3_emit_scoped_barrier(emission, enc, I, retire))
                AGX3_REFUSE(I);
-            const uint16_t completed = retire | (texture_wait ? 1u : 0);
+            const uint16_t completed =
+               retire | (texture_wait && !m5_texture ? 1u : 0);
             outstanding &= (uint16_t)~completed;
             image_write_slots &= (uint16_t)~completed;
             agx3_staging_forget(&staging_banks, completed);
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_SRSQRT) {
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_SRSQRT) {
+            if (enc == AGX_ISA_ENCODING_AGX3C && wait) {
+               if (!agx3_retire_result_wait(emission, enc, I, wait))
+                  AGX3_REFUSE(I);
+               outstanding &= (uint16_t)~wait;
+               wait = 0;
+            }
             const agx_index dest=I->dest[0], src=I->src[0];
             const bool half = src.size == AGX_SIZE_16;
             /* f30 carries the source's absolute value, so `sqrt(abs(x))` --
@@ -11232,7 +12639,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const bool ok =
                half ? agx3_alu_sqrt_step_half(enc, dest.value, src.value,
                                               half_hint,
-                                              true, wait, bytes, &len)
+                                              enc == AGX_ISA_ENCODING_AGX3C ? dest.cache : true,
+                                              wait, bytes, &len)
                     : (src.size == AGX_SIZE_32 && !(src.value & 1) &&
                        src.value < 256 &&
                        agx3_alu_sqrt_step_source_modifiers(enc, dest.value, src.value,
@@ -11248,7 +12656,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * data commonly feeds a full-precision divide in GTK's colour
           * conversion shaders. f10/f26 select the result/source widths, while
           * f30/f31 carry source ABS/NEG without a separate move. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_RCP &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_RCP &&
              I->nr_dests == 1 && I->nr_srcs == 1 && !I->saturate) {
             const agx_index dest=I->dest[0], src=I->src[0];
             if (!agx3_src_is_register(dest) || dest.memory ||
@@ -11270,7 +12679,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          }
 
          /* Reciprocal square root, at either width. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_RSQRT &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_RSQRT &&
              I->nr_dests == 1 && I->nr_srcs == 1 && !I->saturate) {
             const agx_index dest = I->dest[0], src = I->src[0];
             const bool half = dest.size == AGX_SIZE_16;
@@ -11300,10 +12710,22 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         /* C rounding/log/exp share native scalar width, lifetime and SAT
+          * fields. Keep B's existing per-operation admission below. */
+         const int c_unary = I->op == AGX_OPCODE_ROUNDEVEN ? AGX3_FUNOP_RINT :
+                                                          agx3_map_funop(I->op);
+         if (enc == AGX_ISA_ENCODING_AGX3C &&
+             c_unary >= AGX3_FUNOP_FLOOR && c_unary <= AGX3_FUNOP_EXP2) {
+            if (!agx3_emit_funop(emission, enc, I, c_unary, wait))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* f30 and f31 are the source's absolute-value and negate modifiers
           * (NIR folds a negation into the source, e.g. exp(-x)). The half
           * form has a negate bit too. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_EXP2 &&
+         if (enc == AGX_ISA_ENCODING_AGX3B &&
+             I->op == AGX_OPCODE_EXP2 &&
              I->nr_dests == 1 && I->nr_srcs == 1 && !I->saturate) {
             const agx_index dest = I->dest[0], src = I->src[0];
             const bool half = dest.size == AGX_SIZE_16;
@@ -11331,7 +12753,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          /* f10 and f26 independently select scalar result/source width. This
           * matters for mediump texture results promoted before logarithmic
           * transfer functions, as in GTK's component-transfer shader. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_LOG2 &&
+         if (enc == AGX_ISA_ENCODING_AGX3B &&
+             I->op == AGX_OPCODE_LOG2 &&
              I->nr_dests == 1 && I->nr_srcs == 1 && !I->saturate) {
             const agx_index dest = I->dest[0], src = I->src[0];
             if (!agx3_src_is_register(dest) || dest.memory ||
@@ -11355,7 +12778,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          /* floor() and ceil() at either width: the 32-bit form is the half
           * one with the width selectors raised, with the same source
           * modifiers. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FLOOR &&
+         if (enc == AGX_ISA_ENCODING_AGX3B &&
+             I->op == AGX_OPCODE_FLOOR &&
              I->nr_dests == 1 && I->nr_srcs == 1 && !I->saturate) {
             const agx_index dest = I->dest[0], src = I->src[0];
             const bool half = dest.size == AGX_SIZE_16;
@@ -11380,7 +12804,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CEIL &&
+         if (enc == AGX_ISA_ENCODING_AGX3B &&
+             I->op == AGX_OPCODE_CEIL &&
              I->nr_dests == 1 && I->nr_srcs == 1 && !I->saturate) {
             const agx_index dest = I->dest[0], src = I->src[0];
             const bool half = dest.size == AGX_SIZE_16;
@@ -11501,9 +12926,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * desktop shader may nevertheless reach DFDX/DFDY with an
              * outstanding interpolation/coefficient dependency. Retire it
              * through the already-grounded scoreboard no-op, then preserve
-             * the byte-exact zero-wait gradient encoding. */
-            if ((fop == AGX3_FUNOP_SINC || fop == AGX3_FUNOP_DFDX ||
-                 fop == AGX3_FUNOP_DFDY) &&
+             * the byte-exact zero-wait gradient encoding. The same holds for
+             * any funop whose witnessed form has no wait fields (f32sinc, for
+             * instance, scheduled right behind an async producer). */
+            if ((fop == AGX3_FUNOP_DFDX || fop == AGX3_FUNOP_DFDY ||
+                 !agx3_funop_carries_wait(enc, (enum agx3_funop)fop)) &&
                 funop_wait) {
                if (!agx3_retire_result_wait(emission, enc, I, funop_wait))
                   AGX3_REFUSE(I);
@@ -11578,7 +13005,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
             /* Multiplication commutes. Apple keeps the 32-bit operand of a
              * mixed-width 32-bit product in slot A and the half in slot B. */
-            if (I->dest[0].size == AGX_SIZE_32 && a.size == AGX_SIZE_16 &&
+            if ((I->dest[0].size == AGX_SIZE_32 ||
+                 (enc == AGX_ISA_ENCODING_AGX3C && I->dest[0].size == AGX_SIZE_64)) &&
+                a.size == AGX_SIZE_16 &&
                 agx3_src_is_register(m) && m.size == AGX_SIZE_32) {
                const agx_index half = a;
                a = m;
@@ -11604,7 +13033,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const bool mult_imm = m.type == AGX_INDEX_IMMEDIATE && !m.abs &&
                                   !m.neg && m.value <= 0xff;
             const bool widening_half_multiplier =
-               widening && m.size == AGX_SIZE_16 && !m.neg;
+               (widening || (wide && enc == AGX_ISA_ENCODING_AGX3C)) &&
+               m.size == AGX_SIZE_16 && !m.neg;
             const bool mult_reg =
                agx3_src_is_register(m) && !m.memory &&
                ((m.size == (narrow ? AGX_SIZE_16 : AGX_SIZE_32) &&
@@ -11660,7 +13090,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                (!half_a && !half_m) ||
                (!addend_imm &&
                 (no_addend || c.size == AGX_SIZE_32 ||
-                 (!half_m && c.size == AGX_SIZE_16)));
+                 (!half_m && c.size == AGX_SIZE_16) ||
+                 (enc == AGX_ISA_ENCODING_AGX3C && c.size == AGX_SIZE_16 && c.abs)));
 
             if (a_ok && (mult_imm || mult_reg) && addend_ok &&
                 half_operands_ok) {
@@ -11684,8 +13115,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   .src_a_cache = a.cache,
                   .src_c_kill = addend_reg && c.discard,
                   .src_c_cache = addend_reg && c.cache,
-                  .no_addend = no_addend,
-                  .addend_is_immediate = addend_imm,
+                  .no_addend = no_addend && !(wide && enc == AGX_ISA_ENCODING_AGX3C),
+                  .addend_is_immediate = addend_imm ||
+                     (wide && enc == AGX_ISA_ENCODING_AGX3C && no_addend),
                   .addend_immediate = addend_imm ? c.value : 0,
                   .src_c = addend_reg ? c.value : 0,
                   .src_c_16 = addend_reg && c.size == AGX_SIZE_16,
@@ -11710,8 +13142,163 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
+         /* A uniform first comparand (agx3_m5_fcmpsel_uniform_cmp0). */
+         unsigned m5_fsel_cond;
+         bool m5_fsel_swap;
+         if (enc == AGX_ISA_ENCODING_AGX3C &&
+             agx3_m5_fcmpsel_uniform_cmp0(I, &m5_fsel_cond, &m5_fsel_swap)) {
+            const agx_index cmp1 = I->src[1];
+            const agx_index sel1 = I->src[m5_fsel_swap ? 2 : 3];
+            if (!agx3_vary_reg32(I->dest[0]) || !agx3_vary_reg32(cmp1) ||
+                !agx3_vary_reg32(sel1) || cmp1.kill || sel1.kill ||
+                (cmp1.cache && cmp1.discard) || (sel1.cache && sel1.discard))
+               AGX3_REFUSE(I);
+            const struct agx3_fsel32_gpu1_desc d = {
+               .dest = I->dest[0].value, .dest_cache = I->dest[0].cache,
+               .cmp0 = I->src[0].value, .cmp0_uniform = true,
+               .cmp1 = cmp1.value, .cmp1_hint = agx3_post_ra_hint(cmp1),
+               .sel_immediate = {true, false},
+               .sel = {0, sel1.value},
+               .sel_hint = {AGX3_ALU_HINT_NONE, agx3_post_ra_hint(sel1)},
+               .condition = m5_fsel_cond, .wait_mask = wait,
+            };
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            if (!agx3_alu_fcmp_select32_gpu1(&d, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* Keep the uniform half of a native tied min/max in the uniform file.
           * Canonicalize just as the register/minifloat path below does. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FCMPSEL &&
+             I->nr_dests == 1 && I->nr_srcs == 4 && !I->saturate &&
+             (agx3_vary_reg32(I->dest[0]) || agx3_half_reg(I->dest[0])) &&
+             agx_channels(I->dest[0]) == 1 &&
+             (((agx3_vary_reg32(I->src[0]) || agx3_half_reg(I->src[0])) &&
+               (agx3_vary_reg32(I->src[1]) || agx3_half_reg(I->src[1]) ||
+                (I->src[1].type == AGX_INDEX_IMMEDIATE && I->src[1].value <= 255))) ||
+              (I->src[0].type == AGX_INDEX_IMMEDIATE && I->src[0].value <= 255 &&
+               (agx3_vary_reg32(I->src[1]) || agx3_half_reg(I->src[1])))) &&
+             ((I->src[2].type == AGX_INDEX_IMMEDIATE && I->src[2].value <= UINT8_MAX) ||
+              (agx3_half_reg(I->dest[0]) && agx3_half_reg(I->src[2]))) &&
+             ((I->src[3].type == AGX_INDEX_IMMEDIATE && I->src[3].value <= UINT8_MAX) ||
+              (agx3_half_reg(I->dest[0]) && agx3_half_reg(I->src[3])))) {
+            const bool reverse = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index cmp[2] = {I->src[reverse ? 1 : 0],
+                                      I->src[reverse ? 0 : 1]};
+            unsigned condition = I->fcond;
+            bool swap = I->invert_cond;
+            if (!agx3_fcmpsel_tied_minmax(I)) {
+               if (condition == AGX_FCOND_LTN) {
+                  condition = AGX_FCOND_GE;
+                  swap = !swap;
+               } else if (condition == AGX_FCOND_GTN) {
+                  condition = AGX_FCOND_LE;
+                  swap = !swap;
+               }
+            }
+            if (reverse)
+               condition = agx3_fcond_reverse(condition);
+            for (unsigned j = 0; j < 4; ++j) {
+               if (I->src[j].abs || I->src[j].neg || I->src[j].memory ||
+                   agx_channels(I->src[j]) != 1 ||
+                   (I->src[j].cache && I->src[j].discard))
+                  AGX3_REFUSE(I);
+            }
+            const struct agx3_fsel32_gpu1_desc d = {
+               .dest = I->dest[0].value, .dest_cache = I->dest[0].cache,
+               .cmp0 = cmp[0].value, .cmp1 = cmp[1].value,
+               .cmp0_16 = cmp[0].size == AGX_SIZE_16,
+               .cmp1_16 = cmp[1].type != AGX_INDEX_IMMEDIATE &&
+                          cmp[1].size == AGX_SIZE_16,
+               .cmp1_immediate = cmp[1].type == AGX_INDEX_IMMEDIATE,
+               .cmp0_hint = agx3_post_ra_hint(cmp[0]),
+               .cmp1_hint = agx3_post_ra_hint(cmp[1]),
+               .sel_immediate = {
+                  I->src[swap ? 3 : 2].type == AGX_INDEX_IMMEDIATE,
+                  I->src[swap ? 2 : 3].type == AGX_INDEX_IMMEDIATE,
+               },
+               .sel_integer = true,
+               .sel = {I->src[swap ? 3 : 2].value, I->src[swap ? 2 : 3].value},
+               .sel_hint = {agx3_post_ra_hint(I->src[swap ? 3 : 2]),
+                            agx3_post_ra_hint(I->src[swap ? 2 : 3])},
+               .condition = condition, .wait_mask = wait,
+            };
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            const bool encoded = I->dest[0].size == AGX_SIZE_32
+               ? agx3_alu_fcmp_select32_gpu1(&d, bytes, &len)
+               : agx3_alu_fcmp_select16_gpu1(&d, bytes, &len);
+            if (!encoded || !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FCMPSEL &&
+             I->nr_dests == 1 && I->nr_srcs == 4 && !I->saturate &&
+             agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1 &&
+             ((agx3_vary_reg32(I->src[0]) || agx3_half_reg(I->src[0])) ||
+              agx3_src_is_zero_imm(I->src[0])) &&
+             ((agx3_vary_reg32(I->src[1]) || agx3_half_reg(I->src[1])) ||
+              agx3_src_is_zero_imm(I->src[1])) &&
+             (agx3_vary_reg32(I->src[2]) ||
+              (I->src[2].type == AGX_INDEX_IMMEDIATE && I->src[2].value <= UINT8_MAX)) &&
+             (agx3_vary_reg32(I->src[3]) ||
+              (I->src[3].type == AGX_INDEX_IMMEDIATE && I->src[3].value <= UINT8_MAX)) &&
+             !(I->src[2].type == AGX_INDEX_IMMEDIATE &&
+               I->src[3].type == AGX_INDEX_IMMEDIATE)) {
+            const bool reverse = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index cmp[2] = {I->src[reverse ? 1 : 0], I->src[reverse ? 0 : 1]};
+            unsigned condition = I->fcond;
+            bool swap = I->invert_cond;
+            if (!agx3_fcmpsel_tied_minmax(I)) {
+               if (condition == AGX_FCOND_LTN) {
+                  condition = AGX_FCOND_GE;
+                  swap = !swap;
+               } else if (condition == AGX_FCOND_GTN) {
+                  condition = AGX_FCOND_LE;
+                  swap = !swap;
+               }
+            }
+            if (reverse)
+               condition = agx3_fcond_reverse(condition);
+            const agx_index sel[2] = {I->src[swap ? 3 : 2], I->src[swap ? 2 : 3]};
+            /* Generic select immediates are integer bit patterns. Only a
+             * selected operand tied to a compared float carries the
+             * comparison's minifloat encoding (for example a depth clamp).
+             * A fused boolean bit-mask select must retain its raw bits. */
+            const bool tied_float =
+               (agx_is_equiv(I->src[0], I->src[2]) &&
+                agx_is_equiv(I->src[1], I->src[3])) ||
+               (agx_is_equiv(I->src[0], I->src[3]) &&
+                agx_is_equiv(I->src[1], I->src[2]));
+            for (unsigned i = 0; i < 4; ++i) {
+               if (I->src[i].kill || agx_channels(I->src[i]) != 1 ||
+                   (I->src[i].cache && I->src[i].discard))
+                  AGX3_REFUSE(I);
+            }
+            const struct agx3_fsel32_gpu1_desc d = {
+               .dest = I->dest[0].value, .dest_cache = I->dest[0].cache,
+               .cmp0 = cmp[0].value, .cmp1 = cmp[1].value,
+               .cmp0_16 = cmp[0].size == AGX_SIZE_16,
+               .cmp1_16 = cmp[1].type != AGX_INDEX_IMMEDIATE && cmp[1].size == AGX_SIZE_16,
+               .cmp1_immediate = cmp[1].type == AGX_INDEX_IMMEDIATE,
+               .cmp0_hint = agx3_post_ra_hint(cmp[0]),
+               .cmp1_hint = agx3_post_ra_hint(cmp[1]),
+               .sel_immediate = {sel[0].type == AGX_INDEX_IMMEDIATE,
+                                 sel[1].type == AGX_INDEX_IMMEDIATE},
+               .sel_integer = !tied_float,
+               .sel = {sel[0].value, sel[1].value},
+               .sel_hint = {agx3_post_ra_hint(sel[0]), agx3_post_ra_hint(sel[1])},
+               .condition = condition, .wait_mask = wait,
+            };
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            if (!agx3_alu_fcmp_select32_gpu1(&d, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FCMPSEL &&
              I->nr_dests == 1 && I->nr_srcs == 4 && !I->invert_cond &&
              !I->saturate &&
@@ -12859,10 +14446,45 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         /* AGX3C narrow iadd_sat/uadd_sat and isub_sat/usub_sat (subtrahend
+          * negated): C's iadd32/isub32 point with half widths. ABS marks both
+          * halves unsigned, as on G17P. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_IADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && I->saturate &&
+             I->shift == 0 && I->dest[0].size == AGX_SIZE_16 &&
+             agx3_src_is_register(I->src[0]) &&
+             agx3_src_is_register(I->src[1]) &&
+             I->src[0].size == AGX_SIZE_16 && !I->src[0].neg &&
+             I->src[1].size == AGX_SIZE_16) {
+            const agx_index dest = I->dest[0];
+            const agx_index a = I->src[0], b = I->src[1];
+            if (!agx3_src_is_register(dest) || dest.memory ||
+                agx_channels(dest) != 1 || a.memory || b.memory ||
+                a.abs != b.abs || a.kill || b.kill ||
+                agx_channels(a) != 1 || agx_channels(b) != 1 ||
+                (a.cache && a.discard) || (b.cache && b.discard))
+               AGX3_REFUSE(I);
+
+            const uint16_t src[2] = {a.value, b.value};
+            const enum agx3_alu_hint hints[2] = {
+               agx3_post_ra_hint(a), agx3_post_ra_hint(b)};
+            uint16_t w = wait;
+            if (!agx3_retire_extra_waits(emission, enc, sl, res, instr_index,
+                                         I, &w))
+               AGX3_REFUSE(I);
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+            if (!agx3_alu_addsub_sat16_m5(dest.value, src, hints, !a.abs,
+                                          b.neg, dest.cache, w, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* NIR's narrow isub is iadd(a, -b), and ineg is iadd(#0, -a). Apple
           * emits both as the scalar-half isub, the negation with an
           * immediate-zero minuend, so neither needs a widened temporary. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_IADD &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_IADD &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              I->shift == 0 && I->dest[0].size == AGX_SIZE_16 &&
              agx3_src_is_register(I->src[1]) &&
@@ -12884,7 +14506,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                AGX3_REFUSE(I);
 
             uint16_t w = wait;
-            if (!agx3_retire_extra_waits(emission, enc, sl, res, instr_index,
+            if (enc == AGX_ISA_ENCODING_AGX3B &&
+                !agx3_retire_extra_waits(emission, enc, sl, res, instr_index,
                                          I, &w))
                AGX3_REFUSE(I);
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
@@ -12975,38 +14598,150 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
-         /*
-          * `iadd dst32, #imm, src << shift` -- constant address arithmetic,
-          * which is what every shader with more than one memory access is
-          * built out of. The source may be a 16-bit half, zero- or
-          * sign-extended (abs and neg respectively, as on AGX2).
-          *
-          * This carries its own scoreboard wait, so it does not need the
-          * wait-retiring prelude and does not disturb the operands.
-          */
-         if (I->op == AGX_OPCODE_IADD && I->nr_dests == 1 && I->nr_srcs == 2 &&
-             agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
-             I->dest[0].size == AGX_SIZE_32 &&
-             agx_channels(I->dest[0]) == 1) {
-            const bool imm_first = I->src[0].type == AGX_INDEX_IMMEDIATE;
-            const agx_index constant = imm_first ? I->src[0] : I->src[1];
-            const agx_index reg = imm_first ? I->src[1] : I->src[0];
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_IADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             !I->shift && agx3_half_reg(I->dest[0]) &&
+             agx3_half_reg(I->src[0]) && !I->src[0].abs && !I->src[0].neg &&
+             agx_channels(I->dest[0]) == 1 && agx_channels(I->src[0]) == 1 &&
+             I->src[1].type == AGX_INDEX_IMMEDIATE && I->src[1].neg &&
+             !I->src[1].abs && !I->src[1].memory) {
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+            if (!agx3_alu_isub_imm16_m5(I->dest[0].value, I->src[0].value,
+                    I->src[1].value, agx3_post_ra_hint(I->src[0]),
+                    I->dest[0].cache, wait, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
 
-            /* iadd_sat, uadd_sat, isub_sat and usub_sat over 32-bit
-             * registers: the register iadd/isub point with f53. ABS on both
-             * operands selects unsigned saturation; otherwise both sign
-             * fields are set. */
-            if (enc == AGX_ISA_ENCODING_AGX3B && I->saturate) {
-               agx_index a = I->src[0], b = I->src[1];
-               if (a.neg && !b.neg) {
-                  const agx_index t = a;
-                  a = b;
-                  b = t;
-               }
-               if (I->shift != 0 || !agx3_src_is_register(a) ||
-                   !agx3_src_is_register(b) || a.memory || b.memory ||
-                   a.size != AGX_SIZE_32 || b.size != AGX_SIZE_32 ||
-                   agx_channels(a) != 1 || agx_channels(b) != 1 ||
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_IADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_half_reg(I->dest[0]) && agx_channels(I->dest[0]) == 1) {
+            const bool swap = I->src[1].type == AGX_INDEX_IMMEDIATE;
+            const agx_index a = I->src[swap ? 1 : 0], b = I->src[swap ? 0 : 1];
+            if (swap && I->shift)
+               AGX3_REFUSE(I);
+            if ((a.type == AGX_INDEX_IMMEDIATE || agx3_half_reg(a)) &&
+                agx3_half_reg(b) && !a.abs && !a.neg && !b.abs && !b.neg &&
+                !a.memory && !b.memory && !a.kill && !b.kill &&
+                agx_channels(a) == 1 && agx_channels(b) == 1) {
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+               if (!agx3_alu_iadd16_gpu1(I->dest[0].value, I->dest[0].cache,
+                       a.type == AGX_INDEX_IMMEDIATE, a.value,
+                       a.type == AGX_INDEX_IMMEDIATE ? AGX3_ALU_HINT_NONE : agx3_post_ra_hint(a),
+                       b.value, agx3_post_ra_hint(b), I->shift, wait, bytes, &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_IADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             I->shift <= 4 && agx3_vary_reg32(I->dest[0]) &&
+             agx_channels(I->dest[0]) == 1 &&
+             agx3_src_is_register(I->src[0]) && agx3_src_is_register(I->src[1]) &&
+             !I->src[0].memory && !I->src[1].memory &&
+             !I->src[0].neg && !I->src[1].neg &&
+             agx_channels(I->src[0]) == 1 && agx_channels(I->src[1]) == 1 &&
+             (I->src[0].size == AGX_SIZE_16 || I->src[0].size == AGX_SIZE_32) &&
+             (I->src[1].size == AGX_SIZE_16 || I->src[1].size == AGX_SIZE_32) &&
+             (I->src[0].size == AGX_SIZE_16 || !I->src[0].abs) &&
+             (I->src[1].size == AGX_SIZE_16 || !I->src[1].abs)) {
+            const struct agx3_iadd_desc d = {
+               .dest = I->dest[0].value, .dest_cache = I->dest[0].cache,
+               .src_a_is_register = true, .src_a_reg = I->src[0].value,
+               .src_a_16 = I->src[0].size == AGX_SIZE_16,
+               .src_a_sign_extend = I->src[0].size == AGX_SIZE_16 && !I->src[0].abs,
+               .src_a_cache = I->src[0].cache, .src_a_kill = I->src[0].discard,
+               .src = I->src[1].value, .src_16 = I->src[1].size == AGX_SIZE_16,
+               .sign_extend = I->src[1].size == AGX_SIZE_16 && !I->src[1].abs,
+               .src_cache = I->src[1].cache, .src_kill = I->src[1].discard,
+               .shift = I->shift, .wait_mask = wait,
+            };
+            uint8_t bytes[AGX3_IADD_MAX_LENGTH], len;
+            if (!agx3_iadd_encode(enc, &d, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_IADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             I->shift == 0 && agx3_vary_reg32(I->dest[0]) &&
+             agx3_vary_reg32(I->src[0]) && agx3_mod_src_ok(I->src[1]) &&
+             agx_channels(I->dest[0]) == 1 &&
+             agx_channels(I->src[0]) == 1 && agx_channels(I->src[1]) == 1 &&
+             !I->src[0].abs && !I->src[0].neg &&
+             !I->src[1].abs) {
+            const struct agx3_iadd_desc d = {
+               .dest = I->dest[0].value, .dest_cache = I->dest[0].cache,
+               .src_a_is_register = true, .src_a_reg = I->src[0].value,
+               .src_a_cache = I->src[0].cache,
+               .src_a_kill = I->src[0].discard,
+               .src = I->src[1].value, .src_cache = I->src[1].cache,
+               .src_kill = I->src[1].discard, .wait_mask = wait,
+               .subtract = I->src[1].neg,
+            };
+            uint8_t bytes[AGX3_IADD_MAX_LENGTH], len = 0;
+            if (!agx3_iadd_encode(enc, &d, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         /*
+          * `iadd dst32, #imm, src << shift` -- constant address arithmetic,
+          * which is what every shader with more than one memory access is
+          * built out of. The source may be a 16-bit half, zero- or
+          * sign-extended (abs and neg respectively, as on AGX2).
+          *
+          * This carries its own scoreboard wait, so it does not need the
+          * wait-retiring prelude and does not disturb the operands.
+          */
+         if (I->op == AGX_OPCODE_IADD && I->nr_dests == 1 && I->nr_srcs == 2 &&
+             agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
+             I->dest[0].size == AGX_SIZE_32 &&
+             agx_channels(I->dest[0]) == 1) {
+            const bool imm_first = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index constant = imm_first ? I->src[0] : I->src[1];
+            const agx_index reg = imm_first ? I->src[1] : I->src[0];
+
+            /* Native G17G sign extension keeps the half in slot A and
+             * adds immediate zero in slot B, in one instruction. */
+            if (enc == AGX_ISA_ENCODING_AGX3C && !I->saturate && !I->shift &&
+                agx3_plain_immediate(constant) && constant.value == 0 &&
+                agx3_half_reg(reg) && agx_channels(reg) == 1 && !reg.neg &&
+                !reg.kill && !(reg.cache && reg.discard)) {
+               const struct agx3_isub_imm_desc d = {
+                  .dest = I->dest[0].value, .dest_cache = I->dest[0].cache,
+                  .src = reg.value, .src_16 = true, .sign_extend = !reg.abs,
+                  .src_cache = reg.cache, .src_kill = reg.discard,
+                  .wait_mask = wait,
+               };
+               uint8_t bytes[AGX3_IADD_MAX_LENGTH], len;
+               if (!agx3_iadd_immediate_encode(enc, &d, bytes, &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+
+            /* iadd_sat, uadd_sat, isub_sat and usub_sat over 32-bit
+             * registers: the register iadd/isub point with f53. ABS on both
+             * operands selects unsigned saturation; otherwise both sign
+             * fields are set. */
+            if ((enc == AGX_ISA_ENCODING_AGX3B ||
+                 enc == AGX_ISA_ENCODING_AGX3C) && I->saturate) {
+               agx_index a = I->src[0], b = I->src[1];
+               if (a.neg && !b.neg) {
+                  const agx_index t = a;
+                  a = b;
+                  b = t;
+               }
+               if (I->shift != 0 || !agx3_src_is_register(a) ||
+                   !agx3_src_is_register(b) || a.memory || b.memory ||
+                   a.size != AGX_SIZE_32 || b.size != AGX_SIZE_32 ||
+                   agx_channels(a) != 1 || agx_channels(b) != 1 ||
                    a.neg || a.kill || b.kill || a.abs != b.abs ||
                    (a.cache && a.discard) || (b.cache && b.discard))
                   AGX3_REFUSE(I);
@@ -13017,7 +14752,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   AGX3_REFUSE(I);
                const struct agx3_iadd_desc d = {
                   .dest = I->dest[0].value,
-                  .dest_cache = true,
+                  .dest_cache = enc == AGX_ISA_ENCODING_AGX3B || I->dest[0].cache,
                   .src_a_is_register = true,
                   .src_a_reg = a.value,
                   .src_a_kill = a.discard,
@@ -13068,7 +14803,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * form and a non-zero immediate addend alike. Other bare
              * half/shift combinations are refused. */
             const bool signed_half_widen =
-               enc == AGX_ISA_ENCODING_AGX3B && !I->saturate &&
+               (enc == AGX_ISA_ENCODING_AGX3B ||
+                (enc == AGX_ISA_ENCODING_AGX3C && I->shift == 4)) &&
+               !I->saturate &&
                constant.type == AGX_INDEX_IMMEDIATE &&
                reg.size == AGX_SIZE_16 && !reg.abs && !reg.neg &&
                (I->shift == 0 || I->shift == 4);
@@ -13079,7 +14816,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * added, and a negated half was added sign-extended. Only the
              * unshifted subtraction is witnessed. */
             const bool imm_minus_reg =
-               enc == AGX_ISA_ENCODING_AGX3B && !I->saturate &&
+               (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+               !I->saturate &&
                reg.neg && I->shift == 0 &&
                (reg.size == AGX_SIZE_16 ||
                 (reg.size == AGX_SIZE_32 && !reg.abs));
@@ -13096,6 +14834,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   .immediate = (uint8_t)constant.value,
                   .src = reg.value,
                   .src_kill = reg.discard,
+                  .src_cache = enc == AGX_ISA_ENCODING_AGX3C && reg.cache,
                   .src_16 = reg.size == AGX_SIZE_16,
                   .sign_extend = reg.size == AGX_SIZE_16 && !reg.abs,
                   .shift = I->shift,
@@ -13221,12 +14960,12 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   /* Apple's operand cache hints, on the subtraction this
                    * path newly encodes. The additions it already emitted
                    * keep their hardware-proven bytes without them. */
-                  .src_a_cache = b3 && nb.neg && na.cache,
+                  .src_a_cache = nb.neg && na.cache,
                   .src_a_16 = na.size == AGX_SIZE_16,
                   .src_a_sign_extend = na.size == AGX_SIZE_16 && !na.abs,
                   .src = nb.value,
                   .src_kill = nb.discard,
-                  .src_cache = b3 && nb.neg && nb.cache,
+                  .src_cache = nb.neg && nb.cache,
                   .src_16 = nb.size == AGX_SIZE_16,
                   .sign_extend = nb.size == AGX_SIZE_16 && !nb.abs,
                   .subtract = nb.neg,
@@ -13309,6 +15048,25 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITOP &&
+             (I->truth_table & 0xfu) == AGX3_SEL_TT_MOV &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_vary_reg32(I->dest[0]) && agx3_vary_reg32(I->src[0]) &&
+             agx_channels(I->dest[0]) == 1 && agx_channels(I->src[0]) == 1 &&
+             agx3_src_is_zero_imm(I->src[1]) && !I->src[0].kill &&
+             !(I->src[0].cache && I->src[0].discard)) {
+            const struct agx3_iadd_desc d = {
+               .dest=I->dest[0].value, .dest_cache=I->dest[0].cache,
+               .src=I->src[0].value, .src_cache=I->src[0].cache,
+               .src_kill=I->src[0].discard, .wait_mask=wait,
+            };
+            uint8_t bytes[AGX3_IADD_MAX_LENGTH], len = 0;
+            if (!agx3_iadd_encode(enc, &d, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* A narrowing integer move copies the low half of its 32-bit
           * source. The same native 16-bit COPY used above names that low half
           * directly, so no arithmetic conversion or new encoding is needed. */
@@ -13485,7 +15243,10 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                  (a.size == AGX_SIZE_32 || a.size == AGX_SIZE_16) &&
                  agx_channels(a) == 1 &&
                  ((b.type == AGX_INDEX_IMMEDIATE && b.value <= 255) ||
-                  (agx3_src_is_register(b) && b.size == a.size &&
+                  (agx3_src_is_register(b) &&
+                   (b.size == a.size ||
+                    (enc == AGX_ISA_ENCODING_AGX3C &&
+                     (b.size == AGX_SIZE_16 || b.size == AGX_SIZE_32))) &&
                    agx_channels(b) == 1))));
 
             if (!dest_ok || !cmp_ok)
@@ -13499,6 +15260,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             d.cmp0_uniform = a_uniform;
             d.cmp1_uniform = b_uniform;
             d.cmp_16 = a.size == AGX_SIZE_16;
+            d.cmp_mixed_width = b.type != AGX_INDEX_IMMEDIATE && b.size != a.size;
             d.explicit_hints = true;
             d.dest_cache = dest.cache;
             const agx_index hinted[4] = {a, b, t, f};
@@ -13588,7 +15350,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * the exact operand written by that slot. Each carrier has one
              * grounded wait bit; ISEL itself then needs none. */
             d.wait_mask = wait;
-            if (util_bitcount(wait) > 1) {
+            if (enc == AGX_ISA_ENCODING_AGX3B && util_bitcount(wait) > 1) {
                uint16_t remaining = wait;
                while (remaining) {
                   const uint16_t bit = remaining & (uint16_t)-(int16_t)remaining;
@@ -13621,6 +15383,26 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_MOV_IMM &&
+             I->nr_dests == 1 && !I->nr_srcs && !I->saturate &&
+             agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
+             agx_channels(I->dest[0]) == 1 && I->imm <= UINT32_MAX &&
+             (I->dest[0].size == AGX_SIZE_16 || I->dest[0].size == AGX_SIZE_32)) {
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            if (!agx3_alu_m5_ldimm(I->dest[0].value,
+                    I->dest[0].size == AGX_SIZE_16, I->dest[0].cache,
+                    I->imm, wait, bytes, &len)) {
+               if (!wait || !agx3_retire_result_wait(emission, enc, I, wait) ||
+                   !agx3_alu_m5_ldimm(I->dest[0].value,
+                       I->dest[0].size == AGX_SIZE_16, I->dest[0].cache,
+                       I->imm, 0, bytes, &len))
+                  AGX3_REFUSE(I);
+            }
+            if (!agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          if (I->op == AGX_OPCODE_MOV_IMM && I->nr_dests == 1 &&
              I->nr_srcs == 0 && !I->saturate &&
              agx3_src_is_register(I->dest[0]) &&
@@ -13700,7 +15482,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          /* AGX models a narrowing float conversion as add with exact -0.
           * Keep one native operation, with actual source lifetime and waits. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FADD &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_FADD &&
              I->nr_dests == 1 && I->nr_srcs == 2 &&
              agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
              I->dest[0].size == AGX_SIZE_16 && agx_channels(I->dest[0]) == 1) {
@@ -13732,7 +15514,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FMUL &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FMUL &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
              I->dest[0].size == AGX_SIZE_16 && agx_channels(I->dest[0]) == 1 &&
@@ -13759,7 +15542,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * destination width on the ordinary f32mul); let it fall through to
           * the minifloat path below instead of being claimed and refused
           * here. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FMUL &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FMUL &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
              I->dest[0].size == AGX_SIZE_16 &&
@@ -13789,7 +15573,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_HMUL &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_HMUL &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              ((agx3_src_is_register(I->src[0]) &&
                I->src[1].type == AGX_INDEX_IMMEDIATE) ||
@@ -13815,11 +15600,70 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         /* Native f32fma also writes a half destination while consuming a
+          * full register and two minifloats. The result width is independent
+          * of arithmetic/input widths; this is not a mixed f16fma. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FFMA &&
+             I->nr_dests == 1 && I->nr_srcs == 3 &&
+             agx3_half_reg(I->dest[0])) {
+            const bool reverse = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index src = I->src[reverse ? 1 : 0];
+            const agx_index mul = I->src[reverse ? 0 : 1], add = I->src[2];
+            if (agx3_mod_src_ok(src) && agx_channels(src) == 1 &&
+                agx3_plain_immediate(mul) && agx3_plain_immediate(add) &&
+                mul.value <= UINT8_MAX && add.value <= UINT8_MAX) {
+               uint8_t multiplier = mul.value;
+               uint8_t addend = add.value;
+               if (mul.abs)
+                  multiplier &= 0x7f;
+               if (src.neg ^ mul.neg)
+                  multiplier ^= 0x80;
+               if (add.abs)
+                  addend &= 0x7f;
+               if (add.neg)
+                  addend ^= 0x80;
+               const struct agx3_falu_runtime runtime = {
+                  .cache = {src.cache}, .discard = {src.discard},
+                  .dest_cache = I->dest[0].cache, .wait_mask = wait,
+               };
+               uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], len;
+               if (src.kill ||
+                   !agx3_falu_fma_two_imm_narrow_m5(
+                      I->dest[0].value, src.value, multiplier, addend,
+                      src.abs, I->saturate, &runtime, bytes, &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FFMA &&
+             I->nr_dests == 1 && I->nr_srcs == 3 && !I->saturate &&
+             agx3_half_reg(I->dest[0])) {
+            const bool reverse = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index src = I->src[reverse ? 1 : 0];
+            const agx_index mul = I->src[reverse ? 0 : 1], add = I->src[2];
+            if (agx3_half_reg(src) && agx3_plain_immediate(mul) &&
+                agx3_plain_immediate(add) && mul.value <= UINT8_MAX &&
+                add.value <= UINT8_MAX && !src.abs && !src.kill &&
+                !(src.cache && src.discard) && !mul.abs && !add.abs) {
+               const uint8_t multiplier = mul.value ^ ((src.neg ^ mul.neg) ? 0x80 : 0);
+               const uint8_t addend = add.value ^ (add.neg ? 0x80 : 0);
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+               if (!agx3_alu_hfma_immediates_m5(I->dest[0].value, src.value,
+                      multiplier, addend, agx3_post_ra_hint(src),
+                      I->dest[0].cache, wait, bytes, &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
          /* A saturating half multiply is the same instruction with G17P's
           * f14 set, which moves it to the two-byte-longer ep2 point, e.g.
           * clamp(texture(tex, uv) * vcolor, 0.0, 1.0) -> fmul.sat on halves. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_HMUL &&
-             I->nr_dests == 1 && I->nr_srcs == 2) {
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_HMUL && I->nr_dests == 1 && I->nr_srcs == 2) {
             uint16_t src[2]; enum agx3_alu_hint hints[2];
             agx_index dst=I->dest[0];
             if (!agx3_src_is_register(dst) || dst.memory || dst.size != AGX_SIZE_16 ||
@@ -13862,6 +15706,25 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_HADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_half_reg(I->dest[0]) && agx_channels(I->dest[0]) == 1) {
+            const bool swap = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index src = I->src[swap ? 1 : 0], imm = I->src[swap ? 0 : 1];
+            if (agx3_half_reg(src) && agx_channels(src) == 1 && !src.abs &&
+                !src.kill && !(src.cache && src.discard) &&
+                imm.type == AGX_INDEX_IMMEDIATE && imm.value <= UINT8_MAX &&
+                !imm.abs && !imm.neg && !imm.memory) {
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+               if (!agx3_alu_hadd_minifloat(enc, I->dest[0].value, src.value,
+                      imm.value, src.neg, agx3_post_ra_hint(src),
+                      I->dest[0].cache, wait, bytes, &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
          if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_HADD &&
              I->nr_dests == 1 && I->nr_srcs == 2 && I->saturate &&
              I->src[1].type == AGX_INDEX_IMMEDIATE) {
@@ -13910,7 +15773,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          /* Same for the saturating half add. The saturating half add against
           * an immediate is handled above; this is the register/register one. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_HADD &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_HADD &&
              I->nr_dests == 1 && I->nr_srcs == 2) {
             uint16_t src[2];
             enum agx3_alu_hint hints[2];
@@ -13921,15 +15784,22 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             for (unsigned i = 0; i < 2; ++i) {
                agx_index x = I->src[i];
                if (!agx3_src_is_register(x) || x.memory ||
-                   x.size != AGX_SIZE_16 || agx_channels(x) != 1 || x.abs ||
-                   x.neg || x.kill || (x.cache && x.discard))
+                   x.size != AGX_SIZE_16 || agx_channels(x) != 1 || (x.abs && enc != AGX_ISA_ENCODING_AGX3C) ||
+                   (x.neg && (enc != AGX_ISA_ENCODING_AGX3C || I->saturate)) || x.kill || (x.cache && x.discard))
                   AGX3_REFUSE(I);
                src[i] = x.value;
                hints[i] = agx3_post_ra_hint(x);
             }
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
-            const bool encoded =
-               I->saturate ? agx3_alu_hadd_saturate(enc, dst.value, src, hints,
+            const bool src_neg[] = {I->src[0].neg, I->src[1].neg};
+            const bool src_abs[] = {I->src[0].abs, I->src[1].abs};
+            const bool encoded = enc == AGX_ISA_ENCODING_AGX3C
+               ? agx3_alu_hadd_mod_m5(dst.value, src, hints, src_abs, src_neg,
+                                       dst.cache, I->saturate, wait, bytes, &len)
+               : !I->saturate && (src_neg[0] || src_neg[1])
+               ? agx3_alu_hadd_signed(enc, dst.value, src, hints, src_neg,
+                                       dst.cache, wait, bytes, &len)
+               : I->saturate ? agx3_alu_hadd_saturate(enc, dst.value, src, hints,
                                                     dst.cache, wait, bytes, &len)
                            : agx3_alu_hadd(enc, dst.value, src, hints,
                                            dst.cache, wait, bytes, &len);
@@ -13938,11 +15808,15 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_HFMA &&
-             I->nr_dests == 1 && I->nr_srcs == 3 && !I->saturate) {
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_HFMA &&
+             I->nr_dests == 1 && I->nr_srcs == 3 &&
+             (!I->saturate || enc == AGX_ISA_ENCODING_AGX3C)) {
             const bool immediate_addend =
                I->src[2].type == AGX_INDEX_IMMEDIATE;
             if (immediate_addend) {
+               if (I->saturate)
+                  AGX3_REFUSE(I);
                const agx_index dst = I->dest[0];
                const agx_index addend = I->src[2];
                uint16_t src[2];
@@ -14005,8 +15879,12 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              * the extended endpoint's independent wait mask. */
 
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
-            if (!agx3_alu_hfma(enc, dst.value, src, hints, product_neg,
-                               I->src[2].neg, dst.cache, wait, bytes, &len) ||
+            const bool encoded = I->saturate
+               ? agx3_alu_hfma_saturate_m5(dst.value, src, hints, product_neg,
+                                          I->src[2].neg, dst.cache, wait, bytes, &len)
+               : agx3_alu_hfma(enc, dst.value, src, hints, product_neg,
+                               I->src[2].neg, dst.cache, wait, bytes, &len);
+            if (!encoded ||
                 !agx3_append(emission, bytes, len))
                AGX3_REFUSE(I);
             continue;
@@ -14014,7 +15892,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          /* Native scalar f32 x f16 -> f32. Keep the half register distinct
           * from its neighbour, preserve post-RA hints and producer waits. */
-         if (I->op == AGX_OPCODE_FMUL && I->nr_dests == 1 &&
+         if (enc == AGX_ISA_ENCODING_AGX3B &&
+             I->op == AGX_OPCODE_FMUL && I->nr_dests == 1 &&
              I->nr_srcs == 2 && !I->saturate &&
              agx3_src_is_register(I->dest[0]) &&
              !I->dest[0].memory &&
@@ -14081,7 +15960,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          /* Float to a 16-bit integer. The source width is its own field, so
           * a half source is the same instruction. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CONVERT &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_CONVERT &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              I->round == AGX_ROUND_RTZ &&
              I->src[0].type == AGX_INDEX_IMMEDIATE &&
@@ -14095,7 +15975,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
              !I->src[1].abs && !I->src[1].neg && !I->src[1].kill &&
              !(I->src[1].cache && I->src[1].discard)) {
             uint16_t convert_wait = wait;
-            if (convert_wait != 0 && convert_wait != 32) {
+            if (enc == AGX_ISA_ENCODING_AGX3B && convert_wait != 0 && convert_wait != 32) {
                if (!agx3_retire_result_wait(emission, enc, I, convert_wait))
                   AGX3_REFUSE(I);
                convert_wait = 0;
@@ -14115,7 +15995,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          /* shaderFloat16 commonly converts invocation/index values directly
           * from signed or unsigned32 into a scalar half. This has a native
           * result-width form distinct from the established i32-to-f32 form. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CONVERT &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_CONVERT &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              I->round == AGX_ROUND_RTE &&
              I->src[0].type == AGX_INDEX_IMMEDIATE &&
@@ -14143,7 +16024,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * converts without an extension. Apple has no unsigned byte to half
           * form: `half(uchar)` masks the byte in place and converts the
           * resulting ushort, so that sequence is emitted here too. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CONVERT &&
+          if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_CONVERT &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              I->round == AGX_ROUND_RTE &&
              I->src[0].type == AGX_INDEX_IMMEDIATE &&
@@ -14154,7 +16036,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 I->dest[0].size == AGX_SIZE_32)) ||
               ((I->src[0].value == AGX_CONVERT_S16_TO_F ||
                 I->src[0].value == AGX_CONVERT_U16_TO_F) &&
-               I->dest[0].size == AGX_SIZE_16))) {
+               (I->dest[0].size == AGX_SIZE_16 ||
+                (enc == AGX_ISA_ENCODING_AGX3C && I->dest[0].size == AGX_SIZE_32))))) {
             const agx_index dest = I->dest[0], src = I->src[1];
             if (!agx3_src_is_register(dest) || dest.memory ||
                 agx_channels(dest) != 1 || !agx3_src_is_register(src) ||
@@ -14196,8 +16079,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          * value would change both register pressure and rounding. Preserve
          * the signed/unsigned mode, post-RA lifetime/cache state and all six
          * producer dependencies in the native conversion itself. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CONVERT &&
-             I->nr_srcs == 2 &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_CONVERT && I->nr_srcs == 2 &&
              I->src[0].type == AGX_INDEX_IMMEDIATE &&
              (I->src[0].value == AGX_CONVERT_F_TO_S32 ||
               I->src[0].value == AGX_CONVERT_F_TO_U32) &&
@@ -14253,7 +16136,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FADD &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_FADD &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1) {
             const bool swap=I->src[0].type == AGX_INDEX_IMMEDIATE;
@@ -14298,6 +16181,80 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
+         /* G17G multiplies a scalar half by a minifloat straight into a
+          * float (port/hmulwit); a negated half folds into the minifloat's
+          * sign, exactly as for the float-source multiply below. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FMUL &&
+             I->nr_dests == 1 && I->nr_srcs == 2 &&
+             agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1 &&
+             (I->src[0].type == AGX_INDEX_IMMEDIATE) !=
+                (I->src[1].type == AGX_INDEX_IMMEDIATE)) {
+            const bool reverse = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            agx_index imm = I->src[reverse ? 0 : 1];
+            agx_index src = I->src[reverse ? 1 : 0];
+            if (agx3_half_reg(src) && (!src.kill || src.discard) &&
+                !(src.cache && src.discard) && agx3_plain_immediate(imm) &&
+                imm.value <= UINT8_MAX && !imm.abs && !imm.neg) {
+               if (src.neg) {
+                  imm.value ^= 0x80;
+                  src.neg = false;
+               }
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+               if (!agx3_alu_fmul_half_minifloat_m5(
+                      I->dest[0].value, src.value, imm.value, I->dest[0].cache,
+                      src.abs, I->saturate, agx3_post_ra_hint(src), wait, bytes,
+                      &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_vary_reg32(I->dest[0])) {
+            const bool reverse = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index imm = I->src[reverse ? 0 : 1];
+            const agx_index src = I->src[reverse ? 1 : 0];
+            if (agx3_half_reg(src) && !src.kill && !(src.cache && src.discard) &&
+                agx_channels(src) == 1 && agx3_plain_immediate(imm) &&
+                imm.value <= UINT8_MAX && !imm.abs && !imm.neg) {
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+               if (!agx3_alu_half_add_imm_to_f32_m5(I->dest[0].value, src.value,
+                      imm.value, src.abs, src.neg, agx3_post_ra_hint(src),
+                      I->dest[0].cache, wait, bytes, &len) ||
+                   !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 &&
+             (agx3_vary_reg32(I->dest[0]) || agx3_half_reg(I->dest[0])) && agx_channels(I->dest[0]) == 1) {
+            const bool swap = I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index src = I->src[swap ? 1 : 0];
+            const agx_index imm = I->src[swap ? 0 : 1];
+            if (agx3_mod_src_ok(src) && agx_channels(src) == 1 &&
+                imm.type == AGX_INDEX_IMMEDIATE && imm.value <= UINT8_MAX &&
+                !imm.memory && !imm.abs && !imm.neg && !imm.cache &&
+                !imm.discard && !imm.kill && agx_channels(imm) == 1) {
+               const struct agx3_falu_runtime runtime = {
+                  .dest_cache = I->dest[0].cache, .wait_mask = wait,
+                  .cache = {src.cache}, .discard = {src.discard},
+               };
+               uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], length;
+               if (!(I->dest[0].size == AGX_SIZE_16
+                     ? agx3_falu_add_imm_narrow_m5(I->dest[0].value, src.value, imm.value,
+                           src.abs, src.neg, I->saturate, &runtime, bytes, &length)
+                     : agx3_falu_add_imm_m5(I->dest[0].value, src.value, imm.value,
+                           src.abs, src.neg, I->saturate, &runtime, bytes, &length)) ||
+                   !agx3_append(emission, bytes, length))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
          /* Pixel-centre +0.5 and coordinate scaling use minifloat sources,
           * not GPRs. Native f32add/f32mul accept this pair directly, with the
           * register's last-use hint and the compact form's supported waits.
@@ -14310,7 +16267,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          if ((I->op == AGX_OPCODE_FADD || I->op == AGX_OPCODE_FMUL) &&
              I->nr_dests == 1 &&
              I->nr_srcs == 2 &&
-             (I->op == AGX_OPCODE_FADD || !I->saturate) &&
+             (I->op == AGX_OPCODE_FADD || !I->saturate ||
+              enc == AGX_ISA_ENCODING_AGX3C) &&
              agx3_src_is_register(I->dest[0]) &&
              (I->dest[0].size == AGX_SIZE_32 ||
               (I->op == AGX_OPCODE_FMUL &&
@@ -14328,7 +16286,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             /* Use the already-supported signed minifloat encoding for -r * k.
              * This changes neither the multiplication nor its rounding, and
              * leaves ABS, widths, cache/discard hints and waits intact. */
-            if (enc == AGX_ISA_ENCODING_AGX3B && multiply && reg.neg &&
+            if ((enc == AGX_ISA_ENCODING_AGX3B ||
+                 enc == AGX_ISA_ENCODING_AGX3C) && multiply && reg.neg &&
                 imm.value <= UINT8_MAX && !imm.memory && !imm.abs && !imm.neg) {
                imm.value ^= 0x80;
                reg.neg = false;
@@ -14388,7 +16347,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 * destination as a retained intermediate; the plain FMUL then
                 * consumes it in place. This adds no register, spill, wait, or
                 * serialization and preserves the source's real lifetime. */
-               if (reg.abs) {
+               if (reg.abs && enc != AGX_ISA_ENCODING_AGX3C) {
                   uint8_t abs_bytes[AGX3_ALU_MAX_LENGTH], abs_len = 0;
                   uint8_t mul_bytes[AGX3_ALU_MAX_LENGTH], mul_len = 0;
                   if (I->dest[0].size != AGX_SIZE_32 ||
@@ -14408,13 +16367,15 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 * destination widths, so a multi-slot wait needs the identity
                 * prelude here exactly as it does on the compact ADD below. */
                uint16_t encoded_wait = wait;
-               if (!agx3_fuse_compact_minifloat_wait(emission, enc, I,
+               if (enc == AGX_ISA_ENCODING_AGX3B &&
+                   !agx3_fuse_compact_minifloat_wait(emission, enc, I,
                                                      &encoded_wait))
                   AGX3_REFUSE(I);
                encoded = agx3_alu_fmul_minifloat_narrow(
                   enc, I->dest[0].value, reg.value, imm.value,
-                  I->dest[0].cache, false,
-                  I->dest[0].size == AGX_SIZE_16, agx3_post_ra_hint(reg),
+                  I->dest[0].cache, reg.abs,
+                  I->dest[0].size == AGX_SIZE_16, I->saturate,
+                  agx3_post_ra_hint(reg),
                   encoded_wait, bytes, &len);
             } else if (reg.abs) {
                encoded = agx3_alu_fadd_abs_minifloat(
@@ -14439,6 +16400,31 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITOP &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_half_reg(I->dest[0])) {
+            const unsigned tt = I->truth_table & 15;
+            const bool immediate_first =
+               I->src[0].type == AGX_INDEX_IMMEDIATE;
+            const agx_index imm = I->src[immediate_first ? 0 : 1];
+            const agx_index reg = I->src[immediate_first ? 1 : 0];
+            const bool complement = tt == 2 || tt == 4;
+            if ((tt == AGX3_SEL_TT_AND || complement) &&
+                imm.type == AGX_INDEX_IMMEDIATE && imm.value <= 255 &&
+                !imm.memory && !imm.abs && !imm.neg &&
+                agx3_half_reg(reg) && !reg.abs && !reg.neg &&
+                !(reg.cache && reg.discard)) {
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+               if (!agx3_alu_and_imm16_m5(
+                      I->dest[0].value, reg.value, imm.value, complement,
+                      complement && ((tt == 4) != immediate_first),
+                      agx3_post_ra_hint(reg), I->dest[0].cache, wait,
+                      bytes, &len) || !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
          /* A narrow NOT is Apple's unary 16-bit `inv`; the bitop's second
           * source is ignored by the truth table. */
          if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BITOP &&
@@ -14460,7 +16446,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BITOP &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_BITOP &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              I->dest[0].size == AGX_SIZE_16 &&
              (I->src[0].size == AGX_SIZE_16 || agx3_vary_reg32(I->src[0])) &&
@@ -14491,7 +16478,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BITOP &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_BITOP &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_vary_reg32(I->dest[0]) &&
              agx_channels(I->dest[0]) == 1 &&
@@ -14510,16 +16497,69 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                const enum agx3_alu_hint hints[2]={
                   agx3_post_ra_hint(I->src[0]),agx3_post_ra_hint(I->src[1])};
                uint8_t bytes[AGX3_ALU_MAX_LENGTH],len=0;
-               const bool encoded =
-                  agx3_alu_bit_reg_native(enc, op, I->dest[0].value, src,
-                                          hints, I->dest[0].cache, wait,
-                                          bytes, &len);
+               const bool encoded = enc == AGX_ISA_ENCODING_AGX3C
+                  ? agx3_alu_bit_reg(enc, op, I->dest[0].value, src,
+                                      hints, wait, bytes, &len)
+                  : agx3_alu_bit_reg_native(enc, op, I->dest[0].value, src,
+                                             hints, I->dest[0].cache, wait,
+                                             bytes, &len);
                if (!encoded || !agx3_append(emission,bytes,len))
                   AGX3_REFUSE(I);
                continue;
             }
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITOP &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             I->shift == 0 && agx3_vary_reg32(I->dest[0]) &&
+             agx3_half_reg(I->src[0]) && agx3_half_reg(I->src[1]) &&
+             !I->src[0].abs && !I->src[0].neg &&
+             !I->src[1].abs && !I->src[1].neg &&
+             !I->src[0].kill && !I->src[1].kill &&
+             !(I->src[0].cache && I->src[0].discard) &&
+             !(I->src[1].cache && I->src[1].discard)) {
+            const unsigned tt = I->truth_table & 15;
+            if (tt == AGX3_SEL_TT_AND || tt == AGX3_SEL_TT_OR ||
+                tt == AGX3_SEL_TT_XOR) {
+               const uint16_t src[2] = {I->src[0].value, I->src[1].value};
+               const enum agx3_alu_hint hints[2] = {
+                  agx3_post_ra_hint(I->src[0]), agx3_post_ra_hint(I->src[1])};
+               uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+               if (!agx3_alu_bit_half_to_word_m5(
+                      tt == AGX3_SEL_TT_AND ? AGX3_ALU_BAND :
+                      tt == AGX3_SEL_TT_OR ? AGX3_ALU_BOR : AGX3_ALU_BXOR,
+                      I->dest[0].value, src, hints, I->dest[0].cache, wait,
+                      bytes, &len) || !agx3_append(emission, bytes, len))
+                  AGX3_REFUSE(I);
+               continue;
+            }
+         }
+
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITOP &&
+             ((I->truth_table & 0xfu) == AGX3_SEL_TT_AND ||
+              (I->truth_table & 0xfu) == AGX3_SEL_TT_OR ||
+              (I->truth_table & 0xfu) == AGX3_SEL_TT_XOR) &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_vary_reg32(I->dest[0]) &&
+             agx_channels(I->dest[0]) == 1 &&
+             ((agx3_vary_reg32(I->src[0]) && agx3_half_reg(I->src[1])) ||
+              (agx3_half_reg(I->src[0]) && agx3_vary_reg32(I->src[1]))) &&
+             agx_channels(I->src[0]) == 1 && agx_channels(I->src[1]) == 1) {
+            const bool word_first = I->src[0].size == AGX_SIZE_32;
+            const agx_index word = I->src[word_first ? 0 : 1];
+            const agx_index half = I->src[word_first ? 1 : 0];
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+            if (!agx3_alu_bit_mixed_m5(
+                    (I->truth_table & 0xfu) == AGX3_SEL_TT_AND ? AGX3_ALU_BAND :
+                    (I->truth_table & 0xfu) == AGX3_SEL_TT_OR ? AGX3_ALU_BOR : AGX3_ALU_BXOR,
+                    I->dest[0].value, word.value, half.value,
+                    agx3_post_ra_hint(word), agx3_post_ra_hint(half),
+                    I->dest[0].cache, wait, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* AGX3b's compact 32-bit move cannot address full registers above
           * r95; the 16-bit bitwise-immediate form addresses every half, so
           * such a move is two ordered `x | 0` halves. The producer wait goes
@@ -14559,9 +16599,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * `x ^ mask` are that instruction with one field changed -- not a
           * separate form, and not something to fail a shader closed over.
           * AND/OR/XOR all commute, so an immediate in either source slot
-          * lands in the one slot whose kind field takes it.
+          * lands in the one slot whose kind field takes it. AGX3C has the
+          * same point (native G17G witnesses in agx3_alu_bit_imm()).
           */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BITOP &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B ||
+              enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_BITOP &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
              agx_channels(I->dest[0]) == 1 &&
@@ -14601,7 +16643,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * source-discard bit and no cache fields. That destroys a value that
           * remains live for a following compare. Encode the actual post-RA
           * lifetime state through the native flt2int fields instead. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CONVERT &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_CONVERT &&
              I->nr_srcs == 2 &&
              I->src[0].type == AGX_INDEX_IMMEDIATE &&
              (I->src[0].value == AGX_CONVERT_F_TO_S32 ||
@@ -14630,7 +16673,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_CONVERT &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_CONVERT &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1 &&
              agx3_vary_reg32(I->src[1]) && agx_channels(I->src[1]) == 1) {
@@ -14642,14 +16686,55 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
                if (!agx3_alu_int_to_float(enc, I->dest[0].value, I->src[1].value,
                        S.op == AGX3_SEL_OP_I32_TO_F,
-                       agx3_post_ra_hint(I->src[1]), wait, bytes, &len) ||
+                       agx3_post_ra_hint(I->src[1]), I->dest[0].cache,
+                       wait, bytes, &len) ||
                    !agx3_append(emission, bytes, len))
                   AGX3_REFUSE(I);
                continue;
             }
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FFMA &&
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_FFMA &&
+             I->nr_dests == 1 && I->nr_srcs == 3 &&
+             (agx3_vary_reg32(I->dest[0]) || agx3_half_reg(I->dest[0])) && agx_channels(I->dest[0]) == 1 &&
+             agx3_mod_src_any_width(I->src[0]) &&
+             agx3_mod_src_any_width(I->src[1]) &&
+             (I->dest[0].size == AGX_SIZE_16 || I->saturate || I->src[0].size == AGX_SIZE_16 ||
+              I->src[1].size == AGX_SIZE_16) &&
+             agx3_plain_immediate(I->src[2]) && I->src[2].value <= UINT8_MAX) {
+            const struct agx3_falu_mods m = {
+               .src0_abs = I->src[0].abs, .src0_neg = I->src[0].neg,
+               .src0_16 = I->src[0].size == AGX_SIZE_16,
+               .src1_abs = I->src[1].abs, .src1_neg = I->src[1].neg,
+               .src1_16 = I->src[1].size == AGX_SIZE_16,
+               .saturate = I->saturate,
+            };
+            const struct agx3_falu_runtime runtime = {
+               .cache = {I->src[0].cache, I->src[1].cache},
+               .discard = {I->src[0].discard, I->src[1].discard},
+               .dest_cache = I->dest[0].cache, .wait_mask = wait,
+            };
+            uint8_t addend = I->src[2].value;
+            if (I->src[2].abs)
+               addend &= 0x7f;
+            if (I->src[2].neg)
+               addend ^= 0x80;
+            uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], length;
+            if (I->src[0].kill || I->src[1].kill ||
+                !(I->dest[0].size == AGX_SIZE_16
+                   ? agx3_falu_fma_addend_imm_narrow_m5(
+                        I->dest[0].value, I->src[0].value, I->src[1].value,
+                        addend, &m, &runtime, bytes, &length)
+                   : agx3_falu_fma_addend_imm_width_mod(
+                        enc, I->dest[0].value, I->src[0].value, I->src[1].value,
+                        addend, &m, &runtime, bytes, &length)) ||
+                !agx3_append(emission, bytes, length))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FFMA &&
              !I->saturate && I->nr_dests == 1 && I->nr_srcs == 3 &&
              agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1 &&
              agx3_mod_src_ok(I->src[0]) &&
@@ -14685,7 +16770,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FFMA &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FFMA &&
              !I->saturate && I->nr_dests == 1 && I->nr_srcs == 3 &&
              agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1 &&
              I->src[2].type == AGX_INDEX_IMMEDIATE && I->src[2].value <= 255 &&
@@ -14748,9 +16834,12 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * `k * fabs(a) + fabs(b)` is one instruction. A modifier on the
           * immediate is folded into the minifloat (fma(0.5, -b, c) has
           * f34 = -0.5). */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FFMA &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FFMA &&
              I->nr_dests == 1 && I->nr_srcs == 3 &&
-             agx3_vary_reg32(I->dest[0]) && agx_channels(I->dest[0]) == 1 &&
+             (agx3_vary_reg32(I->dest[0]) ||
+              (enc == AGX_ISA_ENCODING_AGX3C && agx3_half_reg(I->dest[0]))) &&
+             agx_channels(I->dest[0]) == 1 &&
              (I->src[0].type == AGX_INDEX_IMMEDIATE) !=
                 (I->src[1].type == AGX_INDEX_IMMEDIATE)) {
             const bool imm_first = I->src[0].type == AGX_INDEX_IMMEDIATE;
@@ -14791,9 +16880,13 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                   .wait_mask = wait,
                };
                uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], length;
-               if (!agx3_falu_fma_mul_imm_mod(enc, I->dest[0].value,
-                       multiplier, reg.value, add.value, &m, &runtime,
-                       bytes, &length) ||
+               const bool narrow = I->dest[0].size == AGX_SIZE_16;
+               if (!(narrow ? agx3_falu_fma_mul_imm_narrow_m5(
+                                 I->dest[0].value, multiplier, reg.value,
+                                 add.value, &m, &runtime, bytes, &length)
+                            : agx3_falu_fma_mul_imm_mod(enc, I->dest[0].value,
+                                 multiplier, reg.value, add.value, &m,
+                                 &runtime, bytes, &length)) ||
                    !agx3_append(emission, bytes, length))
                   AGX3_REFUSE(I);
                continue;
@@ -14803,8 +16896,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          /* The legacy generic packers embed specimen discard flags. Keep
           * G17P's plain register ALU on the post-RA-aware path so an alpha or
           * blend factor reused by later channels is not discarded early. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && !I->saturate &&
-             (I->op == AGX_OPCODE_FADD || I->op == AGX_OPCODE_FMUL ||
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             !I->saturate && (I->op == AGX_OPCODE_FADD || I->op == AGX_OPCODE_FMUL ||
               I->op == AGX_OPCODE_FFMA)) {
             bool plain = I->nr_dests == 1 &&
                          agx3_vary_reg32(I->dest[0]) &&
@@ -14844,7 +16937,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             }
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_ASR &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_ASR &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              I->src[1].type == AGX_INDEX_IMMEDIATE &&
              I->src[1].value >= 1 &&
@@ -14853,15 +16947,22 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const bool half = dest.size == AGX_SIZE_16 &&
                               src.size == AGX_SIZE_16;
             const bool word = agx3_vary_reg32(dest) && agx3_vary_reg32(src);
-            if ((!half && !word) ||
-                I->src[1].value > (half ? 15 : 31) ||
+            const bool mixed = enc == AGX_ISA_ENCODING_AGX3C &&
+                               dest.size == AGX_SIZE_32 && src.size == AGX_SIZE_16;
+            if ((!half && !word && !mixed) ||
+                I->src[1].value > (src.size == AGX_SIZE_16 ? 15 : 31) ||
                 !agx3_src_is_register(dest) || dest.memory ||
                 !agx3_src_is_register(src) || src.memory ||
                 src.abs || src.neg || agx_channels(dest)!=1 || agx_channels(src)!=1)
                AGX3_REFUSE(I);
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
-            const bool encoded = half
-               ? agx3_alu_asr_imm16(enc,dest.value,src.value,I->src[1].value,
+            const bool encoded = enc == AGX_ISA_ENCODING_AGX3C
+               ? agx3_alu_asr_imm_m5(dest.value,src.value,
+                                     dest.size == AGX_SIZE_16,
+                                     src.size == AGX_SIZE_16,I->src[1].value,
+                                     agx3_post_ra_hint(src),dest.cache,wait,
+                                     bytes,&length)
+               : half ? agx3_alu_asr_imm16(enc,dest.value,src.value,I->src[1].value,
                                      agx3_post_ra_hint(src),dest.cache,wait,
                                      bytes,&length)
                : agx3_alu_asr_imm32(enc,dest.value,src.value,I->src[1].value,
@@ -14873,6 +16974,32 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_ASR &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_src_is_register(I->dest[0]) &&
+             agx3_src_is_register(I->src[0]) &&
+             agx3_src_is_register(I->src[1]) &&
+             I->dest[0].size >= I->src[0].size &&
+             I->dest[0].size <= AGX_SIZE_32 && I->src[1].size == AGX_SIZE_32) {
+            const agx_index dest = I->dest[0], value = I->src[0], shift = I->src[1];
+            if (dest.memory || value.memory || shift.memory ||
+                value.abs || value.neg || shift.abs || shift.neg ||
+                agx_channels(dest) != 1 || agx_channels(value) != 1 ||
+                agx_channels(shift) != 1 || value.kill || shift.kill ||
+                (value.cache && value.discard) || (shift.cache && shift.discard))
+               AGX3_REFUSE(I);
+            const enum agx3_alu_hint hints[] = {
+               agx3_post_ra_hint(value), agx3_post_ra_hint(shift),
+            };
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
+            if (!agx3_alu_asr_reg_m5(dest.value, value.value, shift.value,
+                    dest.size == AGX_SIZE_16, value.size == AGX_SIZE_16,
+                    hints, dest.cache, wait, bytes, &length) ||
+                !agx3_append(emission, bytes, length))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* A dynamic signed-short shift stays narrow on G17P. Apple emits a
           * single mixed-width SRA: half value/result and a full register
           * count. Preserve the two source lifetimes and producer waits. */
@@ -14911,7 +17038,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          /* Native immediate-count logical shifts do not require a GPR
           * holding the count. Keep the existing masked/register forms below. */
-         if (enc == AGX_ISA_ENCODING_AGX3B &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
              (I->op == AGX_OPCODE_BFI || I->op == AGX_OPCODE_BFEIL) &&
              I->nr_dests == 1 && I->nr_srcs == 3 && I->bfi_mask == 0 &&
              agx3_src_is_zero_imm(I->src[0]) &&
@@ -14944,7 +17071,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * native `sll` takes an immediate value. Apple reads a half count
           * here; NIR's uint count register supplies its low half. A cached
           * count keeps its register through the clear last-use bit. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BFI &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B ||
+              enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_BFI &&
              I->nr_dests == 1 && I->nr_srcs == 3 && I->bfi_mask == 0 &&
              agx3_src_is_zero_imm(I->src[0]) &&
              I->src[1].type == AGX_INDEX_IMMEDIATE &&
@@ -14969,14 +17097,46 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             const enum agx3_alu_hint shift_hint =
                shift.cache ? AGX3_ALU_HINT_NONE : agx3_post_ra_hint(shift);
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], length = 0;
-            if (!agx3_alu_sll_one_reg32(enc, dest.value, shift.value,
-                                        shift_hint, dest.cache, wait, bytes,
-                                        &length) ||
+            if (!(enc == AGX_ISA_ENCODING_AGX3C
+                     ? agx3_alu_sll_one_m5(dest.value, true, shift.value,
+                                           shift_hint, dest.cache, wait,
+                                           bytes, &length)
+                     : agx3_alu_sll_one_reg32(enc, dest.value, shift.value,
+                                              shift_hint, dest.cache, wait,
+                                              bytes, &length)) ||
                 !agx3_append(emission, bytes, length))
                AGX3_REFUSE(I);
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BFI &&
+             I->nr_dests == 1 && I->nr_srcs == 3 && !I->bfi_mask &&
+             !I->saturate && agx3_src_is_zero_imm(I->src[0]) &&
+             agx3_src_is_register(I->dest[0]) &&
+             agx3_src_is_register(I->src[1]) &&
+             agx3_src_is_register(I->src[2]) &&
+             I->dest[0].size <= AGX_SIZE_32 &&
+             I->src[1].size <= AGX_SIZE_32 && I->src[2].size <= AGX_SIZE_32 &&
+             agx_channels(I->dest[0]) == 1 &&
+             agx_channels(I->src[1]) == 1 && agx_channels(I->src[2]) == 1 &&
+             !I->src[1].abs && !I->src[1].neg &&
+             !I->src[2].abs && !I->src[2].neg) {
+            const uint16_t sources[] = {I->src[1].value, I->src[2].value};
+            const bool half[] = {I->src[1].size == AGX_SIZE_16,
+                                 I->src[2].size == AGX_SIZE_16};
+            const enum agx3_alu_hint hints[] = {
+               agx3_post_ra_hint(I->src[1]),
+               I->src[2].cache ? AGX3_ALU_HINT_NONE : agx3_post_ra_hint(I->src[2]),
+            };
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            if (!agx3_alu_sll_reg(enc, I->dest[0].value,
+                  I->dest[0].size == AGX_SIZE_16, I->dest[0].cache,
+                  sources, half, hints, wait, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* A register-count ushort shift is one native `sll`. The shift
           * operand remains uint in NIR; Apple's mixed-width witness pins
           * source A at width 1 and source B at width 2. */
@@ -15019,6 +17179,28 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BFI &&
+             I->nr_dests == 1 && I->nr_srcs == 3 && !I->bfi_mask && !I->saturate &&
+             agx3_src_is_zero_imm(I->src[0]) && agx3_half_reg(I->dest[0]) &&
+             agx3_half_reg(I->src[1]) && !I->src[1].abs && !I->src[1].neg &&
+             I->src[2].type == AGX_INDEX_IMMEDIATE && I->src[2].value <= 8) {
+            const unsigned amount = I->src[2].value;
+            const bool split = amount > 4;
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            if (!agx3_alu_iadd16_gpu1(I->dest[0].value, split || I->dest[0].cache,
+                    true, 0, AGX3_ALU_HINT_NONE, I->src[1].value,
+                    agx3_post_ra_hint(I->src[1]), split ? amount - 4 : amount,
+                    wait, bytes, &len) || !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            if (split &&
+                (!agx3_alu_iadd16_gpu1(I->dest[0].value, I->dest[0].cache,
+                    true, 0, AGX3_ALU_HINT_NONE, I->dest[0].value,
+                    AGX3_ALU_HINT_DISCARD, 4, 0, bytes, &len) ||
+                 !agx3_append(emission, bytes, len)))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /*
           * G17P has no single 16-bit shift-left form beyond four bits. Apple
           * lowers ushort << {5,6,7,8} to two witnessed shift-adds: first
@@ -15081,7 +17263,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_EXTR &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_EXTR &&
              I->nr_dests==1 && I->nr_srcs==3 && I->bfi_mask==0 &&
              I->src[2].type==AGX_INDEX_IMMEDIATE &&
              I->src[2].value>=1 && I->src[2].value<=31 &&
@@ -15105,8 +17288,9 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * special-register read.  Preserve that narrow value through the
           * exact immediate SRL instead of materializing a full-width shift
           * count, which selects a different G17P instruction domain. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BFEIL &&
-             I->nr_dests == 1 && I->nr_srcs == 3 && I->bfi_mask == 0 &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_BFEIL && I->nr_dests == 1 &&
+             I->nr_srcs == 3 && I->bfi_mask == 0 &&
              agx3_src_is_zero_imm(I->src[0]) &&
              I->src[2].type == AGX_INDEX_IMMEDIATE &&
              I->src[2].value >= 1 && I->src[2].value <= 31 &&
@@ -15134,6 +17318,36 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BFEIL &&
+             I->nr_dests == 1 && I->nr_srcs == 3 && I->bfi_mask == 0 &&
+             I->imm == 0 && agx3_src_is_zero_imm(I->src[0]) &&
+             agx3_src_is_register(I->src[2])) {
+            const agx_index dest = I->dest[0], value = I->src[1];
+            const agx_index shift = I->src[2];
+            if (!agx3_src_is_register(dest) || dest.memory ||
+                (dest.size != AGX_SIZE_16 && dest.size != AGX_SIZE_32) ||
+                agx_channels(dest) != 1 || !agx3_src_is_register(value) ||
+                value.memory || value.abs || value.neg || value.kill ||
+                (value.size != AGX_SIZE_16 && value.size != AGX_SIZE_32) ||
+                agx_channels(value) != 1 || shift.memory ||
+                shift.size != AGX_SIZE_32 || agx_channels(shift) != 1 ||
+                shift.abs || shift.neg || shift.kill ||
+                (value.cache && value.discard) || (shift.cache && shift.discard))
+               AGX3_REFUSE(I);
+            const uint16_t src[2] = {value.value, shift.value};
+            const bool narrow[2] = {value.size == AGX_SIZE_16, false};
+            const enum agx3_alu_hint hints[2] = {
+               agx3_post_ra_hint(value),
+               shift.cache ? AGX3_ALU_HINT_NONE : agx3_post_ra_hint(shift),
+            };
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+            if (!agx3_alu_srl_reg(enc, dest.value, dest.size == AGX_SIZE_16,
+                                  dest.cache, src, narrow, hints, wait, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          /* Dynamic ushort right-shifts use the same mixed-width register
           * domain as SLL. agx_emit_alu has already changed the ordinary NIR
           * count into G17P's signed bitfield boundary. */
@@ -15176,40 +17390,54 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BFEIL &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_BFEIL &&
              I->nr_dests == 1 && I->nr_srcs == 3 && I->imm >= 1 && I->imm <= 31 &&
              agx3_src_is_zero_imm(I->src[0])) {
-            const agx_index dest=I->dest[0], value=I->src[1], shift=I->src[2];
-            const bool reg_shift=agx3_src_is_register(shift);
-            if (!agx3_src_is_register(dest) || dest.memory || dest.size != AGX_SIZE_32 ||
-                agx_channels(dest) != 1 || !agx3_src_is_register(value) || value.memory ||
-                value.size != AGX_SIZE_32 || agx_channels(value) != 1 || value.abs || value.neg ||
-                shift.memory || shift.abs || shift.neg || agx_channels(shift) != 1 ||
-                (reg_shift ? shift.size != AGX_SIZE_32 : shift.type != AGX_INDEX_IMMEDIATE))
+            /* Read packed fields in place: GCC can otherwise read channel
+             * bits from a partially populated local operand copy. */
+            const agx_index *dest = &I->dest[0];
+            const agx_index *value = &I->src[1];
+            const agx_index *shift = &I->src[2];
+            const bool reg_shift=agx3_src_is_register(*shift);
+            if (!agx3_src_is_register(*dest) || dest->memory || dest->size != AGX_SIZE_32 ||
+                agx_channels(*dest) != 1 || !agx3_src_is_register(*value) || value->memory ||
+                (value->size != AGX_SIZE_32 &&
+                 !(enc == AGX_ISA_ENCODING_AGX3C && value->size == AGX_SIZE_16)) || agx_channels(*value) != 1 || value->abs || value->neg ||
+                shift->memory || shift->abs || shift->neg || agx_channels(*shift) != 1 ||
+                (reg_shift ? (shift->size != AGX_SIZE_32 &&
+                 !(enc == AGX_ISA_ENCODING_AGX3C && shift->size == AGX_SIZE_16))
+                 : shift->type != AGX_INDEX_IMMEDIATE))
                AGX3_REFUSE(I);
             const enum agx3_alu_hint hints[2] = {
-               agx3_post_ra_hint(value), reg_shift ? agx3_post_ra_hint(shift) : AGX3_ALU_HINT_NONE,
+               agx3_post_ra_hint(*value), reg_shift ? agx3_post_ra_hint(*shift) : AGX3_ALU_HINT_NONE,
             };
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
-            if (!agx3_alu_bfeil(enc,dest.value,value.value,shift.value,reg_shift,
-                                I->imm,hints,wait,bytes,&length) ||
+            const bool encoded = enc == AGX_ISA_ENCODING_AGX3C
+               ? agx3_alu_bfeil_m5(dest->value,value->value,value->size == AGX_SIZE_16,
+                    shift->value,reg_shift,reg_shift && shift->size == AGX_SIZE_16,
+                    I->imm,hints,wait,bytes,&length)
+               : agx3_alu_bfeil(enc,dest->value,value->value,shift->value,reg_shift,
+                                I->imm,hints,wait,bytes,&length);
+            if (!encoded ||
                 !agx3_append(emission,bytes,length))
                AGX3_REFUSE(I);
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FFS &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FFS &&
              I->nr_dests == 1 && I->nr_srcs == 1) {
             const agx_index dest = I->dest[0], src = I->src[0];
             if (!agx3_vary_reg32(dest) || !agx3_vary_reg32(src) ||
                 agx_channels(dest) != 1 || agx_channels(src) != 1 ||
-                dest.value == src.value || src.abs || src.neg)
+                (enc == AGX_ISA_ENCODING_AGX3B && dest.value == src.value) || src.abs || src.neg)
                AGX3_REFUSE(I);
             /* Apple's G17P clz FT1 carries wait0 and the cached destination
              * form (op-enable 3). The uncached/op-enable-2 combination with
              * a discarded source exposes the internal logarithmic stage
              * instead of the completed bit index. */
-            const uint16_t ft1_wait = wait | 1u;
+            const uint16_t ft1_wait = enc == AGX_ISA_ENCODING_AGX3B ? wait | 1u : wait;
             uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
             if (!agx3_alu_ft1_u32(enc, dest.value, src.value,
                                   agx3_post_ra_hint(src),
@@ -15219,7 +17447,21 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
             continue;
          }
 
-         if (enc == AGX_ISA_ENCODING_AGX3B &&
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITREV &&
+             I->nr_dests == 1 && I->nr_srcs == 1 &&
+             agx3_vary_reg32(I->dest[0]) &&
+             (agx3_vary_reg32(I->src[0]) || agx3_half_reg(I->src[0])) &&
+             !I->src[0].abs && !I->src[0].neg) {
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
+            if (!agx3_alu_bitrev_m5(I->dest[0].value, I->src[0].value,
+                    I->src[0].size == AGX_SIZE_16,
+                    agx3_post_ra_hint(I->src[0]), I->dest[0].cache, wait,
+                    bytes, &length) || !agx3_append(emission, bytes, length))
+               AGX3_REFUSE(I);
+            continue;
+         }
+
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
              I->op == AGX_OPCODE_POPCOUNT && I->nr_dests == 1 &&
              I->nr_srcs == 1) {
             const agx_index dest = I->dest[0], src = I->src[0];
@@ -15265,7 +17507,28 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
           * their own two-register instructions; the one-sided forms take
           * their operands in order. */
          const unsigned bit_tt = I->truth_table & 0xfu;
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_BITOP &&
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITOP &&
+             (bit_tt == 2 || bit_tt == 4 || bit_tt == 11 || bit_tt == 13) &&
+             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+             agx3_half_reg(I->dest[0]) && agx3_half_reg(I->src[0]) &&
+             agx3_half_reg(I->src[1]) && !I->src[0].abs && !I->src[0].neg &&
+             !I->src[1].abs && !I->src[1].neg) {
+            const bool swap = bit_tt == 4 || bit_tt == 13;
+            const agx_index a = I->src[swap ? 1 : 0];
+            const agx_index b = I->src[swap ? 0 : 1];
+            const uint16_t src[] = {a.value, b.value};
+            const enum agx3_alu_hint hints[] = {
+               agx3_post_ra_hint(a), agx3_post_ra_hint(b)};
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+            if (!agx3_alu_complement_reg16_m5(
+                   I->dest[0].value, src, hints, bit_tt == 11 || bit_tt == 13,
+                   I->dest[0].cache, wait, bytes, &len) ||
+                !agx3_append(emission, bytes, len))
+               AGX3_REFUSE(I);
+            continue;
+         }
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_BITOP &&
              (bit_tt == 0x1u || bit_tt == 0x2u || bit_tt == 0x4u ||
               bit_tt == 0x7u || bit_tt == 0x9u || bit_tt == 0xbu ||
               bit_tt == 0xdu) &&
@@ -15275,11 +17538,15 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                AGX3_REFUSE(I);
             const bool swap = bit_tt == 0x4u || bit_tt == 0xdu;
             uint16_t src[2];
+            bool half[2];
             enum agx3_alu_hint hints[2];
             for (unsigned i = 0; i < 2; ++i) {
                const agx_index x = I->src[swap ? 1 - i : i];
-               if (!agx3_vary_reg32(x) || agx_channels(x) != 1 || x.abs ||
-                   x.neg)
+               half[i] = x.size == AGX_SIZE_16;
+               if (!(agx3_vary_reg32(x) ||
+                     (enc == AGX_ISA_ENCODING_AGX3C &&
+                      (bit_tt == 2 || bit_tt == 4) && agx3_half_reg(x))) ||
+                   agx_channels(x) != 1 || x.abs || x.neg)
                   AGX3_REFUSE(I);
                src[i] = x.value;
                hints[i] = agx3_post_ra_hint(x);
@@ -15293,8 +17560,11 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                break;
             case 0x2u:
             case 0x4u:
-               encoded = agx3_alu_bit_andc(enc, dest.value, src, hints,
-                                           dest.cache, wait, bytes, &length);
+               encoded = enc == AGX_ISA_ENCODING_AGX3C
+                  ? agx3_alu_bit_andc_width_m5(dest.value, src, half, hints,
+                                              dest.cache, wait, bytes, &length)
+                  : agx3_alu_bit_andc(enc, dest.value, src, hints,
+                                      dest.cache, wait, bytes, &length);
                break;
             case 0x7u:
                encoded = agx3_alu_bit_nand(enc, dest.value, src, hints,
@@ -15316,9 +17586,12 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
 
          /* Half negation is encoded as -src + -0.0h. This exact f16add.ep2
           * form keeps the source lifetime and producer wait in one native
-          * instruction, including the matrix-inverse and faceforward paths. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FADD &&
-             I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
+          * instruction, including the matrix-inverse and faceforward paths.
+          * On AGX3C saturate(half) is the same form with its saturate bit. */
+         if ((enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C) &&
+             I->op == AGX_OPCODE_FADD &&
+             I->nr_dests == 1 && I->nr_srcs == 2 &&
+             (!I->saturate || enc == AGX_ISA_ENCODING_AGX3C) &&
              agx3_src_is_register(I->dest[0]) && !I->dest[0].memory &&
              I->dest[0].size == AGX_SIZE_16 &&
              agx_channels(I->dest[0]) == 1) {
@@ -15330,13 +17603,19 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                 !imm.memory && !imm.abs && !imm.neg &&
                 agx3_src_is_register(reg) && !reg.memory &&
                 reg.size == AGX_SIZE_16 && agx_channels(reg) == 1 &&
-                !reg.abs && reg.neg && !reg.kill &&
+                ((enc == AGX_ISA_ENCODING_AGX3C) || (!reg.abs && reg.neg)) &&
+                !reg.kill &&
                 !(reg.cache && reg.discard)) {
                uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
-               if (!agx3_alu_hadd_neg_negzero(
+               bool encoded = enc == AGX_ISA_ENCODING_AGX3C
+                  ? agx3_alu_half_add_negzero_m5(
+                       I->dest[0].value, reg.value, reg.abs, reg.neg,
+                       I->saturate, agx3_post_ra_hint(reg), I->dest[0].cache, wait, bytes, &len)
+                  : agx3_alu_hadd_neg_negzero(
                       enc, I->dest[0].value, reg.value,
                       agx3_post_ra_hint(reg), I->dest[0].cache, wait,
-                      bytes, &len) ||
+                      bytes, &len);
+               if (!encoded ||
                    !agx3_append(emission, bytes, len))
                   AGX3_REFUSE(I);
                continue;
@@ -15346,7 +17625,8 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          /* Apple keeps exact half constants in scalar halves through an f32
           * add (widening first would cost a full register). Addition
           * commutes, so either IR order maps to the native f32/f16 order. */
-         if (enc == AGX_ISA_ENCODING_AGX3B && I->op == AGX_OPCODE_FADD &&
+         if ((enc == AGX_ISA_ENCODING_AGX3B ||
+              enc == AGX_ISA_ENCODING_AGX3C) && I->op == AGX_OPCODE_FADD &&
              I->nr_dests == 1 && I->nr_srcs == 2 && !I->saturate &&
              agx3_vary_reg32(I->dest[0]) &&
              agx_channels(I->dest[0]) == 1 &&
@@ -15403,6 +17683,42 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          if (mod < 0)
             AGX3_REFUSE(I);
 
+         /* AGX3C NOT of a half or a 32-bit register that the selector cannot
+          * encode is C's unary `inv` (the bitop's second source is ignored by
+          * the truth table). The selector's own forms are kept wherever they
+          * exist, so only the former refusals change. */
+         if (enc == AGX_ISA_ENCODING_AGX3C && I->op == AGX_OPCODE_BITOP &&
+             (I->truth_table & 0xfu) == AGX3_SEL_TT_NOT &&
+             I->nr_dests == 1 && I->nr_srcs >= 1 && !I->saturate &&
+             I->dest[0].size == I->src[0].size &&
+             (I->dest[0].size == AGX_SIZE_16 ||
+              I->dest[0].size == AGX_SIZE_32) &&
+             agx_channels(I->dest[0]) == 1 && agx_channels(I->src[0]) == 1 &&
+             agx3_src_is_register(I->dest[0]) &&
+             agx3_src_is_register(I->src[0]) && !I->src[0].abs &&
+             !I->src[0].neg && !I->src[0].kill &&
+             !(I->src[0].cache && I->src[0].discard)) {
+            struct agx3_sel_instr S;
+            struct util_dynarray trial;
+            util_dynarray_init(&trial, NULL);
+            const bool selected = agx3_translate_instr(I, &S) &&
+                                  agx3_emit_alu(&trial, enc, &S, wait);
+            uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+            const bool ok = selected
+               ? trial.size <= UINT8_MAX &&
+                    agx3_append(emission, (const uint8_t *)trial.data,
+                                (uint8_t)trial.size)
+               : (agx3_alu_bit_not_m5(I->dest[0].value, I->src[0].value,
+                                      I->dest[0].size == AGX_SIZE_32,
+                                      agx3_post_ra_hint(I->src[0]),
+                                      I->dest[0].cache, wait, bytes, &len) &&
+                  agx3_append(emission, bytes, len));
+            util_dynarray_fini(&trial);
+            if (!ok)
+               AGX3_REFUSE(I);
+            continue;
+         }
+
          struct agx3_sel_instr S;
          if (!agx3_translate_instr(I, &S) ||
              !agx3_emit_alu(emission, enc, &S, wait)) {
@@ -15420,6 +17736,14 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
       }
    }
 
+   if (enc == AGX_ISA_ENCODING_AGX3C && fragment_ray_query &&
+       g17p_colourless_fragment && ray_query_acquired) {
+      uint8_t release[AGX3_PIXEL_MAX_LENGTH], release_len;
+      if (!agx3_pixel_release(enc, 4, false, 0, release, &release_len) ||
+          !agx3_append(emission, release, release_len))
+         AGX3_PACK_FAIL();
+   }
+
    if (early_compare) {
       const unsigned early =
          agx3_apply_early_compares(ctx, emission, initial_size, &spans,
@@ -15428,7 +17752,18 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
          fprintf(stderr, "AGX3 early loop compares: %u\n", early);
    }
 
+   /* Removing an already-open pixel entry is safe when both ends of each
+    * relative branch move together. Control flow elsewhere in the part does
+    * not require retaining a duplicate entry or padding its former bytes. */
+   ctx->agx3_colour_link.entry_unspanned =
+      ctx->agx3_colour_link.entry_emitted;
+   const off_t entry_start = initial_size + ctx->agx3_colour_link.entry_offset;
+   const off_t entry_end = entry_start + AGX3_PIXEL_MAX_LENGTH;
    util_dynarray_foreach(&fixups, struct agx3_branch_fixup, fixup) {
+      const off_t target = agx3_fixup_target(fixup);
+      const bool before = fixup->offset < entry_start && target < entry_start;
+      const bool after = fixup->offset >= entry_end && target >= entry_end;
+      ctx->agx3_colour_link.entry_unspanned &= before || after;
       if (!agx3_fixup_branch(emission, enc, *fixup)) {
          if (agx_compiler_debug & AGX_DBG_SHADERS)
             fprintf(stderr,
@@ -15438,7 +17773,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
                     (long long)fixup->block->offset,
                     (long long)fixup->block->last_offset, fixup->skip_to_end,
                     fixup->loop_exit, fixup->is_bn);
-         goto fail;
+         AGX3_PACK_FAIL();
       }
    }
 
@@ -15455,7 +17790,7 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
     * in-part consumers add no instructions. */
    if (ctx->key->no_stop && !ctx->is_preamble && outstanding != 0 &&
        !agx3_retire_slots_noop(emission, enc, outstanding))
-      goto fail;
+      AGX3_PACK_FAIL();
 
    if (ctx->key->dev.chip == AGX_CHIP_G17P &&
        enc == AGX_ISA_ENCODING_AGX3B &&
@@ -15475,9 +17810,22 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    }
 
    if (!ctx->key->no_stop || ctx->is_preamble) {
+      /* Native G17G's complete empty fragment still opens and releases the
+       * pixel transaction. An empty no-stop main remains empty so its linked
+       * epilog owns this terminal protocol. No registers or staging are used. */
+      if (enc == AGX_ISA_ENCODING_AGX3C &&
+          ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_preamble &&
+          count == 0 && emission->size == initial_size) {
+         uint8_t bytes[AGX3_PIXEL_MAX_LENGTH], len;
+         if (!agx3_pixel_colour_entry(enc, bytes, &len) ||
+             !agx3_append(emission, bytes, len) ||
+             !agx3_pixel_release(enc, 0, true, 0, bytes, &len) ||
+             !agx3_append(emission, bytes, len))
+            AGX3_PACK_FAIL();
+      }
       const struct agx3_sel_instr end = {.op = AGX3_SEL_OP_END};
       if (!agx3_emit(emission, enc, &end))
-         goto fail;
+         AGX3_PACK_FAIL();
    }
 
    util_dynarray_fini(&spans);
@@ -15494,3 +17842,5 @@ agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission)
    util_dynarray_fini(&fixups);
    return false;
 }
+
+#undef AGX3_PACK_FAIL
diff --git a/src/asahi/compiler/agx3_pixel_core.c b/src/asahi/compiler/agx3_pixel_core.c
index 715b8c9e6611..57dd907416d9 100644
--- a/src/asahi/compiler/agx3_pixel_core.c
+++ b/src/asahi/compiler/agx3_pixel_core.c
@@ -73,11 +73,14 @@ agx3_pixel_colour_entry(enum agx_isa_encoding enc,
 
    /* G17P is AGX3b. The pair below is Apple's normalized composer input for
     * the exact 87 02 54 00 06 00 entry gate. */
-   if (enc != AGX_ISA_ENCODING_AGX3B)
+   if (!enc_ok(enc))
       return false;
 
    const struct agx3_field_kv fields[] = {KV(14, 512), KV(15, 1)};
-   if (!agx3_bind_compose(enc, "pixwait", 0, fields, NELEM(fields), NULL, 0,
+   const struct agx3_field_kv base_c[] = {KV(16, 3), KV(17, 122)};
+   if (!agx3_bind_compose(enc, "pixwait", 0, fields, NELEM(fields),
+                          is_b(enc) ? NULL : base_c,
+                          is_b(enc) ? 0 : NELEM(base_c),
                           out, len)) {
       memset(out, 0, AGX3_PIXEL_MAX_LENGTH);
       *len = 0;
@@ -87,6 +90,29 @@ agx3_pixel_colour_entry(enum agx_isa_encoding enc,
    return true;
 }
 
+bool
+agx3_pixel_colour_entry_zs_unreleased(enum agx_isa_encoding enc,
+                                      uint8_t out[AGX3_PIXEL_MAX_LENGTH],
+                                      uint8_t *len)
+{
+   if (out == NULL || len == NULL)
+      return false;
+   memset(out, 0, AGX3_PIXEL_MAX_LENGTH);
+   *len = 0;
+   if (enc != AGX_ISA_ENCODING_AGX3C)
+      return false;
+
+   const struct agx3_field_kv fields[] = {KV(14, 512), KV(15, 1), KV(34, 1)};
+   const struct agx3_field_kv base_c[] = {KV(16, 3), KV(17, 122)};
+   if (!agx3_bind_compose(enc, "pixwait", 0, fields, NELEM(fields), base_c,
+                          NELEM(base_c), out, len)) {
+      memset(out, 0, AGX3_PIXEL_MAX_LENGTH);
+      *len = 0;
+      return false;
+   }
+   return true;
+}
+
 /* metal-nt -arch applegpu_g17p, a fragment function with no colour writes
  * storing the result of atomic_fetch_add_explicit():
  *
@@ -104,12 +130,16 @@ agx3_pixel_atomic_acquire(enum agx_isa_encoding enc,
       return false;
    memset(out, 0, AGX3_PIXEL_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B)
+   if (!enc_ok(enc))
       return false;
 
-   const struct agx3_field_kv fields[] = {KV(14, 4)};
-   if (!agx3_bind_compose(enc, "pixwait", 0, fields, NELEM(fields), NULL, 0,
-                          out, len)) {
+   const struct agx3_field_kv fields[] = {KV(14, 4), KV(34, 0)};
+   const struct agx3_field_kv base_c[] = {KV(16, 3), KV(17, 122)};
+   if (!agx3_bind_compose(enc, "pixwait", 0,
+                          is_b(enc) ? fields : base_c,
+                          is_b(enc) ? 1 : NELEM(base_c),
+                          is_b(enc) ? NULL : fields,
+                          is_b(enc) ? 0 : NELEM(fields), out, len)) {
       memset(out, 0, AGX3_PIXEL_MAX_LENGTH);
       *len = 0;
       return false;
@@ -161,6 +191,8 @@ agx3_pixel_wait(enum agx_isa_encoding enc, uint32_t writeout,
                 uint16_t wait_mask, uint8_t out[AGX3_PIXEL_MAX_LENGTH],
                 uint8_t *len)
 {
+   /* G17P/G17G sparse 4x MRT adds 204 (RT0+RT2) and 240 (RT1+RT2),
+    * with matching final releases. Keep their single native instruction. */
    /* Every writeout mask Apple's fragment corpus emitted on pixwait, less the
     * f15-gated shader-entry form (512). */
    static const uint32_t witnessed[] = {1,   4,   8,   12, 16, 32, 48,  60, 64,
@@ -172,7 +204,8 @@ agx3_pixel_wait(enum agx_isa_encoding enc, uint32_t writeout,
    *len = 0;
 
    if (!enc_ok(enc) || !wait_mask_ok(enc, wait_mask) ||
-       !in_set(writeout, witnessed, NELEM(witnessed)))
+       !(in_set(writeout, witnessed, NELEM(witnessed)) ||
+         (writeout == 204 || writeout == 240)))
       return false;
 
    /* f34 separates the colour writeout classes from the depth/coverage class:
@@ -210,7 +243,9 @@ agx3_pixel_release(enum agx_isa_encoding enc, uint32_t writeout, bool last,
    *len = 0;
 
    if (!enc_ok(enc) || !wait_mask_ok(enc, wait_mask) ||
-       !in_set(writeout, witnessed, NELEM(witnessed)))
+       !(in_set(writeout, witnessed, NELEM(witnessed)) ||
+         (enc == AGX_ISA_ENCODING_AGX3C && writeout == 4) ||
+         (writeout == 204 || writeout == 240)))
       return false;
 
    const struct agx3_field_kv base_c[] = {KV(15, 1), KV(16, 121)};
diff --git a/src/asahi/compiler/agx3_pixel_core.h b/src/asahi/compiler/agx3_pixel_core.h
index 31b27709a985..bf43536278bc 100644
--- a/src/asahi/compiler/agx3_pixel_core.h
+++ b/src/asahi/compiler/agx3_pixel_core.h
@@ -41,6 +41,16 @@ extern "C" {
  */
 bool agx3_pixel_colour_entry(enum agx_isa_encoding enc,
                              uint8_t out[AGX3_PIXEL_MAX_LENGTH], uint8_t *len);
+/*
+ * The same entry gate with f34 set: 077000050000 on AGX3c. Every native G17G
+ * fragment that writes depth or stencil alone at several samples opens with
+ * it, and exactly those never release the depth/coverage class explicitly
+ * (no pixrel 1); every other witnessed fragment uses the plain gate. What
+ * the bit selects in hardware is not decoded. AGX3c only.
+ */
+bool agx3_pixel_colour_entry_zs_unreleased(enum agx_isa_encoding enc,
+                                           uint8_t out[AGX3_PIXEL_MAX_LENGTH],
+                                           uint8_t *len);
 
 /*
  * Bracket a colourless fragment's returning memory atomic. Apple's G17P
diff --git a/src/asahi/compiler/agx3_rt_gen1_triangle.c b/src/asahi/compiler/agx3_rt_gen1_triangle.c
index 94b0786fb90d..737271981415 100644
--- a/src/asahi/compiler/agx3_rt_gen1_triangle.c
+++ b/src/asahi/compiler/agx3_rt_gen1_triangle.c
@@ -51,11 +51,19 @@ static void subnormal_coordinate(struct stream *s,unsigned coordinate){
  assert(agx3_alu_shift_imm32(AGX_ISA_ENCODING_AGX3B,false,98,98,24,AGX3_ALU_HINT_NONE,true,0,bytes,&n));append(s,bytes,n,"coordinate-exponent",-1);
  isel(s,97,98,0,false,97,false,0,true,"coordinate-is-subnormal");setflag(s,32768,"preserve-native-subnormal-coordinate");
 }
-static void prepare_policy(struct stream *s){
+/* Edge and vertex ownership of the record's triangle A = (v0, v1, v2), or
+ * of triangle B = (v2, v1, v3), whose projected vertex v3 occupies v0's
+ * registers when B is tested. B's edges run the other way round: an edge
+ * shared with a neighbour has opposite directions in the two triangles, so
+ * exactly one of them owns it. Projected coordinates: v0/v3 (0, 18),
+ * v1 (14, 2), v2 (4, 22). */
+static void prepare_policy(struct stream *s,bool second){
  imm16(s,96,0,"ownership-state-init");
  subnormal_coordinate(s,0);subnormal_coordinate(s,18);subnormal_coordinate(s,14);subnormal_coordinate(s,2);subnormal_coordinate(s,4);subnormal_coordinate(s,22);
- edge(s,14,2,4,22,8);edge(s,4,22,0,18,16);edge(s,0,18,14,2,32);
- vertex(s,0,14,4,64);vertex(s,14,4,0,128);vertex(s,4,0,14,256);
+ if(!second){edge(s,14,2,4,22,8);edge(s,4,22,0,18,16);edge(s,0,18,14,2,32);
+  vertex(s,0,14,4,64);vertex(s,14,4,0,128);vertex(s,4,0,14,256);}
+ else{edge(s,4,22,14,2,8);edge(s,0,18,4,22,16);edge(s,14,2,0,18,32);
+  vertex(s,0,4,14,64);vertex(s,14,0,4,128);vertex(s,4,14,0,256);}
  zero_factor(s,2,4,512);zero_factor(s,22,14,1024);zero_factor(s,22,0,2048);zero_factor(s,18,4,4096);zero_factor(s,18,14,8192);zero_factor(s,2,0,16384);
 }
 static void finite_product(struct stream *s,unsigned product){
@@ -73,28 +81,91 @@ static void residual_zero(struct stream *s,unsigned p0,unsigned p1,unsigned q0,u
  bitreg(s,AGX3_ALU_BAND,97,97,99,"both-products-qualified");finite_product(s,p0);finite_product(s,p1);
  imm16(s,98,flag,"exact-edge-mask");bitreg(s,AGX3_ALU_BAND,98,96,98,"exact-edge-test");isel(s,97,97,0,true,98,false,0,true,"unsafe-exact-edge");setflag(s,32768,"preserve-native-unsafe-edge");
 }
-static void finish_policy(struct stream *s){
- isel(s,99,72,0,true,0,true,1,true,"save-native-valid");
+static void finish_policy(struct stream *s,unsigned valid){
+ isel(s,99,valid,0,true,0,true,1,true,"save-native-valid");
  bitimm(s,97,96,9,"u-zero-and-eligible");isel(s,97,97,9,true,1,true,0,true,"u-owned");
  bitimm(s,98,96,18,"v-zero-and-eligible");isel(s,97,98,18,true,1,true,97,false,"v-owned-or-u");
  bitimm(s,98,96,36,"w-zero-and-eligible");isel(s,97,98,36,true,1,true,97,false,"w-owned-or-uv");
- bitimm(s,98,96,7,"zero-edge-mask");isel(s,97,98,0,true,1,true,97,false,"no-zero-edge-preserves-native");bitreg(s,AGX3_ALU_BAND,72,72,97,"edge-owner-valid");
+ bitimm(s,98,96,7,"zero-edge-mask");isel(s,97,98,0,true,1,true,97,false,"no-zero-edge-preserves-native");bitreg(s,AGX3_ALU_BAND,valid,valid,97,"edge-owner-valid");
  bitimm(s,97,96,7,"vertex-zero-mask");
- bitimm(s,98,96,64,"vertex-a-owner");isel(s,98,97,6,true,98,false,1,true,"vertex-a-only-on-vw-zero");isel(s,72,98,0,true,0,true,72,false,"vertex-a-valid");
- bitimm(s,98,96,128,"vertex-b-owner");isel(s,98,97,5,true,98,false,1,true,"vertex-b-only-on-uw-zero");isel(s,72,98,0,true,0,true,72,false,"vertex-b-valid");
- imm16(s,98,256,"vertex-c-mask");bitreg(s,AGX3_ALU_BAND,98,96,98,"vertex-c-owner");isel(s,98,97,3,true,98,false,1,true,"vertex-c-only-on-uv-zero");isel(s,72,98,0,true,0,true,72,false,"vertex-c-valid");
- imm16(s,98,32768,"unsafe-edge-mask");bitreg(s,AGX3_ALU_BAND,98,96,98,"unsafe-edge-test");isel(s,72,98,0,true,72,false,99,false,"qualified-policy-or-native");isel(s,72,97,7,true,99,false,72,false,"degenerate-projection-preserves-native");
+ bitimm(s,98,96,64,"vertex-a-owner");isel(s,98,97,6,true,98,false,1,true,"vertex-a-only-on-vw-zero");isel(s,valid,98,0,true,0,true,valid,false,"vertex-a-valid");
+ bitimm(s,98,96,128,"vertex-b-owner");isel(s,98,97,5,true,98,false,1,true,"vertex-b-only-on-uw-zero");isel(s,valid,98,0,true,0,true,valid,false,"vertex-b-valid");
+ imm16(s,98,256,"vertex-c-mask");bitreg(s,AGX3_ALU_BAND,98,96,98,"vertex-c-owner");isel(s,98,97,3,true,98,false,1,true,"vertex-c-only-on-uv-zero");isel(s,valid,98,0,true,0,true,valid,false,"vertex-c-valid");
+ imm16(s,98,32768,"unsafe-edge-mask");bitreg(s,AGX3_ALU_BAND,98,96,98,"unsafe-edge-test");isel(s,valid,98,0,true,valid,false,99,false,"qualified-policy-or-native");isel(s,valid,97,7,true,99,false,valid,false,"degenerate-projection-preserves-native");
+}
+static void fmul(struct stream *s,unsigned dest,unsigned a,unsigned b,const char *name){
+ const uint16_t src[3]={a,b,0};const enum agx3_alu_hint hints[3]={AGX3_ALU_HINT_NONE,AGX3_ALU_HINT_NONE,AGX3_ALU_HINT_NONE};
+ uint8_t bytes[AGX3_ALU_MAX_LENGTH],n;assert(agx3_alu_float_reg(AGX_ISA_ENCODING_AGX3B,AGX3_ALU_FMUL,dest,src,hints,0,bytes,&n));append(s,bytes,n,name,-1);
+}
+static void ffma(struct stream *s,unsigned dest,unsigned a,unsigned b,unsigned c,const char *name){
+ const uint16_t src[3]={a,b,c};const enum agx3_alu_hint hints[3]={AGX3_ALU_HINT_NONE,AGX3_ALU_HINT_NONE,AGX3_ALU_HINT_NONE};
+ uint8_t bytes[AGX3_ALU_MAX_LENGTH],n;assert(agx3_alu_float_reg(AGX_ISA_ENCODING_AGX3B,AGX3_ALU_FMA,dest,src,hints,0,bytes,&n));append(s,bytes,n,name,-1);
+}
+/* Rounded product a * b in p and its exact residual a * b - p in q. */
+static void two_product(struct stream *s,unsigned p,unsigned q,unsigned a,unsigned b){
+ fmul(s,p,a,b,"product");fmul(s,q,p,92,"negated-product");ffma(s,q,a,b,q,"product-residual");
+}
+/* Triangle B's exact path forms the products of its own edges v2-v3 and
+ * v1-v3 but not of the edge v1-v2 it shares with A; form those first, in
+ * registers the rest of the body never reads (84..93), while the projected
+ * coordinates are intact. */
+static void prepare_second(struct stream *s){
+ prepare_policy(s,true);
+ imm32(s,92,UINT32_C(0xbf800000),"negative-one");
+ two_product(s,84,86,2,4);two_product(s,88,90,22,14);
+ residual_zero(s,84,88,86,90,1,512,1024);
+}
+/* B's validity (16) when one of its edge functions is exactly zero. The
+ * native B test applies its own edge rule there, which disagrees with A's
+ * policy and leaves holes along the edges B shares, so decide from the
+ * policy alone: B owns an edge or vertex point by the ownership rules, and
+ * a point on one edge lies on B when its other two edge functions agree in
+ * sign (U 66, V' 44, W' 6 from the native B test). Unsafe products and a
+ * degenerate projection keep the native result, as for A. */
+static void owned(struct stream *s,unsigned mask,unsigned owner,const char *name){
+ if(owner<=255)bitimm(s,98,96,owner,name);else{imm16(s,98,owner,name);bitreg(s,AGX3_ALU_BAND,98,96,98,name);}
+ isel(s,98,98,0,true,0,true,1,true,name);isel(s,16,97,mask,true,98,false,16,false,name);
+}
+static void finish_second(struct stream *s){
+ isel(s,99,16,0,true,0,true,1,true,"save-native-valid");
+ bitimm(s,97,96,7,"zero-edge-mask");
+ imm32(s,92,0,"zero");
+ fsel(s,98,66,92,1,1,true,0,true,false,"u-negative");
+ fsel(s,94,6,92,1,1,true,0,true,false,"w-negative");bitreg(s,AGX3_ALU_BXOR,94,98,94,"u-w-disagree");
+ fsel(s,95,44,92,1,1,true,0,true,false,"v-negative");bitreg(s,AGX3_ALU_BXOR,95,98,95,"u-v-disagree");
+ bitimm(s,98,96,16,"v-edge-eligible");isel(s,98,98,0,true,0,true,1,true,"v-edge-owner");isel(s,98,94,0,true,98,false,0,true,"v-edge-point-on-b");isel(s,16,97,2,true,98,false,16,false,"v-edge-valid");
+ bitimm(s,98,96,32,"w-edge-eligible");isel(s,98,98,0,true,0,true,1,true,"w-edge-owner");isel(s,98,95,0,true,98,false,0,true,"w-edge-point-on-b");isel(s,16,97,4,true,98,false,16,false,"w-edge-valid");
+ bitreg(s,AGX3_ALU_BXOR,94,94,95,"v-w-disagree");
+ bitimm(s,98,96,8,"u-edge-eligible");isel(s,98,98,0,true,0,true,1,true,"u-edge-owner");isel(s,98,94,0,true,98,false,0,true,"u-edge-point-on-b");isel(s,16,97,1,true,98,false,16,false,"u-edge-valid");
+ owned(s,6,64,"vertex-v3");owned(s,5,128,"vertex-v1");owned(s,3,256,"vertex-v2");
+ /* U rounds to zero but is not exactly zero: its exact sign is that of the
+  * residual difference (84/88 products, 86/90 residuals, see
+  * prepare_second); the native B test does not form it. B holds the point
+  * when that sign agrees with V' and W'. */
+ fsel(s,95,86,90,1,1,true,0,true,false,"u-exact-negative");
+ fsel(s,94,44,92,1,1,true,0,true,false,"v-negative-again");bitreg(s,AGX3_ALU_BXOR,95,95,94,"u-v-disagree-exact");
+ fsel(s,98,6,92,1,1,true,0,true,false,"w-negative-again");bitreg(s,AGX3_ALU_BXOR,94,94,98,"v-w-disagree");
+ bitreg(s,AGX3_ALU_BOR,95,95,94,"exact-u-outside");isel(s,95,95,0,true,1,true,0,true,"exact-u-inside");
+ fsel(s,98,84,88,0,1,true,0,true,false,"u-rounds-to-zero");fsel(s,94,86,90,0,0,true,1,true,false,"u-not-exactly-zero");
+ bitreg(s,AGX3_ALU_BAND,98,98,94,"u-sign-from-residuals");isel(s,98,97,0,true,98,false,0,true,"no-exact-zero-edge");
+ isel(s,16,98,0,true,16,false,95,false,"exact-u-valid");
+ imm16(s,98,32768,"unsafe-edge-mask");bitreg(s,AGX3_ALU_BAND,98,96,98,"unsafe-edge-test");isel(s,16,98,0,true,16,false,99,false,"qualified-policy-or-native");isel(s,16,97,7,true,99,false,16,false,"degenerate-projection-preserves-native");
 }
 static void body(struct stream *s,bool corrected){
  size_t positions[TRI_COUNT];
  for(unsigned j=0;j<TRI_COUNT;j++){
   const struct agx3_rt_gen1_native_op *op=&agx3_rt_gen1_triangle_ops[j];positions[j]=s->size;
-  if(corrected&&op->offset==620)prepare_policy(s);
+  if(corrected&&op->offset==620)prepare_policy(s,false);
+  if(corrected&&op->offset==1096)prepare_second(s);
   uint8_t bytes[16],n;
   if(op->target!=UINT32_MAX)assert(agx3_cf_branch(AGX_ISA_ENCODING_AGX3B,true,bytes,&n));
   else assert(agx3_bind_compose(AGX_ISA_ENCODING_AGX3B,op->name,op->ep,op->fields,op->nfields,NULL,0,bytes,&n));
   assert(n==op->size);append(s,bytes,n,op->name,op->offset);
-  if(corrected){switch(op->offset){case 648:residual_zero(s,72,10,76,78,1,512,1024);break;case 700:residual_zero(s,80,82,84,86,2,2048,4096);break;case 722:residual_zero(s,88,18,90,92,4,8192,16384);break;case 908:finish_policy(s);break;default:break;}}
+  if(corrected){switch(op->offset){case 648:residual_zero(s,72,10,76,78,1,512,1024);break;case 700:residual_zero(s,80,82,84,86,2,2048,4096);break;case 722:residual_zero(s,88,18,90,92,4,8192,16384);break;case 908:finish_policy(s,72);break;
+   /* Triangle B: edge v2-v3 (products 38/16, residuals 24/4), edge v1-v3
+    * (76/14, residuals 48/2), then its validity, which the native code
+    * leaves in 16. */
+   case 1148:residual_zero(s,38,16,24,4,2,2048,4096);break;case 1170:residual_zero(s,76,14,48,2,4,8192,16384);break;case 1390:finish_second(s);break;default:break;}}
  }
  for(unsigned j=0;j<TRI_COUNT;j++)if(agx3_rt_gen1_triangle_ops[j].target!=UINT32_MAX){size_t at=positions[j],target=positions[agx3_rt_gen1_triangle_ops[j].target];assert(agx3_cf_patch_branch(AGX_ISA_ENCODING_AGX3B,true,s->bytes+at,(int64_t)target-(int64_t)at));}
 }
diff --git a/src/asahi/compiler/agx3_rt_m5_core.c b/src/asahi/compiler/agx3_rt_m5_core.c
new file mode 100644
index 000000000000..c95a87403281
--- /dev/null
+++ b/src/asahi/compiler/agx3_rt_m5_core.c
@@ -0,0 +1,368 @@
+/* SPDX-License-Identifier: MIT */
+#include "agx3_rt_m5_core.h"
+#include "agx3_binder.h"
+#include "agx3_pack.h"
+#include "agx3_scs_core.h"
+#include <string.h>
+
+#define KV(f, v) {(f), (v)}
+
+bool
+agx3_rt_m5_encode_enter_query(enum agx_isa_encoding encoding,
+                              uint16_t destination, uint16_t allocation_index,
+                              uint8_t out[16], uint8_t *length)
+{
+   if (!out || !length)
+      return false;
+   memset(out, 0, 16);
+   *length = 0;
+   if (encoding != AGX_ISA_ENCODING_AGX3C || destination > 252 ||
+       (destination & 1) || allocation_index < 4 ||
+       allocation_index >= 4 + AGX3_RT_M5_MAX_LIVE_QUERIES)
+      return false;
+   const struct agx3_field_kv fields[] = {
+      KV(2, destination), KV(9, 2),   KV(10, 2),
+      KV(14, 3),          KV(15, 90), KV(24, allocation_index),
+   };
+   return agx3_bind_compose(encoding, "incStk", 0, fields,
+                            sizeof(fields) / sizeof(fields[0]), NULL, 0, out,
+                            length);
+}
+
+bool
+agx3_rt_m5_encode_intersect(enum agx_isa_encoding encoding,
+                           const struct agx3_rt_m5_intersect_desc *d,
+                           uint8_t out[16], uint8_t *length)
+{
+   if (!out || !length)
+      return false;
+   memset(out, 0, 16);
+   *length = 0;
+   if (encoding != AGX_ISA_ENCODING_AGX3C || !d ||
+       d->address_register > 252 || (d->address_register & 1) ||
+       d->bvh_base > (d->bvh_uniform ? 508 : 252) ||
+       (d->bvh_base & 1) || (d->bvh_uniform && d->bvh_kill))
+      return false;
+
+   const struct agx3_field_kv base[] = {
+      KV(9, 5), KV(10, 2), KV(12, 8), KV(13, 1), KV(14, 2),
+      KV(24, 2), KV(25, 3), KV(28, 1), KV(29, 92), KV(30, 1),
+      KV(49, 32), KV(55, 3), KV(56, 3), KV(70, 2), KV(85, 1), KV(99, 1),
+   };
+   const struct agx3_field_kv operands[] = {
+      KV(17, d->address_register),
+      KV(d->bvh_uniform ? 63 : 62, d->bvh_base),
+      KV(69, d->bvh_uniform ? 3 : 2), KV(73, d->bvh_kill),
+   };
+   return agx3_bind_compose(encoding, "intersect_ray", 0, base,
+                           sizeof(base) / sizeof(base[0]), operands,
+                           sizeof(operands) / sizeof(operands[0]), out, length);
+}
+
+bool
+agx3_rt_m5_encode_continue(enum agx_isa_encoding encoding,
+                          uint8_t out[16], uint8_t *length)
+{
+   if (!out || !length)
+      return false;
+   memset(out, 0, 16);
+   *length = 0;
+   if (encoding != AGX_ISA_ENCODING_AGX3C)
+      return false;
+   const struct agx3_field_kv fields[] = {
+      KV(5, 8), KV(9, 5), KV(10, 2), KV(12, 8), KV(13, 1),
+      KV(14, 2), KV(15, 34), KV(16, 1),
+   };
+   return agx3_bind_compose(encoding, "continue_ray", 0, fields,
+                           sizeof(fields) / sizeof(fields[0]), NULL, 0,
+                           out, length);
+}
+
+bool
+agx3_rt_m5_ift_loader_generate(
+   const struct agx3_rt_m5_ift_loader_desc *d,
+   uint8_t out[AGX3_RT_M5_IFT_LOADER_MAX_SIZE], size_t *length)
+{
+   if (!d || !out || !length || !d->constants_va ||
+       (d->constants_va & 3) || (d->constants_va >> 48) ||
+       !d->shader_va || (d->shader_va & 63) || (d->shader_va >> 42) ||
+       d->fetch_span < 1 || d->fetch_span > 3 ||
+       (d->context48 ? (d->constant_dwords != 20 && d->constant_dwords != 24)
+                     : (d->constant_dwords < 2 || d->constant_dwords > 34 ||
+                        d->constant_dwords % 4 != 2)))
+      return false;
+
+   uint8_t staged[AGX3_RT_M5_IFT_LOADER_MAX_SIZE] = {0};
+   size_t used = 0;
+   const uint16_t base = d->context48 ? 28 : 24;
+   const uint16_t dest[] = {8, base, base + 2};
+   const uint32_t value[] = {0x6000, d->constants_va, d->constants_va >> 32};
+   for (unsigned i = 0; i < 3; ++i) {
+      struct agx3_encoded_ldimm imm;
+      if (!agx3_pack_ldimm_u32(AGX_ISA_ENCODING_AGX3C, dest[i], value[i], &imm))
+         return false;
+      memcpy(staged + used, imm.bytes, imm.length);
+      used += imm.length;
+   }
+
+   /* Context fields at +80 and, in the second layout, +48 are runtime
+    * inputs. Keep their numeric names until the wider SCS ABI is decoded. */
+   for (unsigned i = 0; i <= d->context48; ++i) {
+      struct agx3_scs_desc scs = {
+         .data = i ? 20 : base - 4,
+         .width = AGX3_MEM_W32,
+         .byte_offset = i ? 48 : 80,
+         .address_kind = AGX3_SCS_ADDRESS_REGISTER_BASE,
+         .base_register = 8, .component_mask = 3,
+         .has_address_selector = true, .address_selector = 2, .slot = 2,
+      };
+      uint8_t size;
+      if (!agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &scs,
+                               staged + used, &size))
+         return false;
+      used += size;
+   }
+
+   /* The native loader loads the high, possibly partial, constant chunk
+    * first. The final low chunk overwrites the address pair itself. f75=2
+    * is retained as an explicit Ioto load field; it is not reinterpreted as
+    * the generic shader-memory instruction's address scale. */
+   const unsigned chunks = (d->constant_dwords + 3) / 4;
+   for (unsigned c = chunks; c-- > 0;) {
+      unsigned words = d->constant_dwords - 4 * c;
+      if (words > 4) words = 4;
+      const struct agx3_field_kv fields[] = {
+         KV(2, base + 8 * c), KV(9, 2), KV(10, 2), KV(12, 2),
+         KV(23, 1), KV(24, 1), KV(31, base), KV(38, 2), KV(39, 3),
+         KV(45, 15), KV(53, 1), KV(54, 1), KV(59, 104),
+         KV(73, (1u << words) - 1), KV(75, 2), KV(76, 16 * c),
+      };
+      uint8_t size;
+      if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "lddev", 0,
+                             fields, sizeof(fields) / sizeof(fields[0]),
+                             NULL, 0, staged + used, &size))
+         return false;
+      used += size;
+   }
+
+   const struct agx3_field_kv local[] = {KV(0, 3), KV(1, 165)};
+   const struct agx3_field_kv shader[] = {
+      KV(0, d->shader_va), KV(1, d->fetch_span), KV(5, 3), KV(6, 107),
+      KV(23, 1),
+   };
+   uint8_t size;
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "specLM", 0,
+                          local, sizeof(local) / sizeof(local[0]),
+                          NULL, 0, staged + used, &size))
+      return false;
+   used += size;
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "ldshdr", 0,
+                          shader, sizeof(shader) / sizeof(shader[0]),
+                          NULL, 0, staged + used, &size))
+      return false;
+   used += size;
+   memcpy(out, staged, used);
+   *length = used;
+   return true;
+}
+
+bool
+agx3_rt_m5_ift_entry_generate(
+   const struct agx3_rt_m5_ift_entry_desc *d,
+   uint8_t out[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE], size_t *length)
+{
+   if (!d || !out || !length || !d->uniforms || d->uniform_count < 2 ||
+       d->uniform_count > AGX3_RT_M5_IFT_MAX_UNIFORMS)
+      return false;
+   for (unsigned i = 0; i < d->uniform_count; ++i) {
+      if ((d->uniforms[i].uniform_half & 1) ||
+          (d->uniforms[i].register_half & 1) ||
+          d->uniforms[i].uniform_half > 70 ||
+          d->uniforms[i].register_half > 90)
+         return false;
+   }
+   uint8_t staged[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE] = {0};
+   size_t used;
+   if (!agx3_rt_m5_ift_loader_generate(&d->loader, staged, &used))
+      return false;
+   uint8_t n;
+   const struct agx3_field_kv tpr[] = {
+      KV(0, 3), KV(1, 166), KV(20, d->tpr20 ? 20 : 0),
+      KV(21, d->tpr20 ? 20 : 0),
+   };
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "specTPR", 0, tpr,
+                          sizeof(tpr) / sizeof(tpr[0]), NULL, 0,
+                          staged + used, &n))
+      return false;
+   used += n;
+
+   /* Keep the complete native call setup, including the SR14 read. LDSHDR
+    * alone is not the end of this entry program. */
+   for (unsigned i = 0; i < 3; ++i) {
+      const unsigned dest = i == 1 ? 0 : 2;
+      const unsigned value = i == 0 ? 0 : i == 1 ? 1024 : 256;
+      const struct agx3_field_kv imm[] = {
+         KV(2, dest), KV(9, 2), KV(10, 2), KV(14, value),
+         KV(15, 3), KV(16, 105),
+      };
+      if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "ldimm", 2,
+                             imm, sizeof(imm) / sizeof(imm[0]), NULL, 0,
+                             staged + used, &n))
+         return false;
+      used += n;
+      if (i == 0) {
+         const struct agx3_field_kv sr[] = {
+            KV(2, 2), KV(9, 2), KV(10, 1), KV(12, 1), KV(21, 14),
+            KV(23, 7), KV(24, 1), KV(28, 4), KV(29, 144), KV(38, 1),
+         };
+         if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "rld", 1, sr,
+                                sizeof(sr) / sizeof(sr[0]), NULL, 0,
+                                staged + used, &n))
+            return false;
+         used += n;
+      } else if (i == 1) {
+         const struct agx3_field_kv add[] = {
+            KV(9, 2), KV(10, 2), KV(23, 2), KV(24, 2), KV(31, 2),
+            KV(38, 2), KV(39, 2), KV(44, 3), KV(45, 80), KV(46, 1),
+            KV(55, 4),
+         };
+         if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "iadd32", 0, add,
+                                sizeof(add) / sizeof(add[0]), NULL, 0,
+                                staged + used, &n))
+            return false;
+         used += n;
+      }
+   }
+   const struct agx3_field_kv call[] = {
+      KV(9, 2), KV(10, 3), KV(15, 2), KV(16, 27), KV(17, 1), KV(25, 1),
+   };
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "cndcall", 0, call,
+                          sizeof(call) / sizeof(call[0]), NULL, 0,
+                          staged + used, &n))
+      return false;
+   used += n;
+   for (unsigned i = 0; i < d->uniform_count; ++i) {
+      const struct agx3_field_kv copy[] = {
+         KV(3, d->uniforms[i].uniform_half), KV(9, 3), KV(10, 2),
+         KV(16, d->uniforms[i].register_half), KV(23, 2), KV(24, 2),
+         KV(27, 1), KV(28, 1), KV(29, 35), KV(31, i == 0),
+      };
+      if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "copy", 0, copy,
+                             sizeof(copy) / sizeof(copy[0]), NULL, 0,
+                             staged + used, &n))
+         return false;
+      used += n;
+   }
+   if (d->final_copy16) {
+      const struct agx3_field_kv copy[] = {
+         KV(9, 2), KV(10, 1), KV(23, 2), KV(24, 1), KV(28, 3),
+         KV(29, 35), KV(42, 1),
+      };
+      if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "copy", 1, copy,
+                             sizeof(copy) / sizeof(copy[0]), NULL, 0,
+                             staged + used, &n))
+         return false;
+      used += n;
+   }
+   struct agx3_encoded_end end;
+   if (!agx3_pack_end(AGX_ISA_ENCODING_AGX3C, &end))
+      return false;
+   memcpy(staged + used, end.bytes, end.length);
+   used += end.length;
+   memcpy(out, staged, used);
+   *length = used;
+   return true;
+}
+
+/* The type-1 handler has no geometry fetch or triangle intersection. It
+ * commits the hardware candidate in the per-lane packet, publishes continuation
+ * state, and executes the shared traversal return. SR57/SR58/SR7 are kept
+ * numbered: interpreting them more broadly is not needed to encode this ABI. */
+bool
+agx3_rt_m5_hit_commit_generate(
+   const struct agx3_rt_m5_hit_commit_desc *d,
+   uint8_t out[AGX3_RT_M5_HIT_COMMIT_MAX_SIZE], size_t *length)
+{
+   if (!d || !out || !length || d->payload_uniform > 508 ||
+       d->context6_uniform > 510 || d->context8_uniform > 510 ||
+       ((d->payload_uniform | d->context6_uniform | d->context8_uniform) & 1))
+      return false;
+   uint8_t staged[AGX3_RT_M5_HIT_COMMIT_MAX_SIZE];
+   size_t used = 0;
+#define OP(name, ep, ...) do { \
+   const struct agx3_field_kv fields[] = {__VA_ARGS__}; \
+   uint8_t n; \
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, name, ep, fields, \
+                          sizeof(fields) / sizeof(fields[0]), NULL, 0, \
+                          staged + used, &n)) return false; \
+   used += n; \
+} while (0)
+#define RLD(dst, sr, slot, ep, waits) \
+   OP("rld", ep, KV(2,dst), KV(9,2), KV(10,1), KV(11,waits), KV(12,slot), \
+      KV(21,sr), KV(23,7), KV(24,1), KV(28,4), KV(29,144), KV(38,1))
+   RLD(0,58,1,0,0);
+   /* Extract the lane's 64-byte packet offset. */
+   OP("bit_and",1,KV(2,6),KV(9,2),KV(10,1),KV(11,1),KV(23,2),KV(24,1),
+      KV(27,1),KV(31,7),KV(37,3),KV(38,1),KV(42,3),KV(43,10),KV(44,1));
+   OP("imadd32",0,KV(2,10),KV(9,2),KV(10,2),KV(16,6),KV(23,2),KV(24,1),
+      KV(30,64),KV(38,1),KV(39,1),KV(53,1),KV(54,1),KV(59,3),KV(60,84));
+   struct agx3_scs_desc scs = {
+      .data=2,.width=AGX3_MEM_W32,.byte_offset=48,
+      .address_kind=AGX3_SCS_ADDRESS_UNIFORM_INDEXED,.index=10,
+      .uniform_base=d->payload_uniform,.component_mask=1,
+      .has_address_selector=true,.address_selector=0,.slot=8,
+      .c_load_marker=true,
+   };
+   uint8_t n;
+   if (!agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C,&scs,staged+used,&n))
+      return false;
+   used += n;
+   RLD(4,57,1,0,0);
+   RLD(8,7,2,1,1);
+   OP("ldimm",2,KV(9,2),KV(10,2),KV(14,0x40000000),KV(15,3),KV(16,105));
+   OP("imadd32",0,KV(2,4),KV(9,2),KV(10,2),KV(16,4),KV(23,2),KV(24,1),
+      KV(27,1),KV(32,d->context6_uniform),KV(38,3),KV(39,1),
+      KV(53,1),KV(54,1),KV(59,3),KV(60,84),KV(61,1));
+   OP("bit_or",1,KV(2,2),KV(9,2),KV(10,2),KV(11,1),KV(16,2),KV(23,2),
+      KV(24,2),KV(27,1),KV(31,10),KV(37,3),KV(38,2),KV(42,3),
+      KV(43,16),KV(51,1));
+   OP("clr",0,KV(2,9),KV(9,2),KV(10,1),KV(14,3),KV(15,25));
+   OP("iadd32",0,KV(2,6),KV(9,2),KV(10,1),KV(16,6),KV(23,2),KV(24,1),
+      KV(27,1),KV(32,d->context8_uniform),KV(38,3),KV(39,1),KV(44,3),KV(45,80));
+   OP("clr",0,KV(2,7),KV(9,2),KV(10,1),KV(14,3),KV(15,25));
+   scs=(struct agx3_scs_desc){
+      .data=0,.width=AGX3_MEM_W32,.byte_offset=44,
+      .address_kind=AGX3_SCS_ADDRESS_UNIFORM_INDEXED,.index=10,
+      .uniform_base=d->payload_uniform,.component_mask=3,
+      .has_address_selector=true,.address_selector=0,
+      .value_kill=true,.index_kill=true,
+   };
+   if (!agx3_scs_encode_store(AGX_ISA_ENCODING_AGX3C,&scs,staged+used,&n))
+      return false;
+   used += n;
+   /* This form consumes a 16-bit index, unlike the general SCS helpers. */
+   OP("stscs",0,KV(2,6),KV(9,2),KV(10,2),KV(13,1),KV(16,8),KV(23,2),
+      KV(24,2),KV(27,1),KV(31,4),KV(38,2),KV(39,2),KV(42,1),KV(45,15),
+      KV(53,1),KV(54,1),KV(59,188),KV(61,1),KV(68,1),KV(70,2),KV(76,1));
+   /* Shared return: select the per-thread continuation record at
+    * 192 + SR57 * 512, issue its destination-less SCS load, then return. */
+   RLD(4,57,1,0,0);
+   OP("ldimm",2,KV(9,2),KV(10,2),KV(11,1),KV(14,192),KV(15,3),KV(16,105));
+   OP("ldimm",2,KV(2,2),KV(9,2),KV(10,2),KV(11,1),KV(14,512),KV(15,3),KV(16,105));
+   OP("imadd32",0,KV(9,2),KV(10,2),KV(16,4),KV(23,2),KV(24,1),KV(27,1),
+      KV(31,2),KV(38,2),KV(39,2),KV(42,1),KV(53,2),KV(54,2),KV(57,1),
+      KV(59,3),KV(60,84),KV(61,1));
+   /* Immediate-zero slot-A operand; the indexed epilogue API intentionally
+    * does not cover this native return form. */
+   OP("ldscs",0,KV(10,2),KV(23,1),KV(24,1),KV(28,1),KV(38,2),KV(39,2),
+      KV(42,1),KV(45,15),KV(53,1),KV(54,1),KV(59,106),KV(68,3),KV(76,2),KV(80,1));
+   OP("barrier",0,KV(14,3),KV(15,9),KV(25,1),KV(30,1),KV(39,1));
+   OP("end",0,KV(0,3),KV(1,39));
+#undef RLD
+#undef OP
+   if (used > sizeof(staged))
+      return false;
+   memcpy(out,staged,used);
+   *length=used;
+   return true;
+}
diff --git a/src/asahi/compiler/agx3_rt_m5_core.h b/src/asahi/compiler/agx3_rt_m5_core.h
new file mode 100644
index 000000000000..3805f1af7efb
--- /dev/null
+++ b/src/asahi/compiler/agx3_rt_m5_core.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: MIT */
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "asahi/lib/agx_chip.h"
+
+/* M5 native compiler differential: a constant AS uses f63/f69=3; an AS
+ * selected per invocation uses f62/f69=2 and f73 marks its last use.
+ * The first operand is the GPR pair copied from the runtime's uniform pair.
+ * Keep its name neutral until the complete runtime pointer contract is mapped.
+ * All register indices are in half-register units. */
+struct agx3_rt_m5_intersect_desc {
+   uint16_t address_register;
+   uint16_t bvh_base;
+   bool bvh_uniform;
+   bool bvh_kill;
+};
+
+/* Native compilers give N simultaneously live queries consecutive allocation
+ * indices 4..4+N-1; query-model74b witnesses this for N = 2, 3, 4, 5 and 8. */
+#define AGX3_RT_M5_MAX_LIVE_QUERIES 8
+
+/* Live queries the lowering admits: every witnessed allocation index.
+ *
+ * On M5 hardware (R347 tree, R342 kernel) the only CTS cases with 3 to 8
+ * simultaneously live queries are ray_query.multiple_ray_queries.* (3, 4
+ * and 7), ray_query.stress.ahit_shader.* (3) and
+ * ray_query.builtin.rayqueryterminate.* outside any-hit and intersection
+ * shaders (8, from a split rayQueryEXT[8]). All 34 pass with this limit,
+ * also three more times at three workers, and are refused with a limit of
+ * 2. A 3,219-case sample of the other ray query, ray tracing pipeline and
+ * acceleration-structure cases has the same results with either limit;
+ * this value is the only difference between the two builds. An earlier
+ * note that three or more live queries gave wrong results (rayqueryterminate,
+ * 8 live queries) does not reproduce. This is the number of native slots the
+ * candidate-API lowering uses: logical queries beyond it share a slot
+ * (agx_nir_m5_ray_query.c, m5_rq_slot), which is how the any-hit and
+ * intersection rayqueryterminate variants (9 live queries) and
+ * ray_query.misc.dynamic_indexing (48 elements, one selected per thread) are
+ * compiled. A hardware bisection build may still override it
+ * (-DAGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES=N, N <= 8). */
+#ifndef AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES
+#define AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES AGX3_RT_M5_MAX_LIVE_QUERIES
+#endif
+
+/* Native query stack entry: f2 is the 32-bit GPR result (an even half index)
+ * and f24 the allocation index (4 + live query number). */
+bool agx3_rt_m5_encode_enter_query(enum agx_isa_encoding encoding,
+                                   uint16_t destination,
+                                   uint16_t allocation_index, uint8_t out[16],
+                                   uint8_t *length);
+
+bool agx3_rt_m5_encode_intersect(enum agx_isa_encoding encoding,
+                                const struct agx3_rt_m5_intersect_desc *desc,
+                                uint8_t out[16], uint8_t *length);
+
+/* The single continuation site emitted by the captured M5 intersectors.
+ * These fixed bytes do not specify the surrounding polling/barrier protocol. */
+bool agx3_rt_m5_encode_continue(enum agx_isa_encoding encoding,
+                               uint8_t out[16], uint8_t *length);
+
+/* Native Gen3 IFT preamble through LDSHDR (not the complete entry):
+ * reload the context words and per-function constants,
+ * then enter the separately allocated shader body. No shader body is embedded.
+ * The two witnessed context layouts use constants at half24 or half28. */
+#define AGX3_RT_M5_IFT_LOADER_MAX_SIZE 192u
+struct agx3_rt_m5_ift_loader_desc {
+   uint64_t constants_va;
+   uint64_t shader_va;
+   uint8_t constant_dwords;
+   uint8_t fetch_span;
+   bool context48;
+};
+
+bool agx3_rt_m5_ift_loader_generate(
+   const struct agx3_rt_m5_ift_loader_desc *desc,
+   uint8_t out[AGX3_RT_M5_IFT_LOADER_MAX_SIZE], size_t *length);
+
+#define AGX3_RT_M5_IFT_ENTRY_MAX_SIZE 512u
+#define AGX3_RT_M5_IFT_MAX_UNIFORMS 36u
+struct agx3_rt_m5_ift_uniform {
+   uint16_t uniform_half;
+   uint16_t register_half;
+};
+struct agx3_rt_m5_ift_entry_desc {
+   struct agx3_rt_m5_ift_loader_desc loader;
+   const struct agx3_rt_m5_ift_uniform *uniforms;
+   uint8_t uniform_count;
+   /* The two observed TPR fields vary together; their semantics remain open. */
+   bool tpr20;
+   bool final_copy16;
+};
+
+/* Complete entry including resource state, call setup, uniform publication,
+ * and END. The caller owns the separately allocated function body/constants. */
+bool agx3_rt_m5_ift_entry_generate(
+   const struct agx3_rt_m5_ift_entry_desc *desc,
+   uint8_t out[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE], size_t *length);
+
+/* Native IFT type-1 hit commit. Uniform indices and all register operands use
+ * half-register units. The two context uniforms retain numeric names until
+ * their complete lifetime contract is known. No captured code is embedded. */
+#define AGX3_RT_M5_HIT_COMMIT_MAX_SIZE 256u
+struct agx3_rt_m5_hit_commit_desc {
+   uint16_t payload_uniform;
+   uint16_t context6_uniform;
+   uint16_t context8_uniform;
+};
+bool agx3_rt_m5_hit_commit_generate(
+   const struct agx3_rt_m5_hit_commit_desc *desc,
+   uint8_t out[AGX3_RT_M5_HIT_COMMIT_MAX_SIZE], size_t *length);
diff --git a/src/asahi/compiler/agx3_scs_core.h b/src/asahi/compiler/agx3_scs_core.h
index 5ee88279e68a..f824eeadc69e 100644
--- a/src/asahi/compiler/agx3_scs_core.h
+++ b/src/asahi/compiler/agx3_scs_core.h
@@ -9,7 +9,7 @@
  * every combination Apple's G17P ray-query setup emits: direct-absolute, a
  * scalar register index plus a uniform-register base, a scalar register index
  * plus a register base, and a register base with no index -- each with a
- * signed 16-bit byte displacement.  Loads and stores may move one, two or four
+ * signed 16-bit byte displacement.  Loads and stores may move one through four
  * 32-bit components.
  *
  * AGX3c supports W32 vectors and direct, uniform-indexed, and register-base
diff --git a/src/asahi/compiler/agx3_slot_core.c b/src/asahi/compiler/agx3_slot_core.c
index a4b0f1041b33..7c37d1bac16b 100644
--- a/src/asahi/compiler/agx3_slot_core.c
+++ b/src/asahi/compiler/agx3_slot_core.c
@@ -101,6 +101,8 @@ agx3_slot_assign_with_entry(const struct agx3_slot_instr *instrs, unsigned n,
    /* slot_reissuable[s] = the slot's outstanding generation was issued by an
     * instruction that reissues live slots itself. */
    bool slot_reissuable[9] = {false};
+   uint8_t slot_join_class[9] = {0};
+   uint8_t slot_join_count[9] = {0};
 
    unsigned next = 0; /* round-robin cursor into 0..nslots-1 */
 
@@ -112,6 +114,9 @@ agx3_slot_assign_with_entry(const struct agx3_slot_instr *instrs, unsigned n,
          return false;
       if (I->coalesce_src > I->nr_srcs || (!I->is_async && I->coalesce_src))
          return false;
+      if ((I->join_class && (!I->is_async || !I->join_limit || I->join_limit > 8)) ||
+          (!I->join_class && (I->join_limit || I->prefer_join)))
+         return false;
       const uint16_t valid_slots = (uint16_t)((1u << nslots) - 1u);
       if ((I->nr_srcs < 32 && (I->late_src_mask >> I->nr_srcs)) ||
           (!I->is_async && I->late_src_mask))
@@ -225,24 +230,42 @@ agx3_slot_assign_with_entry(const struct agx3_slot_instr *instrs, unsigned n,
           * when every slot is in that state it takes the next one as before.
           */
          uint8_t slot = I->preferred_slot;
+         const unsigned cap =
+            I->max_slot && I->max_slot < nslots ? I->max_slot : nslots;
          if (!slot) {
             if (I->coalesce_src) {
                const unsigned source = I->src[I->coalesce_src - 1];
                const uint8_t producer_slot = reg_slot[source];
-               if (producer_slot && slot_waited[producer_slot] &&
+               if (producer_slot && producer_slot <= cap &&
+                   slot_waited[producer_slot] &&
                    !(I->transient_slots & (1u << (producer_slot - 1))))
                   slot = producer_slot;
             }
+            if (I->prefer_join && !slot) {
+               for (unsigned t = 0; t < nslots && !slot; ++t) {
+                  const uint8_t candidate = (uint8_t)((next + t) % nslots + 1);
+                  if (candidate <= cap && !slot_waited[candidate] &&
+                      slot_join_class[candidate] == I->join_class &&
+                      slot_join_count[candidate] < I->join_limit)
+                     slot = candidate;
+               }
+            }
             for (unsigned t = 0; t < nslots && !slot; ++t) {
                const uint8_t candidate = (uint8_t)((next + t) % nslots + 1);
+               if (candidate > cap)
+                  continue;
                if (slot_waited[candidate] ||
                    !agx3_slot_generation_live(reg_slot, reader_slots, candidate,
                                              token_count) ||
-                   (I->reissues_live_slot && slot_reissuable[candidate]))
+                   (I->reissues_live_slot && slot_reissuable[candidate]) ||
+                   (I->join_class && slot_join_class[candidate] == I->join_class &&
+                    slot_join_count[candidate] < I->join_limit))
                   slot = candidate;
             }
             if (!slot) {
                slot = (uint8_t)(next + 1);
+               if (slot > cap)
+                  slot = 1;
                if (!I->reissues_live_slot) {
                   wait_mask |= (uint16_t)(1u << (slot - 1));
                   slot_waited[slot] = true;
@@ -259,6 +282,16 @@ agx3_slot_assign_with_entry(const struct agx3_slot_instr *instrs, unsigned n,
             slot_waited[slot] = true;
          }
 
+         /* A joined queue must complete before an incompatible producer
+          * replaces it. In particular, an SCS completion or a texture
+          * publication must not inherit a pending device-read generation. */
+         if (!slot_waited[slot] && slot_join_count[slot] &&
+             ((slot_join_class[slot] != I->join_class) ||
+              (I->join_class && slot_join_count[slot] >= I->join_limit))) {
+            wait_mask |= (uint16_t)(1u << (slot - 1));
+            slot_waited[slot] = true;
+         }
+
          out[i].wait_mask = wait_mask;
          out[i].slot = slot;
          entry_pending &= (uint16_t)~wait_mask;
@@ -277,6 +310,8 @@ agx3_slot_assign_with_entry(const struct agx3_slot_instr *instrs, unsigned n,
           * generation. Retire the old generation's register aliases before
           * publishing the new result, otherwise an unrelated later write can
           * consume the new generation's wait and hide it from its consumer. */
+         const bool new_join = slot_waited[slot] ||
+                               slot_join_class[slot] != I->join_class;
          if (slot_waited[slot]) {
             for (unsigned r = 0; r < token_count; ++r) {
                if (reg_slot[r] == slot)
@@ -285,6 +320,9 @@ agx3_slot_assign_with_entry(const struct agx3_slot_instr *instrs, unsigned n,
          }
          slot_waited[slot] = false; /* freshly outstanding */
          slot_reissuable[slot] = I->reissues_live_slot;
+         slot_join_count[slot] = I->join_class ?
+            (new_join ? 1 : slot_join_count[slot] + 1) : 0;
+         slot_join_class[slot] = I->join_class;
          for (unsigned d = 0; d < I->nr_dsts; ++d) {
             if (I->dst[d] >= AGX3_SLOT_NUM_TOKENS)
                return false;
diff --git a/src/asahi/compiler/agx3_slot_core.h b/src/asahi/compiler/agx3_slot_core.h
index d1420d18dca6..0d0d92b49d04 100644
--- a/src/asahi/compiler/agx3_slot_core.h
+++ b/src/asahi/compiler/agx3_slot_core.h
@@ -72,6 +72,9 @@ struct agx3_slot_instr {
     * its descriptor operands retain the existing issue-time contract. */
    uint32_t late_src_mask;
    uint8_t preferred_slot; /* 0=round-robin, otherwise required slot */
+   /* 0 = any slot. Otherwise the round-robin choice is limited to slots
+    * 1..max_slot, for an encoding that can only name those slots. */
+   uint8_t max_slot;
    /* Optional source index plus one whose completed producer slot may be
     * recycled for this event. This is a preference, never a new wait: the
     * source's actual generation must already be synchronized by dataflow.
@@ -90,6 +93,12 @@ struct agx3_slot_instr {
    uint8_t nr_srcs; /* dependency tokens this instruction reads */
    uint16_t dst[AGX3_SLOT_MAX_REGS];
    uint16_t src[AGX3_SLOT_MAX_REGS];
+   /* Compatible asynchronous producers can share a native completion queue.
+    * Zero keeps the one-generation policy. The count is bounded separately
+    * from the number of physical scoreboard slots. */
+   uint8_t join_class;
+   uint8_t join_limit;
+   bool prefer_join;
 };
 
 struct agx3_slot_result {
diff --git a/src/asahi/compiler/agx3_sr_core.c b/src/asahi/compiler/agx3_sr_core.c
index b6082d17a4a4..ce6b56a20497 100644
--- a/src/asahi/compiler/agx3_sr_core.c
+++ b/src/asahi/compiler/agx3_sr_core.c
@@ -64,22 +64,21 @@ agx3_sr_witnessed(enum agx_isa_encoding enc, uint8_t sr, enum agx3_sr_width widt
       switch (sr) {
       case 6: case 7: case 9: case 10: case 11: case 13:
       case 17: case 18: case 19: case 20: case 21: case 22:
-      case 34: case 36:
-      /* fragment-only, see the comment above */
-      case 2: case 39: case 41:
-      /* G17P builtin BG array reloads read their layer with 34c31046. */
-      case 12:
-      /* G17P instance main +0x61a and M5 ray entry read SR59. */
-      case 59:
+      case 34: case 36: case 59:
+      case 2: case 39: case 41: case 12:
          return true;
+      case 63:
+         return enc == AGX_ISA_ENCODING_AGX3C;
       case 65:
-         return enc == AGX_ISA_ENCODING_AGX3B;
+         return enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C;
       default:
          return false;
       }
    }
    if (width == AGX3_SR_W32 && sr == 64)
-      return enc == AGX_ISA_ENCODING_AGX3B;
+      return enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C;
+   if (width == AGX3_SR_W32 && sr == 62)
+      return enc == AGX_ISA_ENCODING_AGX3C;
    if (width == AGX3_SR_W32)
       return (sr >= 23 && sr <= 28) || (sr >= 46 && sr <= 49) || sr == 51;
    return false;
@@ -101,16 +100,17 @@ agx3_sr_encode(enum agx_isa_encoding enc, const struct agx3_sr_desc *d,
    if (d->slot < 1 || d->slot > nslots || (d->wait_mask >> nslots) != 0)
       return false;
 
-   /* The entry marker is witnessed only on G17P's W16 SR39 coverage read:
+   /* The entry marker is witnessed on G17P/G17G's W16 SR39 coverage read:
     * `[[sample_mask]]` and centroid inputs, and a multisample fragment with no
     * other input, each open with `rld ep1 ... f48=1`. */
    /* Apple's throughput fragment (metal-nt applegpu_g17p, fragment.fields
     * offset 64) also opens with a marked 32-bit SR24 read,
     * 1ca1110601000000, so the pixel-coordinate pair qualifies too. */
-   if (d->first &&
-       (enc != AGX_ISA_ENCODING_AGX3B ||
-        !((d->sr == 39 && d->width == AGX3_SR_W16) ||
-          ((d->sr == 23 || d->sr == 24) && d->width == AGX3_SR_W32))))
+   const bool coverage_entry = d->sr == 39 && d->width == AGX3_SR_W16 &&
+      (enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C);
+   const bool pixel_entry = enc == AGX_ISA_ENCODING_AGX3B &&
+      (d->sr == 23 || d->sr == 24) && d->width == AGX3_SR_W32;
+   if (d->first && !coverage_entry && !pixel_entry)
       return false;
 
    struct agx3_field_kv base[5];
diff --git a/src/asahi/compiler/agx3_stage_core.c b/src/asahi/compiler/agx3_stage_core.c
index 59bf5a567e42..e5000477ab1a 100644
--- a/src/asahi/compiler/agx3_stage_core.c
+++ b/src/asahi/compiler/agx3_stage_core.c
@@ -18,11 +18,25 @@ stage_alu(enum agx_isa_encoding enc, bool fma, uint16_t dest, uint8_t slot,
           const bool live[3], const bool discard[3], uint16_t wait_mask,
           uint8_t out[16], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || !src || !live || !discard || !out ||
-       !len || dest >= 256 || (dest & 1) || slot < 1 || slot > 6 ||
-       wait_mask > 0x3f || (unsigned)source0 > AGX3_STAGE_FMA_IMMEDIATE ||
-       (source0 != AGX3_STAGE_FMA_REGISTER && !fma))
+   /* M5 (G17G) terrain vertex mains write their outputs the same
+    * way: staged FMUL R*R, FMA U*R+R and FMA R*R+R with f9=5, the same
+    * operand fields, f95=1/f96=55 (FMA) or f78=1/f79=62 (MUL), eight
+    * completion slots and an eight-bit wait set. Native witnesses are
+    * asserted in test-agx3-uniform-alu.cpp. The coordinate FMA with an
+    * immediate stays an AGX3B form. */
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && enc != AGX_ISA_ENCODING_AGX3B) || !src || !live || !discard ||
+       !out || !len || dest >= 256 || (dest & 1) || slot < 1 ||
+       slot > (m5 ? 8 : 6) || wait_mask > (m5 ? 0xff : 0x3f) ||
+       (unsigned)source0 > AGX3_STAGE_FMA_IMMEDIATE ||
+       (source0 != AGX3_STAGE_FMA_REGISTER && !fma) ||
+       (m5 && source0 == AGX3_STAGE_FMA_IMMEDIATE))
       return false;
+   for (unsigned s = 0; m5 && s < (fma ? 3 : 2); ++s) {
+      const bool uniform = s == 0 && source0 == AGX3_STAGE_FMA_UNIFORM;
+      if (!uniform && src[s] >= 252)
+         return false;
+   }
    for (unsigned s = 0; s < (fma ? 3 : 2); ++s) {
       const bool immediate = s == 0 && source0 == AGX3_STAGE_FMA_IMMEDIATE;
       if (src[s] >= 256 || (!immediate && (src[s] & 1)) ||
@@ -39,7 +53,9 @@ stage_alu(enum agx_isa_encoding enc, bool fma, uint16_t dest, uint8_t slot,
       KV(9, 5), KV(10, 2), KV(25, 2), KV(26, 2), KV(43, 2),
       KV(60, 2),
    };
-   struct agx3_field_kv fields[24] = {
+   /* Seven base fields, two source hints, six FMA fields, two M5
+    * selectors and eight wait bits: the all-register M5 form needs 25. */
+   struct agx3_field_kv fields[25] = {
       KV(5, dest),
       KV(12, slot),
       KV(18, src[1]),
@@ -68,15 +84,35 @@ stage_alu(enum agx_isa_encoding enc, bool fma, uint16_t dest, uint8_t slot,
    } else {
       fields[n++] = (struct agx3_field_kv)KV(74, 2);
    }
-   for (unsigned s = 0; s < 6; ++s)
+   if (m5) {
+      fields[n++] = (struct agx3_field_kv)KV(fma ? 95 : 78, 1);
+      fields[n++] = (struct agx3_field_kv)KV(fma ? 96 : 79, fma ? 55 : 62);
+   }
+   for (unsigned s = 0; s < (m5 ? 8 : 6); ++s)
       fields[n++] = (struct agx3_field_kv)KV((fma ? 97 : 80) + s,
                                            (wait_mask >> s) & 1);
-   for (unsigned ep = 0; ep <= (fma ? 5 : 3); ++ep) {
+   /* M5 endpoints are not ordered by length: keep the shortest. */
+   uint8_t best[16], best_len = 0;
+   for (unsigned ep = 0; ep <= (m5 ? (fma ? 7 : 5) : fma ? 5 : 3); ++ep) {
+      uint8_t bytes[16], size;
       if (agx3_bind_compose(enc, fma ? "f32fma" : "f32mul", ep,
-                           base, 6, fields, n, out, len))
-         return true;
+                           base, 6, fields, n, bytes, &size)) {
+         if (!m5) {
+            memcpy(out, bytes, size);
+            *len = size;
+            return true;
+         }
+         if (!best_len || size < best_len) {
+            memcpy(best, bytes, size);
+            best_len = size;
+         }
+      }
    }
-   return false;
+   if (!best_len)
+      return false;
+   memcpy(out, best, best_len);
+   *len = best_len;
+   return true;
 }
 
 bool
@@ -227,8 +263,15 @@ clr_common(enum agx_isa_encoding enc, const struct agx3_field_kv *ov,
            unsigned n, uint8_t out[AGX3_STAGE_MAX_LENGTH], uint8_t *len)
 {
    const struct agx3_field_kv base[] = {KV(10, 2)};
-   if (agx3_bind_compose(enc, "clr", 0, base, 1, ov, n, out, len))
-      return true;
+   const struct agx3_field_kv m5_base[] = {
+      KV(10, 2), KV(14, 3), KV(15, 25),
+   };
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   for (unsigned ep = 0; ep <= (m5 ? 1 : 0); ++ep) {
+      if (agx3_bind_compose(enc, "clr", ep, m5 ? m5_base : base,
+                           m5 ? 3 : 1, ov, n, out, len))
+         return true;
+   }
    memset(out, 0, AGX3_STAGE_MAX_LENGTH);
    *len = 0;
    return false;
@@ -243,7 +286,8 @@ agx3_clr(enum agx_isa_encoding enc, uint16_t dest, bool half, bool hint,
    memset(out, 0, AGX3_STAGE_MAX_LENGTH);
    *len = 0;
 
-   if (!b_only(enc) || dest >= 256 || (!half && (dest & 1)))
+   if ((!b_only(enc) && enc != AGX_ISA_ENCODING_AGX3C) ||
+       dest >= 256 || (!half && (dest & 1)))
       return false;
 
    struct agx3_field_kv ov[5];
diff --git a/src/asahi/compiler/agx3_staging.h b/src/asahi/compiler/agx3_staging.h
index 8c89fec6dd7c..8d14ce2469ee 100644
--- a/src/asahi/compiler/agx3_staging.h
+++ b/src/asahi/compiler/agx3_staging.h
@@ -64,6 +64,32 @@ agx3_staging_hold(struct agx3_staging_banks *banks, uint16_t base,
    return true;
 }
 
+/* As agx3_staging_plan(), but take a free range at or above `avoid_end`
+ * first: the start of the file belongs to a later fixed-location operand. */
+static inline bool agx3_staging_plan(const struct agx3_staging_banks *banks,
+                                     unsigned span, unsigned alignment,
+                                     unsigned limit, uint16_t *base,
+                                     uint16_t *waits);
+
+static inline bool
+agx3_staging_plan_above(const struct agx3_staging_banks *banks, unsigned span,
+                        unsigned alignment, unsigned limit,
+                        unsigned avoid_end, uint16_t *base, uint16_t *waits)
+{
+   if (avoid_end && alignment && !(alignment & (alignment - 1)) &&
+       limit <= AGX3_STAGING_HALFS && span <= limit) {
+      const unsigned first = (avoid_end + alignment - 1) & ~(alignment - 1);
+      for (unsigned i = first; i + span <= limit; i += alignment) {
+         if (!agx3_staging_conflicts(banks, i, span)) {
+            *base = i;
+            *waits = 0;
+            return true;
+         }
+      }
+   }
+   return agx3_staging_plan(banks, span, alignment, limit, base, waits);
+}
+
 /* Plan an aligned allocation without changing any lifetime. Prefer a free
  * range; otherwise return the range requiring the fewest consumer slots to
  * retire. The caller must emit those waits before overwriting the range and
diff --git a/src/asahi/compiler/agx3_subgroup_core.c b/src/asahi/compiler/agx3_subgroup_core.c
index e13f9c6970c6..2102e93d57e4 100644
--- a/src/asahi/compiler/agx3_subgroup_core.c
+++ b/src/asahi/compiler/agx3_subgroup_core.c
@@ -47,6 +47,64 @@ shuffle_mnemonic(bool quad, enum agx3_subgroup_shuffle_op op)
              : NULL;
 }
 
+bool
+agx3_subgroup_encode_active_mask(uint16_t dest, bool dest_cache,
+                                 uint8_t wait_mask,
+                                 uint8_t out[AGX3_SUBGROUP_MAX_LENGTH],
+                                 uint8_t *length)
+{
+   if (!out || !length || dest >= 256 || (dest & 1))
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9,2},{10,2},{23,1},{24,1},{38,1},{39,1},{44,3},{45,157},
+   };
+   struct agx3_field_kv fields[14] = {
+      {2,dest},{11,dest_cache},{15,1},{30,0},{54,0},{55,1},
+   };
+   unsigned count = 6;
+   for (unsigned bit = 0; bit < 8; ++bit)
+      if (wait_mask & (1u << bit))
+         fields[count++] = (struct agx3_field_kv){46 + bit, 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C,"simdiballot",0,
+                           base,8,fields,count,out,length);
+}
+
+/*
+ * The 16-bit quad ballot of a boolean immediate against zero. Native
+ * native G17G (lane-mesh/cwit-quad, cwit-quad2) keeps a quad vote whose
+ * operand is a constant as a real ballot rather than folding it:
+ *
+ *   quad_any(false), result into a store    370020182d00a61a30000840  (r7l)
+ *   ballot(1 != 0) in q_vote (quad votes)    370120182d00a60230000840  (r1l)
+ *
+ * f23/f24 = 1/1 name an immediate 16-bit source whose value is f15, f38/f39
+ * = 1/1 the immediate zero it is compared with, f54 = 0 the equality and f55
+ * its inversion: the quad counterpart of agx3_subgroup_encode_active_mask().
+ * Apple adds f64 only when the result reaches nothing but the colour output;
+ * it stays clear, as on the register-source quad ballots.
+ */
+bool
+agx3_subgroup_encode_quad_ballot_imm(uint16_t dest, bool dest_cache,
+                                     uint16_t value, uint8_t wait_mask,
+                                     uint8_t out[AGX3_SUBGROUP_MAX_LENGTH],
+                                     uint8_t *length)
+{
+   if (!out || !length || dest >= 256 || value > 1)
+      return false;
+   const struct agx3_field_kv base[] = {
+      {9, 2}, {10, 1}, {23, 1}, {24, 1}, {38, 1}, {39, 1}, {44, 3}, {45, 139},
+   };
+   struct agx3_field_kv fields[13] = {
+      {2, dest}, {11, dest_cache}, {15, value}, {54, 0}, {55, 1},
+   };
+   unsigned count = 5;
+   for (unsigned bit = 0; bit < 8; ++bit)
+      if (wait_mask & (1u << bit))
+         fields[count++] = (struct agx3_field_kv){(uint16_t)(46 + bit), 1};
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "quadiballot", 0, base, 8,
+                            fields, count, out, length);
+}
+
 bool
 agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
                              const struct agx3_subgroup_shuffle_desc *d,
@@ -74,11 +132,12 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
     */
    const bool half_shuffle_op = d->op == AGX3_SUBGROUP_SHUFFLE_BROADCAST ||
                                 d->op == AGX3_SUBGROUP_SHUFFLE_XOR;
-   if (d->half ? enc != AGX_ISA_ENCODING_AGX3B ||
+   if (d->half ? (enc != AGX_ISA_ENCODING_AGX3B && !is_c(enc)) ||
                     (d->lane_kind != AGX3_SUBGROUP_LANE_REGISTER &&
                      !half_shuffle_op) ||
                     (d->quad && d->lane_kind != AGX3_SUBGROUP_LANE_IMMEDIATE) ||
-                    d->dest >= 192 || d->src >= 192
+                    d->dest >= (is_c(enc) ? 256 : 192) ||
+                    d->src >= (is_c(enc) ? 256 : 192)
                : !valid_reg(enc, d->dest) || !valid_reg(enc, d->src))
       return false;
 
@@ -106,16 +165,16 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
     * pinned: a pinned f41 claimed the lane was dead even when the vote and
     * shuffle lowerings read it again for the next component.
     */
-   if (enc == AGX_ISA_ENCODING_AGX3B && !d->quad &&
+   if ((enc == AGX_ISA_ENCODING_AGX3B || is_c(enc)) && !d->quad &&
        d->lane_kind == AGX3_SUBGROUP_LANE_REGISTER) {
-      const unsigned slots = 6;
-      if (d->lane >= 192 || (d->wait_mask >> slots) != 0 ||
+      const unsigned slots = is_c(enc) ? 8 : 6;
+      if (d->lane >= (is_c(enc) ? 256 : 192) || (d->wait_mask >> slots) != 0 ||
           (d->src_cache && d->src_discard) ||
           (d->lane_cache && d->lane_discard))
          return false;
 
       const unsigned width = d->half ? 1 : 2;
-      struct agx3_field_kv hbase[6];
+      struct agx3_field_kv hbase[8];
       unsigned hnb = 0;
       hbase[hnb++] = (struct agx3_field_kv){9, 2};
       hbase[hnb++] = (struct agx3_field_kv){10, width};
@@ -124,6 +183,11 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
       hbase[hnb++] = (struct agx3_field_kv){37, 2};
       hbase[hnb++] = (struct agx3_field_kv){38, 1};
 
+      if (is_c(enc)) {
+         const uint8_t ids[] = {154,155,161,160};
+         hbase[hnb++] = (struct agx3_field_kv){42,3};
+         hbase[hnb++] = (struct agx3_field_kv){43,ids[d->op]};
+      }
       struct agx3_field_kv hov[16];
       unsigned hnv = 0;
       hov[hnv++] = (struct agx3_field_kv){2, d->dest};
@@ -149,7 +213,7 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
     * sets f49 and nothing else.
     */
    if (d->lane_kind == AGX3_SUBGROUP_LANE_IMMEDIATE) {
-      const unsigned slots = 6;
+      const unsigned slots = is_c(enc) ? 8 : 6;
       /*
        * f29 IS the lane, for every immediate-lane shuffle and for every lane
        * the seven-bit field holds.  This used to admit three SIMD broadcast
@@ -180,12 +244,12 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
        * The field is seven bits (half 0xff spills the eighth into f16, the
        * source register), so lanes 0..127 encode and 128 up do not.
        */
-      if (enc != AGX_ISA_ENCODING_AGX3B)
+      if (enc != AGX_ISA_ENCODING_AGX3B && !is_c(enc))
          return false;
       if (d->lane > 0x7f || (d->wait_mask >> slots) != 0)
          return false;
 
-      struct agx3_field_kv ibase[8];
+      struct agx3_field_kv ibase[10];
       unsigned inb = 0;
       /* 16-bit data only narrows the width pair: simd_broadcast(ushort, 5)
        * is 4704560202040a200400 next to the 32-bit words above. */
@@ -201,6 +265,14 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
       ibase[inb++] = (struct agx3_field_kv){37, 1};
       ibase[inb++] = (struct agx3_field_kv){38, 1};
 
+      if (is_c(enc)) {
+         static const uint8_t ids[2][AGX3_SUBGROUP_SHUFFLE_NUM] = {
+            {154,155,161,160},{140,137,142,141},
+         };
+         ibase[inb++] = (struct agx3_field_kv){42,3};
+         ibase[inb++] = (struct agx3_field_kv){43,ids[d->quad][d->op]};
+      }
+
       struct agx3_field_kv iov[14];
       unsigned inv = 0;
       iov[inv++] = (struct agx3_field_kv){2, d->dest};
@@ -223,7 +295,7 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
                                                     : d->lane >= 512))
       return false;
 
-   struct agx3_field_kv base[16];
+   struct agx3_field_kv base[24];
    unsigned nb = 0;
    ADD_FIELD(9, 2);
    ADD_FIELD(10, 2);
@@ -255,15 +327,18 @@ agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
          ADD_FIELD(27, 1);
       }
    } else {
-      if (d->lane_kind == AGX3_SUBGROUP_LANE_REGISTER && !d->quad &&
-          d->op == AGX3_SUBGROUP_SHUFFLE_BROADCAST) {
+      if (d->quad && d->op == AGX3_SUBGROUP_SHUFFLE_BROADCAST) {
+         ADD_FIELD(11, d->dest_hint);
+         ADD_FIELD(d->src_live ? 25 : 27, 1);
+      } else if (d->lane_kind == AGX3_SUBGROUP_LANE_REGISTER && !d->quad &&
+                 d->op == AGX3_SUBGROUP_SHUFFLE_BROADCAST) {
          ADD_FIELD(11, 1);
          ADD_FIELD(25, 1);
       } else {
          ADD_FIELD(27, 1);
       }
       if (d->lane_kind == AGX3_SUBGROUP_LANE_REGISTER)
-         ADD_FIELD(41, 1);
+         ADD_FIELD(d->quad && d->lane_live ? 39 : 41, 1);
       static const uint8_t ids[2][AGX3_SUBGROUP_SHUFFLE_NUM] = {
          {154, 155, 161, 160},
          {140, 137, 142, 141},
@@ -289,6 +364,50 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
    if (d == NULL || out == NULL || length == NULL)
       return false;
 
+   if (d->compare_register) {
+      const bool is_float = d->type == AGX3_SUBGROUP_BALLOT_FLOAT;
+      if (!is_c(enc) || d->quad || d->dest_half || d->compare_immediate ||
+          (unsigned)d->type > AGX3_SUBGROUP_BALLOT_FLOAT ||
+          (is_float ? (d->condition != 0 && d->condition != 1 &&
+                       d->condition != 2 && d->condition != 5 && d->condition != 6)
+                    : (d->condition != 1 && d->condition != 2 &&
+                       d->condition != 4 && d->condition != 5 &&
+                       d->condition != 6)) ||
+          !valid_reg(enc, d->dest) || d->wait_mask > 255 ||
+          (d->src_half ? d->src >= 256 : !valid_reg(enc, d->src)) ||
+          (d->compare_half ? d->compare_value >= 256 :
+                             !valid_reg(enc, d->compare_value)) ||
+          (d->src_cache && !d->src_live) ||
+          (d->compare_cache && d->compare_discard))
+         return false;
+
+      struct agx3_field_kv base[24];
+      unsigned nb = 0;
+      ADD_FIELD(9, 2);
+      ADD_FIELD(10, 2);
+      ADD_FIELD(11, d->dest_cache);
+      ADD_FIELD(23, 2);
+      ADD_FIELD(24, d->src_half ? 1 : 2);
+      ADD_FIELD(25, d->src_cache);
+      ADD_FIELD(27, !d->src_live);
+      ADD_FIELD(is_float ? 40 : 38, 2);
+      ADD_FIELD(is_float ? 41 : 39, d->compare_half ? 1 : 2);
+      ADD_FIELD(is_float ? 42 : 40, d->compare_cache);
+      ADD_FIELD(is_float ? 44 : 42, d->compare_discard);
+      ADD_FIELD(is_float ? 48 : 44, 3);
+      ADD_FIELD(is_float ? 49 : 45, is_float ? 156 : 157);
+      for (unsigned bit = 0; bit < 8; bit++)
+         if (d->wait_mask & (1u << bit))
+            ADD_FIELD((is_float ? 50 : 46) + bit, 1);
+      const struct agx3_field_kv ov[] = {
+         {2, d->dest}, {16, d->src}, {is_float ? 33 : 31, d->compare_value},
+         {is_float ? 58 : 54, d->condition}, {is_float ? 59 : 55, d->invert},
+      };
+      return agx3_bind_compose(enc, is_float ? "simdfballot" : "simdiballot",
+                               0, base, nb, ov, sizeof(ov) / sizeof(ov[0]),
+                               out, length);
+   }
+
    /*
     * The 16-bit quad ballot of a register against zero. metal-nt -arch
     * applegpu_g17p, quad_any(x != 0) and quad_all(x == 0) of a buffer-loaded
@@ -303,17 +422,17 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
     * colour output; like f62 on the quad reductions, that stays clear.
     */
    if (d->quad && d->dest_half) {
-      const unsigned slots = 6;
-      if (enc != AGX_ISA_ENCODING_AGX3B ||
+      const unsigned slots = is_c(enc) ? 8 : 6;
+      if ((enc != AGX_ISA_ENCODING_AGX3B && !is_c(enc)) ||
           d->type != AGX3_SUBGROUP_BALLOT_INTEGER || d->compare_immediate ||
-          d->condition != 4 || d->dest >= 192 ||
-          (d->src_half ? d->src >= 192 : !valid_reg(enc, d->src)) ||
+          d->condition != 4 || d->dest >= (is_c(enc) ? 256 : 192) ||
+          (d->src_half ? d->src >= (is_c(enc) ? 256 : 192) : !valid_reg(enc, d->src)) ||
           (d->wait_mask >> slots) != 0)
          return false;
 
       /* A uint source only widens f24: quad_any(x != 0) of a uint is
        * 17025600030000502212. */
-      struct agx3_field_kv qbase[16];
+      struct agx3_field_kv qbase[24];
       unsigned qnb = 0;
       qbase[qnb++] = (struct agx3_field_kv){9, 2};
       qbase[qnb++] = (struct agx3_field_kv){10, 1};
@@ -322,6 +441,10 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
       qbase[qnb++] = (struct agx3_field_kv){d->src_live ? 25 : 27, 1};
       qbase[qnb++] = (struct agx3_field_kv){38, 1};
       qbase[qnb++] = (struct agx3_field_kv){39, 1};
+      if (is_c(enc)) {
+         qbase[qnb++] = (struct agx3_field_kv){44, 3};
+         qbase[qnb++] = (struct agx3_field_kv){45, 139};
+      }
       for (unsigned i = 0; i < slots; ++i)
          if (d->wait_mask & (1u << i))
             qbase[qnb++] = (struct agx3_field_kv){(uint16_t)(46 + i), 1};
@@ -337,13 +460,14 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
                                sizeof(qov) / sizeof(qov[0]), out, length);
    }
 
-   if (d->wait_mask != 0 || d->dest_half || !valid_reg(enc, d->dest) ||
+   if ((!is_c(enc) && d->wait_mask != 0) || (is_c(enc) && d->wait_mask > 255) ||
+       d->dest_half || !valid_reg(enc, d->dest) ||
        (unsigned)d->type > AGX3_SUBGROUP_BALLOT_FLOAT)
       return false;
 
    /* Only the witnessed 16-bit source shape: equal to zero. */
    if (d->src_half ? !d->compare_immediate || d->condition != 4 ||
-                        d->compare_value != 0 || d->src >= 192
+                        d->compare_value != 0 || d->src >= (is_c(enc) ? 256 : 192)
                    : !valid_reg(enc, d->src))
       return false;
 
@@ -351,10 +475,10 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
       /* Fresh applegpu-nt G17P witnesses use this exact form for unsigned
        * equality/less-than/greater-than against immediates.  The latter is
        * how Apple canonicalizes >= N (as > N - 1). */
-      if (enc != AGX_ISA_ENCODING_AGX3B || d->quad ||
+      if ((enc != AGX_ISA_ENCODING_AGX3B && !is_c(enc)) || d->quad ||
           d->type != AGX3_SUBGROUP_BALLOT_INTEGER ||
           (d->condition != 1 && d->condition != 2 && d->condition != 4) ||
-          d->invert)
+          (d->invert && !is_c(enc)))
          return false;
    } else if (d->type == AGX3_SUBGROUP_BALLOT_INTEGER) {
       if (d->condition != 4 && d->condition != 5 && d->condition != 6)
@@ -364,7 +488,7 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
       return false;
    }
 
-   struct agx3_field_kv base[16];
+   struct agx3_field_kv base[24];
    unsigned nb = 0;
    ADD_FIELD(9, 2);
    ADD_FIELD(10, 2);
@@ -393,7 +517,8 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
       if (d->compare_immediate) {
          ADD_FIELD(38, 1);
          ADD_FIELD(39, 1);
-         ADD_FIELD(46, 1);
+         if (!is_c(enc))
+            ADD_FIELD(46, 1);
       } else if (d->condition == 4) {
          ADD_FIELD(38, 2);
          ADD_FIELD(39, 2);
@@ -420,6 +545,14 @@ agx3_subgroup_encode_ballot(enum agx_isa_encoding enc,
       }
    }
 
+   if (is_c(enc)) {
+      if (d->quad || d->type != AGX3_SUBGROUP_BALLOT_INTEGER)
+         return false;
+      for (unsigned i = 0; i < 8; ++i)
+         if (d->wait_mask & (1u << i))
+            ADD_FIELD(46 + i,1);
+   }
+
    struct agx3_field_kv ov[5] = {
       {2, d->dest},
       {16, d->src},
@@ -442,20 +575,20 @@ struct reduce_form {
 };
 
 static const struct reduce_form reduce_forms[AGX3_SUBGROUP_REDUCE_NUM] = {
-   [AGX3_SUBGROUP_REDUCE_AND] = {"sRedAND", "qRedAND", 169, 0, false, 2},
+   [AGX3_SUBGROUP_REDUCE_AND] = {"sRedAND", "qRedAND", 169, 123, false, 2},
    [AGX3_SUBGROUP_REDUCE_FADD] = {"sRedf32add", "qRedf32add", 172, 126, true,
                                   0},
-   [AGX3_SUBGROUP_REDUCE_OR] = {"sRedOR", "qRedOR", 177, 0, false, 2},
+   [AGX3_SUBGROUP_REDUCE_OR] = {"sRedOR", "qRedOR", 177, 131, false, 2},
    [AGX3_SUBGROUP_REDUCE_FMUL] = {"sRedf32mul", "qRedf32mul", 173, 127, true,
                                   0},
-   [AGX3_SUBGROUP_REDUCE_XOR] = {"sRedXOR", "qRedXOR", 182, 0, false, 2},
+   [AGX3_SUBGROUP_REDUCE_XOR] = {"sRedXOR", "qRedXOR", 182, 136, false, 2},
    [AGX3_SUBGROUP_REDUCE_FMIN] = {"sRedfmin", "qRedfmin", 175, 129, true, 0},
    [AGX3_SUBGROUP_REDUCE_FMAX] = {"sRedfmax", "qRedfmax", 174, 128, true, 0},
    [AGX3_SUBGROUP_REDUCE_IADD] = {"sRediadd", "qRediadd", 176, 130, false, 0},
-   [AGX3_SUBGROUP_REDUCE_SMIN] = {"sRedsmin", "qRedsmin", 179, 0, false, 1},
-   [AGX3_SUBGROUP_REDUCE_SMAX] = {"sRedsmax", "qRedsmax", 178, 0, false, 1},
-   [AGX3_SUBGROUP_REDUCE_UMIN] = {"sRedumin", "qRedumin", 181, 0, false, 2},
-   [AGX3_SUBGROUP_REDUCE_UMAX] = {"sRedumax", "qRedumax", 180, 0, false, 2},
+   [AGX3_SUBGROUP_REDUCE_SMIN] = {"sRedsmin", "qRedsmin", 179, 133, false, 1},
+   [AGX3_SUBGROUP_REDUCE_SMAX] = {"sRedsmax", "qRedsmax", 178, 132, false, 1},
+   [AGX3_SUBGROUP_REDUCE_UMIN] = {"sRedumin", "qRedumin", 181, 135, false, 2},
+   [AGX3_SUBGROUP_REDUCE_UMAX] = {"sRedumax", "qRedumax", 180, 134, false, 2},
 };
 
 bool
@@ -467,7 +600,8 @@ agx3_subgroup_encode_reduce(enum agx_isa_encoding enc,
    if (d == NULL || out == NULL || length == NULL ||
        (unsigned)d->op >= AGX3_SUBGROUP_REDUCE_NUM)
       return false;
-   if (d->half ? is_c(enc) || d->dest >= 192 || d->src >= 192
+   const unsigned reg_limit = is_c(enc) ? 256 : 192;
+   if (d->half ? d->dest >= reg_limit || d->src >= reg_limit
                : !valid_reg(enc, d->dest) || !valid_reg(enc, d->src))
       return false;
 
@@ -493,15 +627,6 @@ agx3_subgroup_encode_reduce(enum agx_isa_encoding enc,
    }
    if (mnemonic == NULL || mnemonic[0] == '\0')
       return false;
-   /* The newly recovered integer min/max and bitwise quad words are
-    * G17P/AGX3b witnesses. Preserve the existing AGX3c boundary until
-    * independently captured on that encoding. */
-   if (d->quad && is_c(enc) &&
-       (d->op == AGX3_SUBGROUP_REDUCE_AND ||
-        d->op == AGX3_SUBGROUP_REDUCE_OR ||
-        d->op == AGX3_SUBGROUP_REDUCE_XOR ||
-        d->op >= AGX3_SUBGROUP_REDUCE_SMIN))
-      return false;
    /*
     * Metal has scans for sums and products only, so those are the witnessed
     * prefix forms: f53 on the integer add, f57 on the f32 and f56 on the f16
@@ -516,14 +641,11 @@ agx3_subgroup_encode_reduce(enum agx_isa_encoding enc,
     */
    if (d->prefix && d->quad)
       return false;
-   if (d->prefix && is_c(enc) && d->op != AGX3_SUBGROUP_REDUCE_FADD &&
-       d->op != AGX3_SUBGROUP_REDUCE_FMUL && d->op != AGX3_SUBGROUP_REDUCE_IADD)
-      return false;
    if (d->prefix && f->is_float && d->op != AGX3_SUBGROUP_REDUCE_FADD &&
        d->op != AGX3_SUBGROUP_REDUCE_FMUL)
       return false;
 
-   struct agx3_field_kv base[16];
+   struct agx3_field_kv base[24];
    unsigned nb = 0;
    ADD_FIELD(9, 2);
    ADD_FIELD(10, d->half ? 1 : 2);
@@ -532,9 +654,15 @@ agx3_subgroup_encode_reduce(enum agx_isa_encoding enc,
       ADD_FIELD(24, 2);
       ADD_FIELD(25, 1);
       ADD_FIELD(d->src_live ? 26 : 28, 1);
-      ADD_FIELD(33, 1);
-      ADD_FIELD(41, 1);
-      ADD_FIELD(42, 1);
+      if (is_c(enc)) {
+         ADD_FIELD(46, 3);
+         ADD_FIELD(47, d->quad ? (d->op == AGX3_SUBGROUP_REDUCE_FADD ? 124 : 125)
+                              : (d->op == AGX3_SUBGROUP_REDUCE_FADD ? 170 : 171));
+      } else {
+         ADD_FIELD(33, 1);
+         ADD_FIELD(41, 1);
+         ADD_FIELD(42, 1);
+      }
       src_field = 17;
       prefix_field = 56;
       wait_base = 48;
@@ -561,7 +689,7 @@ agx3_subgroup_encode_reduce(enum agx_isa_encoding enc,
        * read, for SIMD and quad forms alike: simd_prefix_inclusive_sum(),
        * which adds the source back afterwards, is bf01560403001609 (f25)
        * next to the exclusive bf0156020204100b (f27). */
-      ADD_FIELD(!is_c(enc) && d->src_live ? 25 : 27, 1);
+      ADD_FIELD(d->src_live ? 25 : 27, 1);
       if (f->integer_class == 1)
          ADD_FIELD(28, 1);
       if (is_c(enc)) {
@@ -604,7 +732,7 @@ agx3_subgroup_encode_reduce(enum agx_isa_encoding enc,
     */
    const unsigned slots = is_c(enc) ? 8 : 6;
    if (d->wait_mask != 0) {
-      if (is_c(enc) || (d->wait_mask >> slots) != 0)
+      if ((d->wait_mask >> slots) != 0)
          return false;
       for (unsigned i = 0; i < slots; ++i)
          if (d->wait_mask & (1u << i))
diff --git a/src/asahi/compiler/agx3_subgroup_core.h b/src/asahi/compiler/agx3_subgroup_core.h
index 9e8ef62361bf..02a2bb910ac9 100644
--- a/src/asahi/compiler/agx3_subgroup_core.h
+++ b/src/asahi/compiler/agx3_subgroup_core.h
@@ -81,7 +81,12 @@ struct agx3_subgroup_ballot_desc {
    uint16_t src;
    /* The G17P SIMD integer form can compare against a 16-bit immediate. */
    bool compare_immediate;
+   bool compare_register;
    uint16_t compare_value;
+   bool compare_half;
+   bool compare_cache;
+   bool compare_discard;
+   bool src_cache;
    /* The immediate equal-to-zero form also takes a 16-bit source register
     * (f24 = 1), as Apple's simd_broadcast_first() compares the 16-bit active
     * thread index against zero. */
@@ -136,6 +141,15 @@ struct agx3_subgroup_reduce_desc {
    bool half;
 };
 
+bool agx3_subgroup_encode_active_mask(uint16_t dest, bool dest_cache,
+                                      uint8_t wait_mask,
+                                       uint8_t out[AGX3_SUBGROUP_MAX_LENGTH],
+                                       uint8_t *length);
+bool agx3_subgroup_encode_quad_ballot_imm(uint16_t dest, bool dest_cache,
+                                          uint16_t value, uint8_t wait_mask,
+                                          uint8_t out[AGX3_SUBGROUP_MAX_LENGTH],
+                                          uint8_t *length);
+
 bool agx3_subgroup_encode_shuffle(enum agx_isa_encoding enc,
                                   const struct agx3_subgroup_shuffle_desc *desc,
                                   uint8_t out[AGX3_SUBGROUP_MAX_LENGTH],
diff --git a/src/asahi/compiler/agx3_tex_core.c b/src/asahi/compiler/agx3_tex_core.c
index 2e13b6b9ea79..463a81532ace 100644
--- a/src/asahi/compiler/agx3_tex_core.c
+++ b/src/asahi/compiler/agx3_tex_core.c
@@ -53,32 +53,38 @@ fetch_ok(enum agx_isa_encoding enc, const struct agx3_tex_fetch_desc *d)
       return false;
 
    if ((d->width != AGX3_TEX_W16 && d->width != AGX3_TEX_W32) ||
-       d->dest >= 128 || (d->width == AGX3_TEX_W32 && (d->dest & 1)) ||
-       (d->coord & 3) || d->coord > 60 ||
-       (enc == AGX_ISA_ENCODING_AGX3B ? (!d->mask || d->mask > 15)
-                                    : (d->mask != 1 && d->mask != 15)) ||
+       d->dest >= (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 128) ||
+       d->dest + __builtin_popcount(d->mask) * (d->width == AGX3_TEX_W32 ? 2 : 1) > 256 ||
+       (d->width == AGX3_TEX_W32 && (d->dest & 1)) ||
+       (d->coord & 3) ||
+       d->coord > ((enc == AGX_ISA_ENCODING_AGX3C && d->multisample &&
+                    !d->array) ? 120 : 60) ||
+       (!d->mask || d->mask > 15) ||
        d->texture > 15)
       return false;
 
    /* A18 Pro witnesses both 2D-MS and 2D-MS-array reads.  The non-array
     * form uses staging halves 0/2 and carries a bounded immediate sample.
     * The array form uses halves 16/18 and PACK_SMPIR publishes the packed
-    * sample/layer word separately. */
+    * sample/layer word separately. G17G 2D-MS fragments instead publish the
+    * sample beside each of four independent eight-half coordinate banks. */
    if ((d->sample && !d->multisample) ||
        (d->staged_sample && (!d->multisample || d->sample)) ||
        (d->multisample &&
-        (enc != AGX_ISA_ENCODING_AGX3B || d->sample > 3 ||
+        (d->sample > 3 ||
          (d->array && d->sample) ||
-         d->coord != (d->array && !d->staged_sample ? 16 : 0))))
+         (enc == AGX_ISA_ENCODING_AGX3C
+            ? (!d->staged_sample || (d->coord & 7) ||
+               d->coord > (d->array ? 24 : 120))
+            : d->coord != (d->array && !d->staged_sample ? 16 : 0)))))
       return false;
 
    /* Own-MSL texture3d::read witnesses use f76=3 with a third integer
     * coordinate in the contiguous staging bank. The immediate and indirect
     * texture forms are both captured on G17P. A runtime integer level uses
     * the same f139=4 PACK_SMPIR form as 2D; the dimension remains selected by
-    * f76=3. No AGX3c or combined array/multisample form is grounded. */
-   if (d->three_d &&
-       (enc != AGX_ISA_ENCODING_AGX3B || d->array || d->multisample))
+    * f76=3. M5 retains these dimension and integer-level selectors. */
+   if (d->three_d && (d->array || d->multisample))
       return false;
 
    const unsigned tk = d->texture_kind ?: AGX3_TEX_OPERAND_IMMEDIATE;
@@ -91,27 +97,27 @@ fetch_ok(enum agx_isa_encoding enc, const struct agx3_tex_fetch_desc *d)
        (d->texture_kill && tk != AGX3_TEX_OPERAND_REGISTER) ||
        (d->multisample &&
         (sk != AGX3_TEX_OPERAND_IMMEDIATE ||
-         d->sampler_value != (d->staged_sample ? 1 : 0))))
+         (enc == AGX_ISA_ENCODING_AGX3C ? d->sampler_value > 1
+            : d->sampler_value != (d->staged_sample ? 1 : 0)))))
       return false;
    if (d->three_d &&
-       (sk != AGX3_TEX_OPERAND_IMMEDIATE || d->sampler_value != 0))
+       (sk != AGX3_TEX_OPERAND_IMMEDIATE ||
+        d->sampler_value > (enc == AGX_ISA_ENCODING_AGX3C ? 1 : 0)))
       return false;
 
    /* The level. f29 is 12 bits of 1/64 fixed point; the per-lane form has no
-    * immediate at all, and is witnessed on AGX3b only. */
+    * immediate at all. */
    if (d->lod_x64 > 2047 || (d->register_lod && d->lod_x64) ||
-       (d->register_lod && enc != AGX_ISA_ENCODING_AGX3B) ||
        (d->multisample && (d->lod_x64 || d->register_lod)))
       return false;
 
-   /* Sparse residency is witnessed on AGX3b only, and only for the ordinary
-    * (non-multisample) read.  metal-nt emits no sparse multisample form, and
-    * f142 is beyond the 8-byte AGX3c encoding entirely. */
-   if (d->sparse && (enc != AGX_ISA_ENCODING_AGX3B || d->multisample))
+   /* Sparse reads use AGX3b ep4 or the 22-byte AGX3c ep3 form.
+    * No sparse multisample form is qualified. */
+   if (d->sparse && d->multisample)
       return false;
 
    const unsigned slots = enc == AGX_ISA_ENCODING_AGX3C ? 8 : 6;
-   const uint16_t waits = enc == AGX_ISA_ENCODING_AGX3C ? 0x7f : 0x3f;
+   const uint16_t waits = enc == AGX_ISA_ENCODING_AGX3C ? 0xff : 0x3f;
    return d->slot >= 1 && d->slot <= slots &&
           (d->wait_mask & ~waits) == 0 && one_or_zero(d->wait_mask);
 }
@@ -167,6 +173,7 @@ agx3_tex_encode_fetch(enum agx_isa_encoding enc,
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(2, d->dest);
    ov[n++] = (struct agx3_field_kv)KV(10, d->width);
+   ov[n++] = (struct agx3_field_kv)KV(11, d->dest_cache);
    ov[n++] = (struct agx3_field_kv)KV(12, d->slot);
    ov[n++] = (struct agx3_field_kv)KV(19, d->coord);
    ov[n++] = (struct agx3_field_kv)KV(26, d->slot);
@@ -210,7 +217,8 @@ agx3_tex_encode_fetch(enum agx_isa_encoding enc,
       ov[n++] = (struct agx3_field_kv)KV(155, 1);
    }
    if (d->sparse) {
-      ov[n++] = (struct agx3_field_kv)KV(120, 1);
+      if (enc == AGX_ISA_ENCODING_AGX3B)
+         ov[n++] = (struct agx3_field_kv)KV(120, 1);
       ov[n++] = (struct agx3_field_kv)KV(142, 1);
    }
    for (unsigned i = 0; i < 8; ++i)
@@ -219,18 +227,27 @@ agx3_tex_encode_fetch(enum agx_isa_encoding enc,
 
    /*
     * Immediate operands are ep3; register and uniform operands are ep4. Only
-    * AGX3b is witnessed for the non-immediate kinds, so AGX3c refuses them
-    * rather than assuming its encoding points line up.  f120 deposits
+    * AGX3b uses these encoding points for the non-immediate kinds too.
+    * AGX3c uses its own bounded ep0/1/2 forms below. f120 deposits
     * instruction bit 142, past the 112-bit ep3 form, so a sparse read is
     * always ep4.
     */
-   if ((!immediate_operands || d->sparse) && enc != AGX_ISA_ENCODING_AGX3B)
+   if (enc == AGX_ISA_ENCODING_AGX3C && !immediate_operands &&
+       (d->width != AGX3_TEX_W32 ||
+        sk != AGX3_TEX_OPERAND_IMMEDIATE || d->sampler_value > 1 ||
+        d->texture_kill))
       return false;
    const int ep = enc == AGX_ISA_ENCODING_AGX3B
-                     ? (immediate_operands && !d->sparse &&
-                        !d->heap_uniform_base ? 3 : 4)
-                     : 0;
-   if (!agx3_bind_compose(enc, "smp", ep, base, nb, ov, n, out, len)) {
+                     ? (immediate_operands && !d->sparse && !d->heap_uniform_base ? 3 : 4)
+                     : d->sparse ? 3 : 0;
+   bool composed = agx3_bind_compose(enc, "smp", ep, base, nb, ov, n, out, len);
+   if (!composed && enc == AGX_ISA_ENCODING_AGX3C)
+      composed = agx3_bind_compose(enc, "smp", 1, base, nb, ov, n, out, len);
+   if (!composed && enc == AGX_ISA_ENCODING_AGX3C)
+      composed = agx3_bind_compose(enc, "smp", 2, base, nb, ov, n, out, len);
+   if (!composed && enc == AGX_ISA_ENCODING_AGX3C && d->heap_uniform_base)
+      composed = agx3_bind_compose(enc, "smp", 3, base, nb, ov, n, out, len);
+   if (!composed) {
       memset(out, 0, AGX3_TEX_MAX_LENGTH);
       *len = 0;
       return false;
@@ -271,19 +288,18 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
 
    /* Sparse residency is witnessed on AGX3b for the implicit, immediate
     * level, immediate bias, 2D-array, 3D, cube and gather shapes.  A depth
-    * comparison, a LOD query and every AGX3c form have no witness, and f142
-    * lies past the 14-byte AGX3c sample entirely. */
-   if (d->sparse && (enc != AGX_ISA_ENCODING_AGX3B || d->shadow ||
-                     d->query_lod))
+    * comparison and a LOD query have no witness. AGX3c uses its 22-byte
+    * ep3 form to carry f142. */
+   if (d->sparse && (d->shadow || d->query_lod))
       return false;
 
    /* The native clamp corpus covers fragment 2D, 2D-array and 3D samples,
     * including comparison/offset and indexed descriptor operands, and
     * biased cube, cube-array and comparison cube samples. A staged 3D offset
     * is witnessed only beside a bias. */
+
    if (d->min_lod &&
-       (enc != AGX_ISA_ENCODING_AGX3B ||
-        (d->cube && d->lod != AGX3_TEX_LOD_BIAS_REGISTER) ||
+       ((d->cube && d->lod != AGX3_TEX_LOD_BIAS_REGISTER) ||
         d->vertex_stage || d->query_lod ||
         (d->three_d && d->offset_staged &&
          d->lod != AGX3_TEX_LOD_BIAS_REGISTER) ||
@@ -296,22 +312,22 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
    /* Fresh native calculate_{clamped,unclamped}_lod witnesses cover the
     * ordinary derivative path only, returning packed X/Y float32 results. */
    if (d->query_lod &&
-       (enc != AGX_ISA_ENCODING_AGX3B || d->width != AGX3_TEX_W32 ||
+       (d->width != AGX3_TEX_W32 ||
         d->lod != AGX3_TEX_LOD_AUTO || (d->mask & ~3u) || d->array_layer ||
         d->gather != AGX3_TEX_GATHER_NONE || d->shadow || d->offset ||
         d->sample_abi != AGX3_TEX_SAMPLE_ABI_DRIVER))
       return false;
 
    if ((d->sample_group_nonterminal || d->sample_group_end) &&
-       (enc != AGX_ISA_ENCODING_AGX3B ||
-        d->lod == AGX3_TEX_LOD_GRADIENT ||
+       (d->lod == AGX3_TEX_LOD_GRADIENT ||
         (d->sample_group_nonterminal && d->sample_group_end)))
       return false;
 
    if ((d->width != AGX3_TEX_W16 && d->width != AGX3_TEX_W32) ||
-       d->dest >= 128 || (d->width == AGX3_TEX_W32 && (d->dest & 1)) ||
+       d->dest >= (enc == AGX_ISA_ENCODING_AGX3C ? 256 : 128) ||
+       (d->width == AGX3_TEX_W32 && (d->dest & 1)) ||
        (d->coord & 3) ||
-       d->coord > (enc == AGX_ISA_ENCODING_AGX3B ? 252 : 60) ||
+       d->coord > 252 ||
        !mask_witnessed(d->mask) ||
        d->texture > 15 || d->sampler > 15)
       return false;
@@ -326,30 +342,26 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
        * is 2048) and saturates at 2047 / 2048. The caller passes the field
        * already masked to 12 bits. Apple emits the plain implicit form
        * rather than a zero bias, so refuse the degenerate encoding. */
-      if (enc != AGX_ISA_ENCODING_AGX3B || d->lod_x64 == 0 ||
+      if (d->lod_x64 == 0 ||
           d->lod_x64 > 0xfff)
          return false;
    } else if (d->lod == AGX3_TEX_LOD_REGISTER ||
               d->lod == AGX3_TEX_LOD_BIAS_REGISTER ||
               d->lod == AGX3_TEX_LOD_GRADIENT) {
-      /* The per-lane level, the per-lane bias and the four gradient scalars
-       * are all witnessed on AGX3b only, and none of them carries the
-       * immediate operand. */
-      if (enc != AGX_ISA_ENCODING_AGX3B || d->lod_x64 != 0)
+      /* Per-lane levels/biases use bank 6. Gradients use the shared
+       * interleaved scalar banks 8..18. */
+      if (d->lod_x64 != 0)
          return false;
    } else if (d->lod != AGX3_TEX_LOD_AUTO || d->lod_x64 != 0) {
       return false;
    }
 
-   if (enc == AGX_ISA_ENCODING_AGX3C && d->mask != 1 && d->mask != 3 &&
-       d->mask != 8 && d->mask != 12 && d->mask != 15)
-      return false;
 
    /* G17P names a dynamic array layer directly with f31/f37 in every
     * witnessed LOD mode. Do not accept a value the 32-bit register operand
     * cannot name, or silently ignore a layer on a non-array sample. */
    if ((d->array &&
-        (enc != AGX_ISA_ENCODING_AGX3B || (d->array_layer & 1) ||
+        ((d->array_layer & 1) ||
          d->array_layer > 254)) ||
        (!d->array && d->array_layer != 0))
       return false;
@@ -372,7 +384,7 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
     * literal-gradient kernels (see AGX3_TEX_GRADIENT_BANK_DDX_Z). Array,
     * gather and comparison combinations remain closed. */
    if (d->three_d &&
-       (enc != AGX_ISA_ENCODING_AGX3B || d->array || d->cube ||
+       (d->array || d->cube ||
         d->gather != AGX3_TEX_GATHER_NONE || d->shadow))
       return false;
 
@@ -388,7 +400,7 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
     * Cube and cube-array comparisons reuse that CEM path with their own
     * packed face / layer operands. */
    if (d->cube &&
-       (enc != AGX_ISA_ENCODING_AGX3B || d->three_d ||
+       (d->three_d ||
         d->coord != 0 ||
         (d->lod != AGX3_TEX_LOD_AUTO && d->lod != AGX3_TEX_LOD_LEVEL &&
          d->lod != AGX3_TEX_LOD_REGISTER && d->lod != AGX3_TEX_LOD_BIAS &&
@@ -398,10 +410,10 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
    const unsigned tk = d->texture_kind ? d->texture_kind : 1;
    const unsigned sk = d->sampler_kind ? d->sampler_kind : 1;
    if (tk > 3 || sk > 3 ||
-       ((tk != 1 || sk != 1) && enc != AGX_ISA_ENCODING_AGX3B) ||
        (tk != 1 && ((d->texture_value & 1) || d->texture_value > 252)) ||
        (sk != 1 && ((d->sampler_value & 1) || d->sampler_value > 252)) ||
-       (d->texture_kill && tk != 2) || (d->sampler_kill && sk != 2))
+       (d->texture_kill && tk != 2) || (d->sampler_kill && sk != 2) ||
+       (enc == AGX_ISA_ENCODING_AGX3C && d->sampler_kill))
       return false;
 
    /*
@@ -434,9 +446,11 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
        d->sample_abi != AGX3_TEX_SAMPLE_ABI_DRIVER)
       return false;
 
-   /* Cube operand pairs are witnessed as immediate/immediate,
-    * uniform/uniform and register/register. */
-   if (d->cube && tk != sk)
+   /* C's indexed cube-array samples also mix GPR and uniform descriptor
+    * operands in either direction. B retains its matching-class pairs. */
+   if (d->cube && tk != sk &&
+       !(enc == AGX_ISA_ENCODING_AGX3C &&
+         ((tk == 2 && sk == 3) || (tk == 3 && sk == 2))))
       return false;
 
    /* The coordinate offset is one immediate nibble pair in f43 plus two
@@ -486,10 +500,31 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
     * the X/Y nibbles and the bias into one word, adds the Z nibble at bank-6
     * bit 0, and names the minimum's packed register in SMP. */
    const bool staged_3d_bias_min =
-      d->three_d && d->offset_staged && d->min_lod_register;
+      d->three_d && d->offset_staged &&
+      (d->min_lod_register ||
+       (enc == AGX_ISA_ENCODING_AGX3C && d->min_lod &&
+        d->lod == AGX3_TEX_LOD_BIAS_REGISTER));
    const bool any_staged_offset = d->offset_staged || d->offset_register;
+   /* AGX3C immediate offsets (native G17G, texwit3): exactly {f43, f51 = 1,
+    * f141 = 1} over the offset-free form for 2D, register-layer 2D-array, 3D
+    * (twelve-bit f43), comparison (2D and array) and gather (any component,
+    * array, comparison) samples, in AUTO, LEVEL, REGISTER, BIAS_REGISTER and
+    * GRADIENT LOD shapes, including negative nibbles:
+    *   o_arr_auto  0f071600418062290100030000801100008000000000  f77=2 f43=17
+    *   o_3d_neg    0f05160045806129010003000080780f008000000000  f43=0xf78
+    *   o_cmp_grad  0f071200418000210100030000801100000000000000  f139=5 f43=17
+    *   o_gath_cmp  0f071600418003210118110000801100000000000000  f144=1 f43=17
+    * Runtime (staged, f141 = 2) offsets read bank 6 on C as on B: d_auto,
+    * d_lvlr (f139 = 4) and d_arr (f77 = 2) differ from it exactly as B's
+    * register-LOD and array forms do (M5StagedOffsetSampleWitnesses). */
    if (d->offset &&
-       (enc != AGX_ISA_ENCODING_AGX3B ||
+       ((enc != AGX_ISA_ENCODING_AGX3B &&
+         !(enc == AGX_ISA_ENCODING_AGX3C &&
+           (!any_staged_offset ||
+            /* Runtime (f141 = 2) 2D offsets, texwit3/4 d_*; a 2D array
+             * reads its layer from the same bank-6 word (d_arr). */
+            ((!d->three_d || staged_3d_bias_min) &&
+             (!d->array || d->offset_staged_array))))) ||
         (any_staged_offset
             ? d->offset_xyz != 0 || (d->three_d && !staged_3d_bias_min)
             : d->offset_xyz == 0 ||
@@ -530,7 +565,7 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
     * 3D gathers remain unsupported.
     */
    if (d->gather != AGX3_TEX_GATHER_NONE &&
-       (d->gather > AGX3_TEX_GATHER_A || enc != AGX_ISA_ENCODING_AGX3B ||
+       (d->gather > AGX3_TEX_GATHER_A ||
         d->lod != AGX3_TEX_LOD_AUTO || d->three_d))
       return false;
 
@@ -565,7 +600,10 @@ sample_ok(enum agx_isa_encoding enc, const struct agx3_tex_sample_desc *d)
    const bool staged_register_lod = d->lod == AGX3_TEX_LOD_REGISTER ||
                                     d->lod == AGX3_TEX_LOD_BIAS_REGISTER;
    if (d->shadow &&
-       (enc != AGX_ISA_ENCODING_AGX3B || d->three_d ||
+       (d->three_d ||
+        (enc == AGX_ISA_ENCODING_AGX3C && d->offset && d->array &&
+         (d->offset_register ||
+          (d->offset_staged && !d->offset_staged_array))) ||
         d->mask != (d->gather != AGX3_TEX_GATHER_NONE ? 15u : 1u) ||
         (((d->cube && !staged_register_lod) ||
           (d->array && !d->offset && !d->vertex_stage &&
@@ -736,11 +774,13 @@ agx3_tex_encode_sample(enum agx_isa_encoding enc,
       (d->lod == AGX3_TEX_LOD_AUTO || d->lod == AGX3_TEX_LOD_BIAS ||
        d->lod == AGX3_TEX_LOD_REGISTER ||
        d->lod == AGX3_TEX_LOD_BIAS_REGISTER ||
+       (enc == AGX_ISA_ENCODING_AGX3C && d->lod == AGX3_TEX_LOD_LEVEL) ||
        (d->lod != AGX3_TEX_LOD_GRADIENT && (d->array || d->cube)));
    if (d->sample_group_end ||
        (intrinsic_group && !d->sample_group_nonterminal))
       ov[n++] = (struct agx3_field_kv)KV(156, 1);
-   if (d->array) {
+   if (d->array && !(enc == AGX_ISA_ENCODING_AGX3C &&
+                     d->query_lod && !d->cube)) {
       /* Honeykrisp's driver descriptor ABI pairs arrays with the
        * sampler-private layer staged by PACK_SMPIR.  Native Apple fragment
        * witnesses use the direct f31/f37 path as part of their complete 4/8
@@ -813,8 +853,12 @@ agx3_tex_encode_sample(enum agx_isa_encoding enc,
    }
    if (d->shadow)
       ov[n++] = (struct agx3_field_kv)KV(143, 1);
-   if (d->query_lod)
-      ov[n++] = (struct agx3_field_kv)KV(146, 1);
+   if (d->query_lod) {
+      /* LOD queries use their own selector on each ISA. */
+      ov[n++] = enc == AGX_ISA_ENCODING_AGX3C
+         ? (struct agx3_field_kv)KV(155, 2)
+         : (struct agx3_field_kv)KV(146, 1);
+   }
    if (d->offset) {
       if (d->offset_staged || d->offset_register) {
          ov[n++] = (struct agx3_field_kv)KV(141, 2);
@@ -829,16 +873,33 @@ agx3_tex_encode_sample(enum agx_isa_encoding enc,
    else
       ov[n++] = (struct agx3_field_kv)KV(150, d->final ? 2 : 0);
    if (d->sparse) {
-      ov[n++] = (struct agx3_field_kv)KV(120, 1);
+      if (enc == AGX_ISA_ENCODING_AGX3B)
+         ov[n++] = (struct agx3_field_kv)KV(120, 1);
       ov[n++] = (struct agx3_field_kv)KV(142, 1);
    }
    for (unsigned i = 0; i < 8; ++i)
       if (d->wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(62 + i, 1);
 
-   /* f120 deposits instruction bit 142, past the 112-bit ep3 form, so a
-    * sparse sample takes ep4 even when both descriptors are immediate. */
-   const int ep = (immediate && !d->sparse && !d->heap_uniform_base) ? 3 : 4;
+   /* AGX3b sparse sampling needs ep4 even with immediate descriptors.
+    * AGX3c carries the residency selector in its 22-byte ep3 form. */
+   /* G17G's long form includes register/uniform descriptors in ep3.
+    * Unlike Neo it has no separate ep4 form. */
+   const int ep = enc == AGX_ISA_ENCODING_AGX3C ||
+                        (immediate && !d->sparse && !d->heap_uniform_base) ? 3 : 4;
+   if (enc == AGX_ISA_ENCODING_AGX3C &&
+       (d->query_lod || d->shadow || d->gather != AGX3_TEX_GATHER_NONE)) {
+      /* Query, comparison and gather use compact forms, including indexed
+       * descriptors. Escalate when the operands need a longer form. */
+      const int first_ep = d->query_lod ? (d->cube && !d->array ? 2 : 1)
+                           : d->gather != AGX3_TEX_GATHER_NONE ? 2 : 0;
+      for (int query_ep = first_ep; query_ep <= 3; ++query_ep)
+         if (agx3_bind_compose(enc, "smp", query_ep, base, nb, ov, n, out, len))
+            return true;
+      memset(out, 0, AGX3_TEX_MAX_LENGTH);
+      *len = 0;
+      return false;
+   }
    if (!agx3_bind_compose(enc, "smp", ep, base, nb, ov, n, out, len)) {
       memset(out, 0, AGX3_TEX_MAX_LENGTH);
       *len = 0;
@@ -848,8 +909,9 @@ agx3_tex_encode_sample(enum agx_isa_encoding enc,
    return true;
 }
 
-bool
-agx3_tex_stage_copy_hints(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+static bool
+agx3_tex_stage_copy_hints_impl(enum agx_isa_encoding enc, bool wide_source,
+                    uint16_t dest, uint16_t src,
                     uint8_t slot, uint16_t wait_mask,
                           bool src_cache, bool src_discard,
                     uint8_t out[AGX3_TEX_COPY_MAX_LENGTH], uint8_t *len)
@@ -859,17 +921,18 @@ agx3_tex_stage_copy_hints(enum agx_isa_encoding enc, uint16_t dest, uint16_t src
    memset(out, 0, AGX3_TEX_COPY_MAX_LENGTH);
    *len = 0;
    if ((src_cache && src_discard) ||
-       (enc == AGX_ISA_ENCODING_AGX3C && (src_cache || !src_discard)))
+       (enc == AGX_ISA_ENCODING_AGX3C && src_cache == src_discard))
       return false;
 
    if (enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C)
       return false;
 
-   /* One even 32-bit register each way, as in every witness. The source field
-    * is seven bits wide. Unlike PACK_SMPIR, COPY has no separately recovered
-    * bit 7, so accepting a high-bank source would alias it into the low bank.
+   /* High source registers require the extended M5 encoding. Keep the
+    * original helper's range unchanged for other publication paths.
     */
-   if ((dest & 1) || (src & 1) || dest >= 256 || src >= 128)
+   if ((wide_source && enc != AGX_ISA_ENCODING_AGX3C) ||
+       (dest & 1) || (src & 1) || dest >= 256 ||
+       src >= (wide_source ? 256 : 128))
       return false;
 
    const unsigned slots = enc == AGX_ISA_ENCODING_AGX3C ? 8 : 6;
@@ -903,7 +966,7 @@ agx3_tex_stage_copy_hints(enum agx_isa_encoding enc, uint16_t dest, uint16_t src
    /* The compact form first; Apple escalates the encoding point when a slot
     * or wait bit does not fit, and the composer's first-success order
     * reproduces that choice on every witness. */
-   for (int ep = 0; ep <= 1; ++ep) {
+   for (int ep = wide_source && src >= 128 ? 1 : 0; ep <= 1; ++ep) {
       if (agx3_bind_compose(enc, "copy", ep, base, nb, ov, n, out, len))
          return true;
    }
@@ -912,6 +975,27 @@ agx3_tex_stage_copy_hints(enum agx_isa_encoding enc, uint16_t dest, uint16_t src
    return false;
 }
 
+bool
+agx3_tex_stage_copy_hints(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
+                          uint8_t slot, uint16_t wait_mask,
+                          bool src_cache, bool src_discard,
+                          uint8_t out[AGX3_TEX_COPY_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_tex_stage_copy_hints_impl(enc, false, dest, src, slot, wait_mask,
+                                         src_cache, src_discard, out, len);
+}
+
+bool
+agx3_tex_stage_copy_scalar_m5(uint16_t dest, uint16_t src, uint8_t slot,
+                              uint16_t wait_mask, bool src_cache,
+                              bool src_discard,
+                              uint8_t out[AGX3_TEX_COPY_MAX_LENGTH], uint8_t *len)
+{
+   return agx3_tex_stage_copy_hints_impl(AGX_ISA_ENCODING_AGX3C, true, dest, src,
+                                         slot, wait_mask, src_cache,
+                                         src_discard, out, len);
+}
+
 bool
 agx3_tex_stage_copy(enum agx_isa_encoding enc, uint16_t dest, uint16_t src,
                     uint8_t slot, uint16_t wait_mask,
@@ -1245,35 +1329,43 @@ stage_typed_lod_half(enum agx_isa_encoding enc, uint16_t half, uint16_t src,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B ||
-       src >= 256 || slot < 1 || slot > 6 ||
-       (wait_mask & ~0x3f) || (src_cache && src_discard) ||
+   /* M5 uses a seven-bit source field and eight producer slots. Integer
+    * levels may remain live; floating levels consume their source here. */
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) ||
+       src >= (gpu1 ? 128 : 256) || slot < 1 || slot > slots ||
+       (wait_mask & ~((1u << slots) - 1)) || (src_cache && src_discard) ||
        (kind != AGX3_TEX_LEVEL_MINIMUM && kind != AGX3_TEX_LEVEL_FLOAT &&
-        kind != AGX3_TEX_LEVEL_INTEGER))
+        kind != AGX3_TEX_LEVEL_INTEGER) ||
+       (gpu1 && kind != AGX3_TEX_LEVEL_MINIMUM &&
+        (src_cache || (kind == AGX3_TEX_LEVEL_FLOAT && !src_discard))))
       return false;
 
    const struct agx3_field_kv base[] = {
       KV(5, half), KV(9, 5), KV(10, 2), KV(23, 2), KV(24, 1),
       KV(38, 1), KV(51, 1), KV(52, 2), KV(74, (unsigned)kind), KV(75, 16),
+      KV(56, 1), KV(57, 119),
    };
-   struct agx3_field_kv ov[10];
+   struct agx3_field_kv ov[12];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, src & 0x7f);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(25, src_cache);
    ov[n++] = (struct agx3_field_kv)KV(27, src_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
    if (!agx3_bind_compose(enc, "pack_smpir", 0, base,
-                         sizeof(base) / sizeof(base[0]), ov, n, out, len))
+                         gpu1 ? 12 : 10, ov, n, out, len))
       return false;
 
    /* The source is an eight-bit half-register index (f16 bits 0..6 at
     * instruction bits 41..47, bit 7 at bit 48). Apple's encoder's 191
     * bound is a compiler allocation policy, not a narrower wire field.
     * Keep the same native instruction for Mesa's full 256-half bank. */
-   out[6] |= (src >> 7) & 1;
+   if (!gpu1)
+      out[6] |= (src >> 7) & 1;
    return true;
 }
 
@@ -1284,7 +1376,8 @@ agx3_tex_stage_typed_lod_at(enum agx_isa_encoding enc, uint16_t bank,
                             enum agx3_tex_level_kind kind,
                             uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
 {
-   if (!sampler_bank_valid(bank)) {
+   if (enc == AGX_ISA_ENCODING_AGX3C ? ((bank & 7) || bank > 248)
+                                    : !sampler_bank_valid(bank)) {
       if (out && len) {
          memset(out, 0, AGX3_TEX_MAX_LENGTH);
          *len = 0;
@@ -1350,8 +1443,11 @@ agx3_tex_stage_bias_min(enum agx_isa_encoding enc, uint16_t bias,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || minimum >= 128 || bias >= 256 ||
-       slot < 1 || slot > 6 || (wait_mask & ~0x3f) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) || minimum >= 128 ||
+       bias >= (gpu1 ? 128 : 256) ||
+       slot < 1 || slot > slots || (wait_mask & ~((1u << slots) - 1)) ||
        (src_cache && src_discard))
       return false;
 
@@ -1360,7 +1456,7 @@ agx3_tex_stage_bias_min(enum agx_isa_encoding enc, uint16_t bias,
       KV(37, 2), KV(38, 1), KV(51, 1), KV(52, 2),
       KV(74, 1), KV(76, 2), KV(77, 16),
    };
-   struct agx3_field_kv ov[13];
+   struct agx3_field_kv ov[18];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, minimum);
    ov[n++] = (struct agx3_field_kv)KV(30, bias);
@@ -1369,7 +1465,11 @@ agx3_tex_stage_bias_min(enum agx_isa_encoding enc, uint16_t bias,
    ov[n++] = (struct agx3_field_kv)KV(27, src_discard);
    ov[n++] = (struct agx3_field_kv)KV(39, src_cache);
    ov[n++] = (struct agx3_field_kv)KV(41, src_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   if (gpu1) {
+      ov[n++] = (struct agx3_field_kv)KV(56, 1);
+      ov[n++] = (struct agx3_field_kv)KV(57, 119);
+   }
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
    return agx3_bind_compose(enc, "pack_smpir", 0, base,
@@ -1393,8 +1493,12 @@ agx3_tex_stage_gather_offset(enum agx_isa_encoding enc, uint16_t src,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || (src & 1) || src > 126 ||
-       slot < 1 || slot > 6 || (cache && discard) ||
+   /* AGX3C: the plain (non-array) form of the same PACK_SMPIR with C's
+    * fixed f56 = 1 / f57 = 119 (texwit3 d_gath). Array gathers stage their
+    * layer differently on C (a GPR-packed layer, d_arr) and stay refused. */
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !(m5 && !array)) || (src & 1) ||
+       src > 126 || slot < 1 || slot > (m5 ? 8 : 6) || (cache && discard) ||
        (array && ((packed_layer & 1) || packed_layer > 190)))
       return false;
 
@@ -1405,10 +1509,11 @@ agx3_tex_stage_gather_offset(enum agx_isa_encoding enc, uint16_t src,
       KV(44, array ? packed_layer : 0), KV(51, array ? 2 : 1),
       KV(52, 2), KV(55, array),
       KV(74, 5), KV(75, 12), KV(76, 5), KV(77, 28),
+      KV(56, 1), KV(57, 119),
    };
    return agx3_bind_compose(enc, "pack_smpir", 0, fields,
-                            sizeof(fields) / sizeof(fields[0]), NULL, 0,
-                            out, len);
+                            sizeof(fields) / sizeof(fields[0]) - (m5 ? 0 : 2),
+                            NULL, 0, out, len);
 }
 
 /* The dynamic array-layer witness uses PACK_SMPIR like register LOD, but its
@@ -1427,29 +1532,33 @@ agx3_tex_stage_array_layer_at(enum agx_isa_encoding enc, uint16_t bank,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || ((bank & 3) || bank > 248) ||
-       src >= 256 || slot < 1 || slot > 6 ||
-       (wait_mask & ~0x3f) || (src_cache && src_discard))
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) ||
+       (gpu1 ? ((bank & 7) || bank > 248) : ((bank & 3) || bank > 248)) ||
+       src >= (gpu1 ? 128 : 256) || slot < 1 || slot > slots ||
+       (wait_mask & ~((1u << slots) - 1)) || (src_cache && src_discard))
       return false;
 
    const struct agx3_field_kv base[] = {
       KV(5, bank + 6), KV(9, 5), KV(10, 2), KV(23, 2), KV(24, 1),
-      KV(38, 1), KV(51, 1), KV(52, 2), KV(74, 4),
+      KV(38, 1), KV(51, 1), KV(52, 2), KV(74, 4), KV(56, 1), KV(57, 119),
    };
-   struct agx3_field_kv ov[10];
+   struct agx3_field_kv ov[12];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, src & 0x7f);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(25, src_cache);
    ov[n++] = (struct agx3_field_kv)KV(27, src_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
    if (!agx3_bind_compose(enc, "pack_smpir", 0, base,
-                         sizeof(base) / sizeof(base[0]), ov, n, out, len))
+                         gpu1 ? 11 : 9, ov, n, out, len))
       return false;
 
-   out[6] |= (src >> 7) & 1;
+   if (!gpu1)
+      out[6] |= (src >> 7) & 1;
    return true;
 }
 
@@ -1483,6 +1592,20 @@ agx3_tex_stage_image_write_layer(enum agx_isa_encoding enc, uint16_t src,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      if ((src & 1) || src > 254 || (dest & 1) || dest > 8190 ||
+          slot < 1 || slot > 8 || wait_mask || (src_cache && src_discard))
+         return false;
+      const struct agx3_field_kv fields[] = {
+         KV(5, dest), KV(9, 5), KV(10, 2), KV(12, slot), KV(16, src),
+         KV(23, 2), KV(24, 1), KV(25, src_cache), KV(27, src_discard),
+         KV(38, 1), KV(51, 1), KV(52, 2), KV(56, 1), KV(57, 118), KV(74, 4),
+      };
+      return agx3_bind_compose(enc, "pack_imgwir", 0, fields,
+                               sizeof(fields) / sizeof(fields[0]),
+                               NULL, 0, out, len);
+   }
+
    if (enc != AGX_ISA_ENCODING_AGX3B || (src & 1) || src > 254 ||
        (dest & 1) || dest > 8190 || slot < 1 || slot > 6 ||
        (wait_mask & ~0x3f) || (src_cache && src_discard))
@@ -1533,21 +1656,23 @@ agx3_tex_stage_array_typed_lod(enum agx_isa_encoding enc, uint16_t bank,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || !sampler_bank_valid(bank) ||
-       layer >= 128 || (layer & 1) ||
-       lod >= 256 || slot < 1 || slot > 6 || (wait_mask & ~0x3f) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) || (gpu1 ? ((bank & 7) || bank > 248) : !sampler_bank_valid(bank)) || layer >= 128 ||
+       (!gpu1 && (layer & 1)) || lod >= (gpu1 ? 128 : 256) || slot < 1 || slot > slots ||
+       (wait_mask & ~((1u << slots) - 1)) ||
        (kind != AGX3_TEX_LEVEL_FLOAT && kind != AGX3_TEX_LEVEL_MINIMUM &&
         kind != AGX3_TEX_LEVEL_INTEGER) ||
-       (kind == AGX3_TEX_LEVEL_INTEGER && (lod & 1)) ||
+       (!gpu1 && kind == AGX3_TEX_LEVEL_INTEGER && (lod & 1)) ||
        (layer_cache && layer_discard) || (lod_cache && lod_discard))
       return false;
 
    const struct agx3_field_kv base[] = {
       KV(5, bank + 6), KV(9, 5), KV(10, 2), KV(23, 2), KV(24, 1),
       KV(37, 2), KV(38, 1), KV(51, 1), KV(52, 2),
-      KV(74, 4), KV(76, (unsigned)kind), KV(77, 16),
+      KV(74, 4), KV(76, (unsigned)kind), KV(77, 16), KV(56, 1), KV(57, 119),
    };
-   struct agx3_field_kv ov[13];
+   struct agx3_field_kv ov[15];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, layer);
    ov[n++] = (struct agx3_field_kv)KV(30, lod);
@@ -1556,12 +1681,12 @@ agx3_tex_stage_array_typed_lod(enum agx_isa_encoding enc, uint16_t bank,
    ov[n++] = (struct agx3_field_kv)KV(27, layer_discard);
    ov[n++] = (struct agx3_field_kv)KV(39, lod_cache);
    ov[n++] = (struct agx3_field_kv)KV(41, lod_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
 
-   return agx3_bind_compose(enc, "pack_smpir", 0, base,
-                            sizeof(base) / sizeof(base[0]), ov, n, out, len);
+   return agx3_bind_compose(enc, "pack_smpir", 0, base, gpu1 ? 14 : 12,
+                            ov, n, out, len);
 }
 
 bool
@@ -1622,32 +1747,48 @@ agx3_tex_stage_offset_2d(enum agx_isa_encoding enc, uint16_t x, uint16_t y,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || x >= 256 || y >= 256 ||
-       (x & 1) || (y & 1) || slot < 1 || slot > 6 ||
-       (wait_mask & ~0x3f) || (source_cache && source_discard))
+   /* AGX3C: the same PACK_SMPIR offset form plus C's fixed f56 = 1 and
+    * f57 = 119, eight slots, and seven-bit X (no high bit fixup). Native G17G
+    * runtime offsets (texwit3 d_auto, d_gath): X = h2, Y = h4, slot 1, both
+    * last uses
+    *   3702000a2500aa0a000160300000
+    * and d_cmp reads the two halves of one register (h4, h5). */
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || x >= (m5 ? 128 : 256) ||
+       y >= 256 || (!m5 && ((x & 1) || (y & 1))) || slot < 1 ||
+       slot > (m5 ? 8 : 6) || (wait_mask & ~(m5 ? 0xff : 0x3f)) ||
+       (source_cache && source_discard))
       return false;
 
    const struct agx3_field_kv base[] = {
       KV(5, 6), KV(9, 5), KV(10, 2), KV(23, 2), KV(24, 1),
       KV(37, 2), KV(38, 1), KV(51, 1), KV(52, 2),
       KV(74, 5), KV(75, 12), KV(76, 5), KV(77, 28),
+      KV(56, 1), KV(57, 119),
    };
-   struct agx3_field_kv ov[13];
+   struct agx3_field_kv ov[20];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, x & 0x7f);
    ov[n++] = (struct agx3_field_kv)KV(30, y);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(25, source_cache);
    ov[n++] = (struct agx3_field_kv)KV(41, source_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   if (m5) {
+      /* C names each source's own lifetime hint. */
+      ov[n++] = (struct agx3_field_kv)KV(27, source_discard);
+      ov[n++] = (struct agx3_field_kv)KV(39, source_cache);
+   }
+   for (unsigned i = 0; i < (m5 ? 8u : 6u); ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
 
    if (!agx3_bind_compose(enc, "pack_smpir", 0, base,
-                          sizeof(base) / sizeof(base[0]), ov, n, out, len))
+                          sizeof(base) / sizeof(base[0]) - (m5 ? 0 : 2),
+                          ov, n, out, len))
       return false;
 
-   out[6] |= (x >> 7) & 1;
+   if (!m5)
+      out[6] |= (x >> 7) & 1;
    return true;
 }
 
@@ -1661,57 +1802,92 @@ agx3_tex_stage_offset_2d(enum agx_isa_encoding enc, uint16_t x, uint16_t y,
  *   axis 1  1701540703040d5254219204
  *   axis 2  1701540203040c50502f9208
  */
-bool
-agx3_tex_cube_cem_calc_hints_to(enum agx_isa_encoding enc, uint16_t x,
-                                uint16_t y, uint16_t z, uint16_t dest,
-                                uint8_t axis, const bool discard[3],
-                                uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+/* AGX3B witnesses independent per-coordinate discard hints on the final
+ * axis. On AGX3C only uniform hints are witnessed, and a C comparison
+ * reference aliasing a coordinate must survive CEM until its COPY. */
+static bool
+cube_cem_calc_to(enum agx_isa_encoding enc, uint16_t x, uint16_t y,
+                 uint16_t z, uint16_t dest, uint8_t axis,
+                 const bool discard[3], uint16_t reference,
+                 uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
 {
    if (!out || !len || !discard)
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || x >= 128 || y >= 128 || z >= 128 ||
-       (dest & 1) || dest > 190 || axis > 2 ||
-       ((discard[0] || discard[1] || discard[2]) && axis != 2))
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       x >= 128 || y >= 128 || z >= 128 || (dest & 1) ||
+       dest > 190 || axis > 2 ||
+       ((discard[0] || discard[1] || discard[2]) && axis != 2) ||
+       (enc == AGX_ISA_ENCODING_AGX3C &&
+        (discard[0] != discard[1] || discard[1] != discard[2])))
       return false;
 
    const struct agx3_field_kv base[] = {
       KV(9, 2),  KV(10, 2), KV(11, 1), KV(23, 2), KV(24, 2),
       KV(40, 2), KV(41, 2), KV(57, 2), KV(58, 2),
    };
-   struct agx3_field_kv ov[10];
+   struct agx3_field_kv ov[13];
    unsigned n = 0;
+   if (enc == AGX_ISA_ENCODING_AGX3C) {
+      /* Axis zero writes the packed magnitude/face pair. */
+      ov[n++] = (struct agx3_field_kv)KV(10, axis == 0 ? 1 : 2);
+      ov[n++] = (struct agx3_field_kv)KV(65, 3);
+      ov[n++] = (struct agx3_field_kv)KV(66, 24);
+   }
    ov[n++] = (struct agx3_field_kv)KV(16, x);
    ov[n++] = (struct agx3_field_kv)KV(33, y);
    ov[n++] = (struct agx3_field_kv)KV(50, z);
    ov[n++] = (struct agx3_field_kv)KV(75, axis);
    ov[n++] = (struct agx3_field_kv)KV(2, dest);
-   if (axis == 2) {
-      if (discard[0])
-         ov[n++] = (struct agx3_field_kv)KV(27, 1);
-      if (discard[1])
-         ov[n++] = (struct agx3_field_kv)KV(44, 1);
-      if (discard[2])
-         ov[n++] = (struct agx3_field_kv)KV(61, 1);
-   } else {
-      ov[n++] = (struct agx3_field_kv)KV(25, 1);
-      ov[n++] = (struct agx3_field_kv)KV(42, 1);
-      ov[n++] = (struct agx3_field_kv)KV(59, 1);
+   const uint16_t source[3] = {x, y, z};
+   static const uint16_t cache_field[3] = {25, 42, 59};
+   static const uint16_t discard_field[3] = {27, 44, 61};
+   for (unsigned i = 0; i < 3; ++i) {
+      /* A C comparison publishes its reference after CEM. Preserve an
+       * aliased coordinate until that COPY, including on the final axis. */
+      const bool keep_reference = enc == AGX_ISA_ENCODING_AGX3C &&
+                                  source[i] == reference;
+      if (discard[i] && !keep_reference)
+         ov[n++] = (struct agx3_field_kv)KV(discard_field[i], 1);
+      else if (axis != 2 || keep_reference)
+         ov[n++] = (struct agx3_field_kv)KV(cache_field[i], 1);
    }
 
    return agx3_bind_compose(enc, "cem_calc", 0, base,
                             sizeof(base) / sizeof(base[0]), ov, n, out, len);
 }
 
+bool
+agx3_tex_cube_cem_calc_hints_to(enum agx_isa_encoding enc, uint16_t x,
+                                uint16_t y, uint16_t z, uint16_t dest,
+                                uint8_t axis, const bool discard[3],
+                                uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   return cube_cem_calc_to(enc, x, y, z, dest, axis, discard, UINT16_MAX,
+                           out, len);
+}
+
 bool
 agx3_tex_cube_cem_calc_to(enum agx_isa_encoding enc, uint16_t x, uint16_t y,
                           uint16_t z, uint16_t dest, uint8_t axis, bool discard,
                           uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
 {
    const bool hints[3] = {discard, discard, discard};
-   return agx3_tex_cube_cem_calc_hints_to(enc, x, y, z, dest, axis, hints, out,
-                                          len);
+   return cube_cem_calc_to(enc, x, y, z, dest, axis, hints, UINT16_MAX,
+                           out, len);
+}
+
+bool
+agx3_tex_cube_cem_calc_reference_to(
+   enum agx_isa_encoding enc, uint16_t x, uint16_t y, uint16_t z,
+   uint16_t dest, uint8_t axis, bool discard, uint16_t reference,
+   uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   const bool hints[3] = {discard, discard, discard};
+   return enc == AGX_ISA_ENCODING_AGX3C &&
+          cube_cem_calc_to(enc, x, y, z, dest, axis, hints, reference,
+                           out, len);
 }
 
 bool
@@ -1762,9 +1938,10 @@ agx3_tex_cube_cem_finalize_to(enum agx_isa_encoding enc, uint8_t component,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || component > 1 ||
-       (reciprocal & 1) || reciprocal > 190 || (axis_value & 1) ||
-       axis_value > 190 || slot < 1 || slot > 6)
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       component > 1 || (reciprocal & 1) || reciprocal > 190 ||
+       (axis_value & 1) || axis_value > 190 || slot < 1 ||
+       slot > (enc == AGX_ISA_ENCODING_AGX3C ? 8 : 6))
       return false;
 
    const struct agx3_field_kv base0[] = {
@@ -1780,12 +1957,13 @@ agx3_tex_cube_cem_finalize_to(enum agx_isa_encoding enc, uint8_t component,
    };
    const struct agx3_field_kv ov[] = {
       KV(12, slot), KV(18, axis_value), KV(35, reciprocal),
+      KV(95, 1), KV(96, 55),
    };
    return agx3_bind_compose(
       enc, "f32fma", 5, component ? base1 : base0,
       component ? sizeof(base1) / sizeof(base1[0])
                 : sizeof(base0) / sizeof(base0[0]),
-      ov, sizeof(ov) / sizeof(ov[0]), out, len);
+      ov, enc == AGX_ISA_ENCODING_AGX3C ? 5 : 3, out, len);
 }
 
 bool
@@ -1799,7 +1977,7 @@ agx3_tex_cube_cem_finalize(enum agx_isa_encoding enc, uint8_t component,
    if (!agx3_tex_cube_cem_finalize_to(enc, component, 0,
                                       axis_value[component], 1, out, len))
       return false;
-   if (discard)
+   if (discard && enc == AGX_ISA_ENCODING_AGX3B)
       out[10] |= 0x60;
    return true;
 }
@@ -1817,29 +1995,34 @@ agx3_tex_stage_cube_z(enum agx_isa_encoding enc, uint16_t src,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || src >= 256 || (src & 1) ||
-       slot < 1 || slot > 6 || (wait_mask & ~0x3f) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) ||
+       src >= (gpu1 ? 128 : 256) || (src & 1) ||
+       slot < 1 || slot > slots || (wait_mask & ~((1u << slots) - 1)) ||
        !one_or_zero(wait_mask) || (src_cache && src_discard))
       return false;
 
    const struct agx3_field_kv base[] = {
       KV(5, 6), KV(9, 5), KV(10, 2), KV(23, 2), KV(24, 1),
       KV(38, 1), KV(51, 1), KV(52, 2), KV(74, 10),
+      KV(56, 1), KV(57, 119),
    };
-   struct agx3_field_kv ov[10];
+   struct agx3_field_kv ov[12];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, src & 0x7f);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(25, src_cache);
    ov[n++] = (struct agx3_field_kv)KV(27, src_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
    if (!agx3_bind_compose(enc, "pack_smpir", 0, base,
-                         sizeof(base) / sizeof(base[0]), ov, n, out, len))
+                         gpu1 ? 11 : 9, ov, n, out, len))
       return false;
 
-   out[6] |= (src >> 7) & 1;
+   if (!gpu1)
+      out[6] |= (src >> 7) & 1;
    return true;
 }
 
@@ -1866,9 +2049,11 @@ agx3_tex_stage_cube_array_layer(enum agx_isa_encoding enc, uint16_t face,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || face >= 128 || (face & 1) ||
-       layer >= 128 || slot < 1 || slot > 6 ||
-       (wait_mask & ~0x3f) || !one_or_zero(wait_mask) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) || face >= 128 || (face & 1) ||
+       layer >= 128 || slot < 1 || slot > slots ||
+       (wait_mask & ~((1u << slots) - 1)) || !one_or_zero(wait_mask) ||
        (layer_cache && layer_discard))
       return false;
 
@@ -1877,14 +2062,18 @@ agx3_tex_stage_cube_array_layer(enum agx_isa_encoding enc, uint16_t face,
       KV(37, 2), KV(38, 1), KV(41, 1), KV(51, 1), KV(52, 2),
       KV(74, 4), KV(76, 10), KV(77, 12),
    };
-   struct agx3_field_kv ov[11];
+   struct agx3_field_kv ov[18];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(30, face);
    ov[n++] = (struct agx3_field_kv)KV(16, layer);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(25, layer_cache);
    ov[n++] = (struct agx3_field_kv)KV(27, layer_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   if (gpu1) {
+      ov[n++] = (struct agx3_field_kv)KV(56, 1);
+      ov[n++] = (struct agx3_field_kv)KV(57, 119);
+   }
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
 
@@ -1917,8 +2106,12 @@ agx3_tex_stage_cube_lod(enum agx_isa_encoding enc, uint16_t face,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || face >= 256 || (face & 1) ||
-       lod >= 128 || slot < 1 || slot > 6 || (wait_mask & ~0x3f) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) ||
+       face >= (gpu1 ? 128 : 256) || (face & 1) ||
+       lod >= 128 || slot < 1 || slot > slots ||
+       (wait_mask & ~((1u << slots) - 1)) ||
        !one_or_zero(wait_mask) || (lod_cache && lod_discard))
       return false;
 
@@ -1926,23 +2119,25 @@ agx3_tex_stage_cube_lod(enum agx_isa_encoding enc, uint16_t face,
       KV(5, 6),  KV(9, 5),  KV(10, 2), KV(23, 2), KV(24, 1),
       KV(27, 1), KV(37, 2), KV(38, 1), KV(51, 1), KV(52, 2),
       KV(74, 10), KV(76, (unsigned)AGX3_TEX_LEVEL_FLOAT), KV(77, 16),
+      KV(56, 1), KV(57, 119),
    };
-   struct agx3_field_kv ov[11];
+   struct agx3_field_kv ov[13];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, face & 0x7f);
    ov[n++] = (struct agx3_field_kv)KV(30, lod);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(39, lod_cache);
    ov[n++] = (struct agx3_field_kv)KV(41, lod_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
    if (!agx3_bind_compose(enc, "pack_smpir", 0, base,
-                         sizeof(base) / sizeof(base[0]), ov, n, out, len))
+                         gpu1 ? 15 : 13, ov, n, out, len))
       return false;
 
    /* f16 bit 7, as for the face-only form. */
-   out[6] |= (face >> 7) & 1;
+   if (!gpu1)
+      out[6] |= (face >> 7) & 1;
    return true;
 }
 
@@ -1966,21 +2161,25 @@ agx3_tex_pack_cube_face(enum agx_isa_encoding enc, uint16_t face,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || face >= 256 || (face & 1) ||
-       dest > 190 || (dest & 1))
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) ||
+       face >= (gpu1 ? 128 : 256) || (face & 1) ||
+       dest > (gpu1 ? 254 : 190) || (dest & 1))
       return false;
    const struct agx3_field_kv fields[] = {
       KV(2, dest), KV(9, 2), KV(10, 2), KV(11, 1), KV(16, face & 0x7f),
       KV(23, 2), KV(24, 1), KV(27, 1), KV(38, 1), KV(51, 1), KV(52, 2),
       KV(74, 10), KV(75, 12),
    };
+   const struct agx3_field_kv c_fields[] = {KV(56, 3), KV(57, 119)};
    if (!agx3_bind_compose(enc, "pack_smpir", 0, fields,
-                          sizeof(fields) / sizeof(fields[0]), NULL, 0, out,
-                          len))
+                          sizeof(fields) / sizeof(fields[0]), c_fields,
+                          gpu1 ? 2 : 0, out, len))
       return false;
 
    /* f16 bit 7, as for the face-only staging form. */
-   out[6] |= (face >> 7) & 1;
+   if (!gpu1)
+      out[6] |= (face >> 7) & 1;
    return true;
 }
 
@@ -1995,9 +2194,12 @@ agx3_tex_stage_cube_bias_min(enum agx_isa_encoding enc, uint16_t bias,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || minimum >= 128 || bias >= 256 ||
-       packed_face > 190 || (packed_face & 1) || slot < 1 || slot > 6 ||
-       (wait_mask & ~0x3f) || !one_or_zero(wait_mask) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) || minimum >= 128 ||
+       bias >= (gpu1 ? 128 : 256) ||
+       packed_face > 190 || (packed_face & 1) || slot < 1 || slot > slots ||
+       (wait_mask & ~((1u << slots) - 1)) || !one_or_zero(wait_mask) ||
        (src_cache && src_discard))
       return false;
 
@@ -2007,7 +2209,7 @@ agx3_tex_stage_cube_bias_min(enum agx_isa_encoding enc, uint16_t bias,
       KV(74, (unsigned)AGX3_TEX_LEVEL_MINIMUM),
       KV(76, (unsigned)AGX3_TEX_LEVEL_FLOAT), KV(77, 16),
    };
-   struct agx3_field_kv ov[14];
+   struct agx3_field_kv ov[18];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, minimum);
    ov[n++] = (struct agx3_field_kv)KV(30, bias);
@@ -2017,7 +2219,11 @@ agx3_tex_stage_cube_bias_min(enum agx_isa_encoding enc, uint16_t bias,
    ov[n++] = (struct agx3_field_kv)KV(27, src_discard);
    ov[n++] = (struct agx3_field_kv)KV(39, src_cache);
    ov[n++] = (struct agx3_field_kv)KV(41, src_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   if (gpu1) {
+      ov[n++] = (struct agx3_field_kv)KV(56, 1);
+      ov[n++] = (struct agx3_field_kv)KV(57, 119);
+   }
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
    return agx3_bind_compose(enc, "pack_smpir", 0, base,
@@ -2056,9 +2262,11 @@ agx3_tex_stage_cube_array_lod(enum agx_isa_encoding enc, uint16_t face,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || face >= 128 || (face & 1) ||
+   const bool gpu1 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned slots = gpu1 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !gpu1) || face >= 128 || (face & 1) ||
        lod >= 128 || packed_layer > 190 || (packed_layer & 1) || slot < 1 ||
-       slot > 6 || (wait_mask & ~0x3f) || !one_or_zero(wait_mask) ||
+       slot > slots || (wait_mask & ~((1u << slots) - 1)) || !one_or_zero(wait_mask) ||
        (lod_cache && lod_discard))
       return false;
 
@@ -2068,7 +2276,7 @@ agx3_tex_stage_cube_array_lod(enum agx_isa_encoding enc, uint16_t face,
       KV(55, 1),  KV(74, 10), KV(75, 12),
       KV(76, (unsigned)AGX3_TEX_LEVEL_FLOAT), KV(77, 16),
    };
-   struct agx3_field_kv ov[12];
+   struct agx3_field_kv ov[18];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(16, face);
    ov[n++] = (struct agx3_field_kv)KV(30, lod);
@@ -2076,7 +2284,11 @@ agx3_tex_stage_cube_array_lod(enum agx_isa_encoding enc, uint16_t face,
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(39, lod_cache);
    ov[n++] = (struct agx3_field_kv)KV(41, lod_discard);
-   for (unsigned i = 0; i < 6; ++i)
+   if (gpu1) {
+      ov[n++] = (struct agx3_field_kv)KV(56, 1);
+      ov[n++] = (struct agx3_field_kv)KV(57, 119);
+   }
+   for (unsigned i = 0; i < slots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
 
@@ -2101,6 +2313,37 @@ agx3_tex_stage_cube_array_lod(enum agx_isa_encoding enc, uint16_t face,
  *   05800ca090004200010000080100   SMP f37=2, f30=0, f41=1
  *   0d840ca090244210010000080100   SMP f37=2, f30=2, f41=1
  */
+bool
+agx3_tex_pack_float_lod_m5(uint16_t src, uint16_t dest,
+                          bool cache, bool discard, uint16_t wait_mask,
+                          uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len)
+      return false;
+   memset(out, 0, AGX3_TEX_MAX_LENGTH);
+   *len = 0;
+   if (src >= 128 || dest >= 256 || (dest & 1) ||
+       (cache && discard) || (wait_mask & ~0xffu))
+      return false;
+
+   /* Native shared_level/shared_bias return the packed parameter in a GPR,
+    * f9=2/f11=1 and f56=3, instead of publishing a staging result slot. */
+   const struct agx3_field_kv fields[] = {
+      KV(2, dest), KV(9, 2), KV(10, 2), KV(11, 1),
+      KV(16, src), KV(23, 2), KV(24, 1),
+      KV(25, cache), KV(27, discard), KV(38, 1), KV(51, 1), KV(52, 2),
+      KV(56, 3), KV(57, 119), KV(74, AGX3_TEX_LEVEL_FLOAT), KV(75, 16),
+   };
+   struct agx3_field_kv waits[8];
+   unsigned n = 0;
+   for (unsigned i = 0; i < 8; ++i)
+      if (wait_mask & (1u << i))
+         waits[n++] = (struct agx3_field_kv)KV(58 + i, 1);
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "pack_smpir", 0,
+                            fields, sizeof(fields) / sizeof(fields[0]),
+                            waits, n, out, len);
+}
+
 bool
 agx3_tex_pack_minimum(enum agx_isa_encoding enc, uint16_t src, uint16_t dest,
                       bool cache, bool discard,
@@ -2123,6 +2366,41 @@ agx3_tex_pack_minimum(enum agx_isa_encoding enc, uint16_t src, uint16_t dest,
                            sizeof(fields) / sizeof(fields[0]), NULL, 0, out, len);
 }
 
+bool
+agx3_tex_stage_minimum_bank8_m5(uint16_t src, uint8_t slot, uint16_t wait_mask,
+                                bool cache, bool discard,
+                                uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len)
+      return false;
+   memset(out, 0, AGX3_TEX_MAX_LENGTH);
+   *len = 0;
+   if (src >= 128 || slot < 1 || slot > 8 || (wait_mask & ~0xffu) ||
+       (cache && discard))
+      return false;
+   /* Native G17G comparison/3D minimums with a register bias (port/texwit8
+    * ca_br_min_{d,k,n}, m3_br_min_d): like G17P's minimum pack (minimum
+    * f74 = 1 at 0 beside an immediate at 16), but the immediate's format is
+    * f76 = 2 (G17P: 3), written to bank 8 instead of a GPR the SMP names:
+    *   2702002a2500aa1200004a301800  bank8 <- min(^h6), slot 1 */
+   const struct agx3_field_kv base[] = {
+      KV(5, 8), KV(9, 5), KV(10, 2), KV(23, 2), KV(24, 1),
+      KV(37, 1), KV(38, 1), KV(51, 1), KV(52, 2), KV(56, 1), KV(57, 119),
+      KV(74, (unsigned)AGX3_TEX_LEVEL_MINIMUM), KV(76, 2), KV(77, 16),
+   };
+   struct agx3_field_kv ov[16];
+   unsigned n = 0;
+   ov[n++] = (struct agx3_field_kv)KV(16, src);
+   ov[n++] = (struct agx3_field_kv)KV(12, slot);
+   ov[n++] = (struct agx3_field_kv)KV(25, cache);
+   ov[n++] = (struct agx3_field_kv)KV(27, discard);
+   for (unsigned i = 0; i < 8; ++i)
+      if (wait_mask & (1u << i))
+         ov[n++] = (struct agx3_field_kv)KV(58 + i, 1);
+   return agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "pack_smpir", 0, base,
+                            sizeof(base) / sizeof(base[0]), ov, n, out, len);
+}
+
 bool
 agx3_tex_pack_ms_array_layer(enum agx_isa_encoding enc, uint16_t src,
                             uint16_t dest, bool cache, bool discard,
@@ -2132,7 +2410,8 @@ agx3_tex_pack_ms_array_layer(enum agx_isa_encoding enc, uint16_t src,
       return false;
    memset(out, 0, AGX3_TEX_MAX_LENGTH);
    *len = 0;
-   if (enc != AGX_ISA_ENCODING_AGX3B || src >= 128 || dest > 190 ||
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       src >= 128 || dest > (enc == AGX_ISA_ENCODING_AGX3C ? 254 : 190) ||
        (dest & 1) || (cache && discard))
       return false;
    const struct agx3_field_kv fields[] = {
@@ -2140,8 +2419,10 @@ agx3_tex_pack_ms_array_layer(enum agx_isa_encoding enc, uint16_t src,
       KV(16, src), KV(23, 2), KV(24, 1), KV(25, cache), KV(27, discard),
       KV(38, 1), KV(51, 1), KV(52, 2), KV(74, 4),
    };
+   const struct agx3_field_kv c_fields[] = {KV(56, 3), KV(57, 119)};
    return agx3_bind_compose(enc, "pack_smpir", 0, fields,
-                           sizeof(fields) / sizeof(fields[0]), NULL, 0, out, len);
+                           sizeof(fields) / sizeof(fields[0]), c_fields,
+                           enc == AGX_ISA_ENCODING_AGX3C ? 2 : 0, out, len);
 }
 
 bool
@@ -2166,6 +2447,37 @@ agx3_tex_stage_ms_array_sample(enum agx_isa_encoding enc, uint16_t sample,
                            sizeof(fields) / sizeof(fields[0]), NULL, 0, out, len);
 }
 
+/* G17G arraydynamic/arrayindexed merge a normalized layer word with the
+ * sample half before SMP.ep2. The two native publishers are
+ * 2700003a2d00aa0a000040308100 and2702002a2500aa0a00004a400c00. */
+bool
+agx3_tex_stage_ms_array_sample_at(
+   enum agx_isa_encoding enc, uint16_t bank, uint16_t sample,
+   uint16_t packed_layer, uint8_t slot, bool cache, bool discard,
+   uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   if (enc == AGX_ISA_ENCODING_AGX3B && bank == 0)
+      return agx3_tex_stage_ms_array_sample(enc, sample, packed_layer,
+                                             cache, discard, out, len);
+   if (!out || !len)
+      return false;
+   memset(out, 0, AGX3_TEX_MAX_LENGTH);
+   *len = 0;
+   if (enc != AGX_ISA_ENCODING_AGX3C || (bank & 7) || bank > 24 ||
+       sample >= 128 || (sample & 1) || packed_layer > 254 ||
+       (packed_layer & 1) || slot < 1 || slot > 8 || (cache && discard))
+      return false;
+   const struct agx3_field_kv fields[] = {
+      KV(5, bank + 4), KV(9, 5), KV(10, 2), KV(12, slot),
+      KV(16, sample), KV(23, 2), KV(24, 1), KV(25, cache), KV(27, discard),
+      KV(37, 1), KV(38, 1), KV(44, packed_layer), KV(51, 2),
+      KV(52, 2), KV(55, 1), KV(56, 1), KV(57, 119),
+      KV(74, 9), KV(75, 12), KV(76, 1), KV(77, 16),
+   };
+   return agx3_bind_compose(enc, "pack_smpir", 0, fields,
+                            sizeof(fields) / sizeof(fields[0]), NULL, 0, out, len);
+}
+
 /* Native texture2d_ms reads publish the sample at fixed staging half4.
  * Field24=1 reads the low16 of the sample producer. Field16 uses the same
  * half-register address as the producer, without converting it to words.
@@ -2202,6 +2514,44 @@ agx3_tex_stage_ms_sample(enum agx_isa_encoding enc, uint16_t src,
                             sizeof(base) / sizeof(base[0]), ov, n, out, len);
 }
 
+/* G17G dynamicreg publishes the sample's low half to bank4 with
+ * 2700002a2500aa0a00004a300800. Its per-sample fragment uses the same
+ * publisher at banks4/12/20/28 beside four independent coordinate banks. */
+bool
+agx3_tex_stage_ms_sample_at(enum agx_isa_encoding enc, uint16_t bank,
+                            uint16_t src, uint8_t slot, uint16_t wait_mask,
+                            bool src_cache, bool src_discard,
+                            uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   if (enc == AGX_ISA_ENCODING_AGX3B && bank == 0)
+      return agx3_tex_stage_ms_sample(enc, src, wait_mask, src_cache,
+                                       src_discard, out, len);
+   if (!out || !len)
+      return false;
+   memset(out, 0, AGX3_TEX_MAX_LENGTH);
+   *len = 0;
+   if (enc != AGX_ISA_ENCODING_AGX3C || (bank & 7) || bank > 120 ||
+       src >= 256 || (src & 1) || slot < 1 || slot > 8 ||
+       (wait_mask & ~0xffu) || (src_cache && src_discard))
+      return false;
+
+   const struct agx3_field_kv fields[] = {
+      KV(5, bank + 4), KV(9, 5), KV(10, 2), KV(12, slot),
+      KV(16, src), KV(23, 2), KV(24, 1),
+      KV(25, src_cache), KV(27, src_discard),
+      KV(37, 1), KV(38, 1), KV(51, 1), KV(52, 2),
+      KV(56, 1), KV(57, 119), KV(74, 9), KV(75, 12), KV(76, 1), KV(77, 16),
+   };
+   struct agx3_field_kv waits[8];
+   unsigned n = 0;
+   for (unsigned i = 0; i < 8; ++i)
+      if (wait_mask & (1u << i))
+         waits[n++] = (struct agx3_field_kv)KV(58 + i, 1);
+   return agx3_bind_compose(enc, "pack_smpir", 0, fields,
+                            sizeof(fields) / sizeof(fields[0]), waits, n,
+                            out, len);
+}
+
 /* Fresh native G17P LOD fragment witnesses use this 12-byte PACK_SMPIR for a
  * scalar LOD, preserving the source when it remains live. f5=6 selects
  * operand bank 6, not architectural r3. The FLOAT source is a half, while the
@@ -2248,3 +2598,59 @@ agx3_tex_stage_typed_lod_bank(enum agx_isa_encoding enc, uint16_t bank,
    out[6] |= (src >> 7) & 1;
    return true;
 }
+
+bool
+agx3_tex_encode_image_write_agx3c(
+   const struct agx3_image_write_desc *d, uint16_t uniform_base,
+   uint8_t staging_slot, uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len)
+{
+   if (!out || !len)
+      return false;
+   memset(out, 0, AGX3_TEX_MAX_LENGTH);
+   *len = 0;
+   if (!d)
+      return false;
+
+   const unsigned kind = d->image_kind ?: AGX3_TEX_OPERAND_IMMEDIATE;
+   const bool immediate = kind == AGX3_TEX_OPERAND_IMMEDIATE;
+   const bool reg = kind == AGX3_TEX_OPERAND_REGISTER;
+   const unsigned result_slot = d->result_slot ?: 1;
+   if ((uniform_base & 3) || uniform_base > 252 ||
+       staging_slot < 1 || staging_slot > 8 || result_slot > 8 ||
+       (d->wait_mask & ~UINT16_C(0xff)) ||
+       (kind != AGX3_TEX_OPERAND_IMMEDIATE && !reg &&
+        kind != AGX3_TEX_OPERAND_UNIFORM) ||
+       (immediate ? d->image_value > 15
+                  : ((d->image_value & 1) || d->image_value > (reg ? 126 : 254))) ||
+       (d->image_kill && !reg) ||
+       (d->staging_bank & 15) || d->staging_bank > 240 ||
+       (d->dim != AGX3_IMAGE_WRITE_2D &&
+        d->dim != AGX3_IMAGE_WRITE_2D_ARRAY &&
+        d->dim != AGX3_IMAGE_WRITE_3D))
+      return false;
+
+   /* Native compute-write corpus: float/uint differ only in f70; a
+    * descriptor-set heap at uniform4 sets f79 independently of its index.
+    * writes8/writes12 witness banks0..176 and result slots beyond four.
+    * reuse_index clears f103 until the index's final consumer. */
+   const struct agx3_field_kv fields[] = {
+      KV(5, d->staging_bank), KV(9, 5), KV(10, 2), KV(12, result_slot),
+      KV(13, 1), KV(24, 1), KV(38, 1), KV(51, 1), KV(52, 1),
+      KV(56, 2), KV(58, 87), KV(58 + staging_slot, 1),
+      KV(70, d->normalized), KV(71, 2),
+      KV(72, d->dim == AGX3_IMAGE_WRITE_3D ? 3 : 1),
+      KV(73, d->dim == AGX3_IMAGE_WRITE_2D_ARRAY ? 2 : 0), KV(74, 1),
+      KV(79, uniform_base), KV(85, 3), KV(86, 3),
+      KV(immediate ? 91 : reg ? 92 : 93, d->image_value),
+      KV(99, immediate ? 1 : reg ? 2 : 3), KV(100, immediate ? 1 : 2),
+      KV(103, d->image_kill), KV(113, 1),
+   };
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "imgwci", 0,
+                          fields, sizeof(fields) / sizeof(fields[0]),
+                          NULL, 0, out, len) || *len != 18) {
+      memset(out, 0, AGX3_TEX_MAX_LENGTH);
+      *len = 0;
+      return false;
+   }
+   return true;
+}
diff --git a/src/asahi/compiler/agx3_tex_core.h b/src/asahi/compiler/agx3_tex_core.h
index 4ae492221623..018b8e1f7e36 100644
--- a/src/asahi/compiler/agx3_tex_core.h
+++ b/src/asahi/compiler/agx3_tex_core.h
@@ -23,9 +23,8 @@
 extern "C" {
 #endif
 
-/* Long enough for every emitted form: the texel fetch is 14 bytes (AGX3b) /
- * 8 bytes (AGX3c); the filtered sample is 14 bytes (AGX3b) / 22 bytes
- * (AGX3c, smp.ep3). */
+/* Long enough for dense and sparse operations: AGX3b uses up to 20 bytes,
+ * while AGX3c uses up to 22 bytes (smp.ep3). */
 #define AGX3_TEX_MAX_LENGTH 22u
 
 enum agx3_tex_width {
@@ -77,7 +76,8 @@ enum agx3_tex_level_kind {
 };
 
 struct agx3_tex_fetch_desc {
-   uint16_t heap_uniform_base;
+   uint16_t heap_uniform_base; /* texture at base, sampler at base+4 */
+   bool dest_cache;    /* destination lifetime hint */
    uint16_t dest;       /* half-register index */
    uint16_t coord;      /* four-half aligned pair of f32 coordinates */
    enum agx3_tex_width width;
@@ -110,13 +110,11 @@ struct agx3_tex_fetch_desc {
    uint16_t lod_x64;
    bool register_lod;
    /*
-    * Sparse residency: f142=1 requests it and f120=1 marks the access. The
+    * Sparse residency: f142=1 requests it; AGX3b also sets f120=1. The
     * residency code is one more 32-bit register after the masked channels
-    * (dest + channels * halves), as on AGX2. f120 lies past the 112-bit ep3
-    * form, so a sparse operation is always the 20-byte ep4 form. Apple also
-    * changes the f112 operand profile, but that profile makes sparse fetches
-    * and explicit-LOD sparse samples fail on G17P, so the dense profile is
-    * kept. AGX3b only.
+    * (dest + channels * halves), as on AGX2. AGX3b uses the 20-byte ep4
+    * form and AGX3c uses the 22-byte ep3 form. Keep the driver descriptor
+    * profile: Apple's argument-buffer f112 profile is a different ABI.
     */
    bool sparse;
 };
@@ -141,6 +139,10 @@ bool agx3_tex_decode_fetch(enum agx_isa_encoding enc, const void *instruction,
  *  - REGISTER: per-lane half-float level packed by PACK_SMPIR (f139=4). The
  *    fragment marker is present here too; it does not select derivatives.
  *
+ * G17G also uses these 2D forms, constant/register bias, and immediate XY
+ * offsets. Its fragment immediate LEVEL form sets the group marker too.
+ * The C floating-level publisher consumes its half-register source.
+ *
  * G17P also has 2D-array (register layer) and cube forms, and 3D
  * immediate-level samples (f76=3 selects the third coordinate). Other
  * dimension/LOD/offset/shadow/gather combinations are refused.
@@ -187,6 +189,9 @@ enum agx3_tex_gather {
 };
 
 struct agx3_tex_sample_desc {
+   /* Relocate both uploaded heap pointers together, retaining their +4
+    * separation. Zero preserves every existing native witness byte. M5
+    * query dispatches use168/172 to avoid their captured runtime prefix. */
    uint16_t heap_uniform_base;
    uint16_t dest;       /* half-register index */
    uint16_t coord;      /* four-half aligned pair of f32 coordinates */
@@ -208,12 +213,11 @@ struct agx3_tex_sample_desc {
    uint16_t wait_mask;  /* zero or one compactly encodable dependency */
    bool coord_kill;
    bool final;          /* witnessed final texture-op marker */
-   /* f156 terminates a fragment sample group. Fresh G17P two/three-sample
+   /* f156 terminates a fragment sample group. Native G17P/G17G multi-sample
     * pipelines clear it on every earlier SMP and set it only on the final
     * ordinary sample. Explicit-gradient samples sit outside the group and
-    * reject both flags. These explicit aggregate-group flags are G17P-only;
-    * leaving both clear preserves the independently witnessed per-op encoding
-    * in every other stage/ISA. */
+    * reject both flags. Leaving both clear preserves the independently
+    * witnessed single-operation encoding. */
    bool sample_group_nonterminal;
    bool sample_group_end;
    /* Vertex explicit-LOD samples use the same SMP fields and operand staging
@@ -370,6 +374,12 @@ bool agx3_tex_cube_cem_calc_to(enum agx_isa_encoding enc, uint16_t x,
 
 bool agx3_tex_cube_cem_rcp(enum agx_isa_encoding enc,
                            uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
+/* Keep a C cube comparison's reference live through any aliased XYZ input. */
+bool agx3_tex_cube_cem_calc_reference_to(
+   enum agx_isa_encoding enc, uint16_t x, uint16_t y, uint16_t z,
+   uint16_t dest, uint8_t axis, bool discard, uint16_t reference,
+   uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
+
 bool agx3_tex_cube_cem_rcp_to(enum agx_isa_encoding enc, uint16_t value,
                               uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
 
@@ -424,6 +434,16 @@ bool agx3_tex_stage_ms_sample(enum agx_isa_encoding enc, uint16_t src,
                               bool src_discard,
                               uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
 
+bool agx3_tex_stage_ms_sample_at(enum agx_isa_encoding enc, uint16_t bank,
+                                 uint16_t src, uint8_t slot, uint16_t wait_mask,
+                                 bool src_cache, bool src_discard,
+                                 uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
+
+bool agx3_tex_stage_ms_array_sample_at(
+   enum agx_isa_encoding enc, uint16_t bank, uint16_t sample,
+   uint16_t packed_layer, uint8_t slot, bool cache, bool discard,
+   uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
+
 /* Pack CEM's face alone into the 32-bit GPR `dest` for a biased cube sample
  * with a minimum LOD. The face half is always consumed. */
 bool agx3_tex_pack_cube_face(enum agx_isa_encoding enc, uint16_t face,
@@ -445,6 +465,20 @@ bool agx3_tex_pack_minimum(enum agx_isa_encoding enc, uint16_t src,
                            uint16_t dest, bool cache, bool discard,
                            uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
 
+/* AGX3C: stage a minimum LOD to sampler bank 8 (a comparison owns bank 4 and
+ * the bank-6 word has no room beside its layer/bias or 3D Z nibble). */
+bool agx3_tex_stage_minimum_bank8_m5(uint16_t src, uint8_t slot,
+                                     uint16_t wait_mask, bool cache,
+                                     bool discard,
+                                     uint8_t out[AGX3_TEX_MAX_LENGTH],
+                                     uint8_t *len);
+
+/* G17G PACK_SMPIR-to-GPR: half-float level/bias to a reusable native word.
+ * COPY publishes that word at staging bank+6. */
+bool agx3_tex_pack_float_lod_m5(uint16_t src, uint16_t dest,
+                               bool cache, bool discard, uint16_t wait_mask,
+                               uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
+
 bool agx3_tex_pack_ms_array_layer(enum agx_isa_encoding enc, uint16_t src,
                                  uint16_t dest, bool cache, bool discard,
                                  uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
@@ -507,6 +541,12 @@ bool agx3_tex_encode_image_write_rgba8_2d(
 
 /* A native staging COPY must preserve a coordinate GPR if later work uses
  * it. CACHE/NONE/DISCARD are independent of the staging-bank release bit. */
+/* Full-range 32-bit source for scalar M5 texture publication only. */
+bool agx3_tex_stage_copy_scalar_m5(uint16_t dest, uint16_t src, uint8_t slot,
+                                 uint16_t wait_mask, bool src_cache,
+                                 bool src_discard,
+                                 uint8_t out[AGX3_TEX_COPY_MAX_LENGTH], uint8_t *len);
+
 bool agx3_tex_stage_copy_hints(enum agx_isa_encoding enc, uint16_t dest,
                                uint16_t src, uint8_t slot, uint16_t wait_mask,
                                bool src_cache, bool src_discard,
@@ -516,6 +556,10 @@ bool agx3_tex_stage_copy(enum agx_isa_encoding enc, uint16_t dest,
                          uint16_t src, uint8_t slot, uint16_t wait_mask,
                          uint8_t out[AGX3_TEX_COPY_MAX_LENGTH], uint8_t *len);
 
+bool agx3_tex_encode_image_write_agx3c(
+   const struct agx3_image_write_desc *d, uint16_t uniform_base,
+   uint8_t staging_slot, uint8_t out[AGX3_TEX_MAX_LENGTH], uint8_t *len);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/asahi/compiler/agx3_tile_core.c b/src/asahi/compiler/agx3_tile_core.c
index 25cf76e36deb..596bdde2023c 100644
--- a/src/asahi/compiler/agx3_tile_core.c
+++ b/src/asahi/compiler/agx3_tile_core.c
@@ -6,11 +6,207 @@
 #include "agx3_tile_core.h"
 
 #include "agx3_binder.h"
+#include "agx3_sr_core.h"
+#include "agx3_m5_mrt_eot_data.h"
 
 #include <string.h>
 
+#define AGX_MRT_ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
 #define KV(f, v) {(uint16_t)(f), (uint64_t)(v)}
 
+/* The genuine G17G EOT builtin carries the byte offset in each coordinate
+ * wire and indexes the PBE array separately in IMGWBLK. Its ordinary
+ * registers end at half5; preserve the qualified RT0 eight-half allocation.
+ * Each complete per-target wire occupies eight staging halves. */
+uint16_t
+agx3_m5_mrt_eot_nr_gprs(unsigned targets)
+{
+   return targets >= 1 && targets <= 8 ? 8 : 0;
+}
+
+uint16_t
+agx3_m5_mrt_eot_staging_halfs(unsigned targets)
+{
+   return targets >= 1 && targets <= 8 ? 8 * targets : 0;
+}
+
+static bool
+m5_mrt_append(uint8_t *bytes, size_t *size, const char *op,
+              const struct agx3_field_kv *fields, unsigned count,
+              uint8_t ep, uint8_t expected_length)
+{
+   uint8_t instruction[16], length = 0;
+   if (!agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, op, ep, fields, count,
+                          NULL, 0, instruction, &length) ||
+       (expected_length && length != expected_length) ||
+       length > AGX3_M5_MRT_EOT_MAX_LENGTH - *size)
+      return false;
+   memcpy(bytes + *size, instruction, length);
+   *size += length;
+   return true;
+}
+
+static bool
+m5_mrt_wire(uint8_t *bytes, size_t *size, unsigned rt, unsigned targets,
+             bool layered)
+{
+   struct agx3_field_kv pack[19] = {
+      KV(5, 8 * rt + 4), KV(9, 5), KV(10, 2), KV(12, rt + 1),
+      KV(17, 4 + 4 * rt), KV(23, 3), KV(24, 1), KV(37, 1), KV(38, 1),
+      KV(51, layered ? 2 : 1), KV(52, 2), KV(56, 1), KV(57, 118), KV(74, 8),
+      KV(75, 16), KV(76, 7), KV(77, 27),
+   };
+   unsigned count = 17;
+   if (layered) {
+      pack[count++] = (struct agx3_field_kv)KV(44, 4);
+      pack[count++] = (struct agx3_field_kv)KV(55, rt + 1 == targets);
+   }
+   if (!m5_mrt_append(bytes, size, "pack_imgwir", pack, count, 0, 14))
+      return false;
+   if (targets == 1)
+      return true;
+   for (unsigned xy = 0; xy < 2; ++xy) {
+      const bool last = rt + 1 == targets;
+      const struct agx3_field_kv copy[] = {
+         KV(5, 8 * rt + 2 * xy), KV(9, 5), KV(10, 2), KV(12, rt + 1),
+         KV(16, xy ? 0 : 2), KV(23, 2), KV(24, 2),
+         KV(25, !last), KV(27, last), KV(28, 1), KV(29, 35),
+      };
+      if (!m5_mrt_append(bytes, size, "copy", copy, AGX_MRT_ARRAY_SIZE(copy),
+                           rt == 0 ? 0 : 1, rt == 0 ? 4 : 8))
+         return false;
+   }
+   return true;
+}
+
+static bool
+m5_mrt_store(uint8_t *bytes, size_t *size, unsigned rt, uint8_t selector,
+             bool multisample, bool layered)
+{
+   struct agx3_field_kv img[19] = {
+      KV(5, 8 * rt), KV(9, 5), KV(10, 2), KV(12, rt + 1), KV(13, 1),
+      KV(14, 2), KV(15, 1), KV(16, 86), KV(17 + rt, 1),
+      KV(29, selector == 0x15 ? 1 : selector == 0x35 ? 2 :
+             selector == 0x55 ? 0 : selector == 0x95 ? 4 : 3),
+      KV(30, multisample ? 2 : 1), KV(32, 1), KV(45, 3), KV(46, 3), KV(51, rt),
+      KV(59, 1), KV(60, 1), KV(73, 1),
+   };
+   unsigned count = 18;
+   if (layered)
+      img[count++] = (struct agx3_field_kv)KV(31, 2);
+   return m5_mrt_append(bytes, size, "imgwblk", img, count, 0, 12);
+}
+
+/* Reuse the qualified RT0 array encoding: SR12 is the tile layer, packed
+ * into ordinary half 4 before every target's XY coordinate wire consumes it.
+ * Only the last target releases the shared layer operand. */
+static bool
+m5_mrt_layer(uint8_t *bytes, size_t *size)
+{
+   const struct agx3_sr_desc layer = {
+      .dest = 6, .width = AGX3_SR_W16, .sr = 12, .slot = 2,
+   };
+   uint8_t sr[AGX3_SR_MAX_LENGTH], length;
+   if (!agx3_sr_encode(AGX_ISA_ENCODING_AGX3C, &layer, sr, &length) ||
+       length > AGX3_M5_MRT_EOT_MAX_LENGTH - *size)
+      return false;
+   memcpy(bytes + *size, sr, length);
+   *size += length;
+   const struct agx3_field_kv fields[] = {
+      KV(2, 4), KV(9, 2), KV(10, 2), KV(11, 1), KV(16, 6),
+      KV(23, 2), KV(24, 1), KV(27, 1), KV(38, 1), KV(51, 1),
+      KV(52, 2), KV(56, 3), KV(57, 118), KV(59, 1), KV(74, 4),
+   };
+   return m5_mrt_append(bytes, size, "pack_imgwir", fields,
+                        AGX_MRT_ARRAY_SIZE(fields), 0, 14);
+}
+
+bool
+agx3_m5_mrt_eot_generate_sample_mask(unsigned targets, const uint8_t *selectors,
+                                 unsigned multisample_mask, bool layered, uint8_t *output,
+                                 size_t capacity, size_t *written)
+{
+   if (!output || !written || !selectors || targets < 1 || targets > 8 ||
+       (multisample_mask & ~((1u << targets) - 1)))
+      return false;
+   for (unsigned rt = 0; rt < targets; ++rt) {
+      /* Reuse the RT0 store classes: F16/I16 f29=1, raw32 f29=2,
+       * normalized8 f29=3, normalized16 f29=4 and integer8 f29=0. Descriptor indices and per-target byte offsets
+       * are independent of the tile's scalar width. */
+      if (selectors[rt] != 0x15 && selectors[rt] != 0x35 && selectors[rt] != 0x55 &&
+          selectors[rt] != 0x75 && selectors[rt] != 0x95)
+         return false;
+   }
+   uint8_t bytes[AGX3_M5_MRT_EOT_MAX_LENGTH];
+   size_t size = sizeof(m5_mrt_prefix);
+   memcpy(bytes, targets == 1 ? m5_rt0_prefix : m5_mrt_prefix, size);
+   if (layered && !m5_mrt_layer(bytes, &size))
+      return false;
+   const unsigned first = targets < 2 ? targets : 2;
+   for (unsigned rt = 0; rt < first; ++rt) {
+      if (!m5_mrt_wire(bytes, &size, rt, targets, layered))
+         return false;
+   }
+   for (unsigned rt = 0; rt < first; ++rt) {
+      if (!m5_mrt_store(bytes, &size, rt, selectors[rt], (multisample_mask & (1u << rt)) != 0, layered))
+         return false;
+   }
+   for (unsigned rt = first; rt < targets; ++rt) {
+      if (!m5_mrt_wire(bytes, &size, rt, targets, layered))
+         return false;
+   }
+   for (unsigned rt = first; rt < targets; ++rt) {
+      if (!m5_mrt_store(bytes, &size, rt, selectors[rt], (multisample_mask & (1u << rt)) != 0, layered))
+         return false;
+   }
+   const uint8_t stop[] = {0x0e, 0, 0, 0};
+   if (sizeof(stop) > sizeof(bytes) - size)
+      return false;
+   memcpy(bytes + size, stop, sizeof(stop));
+   size += sizeof(stop);
+   if (capacity < size)
+      return false;
+   memcpy(output, bytes, size);
+   *written = size;
+   return true;
+}
+
+bool
+agx3_m5_mrt_eot_generate_layered(unsigned targets, const uint8_t *selectors,
+                                 bool multisample, bool layered, uint8_t *output,
+                                 size_t capacity, size_t *written)
+{
+   if (targets < 1 || targets > 8)
+      return false;
+   return agx3_m5_mrt_eot_generate_sample_mask(targets, selectors,
+      multisample ? (1u << targets) - 1 : 0, layered, output, capacity, written);
+}
+
+bool
+agx3_m5_mrt_eot_generate_typed_ms(unsigned targets, const uint8_t *selectors,
+                                bool multisample, uint8_t *output,
+                                size_t capacity, size_t *written)
+{
+   return agx3_m5_mrt_eot_generate_layered(targets, selectors, multisample, false,
+                                         output, capacity, written);
+}
+
+bool
+agx3_m5_mrt_eot_generate_typed(unsigned targets, const uint8_t *selectors,
+                                uint8_t *output, size_t capacity, size_t *written)
+{
+   return agx3_m5_mrt_eot_generate_typed_ms(targets, selectors, false,
+                                            output, capacity, written);
+}
+
+bool
+agx3_m5_mrt_eot_generate(unsigned targets, uint8_t *output,
+                          size_t capacity, size_t *written)
+{
+   const uint8_t selectors[8] = {0x75,0x75,0x75,0x75,0x75,0x75,0x75,0x75};
+   return agx3_m5_mrt_eot_generate_typed(targets, selectors, output, capacity, written);
+}
+
 static bool
 is_b(enum agx_isa_encoding e)
 {
@@ -25,15 +221,16 @@ one_or_zero(uint16_t value)
 
 /*
  * The sample-mask operand. The immediate alternative is witnessed at every 1x
- * width; the register alternative is witnessed at 2x and 4x on applegpu_g17p
- * (AGX3b). No AGX3c multisample witness exists, so the register form stays
- * AGX3b-only rather than being asserted for a part nobody has measured.
+ * width. G17P and G17G multisample fragments read W16 SR39 and pass that
+ * register to LSTCOL with f51=2, f44=the mask half, f55=last use. G17G's
+ * native 2x/4x/8x plain fragments use the same register-mask operand fields.
  */
 static bool
 sample_ok(enum agx_isa_encoding enc, struct agx3_tile_sample s)
 {
    if (s.reg)
-      return is_b(enc) && s.value < 192;
+      return (is_b(enc) && s.value < 192) ||
+             (enc == AGX_ISA_ENCODING_AGX3C && s.value < 256);
 
    return !s.kill && s.value <= 255;
 }
@@ -91,7 +288,7 @@ agx3_tile_store(enum agx_isa_encoding enc, uint16_t data,
                 uint8_t *len)
 {
    return agx3_tile_store_f13(enc, data, width, chan_mask, tile_off, sample,
-                              wait_mask, false, out, len);
+                              wait_mask, !is_b(enc), out, len);
 }
 
 bool
@@ -117,7 +314,7 @@ agx3_tile_store_f13(enum agx_isa_encoding enc, uint16_t data,
    struct agx3_field_kv ov[16];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(2, data);
-   if (!is_b(enc) || f13)
+   if (f13)
       ov[n++] = (struct agx3_field_kv)KV(13, 1);
    ov[n++] = (struct agx3_field_kv)KV(10, width);
    ov[n++] = (struct agx3_field_kv)KV(26, chan_mask);
@@ -145,7 +342,8 @@ agx3_tile_load(enum agx_isa_encoding enc, uint16_t dest,
    memset(out, 0, AGX3_TILE_MAX_LENGTH);
    *len = 0;
 
-   /* Full-component loads, plus the G17P scalar, pair, and RGB565 forms.
+   /* G17G R8/RG8 framebuffer-fetch witnesses carry f26=1/3 with the
+    * same native load fields as RGBA8. G17P additionally has RGB565.
     * Fresh applegpu_g17p RG8, RG16{Float,Unorm,Snorm,Uint,Sint}, and
     * RG32{Float,Uint,Sint} render-pipeline witnesses isolate mask3 in f26 at
     * every width. A B5G6R5Unorm framebuffer-fetch witness independently
@@ -153,12 +351,13 @@ agx3_tile_load(enum agx_isa_encoding enc, uint16_t dest,
     *
     *    670e54000000019a02000000
     *
+    * G17G's B5G6R5Unorm framebuffer fetch takes the same mask7 at W16
+    * (port/fbwit: 270105000187bc02000040d82000).
+    *
     * None carries an incoming wait. */
    if (!common_ok(dest, width, chan_mask, sample, enc, wait_mask, tile_off) ||
-       (chan_mask != 15 &&
-        !(is_b(enc) &&
-          (chan_mask == 1 || chan_mask == 3 ||
-           (chan_mask == 7 && width == AGX3_TILE_W16)))) ||
+       (chan_mask != 15 && chan_mask != 1 && chan_mask != 3 &&
+        !(chan_mask == 7 && width == AGX3_TILE_W16)) ||
        wait_mask != 0 || slot < 1 ||
        slot > (is_b(enc) ? 6 : 8))
       return false;
diff --git a/src/asahi/compiler/agx3_tile_core.h b/src/asahi/compiler/agx3_tile_core.h
index 80183accf655..76ed24149bda 100644
--- a/src/asahi/compiler/agx3_tile_core.h
+++ b/src/asahi/compiler/agx3_tile_core.h
@@ -12,6 +12,7 @@
 #pragma once
 
 #include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #include "asahi/lib/agx_chip.h"
@@ -22,6 +23,26 @@ extern "C" {
 
 #define AGX3_TILE_MAX_LENGTH 14u
 
+/* Genuine G17G builtin: base PBE pointer U0, tile offsets U4+4i,
+ * one retained eight-half coordinate wire per target, descriptor indexi. */
+#define AGX3_M5_MRT_EOT_MAX_LENGTH 400u
+uint16_t agx3_m5_mrt_eot_nr_gprs(unsigned targets);
+uint16_t agx3_m5_mrt_eot_staging_halfs(unsigned targets);
+bool agx3_m5_mrt_eot_generate_sample_mask(unsigned targets, const uint8_t *selectors,
+                                      unsigned multisample_mask, bool layered,
+                                      uint8_t *output, size_t capacity, size_t *written);
+bool agx3_m5_mrt_eot_generate_layered(unsigned targets, const uint8_t *selectors,
+                                      bool multisample, bool layered,
+                                      uint8_t *output, size_t capacity,
+                                      size_t *written);
+bool agx3_m5_mrt_eot_generate_typed_ms(unsigned targets, const uint8_t *selectors,
+                                       bool multisample, uint8_t *output,
+                                       size_t capacity, size_t *written);
+bool agx3_m5_mrt_eot_generate_typed(unsigned targets, const uint8_t *selectors,
+                                    uint8_t *output, size_t capacity, size_t *written);
+bool agx3_m5_mrt_eot_generate(unsigned targets, uint8_t *output,
+                              size_t capacity, size_t *written);
+
 enum agx3_tile_width {
    AGX3_TILE_W8 = 0,
    AGX3_TILE_W16 = 1,
@@ -90,7 +111,9 @@ bool agx3_tile_store(enum agx_isa_encoding enc, uint16_t data,
                      uint16_t wait_mask, uint8_t out[AGX3_TILE_MAX_LENGTH],
                      uint8_t *len);
 
-/* agx3_tile_store with AGX3b f13 chosen by the caller; AGX3c always sets it.
+/* agx3_tile_store with native f13 chosen by the caller. The ordinary C
+ * wrapper preserves f13=1; eight-sample fragment payloads may retain it at
+ * f13=0 until their last use, as Apple's sample_id8 shader does.
  * Apple's AGX3b fragments set f13 on the leading stores of colour
  * transactions wider than five targets, see agx3_g17p_mrt_protocol(). */
 bool agx3_tile_store_f13(enum agx_isa_encoding enc, uint16_t data,
diff --git a/src/asahi/compiler/agx3_uniform_alu.c b/src/asahi/compiler/agx3_uniform_alu.c
index f639ef780092..7833a0b883b8 100644
--- a/src/asahi/compiler/agx3_uniform_alu.c
+++ b/src/asahi/compiler/agx3_uniform_alu.c
@@ -11,11 +11,107 @@ uniform_float_encode(enum agx_isa_encoding encoding,
                      bool uniform_dest,
                      uint8_t output[16], uint8_t *length)
 {
-   if (encoding != AGX_ISA_ENCODING_AGX3B || !src || !output || !length ||
+   const bool m5 = encoding == AGX_ISA_ENCODING_AGX3C;
+   if ((!m5 && encoding != AGX_ISA_ENCODING_AGX3B) || !src || !output || !length ||
        (unsigned)op > AGX3_UNIFORM_FMA || dest >= 256 ||
-       (!half && (dest & 1)) || wait_mask > 0x3f)
+       (!half && (dest & 1)) || wait_mask > (m5 ? 0xff : 0x3f))
       return false;
 
+   /* Retained native G17G shaders witness, for a GPR result:
+    *   MUL R*U and U*R (also saturated);
+    *   ADD R+U and U+R, the uniform optionally negated;
+    *   FMA R*U+R, R*U+U and U*R+U, and R*U+imm, the uniform multiplicand
+    *   optionally negated and a register multiplicand negated in U*R+U;
+    *   FMA R*U-R with plain multiplicands (a native compiler probe of
+    *   fma(x, u, -y): 0993060511a00202);
+    *   FMA R*R+U with nothing negated (the agx3-diff-corpus fragments);
+    * and for a uniform result in a constant program:
+    *   FMA R*R+R and MUL imm*R.
+    * Native Minecraft terrain/entity/sky pipelines compiled natively for
+    * G17G (vertex constant programs: 48 U=RRR FMAs, fragment
+    * constant programs: 12 U=IR MULs; mains: 39 ADDs, 29 U-addend FMAs and 5
+    * R*U+imm FMAs) are asserted byte for byte in test-agx3-uniform-alu.cpp.
+    * Half arithmetic stays outside this admission. */
+   if (m5) {
+      const unsigned count = op == AGX3_UNIFORM_FMA ? 3 : 2;
+      unsigned regs = 0, uniforms = 0, immediates = 0;
+      for (unsigned i = 0; i < count; ++i) {
+         regs += src[i].kind == AGX3_FLOAT_REGISTER;
+         uniforms += src[i].kind == AGX3_FLOAT_UNIFORM;
+         immediates += src[i].kind == AGX3_FLOAT_IMMEDIATE;
+         /* A 16-bit uniform multiplicand of an f32 FMA is witnessed (R*U+imm,
+          * f43=1); other half operands stay outside. */
+         if (src[i].abs ||
+             (src[i].half && (src[i].kind != AGX3_FLOAT_UNIFORM ||
+                              op != AGX3_UNIFORM_FMA || i == 2)) ||
+             (src[i].kind == AGX3_FLOAT_REGISTER && src[i].value >= 252))
+            return false;
+      }
+      if (half || dest >= 252 || (saturate && op != AGX3_UNIFORM_FMUL))
+         return false;
+      if (uniform_dest) {
+         /* U=RRR FMA and U=IR MUL, unmodified. */
+         if (op == AGX3_UNIFORM_FADD ||
+             (op == AGX3_UNIFORM_FMA && regs != 3) ||
+             (op == AGX3_UNIFORM_FMUL && (regs != 1 || immediates != 1)))
+            return false;
+         for (unsigned i = 0; i < count; ++i) {
+            if (src[i].neg)
+               return false;
+         }
+      } else {
+         const bool u0 = src[0].kind == AGX3_FLOAT_UNIFORM;
+         const bool u1 = src[1].kind == AGX3_FLOAT_UNIFORM;
+         if (op != AGX3_UNIFORM_FMA) {
+            /* One register and one uniform; only ADD negates (the
+             * uniform). */
+            if (regs != 1 || uniforms != 1)
+               return false;
+            for (unsigned i = 0; i < 2; ++i) {
+               if (src[i].neg && (op != AGX3_UNIFORM_FADD ||
+                                  src[i].kind != AGX3_FLOAT_UNIFORM))
+                  return false;
+            }
+         } else if (!u0 && !u1) {
+            /* R*R+U: register multiplicands and a plain uniform addend
+             * (agx3-diff-corpus native G17G r_heavy_loop fragment). */
+            if (src[0].kind != AGX3_FLOAT_REGISTER ||
+                src[1].kind != AGX3_FLOAT_REGISTER ||
+                src[2].kind != AGX3_FLOAT_UNIFORM || src[0].neg ||
+                src[1].neg || src[2].neg)
+               return false;
+         } else {
+            /* Exactly one uniform multiplicand and one register one. */
+            if (u0 == u1 || (src[0].kind != AGX3_FLOAT_REGISTER &&
+                             src[1].kind != AGX3_FLOAT_REGISTER))
+               return false;
+            switch (src[2].kind) {
+            case AGX3_FLOAT_REGISTER:
+               /* A negated register addend is witnessed beside a plain
+                * register and a plain uniform multiplicand (R*U-R). */
+               if (src[2].neg && (src[0].neg || src[1].neg))
+                  return false;
+               break;
+            case AGX3_FLOAT_UNIFORM:
+               if (src[2].neg || src[2].value >= 512)
+                  return false;
+               break;
+            case AGX3_FLOAT_IMMEDIATE:
+               break;
+            default:
+               return false;
+            }
+            for (unsigned i = 0; i < 2; ++i) {
+               /* Negated register multiplicands are witnessed only with a
+                * uniform addend. */
+               if (src[i].neg && src[i].kind == AGX3_FLOAT_REGISTER &&
+                   src[2].kind != AGX3_FLOAT_UNIFORM)
+                  return false;
+            }
+         }
+      }
+   }
+
    /* Direct uniform destinations are witnessed in constant programs, with
     * aligned low-file f32 results and plain GPR/immediate sources. Keep the
     * independently qualified main-program source contract unchanged. */
@@ -40,7 +136,9 @@ uniform_float_encode(enum agx_isa_encoding encoding,
          if (src[i].value >= 512 || (!source_half && (src[i].value & 1)) ||
              src[i].abs ||
              (src[i].neg &&
-              (source_half || op != AGX3_UNIFORM_FMA || i != 2)) ||
+              (source_half ||
+               (op != AGX3_UNIFORM_FMA && !(m5 && op == AGX3_UNIFORM_FADD)) ||
+               (!m5 && i != 2))) ||
              src[i].cache || src[i].discard)
             return false;
          break;
@@ -78,6 +176,16 @@ uniform_float_encode(enum agx_isa_encoding encoding,
                                                             : "f32fma");
    uint8_t best[16], best_length = 0;
    for (unsigned swap = 0; swap < 2; ++swap) {
+      /* M5 places an FMA's uniform multiplicand in the second operand
+       * unless the addend is a uniform too (U*R+U), and a uniform-result
+       * MUL's immediate in the first. */
+      if (m5 && op == AGX3_UNIFORM_FMA && !uniform_dest &&
+          src[2].kind != AGX3_FLOAT_UNIFORM &&
+          src[1 ^ swap].kind != AGX3_FLOAT_UNIFORM)
+         continue;
+      if (m5 && op == AGX3_UNIFORM_FMUL && uniform_dest &&
+          src[swap].kind != AGX3_FLOAT_IMMEDIATE)
+         continue;
       struct agx3_field_kv fields[48] = {
          {uniform_dest ? 3 : 2, dest},
          {9, uniform_dest ? 3 : 2}, {10, half ? 1 : 2},
@@ -112,24 +220,48 @@ uniform_float_encode(enum agx_isa_encoding encoding,
          fields[n++] = (struct agx3_field_kv){77, 2};
          fields[n++] = (struct agx3_field_kv){91, 2};
       }
+      if (m5) {
+         /* f95/f78 is 3 for a GPR result and 1 for a uniform one; ADD has
+          * f50=3/f51=48 instead. */
+         const unsigned kind = uniform_dest ? 1 : 3;
+         fields[n++] = (struct agx3_field_kv){
+            op == AGX3_UNIFORM_FMA ? 95 : op == AGX3_UNIFORM_FMUL ? 78 : 50,
+            op == AGX3_UNIFORM_FADD ? 3 : kind};
+         fields[n++] = (struct agx3_field_kv){
+            op == AGX3_UNIFORM_FMA ? 96 : op == AGX3_UNIFORM_FMUL ? 79 : 51,
+            op == AGX3_UNIFORM_FMA ? 55 : op == AGX3_UNIFORM_FMUL ? 62 : 48};
+      }
       const unsigned wait_base = half ? (op == AGX3_UNIFORM_FMA ? 68 : 51)
                                  : op == AGX3_UNIFORM_FADD ? 52
                                  : op == AGX3_UNIFORM_FMUL ? 80
                                                            : 97;
-      for (unsigned i = 0; i < 6; ++i)
+      for (unsigned i = 0; i < (m5 ? 8 : 6); ++i)
          fields[n++] =
             (struct agx3_field_kv){wait_base + i, (wait_mask >> i) & 1};
 
-      const unsigned last_ep = op == AGX3_UNIFORM_FMA ? 5 : 3;
+      /* M5 adds the slot-seven/eight endpoints, and its MUL endpoints
+       * are not ordered by length. Use the same complete endpoint domain
+       * as the native register-only FALU encoder. */
+      /* M5 f32add has endpoints 0..3 only; its 12-byte ep3 carries the
+       * independent wait fields, as for the register ADD encoder. */
+      const unsigned last_ep = m5 ? (op == AGX3_UNIFORM_FMA    ? 7
+                                     : op == AGX3_UNIFORM_FADD ? 3
+                                                               : 5)
+                                  : op == AGX3_UNIFORM_FMA ? 5 : 3;
       /* Compact wait selectors express one slot. Use the independent fields
        * for multiple producers, without draining unrelated outstanding work. */
-      const unsigned first_ep = (wait_mask & (wait_mask - 1)) ? last_ep : 0;
+      const unsigned first_ep = (wait_mask & (wait_mask - 1))
+                                   ? (m5 && op == AGX3_UNIFORM_FMUL ? 3 : last_ep)
+                                   : 0;
       for (unsigned ep = first_ep; ep <= last_ep; ++ep) {
          /* Only a strictly shorter encoding can replace the first swap's,
           * and the endpoints grow in length: stop trying once they reach it. */
          if (best_length &&
-             agx3_bind_form_size(encoding, mnemonic, ep) >= best_length)
+             agx3_bind_form_size(encoding, mnemonic, ep) >= best_length) {
+            if (m5)
+               continue;
             break;
+         }
          uint8_t bytes[16], size;
          if (agx3_bind_compose(encoding, mnemonic, ep, NULL, 0, fields, n,
                                bytes, &size)) {
diff --git a/src/asahi/compiler/agx3_uniform_integer.c b/src/asahi/compiler/agx3_uniform_integer.c
index f5eb6d0fb47d..f15ea3780fda 100644
--- a/src/asahi/compiler/agx3_uniform_integer.c
+++ b/src/asahi/compiler/agx3_uniform_integer.c
@@ -4,14 +4,15 @@
 #include "agx3_binder.h"
 
 static bool
-source_valid(struct agx3_uniform_integer_src s, bool half, bool addend)
+source_valid(struct agx3_uniform_integer_src s, bool half, bool addend,
+              unsigned gprs)
 {
    if (s.kind == AGX3_UI_ZERO)
       return addend && !s.value && !s.cache && !s.discard;
    if (s.kind != AGX3_UI_REGISTER && s.kind != AGX3_UI_UNIFORM)
       return false;
    if (s.value >=
-          (s.kind == AGX3_UI_REGISTER ? AGX3_UNIFORM_INTEGER_GPRS : 512) ||
+          (s.kind == AGX3_UI_REGISTER ? gprs : 512) ||
        (!half && (s.value & 1)) || (s.cache && s.discard))
       return false;
    return s.kind != AGX3_UI_UNIFORM || (!s.cache && !s.discard);
@@ -22,20 +23,32 @@ agx3_uniform_integer_encode(enum agx_isa_encoding encoding,
                             const struct agx3_uniform_integer_desc *d,
                             uint8_t output[16], uint8_t *length)
 {
-   if (encoding != AGX_ISA_ENCODING_AGX3B || !d || !output || !length ||
+   const bool m5 = encoding == AGX_ISA_ENCODING_AGX3C;
+   const unsigned gprs = m5 ? AGX3_UNIFORM_INTEGER_M5_GPRS :
+                             AGX3_UNIFORM_INTEGER_GPRS;
+   if ((encoding != AGX_ISA_ENCODING_AGX3B && !m5) || !d || !output || !length ||
        d->op > AGX3_UI_ANDC || (int)d->op < 0 ||
-       d->dest >= AGX3_UNIFORM_INTEGER_GPRS || (!d->half && (d->dest & 1)) ||
-       d->wait_mask >= 64 ||
+       d->dest >= gprs || (!d->half && (d->dest & 1)) ||
+       (!m5 && d->wait_mask >= 64) ||
        (d->shift && (d->op != AGX3_UI_ADD || d->shift != 3)))
       return false;
 
+   /* M5 witnesses: full-width unshifted ADD in either operand order;
+    * SUB has a varying first operand and uniform second operand. */
+   if (m5 && (d->half || d->shift ||
+              (d->op != AGX3_UI_ADD && d->op != AGX3_UI_SUB &&
+               d->op != AGX3_UI_MAD) ||
+              (d->op == AGX3_UI_SUB &&
+               (d->src[0].kind != AGX3_UI_REGISTER ||
+                d->src[1].kind != AGX3_UI_UNIFORM))))
+      return false;
    const bool mad = d->op == AGX3_UI_MAD;
    const bool bitwise = d->op >= AGX3_UI_AND;
    const unsigned count = mad ? 3 : 2, width = d->half ? 1 : 2;
    struct agx3_uniform_integer_src src[3];
    memcpy(src, d->src, sizeof(src));
    for (unsigned i = 0; i < count; ++i)
-      if (!source_valid(src[i], d->half, mad && i == 2))
+      if (!source_valid(src[i], d->half, mad && i == 2, gprs))
          return false;
 
    /* Main-program witnesses have one varying multiplicand/binary operand.
@@ -43,7 +56,7 @@ agx3_uniform_integer_encode(enum agx_isa_encoding encoding,
    if (!((src[0].kind == AGX3_UI_UNIFORM && src[1].kind == AGX3_UI_REGISTER) ||
          (src[0].kind == AGX3_UI_REGISTER && src[1].kind == AGX3_UI_UNIFORM)))
       return false;
-   if (src[1].kind == AGX3_UI_UNIFORM && d->op != AGX3_UI_SUB &&
+   if (!m5 && src[1].kind == AGX3_UI_UNIFORM && d->op != AGX3_UI_SUB &&
        d->op != AGX3_UI_ANDC) {
       /* A shifted ADD is src0 + (src1 << shift); swapping would change it. */
       if (d->shift)
@@ -52,6 +65,19 @@ agx3_uniform_integer_encode(enum agx_isa_encoding encoding,
       src[0] = src[1];
       src[1] = tmp;
    }
+   if (m5 && mad) {
+      /* Native full-width endpoints: R/U/U (cs_loop, ld_constant) and
+       * U/R/R (sr_ids). Keep absent/immediate C and narrow operands closed. */
+      if (src[2].kind != AGX3_UI_REGISTER &&
+          src[2].kind != AGX3_UI_UNIFORM)
+         return false;
+      const bool uniform_first = src[2].kind == AGX3_UI_REGISTER;
+      if ((src[0].kind == AGX3_UI_UNIFORM) != uniform_first) {
+         struct agx3_uniform_integer_src tmp = src[0];
+         src[0] = src[1];
+         src[1] = tmp;
+      }
+   }
    if (d->op == AGX3_UI_ANDC && src[0].kind != AGX3_UI_UNIFORM)
       return false;
 
@@ -59,13 +85,13 @@ agx3_uniform_integer_encode(enum agx_isa_encoding encoding,
    unsigned endpoint = 0;
    switch (d->op) {
    case AGX3_UI_ADD:
-      mnemonic = "iadd";
+      mnemonic = m5 ? "iadd32" : "iadd";
       break;
    case AGX3_UI_SUB:
-      mnemonic = "isub";
+      mnemonic = m5 ? "isub32" : "isub";
       break;
    case AGX3_UI_MAD:
-      mnemonic = "imadd";
+      mnemonic = m5 ? "imadd32" : "imadd";
       break;
    case AGX3_UI_AND:
       mnemonic = "bit_and";
@@ -111,8 +137,12 @@ agx3_uniform_integer_encode(enum agx_isa_encoding encoding,
    }
    if (d->shift)
       FIELD(54, d->shift);
-   for (unsigned slot = 0; slot < 6; ++slot)
-      FIELD((mad       ? 60
+   if (m5) {
+      FIELD(mad ? 59 : 44, 3);
+      FIELD(mad ? 60 : 45, mad ? 84 : d->op == AGX3_UI_SUB ? 100 : 80);
+   }
+   for (unsigned slot = 0; slot < (m5 ? 8 : 6); ++slot)
+      FIELD((m5 ? (mad ? 61 : 46) : mad       ? 60
              : bitwise ? 44
                        : 45) +
                slot,
diff --git a/src/asahi/compiler/agx3_uniform_integer.h b/src/asahi/compiler/agx3_uniform_integer.h
index 02b4141ad775..d8e935ce2520 100644
--- a/src/asahi/compiler/agx3_uniform_integer.h
+++ b/src/asahi/compiler/agx3_uniform_integer.h
@@ -8,6 +8,8 @@
 /* Matches the AGX3B_NUM_REGS allocation limit in agx_compiler.h. Encodable
  * eight-bit register indices above this limit are not allocatable GPRs. */
 #define AGX3_UNIFORM_INTEGER_GPRS 192
+/* M5's allocator uses halves 0..251; the operand encoding is unchanged. */
+#define AGX3_UNIFORM_INTEGER_M5_GPRS 252
 enum agx3_uniform_integer_op {
    AGX3_UI_ADD,
    AGX3_UI_SUB,
diff --git a/src/asahi/compiler/agx3_uniform_integer_instr.h b/src/asahi/compiler/agx3_uniform_integer_instr.h
index 6a641dbf9572..7c539d170565 100644
--- a/src/asahi/compiler/agx3_uniform_integer_instr.h
+++ b/src/asahi/compiler/agx3_uniform_integer_instr.h
@@ -2,11 +2,13 @@
 #pragma once
 #include "agx3_uniform_integer.h"
 #include "agx_compiler.h"
+#include "util/u_debug.h"
 
 /* Shared pre/post-RA domain check: preserve only encodable uniform operands. */
 static inline bool
-agx3_uniform_integer_instruction(const agx_instr *I, bool post_ra,
-                                 struct agx3_uniform_integer_desc *d)
+agx3_uniform_integer_instruction_limit(const agx_instr *I, bool post_ra,
+                                       unsigned gprs,
+                                       struct agx3_uniform_integer_desc *d)
 {
    if ((I->op != AGX_OPCODE_IADD && I->op != AGX_OPCODE_IMAD &&
         I->op != AGX_OPCODE_BITOP) ||
@@ -17,7 +19,7 @@ agx3_uniform_integer_instruction(const agx_instr *I, bool post_ra,
       return false;
 
    if (post_ra &&
-       (I->dest[0].value >= AGX3_UNIFORM_INTEGER_GPRS ||
+       (I->dest[0].value >= gprs ||
         (I->dest[0].size == AGX_SIZE_32 && (I->dest[0].value & 1))))
       return false;
 
@@ -90,7 +92,7 @@ agx3_uniform_integer_instruction(const agx_instr *I, bool post_ra,
          d->src[i].value = src.value;
       } else if (src.type ==
                  (post_ra ? AGX_INDEX_REGISTER : AGX_INDEX_NORMAL)) {
-         if (post_ra && (src.value >= AGX3_UNIFORM_INTEGER_GPRS ||
+         if (post_ra && (src.value >= gprs ||
                          (!d->half && (src.value & 1))))
             return false;
          d->src[i].kind = AGX3_UI_REGISTER;
@@ -111,3 +113,35 @@ agx3_uniform_integer_instruction(const agx_instr *I, bool post_ra,
       return false;
    return true;
 }
+
+static inline bool
+agx3_uniform_integer_instruction(const agx_instr *I, bool post_ra,
+                                 struct agx3_uniform_integer_desc *d)
+{
+   return agx3_uniform_integer_instruction_limit(
+      I, post_ra, AGX3_UNIFORM_INTEGER_GPRS, d);
+}
+
+/* M5 full-width uniform integer domains: ADD/SUB and native IMADD R/U/U
+ * (cs_loop, ld_constant) or U/R/R (sr_ids). Multiplication commutes. */
+static inline bool
+agx3_uniform_integer_for_encoding(enum agx_isa_encoding enc,
+                                  const agx_instr *I, bool post_ra,
+                                  struct agx3_uniform_integer_desc *d)
+{
+   if (enc == AGX_ISA_ENCODING_AGX3B)
+      return agx3_uniform_integer_instruction(I, post_ra, d);
+   if (enc != AGX_ISA_ENCODING_AGX3C ||
+       !agx3_uniform_integer_instruction_limit(
+          I, post_ra, AGX3_UNIFORM_INTEGER_M5_GPRS, d) ||
+       d->half || d->shift ||
+       (d->op != AGX3_UI_ADD && d->op != AGX3_UI_SUB &&
+        d->op != AGX3_UI_MAD))
+      return false;
+   if (d->op == AGX3_UI_MAD)
+      return d->src[2].kind == AGX3_UI_REGISTER ||
+             d->src[2].kind == AGX3_UI_UNIFORM;
+   return d->op != AGX3_UI_SUB ||
+          (d->src[0].kind == AGX3_UI_REGISTER &&
+           d->src[1].kind == AGX3_UI_UNIFORM);
+}
diff --git a/src/asahi/compiler/agx3_vary_core.c b/src/asahi/compiler/agx3_vary_core.c
index 5df3d0131082..30f4aad93b07 100644
--- a/src/asahi/compiler/agx3_vary_core.c
+++ b/src/asahi/compiler/agx3_vary_core.c
@@ -25,12 +25,9 @@ agx3_vary_interp_mode(enum agx_isa_encoding enc, uint16_t dest,
 {
    if ((enc != AGX_ISA_ENCODING_AGX3B &&
         enc != AGX_ISA_ENCODING_AGX3C) ||
-       (first && enc != AGX_ISA_ENCODING_AGX3B) ||
-       (projective_rhw && (enc != AGX_ISA_ENCODING_AGX3B ||
-                           (!explicit_sample && location != AGX3_VARY_CENTER) ||
+       (projective_rhw && ((!explicit_sample && location != AGX3_VARY_CENTER) ||
                            !perspective || coef_index > 0x7f)) ||
-       (explicit_sample && (enc != AGX_ISA_ENCODING_AGX3B || first ||
-                            (coords & 1) != 0 || coords >= 256)) ||
+       (explicit_sample && (first || (coords & 1) != 0 || coords >= 256)) ||
        (dest & 1) != 0 || dest >= 256 ||
        (location != AGX3_VARY_CENTER && location != AGX3_VARY_CENTROID))
       return false;
@@ -43,13 +40,19 @@ agx3_vary_interp_mode(enum agx_isa_encoding enc, uint16_t dest,
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(2, dest);
    ov[n++] = (struct agx3_field_kv)KV(22, coef_index);
-   ov[n++] = (struct agx3_field_kv)KV(11, perspective ? 1 : 0);
+   /* Native G17G token interpolation (lane-mesh/cwit-interp
+    * ias_k1, ias_k1_np, ias_dyn, sp@1) sets f11 for perspective and
+    * no-perspective inputs alike. */
+   ov[n++] = (struct agx3_field_kv)KV(
+      11, (perspective || (explicit_sample && !is_b(enc))) ? 1 : 0);
    if (explicit_sample) {
       /* Apple Metal 32023.917, applegpu_g17p. The source SHA-256
        * 2f0835ef... produces perspective, no-perspective, and mixed
        * center/sample programs:
        * f39/f40 select a 32-bit coordinate register, f32 names the token,
-       * and f41/f43 are its live/final-use alternatives. */
+       * and f41/f43 are its live/final-use alternatives. Native G17G
+       * (lane-mesh/cwit-interp) encodes the same leaf on its common AGX3C
+       * f32interp base (f44/f45). */
       ov[n++] = (struct agx3_field_kv)KV(39, 2);
       ov[n++] = (struct agx3_field_kv)KV(32, coords);
       ov[n++] = (struct agx3_field_kv)KV(coords_kill ? 43 : 41, 1);
@@ -59,16 +62,12 @@ agx3_vary_interp_mode(enum agx_isa_encoding enc, uint16_t dest,
       ov[n++] = (struct agx3_field_kv)KV(41, 0);
       ov[n++] = (struct agx3_field_kv)KV(32, 0);
    } else {
-      /* AGX3C centroid interpolation uses the same register-source leaf, with
-       * the native implicit coordinate register selected by the shade model.
-       * G17P has no implicit form: its centroid coordinates are an explicit
-       * ITRCOORDS token. */
-      if (is_b(enc))
-         return false;
-      ov[n++] = (struct agx3_field_kv)KV(39, 2);
-      ov[n++] = (struct agx3_field_kv)KV(41, 1);
-      ov[n++] =
-         (struct agx3_field_kv)KV(32, perspective ? 0 : 6);
+      /* Neither G17P nor G17G has an implicit centroid leaf: both read SR39
+       * into an explicit ITRCOORDS f38=1 token (lane-mesh/cwit-interp iac,
+       * cwit-interp2 cp/cnp at 1 and 4 samples), consumed through the token
+       * leaf above. The former AGX3C form read its coordinates from a fixed
+       * r0/r6 register instead. */
+      return false;
    }
    if (projective_rhw)
       ov[n++] = (struct agx3_field_kv)KV(54, 1);
@@ -85,26 +84,36 @@ agx3_vary_coords(enum agx_isa_encoding enc, uint16_t dest, uint8_t mode,
                  uint16_t source, bool source_kill, uint16_t wait_mask,
                  uint8_t out[AGX3_VARY_MAX_LENGTH], uint8_t *len)
 {
-   if (enc != AGX_ISA_ENCODING_AGX3B || (dest & 1) != 0 || dest >= 256 ||
-       source >= 256 || (wait_mask & ~0x3f) != 0)
+   /* G17P has six scoreboard slots, f30..f35. The ten-byte G17G form adds
+    * f36 and f37 for slots 6 and 7, at the same bits (22 and 3) as every
+    * other C-class wait field. */
+   const uint16_t slots = is_b(enc) ? 0x3f : 0xff;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && enc != AGX_ISA_ENCODING_AGX3C) ||
+       (dest & 1) != 0 || dest >= 256 || source >= 256 ||
+       (wait_mask & ~slots) != 0)
       return false;
 
    /* ITRCOORDS is an eight-byte G17P instruction: register/32-bit
-    * destination, register/16-bit source, and the mode in f38. */
+    * destination, register/16-bit source, and the mode in f38. G17G
+    * (lane-mesh/cwit-interp, cwit-interp2) uses the same fields in a
+    * ten-byte form that adds f28/f29. Every native G17G instance marks its
+    * source final (f27); the unmarked form keeps G17P's field meaning. */
    const struct agx3_field_kv base[] = {
       KV(9, 2), KV(10, 2), KV(11, 1), KV(23, 2), KV(24, 1), KV(38, mode),
+      KV(28, 3), KV(29, 102),
    };
-   struct agx3_field_kv ov[10];
+   struct agx3_field_kv ov[11];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(2, dest);
    ov[n++] = (struct agx3_field_kv)KV(16, source);
    if (source_kill)
       ov[n++] = (struct agx3_field_kv)KV(27, 1);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < 8; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(30 + i, 1);
 
-   return agx3_bind_compose(enc, "itrcoords", 0, base, 6, ov, n, out, len);
+   return agx3_bind_compose(enc, "itrcoords", 0, base, is_b(enc) ? 6 : 8, ov,
+                            n, out, len);
 }
 
 bool
@@ -251,21 +260,26 @@ agx3_vary_uvsw_register_index(enum agx_isa_encoding enc, uint16_t value,
     * half wide. Metal never computes a vertex output's index, so only the
     * immediate kind is witnessed in compiled programs; the register kind is
     * the table's own form for the same field. */
-   if (enc != AGX_ISA_ENCODING_AGX3B || (value & 1) != 0 || value >= 256 ||
-       index >= 256 || slot < 1 || slot > 6 || (wait_mask & ~0x3fu) != 0)
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned nslots = m5 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || (value & 1) != 0 || value >= 256 ||
+       index >= 256 || slot < 1 || slot > nslots || (wait_mask & ~((1u << nslots) - 1)) != 0)
       return false;
    const struct agx3_field_kv base[] = {KV(9, 5), KV(23, 2), KV(24, 1)};
-   struct agx3_field_kv ov[10];
+   const struct agx3_field_kv base_c[] = {
+      KV(9,5), KV(10,2), KV(23,2), KV(24,1), KV(28,2), KV(29,196),
+   };
+   struct agx3_field_kv ov[12];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(5, value);
    ov[n++] = (struct agx3_field_kv)KV(16, index);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(13, value_kill ? 1 : 0);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < nslots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(30 + i, 1);
 
-   return agx3_bind_compose(enc, "uvsw", 0, base, 3, ov, n, out, len);
+   return agx3_bind_compose(enc, "uvsw", 0, m5 ? base_c : base, m5 ? 6 : 3, ov, n, out, len);
 }
 
 bool
@@ -276,19 +290,24 @@ agx3_vary_uvsw_uniform_index(enum agx_isa_encoding enc, uint16_t value,
 {
    /* The uniform kind of uvsw's index operand (f23 = 3, half index in f17;
     * see agx3_vary_uvsw_register_index()). */
-   if (enc != AGX_ISA_ENCODING_AGX3B || (value & 1) != 0 || value >= 256 ||
-       uniform >= 512 || slot < 1 || slot > 6 || (wait_mask & ~0x3fu) != 0)
+   const bool m5 = enc == AGX_ISA_ENCODING_AGX3C;
+   const unsigned nslots = m5 ? 8 : 6;
+   if ((enc != AGX_ISA_ENCODING_AGX3B && !m5) || (value & 1) != 0 || value >= 256 ||
+       uniform >= 512 || slot < 1 || slot > nslots || (wait_mask & ~((1u << nslots) - 1)) != 0)
       return false;
    const struct agx3_field_kv base[] = {KV(9, 5), KV(23, 3), KV(24, 1)};
-   struct agx3_field_kv ov[10];
+   const struct agx3_field_kv base_c[] = {
+      KV(9,5), KV(10,2), KV(23,3), KV(24,1), KV(28,2), KV(29,196),
+   };
+   struct agx3_field_kv ov[12];
    unsigned n = 0;
    ov[n++] = (struct agx3_field_kv)KV(5, value);
    ov[n++] = (struct agx3_field_kv)KV(17, uniform);
    ov[n++] = (struct agx3_field_kv)KV(12, slot);
    ov[n++] = (struct agx3_field_kv)KV(13, value_kill ? 1 : 0);
-   for (unsigned i = 0; i < 6; ++i)
+   for (unsigned i = 0; i < nslots; ++i)
       if (wait_mask & (1u << i))
          ov[n++] = (struct agx3_field_kv)KV(30 + i, 1);
 
-   return agx3_bind_compose(enc, "uvsw", 0, base, 3, ov, n, out, len);
+   return agx3_bind_compose(enc, "uvsw", 0, m5 ? base_c : base, m5 ? 6 : 3, ov, n, out, len);
 }
diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c
index 3948b43ba90f..5443f4e509f0 100644
--- a/src/asahi/compiler/agx_compile.c
+++ b/src/asahi/compiler/agx_compile.c
@@ -24,6 +24,7 @@
 #include "agx_compiler.h"
 #include "agx_debug.h"
 #include "agx_nir.h"
+#include "agx3_matrix_core.h"
 #include "agx_opcodes.h"
 #include "agx3_pack.h"
 #include "glsl_types.h"
@@ -456,8 +457,9 @@ agx_g17p_reg16(agx_builder *b, agx_index x)
 static agx_index
 agx_recollect_vector(agx_builder *b, nir_src vec)
 {
-   agx_index comps[4];
+   agx_index comps[NIR_MAX_VEC_COMPONENTS];
    unsigned nr = nir_src_num_components(vec);
+   assert(nr <= ARRAY_SIZE(comps));
 
    for (unsigned i = 0; i < nr; ++i)
       comps[i] = agx_extract_nir_src(b, vec, i);
@@ -533,7 +535,10 @@ agx_emit_split(agx_builder *b, agx_index *dests, agx_index vec, unsigned n)
 static void
 agx_emit_cached_split(agx_builder *b, agx_index vec, unsigned n)
 {
-   agx_index dests[4] = {agx_null(), agx_null(), agx_null(), agx_null()};
+   agx_index dests[NIR_MAX_VEC_COMPONENTS];
+   assert(n <= ARRAY_SIZE(dests));
+   for (unsigned i = 0; i < n; ++i)
+      dests[i] = agx_null();
    agx_emit_split(b, dests, vec, n);
    agx_cache_collect(b, vec, n, dests);
 }
@@ -697,9 +702,13 @@ agx_emit_load_vary_agx3(agx_builder *b, agx_index dest, agx_index cf,
 
    agx_index interp_coords = agx_zero();
    if (interp == AGX_INTERPOLATION_SAMPLE) {
-      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
-          (sample_index.size != AGX_SIZE_16 &&
-           sample_index.size != AGX_SIZE_32))
+      /* Native G17G at-sample interpolation (lane-mesh/cwit-interp:
+       * interpolate_at_sample with a constant, uniform or sample-ID index and
+       * [[sample_{,no_}perspective]] at 1, 2 and 4 samples) has G17P's shape:
+       * one ITRCOORDS token from the sample index, then the native projective
+       * RHW/RCP/MUL sequence around token f32interp leaves. */
+      if (sample_index.size != AGX_SIZE_16 &&
+          sample_index.size != AGX_SIZE_32)
          return false;
 
       agx_index sample_half = sample_index;
@@ -715,11 +724,12 @@ agx_emit_load_vary_agx3(agx_builder *b, agx_index dest, agx_index cf,
 
       interp_coords = agx_temp(b->shader, AGX_SIZE_32);
       agx_itrcoords_agx3_to(b, interp_coords, sample_half, interp);
-   } else if (interp == AGX_INTERPOLATION_CENTROID &&
-              b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
-      /* Apple's G17P centroid inputs, single-sample and multisample alike,
-       * pass native W16 SR39 -- the covered samples -- straight to
-       * ITRCOORDS. With one sample that is the pixel centre. */
+   } else if (interp == AGX_INTERPOLATION_CENTROID) {
+      /* Apple's G17P and G17G centroid inputs, single-sample and multisample
+       * alike, pass native W16 SR39 -- the covered samples -- straight to
+       * ITRCOORDS. With one sample that is the pixel centre. G17G witnesses:
+       * lane-mesh/cwit-interp iac/iac_np, cwit-interp2 cp/cnp at 1 and 4
+       * samples ([[centroid_{,no_}perspective]], interpolate_at_centroid()). */
       agx_index coverage = agx_temp(b->shader, AGX_SIZE_16);
       agx_get_sr_to(b, coverage, AGX_SR_INPUT_SAMPLE_MASK);
 
@@ -732,6 +742,11 @@ agx_emit_load_vary_agx3(agx_builder *b, agx_index dest, agx_index cf,
       return false;
    }
 
+   /* G17P and G17G use the native projective RHW/RCP/MUL sequence for centre,
+    * sample and centroid inputs alike. */
+   const bool native_projective =
+      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C;
    agx_index reciprocal_w = agx_null();
    if (perspective) {
       agx_index rhw_cf = agx_get_cf(b->shader, VARYING_SLOT_POS, 3);
@@ -739,15 +754,15 @@ agx_emit_load_vary_agx3(agx_builder *b, agx_index dest, agx_index cf,
          return false;
 
       if (interp == AGX_INTERPOLATION_CENTER &&
-          b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+          native_projective) {
          reciprocal_w = agx3_center_rcp_w(b->shader, rhw_cf);
       } else {
          agx_index rhw = agx_temp(b->shader, AGX_SIZE_32);
-         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+         if (native_projective)
             agx_rhw_interp_agx3_to(b, rhw, rhw_cf, interp_coords, interp);
          else
             agx_interp_agx3_to(b, rhw, rhw_cf, interp_coords, interp, true);
-         reciprocal_w = b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B
+         reciprocal_w = native_projective
                            ? agx_persp_rcp_agx3(b, rhw, rhw_cf.value)
                            : agx_rcp(b, rhw);
       }
@@ -766,7 +781,7 @@ agx_emit_load_vary_agx3(agx_builder *b, agx_index dest, agx_index cf,
       agx_interp_agx3_to(b, raw, channel_cf, interp_coords, interp,
                          perspective);
       if (perspective &&
-          b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+          native_projective) {
          /* Native f32mul.ep4 identifies the coefficient involved in this
           * projective divide. Preserve it explicitly rather than trying to
           * reconstruct SSA provenance after physical register allocation. */
@@ -889,8 +904,7 @@ agx_pack_normalized_agx3(agx_builder *b, const agx_index *channels,
       for (unsigned i = 0; i < 4; ++i) {
          if (channels[i].size == AGX_SIZE_32) {
             pack_sources[i] = channels[i];
-         } else if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-                    channels[i].size == AGX_SIZE_16) {
+         } else if (channels[i].size == AGX_SIZE_16) {
             /* As with the normalized 8-bit epilog below, a texture result
              * can arrive through the half export ABI. The native RGB10A2
              * pack consumes f32, so widen before packing instead of handing
@@ -911,7 +925,7 @@ agx_pack_normalized_agx3(agx_builder *b, const agx_index *channels,
     * store of four channels, as a background clear writes, has no alpha lane
     * to keep. */
    if ((format == AGX_FORMAT_RGB9E5 || format == AGX_FORMAT_RG11B10F) &&
-       b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+       agx_isa_is_agx3(b->shader->key->dev.encoding)) {
       if (nr_channels != 3 && nr_channels != 4)
          return 0;
       for (unsigned i = 0; i < 3; ++i)
@@ -924,11 +938,9 @@ agx_pack_normalized_agx3(agx_builder *b, const agx_index *channels,
       return 1;
    }
 
-   /* An 8-bit sRGB target is U8NORM storage with the transfer function on
-    * the pack. Witnessed on applegpu_g17p only, so keep AGX3c closed. */
+   /* Both native encodings use U8NORM tile storage and select the sRGB
+    * transfer function on FLTNORM, including a linear alpha lane. */
    if (format == AGX_FORMAT_SRGBA8) {
-      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
-         return 0;
       format = AGX_FORMAT_U8NORM;
    } else {
       srgb_lanes = 0;
@@ -939,11 +951,8 @@ agx_pack_normalized_agx3(agx_builder *b, const agx_index *channels,
    const bool norm16 =
       format == AGX_FORMAT_U16NORM || format == AGX_FORMAT_S16NORM;
 
-   /* Fresh Apple G17P R16Unorm/R16Snorm pipelines use the same conversion
-    * opcode with format modes 2/3 and one instruction per 16-bit channel.
-    * Keep AGX3c closed until an equivalent native witness exists. */
-   if (!norm8 &&
-       !(norm16 && b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B))
+   /* Native G17P and G17G use modes 2/3 for normalized 16-bit channels. */
+   if (!norm8 && !norm16)
       return 0;
 
    if (nr_channels < 1 || nr_channels > 4)
@@ -956,9 +965,7 @@ agx_pack_normalized_agx3(agx_builder *b, const agx_index *channels,
    for (unsigned i = 0; i < nr_channels; ++i) {
       if (channels[i].size == AGX_SIZE_32) {
          pack_sources[i] = channels[i];
-      } else if (norm8 &&
-                 b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-                 channels[i].size == AGX_SIZE_16) {
+      } else if (channels[i].size == AGX_SIZE_16) {
          pack_sources[i] = agx_temp(b->shader, AGX_SIZE_32);
          agx_fmov_to(b, pack_sources[i], channels[i]);
       } else {
@@ -999,7 +1006,8 @@ agx_pack_integer_8_agx3(agx_builder *b, const agx_index *channels,
                         unsigned nr_channels, enum agx_format format,
                         agx_index *packed)
 {
-   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if ((b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3C) ||
        format != AGX_FORMAT_I8 || nr_channels < 1 || nr_channels > 4)
       return 0;
 
@@ -1113,11 +1121,9 @@ agx_emit_local_store_pixel(agx_builder *b, nir_intrinsic_instr *instr)
       nr_packed = agx_pack_integer_8_agx3(
          b, compacted, compact_count, format, packed);
 
-   /* G17P's tile store encodes only the lane width.  Unlike AGX2 it cannot
-    * narrow an f32 colour to the f16 lanes named by that width, so make the
-    * conversion explicit before collecting the store payload.  Keep AGX3c
-    * unchanged until its matching narrowing ALU form is grounded. */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   /* Native AGX3 tile stores carry a lane width, with the f32-to-f16
+    * conversion in the shader. Both encodings have the narrowing FMUL form. */
+   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        format == AGX_FORMAT_F16 && nr_packed == 0) {
       nr_packed = compact_count;
       for (unsigned i = 0; i < compact_count; ++i) {
@@ -1152,15 +1158,27 @@ agx_emit_local_store_pixel(agx_builder *b, nir_intrinsic_instr *instr)
 static agx_instr *
 agx_emit_zs_word_agx3(agx_builder *b, nir_intrinsic_instr *instr,
                       agx_index z, agx_index s, unsigned base,
-                      agx_index coverage, agx_index prepared_depth)
+                      agx_index coverage, agx_index prepared_depth,
+                      bool bracketed)
 {
    const bool write_z = base & 1;
    const bool write_s = base & 2;
 
    agx_index word = agx_temp(b->shader, AGX_SIZE_32);
-   if (write_s)
+   if (write_s) {
+      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+         /* Native G17G [[stencil]] and depth+stencil exports put the
+          * target mask above the S8 reference: constant 7 stages 0xff07
+          * in the high half, while coverage occupies the low half. */
+         s = agx_or(b, s, agx_mov_imm(b, 16, 0xff00));
+      }
       agx_emit_collect_to(b, word, 2, (agx_index[]){coverage, s});
-   else
+   } else if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+      /* Native G17G DEPTHF keeps the target mask in bits24..31,
+       * independently of SR2's duplicated low/high coverage bytes. */
+      agx_index target = agx_mov_imm(b, 16, 0xff00);
+      agx_emit_collect_to(b, word, 2, (agx_index[]){coverage, target});
+   } else
       agx_mov_to(b, word, coverage);
 
    if (write_z) {
@@ -1171,13 +1189,26 @@ agx_emit_zs_word_agx3(agx_builder *b, nir_intrinsic_instr *instr,
          const float value = nir_src_as_float(instr->src[1]);
          depth = agx_mov_imm(b, 32, fui(CLAMP(value, 0.0f, 1.0f)));
       } else {
-         depth = agx_temp(b->shader, AGX_SIZE_32);
-         agx_fmov_to(b, depth, z)->saturate = true;
+         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+            agx_index zero = agx_immediate_f(0.0f);
+            agx_index one = agx_immediate_f(1.0f);
+            agx_index nonnegative = agx_fcmpsel(b, z, zero, z, zero,
+                                                AGX_FCOND_GTN);
+            depth = agx_fcmpsel(b, nonnegative, one, nonnegative, one,
+                                 AGX_FCOND_LTN);
+         } else {
+            depth = agx_temp(b->shader, AGX_SIZE_32);
+            agx_fmov_to(b, depth, z)->saturate = true;
+         }
       }
       word = agx_vec2(b, word, depth);
    }
 
    b->shader->out->writes_sample_mask = true;
+   /* A covered-sample loop's transaction is opened and closed around the
+    * loop by zs_transaction_agx. */
+   if (bracketed)
+      return agx_zs_emit_agx3(b, word, base);
    agx_wait_pixel_mask(b, 0x1);
    agx_instr *I = agx_zs_emit_agx3(b, word, base);
    assert(b->shader->zs_writes_left > 0);
@@ -1248,7 +1279,74 @@ agx_emit_store_zs(agx_builder *b, nir_intrinsic_instr *instr)
    const bool sample_words =
       samples > 1 &&
       (mask == 0xff || util_bitcount(mask & BITFIELD_MASK(samples)) == 1);
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (base & AGX_ZS_COVERED_SAMPLE) {
+      /* Native M5 multisample fragments writing only depth or only stencil
+       * (native G17G compiler zs_depth_{2,4,8}x, zs_cdepth_4x,
+       * zs_depth_only_4x, zs_stencil_{2,4}x) issue one DEPTHF in a loop over
+       * the covered samples, rereading SR39 on each iteration: its API mask
+       * byte above that iteration's one sample. The loop and its transaction
+       * come from agx_nir_lower_m5_zs_sample_loop. */
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+          samples < 2 || (base & 3) == 3) {
+         b->shader->unsupported_instruction = true;
+         return NULL;
+      }
+      agx_index sr39 = agx_temp(b->shader, AGX_SIZE_16);
+      agx_get_sr_coverage_to(b, sr39, AGX_SR_INPUT_SAMPLE_MASK);
+      agx_index covered = agx_and(b, sr39, agx_src_index(&instr->src[0]));
+      agx_index api = agx_and(b, sr39, agx_mov_imm(b, 16, 0xff00));
+      /* agx_nir_lower_m5_zs_sample_loop clamps the depth before the loop. */
+      return agx_emit_zs_word_agx3(b, instr, z, s, base & 7,
+                                  agx_or(b, api, covered),
+                                  write_z ? z : agx_null(), true);
+   }
+   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       samples > 1 && write_z && write_s && !dynamic_mask &&
+       (mask & BITFIELD_MASK(samples)) == BITFIELD_MASK(samples)) {
+      /* Native M5 whole-pixel depth and stencil (native G17G compiler
+       * zs_depth_stencil_{2,4}x) is one DEPTHF naming every covered sample,
+       * where depth or stencil alone loops over them. */
+      agx_index sr39 = agx_temp(b->shader, AGX_SIZE_16);
+      agx_get_sr_coverage_to(b, sr39, AGX_SR_INPUT_SAMPLE_MASK);
+      agx_index covered =
+         agx_and(b, sr39, agx_immediate(BITFIELD_MASK(samples)));
+      agx_index api = agx_and(b, sr39, agx_mov_imm(b, 16, 0xff00));
+      return agx_emit_zs_word_agx3(b, instr, z, s, base,
+                                  agx_or(b, api, covered), agx_null(), false);
+   }
+   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       samples > 1) {
+      /* G17G keeps SR39's API mask in the high byte and the changing
+       * coverage in the low byte. Native multisample DEPTHF loops preserve
+       * the former and select one covered sample per payload. The target
+       * mask and optional stencil reference are added by the shared word
+       * builder, independently of this coverage pair. */
+      unsigned reach = agx_possible_zs_samples(instr->src[0].ssa) &
+                       BITFIELD_MASK(samples);
+      if (!reach)
+         reach = BITFIELD_MASK(samples);
+
+      agx_index keep = dynamic_mask
+         ? agx_and(b, agx_src_index(&instr->src[0]),
+                      agx_immediate(BITFIELD_MASK(samples)))
+         : agx_immediate(mask & BITFIELD_MASK(samples));
+      agx_instr *last = NULL;
+      b->shader->zs_writes_left += util_bitcount(reach) - 1;
+      u_foreach_bit(i, reach) {
+         agx_index sr39 = agx_temp(b->shader, AGX_SIZE_16);
+         agx_get_sr_coverage_to(b, sr39, AGX_SR_INPUT_SAMPLE_MASK);
+         agx_index covered = agx_and(b, sr39, agx_immediate(BITFIELD_BIT(i)));
+         if (dynamic_mask)
+            covered = agx_and(b, covered, keep);
+         else if (!(mask & BITFIELD_BIT(i)))
+            covered = agx_zero();
+         agx_index api = agx_and(b, sr39, agx_mov_imm(b, 16, 0xff00));
+         last = agx_emit_zs_word_agx3(b, instr, z, s, base,
+                                     agx_or(b, api, covered), agx_null(), false);
+      }
+      return last;
+   }
+   if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
        samples > 1 && dynamic_mask) {
       /* A split late shader keeps its discard predicate in a register. Fold
        * that predicate into the same per-sample words a compile-time mask
@@ -1284,21 +1382,23 @@ agx_emit_store_zs(agx_builder *b, nir_intrinsic_instr *instr)
             b, instr, z, s, base,
             agx_iadd(b, agx_immediate(15),
                      agx_iadd(b, upper, covered, 4), 4),
-            agx_null());
+            agx_null(), false);
       }
       return last;
    }
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+        (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C && samples == 1)) &&
        !sample_words) {
       /* At several samples, only a mask known at compile time has a
        * witnessed word; any other reads the one-sample coverage, which the
        * packer refuses there. */
       agx_index sr2 = agx_temp(b->shader, AGX_SIZE_16);
       agx_get_coverage_1x_agx3_to(b, sr2);
-      return agx_emit_zs_word_agx3(
-         b, instr, z, s, base,
-         agx_iadd(b, agx_immediate(15), agx_iadd(b, sr2, sr2, 4), 4),
-         agx_null());
+      agx_index coverage = b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C
+         ? agx_iadd(b, sr2, agx_iadd(b, agx_zero(), sr2, 4), 4)
+         : agx_iadd(b, agx_immediate(15), agx_iadd(b, sr2, sr2, 4), 4);
+      return agx_emit_zs_word_agx3(b, instr, z, s, base, coverage, agx_null(),
+                                   false);
    } else if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
       /* Apple's 2x/4x fragments writing [[depth(any)]] or [[stencil]] per
        * [[sample_id]] write one DEPTHF per sample, each word built from SR39:
@@ -1328,7 +1428,7 @@ agx_emit_store_zs(agx_builder *b, nir_intrinsic_instr *instr)
             b, instr, z, s, base,
             agx_iadd(b, agx_immediate(15),
                      agx_iadd(b, upper, covered, 4), 4),
-            agx_null());
+            agx_null(), false);
       }
       return last;
    }
@@ -1368,10 +1468,10 @@ agx_unpack_normalized_halves_agx3(agx_builder *b, agx_index dest,
    /* An 8-bit sRGB target is U8NORM storage whose colour channels `normflt`
     * decodes (agx3_fltnorm_core.h, "The sRGB conversion modes"); the fourth
     * channel, alpha, is linear. The packer reads the lane set from the mask.
-    * Witnessed on applegpu_g17p only. */
+    * Witnessed on both native encodings. */
    unsigned srgb_lanes = 0;
    if (format == AGX_FORMAT_SRGBA8) {
-      assert(b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+      assert(b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2);
       format = AGX_FORMAT_U8NORM;
       srgb_lanes = BITFIELD_MASK(MIN2(nr_comps, 3));
    }
@@ -1453,7 +1553,7 @@ agx_unpack_normalized_agx3(agx_builder *b, agx_index dest,
     * format has alpha, so a four-component read sees 1.0 there; a narrower
     * read takes the leading lanes. */
    if ((format == AGX_FORMAT_RGB9E5 || format == AGX_FORMAT_RG11B10F) &&
-       b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+       agx_isa_is_agx3(b->shader->key->dev.encoding)) {
       if (nr_comps < 1 || nr_comps > 4 || dest.size != AGX_SIZE_32 ||
           (format == AGX_FORMAT_RGB9E5 && nr_comps < 3))
          return false;
@@ -1476,14 +1576,14 @@ agx_unpack_normalized_agx3(agx_builder *b, agx_index dest,
    /* sRGB: a U8NORM tile read, decoded by normflt (see
     * agx_unpack_normalized_halves_agx3). */
    const bool srgb = format == AGX_FORMAT_SRGBA8 &&
-                     b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B;
+                     b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2;
    const enum agx_format tile_format = srgb ? AGX_FORMAT_U8NORM : format;
    const bool norm8 =
       tile_format == AGX_FORMAT_U8NORM || tile_format == AGX_FORMAT_S8NORM;
    const bool norm16 =
       format == AGX_FORMAT_U16NORM || format == AGX_FORMAT_S16NORM;
    if (!norm8 &&
-       !(norm16 && b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B))
+       !(norm16 && agx_isa_is_agx3(b->shader->key->dev.encoding)))
       return false;
    if (nr_comps < 1 || nr_comps > 4 ||
        (dest.size != AGX_SIZE_16 && dest.size != AGX_SIZE_32))
@@ -1523,7 +1623,7 @@ agx_emit_local_load_pixel(agx_builder *b, agx_index dest,
    /* Apple's reads of 8-bit integer attachments load the bytes packed, two
     * per 16-bit register, and then shift and mask the channels out
     * (blend/native/tilemask.py uread_*). */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
        format == AGX_FORMAT_I8) {
       const unsigned nr_halfs = DIV_ROUND_UP(nr_comps, 2);
       agx_index packed = agx_temp(b->shader, AGX_SIZE_16);
@@ -1561,11 +1661,11 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
    enum agx_format fmt = agx_format_for_pipe(nir_intrinsic_format(instr));
    unsigned shift = nir_intrinsic_base(instr);
    const bool index_bytes = shift & AGX_NIR_MEMORY_INDEX_BYTES;
-   assert(!index_bytes || b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+   assert(!index_bytes || b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2);
    const unsigned packed_count = (shift & AGX_NIR_MEMORY_PACKED_COUNT_MASK) >>
                                  AGX_NIR_MEMORY_PACKED_COUNT_SHIFT;
    assert(packed_count <= 4);
-   assert(!packed_count || b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+   assert(!packed_count || b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2);
    shift &= ~(AGX_NIR_MEMORY_INDEX_BYTES | AGX_NIR_MEMORY_PACKED_COUNT_MASK);
 
    assert(shift <= 3);
@@ -1576,8 +1676,8 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
 
    const bool norm8 = fmt == AGX_FORMAT_U8NORM || fmt == AGX_FORMAT_S8NORM;
    const bool norm16 = fmt == AGX_FORMAT_U16NORM || fmt == AGX_FORMAT_S16NORM;
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-       (norm8 || norm16)) {
+   if ((b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 && norm8) ||
+       (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 && norm16)) {
       const unsigned nr = instr->def.num_components;
       assert(nr >= 1 && nr <= 4 && packed_count == 0);
       assert(dest.size == AGX_SIZE_16 || dest.size == AGX_SIZE_32);
@@ -1618,7 +1718,7 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
 
    /* AGX3 memory traffic is raw. Convert a normalized packed texel with
     * the same native instruction used by tile loads, after its real load. */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        fmt == AGX_FORMAT_RGB10A2) {
       assert(dest.size == AGX_SIZE_32 && instr->def.num_components == 4);
       assert(packed_count == 0);
@@ -1633,7 +1733,7 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
 
    /* The same one-word load and native unpack for R11G11B10_FLOAT: Apple's
     * G17P FloatRG11B10 vertex fetch is LDDEV W32 + `unpack` layout 2. */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        fmt == AGX_FORMAT_RG11B10F) {
       assert(dest.size == AGX_SIZE_32 && instr->def.num_components == 3);
       assert(packed_count == 0);
@@ -1648,8 +1748,12 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
 
    /* A scalar W8 load writes an aligned low half. Multi-component W8 loads
     * instead return a byte-packed payload, like G17P tile traffic. */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-       fmt == AGX_FORMAT_I8 && dest.size == AGX_SIZE_16 && packed_count == 0) {
+   if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+        b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
+       fmt == AGX_FORMAT_I8 &&
+       (dest.size == AGX_SIZE_16 ||
+        (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+         dest.size == AGX_SIZE_32)) && packed_count == 0) {
       const unsigned nr = instr->def.num_components;
       agx_index payload = agx_temp(b->shader, AGX_SIZE_32);
       agx_instr *load = agx_device_load_to(b, payload, addr, offset, fmt,
@@ -1657,7 +1761,7 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
                                            nir_is_coherent(instr));
       load->memory_index_is_bytes = index_bytes;
 
-      if (nr == 1) {
+      if (nr == 1 && dest.size == AGX_SIZE_16) {
          agx_subdivide_to(b, dest, payload, 0);
       } else {
          load->memory_is_packed = true;
@@ -1682,7 +1786,7 @@ agx_emit_store(agx_builder *b, nir_intrinsic_instr *instr)
    enum agx_format fmt = agx_format_for_pipe(nir_intrinsic_format(instr));
    unsigned shift = nir_intrinsic_base(instr);
    const bool index_bytes = shift & AGX_NIR_MEMORY_INDEX_BYTES;
-   assert(!index_bytes || b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+   assert(!index_bytes || b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2);
    shift &= ~AGX_NIR_MEMORY_INDEX_BYTES;
 
    assert(shift <= 3);
@@ -1693,7 +1797,8 @@ agx_emit_store(agx_builder *b, nir_intrinsic_instr *instr)
 
    agx_index value;
    unsigned mask;
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+        b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
        fmt == AGX_FORMAT_RGB10A2) {
       assert(nir_src_num_components(instr->src[0]) == 4 &&
              nir_src_bit_size(instr->src[0]) == 32);
@@ -1704,7 +1809,8 @@ agx_emit_store(agx_builder *b, nir_intrinsic_instr *instr)
          agx_extract_nir_src(b, instr->src[0], 3));
       fmt = AGX_FORMAT_I32;
       mask = 1;
-   } else if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   } else if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+               b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
               fmt == AGX_FORMAT_I8 &&
               nir_src_bit_size(instr->src[0]) == 8 &&
               nir_src_num_components(instr->src[0]) > 1) {
@@ -1729,7 +1835,8 @@ agx_emit_store(agx_builder *b, nir_intrinsic_instr *instr)
                                       shift, nir_is_coherent(instr));
    store->memory_index_is_bytes = index_bytes;
    store->memory_is_packed =
-      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+       b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
       fmt == AGX_FORMAT_I8 && nir_src_num_components(instr->src[0]) > 1;
 }
 
@@ -1754,7 +1861,7 @@ static agx_instr *
 agx_emit_load_preamble(agx_builder *b, agx_index dst,
                        nir_intrinsic_instr *instr)
 {
-   agx_index srcs[4] = {agx_null()};
+   agx_index srcs[NIR_MAX_VEC_COMPONENTS];
    unsigned dim = instr->def.num_components;
    assert(dim <= ARRAY_SIZE(srcs) && "shouldn't see larger vectors");
 
@@ -1790,6 +1897,26 @@ agx_emit_store_preamble(agx_builder *b, nir_intrinsic_instr *instr)
    unsigned stride = agx_size_align_16(vec.size);
    unsigned nr = nir_src_num_components(instr->src[0]);
 
+   /* An AGX3C uniform export is one 32-bit copy per word, so a 64-bit export
+    * is two copies however it is spelled (agx3_pack_binary). Most 64-bit
+    * fragment exports are texture and sampler handles built as
+    * pack_64_2x32_split(word, 0): collecting the two words into an aligned
+    * register pair first costs a move per word on top of the copies. Export
+    * each word straight from its own register instead, into the same
+    * uniform halves, in the same order and at the same program point. */
+   nir_alu_instr *pack = nir_def_as_alu_or_null(instr->src[0].ssa);
+   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C && nr == 1 &&
+       vec.size == AGX_SIZE_64 && pack &&
+       pack->op == nir_op_pack_64_2x32_split) {
+      for (unsigned w = 0; w < 2; ++w) {
+         agx_index word = agx_extract_nir_src(b, pack->src[w].src,
+                                              pack->src[w].swizzle[0]);
+         agx_uniform_store(b, word, agx_immediate(base + 2 * w), 1);
+      }
+
+      return NULL;
+   }
+
    for (unsigned i = 0; i < nr; i += (4 / stride)) {
       agx_index data[4] = {0};
       unsigned count = MIN2(4 / stride, nr - i);
@@ -1878,6 +2005,9 @@ agx_emit_block_image_store(agx_builder *b, nir_intrinsic_instr *instr)
    // XXX: how does this possibly work
    if (format == AGX_FORMAT_F16)
       format = AGX_FORMAT_I16;
+   else if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+            format == AGX_FORMAT_SRGBA8)
+      format = AGX_FORMAT_U8NORM;
 
    return agx_block_image_store(b, base, index, offset, coords, format, dim,
                                 explicit);
@@ -1898,13 +2028,13 @@ agx_get_sr_native_width_to(agx_builder *b, agx_index dst, enum agx_sr sr)
    case AGX_SR_THREAD_POSITION_IN_THREADGROUP_X:
    case AGX_SR_THREAD_POSITION_IN_THREADGROUP_Y:
    case AGX_SR_THREAD_POSITION_IN_THREADGROUP_Z:
+   case AGX_SR_SCS_THREAD_INDEX:
    case AGX_SR_THREAD_INDEX_IN_THREADGROUP:
    case AGX_SR_THREAD_INDEX_IN_SUBGROUP:
    case AGX_SR_SUBGROUP_INDEX_IN_THREADGROUP:
    case AGX_SR_MESH_PAYLOAD_HI:
    case AGX_SR_MESH_PRIMITIVE_STORAGE_HI:
    case AGX_SR_BACKGROUND_LAYER_ID:
-   case AGX_SR_SCS_THREAD_INDEX:
       narrow_agx3 = true;
       break;
    default:
@@ -1931,6 +2061,11 @@ static agx_instr *
 agx_get_sr_coverage_native_width_to(agx_builder *b, agx_index dst,
                                     enum agx_sr sr)
 {
+   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       sr == AGX_SR_TOTAL_ACTIVE_THREADS_IN_SUBGROUP) {
+      return agx_simd_reduce_to(b, dst,
+                               agx_mov_imm(b, 32, 1), AGX_SIMD_OP_IADD);
+   }
    if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        dst.size == AGX_SIZE_32 &&
        (sr == AGX_SR_ACTIVE_THREAD_INDEX_IN_SUBGROUP ||
@@ -2000,7 +2135,7 @@ agx_local_base(nir_src src)
 static agx_index
 agx_local_atomic_base(agx_builder *b, nir_src src)
 {
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
        nir_src_is_const(src)) {
       const uint64_t address = nir_src_as_uint(src);
 
@@ -2053,7 +2188,7 @@ agx_emit_atomic(agx_builder *b, agx_index dst, nir_intrinsic_instr *instr,
 static bool
 agx_lower_agx3_device_atomic_sources(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return false;
 
    bool progress = false;
@@ -2081,6 +2216,22 @@ agx_lower_agx3_device_atomic_sources(agx_context *ctx)
    return progress;
 }
 
+static bool
+agx_lower_agx3_matrix_sources(agx_context *ctx)
+{
+   bool progress = false;
+   agx_foreach_instr_global_safe(ctx, I) {
+      if (I->op != AGX_OPCODE_MATMAD16_AGX3)
+         continue;
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+      agx_index tied = agx_vec_temp(ctx, AGX_SIZE_32, 8);
+      agx_mov_to(&b, tied, I->src[2]);
+      I->src[2] = tied;
+      progress = true;
+   }
+   return progress;
+}
+
 static enum agx_format
 format_for_bitsize(unsigned bitsize)
 {
@@ -2096,18 +2247,33 @@ format_for_bitsize(unsigned bitsize)
    }
 }
 
+/* AGX3 ldtg/sttg carry an immediate byte offset next to the element index.
+ * agx_nir_opt_shared_byte_offsets() folds a constant addend of the byte
+ * address into the intrinsic's base; it rides in the base operand here
+ * (agx3_translate_local() moves it into the descriptor's byte_offset).
+ * AGX2 keeps its byte-addressed base register and never sees a nonzero base.
+ */
+static agx_index
+agx_local_immediate_base(agx_builder *b, nir_intrinsic_instr *instr)
+{
+   const unsigned base = nir_intrinsic_base(instr);
+   assert(base == 0 ||
+          b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2);
+   if (base == 0)
+      return agx_zero();
+
+   agx_index imm = agx_immediate(base);
+   imm.size = AGX_SIZE_16;
+   return imm;
+}
+
 static void
 agx_emit_local_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *instr)
 {
    agx_index address = agx_local_base(instr->src[0]);
    const bool agx3 = b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2;
-   /* NIR shared addresses are offsets into the statically allocated
-    * workgroup storage. Uniform half 8 is ordinary argument storage on
-    * G17P (for example the tessellation prefix kernel's third pointer),
-    * so it cannot serve as an implicit shared-memory base. */
-   agx_index base = b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B
-                      ? agx_zero()
-                      : agx3 ? agx_uniform(8, AGX_SIZE_16) : address;
+   /* Shared offsets are relative to the workgroup allocation. */
+   agx_index base = agx3 ? agx_local_immediate_base(b, instr) : address;
    agx_index index = agx3 ? address : agx_zero();
    assert(address.size == AGX_SIZE_16);
 
@@ -2115,7 +2281,7 @@ agx_emit_local_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *instr)
    unsigned nr = instr->def.num_components;
    unsigned mask = BITFIELD_MASK(nr);
 
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        format == AGX_FORMAT_I8 && nr > 1) {
       /* LDTG returns packed bytes, not one byte in each half register.
        * Keep the vector memory transaction and unpack its physical payload. */
@@ -2140,9 +2306,7 @@ agx_emit_local_store(agx_builder *b, nir_intrinsic_instr *instr)
    agx_index value = agx_src_index(&instr->src[0]);
    agx_index address = agx_local_base(instr->src[1]);
    const bool agx3 = b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2;
-   agx_index base = b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B
-                      ? agx_zero()
-                      : agx3 ? agx_uniform(8, AGX_SIZE_16) : address;
+   agx_index base = agx3 ? agx_local_immediate_base(b, instr) : address;
    agx_index index = agx3 ? address : agx_zero();
    assert(address.size == AGX_SIZE_16);
 
@@ -2151,7 +2315,7 @@ agx_emit_local_store(agx_builder *b, nir_intrinsic_instr *instr)
       nir_src_num_components(instr->src[0])); /* XXX: there's a write mask */
 
    const bool packed_bytes =
-      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
       format == AGX_FORMAT_I8 && nir_src_num_components(instr->src[0]) > 1;
    if (packed_bytes) {
       const unsigned nr = nir_src_num_components(instr->src[0]);
@@ -2171,10 +2335,11 @@ agx_emit_local_store(agx_builder *b, nir_intrinsic_instr *instr)
 /* Scalarize static 32-bit stack vectors after RA so both NIR scratch and
  * allocator spills use the native scalar TPS copy/rld protocol. No temporary
  * GPR is required: component registers and frame offsets are already fixed. */
+
 static void
 agx_lower_agx3_stack_vectors(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_instr_global_safe(ctx, I) {
@@ -2218,7 +2383,17 @@ agx_emit_load_scratch(agx_builder *b, agx_index dst, nir_intrinsic_instr *instr)
    unsigned nr = instr->def.num_components;
    unsigned mask = BITFIELD_MASK(nr);
 
-   agx_stack_load_to(b, dst, offset, format, mask);
+   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       format == AGX_FORMAT_I8 && nr > 1) {
+      const unsigned n = DIV_ROUND_UP(nr, 2);
+      agx_index packed = agx_vec_temp(b->shader, AGX_SIZE_16, n);
+      agx_stack_load_to(b, packed, offset, format, mask)->memory_is_packed = true;
+      agx_index halves[2] = {agx_null()};
+      agx_emit_split(b, halves, packed, n);
+      agx_unpack_integer_8_agx3(b, dst, halves, nr);
+   } else {
+      agx_stack_load_to(b, dst, offset, format, mask);
+   }
    agx_emit_cached_split(b, dst, nr);
    b->shader->any_scratch = true;
 }
@@ -2231,7 +2406,18 @@ agx_emit_store_scratch(agx_builder *b, nir_intrinsic_instr *instr)
    enum agx_format format = format_for_bitsize(nir_src_bit_size(instr->src[0]));
    unsigned mask = BITFIELD_MASK(nir_src_num_components(instr->src[0]));
 
-   agx_stack_store(b, value, offset, format, mask);
+   const unsigned nr = nir_src_num_components(instr->src[0]);
+   const bool packed_bytes =
+      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+      format == AGX_FORMAT_I8 && nr > 1;
+   if (packed_bytes) {
+      agx_index channels[4] = {agx_null()}, packed[2] = {agx_null()};
+      for (unsigned i = 0; i < nr; ++i)
+         channels[i] = agx_extract_nir_src(b, instr->src[0], i);
+      const unsigned n = agx_pack_integer_8_agx3(b, channels, nr, format, packed);
+      value = agx_emit_collect(b, n, packed);
+   }
+   agx_stack_store(b, value, offset, format, mask)->memory_is_packed = packed_bytes;
    b->shader->any_scratch = true;
 }
 
@@ -2327,8 +2513,12 @@ agx_emit_image_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *intr)
    if (is_ms) {
       lod_mode = AGX_LOD_MODE_AUTO_LOD;
       lod = agx_zero();
-   } else if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-              dim == GLSL_SAMPLER_DIM_2D && !is_array && !sparse &&
+   } else if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+               b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
+              (dim == GLSL_SAMPLER_DIM_2D ||
+               (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+                dim == GLSL_SAMPLER_DIM_3D)) &&
+              !is_array && !sparse &&
               nir_src_is_const(intr->src[3]) &&
               nir_src_as_uint(intr->src[3]) == 0) {
       /* Match texelFetch's constant-level handling. Image reads also use
@@ -2345,7 +2535,8 @@ agx_emit_image_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *intr)
     * otherwise byte-identical staging/fetch sequence. Robust bindless image
     * lowering can synthesize the same semantic uint level in a 16-bit
     * register, so make that required widening explicit before RA. */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+        b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
        lod_mode == AGX_LOD_MODE_LOD_MIN && lod.size == AGX_SIZE_16 &&
        lod.type != AGX_INDEX_IMMEDIATE) {
       agx_index lod32 = agx_temp(b->shader, AGX_SIZE_32);
@@ -2358,7 +2549,7 @@ agx_emit_image_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *intr)
     * base. Drop that bookkeeping operand before CProg input import, so it
     * does not become an unused register operand or an extra TPS load.
     */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
        texture.size == AGX_SIZE_64)
       bindless = agx_zero();
 
@@ -2369,6 +2560,19 @@ agx_emit_image_load(agx_builder *b, agx_index dst, nir_intrinsic_instr *intr)
                                     agx_immediate(0), agx_null(),
                                     agx_tex_dim(dim, is_array), lod_mode, 0,
                                     false, sparse, nir_is_coherent(intr));
+   /* ACCESS_CAN_REORDER (nir_opt_access: the image is NonWritable, so
+    * nothing in this shader writes it) and ACCESS_RESTRICT (vtn: an image
+    * not decorated Aliased, which SPIR-V's Simple/GLSL/Vulkan memory models
+    * let the consumer assume does not alias another memory object
+    * declaration) together mean no store of this shader can write what the
+    * load reads, so the M5 copy schedule may move it across them. M5 encodes
+    * the read as a native SMP, which Apple's compiler issues ahead of earlier
+    * image writes (native725 vs_copy). */
+   const enum gl_access_qualifier access = nir_intrinsic_access(intr);
+   I->memory_can_reorder =
+      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+      (access & ACCESS_CAN_REORDER) && (access & ACCESS_RESTRICT) &&
+      !nir_is_coherent(intr);
    I->mask = agx_expand_tex_to(b, &intr->def, tmp, !sparse);
 
    b->shader->out->uses_txf = true;
@@ -2475,7 +2679,17 @@ agx_emit_image_store(agx_builder *b, nir_intrinsic_instr *instr)
 
    agx_instr *write = agx_image_write(b, data, coords, lod, base, index, dim,
                                       nir_is_coherent(instr));
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+   /* The selector below is native G17P's witnessed imgwci f70 rule. The
+    * AGX3C packer encodes its float/UNORM and I32 forms for 1D/2D stores
+    * only; 3D, array and cube stores keep the integer format it encodes for
+    * every dimension. Selecting UNORM for those made every float 3D, array
+    * and cube store unencodable on M5 (image.format_reinterpret.3d,
+    * geometry.layered secondary_cmd_buffer, image.store.without_format:
+    * Pass on the pre-widening build, packer refusals after it), while 2D
+    * stores pass with it (glsl.builtin_var.fragcoord_msaa*). */
+   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+       (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+        (dim == AGX_DIM_1D || dim == AGX_DIM_2D))) {
       /* imgwci.f70 follows the TYPE OF THE DATA the shader stores, not the
        * destination surface's format. Metal never puts a pixel format in a
        * shader -- a texture2d<float, access::write> is written the same way
@@ -2574,6 +2788,29 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
    mesa_shader_stage stage = b->shader->stage;
 
    switch (instr->intrinsic) {
+   case nir_intrinsic_cmat_muladd_agx: {
+      unsigned type = nir_intrinsic_src_type(instr) == nir_type_uint16
+         ? AGX3_MATRIX_INPUT_BF16 : nir_intrinsic_src_type(instr) == nir_type_float32
+         ? AGX3_MATRIX_INPUT_F32 : AGX3_MATRIX_INPUT_F16;
+      unsigned type_b = nir_intrinsic_src_type2(instr) == nir_type_uint16
+         ? AGX3_MATRIX_INPUT_BF16 : nir_intrinsic_src_type2(instr) == nir_type_float32
+         ? AGX3_MATRIX_INPUT_F32 : AGX3_MATRIX_INPUT_F16;
+      type |= type_b << 4;
+      assert(b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+             stage == MESA_SHADER_COMPUTE && !b->shader->is_preamble);
+      if (instr->def.num_components == 8) {
+         agx_matmad16_agx3_to(b, dst, agx_src_index(&instr->src[0]),
+                            agx_src_index(&instr->src[1]),
+                            agx_src_index(&instr->src[2]), type);
+      } else {
+         assert(instr->def.num_components == 2);
+         agx_matmad8_agx3_to(b, dst, agx_src_index(&instr->src[0]),
+                            agx_src_index(&instr->src[1]),
+                            agx_src_index(&instr->src[2]), type);
+      }
+      agx_emit_cached_split(b, dst, instr->def.num_components);
+      return NULL;
+   }
    case nir_intrinsic_load_barycentric_pixel:
    case nir_intrinsic_load_barycentric_centroid:
    case nir_intrinsic_load_barycentric_at_sample:
@@ -2627,6 +2864,15 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
       assert(stage == MESA_SHADER_FRAGMENT);
       return agx_emit_store_zs(b, instr);
 
+   case nir_intrinsic_zs_transaction_agx:
+      assert(stage == MESA_SHADER_FRAGMENT);
+      if (nir_intrinsic_flags(instr) == 0) {
+         b->shader->out->writes_sample_mask = true;
+         agx_wait_pixel_mask(b, 0x1);
+         return NULL;
+      }
+      return agx_signal_pix(b, 0x1);
+
    case nir_intrinsic_store_local_pixel_agx:
       return agx_emit_local_store_pixel(b, instr);
 
@@ -2660,7 +2906,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
    case nir_intrinsic_load_frag_coord_z: {
       agx_index cf = agx_get_cf(b->shader, VARYING_SLOT_POS, 2);
 
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2)
          return agx_interp_agx3_to(b, dst, cf, agx_zero(),
                                      AGX_INTERPOLATION_CENTER, false);
 
@@ -2673,7 +2919,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
       /* Native position.w uses ordinary interpolation (f54=0). The
        * projective RHW form is private to reciprocal interpolation and
        * cannot be reused as a public arithmetic value. */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2)
          return agx_interp_agx3_to(b, dst, cf, agx_zero(),
                                      AGX_INTERPOLATION_CENTER, false);
 
@@ -2682,6 +2928,11 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
 
    case nir_intrinsic_sample_mask_agx: {
       assert(stage == MESA_SHADER_FRAGMENT);
+      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          b->shader->key->fs.nr_samples > 1 &&
+          nir_src_is_const(instr->src[0]) &&
+          !(nir_src_as_uint(instr->src[0]) & 255))
+         return NULL;
       b->shader->out->writes_sample_mask = true;
 
       /* We need to wait_pix before running Z/S tests, but we don't need to
@@ -2700,7 +2951,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
            !(nir_src_as_uint(instr->src[0]) &
              nir_src_as_uint(instr->src[1]))));
 
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
           !b->shader->key->fs.inside_sample_loop) {
          /* The all/all update inserted at the start of an explicit-early
           * shader is a distinct native protocol: acquire class 1 before
@@ -2715,8 +2966,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          if (force_early)
             agx_wait_pixel_mask(b, 0x1);
 
-         /* MOVMSK takes one coverage word: the samples that stay alive in
-          * nibble 0, above the pixel's coverage and API input mask. Native
+         /* MOVMSK takes one staged coverage word. Native
           * early-fragment-tests code uses word zero for a no-op. Keep TARGET
           * dynamic and select that word without changing EXEC around a
           * coverage operation. */
@@ -2734,6 +2984,122 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          agx_index live_in = agx_src_index(&instr->src[1]);
          agx_index target, keep, mid;
 
+         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+             samples > 1) {
+            /* Native G17G 4x/8x mask0 publishes the killed byte directly.
+             * Mask1/all/variable retain the complete W16 SR39 word in the
+             * low half and publish the live byte at bits 24..31. The native
+             * AND255 + ANDC255 + ADD reconstructs SR39, not two nibbles.
+             * The byte ABI is independent of the raster sample count. */
+            const bool constant_target = nir_src_is_const(instr->src[0]);
+            const unsigned target_value = constant_target
+               ? nir_src_as_uint(instr->src[0]) & 255 : 0;
+            agx_index affected = constant_target
+               ? agx_mov_imm(b, 16, target_value)
+               : agx_and(b, target_in, agx_immediate(255));
+            agx_index selected = agx_temp(b->shader, AGX_SIZE_32);
+            if (no_tests) {
+               agx_mov_to(b, selected, affected);
+            } else {
+               agx_index live;
+               if (constant_target && target_value == 255) {
+                  live = agx_and(b, live_in, agx_immediate(255));
+               } else {
+                  agx_index untargeted = constant_target
+                     ? agx_immediate(target_value ^ 255)
+                     : agx_xor(b, affected, agx_immediate(255));
+                  live = agx_and(b, agx_or(b, untargeted, live_in),
+                                 agx_immediate(255));
+               }
+               agx_index coverage = agx_temp(b->shader, AGX_SIZE_16);
+               agx_get_sr_coverage_to(b, coverage, AGX_SR_INPUT_SAMPLE_MASK);
+               agx_index high;
+               if (constant_target && nir_src_is_const(instr->src[1])) {
+                  const unsigned live_value =
+                     ((target_value ^ 255) | nir_src_as_uint(instr->src[1])) & 255;
+                  high = agx_mov_imm(b, 16, live_value << 8);
+               } else {
+                  high = agx_iadd(b, agx_zero(),
+                     agx_iadd(b, agx_zero(), live, 4), 4);
+               }
+               agx_index word = agx_temp(b->shader, AGX_SIZE_32);
+               agx_emit_collect_to(b, word, 2,
+                                  (agx_index[]){coverage, high});
+               if (!(constant_target && nir_src_is_const(instr->src[1]))) {
+                  /* The testing word carries the whole SR39 coverage, so it
+                   * tests every covered sample, and naming a targeted sample
+                   * dead in its live byte does not keep that sample from
+                   * being tested and written. A per-sample alpha-to-coverage
+                   * epilog at alpha 0 wrote depth on M5
+                   * (pipeline.*.multisample.alpha_to_coverage.samples_N.
+                   * alpha_invisible_check_depth) once the per-lane split no
+                   * longer routed such updates to its kill arm. When no
+                   * targeted sample is live, publish the killed byte without
+                   * testing instead, as that arm and a constant kill do. */
+                  agx_index dead = agx_temp(b->shader, AGX_SIZE_32);
+                  agx_mov_to(b, dead, agx_and(b, live_in, affected));
+                  agx_index killed = agx_temp(b->shader, AGX_SIZE_32);
+                  agx_mov_to(b, killed, affected);
+                  agx_index tested = agx_temp(b->shader, AGX_SIZE_32);
+                  agx_icmpsel_to(b, tested, dead, agx_zero(), killed, word,
+                                 AGX_ICOND_UEQ);
+                  word = tested;
+               }
+               if (constant_target) {
+                  agx_mov_to(b, selected, word);
+               } else {
+                  agx_index gate = agx_temp(b->shader, AGX_SIZE_32);
+                  agx_mov_to(b, gate, affected);
+                  agx_icmpsel_to(b, selected, gate, agx_zero(), word,
+                                 agx_zero(), AGX_ICOND_UGT);
+               }
+            }
+            return agx_sample_mask_agx3(b, affected, selected,
+               (!no_tests ? 2 : 0) | (force_early ? 1 : 0));
+         }
+
+         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+             samples == 1) {
+            /* G17G MOVMSK is not the G17P nibble layout. Native mask0
+             * publishes its kill bits directly. Mask1 and a variable mask
+             * duplicate SR2 into the low two bytes and publish the live mask
+             * in byte3 (metal-mask209); DEPTHF uses the same byte layout. */
+            /* agx_m5_discard_word() gives LIVE as the high half (bit 0
+             * at bit 8) beside a constant target naming the sample. */
+            const bool high_live =
+               nir_intrinsic_flags(instr) & BITFIELD_BIT(2);
+            const bool whole_target = nir_src_is_const(instr->src[0]) &&
+               (nir_src_as_uint(instr->src[0]) & 1);
+            agx_index affected = whole_target
+               ? agx_mov_imm(b, 16, 1)
+               : agx_and(b, target_in, agx_immediate(1));
+            agx_index selected = agx_temp(b->shader, AGX_SIZE_32);
+            if (no_tests) {
+               agx_mov_to(b, selected, affected);
+            } else {
+               agx_index sr2 = agx_temp(b->shader, AGX_SIZE_16);
+               agx_get_coverage_1x_agx3_to(b, sr2);
+               agx_index repeated = agx_iadd(b, sr2,
+                  agx_iadd(b, agx_zero(), sr2, 4), 4);
+               agx_index high = high_live ? live_in : agx_iadd(b, agx_zero(),
+                  agx_iadd(b, agx_zero(),
+                           agx_and(b, live_in, agx_immediate(1)), 4), 4);
+               agx_index word = agx_temp(b->shader, AGX_SIZE_32);
+               agx_emit_collect_to(b, word, 2, (agx_index[]){repeated, high});
+               if (whole_target) {
+                  /* The selector below is constant: it picks the word. */
+                  agx_mov_to(b, selected, word);
+               } else {
+                  agx_index gate = agx_temp(b->shader, AGX_SIZE_32);
+                  agx_mov_to(b, gate, affected);
+                  agx_icmpsel_to(b, selected, gate, agx_zero(), word,
+                                 agx_zero(), AGX_ICOND_UGT);
+               }
+            }
+            return agx_sample_mask_agx3(b, affected, selected,
+               (!no_tests ? 2 : 0) | (force_early ? 1 : 0));
+         }
+
          /* agx_g17p_discard_word() reduced these sources to bit 0 already
           * (FLAGS) at one sample, and the select below is one widening
           * 16x16->32 multiply. */
@@ -2806,27 +3172,31 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
       return agx_get_sr_to(b, dst, AGX_SR_SAMPLES_LOG2);
 
    case nir_intrinsic_load_sample_mask_in:
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2) {
          /* Apple's G17P fragment ABI exposes the API-visible sample mask in
           * bits 4..7 of native W16 SR39.  Extract it before widening to NIR's
           * uint32_t result; reading SR39 as W32 selects an unwitnessed form. */
          agx_index native = agx_temp(b->shader, AGX_SIZE_16);
          agx_index shifted = agx_temp(b->shader, AGX_SIZE_16);
          agx_get_sr_to(b, native, AGX_SR_INPUT_SAMPLE_MASK);
-         agx_ushr_to(b, shifted, native, agx_immediate(4));
+         /* G17G exposes the API mask in the high byte (native input-mask
+          * witness); G17P exposes it in the high nibble. */
+         agx_ushr_to(b, shifted, native, agx_immediate(
+            b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C ? 8 : 4));
          return agx_mov_to(b, dst, shifted);
       }
       return agx_get_sr_to(b, dst, AGX_SR_INPUT_SAMPLE_MASK);
 
    case nir_intrinsic_load_sample_mask:
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2) {
          /* G17P's covered samples are bits 0..3 of native W16 SR39, the word
           * Apple's multisample fragments AND with each sample's bit before
           * that sample's tile store. Unlike the API input mask, this changes
           * after depth/stencil tests and cannot be CSE'd across them. */
          agx_index native = agx_temp(b->shader, AGX_SIZE_16);
          agx_get_sr_coverage_to(b, native, AGX_SR_INPUT_SAMPLE_MASK);
-         return agx_mov_to(b, dst, agx_and(b, native, agx_immediate(15)));
+         return agx_mov_to(b, dst, agx_and(b, native, agx_immediate(
+            b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C ? 255 : 15)));
       }
       return agx_get_sr_coverage_to(b, dst, AGX_SR_COVERAGE_MASK);
 
@@ -2839,7 +3209,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
        * the tile access. Bits 0..3 are the covered samples -- the per-sample
        * form ANDs this same word with 1/2/4/8. Use the ordered coverage read
        * so scheduling and CSE cannot reuse pre-depth coverage for tile stores. */
-      assert(b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      assert(b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
              "only the G17P tile lowering leaves this for the backend");
       assert(instr->def.bit_size == 16 && "16-bit sample mask");
       return agx_get_sr_coverage_to(b, dst, AGX_SR_INPUT_SAMPLE_MASK);
@@ -2956,7 +3326,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
        * API gl_Layer, and native SR 0xc3 (normalized SR12, W16) is what
        * supplies it:
        *
-       *  - the builtin background reload (fs.ignore_tib_dependencies), and
+       *  - the builtin background reload (fs.background), and
        *  - with a spilled render target, the fragment epilog's eMRT image
        *    store, whose layer coordinate image_coords() in
        *    agx_nir_lower_tilebuffer.c builds from this intrinsic. Reading the
@@ -2969,9 +3339,10 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
        * first (hk_shader.c) -- but it does from gallium, which lowers
        * VARYING_SLOT_LAYER back to this intrinsic with layer_as_varying
        * false and sets neither key bit. Leave that path alone. */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
           stage == MESA_SHADER_FRAGMENT &&
-          (b->shader->key->fs.ignore_tib_dependencies ||
+          (b->shader->key->fs.background ||
+           b->shader->key->fs.ignore_tib_dependencies ||
            b->shader->key->g17p_emrt))
          return agx_get_sr_native_width_to(b, dst, AGX_SR_BACKGROUND_LAYER_ID);
       return agx_get_sr_to(b, dst, AGX_SR_THREADGROUP_POSITION_IN_GRID_Z);
@@ -2985,9 +3356,18 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          if (nir_intrinsic_memory_scope(instr) != SCOPE_NONE) {
             const nir_variable_mode modes =
                nir_intrinsic_memory_modes(instr);
+            const bool memory_only =
+               nir_intrinsic_execution_scope(instr) < SCOPE_WORKGROUP;
             const bool memory_only_g17p =
                b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-               nir_intrinsic_execution_scope(instr) < SCOPE_WORKGROUP;
+               memory_only;
+            /* G17G: atomic_thread_fence(mem_texture) is the threadgroup
+             * fence pair at threadgroup scope and texture.fence()'s pair at
+             * device and simdgroup scope; threadgroup_barrier(mem_texture)
+             * is the rendezvous pair (port/fencewit). */
+            const bool memory_only_m5 =
+               b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+               memory_only;
 
             /* The AGX2 cache-operation train collapses on AGX3 into one
              * exact operation per semantic memory scope. G17P distinguishes
@@ -3003,8 +3383,9 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
             if ((modes & (nir_var_mem_global | nir_var_image)) ||
                 tcs_outputs) {
                agx_scoped_barrier_agx3(
-                  b, memory_only_g17p ? AGX3_BARRIER_SCOPE_DEVICE_FENCE
-                                     : AGX3_BARRIER_SCOPE_DEVICE);
+                  b, nir_intrinsic_execution_scope(instr) < SCOPE_WORKGROUP
+                        ? AGX3_BARRIER_SCOPE_DEVICE_FENCE
+                        : AGX3_BARRIER_SCOPE_DEVICE);
                emitted = true;
             }
 
@@ -3017,6 +3398,10 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
                         ? (wait_for_prior_texture
                               ? AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT
                               : AGX3_BARRIER_SCOPE_TEXTURE_FENCE)
+                     : memory_only_m5
+                        ? (nir_intrinsic_memory_scope(instr) == SCOPE_WORKGROUP
+                              ? AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP
+                              : AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT)
                         : AGX3_BARRIER_SCOPE_TEXTURE);
                emitted = true;
             }
@@ -3104,10 +3489,8 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
 
    case nir_intrinsic_fence_pbe_to_tex_agx: {
       if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2) {
-         agx_scoped_barrier_agx3(
-            b, b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B
-                  ? AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT
-                  : AGX3_BARRIER_SCOPE_TEXTURE);
+         /* texture.fence(): on AGX3C it waits on the writes' own slots. */
+         agx_scoped_barrier_agx3(b, AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT);
          return NULL;
       }
 
@@ -3187,6 +3570,10 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
       return agx_get_sr_native_width_to(b, dst,
                                         AGX_SR_THREAD_INDEX_IN_SUBGROUP);
 
+   case nir_intrinsic_object_payload_exit_barrier_agx:
+      return agx_scoped_barrier_agx3(b,AGX3_BARRIER_SCOPE_OBJECT_PAYLOAD_EXIT);
+   case nir_intrinsic_mesh_payload_entry_barrier_agx:
+      return agx_scoped_barrier_agx3(b,AGX3_BARRIER_SCOPE_MESH_PAYLOAD_ENTRY);
    case nir_intrinsic_load_mesh_payload_address_agx: {
       agx_index parts[2]={agx_temp(b->shader,AGX_SIZE_32),
                           agx_temp(b->shader,AGX_SIZE_32)};
@@ -3264,7 +3651,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
        * program we must reject, so materialise it into the register-lane
        * form, which takes any value, instead of refusing the shader.
        */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
           lane.type == AGX_INDEX_IMMEDIATE && lane.value > 0x7f) {
          agx_index reg = agx_temp(b->shader, AGX_SIZE_32);
          agx_mov_imm_to(b, reg, lane.value);
@@ -3375,7 +3762,7 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          agx_src_index(&instr->src[1]), nir_intrinsic_base(instr),
          instr->def.bit_size == 16 ? AGX_FORMAT_I16 : AGX_FORMAT_I32,
          BITFIELD_MASK(instr->def.num_components),
-         nir_intrinsic_component(instr), nir_intrinsic_flags(instr) & 1);
+         nir_intrinsic_component(instr), nir_intrinsic_flags(instr) & 1, false);
       agx_emit_cached_split(b, dst, instr->def.num_components);
       return load;
    }
@@ -3387,6 +3774,16 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          AGX_FORMAT_I32, BITFIELD_MASK(instr->num_components),
          nir_intrinsic_component(instr));
 
+   case nir_intrinsic_load_scs_register_agx: {
+      agx_instr *load = agx_scs_load_register_indexed_agx3_to(
+         b, dst, agx_src_index(&instr->src[0]), agx_src_index(&instr->src[1]),
+         nir_intrinsic_base(instr), agx_format_for_pipe(nir_intrinsic_format(instr)),
+         BITFIELD_MASK(instr->def.num_components), nir_intrinsic_component(instr),
+         !!(nir_intrinsic_flags(instr) & 1), !!(nir_intrinsic_flags(instr) & 8));
+      agx_emit_cached_split(b, dst, instr->def.num_components);
+      return load;
+   }
+
    case nir_intrinsic_load_scs_uniform_agx:
    case nir_intrinsic_load_scs_base_agx: {
       const bool uniform = instr->intrinsic == nir_intrinsic_load_scs_uniform_agx;
@@ -3400,7 +3797,8 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          agx_scs_load_register_base_agx3_to(
          b, dst, agx_src_index(&instr->src[0]), nir_intrinsic_base(instr),
          fmt, mask, nir_intrinsic_component(instr), !!(flags & 1), !!(flags & 4));
-      load->scs_rt_wait = !!(flags & 8);
+      load->scs_rt_wait = b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B && (flags & 8);
+      load->scs_rt_completion = b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C && !uniform && (flags & 8);
       agx_emit_cached_split(b, dst, instr->def.num_components);
       return load;
    }
@@ -3419,7 +3817,19 @@ agx_emit_intrinsic(agx_builder *b, nir_intrinsic_instr *instr)
          nir_intrinsic_base(instr), fmt, mask, nir_intrinsic_component(instr));
    }
 
+   case nir_intrinsic_intersect_ray_agx:
+      return agx_intersect_ray_m5(b, agx_src_index(&instr->src[0]),
+                                agx_src_index(&instr->src[1]),
+                                agx_src_index(&instr->src[2]),
+                                agx_src_index(&instr->src[3]),
+                                agx_src_index(&instr->src[4]));
+
+   case nir_intrinsic_enter_query_agx:
+      return agx_enter_query_m5_to(b, dst,
+                                   nir_intrinsic_base(instr) ?: 4);
 
+   case nir_intrinsic_continue_ray_agx:
+      return agx_continue_ray_m5(b, agx_src_index(&instr->src[0]));
 
    case nir_intrinsic_doorbell_agx: {
       return agx_doorbell(b, nir_src_as_uint(instr->src[0]));
@@ -3603,7 +4013,7 @@ agx_fminmax_to(agx_builder *b, agx_index dst, agx_index s0, agx_index s1,
 static void
 agx_lower_agx3_half_converts(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_instr_global_safe(ctx, I) {
@@ -3612,6 +4022,12 @@ agx_lower_agx3_half_converts(agx_context *ctx)
           I->src[0].type != AGX_INDEX_IMMEDIATE || I->saturate)
          continue;
 
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->src[0].value == AGX_CONVERT_S8_TO_F &&
+          I->round == AGX_ROUND_RTE && I->dest[0].size == AGX_SIZE_16 &&
+          !I->src[1].abs && !I->src[1].neg && !I->src[1].memory)
+         continue;
+
       if (I->src[0].value == AGX_CONVERT_U8_TO_F &&
           I->round == AGX_ROUND_RTE) {
          /* Byte integers occupy a half register; discard the other byte
@@ -3703,14 +4119,170 @@ agx_lower_agx3_half_converts(agx_context *ctx)
    }
 }
 
+/* A modified comparand and the plain value materializing it */
+struct agx_m5_comparand {
+   agx_index source, plain;
+};
+
+static agx_index
+agx_m5_find_comparand(const struct util_dynarray *comparands, agx_index src)
+{
+   util_dynarray_foreach(comparands, struct agx_m5_comparand, c) {
+      if (agx_is_equiv(c->source, src) && c->source.size == src.size &&
+          c->source.abs == src.abs && c->source.neg == src.neg)
+         return c->plain;
+   }
+
+   return agx_null();
+}
+
 /* A fragment shader that reads a varying array at a run-time index computes
  * its coefficient index as dest16 = imm + (index << 2), because the ITERPROJ
  * coefficient operand is sixteen bits wide. AGX3B's shifted add writes a
  * 32-bit register, so widen the destination and narrow the exact low half. */
+/* Form a full-width product from narrow indices with two shifted adds.
+ * Keep the extension modifiers on their original operands. */
+static void
+agx_lower_m5_alu_operands(agx_context *ctx)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C)
+      return;
+
+   /* Comparands materialized in the current block. Shaders often compare one
+    * |x| against several bounds, and no CSE runs after this pass for M5
+    * fragment shaders, so reuse the first move rather than repeat it.
+    */
+   struct util_dynarray comparands = UTIL_DYNARRAY_INIT;
+   agx_block *comparand_block = NULL;
+
+   agx_foreach_instr_global_safe(ctx, I) {
+      /* M5 has no grounded compare-select whose compared operands carry |x|
+       * or -x: the AGX3C packer refuses every such FCMPSEL form, and FCMP and
+       * BREAK_IF_FCMP become FCMPSEL after register allocation. Materialize a
+       * modified register/uniform comparand as its own exact move here, after
+       * the optimizer has folded what it can, as the agx3c series did. */
+      if (I->op == AGX_OPCODE_FCMP || I->op == AGX_OPCODE_FCMPSEL ||
+          I->op == AGX_OPCODE_BREAK_IF_FCMP) {
+         if (comparand_block != v_block) {
+            util_dynarray_clear(&comparands);
+            comparand_block = v_block;
+         }
+
+         for (unsigned s = 0; s < 2 && s < I->nr_srcs; ++s) {
+            const agx_index src = I->src[s];
+            if (src.type == AGX_INDEX_IMMEDIATE || !(src.abs || src.neg))
+               continue;
+            agx_index plain = agx_m5_find_comparand(&comparands, src);
+            if (agx_is_null(plain)) {
+               agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+               plain = agx_temp(ctx, src.size);
+               agx_fmov_to(&b, plain, src);
+               util_dynarray_append(
+                  &comparands,
+                  ((struct agx_m5_comparand){.source = src, .plain = plain}));
+            }
+            I->src[s] = plain;
+         }
+      }
+
+      /* A zero-extended low half contributes no bits to a funnel shift
+       * by 16 or more. Keep the remaining high-word shift in native form. */
+      if (I->op == AGX_OPCODE_EXTR && I->nr_dests == 1 && I->nr_srcs == 3 &&
+          I->bfi_mask == 0 && !I->saturate &&
+          I->dest[0].size == AGX_SIZE_32 &&
+          I->src[0].size == AGX_SIZE_16 &&
+          !I->src[0].abs && !I->src[0].neg &&
+          I->src[1].size == AGX_SIZE_32 &&
+          !I->src[1].abs && !I->src[1].neg &&
+          I->src[2].type == AGX_INDEX_IMMEDIATE &&
+          !I->src[2].abs && !I->src[2].neg &&
+          I->src[2].value >= 16 && I->src[2].value <= 31) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         agx_bfi_to(&b, I->dest[0], agx_zero(), I->src[1],
+                    agx_immediate(32 - I->src[2].value), 0);
+         agx_remove_instruction(I);
+         continue;
+      }
+
+      /* Put a multiplicand's sign into an exact minifloat multiplier. This
+       * retains one FMA and avoids materializing a separate negation. */
+      if (I->op == AGX_OPCODE_FFMA && I->nr_srcs == 3) {
+         const unsigned immediate = I->src[0].type == AGX_INDEX_IMMEDIATE ? 0 : 1;
+         agx_index *reg = &I->src[1 - immediate];
+         agx_index *imm = &I->src[immediate];
+         if (imm->type == AGX_INDEX_IMMEDIATE && imm->value <= UINT8_MAX &&
+             !imm->abs && !imm->neg && reg->neg &&
+             reg->type != AGX_INDEX_IMMEDIATE) {
+            imm->value ^= 0x80;
+            reg->neg = false;
+         }
+      }
+
+      if (I->op == AGX_OPCODE_IADD && I->shift && !I->saturate &&
+          I->dest[0].size == AGX_SIZE_32 && I->src[1].size == AGX_SIZE_32 &&
+          I->src[1].neg && !I->src[1].abs) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         agx_index source = I->src[1];
+         source.neg = false;
+         agx_index shifted = agx_temp(ctx, AGX_SIZE_32);
+         agx_bfi_to(&b, shifted, agx_zero(), source, agx_immediate(I->shift), 0);
+         I->src[1] = agx_neg(shifted);
+         I->shift = 0;
+         continue;
+      }
+      if (I->op == AGX_OPCODE_FFS && I->src[0].size == AGX_SIZE_16) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         I->src[0] = agx_pad_to_32(&b, I->src[0]);
+         continue;
+      }
+
+      if (I->op != AGX_OPCODE_IMAD || I->nr_dests != 1 || I->nr_srcs != 3 ||
+          I->saturate || I->shift || I->dest[0].size != AGX_SIZE_32)
+         continue;
+      /* Metal537 lowers a*b-c to a product and native ISUB. M5's IMADD
+       * has no subtract-addend form, so preserve the exact modulo result
+       * with those two native operations before register allocation. */
+      if (I->src[2].neg) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         agx_index addend = I->src[2];
+         agx_index product = agx_temp(ctx, I->dest[0].size);
+         I->src[2] = agx_zero();
+         agx_index dest = I->dest[0];
+         I->dest[0] = product;
+         b.cursor = agx_after_instr(I);
+         agx_iadd_to(&b, dest, product, addend, 0);
+         continue;
+      }
+      /* Native mixed-width multiply-add takes its addend from a register.
+       * Materialize a nonzero literal before register allocation. */
+      if (I->src[2].type == AGX_INDEX_IMMEDIATE && I->src[2].value &&
+          (I->src[0].size == AGX_SIZE_16 || I->src[1].size == AGX_SIZE_16)) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         agx_index addend = agx_temp(ctx, AGX_SIZE_32);
+         agx_mov_imm_to(&b, addend, I->src[2].value);
+         addend.neg = I->src[2].neg;
+         I->src[2] = addend;
+      }
+      const bool swap = I->src[0].type == AGX_INDEX_IMMEDIATE;
+      const agx_index a = I->src[swap ? 1 : 0], m = I->src[swap ? 0 : 1];
+      if (a.size != AGX_SIZE_16 || a.neg || I->src[2].neg ||
+          m.type != AGX_INDEX_IMMEDIATE || m.abs || m.neg ||
+          (m.value != 3 && m.value != 6))
+         continue;
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+      agx_index product = agx_temp(ctx, AGX_SIZE_32);
+      agx_iadd_to(&b, product, a, a, 1);
+      agx_iadd_to(&b, I->dest[0], I->src[2], product, m.value == 6 ? 1 : 0);
+      agx_remove_instruction(I);
+   }
+
+   util_dynarray_fini(&comparands);
+}
+
 static void
 agx_lower_agx3_shifted_iadd_dest(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_instr_global_safe(ctx, I) {
@@ -3751,7 +4323,7 @@ agx_lower_agx3_shifted_iadd_dest(agx_context *ctx)
 void
 agx_lower_agx3_mixed_width_unary(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_instr_global_safe(ctx, I) {
@@ -3804,10 +4376,64 @@ agx_lower_agx3_mixed_width_unary(agx_context *ctx)
       }
    }
 }
+
+/* M5 legalization widens a half operand with a float move, fadd(x, -0.0),
+ * where an f32 operation cannot read it. agx_opt_compact_constants made
+ * such an operand a half constant (1/256 in Minecraft's terrain vertex
+ * shader), so the vertex shader materialized the half and then moved it to
+ * f32 twice for every vertex. A float move of a known finite, normal
+ * constant is that constant exactly (f16 -> f32 is exact; x + -0.0 == x
+ * for every non-NaN x, including +-0), so materialize the f32 bits once
+ * instead. NaNs, infinities and f32 denormals keep their moves. */
+static void
+agx3_fold_constant_fmovs(agx_context *ctx)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C)
+      return;
+
+   agx_instr **defs = calloc(ctx->alloc, sizeof(*defs));
+   if (!defs)
+      return;
+
+   agx_foreach_instr_global_safe(ctx, I) {
+      agx_foreach_ssa_dest(I, d)
+         defs[I->dest[d].value] = I;
+
+      if (I->op != AGX_OPCODE_FADD || I->nr_dests != 1 || I->saturate ||
+          I->dest[0].size != AGX_SIZE_32 || agx_channels(I->dest[0]) != 1 ||
+          I->src[0].type != AGX_INDEX_NORMAL || I->src[0].abs ||
+          I->src[0].neg || I->src[0].memory ||
+          !agx_is_equiv(I->src[1], agx_negzero()) || I->src[1].abs ||
+          I->src[1].neg)
+         continue;
+
+      const agx_instr *def = defs[I->src[0].value];
+      if (!def || def->op != AGX_OPCODE_MOV_IMM || def->nr_dests != 1 ||
+          agx_channels(def->dest[0]) != 1 ||
+          (def->dest[0].size != AGX_SIZE_16 &&
+           def->dest[0].size != AGX_SIZE_32))
+         continue;
+
+      const float value = def->dest[0].size == AGX_SIZE_16
+                             ? _mesa_half_to_float((uint16_t)def->imm)
+                             : uif((uint32_t)def->imm);
+      const int cls = fpclassify(value);
+      if (cls != FP_NORMAL && cls != FP_ZERO)
+         continue;
+
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+      agx_instr *mov = agx_mov_imm_to(&b, I->dest[0], fui(value));
+      defs[I->dest[0].value] = mov;
+      agx_remove_instruction(I);
+   }
+
+   free(defs);
+}
+
 void
 agx_lower_agx3_fma_sources(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_instr_global_safe(ctx, I) {
@@ -3952,7 +4578,7 @@ agx_lower_agx3_fma_sources(agx_context *ctx)
 static bool
 agx_lower_agx3_cf_constants(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return false;
 
    bool progress = false;
@@ -4078,7 +4704,7 @@ agx_lower_agx3_cf_constants(agx_context *ctx)
 static bool
 agx_lower_agx3_half_iadd_constants(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return false;
 
    bool progress = false;
@@ -4118,7 +4744,7 @@ agx_lower_agx3_half_iadd_constants(agx_context *ctx)
 static bool
 agx_lower_agx3_widening_int(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return false;
 
    bool progress = false;
@@ -4157,7 +4783,7 @@ agx_lower_agx3_widening_int(agx_context *ctx)
 static bool
 agx3_lower_large_fetch_indices(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return false;
    bool progress = false;
    agx_foreach_instr_global_safe(ctx, I) {
@@ -4188,7 +4814,7 @@ agx3_lower_large_fetch_indices(agx_context *ctx)
 static bool
 agx_lower_agx3_wide_iadd(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return false;
 
    bool progress = false;
@@ -4237,7 +4863,7 @@ agx_lower_agx3_wide_iadd(agx_context *ctx)
 static bool
 agx_agx3_bitop_immediate_ok(const agx_context *ctx, const agx_instr *I)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        I->op != AGX_OPCODE_BITOP || I->nr_dests != 1 || I->nr_srcs != 2 ||
        I->saturate || agx_channels(I->dest[0]) != 1 ||
        (I->dest[0].size != AGX_SIZE_16 && I->dest[0].size != AGX_SIZE_32))
@@ -4320,15 +4946,56 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
          }
       }
 
-      /* G17P's grounded 32-bit saturating IADD encoding takes two registers.
+      /* XNOR with a constant is XOR with the complemented constant. Native
+       * G17G uses the same identity for ushort masks, sourcing the resulting
+       * full-width mask from its constant program. Materialize before RA so
+       * the XOR's register operand and lifetime are allocated normally. */
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_BITOP && I->nr_dests == 1 && I->nr_srcs == 2 &&
+          !I->saturate && I->dest[0].size == AGX_SIZE_16 &&
+          (I->truth_table & 0xf) == 9) {
+         for (unsigned i = 0; i < 2; ++i) {
+            const agx_index imm = I->src[i];
+            if (imm.type != AGX_INDEX_IMMEDIATE || imm.abs || imm.neg ||
+                imm.memory)
+               continue;
+            I->src[i] = agx_mov_imm(&b, 16, (~imm.value) & 0xffff);
+            I->truth_table = 6; /* XOR */
+            break;
+         }
+      }
+
+      /* A 32-bit FADD of a half and an immediate has no single AGX3C form.
+       * G17G converts first (f32add of the half and -0.0) and then adds in
+       * f32: saturate(float(h)) is f32add, f32add.sat (port/hfwit k_sat_h2f,
+       * k_h2f_mul). Widen the half the same exact way. */
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_FADD && I->nr_dests == 1 && I->nr_srcs == 2 &&
+          I->dest[0].size == AGX_SIZE_32 &&
+          (I->src[0].type == AGX_INDEX_IMMEDIATE ||
+           I->src[1].type == AGX_INDEX_IMMEDIATE)) {
+         for (unsigned s = 0; s < 2; ++s) {
+            if (I->src[s].type == AGX_INDEX_IMMEDIATE ||
+                I->src[s].type == AGX_INDEX_UNIFORM ||
+                I->src[s].size != AGX_SIZE_16)
+               continue;
+            agx_index wide = agx_temp(ctx, AGX_SIZE_32);
+            agx_fmov_to(&b, wide, I->src[s]);
+            I->src[s] = wide;
+         }
+      }
+
+      /* The shared 32-bit saturating IADD path takes two registers.
        * Robust buffer bounds checks naturally produce uadd_sat(offset, #size),
        * where an immediate is implicitly unsigned and therefore lacks the ABS
        * modifier carried by the register operand. Materialize the immediate
        * before RA and copy the operation's signedness onto the new register so
        * the existing register/register iadd_sat/uadd_sat encoding applies. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-          I->op == AGX_OPCODE_IADD && I->saturate && I->nr_dests == 1 &&
-          I->nr_srcs == 2 && I->dest[0].size == AGX_SIZE_32) {
+      if (I->op == AGX_OPCODE_IADD && I->saturate && I->nr_dests == 1 &&
+          I->nr_srcs == 2 && I->dest[0].size == AGX_SIZE_32 &&
+          (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+            (I->src[0].abs || I->src[1].abs)))) {
          for (unsigned s = 0; s < 2; ++s) {
             if (I->src[s].type != AGX_INDEX_IMMEDIATE)
                continue;
@@ -4348,7 +5015,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * packer refuse frexp/modf precision shaders.  fabs on an IEEE half is
        * exactly a sign-bit clear, so lower that exact move to the grounded
        * half BITOP path while the mask can still receive an SSA register. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FADD && I->nr_dests == 1 && I->nr_srcs == 2 &&
           I->dest[0].size == AGX_SIZE_16 && !I->saturate &&
           I->src[0].size == AGX_SIZE_16 && I->src[0].abs &&
@@ -4370,14 +5037,14 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * materializing abs/neg as sign-bit operations before RA instead of
        * refusing the post-RA instruction. */
       const bool half_fsel =
-         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(ctx->key->dev.encoding) &&
          I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
          (I->dest[0].size == AGX_SIZE_16 ||
           (I->dest[0].size == AGX_SIZE_32 &&
            I->src[0].size == AGX_SIZE_16 &&
            I->src[1].size == AGX_SIZE_16));
       const bool half_round =
-         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(ctx->key->dev.encoding) &&
          (I->op == AGX_OPCODE_FLOOR || I->op == AGX_OPCODE_CEIL) &&
          I->nr_srcs == 1 && I->dest[0].size == AGX_SIZE_16;
       /* The modified add form needs a 32-bit destination, so a half add of
@@ -4387,7 +5054,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * witnessed half form (agx3_alu_hadd_one_neg) and a plain x + #imm
        * does not. */
       const bool half_add_candidate =
-         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(ctx->key->dev.encoding) &&
          I->op == AGX_OPCODE_FADD && I->nr_srcs == 2 && !I->saturate &&
          I->dest[0].size == AGX_SIZE_16;
       const int add_imm = !half_add_candidate                       ? -1
@@ -4425,7 +5092,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * Keep half multiplication and its rounding, but use two registers
        * with the exact sign-bit operations shared by the half add/select. */
       const bool half_mul =
-         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(ctx->key->dev.encoding) &&
          I->op == AGX_OPCODE_HMUL && I->nr_srcs == 2 && !I->saturate &&
          I->dest[0].size == AGX_SIZE_16 &&
          ((I->src[0].type == AGX_INDEX_IMMEDIATE &&
@@ -4488,7 +5155,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * selects two full floats.  G17P's zero-comparison FSEL is grounded
        * only without source modifiers, so make fabs/fneg exact with integer
        * sign-bit operations while the modified comparator is still SSA. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_32) {
          const bool zero_first =
@@ -4533,7 +5200,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * modifier across result shapes. Materialize IEEE fneg exactly by
        * flipping the sign bit, using the already-grounded MOV-immediate and
        * BITOP XOR forms, before register allocation. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_32 && !I->invert_cond && !I->saturate &&
           I->fcond == AGX_FCOND_EQ && I->src[0].size == AGX_SIZE_32 &&
@@ -4553,7 +5220,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
          I->src[0] = flipped;
       }
 
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_BITREV && I->src[0].size == AGX_SIZE_16) {
          /* Copy propagation may narrow an otherwise word-wide reversal. */
          agx_index wide = agx_temp(ctx, AGX_SIZE_32);
@@ -4563,7 +5230,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
       /* ISEL accepts a register comparator and an immediate comparator,
        * including ICMP pseudos lowered to ISEL after register allocation.
        */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           (I->op == AGX_OPCODE_ICMP || I->op == AGX_OPCODE_ICMPSEL) &&
           I->src[0].type == AGX_INDEX_IMMEDIATE &&
           I->src[1].type == AGX_INDEX_IMMEDIATE &&
@@ -4573,7 +5240,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * extension. Materialize the minuend so the native register ISUB can
        * implement constant - value, including the negation in udiv refinement.
        * Shifted subtraction has already been lowered to a separate shift. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_IADD && I->nr_dests == 1 && I->nr_srcs == 2 &&
           !I->saturate && !I->shift && I->dest[0].size == AGX_SIZE_32 &&
           I->src[0].type == AGX_INDEX_IMMEDIATE && !I->src[0].abs &&
@@ -4597,6 +5264,22 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
          agx_remove_instruction(I);
          continue;
       }
+      /* AGX3C's tied min/max consumes full register operands in both
+       * positions. Keep constants tied after compact-constant folding. */
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
+          I->dest[0].size == AGX_SIZE_32 && !I->invert_cond &&
+          (I->fcond == AGX_FCOND_LTN || I->fcond == AGX_FCOND_GTN)) {
+         for (unsigned c = 0; c < 2; ++c) {
+            if (I->src[c].type != AGX_INDEX_IMMEDIATE ||
+                !agx_is_equiv(I->src[c], I->src[c + 2]))
+               continue;
+            const agx_index constant = agx_mov_imm(
+               &b, 32, fui(agx_minifloat_decode(I->src[c].value)));
+            I->src[c] = I->src[c + 2] = constant;
+         }
+      }
+
       /* General FSEL has register comparators or an implicit zero. Nonzero
        * minifloats in compare slots must first become floating registers;
        * selected values in slots 2/3 have integer bit-pattern semantics.
@@ -4604,7 +5287,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        */
       if (ctx->key->dev.chip != AGX_CHIP_G17P &&
           (I->op == AGX_OPCODE_FCMPSEL || I->op == AGX_OPCODE_FCMP)) {
-         if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
              I->op == AGX_OPCODE_FCMP) {
             /* Boolean FCMP becomes an immediate-alternative FSEL after this
              * pass; that form also needs full-width comparators. */
@@ -4617,7 +5300,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
                }
             }
          }
-         if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
              I->op == AGX_OPCODE_FCMPSEL && I->dest[0].size == AGX_SIZE_16) {
             /* Select alternatives are bit patterns, not minifloats -- except
              * in a tied minimum/maximum, where the optimizer deposited the
@@ -4645,7 +5328,9 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
          /* The native F32/F16 route is qualified for EQ/LT and two full
           * register alternatives. Other full-result shapes use the general
           * F32 comparator route. F16->F32 is exact, including signed zero;
-          * retain the condition, inversion, and selected bit patterns. */
+          * retain the condition, inversion, and selected bit patterns.
+          * AGX3B only: G17G selects 32-bit values on half comparands
+          * directly (port/rawit k_fcmpsel_mixed, k_fcmpsel_mixed2). */
          if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
              I->op == AGX_OPCODE_FCMPSEL && I->dest[0].size == AGX_SIZE_32) {
             const bool native_mixed =
@@ -4672,7 +5357,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
                }
             }
          }
-         if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
              I->op == AGX_OPCODE_FCMPSEL &&
              I->dest[0].size == AGX_SIZE_16 &&
              !(agx_is_equiv(I->src[0], I->src[2]) &&
@@ -4767,7 +5452,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * the sign and -x flips it, so an unconditional XOR is only correct
        * when a negate is actually present. Apply each modifier that is set,
        * and leave an unmodified move alone. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FADD && !I->saturate &&
           (I->src[0].abs || I->src[0].neg) &&
           ((I->dest[0].size == AGX_SIZE_16 &&
@@ -4834,7 +5519,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * exist; the resulting IF/WHILE/BREAK_IF_FCMP uses the already-grounded
        * register/register compare. Keep zero inline for the distinct compact
        * zero-comparison form. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           (I->op == AGX_OPCODE_IF_FCMP ||
            I->op == AGX_OPCODE_WHILE_FCMP ||
            I->op == AGX_OPCODE_BREAK_IF_FCMP) &&
@@ -4867,7 +5552,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * G17P, so widen the half input explicitly and use the ordinary f32
        * register form. Keep the separately witnessed abs(f32) + f16 endpoint
        * available for its exact modifier shape. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FADD && I->nr_srcs == 2 && !I->saturate &&
           I->dest[0].size == AGX_SIZE_32) {
          for (unsigned s = 0; s < 2; ++s) {
@@ -4940,7 +5625,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
       /* Only saturation through -0.0h and 1.0h - x have grounded immediate
        * half-add forms. Materialize every other minifloat operand so general
        * mediump offsets use the exact register-register add encoding. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_HADD && I->nr_srcs == 2) {
          const bool saturate_negzero =
             I->saturate && I->src[1].type == AGX_INDEX_IMMEDIATE &&
@@ -4969,7 +5654,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
       /* Half add has no register absolute field. Materialize abs by clearing
        * the sign bit, retaining a following negate for the FMA subtraction
        * lowering below. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_HADD && I->nr_srcs == 2) {
          for (unsigned s = 0; s < 2; ++s) {
             if (I->src[s].type == AGX_INDEX_IMMEDIATE || !I->src[s].abs)
@@ -4992,7 +5677,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * a - b = fma(a, 1.0h, -b), while -a - b uses both grounded FMA sign
        * bits. Do this after materializing constants so terrain's 0.5h - x
        * shape reaches the same register FMA path. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_HADD && I->nr_srcs == 2 &&
           (I->src[0].neg || I->src[1].neg) && !I->src[0].abs &&
           !I->src[1].abs && I->src[0].type != AGX_INDEX_IMMEDIATE &&
@@ -5017,7 +5702,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * represents max(x, k) as a tied compare/select, so load one exact half
        * constant and reuse it in both tied positions. In particular, this is
        * the mediump max(x, 0.0h) generated by glmark2's bump fragment. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_16 && !I->invert_cond &&
           !I->saturate &&
@@ -5050,7 +5735,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * point-size clamp generated by glmark2's ideas vertex shader. A zero
        * first comparator stays inline: the register/zero FSEL encodes it by
        * exchanging the comparators, as Apple does for max(0.0, x). */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_32 &&
           ((I->src[0].type == AGX_INDEX_IMMEDIATE && I->src[0].value != 0) ||
@@ -5090,7 +5775,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * select a half-register boolean or a raw 16-bit immediate. Neither
        * mixed axis is grounded together, so materialize both constants and
        * use the general four-register half FSEL. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_16 && !I->saturate) {
          const bool compare_immediate_first =
@@ -5133,7 +5818,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * the grounded full-register/zero mixed-result FSEL once its comparator
        * is widened. Leave both zero immediates inline so existing compact
        * encodings, including stock vkcube, remain byte-identical. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_32 && !I->saturate &&
           ((I->src[0].type == AGX_INDEX_IMMEDIATE &&
@@ -5165,7 +5850,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * independent highp registers. Normalize those comparator-only shapes
        * to the general four-register f32 select. This avoids extending a
        * modifier or mixed-width field beyond the endpoints witnessed for it. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_32 && !I->saturate &&
           I->fcond == AGX_FCOND_GE &&
@@ -5217,7 +5902,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * after register allocation. Materialize a nonzero minifloat comparator
        * while a temporary can still be allocated, so Hyprland's alpha == 1.0
        * predicate reaches the grounded two-register comparison form. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMP && I->nr_srcs == 2 &&
           I->dest[0].size == AGX_SIZE_16 &&
           ctx->stage == MESA_SHADER_FRAGMENT &&
@@ -5248,7 +5933,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
       /* Standalone half comparisons lower after RA to a byte-constant FSEL.
        * Normalize their zero comparison here, while a temporary can still be
        * allocated, so late pseudo lowering reaches the same grounded form. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMP && I->nr_srcs == 2 &&
           I->dest[0].size == AGX_SIZE_16 &&
           (I->fcond == AGX_FCOND_EQ || I->fcond == AGX_FCOND_LT ||
@@ -5274,7 +5959,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * immediates are integer bit patterns, including discard's 0/1 masks.
        * Tied min/max constants were materialized above. Keep f32 comparisons
        * selecting byte constants on their separate grounded form. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_16 &&
           I->src[0].type != AGX_INDEX_IMMEDIATE &&
@@ -5310,7 +5995,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * explicitly for a half result selected from registers and/or
        * minifloats instead of treating it as a nonexistent mixed-width
        * immediate form. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_16 &&
           (I->src[2].type == AGX_INDEX_IMMEDIATE ||
@@ -5338,7 +6023,7 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
        * has no separately grounded immediate form. Materialize the compare
        * constant so it uses the general four-register half select whose
        * conditions and lifetime fields are already recovered. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_FCMPSEL && I->nr_srcs == 4 &&
           I->dest[0].size == AGX_SIZE_16 &&
           I->src[2].type != AGX_INDEX_IMMEDIATE &&
@@ -5399,7 +6084,8 @@ agx_materialize_agx3_alu_constants(agx_context *ctx)
           I->dest[0].size == AGX_SIZE_32 &&
           I->src[2].type == AGX_INDEX_IMMEDIATE) {
          const bool immediate_shift =
-            ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+            (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+             ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
             I->bfi_mask == 0 && I->src[0].type == AGX_INDEX_IMMEDIATE &&
             I->src[0].value == 0 && I->src[1].size == AGX_SIZE_32 &&
             I->src[2].value >= 1 && I->src[2].value <= 31 &&
@@ -5472,7 +6158,7 @@ agx_coerce_f16_src(agx_builder *b, agx_index src)
 static agx_index
 agx_g17p_coerce_f32_src(agx_builder *b, agx_index src)
 {
-   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(b->shader->key->dev.encoding) ||
        src.size == AGX_SIZE_32 || src.type == AGX_INDEX_IMMEDIATE ||
        src.type == AGX_INDEX_NULL)
       return src;
@@ -5668,7 +6354,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
        * its native special register is already zero-extended to 32 bits, so
        * reuse that register instead of copying the half and clearing the
        * high half (two instructions per coordinate). */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
           s0.size == AGX_SIZE_16 && dst.size == AGX_SIZE_32) {
          for (unsigned i = 0; i < 2; ++i) {
             if (!agx_is_null(b->shader->agx3_pixel_coord_native[i]) &&
@@ -5679,12 +6365,14 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
       }
       return agx_mov_to(b, dst, s0);
    case nir_op_bitfield_reverse:
+      /* AGX3B only: G17P's emulated sequence uses an immediate+immediate IADD
+       * that AGX3C cannot encode; AGX3C keeps its native BITREV. */
       if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
           src_sz == 32)
          return agx_emit_g17p_bitrev(b, dst, s0);
       return agx_bitrev_to(b, dst, s0);
    case nir_op_bit_count:
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
           src_sz == 16 && sz == 32) {
          /* Apple's native ushort popcount writes a ushort.  Count(16 bits)
           * is at most 16, so widen its exact result after the operation. */
@@ -5702,7 +6390,8 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
       BINOP(ior, or);
       BINOP(ixor, xor);
    case nir_op_interleave:
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+          b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C)
          return agx_emit_g17p_interleave(b, dst, s0, s1);
       return agx_intl_to(b, dst, s0, s1);
 
@@ -5710,9 +6399,12 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
       if (instr->def.bit_size == 16)
          return agx_hadd_to(b, dst, agx_coerce_f16_src(b, s0),
                             agx_coerce_f16_src(b, s1));
-      else
-         return agx_fadd_to(b, dst, agx_g17p_coerce_f32_src(b, s0),
-                            agx_g17p_coerce_f32_src(b, s1));
+      else {
+         agx_instr *add = agx_fadd_to(b, dst, agx_g17p_coerce_f32_src(b, s0),
+                                     agx_g17p_coerce_f32_src(b, s1));
+         add->no_signed_zero = !nir_alu_instr_is_signed_zero_preserve(instr);
+         return add;
+      }
 
    case nir_op_fmul:
       if (instr->def.bit_size == 16)
@@ -5829,7 +6521,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
        * that bit is not arithmetic negation and would turn `c - x` into
        * `c + x`.
        */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
           sz == 32 && s0.type == AGX_INDEX_IMMEDIATE)
          s0 = agx_mov_imm(b, 32, s0.value);
       return agx_iadd_to(b, dst, s0, agx_neg(s1), 0);
@@ -5839,7 +6531,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
        * instruction selection reaches the native register-register isub
        * form instead of silently encoding `0 + x`.
        */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
           sz == 32) {
          agx_index zero = agx_mov_imm(b, 32, 0);
          return agx_iadd_to(b, dst, zero, agx_neg(s0), 0);
@@ -5863,7 +6555,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
        * throughput shader's x << 5). The adds take the zero immediate in
        * slot A.
        */
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding) &&
           sz == 32 && nir_src_is_const(instr->src[1].src)) {
          const unsigned shift = nir_alu_src_as_uint(instr->src[1]) & 31;
          if (shift >= 1 && shift <= 4)
@@ -5876,6 +6568,8 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
       }
       return agx_bfi_to(b, dst, i0, s0, s1, 0);
    case nir_op_ushr: {
+      /* AGX3B only (this and the SRL count path below): the G17P shift
+       * boundary materialization emits IADD forms AGX3C cannot encode. */
       if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
           sz == 32 && nir_src_is_const(instr->src[1].src)) {
          const unsigned shift = nir_alu_src_as_uint(instr->src[1]) & 31;
@@ -5925,7 +6619,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
 
    case nir_op_i2i32: {
       if (src_sz == 8) {
-         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+         if (agx_isa_is_agx3(b->shader->key->dev.encoding)) {
             /* G17P's grounded shifts operate on complete 32-bit registers.
              * Widen first and sign-extend from bit 7 as Apple does, rather
              * than inventing a half-register BFI/ASR encoding. */
@@ -5948,7 +6642,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
 
    case nir_op_i2i16: {
       if (src_sz == 8) {
-         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+         if (agx_isa_is_agx3(b->shader->key->dev.encoding)) {
             /* Perform the same exact 32-bit sign extension and expose its
              * low half only after the shift pair has completed. */
             agx_index wide = agx_pad_to_32(b, s0);
@@ -6009,7 +6703,7 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
    }
 
    case nir_op_fsin_agx: {
-      if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+      if (agx_isa_is_agx3(b->shader->key->dev.encoding)) {
          /* Compatibility for callers which already performed the AGX2 NIR
           * quadrant reduction. Fold [0,4] exactly to [-1,1], then use the
           * same native sinc operation as the G17P radians lowering. The
@@ -6110,8 +6804,13 @@ agx_emit_alu(agx_builder *b, nir_alu_instr *instr)
    }
 
    case nir_op_vec8:
-   case nir_op_vec16:
-      UNREACHABLE("should've been lowered");
+   case nir_op_vec16: {
+      agx_index idx[NIR_MAX_VEC_COMPONENTS];
+      assert(srcs <= ARRAY_SIZE(idx));
+      for (unsigned i = 0; i < srcs; ++i)
+         idx[i] = agx_alu_src_index(b, instr->src[i]);
+      return agx_emit_collect_to(b, dst, srcs, idx);
+   }
 
    default:
       /* For instance 64-bit float conversions, which need shaderFloat64.
@@ -6188,6 +6887,7 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
    }
 
    bool lod_is_zero = true;
+   bool reused_lod = false;
 
    for (unsigned i = 0; i < instr->num_srcs; ++i) {
       agx_index index = agx_src_index(&instr->src[i].src);
@@ -6203,12 +6903,23 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
           * agx_nir_lower_texture in that form instead of materializing a GPR
           * that the native instruction does not read.
           */
-         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         /* AGX3C (G17G witnesses, texwit3): the same immediate f43/f51/f141
+          * offset on 2D, 2D-array, 3D, comparison and gather samples, in every
+          * LOD mode. Cubes have no offset form. */
+         const bool m5_offset =
+            b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+            (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
+             instr->sampler_dim == GLSL_SAMPLER_DIM_3D);
+         if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+              m5_offset) &&
              nir_src_is_const(instr->src[i].src) &&
              (instr->op == nir_texop_tg4 ||
               nir_src_as_uint(instr->src[i].src) <= 0xfff)) {
             uint32_t value = nir_src_as_uint(instr->src[i].src);
-            if (instr->op == nir_texop_tg4)
+            /* Only G17P's lowering keeps gather X/Y in separate 16-bit lanes;
+             * C (generation 10) already packed the compact (y << 4) | x. */
+            if (instr->op == nir_texop_tg4 &&
+                b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
                value = (value & 0xf) | ((value >> 12) & 0xf0);
             /* Apple's compiler emits an all-zero offset as a plain sample. */
             packed_offset = value ? agx_immediate(value) : agx_null();
@@ -6221,12 +6932,14 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
 
       case nir_tex_src_lod:
       case nir_tex_src_bias:
+         reused_lod = !list_is_singular(&instr->src[i].src.ssa->uses);
          /* G17P has an exact immediate texel-fetch level encoding. Keep
           * compile-time levels in that form instead of promoting them into
           * the per-lane PACK_SMPIR path. Besides being smaller, this matches
           * Apple's constant read(..., level) programs and avoids introducing
           * an asynchronous staging dependency for a literal. */
-         if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         if ((b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+              b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
              instr->op == nir_texop_txf &&
              nir_src_is_const(instr->src[i].src) &&
              nir_src_as_uint(instr->src[i].src) <= 31)
@@ -6334,12 +7047,12 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
    enum agx_lod_mode lod_mode =
       agx_lod_mode_for_nir(instr->op, has_bias, has_min_lod, lod_is_zero);
 
-   /* AGX2's AUTO_LOD encoding also covers txl(..., 0). G17P's AUTO form is
+   /* AGX2's AUTO_LOD encoding also covers txl(..., 0). AGX3's AUTO form is
     * specifically the fragment derivative form, while Apple's compute and
     * vertex witnesses use the explicit immediate-level encoding. Keep every
     * non-fragment txl at level zero explicit so the packer selects that
     * native form. */
-   if (b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (b->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        b->shader->stage != MESA_SHADER_FRAGMENT &&
        (instr->op == nir_texop_txl || instr->op == nir_texop_txf) &&
        lod_is_zero) {
@@ -6356,6 +7069,25 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
       lod = agx_immediate(0);
    }
 
+   /* Reused G17G parameters are packed once in the normal register file.
+    * CSE shares identical pure producers and liveness preserves the word
+    * until its last publication. Single-use half-floats retain the direct
+    * staging path. */
+   const bool packed_lod =
+      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+      reused_lod && lod.type != AGX_INDEX_IMMEDIATE &&
+      lod.size == AGX_SIZE_16 &&
+      instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms &&
+      (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
+       instr->sampler_dim == GLSL_SAMPLER_DIM_3D) && !instr->is_array &&
+      !has_min_lod && (lod_mode == AGX_LOD_MODE_LOD_MIN ||
+                      lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS);
+   if (packed_lod) {
+      agx_index word = agx_temp(b->shader, AGX_SIZE_32);
+      agx_pack_sampler_lod_agx3_to(b, word, lod);
+      lod = word;
+   }
+
    agx_index dst = agx_def_index(&instr->def);
 
    /* Pack the shadow reference and offset sources into the one texture IR
@@ -6373,7 +7105,7 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
          ? (uint16_t)packed_offset.value : 0;
 
    const bool g17p_split_offset =
-      b->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      agx_isa_is_agx3(b->shader->key->dev.encoding) &&
       packed_offset.type == AGX_INDEX_IMMEDIATE;
 
    if (g17p_split_offset && !agx_is_null(compare))
@@ -6421,6 +7153,8 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
    /* The packer reads the G17P offset from the instruction, not from src[5],
     * so it survives being split away from the comparator above. */
    I->offset_xyz = offset_xyz;
+   /* A C sample's packed parameter is a word, not an unpacked half-float. */
+   I->memory_is_packed = packed_lod;
 
    if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms) {
       I->op = AGX_OPCODE_TEXTURE_LOAD;
@@ -6507,12 +7241,13 @@ agx_emit_undef(agx_builder *b, nir_undef_instr *instr)
     * to result in undefs.
     */
    if (instr->def.num_components > 1) {
-      assert(instr->def.num_components <= 4);
+      agx_index zeros[NIR_MAX_VEC_COMPONENTS];
+      assert(instr->def.num_components <= ARRAY_SIZE(zeros));
       agx_index zero = agx_mov_imm(b, instr->def.bit_size, 0);
-
+      for (unsigned i = 0; i < instr->def.num_components; ++i)
+         zeros[i] = zero;
       agx_emit_collect_to(b, agx_def_index(&instr->def),
-                          instr->def.num_components,
-                          (agx_index[4]){zero, zero, zero, zero});
+                          instr->def.num_components, zeros);
    } else {
       agx_mov_imm_to(b, agx_def_index(&instr->def), 0);
    }
@@ -6849,11 +7584,19 @@ agx_lower_sincos_impl(struct nir_builder *b, nir_instr *instr, void *data)
       r = nir_ffma(b, x, nir_imm_float(b, uif(0x25a94fe1)), r);
       nir_def *complement = nir_fsub(b, nir_imm_float(b, 1), nir_fabs(b, r));
       const bool cosine = alu->op == nir_op_fcos;
-      r = nir_bittest_select_agx3(b, quadrant, nir_imm_int(b, 1),
+      if ((uintptr_t)data == 2) {
+         nir_def *odd = nir_ine_imm(b, nir_iand_imm(b, quadrant, 1), 0);
+         nir_def *negative = nir_ine_imm(b, nir_iand_imm(b, quadrant, 2), 0);
+         r = nir_bcsel(b, odd, cosine ? nir_fneg(b, r) : complement,
+                       cosine ? complement : r);
+         r = nir_bcsel(b, negative, nir_fneg(b, r), r);
+      } else {
+         r = nir_bittest_select_agx3(b, quadrant, nir_imm_int(b, 1),
                                  cosine ? nir_fneg(b, r) : complement,
                                  cosine ? complement : r);
-      r = nir_bittest_select_agx3(b, quadrant, nir_imm_int(b, 2),
+         r = nir_bittest_select_agx3(b, quadrant, nir_imm_int(b, 2),
                                  nir_fneg(b, r), r);
+      }
       nir_def *result = nir_fmul(b, nir_fsinc_agx3(b, r), r);
       nir_def *small = cosine ? nir_imm_float(b, 1) : x;
 
@@ -6916,11 +7659,11 @@ agx_lower_sincos_impl(struct nir_builder *b, nir_instr *instr, void *data)
 }
 
 static bool
-agx_lower_sincos(nir_shader *shader, bool g17p)
+agx_lower_sincos(nir_shader *shader, unsigned native)
 {
    return nir_shader_lower_instructions(shader, agx_lower_sincos_filter,
                                         agx_lower_sincos_impl,
-                                        (void *)(uintptr_t)g17p);
+                                        (void *)(uintptr_t)native);
 }
 
 static bool
@@ -6944,8 +7687,17 @@ agx_lower_front_face(struct nir_builder *b, nir_intrinsic_instr *intr,
  * the latter run should be almost a no-op.
  */
 static void
-agx_optimize_loop_nir(nir_shader *nir)
+agx_optimize_loop_nir(nir_shader *nir, bool m5)
 {
+   const nir_shader_compiler_options *saved_options = nir->options;
+   nir_shader_compiler_options options;
+   /* Native M5 ALU throughput benefits from exposing its 64-iteration
+    * loops before scheduling. Keep earlier GPU generations unchanged. */
+   if (m5) {
+      options = *saved_options;
+      options.max_unroll_iterations = 64;
+      nir->options = &options;
+   }
    bool progress;
 
    do {
@@ -6969,6 +7721,7 @@ agx_optimize_loop_nir(nir_shader *nir)
       NIR_PASS(progress, nir, nir_opt_undef);
       NIR_PASS(progress, nir, nir_opt_loop_unroll);
    } while (progress);
+   nir->options = saved_options;
 }
 
 static bool
@@ -7056,6 +7809,86 @@ agx_nir_has_alu64(const nir_shader *nir, nir_op opcode)
    return false;
 }
 
+struct agx_position_order_state {
+   struct set *seen;
+   struct util_dynarray todo;
+};
+
+static void
+agx_position_order_add(struct agx_position_order_state *state, nir_def *def)
+{
+   if (_mesa_set_search(state->seen, def))
+      return;
+   _mesa_set_add(state->seen, def);
+   util_dynarray_append(&state->todo, def);
+}
+
+static bool
+agx_position_order_src(nir_src *src, void *data)
+{
+   agx_position_order_add(data, src->ssa);
+   return true;
+}
+
+static void
+agx_position_order_control(struct agx_position_order_state *state,
+                           nir_block *block)
+{
+   for (nir_cf_node *node = block->cf_node.parent; node; node = node->parent) {
+      if (node->type == nir_cf_node_if)
+         agx_position_order_add(state, nir_cf_node_as_if(node)->condition.ssa);
+   }
+}
+
+/* Preserve the source addition order of position, including shared operands.
+ * A depth-only link must not reassociate its position differently merely
+ * because a color link also uses an intermediate as a varying. Contracting
+ * a multiply/add remains allowed; no other output is made invariant here. */
+bool
+agx_nir_preserve_position_order(nir_shader *nir)
+{
+   struct agx_position_order_state state = {
+      .seen = _mesa_pointer_set_create(NULL),
+   };
+   util_dynarray_init(&state.todo, NULL);
+   bool progress = false;
+   nir_foreach_function_impl(impl, nir) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+            if (intr->intrinsic == nir_intrinsic_store_output &&
+                nir_intrinsic_io_semantics(intr).location == VARYING_SLOT_POS) {
+               agx_position_order_add(&state, intr->src[0].ssa);
+               agx_position_order_control(&state, block);
+            }
+         }
+      }
+   }
+
+   while (util_dynarray_num_elements(&state.todo, nir_def *)) {
+      nir_def *def = util_dynarray_pop(&state.todo, nir_def *);
+      nir_instr *instr = nir_def_instr(def);
+      if (instr->type == nir_instr_type_alu) {
+         nir_alu_instr *alu = nir_instr_as_alu(instr);
+         unsigned flags = nir_op_valid_fp_math_ctrl(
+            alu->op, nir_fp_no_reassoc | nir_fp_no_transform);
+         progress |= (alu->fp_math_ctrl & flags) != flags;
+         alu->fp_math_ctrl |= flags;
+      } else if (instr->type == nir_instr_type_phi) {
+         nir_foreach_phi_src(src, nir_instr_as_phi(instr))
+            agx_position_order_control(&state, src->pred);
+      }
+      if (instr->block)
+         agx_position_order_control(&state, instr->block);
+      nir_foreach_src(instr, agx_position_order_src, &state);
+   }
+   util_dynarray_fini(&state.todo);
+   _mesa_set_destroy(state.seen, NULL);
+   return progress;
+}
+
 bool
 agx_mem_vectorize_cb(unsigned align_mul, unsigned align_offset,
                      unsigned bit_size, unsigned num_components,
@@ -7085,6 +7918,8 @@ set_speculate(nir_builder *b, nir_instr *instr, UNUSED void *_)
    if (instr->type == nir_instr_type_intrinsic &&
        nir_intrinsic_has_access(nir_instr_as_intrinsic(instr))) {
       nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+      if (intr->intrinsic == nir_intrinsic_load_global_predicated_agx)
+         return false;
       nir_intrinsic_set_access(
          intr, ACCESS_CAN_SPECULATE | nir_intrinsic_access(intr));
       return true;
@@ -7099,6 +7934,7 @@ set_speculate(nir_builder *b, nir_instr *instr, UNUSED void *_)
 }
 
 static bool agx_g17p_discard_word(nir_shader *nir);
+static bool agx_m5_discard_word(nir_shader *nir);
 
 /* Drop a constant program left without instructions. */
 static void
@@ -7290,7 +8126,7 @@ static bool agx3_nir_lower_uniform_dma(nir_shader *nir,
 
 static void
 agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
-                 bool byte_indices, bool discard_word, bool g17p,
+                 bool byte_indices, bool discard_word, bool g17p, bool m5,
                  uint16_t *preamble_size, uint8_t *ts_count, uint8_t *ss_count,
                  const struct agx_shader_key *dma_key,
                  struct agx_shader_info *dma_info)
@@ -7298,7 +8134,7 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
    /* This runs only once up front since other optimizations don't affect it */
    NIR_PASS(_, nir, nir_opt_shrink_stores, true);
 
-   agx_optimize_loop_nir(nir);
+   agx_optimize_loop_nir(nir, m5);
 
    /* If soft fault is enabled, we can freely speculate everything. That lets us
     * peephole select and form preambles more aggressively.
@@ -7341,16 +8177,9 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
    /*
     * Rewrites a uniform-address atomic into "one lane of the subgroup performs
     * the whole operation", which needs ACTIVE_THREAD_INDEX_IN_SUBGROUP and
-    * TOTAL_ACTIVE_THREADS_IN_SUBGROUP. Their AGX3 numbers (SR 34/36) are now
-    * witnessed and grounded in agx3_map_sr(), but the optimized form is kept
-    * off on AGX3 until a CTS campaign validates it; before the witness every
-    * shader containing a uniform-address atomic failed to compile.
-    *
-    * Skipping the optimization keeps atomics CORRECT -- each lane performs its
-    * own atomic, which is what the unoptimized program means -- at the cost of
-    * one atomic per lane instead of one per subgroup on uniform addresses.
-    * Re-enable once those two SR numbers are witnessed rather than guessed; a
-    * wrong SR number would silently elect the wrong lane.
+    * TOTAL_ACTIVE_THREADS_IN_SUBGROUP. On AGX3 the caller enables it only in
+    * the stages where the packer admits those (SR 34/36). Skipping it keeps
+    * atomics correct, at one atomic per lane instead of one per subgroup.
     */
    NIR_PASS(_, nir, agx_nir_lower_atomics, byte_indices);
 
@@ -7408,10 +8237,11 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
    progress = false;
    if (byte_indices)
       NIR_PASS(progress, nir, agx_nir_scalarize_int64);
-   /* G17P: normalized 64-bit division instead of nir_lower_int64's fully
-    * unrolled 64-step division (agx_nir_lower_divmod64.c). */
-   if (g17p)
-      NIR_PASS(progress, nir, agx_nir_lower_divmod64);
+   /* G17P and M5: normalized 64-bit division instead of nir_lower_int64's
+    * fully unrolled 64-step division (agx_nir_lower_divmod64.c). M5 keeps
+    * the loop's 64-bit add and subtract native. */
+   if (g17p || m5)
+      NIR_PASS(progress, nir, agx_nir_lower_divmod64, m5);
    NIR_PASS(progress, nir, nir_lower_int64);
 
    /* If we lowered actual int64 arithmetic (not folded into the address
@@ -7428,16 +8258,20 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
    }
 
    /* Reassociate before forming preambles because it makes preambles more
-    * effective. Clean up after.
+    * effective. Clean up after. On M5, also order the loop-invariant terms of
+    * a chain in a loop first, so accumulations of per-iteration constants
+    * (robustness loops add the same size queries every iteration) are summed
+    * once in the preamble instead of once per term and iteration.
     */
    nir_opt_reassociate_loop(
-      nir, nir_reassociate_scalar_math | nir_reassociate_cse_heuristic);
+      nir, nir_reassociate_scalar_math | nir_reassociate_cse_heuristic |
+              (m5 ? nir_reassociate_loop_invariant : 0));
 
    /* Lower fmin/fmax before optimizing preambles so we can see across uniform
     * expressions. Do it after nir_opt_reassociate because nir_opt_reassociate
     * should work on unlowered fmin/fmax.
     */
-   NIR_PASS(_, nir, agx_nir_lower_fminmax, g17p);
+   NIR_PASS(_, nir, agx_nir_lower_fminmax, g17p || m5);
 
    /* AGX_G17P_DISCARD_WORD: reduce one-sample coverage-update sources to bit
     * 0 after the algebraic loops and right before preamble formation, so the
@@ -7446,7 +8280,10 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
     * AND per fragment). */
    if (discard_word) {
       bool word = false;
-      NIR_PASS(word, nir, agx_g17p_discard_word);
+      if (m5)
+         NIR_PASS(word, nir, agx_m5_discard_word);
+      else
+         NIR_PASS(word, nir, agx_g17p_discard_word);
       if (word) {
          NIR_PASS(_, nir, nir_opt_constant_folding);
          NIR_PASS(_, nir, nir_opt_dce);
@@ -7466,7 +8303,7 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
        * through a bindless image failed vkCreateGraphicsPipelines). Keep
        * bindless handles in the main shader there; nothing that compiles
        * today hoists either class. */
-      const bool state_promotion = !byte_indices;
+      const bool state_promotion = !(byte_indices || m5);
       unsigned sizes[] = {
          *preamble_size,
          (ts_count && state_promotion) ? *ts_count : 1000 /* large finite */,
@@ -7477,10 +8314,35 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
       if (sizes[2] == 0)
          sizes[2]++;
 
-      NIR_PASS(_, nir, agx_nir_opt_preamble, sizes, byte_indices);
+      NIR_PASS(_, nir, agx_nir_opt_preamble, sizes, byte_indices || m5, m5);
 
       *preamble_size = sizes[0];
 
+      /* M5: check the constant program's constant-offset buffer reads past
+       * 255 bytes in words, so their ends stay immediates
+       * (agx_nir_lower_word_bounds.c). The main program keeps its checks.
+       * A uniform DMA launch drops the byte checks of the reads it copies by
+       * matching their byte form, so leave those programs alone. */
+      if (m5 && !(dma_key && dma_key->g17p_uniform_dma)) {
+         bool words = false;
+         NIR_PASS(words, nir, agx_nir_lower_word_bounds);
+         if (words) {
+            NIR_PASS(_, nir, nir_opt_cse);
+            NIR_PASS(_, nir, nir_opt_dce);
+         }
+      }
+
+      /* M5: let the main program's robust reads of one structure share their
+       * end computation (agx_nir_share_bounds.c). */
+      if (m5) {
+         bool shared = false;
+         NIR_PASS(shared, nir, agx_nir_share_bounds);
+         if (shared) {
+            NIR_PASS(_, nir, nir_opt_copy_prop);
+            NIR_PASS(_, nir, nir_opt_dce);
+         }
+      }
+
       /* Copy what the launch can copy into the uniforms the constant
        * program leaves free, which then computes on the copies instead of
        * loading. Copying first would take uniforms the constant program's
@@ -7517,12 +8379,22 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
       NIR_PASS(_, nir, agx_nir_scalarize_int64);
    NIR_PASS(_, nir, nir_lower_int64);
 
+   /* After the selects of trivial if-else statements exist. */
+   if (m5) {
+      bool folded = false;
+      NIR_PASS(folded, nir, agx_nir_m5_split_or_selects);
+      if (folded) {
+         NIR_PASS(_, nir, nir_opt_copy_prop);
+         NIR_PASS(_, nir, nir_opt_dce);
+      }
+   }
+
    /* We need to lower fmin/fmax again after nir_opt_algebraic_late due to f2fmp
     * wackiness. This is usually a no-op but is required for correctness in
     * GLES.
     */
    NIR_PASS(_, nir, nir_opt_algebraic_late);
-   NIR_PASS(_, nir, agx_nir_lower_fminmax, g17p);
+   NIR_PASS(_, nir, agx_nir_lower_fminmax, g17p || m5);
 
    /* The AGX3 byte-address backend cannot encode 64-bit shifted adds.
     * Lower remaining multiplies before fusion turns x * 3 into one; the
@@ -7539,6 +8411,8 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, bool uniform_atomics,
       NIR_PASS(progress, nir, nir_opt_dce);
       NIR_PASS(progress, nir, nir_opt_cse);
       NIR_PASS(progress, nir, agx_nir_fuse_algebraic_late);
+      if (m5)
+         NIR_PASS(progress, nir, agx_nir_fuse_doubled_add);
    } while (progress);
 
    /* Before optimizing bounds checks, we need to clean up and index defs so
@@ -7846,7 +8720,7 @@ agx_should_dump(nir_shader *nir, unsigned agx_dbg_bit)
 static bool
 agx3_coalesce_uniform_half_stores(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        !ctx->is_preamble)
       return false;
 
@@ -8001,7 +8875,7 @@ agx3_coalesce_uniform_half_stores(agx_context *ctx)
 static bool
 agx3_order_fragment_device_loads(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT)
       return false;
 
@@ -8071,7 +8945,7 @@ agx3_order_fragment_device_loads(agx_context *ctx)
 static bool
 agx3_sink_vertex_outputs(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_VERTEX || ctx->is_preamble ||
        ctx->key->secondary)
       return false;
@@ -8206,7 +9080,10 @@ agx3_vertex_alu_operand(agx_index x, bool allow_uniform)
 static bool
 agx3_fuse_vertex_output_alu(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   /* ST_VARY_FMA/MUL_AGX3 are the G17P vertex-output forms; native M5
+    * terrain vertex mains use the same three (agx3_stage_vertex_alu). */
+   if ((ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C) ||
        ctx->stage != MESA_SHADER_VERTEX || ctx->is_preamble)
       return false;
 
@@ -8353,7 +9230,7 @@ agx3_order_g17p_mrt_tile_ops(agx_context *ctx)
 {
    agx_block *tail = agx_reconverged_mrt_tail(ctx);
    if (ctx->key->g17p_emrt && ctx->key->secondary &&
-       ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+       agx_isa_is_agx3(ctx->key->dev.encoding) &&
        ctx->stage == MESA_SHADER_FRAGMENT && ctx->key->fs.nr_samples >= 1 &&
        list_length(&ctx->blocks) > 1 && !tail) {
       /* Keep each sample within its block. Sorting stores before that
@@ -8364,7 +9241,7 @@ agx3_order_g17p_mrt_tile_ops(agx_context *ctx)
          BITSET_WORDS(AGX_G17P_MAX_BYTES_PER_SAMPLE));
       if (partitions >= 2) {
          agx_foreach_block(ctx, block) {
-            agx_instr *group[AGX_MAX_RTS * 4];
+            agx_instr *group[AGX_MAX_RTS * 8];
             unsigned count = 0;
             agx_foreach_instr_in_block(block, I) {
                if (I->op != AGX_OPCODE_ST_TILE)
@@ -8407,7 +9284,9 @@ agx3_order_g17p_mrt_tile_ops(agx_context *ctx)
       return true;
    }
 
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if ((ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B &&
+        !(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          ctx->key->fs.nr_samples <= 8)) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        (!ctx->key->secondary && !ctx->key->fs.ignore_tib_dependencies &&
         (ctx->key->no_stop ||
@@ -8419,8 +9298,8 @@ agx3_order_g17p_mrt_tile_ops(agx_context *ctx)
     * register (agx_nir_lower_g17p_tile_samples). */
    const bool multisample_stores =
       ctx->key->fs.nr_samples > 1 && !ctx->key->fs.ignore_tib_dependencies;
-   agx_instr *loads[AGX_MAX_RTS * 4] = {0};
-   agx_instr *stores[AGX_MAX_RTS * 4] = {0};
+   agx_instr *loads[AGX_MAX_RTS * 8] = {0};
+   agx_instr *stores[AGX_MAX_RTS * 8] = {0};
    agx_instr *stop = NULL;
    agx_instr *first_wait = NULL;
    unsigned load_count = 0, store_count = 0;
@@ -8680,7 +9559,7 @@ agx3_order_g17p_mrt_tile_ops(agx_context *ctx)
    const bool unrolled_samples = multisample_stores && sample_groups > 1;
    const unsigned group_count = sample_groups;
    const unsigned group_size = store_count / group_count;
-   struct list_head *group_anchor[4] = {0};
+   struct list_head *group_anchor[8] = {0};
    for (unsigned group = 0; group < group_count; ++group) {
       agx_instr **group_stores = &stores[group * group_size];
       if (unrolled_samples)
@@ -8763,7 +9642,7 @@ agx3_unchanged_tile_load(agx_instr **defs, const agx_instr *store)
 static bool
 agx3_drop_unchanged_tile_stores(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        list_length(&ctx->blocks) != 1)
       return false;
@@ -8819,7 +9698,7 @@ agx3_drop_unchanged_tile_stores(agx_context *ctx)
 static bool
 agx3_merge_unread_pixel_waits(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        ctx->key->fs.ignore_tib_dependencies)
       return false;
@@ -8866,7 +9745,7 @@ agx3_merge_unread_pixel_waits(agx_context *ctx)
 static bool
 agx3_tile_store_payloads_fit(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble)
       return true;
 
@@ -8926,7 +9805,7 @@ agx3_tile_store_payloads_fit(agx_context *ctx)
 static bool
 agx3_bank_scalar_tile_store_inputs(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble)
       return false;
 
@@ -9004,6 +9883,7 @@ agx3_implicit_coord_stage_producer(const agx_instr *I)
 static bool
 agx3_mark_implicit_tex_coord_staging(agx_context *ctx)
 {
+   /* AGX3B only: the staging-destination PERSP_MUL form is G17P's. */
    if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        ctx->key->fs.ignore_tib_dependencies || list_length(&ctx->blocks) != 1)
@@ -9210,20 +10090,46 @@ agx_nir_fragment_output_temporaries(nir_shader *nir)
    return true;
 }
 
+/* This pass runs after all NIR speculation and movement. The actual global
+ * load is introduced only within its valid arm, and cannot be speculated or
+ * reordered. The frontend predicate retains its original overflow semantics. */
 static bool
-agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
+m5_lower_predicated_load(nir_builder *b, nir_intrinsic_instr *intr,
+                         UNUSED void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_load_global_predicated_agx)
+      return false;
+   b->cursor = nir_before_instr(&intr->instr);
+   enum gl_access_qualifier access = nir_intrinsic_access(intr) &
+      ~(ACCESS_CAN_REORDER | ACCESS_CAN_SPECULATE);
+   nir_push_if(b, intr->src[2].ssa);
+   nir_def *value = nir_load_global(b, intr->num_components, intr->def.bit_size,
+      nir_iadd(b, intr->src[0].ssa, nir_u2u64(b, intr->src[1].ssa)),
+      .access = access, .align_mul = nir_intrinsic_align_mul(intr),
+      .align_offset = nir_intrinsic_align_offset(intr));
+   nir_push_else(b, NULL);
+   nir_def *zero = nir_imm_zero(b, intr->num_components, intr->def.bit_size);
+   nir_pop_if(b, NULL);
+   nir_def_replace(&intr->def, nir_if_phi(b, value, zero));
+   return true;
+}
+
+static bool
+agx_compile_function_nir_attempt(nir_shader *nir, nir_function_impl *impl,
                          struct agx_shader_key *key,
                          struct util_dynarray *binary,
-                         struct agx_shader_info *out, unsigned *offset_out)
+                         struct agx_shader_info *out, unsigned *offset_out,
+                         unsigned ra_budget_cut, bool disable_remat, bool *retry_without_remat)
 {
    const unsigned initial_binary_size = binary->size;
-   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-       !impl->function->is_preamble)
+   const struct agx_shader_info initial_out = *out;
+   if (agx_isa_is_agx3(key->dev.encoding) &&
+       !impl->function->is_preamble && !ra_budget_cut)
       NIR_PASS(_, nir, agx_nir_fragment_output_temporaries);
 
-   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(key->dev.encoding) &&
        nir->info.stage == MESA_SHADER_VERTEX && !impl->function->is_preamble &&
-       exec_list_length(&impl->body) > 1)
+       exec_list_length(&impl->body) > 1 && !ra_budget_cut)
       NIR_PASS(_, nir, nir_move_output_stores_to_end);
 
    nir_index_blocks(impl);
@@ -9231,6 +10137,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
 
    agx_context *ctx = rzalloc(NULL, agx_context);
    ctx->nir = nir;
+   ctx->ra_budget_cut = ra_budget_cut;
    ctx->is_preamble = impl->function->is_preamble;
    if (ctx->is_preamble)
       out->agx3_preamble_static_spill_version = 0;
@@ -9250,7 +10157,9 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
             ctx->zs_writes_left +=
                instr->type == nir_instr_type_intrinsic &&
                nir_instr_as_intrinsic(instr)->intrinsic ==
-                  nir_intrinsic_store_zs_agx;
+                  nir_intrinsic_store_zs_agx &&
+               !(nir_intrinsic_base(nir_instr_as_intrinsic(instr)) &
+                 AGX_ZS_COVERED_SAMPLE);
          }
       }
    }
@@ -9323,6 +10232,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    agx_lower_agx3_fma_sources(ctx);
    agx_lower_agx3_mixed_width_unary(ctx);
    agx_lower_agx3_shifted_iadd_dest(ctx);
+   agx_lower_m5_alu_operands(ctx);
 
    /* Before materialization, which lowers the SIGNEXT this introduces. */
    AGX_PASS(ctx, agx_lower_agx3_widening_int);
@@ -9330,8 +10240,46 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    if (key->dev.encoding != AGX_ISA_ENCODING_AGX2)
       agx_materialize_agx3_alu_constants(ctx);
 
+   /* M5 (AGX3C): an f16-destination FADD/FFMA whose sources are 32-bit
+    * registers or immediates has no grounded encoding (the packer refused
+    * `ffma d16, r32, #imm16, r32` in RayTracedGL1's IndirectInit raygen).
+    * agx_optimizer's fold guard only sees immediates that exist at fold
+    * time; a constant inlined afterwards by agx_opt_compact_constants slips
+    * through.  Compute such an operation at its sources' width and narrow
+    * the result with the explicit f32-to-f16 move the optimizer would have
+    * kept, which rounds once exactly as the unfused sequence did. */
+   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+      agx_foreach_instr_global_safe(ctx, I) {
+         if ((I->op != AGX_OPCODE_FFMA && I->op != AGX_OPCODE_FADD) ||
+             I->nr_dests != 1 || I->dest[0].size != AGX_SIZE_16 ||
+             agx_channels(I->dest[0]) != 1)
+            continue;
+
+         bool wide = false, immediate = false;
+         agx_foreach_src(I, s) {
+            if (I->src[s].type == AGX_INDEX_IMMEDIATE)
+               immediate = true;
+            else if (I->src[s].type != AGX_INDEX_NULL &&
+                     I->src[s].size == AGX_SIZE_32)
+               wide = true;
+         }
+         if (!(wide && immediate))
+            continue;
+
+         agx_index narrow = I->dest[0];
+         agx_index result = agx_temp(ctx, AGX_SIZE_32);
+         I->dest[0] = result;
+         agx_builder b = agx_init_builder(ctx, agx_after_instr(I));
+         /* Saturation stays on the wide operation: clamping to [0, 1] and
+          * then rounding to binary16 equals rounding then clamping, since
+          * 0 and 1 are exact halves and rounding is monotonic. */
+         agx_fmov_to(&b, narrow, result);
+      }
+   }
+
    /* Constant materialization can introduce mixed-width FADD operands. */
    agx_lower_agx3_fma_sources(ctx);
+   agx3_fold_constant_fmovs(ctx);
 
    AGX_PASS(ctx, agx_lower_agx3_cf_constants);
    AGX_PASS(ctx, agx_lower_agx3_half_iadd_constants);
@@ -9345,7 +10293,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    const bool native_cprog =
       key->g17p_native_uniforms || key->agx3_native_cprog;
    if (native_cprog && ctx->is_preamble &&
-       key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+       agx_isa_is_agx3(key->dev.encoding)) {
       if (!agx3_lower_cprog_inputs(ctx)) {
          agx_set_refusal(ctx, "constant-program input lowering refused",
                          __LINE__);
@@ -9360,31 +10308,52 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
 
    AGX_PASS(ctx, agx_lower_uniform_sources);
 
-
-   if (((key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-         (ctx->stage == MESA_SHADER_VERTEX || !ctx->is_preamble)) ||
-        (key->g17p_uniform_dma && out->agx3_uniform_dma_count)) &&
+   /* Source legalization materializes the M5 float immediates no native form
+    * reads in place; read them from constant uniforms where one does. */
+   if (key->promote_constants && !key->secondary &&
+       !(agx_compiler_debug & AGX_DBG_NOPROMOTE) &&
+       likely(!(agx_compiler_debug & AGX_DBG_NOOPT)))
+      AGX_PASS(ctx, agx_opt_promote_m5_float_constants);
+
+   /* Legalization can introduce repeated half-to-float conversions and
+    * uniform-source copies after the main CSE pass. Merge them before vertex
+    * register allocation, retaining the legalized operand types. The existing
+    * DMA path also needs this for descriptor words used by several operations.
+    * Do not run copy propagation here: it could undo source legalization.
+    * AGX3B main programs additionally use the pressure-aware CSE, and M5
+    * main programs the bounded one described below. */
+   const bool agx3_cse =
+      (agx_isa_is_agx3(key->dev.encoding) &&
+       ctx->stage == MESA_SHADER_VERTEX) ||
+      (key->dev.encoding == AGX_ISA_ENCODING_AGX3B && !ctx->is_preamble) ||
+      (key->g17p_uniform_dma && out->agx3_uniform_dma_count);
+   const bool m5_bounded_cse =
+      !agx3_cse && key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+      !ctx->is_preamble;
+   if ((agx3_cse || m5_bounded_cse) &&
        !(agx_compiler_debug & AGX_DBG_NOOPT)) {
       /* AGX3B main programs use the pressure-aware CSE, which merges only
        * while the earlier result is still live. Uniform source lowering
        * copies a uniform into a GPR for each operation that cannot read it in
        * place, so a value used by a run of selects is copied once per select,
-       * and each copy is read right after it is made. On G17P, also merge
-       * such copies where that keeps the register pressure within the
-       * shader's existing peak. */
+       * and each copy is read right after it is made. On G17P and in M5
+       * main programs, also merge such copies where that keeps the register
+       * pressure within the shader's existing peak. */
       if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B && !ctx->is_preamble &&
           ctx->stage != MESA_SHADER_VERTEX) {
          if (key->dev.chip == AGX_CHIP_G17P)
             AGX_PASS(ctx, agx_opt_cse_bounded);
          else
             AGX_PASS(ctx, agx_opt_cse_pressure);
+      } else if (m5_bounded_cse) {
+         AGX_PASS(ctx, agx_opt_cse_bounded);
       } else {
          AGX_PASS(ctx, agx_opt_cse);
       }
    }
 
    if (native_cprog && ctx->is_preamble &&
-       key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+       agx_isa_is_agx3(key->dev.encoding)) {
       /* Selection requires a single live use of an arithmetic result.
        * Dead uses left by earlier lowering must not hide that opportunity. */
       AGX_PASS(ctx, agx_dce, true);
@@ -9402,13 +10371,14 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    AGX_PASS(ctx, agx3_merge_unread_pixel_waits);
 
    AGX_PASS(ctx, agx_lower_agx3_device_atomic_sources);
+   AGX_PASS(ctx, agx_lower_agx3_matrix_sources);
 
    if (agx_should_dump(nir, AGX_DBG_SHADERS))
       agx_print_shader(ctx, stdout);
 
    if (likely(!(agx_compiler_debug & AGX_DBG_NOSCHED))) {
       AGX_PASS(ctx, agx_pressure_schedule);
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+      if (agx_isa_is_agx3(ctx->key->dev.encoding))
          AGX_PASS(ctx, agx_ilp_schedule);
    }
 
@@ -9434,6 +10404,10 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
       agx_validate(ctx, "scalar texture publications");
       AGX_PASS(ctx, agx_dce, true);
    }
+   if (agx3_split_texture_coordinates(ctx)) {
+      agx_validate(ctx, "split texture coordinates");
+      AGX_PASS(ctx, agx_dce, true);
+   }
 
 
    if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
@@ -9452,8 +10426,50 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
 
    /* Record provenance before RA can create stack instructions. */
    const bool imports_only_before_ra = agx3_cprog_spill_provenance(ctx);
-   AGX_PASS(ctx, agx_ra);
+   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       ctx->stage == MESA_SHADER_FRAGMENT && ctx->key->fs.nr_samples > 1 &&
+       debug_get_bool_option("ASAHI_M5_TILE_DATA_COPY_PROBE", false)) {
+      agx_foreach_instr_global_safe(ctx, I) {
+         if (I->op != AGX_OPCODE_ST_TILE ||
+             I->src[0].type != AGX_INDEX_NORMAL || I->src[0].memory)
+            continue;
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         const unsigned channels = agx_channels(I->src[0]);
+         if (channels > 4)
+            continue;
+         agx_index parts[4];
+         agx_emit_split(&b, parts, I->src[0], channels);
+         for (unsigned c = 0; c < channels; ++c)
+            parts[c] = agx_mov(&b, parts[c]);
+         agx_replace_src(I, 0, agx_emit_collect(&b, channels, parts));
+      }
+   }
+   if (disable_remat)
+      AGX_PASS(ctx, agx_ra_without_remat);
+   else
+      AGX_PASS(ctx, agx_ra);
+
+   /* The occupancy rematerialization attempt has already modified AIR.
+    * Discard the entire context; changing its budget cannot undo spills. */
+   if (retry_without_remat && ctx->remat_fallback_spilled) {
+      *retry_without_remat = true;
+      binary->size = initial_binary_size;
+      ralloc_free(ctx);
+      return false;
+   }
    if (ctx->ra_failed) {
+      /* Spilling only targets total demand; pinned values can still leave the
+       * shuffle no room. Recompile with an ever smaller spill budget. */
+      if (key->has_scratch && !key->is_helper && ra_budget_cut < 64) {
+         ralloc_free(ctx);
+         binary->size = initial_binary_size;
+         /* Constant promotion and others accumulate into out. */
+         *out = initial_out;
+         return agx_compile_function_nir_attempt(
+            nir, impl, key, binary, out, offset_out,
+            ra_budget_cut ? ra_budget_cut * 2 : 16, disable_remat,
+            retry_without_remat);
+      }
       agx_set_refusal(ctx, "register allocation found no legal placement",
                       __LINE__);
       ralloc_free(ctx);
@@ -9470,7 +10486,9 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
     * allocator-created spill size. */
    /* Software geometry and tessellation-control shaders keep their API
     * stage labels while running through CDM. Their returned atomics consume
-    * the same thread-private word and require the same reservation. */
+    * the same thread-private word and require the same reservation.
+    * AGX3B only: this is native G17P's ABI; on M5 the reservation makes the
+    * precompiled helpers carry thread-private storage their launch refuses. */
    if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
       agx_foreach_instr_global(ctx, I) {
          if (I->op == AGX_OPCODE_ATOMIC && I->nr_dests == 1 &&
@@ -9519,7 +10537,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
        * kernels whose highest reloaded TPS half is 470 and 790 records 944
        * and 1584 bytes, the halves up to and including that 32-bit word.
        * SPECTPR's spill fields are those bytes in quads. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+      if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX2)
          stack_size = align(ctx->scratch_size_B, 4);
 
       if (ctx->is_preamble)
@@ -9551,7 +10569,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
     * instruction, so a vector spill or scratch access has to become a run of
     * scalar ones. Do it before the last-use pass, so each component's hints
     * are computed against the instructions that actually survive. */
-   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX2)
       agx3_split_vector_stack_access(ctx);
 
    /* Set last-use bits after lowering pseudo for best results. */
@@ -9565,7 +10583,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    agx3_lower_texture_publications(ctx);
 
    if (ctx->is_preamble &&
-       ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+       agx_isa_is_agx3(ctx->key->dev.encoding) &&
        !ctx->agx3_cprog_inputs_lowered &&
        !agx3_analyze_preamble_uniform_inputs(
           ctx, &out->agx3_preamble_uniform_inputs)) {
@@ -9600,8 +10618,11 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
        * If the encoder refuses an instruction, discard the emitted prefix and
        * report compilation failure. */
       if (!agx3_pack_binary(ctx, binary)) {
-         agx_set_refusal(ctx, "AGX3 binary packer refused an instruction "
-                         "(AGX_MESA_DEBUG=shaders names it)", __LINE__);
+         char what[160];
+         snprintf(what, sizeof(what), "AGX3 binary packer refused %s",
+                  agx3_last_pack_refusal()[0] ? agx3_last_pack_refusal()
+                                              : "an instruction");
+         agx_set_refusal(ctx, what, __LINE__);
          binary->size = initial_binary_size;
          ralloc_free(ctx);
          return false;
@@ -9624,7 +10645,7 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
     * reserve every GPR; report the allocation that actually reached packing. */
    if (impl->function->is_preamble) {
       const bool legacy_scratch = ctx->scratch_size_B &&
-         ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B;
+         !agx_isa_is_agx3(ctx->key->dev.encoding);
       out->nr_preamble_gprs = legacy_scratch ? 256 : nr_gprs;
       out->agx3_preamble_staging_halfs = ctx->agx3_staging_halfs;
       out->agx3_preamble_staging_known = ctx->agx3_staging_known;
@@ -9707,6 +10728,56 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    return true;
 }
 
+/* Retry eligible preambles without changing scratch admission or launch rules. */
+static bool
+agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
+                         struct agx_shader_key *key,
+                         struct util_dynarray *binary,
+                         struct agx_shader_info *out, unsigned *offset_out)
+{
+   const bool eligible =
+      impl->function->is_preamble && nir->info.stage == MESA_SHADER_FRAGMENT &&
+      key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+      (key->g17p_native_uniforms || key->agx3_native_cprog) &&
+      debug_get_bool_option("ASAHI_M5_CPROG_REMAT_RETRY", true);
+   if (!eligible)
+      return agx_compile_function_nir_attempt(nir, impl, key, binary, out,
+                                             offset_out, 0, false, NULL);
+
+   /* Preserve all compiler-owned state, including earlier main code/rodata.
+    * The first attempt owns a deep NIR clone and a fresh AIR context. */
+   nir_shader *attempt = nir_shader_clone(NULL, nir);
+   if (!attempt)
+      return agx_compile_function_nir_attempt(nir, impl, key, binary, out,
+                                             offset_out, 0, false, NULL);
+
+   nir_function_impl *attempt_impl = NULL;
+   nir_foreach_function_with_impl(func, candidate, attempt) {
+      if (func->is_preamble) {
+         assert(attempt_impl == NULL);
+         attempt_impl = candidate;
+      }
+   }
+   assert(attempt_impl != NULL);
+   const struct agx_shader_info saved_info = *out;
+   const struct agx_shader_key saved_key = *key;
+   const size_t saved_size = binary->size;
+   bool retry = false;
+   bool success = agx_compile_function_nir_attempt(
+      attempt, attempt_impl, key, binary, out, offset_out, 0, false, &retry);
+   ralloc_free(attempt);
+   if (!retry)
+      return success;
+
+   assert(!success && binary->size == saved_size);
+   *out = saved_info;
+   *key = saved_key;
+   binary->size = saved_size;
+   /* Exactly one retry. Real full-budget spills retain the old refusal. */
+   return agx_compile_function_nir_attempt(nir, impl, key, binary, out,
+                                          offset_out, 0, true, NULL);
+}
+
 /*
  * The hardware frcp instruction is sometimes off by 1 ULP. For correctly
  * rounded frcp, a refinement step is required. This routine has been
@@ -9855,7 +10926,7 @@ agx3_nir_lower_scratch_words(nir_shader *nir)
 }
 
 static void
-agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
+agx_preprocess_nir_internal(nir_shader *nir, bool g17p, bool native_fragment_depth)
 {
    if (!nir)
       return;
@@ -9900,7 +10971,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
    NIR_PASS(_, nir, nir_lower_var_copies);
 
    if (nir->info.stage == MESA_SHADER_FRAGMENT) {
-      NIR_PASS(_, nir, agx_nir_lower_frag_sidefx, g17p);
+      NIR_PASS(_, nir, agx_nir_lower_frag_sidefx, g17p || native_fragment_depth);
       NIR_PASS(_, nir, nir_lower_is_helper_invocation);
    }
 
@@ -9908,7 +10979,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
    NIR_PASS(_, nir, nir_opt_dce);
 
    NIR_PASS(_, nir, nir_lower_alu_to_scalar, NULL, NULL);
-   if (g17p) {
+   if (g17p || native_fragment_depth) {
       NIR_PASS(_, nir, nir_shader_alu_pass,
                agx_nir_lower_g17p_f16_classify,
                nir_metadata_control_flow, NULL);
@@ -9917,7 +10988,11 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
    NIR_PASS(_, nir, nir_lower_alu);
    NIR_PASS(_, nir, nir_lower_load_const_to_scalar);
    NIR_PASS(_, nir, nir_lower_flrp, 16 | 32 | 64, false);
-   NIR_PASS(_, nir, agx_lower_sincos, g17p);
+   /* When the explicit-RTE FMA repair applies, G17P lowers sin/cos in
+    * agx_compile_shader_nir after it, so the repair covers the shader's own
+    * FMAs only. Other shaders keep the lowering here. */
+   if (!g17p || !agx_nir_fma_rte_applies(nir))
+      NIR_PASS(_, nir, agx_lower_sincos, g17p ? 1 : native_fragment_depth ? 2 : 0);
    NIR_PASS(_, nir, nir_shader_intrinsics_pass, agx_lower_front_face,
             nir_metadata_control_flow, NULL);
    if (g17p)
@@ -9930,7 +11005,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
     * will run through the loop later, once we have the shader key, but if we
     * run now, that run will ideally be almost a no-op.
     */
-   agx_optimize_loop_nir(nir);
+   agx_optimize_loop_nir(nir, !g17p && native_fragment_depth);
 
    /* Lower idiv after an optimization loop so we can constant fold more before
     * nir_opt_idiv_const.
@@ -9975,7 +11050,7 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
     * into 32-bit halves, leaving each half addressing half its element.
     * Accesses of 32 bits and narrower are not split, so they are unaffected.
     */
-   if (g17p) {
+   if (g17p || native_fragment_depth) {
       nir_lower_mem_access_bit_sizes_options shared_split = {
          .modes = nir_var_mem_shared,
          .callback = mem_access_size_align_cb,
@@ -9983,7 +11058,8 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
       NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &shared_split);
    }
 
-   NIR_PASS(_, nir, agx_nir_lower_shared_bitsize, g17p);
+   NIR_PASS(_, nir, agx_nir_lower_shared_bitsize,
+            g17p || native_fragment_depth);
 
    /* A frontend that reaches this helper with `g17p` set owns the element-index
     * conversion for the whole shader, including shared accesses that do not
@@ -9995,20 +11071,20 @@ agx_preprocess_nir_internal(nir_shader *nir, bool g17p)
     * suppress both the real conversion and the backstop below, which is
     * exactly how the Vulkan front end lost its conversion once already.
     */
-   if (g17p)
+   if (g17p || native_fragment_depth)
       nir->info.shared_element_index_agx = true;
 }
 
 void
 agx_preprocess_nir(nir_shader *nir)
 {
-   agx_preprocess_nir_internal(nir, false);
+   agx_preprocess_nir_internal(nir, false, false);
 }
 
 void
 agx_preprocess_nir_g17p(nir_shader *nir)
 {
-   agx_preprocess_nir_internal(nir, true);
+   agx_preprocess_nir_internal(nir, true, true);
 }
 
 static bool
@@ -10359,6 +11435,13 @@ agx3_uniform_dma_collect(nir_shader *nir, const struct agx_shader_key *key,
                c.pointer = pointer;
             } else if (!agx3_uniform_dma_buffer(key, intr->src[0].ssa, &c)) {
                continue;
+            } else if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+                       nir->info.stage == MESA_SHADER_COMPUTE &&
+                       intr->def.bit_size != 32) {
+               /* M5 compute copies complete bounded dwords only. Preserve
+                * narrow loads: a valid byte/half in an incomplete trailing
+                * word cannot be represented by a whole-word zero tail. */
+               continue;
             }
             util_dynarray_append(cands, c);
          }
@@ -11437,11 +12520,132 @@ agx_g17p_discard_word(nir_shader *nir)
                                      nir_metadata_control_flow, NULL);
 }
 
+/* AGX_M5_DISCARD_WORD.
+ *
+ * M5 keeps a whole-pixel coverage update as one testing MOVMSK whose word is
+ * SR2 repeated in the low two bytes and the live bit in byte 3 (the
+ * sample_mask_agx case). For Minecraft's `if (color.a < 0.5) discard;` the
+ * backend received sample_mask_agx(0xff, bcsel(c, 0, 0xff)) and spent an AND,
+ * two shifting IADDs and a select per fragment on the live byte, plus an AND
+ * and a select on the constant target. Apple's native conditional discard
+ * selects byte 3's half directly (one FSEL). With a constant target naming
+ * the one sample, rebuild LIVE as its bit 0 shifted to bit 8 of the high half
+ * -- selects of constants keep one select, now of shifted constants -- and
+ * record that in FLAGS bit 2, so the backend collects it as the high half
+ * unchanged. Bit 0 is the only bit the one-sample word reads, so the update
+ * is unchanged by construction.
+ */
+/* Whether component d_comp of d is b2f of component c_comp of c or its
+ * inversion. Keep the consumer and producer swizzles composed. */
+static bool
+agx_b2f_of(nir_def *d, unsigned d_comp, nir_def *c, unsigned c_comp,
+           bool *inverted)
+{
+   nir_alu_instr *b2f = nir_def_as_alu_or_null(d);
+   if (!b2f || (b2f->op != nir_op_b2f32 && b2f->op != nir_op_b2f16))
+      return false;
+   nir_def *x = b2f->src[0].src.ssa;
+   unsigned xc = b2f->src[0].swizzle[d_comp];
+   nir_alu_instr *inot = nir_def_as_alu_or_null(x);
+   *inverted = inot && inot->op == nir_op_inot;
+   if (*inverted) {
+      x = inot->src[0].src.ssa;
+      xc = inot->src[0].swizzle[xc];
+   }
+   return x == c && xc == c_comp;
+}
+
+/*
+ * bcsel(a || b, b2f(!a), r) is 0.0 when a holds, else 1.0 when b holds, else
+ * r: split it into bcsel(a, 0.0, bcsel(b, 1.0, r)) (likewise for b2f(a) and
+ * for b in place of a). Minecraft's linear fog builds this per distance (0
+ * before the start, 1 past the end, the ramp between); AGX has no boolean
+ * OR or NOT, so the backend built it from three selects, two of them needing
+ * their uniform comparand copied into a GPR on M5. Apple's fragment uses two
+ * selects. The values are exact (b2f gives +0.0 and 1.0). M5 only, so other
+ * targets keep their compiles.
+ */
+static bool
+agx_m5_split_or_select(nir_builder *b, nir_alu_instr *sel, void *data)
+{
+   if (sel->op != nir_op_bcsel || sel->def.num_components != 1)
+      return false;
+   nir_alu_instr *ior = nir_def_as_alu_or_null(sel->src[0].src.ssa);
+   if (!ior || ior->op != nir_op_ior || ior->def.num_components != 1 ||
+       ior->def.bit_size != 1)
+      return false;
+
+   for (unsigned first = 0; first < 2; ++first) {
+      nir_def *a = ior->src[first].src.ssa;
+      unsigned ac = ior->src[first].swizzle[0];
+      bool inverted;
+      if (!agx_b2f_of(sel->src[1].src.ssa, sel->src[1].swizzle[0],
+                      a, ac, &inverted) ||
+          a->num_components <= ac)
+         continue;
+
+      b->cursor = nir_before_instr(&sel->instr);
+      const unsigned bits = sel->def.bit_size;
+      nir_def *ca = nir_channel(b, a, ac);
+      nir_def *cb = nir_mov_alu(b, ior->src[!first], 1);
+      nir_def *r = nir_mov_alu(b, sel->src[2], 1);
+      /* a holds: b2f(a) = 1, b2f(!a) = 0. Otherwise b holds: the opposite. */
+      nir_def *when_a = nir_imm_floatN_t(b, inverted ? 0.0 : 1.0, bits);
+      nir_def *when_b = nir_imm_floatN_t(b, inverted ? 1.0 : 0.0, bits);
+      nir_def *split = nir_bcsel(b, ca, when_a, nir_bcsel(b, cb, when_b, r));
+      nir_def_replace(&sel->def, split);
+      return true;
+   }
+   return false;
+}
+
+bool
+agx_nir_m5_split_or_selects(nir_shader *nir)
+{
+   return nir_shader_alu_pass(nir, agx_m5_split_or_select,
+                              nir_metadata_control_flow, NULL);
+}
+
+static bool
+agx_m5_discard_word_instr(nir_builder *b, nir_intrinsic_instr *intr,
+                          void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_sample_mask_agx ||
+       nir_intrinsic_flags(intr) || !nir_src_is_const(intr->src[0]) ||
+       !(nir_src_as_uint(intr->src[0]) & 1) || nir_src_is_const(intr->src[1]))
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_def *bit0 = agx_g17p_bit0(b, nir_get_scalar(intr->src[1].ssa, 0), 0);
+   unsigned v, x, y;
+   nir_def *c;
+   /* A live mask that reduces to a constant keeps its original form; the
+    * no-tests decision reads constant sources. */
+   if (agx_g17p_bit0_const(bit0, &v))
+      return false;
+   nir_def *high = agx_g17p_bit0_const_select(bit0, &c, &x, &y)
+      ? nir_bcsel(b, c, nir_imm_intN_t(b, x << 8, 16),
+                  nir_imm_intN_t(b, y << 8, 16))
+      : nir_ishl_imm(b, bit0, 8);
+   nir_src_rewrite(&intr->src[1], high);
+   nir_intrinsic_set_flags(intr, BITFIELD_BIT(2));
+   return true;
+}
+
+static bool
+agx_m5_discard_word(nir_shader *nir)
+{
+   return nir_shader_intrinsics_pass(nir, agx_m5_discard_word_instr,
+                                     nir_metadata_control_flow, NULL);
+}
+
 unsigned
 agx_max_vertex_attribs(const struct agx_device_key *dev)
 {
    const unsigned halfs = dev->encoding == AGX_ISA_ENCODING_AGX3B
                              ? AGX3B_NUM_REGS
+                          : dev->encoding == AGX_ISA_ENCODING_AGX3C
+                             ? AGX3C_NUM_REGS
                              : AGX_NUM_REGS;
 
    /* The last component's 32-bit value must end inside the file, and the
@@ -11456,17 +12660,31 @@ agx_max_vertex_attribs(const struct agx_device_key *dev)
    return MIN2(attribs, AGX_MAX_ATTRIBS);
 }
 
+struct agx_split_sample_tests {
+   unsigned samples;
+   bool m5;
+};
+
 /* Separate pure kills from testing updates before lowering their different
- * pixel-class lifetimes. The testing arm needs a real empty-EXEC skip. */
+ * pixel-class lifetimes. The testing arm needs a real empty-EXEC skip.
+ *
+ * M5 keeps a whole-pixel update as one unconditional testing MOVMSK, as
+ * native G17G does (agx_nir_lower_sample_mask_m5). An update naming only some
+ * samples (one per sample in a sample-rate epilog) is still split.
+ */
 static bool
 agx_nir_split_g17p_sample_tests(nir_builder *b, nir_intrinsic_instr *intr,
                                 void *data)
 {
+   const struct agx_split_sample_tests *split = data;
    if (intr->intrinsic != nir_intrinsic_sample_mask_agx ||
        nir_src_is_const(intr->src[1]))
       return false;
 
-   unsigned all = BITFIELD_MASK(*(unsigned *)data);
+   unsigned all = BITFIELD_MASK(split->samples);
+   if (split->m5 && nir_src_is_const(intr->src[0]) &&
+       (nir_src_as_uint(intr->src[0]) & all) == all)
+      return false;
    nir_def *target = intr->src[0].ssa;
    nir_def *live = intr->src[1].ssa;
    b->cursor = nir_before_instr(&intr->instr);
@@ -11488,6 +12706,9 @@ agx_nir_split_g17p_sample_tests(nir_builder *b, nir_intrinsic_instr *intr,
    return true;
 }
 
+/* Keep unsupported private SCS forms transactional, before instruction
+ * selection. These operations never reinterpret NIR scratch/global memory.
+ * M5 launch setup must allocate SCS even for a load/store-only program. */
 static bool
 agx_validate_scs_nir(nir_shader *nir, const struct agx_shader_key *key,
                      bool *uses_scs)
@@ -11527,6 +12748,25 @@ agx_validate_scs_nir(nir_shader *nir, const struct agx_shader_key *key,
                        (store || !(nir_intrinsic_flags(in) & ~1u));
                break;
             }
+            case nir_intrinsic_load_scs_register_agx: {
+               unsigned format = nir_intrinsic_format(in);
+               unsigned width = format == PIPE_FORMAT_R8_UINT ? 8 :
+                                format == PIPE_FORMAT_R16_UINT ? 16 :
+                                format == PIPE_FORMAT_R32_UINT ? 32 : 0;
+               memory = true;
+               valid = width && in->def.num_components <= 4 &&
+                  (width == 32 || in->def.num_components == 1) &&
+                  in->def.bit_size == (width == 32 ? 32 : 16) &&
+                  in->src[0].ssa->bit_size == 32 &&
+                  in->src[1].ssa->bit_size == 32 &&
+                  !(nir_intrinsic_base(in) & (width / 8 - 1)) &&
+                  nir_intrinsic_component(in) <= 4 &&
+                  !(nir_intrinsic_flags(in) & ~9u) &&
+                  (!(nir_intrinsic_flags(in) & 8) ||
+                   (width == 8 && nir_intrinsic_component(in) == 3 &&
+                    (nir_intrinsic_flags(in) & 1)));
+               break;
+            }
             case nir_intrinsic_load_scs_uniform_agx:
             case nir_intrinsic_store_scs_uniform_agx:
             case nir_intrinsic_load_scs_base_agx:
@@ -11552,9 +12792,10 @@ agx_validate_scs_nir(nir_shader *nir, const struct agx_shader_key *key,
                            nir_intrinsic_component(in) <= 4 && !(flags & ~3u) && (!(flags & 2) || width < 32);
                else
                   valid &= nir_intrinsic_component(in) <= 4 && !(flags & ~13u) &&
-                           (!(flags & 8) || key->dev.encoding == AGX_ISA_ENCODING_AGX3B) &&
                            (!(flags & 4) || ((flags & 1 || key->dev.encoding == AGX_ISA_ENCODING_AGX3B) && width == 8 &&
-                                            nir_intrinsic_component(in) == 4));
+                                            nir_intrinsic_component(in) == 4)) &&
+                           (!(flags & 8) || (key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+                                            (width == 8 && nir_intrinsic_component(in) == 4)));
                break;
             }
             case nir_intrinsic_intersect_ray_gen1_agx:
@@ -11569,6 +12810,18 @@ agx_validate_scs_nir(nir_shader *nir, const struct agx_shader_key *key,
                valid = key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
                   in->src[0].ssa->bit_size == 32 && nir_intrinsic_base(in) <= 1;
                break;
+            case nir_intrinsic_intersect_ray_agx:
+               valid = in->src[0].ssa->bit_size == 64 &&
+                       in->src[1].ssa->bit_size == 64 &&
+                       in->src[2].ssa->bit_size == 32 &&
+                       in->src[3].ssa->bit_size == 32 &&
+                       in->src[4].ssa->bit_size == 32;
+               break;
+            case nir_intrinsic_enter_query_agx:
+               break;
+            case nir_intrinsic_continue_ray_agx:
+               valid = in->src[0].ssa->bit_size == 32;
+               break;
             default:
                continue;
             }
@@ -11653,18 +12906,21 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_printf_buffer,
             nir_metadata_control_flow, &key->dev);
 
-   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (key->dev.encoding != AGX_ISA_ENCODING_AGX2 &&
        nir->info.stage == MESA_SHADER_FRAGMENT) {
       NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_indirect_fs_input,
                nir_metadata_none, NULL);
    }
 
-   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(key->dev.encoding) &&
        nir->info.stage == MESA_SHADER_FRAGMENT &&
        key->fs.nr_samples > 1 && !key->fs.inside_sample_loop) {
-      unsigned samples = key->fs.nr_samples;
+      struct agx_split_sample_tests split = {
+         .samples = key->fs.nr_samples,
+         .m5 = key->dev.encoding == AGX_ISA_ENCODING_AGX3C,
+      };
       NIR_PASS(_, nir, nir_shader_intrinsics_pass, agx_nir_split_g17p_sample_tests,
-               nir_metadata_none, &samples);
+               nir_metadata_none, &split);
    }
 
    NIR_PASS(_, nir, nir_lower_frag_coord_to_pixel_coord);
@@ -11679,7 +12935,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    };
    NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &lower_mem_access_options);
 
-   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+   if (agx_isa_is_agx3(key->dev.encoding)) {
       NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_g17p_halfword_tile_load,
                nir_metadata_control_flow, NULL);
    }
@@ -11713,6 +12969,12 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    if (key->dev.chip == AGX_CHIP_G17P)
       NIR_PASS(_, nir, agx_nir_lower_fma_rte);
 
+   /* Sin/cos that agx_preprocess_nir_g17p left for an explicit-RTE shader.
+    * Their reduction FMAs serve an absolute-error bound, which a tininess
+    * repair of at most 2^-126 cannot affect, so they are lowered after it.
+    * Every other shader has already lowered them. */
+   NIR_PASS(_, nir, agx_lower_sincos, 1);
+
    /* Late VBO lowering creates constant udiv instructions */
    NIR_PASS(_, nir, nir_opt_idiv_const, 16);
 
@@ -11729,31 +12991,14 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
                        key->dev.encoding == AGX_ISA_ENCODING_AGX3C;
    const bool is_g17p = key->dev.encoding == AGX_ISA_ENCODING_AGX3B;
 
-   /* AGX3's ldtg/sttg index counts scalar elements and the instruction's own
-    * shift field scales it by the access width, so a NIR byte offset has to be
-    * divided by that width.  A frontend that knows its target does this during
-    * preprocessing, where the optimizer can fold the division into the address
-    * arithmetic and claims it with the shader_info bit above, so this backstop
-    * only fires for a frontend that reaches an AGX3 target without having
-    * claimed it -- which is how every internal libagx kernel used to reach
-    * G17P with a byte offset in the element-index field, a fourfold
-    * over-stride for a 32-bit element.
-    *
-    * AGX3C shares the convention (agx_emit_local_load/store put the address in
-    * the index for every AGX3 encoding) but is deliberately left out: applying
-    * it there turns a shared read-modify-write into a refusal, because the
-    * resulting 16-bit index reaches the AGX3C
-    * zero-extend path -- `bitop [d32:reg0 s0=16:reg0 s1=16:imm#0]`,
-    * agx3_pack_binary.c:11641, the copy16 + clr pair -- which has no AGX3C
-    * form.  That gap is unrelated to threadgroup memory and no frontend in
-    * this tree compiles a shared-memory shader for AGX3C, so it is a separate
-    * fix; until then AGX3C keeps the over-stride it already had. */
-   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-       !nir->info.shared_element_index_agx)
+   /* Shared addresses arrive as byte offsets; AGX3 local memory instructions
+    * scale scalar element indices by their access width. */
+   if (is_agx3 && !nir->info.shared_element_index_agx)
       NIR_PASS(_, nir, agx_nir_lower_shared_element_index);
 
+
    /* AGX2 has native 64-bit add and 32x32->64 multiply forms, so the shared
-    * compiler options preserve them.  G17P has no grounded 64-bit add, so
+    * compiler options preserve them.  AGX3 has no grounded 64-bit add, so
     * lower it to low/add, carry, high/add.
     *
     * G17P's 32x32->64 multiply IS grounded (agx3_iadd_core.h, dest_64), but a
@@ -11768,18 +13013,17 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
     * their paths and frontends continue sharing agx_nir_options. */
    const nir_shader_compiler_options *original_options = nir->options;
    nir_shader_compiler_options agx3_options;
-   if (is_g17p) {
+   if (is_agx3) {
       agx3_options = *original_options;
-      agx3_options.lower_iabs = false;
-      /* G17P's exact saturate form is the witnessed standalone f32add.ep2
-       * operation. Preserve fsat so isel emits that form; the backend
-       * optimizer keeps it separate from producers with no saturation bit. */
-      agx3_options.lower_fsat = false;
-      /* The G17P integer encoder has no witnessed saturating-subtract form.
-       * Lower it to the canonical unsigned compare, subtract, and select;
-       * each resulting operation already has an Apple-exact encoding.  XFB
-       * uses this for bounds-safe counter arithmetic. */
-      agx3_options.lower_usub_sat = true;
+      if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C)
+         agx3_options.lower_bfloat16_conversions = true;
+      if (is_g17p) {
+         agx3_options.lower_iabs = false;
+         /* G17P's saturate form is a standalone f32add.ep2 operation. */
+         agx3_options.lower_fsat = false;
+         /* G17P lowers unsigned saturating subtract to compare/sub/select. */
+         agx3_options.lower_usub_sat = true;
+      }
       if (source_iadd64)
          agx3_options.lower_int64_options |= nir_lower_iadd64;
       if (source_imul64)
@@ -11816,14 +13060,24 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
          NIR_PASS(_, nir, agx_nir_lower_imul64);
    }
 
-   const bool uniform_atomics = !is_agx3;
+   /* The AGX3 packer admits SR 34/36 only on AGX3B and in these stages
+    * (agx3_pack_binary.c, GET_SR_COVERAGE); a geometry or tessellation
+    * evaluation shader with a uniform-address atomic would fail to compile.
+    */
+   const mesa_shader_stage stage = nir->info.stage;
+   const bool uniform_atomics =
+      !is_agx3 ||
+      (is_g17p && (stage == MESA_SHADER_COMPUTE ||
+                   stage == MESA_SHADER_VERTEX ||
+                   stage == MESA_SHADER_TESS_CTRL ||
+                   stage == MESA_SHADER_FRAGMENT));
 
    /* Apple's G17P loops are a
     * guard plus a bottom-tested body (fs/cs-apple.bin: the backedge target
     * is the first xorshift `sll`, `icmp` sits mid-body and `cndlt; b` end
     * the body). Rotate single-exit counted loops the same way before the
     * main optimization loop, so the copied test is optimized with the rest. */
-   if (is_g17p) {
+   if (is_agx3) {
       /* Uniform loops only (bw_copy regression, see nir_opt_loop.c). */
       nir_divergence_analysis(nir);
       bool rotated = false;
@@ -11840,13 +13094,14 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
     * plus a CNDEF (else flip) per invocation; inverted it is one inverted
     * CNDST, the empty else is removed by agx_opt_empty_else, and the guard
     * matches Apple's (fs-apple.bin +0x76 cndst invert, no else). */
-   if (is_g17p)
+   if (is_agx3)
       NIR_PASS(_, nir, nir_opt_if_empty_then);
 
    /* Only the one-sample AGX3B coverage path reads
     * bit 0 alone; sample loops and multisampled variants keep whole masks. */
    const bool discard_word =
-      is_g17p && key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      ((is_g17p && key->dev.encoding == AGX_ISA_ENCODING_AGX3B) ||
+       key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
       nir->info.stage == MESA_SHADER_FRAGMENT &&
       MAX2(key->fs.nr_samples, 1) == 1 && !key->fs.inside_sample_loop;
 
@@ -11854,6 +13109,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
       nir, key->dev.soft_fault, uniform_atomics,
       key->dev.encoding == AGX_ISA_ENCODING_AGX3B, discard_word,
       key->dev.chip == AGX_CHIP_G17P,
+      key->dev.encoding == AGX_ISA_ENCODING_AGX3C,
       form_preamble ? &info->push_count : NULL,
       (form_preamble && key->promote_textures) ? &info->texture_state_count
                                               : NULL,
@@ -11861,14 +13117,14 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
                                               : NULL,
       form_preamble ? key : NULL, info);
 
-   if (is_g17p && source_imul64)
+   if (is_agx3 && source_imul64)
       NIR_PASS(_, nir, agx_nir_lower_imul64);
 
-   if (is_g17p)
+   if (is_agx3)
       nir->options = original_options;
 
    /* AGX_G17P_UNIFORM_DMA: only when no preamble is formed. */
-   if (is_g17p && key->g17p_uniform_dma && !form_preamble &&
+   if (is_agx3 && key->g17p_uniform_dma && !form_preamble &&
        agx3_nir_lower_uniform_dma(nir, key, info))
       NIR_PASS(_, nir, nir_opt_dce);
 
@@ -11876,7 +13132,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
     * preamble extraction. Preserve their SSA uses inside the CProg, while
     * dropping writes to general uniform slots nobody reads anymore.
     */
-   if (is_g17p && form_preamble) {
+   if (is_agx3 && form_preamble) {
       NIR_PASS(_, nir, agx_nir_remove_unused_preamble_exports,
                key->reserved_preamble);
       NIR_PASS(_, nir, agx_nir_coalesce_preamble_half_exports,
@@ -11897,6 +13153,67 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
       assign_coefficient_regs(nir, &info->varyings.fs);
    }
 
+   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C)
+      NIR_PASS(_, nir, nir_opt_licm, NULL);
+
+   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       debug_get_bool_option("ASAHI_M5_SHARED_OFFSETS", true)) {
+      NIR_PASS(_, nir, agx_nir_opt_shared_byte_offsets);
+      NIR_PASS(_, nir, nir_opt_cse);
+      NIR_PASS(_, nir, nir_opt_dce);
+   }
+
+   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       debug_get_bool_option("ASAHI_M5_MINMAX_PROVENANCE", true)) {
+      NIR_PASS(_, nir, agx_nir_opt_canonical_minmax);
+      NIR_PASS(_, nir, nir_opt_dce);
+   }
+
+
+   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+      bool guarded = false;
+      NIR_PASS(guarded, nir, nir_shader_intrinsics_pass,
+               m5_lower_predicated_load, nir_metadata_none, NULL);
+      if (guarded) {
+         /* Address/ALU/phi lowering and dead-code cleanup do not move loads
+          * out of these arms. No speculation, LICM or if-flattening follows. */
+         NIR_PASS(_, nir, agx_nir_lower_address, true);
+         NIR_PASS(_, nir, nir_lower_int64);
+         NIR_PASS(_, nir, nir_opt_constant_folding);
+         NIR_PASS(_, nir, nir_lower_alu_to_scalar, NULL, NULL);
+         NIR_PASS(_, nir, nir_lower_load_const_to_scalar);
+         NIR_PASS(_, nir, nir_lower_all_phis_to_scalar);
+         NIR_PASS(_, nir, nir_opt_copy_prop);
+         NIR_PASS(_, nir, nir_opt_dce);
+         nir_divergence_analysis(nir);
+      }
+   }
+
+   /* Apple issues a loop body's samples in source order and consumes all of
+    * them after the last one (native robust_loop witness); Mesa consumed each
+    * right after its sample. Geometry-side M5 stages, like the M5 copy-loop
+    * schedule; see agx_nir_opt_sink_loop_sample_uses.c. */
+   if (key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       nir->info.stage != MESA_SHADER_FRAGMENT &&
+       nir->info.stage != MESA_SHADER_COMPUTE) {
+      NIR_PASS(_, nir, agx_nir_opt_sink_loop_sample_uses);
+
+      /* Every control-flow instruction retires the samples in flight, so
+       * also issue the body's unconditional samples ahead of the
+       * side-effect-free conditional blocks before them, then merge the
+       * conditional blocks this leaves adjacent; see
+       * agx_nir_opt_hoist_loop_samples. */
+      bool hoisted = false;
+      NIR_PASS(hoisted, nir, agx_nir_opt_hoist_loop_samples);
+      bool merged = false;
+      if (hoisted)
+         NIR_PASS(merged, nir, nir_opt_if_merge);
+      if (merged) {
+         NIR_PASS(_, nir, nir_opt_cse);
+         NIR_PASS(_, nir, nir_opt_dce);
+      }
+   }
+
    if (agx_should_dump(nir, AGX_DBG_SHADERS))
       nir_print_shader(nir, stdout);
 
@@ -11926,6 +13243,18 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    }
 
    info->local_size = nir->info.shared_size;
+   nir_foreach_function_impl(impl, nir) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type == nir_instr_type_intrinsic) {
+               nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+               if (intr->intrinsic == nir_intrinsic_cmat_muladd_agx &&
+                   intr->def.num_components == 8)
+                  info->uses_matrix16 = true;
+            }
+         }
+      }
+   }
    info->uses_resource_info_query = nir->info.uses_resource_info_query;
    info->uses_ray_traversal = nir->info.ray_queries != 0 || uses_scs;
    /* NIR ray_queries counts inline intersection_query variables, matching the
@@ -11962,7 +13291,7 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    if (agx_should_dump(nir, AGX_DBG_SHADERDB)) {
       agx2_stats_fprintf(stderr, _mesa_shader_stage_to_abbrev(nir->info.stage),
                          &out->info.stats);
-      if (key->dev.encoding == AGX_ISA_ENCODING_AGX3B && info->has_preamble) {
+      if (agx_isa_is_agx3(key->dev.encoding) && info->has_preamble) {
          fprintf(stderr, "AGX3 CProg: inputs=%u storage=%u scratch=%u "
                          "gpr_halves=%u bytes=%u native_tps=%u "
                          "staging_halfs=%u staging_known=%u\n",
@@ -12037,3 +13366,9 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    info->binary_size = binary.size;
    return true;
 }
+
+void
+agx_preprocess_nir_m5(nir_shader *nir)
+{
+   agx_preprocess_nir_internal(nir, false, true);
+}
diff --git a/src/asahi/compiler/agx_compile.h b/src/asahi/compiler/agx_compile.h
index c7bd651294b2..8e33177e4365 100644
--- a/src/asahi/compiler/agx_compile.h
+++ b/src/asahi/compiler/agx_compile.h
@@ -264,6 +264,7 @@ struct agx_shader_info {
     * opcode must set uses_ray_traversal as well; ray_queries alone is not a
     * future-proof proxy for full RT pipelines. ShaderInfo ID32 is independent
     * from traversal use and must have its own committed-path producer. */
+   bool uses_matrix16;
    bool uses_resource_info_query;
    bool uses_ray_traversal;
    bool ray_traversal_depth_nonzero;
@@ -381,6 +382,9 @@ struct agx_fs_shader_key {
     */
    bool ignore_tib_dependencies;
 
+   /* Driver background programs use the tile layer as their layer input. */
+   bool background;
+
    /* When dynamic sample shading is used, the fragment shader is wrapped in a
     * loop external to the API shader. This bit indicates that we are compiling
     * inside the sample loop, meaning the execution nesting counter is already
@@ -435,6 +439,7 @@ struct agx_shader_key {
    /* Number of reserved preamble slots at the start */
    unsigned reserved_preamble;
 
+   /* Uniform half containing the indexed texture heap; sampler follows at+4. */
    uint16_t g17p_texture_uniform_base;
 
    /* Whether scratch memory is available in the given shader stage */
@@ -501,6 +506,10 @@ struct agx_shader_key {
     * logical layout from the final AGX IR alone. */
    uint8_t g17p_rt0_eot_words;
 
+   /* Select the M5 tile-store ABI for a compiler-owned end-of-tile program. */
+   bool m5_rt0_eot;
+   bool m5_rt0_eot_store_and_resolve;
+
    /* Store the tile without resolving it. Apple's own builtin end-of-tile
     * factory emits a second program per selector whose only differences are
     * the PACK_IMGWIR wire width and the IMGWBLK dimension class (2D -> 2D
@@ -590,9 +599,17 @@ void agx_preprocess_nir_g17p(nir_shader *nir);
 /* Access private (scratch) memory in whole 32-bit words, the only width the
  * AGX3 stack protocol encodes. agx_preprocess_nir_g17p() runs it. */
 bool agx3_nir_lower_scratch_words(nir_shader *nir);
+void agx_preprocess_nir_m5(nir_shader *nir);
 bool agx_nir_lower_discard_zs_emit(nir_shader *s);
 bool agx_nir_lower_sample_mask(nir_shader *s);
+bool agx_nir_lower_sample_mask_m5(nir_shader *s);
+
+/* store_zs_agx base bit: the sample mask names exactly one covered sample,
+ * and the store sits in a covered-sample loop that zs_transaction_agx
+ * brackets (M5 only). The backend stages one DEPTHF word for it. */
+#define AGX_ZS_COVERED_SAMPLE (4)
 bool agx_nir_lower_sample_mask_deferred(nir_shader *s);
+bool agx_nir_lower_sample_mask_deferred_m5(nir_shader *s);
 bool agx_nir_lower_interpolation(nir_shader *s, bool layer_as_varying);
 
 bool agx_nir_lower_cull_distance_vs(struct nir_shader *s);
@@ -702,7 +719,9 @@ static const nir_shader_compiler_options agx_nir_options = {
                               BITFIELD_BIT(MESA_SHADER_TESS_EVAL) |
                               BITFIELD_BIT(MESA_SHADER_FRAGMENT),
    .support_indirect_outputs = (uint8_t)BITFIELD_MASK(MESA_SHADER_STAGES),
-   .max_samples = 4,
+   /* Compiler range bound covers M5's native eight-sample path. Device
+    * properties retain each chip's supported sample counts. */
+   .max_samples = 8,
    .lower_fquantize2f16 = true,
    .compact_arrays = true,
    .discard_is_demote = true,
diff --git a/src/asahi/compiler/agx_compiler.h b/src/asahi/compiler/agx_compiler.h
index effe1f7914e7..20461fb035fd 100644
--- a/src/asahi/compiler/agx_compiler.h
+++ b/src/asahi/compiler/agx_compiler.h
@@ -24,6 +24,9 @@ extern "C" {
 #define AGX_NUM_REGS (256)
 
 
+/* G17G/S native allocation ends at r125; r126/r127 are not GPRs. */
+#define AGX3C_NUM_REGS (252)
+
 /* u0-u255 inclusive, as pairs of 16-bits */
 #define AGX_NUM_UNIFORMS (512)
 
@@ -376,6 +379,7 @@ typedef struct {
       uint64_t imm;
       uint32_t writeout;
       uint32_t truth_table;
+      uint32_t matrix_type;
       uint32_t component;
       uint32_t channels;
       uint32_t bfi_mask;
@@ -440,6 +444,10 @@ typedef struct {
 
    /* TODO: Handle loads more efficiently */
    bool coherent : 1;
+   /* M5 image load from memory no store of this shader can write (NIR
+    * ACCESS_CAN_REORDER and ACCESS_RESTRICT, not coherent): the M5 copy
+    * schedule may move it across memory stores. */
+   bool memory_can_reorder : 1;
    /* AGX3 memory index scale excludes the access-format size. */
    bool memory_index_is_bytes : 1;
    /* Raw subword memory elements use packed payloads (device loads use
@@ -450,9 +458,9 @@ typedef struct {
     * and are never inferred from generic memory IR. */
    bool scs_zero_extend : 1;
    bool scs_c_load_marker : 1;
-   bool scs_c_status_load;
-   /* Native G17P traversal publishes completion on scoreboard slot 6. */
+   bool scs_c_status_load : 1;
    bool scs_rt_wait : 1;
+   bool scs_rt_completion : 1;
    uint8_t scs_address_selector;
 
    /* Final st_vary op */
@@ -465,6 +473,9 @@ typedef struct {
     * scoreboarding (everything but memory load/store and texturing). */
    unsigned scoreboard : 1;
 
+   /* NIR permits either sign for a zero result of this floating add. */
+   bool no_signed_zero : 1;
+
    /* Output modifiers */
    bool saturate : 1;
    unsigned mask : 4;
@@ -481,6 +492,13 @@ typedef struct {
    bool agx3_tex_coords_staged : 1;
    bool agx3_cube_scalar_coords  : 1;
 
+   /* M5 texel fetch or 2D sample with independent scalar coordinates:
+    * src[0] is x, src[6] is y and src[7] is the sample index of a
+    * multisampled read or the layer of an array read (null otherwise). The
+    * staging COPYs read each component where it lives, so no contiguous
+    * coordinate vector has to be collected for the read. */
+   bool agx3_split_coords : 1;
+
 } agx_instr;
 
 static inline uint32_t
@@ -667,6 +685,9 @@ typedef struct {
    /* Beginning of stack allocation used for parallel copy lowering */
    bool has_spill_pcopy_reserved;
 
+   /* M5 CEM owns halves 8..15, beyond CF/shuffle/parallel-copy scratch. */
+   bool agx3_cem_low_scratch;
+
    /* The remat-only spiller had to spill a value to scratch; register
     * allocation must finish with real spilling under the full budget. */
    bool remat_fallback_spilled;
@@ -692,6 +713,9 @@ typedef struct {
    /* Register allocation found no legal placement for a constrained value
     * (agx_register_allocate.c); the compile is refused after agx_ra(). */
    bool ra_failed;
+   /* Halves taken off the spill budget after a failed allocation, so the
+    * live-range shuffle gets free room (agx_compile_function_nir retries). */
+   unsigned ra_budget_cut;
 } agx_context;
 
 static inline void
@@ -1158,6 +1182,12 @@ agx_builder_insert(agx_cursor *cursor, agx_instr *I)
 bool agx3_tex_uniform_pair_unwitnessed(const agx_context *ctx,
                                        const agx_instr *I, agx_index texture,
                                        agx_index sampler);
+/* Whether replacing source s of an M5 main-program FADD/FMUL/FFMA with
+ * `uniform` gives a native uniform-operand form (agx3_uniform_float_encode). */
+bool agx3_m5_float_accepts_uniform(const agx_context *ctx, const agx_instr *I,
+                                   unsigned s, unsigned uniform,
+                                   enum agx_size size);
+
 bool agx_instr_accepts_uniform(enum agx_isa_encoding encoding,
                                enum agx_opcode op, unsigned src_index,
                                unsigned value, enum agx_size size);
@@ -1179,6 +1209,7 @@ bool agx3_lower_texture_publications(agx_context *ctx);
 bool agx3_fuse_fma_texture_publications(agx_context *ctx);
 bool agx3_lower_scalar_texture_publications(agx_context *ctx);
 bool agx3_lower_scalar_cube_coordinates(agx_context *ctx);
+bool agx3_split_texture_coordinates(agx_context *ctx);
 void agx_lower_uniform_sources(agx_context *ctx);
 void agx_opt_cse(agx_context *ctx);
 bool agx_opt_collect_split(agx_context *ctx);
@@ -1188,6 +1219,7 @@ bool agx_can_rematerialize_uniform(const agx_context *ctx, const agx_instr *I);
 bool agx_has_uniform_store(const agx_context *ctx);
 void agx_opt_compact_constants(agx_context *ctx);
 void agx_opt_promote_constants(agx_context *ctx);
+void agx_opt_promote_m5_float_constants(agx_context *ctx);
 void agx_dce(agx_context *ctx, bool partial);
 void agx_pressure_schedule(agx_context *ctx);
 void agx_ilp_schedule(agx_context *ctx);
@@ -1232,7 +1264,9 @@ bool agx3_analyze_preamble_uniform_inputs(agx_context *ctx, bool *has_inputs);
 static inline unsigned
 agx_reserved_registers(const agx_context *ctx)
 {
-   if (ctx->has_spill_pcopy_reserved)
+   if (ctx->agx3_cem_low_scratch)
+      return 16;
+   else if (ctx->has_spill_pcopy_reserved)
       return 8;
    else if (ctx->any_quad_divergent_shuffle)
       return ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX2 ? 2 : 4;
@@ -1243,9 +1277,11 @@ agx_reserved_registers(const agx_context *ctx)
 }
 
 void agx_spill(agx_context *ctx, unsigned k, bool remat_only);
+bool agx_fold_m5_spill_constant_phis(agx_context *ctx);
 void agx_repair_ssa(agx_context *ctx);
 void agx_reindex_ssa(agx_context *ctx);
 void agx_ra(agx_context *ctx);
+void agx_ra_without_remat(agx_context *ctx);
 bool agx3_is_async(const agx_instr *I);
 unsigned agx_calc_register_demand(agx_context *ctx, bool remat);
 unsigned agx3_tile_store_bank(agx_context *ctx);
@@ -1264,6 +1300,7 @@ void agx_pack_binary(agx_context *ctx, struct util_dynarray *emission);
  * encoding. */
 bool agx3_g17p_interleaved_zs(const agx_context *ctx);
 bool agx3_pack_binary(agx_context *ctx, struct util_dynarray *emission);
+const char *agx3_last_pack_refusal(void);
 
 #ifndef NDEBUG
 void agx_validate(agx_context *ctx, const char *after_str);
@@ -1294,6 +1331,50 @@ agx_is_float_src(const agx_instr *I, unsigned s)
    return info.is_float || (s < 2 && fcmp);
 }
 
+/* The M5 select with a uniform first comparand that native G17G
+ * fragments use (agx3_alu_fcmp_select32_gpu1 cmp0_uniform): a 32-bit
+ * result choosing a zero immediate or a 32-bit register after comparing an
+ * aligned 32-bit uniform with a 32-bit register, by GE or LE once the
+ * NaN-ignoring conditions are normalized as the packer does. Returns the
+ * normalized condition and whether the selected operands are exchanged. */
+static inline bool
+agx3_m5_fcmpsel_uniform_cmp0(const agx_instr *I, unsigned *condition,
+                             bool *swap)
+{
+   if (I->op != AGX_OPCODE_FCMPSEL || I->nr_dests != 1 || I->nr_srcs != 4 ||
+       I->saturate || I->dest[0].size != AGX_SIZE_32 ||
+       agx_channels(I->dest[0]) != 1)
+      return false;
+   const agx_index u = I->src[0], r = I->src[1];
+   if (u.type != AGX_INDEX_UNIFORM || u.size != AGX_SIZE_32 || (u.value & 1) ||
+       u.value >= 256 || u.abs || u.neg || u.memory ||
+       r.type == AGX_INDEX_UNIFORM || r.type == AGX_INDEX_IMMEDIATE ||
+       r.size != AGX_SIZE_32 || agx_channels(r) != 1 || r.abs || r.neg ||
+       r.memory)
+      return false;
+   unsigned cond = I->fcond;
+   bool exchanged = I->invert_cond;
+   if (cond == AGX_FCOND_LTN) {
+      cond = AGX_FCOND_GE;
+      exchanged = !exchanged;
+   } else if (cond == AGX_FCOND_GTN) {
+      cond = AGX_FCOND_LE;
+      exchanged = !exchanged;
+   }
+   if (cond != AGX_FCOND_GE && cond != AGX_FCOND_LE)
+      return false;
+   const agx_index imm = I->src[exchanged ? 3 : 2];
+   const agx_index reg = I->src[exchanged ? 2 : 3];
+   if (imm.type != AGX_INDEX_IMMEDIATE || imm.value != 0 || imm.abs ||
+       imm.neg || reg.type == AGX_INDEX_UNIFORM ||
+       reg.type == AGX_INDEX_IMMEDIATE || reg.size != AGX_SIZE_32 ||
+       agx_channels(reg) != 1 || reg.abs || reg.neg || reg.memory)
+      return false;
+   *condition = cond;
+   *swap = exchanged;
+   return true;
+}
+
 struct agx_copy {
    /* Base register destination of the copy */
    unsigned dest;
@@ -1311,10 +1392,18 @@ struct agx_copy {
 void agx_emit_parallel_copies(agx_builder *b, struct agx_copy *copies,
                               unsigned n);
 
+/* As above, with an optional free 32-bit GPR (two aligned halves that no
+ * copy reads or writes and that holds no live value at this point).  Register
+ * cycles are then broken with one move through it instead of a three-XOR
+ * swap per element.  Pass agx_null() when no such register is known. */
+void agx_emit_parallel_copies_temp(agx_builder *b, struct agx_copy *copies,
+                                   unsigned num_copies, agx_index temp);
+
 void agx_compute_liveness(agx_context *ctx);
 void agx_liveness_ins_update(struct u_sparse_bitset *live, agx_instr *I);
 
-bool agx_nir_opt_preamble(nir_shader *s, unsigned *sizes, bool g17p);
+bool agx_nir_opt_preamble(nir_shader *s, unsigned *sizes, bool g17p,
+                          bool agx3c);
 bool agx_nir_remove_unused_preamble_exports(nir_shader *s, unsigned reserved);
 bool agx_nir_coalesce_preamble_half_exports(nir_shader *s, unsigned reserved,
                                           unsigned allocated);
diff --git a/src/asahi/compiler/agx_dce.c b/src/asahi/compiler/agx_dce.c
index 2d18f0934de4..b62d6a6f4387 100644
--- a/src/asahi/compiler/agx_dce.c
+++ b/src/asahi/compiler/agx_dce.c
@@ -29,6 +29,13 @@ agx_dce(agx_context *ctx, bool partial)
          if (block->loop_header && I->op == AGX_OPCODE_PHI)
             break;
 
+         /* RT-completion loads wait for a traversal producer even when their
+          * byte result is unread. Ordinary dead SCS metadata loads remain
+          * removable. Keep a real destination, as the native load writes it. */
+         const bool rt_completion =
+            (I->op == AGX_OPCODE_SCS_LOAD_REGISTER_INDEXED_AGX3 ||
+             I->op == AGX_OPCODE_SCS_LOAD_REGISTER_BASE_AGX3) &&
+            I->scs_rt_completion;
          bool needed = false;
 
          agx_foreach_ssa_dest(I, d) {
@@ -39,13 +46,17 @@ agx_dce(agx_context *ctx, bool partial)
              */
             if (BITSET_TEST(seen, I->dest[d].value)) {
                needed = true;
-            } else if (partial) {
+            } else if (partial && I->op != AGX_OPCODE_ENTER_QUERY_M5 &&
+                       !rt_completion) {
+               /* The M5 query entry always writes its result register, so
+                * it must keep a real destination even when it is unread.
+                */
                I->dest[d] = agx_null();
             }
          }
 
          if (!needed && agx_opcodes_info[I->op].can_eliminate &&
-             !I->agx3_tex_coord_stage) {
+             !I->agx3_tex_coord_stage && !rt_completion) {
             agx_remove_instruction(I);
          } else {
             agx_foreach_ssa_src(I, s) {
diff --git a/src/asahi/compiler/agx_ir.c b/src/asahi/compiler/agx_ir.c
index 48555f39acce..09aa78fe7b95 100644
--- a/src/asahi/compiler/agx_ir.c
+++ b/src/asahi/compiler/agx_ir.c
@@ -43,7 +43,7 @@ agx_reconverged_mrt_tail(agx_context *ctx)
 {
    const bool multisample_epilog = ctx->key->secondary &&
       (ctx->key->fs.nr_samples == 2 || ctx->key->fs.nr_samples == 4);
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
        (ctx->key->secondary && !multisample_epilog) || ctx->key->no_stop ||
        ctx->key->fs.ignore_tib_dependencies ||
diff --git a/src/asahi/compiler/agx_lower_parallel_copy.c b/src/asahi/compiler/agx_lower_parallel_copy.c
index 0c673aa8b03a..e38fe454cdd9 100644
--- a/src/asahi/compiler/agx_lower_parallel_copy.c
+++ b/src/asahi/compiler/agx_lower_parallel_copy.c
@@ -171,7 +171,8 @@ split_32bit_copy(struct copy_ctx *ctx, struct agx_copy *entry)
 
 static void
 agx_emit_parallel_copies_for_class(agx_builder *b, struct agx_copy *copies,
-                                   unsigned num_copies, bool cls)
+                                   unsigned num_copies, bool cls,
+                                   agx_index temp)
 {
    /* First, lower away 64-bit copies to smaller chunks, since we don't have
     * 64-bit ALU so we always want to split.
@@ -281,6 +282,13 @@ agx_emit_parallel_copies_for_class(agx_builder *b, struct agx_copy *copies,
       next_copy->done = true;
    }
 
+   /* A GPR temporary lets a cycle be broken with one extra move: save the
+    * value a blocked copy is about to overwrite, retarget the copies that read
+    * it, and the cycle becomes a path for step 1.  Only GPR copies use it;
+    * memory copies keep the swap path. */
+   const bool use_temp = !cls && !agx_is_null(temp) &&
+                         temp.type == AGX_INDEX_REGISTER && !temp.memory;
+
    bool progress = true;
    while (progress) {
       progress = false;
@@ -333,6 +341,75 @@ agx_emit_parallel_copies_for_class(agx_builder *b, struct agx_copy *copies,
             progress = true;
          }
       }
+
+      if (progress || !use_temp)
+         continue;
+
+      /* Step 2b: only cycles remain.  Break one through the temporary: the
+       * value in a blocked copy's destination is saved, the copy is emitted,
+       * and every copy reading that destination now reads the temporary.
+       * The cycle is then a path, so step 1 resumes.  Costs k + 1 moves for a
+       * k-element cycle instead of 3 (k - 1) XORs.
+       */
+      for (unsigned i = 0; i < ctx->entry_count; i++) {
+         struct agx_copy *entry = &ctx->entries[i];
+         if (entry->done)
+            continue;
+
+         assert(is_real(entry));
+         if (entry->dest == entry->src.value) {
+            entry->done = true;
+            progress = true;
+            break;
+         }
+
+         const unsigned width = agx_size_align_16(entry->src.size);
+         agx_index saved = agx_register(temp.value, entry->src.size);
+         agx_index victim = agx_register(entry->dest, entry->src.size);
+         agx_mov_to(b, saved, victim);
+
+         do_copy(b, entry);
+         entry->done = true;
+         for (unsigned j = 0; j < width; j++) {
+            ctx->physreg_use_count[entry->src.value + j]--;
+            ctx->physreg_dest[entry->dest + j] = NULL;
+         }
+
+         /* Copies whose source lay (wholly or partly) inside the destination
+          * now read the saved value.  A 32-bit reader straddling the
+          * destination boundary is split first so each half can be redirected
+          * independently. */
+         if (width == 1) {
+            for (unsigned j = 0; j < ctx->entry_count; j++) {
+               struct agx_copy *blocking = &ctx->entries[j];
+               if (blocking->done || !is_real(blocking))
+                  continue;
+               if (blocking->src.value <= entry->dest &&
+                   blocking->src.value + 1 >= entry->dest &&
+                   agx_size_align_16(blocking->src.size) == 2)
+                  split_32bit_copy(ctx, blocking);
+            }
+         }
+
+         for (unsigned j = 0; j < ctx->entry_count; j++) {
+            struct agx_copy *blocking = &ctx->entries[j];
+            if (blocking->done || !is_real(blocking))
+               continue;
+            if (blocking->src.value >= entry->dest &&
+                blocking->src.value < entry->dest + width) {
+               const unsigned offset = blocking->src.value - entry->dest;
+               /* The temporary is live until the retargeted copies drain,
+                * which step 1 does before the next cycle break: the cycle is
+                * now a path whose every copy becomes unblocked in order. */
+               ctx->physreg_use_count[blocking->src.value]--;
+               blocking->src.value = temp.value + offset;
+               ctx->physreg_use_count[blocking->src.value]++;
+            }
+         }
+
+         progress = true;
+         break;
+      }
    }
 
    /* Step 3: resolve cycles through swapping.
@@ -427,8 +504,8 @@ agx_emit_parallel_copies_for_class(agx_builder *b, struct agx_copy *copies,
 }
 
 void
-agx_emit_parallel_copies(agx_builder *b, struct agx_copy *copies,
-                         unsigned num_copies)
+agx_emit_parallel_copies_temp(agx_builder *b, struct agx_copy *copies,
+                              unsigned num_copies, agx_index temp)
 {
    /* Emit copies fo reach register class separately because we don't have
     * register class awareness in the parallel copy lowering data structure.
@@ -436,6 +513,13 @@ agx_emit_parallel_copies(agx_builder *b, struct agx_copy *copies,
    if (num_copies == 0)
       return;
 
-   agx_emit_parallel_copies_for_class(b, copies, num_copies, false);
-   agx_emit_parallel_copies_for_class(b, copies, num_copies, true);
+   agx_emit_parallel_copies_for_class(b, copies, num_copies, false, temp);
+   agx_emit_parallel_copies_for_class(b, copies, num_copies, true, temp);
+}
+
+void
+agx_emit_parallel_copies(agx_builder *b, struct agx_copy *copies,
+                         unsigned num_copies)
+{
+   agx_emit_parallel_copies_temp(b, copies, num_copies, agx_null());
 }
diff --git a/src/asahi/compiler/agx_lower_pseudo.c b/src/asahi/compiler/agx_lower_pseudo.c
index 96d65e2e09f9..9e51b343a131 100644
--- a/src/asahi/compiler/agx_lower_pseudo.c
+++ b/src/asahi/compiler/agx_lower_pseudo.c
@@ -134,6 +134,32 @@ lower(agx_builder *b, agx_instr *I)
 
    /* Various instructions are implemented as bitwise truth tables */
    case AGX_OPCODE_MOV:
+      if (agx3 && agx_channels(I->dest[0]) > 1) {
+         const unsigned channels = agx_channels(I->dest[0]);
+         const unsigned stride = agx_size_align_16(I->dest[0].size);
+         const agx_index dest = I->dest[0], src = I->src[0];
+         assert(!dest.memory && !src.memory);
+         assert(src.type == AGX_INDEX_IMMEDIATE ||
+                (src.size == dest.size && agx_channels(src) == channels));
+         const bool backwards = src.type == AGX_INDEX_REGISTER &&
+            dest.value > src.value &&
+            dest.value < src.value + channels * stride;
+
+         for (unsigned i = 0; i < channels; ++i) {
+            const unsigned c = backwards ? channels - 1 - i : i;
+            agx_index d = dest, s = src;
+            d.channels_m1 = s.channels_m1 = 0;
+            d.value += c * stride;
+            if (s.type != AGX_INDEX_IMMEDIATE)
+               s.value += c * stride;
+            agx_instr *copy = agx_mov_to(b, d, s);
+            agx_instr *lowered = lower(b, copy);
+            assert(lowered);
+            (void)lowered;
+            agx_remove_instruction(copy);
+         }
+         return I;
+      }
       if (agx3 && I->src[0].type == AGX_INDEX_IMMEDIATE)
          return agx_mov_imm_to(b, I->dest[0], I->src[0].value);
       /*
@@ -261,7 +287,7 @@ agx_lower_pseudo(agx_context *ctx)
    agx_foreach_instr_global_safe(ctx, I) {
       agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
 
-      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+      if (ctx->key && agx_isa_is_agx3(ctx->key->dev.encoding))
          agx3_lower_constant_cf_compare(ctx, I);
 
       if (lower(&b, I))
diff --git a/src/asahi/compiler/agx_lower_uniform_sources.c b/src/asahi/compiler/agx_lower_uniform_sources.c
index bf061d3c059d..7f61e9ba9392 100644
--- a/src/asahi/compiler/agx_lower_uniform_sources.c
+++ b/src/asahi/compiler/agx_lower_uniform_sources.c
@@ -3,15 +3,19 @@
  * SPDX-License-Identifier: MIT
  */
 #include "agx_builder.h"
+#include "util/half_float.h"
 #include "agx_compiler.h"
 #include "agx_debug.h"
+#include "util/u_debug.h"
 #include "agx3_uniform_integer_instr.h"
 #include "util/u_debug.h"
+#include "agx3_uniform_alu.h"
+#include "agx3_isel_core.h"
 
 /*
  * Not all instructions can take uniforms. Memory instructions can take
- * uniforms, but only for their base (first) source and only in the
- * low-half of the uniform file.
+ * uniforms for their base source. Most forms use the lower half of the
+ * uniform file; M5 device loads also accept aligned bases in the upper half.
  *
  * This pass lowers invalid uniforms.
  */
@@ -27,7 +31,11 @@ agx_instr_accepts_uniform(enum agx_isa_encoding encoding, enum agx_opcode op,
    bool is_64 = size == AGX_SIZE_64;
    /* AGX3 memory bases address four-half uniform groups. A pointer packed
     * at a two-half boundary must be copied to a GPR pair before use. */
-   bool memory_base = !high &&
+   const bool m5_high_load_base =
+      encoding == AGX_ISA_ENCODING_AGX3C && op == AGX_OPCODE_DEVICE_LOAD &&
+      size == AGX_SIZE_64 && value <= 508 &&
+      debug_get_bool_option("ASAHI_M5_HIGH_UNIFORM_LOAD", true);
+   bool memory_base = (!high || m5_high_load_base) &&
       (encoding == AGX_ISA_ENCODING_AGX2 || !(value & 3));
 
    switch (op) {
@@ -61,6 +69,13 @@ agx_instr_accepts_uniform(enum agx_isa_encoding encoding, enum agx_opcode op,
              size == AGX_SIZE_32 && value < 512 && !(value & 1);
    case AGX_OPCODE_IMAGE_LOAD:
    case AGX_OPCODE_TEXTURE_LOAD:
+      /* The saved G17S Atrous programs read texture handles directly from
+       * aligned 64-bit uniforms (SMP kind 3), without per-lane copies.
+       * Keep the environment override for isolated regression controls. */
+      if (encoding == AGX_ISA_ENCODING_AGX3C && op == AGX_OPCODE_TEXTURE_LOAD &&
+          src_index == 3 &&
+          debug_get_bool_option("ASAHI_M5_UNIFORM_TEXTURE_OPERANDS", true))
+         return !high && !(value & 3) && value <= 252 && size == AGX_SIZE_64;
       /* G17P publishes a dynamic integer LOD through PACK_SMPIR from a GPR;
        * there is no witnessed uniform-file source for that operand.  Keep
        * the heap in uniforms, but materialize the LOD into a register. */
@@ -91,9 +106,13 @@ agx_instr_accepts_uniform(enum agx_isa_encoding encoding, enum agx_opcode op,
    case AGX_OPCODE_INTERSECT_RAY_GEN1:
       return encoding == AGX_ISA_ENCODING_AGX3B && src_index < 2 &&
              !high && !(value & 1) && value <= 508 && size == AGX_SIZE_64;
+   case AGX_OPCODE_INTERSECT_RAY_M5:
+      return encoding == AGX_ISA_ENCODING_AGX3C && src_index == 1 &&
+             size == AGX_SIZE_64 && !(value & 1) && value < 512;
    case AGX_OPCODE_SCS_LOAD_UNIFORM_INDEXED_AGX3:
    case AGX_OPCODE_SCS_STORE_UNIFORM_INDEXED_AGX3:
-      return encoding != AGX_ISA_ENCODING_AGX2 && !high &&
+      return (encoding == AGX_ISA_ENCODING_AGX3C ||
+              (encoding == AGX_ISA_ENCODING_AGX3B && !high)) &&
              src_index == (op == AGX_OPCODE_SCS_LOAD_UNIFORM_INDEXED_AGX3 ? 0 : 1) &&
              size == AGX_SIZE_64 && !(value & 1) && value < 512;
    case AGX_OPCODE_DEVICE_LOAD:
@@ -161,7 +180,9 @@ agx_instr_accepts_uniform(enum agx_isa_encoding encoding, enum agx_opcode op,
        * (fs-apple.bin +0xbc 0a832307060007020000: f18 = u6, f24 = 3), so
        * keep the uniform as the second operand instead of copying it into
        * a GPR on every iteration. */
-      return encoding == AGX_ISA_ENCODING_AGX3B &&
+      return (encoding == AGX_ISA_ENCODING_AGX3B ||
+              (encoding == AGX_ISA_ENCODING_AGX3C &&
+               debug_get_bool_option("ASAHI_M5_UNIFORM_BRANCH", true))) &&
              src_index == 1 && size == AGX_SIZE_32 && !(value & 1) &&
              value < 256;
    case AGX_OPCODE_PHI:
@@ -263,18 +284,417 @@ agx3_move_uniform_negation(agx_context *ctx, agx_instr *I)
    return I;
 }
 
+/* Modified M5 FMA uses the witnessed register-only modifier encoder.
+ * Convert minifloats exactly before RA, preserving the one fused operation.
+ * Unmodified immediate FMA keeps its existing compact operand forms. */
+static void
+agx3_lower_m5_modified_fma_immediates(agx_context *ctx, agx_instr *I)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       I->op != AGX_OPCODE_FFMA || I->nr_dests != 1 || I->nr_srcs != 3)
+      return;
+
+   bool modified = I->saturate;
+   agx_foreach_src(I, s)
+      modified |= I->src[s].abs || I->src[s].neg;
+   if (!modified)
+      return;
+
+   /* Native G17G terrain fragments saturate a*b + 0.5 with the
+    * minifloat addend in place (f51 = 32, f14 = 1, both multipliers plain
+    * GPRs): agx3_falu_fma_addend_imm_width_mod encodes exactly that, so
+    * keep the literal instead of rematerializing it before every use. */
+   if (I->saturate && I->dest[0].size == AGX_SIZE_32 &&
+       I->src[2].type == AGX_INDEX_IMMEDIATE && !I->src[2].abs &&
+       !I->src[2].neg) {
+      bool plain = true;
+      for (unsigned s = 0; s < 2; ++s) {
+         const agx_index x = I->src[s];
+         plain &= x.type == AGX_INDEX_NORMAL && x.size == AGX_SIZE_32 &&
+                  agx_channels(x) == 1 && !x.abs && !x.neg && !x.memory;
+      }
+      if (plain)
+         return;
+   }
+
+   agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+   agx_foreach_src(I, s) {
+      const agx_index literal = I->src[s];
+      if (literal.type != AGX_INDEX_IMMEDIATE)
+         continue;
+
+      agx_index value = agx_mov_imm(
+         &b, 32, fui(agx_minifloat_decode(literal.value)));
+      value.abs = literal.abs;
+      value.neg = literal.neg;
+      I->src[s] = value;
+   }
+}
+
+/* The direct M5 scalar LOD publisher consumes its half-float source.
+ * Final physical divergence disables discard hints, and post-optimizer SSA
+ * reuse may differ from the original NIR use count. Pack only those operands
+ * that cannot use the consuming publisher, after optimization and before RA. */
+DEBUG_GET_ONCE_BOOL_OPTION(m5_fold_constant_lod,
+                           "ASAHI_M5_FOLD_CONSTANT_LOD", true)
+
+static bool
+agx3_m5_exact_lod_immediate(float value, bool explicit_lod, uint8_t *immediate)
+{
+   if (!isfinite(value) || !agx_minifloat_exact(value) ||
+       value * 64.0f < -2048.0f || value * 64.0f > 2047.0f ||
+       (explicit_lod && value < 0.0f))
+      return false;
+
+   const uint8_t imm = agx_minifloat_encode(value);
+   const float decoded = agx_minifloat_decode(imm);
+   const int encoded = (int)(decoded * 64.0f);
+   /* Reject signed zero or fractions lost by the hardware immediate. */
+   if (fui(decoded) != fui(value) ||
+       fui((float)encoded / 64.0f) != fui(value))
+      return false;
+
+   *immediate = imm;
+   return true;
+}
+
+/* Recover an exactly representable immediate before live-parameter
+ * legalization gives a constant a register lifetime. */
+static void
+agx3_fold_m5_constant_lods(agx_context *ctx)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       ctx->stage != MESA_SHADER_FRAGMENT ||
+       !debug_get_option_m5_fold_constant_lod())
+      return;
+
+   agx_instr **defs = calloc(ctx->alloc, sizeof(*defs));
+   if (!defs)
+      return;
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_dest(I, d)
+         defs[I->dest[d].value] = I;
+   }
+
+   agx_foreach_instr_global(ctx, I) {
+      if (I->op != AGX_OPCODE_TEXTURE_SAMPLE || I->nr_srcs != 6 ||
+          (I->dim != AGX_DIM_2D && I->dim != AGX_DIM_3D) ||
+          I->shadow || I->offset || I->sparse || I->query_lod ||
+          I->gather != AGX_GATHER_NONE ||
+          (I->lod_mode != AGX_LOD_MODE_LOD_MIN &&
+           I->lod_mode != AGX_LOD_MODE_AUTO_LOD_BIAS))
+         continue;
+
+      agx_index lod = I->src[1];
+      if (lod.type != AGX_INDEX_NORMAL || lod.abs || lod.neg || lod.memory ||
+          agx_channels(lod) != 1)
+         continue;
+      agx_instr *def = defs[lod.value];
+      if (I->memory_is_packed) {
+         if (!def || def->op != AGX_OPCODE_PACK_SAMPLER_LOD_AGX3 ||
+             def->nr_srcs != 1 || lod.size != AGX_SIZE_32)
+            continue;
+         lod = def->src[0];
+         if (lod.type != AGX_INDEX_NORMAL || lod.abs || lod.neg || lod.memory ||
+             agx_channels(lod) != 1)
+            continue;
+         def = defs[lod.value];
+      }
+      if (!def || def->op != AGX_OPCODE_MOV_IMM ||
+          lod.size != AGX_SIZE_16 || def->nr_dests != 1 ||
+          def->dest[0].size != AGX_SIZE_16 || def->imm > UINT16_MAX)
+         continue;
+
+      uint8_t immediate;
+      if (!agx3_m5_exact_lod_immediate(_mesa_half_to_float(def->imm),
+                                      I->lod_mode == AGX_LOD_MODE_LOD_MIN,
+                                      &immediate))
+         continue;
+
+      I->src[1] = agx_immediate(immediate);
+      I->memory_is_packed = false;
+   }
+   free(defs);
+}
+
+static void
+agx3_lower_m5_live_scalar_lods(agx_context *ctx)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C)
+      return;
+
+   bool needed = false;
+   agx_foreach_instr_global(ctx, I) {
+      needed |= I->op == AGX_OPCODE_TEXTURE_SAMPLE && I->nr_srcs == 6 &&
+                !I->memory_is_packed &&
+                (I->dim == AGX_DIM_2D || I->dim == AGX_DIM_3D) &&
+                (I->lod_mode == AGX_LOD_MODE_LOD_MIN ||
+                 I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS) &&
+                I->src[1].type == AGX_INDEX_NORMAL &&
+                I->src[1].size == AGX_SIZE_16;
+   }
+   if (!needed)
+      return;
+
+   const unsigned alloc = ctx->alloc;
+   unsigned *uses = calloc(alloc, sizeof(*uses));
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s)
+         uses[I->src[s].value]++;
+   }
+
+   agx_index *packed = NULL;
+   agx_block **owner = NULL;
+   agx_foreach_instr_global_safe(ctx, I) {
+      if (I->op != AGX_OPCODE_TEXTURE_SAMPLE || I->nr_srcs != 6 ||
+          I->memory_is_packed ||
+          (I->dim != AGX_DIM_2D && I->dim != AGX_DIM_3D) ||
+          (I->lod_mode != AGX_LOD_MODE_LOD_MIN &&
+           I->lod_mode != AGX_LOD_MODE_AUTO_LOD_BIAS))
+         continue;
+
+      const agx_index lod = I->src[1];
+      if (lod.type != AGX_INDEX_NORMAL || lod.size != AGX_SIZE_16 ||
+          agx_channels(lod) != 1 ||
+          (!v_block->cf_divergent && uses[lod.value] == 1))
+         continue;
+
+      if (!owner) {
+         packed = calloc(alloc, sizeof(*packed));
+         owner = calloc(alloc, sizeof(*owner));
+      }
+
+      /* Share identical pure producers within this block. Sibling blocks
+       * cannot share a producer inserted at a use without dominance proof. */
+      if (owner[lod.value] != v_block) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         packed[lod.value] = agx_temp(ctx, AGX_SIZE_32);
+         agx_pack_sampler_lod_agx3_to(&b, packed[lod.value], lod);
+         owner[lod.value] = v_block;
+      }
+      I->src[1] = packed[lod.value];
+      I->memory_is_packed = true;
+   }
+   free(owner);
+   free(packed);
+   free(uses);
+}
+
+/* Validate the narrow native M5 FALU forms before retaining uniform
+ * operands. The encoder owns the exact source/modifier domain; dummy GPRs
+ * suffice here because allocation has not assigned their indices yet. */
+static bool
+m5_native_float_uniform(const agx_context *ctx, const agx_instr *I)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       ctx->is_preamble || I->nr_dests != 1 ||
+       I->dest[0].size != AGX_SIZE_32 || agx_channels(I->dest[0]) != 1 ||
+       (I->op != AGX_OPCODE_FADD && I->op != AGX_OPCODE_FMUL &&
+        I->op != AGX_OPCODE_FFMA))
+      return false;
+   const bool fma = I->op == AGX_OPCODE_FFMA;
+   if (I->nr_srcs != (fma ? 3 : 2))
+      return false;
+   struct agx3_uniform_float_src src[3] = {0};
+   agx_foreach_src(I, s) {
+      const agx_index v = I->src[s];
+      const bool uniform = v.type == AGX_INDEX_UNIFORM;
+      if (v.memory || agx_channels(v) != 1 ||
+          (v.type == AGX_INDEX_IMMEDIATE ? v.value > 255
+           : v.type != AGX_INDEX_NORMAL && !uniform) ||
+          (v.type != AGX_INDEX_IMMEDIATE && v.size != AGX_SIZE_32 &&
+           !(uniform && v.size == AGX_SIZE_16)))
+         return false;
+      src[s] = (struct agx3_uniform_float_src){
+         .kind = uniform ? AGX3_FLOAT_UNIFORM
+                 : v.type == AGX_INDEX_IMMEDIATE ? AGX3_FLOAT_IMMEDIATE
+                                                 : AGX3_FLOAT_REGISTER,
+         .value = v.type == AGX_INDEX_NORMAL ? 0 : v.value,
+         .abs = v.abs, .neg = v.neg,
+         .half = uniform && v.size == AGX_SIZE_16,
+      };
+   }
+   uint8_t out[16], length;
+   return agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C,
+      fma ? AGX3_UNIFORM_FMA
+          : I->op == AGX_OPCODE_FMUL ? AGX3_UNIFORM_FMUL : AGX3_UNIFORM_FADD,
+      0, src, I->saturate, false, 0, out, &length);
+}
+
+/*
+ * Every native M5 float form with a uniform operand also has a GPR operand,
+ * so an operation on uniforms and immediates alone has to copy a uniform. One
+ * copy suffices whenever the rest is then a native form: u0 - u1 is R + -U
+ * after copying u0, whereas copying u1 would leave U - R, which has none.
+ * Mark the uniforms that can stay when one is copied.
+ */
+static bool
+m5_float_uniforms_to_keep(const agx_context *ctx, const agx_instr *I,
+                          bool keep[3])
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       ctx->is_preamble || I->nr_srcs > 3 ||
+       (I->op != AGX_OPCODE_FADD && I->op != AGX_OPCODE_FMUL &&
+        I->op != AGX_OPCODE_FFMA))
+      return false;
+
+   unsigned uniforms = 0;
+   agx_foreach_src(I, s)
+      uniforms += I->src[s].type == AGX_INDEX_UNIFORM;
+   if (uniforms < 2)
+      return false;
+
+   agx_foreach_src(I, s) {
+      if (I->src[s].type != AGX_INDEX_UNIFORM)
+         continue;
+
+      /* The copy is a register of the uniform's size. The encoder query
+       * looks at a register's size and modifiers, not at its index. */
+      agx_instr candidate = *I;
+      agx_index sources[3];
+      memcpy(sources, I->src, I->nr_srcs * sizeof(*sources));
+      candidate.src = sources;
+      sources[s] =
+         agx_replace_index(sources[s], agx_get_index(0, I->src[s].size));
+
+      if (m5_native_float_uniform(ctx, &candidate)) {
+         agx_foreach_src(I, t)
+            keep[t] = t != s && I->src[t].type == AGX_INDEX_UNIFORM;
+         return true;
+      }
+   }
+
+   return false;
+}
+
+bool
+agx3_m5_float_accepts_uniform(const agx_context *ctx, const agx_instr *I,
+                              unsigned s, unsigned uniform,
+                              enum agx_size size)
+{
+   if (s >= I->nr_srcs || I->nr_srcs > 3 || uniform >= 512)
+      return false;
+   agx_instr candidate = *I;
+   agx_index sources[3];
+   memcpy(sources, I->src, I->nr_srcs * sizeof(*sources));
+   candidate.src = sources;
+   sources[s] = agx_replace_index(sources[s], agx_uniform(uniform, size));
+   return m5_native_float_uniform(ctx, &candidate);
+}
+
+/*
+ * M5 adds a uniform to a register with the uniform optionally negated, not
+ * the register: u - r has no native form and would copy u into a GPR. When
+ * every use of u - r is a float operation reading it as a plain register
+ * operand, compute r - u instead (a native R + -U) and negate it at each use,
+ * only when NIR permits a zero result of either sign. Equal finite operands
+ * otherwise change from +0 to -0. Minecraft's fog mixes the fragment colour
+ * toward the fog colour as (fog - c) * t + c, three times per fragment.
+ */
+static bool
+agx3_m5_negate_uniform_difference(agx_context *ctx, agx_instr *I)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C || ctx->is_preamble ||
+       I->op != AGX_OPCODE_FADD || I->nr_dests != 1 || I->nr_srcs != 2 ||
+       I->saturate || !I->no_signed_zero ||
+       I->dest[0].type != AGX_INDEX_NORMAL || I->dest[0].size != AGX_SIZE_32)
+      return false;
+   unsigned u = I->src[0].type == AGX_INDEX_UNIFORM ? 0 : 1;
+   const agx_index uni = I->src[u], reg = I->src[!u];
+   if (uni.type != AGX_INDEX_UNIFORM || uni.neg || uni.abs ||
+       reg.type != AGX_INDEX_NORMAL || !reg.neg || reg.abs)
+      return false;
+
+   const unsigned value = I->dest[0].value;
+   agx_foreach_instr_global(ctx, J) {
+      agx_foreach_ssa_src(J, s) {
+         if (J->src[s].value != value)
+            continue;
+         if ((J->op != AGX_OPCODE_FADD && J->op != AGX_OPCODE_FMUL &&
+              J->op != AGX_OPCODE_FFMA) || J->src[s].abs)
+            return false;
+      }
+   }
+
+   agx_index r = reg, n = uni;
+   r.neg = false;
+   n.neg = true;
+   I->src[0] = r;
+   I->src[1] = n;
+   agx_foreach_instr_global(ctx, J) {
+      agx_foreach_ssa_src(J, s) {
+         if (J->src[s].value == value)
+            J->src[s].neg = !J->src[s].neg;
+      }
+   }
+   return true;
+}
+
+/* Native M5 bounded-buffer code uses uniform compare/select operands.
+ * Query the encoder's full32 contract before retaining any of them. */
+static bool
+m5_native_isel_uniform(const agx_context *ctx, const agx_instr *I)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       ctx->is_preamble || I->op != AGX_OPCODE_ICMPSEL || I->saturate ||
+       I->invert_cond || I->nr_dests != 1 ||
+       I->nr_srcs != 4 || I->dest[0].size != AGX_SIZE_32 ||
+       agx_channels(I->dest[0]) != 1 || I->src[0].type != AGX_INDEX_NORMAL)
+      return false;
+   bool uniform = false;
+   for (unsigned i = 0; i < 4; ++i) {
+      const agx_index x = I->src[i];
+      uniform |= x.type == AGX_INDEX_UNIFORM;
+      if (x.memory || x.abs || x.neg || agx_channels(x) != 1 ||
+          (x.type != AGX_INDEX_IMMEDIATE && x.size != AGX_SIZE_32) ||
+          (x.type != AGX_INDEX_NORMAL && x.type != AGX_INDEX_UNIFORM &&
+           x.type != AGX_INDEX_IMMEDIATE))
+         return false;
+   }
+   if (!uniform)
+      return false;
+   struct agx3_isel_desc d = {
+      .dest = 0, .cmp0 = 0,
+      .cmp1 = I->src[1].type == AGX_INDEX_NORMAL ? 0 : I->src[1].value,
+      .cmp1_immediate = I->src[1].type == AGX_INDEX_IMMEDIATE,
+      .cmp1_uniform = I->src[1].type == AGX_INDEX_UNIFORM,
+      .cond = I->icond == AGX_ICOND_UEQ ? AGX3_ISEL_ICOND_EQ :
+                                       (enum agx3_isel_icond)I->icond,
+      .explicit_hints = true,
+   };
+   enum agx3_isel_operand *kind[2] = {&d.true_kind, &d.false_kind};
+   uint16_t *value[2] = {&d.sel_true, &d.sel_false};
+   for (unsigned i = 0; i < 2; ++i) {
+      const agx_index x = I->src[2 + i];
+      *kind[i] = x.type == AGX_INDEX_UNIFORM ? AGX3_ISEL_UNIFORM :
+                 x.type == AGX_INDEX_IMMEDIATE ? AGX3_ISEL_IMMEDIATE :
+                                                AGX3_ISEL_REGISTER;
+      *value[i] = x.type == AGX_INDEX_NORMAL ? 0 : x.value;
+   }
+   uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length;
+   return agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length);
+}
+
 void
 agx_lower_uniform_sources(agx_context *ctx)
 {
    agx_foreach_instr_global_safe(ctx, I) {
+      agx3_lower_m5_modified_fma_immediates(ctx, I);
+
       if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
          I = agx3_move_uniform_negation(ctx, I);
 
       agx_instr *at = I;
       struct agx3_uniform_integer_desc integer_desc;
       const bool native_integer =
-         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-         agx3_uniform_integer_instruction(I, false, &integer_desc);
+         agx3_uniform_integer_for_encoding(ctx->key->dev.encoding,
+                                           I, false, &integer_desc);
+      if (!m5_native_float_uniform(ctx, I))
+         agx3_m5_negate_uniform_difference(ctx, I);
+      const bool native_m5_float = m5_native_float_uniform(ctx, I);
+      bool keep_m5_float_uniform[3] = {false};
+      if (!native_m5_float)
+         m5_float_uniforms_to_keep(ctx, I, keep_m5_float_uniform);
       bool copy_float_uniforms = false;
       unsigned first_uniform = UINT_MAX;
       const bool half_float = I->op == AGX_OPCODE_HADD ||
@@ -333,6 +753,16 @@ agx_lower_uniform_sources(agx_context *ctx)
        * GPR per use; prefer the selects when both kinds are present, since
        * a uniform comparand is only witnessed beside register selects. */
       bool keep_isel_uniform[4] = {false};
+      unsigned m5_fsel_cond;
+      bool m5_fsel_swap;
+      const bool m5_fsel_uniform =
+         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+         !ctx->is_preamble &&
+         agx3_m5_fcmpsel_uniform_cmp0(I, &m5_fsel_cond, &m5_fsel_swap);
+      if (m5_native_isel_uniform(ctx, I)) {
+         for (unsigned i = 0; i < 4; ++i)
+            keep_isel_uniform[i] = I->src[i].type == AGX_INDEX_UNIFORM;
+      }
       if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
           !ctx->is_preamble &&
           I->op == AGX_OPCODE_ICMPSEL && I->nr_dests == 1 &&
@@ -366,6 +796,22 @@ agx_lower_uniform_sources(agx_context *ctx)
          I->op == AGX_OPCODE_TEXTURE_SAMPLE &&
          agx3_tex_uniform_pair_unwitnessed(ctx, I, I->src[3], I->src[4]);
 
+      /* M5 fragment SMP reads both its texture and its sampler handle from
+       * aligned 64-bit uniforms (f97/f103=3 and f126/f132=3) when Metal
+       * passes them through an argument buffer; the native
+       * G17G compiler compiles texture2d::sample, level and bias that way.
+       * Only that pair is witnessed: keep both or copy both. */
+      bool m5_tex_uniform_pair =
+         ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+         ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_preamble &&
+         I->op == AGX_OPCODE_TEXTURE_SAMPLE && I->nr_srcs >= 5;
+      for (unsigned j = 3; m5_tex_uniform_pair && j < 5; ++j) {
+         const agx_index x = I->src[j];
+         m5_tex_uniform_pair = x.type == AGX_INDEX_UNIFORM &&
+            x.size == AGX_SIZE_64 && !(x.value & 3) && x.value <= 252 &&
+            !x.abs && !x.neg && !x.memory;
+      }
+
       /* Exports end the block and admit no instruction between them; place
        * their materialized sources before the first export of the run. */
       if (I->op == AGX_OPCODE_EXPORT) {
@@ -432,7 +878,10 @@ agx_lower_uniform_sources(agx_context *ctx)
           * in place instead of copying it into a GPR first; the packer
           * emits agx3_cf_icmp_uniform_imm() for either operand order. */
          const bool uniform_imm_compare =
-            ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+            (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+             (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+              ctx->stage == MESA_SHADER_COMPUTE && !ctx->is_preamble &&
+              debug_get_bool_option("ASAHI_M5_UNIFORM_BRANCH", true))) &&
             is_int_compare && I->nr_srcs == 2 &&
             I->src[s].type == AGX_INDEX_UNIFORM &&
             I->src[s].size == AGX_SIZE_32 && !(I->src[s].value & 1) &&
@@ -445,7 +894,9 @@ agx_lower_uniform_sources(agx_context *ctx)
             is_int_compare && !uniform_imm_compare &&
             (I->src[0].type != AGX_INDEX_NORMAL ||
              I->src[0].size != AGX_SIZE_32 || I->src[0].abs ||
-             I->src[0].neg);
+             I->src[0].neg ||
+             (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+              (ctx->stage != MESA_SHADER_COMPUTE || ctx->is_preamble)));
          const bool copy_tex = copy_tex_uniform && (s == 3 || s == 4);
          /* A G17P select compares a uniform (a half, or an aligned word)
           * against an immediate zero directly, as Apple does for constant
@@ -528,8 +979,11 @@ agx_lower_uniform_sources(agx_context *ctx)
              (s == first_uniform ||
               copy_minmax || copy_float_uniforms || copy_float_modifier ||
               copy_compare_uniform || copy_tex ||
-              !(native_integer || uniform_imm_compare ||
+              !(native_integer || native_m5_float || uniform_imm_compare ||
+                (s < 3 && keep_m5_float_uniform[s]) ||
                 (s < 4 && keep_isel_uniform[s]) ||
+                (m5_tex_uniform_pair && (s == 3 || s == 4)) ||
+                (m5_fsel_uniform && s == 0) ||
                 agx_instr_accepts_uniform(ctx->key->dev.encoding, I->op, s,
                                          I->src[s].value, I->src[s].size)))) {
             agx_index idx = I->src[s];
@@ -572,4 +1026,9 @@ agx_lower_uniform_sources(agx_context *ctx)
          }
       }
    }
+
+   agx3_fold_m5_constant_lods(ctx);
+
+   /* Uniform materialization can also create a scalar LOD SSA source. */
+   agx3_lower_m5_live_scalar_lods(ctx);
 }
diff --git a/src/asahi/compiler/agx_nir.h b/src/asahi/compiler/agx_nir.h
index ce0fcf3d8c8d..d38cdbb132cd 100644
--- a/src/asahi/compiler/agx_nir.h
+++ b/src/asahi/compiler/agx_nir.h
@@ -19,8 +19,9 @@ struct nir_def;
 #define AGX_NIR_MEMORY_PACKED_COUNT_SHIFT 4u
 #define AGX_NIR_MEMORY_PACKED_COUNT_MASK (7u << AGX_NIR_MEMORY_PACKED_COUNT_SHIFT)
 bool agx_nir_lower_address(struct nir_shader *shader, bool byte_indices);
+bool agx_nir_preserve_position_order(struct nir_shader *shader);
 bool agx_nir_scalarize_int64(struct nir_shader *shader);
-bool agx_nir_lower_divmod64(struct nir_shader *shader);
+bool agx_nir_lower_divmod64(struct nir_shader *shader, bool native_add64);
 bool agx_nir_lower_algebraic_late(struct nir_shader *shader);
 bool agx_nir_fold_pixel_coord(struct nir_shader *shader);
 bool agx_nir_fold_extract16_bfe(struct nir_shader *shader);
@@ -28,8 +29,12 @@ bool agx_nir_cleanup_amul(struct nir_shader *shader);
 bool agx_nir_fuse_lea(struct nir_shader *shader);
 bool agx_nir_lower_lea(struct nir_shader *shader);
 bool agx_nir_fuse_selects(struct nir_shader *shader);
+bool agx_nir_m5_split_or_selects(struct nir_shader *shader);
 bool agx_nir_opt_select_load(struct nir_shader *shader);
+bool agx_nir_opt_sink_loop_sample_uses(struct nir_shader *shader);
+bool agx_nir_opt_hoist_loop_samples(struct nir_shader *shader);
 bool agx_nir_fuse_algebraic_late(struct nir_shader *shader);
+bool agx_nir_fuse_doubled_add(struct nir_shader *shader);
 bool agx_nir_fence_images(struct nir_shader *shader);
 bool agx_nir_lower_layer(struct nir_shader *s);
 bool agx_nir_lower_clip_distance(struct nir_shader *s);
@@ -46,8 +51,12 @@ struct nir_def *agx_nir_quad_ddy(struct nir_builder *b, struct nir_def *v);
  * run it as early as the optimizer can use it and agx_compile_shader_nir()
  * still guarantees it for any AGX3 target. */
 bool agx_nir_lower_shared_element_index(struct nir_shader *s);
+bool agx_nir_opt_shared_byte_offsets(struct nir_shader *s);
 bool agx_nir_lower_fminmax(struct nir_shader *s, bool ftz_only);
+bool agx_nir_lower_word_bounds(struct nir_shader *s);
+bool agx_nir_share_bounds(struct nir_shader *s);
 bool agx_nir_lower_atomics(struct nir_shader *s, bool native_fadd);
+bool agx_nir_opt_canonical_minmax(struct nir_shader *s);
 
 bool agx_nir_lower_texture_early(struct nir_shader *s, bool support_lod_bias);
 bool agx_nir_lower_g17p_helper_samples(struct nir_shader *s);
@@ -137,6 +146,7 @@ agx_g17p_pack_image_atomic_sw(uint64_t level_offset_B, unsigned tile_width_px,
 #define AGX_G17P_EMRT_ALIGNED_WIDTH_OFFSET 80
 /* Declines without changing the shader, so callers need no copy. */
 bool agx_nir_lower_g17p_indexed_samplers(struct nir_shader *s);
+bool agx_nir_lower_g17g_indexed_reads(struct nir_shader *s);
 /* Packed native32 array handles are byte offsets; sampler handles already
  * name their native entry, including the reserved texel-fetch sampler0. */
 bool agx_nir_lower_g17p_native32_samplers(struct nir_shader *s, uint8_t texture_set);
@@ -220,7 +230,9 @@ bool agx_nir_lower_g17p_sampled_storage(
 bool agx_nir_lower_to_per_sample_agx2(struct nir_shader *shader);
 bool agx_nir_lower_monolithic_msaa_agx2(struct nir_shader *shader, uint8_t nr_samples);
 
+bool agx_nir_fma_rte_applies(const struct nir_shader *shader);
 bool agx_nir_lower_fma_rte(struct nir_shader *shader);
 struct nir_def *agx_nir_fma_rte_fixup(struct nir_builder *b, struct nir_def *a,
                                     struct nir_def *c, struct nir_def *z,
                                     struct nir_def *native);
+bool agx_nir_lower_cmat(struct nir_shader *s);
diff --git a/src/asahi/compiler/agx_nir_algebraic.py b/src/asahi/compiler/agx_nir_algebraic.py
index 6425ad415cfc..742c7fb1e793 100644
--- a/src/asahi/compiler/agx_nir_algebraic.py
+++ b/src/asahi/compiler/agx_nir_algebraic.py
@@ -234,6 +234,14 @@ def isubshl(x, y, s):
     (('iadd', a, ('ishl(is_used_once)', b, '#c')), imad(b, ('ishl', 1, c), a)),
 ]
 
+# A value added twice is one shifted add: reassociation groups the duplicated
+# sources of a chain as x + (y + y), an accumulation it leaves alone adds
+# them in turn as (x + y) + y.
+fuse_doubled_add = [
+    (('iadd', 'a@32', ('iadd(is_used_once)', 'b@32', b)), iaddshl(a, b, 1)),
+    (('iadd', ('iadd(is_used_once)', 'a@32', 'b@32'), b), iaddshl(a, b, 1)),
+]
+
 # Discard lowering generates this pattern, clean it up
 ixor_bcsel = [
    (('ixor', ('bcsel', a, '#b', '#c'), '#d'),
@@ -439,6 +447,8 @@ def run():
                                       fold_pixel_coord).render())
     print(nir_algebraic.AlgebraicPass("agx_nir_fold_extract16_bfe",
                                       fold_extract16_bfe).render())
+    print(nir_algebraic.AlgebraicPass("agx_nir_fuse_doubled_add",
+                                      fuse_doubled_add).render())
     print(nir_algebraic.AlgebraicPass("agx_nir_fuse_algebraic_late",
                                       fuse_extr + fuse_ubfe +
                                       fuse_imad + ixor_bcsel +
diff --git a/src/asahi/compiler/agx_nir_gen1_mesh.c b/src/asahi/compiler/agx_nir_gen1_mesh.c
new file mode 100644
index 000000000000..da49bdf950a0
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_gen1_mesh.c
@@ -0,0 +1,994 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT */
+#include "agx_nir_m5_mesh.h"
+#include "compiler/nir/nir_builder.h"
+#include "compiler/nir/nir_control_flow.h"
+#include "util/u_math.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#define M5_REFUSE() do { if (getenv("ASAHI_MESH_CONTRACT_DEBUG")) fprintf(stderr, "m5 mesh refuse: %s:%d\n", __func__, __LINE__); return false; } while (0)
+
+struct mesh_layout {
+   unsigned output_base, primitive_count, vertex_count, position;
+   unsigned prim_vertices;
+   unsigned plane_stride, planes, bytes;
+   bool storage_probe;
+   uint64_t primitive_audit_va;
+   const struct agx_m5_mesh_varyings *varyings;
+};
+
+static uint64_t
+primitive_audit_va(void)
+{
+   const char *enable = getenv("ASAHI_MESH_PP_AUDIT");
+   const char *address = getenv("ASAHI_MESH_MARKER_VA");
+   return enable && !strcmp(enable, "1") && address
+      ? strtoull(address, NULL, 0) : 0;
+}
+
+static bool
+primitive_audit_fan(void)
+{
+   const char *fan = getenv("ASAHI_MESH_PP_AUDIT_FAN");
+   return fan && !strcmp(fan, "1");
+}
+
+static unsigned
+primitive_audit_primitives(void)
+{
+   return primitive_audit_fan() ? 4 : 2;
+}
+
+static unsigned
+primitive_audit_lanes(void)
+{
+   return primitive_audit_fan() ? 8 : 4;
+}
+
+static unsigned
+primitive_audit_lane_first(void)
+{
+   return 16 * 4 * primitive_audit_primitives();
+}
+
+static unsigned
+primitive_audit_fragment_first(void)
+{
+   return primitive_audit_lane_first() + 8 * 4 * primitive_audit_lanes();
+}
+
+static bool
+native_primitive_address(void)
+{
+   const char *enable = getenv("ASAHI_MESH_NATIVE_PRIMITIVE_ADDRESS_DIAGNOSTIC");
+   return enable && !strcmp(enable, "1");
+}
+
+static bool
+native_page_carrier(void)
+{
+   const char *enable = getenv("ASAHI_MESH_NATIVE_PAGE_CARRIER");
+   return enable && !strcmp(enable, "1");
+}
+
+static unsigned
+page_carrier_words(void)
+{
+   return native_page_carrier() ? 1 : 2;
+}
+
+static void
+primitive_capture(nir_builder *b, uint64_t va, unsigned first,
+                  nir_def *record, unsigned words, nir_def **values)
+{
+   nir_def *address = nir_iadd(b, nir_imm_int64(b, va + 32 + 4 * first),
+      nir_imul_imm(b, nir_u2u64(b, record), 4 * words));
+   for (unsigned c = 0; c < words; ++c) {
+      if (values[c])
+         nir_global_atomic(b, 32, nir_iadd_imm(b, address, 4 * c),
+            nir_u2u32(b, values[c]), .atomic_op = nir_atomic_op_xchg);
+   }
+}
+
+static void
+primitive_order(nir_builder *b)
+{
+   nir_barrier(b, .execution_scope = SCOPE_NONE, .memory_scope = SCOPE_WORKGROUP,
+      .memory_semantics = NIR_MEMORY_ACQ_REL,
+      .memory_modes = nir_var_mem_global | nir_var_mem_shared);
+}
+
+static unsigned
+prim_vertices(const nir_shader *s)
+{
+   switch (s->info.mesh.primitive_type) {
+   case MESA_PRIM_POINTS:
+      return 1;
+   case MESA_PRIM_LINES:
+      return 2;
+   case MESA_PRIM_TRIANGLES:
+      return 3;
+   default:
+      return 0;
+   }
+}
+
+static struct mesh_layout
+mesh_layout(const nir_shader *s, const struct agx_m5_mesh_varyings *varyings)
+{
+   const unsigned base = ALIGN_POT(s->info.shared_size, 64);
+   
+
+
+
+
+   const unsigned count = ALIGN_POT(prim_vertices(s) * s->info.mesh.max_primitives_out, 16);
+   const unsigned position = count + 16;
+   const unsigned stride = ALIGN_POT(s->info.mesh.max_vertices_out, 4) * 4;
+   return (struct mesh_layout) {
+      .primitive_audit_va = primitive_audit_va(),
+      .prim_vertices = prim_vertices(s),
+      .output_base = base, .primitive_count = base + count,
+      .vertex_count = base + count + 4, .position = base + position,
+      .plane_stride = stride, .planes = varyings->size,
+      .bytes = ALIGN_POT(position + varyings->size * stride, 64),
+      .storage_probe = getenv("ASAHI_MESH_PP_PROBE") != NULL &&
+                       !strcmp(getenv("ASAHI_MESH_PP_PROBE"), "1"),
+      .varyings = varyings,
+   };
+}
+
+static unsigned
+vertex_output_slot(nir_intrinsic_instr *i)
+{
+   return nir_intrinsic_io_semantics(i).location + nir_src_as_uint(i->src[2]);
+}
+
+
+static unsigned
+first_plane(const struct agx_m5_mesh_varyings *varyings, unsigned slot)
+{
+   return slot == VARYING_SLOT_POS ? 0 : varyings->first[slot];
+}
+
+static nir_def *
+widen_output(nir_builder *b, nir_intrinsic_instr *i, nir_def *value)
+{
+   
+
+
+   if (value->bit_size == 32)
+      return value;
+   nir_alu_type type = nir_alu_type_get_base_type(nir_intrinsic_src_type(i));
+   return nir_convert_to_bit_size(b, value, type, 32);
+}
+
+static bool
+lower(nir_builder *b,nir_intrinsic_instr *i,void *data)
+{
+   const struct mesh_layout *layout = data;
+   const unsigned output_base = layout->output_base;
+   switch(i->intrinsic) {
+   case nir_intrinsic_store_per_vertex_output: {
+      b->cursor=nir_before_instr(&i->instr);
+      nir_def *element=nir_imul_imm(b,i->src[1].ssa,4);
+      const unsigned slot=vertex_output_slot(i);
+      if (slot != VARYING_SLOT_POS && !layout->varyings->components[slot])
+         break;
+      const unsigned first=first_plane(layout->varyings,slot);
+      unsigned component=nir_intrinsic_component(i);
+      nir_def *value=widen_output(b,i,i->src[0].ssa), *probe=NULL;
+      if (slot == VARYING_SLOT_VAR0 && layout->storage_probe) {
+         
+
+
+
+         nir_def *storage=nir_load_mesh_primitive_storage_agx(b);
+         probe=nir_vec4(b,nir_channel(b,storage,0),nir_channel(b,storage,1),
+                        nir_channel(b,nir_load_workgroup_id(b),0),
+                        nir_load_local_invocation_index(b));
+      }
+      u_foreach_bit(c,nir_intrinsic_write_mask(i)) {
+         nir_def *address=nir_iadd_imm(b,element,
+            layout->position+layout->plane_stride*(first+component+c));
+         nir_store_shared(b,probe ? nir_channel(b,probe,component+c)
+                                  : nir_channel(b,value,c),address,
+                            .write_mask=1,.align_mul=4);
+         if (slot == VARYING_SLOT_CLIP_DIST0 || slot == VARYING_SLOT_CLIP_DIST1) {
+            const unsigned clip = 4 * (slot - VARYING_SLOT_CLIP_DIST0) + component + c;
+            if (clip < layout->varyings->clip_count) {
+               nir_store_shared(b, nir_channel(b, value, c),
+                  nir_iadd_imm(b, element, layout->position + layout->plane_stride *
+                                              (layout->varyings->clip_first + clip)),
+                  .write_mask = 1, .align_mul = 4);
+            }
+         }
+      }
+      break;
+   }
+   case nir_intrinsic_store_per_primitive_output: {
+      b->cursor=nir_before_instr(&i->instr);
+      const unsigned slot = vertex_output_slot(i);
+      if (slot != VARYING_SLOT_PRIMITIVE_INDICES) {
+         nir_def *storage = nir_pack_64_2x32(b, nir_load_mesh_primitive_storage_agx(b));
+         nir_def *offset = nir_imul_imm(b, nir_u2u64(b, i->src[1].ssa),
+                                      layout->varyings->primitive_stride_B);
+         nir_def *value = widen_output(b, i, i->src[0].ssa);
+         u_foreach_bit(c, nir_intrinsic_write_mask(i)) {
+            nir_def *address = nir_iadd_imm(b, nir_iadd(b, storage, offset),
+               4 * (layout->varyings->primitive_first[slot] +
+                    nir_intrinsic_component(i) + c));
+            nir_store_global(b, nir_channel(b, value, c), address,
+                             .write_mask = 1, .align_mul = 4);
+            if (layout->primitive_audit_va && c == 0) {
+               nir_def *group = nir_channel(b, nir_load_workgroup_id(b), 0);
+               const unsigned primitives = primitive_audit_primitives();
+               nir_push_if(b, nir_iand(b, nir_ult_imm(b, group, 4),
+                                      nir_ult_imm(b, i->src[1].ssa, primitives)));
+               primitive_order(b);
+               nir_def *post = nir_load_global(b, 1, 32, address,
+                  .align_mul = 4, .access = ACCESS_VOLATILE);
+               nir_def *fields[16] = {nir_imm_int(b, 0x50510001), group,
+                  i->src[1].ssa, nir_load_local_invocation_index(b),
+                  nir_u2u32(b, storage), nir_u2u32(b, nir_ushr_imm(b, storage, 32)),
+                  nir_u2u32(b, address), nir_u2u32(b, nir_ushr_imm(b, address, 32)),
+                  nir_imm_int(b, nir_intrinsic_component(i) + c), nir_channel(b, value, c), post,
+                  nir_imm_int(b, 0x50510002)};
+               primitive_capture(b, layout->primitive_audit_va, 0,
+                  nir_iadd(b, nir_imul_imm(b, group, primitives), i->src[1].ssa), 16, fields);
+               nir_pop_if(b, NULL);
+            }
+         }
+         break;
+      }
+      nir_def *element=nir_imul_imm(b,i->src[1].ssa,layout->prim_vertices);
+      unsigned component=nir_intrinsic_component(i);
+      u_foreach_bit(c,nir_intrinsic_write_mask(i)) {
+         nir_def *index=nir_channel(b,i->src[0].ssa,c);
+         
+
+
+         nir_def *packed=nir_u2u8(b,index);
+         nir_store_shared(b,packed,nir_iadd_imm(b,element,output_base+component+c),
+                            .write_mask=1,.align_mul=1);
+      }
+      break;
+   }
+   case nir_intrinsic_set_vertex_and_primitive_count: {
+      b->cursor=nir_before_instr(&i->instr);
+      nir_push_if(b,nir_ieq_imm(b,nir_load_local_invocation_index(b),0));
+      nir_store_shared(b,nir_u2u16(b,i->src[1].ssa),nir_imm_int(b,layout->primitive_count),
+                         .write_mask=1,.align_mul=2);
+      nir_store_shared(b,nir_u2u16(b,i->src[0].ssa),nir_imm_int(b,layout->vertex_count),
+                         .write_mask=1,.align_mul=2);
+      nir_pop_if(b,NULL);
+      break;
+   }
+   default:M5_REFUSE();
+   }
+   nir_instr_remove(&i->instr);
+   return true;
+}
+
+#define USER_VARYINGS BITFIELD64_RANGE(VARYING_SLOT_VAR0, 32)
+#define MESH_PSIZ_PLANE 4
+#define MESH_CLIP_SLOTS (VARYING_BIT_CLIP_DIST0 | VARYING_BIT_CLIP_DIST1)
+#define PRIMITIVE_SLOTS (USER_VARYINGS | VARYING_BIT_PRIMITIVE_ID | \
+                         VARYING_BIT_LAYER | VARYING_BIT_VIEWPORT)
+
+bool
+agx_gen1_mesh_gather_varyings(const nir_shader *s,
+                            struct agx_m5_mesh_varyings *varyings)
+{
+   memset(varyings, 0, sizeof(*varyings));
+   nir_foreach_function_impl(impl, s) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(in, block) {
+            if (in->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+            if (i->intrinsic != nir_intrinsic_store_per_vertex_output &&
+                i->intrinsic != nir_intrinsic_store_per_primitive_output)
+               continue;
+            if (!nir_src_is_const(i->src[2]))
+               M5_REFUSE();
+            const unsigned slot = vertex_output_slot(i);
+            if (i->intrinsic == nir_intrinsic_store_per_primitive_output) {
+               if (slot == VARYING_SLOT_PRIMITIVE_INDICES)
+                  continue;
+               const unsigned end = nir_intrinsic_component(i) +
+                                    util_last_bit(nir_intrinsic_write_mask(i));
+               if (slot >= 64 || !(PRIMITIVE_SLOTS & BITFIELD64_BIT(slot)) ||
+                   end > 4 ||
+                   (i->src[0].ssa->bit_size != 32 && i->src[0].ssa->bit_size != 16))
+                  M5_REFUSE();
+               varyings->primitive_comps[slot] =
+                  MAX2(varyings->primitive_comps[slot], end);
+               continue;
+            }
+            if (slot == VARYING_SLOT_POS)
+               continue;
+            if (slot == VARYING_SLOT_PSIZ &&
+                s->info.mesh.primitive_type != MESA_PRIM_POINTS)
+               continue;
+            if (slot >= 64 || !((USER_VARYINGS | VARYING_BIT_PSIZ | MESH_CLIP_SLOTS) &
+                                BITFIELD64_BIT(slot))) {
+               if (getenv("ASAHI_MESH_CONTRACT_DEBUG"))
+                  fprintf(stderr, "m5 mesh vertex slot %u\n", slot);
+               M5_REFUSE();
+            }
+            const unsigned end = nir_intrinsic_component(i) +
+                                 util_last_bit(nir_intrinsic_write_mask(i));
+            if (end > 4)
+               M5_REFUSE();
+            varyings->components[slot] = MAX2(varyings->components[slot], end);
+         }
+      }
+   }
+   
+
+
+   unsigned record = 0;
+   for (unsigned slot = 0; slot < 64; ++slot) {
+      if (!varyings->primitive_comps[slot])
+         continue;
+      varyings->primitive_first[slot] = record;
+      record += varyings->primitive_comps[slot];
+   }
+   varyings->primitive_components = varyings->primitive_comps[VARYING_SLOT_VAR0];
+   varyings->primitive_stride_B = 4 * record;
+   if ((uint64_t)varyings->primitive_stride_B * s->info.mesh.max_primitives_out >
+       UINT64_C(0x10000))
+      M5_REFUSE();
+
+   if (s->info.cull_distance_array_size)
+      M5_REFUSE();
+   const bool clip_written = varyings->components[VARYING_SLOT_CLIP_DIST0] ||
+                             varyings->components[VARYING_SLOT_CLIP_DIST1];
+   if (clip_written != (s->info.clip_distance_array_size != 0))
+      M5_REFUSE();
+
+   unsigned next = 4, slots = 0;
+   if (varyings->components[VARYING_SLOT_PSIZ]) {
+      varyings->first[VARYING_SLOT_PSIZ] = MESH_PSIZ_PLANE;
+      next++;
+   }
+   if (varyings->primitive_comps[VARYING_SLOT_LAYER] ||
+       varyings->primitive_comps[VARYING_SLOT_VIEWPORT]) {
+      varyings->lv = true;
+      varyings->lv_first = next++;
+   }
+   if (clip_written) {
+      varyings->clip_first = next;
+      varyings->clip_count = s->info.clip_distance_array_size;
+      next += varyings->clip_count;
+   }
+   if (varyings->primitive_stride_B)
+      varyings->components[AGX_MESH_PRIMITIVE_PAGE_SLOT] = page_carrier_words();
+   for (unsigned slot = VARYING_SLOT_CLIP_DIST0; slot < 64; ++slot) {
+      if (!varyings->components[slot])
+         continue;
+      varyings->first[slot] = next;
+      next += varyings->components[slot];
+      slots++;
+   }
+   varyings->size = next;
+   varyings->dynamic = slots > 1 || varyings->lv;
+   return true;
+}
+
+
+static bool
+varyings_valid(const struct agx_m5_mesh_varyings *v)
+{
+   if (getenv("ASAHI_MESH_CONTRACT_DEBUG")) {
+      fprintf(stderr, "m5 mesh layout size=%u clip=%u@%u lv=%u@%u:", v->size, v->clip_count, v->clip_first, v->lv, v->lv_first);
+      for (unsigned slot = 0; slot < 64; ++slot)
+         if (v->components[slot])
+            fprintf(stderr, " %u:%u@%u", slot, v->components[slot], v->first[slot]);
+      fprintf(stderr, "\n");
+   }
+   if (v->size < 4 || v->size > 4 + 128)
+      M5_REFUSE();
+   uint8_t used[4 + 128] = {0};
+   unsigned user = 0;
+   if (v->components[VARYING_SLOT_PSIZ]) {
+      if (v->components[VARYING_SLOT_PSIZ] != 1 ||
+          v->first[VARYING_SLOT_PSIZ] != MESH_PSIZ_PLANE)
+         M5_REFUSE();
+      used[MESH_PSIZ_PLANE] = 1;
+      user++;
+   }
+   if (v->lv) {
+      if (v->lv_first < 4 || v->lv_first >= v->size || used[v->lv_first]++)
+         M5_REFUSE();
+      user++;
+   }
+   if (v->clip_count) {
+      if (v->clip_count > 8 || v->clip_first < 4 ||
+          v->clip_first + v->clip_count > v->size)
+         M5_REFUSE();
+      for (unsigned c = 0; c < v->clip_count; ++c) {
+         if (used[v->clip_first + c]++)
+            M5_REFUSE();
+      }
+      user += v->clip_count;
+   }
+   unsigned hidden = 0;
+   for (unsigned slot = 0; slot < 64; ++slot) {
+      if (!v->components[slot])
+         continue;
+      if (slot == VARYING_SLOT_PSIZ)
+         continue;
+      if ((MESH_CLIP_SLOTS & BITFIELD64_BIT(slot)) && v->clip_count &&
+          v->first[slot] >= v->clip_first &&
+          v->first[slot] + v->components[slot] <= v->clip_first + v->clip_count) {
+         hidden += v->components[slot];
+         continue;
+      }
+      if ((!((USER_VARYINGS | MESH_CLIP_SLOTS) & BITFIELD64_BIT(slot)) &&
+           !(slot == AGX_MESH_PRIMITIVE_PAGE_SLOT && v->primitive_stride_B != 0 &&
+             v->components[slot] == page_carrier_words())) || v->components[slot] > 4 ||
+          v->first[slot] < 4 || v->first[slot] + v->components[slot] > v->size)
+         M5_REFUSE();
+      for (unsigned c = 0; c < v->components[slot]; ++c) {
+         if (used[v->first[slot] + c]++)
+            M5_REFUSE();
+      }
+      user += v->components[slot];
+   }
+   return user + hidden + 4 == v->size;
+}
+
+bool
+agx_nir_lower_gen1_mesh_triangle_probe(nir_shader *s)
+{
+   struct agx_m5_mesh_varyings varyings;
+   if (!agx_gen1_mesh_gather_varyings(s, &varyings))
+      M5_REFUSE();
+   return agx_nir_lower_gen1_mesh(s, &varyings);
+}
+
+bool
+agx_nir_lower_gen1_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings)
+{
+   unsigned threads=s->info.workgroup_size[0]*s->info.workgroup_size[1]*
+                    s->info.workgroup_size[2];
+   if(s->info.stage!=MESA_SHADER_MESH || s->info.shared_size>32768 ||
+      s->info.workgroup_size_variable || threads==0 || threads>128 ||
+      s->info.mesh.max_vertices_out==0 || s->info.mesh.max_vertices_out>256 ||
+      s->info.mesh.max_primitives_out==0 || s->info.mesh.max_primitives_out>256 ||
+      prim_vertices(s)==0 ||
+      (s->info.per_primitive_outputs & ~(VARYING_BIT_PRIMITIVE_INDICES | PRIMITIVE_SLOTS)) ||
+      (s->info.outputs_written & ~s->info.per_primitive_outputs) & ~(VARYING_BIT_POS|VARYING_BIT_PSIZ|MESH_CLIP_SLOTS|USER_VARYINGS) ||
+      !varyings_valid(varyings))
+      M5_REFUSE();
+   if (s->info.mesh.primitive_type == MESA_PRIM_POINTS &&
+       !varyings->components[VARYING_SLOT_PSIZ])
+      M5_REFUSE();
+   struct mesh_layout layout = mesh_layout(s, varyings);
+   if (getenv("ASAHI_MESH_PP_AUDIT") &&
+       !strcmp(getenv("ASAHI_MESH_PP_AUDIT"), "1") &&
+       (!layout.primitive_audit_va || (layout.primitive_audit_va & 15) ||
+        s->info.mesh.max_vertices_out > primitive_audit_lanes() ||
+        s->info.mesh.max_primitives_out > primitive_audit_primitives() ||
+        threads > primitive_audit_lanes()))
+      M5_REFUSE();
+   if (layout.output_base + layout.bytes > 32768)
+      M5_REFUSE();
+   unsigned physical_threads=MAX2(threads,MIN2(s->info.mesh.max_vertices_out,32));
+   bool counts=false;
+   nir_foreach_function_impl(impl,s) {
+      nir_foreach_block(block,impl) {
+         nir_foreach_instr(in,block) {
+            if(in->type!=nir_instr_type_intrinsic)continue;
+            nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+            switch(i->intrinsic) {
+            case nir_intrinsic_set_vertex_and_primitive_count:counts=true;break;
+            case nir_intrinsic_barrier:
+               if (physical_threads != threads) {
+                  if (nir_intrinsic_execution_scope(i) == SCOPE_WORKGROUP)
+                     nir_intrinsic_set_execution_scope(i, SCOPE_SUBGROUP);
+                  if (nir_intrinsic_memory_scope(i) == SCOPE_WORKGROUP)
+                     nir_intrinsic_set_memory_scope(i, SCOPE_SUBGROUP);
+               }
+               break;
+            case nir_intrinsic_store_per_vertex_output: {
+               if (!nir_src_is_const(i->src[2]))
+                  M5_REFUSE();
+               const unsigned slot=vertex_output_slot(i);
+               const unsigned bits=i->src[0].ssa->bit_size;
+               const unsigned end=nir_intrinsic_component(i)+
+                                  util_last_bit(nir_intrinsic_write_mask(i));
+               if(slot>=64 || (bits!=32 && bits!=16) || end>4)
+                  M5_REFUSE();
+               if(slot==VARYING_SLOT_POS) {
+                  if(bits!=32)M5_REFUSE();
+               } else if(slot==VARYING_SLOT_PSIZ) {
+                  if(varyings->components[slot] && (bits!=32 || end>1))M5_REFUSE();
+               } else if(!varyings->components[slot] ||
+                         end>varyings->components[slot]) {
+                  M5_REFUSE();
+               }
+               break;
+            }
+            case nir_intrinsic_store_per_primitive_output: {
+               if (!nir_src_is_const(i->src[2]))
+                  M5_REFUSE();
+               const unsigned slot = vertex_output_slot(i);
+               const unsigned end = nir_intrinsic_component(i) +
+                                    util_last_bit(nir_intrinsic_write_mask(i));
+               if (slot == VARYING_SLOT_PRIMITIVE_INDICES) {
+                  if (end > prim_vertices(s) || nir_src_as_uint(i->src[2]) != 0)
+                     M5_REFUSE();
+               } else if (slot >= 64 || !varyings->primitive_stride_B ||
+                          end > varyings->primitive_comps[slot] ||
+                          (i->src[0].ssa->bit_size != 32 && i->src[0].ssa->bit_size != 16)) {
+                  M5_REFUSE();
+               }
+               break;
+            }
+            case nir_intrinsic_load_per_vertex_output:
+            case nir_intrinsic_load_per_primitive_output:
+            case nir_intrinsic_launch_mesh_workgroups:M5_REFUSE();
+            default:break;
+            }
+         }
+      }
+   }
+   if(!counts)M5_REFUSE();
+   if (physical_threads != threads) {
+      
+
+
+      nir_function_impl *entry=nir_shader_get_entrypoint(s);
+      nir_foreach_block(block,entry) {
+         nir_foreach_instr_safe(in,block) {
+            if (in->type != nir_instr_type_intrinsic) continue;
+            nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+            nir_builder b=nir_builder_at(nir_before_instr(in));
+            nir_def *replacement=NULL;
+            if (i->intrinsic == nir_intrinsic_load_workgroup_size) {
+               replacement=nir_imm_ivec3(&b,s->info.workgroup_size[0],
+                  s->info.workgroup_size[1],s->info.workgroup_size[2]);
+            } else if (i->intrinsic == nir_intrinsic_load_local_invocation_id ||
+                       i->intrinsic == nir_intrinsic_load_global_invocation_id) {
+               nir_def *index=nir_load_local_invocation_index(&b);
+               unsigned x=s->info.workgroup_size[0], y=s->info.workgroup_size[1];
+               replacement=nir_vec3(&b,nir_umod_imm(&b,index,x),
+                  nir_umod_imm(&b,nir_udiv_imm(&b,index,x),y),
+                  nir_udiv_imm(&b,index,x*y));
+               if (i->intrinsic == nir_intrinsic_load_global_invocation_id) {
+                  
+
+                  nir_def *size=nir_imm_ivec3(&b,s->info.workgroup_size[0],
+                     s->info.workgroup_size[1],s->info.workgroup_size[2]);
+                  replacement=nir_iadd(&b,nir_imul(&b,nir_load_workgroup_id(&b),size),
+                                       replacement);
+                  replacement=nir_u2uN(&b,replacement,i->def.bit_size);
+               }
+            }
+            if (replacement) {
+               nir_def_rewrite_uses(&i->def,replacement);
+               nir_instr_remove(in);
+            }
+         }
+      }
+      nir_cf_list body;
+      nir_cf_extract(&body,nir_before_cf_list(&entry->body),
+                            nir_after_cf_list(&entry->body));
+      nir_builder b=nir_builder_at(nir_before_impl(entry));
+      nir_if *active=nir_push_if(&b,nir_ult_imm(&b,
+                                  nir_load_local_invocation_index(&b),threads));
+      b.cursor=nir_cf_reinsert(&body,b.cursor);
+      nir_pop_if(&b,active);
+      s->info.workgroup_size[0]=physical_threads;
+      s->info.workgroup_size[1]=s->info.workgroup_size[2]=1;
+   }
+   
+
+
+   nir_shader_intrinsics_pass(s,lower,nir_metadata_none,&layout);
+   nir_function_impl *impl=nir_shader_get_entrypoint(s);
+   nir_builder b=nir_builder_at(nir_after_cf_list(&impl->body));
+   nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+                  .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
+   
+   nir_push_if(&b,nir_ieq_imm(&b,nir_load_subgroup_id(&b),0));
+   nir_def *p=nir_load_shared(&b,1,16,nir_imm_int(&b,layout.primitive_count),.align_mul=2);
+   nir_def *v=nir_load_shared(&b,1,16,nir_imm_int(&b,layout.vertex_count),.align_mul=2);
+   if (varyings->lv) {
+      nir_def *storage = nir_pack_64_2x32(&b, nir_load_mesh_primitive_storage_agx(&b));
+      nir_def *lane = nir_load_subgroup_invocation(&b);
+      for (unsigned prim = 0; prim < s->info.mesh.max_primitives_out; prim += 32) {
+         nir_def *index = nir_iadd_imm(&b, lane, prim);
+         nir_push_if(&b, nir_ult(&b, index, nir_u2u32(&b, p)));
+         nir_def *record = nir_iadd(&b, storage,
+            nir_u2u64(&b, nir_imul_imm(&b, index, varyings->primitive_stride_B)));
+         nir_def *layer = nir_imm_int(&b, 0), *viewport = nir_imm_int(&b, 0);
+         if (varyings->primitive_comps[VARYING_SLOT_LAYER])
+            layer = nir_load_global(&b, 1, 32, nir_iadd_imm(&b, record,
+               4 * varyings->primitive_first[VARYING_SLOT_LAYER]), .align_mul = 4);
+         if (varyings->primitive_comps[VARYING_SLOT_VIEWPORT])
+            viewport = nir_load_global(&b, 1, 32, nir_iadd_imm(&b, record,
+               4 * varyings->primitive_first[VARYING_SLOT_VIEWPORT]), .align_mul = 4);
+         nir_def *packed = nir_pack_32_2x16_split(&b, nir_u2u16(&b, layer),
+                                                  nir_u2u16(&b, viewport));
+         for (unsigned k = 0; k < layout.prim_vertices; ++k) {
+            nir_def *vertex = nir_u2u32(&b, nir_load_shared(&b, 1, 8,
+               nir_iadd_imm(&b, nir_imul_imm(&b, index, layout.prim_vertices),
+                            layout.output_base + k), .align_mul = 1));
+            nir_store_shared(&b, packed,
+               nir_iadd_imm(&b, nir_imul_imm(&b, vertex, 4),
+                            layout.position + layout.plane_stride * varyings->lv_first),
+               .write_mask = 1, .align_mul = 4);
+         }
+         nir_pop_if(&b, NULL);
+      }
+      nir_barrier(&b, .execution_scope = SCOPE_SUBGROUP, .memory_scope = SCOPE_SUBGROUP,
+                  .memory_semantics = NIR_MEMORY_ACQ_REL, .memory_modes = nir_var_mem_shared);
+   }
+   if (varyings->primitive_stride_B) {
+      nir_def *storage = nir_load_mesh_primitive_storage_agx(&b);
+      nir_def *lane = nir_load_subgroup_invocation(&b);
+      for (unsigned vertex = 0; vertex < s->info.mesh.max_vertices_out; vertex += 32) {
+         nir_def *index = nir_iadd_imm(&b, lane, vertex);
+         nir_push_if(&b, nir_ult(&b, index, nir_u2u32(&b, v)));
+         if (native_page_carrier()) {
+            nir_def *page = nir_unpack_64_2x32_split_x(&b,
+               nir_ushr_imm(&b, nir_pack_64_2x32(&b, storage), 16));
+            nir_store_shared(&b, page,
+               nir_iadd_imm(&b, nir_imul_imm(&b, index, 4),
+                  layout.position + layout.plane_stride *
+                     varyings->first[AGX_MESH_PRIMITIVE_PAGE_SLOT]),
+               .write_mask = 1, .align_mul = 4);
+         } else {
+            for (unsigned c = 0; c < 2; ++c) {
+               nir_store_shared(&b, nir_channel(&b, storage, c),
+                  nir_iadd_imm(&b, nir_imul_imm(&b, index, 4),
+                     layout.position + layout.plane_stride *
+                        (varyings->first[AGX_MESH_PRIMITIVE_PAGE_SLOT] + c)),
+                  .write_mask = 1, .align_mul = 4);
+            }
+         }
+         if (layout.primitive_audit_va) {
+            nir_def *group = nir_channel(&b, nir_load_workgroup_id(&b), 0);
+            nir_push_if(&b, nir_ult_imm(&b, group, 4));
+            primitive_order(&b);
+            nir_def *shared = nir_iadd_imm(&b, nir_imul_imm(&b, index, 4),
+               layout.position + layout.plane_stride *
+                  varyings->first[AGX_MESH_PRIMITIVE_PAGE_SLOT]);
+            nir_def *fields[8] = {nir_imm_int(&b, 0x50510010), index,
+               nir_channel(&b, storage, 0), nir_channel(&b, storage, 1),
+               nir_load_shared(&b, 1, 32, shared,
+                  .align_mul = 4, .access = ACCESS_VOLATILE),
+               nir_load_shared(&b, 1, 32, nir_iadd_imm(&b, shared, layout.plane_stride),
+                  .align_mul = 4, .access = ACCESS_VOLATILE), group,
+               nir_imm_int(&b, 0x50510011)};
+            primitive_capture(&b, layout.primitive_audit_va, primitive_audit_lane_first(),
+               nir_iadd(&b, nir_imul_imm(&b, group, primitive_audit_lanes()), index), 8, fields);
+            nir_pop_if(&b, NULL);
+         }
+         nir_pop_if(&b, NULL);
+      }
+      nir_barrier(&b, .execution_scope = SCOPE_SUBGROUP,
+         .memory_scope = SCOPE_SUBGROUP, .memory_semantics = NIR_MEMORY_ACQ_REL,
+         .memory_modes = nir_var_mem_shared);
+   }
+   nir_allocate_mesh_uvb_agx(&b,p,v);
+   nir_def *lane=nir_load_subgroup_invocation(&b);
+   nir_def *primitive_words=nir_udiv_imm(&b,
+      nir_iadd_imm(&b,nir_imul_imm(&b,nir_u2u32(&b,p),layout.prim_vertices),3),4);
+   for (unsigned word=0;
+        word<DIV_ROUND_UP(layout.prim_vertices*s->info.mesh.max_primitives_out,4);
+        word+=32) {
+      nir_push_if(&b,nir_ult(&b,nir_iadd_imm(&b,lane,word),primitive_words));
+      
+
+      nir_store_mesh_uvb_agx(&b,nir_imm_ivec2(&b,word<<16,
+         layout.output_base+4*word),.flags=1);
+      nir_pop_if(&b,NULL);
+   }
+   
+
+
+   nir_def *selector[4 + 128] = {NULL};
+   if (varyings->dynamic) {
+      for (unsigned slot = 0; slot < 64; ++slot) {
+         if (!varyings->components[slot])
+            continue;
+         nir_def *base = nir_u2u32(&b, nir_load_uvs_index_agx(&b,
+            .io_semantics = {.location = slot, .num_slots = 1}));
+         for (unsigned c = 0; c < varyings->components[slot]; ++c)
+            selector[varyings->first[slot] + c] = nir_iadd_imm(&b, base, c);
+      }
+   }
+   for(unsigned vertex=0;vertex<s->info.mesh.max_vertices_out;vertex+=32) {
+      nir_push_if(&b,nir_ult(&b,nir_iadd_imm(&b,lane,vertex),nir_u2u32(&b,v)));
+      for(unsigned plane=0;plane<layout.planes;plane++) {
+         const unsigned address=layout.position+layout.plane_stride*plane+4*vertex;
+         
+
+         nir_def *descriptor = selector[plane]
+            ? nir_vec2(&b, nir_ior_imm(&b, selector[plane], vertex << 16),
+                       nir_imm_int(&b, address))
+            : nir_imm_ivec2(&b, (vertex << 16) | plane, address);
+         nir_store_mesh_uvb_agx(&b,descriptor,.flags=0);
+      }
+      nir_pop_if(&b,NULL);
+   }
+   nir_pop_if(&b,NULL);
+   if (layout.primitive_audit_va)
+      nir_shader_gather_info(s, nir_shader_get_entrypoint(s));
+   s->info.shared_size=layout.output_base+layout.bytes;
+   s->info.outputs_written=0;s->info.per_primitive_outputs=0;
+   nir_progress(true,impl,nir_metadata_none);
+   return true;
+}
+
+
+
+
+static bool
+lower_payload(nir_builder *b,nir_intrinsic_instr *i,void *data)
+{
+   (void)data;
+   if (i->intrinsic==nir_intrinsic_barrier &&
+       (nir_intrinsic_memory_modes(i)&nir_var_mem_task_payload)) {
+      nir_intrinsic_set_memory_modes(i,
+         (nir_intrinsic_memory_modes(i)&~nir_var_mem_task_payload)|nir_var_mem_global);
+      return true;
+   }
+   bool store=i->intrinsic==nir_intrinsic_store_task_payload;
+   if (!store && i->intrinsic!=nir_intrinsic_load_task_payload) M5_REFUSE();
+   b->cursor=nir_before_instr(&i->instr);
+   nir_def *pointer=nir_pack_64_2x32(b,nir_load_mesh_payload_address_agx(b));
+   nir_def *offset=i->src[store ? 1 : 0].ssa;
+   nir_def *addr=nir_iadd(b,pointer,nir_u2u64(b,
+                            nir_iadd_imm(b,offset,nir_intrinsic_base(i))));
+   if (store) {
+      nir_store_global(b,i->src[0].ssa,addr,
+         .write_mask=nir_intrinsic_write_mask(i),
+         .align_mul=nir_intrinsic_align_mul(i),
+         .align_offset=nir_intrinsic_align_offset(i),
+         .access=nir_intrinsic_access(i));
+      nir_instr_remove(&i->instr);
+   } else {
+      nir_def_replace(&i->def,nir_load_global(b,i->def.num_components,
+         i->def.bit_size,addr,.align_mul=nir_intrinsic_align_mul(i),
+         .align_offset=nir_intrinsic_align_offset(i),
+         .access=nir_intrinsic_access(i)));
+   }
+   return true;
+}
+
+bool
+agx_nir_lower_gen1_task_payload_probe(nir_shader *s)
+{
+   if (s->info.task_payload_size>4) M5_REFUSE();
+   bool reads=false;
+   nir_foreach_function_impl(impl,s) {
+      nir_foreach_block(block,impl) {
+         nir_foreach_instr(in,block) {
+            if (in->type!=nir_instr_type_intrinsic) continue;
+            nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+            reads |= i->intrinsic==nir_intrinsic_load_task_payload;
+            if (i->intrinsic==nir_intrinsic_task_payload_atomic ||
+                i->intrinsic==nir_intrinsic_task_payload_atomic_swap ||
+                i->intrinsic==nir_intrinsic_launch_mesh_workgroups_with_payload_deref)
+               M5_REFUSE();
+         }
+      }
+   }
+   if (reads && s->info.stage==MESA_SHADER_MESH)
+      M5_REFUSE();
+   nir_shader_intrinsics_pass(s,lower_payload,nir_metadata_control_flow,NULL);
+   return true;
+}
+
+static bool
+lower_launch(nir_builder *b,nir_intrinsic_instr *i,void *data)
+{
+   if (i->intrinsic!=nir_intrinsic_launch_mesh_workgroups) M5_REFUSE();
+   const unsigned grid=*(unsigned *)data;
+   b->cursor=nir_before_instr(&i->instr);
+   nir_push_if(b,nir_ieq_imm(b,nir_load_local_invocation_index(b),0));
+   nir_store_shared(b,i->src[0].ssa,nir_imm_int(b,grid),
+                       .write_mask=7,.align_mul=4);
+   nir_pop_if(b,NULL);
+   nir_instr_remove(&i->instr);
+   return true;
+}
+
+bool
+agx_nir_lower_gen1_task_probe(nir_shader *s)
+{
+   unsigned threads=s->info.workgroup_size[0]*s->info.workgroup_size[1]*
+                    s->info.workgroup_size[2];
+   if (s->info.stage!=MESA_SHADER_TASK || s->info.workgroup_size_variable ||
+       threads<3 || threads>128 || s->info.shared_size>48 ||
+       !agx_nir_lower_gen1_task_payload_probe(s)) M5_REFUSE();
+   unsigned launches=0;
+   nir_foreach_function_impl(impl,s) {
+      nir_foreach_block(block,impl) {
+         nir_foreach_instr(in,block) {
+            if (in->type==nir_instr_type_intrinsic &&
+                nir_instr_as_intrinsic(in)->intrinsic==nir_intrinsic_launch_mesh_workgroups)
+               launches++;
+         }
+      }
+   }
+   if (!launches) M5_REFUSE();
+   unsigned grid=ALIGN_POT(s->info.shared_size,16);
+   nir_shader_intrinsics_pass(s,lower_launch,nir_metadata_none,&grid);
+   nir_builder b=nir_builder_at(nir_after_cf_list(&nir_shader_get_entrypoint(s)->body));
+   nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+      .memory_semantics=NIR_MEMORY_ACQ_REL,
+      .memory_modes=nir_var_mem_shared);
+
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_push_if(&b,nir_ult_imm(&b,lane,3));
+   nir_def *value=nir_load_shared(&b,1,32,
+      nir_iadd_imm(&b,nir_imul_imm(&b,lane,4),grid),.align_mul=4);
+   nir_store_object_childcount_agx(&b,value);
+   nir_pop_if(&b,NULL);
+   s->info.shared_size=64;
+   nir_progress(true,b.impl,nir_metadata_none);
+   return true;
+}
+
+static bool
+lower_primitive_input(nir_builder *b, nir_intrinsic_instr *i, void *data)
+{
+   if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
+      M5_REFUSE();
+   b->cursor = nir_before_instr(&i->instr);
+   const struct agx_m5_mesh_varyings *mesh = data;
+   const unsigned location = nir_intrinsic_io_semantics(i).location;
+   if (mesh && (location >= 64 || !mesh->primitive_comps[location])) {
+      nir_def_rewrite_uses(&i->def, nir_imm_zero(b, i->def.num_components,
+                                                 i->def.bit_size));
+      nir_instr_remove(&i->instr);
+      return true;
+   }
+   const unsigned record_first = mesh ? mesh->primitive_first[location] : 0;
+   nir_def *primitive_cf = nir_load_coefficients_agx(b, nir_imm_int(b, 0),
+      .component = 0, .interp_mode = INTERP_MODE_FLAT,
+      .io_semantics = {.location = VARYING_SLOT_PRIMITIVE_ID, .num_slots = 1});
+   nir_def *base_low_cf = nir_load_coefficients_agx(b, nir_imm_int(b, 0),
+      .component = 0, .interp_mode = INTERP_MODE_FLAT,
+      .io_semantics = {.location = AGX_MESH_PRIMITIVE_PAGE_SLOT, .num_slots = 1});
+   nir_def *base_high_cf = native_page_carrier() ? base_low_cf :
+      nir_load_coefficients_agx(b, nir_imm_int(b, 0),
+      .component = 1, .interp_mode = INTERP_MODE_FLAT,
+      .io_semantics = {.location = AGX_MESH_PRIMITIVE_PAGE_SLOT, .num_slots = 1});
+   nir_def *low = nir_channel(b, primitive_cf, 2);
+   nir_def *base_low = nir_channel(b, base_low_cf, 2);
+   nir_def *base_high = nir_channel(b, base_high_cf, 2);
+   nir_def *base = native_page_carrier()
+      ? nir_ishl_imm(b, nir_u2u64(b, base_low), 16)
+      : nir_pack_64_2x32(b, nir_vec2(b, base_low, base_high));
+   nir_def *address = native_primitive_address()
+      ? nir_ior(b, nir_iand_imm(b, base, ~UINT64_C(0xffff)),
+                nir_u2u64(b, nir_iand_imm(b, nir_ishl_imm(b, low, 2), 0xffff)))
+      : nir_iadd(b, base, nir_u2u64(b, nir_iand_imm(b, low, 0xffff)));
+   address = nir_iadd_imm(b, address,
+                          4 * (record_first + nir_intrinsic_component(i)));
+   nir_def *value = nir_load_global(b, i->def.num_components, 32, address,
+                                  .align_mul = 4);
+   const uint64_t va = primitive_audit_va();
+   if (va) {
+      nir_def *position = nir_load_frag_coord(b);
+      nir_def *x = nir_f2u32(b, nir_channel(b, position, 0));
+      nir_def *y = nir_f2u32(b, nir_channel(b, position, 1));
+      nir_def *group = nir_ushr_imm(b, x, 4);
+      nir_def *bottom = nir_ieq_imm(b, y, 60);
+      nir_def *sample, *right = NULL, *below = NULL, *left = NULL;
+      const bool fan = primitive_audit_fan();
+      if (fan) {
+         nir_def *column = nir_iand_imm(b, x, 15);
+         nir_def *centre = nir_ieq_imm(b, column, 8);
+         nir_def *middle = nir_ieq_imm(b, y, 32);
+         nir_def *top = nir_iand(b, centre, nir_ieq_imm(b, y, 4));
+         right = nir_iand(b, nir_ieq_imm(b, column, 14), middle);
+         below = nir_iand(b, centre, bottom);
+         left = nir_iand(b, nir_ieq_imm(b, column, 2), middle);
+         sample = nir_ior(b, nir_ior(b, top, right), nir_ior(b, below, left));
+      } else {
+         sample = nir_iand(b, nir_ieq_imm(b, nir_iand_imm(b, x, 15), 8),
+            nir_ior(b, nir_ieq_imm(b, y, 4), bottom));
+      }
+      nir_push_if(b, nir_iand(b, nir_ult_imm(b, group, 4), sample));
+      nir_def *fields[32] = {nir_imm_int(b, 0x50510030), x, y, group,
+         nir_channel(b, primitive_cf, 0), nir_channel(b, primitive_cf, 1), low,
+         nir_channel(b, base_low_cf, 0), nir_channel(b, base_low_cf, 1), base_low,
+         nir_channel(b, base_high_cf, 0), nir_channel(b, base_high_cf, 1), base_high,
+         nir_u2u32(b, address), nir_u2u32(b, nir_ushr_imm(b, address, 32)),
+         nir_channel(b, value, 0), nir_imm_int(b, nir_intrinsic_component(i)),
+         nir_imm_int(b, 0x50510031)};
+      nir_def *record = fan
+         ? nir_iadd(b, nir_imul_imm(b, group, 4),
+              nir_bcsel(b, right, nir_imm_int(b, 1),
+                 nir_bcsel(b, below, nir_imm_int(b, 2),
+                    nir_bcsel(b, left, nir_imm_int(b, 3), nir_imm_int(b, 0)))))
+         : nir_iadd(b, nir_imul_imm(b, group, 2), nir_b2i32(b, bottom));
+      primitive_capture(b, va, primitive_audit_fragment_first(), record, 32, fields);
+      nir_pop_if(b, NULL);
+   }
+   if (i->def.bit_size != 32)
+      value = nir_convert_to_bit_size(b, value,
+         nir_alu_type_get_base_type(nir_intrinsic_dest_type(i)), i->def.bit_size);
+   nir_def_rewrite_uses(&i->def, value);
+   nir_instr_remove(&i->instr);
+   return true;
+}
+
+static bool
+fragment_inputs_supported(nir_shader *s, const struct agx_m5_mesh_varyings *mesh)
+{
+   nir_foreach_function_impl(impl, s) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(in, block) {
+            if (in->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+            if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
+               continue;
+            const unsigned location = nir_intrinsic_io_semantics(i).location;
+            if (!nir_src_is_const(i->src[0]) || nir_src_as_uint(i->src[0]) != 0 ||
+                nir_intrinsic_component(i) + i->def.num_components > 4 ||
+                (i->def.bit_size != 32 && i->def.bit_size != 16))
+               M5_REFUSE();
+            if (location >= 64 || !(PRIMITIVE_SLOTS & BITFIELD64_BIT(location)))
+               M5_REFUSE();
+            if (!mesh && location != VARYING_SLOT_VAR0)
+               M5_REFUSE();
+         }
+      }
+   }
+   return true;
+}
+
+bool
+agx_nir_lower_gen1_mesh_fragment_layout(nir_shader *s,
+                                      const struct agx_m5_mesh_varyings *mesh)
+{
+   if (s->info.stage != MESA_SHADER_FRAGMENT || !mesh ||
+       !fragment_inputs_supported(s, mesh))
+      M5_REFUSE();
+   nir_shader_intrinsics_pass(s, lower_primitive_input,
+      primitive_audit_va() ? nir_metadata_none : nir_metadata_control_flow,
+      (void *)mesh);
+   if (primitive_audit_va())
+      nir_shader_gather_info(s, nir_shader_get_entrypoint(s));
+   return true;
+}
+
+bool
+agx_nir_lower_gen1_mesh_fragment(nir_shader *s)
+{
+   if (s->info.stage != MESA_SHADER_FRAGMENT)
+      M5_REFUSE();
+   nir_foreach_function_impl(impl, s) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(in, block) {
+            if (in->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+            if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
+               continue;
+            if (nir_intrinsic_io_semantics(i).location != VARYING_SLOT_VAR0 ||
+                !nir_src_is_const(i->src[0]) || nir_src_as_uint(i->src[0]) != 0 ||
+                nir_intrinsic_component(i) + i->def.num_components > 4 ||
+                (i->def.bit_size != 32 && i->def.bit_size != 16))
+               M5_REFUSE();
+         }
+      }
+   }
+   nir_shader_intrinsics_pass(s, lower_primitive_input,
+      primitive_audit_va() ? nir_metadata_none : nir_metadata_control_flow, NULL);
+   if (primitive_audit_va())
+      nir_shader_gather_info(s, nir_shader_get_entrypoint(s));
+   return true;
+}
diff --git a/src/asahi/compiler/agx_nir_lower_cmat.c b/src/asahi/compiler/agx_nir_lower_cmat.c
new file mode 100644
index 000000000000..eb7e0aabebe5
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_lower_cmat.c
@@ -0,0 +1,333 @@
+/*
+ * Copyright © 2023 Bas Nieuwenhuizen
+ * Copyright © 2024 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+
+#include "agx_nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "util/hash_table.h"
+
+static const struct glsl_type *
+remap_matrix_type(struct hash_table *mapping, const struct glsl_type *orig)
+{
+   struct hash_entry *entry = _mesa_hash_table_search(mapping, orig);
+
+   if (entry)
+      return entry->data;
+
+   const struct glsl_type *new_type = orig;
+
+   if (glsl_type_is_cmat(orig)) {
+      struct glsl_cmat_description matrix_desc =
+         *glsl_get_cmat_description(orig);
+
+      new_type = glsl_vector_type(matrix_desc.element_type,
+                                  matrix_desc.rows * matrix_desc.cols / 32);
+   } else if (glsl_type_is_array(orig)) {
+      const struct glsl_type *elem_type = glsl_get_array_element(orig);
+      const struct glsl_type *new_elem_type =
+         remap_matrix_type(mapping, elem_type);
+
+      if (elem_type != new_elem_type) {
+         new_type = glsl_array_type(new_elem_type, glsl_get_length(orig),
+                                    glsl_get_explicit_stride(orig));
+      }
+   } else if (glsl_type_is_struct(orig)) {
+      unsigned i;
+      for (i = 0; i < orig->length; i++) {
+         const struct glsl_type *field_type = glsl_get_struct_field(orig, i);
+         const struct glsl_type *new_field_type =
+            remap_matrix_type(mapping, field_type);
+
+         if (field_type != new_field_type) {
+            break;
+         }
+      }
+
+      
+      if (i < orig->length) {
+         struct glsl_struct_field *fields =
+            malloc(sizeof(struct glsl_struct_field) * orig->length);
+
+         
+         memcpy(fields, orig->fields.structure,
+                sizeof(struct glsl_struct_field) * i);
+
+         
+         for (; i < orig->length; i++) {
+            fields[i] = *glsl_get_struct_field_data(orig, i);
+            fields[i].type = remap_matrix_type(mapping, fields[i].type);
+         }
+
+         new_type =
+            glsl_struct_type(fields, orig->length, glsl_get_type_name(orig),
+                             glsl_struct_type_is_packed(orig));
+
+         free(fields);
+      }
+   }
+
+   _mesa_hash_table_insert(mapping, orig, (void *)new_type);
+   return new_type;
+}
+
+static struct glsl_cmat_description
+matrix_desc(nir_src src)
+{
+   return *glsl_get_cmat_description(nir_src_as_deref(src)->type);
+}
+
+static nir_def *
+load_matrix(nir_builder *b, nir_src src)
+{
+   struct glsl_cmat_description desc = matrix_desc(src);
+   assert((desc.rows == 8 || desc.rows == 16) &&
+          desc.cols == desc.rows && desc.scope == SCOPE_SUBGROUP);
+   return nir_build_load_deref(b, desc.rows * desc.cols / 32, glsl_base_type_bit_size(desc.element_type),
+                               src.ssa, 0);
+}
+
+static void
+store_matrix(nir_builder *b, nir_src dst, nir_def *value)
+{
+   nir_build_store_deref(b, dst.ssa, value, .write_mask = BITFIELD_MASK(value->num_components));
+}
+
+static nir_def *
+matrix_row(nir_builder *b, nir_def *lane,
+           struct glsl_cmat_description desc, unsigned element)
+{
+   if (desc.rows == 16)
+      return nir_ior_imm(b, nir_ior(b, nir_iand_imm(b, lane, 6),
+                 nir_iand_imm(b, nir_ushr_imm(b, lane, 1), 8)), element >> 2);
+   return nir_ior(b, nir_iand_imm(b, nir_ushr_imm(b, lane, 1), 3),
+                    nir_iand_imm(b, nir_ushr_imm(b, lane, 2), 4));
+}
+
+static nir_def *
+matrix_col(nir_builder *b, nir_def *lane,
+           struct glsl_cmat_description desc, unsigned element)
+{
+   if (desc.cols == 16) {
+      if (desc.use == GLSL_CMAT_USE_A)
+         return nir_ior_imm(b, nir_ior(b,
+            nir_iand_imm(b, nir_ushr_imm(b, lane, 3), 1),
+            nir_ishl_imm(b, nir_iand_imm(b, lane, 1), 3)),
+            (element & 3) << 1);
+      return nir_ior_imm(b, nir_ior(b,
+         nir_ishl_imm(b, nir_iand_imm(b, lane, 1), 2),
+         nir_iand_imm(b, lane, 8)), element & 3);
+   }
+   return nir_ior_imm(b, nir_ior(b, nir_ishl_imm(b, nir_iand_imm(b, lane, 1), 1),
+                                   nir_iand_imm(b, nir_ushr_imm(b, lane, 1), 4)),
+                         element);
+}
+
+static nir_deref_instr *
+matrix_memory(nir_builder *b, nir_intrinsic_instr *intr, unsigned element)
+{
+   const bool store = intr->intrinsic == nir_intrinsic_cmat_store;
+   struct glsl_cmat_description desc = matrix_desc(intr->src[store ? 1 : 0]);
+   nir_deref_instr *ptr = nir_src_as_deref(intr->src[store ? 0 : 1]);
+   nir_def *lane = nir_load_subgroup_invocation(b);
+   nir_def *row = matrix_row(b, lane, desc, element),
+           *col = matrix_col(b, lane, desc, element);
+   if (nir_intrinsic_matrix_layout(intr) == GLSL_MATRIX_LAYOUT_COLUMN_MAJOR) {
+      nir_def *tmp = row; row = col; col = tmp;
+   }
+   unsigned bits = ptr->def.bit_size;
+   nir_def *stride = nir_u2uN(b, intr->src[2].ssa, bits);
+   nir_def *row_offset = nir_imul(b, nir_u2uN(b, row, bits), stride);
+   ptr = nir_build_deref_cast(b, &ptr->def, ptr->modes, ptr->type,
+                              glsl_get_vector_elements(ptr->type) *
+                              glsl_get_bit_size(ptr->type) / 8);
+   ptr = nir_build_deref_ptr_as_array(b, ptr, row_offset);
+   ptr = nir_build_deref_cast(b, &ptr->def, ptr->modes,
+                              glsl_scalar_type(desc.element_type),
+                              glsl_base_type_bit_size(desc.element_type) / 8);
+   return nir_build_deref_ptr_as_array(b, ptr, nir_u2uN(b, col, bits));
+}
+
+static nir_def *
+convert_matrix_value(nir_builder *b, nir_def *value,
+                     enum glsl_base_type src, enum glsl_base_type dst)
+{
+   if (src == dst)
+      return value;
+   if (src == GLSL_TYPE_BFLOAT16) {
+      value = nir_bf2f(b, value);
+      src = GLSL_TYPE_FLOAT;
+   }
+   if (dst == GLSL_TYPE_BFLOAT16) {
+      if (src != GLSL_TYPE_FLOAT)
+         value = nir_f2f32(b, value);
+      return nir_f2bf(b, value);
+   }
+   nir_op op = nir_type_conversion_op(nir_get_nir_type_for_glsl_base_type(src),
+                                      nir_get_nir_type_for_glsl_base_type(dst),
+                                      nir_rounding_mode_undef);
+   return nir_build_alu1(b, op, value);
+}
+
+static bool
+lower_matrix_instr(nir_builder *b, nir_instr *instr, struct hash_table *types)
+{
+   if (instr->type == nir_instr_type_deref) {
+      nir_deref_instr *deref = nir_instr_as_deref(instr);
+      const glsl_type *type = remap_matrix_type(types, deref->type);
+      bool changed = type != deref->type;
+      deref->type = type;
+      return changed;
+   }
+   if (instr->type != nir_instr_type_intrinsic)
+      return false;
+   nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+   b->cursor = nir_before_instr(instr);
+   switch (intr->intrinsic) {
+   case nir_intrinsic_cmat_construct:
+      store_matrix(b, intr->src[0], nir_replicate(b, intr->src[1].ssa,
+         matrix_desc(intr->src[0]).rows * matrix_desc(intr->src[0]).cols / 32));
+      break;
+   case nir_intrinsic_cmat_load:
+   case nir_intrinsic_cmat_store: {
+      bool store = intr->intrinsic == nir_intrinsic_cmat_store;
+      struct glsl_cmat_description desc = matrix_desc(intr->src[store ? 1 : 0]);
+      nir_def *data = store ? load_matrix(b, intr->src[1]) : NULL;
+      nir_def *values[8];
+      unsigned count = desc.rows * desc.cols / 32;
+      bool row_major = nir_intrinsic_matrix_layout(intr) == GLSL_MATRIX_LAYOUT_ROW_MAJOR;
+      unsigned width = desc.rows == 8 ? (row_major ? 2 : 1)
+         : (!row_major ? 2 : desc.use == GLSL_CMAT_USE_A ? 1 : 4);
+      unsigned component_step = desc.rows == 16 && !row_major ? 4 : 1;
+      unsigned component_mask = (width - 1) * component_step;
+      for (unsigned i = 0; i < count; ++i) {
+         if (i & component_mask)
+            continue;
+         nir_deref_instr *ptr = matrix_memory(b, intr, i);
+         if (width > 1) {
+            unsigned align = width * glsl_base_type_bit_size(desc.element_type) / 8;
+            ptr = nir_build_deref_cast_with_alignment(b, &ptr->def, ptr->modes,
+               glsl_vector_type(desc.element_type, width), 0, align, 0);
+         }
+         if (store) {
+            nir_def *channels[4];
+            for (unsigned j = 0; j < width; ++j)
+               channels[j] = nir_channel(b, data, i + j * component_step);
+            nir_store_deref(b, ptr, nir_vec(b, channels, width), ~0);
+         } else {
+            nir_def *loaded = nir_load_deref(b, ptr);
+            for (unsigned j = 0; j < width; ++j)
+               values[i + j * component_step] = nir_channel(b, loaded, j);
+         }
+      }
+      if (!store)
+         store_matrix(b, intr->src[0], nir_vec(b, values, count));
+      break;
+   }
+   case nir_intrinsic_cmat_length: {
+      struct glsl_cmat_description desc = nir_intrinsic_cmat_desc(intr);
+      nir_def_replace(&intr->def, nir_imm_int(b, desc.rows * desc.cols / 32));
+      return true;
+   }
+   case nir_intrinsic_cmat_muladd: {
+      assert(!nir_intrinsic_saturate(intr));
+      enum glsl_base_type input = matrix_desc(intr->src[1]).element_type;
+      enum glsl_base_type input_b = matrix_desc(intr->src[2]).element_type;
+      enum glsl_base_type accum = matrix_desc(intr->src[3]).element_type;
+      enum glsl_base_type result = matrix_desc(intr->src[0]).element_type;
+      assert((input == GLSL_TYPE_FLOAT16 || input == GLSL_TYPE_BFLOAT16 ||
+              (input == GLSL_TYPE_FLOAT && matrix_desc(intr->src[1]).rows == 8)) &&
+             (input_b == input || (matrix_desc(intr->src[1]).rows == 16 &&
+                ((input == GLSL_TYPE_FLOAT16 && input_b == GLSL_TYPE_BFLOAT16) ||
+                 (input == GLSL_TYPE_BFLOAT16 && input_b == GLSL_TYPE_FLOAT16)))) &&
+             (accum == GLSL_TYPE_FLOAT || accum == GLSL_TYPE_BFLOAT16) && result == accum);
+      nir_def *a = load_matrix(b, intr->src[1]);
+      nir_def *bb = load_matrix(b, intr->src[2]);
+      nir_def *c = convert_matrix_value(b, load_matrix(b, intr->src[3]),
+                                         accum, GLSL_TYPE_FLOAT);
+      nir_def *value = nir_cmat_muladd_agx(b, a, bb, c, .src_type =
+         input == GLSL_TYPE_BFLOAT16 ? nir_type_uint16 :
+         input == GLSL_TYPE_FLOAT ? nir_type_float32 : nir_type_float16,
+         .src_type2 = input_b == GLSL_TYPE_BFLOAT16 ? nir_type_uint16 :
+         input_b == GLSL_TYPE_FLOAT ? nir_type_float32 : nir_type_float16);
+      store_matrix(b, intr->src[0], convert_matrix_value(b, value,
+                                                      GLSL_TYPE_FLOAT, result));
+      break;
+   }
+   case nir_intrinsic_cmat_unary_op:
+   case nir_intrinsic_cmat_binary_op:
+   case nir_intrinsic_cmat_scalar_op: {
+      b->fp_math_ctrl = nir_intrinsic_fp_math_ctrl(intr);
+      nir_op op = nir_intrinsic_alu_op(intr);
+      enum glsl_base_type type = matrix_desc(intr->src[1]).element_type;
+      nir_def *a = load_matrix(b, intr->src[1]);
+      if (type == GLSL_TYPE_BFLOAT16)
+         a = nir_bf2f(b, a);
+      nir_def *value;
+      if (intr->intrinsic == nir_intrinsic_cmat_unary_op) {
+         value = nir_build_alu1(b, op, a);
+      } else {
+         nir_def *bb = intr->intrinsic == nir_intrinsic_cmat_scalar_op
+            ? intr->src[2].ssa : load_matrix(b, intr->src[2]);
+         if (type == GLSL_TYPE_BFLOAT16)
+            bb = nir_bf2f(b, bb);
+         value = nir_build_alu2(b, op, a, bb);
+      }
+      if (type == GLSL_TYPE_BFLOAT16)
+         value = nir_f2bf(b, value);
+      store_matrix(b, intr->src[0], value);
+      b->fp_math_ctrl = nir_fp_fast_math;
+      break;
+   }
+   case nir_intrinsic_cmat_convert: {
+      struct glsl_cmat_description src = matrix_desc(intr->src[1]);
+      struct glsl_cmat_description dst = matrix_desc(intr->src[0]);
+      b->fp_math_ctrl = nir_intrinsic_fp_math_ctrl(intr);
+      store_matrix(b, intr->src[0], convert_matrix_value(b,
+         load_matrix(b, intr->src[1]), src.element_type, dst.element_type));
+      b->fp_math_ctrl = nir_fp_fast_math;
+      break;
+   }
+   case nir_intrinsic_cmat_bitcast:
+      store_matrix(b, intr->src[0], load_matrix(b, intr->src[1]));
+      break;
+   case nir_intrinsic_cmat_extract:
+      nir_def_replace(&intr->def, nir_vector_extract(b, load_matrix(b, intr->src[0]), intr->src[1].ssa));
+      return true;
+   case nir_intrinsic_cmat_insert:
+      store_matrix(b, intr->src[0], nir_vector_insert(b, load_matrix(b, intr->src[2]),
+                                                     intr->src[1].ssa, intr->src[3].ssa));
+      break;
+   case nir_intrinsic_cmat_copy:
+      store_matrix(b, intr->src[0], load_matrix(b, intr->src[1]));
+      break;
+   default:
+      return false;
+   }
+   nir_instr_remove(instr);
+   return true;
+}
+
+bool
+agx_nir_lower_cmat(nir_shader *nir)
+{
+   if (nir->info.stage != MESA_SHADER_COMPUTE || !nir->info.cs.has_cooperative_matrix)
+      return false;
+   struct hash_table *types = _mesa_pointer_hash_table_create(NULL);
+   nir_foreach_variable_with_modes(var, nir, nir_var_shader_temp)
+      var->type = remap_matrix_type(types, var->type);
+   nir_foreach_function_impl(impl, nir) {
+      nir_foreach_function_temp_variable(var, impl)
+         var->type = remap_matrix_type(types, var->type);
+      nir_builder b = nir_builder_create(impl);
+      nir_foreach_block_reverse_safe(block, impl) {
+         nir_foreach_instr_reverse_safe(instr, block)
+            lower_matrix_instr(&b, instr, types);
+      }
+      nir_progress(true, impl, nir_metadata_control_flow);
+   }
+   _mesa_hash_table_destroy(types, NULL);
+   return true;
+}
diff --git a/src/asahi/compiler/agx_nir_lower_cull_distance.c b/src/asahi/compiler/agx_nir_lower_cull_distance.c
index a009f4b3f5d7..b56cafab110a 100644
--- a/src/asahi/compiler/agx_nir_lower_cull_distance.c
+++ b/src/asahi/compiler/agx_nir_lower_cull_distance.c
@@ -39,7 +39,8 @@
 static bool
 lower_write(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
 {
-   if (intr->intrinsic != nir_intrinsic_store_output)
+   const bool mesh = intr->intrinsic == nir_intrinsic_store_per_vertex_output;
+   if (intr->intrinsic != nir_intrinsic_store_output && !mesh)
       return false;
 
    nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
@@ -47,7 +48,7 @@ lower_write(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
        sem.location != VARYING_SLOT_CLIP_DIST1)
       return false;
 
-   signed loc = sem.location + nir_src_as_uint(intr->src[1]);
+   signed loc = sem.location + nir_src_as_uint(intr->src[mesh ? 2 : 1]);
    unsigned total_component =
       (loc - VARYING_SLOT_CLIP_DIST0) * 4 +
       nir_intrinsic_component(intr);
@@ -67,10 +68,18 @@ lower_write(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
    nir_def *offs = nir_imm_int(b, component / 4);
    nir_def *v = nir_b2f32(b, nir_fge_imm(b, intr->src[0].ssa, 0.0));
 
-   nir_store_output(b, v, offs, .component = component % 4,
-                    .src_type = nir_type_float32,
-                    .io_semantics.location = VARYING_SLOT_CULL_PRIMITIVE,
-                    .io_semantics.num_slots = 2);
+   if (mesh) {
+      nir_store_per_vertex_output(b, v, intr->src[1].ssa, offs,
+                                 .component = component % 4,
+                                 .src_type = nir_type_float32,
+                                 .io_semantics.location = VARYING_SLOT_CULL_PRIMITIVE,
+                                 .io_semantics.num_slots = 2);
+   } else {
+      nir_store_output(b, v, offs, .component = component % 4,
+                       .src_type = nir_type_float32,
+                       .io_semantics.location = VARYING_SLOT_CULL_PRIMITIVE,
+                       .io_semantics.num_slots = 2);
+   }
 
    return true;
 }
@@ -79,7 +88,8 @@ bool
 agx_nir_lower_cull_distance_vs(nir_shader *s)
 {
    assert(s->info.stage == MESA_SHADER_VERTEX ||
-          s->info.stage == MESA_SHADER_TESS_EVAL);
+          s->info.stage == MESA_SHADER_TESS_EVAL ||
+          s->info.stage == MESA_SHADER_MESH);
 
    nir_shader_intrinsics_pass(s, lower_write, nir_metadata_control_flow, NULL);
 
diff --git a/src/asahi/compiler/agx_nir_lower_divmod64.c b/src/asahi/compiler/agx_nir_lower_divmod64.c
index 1496e075a7b9..5da950a0b9a1 100644
--- a/src/asahi/compiler/agx_nir_lower_divmod64.c
+++ b/src/asahi/compiler/agx_nir_lower_divmod64.c
@@ -27,10 +27,14 @@
 
 /* G17P has no 64-bit add or subtract, and nir_lower_int64 only lowers those
  * for shaders whose source has a 64-bit add, so build them from 32-bit halves
- * here as nir_lower_int64 would. */
+ * here as nir_lower_int64 would. M5 adds and subtracts 64-bit values natively
+ * (native_add64). */
 static nir_def *
-iadd64(nir_builder *b, nir_def *x, nir_def *y)
+iadd64(nir_builder *b, nir_def *x, nir_def *y, bool native_add64)
 {
+   if (native_add64)
+      return nir_iadd(b, x, y);
+
    nir_def *x_lo = nir_unpack_64_2x32_split_x(b, x);
    nir_def *x_hi = nir_unpack_64_2x32_split_y(b, x);
    nir_def *y_lo = nir_unpack_64_2x32_split_x(b, y);
@@ -43,8 +47,11 @@ iadd64(nir_builder *b, nir_def *x, nir_def *y)
 }
 
 static nir_def *
-isub64(nir_builder *b, nir_def *x, nir_def *y)
+isub64(nir_builder *b, nir_def *x, nir_def *y, bool native_add64)
 {
+   if (native_add64)
+      return nir_isub(b, x, y);
+
    nir_def *x_lo = nir_unpack_64_2x32_split_x(b, x);
    nir_def *x_hi = nir_unpack_64_2x32_split_y(b, x);
    nir_def *y_lo = nir_unpack_64_2x32_split_x(b, y);
@@ -57,8 +64,8 @@ isub64(nir_builder *b, nir_def *x, nir_def *y)
 }
 
 static void
-udivmod64(nir_builder *b, nir_def *n, nir_def *d, nir_def **q_out,
-          nir_def **r_out)
+udivmod64(nir_builder *b, nir_def *n, nir_def *d, bool native_add64,
+          nir_def **q_out, nir_def **r_out)
 {
    nir_variable *q_var =
       nir_local_variable_create(b->impl, glsl_uint64_t_type(), "divmod64_q");
@@ -86,7 +93,7 @@ udivmod64(nir_builder *b, nir_def *n, nir_def *d, nir_def **q_out,
          nir_def *dd = nir_load_var(b, d_var);
          nir_def *ge = nir_uge(b, r, dd);
 
-         nir_store_var(b, r_var, nir_bcsel(b, ge, isub64(b, r, dd), r), 1);
+         nir_store_var(b, r_var, nir_bcsel(b, ge, isub64(b, r, dd, native_add64), r), 1);
          nir_store_var(b, q_var,
                        nir_ior(b, nir_ishl_imm(b, nir_load_var(b, q_var), 1),
                                nir_b2i64(b, ge)),
@@ -106,16 +113,17 @@ udivmod64(nir_builder *b, nir_def *n, nir_def *d, nir_def **q_out,
 }
 
 static nir_def *
-lower_scalar(nir_builder *b, nir_op op, nir_def *n, nir_def *d)
+lower_scalar(nir_builder *b, nir_op op, nir_def *n, nir_def *d,
+             bool native_add64)
 {
    nir_def *q, *r;
 
    switch (op) {
    case nir_op_udiv:
-      udivmod64(b, n, d, &q, &r);
+      udivmod64(b, n, d, native_add64, &q, &r);
       return q;
    case nir_op_umod:
-      udivmod64(b, n, d, &q, &r);
+      udivmod64(b, n, d, native_add64, &q, &r);
       return r;
    default:
       break;
@@ -126,7 +134,7 @@ lower_scalar(nir_builder *b, nir_op op, nir_def *n, nir_def *d)
    nir_def *d_neg = nir_ilt_imm(b, d, 0);
    nir_def *n_abs = nir_bcsel(b, n_neg, nir_ineg(b, n), n);
    nir_def *d_abs = nir_bcsel(b, d_neg, nir_ineg(b, d), d);
-   udivmod64(b, n_abs, d_abs, &q, &r);
+   udivmod64(b, n_abs, d_abs, native_add64, &q, &r);
 
    switch (op) {
    case nir_op_idiv:
@@ -141,7 +149,7 @@ lower_scalar(nir_builder *b, nir_op op, nir_def *n, nir_def *d)
       nir_def *rem = nir_bcsel(b, n_neg, nir_ineg(b, r), r);
       nir_def *fix = nir_iand(b, nir_ine_imm(b, rem, 0),
                               nir_ine(b, n_neg, d_neg));
-      return nir_bcsel(b, fix, iadd64(b, rem, d), rem);
+      return nir_bcsel(b, fix, iadd64(b, rem, d, native_add64), rem);
    }
    default:
       UNREACHABLE("checked by the caller");
@@ -149,7 +157,7 @@ lower_scalar(nir_builder *b, nir_op op, nir_def *n, nir_def *d)
 }
 
 static bool
-lower_divmod64(nir_builder *b, nir_alu_instr *alu, UNUSED void *data)
+lower_divmod64(nir_builder *b, nir_alu_instr *alu, bool native_add64)
 {
    switch (alu->op) {
    case nir_op_udiv:
@@ -171,7 +179,7 @@ lower_divmod64(nir_builder *b, nir_alu_instr *alu, UNUSED void *data)
    nir_def *comps[NIR_MAX_VEC_COMPONENTS];
    for (unsigned c = 0; c < alu->def.num_components; ++c) {
       comps[c] = lower_scalar(b, alu->op, nir_channel(b, n, c),
-                              nir_channel(b, d, c));
+                              nir_channel(b, d, c), native_add64);
    }
 
    nir_def_replace(&alu->def, nir_vec(b, comps, alu->def.num_components));
@@ -179,7 +187,7 @@ lower_divmod64(nir_builder *b, nir_alu_instr *alu, UNUSED void *data)
 }
 
 bool
-agx_nir_lower_divmod64(nir_shader *shader)
+agx_nir_lower_divmod64(nir_shader *shader, bool native_add64)
 {
    bool progress = false;
 
@@ -196,7 +204,7 @@ agx_nir_lower_divmod64(nir_shader *shader)
       bool impl_progress = false;
       nir_builder b = nir_builder_create(impl);
       util_dynarray_foreach(&sites, nir_alu_instr *, alu)
-         impl_progress |= lower_divmod64(&b, *alu, NULL);
+         impl_progress |= lower_divmod64(&b, *alu, native_add64);
       util_dynarray_fini(&sites);
 
       nir_progress(impl_progress, impl, nir_metadata_none);
diff --git a/src/asahi/compiler/agx_nir_lower_fma_rte.c b/src/asahi/compiler/agx_nir_lower_fma_rte.c
index 5211bc34db4b..aece4dbc8ee0 100644
--- a/src/asahi/compiler/agx_nir_lower_fma_rte.c
+++ b/src/asahi/compiler/agx_nir_lower_fma_rte.c
@@ -4,9 +4,9 @@
 #include "compiler/nir/nir_builder.h"
 #include "agx_nir.h"
 
-nir_def *
-agx_nir_fma_rte_fixup(nir_builder *b, nir_def *a, nir_def *c, nir_def *z,
-                      nir_def *native)
+static nir_def *
+repair_tininess(nir_builder *b, nir_def *a, nir_def *c, nir_def *z,
+                nir_def *native)
 {
    /* G17P detects FMA tininess before rounding. Repair only a flushed result
     * whose exact magnitude lies in [2^-126 - 2^-150, 2^-126). RTE rounds that
@@ -87,6 +87,24 @@ agx_nir_fma_rte_fixup(nir_builder *b, nir_def *a, nir_def *c, nir_def *z,
    return nir_bcsel(b, fix, normal, native);
 }
 
+nir_def *
+agx_nir_fma_rte_fixup(nir_builder *b, nir_def *a, nir_def *c, nir_def *z,
+                      nir_def *native)
+{
+   /* The repair only changes a flushed zero. Keep its multiword integer
+    * arithmetic off the normal-result path, particularly for shaders which
+    * explicitly request RTE for every FP32 operation. A select would execute
+    * that arithmetic for every FMA even when the native result is normal.
+    */
+   nir_def *is_zero =
+      nir_ieq_imm(b, nir_iand_imm(b, native, 0x7fffffff), 0);
+   nir_if *repair = nir_push_if(b, is_zero);
+   nir_def *fixed = repair_tininess(b, a, c, z, native);
+   nir_push_else(b, repair);
+   nir_pop_if(b, repair);
+   return nir_if_phi(b, fixed, native);
+}
+
 static bool
 filter(const nir_instr *instr, const void *data)
 {
@@ -105,11 +123,17 @@ lower(nir_builder *b, nir_instr *instr, void *data)
 }
 
 bool
-agx_nir_lower_fma_rte(nir_shader *shader)
+agx_nir_fma_rte_applies(const nir_shader *shader)
 {
    unsigned mode = shader->info.float_controls_execution_mode;
-   if (!(mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32) ||
-       (mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP32))
+   return (mode & FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32) &&
+          !(mode & FLOAT_CONTROLS_DENORM_PRESERVE_FP32);
+}
+
+bool
+agx_nir_lower_fma_rte(nir_shader *shader)
+{
+   if (!agx_nir_fma_rte_applies(shader))
       return false;
    return nir_shader_lower_instructions(shader, filter, lower, NULL);
 }
diff --git a/src/asahi/compiler/agx_nir_lower_fminmax.c b/src/asahi/compiler/agx_nir_lower_fminmax.c
index 74691a863c1e..942c4cdfccfa 100644
--- a/src/asahi/compiler/agx_nir_lower_fminmax.c
+++ b/src/asahi/compiler/agx_nir_lower_fminmax.c
@@ -6,6 +6,7 @@
 #include "compiler/nir/nir.h"
 #include "compiler/nir/nir_builder.h"
 #include "agx_nir.h"
+#include <string.h>
 
 /*
  * AGX generally flushes FP32 denorms. However, the min/max instructions do not
@@ -132,3 +133,180 @@ agx_nir_lower_fminmax(nir_shader *s, bool ftz_only)
 {
    return nir_shader_alu_pass(s, lower, nir_metadata_control_flow, &ftz_only);
 }
+
+/* A shared-memory load cannot introduce a denormal if every writer stores
+ * canonical FP32 values. Uninitialized shared reads have undefined values;
+ * this analysis never relies on a particular value for them. Integer/partial
+ * writes and atomics conservatively disable the shared-memory proof. */
+struct canonical_state {
+   bool shared_canonical;
+   unsigned removed;
+};
+
+static bool
+canonical_value(nir_scalar value, bool shared_canonical, nir_scalar *path,
+                unsigned depth, unsigned *budget)
+{
+   if (!*budget)
+      return false;
+   (*budget)--;
+   value = nir_scalar_chase_movs(value);
+   if (value.def->bit_size != 32 || depth >= 64)
+      return false;
+   for (unsigned i = 0; i < depth; ++i)
+      if (path[i].def == value.def && path[i].comp == value.comp)
+         return false;
+   path[depth++] = value;
+
+   if (nir_scalar_is_const(value)) {
+      uint32_t bits = nir_scalar_as_uint(value);
+      if ((bits & 0x7f800000u) == 0x7f800000u && (bits & 0x007fffffu) != 0)
+         return false; /* Do not remove a possible signalling-NaN quieting. */
+      return (bits & 0x7f800000u) != 0 || (bits & 0x007fffffu) == 0;
+   }
+
+   if (nir_def_instr(value.def)->type == nir_instr_type_phi) {
+      nir_phi_instr *phi = nir_def_as_phi(value.def);
+      nir_foreach_phi_src(src, phi) {
+         if (!canonical_value(nir_get_scalar(src->src.ssa, value.comp),
+                              shared_canonical, path, depth, budget))
+            return false;
+      }
+      return true;
+   }
+
+   if (nir_scalar_is_intrinsic(value)) {
+      if (!shared_canonical ||
+          nir_scalar_intrinsic_op(value) != nir_intrinsic_load_shared)
+         return false;
+      nir_intrinsic_instr *load = nir_def_as_intrinsic(value.def);
+      return nir_intrinsic_align_mul(load) >= 4 &&
+             (nir_intrinsic_align_offset(load) & 3) == 0;
+   }
+
+   if (!nir_scalar_is_alu(value))
+      return false;
+
+   nir_op op = nir_scalar_alu_op(value);
+   switch (op) {
+   case nir_op_bcsel:
+   case nir_op_b32csel:
+      return canonical_value(nir_scalar_chase_alu_src(value, 1),
+                             shared_canonical, path, depth, budget) &&
+             canonical_value(nir_scalar_chase_alu_src(value, 2),
+                             shared_canonical, path, depth, budget);
+   case nir_op_fmin:
+   case nir_op_fmax:
+   case nir_op_fmin_agx:
+   case nir_op_fmax_agx:
+      return canonical_value(nir_scalar_chase_alu_src(value, 0),
+                             shared_canonical, path, depth, budget) &&
+             canonical_value(nir_scalar_chase_alu_src(value, 1),
+                             shared_canonical, path, depth, budget);
+   case nir_op_fneg:
+   case nir_op_fabs:
+      return canonical_value(nir_scalar_chase_alu_src(value, 0),
+                             shared_canonical, path, depth, budget);
+   case nir_op_f2f32:
+      return nir_scalar_chase_alu_src(value, 0).def->bit_size == 16;
+   case nir_op_fadd:
+   case nir_op_fmul:
+   case nir_op_ffma:
+   case nir_op_fdiv:
+   case nir_op_frcp:
+   case nir_op_frsq:
+   case nir_op_fsqrt:
+   case nir_op_fexp2:
+   case nir_op_flog2:
+   case nir_op_ffloor:
+   case nir_op_fceil:
+   case nir_op_ftrunc:
+   case nir_op_fround_even:
+   case nir_op_i2f32:
+   case nir_op_u2f32:
+      return true;
+   default:
+      return false;
+   }
+}
+
+static bool
+canonical(nir_def *def, unsigned component, bool shared_canonical)
+{
+   nir_scalar path[64];
+   unsigned budget = 1024;
+   return canonical_value(nir_get_scalar(def, component), shared_canonical,
+                          path, 0, &budget);
+}
+
+static bool
+remove_canonical_minmax(nir_builder *b, nir_alu_instr *alu, void *data)
+{
+   struct canonical_state *state = data;
+   if (alu->op != nir_op_fadd || alu->def.bit_size != 32 ||
+       alu->def.num_components != 1)
+      return false;
+
+   nir_scalar self = nir_get_scalar(&alu->def, 0);
+   for (unsigned i = 0; i < 2; ++i) {
+      nir_scalar zero = nir_scalar_chase_alu_src(self, i);
+      nir_scalar value = nir_scalar_chase_alu_src(self, 1 - i);
+      if (!nir_scalar_is_const(zero) ||
+          nir_scalar_as_uint(zero) != 0x80000000u || !nir_scalar_is_alu(value))
+         continue;
+      nir_op op = nir_scalar_alu_op(value);
+      if (op != nir_op_fmin_agx && op != nir_op_fmax_agx)
+         continue;
+      if (!canonical(value.def, value.comp, state->shared_canonical))
+         continue;
+      b->cursor = nir_before_instr(&alu->instr);
+      nir_def *replacement = nir_channel(b, value.def, value.comp);
+      nir_def_rewrite_uses(&alu->def, replacement);
+      nir_instr_remove(&alu->instr);
+      state->removed++;
+      return true;
+   }
+   return false;
+}
+
+bool
+agx_nir_opt_canonical_minmax(nir_shader *s)
+{
+   if (nir_is_denorm_preserve(s->info.float_controls_execution_mode, 32))
+      return false;
+
+   bool seen_store = false, shared_canonical = true;
+   nir_foreach_function_impl(impl, s) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+            if (intr->intrinsic == nir_intrinsic_store_shared) {
+               seen_store = true;
+               nir_def *value = intr->src[0].ssa;
+               if (value->bit_size != 32 || nir_intrinsic_align_mul(intr) < 4 ||
+                   (nir_intrinsic_align_offset(intr) & 3) != 0) {
+                  shared_canonical = false;
+                  continue;
+               }
+               for (unsigned c = 0; c < value->num_components; ++c)
+                  if ((nir_intrinsic_write_mask(intr) & (1u << c)) &&
+                      !canonical(value, c, false))
+                     shared_canonical = false;
+            } else {
+               const char *name = nir_intrinsic_infos[intr->intrinsic].name;
+               if ((strstr(name, "shared") &&
+                    (strstr(name, "atomic") || strstr(name, "store"))) ||
+                   (strstr(name, "deref") &&
+                    (strstr(name, "atomic") || strstr(name, "store") ||
+                     strstr(name, "copy"))))
+                  shared_canonical = false;
+            }
+         }
+      }
+   }
+   struct canonical_state state = {seen_store && shared_canonical, 0};
+   return nir_shader_alu_pass(s, remove_canonical_minmax,
+                              nir_metadata_control_flow, &state);
+}
diff --git a/src/asahi/compiler/agx_nir_lower_g17p_combined_sampler.c b/src/asahi/compiler/agx_nir_lower_g17p_combined_sampler.c
index 148da4438221..d5c0d07e06ce 100644
--- a/src/asahi/compiler/agx_nir_lower_g17p_combined_sampler.c
+++ b/src/asahi/compiler/agx_nir_lower_g17p_combined_sampler.c
@@ -26,6 +26,14 @@ g17p_explicit_lod_stage(mesa_shader_stage stage)
           stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_TESS_EVAL;
 }
 
+/* M5 TA programs (API mesh and object/task) have no fragment derivatives and
+ * take the same heap-indexed texture/image operands as compute. */
+static bool
+m5_ta_stage(mesa_shader_stage stage)
+{
+   return stage == MESA_SHADER_MESH || stage == MESA_SHADER_TASK;
+}
+
 /* agx_nir_lower_g17p_indexed_samplers() is all-or-nothing per shader: one
  * unadmitted texture or image instruction and NOTHING in that shader gets a
  * heap index, so every texture instruction in it then reaches the packer with
@@ -70,7 +78,7 @@ g17p_decline(const char *why, const nir_tex_instr *tex,
 }
 
 static bool
-g17p_sample_shape(const nir_tex_instr *tex)
+g17p_sample_shape(const nir_tex_instr *tex, bool m5)
 {
    const bool query_lod = tex->op == nir_texop_lod;
    const int backend_coord =
@@ -243,8 +251,10 @@ g17p_sample_shape(const nir_tex_instr *tex)
     * (+8,+8) is 0x00080008), not in the compact SMP byte used after
     * agx_emit_tex() folds it.  Admit exactly that generated representation;
     * the backend later maps it to (y << 4) | x. */
+   /* C's texture lowering (generation 10) packs gather offsets like any
+    * other constant offset: compact (y << 4) | x. */
    const bool valid_constant_offset =
-      tex->op == nir_texop_tg4
+      tex->op == nir_texop_tg4 && !m5
          ? (constant_offset & ~0x000f000fu) == 0
          : constant_offset <= (three_d ? 0xfffu : 0xffu);
    /* Native bank-6 minimum operands are grounded for 2D, 2D-array and 3D
@@ -509,7 +519,7 @@ g17p_combined_sampler_candidate(const nir_shader *shader,
             if (instr->type != nir_instr_type_tex)
                continue;
 
-            if (sample || !g17p_sample_shape(nir_instr_as_tex(instr)))
+            if (sample || !g17p_sample_shape(nir_instr_as_tex(instr), false))
                return NULL;
             sample = nir_instr_as_tex(instr);
          }
@@ -781,11 +791,12 @@ g17p_indexed_image_fetch_shape(const nir_intrinsic_instr *intr)
  * after command recording and dynamically selected array elements are visible.
  */
 static bool
-lower_indexed_samplers(nir_shader *shader, int native32_set)
+lower_indexed_samplers(nir_shader *shader, int native32_set, bool m5)
 {
    if (!shader || (!g17p_explicit_lod_stage(shader->info.stage) &&
                    shader->info.stage != MESA_SHADER_FRAGMENT &&
-                   shader->info.stage != MESA_SHADER_COMPUTE))
+                   shader->info.stage != MESA_SHADER_COMPUTE &&
+                   !(m5 && m5_ta_stage(shader->info.stage))))
       return false;
 
    unsigned samples = 0, image_reads = 0;
@@ -814,12 +825,13 @@ lower_indexed_samplers(nir_shader *shader, int native32_set)
             /* Explicit levels and gradients need no fragment derivatives.
              * Gather uses the packer's existing non-fragment level-zero form.
              * Ordinary implicit samples and biases still remain closed. */
-            if (!fetch && g17p_explicit_lod_stage(shader->info.stage) &&
+            if (!fetch && (g17p_explicit_lod_stage(shader->info.stage) ||
+                           (m5 && m5_ta_stage(shader->info.stage))) &&
                 tex->op != nir_texop_txl && tex->op != nir_texop_txd &&
                 tex->op != nir_texop_tg4)
                return g17p_decline("vertex_implicit_sample", tex, NULL,
                                    shader->info.stage);
-            if (!fetch && !g17p_sample_shape(tex))
+            if (!fetch && !g17p_sample_shape(tex, m5))
                return g17p_decline("sample_shape", tex, NULL,
                                    shader->info.stage);
             nir_intrinsic_instr *image = nir_src_as_intrinsic(
@@ -982,13 +994,145 @@ lower_indexed_samplers(nir_shader *shader, int native32_set)
 bool
 agx_nir_lower_g17p_indexed_samplers(nir_shader *shader)
 {
-   return lower_indexed_samplers(shader, -1);
+   return lower_indexed_samplers(shader, -1, false);
+}
+
+
+/* G17G multisample texel fetches: fragment, compute and vertex. The vertex
+ * form is the fragment form with sampler index 0 (port/vmswit v_ms_f, v_ms_u,
+ * v_msa_f, v_ms_k against f_ms_f). */
+static bool
+g17g_ms_read_stage(mesa_shader_stage stage)
+{
+   return stage == MESA_SHADER_FRAGMENT || stage == MESA_SHADER_COMPUTE ||
+          stage == MESA_SHADER_VERTEX;
+}
+
+bool
+agx_nir_lower_g17g_indexed_reads(nir_shader *shader)
+{
+   if (!shader || (!g17p_explicit_lod_stage(shader->info.stage) &&
+                   shader->info.stage != MESA_SHADER_FRAGMENT &&
+                   shader->info.stage != MESA_SHADER_COMPUTE &&
+                   !m5_ta_stage(shader->info.stage)))
+      return false;
+
+   unsigned reads = 0;
+   nir_foreach_function_impl(impl, shader) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type == nir_instr_type_intrinsic) {
+               const nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+               if (intr->intrinsic == nir_intrinsic_bindless_image_load ||
+                   intr->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
+                  if (nir_intrinsic_image_dim(intr) == GLSL_SAMPLER_DIM_MS &&
+                      !g17g_ms_read_stage(shader->info.stage))
+                     return false;
+                  if ((nir_intrinsic_image_dim(intr) != GLSL_SAMPLER_DIM_2D &&
+                       nir_intrinsic_image_dim(intr) != GLSL_SAMPLER_DIM_3D &&
+                       nir_intrinsic_image_dim(intr) != GLSL_SAMPLER_DIM_CUBE &&
+                       nir_intrinsic_image_dim(intr) != GLSL_SAMPLER_DIM_MS) ||
+                      intr->def.bit_size != 32)
+                     return false;
+                  reads++;
+               }
+            }
+            if (instr->type != nir_instr_type_tex)
+               continue;
+            const nir_tex_instr *tex = nir_instr_as_tex(instr);
+            const int lod = nir_tex_instr_src_index(tex, nir_tex_src_lod);
+            const bool multisample = tex->sampler_dim == GLSL_SAMPLER_DIM_MS;
+            const bool query_lod = tex->op == nir_texop_lod &&
+               shader->info.stage == MESA_SHADER_FRAGMENT &&
+               tex->dest_type == nir_type_float32 && tex->def.bit_size == 32 &&
+               (tex->sampler_dim == GLSL_SAMPLER_DIM_2D ||
+                tex->sampler_dim == GLSL_SAMPLER_DIM_3D ||
+                tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE);
+            if (multisample && !g17g_ms_read_stage(shader->info.stage))
+               return false;
+            if ((!multisample && !query_lod &&
+                 tex->sampler_dim != GLSL_SAMPLER_DIM_2D &&
+                 tex->sampler_dim != GLSL_SAMPLER_DIM_3D &&
+                 tex->sampler_dim != GLSL_SAMPLER_DIM_CUBE))
+               return false;
+
+            /* C uses the common heap indices but its own SMP selectors and
+             * staging. Ordinary 2D/array/3D/cube samples share the supported
+             * LOD forms and interleaved gradient banks. Minimum LOD uses
+             * typed publishers shared with the dimension's layer/face.
+             * Runtime offsets retain their generation-specific requirements. */
+            const int offset = nir_tex_instr_src_index(tex, nir_tex_src_offset);
+            const int packed_offset =
+               nir_tex_instr_src_index(tex, nir_tex_src_backend2);
+            const bool constant_offset =
+               (offset < 0 || nir_src_is_const(tex->src[offset].src)) &&
+               (packed_offset < 0 ||
+                nir_src_is_const(tex->src[packed_offset].src));
+            const bool ms_fetch = multisample &&
+               g17p_indexed_fetch_shape(tex) && offset < 0 && packed_offset < 0;
+            const bool fetch = ms_fetch || (!multisample &&
+               tex->op == nir_texop_txf && lod >= 0 &&
+               (tex->sampler_dim == GLSL_SAMPLER_DIM_2D ||
+                tex->sampler_dim == GLSL_SAMPLER_DIM_3D) &&
+               offset < 0 && packed_offset < 0 &&
+               (tex->dest_type == nir_type_uint32 ||
+                tex->dest_type == nir_type_float32 ||
+                tex->dest_type == nir_type_int32));
+            const bool minimum =
+               nir_tex_instr_src_index(tex, nir_tex_src_min_lod) >= 0 ||
+               nir_tex_instr_src_index(tex, nir_tex_src_lod_bias_min_agx) >= 0;
+            /* Constant offsets are SMP immediates on C as on B: native G17G
+             * (texwit3) carries {f43, f51, f141} on 2D, 2D-array, 3D,
+             * comparison and gather samples alike. Cubes have no offset. */
+            /* Runtime offsets: the generation-10 lowering keeps X/Y separate
+             * (a component vector, or a 2x16 pair for a gather) on non-array
+             * 2D samples without a register LOD/bias/minimum; the packer
+             * publishes them with PACK_SMPIR into bank 6 (texwit3/4 d_*). */
+            const bool runtime_offset = !constant_offset &&
+               offset < 0 && packed_offset >= 0 && !tex->is_array &&
+               tex->sampler_dim == GLSL_SAMPLER_DIM_2D && !minimum &&
+               nir_src_num_components(tex->src[packed_offset].src) ==
+                  (tex->op == nir_texop_tg4 ? 1u : 2u) &&
+               (lod < 0 || nir_src_is_const(tex->src[lod].src)) &&
+               (nir_tex_instr_src_index(tex, nir_tex_src_bias) < 0 ||
+                nir_src_is_const(tex->src[nir_tex_instr_src_index(
+                   tex, nir_tex_src_bias)].src));
+            /* A 2D or 2D-array sample whose runtime offset shares bank 6
+             * with its layer or register LOD/bias arrives as G17P's complete
+             * 32-bit word (one component), staged by COPY; a comparison's
+             * reference rides in front of it (texwit6 c_arr, c_lvlr,
+             * g_cmp_arr). */
+            const bool packed_runtime_offset = !constant_offset &&
+               offset < 0 && packed_offset >= 0 &&
+               (tex->op != nir_texop_tg4 || tex->is_array) &&
+               (tex->sampler_dim == GLSL_SAMPLER_DIM_2D ||
+                (tex->sampler_dim == GLSL_SAMPLER_DIM_3D &&
+                 nir_tex_instr_src_index(tex, nir_tex_src_lod_bias_min_agx) >= 0)) &&
+               nir_src_num_components(tex->src[packed_offset].src) == 1;
+            const bool sample = !multisample &&
+               (constant_offset || runtime_offset || packed_runtime_offset) &&
+               (tex->sampler_dim != GLSL_SAMPLER_DIM_CUBE ||
+                (offset < 0 && packed_offset < 0)) &&
+               (tex->dest_type == nir_type_float32 ||
+                tex->dest_type == nir_type_uint32 ||
+                tex->dest_type == nir_type_int32) &&
+               (tex->op == nir_texop_tex || tex->op == nir_texop_txl ||
+                tex->op == nir_texop_txb || tex->op == nir_texop_tg4 ||
+                (tex->op == nir_texop_txd &&
+                 tex->sampler_dim != GLSL_SAMPLER_DIM_CUBE));
+            if (!fetch && !sample && !query_lod)
+               return false;
+            reads++;
+         }
+      }
+   }
+   return reads > 0 && lower_indexed_samplers(shader, -1, true);
 }
 
 bool
 agx_nir_lower_g17p_native32_samplers(nir_shader *shader, uint8_t texture_set)
 {
-   return lower_indexed_samplers(shader, texture_set);
+   return lower_indexed_samplers(shader, texture_set, false);
 }
 
 static bool
@@ -1499,7 +1643,7 @@ agx_nir_lower_g17p_sampled_storage(
          block_count++;
          nir_foreach_instr(instr, block) {
             if (instr->type == nir_instr_type_tex) {
-               if (sample || !g17p_sample_shape(nir_instr_as_tex(instr)))
+               if (sample || !g17p_sample_shape(nir_instr_as_tex(instr), false))
                   return false;
                sample = nir_instr_as_tex(instr);
                continue;
diff --git a/src/asahi/compiler/agx_nir_lower_sample_mask.c b/src/asahi/compiler/agx_nir_lower_sample_mask.c
index 6b0c77784439..6d374841455e 100644
--- a/src/asahi/compiler/agx_nir_lower_sample_mask.c
+++ b/src/asahi/compiler/agx_nir_lower_sample_mask.c
@@ -88,6 +88,33 @@ lower_discard_to_sample_mask_0(nir_builder *b, nir_intrinsic_instr *intr,
    return true;
 }
 
+/* M5 (G17G/G17S) form of the final coverage update. Native M5 fragments
+ * (native G17G compiler: discard + alpha-to-coverage at 4x) issue exactly
+ * one testing MOVMSK, unconditionally, after their kills; a lane whose live
+ * set is empty still executes it. Whether this shader part triggers the
+ * tests is the uniform tests_zs (another linked part may own them), so pick
+ * the testing or the pure-kill form with a uniform branch instead of
+ * splitting per lane (agx_nir_split_g17p_sample_tests): a SIMD group never
+ * reaches the testing update with only some of its live lanes. */
+static void
+final_coverage_update(nir_builder *b, nir_def *killed, bool uniform_tests)
+{
+   nir_def *all_samples = nir_imm_intN_t(b, ALL_SAMPLES, 16);
+   nir_def *live = nir_ixor(b, killed, all_samples);
+   nir_def *testing = nir_load_shader_part_tests_zs_agx(b);
+
+   if (!uniform_tests) {
+      nir_sample_mask_agx(b, nir_ior(b, testing, killed), live);
+      return;
+   }
+
+   nir_push_if(b, nir_ine_imm(b, testing, 0));
+   nir_sample_mask_agx(b, all_samples, live);
+   nir_push_else(b, NULL);
+   nir_sample_mask_agx(b, killed, nir_imm_intN_t(b, 0, 16));
+   nir_pop_if(b, NULL);
+}
+
 static nir_intrinsic_instr *
 last_discard_in_block(nir_block *block)
 {
@@ -125,7 +152,7 @@ cf_node_contains_discard(nir_cf_node *node)
  * earliest unconditionally executed instruction after all discard.
  */
 static void
-run_tests_after_last_discard(nir_builder *b)
+run_tests_after_last_discard(nir_builder *b, bool uniform_tests)
 {
    foreach_list_typed_reverse(nir_cf_node, node, node, &b->impl->body) {
       if (node->type == nir_cf_node_block) {
@@ -145,15 +172,8 @@ run_tests_after_last_discard(nir_builder *b)
              * equivalent to `sample_mask killed, 0`, killing without testing.
              */
             b->cursor = nir_before_instr(&intr->instr);
-
-            nir_def *all_samples = nir_imm_intN_t(b, ALL_SAMPLES, 16);
-            nir_def *killed = nir_u2u16(b, intr->src[0].ssa);
-            nir_def *live = nir_ixor(b, killed, all_samples);
-
-            nir_def *testing = nir_load_shader_part_tests_zs_agx(b);
-            nir_def *affected = nir_ior(b, testing, killed);
-
-            nir_sample_mask_agx(b, affected, live);
+            final_coverage_update(b, nir_u2u16(b, intr->src[0].ssa),
+                                  uniform_tests);
             nir_instr_remove(&intr->instr);
             return;
          } else {
@@ -169,8 +189,15 @@ run_tests_after_last_discard(nir_builder *b)
           * If we're not testing, this turns into sample_mask(0, ~0) which is a
           * no-op.
           */
-         nir_sample_mask_agx(b, nir_load_shader_part_tests_zs_agx(b),
-                             nir_imm_intN_t(b, ALL_SAMPLES, 16));
+         if (uniform_tests) {
+            nir_push_if(b, nir_ine_imm(b, nir_load_shader_part_tests_zs_agx(b), 0));
+            nir_sample_mask_agx(b, nir_imm_intN_t(b, ALL_SAMPLES, 16),
+                                nir_imm_intN_t(b, ALL_SAMPLES, 16));
+            nir_pop_if(b, NULL);
+         } else {
+            nir_sample_mask_agx(b, nir_load_shader_part_tests_zs_agx(b),
+                                nir_imm_intN_t(b, ALL_SAMPLES, 16));
+         }
          return;
       }
    }
@@ -186,8 +213,8 @@ run_tests_at_start(nir_shader *shader)
                        nir_imm_intN_t(&b, ALL_SAMPLES, 16));
 }
 
-bool
-agx_nir_lower_sample_mask(nir_shader *shader)
+static bool
+lower_sample_mask(nir_shader *shader, bool uniform_tests)
 {
    nir_function_impl *impl = nir_shader_get_entrypoint(shader);
 
@@ -210,14 +237,15 @@ agx_nir_lower_sample_mask(nir_shader *shader)
          nir_builder b = nir_builder_create(impl);
 
          /* run tests late */
-         run_tests_after_last_discard(&b);
+         run_tests_after_last_discard(&b, uniform_tests);
       }
    } else {
       /* regular shaders that don't use discard have nothing to lower */
       return nir_no_progress(impl);
    }
 
-   nir_progress(true, impl, nir_metadata_control_flow);
+   nir_progress(true, impl,
+                uniform_tests ? nir_metadata_none : nir_metadata_control_flow);
 
    nir_shader_intrinsics_pass(shader, lower_discard_to_sample_mask_0,
                               nir_metadata_control_flow, NULL);
@@ -225,6 +253,19 @@ agx_nir_lower_sample_mask(nir_shader *shader)
    return true;
 }
 
+bool
+agx_nir_lower_sample_mask(nir_shader *shader)
+{
+   return lower_sample_mask(shader, false);
+}
+
+/* M5 shader parts: one uniformly selected final coverage update. */
+bool
+agx_nir_lower_sample_mask_m5(nir_shader *shader)
+{
+   return lower_sample_mask(shader, true);
+}
+
 struct deferred_ctx {
    nir_variable *killed;
 };
@@ -285,6 +326,9 @@ quad_dependent_instr(const nir_instr *instr)
    case nir_intrinsic_ddy_coarse:
    case nir_intrinsic_is_helper_invocation:
    case nir_intrinsic_load_helper_invocation:
+   case nir_intrinsic_load_active_subgroup_invocation_agx:
+   case nir_intrinsic_load_active_subgroup_count_agx:
+   case nir_intrinsic_quad_ballot_agx:
       return true;
    default:
       return false;
@@ -553,6 +597,10 @@ subgroup_instr(const nir_instr *instr)
 {
    if (instr->type != nir_instr_type_intrinsic)
       return false;
+   const nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;
+   if (op == nir_intrinsic_load_active_subgroup_invocation_agx ||
+       op == nir_intrinsic_load_active_subgroup_count_agx)
+      return true;
    /* Quad operations are also flagged SUBGROUP; they only read their own
     * quad, which a dead-quad kill never leaves partially alive. */
    const unsigned flags =
@@ -613,8 +661,8 @@ kill_dead_quads(nir_function_impl *impl, nir_variable *killed)
  * the fused form run_tests_after_last_discard() builds for an unconditional
  * last discard.
  */
-bool
-agx_nir_lower_sample_mask_deferred(nir_shader *shader)
+static bool
+lower_sample_mask_deferred(nir_shader *shader, bool uniform_tests)
 {
    nir_function_impl *impl = nir_shader_get_entrypoint(shader);
    const bool lanes_reach_sensitive_work =
@@ -630,7 +678,7 @@ agx_nir_lower_sample_mask_deferred(nir_shader *shader)
    if (!shader->info.fs.needs_coarse_quad_helper_invocations &&
        !shader->info.fs.needs_full_quad_helper_invocations &&
        !lanes_reach_sensitive_work)
-      return agx_nir_lower_sample_mask(shader);
+      return lower_sample_mask(shader, uniform_tests);
 
    bool writes_zs =
       shader->info.outputs_written &
@@ -639,7 +687,7 @@ agx_nir_lower_sample_mask_deferred(nir_shader *shader)
    const bool early = shader->info.fs.early_fragment_tests;
    if (!early && !writes_zs && shader->info.fs.uses_discard &&
        !lanes_reach_sensitive_work)
-      return agx_nir_lower_sample_mask(shader);
+      return lower_sample_mask(shader, uniform_tests);
 
    if (early) {
       if (shader->info.fs.uses_discard || writes_zs ||
@@ -696,13 +744,23 @@ agx_nir_lower_sample_mask_deferred(nir_shader *shader)
       /* Tests already ran (or zs_emit runs them): kill without testing. */
       nir_sample_mask_agx(&b, killed, nir_imm_intN_t(&b, 0, 16));
    } else {
-      nir_def *all_samples = nir_imm_intN_t(&b, ALL_SAMPLES, 16);
-      nir_def *live = nir_ixor(&b, killed, all_samples);
-      nir_def *testing = nir_load_shader_part_tests_zs_agx(&b);
-      nir_sample_mask_agx(&b, nir_ior(&b, testing, killed), live);
+      final_coverage_update(&b, killed, uniform_tests);
    }
 
    nir_progress(true, impl, nir_metadata_none);
    NIR_PASS(_, shader, nir_lower_vars_to_ssa);
    return true;
 }
+
+bool
+agx_nir_lower_sample_mask_deferred(nir_shader *shader)
+{
+   return lower_sample_mask_deferred(shader, false);
+}
+
+/* M5: one uniformly selected final coverage update (final_coverage_update). */
+bool
+agx_nir_lower_sample_mask_deferred_m5(nir_shader *shader)
+{
+   return lower_sample_mask_deferred(shader, true);
+}
diff --git a/src/asahi/compiler/agx_nir_lower_shared_bitsize.c b/src/asahi/compiler/agx_nir_lower_shared_bitsize.c
index aea89c56ae01..a9ebf30f5018 100644
--- a/src/asahi/compiler/agx_nir_lower_shared_bitsize.c
+++ b/src/asahi/compiler/agx_nir_lower_shared_bitsize.c
@@ -6,6 +6,10 @@
 #include "compiler/nir/nir_builder.h"
 #include "agx_compiler.h"
 #include "agx_nir.h"
+#include <stdlib.h>
+
+static nir_def *divide_exact(nir_builder *b, nir_scalar s, unsigned shift,
+                             unsigned depth);
 
 /* Local memory instructions require 16-bit offsets, so we add conversions. */
 static bool
@@ -30,7 +34,11 @@ pass(struct nir_builder *b, nir_intrinsic_instr *intr, void *data)
             ? intr->def.bit_size
             : nir_src_bit_size(intr->src[0]);
       const unsigned shift = util_logbase2(bit_size / 8);
-      address = nir_ushr_imm(b, address, shift);
+      /* Divide the byte arithmetic itself where that is exact, so a scaled
+       * thread index becomes an element index without a shift per access. */
+      nir_def *elements =
+         shift ? divide_exact(b, nir_get_scalar(address, 0), shift, 4) : NULL;
+      address = elements ? elements : nir_ushr_imm(b, address, shift);
    }
 
    if (address->bit_size != 16)
@@ -88,6 +96,7 @@ divide_exact(nir_builder *b, nir_scalar s, unsigned shift, unsigned depth)
       return k == shift ? kept : nir_ishl_imm(b, kept, k - shift);
    }
 
+   case nir_op_amul:
    case nir_op_imul: {
       for (unsigned i = 0; i < 2; ++i) {
          const nir_scalar factor = nir_scalar_chase_alu_src(s, i);
@@ -103,6 +112,41 @@ divide_exact(nir_builder *b, nir_scalar s, unsigned shift, unsigned depth)
       return NULL;
    }
 
+   case nir_op_imadshl_agx:
+   case nir_op_imsubshl_agx: {
+      /* (src0 * src1) +/- (src2 << src3): the shifted term divides when it
+       * shifts at least as far, the product when either factor is a constant
+       * multiple of the divisor (or zero). */
+      const nir_scalar amount = nir_scalar_chase_alu_src(s, 3);
+      if (!nir_scalar_is_const(amount))
+         return NULL;
+      const uint64_t k = nir_scalar_as_uint(amount);
+      if (k < shift)
+         return NULL;
+      nir_scalar factor[2] = {nir_scalar_chase_alu_src(s, 0),
+                              nir_scalar_chase_alu_src(s, 1)};
+      nir_def *f[2] = {nir_channel(b, factor[0].def, factor[0].comp),
+                       nir_channel(b, factor[1].def, factor[1].comp)};
+      bool divided = false;
+      for (unsigned i = 0; i < 2 && !divided; ++i) {
+         if (!nir_scalar_is_const(factor[i]))
+            continue;
+         const uint64_t c = nir_scalar_as_uint(factor[i]);
+         if (c & mask)
+            continue;
+         f[i] = nir_imm_intN_t(b, c >> shift, s.def->bit_size);
+         divided = true;
+      }
+      if (!divided)
+         return NULL;
+      const nir_scalar value = nir_scalar_chase_alu_src(s, 2);
+      nir_def *v = nir_channel(b, value.def, value.comp);
+      nir_def *k_s = nir_imm_int(b, k - shift);
+      return nir_scalar_alu_op(s) == nir_op_imadshl_agx
+                ? nir_imadshl_agx(b, f[0], f[1], v, k_s)
+                : nir_imsubshl_agx(b, f[0], f[1], v, k_s);
+   }
+
    case nir_op_iadd: {
       nir_def *x =
          divide_exact(b, nir_scalar_chase_alu_src(s, 0), shift, depth - 1);
@@ -188,3 +232,159 @@ agx_nir_lower_shared_element_index(nir_shader *shader)
    return nir_shader_intrinsics_pass(shader, element_index,
                                      nir_metadata_control_flow, NULL);
 }
+
+/* Flatten a 32-bit iadd tree into at most four non-constant terms and the sum
+ * of its constants. */
+static bool
+gather_sum(nir_scalar s, unsigned depth, nir_scalar *terms, unsigned *count,
+           uint64_t *constant)
+{
+   if (nir_scalar_is_const(s)) {
+      *constant += nir_scalar_as_uint(s);
+      return *constant <= UINT16_MAX;
+   }
+   if (depth && nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == nir_op_iadd) {
+      return gather_sum(nir_scalar_chase_alu_src(s, 0), depth - 1, terms, count,
+                        constant) &&
+             gather_sum(nir_scalar_chase_alu_src(s, 1), depth - 1, terms, count,
+                        constant);
+   }
+   if (*count == 4)
+      return false;
+   terms[(*count)++] = s;
+   return true;
+}
+
+/* nir_unsigned_upper_bound() with the AGX address multiply-add understood:
+ * (src0 * src1) + (src2 << src3), bounded termwise. */
+static uint64_t
+term_upper_bound(nir_shader *shader, struct hash_table *ranges, nir_scalar s,
+                 unsigned depth)
+{
+   if (depth && nir_scalar_is_alu(s) &&
+       nir_scalar_alu_op(s) == nir_op_imadshl_agx) {
+      const nir_scalar amount = nir_scalar_chase_alu_src(s, 3);
+      if (!nir_scalar_is_const(amount) || nir_scalar_as_uint(amount) > 31)
+         return UINT32_MAX;
+      const uint64_t x =
+         term_upper_bound(shader, ranges, nir_scalar_chase_alu_src(s, 0), depth - 1);
+      const uint64_t y =
+         term_upper_bound(shader, ranges, nir_scalar_chase_alu_src(s, 1), depth - 1);
+      const uint64_t z =
+         term_upper_bound(shader, ranges, nir_scalar_chase_alu_src(s, 2), depth - 1);
+      const unsigned k = nir_scalar_as_uint(amount);
+      if ((x && y > UINT32_MAX / x) || z > (UINT32_MAX >> k))
+         return UINT32_MAX;
+      return MIN2(x * y + (z << k), UINT32_MAX);
+   }
+   return nir_unsigned_upper_bound(shader, ranges, s);
+}
+
+static int
+compare_scalars(const void *a, const void *b)
+{
+   const nir_scalar *x = a, *y = b;
+   if (x->def->index != y->def->index)
+      return x->def->index < y->def->index ? -1 : 1;
+   return (int)x->comp - (int)y->comp;
+}
+
+/* Element-index lowering leaves byte additions beneath the shift/narrowing:
+ * u2u16((byte_address + constant) >> log2(element_size)).  Extract a small,
+ * aligned constant into LDTG/STTG's byte offset.  Both the source addition and
+ * the narrowed element index must be proven not to wrap: moving an add across
+ * either wrap would change the accessed address. */
+static bool
+fold_byte_offset(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_load_shared &&
+       intr->intrinsic != nir_intrinsic_store_shared)
+      return false;
+
+   const unsigned bits = intr->intrinsic == nir_intrinsic_load_shared
+                            ? intr->def.bit_size
+                            : nir_src_bit_size(intr->src[0]);
+   if (bits != 16 && bits != 32)
+      return false;
+   const unsigned shift = util_logbase2(bits / 8);
+   nir_src *offset = nir_get_io_offset_src(intr);
+   nir_scalar s = nir_scalar_resolved(offset->ssa, 0);
+   if (s.def->bit_size != 16 || !nir_scalar_is_alu(s) ||
+       nir_scalar_alu_op(s) != nir_op_u2u16)
+      return false;
+   s = nir_scalar_chase_alu_src(s, 0);
+   const unsigned base = nir_intrinsic_base(intr);
+   if (base > 255)
+      return false;
+
+   /* An element index the lowering divided exactly keeps its constant
+    * addends inside a sum: u2u16(a + (b + constant)). Gather the sum's terms,
+    * and move the constant to the byte offset when neither the remaining sum
+    * nor the narrowing can wrap. Summing the variable terms in a fixed order
+    * lets accesses that differ only in the constant share one index. */
+   if (s.def->bit_size == 32 && nir_scalar_is_alu(s) &&
+       nir_scalar_alu_op(s) == nir_op_iadd) {
+      nir_scalar terms[4];
+      unsigned count = 0;
+      uint64_t elements = 0;
+      if (!gather_sum(s, 3, terms, &count, &elements) || !elements || !count ||
+          elements > (255 - base) >> shift)
+         return false;
+
+      uint64_t bound = 0;
+      for (unsigned t = 0; t < count; ++t)
+         bound += term_upper_bound(b->shader, data, terms[t], 2);
+      if (bound > UINT16_MAX - elements)
+         return false;
+
+      qsort(terms, count, sizeof(terms[0]), compare_scalars);
+      b->cursor = nir_before_instr(&intr->instr);
+      nir_def *variable = nir_mov_scalar(b, terms[0]);
+      for (unsigned t = 1; t < count; ++t)
+         variable = nir_iadd(b, variable, nir_mov_scalar(b, terms[t]));
+      nir_src_rewrite(offset, nir_u2u16(b, variable));
+      nir_intrinsic_set_base(intr, base + (elements << shift));
+      return true;
+   }
+
+   if (!nir_scalar_is_alu(s) || nir_scalar_alu_op(s) != nir_op_ushr)
+      return false;
+   nir_scalar amount = nir_scalar_chase_alu_src(s, 1);
+   if (!nir_scalar_is_const(amount) || nir_scalar_as_uint(amount) != shift)
+      return false;
+   s = nir_scalar_chase_alu_src(s, 0);
+   if (s.def->bit_size != 32 || !nir_scalar_is_alu(s) ||
+       nir_scalar_alu_op(s) != nir_op_iadd)
+      return false;
+
+   for (unsigned c = 0; c < 2; ++c) {
+      nir_scalar constant = nir_scalar_chase_alu_src(s, c);
+      if (!nir_scalar_is_const(constant))
+         continue;
+      const uint32_t add = nir_scalar_as_uint(constant);
+      if (!add || add > 255 - base || (add & ((1u << shift) - 1)))
+         continue;
+      nir_scalar variable = nir_scalar_chase_alu_src(s, 1 - c);
+      const uint32_t limit = ((UINT32_C(1) << (16 + shift)) - 1) - add;
+      if (nir_unsigned_upper_bound(b->shader, data, variable) > limit)
+         continue;
+
+      b->cursor = nir_before_instr(&intr->instr);
+      nir_def *index = nir_ushr_imm(b, nir_mov_scalar(b, variable), shift);
+      nir_src_rewrite(offset, nir_u2u16(b, index));
+      nir_intrinsic_set_base(intr, base + add);
+      return true;
+   }
+   return false;
+}
+
+bool
+agx_nir_opt_shared_byte_offsets(nir_shader *shader)
+{
+   struct hash_table *ranges = _mesa_pointer_hash_table_create(NULL);
+   bool progress = nir_shader_intrinsics_pass(shader, fold_byte_offset,
+                                              nir_metadata_control_flow,
+                                              ranges);
+   _mesa_hash_table_destroy(ranges, NULL);
+   return progress;
+}
diff --git a/src/asahi/compiler/agx_nir_lower_texture.c b/src/asahi/compiler/agx_nir_lower_texture.c
index d5faecc07c5c..92c22a9a5c8c 100644
--- a/src/asahi/compiler/agx_nir_lower_texture.c
+++ b/src/asahi/compiler/agx_nir_lower_texture.c
@@ -22,6 +22,7 @@
 
 struct agx_texture_lower_options {
    bool gen9;
+   bool gen10;
    bool buffer_metadata;
    agx_buffer_texture_metadata_cb buffer_metadata_cb;
    /* Formatless image loads still need view-bounds robustness. */
@@ -172,6 +173,16 @@ lower_tex_crawl(nir_builder *b, nir_instr *instr, void *data)
       res = libagx_texture_levels_gen9(b, ptr);
    } else if (gen9 && tex->op == nir_texop_texture_samples) {
       res = libagx_texture_samples_gen9(b, ptr);
+   } else if (options && options->gen10 && tex->op == nir_texop_txs) {
+      res = libagx_txs_gen10(b, ptr, lod, nir_imm_int(b, nr_comps),
+                           nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_1D),
+                           nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_2D),
+                           nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE),
+                           nir_imm_bool(b, tex->is_array));
+   } else if (options && options->gen10 && tex->op == nir_texop_query_levels) {
+      res = libagx_texture_levels_gen10(b, ptr);
+   } else if (options && options->gen10 && tex->op == nir_texop_texture_samples) {
+      res = libagx_texture_samples_gen10(b, ptr);
    } else if (tex->op == nir_texop_txs) {
       res =
          libagx_txs(b, ptr, lod, nir_imm_int(b, nr_comps),
@@ -736,6 +747,7 @@ lower_regular_texture(nir_builder *b, nir_instr *instr, void *data)
    nir_tex_instr *tex = nir_instr_as_tex(instr);
    const struct agx_texture_lower_options *options = data;
    const bool gen9 = options && options->gen9;
+   const bool native_bounds = options && (options->gen9 || options->gen10);
    const bool robust_texel_fetch = !options || options->robust_texel_fetch;
    b->cursor = nir_before_instr(instr);
 
@@ -779,16 +791,36 @@ lower_regular_texture(nir_builder *b, nir_instr *instr, void *data)
     * access explicitly, fetch texel (0,0) of level 0 for an invalid one so the
     * hardware access itself stays inside the view, and select the
     * robustImageAccess2 value after the fetch. */
-   if (gen9 && robust_texel_fetch && tex->op == nir_texop_txf &&
+   if (native_bounds && robust_texel_fetch && tex->op == nir_texop_txf &&
        (tex->sampler_dim == GLSL_SAMPLER_DIM_2D ||
         tex->sampler_dim == GLSL_SAMPLER_DIM_3D) &&
        nir_tex_instr_src_index(tex, nir_tex_src_min_lod) < 0 &&
        !(tex->backend_flags & AGX_TEXTURE_FLAG_NO_CLAMP)) {
-      /* The size at the fetched level also counts array layers. */
-      const unsigned n = tex->coord_components;
-      nir_def *size = nir_trim_vector(b, nir_get_texture_size(b, tex), n);
-      nir_def *checked = nir_trim_vector(b, coord, n);
-      g17p_txf_oob = nir_bany(b, nir_uge(b, checked, size));
+      /* The size at the fetched level also counts array layers. A constant
+       * zero component lies inside every valid level and layer, so only the
+       * level check below can fail for it; a null descriptor already swizzles
+       * every channel to zero. Skipping those components drops the size query,
+       * which depends on a varying level, from a fetch of texel 0. */
+      if (options->gen10) {
+         nir_def *size = NULL;
+         g17p_txf_oob = nir_imm_false(b);
+         for (unsigned c = 0; c < tex->coord_components; ++c) {
+            if (nir_scalar_is_zero(nir_get_scalar(coord, c)))
+               continue;
+
+            size = size ?: nir_get_texture_size(b, tex);
+            g17p_txf_oob = nir_ior(
+               b, g17p_txf_oob,
+               nir_uge(b, nir_channel(b, coord, c), nir_channel(b, size, c)));
+         }
+      } else {
+         /* Gen9 keeps the whole-vector check: the zero-component skip is
+          * qualified on M5 only. */
+         const unsigned n = tex->coord_components;
+         nir_def *size = nir_trim_vector(b, nir_get_texture_size(b, tex), n);
+         nir_def *checked = nir_trim_vector(b, coord, n);
+         g17p_txf_oob = nir_bany(b, nir_uge(b, checked, size));
+      }
 
       if (has_nonzero_lod(tex)) {
          /* The fetch consumes a 32-bit level, see the end of lower_texture().
@@ -829,10 +861,13 @@ lower_regular_texture(nir_builder *b, nir_instr *instr, void *data)
       unsigned lidx = coord->num_components - 1;
       nir_def *layer = nir_channel(b, coord, lidx);
 
+      /* Gen9/10 share the mip range, layout and layer-bound fields this
+       * helper reads. Their width/height fields differ and are not read here.
+       * Legacy mip fields overlap Gen10's sample/mipmap flags. */
       nir_def *(*lower)(nir_builder *, nir_def *, nir_def *, nir_def *,
                          nir_def *, nir_def *, nir_def *, nir_def *, nir_def *) =
-         data && *(const bool *)data ? libagx_lower_txf_robustness_gen9
-                                    : libagx_lower_txf_robustness;
+         native_bounds ? libagx_lower_txf_robustness_gen9
+                       : libagx_lower_txf_robustness;
       nir_def *replaced = lower(
          b, texture_descriptor_ptr(b, tex),
          nir_imm_bool(b, has_nonzero_lod(tex)), lod,
@@ -934,7 +969,39 @@ lower_regular_texture(nir_builder *b, nir_instr *instr, void *data)
    nir_def *offset = nir_steal_tex_src(tex, nir_tex_src_offset);
 
    if (offset != NULL) {
-      if (gen9 && !g17p_offset_is_constant(offset)) {
+      /* G17G (generation 10) runtime offsets use the same PACK_SMPIR into
+       * bank 6 as G17P, reading X and Y as separate 16-bit sources (native
+       * texwit3/texwit4 d_auto, d_lvl2, d_bias1, d_grad, d_cmp*, d_gath*).
+       * Keep the component vector, or the 2x16 pair for a gather, wherever
+       * no array layer or register LOD/bias shares that bank. Those share
+       * G17P's complete bank-6 word below: native G17G builds the same
+       * layout (texwit3 d_lvlr: LOD f74=2 at bit 16, d_arr: layer f74=4 at
+       * bit 0, X/Y f74=f76=5 at bits 12/28) with a register PACK_SMPIR
+       * (f56=3) that the bank-6 PACK_SMPIR then merges (f51=2, f55=1). */
+      if (options && options->gen10 && !g17p_offset_is_constant(offset) &&
+          !tex->is_array && offset->num_components == 2) {
+         const int lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_lod);
+         const int bias_idx = nir_tex_instr_src_index(tex, nir_tex_src_bias);
+         const int min_idx = nir_tex_instr_src_index(tex, nir_tex_src_min_lod);
+         const bool register_level =
+            (lod_idx >= 0 && !nir_src_is_const(tex->src[lod_idx].src)) ||
+            (bias_idx >= 0 && !nir_src_is_const(tex->src[bias_idx].src)) ||
+            min_idx >= 0;
+         if (tex->op == nir_texop_tg4) {
+            nir_def *packed = nir_pack_32_2x16_split(
+               b, nir_i2i16(b, nir_channel(b, offset, 0)),
+               nir_i2i16(b, nir_channel(b, offset, 1)));
+            nir_tex_instr_add_src(tex, nir_tex_src_backend2, packed);
+            goto offset_lowered;
+         }
+         if (!register_level) {
+            nir_tex_instr_add_src(tex, nir_tex_src_backend2, offset);
+            goto offset_lowered;
+         }
+      }
+
+      if ((gen9 || (options && options->gen10)) &&
+          !g17p_offset_is_constant(offset)) {
          /* Gather's PACK_SMPIR form consumes X and Y as adjacent 16-bit
           * inputs.  Keep them in one 32-bit SSA value so register allocation
           * preserves that exact pair.  The generic runtime-offset path below
@@ -942,7 +1009,10 @@ lower_regular_texture(nir_builder *b, nir_instr *instr, void *data)
           * it for an array gather made the layer occupy X's low nibble and
           * produced wrong texels, while a plain gather arrived as two 32-bit
           * channels and was refused by the packer. */
-         if (tex->op == nir_texop_tg4) {
+         /* AGX3C (generation 10) stages an array gather's offset like an
+          * array sample's: the layer word below (native texwit6 g_arr). */
+         if (tex->op == nir_texop_tg4 &&
+             !(options && options->gen10 && tex->is_array)) {
             assert(offset->num_components == 2);
             nir_def *packed = nir_pack_32_2x16_split(
                b, nir_i2i16(b, nir_channel(b, offset, 0)),
@@ -1164,7 +1234,8 @@ txs_for_image(nir_builder *b, nir_intrinsic_instr *intr,
 
 static nir_def *
 image_texel_address(nir_builder *b, nir_intrinsic_instr *intr,
-                    bool return_index, bool gen9, bool buffer_metadata)
+                    bool return_index, bool gen9, bool gen10,
+                    bool buffer_metadata)
 {
    /* First, calculate the address of the PBE descriptor */
    nir_def *desc_address =
@@ -1188,7 +1259,7 @@ image_texel_address(nir_builder *b, nir_intrinsic_instr *intr,
 
    if (dim == GLSL_SAMPLER_DIM_BUF && return_index) {
       return nir_channel(b, coord, 0);
-   } else if (dim == GLSL_SAMPLER_DIM_BUF && (buffer_metadata || gen9)) {
+   } else if (dim == GLSL_SAMPLER_DIM_BUF && (buffer_metadata || gen9 || gen10)) {
       /* Buffer atomics receive the PBE handle, but Gen9 has no AGX2
        * software address/offset fields. Use the same sideband as buffer
        * loads and stores, relative to the start of the descriptor.
@@ -1205,7 +1276,7 @@ image_texel_address(nir_builder *b, nir_intrinsic_instr *intr,
                                nir_u2u64(b, blocksize_B)));
    } else if (dim == GLSL_SAMPLER_DIM_BUF) {
       return libagx_buffer_texel_address(b, desc_address, coord, blocksize_B);
-   } else if (gen9) {
+   } else if (gen9 || gen10) {
       nir_def *sw_address =
          nir_iadd_imm(b, desc_address, AGX_G17P_IMAGE_ATOMIC_SW_FROM_PBE);
       if (dim == GLSL_SAMPLER_DIM_MS) {
@@ -1234,7 +1305,7 @@ image_texel_address(nir_builder *b, nir_intrinsic_instr *intr,
          b, desc_address, sw_address, coord, nir_u2u32(b, intr->src[2].ssa),
          blocksize_B, nir_imm_bool(b, dim == GLSL_SAMPLER_DIM_1D),
          nir_imm_bool(b, dim == GLSL_SAMPLER_DIM_MS), nir_imm_bool(b, layered),
-         nir_imm_bool(b, return_index));
+         nir_imm_bool(b, return_index), nir_imm_bool(b, gen10));
    } else {
       return libagx_image_texel_address(
          b, desc_address, coord, nir_u2u32(b, intr->src[2].ssa), blocksize_B,
@@ -1457,7 +1528,8 @@ lower_images(nir_builder *b, nir_intrinsic_instr *intr, void *data)
       } else if (intr->intrinsic == nir_intrinsic_image_load ||
                  intr->intrinsic == nir_intrinsic_bindless_image_load ||
                  intr->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
-         lower_image_load_robustness(b, intr, gen9);
+         lower_image_load_robustness(b, intr,
+                                      gen9 || (options && options->gen10));
       }
 
       switch (nir_intrinsic_image_dim(intr)) {
@@ -1488,6 +1560,7 @@ lower_images(nir_builder *b, nir_intrinsic_instr *intr, void *data)
    case nir_intrinsic_bindless_image_texel_address:
       nir_def_rewrite_uses(&intr->def,
                            image_texel_address(b, intr, false, gen9,
+                                               options && options->gen10,
                                                options && options->buffer_metadata));
       return true;
 
@@ -1839,6 +1912,7 @@ lower_texture(nir_shader *s, unsigned generation, bool buffer_metadata,
    const bool gen9 = generation == 9;
    const struct agx_texture_lower_options options = {
       .gen9 = gen9,
+      .gen10 = generation == 10,
       .buffer_metadata = buffer_metadata,
       .buffer_metadata_cb = buffer_metadata_cb,
       .robust_formatless_images = robust_formatless_images,
@@ -1895,8 +1969,8 @@ lower_texture(nir_shader *s, unsigned generation, bool buffer_metadata,
    NIR_PASS(progress, s, nir_shader_instructions_pass, lower_tex_crawl,
             nir_metadata_control_flow, (void *)&options);
 
-   if (gen9) {
-      /* HAL200's texture2d::read consumes a full uint LOD through the integer
+   if (gen9 || generation == 10) {
+      /* AGX3's texture2d::read consumes a full uint LOD through the integer
        * PACK_SMPIR operand. Keep the shared 16-bit lowering above for its
        * robustness arithmetic, then restore the hardware ABI on the final
        * texel-fetch instruction. */
@@ -1979,7 +2053,7 @@ lower_multisampled_store(nir_builder *b, nir_intrinsic_instr *intr,
          intr->src[1].ssa, nir_u2u32(b, intr->src[2].ssa),
          nir_imm_int(b, util_format_get_blocksize(nir_intrinsic_format(intr))));
    } else {
-      index_px = nir_u2u32(b, image_texel_address(b, intr, true, false, false));
+      index_px = nir_u2u32(b, image_texel_address(b, intr, true, false, false, false));
    }
    nir_def *coord2d = coords_for_buffer_texture(b, index_px);
 
diff --git a/src/asahi/compiler/agx_nir_lower_word_bounds.c b/src/asahi/compiler/agx_nir_lower_word_bounds.c
new file mode 100644
index 000000000000..9a684c2c0f38
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_lower_word_bounds.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "util/u_dynarray.h"
+#include "agx_nir.h"
+#include "nir.h"
+
+/*
+ * Check a constant program's constant-offset buffer reads in words.
+ *
+ * A robust read at a constant offset keeps its value only when its exclusive
+ * end lies within the descriptor's byte size:
+ *
+ *    bounds_agx(data, end, size) = end <= size ? data : 0
+ *
+ * which is one compare-select taking `end` as an 8-bit immediate. Every check
+ * with an end past 255 bytes therefore first materializes its end with a
+ * constant move. Large constant buffers (a G-buffer pass reading a 2 KiB D3D
+ * constant buffer) pay that move for most of their reads, and a constant
+ * program reruns for every tile its draw touches.
+ *
+ * For a word-aligned end, end <= size exactly when end/4 <= size/4 (rounded
+ * down): end/4 is an integer, and an integer is at most a real exactly when it
+ * is at most its floor. A size is a 32-bit unsigned value, so size/4 < 2^30,
+ * and every end/4 here is below 2^14. Subtracting a common base C < 2^14 from
+ * both sides of a signed comparison is then exact too, as no term leaves
+ * (-2^31, 2^31):
+ *
+ *    end <= size  <=>  end/4 - C <= size/4 - C        (signed)
+ *
+ * Choose C per 256-word window so that end/4 - C is an 8-bit immediate. The
+ * checks of one descriptor then share one shift (size/4) and one subtraction
+ * per window past the first, instead of a constant move per check. Each check
+ * keeps its own exact predicate: no value is read, kept or zeroed
+ * differently. A descriptor's checks are rewritten only when that is cheaper.
+ *
+ * Main programs keep their byte checks: there, the shared word sizes would be
+ * more live uniform values competing for registers.
+ */
+
+/* Constant-buffer offsets stay well below 64 KiB: 64 windows of 1 KiB. */
+#define WORD_BOUND_WINDOWS 64
+
+struct word_bound {
+   nir_alu_instr *alu;
+   uint32_t end;
+};
+
+struct word_bound_group {
+   nir_scalar size;
+   struct util_dynarray checks;
+};
+
+static nir_cursor
+after_scalar_def(nir_scalar s)
+{
+   nir_instr *instr = nir_def_instr(s.def);
+   return instr->type == nir_instr_type_phi ? nir_after_phis(instr->block)
+                                            : nir_after_instr(instr);
+}
+
+static void
+collect_checks(nir_function_impl *impl, struct util_dynarray *groups,
+               void *mem)
+{
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_alu)
+            continue;
+         nir_alu_instr *alu = nir_instr_as_alu(instr);
+         if (alu->op != nir_op_bounds_agx || alu->def.bit_size != 32 ||
+             alu->def.num_components != 1 ||
+             !nir_src_is_const(alu->src[1].src))
+            continue;
+
+         const uint32_t end = nir_alu_src_as_uint(alu->src[1]);
+         if (end == 0 || (end % 4) != 0 ||
+             (end / 4) >= (WORD_BOUND_WINDOWS << 8))
+            continue;
+
+         nir_scalar size =
+            nir_get_scalar(alu->src[2].src.ssa, alu->src[2].swizzle[0]);
+         struct word_bound_group *group = NULL;
+         util_dynarray_foreach(groups, struct word_bound_group, g) {
+            if (nir_scalar_equal(g->size, size)) {
+               group = g;
+               break;
+            }
+         }
+         if (!group) {
+            group = util_dynarray_grow(groups, struct word_bound_group, 1);
+            *group = (struct word_bound_group){.size = size};
+            util_dynarray_init(&group->checks, mem);
+         }
+         struct word_bound check = {.alu = alu, .end = end};
+         util_dynarray_append(&group->checks, check);
+      }
+   }
+}
+
+static bool
+words_cheaper(const struct word_bound_group *g)
+{
+   /* A byte check costs a constant move once its end is past 255. In words,
+    * the descriptor costs a shift, plus a constant move and a subtraction for
+    * each 256-word window past the first. */
+   unsigned byte_cost = 0, word_cost = 1;
+   bool windows[WORD_BOUND_WINDOWS] = {false};
+   util_dynarray_foreach(&g->checks, struct word_bound, c) {
+      byte_cost += c->end > 255;
+      const unsigned window = (c->end / 4) >> 8;
+      if (window && !windows[window]) {
+         windows[window] = true;
+         word_cost += 2;
+      }
+   }
+   return word_cost < byte_cost;
+}
+
+static void
+rewrite_group(const struct word_bound_group *g)
+{
+   nir_builder b = nir_builder_at(after_scalar_def(g->size));
+   nir_def *window_words[WORD_BOUND_WINDOWS] = {NULL};
+   window_words[0] = nir_ushr_imm(&b, nir_mov_scalar(&b, g->size), 2);
+
+   util_dynarray_foreach(&g->checks, struct word_bound, c) {
+      const unsigned window = (c->end / 4) >> 8;
+      if (!window_words[window]) {
+         window_words[window] =
+            nir_iadd_imm(&b, window_words[0], -(int64_t)(window << 8));
+      }
+   }
+
+   util_dynarray_foreach(&g->checks, struct word_bound, c) {
+      const uint32_t end_words = c->end / 4;
+      const unsigned window = end_words >> 8;
+      b.cursor = nir_before_instr(&c->alu->instr);
+      nir_def *data = nir_ssa_for_alu_src(&b, c->alu, 0);
+      nir_def *checked;
+      if (window == 0) {
+         /* Unsigned is exact here: size/4 is never negative. */
+         checked = nir_bounds_agx(&b, data, nir_imm_int(&b, end_words),
+                                  window_words[0]);
+      } else {
+         nir_def *outside = nir_ilt_imm(&b, window_words[window],
+                                        end_words - (window << 8));
+         checked = nir_bcsel(&b, outside, nir_imm_int(&b, 0), data);
+      }
+      nir_def_replace(&c->alu->def, checked);
+   }
+}
+
+bool
+agx_nir_lower_word_bounds(nir_shader *shader)
+{
+   /* nir_shader_get_preamble() would create an empty one. */
+   nir_function_impl *entrypoint = nir_shader_get_entrypoint(shader);
+   if (!entrypoint->preamble)
+      return false;
+   nir_function_impl *impl = entrypoint->preamble->impl;
+
+   void *mem = ralloc_context(NULL);
+   struct util_dynarray groups;
+   util_dynarray_init(&groups, mem);
+   collect_checks(impl, &groups, mem);
+
+   bool progress = false;
+   util_dynarray_foreach(&groups, struct word_bound_group, g) {
+      if (words_cheaper(g)) {
+         rewrite_group(g);
+         progress = true;
+      }
+   }
+
+   ralloc_free(mem);
+   return nir_progress(progress, impl, nir_metadata_control_flow);
+}
diff --git a/src/asahi/compiler/agx_nir_m5_mesh.c b/src/asahi/compiler/agx_nir_m5_mesh.c
index d668adf57e27..3743377f9f44 100644
--- a/src/asahi/compiler/agx_nir_m5_mesh.c
+++ b/src/asahi/compiler/agx_nir_m5_mesh.c
@@ -6,137 +6,240 @@
 #include "util/u_math.h"
 #include <stdlib.h>
 #include <string.h>
-#include <stdio.h>
-#define M5_REFUSE() do { if (getenv("ASAHI_MESH_CONTRACT_DEBUG")) fprintf(stderr, "m5 mesh refuse: %s:%d\n", __func__, __LINE__); return false; } while (0)
-
-struct mesh_layout {
-   unsigned output_base, primitive_count, vertex_count, position;
-   unsigned prim_vertices;
-   unsigned plane_stride, planes, bytes;
-   bool storage_probe;
-   uint64_t primitive_audit_va;
-   const struct agx_m5_mesh_varyings *varyings;
-};
-
-static uint64_t
-primitive_audit_va(void)
-{
-   const char *enable = getenv("ASAHI_MESH_PP_AUDIT");
-   const char *address = getenv("ASAHI_MESH_MARKER_VA");
-   return enable && !strcmp(enable, "1") && address
-      ? strtoull(address, NULL, 0) : 0;
-}
 
+/* An execution barrier is admitted in a padded workgroup only at the top
+ * level of main, where every physical lane can pass it. */
 static bool
-primitive_audit_fan(void)
-{
-   const char *fan = getenv("ASAHI_MESH_PP_AUDIT_FAN");
-   return fan && !strcmp(fan, "1");
-}
-
-static unsigned
-primitive_audit_primitives(void)
+m5_padded_barrier_ok(const nir_block *block, const nir_intrinsic_instr *i)
 {
-   return primitive_audit_fan() ? 4 : 2;
+   return nir_intrinsic_execution_scope(i) == SCOPE_NONE ||
+          block->cf_node.parent->type == nir_cf_node_function;
 }
 
-static unsigned
-primitive_audit_lanes(void)
+/* Run the API body of a padded workgroup under its exact invocation mask
+ * (local index < threads). Top-level execution barriers split the body and
+ * stay outside the masked segments, so every physical lane passes them, as
+ * Apple passes barriers with all lanes enabled; values that cross a barrier
+ * are only consumed by the API lanes. */
+static void
+m5_mask_padded_body(nir_function_impl *entry, unsigned threads)
 {
-   return primitive_audit_fan() ? 8 : 4;
+   nir_intrinsic_instr **barriers = NULL;
+   unsigned count = 0;
+   nir_foreach_block(block, entry) {
+      if (block->cf_node.parent->type != nir_cf_node_function)
+         continue;
+      nir_foreach_instr(in, block) {
+         if (in->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+         if (i->intrinsic == nir_intrinsic_barrier &&
+             nir_intrinsic_execution_scope(i) != SCOPE_NONE) {
+            barriers = realloc(barriers, (count + 1) * sizeof(*barriers));
+            barriers[count++] = i;
+         }
+      }
+   }
+   nir_cursor start = nir_before_cf_list(&entry->body);
+   for (unsigned k = 0; k <= count; ++k) {
+      nir_cf_list segment;
+      nir_cf_extract(&segment, start,
+                     k < count ? nir_before_instr(&barriers[k]->instr)
+                               : nir_after_cf_list(&entry->body));
+      nir_builder b = nir_builder_at(
+         k < count ? nir_before_instr(&barriers[k]->instr)
+                   : nir_after_cf_list(&entry->body));
+      nir_if *active = nir_push_if(&b, nir_ult_imm(&b,
+                          nir_load_local_invocation_index(&b), threads));
+      b.cursor = nir_cf_reinsert(&segment, b.cursor);
+      nir_pop_if(&b, active);
+      if (k < count)
+         start = nir_after_instr(&barriers[k]->instr);
+   }
+   free(barriers);
+   if (count)
+      nir_repair_ssa_impl(entry);
 }
 
-static unsigned
-primitive_audit_lane_first(void)
+bool
+agx_nir_m5_canonicalize_empty_mesh(nir_shader *s)
 {
-   return 16 * 4 * primitive_audit_primitives();
-}
+   if (s->info.stage != MESA_SHADER_MESH ||
+       (s->info.mesh.primitive_type != MESA_PRIM_POINTS &&
+        s->info.mesh.primitive_type != MESA_PRIM_LINES) ||
+       s->info.outputs_written || s->info.outputs_read)
+      return false;
 
-static unsigned
-primitive_audit_fragment_first(void)
-{
-   return primitive_audit_lane_first() + 8 * 4 * primitive_audit_lanes();
-}
+   nir_function_impl *impl = nir_shader_get_entrypoint(s);
+   nir_intrinsic_instr *count = NULL;
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         switch (intr->intrinsic) {
+         case nir_intrinsic_set_vertex_and_primitive_count:
+            if (count || nir_intrinsic_stream_id(intr))
+               return false;
+            for (unsigned i = 0; i < 2; ++i) {
+               if (!nir_src_is_const(intr->src[i]) ||
+                   nir_src_as_uint(intr->src[i]) != 0)
+                  return false;
+            }
+            count = intr;
+            break;
+         case nir_intrinsic_store_output:
+         case nir_intrinsic_store_per_vertex_output:
+         case nir_intrinsic_store_per_primitive_output:
+            return false;
+         default:
+            break;
+         }
+      }
+   }
+   if (!count)
+      return false;
 
-static bool
-native_primitive_address(void)
-{
-   const char *enable = getenv("ASAHI_MESH_NATIVE_PRIMITIVE_ADDRESS_DIAGNOSTIC");
-   return enable && !strcmp(enable, "1");
-}
+   nir_metadata_require(impl, nir_metadata_dominance);
+   if (!nir_block_dominates(count->instr.block, nir_impl_last_block(impl)))
+      return false;
 
-static bool
-native_page_carrier(void)
-{
-   const char *enable = getenv("ASAHI_MESH_NATIVE_PAGE_CARRIER");
-   return enable && !strcmp(enable, "1");
+   /* No geometry reaches rasterization. Keep the shader's memory work and
+    * use the existing native triangle launch for the proven-empty output. */
+   s->info.mesh.primitive_type = MESA_PRIM_TRIANGLES;
+   return true;
 }
 
-static unsigned
-page_carrier_words(void)
-{
-   return native_page_carrier() ? 1 : 2;
-}
+#define M5_MESH_MAX_PLANES (4 + 128 + 1 + 8)
 
-static void
-primitive_capture(nir_builder *b, uint64_t va, unsigned first,
-                  nir_def *record, unsigned words, nir_def **values)
-{
-   nir_def *address = nir_iadd(b, nir_imm_int64(b, va + 32 + 4 * first),
-      nir_imul_imm(b, nir_u2u64(b, record), 4 * words));
-   for (unsigned c = 0; c < words; ++c) {
-      if (values[c])
-         nir_global_atomic(b, 32, nir_iadd_imm(b, address, 4 * c),
-            nir_u2u32(b, values[c]), .atomic_op = nir_atomic_op_xchg);
-   }
-}
+struct mesh_layout {
+   unsigned output_base, primitive_count, vertex_count, position;
+   unsigned plane_stride, planes, bytes;
+   bool storage_probe;
+   const struct agx_m5_mesh_varyings *varyings;
 
-static void
-primitive_order(nir_builder *b)
-{
-   nir_barrier(b, .execution_scope = SCOPE_NONE, .memory_scope = SCOPE_WORKGROUP,
-      .memory_semantics = NIR_MEMORY_ACQ_REL,
-      .memory_modes = nir_var_mem_global | nir_var_mem_shared);
-}
+   /* Per-primitive outputs (see agx_m5_mesh_unwelds): the API body writes
+    * vertex planes to vertex_source and per-primitive planes to
+    * primitive_source; the export region at position then holds one
+    * private vertex per primitive corner (prim_vertices). */
+   bool unweld;
+   unsigned export_vertices;
+   /* One byte per primitive: nonzero when gl_CullPrimitiveEXT is set. */
+   unsigned cull;
+   unsigned vertex_source, vertex_stride;
+   unsigned primitive_source, primitive_stride;
+   int16_t layer_plane, viewport_plane;
+   int16_t vertex_plane[M5_MESH_MAX_PLANES], primitive_plane[M5_MESH_MAX_PLANES];
+};
 
+/* Index bytes per primitive. Native G17G point and line witnesses
+ * (lane-mesh/cwit-meshtopo mpt/mln) store one and two packed index bytes per
+ * primitive in the same 16-byte-aligned index region as triangles. */
 static unsigned
 prim_vertices(const nir_shader *s)
 {
    switch (s->info.mesh.primitive_type) {
-   case MESA_PRIM_POINTS:
-      return 1;
-   case MESA_PRIM_LINES:
-      return 2;
-   case MESA_PRIM_TRIANGLES:
-      return 3;
-   default:
-      return 0;
+   case MESA_PRIM_POINTS: return 1;
+   case MESA_PRIM_LINES: return 2;
+   case MESA_PRIM_TRIANGLES: return 3;
+   default: return 0;
+   }
+}
+
+static bool
+agx_m5_mesh_unwelds(const struct agx_m5_mesh_varyings *varyings)
+{
+   if (varyings->lv)
+      return true;
+   for (unsigned slot = 0; slot < 64; ++slot) {
+      if (varyings->primitive_comps[slot])
+         return true;
    }
+   return false;
 }
 
 static struct mesh_layout
 mesh_layout(const nir_shader *s, const struct agx_m5_mesh_varyings *varyings)
 {
    const unsigned base = ALIGN_POT(s->info.shared_size, 64);
-   
-
-
-
-
-   const unsigned count = ALIGN_POT(prim_vertices(s) * s->info.mesh.max_primitives_out, 16);
+   /* CPU-native maxV3/4/5/8/9/16/32/33/64/256 witnesses establish
+    * a 16-byte-aligned SoA plane and aligned triangle-byte/count regions.
+    * Native multi-varying witnesses (lane-mesh/native-var objects 9..17)
+    * place one plane per UVS word directly after the four position planes,
+    * in selector order. */
+   const unsigned count =
+      ALIGN_POT(prim_vertices(s) * s->info.mesh.max_primitives_out, 16);
    const unsigned position = count + 16;
-   const unsigned stride = ALIGN_POT(s->info.mesh.max_vertices_out, 4) * 4;
-   return (struct mesh_layout) {
-      .primitive_audit_va = primitive_audit_va(),
-      .prim_vertices = prim_vertices(s),
+   const bool unweld = agx_m5_mesh_unwelds(varyings);
+   const unsigned export_vertices = unweld
+      ? prim_vertices(s) * s->info.mesh.max_primitives_out
+      : s->info.mesh.max_vertices_out;
+   const unsigned stride = ALIGN_POT(export_vertices, 4) * 4;
+   struct mesh_layout layout = {
       .output_base = base, .primitive_count = base + count,
       .vertex_count = base + count + 4, .position = base + position,
       .plane_stride = stride, .planes = varyings->size,
       .bytes = ALIGN_POT(position + varyings->size * stride, 64),
       .storage_probe = getenv("ASAHI_MESH_PP_PROBE") != NULL &&
-                       !strcmp(getenv("ASAHI_MESH_PP_PROBE"), "1"),
+                       (!strcmp(getenv("ASAHI_MESH_PP_PROBE"), "1") ||
+                        !strcmp(getenv("ASAHI_MESH_PP_PROBE"), "payload")),
       .varyings = varyings,
+      .unweld = unweld,
+      .export_vertices = export_vertices,
+      .layer_plane = -1, .viewport_plane = -1,
    };
+   if (!unweld)
+      return layout;
+
+   /* Classify each exported UVS word as a vertex or primitive word. */
+   unsigned vertex_planes = 0, primitive_planes = 0;
+   for (unsigned plane = 0; plane < M5_MESH_MAX_PLANES; ++plane)
+      layout.vertex_plane[plane] = layout.primitive_plane[plane] = -1;
+   for (unsigned plane = 0; plane < MIN2(varyings->size, M5_MESH_MAX_PLANES); ++plane) {
+      if (varyings->lv && plane == varyings->lv_first)
+         continue; /* packed from the layer/viewport words */
+      int primitive_slot = -1;
+      for (unsigned slot = 0; slot < 64; ++slot) {
+         if ((varyings->primitive_comps[slot] ||
+              (varyings->lv && slot == VARYING_SLOT_LAYER)) &&
+             plane >= varyings->first[slot] &&
+             plane < varyings->first[slot] + varyings->components[slot])
+            primitive_slot = slot;
+      }
+      if (primitive_slot < 0)
+         layout.vertex_plane[plane] = vertex_planes++;
+      else if (varyings->primitive_comps[primitive_slot])
+         layout.primitive_plane[plane] = primitive_planes++;
+      /* else: the unwritten layer word, exported as zero */
+   }
+   if (varyings->primitive_comps[VARYING_SLOT_LAYER])
+      layout.layer_plane =
+         layout.primitive_plane[varyings->first[VARYING_SLOT_LAYER]];
+   if (varyings->primitive_comps[VARYING_SLOT_VIEWPORT])
+      layout.viewport_plane =
+         layout.primitive_plane[varyings->first[VARYING_SLOT_VIEWPORT]];
+
+   layout.vertex_stride = ALIGN_POT(s->info.mesh.max_vertices_out, 4) * 4;
+   layout.primitive_stride = ALIGN_POT(s->info.mesh.max_primitives_out, 4) * 4;
+   const unsigned vertex_source = position + varyings->size * stride;
+   const unsigned primitive_source =
+      vertex_source + vertex_planes * layout.vertex_stride;
+   layout.vertex_source = base + vertex_source;
+   layout.primitive_source = base + primitive_source;
+   layout.bytes = ALIGN_POT(primitive_source +
+                            primitive_planes * layout.primitive_stride, 64);
+   return layout;
+}
+
+/* Shared address of a vertex-output word written by the API body. */
+static unsigned
+vertex_plane_address(const struct mesh_layout *layout, unsigned plane)
+{
+   if (!layout->unweld)
+      return layout->position + layout->plane_stride * plane;
+   assert(layout->vertex_plane[plane] >= 0);
+   return layout->vertex_source +
+          layout->vertex_stride * layout->vertex_plane[plane];
 }
 
 static unsigned
@@ -145,7 +248,7 @@ vertex_output_slot(nir_intrinsic_instr *i)
    return nir_intrinsic_io_semantics(i).location + nir_src_as_uint(i->src[2]);
 }
 
-
+/* Plane (UVS word) of the first component of a per-vertex output slot. */
 static unsigned
 first_plane(const struct agx_m5_mesh_varyings *varyings, unsigned slot)
 {
@@ -155,9 +258,9 @@ first_plane(const struct agx_m5_mesh_varyings *varyings, unsigned slot)
 static nir_def *
 widen_output(nir_builder *b, nir_intrinsic_instr *i, nir_def *value)
 {
-   
-
-
+   /* The UVS carries 32-bit words; FP16/INT16 varyings are converted the
+    * same way agx_nir_lower_uvs does for the vertex stage, and the fragment
+    * linkage (32-bit counts only) narrows them again on load. */
    if (value->bit_size == 32)
       return value;
    nir_alu_type type = nir_alu_type_get_base_type(nir_intrinsic_src_type(i));
@@ -174,34 +277,50 @@ lower(nir_builder *b,nir_intrinsic_instr *i,void *data)
       b->cursor=nir_before_instr(&i->instr);
       nir_def *element=nir_imul_imm(b,i->src[1].ssa,4);
       const unsigned slot=vertex_output_slot(i);
-      if (slot != VARYING_SLOT_POS && !layout->varyings->components[slot])
-         break;
       const unsigned first=first_plane(layout->varyings,slot);
       unsigned component=nir_intrinsic_component(i);
       nir_def *value=widen_output(b,i,i->src[0].ssa), *probe=NULL;
       if (slot == VARYING_SLOT_VAR0 && layout->storage_probe) {
-         
-
-
-
-         nir_def *storage=nir_load_mesh_primitive_storage_agx(b);
+         /* Opt-in read-only hardware probe (ASAHI_MESH_PP_PROBE=1): publish the
+          * native per-primitive record base SR64/SR65 and the workgroup index
+          * through the flat VAR0 planes instead of the API colour. No pointer
+          * is dereferenced; the fragment dumps the raw bits as pixels. */
+         const char *pe = getenv("ASAHI_MESH_PP_PROBE");
+         const bool payload_probe = pe && !strcmp(pe, "payload");
+         nir_def *storage = payload_probe
+            ? nir_load_mesh_payload_address_agx(b)
+            : nir_load_mesh_primitive_storage_agx(b);
+         if (payload_probe) {
+            /* Bytes of SR62 (even workgroups) or SR63 (odd) as UNORM8. */
+            nir_def *odd = nir_i2b(b, nir_iand_imm(b, nir_channel(b, nir_load_workgroup_id(b), 0), 1));
+            nir_def *word = nir_bcsel(b, odd, nir_channel(b, storage, 1), nir_channel(b, storage, 0));
+            nir_def *bytes[4];
+            for (unsigned k = 0; k < 4; ++k)
+               bytes[k] = nir_fmul_imm(b, nir_u2f32(b, nir_iand_imm(b, nir_ushr_imm(b, word, 8 * k), 0xff)), 1.0 / 255.0);
+            storage = NULL;
+            probe = nir_vec(b, bytes, 4);
+         }
+         if (storage)
          probe=nir_vec4(b,nir_channel(b,storage,0),nir_channel(b,storage,1),
                         nir_channel(b,nir_load_workgroup_id(b),0),
                         nir_load_local_invocation_index(b));
       }
       u_foreach_bit(c,nir_intrinsic_write_mask(i)) {
          nir_def *address=nir_iadd_imm(b,element,
-            layout->position+layout->plane_stride*(first+component+c));
+            vertex_plane_address(layout,first+component+c));
          nir_store_shared(b,probe ? nir_channel(b,probe,component+c)
                                   : nir_channel(b,value,c),address,
                             .write_mask=1,.align_mul=4);
-         if (slot == VARYING_SLOT_CLIP_DIST0 || slot == VARYING_SLOT_CLIP_DIST1) {
-            const unsigned clip = 4 * (slot - VARYING_SLOT_CLIP_DIST0) + component + c;
-            if (clip < layout->varyings->clip_count) {
-               nir_store_shared(b, nir_channel(b, value, c),
-                  nir_iadd_imm(b, element, layout->position + layout->plane_stride *
-                                              (layout->varyings->clip_first + clip)),
-                  .write_mask = 1, .align_mul = 4);
+         if (slot == VARYING_SLOT_CLIP_DIST0 ||
+             slot == VARYING_SLOT_CLIP_DIST1) {
+            const unsigned clip =
+               4 * (slot - VARYING_SLOT_CLIP_DIST0) + component + c;
+            if (clip < layout->varyings->clip_distance_count) {
+               nir_def *clip_address = nir_iadd_imm(b, element,
+                  vertex_plane_address(layout,
+                     layout->varyings->clip_distance_first + clip));
+               nir_store_shared(b, nir_channel(b, value, c), clip_address,
+                                .write_mask = 1, .align_mul = 4);
             }
          }
       }
@@ -210,45 +329,38 @@ lower(nir_builder *b,nir_intrinsic_instr *i,void *data)
    case nir_intrinsic_store_per_primitive_output: {
       b->cursor=nir_before_instr(&i->instr);
       const unsigned slot = vertex_output_slot(i);
+      if (slot == VARYING_SLOT_CULL_PRIMITIVE) {
+         nir_def *value = i->src[0].ssa;
+         nir_def *culled = value->bit_size == 1 ? value : nir_ine_imm(b, value, 0);
+         nir_store_shared(b, nir_b2i8(b, culled),
+            nir_iadd_imm(b, i->src[1].ssa, layout->cull), .write_mask = 1,
+            .align_mul = 1);
+         break;
+      }
       if (slot != VARYING_SLOT_PRIMITIVE_INDICES) {
-         nir_def *storage = nir_pack_64_2x32(b, nir_load_mesh_primitive_storage_agx(b));
-         nir_def *offset = nir_imul_imm(b, nir_u2u64(b, i->src[1].ssa),
-                                      layout->varyings->primitive_stride_B);
+         /* A primitive word: the export below replicates it to the
+          * private vertices of its primitive. */
+         nir_def *element = nir_imul_imm(b, i->src[1].ssa, 4);
          nir_def *value = widen_output(b, i, i->src[0].ssa);
          u_foreach_bit(c, nir_intrinsic_write_mask(i)) {
-            nir_def *address = nir_iadd_imm(b, nir_iadd(b, storage, offset),
-               4 * (layout->varyings->primitive_first[slot] +
-                    nir_intrinsic_component(i) + c));
-            nir_store_global(b, nir_channel(b, value, c), address,
-                             .write_mask = 1, .align_mul = 4);
-            if (layout->primitive_audit_va && c == 0) {
-               nir_def *group = nir_channel(b, nir_load_workgroup_id(b), 0);
-               const unsigned primitives = primitive_audit_primitives();
-               nir_push_if(b, nir_iand(b, nir_ult_imm(b, group, 4),
-                                      nir_ult_imm(b, i->src[1].ssa, primitives)));
-               primitive_order(b);
-               nir_def *post = nir_load_global(b, 1, 32, address,
-                  .align_mul = 4, .access = ACCESS_VOLATILE);
-               nir_def *fields[16] = {nir_imm_int(b, 0x50510001), group,
-                  i->src[1].ssa, nir_load_local_invocation_index(b),
-                  nir_u2u32(b, storage), nir_u2u32(b, nir_ushr_imm(b, storage, 32)),
-                  nir_u2u32(b, address), nir_u2u32(b, nir_ushr_imm(b, address, 32)),
-                  nir_imm_int(b, nir_intrinsic_component(i) + c), nir_channel(b, value, c), post,
-                  nir_imm_int(b, 0x50510002)};
-               primitive_capture(b, layout->primitive_audit_va, 0,
-                  nir_iadd(b, nir_imul_imm(b, group, primitives), i->src[1].ssa), 16, fields);
-               nir_pop_if(b, NULL);
-            }
+            int plane = layout->primitive_plane[layout->varyings->first[slot] +
+                                                nir_intrinsic_component(i) + c];
+            assert(plane >= 0);
+            nir_store_shared(b, nir_channel(b, value, c),
+               nir_iadd_imm(b, element, layout->primitive_source +
+                                        layout->primitive_stride * plane),
+               .write_mask = 1, .align_mul = 4);
          }
          break;
       }
-      nir_def *element=nir_imul_imm(b,i->src[1].ssa,layout->prim_vertices);
+      nir_def *element=nir_imul_imm(b,i->src[1].ssa,
+                                    prim_vertices(b->shader));
       unsigned component=nir_intrinsic_component(i);
       u_foreach_bit(c,nir_intrinsic_write_mask(i)) {
          nir_def *index=nir_channel(b,i->src[0].ssa,c);
-         
-
-
+         /* Native helper creates per-byte validity masks with two ISELs,
+          * adds low0xff/high0xff00, then ANDs the original index word.
+          * Valid index bytes pass through unchanged. Cull is not mapped. */
          nir_def *packed=nir_u2u8(b,index);
          nir_store_shared(b,packed,nir_iadd_imm(b,element,output_base+component+c),
                             .write_mask=1,.align_mul=1);
@@ -265,23 +377,28 @@ lower(nir_builder *b,nir_intrinsic_instr *i,void *data)
       nir_pop_if(b,NULL);
       break;
    }
-   default:M5_REFUSE();
+   default:return false;
    }
    nir_instr_remove(&i->instr);
    return true;
 }
 
 #define USER_VARYINGS BITFIELD64_RANGE(VARYING_SLOT_VAR0, 32)
-#define MESH_PSIZ_PLANE 4
-#define MESH_CLIP_SLOTS (VARYING_BIT_CLIP_DIST0 | VARYING_BIT_CLIP_DIST1)
-#define PRIMITIVE_SLOTS (USER_VARYINGS | VARYING_BIT_PRIMITIVE_ID | \
-                         VARYING_BIT_LAYER | VARYING_BIT_VIEWPORT)
+#define MESH_VERTEX_VARYINGS \
+   (USER_VARYINGS | VARYING_BIT_CLIP_DIST0 | VARYING_BIT_CLIP_DIST1 | \
+    BITFIELD64_RANGE(VARYING_SLOT_CULL_PRIMITIVE, 2))
+/* Per-primitive outputs exported as user words of private vertices. */
+#define MESH_PRIMITIVE_WORDS (USER_VARYINGS | VARYING_BIT_PRIMITIVE_ID)
 
 bool
 agx_m5_mesh_gather_varyings(const nir_shader *s,
                             struct agx_m5_mesh_varyings *varyings)
 {
    memset(varyings, 0, sizeof(*varyings));
+   if (s->info.clip_distance_array_size +
+       s->info.cull_distance_array_size > 8)
+      return false;
+   varyings->clip_distance_count = s->info.clip_distance_array_size;
    nir_foreach_function_impl(impl, s) {
       nir_foreach_block(block, impl) {
          nir_foreach_instr(in, block) {
@@ -292,153 +409,161 @@ agx_m5_mesh_gather_varyings(const nir_shader *s,
                 i->intrinsic != nir_intrinsic_store_per_primitive_output)
                continue;
             if (!nir_src_is_const(i->src[2]))
-               M5_REFUSE();
+               return false;
             const unsigned slot = vertex_output_slot(i);
             if (i->intrinsic == nir_intrinsic_store_per_primitive_output) {
                if (slot == VARYING_SLOT_PRIMITIVE_INDICES)
                   continue;
+               if (slot == VARYING_SLOT_CULL_PRIMITIVE) {
+                  if (nir_intrinsic_component(i) != 0 ||
+                      nir_intrinsic_write_mask(i) != 1)
+                     return false;
+                  varyings->cull_primitive = true;
+                  continue;
+               }
+               /* Per-primitive user words become flat words of the
+                * primitive's private vertices; layer/viewport fill the
+                * vertex layer/viewport word. The fragment side reads both
+                * as flat inputs (agx_nir_lower_m5_mesh_fragment). */
                const unsigned end = nir_intrinsic_component(i) +
                                     util_last_bit(nir_intrinsic_write_mask(i));
-               if (slot >= 64 || !(PRIMITIVE_SLOTS & BITFIELD64_BIT(slot)) ||
-                   end > 4 ||
-                   (i->src[0].ssa->bit_size != 32 && i->src[0].ssa->bit_size != 16))
-                  M5_REFUSE();
-               varyings->primitive_comps[slot] =
-                  MAX2(varyings->primitive_comps[slot], end);
+               const unsigned bits = i->src[0].ssa->bit_size;
+               if (slot >= 64 || end > 4 || (bits != 32 && bits != 16))
+                  return false;
+               if (USER_VARYINGS & BITFIELD64_BIT(slot)) {
+                  varyings->primitive_comps[slot] =
+                     MAX2(varyings->primitive_comps[slot], end);
+               } else if (slot == VARYING_SLOT_LAYER ||
+                          slot == VARYING_SLOT_VIEWPORT) {
+                  if (end != 1)
+                     return false;
+                  varyings->primitive_comps[slot] = 1;
+                  varyings->lv = true;
+               } else if (slot == VARYING_SLOT_PRIMITIVE_ID) {
+                  /* A flat user word of the primitive's private vertices,
+                   * which the fragment shader's gl_PrimitiveID links to. */
+                  if (end != 1)
+                     return false;
+                  varyings->primitive_comps[slot] = 1;
+               } else {
+                  return false;
+               }
                continue;
             }
             if (slot == VARYING_SLOT_POS)
                continue;
-            if (slot == VARYING_SLOT_PSIZ &&
-                s->info.mesh.primitive_type != MESA_PRIM_POINTS)
+            /* Point size also reaches the raster state of triangle and line
+             * meshes (a shader object's polygon mode can be POINT). */
+            if (slot == VARYING_SLOT_PSIZ) {
+               if (nir_intrinsic_component(i) != 0 ||
+                   nir_intrinsic_write_mask(i) != 1)
+                  return false;
+               varyings->components[slot] = 1;
                continue;
-            if (slot >= 64 || !((USER_VARYINGS | VARYING_BIT_PSIZ | MESH_CLIP_SLOTS) &
-                                BITFIELD64_BIT(slot))) {
-               if (getenv("ASAHI_MESH_CONTRACT_DEBUG"))
-                  fprintf(stderr, "m5 mesh vertex slot %u\n", slot);
-               M5_REFUSE();
             }
+            if (slot >= 64 || !(MESH_VERTEX_VARYINGS & BITFIELD64_BIT(slot)))
+               return false;
             const unsigned end = nir_intrinsic_component(i) +
                                  util_last_bit(nir_intrinsic_write_mask(i));
             if (end > 4)
-               M5_REFUSE();
+               return false;
             varyings->components[slot] = MAX2(varyings->components[slot], end);
          }
       }
    }
-   
-
-
-   unsigned record = 0;
    for (unsigned slot = 0; slot < 64; ++slot) {
-      if (!varyings->primitive_comps[slot])
+      if (!(MESH_PRIMITIVE_WORDS & BITFIELD64_BIT(slot)) ||
+          !varyings->primitive_comps[slot])
          continue;
-      varyings->primitive_first[slot] = record;
-      record += varyings->primitive_comps[slot];
-   }
-   varyings->primitive_components = varyings->primitive_comps[VARYING_SLOT_VAR0];
-   varyings->primitive_stride_B = 4 * record;
-   if ((uint64_t)varyings->primitive_stride_B * s->info.mesh.max_primitives_out >
-       UINT64_C(0x10000))
-      M5_REFUSE();
-
-   if (s->info.cull_distance_array_size)
-      M5_REFUSE();
-   const bool clip_written = varyings->components[VARYING_SLOT_CLIP_DIST0] ||
-                             varyings->components[VARYING_SLOT_CLIP_DIST1];
-   if (clip_written != (s->info.clip_distance_array_size != 0))
-      M5_REFUSE();
-
-   unsigned next = 4, slots = 0;
-   if (varyings->components[VARYING_SLOT_PSIZ]) {
-      varyings->first[VARYING_SLOT_PSIZ] = MESH_PSIZ_PLANE;
-      next++;
-   }
-   if (varyings->primitive_comps[VARYING_SLOT_LAYER] ||
-       varyings->primitive_comps[VARYING_SLOT_VIEWPORT]) {
-      varyings->lv = true;
-      varyings->lv_first = next++;
+      /* A location is either per-vertex or per-primitive. */
+      if (varyings->components[slot])
+         return false;
+      varyings->components[slot] = varyings->primitive_comps[slot];
    }
-   if (clip_written) {
-      varyings->clip_first = next;
-      varyings->clip_count = s->info.clip_distance_array_size;
-      next += varyings->clip_count;
+   /* Like the M5 vertex stage (hk_lower_hw_vs with layer_as_varying), the
+    * layer is also a flat user word for fragment reads, zero when only the
+    * viewport is written, and a written viewport is one as well. */
+   if (varyings->lv) {
+      varyings->components[VARYING_SLOT_LAYER] = 1;
+      varyings->components[VARYING_SLOT_VIEWPORT] =
+         varyings->primitive_comps[VARYING_SLOT_VIEWPORT];
    }
-   if (varyings->primitive_stride_B)
-      varyings->components[AGX_MESH_PRIMITIVE_PAGE_SLOT] = page_carrier_words();
-   for (unsigned slot = VARYING_SLOT_CLIP_DIST0; slot < 64; ++slot) {
-      if (!varyings->components[slot])
+   /* Declaration order. A single user slot (or one interpolation class) sits
+    * at the base of the user section for every fragment shader; otherwise the
+    * fragment shader chooses the class order at draw time. The UVS groups
+    * that follow are layer/viewport, then clip distances. */
+   unsigned next = 4, slots = 0;
+   for (unsigned slot = 0; slot < 64; ++slot) {
+      if (!varyings->components[slot] || slot == VARYING_SLOT_PSIZ)
          continue;
       varyings->first[slot] = next;
       next += varyings->components[slot];
       slots++;
    }
-   varyings->size = next;
-   varyings->dynamic = slots > 1 || varyings->lv;
+   /* Point size follows the user words, in the UVS group order of
+    * agx_nir_lower_uvs that programs the raster state; with no user words
+    * it is the plane after the position, as in native size-only points
+    * (cwit-meshtopo mpt). */
+   if (varyings->components[VARYING_SLOT_PSIZ])
+      varyings->first[VARYING_SLOT_PSIZ] = next++;
+   if (varyings->lv)
+      varyings->lv_first = next++;
+   varyings->clip_distance_first = next;
+   varyings->size = next + varyings->clip_distance_count;
+   varyings->dynamic = slots > 1;
    return true;
 }
 
-
+/* Every user plane is covered exactly once by the slot placement. */
 static bool
 varyings_valid(const struct agx_m5_mesh_varyings *v)
 {
-   if (getenv("ASAHI_MESH_CONTRACT_DEBUG")) {
-      fprintf(stderr, "m5 mesh layout size=%u clip=%u@%u lv=%u@%u:", v->size, v->clip_count, v->clip_first, v->lv, v->lv_first);
-      for (unsigned slot = 0; slot < 64; ++slot)
-         if (v->components[slot])
-            fprintf(stderr, " %u:%u@%u", slot, v->components[slot], v->first[slot]);
-      fprintf(stderr, "\n");
-   }
-   if (v->size < 4 || v->size > 4 + 128)
-      M5_REFUSE();
-   uint8_t used[4 + 128] = {0};
+   const unsigned lv = v->lv ? 1 : 0;
+   if (v->size < 4 || v->size > 4 + 128 + 8 + lv ||
+       v->clip_distance_count > 8 ||
+       v->size < 4 + lv + v->clip_distance_count ||
+       (v->clip_distance_count &&
+        v->clip_distance_first + v->clip_distance_count != v->size))
+      return false;
+   /* User words, then the optional layer/viewport word, then clip words. */
+   const unsigned user_end = v->size - v->clip_distance_count - lv;
+   if (v->lv && v->lv_first != user_end)
+      return false;
+   uint8_t used[M5_MESH_MAX_PLANES] = {0};
    unsigned user = 0;
-   if (v->components[VARYING_SLOT_PSIZ]) {
-      if (v->components[VARYING_SLOT_PSIZ] != 1 ||
-          v->first[VARYING_SLOT_PSIZ] != MESH_PSIZ_PLANE)
-         M5_REFUSE();
-      used[MESH_PSIZ_PLANE] = 1;
-      user++;
-   }
-   if (v->lv) {
-      if (v->lv_first < 4 || v->lv_first >= v->size || used[v->lv_first]++)
-         M5_REFUSE();
-      user++;
-   }
-   if (v->clip_count) {
-      if (v->clip_count > 8 || v->clip_first < 4 ||
-          v->clip_first + v->clip_count > v->size)
-         M5_REFUSE();
-      for (unsigned c = 0; c < v->clip_count; ++c) {
-         if (used[v->clip_first + c]++)
-            M5_REFUSE();
-      }
-      user += v->clip_count;
-   }
-   unsigned hidden = 0;
    for (unsigned slot = 0; slot < 64; ++slot) {
       if (!v->components[slot])
          continue;
-      if (slot == VARYING_SLOT_PSIZ)
-         continue;
-      if ((MESH_CLIP_SLOTS & BITFIELD64_BIT(slot)) && v->clip_count &&
-          v->first[slot] >= v->clip_first &&
-          v->first[slot] + v->components[slot] <= v->clip_first + v->clip_count) {
-         hidden += v->components[slot];
-         continue;
+      const bool lv_word = v->lv && (slot == VARYING_SLOT_LAYER ||
+                                     slot == VARYING_SLOT_VIEWPORT);
+      if (lv_word) {
+         if (v->components[slot] != 1 ||
+             (slot == VARYING_SLOT_VIEWPORT && !v->primitive_comps[slot]))
+            return false;
+      } else if (!((MESH_VERTEX_VARYINGS | VARYING_BIT_PSIZ |
+                    VARYING_BIT_PRIMITIVE_ID) & BITFIELD64_BIT(slot)) ||
+                 (slot == VARYING_SLOT_PRIMITIVE_ID &&
+                  !v->primitive_comps[slot]) ||
+                 (v->primitive_comps[slot] &&
+                  (!(MESH_PRIMITIVE_WORDS & BITFIELD64_BIT(slot)) ||
+                   v->primitive_comps[slot] != v->components[slot]))) {
+         return false;
       }
-      if ((!((USER_VARYINGS | MESH_CLIP_SLOTS) & BITFIELD64_BIT(slot)) &&
-           !(slot == AGX_MESH_PRIMITIVE_PAGE_SLOT && v->primitive_stride_B != 0 &&
-             v->components[slot] == page_carrier_words())) || v->components[slot] > 4 ||
-          v->first[slot] < 4 || v->first[slot] + v->components[slot] > v->size)
-         M5_REFUSE();
+      if (v->components[slot] > 4 ||
+          v->first[slot] < 4 ||
+          v->first[slot] + v->components[slot] > user_end)
+         return false;
       for (unsigned c = 0; c < v->components[slot]; ++c) {
          if (used[v->first[slot] + c]++)
-            M5_REFUSE();
+            return false;
       }
       user += v->components[slot];
    }
-   return user + hidden + 4 == v->size;
+   if (v->lv != (v->primitive_comps[VARYING_SLOT_LAYER] ||
+                 v->primitive_comps[VARYING_SLOT_VIEWPORT]) ||
+       (v->lv && !v->components[VARYING_SLOT_LAYER]))
+      return false;
+   return user + 4 + lv + v->clip_distance_count == v->size;
 }
 
 bool
@@ -446,38 +571,59 @@ agx_nir_lower_m5_mesh_triangle_probe(nir_shader *s)
 {
    struct agx_m5_mesh_varyings varyings;
    if (!agx_m5_mesh_gather_varyings(s, &varyings))
-      M5_REFUSE();
+      return false;
    return agx_nir_lower_m5_mesh(s, &varyings);
 }
 
 bool
 agx_nir_lower_m5_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings)
 {
+   const uint64_t primitive_outputs =
+      s->info.per_primitive_outputs & s->info.outputs_written;
+   uint64_t primitive_slots = VARYING_BIT_PRIMITIVE_INDICES;
+   for (unsigned slot = 0; slot < 64; ++slot) {
+      if (varyings->primitive_comps[slot])
+         primitive_slots |= BITFIELD64_BIT(slot);
+   }
+   if (varyings->cull_primitive)
+      primitive_slots |= VARYING_BIT_CULL_PRIMITIVE;
+   const bool unweld = agx_m5_mesh_unwelds(varyings);
    unsigned threads=s->info.workgroup_size[0]*s->info.workgroup_size[1]*
                     s->info.workgroup_size[2];
    if(s->info.stage!=MESA_SHADER_MESH || s->info.shared_size>32768 ||
-      s->info.workgroup_size_variable || threads==0 || threads>128 ||
+      s->info.workgroup_size_variable || threads==0 || threads>1024 ||
       s->info.mesh.max_vertices_out==0 || s->info.mesh.max_vertices_out>256 ||
       s->info.mesh.max_primitives_out==0 || s->info.mesh.max_primitives_out>256 ||
       prim_vertices(s)==0 ||
-      (s->info.per_primitive_outputs & ~(VARYING_BIT_PRIMITIVE_INDICES | PRIMITIVE_SLOTS)) ||
-      (s->info.outputs_written & ~s->info.per_primitive_outputs) & ~(VARYING_BIT_POS|VARYING_BIT_PSIZ|MESH_CLIP_SLOTS|USER_VARYINGS) ||
+      /* Points carry their size; a point size word follows the user words. */
+      (s->info.mesh.primitive_type==MESA_PRIM_POINTS &&
+       !varyings->components[VARYING_SLOT_PSIZ]) ||
+      (varyings->components[VARYING_SLOT_PSIZ] &&
+       varyings->first[VARYING_SLOT_PSIZ] + 1 !=
+          varyings->size - varyings->clip_distance_count -
+          (varyings->lv ? 1 : 0)) ||
+      /* Interstage compaction can leave per_primitive_outputs stale; with
+       * per-primitive words every store is checked against them below. */
+      (primitive_outputs&~(unweld ? ~UINT64_C(0) : primitive_slots)) ||
+      s->info.outputs_written & ~(VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES|MESH_VERTEX_VARYINGS|
+                                  (primitive_slots & (VARYING_BIT_LAYER|VARYING_BIT_VIEWPORT|
+                                                      VARYING_BIT_CULL_PRIMITIVE|
+                                                      VARYING_BIT_PRIMITIVE_ID)) |
+                                  VARYING_BIT_PSIZ) ||
+      /* Every primitive gets private vertices within the native
+       * 256-vertex output limit. */
+      (unweld && prim_vertices(s) * s->info.mesh.max_primitives_out > 256) ||
+      varyings->clip_distance_count != s->info.clip_distance_array_size ||
       !varyings_valid(varyings))
-      M5_REFUSE();
-   if (s->info.mesh.primitive_type == MESA_PRIM_POINTS &&
-       !varyings->components[VARYING_SLOT_PSIZ])
-      M5_REFUSE();
+      return false;
    struct mesh_layout layout = mesh_layout(s, varyings);
-   if (getenv("ASAHI_MESH_PP_AUDIT") &&
-       !strcmp(getenv("ASAHI_MESH_PP_AUDIT"), "1") &&
-       (!layout.primitive_audit_va || (layout.primitive_audit_va & 15) ||
-        s->info.mesh.max_vertices_out > primitive_audit_lanes() ||
-        s->info.mesh.max_primitives_out > primitive_audit_primitives() ||
-        threads > primitive_audit_lanes()))
-      M5_REFUSE();
+   if (varyings->cull_primitive) {
+      layout.cull = layout.output_base + layout.bytes;
+      layout.bytes += ALIGN_POT(s->info.mesh.max_primitives_out, 64);
+   }
    if (layout.output_base + layout.bytes > 32768)
-      M5_REFUSE();
-   unsigned physical_threads=MAX2(threads,MIN2(s->info.mesh.max_vertices_out,32));
+      return false;
+   unsigned physical_threads=MAX2(threads,MIN2(layout.export_vertices,32));
    bool counts=false;
    nir_foreach_function_impl(impl,s) {
       nir_foreach_block(block,impl) {
@@ -487,61 +633,70 @@ agx_nir_lower_m5_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings
             switch(i->intrinsic) {
             case nir_intrinsic_set_vertex_and_primitive_count:counts=true;break;
             case nir_intrinsic_barrier:
-               if (physical_threads != threads) {
-                  if (nir_intrinsic_execution_scope(i) == SCOPE_WORKGROUP)
-                     nir_intrinsic_set_execution_scope(i, SCOPE_SUBGROUP);
-                  if (nir_intrinsic_memory_scope(i) == SCOPE_WORKGROUP)
-                     nir_intrinsic_set_memory_scope(i, SCOPE_SUBGROUP);
-               }
+               /* Padding moves the API body under its exact active mask;
+                * see m5_mask_padded_body. */
+               if (physical_threads != threads &&
+                   !m5_padded_barrier_ok(block, i))
+                  return false;
                break;
             case nir_intrinsic_store_per_vertex_output: {
                if (!nir_src_is_const(i->src[2]))
-                  M5_REFUSE();
+                  return false;
                const unsigned slot=vertex_output_slot(i);
                const unsigned bits=i->src[0].ssa->bit_size;
                const unsigned end=nir_intrinsic_component(i)+
                                   util_last_bit(nir_intrinsic_write_mask(i));
                if(slot>=64 || (bits!=32 && bits!=16) || end>4)
-                  M5_REFUSE();
+                  return false;
                if(slot==VARYING_SLOT_POS) {
-                  if(bits!=32)M5_REFUSE();
-               } else if(slot==VARYING_SLOT_PSIZ) {
-                  if(varyings->components[slot] && (bits!=32 || end>1))M5_REFUSE();
+                  if(bits!=32)return false;
                } else if(!varyings->components[slot] ||
                          end>varyings->components[slot]) {
-                  M5_REFUSE();
+                  return false;
                }
                break;
             }
             case nir_intrinsic_store_per_primitive_output: {
                if (!nir_src_is_const(i->src[2]))
-                  M5_REFUSE();
-               const unsigned slot = vertex_output_slot(i);
-               const unsigned end = nir_intrinsic_component(i) +
-                                    util_last_bit(nir_intrinsic_write_mask(i));
-               if (slot == VARYING_SLOT_PRIMITIVE_INDICES) {
-                  if (end > prim_vertices(s) || nir_src_as_uint(i->src[2]) != 0)
-                     M5_REFUSE();
-               } else if (slot >= 64 || !varyings->primitive_stride_B ||
-                          end > varyings->primitive_comps[slot] ||
-                          (i->src[0].ssa->bit_size != 32 && i->src[0].ssa->bit_size != 16)) {
-                  M5_REFUSE();
+                  return false;
+               const unsigned slot=vertex_output_slot(i);
+               const unsigned bits=i->src[0].ssa->bit_size;
+               const unsigned end=nir_intrinsic_component(i)+
+                                  util_last_bit(nir_intrinsic_write_mask(i));
+               if (slot==VARYING_SLOT_CULL_PRIMITIVE) {
+                  if (!varyings->cull_primitive || !nir_src_is_const(i->src[2]))
+                     return false;
+               } else if (slot==VARYING_SLOT_PRIMITIVE_INDICES) {
+                  if(nir_intrinsic_io_semantics(i).location!=VARYING_SLOT_PRIMITIVE_INDICES ||
+                     nir_intrinsic_component(i)+i->src[0].ssa->num_components>prim_vertices(s) ||
+                     nir_src_as_uint(i->src[2])!=0)
+                     return false;
+               } else if (slot>=64 || !varyings->primitive_comps[slot] ||
+                          end>varyings->primitive_comps[slot] ||
+                          (bits!=32 && bits!=16)) {
+                  return false;
                }
                break;
             }
             case nir_intrinsic_load_per_vertex_output:
             case nir_intrinsic_load_per_primitive_output:
-            case nir_intrinsic_launch_mesh_workgroups:M5_REFUSE();
+            case nir_intrinsic_launch_mesh_workgroups:return false;
             default:break;
             }
          }
       }
    }
-   if(!counts)M5_REFUSE();
-   if (physical_threads != threads) {
-      
-
-
+   if (!counts) {
+      if (s->info.outputs_written || s->info.outputs_read)
+         return false;
+      nir_builder b = nir_builder_at(nir_before_impl(nir_shader_get_entrypoint(s)));
+      nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, 0),
+         nir_imm_int(&b, 0), nir_imm_int(&b, 0));
+   }
+   if (physical_threads != threads || s->info.workgroup_size[2] > 1023) {
+      /* Native output batches target vertex_base + physical lane. Pad at
+       * most the first output SIMD group; later batches reuse it with the
+       * captured high16 vertex_base field. Keep the original API body mask. */
       nir_function_impl *entry=nir_shader_get_entrypoint(s);
       nir_foreach_block(block,entry) {
          nir_foreach_instr_safe(in,block) {
@@ -560,8 +715,8 @@ agx_nir_lower_m5_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings
                   nir_umod_imm(&b,nir_udiv_imm(&b,index,x),y),
                   nir_udiv_imm(&b,index,x*y));
                if (i->intrinsic == nir_intrinsic_load_global_invocation_id) {
-                  
-
+                  /* The hardware thread position in grid counts the padded
+                   * physical workgroup; the API id uses the declared size. */
                   nir_def *size=nir_imm_ivec3(&b,s->info.workgroup_size[0],
                      s->info.workgroup_size[1],s->info.workgroup_size[2]);
                   replacement=nir_iadd(&b,nir_imul(&b,nir_load_workgroup_id(&b),size),
@@ -575,125 +730,204 @@ agx_nir_lower_m5_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings
             }
          }
       }
-      nir_cf_list body;
-      nir_cf_extract(&body,nir_before_cf_list(&entry->body),
-                            nir_after_cf_list(&entry->body));
-      nir_builder b=nir_builder_at(nir_before_impl(entry));
-      nir_if *active=nir_push_if(&b,nir_ult_imm(&b,
-                                  nir_load_local_invocation_index(&b),threads));
-      b.cursor=nir_cf_reinsert(&body,b.cursor);
-      nir_pop_if(&b,active);
+      /* The payload-entry barrier stays outside the API body mask: native
+       * mesh main functions pass it with every lane before the thread-index
+       * branch (lane-mesh nwit mpay, nwit2 mtpg). */
+      bool entry_barrier = false;
+      nir_foreach_instr_safe(in, nir_start_block(entry)) {
+         if (in->type == nir_instr_type_intrinsic &&
+             nir_instr_as_intrinsic(in)->intrinsic ==
+                nir_intrinsic_mesh_payload_entry_barrier_agx) {
+            nir_instr_remove(in);
+            entry_barrier = true;
+         }
+      }
+      m5_mask_padded_body(entry, threads);
+      if (entry_barrier) {
+         nir_builder b=nir_builder_at(nir_before_impl(entry));
+         nir_mesh_payload_entry_barrier_agx(&b);
+      }
       s->info.workgroup_size[0]=physical_threads;
       s->info.workgroup_size[1]=s->info.workgroup_size[2]=1;
    }
-   
-
-
+   /* Keep application shared storage disjoint from the native output arena.
+    * The UVST descriptor carries a byte address, so relocation changes no
+    * primitive/position plane layout. Existing barriers preserve visibility. */
    nir_shader_intrinsics_pass(s,lower,nir_metadata_none,&layout);
    nir_function_impl *impl=nir_shader_get_entrypoint(s);
+   const unsigned max_primitives = s->info.mesh.max_primitives_out;
+   if (varyings->cull_primitive) {
+      /* An unwritten gl_CullPrimitiveEXT keeps its primitive: clear the cull
+       * bytes before the body writes any. */
+      nir_builder z = nir_builder_at(nir_before_impl(impl));
+      const unsigned words = ALIGN_POT(max_primitives, 64) / 4;
+      const unsigned lanes = s->info.workgroup_size[0] *
+                             s->info.workgroup_size[1] *
+                             s->info.workgroup_size[2];
+      nir_def *lane = nir_load_local_invocation_index(&z);
+      for (unsigned w = 0; w < words; w += lanes) {
+         nir_def *word = nir_iadd_imm(&z, lane, w);
+         nir_push_if(&z, nir_ult_imm(&z, word, words));
+         nir_store_shared(&z, nir_imm_int(&z, 0),
+            nir_iadd_imm(&z, nir_imul_imm(&z, word, 4), layout.cull),
+            .write_mask = 1, .align_mul = 4);
+         nir_pop_if(&z, NULL);
+      }
+      nir_barrier(&z, .execution_scope = SCOPE_WORKGROUP,
+                  .memory_scope = SCOPE_WORKGROUP,
+                  .memory_semantics = NIR_MEMORY_ACQ_REL,
+                  .memory_modes = nir_var_mem_shared);
+   }
    nir_builder b=nir_builder_at(nir_after_cf_list(&impl->body));
-   nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
-                  .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
-   
-   nir_push_if(&b,nir_ieq_imm(&b,nir_load_subgroup_id(&b),0));
-   nir_def *p=nir_load_shared(&b,1,16,nir_imm_int(&b,layout.primitive_count),.align_mul=2);
-   nir_def *v=nir_load_shared(&b,1,16,nir_imm_int(&b,layout.vertex_count),.align_mul=2);
-   if (varyings->lv) {
-      nir_def *storage = nir_pack_64_2x32(&b, nir_load_mesh_primitive_storage_agx(&b));
-      nir_def *lane = nir_load_subgroup_invocation(&b);
-      for (unsigned prim = 0; prim < s->info.mesh.max_primitives_out; prim += 32) {
-         nir_def *index = nir_iadd_imm(&b, lane, prim);
-         nir_push_if(&b, nir_ult(&b, index, nir_u2u32(&b, p)));
-         nir_def *record = nir_iadd(&b, storage,
-            nir_u2u64(&b, nir_imul_imm(&b, index, varyings->primitive_stride_B)));
-         nir_def *layer = nir_imm_int(&b, 0), *viewport = nir_imm_int(&b, 0);
-         if (varyings->primitive_comps[VARYING_SLOT_LAYER])
-            layer = nir_load_global(&b, 1, 32, nir_iadd_imm(&b, record,
-               4 * varyings->primitive_first[VARYING_SLOT_LAYER]), .align_mul = 4);
-         if (varyings->primitive_comps[VARYING_SLOT_VIEWPORT])
-            viewport = nir_load_global(&b, 1, 32, nir_iadd_imm(&b, record,
-               4 * varyings->primitive_first[VARYING_SLOT_VIEWPORT]), .align_mul = 4);
-         nir_def *packed = nir_pack_32_2x16_split(&b, nir_u2u16(&b, layer),
-                                                  nir_u2u16(&b, viewport));
-         for (unsigned k = 0; k < layout.prim_vertices; ++k) {
-            nir_def *vertex = nir_u2u32(&b, nir_load_shared(&b, 1, 8,
-               nir_iadd_imm(&b, nir_imul_imm(&b, index, layout.prim_vertices),
-                            layout.output_base + k), .align_mul = 1));
-            nir_store_shared(&b, packed,
-               nir_iadd_imm(&b, nir_imul_imm(&b, vertex, 4),
-                            layout.position + layout.plane_stride * varyings->lv_first),
-               .write_mask = 1, .align_mul = 4);
+   if (varyings->cull_primitive) {
+      /* Drop culled primitives before the export: one lane moves the index
+       * bytes (and the words of unwelded primitives) of every kept
+       * primitive down, in API order, and stores the new primitive count.
+       * The rendezvous that follows (unweld copy or export) publishes them. */
+      nir_barrier(&b, .execution_scope = SCOPE_WORKGROUP,
+                  .memory_scope = SCOPE_WORKGROUP,
+                  .memory_semantics = NIR_MEMORY_ACQ_REL,
+                  .memory_modes = nir_var_mem_shared);
+      const unsigned corners = prim_vertices(s);
+      unsigned primitive_planes = 0;
+      for (unsigned plane = 0; plane < M5_MESH_MAX_PLANES; ++plane)
+         primitive_planes = MAX2(primitive_planes, layout.primitive_plane[plane] + 1);
+      nir_variable *jv = nir_local_variable_create(impl, glsl_uint_type(), "cull_j");
+      nir_variable *kv = nir_local_variable_create(impl, glsl_uint_type(), "cull_k");
+      nir_push_if(&b, nir_ieq_imm(&b, nir_load_local_invocation_index(&b), 0));
+      nir_store_var(&b, jv, nir_imm_int(&b, 0), 1);
+      nir_store_var(&b, kv, nir_imm_int(&b, 0), 1);
+      nir_def *count = nir_umin(&b, nir_u2u32(&b, nir_load_shared(&b, 1, 16,
+         nir_imm_int(&b, layout.primitive_count), .align_mul = 2)),
+         nir_imm_int(&b, max_primitives));
+      nir_push_loop(&b);
+      {
+         nir_def *j = nir_load_var(&b, jv);
+         nir_break_if(&b, nir_uge(&b, j, count));
+         nir_def *culled = nir_load_shared(&b, 1, 8,
+            nir_iadd_imm(&b, j, layout.cull), .align_mul = 1);
+         nir_push_if(&b, nir_ieq_imm(&b, culled, 0));
+         {
+            nir_def *k = nir_load_var(&b, kv);
+            for (unsigned c = 0; c < corners; ++c) {
+               nir_def *index = nir_load_shared(&b, 1, 8, nir_iadd_imm(&b,
+                  nir_imul_imm(&b, j, corners), layout.output_base + c), .align_mul = 1);
+               nir_store_shared(&b, index, nir_iadd_imm(&b,
+                  nir_imul_imm(&b, k, corners), layout.output_base + c),
+                  .write_mask = 1, .align_mul = 1);
+            }
+            for (unsigned plane = 0; layout.unweld && plane < primitive_planes; ++plane) {
+               const unsigned base = layout.primitive_source + layout.primitive_stride * plane;
+               nir_def *word = nir_load_shared(&b, 1, 32,
+                  nir_iadd_imm(&b, nir_imul_imm(&b, j, 4), base), .align_mul = 4);
+               nir_store_shared(&b, word, nir_iadd_imm(&b, nir_imul_imm(&b, k, 4), base),
+                  .write_mask = 1, .align_mul = 4);
+            }
+            nir_store_var(&b, kv, nir_iadd_imm(&b, k, 1), 1);
          }
          nir_pop_if(&b, NULL);
+         nir_store_var(&b, jv, nir_iadd_imm(&b, j, 1), 1);
       }
-      nir_barrier(&b, .execution_scope = SCOPE_SUBGROUP, .memory_scope = SCOPE_SUBGROUP,
-                  .memory_semantics = NIR_MEMORY_ACQ_REL, .memory_modes = nir_var_mem_shared);
+      nir_pop_loop(&b, NULL);
+      nir_store_shared(&b, nir_u2u16(&b, nir_load_var(&b, kv)),
+         nir_imm_int(&b, layout.primitive_count), .write_mask = 1, .align_mul = 2);
+      nir_pop_if(&b, NULL);
    }
-   if (varyings->primitive_stride_B) {
-      nir_def *storage = nir_load_mesh_primitive_storage_agx(&b);
+   if (layout.unweld) {
+      /* Per-primitive words have no native vertex form here. Give every
+       * primitive private vertices: copy each referenced vertex's
+       * words, replicate the primitive's words (flat on every corner, so
+       * any provoking vertex selects them), and rewrite the index list to
+       * the identity. Primitive order, winding and the interpolated values
+       * are unchanged. SIMD group 0 copies, in the same 32-lane batches
+       * as the export, between two rendezvous: UVB stores read only words
+       * published by a threadgroup barrier, as for API output stores (a
+       * SIMD-group barrier between the copy and the export left stale
+       * words on hardware). Each lane reads and then rewrites only the
+       * index bytes of its own private vertices. */
+      nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+                     .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
+      nir_push_if(&b, nir_ieq_imm(&b, nir_load_subgroup_id(&b), 0));
+      nir_def *primitives = nir_umin(&b, nir_u2u32(&b, nir_load_shared(&b, 1, 16,
+         nir_imm_int(&b, layout.primitive_count), .align_mul = 2)),
+         nir_imm_int(&b, max_primitives));
+      const unsigned corners = prim_vertices(s);
+      nir_def *vertices = nir_imul_imm(&b, primitives, corners);
       nir_def *lane = nir_load_subgroup_invocation(&b);
-      for (unsigned vertex = 0; vertex < s->info.mesh.max_vertices_out; vertex += 32) {
-         nir_def *index = nir_iadd_imm(&b, lane, vertex);
-         nir_push_if(&b, nir_ult(&b, index, nir_u2u32(&b, v)));
-         if (native_page_carrier()) {
-            nir_def *page = nir_unpack_64_2x32_split_x(&b,
-               nir_ushr_imm(&b, nir_pack_64_2x32(&b, storage), 16));
-            nir_store_shared(&b, page,
-               nir_iadd_imm(&b, nir_imul_imm(&b, index, 4),
-                  layout.position + layout.plane_stride *
-                     varyings->first[AGX_MESH_PRIMITIVE_PAGE_SLOT]),
-               .write_mask = 1, .align_mul = 4);
-         } else {
-            for (unsigned c = 0; c < 2; ++c) {
-               nir_store_shared(&b, nir_channel(&b, storage, c),
-                  nir_iadd_imm(&b, nir_imul_imm(&b, index, 4),
-                     layout.position + layout.plane_stride *
-                        (varyings->first[AGX_MESH_PRIMITIVE_PAGE_SLOT] + c)),
-                  .write_mask = 1, .align_mul = 4);
+      for (unsigned base = 0; base < corners * max_primitives; base += 32) {
+         nir_def *j = nir_iadd_imm(&b, lane, base);
+         nir_push_if(&b, nir_ult(&b, j, vertices));
+         nir_def *index = nir_u2u32(&b, nir_load_shared(&b, 1, 8,
+            nir_iadd_imm(&b, j, layout.output_base), .align_mul = 1));
+         nir_def *source = nir_imul_imm(&b,
+            nir_umin(&b, index, nir_imm_int(&b, s->info.mesh.max_vertices_out - 1)), 4);
+         nir_def *primitive = nir_imul_imm(&b, nir_udiv_imm(&b, j, corners), 4);
+         nir_def *target = nir_imul_imm(&b, j, 4);
+         for (unsigned plane = 0; plane < layout.planes; ++plane) {
+            nir_def *value;
+            if (varyings->lv && plane == varyings->lv_first) {
+               nir_def *layer = nir_imm_int(&b, 0), *viewport = nir_imm_int(&b, 0);
+               if (layout.layer_plane >= 0)
+                  layer = nir_load_shared(&b, 1, 32, nir_iadd_imm(&b, primitive,
+                     layout.primitive_source + layout.primitive_stride * layout.layer_plane),
+                     .align_mul = 4);
+               if (layout.viewport_plane >= 0)
+                  viewport = nir_load_shared(&b, 1, 32, nir_iadd_imm(&b, primitive,
+                     layout.primitive_source + layout.primitive_stride * layout.viewport_plane),
+                     .align_mul = 4);
+               /* render_target_array_index low, viewport_array_index high,
+                * as agx_nir_lower_uvs packs the vertex-stage word. */
+               value = nir_pack_32_2x16_split(&b, nir_u2u16(&b, layer),
+                                              nir_u2u16(&b, viewport));
+            } else if (layout.vertex_plane[plane] >= 0) {
+               value = nir_load_shared(&b, 1, 32, nir_iadd_imm(&b, source,
+                  layout.vertex_source + layout.vertex_stride * layout.vertex_plane[plane]),
+                  .align_mul = 4);
+            } else if (layout.primitive_plane[plane] < 0) {
+               value = nir_imm_int(&b, 0);
+            } else {
+               value = nir_load_shared(&b, 1, 32, nir_iadd_imm(&b, primitive,
+                  layout.primitive_source +
+                  layout.primitive_stride * layout.primitive_plane[plane]),
+                  .align_mul = 4);
             }
+            nir_store_shared(&b, value, nir_iadd_imm(&b, target,
+               layout.position + layout.plane_stride * plane), .write_mask = 1,
+               .align_mul = 4);
          }
-         if (layout.primitive_audit_va) {
-            nir_def *group = nir_channel(&b, nir_load_workgroup_id(&b), 0);
-            nir_push_if(&b, nir_ult_imm(&b, group, 4));
-            primitive_order(&b);
-            nir_def *shared = nir_iadd_imm(&b, nir_imul_imm(&b, index, 4),
-               layout.position + layout.plane_stride *
-                  varyings->first[AGX_MESH_PRIMITIVE_PAGE_SLOT]);
-            nir_def *fields[8] = {nir_imm_int(&b, 0x50510010), index,
-               nir_channel(&b, storage, 0), nir_channel(&b, storage, 1),
-               nir_load_shared(&b, 1, 32, shared,
-                  .align_mul = 4, .access = ACCESS_VOLATILE),
-               nir_load_shared(&b, 1, 32, nir_iadd_imm(&b, shared, layout.plane_stride),
-                  .align_mul = 4, .access = ACCESS_VOLATILE), group,
-               nir_imm_int(&b, 0x50510011)};
-            primitive_capture(&b, layout.primitive_audit_va, primitive_audit_lane_first(),
-               nir_iadd(&b, nir_imul_imm(&b, group, primitive_audit_lanes()), index), 8, fields);
-            nir_pop_if(&b, NULL);
-         }
+         nir_store_shared(&b, nir_u2u8(&b, j),
+            nir_iadd_imm(&b, j, layout.output_base), .write_mask = 1,
+            .align_mul = 1);
          nir_pop_if(&b, NULL);
       }
-      nir_barrier(&b, .execution_scope = SCOPE_SUBGROUP,
-         .memory_scope = SCOPE_SUBGROUP, .memory_semantics = NIR_MEMORY_ACQ_REL,
-         .memory_modes = nir_var_mem_shared);
+      nir_pop_if(&b, NULL);
+   }
+   nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+                  .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
+   /* Native write_uvb helpers execute on SIMD group0 after rendezvous. */
+   nir_push_if(&b,nir_ieq_imm(&b,nir_load_subgroup_id(&b),0));
+   nir_def *p=nir_load_shared(&b,1,16,nir_imm_int(&b,layout.primitive_count),.align_mul=2);
+   nir_def *v=nir_load_shared(&b,1,16,nir_imm_int(&b,layout.vertex_count),.align_mul=2);
+   if (layout.unweld) {
+      p = nir_umin(&b, p, nir_imm_intN_t(&b, max_primitives, 16));
+      v = nir_imul_imm(&b, p, prim_vertices(s));
    }
    nir_allocate_mesh_uvb_agx(&b,p,v);
    nir_def *lane=nir_load_subgroup_invocation(&b);
    nir_def *primitive_words=nir_udiv_imm(&b,
-      nir_iadd_imm(&b,nir_imul_imm(&b,nir_u2u32(&b,p),layout.prim_vertices),3),4);
-   for (unsigned word=0;
-        word<DIV_ROUND_UP(layout.prim_vertices*s->info.mesh.max_primitives_out,4);
-        word+=32) {
+      nir_iadd_imm(&b,nir_imul_imm(&b,nir_u2u32(&b,p),prim_vertices(s)),3),4);
+   for (unsigned word=0;word<DIV_ROUND_UP(prim_vertices(s)*s->info.mesh.max_primitives_out,4);word+=32) {
       nir_push_if(&b,nir_ult(&b,nir_iadd_imm(&b,lane,word),primitive_words));
-      
-
+      /* Native64/256 helper increments high16 target word by32 and
+       * shared-byte address by128, while each active lane emits one word. */
       nir_store_mesh_uvb_agx(&b,nir_imm_ivec2(&b,word<<16,
          layout.output_base+4*word),.flags=1);
       nir_pop_if(&b,NULL);
    }
-   
-
-
-   nir_def *selector[4 + 128] = {NULL};
+   /* Plane k publishes UVS word k unless the fragment interpolation order is
+    * a draw-time choice; then user planes take the root UVS index of their
+    * slot, like an unlinked vertex shader's store_uvs_agx. */
+   nir_def *selector[M5_MESH_MAX_PLANES] = {NULL};
    if (varyings->dynamic) {
       for (unsigned slot = 0; slot < 64; ++slot) {
          if (!varyings->components[slot])
@@ -704,12 +938,12 @@ agx_nir_lower_m5_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings
             selector[varyings->first[slot] + c] = nir_iadd_imm(&b, base, c);
       }
    }
-   for(unsigned vertex=0;vertex<s->info.mesh.max_vertices_out;vertex+=32) {
+   for(unsigned vertex=0;vertex<layout.export_vertices;vertex+=32) {
       nir_push_if(&b,nir_ult(&b,nir_iadd_imm(&b,lane,vertex),nir_u2u32(&b,v)));
       for(unsigned plane=0;plane<layout.planes;plane++) {
          const unsigned address=layout.position+layout.plane_stride*plane+4*vertex;
-         
-
+         /* Native mesh64 stores0x200000|component in round2. The
+          * descriptor shared address names the batch base, as native does. */
          nir_def *descriptor = selector[plane]
             ? nir_vec2(&b, nir_ior_imm(&b, selector[plane], vertex << 16),
                        nir_imm_int(&b, address))
@@ -719,35 +953,91 @@ agx_nir_lower_m5_mesh(nir_shader *s, const struct agx_m5_mesh_varyings *varyings
       nir_pop_if(&b,NULL);
    }
    nir_pop_if(&b,NULL);
-   if (layout.primitive_audit_va)
-      nir_shader_gather_info(s, nir_shader_get_entrypoint(s));
    s->info.shared_size=layout.output_base+layout.bytes;
    s->info.outputs_written=0;s->info.per_primitive_outputs=0;
+   /* The native output limits describe the exported vertices. */
+   s->info.mesh.max_vertices_out=layout.export_vertices;
    nir_progress(true,impl,nir_metadata_none);
+   if (varyings->cull_primitive)
+      NIR_PASS(_, s, nir_lower_vars_to_ssa);
    return true;
 }
 
+/* agx_nir_lower_m5_mesh gives every primitive private vertices and stores
+ * its per-primitive user, layer and viewport words on all of them, so the
+ * fragment shader reads them as ordinary flat inputs. Like the mesh side,
+ * UVS words are 32-bit; narrower inputs are narrowed after the load. */
+static bool
+lower_m5_primitive_input(nir_builder *b, nir_intrinsic_instr *i, void *data)
+{
+   (void)data;
+   if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
+      return false;
+   nir_io_semantics sem = nir_intrinsic_io_semantics(i);
+   const unsigned location = sem.location;
+   if (!((location >= VARYING_SLOT_VAR0 && location < VARYING_SLOT_VAR0 + 32) ||
+         location == VARYING_SLOT_LAYER || location == VARYING_SLOT_VIEWPORT ||
+         location == VARYING_SLOT_PRIMITIVE_ID) ||
+       (i->def.bit_size != 32 && i->def.bit_size != 16))
+      return false;
+   b->cursor = nir_before_instr(&i->instr);
+   const nir_alu_type base =
+      nir_alu_type_get_base_type(nir_intrinsic_dest_type(i));
+   nir_def *value = nir_load_input(b, i->def.num_components, 32,
+      i->src[0].ssa, .base = nir_intrinsic_base(i),
+      .component = nir_intrinsic_component(i), .dest_type = base | 32,
+      .io_semantics = sem);
+   if (i->def.bit_size != 32)
+      value = nir_convert_to_bit_size(b, value, base, i->def.bit_size);
+   b->shader->info.per_primitive_inputs &=
+      ~BITFIELD64_RANGE(location, MAX2(sem.num_slots, 1));
+   nir_def_replace(&i->def, value);
+   return true;
+}
 
+bool
+agx_nir_lower_m5_mesh_fragment(nir_shader *s)
+{
+   if (s->info.stage != MESA_SHADER_FRAGMENT)
+      return false;
+   return nir_shader_intrinsics_pass(s, lower_m5_primitive_input,
+                                     nir_metadata_control_flow, NULL);
+}
 
-
+/* The retained native object and matching mesh both access the same pointer:
+ * normalized SR62/W32 + SR63/W16. The four-byte payload limit is the actual
+ * captured source contract, not a claimed hardware maximum. */
 static bool
 lower_payload(nir_builder *b,nir_intrinsic_instr *i,void *data)
 {
-   (void)data;
+   const unsigned payload_base = *(const unsigned *)data;
    if (i->intrinsic==nir_intrinsic_barrier &&
        (nir_intrinsic_memory_modes(i)&nir_var_mem_task_payload)) {
       nir_intrinsic_set_memory_modes(i,
          (nir_intrinsic_memory_modes(i)&~nir_var_mem_task_payload)|nir_var_mem_global);
       return true;
    }
+   const bool atomic=i->intrinsic==nir_intrinsic_task_payload_atomic ||
+                     i->intrinsic==nir_intrinsic_task_payload_atomic_swap;
    bool store=i->intrinsic==nir_intrinsic_store_task_payload;
-   if (!store && i->intrinsic!=nir_intrinsic_load_task_payload) M5_REFUSE();
+   if (!store && !atomic && i->intrinsic!=nir_intrinsic_load_task_payload)
+      return false;
    b->cursor=nir_before_instr(&i->instr);
    nir_def *pointer=nir_pack_64_2x32(b,nir_load_mesh_payload_address_agx(b));
    nir_def *offset=i->src[store ? 1 : 0].ssa;
    nir_def *addr=nir_iadd(b,pointer,nir_u2u64(b,
-                            nir_iadd_imm(b,offset,nir_intrinsic_base(i))));
-   if (store) {
+                            nir_iadd_imm(b,offset,nir_intrinsic_base(i)+payload_base)));
+   if (atomic) {
+      /* The payload is ordinary memory at the payload address: its atomics
+       * are global atomics there, lowered further like any other. */
+      const nir_atomic_op op=nir_intrinsic_atomic_op(i);
+      nir_def *r=i->intrinsic==nir_intrinsic_task_payload_atomic_swap
+         ? nir_global_atomic_swap(b,i->def.bit_size,addr,i->src[1].ssa,
+                                  i->src[2].ssa,.atomic_op=op)
+         : nir_global_atomic(b,i->def.bit_size,addr,i->src[1].ssa,
+                             .atomic_op=op);
+      nir_def_replace(&i->def,r);
+   } else if (store) {
       nir_store_global(b,i->src[0].ssa,addr,
          .write_mask=nir_intrinsic_write_mask(i),
          .align_mul=nir_intrinsic_align_mul(i),
@@ -766,31 +1056,70 @@ lower_payload(nir_builder *b,nir_intrinsic_instr *i,void *data)
 bool
 agx_nir_lower_m5_task_payload_probe(nir_shader *s)
 {
-   if (s->info.task_payload_size>4) M5_REFUSE();
+   return agx_nir_lower_m5_task_payload(s, 4, 0);
+}
+
+bool
+agx_nir_lower_m5_task_payload(nir_shader *s, unsigned max_payload_B,
+                              unsigned payload_base_B)
+{
+   if (s->info.task_payload_size>max_payload_B) return false;
    bool reads=false;
    nir_foreach_function_impl(impl,s) {
       nir_foreach_block(block,impl) {
          nir_foreach_instr(in,block) {
             if (in->type!=nir_instr_type_intrinsic) continue;
             nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
-            reads |= i->intrinsic==nir_intrinsic_load_task_payload;
-            if (i->intrinsic==nir_intrinsic_task_payload_atomic ||
-                i->intrinsic==nir_intrinsic_task_payload_atomic_swap ||
-                i->intrinsic==nir_intrinsic_launch_mesh_workgroups_with_payload_deref)
-               M5_REFUSE();
+            reads |= i->intrinsic==nir_intrinsic_load_task_payload ||
+                     i->intrinsic==nir_intrinsic_load_mesh_payload_address_agx;
+            reads |= i->intrinsic==nir_intrinsic_task_payload_atomic ||
+                     i->intrinsic==nir_intrinsic_task_payload_atomic_swap;
+            if (i->intrinsic==nir_intrinsic_launch_mesh_workgroups_with_payload_deref)
+               return false;
          }
       }
    }
-   if (reads && s->info.stage==MESA_SHADER_MESH)
-      M5_REFUSE();
-   nir_shader_intrinsics_pass(s,lower_payload,nir_metadata_control_flow,NULL);
+   if (reads && s->info.stage==MESA_SHADER_MESH) {
+      nir_builder b=nir_builder_at(nir_before_impl(nir_shader_get_entrypoint(s)));
+      nir_mesh_payload_entry_barrier_agx(&b);
+   }
+   nir_shader_intrinsics_pass(s,lower_payload,nir_metadata_control_flow,
+                              &payload_base_B);
+   return true;
+}
+
+/* Native object->mesh record at the payload address (SR62/SR63): native G17G
+ * witnesses (lane-mesh nwit2 otpg/mtpg) store the child grid
+ * (threadgroups_per_grid) at +0 and the payload from +16; the mesh reads its
+ * threadgroups_per_grid from +0 after the payload-entry barrier. */
+#define M5_TASK_RECORD_PAYLOAD 16
+
+static bool
+lower_mesh_record_grid(nir_builder *b, nir_intrinsic_instr *i, void *data)
+{
+   (void)data;
+   if (i->intrinsic != nir_intrinsic_load_num_workgroups)
+      return false;
+   b->cursor = nir_before_instr(&i->instr);
+   nir_def *pointer = nir_pack_64_2x32(b, nir_load_mesh_payload_address_agx(b));
+   nir_def *grid = nir_load_global(b, 3, 32, pointer, .align_mul = 16);
+   nir_def_replace(&i->def, nir_u2uN(b, grid, i->def.bit_size));
    return true;
 }
 
+bool
+agx_nir_lower_m5_mesh_task_record(nir_shader *s)
+{
+   if (s->info.stage != MESA_SHADER_MESH)
+      return false;
+   return nir_shader_intrinsics_pass(s, lower_mesh_record_grid,
+                                     nir_metadata_control_flow, NULL);
+}
+
 static bool
 lower_launch(nir_builder *b,nir_intrinsic_instr *i,void *data)
 {
-   if (i->intrinsic!=nir_intrinsic_launch_mesh_workgroups) M5_REFUSE();
+   if (i->intrinsic!=nir_intrinsic_launch_mesh_workgroups) return false;
    const unsigned grid=*(unsigned *)data;
    b->cursor=nir_before_instr(&i->instr);
    nir_push_if(b,nir_ieq_imm(b,nir_load_local_invocation_index(b),0));
@@ -808,7 +1137,7 @@ agx_nir_lower_m5_task_probe(nir_shader *s)
                     s->info.workgroup_size[2];
    if (s->info.stage!=MESA_SHADER_TASK || s->info.workgroup_size_variable ||
        threads<3 || threads>128 || s->info.shared_size>48 ||
-       !agx_nir_lower_m5_task_payload_probe(s)) M5_REFUSE();
+       !agx_nir_lower_m5_task_payload_probe(s)) return false;
    unsigned launches=0;
    nir_foreach_function_impl(impl,s) {
       nir_foreach_block(block,impl) {
@@ -819,14 +1148,14 @@ agx_nir_lower_m5_task_probe(nir_shader *s)
          }
       }
    }
-   if (!launches) M5_REFUSE();
+   if (!launches) return false;
    unsigned grid=ALIGN_POT(s->info.shared_size,16);
    nir_shader_intrinsics_pass(s,lower_launch,nir_metadata_none,&grid);
    nir_builder b=nir_builder_at(nir_after_cf_list(&nir_shader_get_entrypoint(s)->body));
    nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
       .memory_semantics=NIR_MEMORY_ACQ_REL,
       .memory_modes=nir_var_mem_shared);
-
+   nir_object_payload_exit_barrier_agx(&b);
    nir_def *lane=nir_load_local_invocation_index(&b);
    nir_push_if(&b,nir_ult_imm(&b,lane,3));
    nir_def *value=nir_load_shared(&b,1,32,
@@ -838,157 +1167,236 @@ agx_nir_lower_m5_task_probe(nir_shader *s)
    return true;
 }
 
-static bool
-lower_primitive_input(nir_builder *b, nir_intrinsic_instr *i, void *data)
-{
-   if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
-      M5_REFUSE();
-   b->cursor = nir_before_instr(&i->instr);
-   const struct agx_m5_mesh_varyings *mesh = data;
-   const unsigned location = nir_intrinsic_io_semantics(i).location;
-   if (mesh && (location >= 64 || !mesh->primitive_comps[location])) {
-      nir_def_rewrite_uses(&i->def, nir_imm_zero(b, i->def.num_components,
-                                                 i->def.bit_size));
-      nir_instr_remove(&i->instr);
-      return true;
-   }
-   const unsigned record_first = mesh ? mesh->primitive_first[location] : 0;
-   nir_def *primitive_cf = nir_load_coefficients_agx(b, nir_imm_int(b, 0),
-      .component = 0, .interp_mode = INTERP_MODE_FLAT,
-      .io_semantics = {.location = VARYING_SLOT_PRIMITIVE_ID, .num_slots = 1});
-   nir_def *base_low_cf = nir_load_coefficients_agx(b, nir_imm_int(b, 0),
-      .component = 0, .interp_mode = INTERP_MODE_FLAT,
-      .io_semantics = {.location = AGX_MESH_PRIMITIVE_PAGE_SLOT, .num_slots = 1});
-   nir_def *base_high_cf = native_page_carrier() ? base_low_cf :
-      nir_load_coefficients_agx(b, nir_imm_int(b, 0),
-      .component = 1, .interp_mode = INTERP_MODE_FLAT,
-      .io_semantics = {.location = AGX_MESH_PRIMITIVE_PAGE_SLOT, .num_slots = 1});
-   nir_def *low = nir_channel(b, primitive_cf, 2);
-   nir_def *base_low = nir_channel(b, base_low_cf, 2);
-   nir_def *base_high = nir_channel(b, base_high_cf, 2);
-   nir_def *base = native_page_carrier()
-      ? nir_ishl_imm(b, nir_u2u64(b, base_low), 16)
-      : nir_pack_64_2x32(b, nir_vec2(b, base_low, base_high));
-   nir_def *address = native_primitive_address()
-      ? nir_ior(b, nir_iand_imm(b, base, ~UINT64_C(0xffff)),
-                nir_u2u64(b, nir_iand_imm(b, nir_ishl_imm(b, low, 2), 0xffff)))
-      : nir_iadd(b, base, nir_u2u64(b, nir_iand_imm(b, low, 0xffff)));
-   address = nir_iadd_imm(b, address,
-                          4 * (record_first + nir_intrinsic_component(i)));
-   nir_def *value = nir_load_global(b, i->def.num_components, 32, address,
-                                  .align_mul = 4);
-   const uint64_t va = primitive_audit_va();
-   if (va) {
-      nir_def *position = nir_load_frag_coord(b);
-      nir_def *x = nir_f2u32(b, nir_channel(b, position, 0));
-      nir_def *y = nir_f2u32(b, nir_channel(b, position, 1));
-      nir_def *group = nir_ushr_imm(b, x, 4);
-      nir_def *bottom = nir_ieq_imm(b, y, 60);
-      nir_def *sample, *right = NULL, *below = NULL, *left = NULL;
-      const bool fan = primitive_audit_fan();
-      if (fan) {
-         nir_def *column = nir_iand_imm(b, x, 15);
-         nir_def *centre = nir_ieq_imm(b, column, 8);
-         nir_def *middle = nir_ieq_imm(b, y, 32);
-         nir_def *top = nir_iand(b, centre, nir_ieq_imm(b, y, 4));
-         right = nir_iand(b, nir_ieq_imm(b, column, 14), middle);
-         below = nir_iand(b, centre, bottom);
-         left = nir_iand(b, nir_ieq_imm(b, column, 2), middle);
-         sample = nir_ior(b, nir_ior(b, top, right), nir_ior(b, below, left));
-      } else {
-         sample = nir_iand(b, nir_ieq_imm(b, nir_iand_imm(b, x, 15), 8),
-            nir_ior(b, nir_ieq_imm(b, y, 4), bottom));
-      }
-      nir_push_if(b, nir_iand(b, nir_ult_imm(b, group, 4), sample));
-      nir_def *fields[32] = {nir_imm_int(b, 0x50510030), x, y, group,
-         nir_channel(b, primitive_cf, 0), nir_channel(b, primitive_cf, 1), low,
-         nir_channel(b, base_low_cf, 0), nir_channel(b, base_low_cf, 1), base_low,
-         nir_channel(b, base_high_cf, 0), nir_channel(b, base_high_cf, 1), base_high,
-         nir_u2u32(b, address), nir_u2u32(b, nir_ushr_imm(b, address, 32)),
-         nir_channel(b, value, 0), nir_imm_int(b, nir_intrinsic_component(i)),
-         nir_imm_int(b, 0x50510031)};
-      nir_def *record = fan
-         ? nir_iadd(b, nir_imul_imm(b, group, 4),
-              nir_bcsel(b, right, nir_imm_int(b, 1),
-                 nir_bcsel(b, below, nir_imm_int(b, 2),
-                    nir_bcsel(b, left, nir_imm_int(b, 3), nir_imm_int(b, 0)))))
-         : nir_iadd(b, nir_imul_imm(b, group, 2), nir_b2i32(b, bottom));
-      primitive_capture(b, va, primitive_audit_fragment_first(), record, 32, fields);
-      nir_pop_if(b, NULL);
-   }
-   if (i->def.bit_size != 32)
-      value = nir_convert_to_bit_size(b, value,
-         nir_alu_type_get_base_type(nir_intrinsic_dest_type(i)), i->def.bit_size);
-   nir_def_rewrite_uses(&i->def, value);
-   nir_instr_remove(&i->instr);
-   return true;
-}
-
-static bool
-fragment_inputs_supported(nir_shader *s, const struct agx_m5_mesh_varyings *mesh)
+/* The opt-in API task (object) stage. The child grid is published with UVSW
+ * from lanes 0..2 of SIMD group 0. Payload-carrying Apple object shaders
+ * store the EmitMeshTasks grid in threadgroup memory, rendezvous and pass
+ * the object payload-exit barrier first (_agc.object.write_childcount);
+ * payload-free objects take the grid of invocation 0 from registers, like
+ * the mesh-only passthrough object. Workgroups of fewer than three
+ * invocations run on three physical lanes with the API body under its exact
+ * invocation mask. Payload accesses use the native SR62/SR63 payload
+ * address; callers bound max_payload_B to what the running kernel backs
+ * (zero refuses every payload access).
+ */
+bool
+agx_nir_lower_m5_task(nir_shader *s, unsigned max_payload_B, bool record)
 {
+   const unsigned threads = s->info.workgroup_size[0] *
+                            s->info.workgroup_size[1] *
+                            s->info.workgroup_size[2];
+   if (s->info.stage != MESA_SHADER_TASK || s->info.workgroup_size_variable ||
+       threads == 0 || threads > 1024)
+      return false;
+   const unsigned physical_threads = MAX2(threads, 3);
+   unsigned launches = 0;
+   bool payload_access = false;
    nir_foreach_function_impl(impl, s) {
       nir_foreach_block(block, impl) {
          nir_foreach_instr(in, block) {
             if (in->type != nir_instr_type_intrinsic)
                continue;
             nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
-            if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
-               continue;
-            const unsigned location = nir_intrinsic_io_semantics(i).location;
-            if (!nir_src_is_const(i->src[0]) || nir_src_as_uint(i->src[0]) != 0 ||
-                nir_intrinsic_component(i) + i->def.num_components > 4 ||
-                (i->def.bit_size != 32 && i->def.bit_size != 16))
-               M5_REFUSE();
-            if (location >= 64 || !(PRIMITIVE_SLOTS & BITFIELD64_BIT(location)))
-               M5_REFUSE();
-            if (!mesh && location != VARYING_SLOT_VAR0)
-               M5_REFUSE();
+            switch (i->intrinsic) {
+            case nir_intrinsic_launch_mesh_workgroups:
+               launches++;
+               break;
+            case nir_intrinsic_load_task_payload:
+            case nir_intrinsic_store_task_payload:
+            case nir_intrinsic_task_payload_atomic:
+            case nir_intrinsic_task_payload_atomic_swap:
+               if (!max_payload_B)
+                  return false;
+               payload_access = true;
+               break;
+            case nir_intrinsic_barrier:
+               /* As for padded mesh workgroups. */
+               if (physical_threads != threads &&
+                   !m5_padded_barrier_ok(block, i))
+                  return false;
+               break;
+            default:
+               break;
+            }
          }
       }
    }
-   return true;
-}
-
-bool
-agx_nir_lower_m5_mesh_fragment_layout(nir_shader *s,
-                                      const struct agx_m5_mesh_varyings *mesh)
-{
-   if (s->info.stage != MESA_SHADER_FRAGMENT || !mesh ||
-       !fragment_inputs_supported(s, mesh))
-      M5_REFUSE();
-   nir_shader_intrinsics_pass(s, lower_primitive_input,
-      primitive_audit_va() ? nir_metadata_none : nir_metadata_control_flow,
-      (void *)mesh);
-   if (primitive_audit_va())
-      nir_shader_gather_info(s, nir_shader_get_entrypoint(s));
-   return true;
-}
-
-bool
-agx_nir_lower_m5_mesh_fragment(nir_shader *s)
-{
-   if (s->info.stage != MESA_SHADER_FRAGMENT)
-      M5_REFUSE();
-   nir_foreach_function_impl(impl, s) {
-      nir_foreach_block(block, impl) {
+   const unsigned grid = ALIGN_POT(s->info.shared_size, 16);
+   /* A body that uses all 32 KiB of threadgroup memory leaves no room for
+    * the grid words; they then alias word 0 once the body is done. */
+   const bool alias_grid = grid + 16 > 32768;
+   if (!launches || s->info.shared_size > 32768 ||
+       !agx_nir_lower_m5_task_payload(s, max_payload_B,
+                                      record ? M5_TASK_RECORD_PAYLOAD : 0))
+      return false;
+   /* SPIR-V EmitMeshTasksEXT terminates the invocation: vtn follows the
+    * launch with a halt, which would skip the child-count epilogue appended
+    * below (EmitMeshTasks inside an if/else left both branches halting and
+    * published undefined counts). A task shader has no other halt source, so
+    * the launch ends main instead. */
+   {
+      bool halts = false;
+      nir_foreach_block(block, nir_shader_get_entrypoint(s)) {
          nir_foreach_instr(in, block) {
+            if (in->type == nir_instr_type_jump &&
+                nir_instr_as_jump(in)->type == nir_jump_halt) {
+               nir_instr_as_jump(in)->type = nir_jump_return;
+               halts = true;
+            }
+         }
+      }
+      if (halts)
+         NIR_PASS(_, s, nir_lower_returns);
+   }
+   /* A payload-free object publishes the grid of invocation 0 from
+    * registers, like Apple's mesh-only passthrough object: no threadgroup
+    * memory and no rendezvous. Only payload-carrying objects need the
+    * threadgroup grid, the rendezvous and the payload-exit barrier of
+    * _agc.object.write_childcount. (The latter shape faulted once on R239
+    * with no 0x1ca48 mesh rendering buffer bound; it stays behind the
+    * payload limit.) */
+   const bool direct = !record && !s->info.task_payload_size && !payload_access;
+   nir_function_impl *entry = nir_shader_get_entrypoint(s);
+   nir_variable *grid_var = NULL;
+   if (direct) {
+      grid_var = nir_local_variable_create(entry, glsl_uvec_type(3), "task_grid");
+      nir_foreach_block(block, entry) {
+         nir_foreach_instr_safe(in, block) {
             if (in->type != nir_instr_type_intrinsic)
                continue;
             nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
-            if (i->intrinsic != nir_intrinsic_load_per_primitive_input)
+            if (i->intrinsic != nir_intrinsic_launch_mesh_workgroups)
+               continue;
+            nir_builder b = nir_builder_at(nir_before_instr(in));
+            nir_store_var(&b, grid_var, i->src[0].ssa, 7);
+            nir_instr_remove(in);
+         }
+      }
+      nir_builder b = nir_builder_at(nir_before_impl(entry));
+      nir_store_var(&b, grid_var, nir_imm_ivec3(&b, 0, 0, 0), 7);
+   } else {
+      if (record) {
+         /* Native object shaders also publish the child grid at +0 of the
+          * record for the mesh stage's threadgroups_per_grid. */
+         nir_foreach_block(block, entry) {
+            nir_foreach_instr(in, block) {
+               if (in->type != nir_instr_type_intrinsic ||
+                   nir_instr_as_intrinsic(in)->intrinsic !=
+                      nir_intrinsic_launch_mesh_workgroups)
+                  continue;
+               nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+               nir_builder b = nir_builder_at(nir_before_instr(in));
+               nir_push_if(&b, nir_ieq_imm(&b, nir_load_local_invocation_index(&b), 0));
+               nir_store_global(&b, i->src[0].ssa,
+                  nir_pack_64_2x32(&b, nir_load_mesh_payload_address_agx(&b)),
+                  .write_mask = 7, .align_mul = 16);
+               nir_pop_if(&b, NULL);
+            }
+         }
+      }
+      if (alias_grid) {
+         grid_var = nir_local_variable_create(entry, glsl_uvec_type(3), "task_grid");
+         nir_foreach_block(block, entry) {
+            nir_foreach_instr_safe(in, block) {
+               if (in->type != nir_instr_type_intrinsic ||
+                   nir_instr_as_intrinsic(in)->intrinsic !=
+                      nir_intrinsic_launch_mesh_workgroups)
+                  continue;
+               nir_builder b = nir_builder_at(nir_before_instr(in));
+               nir_store_var(&b, grid_var, nir_instr_as_intrinsic(in)->src[0].ssa, 7);
+               nir_instr_remove(in);
+            }
+         }
+         nir_builder b = nir_builder_at(nir_before_impl(entry));
+         nir_store_var(&b, grid_var, nir_imm_ivec3(&b, 0, 0, 0), 7);
+      } else {
+         unsigned grid_offset = grid;
+         nir_shader_intrinsics_pass(s, lower_launch, nir_metadata_none, &grid_offset);
+      }
+   }
+   if (physical_threads != threads) {
+      nir_foreach_block(block, entry) {
+         nir_foreach_instr_safe(in, block) {
+            if (in->type != nir_instr_type_intrinsic)
                continue;
-            if (nir_intrinsic_io_semantics(i).location != VARYING_SLOT_VAR0 ||
-                !nir_src_is_const(i->src[0]) || nir_src_as_uint(i->src[0]) != 0 ||
-                nir_intrinsic_component(i) + i->def.num_components > 4 ||
-                (i->def.bit_size != 32 && i->def.bit_size != 16))
-               M5_REFUSE();
+            nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+            nir_builder b = nir_builder_at(nir_before_instr(in));
+            nir_def *replacement = NULL;
+            if (i->intrinsic == nir_intrinsic_load_workgroup_size) {
+               replacement = nir_imm_ivec3(&b, s->info.workgroup_size[0],
+                  s->info.workgroup_size[1], s->info.workgroup_size[2]);
+            } else if (i->intrinsic == nir_intrinsic_load_local_invocation_id ||
+                       i->intrinsic == nir_intrinsic_load_global_invocation_id) {
+               nir_def *index = nir_load_local_invocation_index(&b);
+               const unsigned x = s->info.workgroup_size[0];
+               const unsigned y = s->info.workgroup_size[1];
+               replacement = nir_vec3(&b, nir_umod_imm(&b, index, x),
+                  nir_umod_imm(&b, nir_udiv_imm(&b, index, x), y),
+                  nir_udiv_imm(&b, index, x * y));
+               if (i->intrinsic == nir_intrinsic_load_global_invocation_id) {
+                  nir_def *size = nir_imm_ivec3(&b, s->info.workgroup_size[0],
+                     s->info.workgroup_size[1], s->info.workgroup_size[2]);
+                  replacement = nir_iadd(&b,
+                     nir_imul(&b, nir_load_workgroup_id(&b), size), replacement);
+                  replacement = nir_u2uN(&b, replacement, i->def.bit_size);
+               }
+            }
+            if (replacement) {
+               nir_def_rewrite_uses(&i->def, replacement);
+               nir_instr_remove(in);
+            }
          }
       }
+      m5_mask_padded_body(entry, threads);
+      s->info.workgroup_size[0] = physical_threads;
+      s->info.workgroup_size[1] = s->info.workgroup_size[2] = 1;
+   }
+   nir_builder b = nir_builder_at(nir_after_cf_list(&entry->body));
+   if (direct) {
+      nir_def *value = nir_read_invocation(&b, nir_load_var(&b, grid_var),
+                                           nir_imm_int(&b, 0));
+      nir_def *lane = nir_load_subgroup_invocation(&b);
+      nir_push_if(&b, nir_iand(&b, nir_ieq_imm(&b, nir_load_subgroup_id(&b), 0),
+                               nir_ult_imm(&b, lane, 3)));
+      nir_def *count = nir_bcsel(&b, nir_ieq_imm(&b, lane, 0),
+         nir_channel(&b, value, 0),
+         nir_bcsel(&b, nir_ieq_imm(&b, lane, 1), nir_channel(&b, value, 1),
+                   nir_channel(&b, value, 2)));
+      nir_store_object_childcount_agx(&b, count);
+      nir_pop_if(&b, NULL);
+      /* Object threadgroup memory is allocated in 64-byte units. */
+      if (s->info.shared_size)
+         s->info.shared_size = ALIGN_POT(s->info.shared_size, 64);
+      nir_progress(true, entry, nir_metadata_none);
+      NIR_PASS(_, s, nir_lower_vars_to_ssa);
+      return true;
+   }
+   nir_barrier(&b, .execution_scope = SCOPE_WORKGROUP,
+               .memory_scope = SCOPE_WORKGROUP,
+               .memory_semantics = NIR_MEMORY_ACQ_REL,
+               .memory_modes = nir_var_mem_shared);
+   if (alias_grid) {
+      /* Every invocation has finished with the body's shared memory. */
+      nir_push_if(&b, nir_ieq_imm(&b, nir_load_local_invocation_index(&b), 0));
+      nir_store_shared(&b, nir_load_var(&b, grid_var), nir_imm_int(&b, 0),
+                       .write_mask = 7, .align_mul = 4);
+      nir_pop_if(&b, NULL);
+      nir_barrier(&b, .execution_scope = SCOPE_WORKGROUP,
+                  .memory_scope = SCOPE_WORKGROUP,
+                  .memory_semantics = NIR_MEMORY_ACQ_REL,
+                  .memory_modes = nir_var_mem_shared);
    }
-   nir_shader_intrinsics_pass(s, lower_primitive_input,
-      primitive_audit_va() ? nir_metadata_none : nir_metadata_control_flow, NULL);
-   if (primitive_audit_va())
-      nir_shader_gather_info(s, nir_shader_get_entrypoint(s));
+   nir_object_payload_exit_barrier_agx(&b);
+   nir_def *lane = nir_load_local_invocation_index(&b);
+   nir_push_if(&b, nir_ult_imm(&b, lane, 3));
+   nir_def *value = nir_load_shared(&b, 1, 32,
+      nir_iadd_imm(&b, nir_imul_imm(&b, lane, 4), alias_grid ? 0 : grid),
+      .align_mul = 4);
+   nir_store_object_childcount_agx(&b, value);
+   nir_pop_if(&b, NULL);
+   s->info.shared_size = alias_grid ? ALIGN_POT(s->info.shared_size, 64)
+                                    : ALIGN_POT(grid + 16, 64);
+   nir_progress(true, entry, nir_metadata_none);
+   if (alias_grid)
+      NIR_PASS(_, s, nir_lower_vars_to_ssa);
    return true;
 }
diff --git a/src/asahi/compiler/agx_nir_m5_mesh.h b/src/asahi/compiler/agx_nir_m5_mesh.h
index f0848918dc0a..bbb0aa5c9ef3 100644
--- a/src/asahi/compiler/agx_nir_m5_mesh.h
+++ b/src/asahi/compiler/agx_nir_m5_mesh.h
@@ -12,6 +12,7 @@ extern "C" {
 
 
 
+bool agx_nir_m5_canonicalize_empty_mesh(nir_shader *shader);
 bool agx_nir_lower_m5_mesh_triangle_probe(nir_shader *shader);
 
 
@@ -32,13 +33,16 @@ struct agx_m5_mesh_varyings {
    bool dynamic;
    uint8_t primitive_components;
    uint16_t primitive_stride_B;
-   uint8_t clip_first, clip_count;
+   union { uint8_t clip_first; uint8_t clip_distance_first; };
+   union { uint8_t clip_count; uint8_t clip_distance_count; };
    uint8_t lv_first;
    bool lv;
    uint8_t first[64];
    uint8_t components[64];
    uint8_t primitive_first[64];
    uint8_t primitive_comps[64];
+   /* gl_CullPrimitiveEXT is written (M5: dropped before the export). */
+   bool cull_primitive;
 };
 
 
@@ -56,6 +60,19 @@ bool agx_nir_lower_m5_mesh(nir_shader *shader,
 
 bool agx_nir_lower_m5_task_payload_probe(nir_shader *shader);
 bool agx_nir_lower_m5_task_probe(nir_shader *shader);
+bool agx_nir_lower_m5_task_payload(nir_shader *shader, unsigned max_payload_B,
+                                   unsigned payload_base_B);
+bool agx_nir_lower_m5_task(nir_shader *shader, unsigned max_payload_B,
+                           bool record);
+bool agx_nir_lower_m5_mesh_task_record(nir_shader *shader);
+bool agx_gen1_mesh_gather_varyings(const nir_shader *, struct agx_m5_mesh_varyings *);
+bool agx_nir_lower_gen1_mesh(nir_shader *, const struct agx_m5_mesh_varyings *);
+bool agx_nir_lower_gen1_mesh_triangle_probe(nir_shader *);
+bool agx_nir_lower_gen1_mesh_fragment(nir_shader *);
+bool agx_nir_lower_gen1_mesh_fragment_layout(nir_shader *, const struct agx_m5_mesh_varyings *);
+bool agx_nir_lower_gen1_task_payload_probe(nir_shader *);
+bool agx_nir_lower_gen1_task_probe(nir_shader *);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/asahi/compiler/agx_nir_m5_ray_query.c b/src/asahi/compiler/agx_nir_m5_ray_query.c
new file mode 100644
index 000000000000..a420159eac5c
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_m5_ray_query.c
@@ -0,0 +1,2564 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#include "agx_nir_m5_ray_query.h"
+#include "agx3_rt_m5_core.h"
+#include "compiler/nir/nir_builder.h"
+#include "util/u_debug.h"
+
+static bool
+m5_candidate_api(void)
+{
+   return debug_get_bool_option("ASAHI_M5_RQ_CANDIDATE_API", false);
+}
+
+static bool
+m5_candidate_mode(void)
+{
+   return m5_candidate_api() ||
+          debug_get_bool_option("ASAHI_M5_RQ_CANDIDATE_PROBE", false);
+}
+
+/* Keep native geometry opacity so opaque triangles commit in the
+ * intersector, while transparent triangles continue through the native
+ * candidate queue and Vulkan application any-hit code. This is the
+ * configuration the M5 RT gate qualifies (455 cases). */
+static bool
+m5_native_opaque(void)
+{
+   return m5_candidate_api();
+}
+
+static bool
+m5_defer_completion(nir_builder *b)
+{
+   return m5_candidate_api() &&
+          b->shader->info.stage == MESA_SHADER_COMPUTE &&
+          b->shader->info.ray_queries == 1 &&
+          !getenv("ASAHI_M5_RQ_STATE_ADDRESS") &&
+          !getenv("ASAHI_M5_RQ_ZERO_SPAN_DIAG_ADDRESS") &&
+          debug_get_bool_option("ASAHI_M5_RQ_DEFER_COMPLETION", true);
+}
+
+/* The live raw IFT50 callback stores its final 48-bit instance pointer at
+ * driver+8640+6*lane. Its complete per-clique tail is 8832 bytes (276*32),
+ * which overlaps the next clique with the 272-entry candidate stride, so the
+ * candidate profile uses 276 (and the matching packed SCS layout in
+ * hk_cmd_dispatch.c/hk_queue.c). */
+static bool
+m5_raw50_layout_probe(void)
+{
+   return m5_candidate_mode();
+}
+
+static unsigned
+m5_candidate_driver_stride(void)
+{
+   return m5_raw50_layout_probe() ? 276 : 272;
+}
+
+static nir_def *
+m5_native_opaque_eligible(nir_builder *b, nir_def *flags)
+{
+   /* The captured native query policy implements geometry/instance opacity,
+    * but these Vulkan policies require our candidate filter or first-hit
+    * ordering. Face culling remains encoded in the native ray packet. */
+   /* Owner replay counts application candidates. Native opaque pruning can
+    * alter that prefix, so qualify this policy only for a lexical single
+    * query until the mixed-owner prefix protocol is separately established. */
+   /* Native RT-pipeline shaders arrive here after lowering to COMPUTE.
+    * Graphics query resources and callback layouts require their own
+    * hardware qualification before using this optimization. */
+   return m5_native_opaque() && b->shader->info.stage == MESA_SHADER_COMPUTE &&
+          b->shader->info.ray_queries == 1
+      ? nir_ieq_imm(b, nir_iand_imm(b, flags, 2 | 4 | 0x40 | 0x100), 0)
+      : nir_imm_false(b);
+}
+
+nir_def *
+agx_nir_m5_candidate_count(nir_builder *b, nir_def *driver, nir_def *lane)
+{
+   assert(driver->bit_size == 32 && lane->bit_size == 32);
+   /* RIA3 payload segment14: the four-bit count starts at bit21. */
+   /* The captured IFT runtime writes its query count at 1728. The newer
+    * query-once compiler witness uses 1600: its payload layout is 128 bytes
+    * smaller before the candidate queue. Operand shape is the native +0x6ac
+    * two-register selector-2 form. */
+   nir_def *control = nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+      .base = 1728, .component = 2);
+   return nir_iand_imm(b, nir_ushr_imm(b, control, 21), 15);
+}
+
+struct agx_m5_ray_candidate
+agx_nir_m5_load_candidate(nir_builder *b, nir_def *driver, nir_def *lane,
+                          nir_def *index)
+{
+   assert(driver->bit_size == 32 && lane->bit_size == 32 &&
+          index->bit_size == 32);
+   /* The native program multiplies the lane by200 and its candidate cursor
+    * by20. Each record is a 48-bit compressed address, 16-bit flags, two
+    * barycentrics and distance. Do not decode it as a committed hit packet. */
+   nir_def *offset = nir_iadd(b, nir_imul_imm(b, lane, 200),
+                                nir_imul_imm(b, index, 20));
+   if (debug_get_bool_option("ASAHI_M5_RQ_CANDIDATE_FLAT", false)) {
+      nir_def *address = nir_iadd(b, driver, offset);
+      nir_def *prefix = nir_load_scs_base_agx(b, 2, 32, address,
+         .base = 1856, .format = PIPE_FORMAT_R32_UINT, .component = 4);
+      return (struct agx_m5_ray_candidate){
+         .compressed_address = nir_iand_imm(b, nir_pack_64_2x32(b, prefix),
+                                            UINT64_C(0xffffffffffff)),
+         .flags = nir_ushr_imm(b, nir_channel(b, prefix, 1), 16),
+         .barycentrics = nir_load_scs_base_agx(b, 2, 32, address,
+            .base = 1864, .format = PIPE_FORMAT_R32_UINT, .component = 4),
+         .distance = nir_load_scs_base_agx(b, 1, 32, address,
+            .base = 1872, .format = PIPE_FORMAT_R32_UINT, .component = 4),
+      };
+   }
+   /* Native +0x7d0..+0x82e uses selector 0 with the driver in f16 and
+    * per-lane record offset in f31. Preserve that operand form. */
+   nir_def *prefix = nir_load_scs_register_agx(b, 2, 32, offset, driver,
+      .base = 1856, .format = PIPE_FORMAT_R32_UINT, .component = 0);
+   return (struct agx_m5_ray_candidate){
+      .compressed_address = nir_iand_imm(b, nir_pack_64_2x32(b, prefix),
+                                         UINT64_C(0xffffffffffff)),
+      .flags = nir_ushr_imm(b, nir_channel(b, prefix, 1), 16),
+      .barycentrics = nir_load_scs_register_agx(b, 2, 32, offset, driver,
+         .base = 1864, .format = PIPE_FORMAT_R32_UINT, .component = 0),
+      .distance = nir_load_scs_register_agx(b, 1, 32, offset, driver,
+         .base = 1872, .format = PIPE_FORMAT_R32_UINT, .component = 0),
+   };
+}
+
+/* Private qualification of native zero/FTZ maximum-distance admission.
+ * API bounds stay in the query variables; only the traversal packet changes.
+ * Root hardware receipts distinguish positive zero-span rays (admitted) from
+ * zero/subnormal maxima (no native candidate), with FLT_MIN admitted. */
+static nir_def *
+m5_native_admission_tmax(nir_builder *b, nir_def *api_tmax)
+{
+   if (!m5_candidate_api())
+      return api_tmax;
+
+   /* Classify the bit representation, avoiding a floating comparison that
+    * could itself flush a subnormal. Preserve negative finite values, NaNs,
+    * infinity, and every normal maximum bit-for-bit. Both signed zeros are
+    * valid nonnegative endpoints. */
+   nir_def *zero_or_positive_subnormal = nir_ior(b,
+      nir_ult_imm(b, api_tmax, 0x00800000u),
+      nir_ieq_imm(b, api_tmax, 0x80000000u));
+   return nir_bcsel(b, zero_or_positive_subnormal,
+                   nir_imm_int(b, 0x00800000u), api_tmax);
+}
+
+/* Request the next candidate batch exactly as the native runtime does. The
+ * RIA runtime IR agx.intersection_query.next (work/neo-rt-20261011/native-ria2/
+ * native-rt-ir) performs, before every agc.continue.ray: cursor=0, control
+ * word &= ~0x1e00001 (count bits 21..24 and the active bit0), both 32-bit words
+ * of the per-lane completion record (its payload segment 7; driver+256+8*lane
+ * here) stored as zero, and the committed distance published to the ray
+ * packet when it changed. The record's bit 6 is only its busy poll; the
+ * runtime never derives the resume decision from it. The installed lowering
+ * cleared bit 6 of byte 7 alone, leaving any stale committed-hit words from
+ * the previous batch in place. */
+static void
+m5_candidate_request_batch(nir_builder *b, nir_def *id, nir_def *driver,
+                           nir_def *lane, nir_def *status_address,
+                           nir_def *control, nir_def *committed_t)
+{
+   nir_store_scs_indexed_agx(b, nir_iand_imm(b, control, ~0x1e00001u),
+      driver, lane, .base = 1728, .component = 2);
+   if (committed_t) {
+      nir_store_scs_uniform_agx(b, m5_native_admission_tmax(b, committed_t),
+         nir_load_preamble(b, 1, 64, .base = 152), nir_ishl_imm(b, id, 6),
+         .base = 24, .format = PIPE_FORMAT_R32_UINT, .component = 0);
+   }
+   nir_store_scs_base_agx(b, nir_imm_ivec2(b, 0, 0), status_address,
+      .base = 0, .format = PIPE_FORMAT_R32_UINT, .component = 4);
+   nir_continue_ray_agx(b, id);
+}
+
+void
+agx_nir_m5_trace_opaque(nir_builder *b, nir_def *request, nir_def *bvh_state,
+                        nir_def *payload_base, nir_def *payload_index,
+                        nir_def *status_base, nir_def *launch_words,
+                        nir_def **initial_pending,
+                        nir_def **continuations)
+{
+   assert(request->bit_size == 64 && bvh_state->bit_size == 64);
+   assert(payload_base->bit_size == 64 && payload_index->bit_size == 32);
+   assert(status_base->bit_size == 32);
+   nir_variable *steps = NULL;
+   if (continuations) {
+      steps = nir_local_variable_create(b->impl, glsl_uint_type(), "rt_steps");
+      nir_store_var(b, steps, nir_imm_int(b, 0), 1);
+   }
+
+   assert(launch_words->bit_size == 32 && launch_words->num_components == 3);
+   nir_intersect_ray_agx(b, request, bvh_state,
+                         nir_channel(b, launch_words, 0),
+                         nir_channel(b, launch_words, 1),
+                         nir_channel(b, launch_words, 2));
+   nir_def *pending = nir_load_scs_uniform_agx(
+      b, 1, 16, payload_base, payload_index, .base = 51,
+      .format = PIPE_FORMAT_R8_UINT, .component = 0);
+   if (initial_pending)
+      *initial_pending = nir_u2u32(b, pending);
+
+   /* Native rt-wide main +0x4f2..+0x596: byte51 selects the continuation
+    * path. Its status record is addressed separately, through uniform134.
+    * Use the special status-load form only for the read/modify/write pair;
+    * the completion check is an ordinary byte load, as in the capture.
+    */
+   nir_push_if(b, nir_ine_imm(b, pending, 0));
+   nir_def *status_address = nir_iadd(b, status_base, payload_index);
+   nir_push_loop(b);
+   nir_def *status = nir_load_scs_base_agx(
+      b, 1, 16, status_address, .base = 7, .format = PIPE_FORMAT_R8_UINT,
+      .component = 4, .flags = 5);
+   nir_def *resume = nir_ior_imm(b, nir_iand_imm(b, status, ~64u), 16);
+   nir_store_scs_base_agx(b, resume, status_address, .base = 7,
+                         .format = PIPE_FORMAT_R8_UINT, .component = 4);
+   nir_continue_ray_agx(b, nir_ushr_imm(b, payload_index, 6));
+   if (steps)
+      nir_store_var(b, steps, nir_iadd_imm(b, nir_load_var(b, steps), 1), 1);
+   status = nir_load_scs_base_agx(b, 1, 16, status_address, .base = 7,
+                                 .format = PIPE_FORMAT_R8_UINT, .component = 4);
+   nir_break_if(b, nir_ieq_imm(b, status, 0));
+   nir_pop_loop(b, NULL);
+   nir_pop_if(b, NULL);
+   if (continuations)
+      *continuations = nir_load_var(b, steps);
+}
+
+/* ScsLayoutGen2 stores the level records before end[6], not at end[6].
+ * This entered-query profile has one256-byte record per resident clique. */
+static nir_def *
+m5_query_level_base(nir_builder *b)
+{
+   nir_def *packed = nir_load_preamble(b, 1, 32, .base = 52);
+   nir_def *cliques = nir_ushr_imm(b, packed, 24);
+   return nir_isub(b, nir_load_preamble(b, 1, 32, .base = 48),
+                      nir_ishl_imm(b, cliques, 8));
+}
+
+/* Native leaf records: the triangle record starts with 0x40|type (type 8
+ * opaque, 16 nonopaque) and keeps its geometry/primitive ids at 52/56; the
+ * 48-byte bounding-box record starts with marker 0x18 and keeps geometry id
+ * (low 30 bits, opaque in bit 31) at 28 and primitive id at 32. The runtime's
+ * agx.bounding_box_test reads the box record from the token address plus 4,
+ * the same past-the-marker convention the triangle candidate address uses.
+ * ASAHI_M5_RQ_AABB_RECORD_BIAS is a diagnostic to A/B that convention on
+ * hardware; the production default is 4. */
+static unsigned
+m5_aabb_record_bias(void)
+{
+   return debug_get_num_option("ASAHI_M5_RQ_AABB_RECORD_BIAS", 4);
+}
+
+/* Caller has established that this queue entry exists. The triangle leaf
+ * and instance-node offsets are independently checked by candidate43's
+ * nonzero-ID hardware fixture. */
+static void
+m5_candidate_packet(nir_builder *b, nir_def *driver, nir_def *id,
+                    nir_def *index, nir_def *packet[5],
+                    nir_def *ray_flags, nir_def **opaque, nir_def **aabb)
+{
+   nir_def *lane = nir_iand_imm(b, id, 31);
+   struct agx_m5_ray_candidate c =
+      agx_nir_m5_load_candidate(b, driver, lane, index);
+   /* The captured Gen1 extractor identifies triangle type8 as opaque and
+    * type16 as nonopaque, and bounding boxes by marker 24 (0x18). The
+    * triangle candidate points past this marker. */
+   const int bias = m5_aabb_record_bias();
+   nir_def *marker = nir_load_global(b, 1, 32,
+      nir_iadd_imm(b, c.compressed_address, -4), .align_mul = 4);
+   nir_def *is_aabb = NULL;
+   if (bias == 4) {
+      is_aabb = nir_ieq_imm(b, nir_iand_imm(b, marker, 0xff), 0x18);
+   } else {
+      nir_def *base_marker = nir_load_global(b, 1, 32,
+         nir_iadd_imm(b, c.compressed_address, bias - 4), .align_mul = 4);
+      is_aabb = nir_ieq_imm(b, nir_iand_imm(b, base_marker, 0xff), 0x18);
+   }
+   nir_def *id_offset = nir_bcsel(b, is_aabb,
+      nir_imm_int(b, 28 - bias), nir_imm_int(b, 48));
+   nir_def *ids = nir_load_global(b, 2, 32,
+      nir_iadd(b, c.compressed_address, nir_i2i64(b, id_offset)),
+      .align_mul = 4);
+   nir_def *level_address = nir_iadd(b, m5_query_level_base(b),
+                                    nir_ishl_imm(b, id, 3));
+   nir_def *halves[3];
+   for (unsigned i = 0; i < 3; ++i)
+      halves[i] = nir_u2u32(b, nir_load_scs_base_agx(b, 1, 16,
+         level_address, .base = 2 * i, .format = PIPE_FORMAT_R16_UINT,
+         .component = 4));
+   nir_def *node = nir_pack_64_2x32_split(b,
+      nir_ior(b, halves[0], nir_ishl_imm(b, halves[1], 16)), halves[2]);
+   nir_def *meta = nir_load_global(b, 2, 32, nir_iadd_imm(b, node, 96),
+                                   .align_mul = 4);
+   nir_def *zero = nir_imm_int(b, 0);
+   nir_def *opaque_value = zero;
+   if (opaque) {
+      /* Native instance records retain Vulkan instance flags at byte8. */
+      nir_def *instance_flags = nir_load_global(b, 1, 32,
+         nir_iadd_imm(b, node, 8), .align_mul = 4);
+      nir_def *geometry_opaque = nir_bcsel(b, is_aabb,
+         nir_test_mask(b, nir_channel(b, ids, 0), 0x80000000u),
+         nir_ieq_imm(b, nir_iand_imm(b, marker, 31), 8));
+      nir_def *instance_opaque = nir_bcsel(b,
+         nir_test_mask(b, instance_flags, 8), nir_imm_false(b), geometry_opaque);
+      instance_opaque = nir_ior(b, nir_test_mask(b, instance_flags, 4), instance_opaque);
+      *opaque = nir_bcsel(b, nir_test_mask(b, ray_flags, 2),
+         nir_imm_false(b), nir_ior(b, nir_test_mask(b, ray_flags, 1), instance_opaque));
+      opaque_value = nir_b2i32(b, *opaque);
+   }
+   if (aabb)
+      *aabb = is_aabb;
+   nir_def *geometry_mask = nir_bcsel(b, is_aabb,
+      nir_imm_int(b, 0x3fffffff), nir_imm_int(b, 0xffffff));
+   nir_def *face_geom = nir_ior(b,
+      nir_iand(b, nir_channel(b, ids, 0), geometry_mask),
+      /* Native candidate_vectors +0xf58 shifts the16-bit flags by8. Bit0
+       * stays set on both windings and is not the candidate facing bit. */
+      nir_bcsel(b, is_aabb, zero,
+         nir_ishl_imm(b, nir_iand_imm(b, c.flags, 1u << 8), 23)));
+   /* Word0.x keeps the candidate AABB opacity for
+    * rayQueryGetIntersectionCandidateAABBOpaqueEXT. */
+   packet[0] = nir_vec4(b, opaque_value, zero, zero, zero);
+   packet[1] = nir_vec4(b, zero, zero, c.distance, zero);
+   /* Type bits 30..31: 1 triangle, 2 bounding box (the native hit record
+    * uses the same two-bit kind, see agx.commit_serial.bounding_box). */
+   packet[2] = nir_vec4(b, face_geom, zero, zero,
+      nir_ior(b, nir_iand(b, nir_channel(b, ids, 1), geometry_mask),
+              nir_bcsel(b, is_aabb, nir_imm_int(b, 0x80000000u),
+                        nir_imm_int(b, 0x40000000))));
+   packet[3] = nir_vec4(b, zero, nir_channel(b, c.barycentrics, 0),
+                       nir_channel(b, c.barycentrics, 1), zero);
+   packet[4] = nir_vec4(b, nir_unpack_64_2x32_split_x(b, node),
+      nir_unpack_64_2x32_split_y(b, node), nir_channel(b, meta, 1),
+      nir_channel(b, meta, 0));
+}
+
+/* Diagnostic capture of the shared context/root area and driver payloads.
+ * Two 64-lane snapshots, before launch and after completion, 64 KiB each.
+ * The explicit address is supplied by the private host fixture. */
+static void
+m5_capture_query_state(nir_builder *b, nir_def *id, nir_def *driver, bool after)
+{
+   const char *text = getenv("ASAHI_M5_RQ_STATE_ADDRESS");
+   if (!text)
+      return;
+   uint64_t output = strtoull(text, NULL, 0);
+   if (!output)
+      return;
+   nir_def *capture_mask = nir_load_global(b, 1, 32,
+      nir_imm_int64(b, output + 131072), .align_mul = 4);
+   nir_push_if(b, nir_test_mask(b, capture_mask, after ? 2 : 1));
+   nir_variable *cursor = nir_local_variable_create(b->impl, glsl_uint_type(),
+                                                    "capture_cursor");
+   nir_store_var(b, cursor, nir_imm_int(b, 0), 1);
+   nir_push_loop(b);
+   nir_def *i = nir_load_var(b, cursor);
+   nir_break_if(b, nir_uge_imm(b, i, 32));
+   nir_def *offset = nir_iadd(b, nir_ishl_imm(b, id, 9),
+                                nir_ishl_imm(b, i, 4));
+   for (unsigned region = 0; region < 2; ++region) {
+      nir_def *first_base = debug_get_bool_option(
+         "ASAHI_M5_RQ_CAPTURE_LEVELS", false)
+         ? m5_query_level_base(b) : nir_imm_int(b, 0x6000);
+      nir_def *address = nir_iadd(b, offset, region ? driver : first_base);
+      nir_def *value = nir_load_scs_base_agx(b, 4, 32, address,
+         .base = 0, .format = PIPE_FORMAT_R32_UINT, .component = 4);
+      nir_store_global(b, value, nir_iadd_imm(b, nir_u2u64(b, offset),
+         output + (after ? 65536 : 0) + region * 32768),
+         .align_mul = 16, .write_mask = 15);
+   }
+   nir_store_var(b, cursor, nir_iadd_imm(b, i, 1), 1);
+   nir_pop_loop(b, NULL);
+   nir_pop_if(b, NULL);
+}
+
+/* Native rays_instance uses a driver record in bank 3, separate from the
+ * bank-0 ray. Offsets and strides are from candidate26's live SCS snapshots. */
+static void
+m5_trace_instance(nir_builder *b, nir_def *selector, nir_def *id,
+                  nir_def *opaque, nir_def *query_owner, nir_def *minimum,
+                  nir_def *native_opaque_eligible, nir_def **type_prim,
+                  nir_def **bary, nir_def **instance, nir_def **front_face,
+                  nir_def **distance)
+{
+   const bool candidate_probe = m5_candidate_mode();
+   const bool native_init_parity = candidate_probe &&
+      debug_get_bool_option("ASAHI_M5_RQ_NATIVE_INIT_PARITY", false);
+   nir_def *lane = nir_iand_imm(b, id, 31);
+   nir_def *driver = nir_iadd_imm(b,
+      nir_imul_imm(b, nir_iand_imm(b, id, ~31u), 72), 0x5d8000);
+   if (candidate_probe)
+      driver = nir_iadd(b, nir_load_preamble(b, 1, 32, .base = 56),
+         nir_imul_imm(b, nir_iand_imm(b, id, ~31u), m5_candidate_driver_stride()));
+   nir_def *zero = nir_imm_int(b, 0);
+   if (debug_get_num_option("ASAHI_M5_RQ_ADDRESS_PROBE", 0) == 2) {
+      *type_prim = nir_imm_int(b, 0x40000001);
+      *bary = nir_vec2(b, nir_u2f32(b, lane), nir_u2f32(b, id));
+      *instance = nir_vec4(b, zero, zero, lane, driver);
+      *front_face = nir_imul_imm(b, lane, 200);
+      return;
+   }
+   /* RIA3 agx.intersect.ray's payload_segment.41 control uses bit1
+    * for forced-opacity=opaque; its feature bits remain 0x120. */
+   nir_def *pair = nir_vec2(b, nir_bcsel(b, opaque,
+      nir_imm_int(b, 0x122), nir_imm_int(b, 0x120)), zero);
+   if (candidate_probe) {
+      pair = nir_vec2(b, nir_ior(b, nir_imm_int(b, 0x142),
+         nir_b2i32(b, nir_ine_imm(b, query_owner, 0))),
+         native_init_parity ? minimum : zero);
+   }
+   nir_store_scs_indexed_agx(b, pair, driver, lane,
+                            .base = 0, .component = 3);
+   nir_store_scs_indexed_agx(b, nir_vec2(b, zero, zero), driver, lane,
+                            .base = 256, .component = 3);
+
+   /* One per-clique query descriptor, with a fresh active query at +256. */
+   nir_def *clique = nir_imul_imm(b, nir_ushr_imm(b, id, 5), 12);
+   nir_def *state = nir_load_preamble(b, 1, 64, .base = 140);
+   nir_store_scs_uniform_agx(b,
+      nir_imm_ivec4(b, 0, 1, 0x600100e8, 0), state, clique,
+      .base = 256, .format = PIPE_FORMAT_R32_UINT, .component = 2);
+   nir_store_scs_uniform_agx(b, nir_imm_ivec4(b, 0, 0, 0, 0), state, clique,
+      .base = 272, .format = PIPE_FORMAT_R32_UINT, .component = 2);
+   nir_store_scs_uniform_agx(b,
+      nir_load_preamble(b, 2, 32, .base = 80), state, clique,
+      .base = 288, .format = PIPE_FORMAT_R32_UINT, .component = 2);
+   /* Clear the level's pending bit. The fresh profile has one level. */
+   nir_def *level = nir_iadd_imm(b, nir_ishl_imm(b, id, 3), 0x60e000);
+   if (candidate_probe)
+      level = nir_iadd(b, nir_ishl_imm(b, id, 3), m5_query_level_base(b));
+   /* Gen2 native next and the deployed instance callback both clear only
+    * bit0 of the compressed instance frame's trailing16-bit field. Its
+    * remaining bits belong to the runtime and survive query activation. */
+   nir_def *level_flags = nir_imm_intN_t(b, 0, 16);
+   if (native_init_parity) {
+      level_flags = nir_load_scs_base_agx(b, 1, 16, level,
+         .base = 6, .format = PIPE_FORMAT_R16_UINT, .component = 0);
+      level_flags = nir_iand_imm(b, level_flags, ~1u);
+   }
+   nir_store_scs_base_agx(b, level_flags, level,
+                         .base = 6, .format = PIPE_FORMAT_R16_UINT, .component = 0);
+
+   /* Native instance publication inserts the high nibble into the live
+    * 0x111001 control word. Replacing this with the primitive-only
+    * 0x10000000 header makes even the otherwise native shader miss.
+    * This constant belongs to the current one-level instance profile. */
+   m5_capture_query_state(b, id, driver, false);
+   if (candidate_probe) {
+      /* Runtime next identifies its resident owner from bits 1..20 of this
+       * control word. Native query-model71 masks the entry result to 20 bits
+       * and shifts it by1 before publishing, preserving the count/active bits. */
+      nir_store_scs_indexed_agx(b,
+         nir_ishl_imm(b, nir_iand_imm(b, query_owner, 0xfffff), 1), driver, lane,
+         .base = 1728, .component = 2);
+      nir_intersect_ray_agx(b, nir_imm_int64(b, 0), selector,
+                            nir_imm_int(b, debug_get_bool_option(
+                               "ASAHI_M5_RQ_INLINE_PACKET", false)
+                                  ? 0x10216001 : 0x00216000), zero, id);
+
+   } else {
+      nir_intersect_ray_agx(b, nir_imm_int64(b, 0), selector,
+                            nir_imm_int(b, 0x10111001), zero, id);
+   }
+   if (m5_defer_completion(b)) {
+      *type_prim = zero;
+      *bary = nir_vec2(b, zero, zero);
+      *instance = nir_vec4(b, zero, zero, zero, zero);
+      *front_face = zero;
+      *distance = zero;
+      return;
+   }
+   nir_def *pending = nir_load_scs_register_agx(b, 1, 16, driver, lane,
+      .base = 263, .format = PIPE_FORMAT_R8_UINT, .component = 3, .flags = 9);
+   /* Bit 7 also occurs on completed hits. Only bit 6 requests a
+    * continuation; testing the entire byte resumes completed traversal. */
+   nir_def *resume_needed = nir_ine_imm(b, nir_iand_imm(b, pending, 0x40), 0);
+   if (m5_candidate_api())
+      resume_needed = nir_iand(b, resume_needed,
+         nir_ieq_imm(b, agx_nir_m5_candidate_count(b, driver, lane), 0));
+   /* candidate_next imports the native committed packet before clearing
+    * completion words on an empty-batch continuation. Eagerly continuing
+    * here would discard an opaque hit that has no application candidate. */
+   if (m5_native_opaque())
+      resume_needed = nir_iand(b, resume_needed,
+                                  nir_inot(b, native_opaque_eligible));
+   nir_push_if(b, resume_needed);
+   nir_def *status_address = nir_iadd_imm(b,
+      nir_iadd(b, driver, nir_ishl_imm(b, lane, 3)), 256);
+   nir_push_loop(b);
+   nir_def *status = nir_load_scs_base_agx(b, 1, 16, status_address,
+      .base = 7, .format = PIPE_FORMAT_R8_UINT, .component = 4, .flags = 5);
+   if (candidate_probe) {
+      /* The queue is empty here (count == 0), so clearing the count bits is
+       * a no-op; the active bit and completion words follow the runtime. */
+      m5_candidate_request_batch(b, id, driver, lane, status_address,
+         nir_load_scs_indexed_agx(b, 1, 32, driver, lane, .base = 1728,
+                                  .component = 2), NULL);
+   } else {
+      nir_def *resume_status = nir_ior_imm(b, nir_iand_imm(b, status, ~64u), 16);
+      nir_store_scs_base_agx(b, resume_status, status_address,
+         .base = 7, .format = PIPE_FORMAT_R8_UINT, .component = 4);
+      nir_continue_ray_agx(b, id);
+   }
+   status = nir_load_scs_base_agx(b, 1, 16, status_address,
+      .base = 7, .format = PIPE_FORMAT_R8_UINT, .component = 4, .flags = 8);
+   nir_def *finished = nir_ieq_imm(b, nir_iand_imm(b, status, 0x40), 0);
+   if (m5_candidate_api())
+      finished = nir_ior(b, finished,
+         nir_ine_imm(b, agx_nir_m5_candidate_count(b, driver, lane), 0));
+   nir_break_if(b, finished);
+   nir_pop_loop(b, NULL);
+   nir_pop_if(b, NULL);
+   m5_capture_query_state(b, id, driver, true);
+   if (candidate_probe) {
+      nir_def *count = agx_nir_m5_candidate_count(b, driver, lane);
+      struct agx_m5_ray_candidate first =
+         agx_nir_m5_load_candidate(b, driver, lane, zero);
+      /* Diagnostic transport only: expose queue fields through existing
+       * committed-result slots. This is not Vulkan candidate support. */
+      *type_prim = nir_ior_imm(b, count, 0x40000000);
+      *bary = first.barycentrics;
+      *instance = nir_vec4(b, zero, zero, nir_u2u32(b, pending),
+                           first.distance);
+      *front_face = nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+         .base = 0, .component = 3);
+      if (!m5_candidate_api() &&
+          debug_get_bool_option("ASAHI_M5_RQ_METADATA_PROBE", false)) {
+         nir_def *packet[5];
+         nir_def *empty = nir_imm_ivec4(b, 0, 0, 0, 0);
+         nir_push_if(b, nir_ine_imm(b, count, 0));
+         m5_candidate_packet(b, driver, id, zero, packet, NULL, NULL, NULL);
+         nir_push_else(b, NULL);
+         nir_pop_if(b, NULL);
+         for (unsigned i = 0; i < 5; ++i)
+            packet[i] = nir_if_phi(b, packet[i], empty);
+         *type_prim = nir_channel(b, packet[2], 3);
+         *front_face = nir_channel(b, packet[2], 0);
+         *instance = packet[4];
+         *bary = nir_channels(b, packet[3], 6);
+         *distance = nir_channel(b, packet[1], 2);
+      }
+      if (debug_get_bool_option("ASAHI_M5_RQ_ADDRESS_PROBE", false)) {
+         *bary = nir_vec2(b, nir_u2f32(b, lane), nir_u2f32(b, id));
+         *instance = nir_vec4(b, zero, zero, lane, driver);
+         *front_face = nir_imul_imm(b, lane, 200);
+      }
+      return;
+   }
+   nir_def *hit_words = nir_load_scs_indexed_agx(b, 2, 32, driver, lane,
+      .base = 256, .component = 3, .flags = 1);
+   *type_prim = nir_channel(b, hit_words, 0);
+   /* Fresh G17G triangle_front_facing witness selects1 when the signed
+    * second hit word is negative: the native front-face bit is bit31. */
+   *front_face = nir_channel(b, hit_words, 1);
+   *bary = nir_load_scs_indexed_agx(b, 2, 32, driver, lane,
+      .base = 512, .component = 3, .flags = 1);
+
+   /* The captured runtime native-main.bin reads the committed instance
+    * pointer at driver + 2112 + 6*lane (selector0). Fresh Metal compiler
+    * witnesses use1984 with their own layout; they cannot override this
+    * captured depth232/72-byte-driver profile. Do not confuse these with the
+    * current traversal instance or the low24 primitive-ID bits. */
+   nir_push_if(b, nir_ine_imm(b, nir_ushr_imm(b, *type_prim, 30), 0));
+   nir_def *pointer_halves[3];
+   for (unsigned i = 0; i < 3; ++i)
+      pointer_halves[i] = nir_u2u32(b, nir_load_scs_register_agx(
+         b, 1, 16, nir_imul_imm(b, lane, 6), driver,
+         .base = 2112 + 2*i, .format = PIPE_FORMAT_R16_UINT, .component = 0));
+   nir_def *pointer = nir_pack_64_2x32_split(b,
+      nir_ior(b, pointer_halves[0], nir_ishl_imm(b, pointer_halves[1], 16)),
+      pointer_halves[2]);
+   nir_def *metadata = nir_load_global(b, 2, 32, nir_iadd_imm(b, pointer, 96),
+                                       .align_mul = 4);
+   nir_def *hit_instance = nir_vec4(b, nir_unpack_64_2x32_split_x(b, pointer),
+      nir_unpack_64_2x32_split_y(b, pointer), nir_channel(b, metadata, 1),
+      nir_channel(b, metadata, 0));
+   nir_push_else(b, NULL);
+   nir_def *no_instance = nir_imm_ivec4(b, 0, 0, 0, 0);
+   nir_pop_if(b, NULL);
+   *instance = nir_if_phi(b, hit_instance, no_instance);
+}
+
+static void
+m5_init_triangle_packet(nir_builder *b, nir_def *payload_base,
+                                nir_def *payload_index, nir_def *origin,
+                                nir_def *direction, nir_def *t_min,
+                                nir_def *t_max, nir_def *bounds_min,
+                                nir_def *bounds_max, nir_def *mask,
+                                nir_def *query_flags, nir_def *opaque, nir_def *first_hit,
+                                nir_def *triangle_cull,
+                                nir_def *native_opaque_eligible)
+{
+   nir_def *o[3], *d[3], *far[3];
+   for (unsigned i = 0; i < 3; ++i) {
+      o[i] = nir_channel(b, origin, i);
+      d[i] = nir_channel(b, direction, i);
+      nir_def *lo = nir_fsub(b, o[i], nir_channel(b, bounds_min, i));
+      nir_def *hi = nir_fsub(b, o[i], nir_channel(b, bounds_max, i));
+      /* One-ULP outward bias, as in the captured native woop setup. */
+      lo = nir_fmul(b, lo, nir_bcsel(b, nir_flt(b, nir_imm_float(b, 0), lo),
+                                   nir_imm_float(b, 0x1.fffffcp-1f),
+                                   nir_imm_float(b, 0x1.000002p0f)));
+      hi = nir_fmul(b, hi, nir_bcsel(b, nir_flt(b, nir_imm_float(b, 0), hi),
+                                   nir_imm_float(b, 0x1.000002p0f),
+                                   nir_imm_float(b, 0x1.fffffcp-1f)));
+      far[i] = nir_fmax(b, nir_fabs(b, lo), nir_fabs(b, hi));
+   }
+   nir_def *y_major = nir_flt(b, nir_fabs(b, d[0]), nir_fabs(b, d[1]));
+   nir_def *xy_major = nir_bcsel(b, y_major, d[1], d[0]);
+   nir_def *z_major = nir_flt(b, nir_fabs(b, xy_major), nir_fabs(b, d[2]));
+   nir_def *major = nir_bcsel(b, z_major, nir_imm_int(b, 2),
+                             nir_b2i32(b, y_major));
+   nir_def *major_dir = nir_bcsel(b, z_major, d[2], xy_major);
+   nir_def *major_far = nir_bcsel(b, z_major, far[2],
+                                 nir_bcsel(b, y_major, far[1], far[0]));
+   nir_def *other_far = nir_bcsel(
+      b, z_major, nir_fmax(b, far[0], far[1]),
+      nir_fmax(b, far[2], nir_bcsel(b, y_major, far[0], far[1])));
+   nir_def *epsilon = nir_fmul_imm(b, nir_fadd(b, major_far, other_far), 0x1.4p-22f);
+   nir_def *quantized = nir_iand_imm(
+      b, nir_ushr_imm(b, nir_iadd_imm(b, epsilon, 131071), 17), 0x3fff);
+   nir_def *woop = nir_ior(b, nir_ishl_imm(b, major, 16),
+                         nir_ishl_imm(b, quantized, 18));
+   nir_def *u = nir_bcsel(b, nir_ieq_imm(b, major, 2), nir_imm_int(b, 0),
+                         nir_iadd_imm(b, major, 1));
+   nir_def *v = nir_bcsel(b, nir_ieq_imm(b, major, 0), nir_imm_int(b, 2),
+                         nir_iadd_imm(b, major, -1));
+   nir_def *negative = nir_flt(b, major_dir, nir_imm_float(b, 0));
+   nir_def *sx = nir_bcsel(b, negative, v, u);
+   nir_def *sy = nir_bcsel(b, negative, u, v);
+   /* Matching RIA3 runtime (03cb21ad.ll): payload_segment's ray
+    * control stores opacity in bits0..1 and enables per-geometry opacity
+    * with bit14 when the forced opacity is zero. */
+   nir_def *control = nir_ior(
+      b, nir_ior(b, nir_ishl_imm(b, nir_iand_imm(b, mask, 0xff), 18),
+                 nir_ior(b, nir_ishl_imm(b, sx, 2), nir_ishl_imm(b, sy, 4))),
+      nir_bcsel(b, opaque, nir_imm_int(b, 1), nir_imm_int(b, 1 << 14)));
+   if (m5_candidate_mode()) {
+      nir_def *nonopaque = nir_ior_imm(b,
+         nir_iand_imm(b, control, ~0x4003u), 0x402);
+      control = nir_bcsel(b, native_opaque_eligible,
+                            nir_ior_imm(b, control, 0x400), nonopaque);
+   }
+   /* The compiler witness toggles metadata bit11 for accept-any;
+    * matching RIA3 IR stores its policy in root packet bit11 (%211).
+    * Bit6 changes facing and must not be used for this policy. */
+   control = nir_ior(b, control, nir_bcsel(b, first_hit,
+      nir_imm_int(b, 1 << 11), nir_imm_int(b, 0)));
+   /* RIA3 intersect.ray %224..%238 publishes triangle culling at bits8..9.
+    * With the captured winding, Metal back/front map to 1/2 respectively. */
+   control = nir_ior(b, control, triangle_cull);
+   nir_def *zero = nir_imm_int(b, 0);
+   nir_def *words[4] = {
+      nir_vec4(b, o[0], d[0], o[1], d[1]),
+      nir_vec4(b, o[2], d[2], m5_native_admission_tmax(b, t_max), control),
+      nir_vec4(b, woop, query_flags, t_min, zero),
+      nir_vec4(b, zero, zero, zero, zero),
+   };
+   for (unsigned i = 0; i < 4; ++i)
+      nir_store_scs_uniform_agx(b, words[i], payload_base, payload_index,
+                                .base = 16 * i, .format = PIPE_FORMAT_R32_UINT,
+                                .component = 0);
+}
+
+/* Preserve the compiler fixture/opaque-only packet API. Vulkan candidate
+ * lowering supplies its actual ray flags to the private policy-aware helper. */
+void
+agx_nir_m5_init_triangle_packet(nir_builder *b, nir_def *payload_base,
+                              nir_def *payload_index, nir_def *origin,
+                              nir_def *direction, nir_def *t_min,
+                              nir_def *t_max, nir_def *bounds_min,
+                              nir_def *bounds_max, nir_def *mask,
+                              nir_def *query_flags, nir_def *opaque,
+                              nir_def *first_hit, nir_def *triangle_cull)
+{
+   m5_init_triangle_packet(b, payload_base, payload_index, origin, direction,
+      t_min, t_max, bounds_min, bounds_max, mask, query_flags, opaque,
+      first_hit, triangle_cull, nir_imm_false(b));
+}
+
+#include "util/u_dynarray.h"
+
+struct m5_rq_state {
+   nir_variable *query;
+   nir_variable *as, *origin, *direction, *tmin, *tmax, *mask, *active, *flags;
+   nir_variable *result[5];
+   nir_variable *initial_completion;
+   bool queue_first;
+   bool native_opaque_possible;
+   nir_variable *candidate[5];
+   nir_variable *started, *resident_id, *cursor, *owner, *capture_event;
+   nir_variable *processed, *replay, *commit_history, *prune_t;
+   nir_variable *batch_completed;
+   /* Logical query numbers of this variable's elements: first_logical for a
+    * scalar query, first_logical..first_logical+logical_count-1 for a query
+    * array. Native allocation slots are assigned from the logical number by
+    * m5_rq_slot(). */
+   unsigned first_logical;
+   unsigned logical_count;
+   /* Some use of the query array has a non-constant element index. */
+   bool dynamic;
+};
+
+/* Logical queries beyond the witnessed native allocation indices share a
+ * slot. Every use publishes its own owner token and a slot switch between
+ * logical queries restarts traversal through the existing owner-mismatch
+ * replay, so a shared slot only costs that replay. */
+static unsigned
+m5_rq_slot(unsigned logical)
+{
+   return 4 + logical % AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES;
+}
+
+/* Logical number of the query element a deref selects, or NULL when the
+ * deref is a scalar query or a constant element (then *constant is set). */
+static nir_def *
+m5_rq_logical(nir_builder *b, const struct m5_rq_state *state,
+              nir_deref_instr *query, unsigned *constant)
+{
+   *constant = state->first_logical;
+   if (query->deref_type == nir_deref_type_var)
+      return NULL;
+   assert(query->deref_type == nir_deref_type_array &&
+          nir_deref_instr_parent(query)->deref_type == nir_deref_type_var);
+   if (nir_src_is_const(query->arr.index)) {
+      *constant += nir_src_as_uint(query->arr.index);
+      return NULL;
+   }
+   return nir_iadd_imm(b, nir_u2u32(b, query->arr.index.ssa),
+                       state->first_logical);
+}
+
+static nir_def *
+m5_rq_enter_reachable(nir_builder *b, nir_def *slot, unsigned first,
+                      unsigned i, unsigned slots)
+{
+   const unsigned candidate =
+      (first + i) % AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES;
+   if (i + 1 == slots)
+      return nir_enter_query_agx(b, .base = 4 + candidate);
+   nir_push_if(b, nir_ieq_imm(b, slot, candidate));
+   nir_def *then_value = nir_enter_query_agx(b, .base = 4 + candidate);
+   nir_push_else(b, NULL);
+   nir_def *else_value = m5_rq_enter_reachable(b, slot, first, i + 1, slots);
+   nir_pop_if(b, NULL);
+   return nir_if_phi(b, then_value, else_value);
+}
+
+/* Enter the native query slot of a logical query. A constant logical number
+ * enters its slot directly; a dynamically selected element chooses among the
+ * slots its elements can map to. */
+static nir_def *
+m5_rq_enter(nir_builder *b, const struct m5_rq_state *state,
+            nir_deref_instr *query)
+{
+   unsigned constant;
+   nir_def *logical = m5_rq_logical(b, state, query, &constant);
+   if (!logical)
+      return nir_enter_query_agx(b, .base = m5_rq_slot(constant));
+
+   /* The elements reach min(count, slot count) consecutive slots starting at
+    * the first element's slot; the last reachable slot needs no test. */
+   const unsigned slots = MIN2(state->logical_count,
+                               AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES);
+   nir_def *slot = nir_umod_imm(b, logical,
+                                AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES);
+   return m5_rq_enter_reachable(b, slot, m5_rq_slot(state->first_logical) - 4,
+                                0, slots);
+}
+
+/* Shared-edge/vertex candidate filtering, see m5_boundary_duplicate. On by
+ * default (Vulkan requires watertight single hits); ASAHI_M5_RQ_WATERTIGHT=0
+ * disables it for diagnostics. */
+static bool
+m5_rq_watertight(void)
+{
+   return debug_get_bool_option("ASAHI_M5_RQ_WATERTIGHT", true);
+}
+
+/* The candidate cursor indexes a queue of at most 15 entries (a four-bit
+ * count). With m5_rq_watertight its upper bits carry the boundary distance
+ * key; otherwise they stay zero and the cursor is used unmasked. */
+#define M5_CURSOR_INDEX 0xfu
+/* Boundary distance key: distance bits 8..31. Partners compute t from their
+ * own planes; on hardware they differ by up to one ulp in probes but by more
+ * than 31 ulps on the CTS random-height mesh, all inside one 256-ulp step. */
+#define M5_BOUNDARY_KEY_LOW 0xffu
+
+static nir_def *
+m5_cursor_index(nir_builder *b, nir_deref_instr *cursor)
+{
+   nir_def *packed = nir_load_deref(b, cursor);
+   return m5_rq_watertight() ? nir_iand_imm(b, packed, M5_CURSOR_INDEX)
+                             : packed;
+}
+
+static void
+m5_cursor_set_index(nir_builder *b, nir_deref_instr *cursor, nir_def *index)
+{
+   if (m5_rq_watertight())
+      index = nir_ior(b, index,
+         nir_iand_imm(b, nir_load_deref(b, cursor), ~M5_CURSOR_INDEX));
+   nir_store_deref(b, cursor, index, 1);
+}
+
+/* Private opt-in bounded owner recorder. 64 lanes, 8 lexical queries,
+ * 64 events/query, 32 bytes/event = 1 MiB. The host reserves the device-address
+ * range and supplies its address before compiling the query shader. No atomics,
+ * extra submissions, or global waits. Records are private to a lane/query. */
+static uint64_t
+m5_model_capture_address(void)
+{
+   const char *text = getenv("ASAHI_M5_RQ_MODEL_ADDRESS");
+   return text ? strtoull(text, NULL, 0) : 0;
+}
+
+static void
+m5_model_capture(nir_builder *b, struct m5_rq_state *state,
+                 nir_deref_instr *capture_event,
+                 nir_deref_instr *owner, nir_deref_instr *resident_id,
+                 nir_deref_instr *cursor, nir_deref_instr *active,
+                 nir_deref_instr *result, unsigned phase)
+{
+   const uint64_t output = m5_model_capture_address();
+   if (!output || !capture_event)
+      return;
+   nir_def *id = nir_load_deref(b, resident_id);
+   nir_def *event = nir_load_deref(b, capture_event);
+   nir_push_if(b, nir_iand(b, nir_ult_imm(b, id, 64),
+                             nir_ult_imm(b, event, 64)));
+   nir_def *lane = nir_iand_imm(b, id, 31);
+   nir_def *driver = nir_iadd(b, nir_load_preamble(b, 1, 32, .base = 56),
+      nir_imul_imm(b, nir_iand_imm(b, id, ~31u), m5_candidate_driver_stride()));
+   nir_def *control = nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+      .base = 1728, .component = 2);
+   nir_def *status = nir_load_scs_indexed_agx(b, 2, 32, driver, lane,
+      .base = 256, .component = 3);
+   nir_def *offset = nir_iadd(b, nir_imul_imm(b, id, 8 * 64 * 32),
+      nir_iadd_imm(b, nir_imul_imm(b, event, 32),
+         (m5_rq_slot(state->first_logical) - 4) * 64 * 32));
+   nir_def *address = nir_iadd_imm(b, nir_u2u64(b, offset), output);
+   nir_store_global(b, nir_vec4(b, nir_imm_int(b, phase),
+      nir_load_deref(b, owner), id, control), address,
+      .align_mul = 16, .write_mask = 15);
+   nir_store_global(b, nir_vec4(b, nir_channel(b, status, 0),
+      nir_channel(b, status, 1), m5_cursor_index(b, cursor),
+      nir_ior(b, nir_iand_imm(b, nir_channel(b, nir_load_deref(b, result), 2),
+         ~1u), nir_b2i32(b, nir_load_deref(b, active)))),
+      nir_iadd_imm(b, address, 16), .align_mul = 16, .write_mask = 15);
+   nir_store_deref(b, capture_event, nir_iadd_imm(b, event, 1), 1);
+   nir_pop_if(b, NULL);
+}
+
+static const struct glsl_type *
+m5_rq_field_type(const struct glsl_type *query, const struct glsl_type *field)
+{
+   if (!glsl_type_is_array(query))
+      return field;
+   return glsl_array_type(m5_rq_field_type(glsl_get_array_element(query), field),
+                          glsl_get_length(query), 0);
+}
+
+static nir_deref_instr *
+m5_rq_field_deref(nir_builder *b, nir_deref_instr *query, nir_variable *field)
+{
+   if (query->deref_type == nir_deref_type_var)
+      return nir_build_deref_var(b, field);
+   assert(query->deref_type == nir_deref_type_array);
+   return nir_build_deref_follower(b,
+      m5_rq_field_deref(b, nir_deref_instr_parent(query), field), query);
+}
+
+static bool
+m5_rq_supported_load(const nir_intrinsic_instr *intr)
+{
+   switch (nir_intrinsic_ray_query_value(intr)) {
+   case nir_ray_query_value_flags:
+   case nir_ray_query_value_tmin:
+   case nir_ray_query_value_world_ray_origin:
+   case nir_ray_query_value_world_ray_direction:
+      return true;
+   case nir_ray_query_value_intersection_type:
+   case nir_ray_query_value_intersection_t:
+   case nir_ray_query_value_intersection_primitive_index:
+   case nir_ray_query_value_intersection_geometry_index:
+   case nir_ray_query_value_intersection_barycentrics:
+   case nir_ray_query_value_intersection_instance_id:
+   case nir_ray_query_value_intersection_instance_custom_index:
+   case nir_ray_query_value_intersection_front_face:
+   case nir_ray_query_value_intersection_instance_sbt_index:
+   case nir_ray_query_value_intersection_object_ray_direction:
+   case nir_ray_query_value_intersection_object_ray_origin:
+   case nir_ray_query_value_intersection_object_to_world:
+   case nir_ray_query_value_intersection_world_to_object:
+      return nir_intrinsic_committed(intr) || m5_candidate_api();
+   case nir_ray_query_value_intersection_candidate_aabb_opaque:
+      return !nir_intrinsic_committed(intr) && m5_candidate_api();
+   default:
+      return false;
+   }
+}
+
+/* Native Gen2 commit records the first32 candidate positions where it
+ * publishes a new pruning distance. Replay restores those updates at the
+ * same positions instead of restarting with the final bound. */
+static void
+m5_replay_commit(nir_builder *b, nir_deref_instr *processed,
+                 nir_deref_instr *history, nir_deref_instr *prune_t,
+                 nir_def *distance)
+{
+   if (!processed)
+      return;
+   nir_def *count = nir_load_deref(b, processed);
+   nir_push_if(b, nir_ule_imm(b, count, 32));
+   nir_def *bit = nir_ishl(b, nir_imm_int(b, 1), nir_iadd_imm(b, count, -1));
+   nir_store_deref(b, history, nir_ior(b, nir_load_deref(b, history), bit), 1);
+   nir_store_deref(b, prune_t, distance, 1);
+   nir_pop_if(b, NULL);
+}
+
+/* Consume one candidate at a time. RIA3 intersection_query.next clears both
+ * count bits21..24 and the queue's low active bit before requesting another
+ * batch. The committed distance is published before continuation so that the
+ * intersector can prune work, while the committed API result stays private. */
+/* Gen2 intersection_query.next copies segment7 hit words, segment8 bary,
+ * segment11 committed instance and the ray-root max-distance into its API
+ * result before requesting another batch. The retained gfxrq-both helper8
+ * (VA0x100000a800) stores these exact captured-layout offsets: 256/260,
+ * 512, 2112/2116 and ray24. Fresh compiler witnesses with a smaller driver
+ * layout use1984 instead of2112 and are not this IFT runtime. */
+/* Admit the initial upper endpoint, then retain the current winner at ties.
+ * A missing committed hit must not turn t==tmax into an application miss. */
+nir_def *
+agx_nir_m5_should_import_triangle(nir_builder *b, nir_def *native_type,
+                                  nir_def *prior_type, nir_def *distance,
+                                  nir_def *best_t, nir_def *minimum)
+{
+   nir_def *before_best = nir_bcsel(b,
+      nir_ieq_imm(b, nir_ushr_imm(b, prior_type, 30), 0),
+      nir_fge(b, best_t, distance), nir_flt(b, distance, best_t));
+   return nir_iand(b, nir_ieq_imm(b, nir_ushr_imm(b, native_type, 30), 1),
+      nir_iand(b, before_best, nir_fge(b, distance, minimum)));
+}
+
+static void
+m5_import_native_opaque(nir_builder *b, nir_def *id, nir_def *driver,
+                        nir_def *lane, nir_deref_instr *result[5],
+                        nir_deref_instr *minimum, nir_deref_instr *flags,
+                        nir_def *completion_status)
+{
+   nir_push_if(b, m5_native_opaque_eligible(b, nir_load_deref(b, flags)));
+   /* Canonical native decoder byte-reencodes REGISTER_INDEXED selector3:
+    * type256 scalar marker1; geom260 scalar marker0. */
+   nir_def *hit = nir_vec2(b,
+      nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+         .base = 256, .component = 3, .flags = 1),
+      nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+         .base = 260, .component = 3));
+   nir_def *type_prim = nir_channel(b, hit, 0);
+   nir_def *distance = nir_load_scs_uniform_agx(b, 1, 32,
+      nir_load_preamble(b, 1, 64, .base = 152), nir_ishl_imm(b, id, 6),
+      .base = 24, .format = PIPE_FORMAT_R32_UINT, .component = 0);
+   nir_def *best = nir_channel(b, nir_load_deref(b, result[1]), 2);
+   /* Private benchmark transport: its 64-byte output record leaves words
+    * 1..10 unused. Capture the raw operands before interpreting type or t,
+    * keeping the public query result and flags unchanged. The caller pins
+    * the supplied allocation for the entire dispatch. */
+   const char *diag_text = getenv("ASAHI_M5_RQ_NATIVE_OPAQUE_DIAG_ADDRESS");
+   uint64_t diag_address = diag_text ? strtoull(diag_text, NULL, 0) : 0;
+   if (diag_address && b->shader->info.stage == MESA_SHADER_COMPUTE) {
+      nir_def *invocation = nir_channel(b, nir_load_global_invocation_id(b, 32), 0);
+      nir_def *output = nir_iadd_imm(b,
+         nir_ishl_imm(b, nir_u2u64(b, invocation), 6), diag_address);
+      if (debug_get_bool_option("ASAHI_M5_RQ_NATIVE_OPAQUE_DIAG_TAIL", false)) {
+         /* Raw root bytes32..63 expose both coordinate slots. Offset60 is
+          * left uninterpreted: the native union declares an i32 there. */
+         nir_def *base = nir_load_preamble(b, 1, 64, .base = 152);
+         nir_def *index = nir_ishl_imm(b, id, 6);
+         for (unsigned i = 0; i < 2; ++i) {
+            nir_def *tail = nir_load_scs_uniform_agx(b, 4, 32, base, index,
+               .base = 32 + 16 * i, .format = PIPE_FORMAT_R32_UINT,
+               .component = 0);
+            nir_store_global(b, tail, nir_iadd_imm(b, output, 4 + 16 * i),
+               .align_mul = 4, .write_mask = 15);
+         }
+         nir_store_global(b, nir_vec2(b, type_prim, distance),
+            nir_iadd_imm(b, output, 36), .align_mul = 4, .write_mask = 3);
+         nir_store_global(b, nir_u2u32(b, completion_status),
+            nir_iadd_imm(b, output, 48), .align_mul = 4, .write_mask = 1);
+         nir_def *control = nir_load_scs_uniform_agx(b, 1, 32, base, index,
+            .base = 28, .format = PIPE_FORMAT_R32_UINT, .component = 0);
+         nir_store_global(b, control, nir_iadd_imm(b, output, 56),
+            .align_mul = 4, .write_mask = 1);
+      } else if (debug_get_bool_option("ASAHI_M5_RQ_NATIVE_OPAQUE_DIAG_PACKET", false)) {
+         nir_def *base = nir_load_preamble(b, 1, 64, .base = 152);
+         nir_def *index = nir_ishl_imm(b, id, 6);
+         nir_def *ray[3];
+         for (unsigned i = 0; i < 3; ++i)
+            ray[i] = nir_load_scs_uniform_agx(b, 4, 32, base, index,
+               .base = 16 * i, .format = PIPE_FORMAT_R32_UINT, .component = 0);
+         nir_store_global(b, ray[0], nir_iadd_imm(b, output, 4),
+            .align_mul = 4, .write_mask = 15);
+         nir_store_global(b, ray[1], nir_iadd_imm(b, output, 20),
+            .align_mul = 4, .write_mask = 15);
+         nir_store_global(b, nir_channels(b, ray[2], 3),
+            nir_iadd_imm(b, output, 36), .align_mul = 4, .write_mask = 3);
+         nir_store_global(b, nir_channel(b, ray[2], 2),
+            nir_iadd_imm(b, output, 48), .align_mul = 4, .write_mask = 1);
+         nir_store_global(b, nir_channel(b, ray[2], 3),
+            nir_iadd_imm(b, output, 56), .align_mul = 4, .write_mask = 1);
+      } else {
+      nir_store_global(b, nir_vec4(b, type_prim, nir_channel(b, hit, 1),
+                                  distance, nir_u2u32(b, completion_status)),
+         nir_iadd_imm(b, output, 4), .align_mul = 4, .write_mask = 15);
+      nir_def *control = nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+         .base = 1728, .component = 2);
+      nir_store_global(b, nir_vec4(b, driver, id, lane, control),
+         nir_iadd_imm(b, output, 20), .align_mul = 4, .write_mask = 15);
+      nir_store_global(b, nir_vec2(b, best,
+         nir_b2i32(b, m5_native_opaque_eligible(b, nir_load_deref(b, flags)))),
+         nir_iadd_imm(b, output, 36), .align_mul = 4, .write_mask = 3);
+      }
+   }
+   /* AABB candidates are still returned to the application. Native triangle
+    * kind1 is the only packet auto-committed by this qualification path. */
+   nir_push_if(b, agx_nir_m5_should_import_triangle(b, type_prim,
+      nir_channel(b, nir_load_deref(b, result[2]), 3), distance, best,
+      nir_load_deref(b, minimum)));
+   nir_def *bary = nir_load_scs_indexed_agx(b, 2, 32, driver, lane,
+      .base = 512, .component = 3, .flags = 1);
+   nir_def *halves[3];
+   for (unsigned i = 0; i < 3; ++i)
+      halves[i] = nir_u2u32(b, nir_load_scs_register_agx(b, 1, 16,
+         nir_imul_imm(b, lane, 6), driver, .base = (m5_raw50_layout_probe() ? 8640 : 2112) + 2 * i,
+         .format = PIPE_FORMAT_R16_UINT, .component = 0));
+   nir_def *pointer = nir_pack_64_2x32_split(b,
+      nir_ior(b, halves[0], nir_ishl_imm(b, halves[1], 16)), halves[2]);
+   nir_def *metadata = nir_load_global(b, 2, 32,
+      nir_iadd_imm(b, pointer, 96), .align_mul = 4);
+   nir_def *zero = nir_imm_int(b, 0);
+   nir_def *packet[5] = {
+      nir_imm_ivec4(b, 1, 0, 0, 0),
+      nir_vec4(b, zero, zero, distance, zero),
+      nir_vec4(b, nir_channel(b, hit, 1), zero, zero, type_prim),
+      nir_vec4(b, zero, nir_channel(b, bary, 0),
+                   nir_channel(b, bary, 1), zero),
+      nir_vec4(b, nir_unpack_64_2x32_split_x(b, pointer),
+                   nir_unpack_64_2x32_split_y(b, pointer),
+                   nir_channel(b, metadata, 1), nir_channel(b, metadata, 0)),
+   };
+   for (unsigned i = 0; i < 5; ++i)
+      nir_store_deref(b, result[i], packet[i], 15);
+   nir_pop_if(b, NULL);
+   nir_pop_if(b, NULL);
+}
+
+/* Native Gen2 copy_committed_data latches query state1 when a completed
+ * batch contains candidates, even when control bit0 is clear. It then clears
+ * that bit; exhausting the batch resumes from the latched state1. Every
+ * activation/continuation resets cursor0 and keeps count until the next
+ * request, so count!=0 retains that state without another per-query variable.
+ * Preserve existing busy/control admission while qualifying full parity. */
+nir_def *
+agx_nir_m5_should_resume_batch(nir_builder *b, nir_def *status,
+                              nir_def *control, nir_def *count)
+{
+   return nir_ior(b, nir_ine_imm(b, count, 0),
+      nir_ior(b, nir_test_mask(b, status, 64), nir_test_mask(b, control, 1)));
+}
+
+/* Vulkan watertightness: a ray through an edge shared by two triangles, or
+ * through the shared vertex of a closed fan, must not report a double hit.
+ *
+ * M5's triangle test is inclusive. When its edge function is exactly zero it
+ * returns every triangle at that edge or vertex, each with a zero
+ * barycentric there, up to the hardware's barycentric rounding (~2^-24
+ * observed; diagonal edges can come back as a tiny nonzero u or v, and
+ * u + v as 1 +- an ulp). Its arithmetic is not exact:
+ * hardware probes of the closed-fan rays report both spoke neighbours for
+ * rays that exact arithmetic places strictly inside one triangle. The shader
+ * therefore cannot predict which candidates the hardware returns, and the
+ * native runtime's query.next does not filter them either.
+ *
+ * Observe instead: remember the distance of the last boundary triangle this
+ * query let through, and drop a later boundary triangle at the same distance
+ * (see M5_BOUNDARY_KEY_LOW). Both are then at the same point of the shared
+ * edge or vertex, so the ray still hits there exactly once. A drop only
+ * happens after its partner was delivered, so it never turns a hit into a
+ * miss.
+ *
+ * The key lives in the candidate cursor's spare bits: the cursor never
+ * exceeds the four-bit queue count. No memory access; it runs only for
+ * admitted triangles. It still adds about 35 instructions per proceed site
+ * and can cost occupancy in some shaders; it is on by default
+ * (ASAHI_M5_RQ_WATERTIGHT=0 disables it, see m5_rq_watertight). */
+static nir_def *
+m5_boundary_duplicate(nir_builder *b, nir_def *bary, nir_def *distance,
+                      nir_def *is_aabb, nir_deref_instr *cursor)
+{
+   nir_def *u = nir_channel(b, bary, 0), *v = nir_channel(b, bary, 1);
+   nir_def *boundary = nir_iand(b, nir_inot(b, is_aabb), nir_fle_imm(b,
+      nir_fmin(b, nir_fmin(b, u, v), nir_fsub_imm(b, 1.0f, nir_fadd(b, u, v))),
+      0x1p-22f));
+   nir_def *packed = nir_load_deref(b, cursor);
+   nir_def *key = nir_iand_imm(b, distance, ~M5_BOUNDARY_KEY_LOW);
+   /* Equal or adjacent keys, |key - last| <= one key step. */
+   nir_def *same_point = nir_ule_imm(b, nir_iadd_imm(b, nir_isub(b, key,
+      nir_iand_imm(b, packed, ~M5_BOUNDARY_KEY_LOW)),
+      M5_BOUNDARY_KEY_LOW + 1), 2 * (M5_BOUNDARY_KEY_LOW + 1));
+   nir_store_deref(b, cursor, nir_bcsel(b,
+      nir_iand(b, boundary, nir_inot(b, same_point)),
+      nir_ior(b, key, nir_iand_imm(b, packed, M5_CURSOR_INDEX)), packed), 1);
+   return nir_iand(b, boundary, same_point);
+}
+
+static void
+m5_candidate_next(nir_builder *b, nir_deref_instr *active,
+                  nir_deref_instr *saved_id, nir_deref_instr *cursor,
+                  nir_deref_instr *result[5], nir_deref_instr *minimum,
+                  nir_deref_instr *flags, nir_deref_instr *candidate[5],
+                  nir_deref_instr *initial_completion, nir_deref_instr *batch_completed,
+                  bool queue_first,
+                  nir_variable *more, nir_deref_instr *processed, nir_deref_instr *replay,
+                  nir_deref_instr *history, nir_deref_instr *prune_t)
+{
+   nir_def *id = nir_load_deref(b, saved_id);
+   nir_def *lane = nir_iand_imm(b, id, 31);
+   nir_def *driver = nir_iadd(b, nir_load_preamble(b, 1, 32, .base = 56),
+      nir_imul_imm(b, nir_iand_imm(b, id, ~31u), m5_candidate_driver_stride()));
+   nir_def *status_address = nir_iadd_imm(b,
+      nir_iadd(b, driver, nir_ishl_imm(b, lane, 3)), 256);
+
+   nir_variable *status_value = initial_completion
+      ? nir_local_variable_create(b->impl, glsl_uint16_t_type(), "rq_status")
+      : NULL;
+   nir_push_loop(b);
+   nir_def *status = NULL;
+   if (!queue_first) {
+      if (batch_completed)
+         nir_push_if(b, nir_inot(b, nir_load_deref(b, batch_completed)));
+      if (initial_completion) {
+         /* The native first read is indexed263/selector3/marker1/wait128.
+          * BASE7/selector4/marker0 is witnessed only after continue_ray.
+          * Keep the marked read's value live in the actual query decision. */
+         nir_push_if(b, nir_load_deref(b, initial_completion));
+         nir_store_var(b, status_value,
+            nir_load_scs_register_agx(b, 1, 16, driver, lane,
+               .base = 263, .format = PIPE_FORMAT_R8_UINT,
+               .component = 3, .flags = 9), 1);
+         nir_store_deref(b, initial_completion, nir_imm_false(b), 1);
+         nir_push_else(b, NULL);
+         nir_store_var(b, status_value,
+            nir_load_scs_base_agx(b, 1, 16, status_address,
+               .base = 7, .format = PIPE_FORMAT_R8_UINT,
+               .component = 4, .flags = 8), 1);
+         nir_pop_if(b, NULL);
+         status = nir_load_var(b, status_value);
+      } else {
+         status = nir_load_scs_base_agx(b, 1, 16, status_address,
+            .base = 7, .format = PIPE_FORMAT_R8_UINT, .component = 4, .flags = 8);
+      }
+      /* flags8 emits the captured traversal completion wait. Committed fields
+       * are not readable until it completes, even when the candidate count is
+       * zero and no application candidate will be produced. */
+      if (initial_completion)
+         m5_import_native_opaque(b, id, driver, lane, result, minimum, flags, status);
+      if (batch_completed) {
+         /* Gen2 next consumes a completed queue before reading completion
+          * again. Import its immutable opaque result once, before exposing
+          * this batch's first application candidate. No traversal runs while
+          * the singleton query consumes the rest of that completed batch. */
+         nir_store_deref(b, batch_completed, nir_imm_true(b), 1);
+         nir_pop_if(b, NULL);
+      }
+   }
+   nir_def *index = m5_cursor_index(b, cursor);
+   nir_def *control = nir_load_scs_indexed_agx(b, 1, 32, driver, lane,
+      .base = 1728, .component = 2);
+   nir_def *count = nir_iand_imm(b, nir_ushr_imm(b, control, 21), 15);
+   /* Private first-queue decode transport for the pinned standalone
+    * compute fixture. No additional completion wait or queue mutation. */
+   const char *diag_text = getenv("ASAHI_M5_RQ_ZERO_SPAN_DIAG_ADDRESS");
+   uint64_t diag_address = diag_text ? strtoull(diag_text, NULL, 0) : 0;
+   if (diag_address && b->shader->info.stage == MESA_SHADER_COMPUTE) {
+      nir_push_if(b, nir_ieq_imm(b, index, 0));
+      nir_def *invocation = nir_channel(b,
+         nir_load_global_invocation_id(b, 32), 0);
+      nir_def *output = nir_iadd_imm(b,
+         nir_ishl_imm(b, nir_u2u64(b, invocation), 6), diag_address);
+      nir_def *ray_tmax = nir_load_scs_uniform_agx(b, 1, 32,
+         nir_load_preamble(b, 1, 64, .base = 152),
+         nir_ishl_imm(b, id, 6), .base = 24,
+         .format = PIPE_FORMAT_R32_UINT, .component = 0);
+      nir_store_global(b, nir_vec4(b, control, count,
+         nir_u2u32(b, status), index), nir_iadd_imm(b, output, 12),
+         .align_mul = 4, .write_mask = 15);
+      nir_store_global(b, nir_vec4(b, nir_b2i32(b, nir_load_deref(b, active)),
+         ray_tmax, id, driver), nir_iadd_imm(b, output, 28),
+         .align_mul = 4, .write_mask = 15);
+      nir_pop_if(b, NULL);
+   }
+   nir_push_if(b, nir_ult(b, index, count));
+   /* Gen2 native next restarts on an owner mismatch, then pops the
+    * previously processed candidate prefix before exposing a new candidate.
+    * Private opt-in diagnostic: qualify the uncommitted replay path first. */
+   if (replay) {
+      nir_push_if(b, nir_ine_imm(b, nir_load_deref(b, replay), 0));
+      nir_def *replay_index = nir_isub(b, nir_load_deref(b, processed), nir_load_deref(b, replay));
+      nir_def *bit = nir_ishl(b, nir_imm_int(b, 1), replay_index);
+      nir_push_if(b, nir_iand(b, nir_ult_imm(b, replay_index, 32),
+         nir_ine_imm(b, nir_iand(b, nir_load_deref(b, history), bit), 0)));
+      struct agx_m5_ray_candidate old = agx_nir_m5_load_candidate(b, driver, lane, index);
+      nir_store_deref(b, prune_t, old.distance, 1);
+      nir_pop_if(b, NULL);
+      nir_store_deref(b, replay, nir_iadd_imm(b, nir_load_deref(b, replay), -1), 1);
+      m5_cursor_set_index(b, cursor, nir_iadd_imm(b, index, 1));
+      nir_push_else(b, NULL);
+      nir_store_deref(b, processed, nir_iadd_imm(b, nir_load_deref(b, processed), 1), 1);
+   }
+   nir_def *packet[5];
+   nir_def *ray_flags = nir_load_deref(b, flags);
+   nir_def *opaque, *is_aabb;
+   m5_candidate_packet(b, driver, id, index, packet, ray_flags, &opaque,
+                       &is_aabb);
+   m5_cursor_set_index(b, cursor, nir_iadd_imm(b, index, 1));
+   nir_def *distance = nir_channel(b, packet[1], 2);
+   nir_def *best_t = nir_channel(b, nir_load_deref(b, result[1]), 2);
+   /* A procedural candidate is always handed to the application: the
+    * shader computes its own intersection distance. */
+   nir_def *before_best = nir_fge(b, best_t, distance);
+   {
+      /* A native batch may already contain another triangle at the exact
+       * distance just committed. Gen2 query.next clips queued candidates
+       * strictly against its committed distance before exposing them.
+       * Preserve the inclusive initial API endpoint when no hit exists;
+       * ignored/unconfirmed candidates therefore do not suppress ties. */
+      nir_def *committed_kind = nir_ushr_imm(b,
+         nir_channel(b, nir_load_deref(b, result[2]), 3), 30);
+      before_best = nir_bcsel(b, nir_ieq_imm(b, committed_kind, 0),
+         before_best, nir_flt(b, distance, best_t));
+   }
+   nir_def *in_interval = nir_ior(b, is_aabb, nir_iand(b,
+      nir_fge(b, distance, nir_load_deref(b, minimum)), before_best));
+   /* RayFlagsCullOpaque/CullNoOpaque, then SpvRayFlagsSkipTrianglesKHR
+    * (0x100) and SpvRayFlagsSkipAABBsKHR (0x200) on the candidate kind. */
+   nir_def *culled = nir_bcsel(b, opaque,
+      nir_test_mask(b, ray_flags, 0x40), nir_test_mask(b, ray_flags, 0x80));
+   culled = nir_ior(b, culled, nir_bcsel(b, is_aabb,
+      nir_test_mask(b, ray_flags, 0x200), nir_test_mask(b, ray_flags, 0x100)));
+   nir_push_if(b, nir_iand(b, in_interval, nir_inot(b, culled)));
+   const bool watertight = m5_rq_watertight();
+   if (watertight)
+      nir_push_if(b, nir_inot(b, m5_boundary_duplicate(b,
+         nir_channels(b, packet[3], 6), distance, is_aabb, cursor)));
+   /* Opaque triangles commit without application involvement; an opaque
+    * AABB is still a candidate (rayQueryGetIntersectionCandidateAABBOpaqueEXT). */
+   nir_push_if(b, nir_iand(b, opaque, nir_inot(b, is_aabb)));
+   for (unsigned i = 0; i < 5; ++i)
+      nir_store_deref(b, result[i], packet[i], 15);
+   m5_replay_commit(b, processed, history, prune_t, distance);
+   nir_push_if(b, nir_test_mask(b, ray_flags, 4));
+   nir_store_deref(b, active, nir_imm_false(b), 1);
+   nir_jump(b, nir_jump_break);
+   nir_pop_if(b, NULL);
+   nir_push_else(b, NULL);
+   for (unsigned i = 0; i < 5; ++i)
+      nir_store_deref(b, candidate[i], packet[i], 15);
+   nir_store_var(b, more, nir_imm_true(b), 1);
+   nir_jump(b, nir_jump_break);
+   nir_pop_if(b, NULL);
+   if (watertight)
+      nir_pop_if(b, NULL);
+   nir_pop_if(b, NULL);
+   if (replay)
+      nir_pop_if(b, NULL);
+   /* This lowering runs after continue lowering. Keep queue exhaustion in
+    * the else branch so both paths reach the natural loop backedge; emitting
+    * nir_jump_continue here would reach a backend that only accepts breaks. */
+   nir_push_else(b, NULL);
+
+   if (queue_first || batch_completed)
+      /* Queue exhaustion needs the returned status, but a latched completed
+       * batch needs no new completion dependency. Continuation invalidates
+       * that latch and retains its real completion wait on the next loop. */
+      status = nir_load_scs_base_agx(b, 1, 16, status_address,
+         .base = 7, .format = PIPE_FORMAT_R8_UINT, .component = 4);
+   /* Native Gen2 state1 survives consumption of a nonempty completed queue,
+    * including the one-AABB control0x200008/status0 batch. Re-testing only the
+    * busy byte and cleared low control bit would lose later intersections. */
+   nir_def *resume = agx_nir_m5_should_resume_batch(b, status, control, count);
+   nir_push_if(b, nir_inot(b, resume));
+   nir_store_deref(b, active, nir_imm_false(b), 1);
+   nir_jump(b, nir_jump_break);
+   nir_pop_if(b, NULL);
+
+   m5_candidate_request_batch(b, id, driver, lane, status_address, control,
+      prune_t ? nir_load_deref(b, prune_t) : nir_channel(b, nir_load_deref(b, result[1]), 2));
+   if (batch_completed)
+      nir_store_deref(b, batch_completed, nir_imm_false(b), 1);
+   if (queue_first) {
+      /* Continuation starts a new asynchronous batch. Keep this completion
+       * wait even if its returned byte is unread; compiler DCE must retain
+       * the scs_rt_completion hardware side effect and real destination. */
+      nir_load_scs_base_agx(b, 1, 16, status_address,
+         .base = 7, .format = PIPE_FORMAT_R8_UINT,
+         .component = 4, .flags = 8);
+   }
+   m5_cursor_set_index(b, cursor, nir_imm_int(b, 0));
+   nir_pop_if(b, NULL);
+   nir_pop_loop(b, NULL);
+}
+
+bool
+agx_nir_m5_split_ray_query_arrays(nir_shader *shader)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+   bool progress = false;
+
+   nir_foreach_function_temp_variable_safe(var, impl) {
+      if (!var->data.ray_query || !glsl_type_is_array(var->type))
+         continue;
+      const struct glsl_type *elem = glsl_get_array_element(var->type);
+      const unsigned len = glsl_get_length(var->type);
+      if (glsl_type_is_array(elem) || len == 0)
+         continue;
+
+      /* A dynamically selected element of a small array is resolved into
+       * control flow before splitting, so each element becomes a separate
+       * query with its own copy of the operation (the qualified form for
+       * arrays of up to the native slot count). Larger arrays keep their
+       * array form: agx_nir_lower_m5_ray_query gives every element its own
+       * state and selects the native slot per use. */
+      nir_builder indirect = nir_builder_create(impl);
+      if (len <= AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES)
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr_safe(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr);
+            if (!nir_intrinsic_is_ray_query(op->intrinsic))
+               continue;
+            nir_deref_instr *d = nir_src_as_deref(op->src[0]);
+            if (d->deref_type != nir_deref_type_array ||
+                nir_src_is_const(d->arr.index))
+               continue;
+            nir_deref_instr *parent = nir_deref_instr_parent(d);
+            if (parent->deref_type != nir_deref_type_var || parent->var != var)
+               continue;
+            indirect.cursor = nir_before_instr(instr);
+            nir_def **values = nir_intrinsic_infos[op->intrinsic].has_dest ? calloc(len, sizeof(*values)) : NULL;
+            for (unsigned i = 0; i < len; ++i) {
+               nir_push_if(&indirect, nir_ieq_imm(&indirect, d->arr.index.ssa, i));
+               nir_deref_instr *selected = nir_build_deref_array_imm(&indirect, parent, i);
+               nir_intrinsic_instr *copy = nir_instr_as_intrinsic(nir_instr_clone(shader, instr));
+               copy->src[0] = nir_src_for_ssa(&selected->def);
+               nir_builder_instr_insert(&indirect, &copy->instr);
+               if (values)
+                  values[i] = &copy->def;
+               nir_push_else(&indirect, NULL);
+            }
+            nir_def *value = values ? nir_undef(&indirect, op->def.num_components, op->def.bit_size) : NULL;
+            for (unsigned i = len; i-- > 0;) {
+               nir_pop_if(&indirect, NULL);
+               if (values)
+                  value = nir_if_phi(&indirect, values[i], value);
+            }
+            if (value)
+               nir_def_rewrite_uses(&op->def, value);
+            nir_instr_remove(instr);
+            free(values);
+            progress = true;
+         }
+      }
+      nir_remove_dead_derefs_impl(impl);
+      /* Every use of the variable must be a constant, in-range element. */
+      bool constant_only = true;
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_deref)
+               continue;
+            nir_deref_instr *d = nir_instr_as_deref(instr);
+            if (d->deref_type != nir_deref_type_var || d->var != var)
+               continue;
+            nir_foreach_use(src, &d->def) {
+               nir_instr *use = nir_src_use_instr(src);
+               nir_deref_instr *a = use->type == nir_instr_type_deref
+                  ? nir_instr_as_deref(use) : NULL;
+               if (!a || a->deref_type != nir_deref_type_array ||
+                   !nir_src_is_const(a->arr.index) ||
+                   nir_src_as_uint(a->arr.index) >= len)
+                  constant_only = false;
+            }
+         }
+      }
+      if (!constant_only)
+         continue;
+
+      nir_variable **elems = calloc(len, sizeof(*elems));
+      for (unsigned i = 0; i < len; ++i) {
+         elems[i] = nir_local_variable_create(impl, elem, var->name);
+         elems[i]->data.ray_query = true;
+      }
+
+      nir_builder b = nir_builder_create(impl);
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr_safe(instr, block) {
+            if (instr->type != nir_instr_type_deref)
+               continue;
+            nir_deref_instr *a = nir_instr_as_deref(instr);
+            if (a->deref_type != nir_deref_type_array)
+               continue;
+            nir_deref_instr *parent = nir_deref_instr_parent(a);
+            if (!parent || parent->deref_type != nir_deref_type_var ||
+                parent->var != var)
+               continue;
+            b.cursor = nir_before_instr(&a->instr);
+            nir_deref_instr *split = nir_build_deref_var(
+               &b, elems[nir_src_as_uint(a->arr.index)]);
+            nir_def_rewrite_uses(&a->def, &split->def);
+            nir_instr_remove(&a->instr);
+         }
+      }
+      free(elems);
+      progress = true;
+   }
+
+   if (progress) {
+      nir_remove_dead_derefs_impl(impl);
+      nir_remove_dead_variables(shader, nir_var_function_temp, NULL);
+      nir_progress(true, impl, nir_metadata_control_flow);
+   } else {
+      nir_no_progress(impl);
+   }
+   return progress;
+}
+
+/* GLSL passes rayQueryEXT to functions as inout, so after inlining the callee
+ * works on a temporary query that is assigned from the caller's query on entry
+ * (and copied back on return). Ray queries are opaque objects: such a
+ * temporary is an alias of the caller's query, not a second live query.
+ * Rewrite every use of a whole-variable copy to the source variable and drop
+ * the copies, so a wrapper function does not look like two queries.
+ */
+static bool
+m5_rq_alias_copies(nir_function_impl *impl)
+{
+   bool progress = false;
+
+   for (bool again = true; again;) {
+      again = false;
+
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr_safe(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *store = nir_instr_as_intrinsic(instr);
+            if (store->intrinsic != nir_intrinsic_store_deref)
+               continue;
+
+            nir_deref_instr *dst = nir_src_as_deref(store->src[0]);
+            nir_intrinsic_instr *load = nir_src_as_intrinsic(store->src[1]);
+            if (dst->deref_type != nir_deref_type_var ||
+                !dst->var->data.ray_query || !load ||
+                load->intrinsic != nir_intrinsic_load_deref)
+               continue;
+
+            nir_deref_instr *src = nir_src_as_deref(load->src[0]);
+            if (src->deref_type != nir_deref_type_var ||
+                !src->var->data.ray_query)
+               continue;
+
+            /* Self copy, e.g. a copy-back after aliasing. */
+            if (src->var == dst->var) {
+               nir_instr_remove(&store->instr);
+               progress = again = true;
+               continue;
+            }
+
+            /* Only alias a temporary whose sole definition is this copy. */
+            unsigned stores = 0;
+            nir_foreach_block(b2, impl) {
+               nir_foreach_instr(i2, b2) {
+                  if (i2->type != nir_instr_type_intrinsic)
+                     continue;
+                  nir_intrinsic_instr *s2 = nir_instr_as_intrinsic(i2);
+                  if (s2->intrinsic != nir_intrinsic_store_deref &&
+                      s2->intrinsic != nir_intrinsic_copy_deref)
+                     continue;
+                  nir_deref_instr *d2 = nir_src_as_deref(s2->src[0]);
+                  if (nir_deref_instr_get_variable(d2) == dst->var)
+                     stores++;
+               }
+            }
+            if (stores != 1 || dst->var->data.mode != nir_var_function_temp)
+               continue;
+
+            nir_variable *alias = dst->var, *target = src->var;
+            nir_foreach_block(b2, impl) {
+               nir_foreach_instr_safe(i2, b2) {
+                  if (i2->type != nir_instr_type_deref)
+                     continue;
+                  nir_deref_instr *d = nir_instr_as_deref(i2);
+                  if (d->deref_type != nir_deref_type_var || d->var != alias)
+                     continue;
+                  d->var = target;
+                  d->modes = target->data.mode;
+               }
+            }
+            nir_instr_remove(&store->instr);
+            progress = again = true;
+         }
+      }
+   }
+
+   if (progress)
+      nir_progress(true, impl, nir_metadata_control_flow);
+   return progress;
+}
+
+/* A dynamically selected element's state is staged through per-operation
+ * scalar shadows: each field is copied in once before the operation and the
+ * fields the operation writes are copied back once after it, so the element
+ * selection costs one select tree per field rather than one per access. */
+struct m5_rq_shadows {
+   nir_function_impl *impl;
+   nir_deref_instr *query;
+   unsigned count;
+   struct {
+      nir_variable *field;
+      nir_variable *shadow;
+   } entries[48];
+};
+
+static nir_deref_instr *
+m5_rq_op_field(nir_builder *b, struct m5_rq_shadows *shadows,
+               nir_variable *field)
+{
+   nir_deref_instr *query = shadows->query;
+   nir_deref_instr *elem = m5_rq_field_deref(b, query, field);
+   if (query->deref_type != nir_deref_type_array ||
+       nir_src_is_const(query->arr.index))
+      return elem;
+   assert(shadows->count < ARRAY_SIZE(shadows->entries));
+   nir_variable *shadow =
+      nir_local_variable_create(shadows->impl, elem->type, field->name);
+   nir_deref_instr *staged = nir_build_deref_var(b, shadow);
+   nir_copy_deref(b, staged, elem);
+   shadows->entries[shadows->count].field = field;
+   shadows->entries[shadows->count].shadow = shadow;
+   shadows->count++;
+   return staged;
+}
+
+/* Whether a lowered operation may write a state field. */
+static bool
+m5_rq_op_writes(nir_intrinsic_op op, const struct m5_rq_state *state,
+                nir_variable *field)
+{
+   bool result = false, candidate = false;
+   for (unsigned i = 0; i < 5; ++i) {
+      result |= field == state->result[i];
+      candidate |= field == state->candidate[i];
+   }
+   switch (op) {
+   case nir_intrinsic_rq_initialize:
+      return true;
+   case nir_intrinsic_rq_proceed:
+      return field == state->active || field == state->started ||
+             field == state->owner || field == state->resident_id ||
+             field == state->cursor || result || candidate ||
+             field == state->batch_completed ||
+             field == state->initial_completion ||
+             field == state->processed || field == state->replay ||
+             field == state->commit_history || field == state->prune_t ||
+             field == state->capture_event;
+   case nir_intrinsic_rq_terminate:
+      return field == state->active;
+   case nir_intrinsic_rq_confirm_intersection:
+   case nir_intrinsic_rq_generate_intersection:
+      return field == state->active || result ||
+             field == state->commit_history || field == state->prune_t;
+   default:
+      return false;
+   }
+}
+
+static void
+m5_rq_op_writeback(nir_builder *b, struct m5_rq_shadows *shadows,
+                   nir_intrinsic_op op, const struct m5_rq_state *state)
+{
+   for (unsigned i = 0; i < shadows->count; ++i) {
+      if (!m5_rq_op_writes(op, state, shadows->entries[i].field))
+         continue;
+      nir_copy_deref(b,
+         m5_rq_field_deref(b, shadows->query, shadows->entries[i].field),
+         nir_build_deref_var(b, shadows->entries[i].shadow));
+   }
+}
+
+/* Per-query state: one value per logical query, so an array query gets an
+ * array of the field type. The state of a dynamically selected element is
+ * resolved to selects over the element states after lowering
+ * (nir_lower_indirect_derefs_to_if_else_trees over shader temporaries), so
+ * query state never reaches thread-private memory. */
+static nir_variable *
+m5_rq_var(nir_shader *shader, nir_function_impl *impl,
+          const struct m5_rq_state *state, const struct glsl_type *type,
+          const char *name)
+{
+   const struct glsl_type *field = m5_rq_field_type(state->query->type, type);
+   if (state->dynamic)
+      return nir_variable_create(shader, nir_var_shader_temp, field, name);
+   return nir_local_variable_create(impl, field, name);
+}
+
+bool
+agx_nir_lower_m5_ray_query(nir_shader *shader,
+                           const struct agx_m5_ray_query_layout *layout)
+{
+   const bool kernel = layout && layout->compute_kernel &&
+      (shader->info.stage == MESA_SHADER_TESS_CTRL ||
+       shader->info.stage == MESA_SHADER_TESS_EVAL ||
+       shader->info.stage == MESA_SHADER_GEOMETRY ||
+       shader->info.stage == MESA_SHADER_VERTEX ||
+       shader->info.stage == MESA_SHADER_FRAGMENT);
+   if (!layout || (!kernel && shader->info.stage != MESA_SHADER_COMPUTE) ||
+       (!kernel && shader->info.workgroup_size_variable) ||
+       layout->payload_uniform > 508 ||
+       (layout->payload_uniform & 1) || layout->bvh_uniform > 508 ||
+       (layout->bvh_uniform & 1) || layout->status_uniform > 510 ||
+       (layout->status_uniform & 1) || layout->context_uniform > 508 ||
+       (layout->context_uniform & 1) || layout->as_table_uniform > 508 ||
+       (layout->as_table_uniform & 1) ||
+       (!!layout->context_uniform != !!layout->as_table_uniform))
+      return false;
+
+   if (debug_get_bool_option("ASAHI_M5_RQ_DIAG", false))
+      nir_print_shader(shader, stderr);
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+   if (m5_rq_alias_copies(impl))
+      nir_remove_dead_variables(shader, nir_var_function_temp, NULL);
+   struct util_dynarray queries = UTIL_DYNARRAY_INIT;
+   struct util_dynarray ops = UTIL_DYNARRAY_INIT;
+   /* Logical queries: scalar query variables plus every element of a query
+    * array. Native slots are assigned per logical query by m5_rq_slot(). */
+   unsigned logical_total = 0;
+   nir_foreach_function(f, shader) {
+      if (!f->impl)
+         continue;
+      nir_foreach_block(block, f->impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+            if (!nir_intrinsic_is_ray_query(intr->intrinsic))
+               continue;
+            nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
+            if (f->impl != impl || !deref)
+               do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+            /* A query is a scalar variable or one element of a
+             * one-dimensional query array. */
+            nir_deref_instr *root = deref;
+            if (root->deref_type == nir_deref_type_array)
+               root = nir_deref_instr_parent(root);
+            if (root->deref_type != nir_deref_type_var ||
+                !root->var->data.ray_query ||
+                (glsl_type_is_array(root->var->type) &&
+                 (glsl_type_is_array(glsl_get_array_element(root->var->type)) ||
+                  glsl_get_length(root->var->type) == 0)) ||
+                (deref != root && !glsl_type_is_array(root->var->type)))
+               do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+            struct m5_rq_state *owner_state = NULL;
+            util_dynarray_foreach(&queries, struct m5_rq_state, state) {
+               if (state->query == root->var)
+                  owner_state = state;
+            }
+            if (!owner_state) {
+               struct m5_rq_state state = {
+                  .query = root->var,
+                  .first_logical = logical_total,
+                  .logical_count = glsl_type_is_array(root->var->type)
+                     ? glsl_get_length(root->var->type) : 1,
+               };
+               logical_total += state.logical_count;
+               util_dynarray_append(&queries, state);
+               owner_state = util_dynarray_top_ptr(&queries, struct m5_rq_state);
+            }
+            if (deref != root) {
+               if (!nir_src_is_const(deref->arr.index))
+                  owner_state->dynamic = true;
+               else if (nir_src_as_uint(deref->arr.index) >=
+                        owner_state->logical_count)
+                  do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+            }
+            switch (intr->intrinsic) {
+            case nir_intrinsic_rq_initialize:
+               /* Candidate policy is evaluated per invocation, so valid
+                * dynamically selected Vulkan flags need no specialization.
+                * Retain the narrower normal-path contract until qualified. */
+               if ((!m5_candidate_api() && !nir_src_is_const(intr->src[2])) ||
+                   (nir_src_is_const(intr->src[2]) &&
+                    ((nir_src_as_uint(intr->src[2]) &
+                      ~(m5_candidate_api() ? 0x3ffu : 0x3du)) ||
+                     ((nir_src_as_uint(intr->src[2]) & 0x30u) == 0x30u) ||
+                     ((nir_src_as_uint(intr->src[2]) & 3u) == 3u))) ||
+                   intr->src[1].ssa->bit_size != 64)
+                  do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+               break;
+            case nir_intrinsic_rq_load:
+               if (nir_def_is_unused(&intr->def))
+                  break;
+               if (!m5_rq_supported_load(intr) ||
+                   (!layout->instance_traversal &&
+                    (nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_instance_id ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_instance_custom_index ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_front_face ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_instance_sbt_index ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_object_ray_origin ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_object_ray_direction ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_object_to_world ||
+                     nir_intrinsic_ray_query_value(intr) == nir_ray_query_value_intersection_world_to_object)))
+                  do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+               break;
+            case nir_intrinsic_rq_proceed:
+            case nir_intrinsic_rq_terminate:
+               break;
+            case nir_intrinsic_rq_confirm_intersection:
+            case nir_intrinsic_rq_generate_intersection:
+               if (!m5_candidate_api())
+                  do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+               break;
+            default:
+               do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+            }
+            util_dynarray_append(&ops, intr);
+         }
+      }
+   }
+   if (!util_dynarray_num_elements(&queries, struct m5_rq_state))
+      do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+   if (m5_candidate_api() && !layout->instance_traversal)
+      do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+   /* The ordinary path keeps one native slot per query variable. */
+   if (!m5_candidate_api() &&
+       util_dynarray_num_elements(&queries, struct m5_rq_state) >
+          AGX3_RT_M5_MAX_QUALIFIED_LIVE_QUERIES)
+      do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+   if (m5_candidate_api() &&
+       util_dynarray_num_elements(&queries, struct m5_rq_state) > 1 &&
+       debug_get_bool_option("ASAHI_M5_RQ_INLINE_PACKET", false))
+      do { if (debug_get_bool_option("ASAHI_M5_RQ_REFUSAL_DIAG", false)) fprintf(stderr, "M5-RQ refusal line%d queries=%zu\n", __LINE__, util_dynarray_num_elements(&queries, struct m5_rq_state)); goto unsupported; } while (0);
+
+   const unsigned query_count =
+      util_dynarray_num_elements(&queries, struct m5_rq_state);
+   /* Remove native-only state when every initializer of this query has a
+    * constant policy that requires candidate traversal. Unknown or mixed
+    * policies retain the native path; multiple owners remain ineligible. */
+   if (m5_native_opaque() && shader->info.stage == MESA_SHADER_COMPUTE &&
+       query_count == 1 && logical_total == 1 &&
+       shader->info.ray_queries == 1) {
+      util_dynarray_foreach(&queries, struct m5_rq_state, state) {
+         util_dynarray_foreach(&ops, nir_intrinsic_instr *, op) {
+            nir_intrinsic_instr *intr = *op;
+            if (intr->intrinsic != nir_intrinsic_rq_initialize ||
+                nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0])) != state->query)
+               continue;
+            state->native_opaque_possible |= !nir_src_is_const(intr->src[2]) ||
+               !(nir_src_as_uint(intr->src[2]) & (2 | 4 | 0x40 | 0x100));
+         }
+      }
+   }
+   if (debug_get_bool_option("ASAHI_M5_RQ_CODEGEN_DIAG", false))
+      fprintf(stderr, "M5-RQ codegen queries=%u logical=%u replay=%u\n",
+              query_count, logical_total, logical_total > 1);
+
+   /* The logical query whose traversal is resident in the shared native
+    * payload. Logical queries can share a native slot, so residency is
+    * tracked here as well as through the published owner token. */
+   nir_variable *resident_logical = NULL;
+   if (m5_candidate_api() && logical_total > 1) {
+      resident_logical = nir_local_variable_create(impl, glsl_uint_type(),
+                                                   "rq_resident_logical");
+      nir_builder init = nir_builder_at(nir_before_impl(impl));
+      nir_store_var(&init, resident_logical, nir_imm_int(&init, ~0u), 1);
+   }
+   bool any_dynamic = false;
+   util_dynarray_foreach(&queries, struct m5_rq_state, state) {
+      any_dynamic |= state->dynamic;
+      state->as = m5_rq_var(shader, impl, state, glsl_uint64_t_type(), "rq_as");
+      state->origin = m5_rq_var(shader, impl, state,
+         glsl_vector_type(GLSL_TYPE_FLOAT, 3), "rq_origin");
+      state->direction = m5_rq_var(shader, impl, state,
+         glsl_vector_type(GLSL_TYPE_FLOAT, 3), "rq_direction");
+      state->tmin = m5_rq_var(shader, impl, state, glsl_float_type(), "rq_tmin");
+      state->tmax = m5_rq_var(shader, impl, state, glsl_float_type(), "rq_tmax");
+      state->mask = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_mask");
+      state->flags = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_flags");
+      state->active = m5_rq_var(shader, impl, state, glsl_bool_type(), "rq_active");
+      for (unsigned i = 0; i < 5; ++i)
+         state->result[i] = m5_rq_var(shader, impl, state, glsl_uvec4_type(),
+                                      "rq_packet");
+      if (m5_candidate_api()) {
+         state->started = m5_rq_var(shader, impl, state, glsl_bool_type(), "rq_started");
+         state->owner = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_owner");
+         state->resident_id = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_resident");
+         state->cursor = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_cursor");
+         /* Native next consumes same-owner completed queue entries before
+          * reading completion. Activation and continuation finish their batch
+          * before queue access; only queue exhaustion reads its status. Multiple owners and the
+          * unqualified native opaque import keep the ordinary path. */
+         if (logical_total == 1 && !state->native_opaque_possible &&
+             debug_get_bool_option("ASAHI_M5_RQ_QUEUE_FIRST_PROBE", false))
+            state->queue_first =
+               !getenv("ASAHI_M5_RQ_ZERO_SPAN_DIAG_ADDRESS");
+         /* Each singleton compute query owns its completed candidate batch.
+          * Owner switching, graphics layouts and the raw-status recorder keep
+          * their existing completion/import path. */
+         if (logical_total == 1 && shader->info.stage == MESA_SHADER_COMPUTE &&
+             !state->queue_first && !getenv("ASAHI_M5_RQ_ZERO_SPAN_DIAG_ADDRESS"))
+            state->batch_completed = m5_rq_var(shader, impl, state,
+               glsl_bool_type(), "rq_batch_completed");
+         if (state->native_opaque_possible)
+            state->initial_completion = m5_rq_var(shader, impl, state,
+               glsl_bool_type(), "rq_initial_completion");
+         /* A single logical query cannot lose its owner to another query.
+          * Reinitialization clears traversal state, so there is no candidate
+          * prefix to restore. Keep replay state only for multiple owners. */
+         if (logical_total > 1) {
+            state->processed = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_processed");
+            state->replay = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_replay");
+            state->commit_history = m5_rq_var(shader, impl, state, glsl_uint_type(), "rq_commit_history");
+            state->prune_t = m5_rq_var(shader, impl, state, glsl_float_type(), "rq_prune_t");
+         }
+         if (m5_model_capture_address())
+            state->capture_event = m5_rq_var(shader, impl, state,
+               glsl_uint_type(), "rq_capture_event");
+         for (unsigned i = 0; i < 5; ++i)
+            state->candidate[i] = m5_rq_var(shader, impl, state,
+               glsl_uvec4_type(), "rq_candidate");
+      }
+   }
+
+   nir_builder b = nir_builder_create(impl);
+   util_dynarray_foreach(&ops, nir_intrinsic_instr *, op) {
+      nir_intrinsic_instr *intr = *op;
+      if (intr->intrinsic == nir_intrinsic_rq_load &&
+          nir_def_is_unused(&intr->def)) {
+         nir_instr_remove(&intr->instr);
+         continue;
+      }
+      b.cursor = nir_before_instr(&intr->instr);
+      nir_deref_instr *query = nir_src_as_deref(intr->src[0]);
+      struct m5_rq_shadows shadows = {.impl = impl, .query = query};
+      struct m5_rq_state *state = NULL;
+      util_dynarray_foreach(&queries, struct m5_rq_state, candidate) {
+         if (candidate->query == nir_deref_instr_get_variable(query)) {
+            state = candidate;
+            break;
+         }
+      }
+      assert(state);
+      nir_deref_instr *as = m5_rq_op_field(&b, &shadows, state->as);
+      nir_deref_instr *origin = m5_rq_op_field(&b, &shadows, state->origin);
+      nir_deref_instr *direction = m5_rq_op_field(&b, &shadows, state->direction);
+      nir_deref_instr *tmin = m5_rq_op_field(&b, &shadows, state->tmin);
+      nir_deref_instr *tmax = m5_rq_op_field(&b, &shadows, state->tmax);
+      nir_deref_instr *mask = m5_rq_op_field(&b, &shadows, state->mask);
+      nir_deref_instr *active = m5_rq_op_field(&b, &shadows, state->active);
+      nir_deref_instr *result[5];
+      for (unsigned i = 0; i < 5; ++i)
+         result[i] = m5_rq_op_field(&b, &shadows, state->result[i]);
+      nir_deref_instr *started = NULL, *resident_id = NULL, *cursor = NULL, *owner = NULL;
+      nir_deref_instr *batch_completed = state->batch_completed
+         ? m5_rq_op_field(&b, &shadows, state->batch_completed) : NULL;
+      nir_deref_instr *candidate[5] = {0};
+      nir_deref_instr *initial_completion = state->initial_completion
+         ? m5_rq_op_field(&b, &shadows, state->initial_completion) : NULL;
+      nir_deref_instr *capture_event = state->capture_event
+         ? m5_rq_op_field(&b, &shadows, state->capture_event) : NULL;
+      if (m5_candidate_api()) {
+         started = m5_rq_op_field(&b, &shadows, state->started);
+         owner = m5_rq_op_field(&b, &shadows, state->owner);
+         resident_id = m5_rq_op_field(&b, &shadows, state->resident_id);
+         cursor = m5_rq_op_field(&b, &shadows, state->cursor);
+         for (unsigned i = 0; i < 5; ++i)
+            candidate[i] = m5_rq_op_field(&b, &shadows, state->candidate[i]);
+      }
+      nir_deref_instr *processed = state->processed ? m5_rq_op_field(&b, &shadows, state->processed) : NULL;
+      nir_deref_instr *replay = state->replay ? m5_rq_op_field(&b, &shadows, state->replay) : NULL;
+      nir_deref_instr *history = state->commit_history ? m5_rq_op_field(&b, &shadows, state->commit_history) : NULL;
+      nir_deref_instr *prune_t = state->prune_t ? m5_rq_op_field(&b, &shadows, state->prune_t) : NULL;
+      nir_deref_instr *flags = m5_rq_op_field(&b, &shadows, state->flags);
+      nir_def *value = NULL;
+      switch (intr->intrinsic) {
+      case nir_intrinsic_rq_initialize:
+         if (m5_candidate_api()) {
+            nir_store_deref(&b, started, nir_imm_false(&b), 1);
+            if (batch_completed)
+               nir_store_deref(&b, batch_completed, nir_imm_false(&b), 1);
+            if (capture_event)
+               nir_store_deref(&b, capture_event, nir_imm_int(&b, 0), 1);
+            /* Cursor index 0; with m5_rq_watertight also a NaN boundary
+             * distance key, which matches no candidate distance. */
+            nir_store_deref(&b, cursor,
+               nir_imm_int(&b, m5_rq_watertight() ? 0x7ffffff0 : 0), 1);
+            if (processed) {
+               nir_store_deref(&b, processed, nir_imm_int(&b, 0), 1);
+               nir_store_deref(&b, replay, nir_imm_int(&b, 0), 1);
+               nir_store_deref(&b, history, nir_imm_int(&b, 0), 1);
+               nir_store_deref(&b, prune_t, intr->src[7].ssa, 1);
+            }
+            for (unsigned i = 0; i < 5; ++i)
+               nir_store_deref(&b, candidate[i], nir_imm_ivec4(&b, 0, 0, 0, 0), 15);
+         }
+         nir_store_deref(&b, flags, intr->src[2].ssa, 1);
+         nir_store_deref(&b, as, intr->src[1].ssa, 1);
+         nir_store_deref(&b, origin, intr->src[4].ssa, 7);
+         nir_store_deref(&b, direction, intr->src[6].ssa, 7);
+         nir_store_deref(&b, tmin, intr->src[5].ssa, 1);
+         nir_store_deref(&b, tmax, intr->src[7].ssa, 1);
+         /* Only the low eight cull-mask bits take part in traversal. */
+         nir_store_deref(&b, mask, nir_iand_imm(&b, intr->src[3].ssa, 0xff), 1);
+         nir_def *has_as = nir_ine_imm(&b, intr->src[1].ssa, 0);
+         if (m5_candidate_api()) {
+            /* SkipTriangles (0x100) / SkipAABBs (0x200) are applied per
+             * candidate kind in m5_candidate_next; the candidate profile
+             * forces non-opaque traversal (control 0x402), so every triangle
+             * reaches that filter. Metal's assume_geometry_type compiles to
+             * byte-identical code for every mask (witness-geom), so no packet
+             * level geometry filter is known. */
+         } else {
+            /* The opaque profile commits triangles in hardware and carries
+             * triangle leaves only: SkipTriangles makes it empty. */
+            has_as = nir_iand(&b, has_as,
+               nir_inot(&b, nir_test_mask(&b, intr->src[2].ssa, 0x100)));
+         }
+         nir_store_deref(&b, active, has_as, 1);
+         if (layout->context_uniform) {
+            /* The driver-owned empty header clears readiness; ordinary
+             * native envelopes publish 1 at AS byte0xa0. Do this before any
+             * query entry, packet publication, or intersector operation. */
+            nir_push_if(&b, has_as);
+            nir_def *ready = nir_load_global(&b, 1, 32,
+               nir_iadd_imm(&b, intr->src[1].ssa, 0xa0), .align_mul = 4);
+            nir_store_deref(&b, active, nir_ine_imm(&b, ready, 0), 1);
+            nir_pop_if(&b, NULL);
+         }
+         for (unsigned i = 0; i < 5; ++i)
+            nir_store_deref(&b, result[i], nir_imm_ivec4(&b, 0, 0, 0, 0), 15);
+         /* Committed distance is initialized to tmax before traversal. */
+         nir_store_deref(&b, result[1],
+                       nir_vec4(&b, nir_imm_int(&b, 0), nir_imm_int(&b, 0),
+                                intr->src[7].ssa, nir_imm_int(&b, 0)),
+                       15);
+         break;
+      case nir_intrinsic_rq_proceed: {
+         nir_variable *more = NULL;
+         if (m5_candidate_api()) {
+            more = nir_local_variable_create(impl, glsl_bool_type(), "rq_more");
+            nir_store_var(&b, more, nir_imm_false(&b), 1);
+         }
+         const bool fresh_query = m5_candidate_mode() &&
+            !debug_get_bool_option("ASAHI_M5_RQ_INLINE_PACKET", false);
+         nir_push_if(&b, nir_load_deref(&b, active));
+         if (m5_candidate_api()) {
+            nir_push_if(&b, nir_inot(&b, nir_load_deref(&b, started)));
+            nir_store_deref(&b, owner,
+               fresh_query ? m5_rq_enter(&b, state, query)
+                           : nir_imm_int(&b, 0), 1);
+            nir_store_deref(&b, resident_id, nir_load_scs_thread_index_agx(&b), 1);
+            nir_pop_if(&b, NULL);
+            m5_model_capture(&b, state, capture_event, owner, resident_id, cursor, active,
+                             result[1], 1);
+            nir_def *activate = nir_inot(&b, nir_load_deref(&b, started));
+            nir_def *logical = NULL;
+            if (logical_total > 1) {
+               unsigned constant_logical;
+               logical = m5_rq_logical(&b, state, query, &constant_logical);
+               if (!logical)
+                  logical = nir_imm_int(&b, constant_logical);
+               /* Another logical query took the payload since this one
+                * last traversed, possibly through the same native slot. */
+               activate = nir_ior(&b, activate,
+                  nir_ine(&b, nir_load_var(&b, resident_logical), logical));
+               nir_def *saved_id = nir_load_deref(&b, resident_id);
+               nir_def *driver = nir_iadd(&b,
+                  nir_load_preamble(&b, 1, 32, .base = 56),
+                  nir_imul_imm(&b, nir_iand_imm(&b, saved_id, ~31u), m5_candidate_driver_stride()));
+               nir_def *control = nir_load_scs_indexed_agx(&b, 1, 32, driver,
+                  nir_iand_imm(&b, saved_id, 31), .base = 1728, .component = 2);
+               nir_def *resident_owner = nir_iand_imm(&b,
+                  nir_ushr_imm(&b, control, 1), 0xfffff);
+               activate = nir_ior(&b, activate, nir_ine(&b, resident_owner,
+                  nir_iand_imm(&b, nir_load_deref(&b, owner), 0xfffff)));
+            }
+            /* Native next reactivates a query through intersect_ray when the
+             * shared payload belongs to a different query. Its token selects
+             * the saved traversal checkpoint; each logical query keeps its
+             * committed/candidate data in separate SSA variables. The single
+             * query path needs no ownership load or extra activation branch. */
+            nir_push_if(&b, activate);
+            if (resident_logical)
+               nir_store_var(&b, resident_logical, logical, 1);
+            if (replay) {
+               nir_store_deref(&b, replay, nir_load_deref(&b, processed), 1);
+               nir_store_deref(&b, prune_t, nir_load_deref(&b, tmax), 1);
+            }
+         }
+         nir_def *address = nir_load_deref(&b, as);
+         nir_def *selector = nir_load_preamble(&b, 1, 64, .base = layout->bvh_uniform);
+         if (layout->context_uniform) {
+            nir_def *context = nir_load_preamble(&b, 1, 64,
+                                                  .base = layout->context_uniform);
+            for (unsigned i = 0; i < (layout->instance_traversal ? 8 : 6); ++i) {
+               nir_def *words = nir_load_global(&b, 4, 32,
+                  nir_iadd_imm(&b, context, 16 * i), .align_mul = 16);
+               nir_store_scs_agx(&b, words, .base = 0x6000 + 16 * i);
+            }
+         }
+         if (layout->instance_traversal) {
+            selector = nir_u2u64(&b, nir_load_global(&b, 1, 32,
+               nir_iadd_imm(&b, address, 0xb0), .align_mul = 4));
+         } else if (layout->as_table_uniform) {
+            nir_def *entry = nir_iadd_imm(&b, nir_load_scs_thread_index_agx(&b), 1);
+            nir_def *table = nir_load_preamble(&b, 1, 64,
+                                                .base = layout->as_table_uniform);
+            nir_store_global(&b, address, nir_iadd(&b, table,
+               nir_u2u64(&b, nir_ishl_imm(&b, entry, 4))),
+               .align_mul = 16, .write_mask = 1);
+            nir_barrier(&b, .execution_scope = SCOPE_INVOCATION,
+               .memory_scope = SCOPE_DEVICE,
+               .memory_semantics = NIR_MEMORY_RELEASE,
+               .memory_modes = nir_var_mem_global);
+            selector = nir_u2u64(&b, entry);
+         }
+         /* Entry produces a real GPR value. Tracking it through SSA/RA also
+          * makes any SR59 value live across entry safe from an implicit r0
+          * clobber; no architectural change to SR59 is assumed. */
+         nir_def *query_owner = m5_candidate_api()
+            ? nir_load_deref(&b, owner)
+            : fresh_query ? nir_enter_query_agx(&b, .base = 4)
+                          : nir_imm_int(&b, 0);
+         nir_def *lo = nir_load_global(
+            &b, 3, 32, nir_iadd_imm(&b, address, 0x50), .align_mul = 16);
+         nir_def *hi = nir_load_global(
+            &b, 3, 32, nir_iadd_imm(&b, address, 0x5c), .align_mul = 4);
+         nir_def *base =
+            nir_load_preamble(&b, 1, 64, .base = layout->payload_uniform);
+         nir_def *id = m5_candidate_api() ? nir_load_deref(&b, resident_id)
+                                           : nir_load_scs_thread_index_agx(&b);
+         nir_def *index = nir_ishl_imm(&b, id, 6);
+         nir_def *ray_flags = nir_load_deref(&b, flags);
+         nir_def *opaque = nir_test_mask(&b, ray_flags, 1);
+         nir_def *first_hit = nir_test_mask(&b, ray_flags, 4);
+         nir_def *query_control = nir_bcsel(&b, opaque,
+            nir_imm_int(&b, 0x122), nir_imm_int(&b, 0x120));
+         if (fresh_query)
+            base = nir_load_preamble(&b, 1, 64, .base = 152);
+         nir_def *native_opaque_eligible = state->native_opaque_possible
+            ? m5_native_opaque_eligible(&b, ray_flags) : nir_imm_false(&b);
+         m5_init_triangle_packet(
+            &b, base, index, nir_load_deref(&b, origin),
+            nir_load_deref(&b, direction), nir_load_deref(&b, tmin),
+            prune_t ? nir_load_deref(&b, tmax) : m5_candidate_api() ? nir_channel(&b, nir_load_deref(&b, result[1]), 2)
+                               : nir_load_deref(&b, tmax),
+            lo, hi, nir_load_deref(&b, mask),
+            layout->instance_traversal ? nir_imm_int(&b, 0) : query_control,
+            opaque, m5_candidate_api() ? nir_imm_false(&b) : first_hit,
+            nir_ishl_imm(&b, nir_iand_imm(&b, ray_flags, 0x30), 4),
+            native_opaque_eligible);
+         nir_def *instance_type = NULL, *instance_bary = NULL, *instance_metadata = NULL, *instance_face = NULL;
+         nir_def *candidate_distance = NULL;
+         if (layout->instance_traversal)
+            m5_trace_instance(&b, selector, id, opaque, query_owner,
+                               nir_load_deref(&b, tmin), native_opaque_eligible, &instance_type, &instance_bary, &instance_metadata, &instance_face,
+                               &candidate_distance);
+         else {
+            agx_nir_m5_trace_opaque(
+               &b, nir_imm_int64(&b, 0), selector, base, index,
+               nir_load_preamble(&b, 1, 32, .base = layout->status_uniform),
+               nir_vec3(&b, nir_imm_int(&b, layout->launch_header),
+                        nir_imm_int(&b, 0), id),
+               NULL, NULL);
+         }
+         if (m5_candidate_api()) {
+            if (initial_completion)
+               nir_store_deref(&b, initial_completion,
+                  m5_defer_completion(&b) ? nir_imm_true(&b) : native_opaque_eligible, 1);
+            nir_store_deref(&b, resident_id, id, 1);
+            nir_store_deref(&b, started, nir_imm_true(&b), 1);
+            m5_cursor_set_index(&b, cursor, nir_imm_int(&b, 0));
+            nir_pop_if(&b, NULL);
+            m5_candidate_next(&b, active, resident_id, cursor, result,
+                              tmin, flags, candidate, initial_completion, batch_completed, state->queue_first, more, processed, replay, history, prune_t);
+            m5_model_capture(&b, state, capture_event, owner, resident_id, cursor, active,
+                             result[1], 2);
+            nir_pop_if(&b, NULL);
+            value = nir_load_var(&b, more);
+            break;
+         }
+         for (unsigned i = 0; i < 4; ++i)
+            nir_store_deref(&b, result[i],
+                          nir_load_scs_uniform_agx(
+                             &b, 4, 32, base, index, .base = 16 * i,
+                             .format = PIPE_FORMAT_R32_UINT, .component = 0),
+                          15);
+         if (layout->instance_traversal) {
+            if (candidate_distance)
+               nir_store_deref(&b, result[1], nir_vector_insert_imm(&b,
+                  nir_load_deref(&b, result[1]), candidate_distance, 2), 15);
+            nir_store_deref(&b, result[4], instance_metadata, 15);
+            nir_store_deref(&b, result[2], nir_vec4(&b,
+               instance_face, nir_imm_int(&b, 0), nir_imm_int(&b, 0),
+               instance_type), 15);
+            nir_store_deref(&b, result[3], nir_vec4(&b, nir_imm_int(&b, 0),
+               nir_channel(&b, instance_bary, 0),
+               nir_channel(&b, instance_bary, 1), nir_imm_int(&b, 0)), 15);
+         }
+         nir_store_deref(&b, active, nir_imm_false(&b), 1);
+         nir_pop_if(&b, NULL);
+         /* Opaque traversal consumes the whole query, with no candidate for
+          * application code to confirm. Subsequent proceed calls are inert. */
+         value = nir_imm_false(&b);
+         break;
+      }
+      case nir_intrinsic_rq_terminate:
+         nir_store_deref(&b, active, nir_imm_false(&b), 1);
+         break;
+      case nir_intrinsic_rq_confirm_intersection:
+         m5_replay_commit(&b, processed, history, prune_t,
+            nir_channel(&b, nir_load_deref(&b, candidate[1]), 2));
+         for (unsigned i = 0; i < 5; ++i)
+            nir_store_deref(&b, result[i], nir_load_deref(&b, candidate[i]), 15);
+         nir_push_if(&b, nir_test_mask(&b, nir_load_deref(&b, flags), 4));
+         nir_store_deref(&b, active, nir_imm_false(&b), 1);
+         nir_pop_if(&b, NULL);
+         break;
+      case nir_intrinsic_rq_generate_intersection: {
+         /* Commit the candidate AABB at the application's distance when it
+          * lies inside [tmin, committed t]; the candidate packet already
+          * carries the generated (bounding box) kind in its type bits. The
+          * committed distance is published to the native packet before the
+          * next continuation by m5_candidate_next. */
+         nir_def *t = intr->src[1].ssa;
+         nir_def *best = nir_channel(&b, nir_load_deref(&b, result[1]), 2);
+         nir_push_if(&b, nir_iand(&b, nir_fge(&b, best, t),
+                                  nir_fge(&b, t, nir_load_deref(&b, tmin))));
+         /* Gen2 commit stores the application's distance at112, but saves
+          * the original candidate distance at104. Prefix replay publishes
+          * the latter at116, preserving the native candidate order while
+          * the API's generated distance remains private. */
+         m5_replay_commit(&b, processed, history, prune_t,
+            nir_channel(&b, nir_load_deref(&b, candidate[1]), 2));
+         for (unsigned i = 0; i < 5; ++i)
+            nir_store_deref(&b, result[i], nir_load_deref(&b, candidate[i]), 15);
+         nir_store_deref(&b, result[1], nir_vector_insert_imm(&b,
+            nir_load_deref(&b, result[1]), t, 2), 15);
+         nir_push_if(&b, nir_test_mask(&b, nir_load_deref(&b, flags), 4));
+         nir_store_deref(&b, active, nir_imm_false(&b), 1);
+         nir_pop_if(&b, NULL);
+         nir_pop_if(&b, NULL);
+         break;
+      }
+      case nir_intrinsic_rq_load:
+         if (!nir_intrinsic_committed(intr) && m5_candidate_api())
+            for (unsigned i = 0; i < 5; ++i)
+               result[i] = candidate[i];
+         switch (nir_intrinsic_ray_query_value(intr)) {
+         case nir_ray_query_value_flags:
+            value = nir_load_deref(&b, flags);
+
+            break;
+         case nir_ray_query_value_tmin:
+            value = nir_load_deref(&b, tmin);
+            break;
+         case nir_ray_query_value_world_ray_origin:
+            value = nir_load_deref(&b, origin);
+            break;
+         case nir_ray_query_value_world_ray_direction:
+            value = nir_load_deref(&b, direction);
+            break;
+         case nir_ray_query_value_intersection_type:
+            value = nir_ushr_imm(
+               &b, nir_channel(&b, nir_load_deref(&b, result[2]), 3), 30);
+            if (!nir_intrinsic_committed(intr))
+               value = nir_iadd_imm(&b, value, -1);
+            break;
+         case nir_ray_query_value_intersection_t:
+            value = nir_channel(&b, nir_load_deref(&b, result[1]), 2);
+            break;
+         case nir_ray_query_value_intersection_primitive_index:
+            value = nir_iand_imm(
+               &b, nir_channel(&b, nir_load_deref(&b, result[2]), 3), 0x3fffffff);
+            break;
+         case nir_ray_query_value_intersection_geometry_index:
+            /* Native geometry-id witness masks the second hit word read
+             * together with primitive/type at driver+256. */
+            value = nir_iand_imm(&b,
+               nir_channel(&b, nir_load_deref(&b, result[2]), 0), 0xffffff);
+            break;
+         case nir_ray_query_value_intersection_front_face:
+            value = nir_ilt_imm(&b, nir_channel(&b, nir_load_deref(&b, result[2]), 0), 0);
+            break;
+         case nir_ray_query_value_intersection_candidate_aabb_opaque:
+            value = nir_ine_imm(&b, nir_channel(&b, nir_load_deref(&b, result[0]), 0), 0);
+            break;
+         case nir_ray_query_value_intersection_instance_sbt_index:
+         case nir_ray_query_value_intersection_object_ray_direction:
+         case nir_ray_query_value_intersection_object_ray_origin:
+         case nir_ray_query_value_intersection_object_to_world:
+         case nir_ray_query_value_intersection_world_to_object: {
+            nir_def *node = nir_pack_64_2x32(&b,
+               nir_channels(&b, nir_load_deref(&b, result[4]), 3));
+            const nir_ray_query_value field = nir_intrinsic_ray_query_value(intr);
+            if (field == nir_ray_query_value_intersection_instance_sbt_index ||
+                field == nir_ray_query_value_intersection_object_to_world) {
+               nir_def *relative = nir_load_global(&b, 1, 64,
+                  nir_iadd_imm(&b, node, 104), .align_mul = 8);
+               nir_def *transform = nir_iadd(&b, node, relative);
+               if (field == nir_ray_query_value_intersection_instance_sbt_index) {
+                  value = nir_iand_imm(&b, nir_load_global(&b, 1, 32,
+                     nir_iadd_imm(&b, transform, 48), .align_mul = 4), 0xffffff);
+               } else {
+                  value = nir_load_global(&b, 3, 32,
+                     nir_iadd_imm(&b, transform, 12 * nir_intrinsic_column(intr)),
+                     .align_mul = 4);
+               }
+            } else if (field == nir_ray_query_value_intersection_world_to_object) {
+               value = nir_load_global(&b, 3, 32,
+                  nir_iadd_imm(&b, node, 16 + 12 * nir_intrinsic_column(intr)),
+                  .align_mul = 4);
+            } else {
+               /* Native node stores four packed vec3 columns of world-to-
+                * object at16. Translation applies to origins only. */
+               const bool is_origin =
+                  field == nir_ray_query_value_intersection_object_ray_origin;
+               nir_def *ray = nir_load_deref(&b, is_origin ? origin : direction);
+               nir_def *column[4];
+               for (unsigned c = 0; c < 4; ++c)
+                  column[c] = nir_load_global(&b, 3, 32,
+                     nir_iadd_imm(&b, node, 16 + 12 * c), .align_mul = 4);
+               nir_def *component[3];
+               for (unsigned r = 0; r < 3; ++r) {
+                  component[r] = is_origin ? nir_channel(&b, column[3], r)
+                                           : nir_imm_float(&b, 0);
+                  for (unsigned c = 0; c < 3; ++c)
+                     component[r] = nir_ffma(&b, nir_channel(&b, ray, c),
+                        nir_channel(&b, column[c], r), component[r]);
+               }
+               value = nir_vec3(&b, component[0], component[1], component[2]);
+            }
+            break;
+         }
+         case nir_ray_query_value_intersection_instance_id:
+            value = nir_channel(&b, nir_load_deref(&b, result[4]), 2);
+            break;
+         case nir_ray_query_value_intersection_instance_custom_index:
+            value = nir_channel(&b, nir_load_deref(&b, result[4]), 3);
+            break;
+         case nir_ray_query_value_intersection_barycentrics:
+            value = nir_channels(&b, nir_load_deref(&b, result[3]), 6);
+            break;
+         default:
+            UNREACHABLE("preflighted M5 query load");
+         }
+         break;
+      default:
+         UNREACHABLE("preflighted M5 query operation");
+      }
+      m5_rq_op_writeback(&b, &shadows, intr->intrinsic, state);
+      if (value)
+         nir_def_rewrite_uses(&intr->def, value);
+      nir_instr_remove(&intr->instr);
+   }
+   util_dynarray_fini(&ops);
+   util_dynarray_foreach(&queries, struct m5_rq_state, state)
+      state->query->data.ray_query = false;
+   util_dynarray_fini(&queries);
+   shader->info.ray_queries = 0;
+   nir_progress(true, impl, nir_metadata_none);
+   if (any_dynamic) {
+      NIR_PASS(_, shader, nir_lower_var_copies);
+      NIR_PASS(_, shader, nir_lower_indirect_derefs_to_if_else_trees,
+               nir_var_shader_temp, ~0u);
+   }
+   NIR_PASS(_, shader, nir_lower_global_vars_to_local);
+   NIR_PASS(_, shader, nir_lower_vars_to_ssa);
+   NIR_PASS(_, shader, nir_opt_dce);
+   NIR_PASS(_, shader, nir_remove_dead_variables,
+            nir_var_shader_temp | nir_var_function_temp, NULL);
+   return true;
+unsupported:
+   util_dynarray_fini(&ops);
+   util_dynarray_fini(&queries);
+   return false;
+}
+
+/* Native query operations only touch the invocation's SCS stack and the
+ * per-query state they own; they have no observer other than other query
+ * operations. */
+static bool
+m5_rq_native_query_op(const nir_instr *instr)
+{
+   if (instr->type != nir_instr_type_intrinsic)
+      return false;
+   switch (nir_instr_as_intrinsic(instr)->intrinsic) {
+   case nir_intrinsic_enter_query_agx:
+   case nir_intrinsic_intersect_ray_agx:
+   case nir_intrinsic_continue_ray_agx:
+   case nir_intrinsic_load_scs_agx:
+   case nir_intrinsic_store_scs_agx:
+   case nir_intrinsic_load_scs_indexed_agx:
+   case nir_intrinsic_store_scs_indexed_agx:
+   case nir_intrinsic_load_scs_uniform_agx:
+   case nir_intrinsic_store_scs_uniform_agx:
+   case nir_intrinsic_load_scs_base_agx:
+   case nir_intrinsic_store_scs_base_agx:
+   case nir_intrinsic_load_scs_register_agx:
+   case nir_intrinsic_load_scs_thread_index_agx:
+      return true;
+   default:
+      return false;
+   }
+}
+
+struct m5_rq_taint {
+   struct set *loops;
+   bool progress;
+   bool external;
+};
+
+static bool
+m5_rq_src_clean(nir_src *src, void *data)
+{
+   return !nir_def_instr(src->ssa)->pass_flags;
+}
+
+static void
+m5_rq_taint_instr(nir_instr *instr, struct m5_rq_taint *t)
+{
+   if (!instr->pass_flags) {
+      instr->pass_flags = 1;
+      t->progress = true;
+   }
+}
+
+/* Mark everything data- or control-dependent on a native query operation.
+ * region is true inside control flow selected by such a value. Sets *jump
+ * when a break/continue of the innermost enclosing loop is so selected. */
+static void
+m5_rq_taint_list(struct exec_list *list, bool region, bool *jump,
+                 struct m5_rq_taint *t)
+{
+   foreach_list_typed(nir_cf_node, node, node, list) {
+      switch (node->type) {
+      case nir_cf_node_block:
+         nir_foreach_instr(instr, nir_cf_node_as_block(node)) {
+            if (region || m5_rq_native_query_op(instr) ||
+                !nir_foreach_src(instr, m5_rq_src_clean, NULL))
+               m5_rq_taint_instr(instr, t);
+            if (instr->pass_flags && instr->type == nir_instr_type_jump) {
+               nir_jump_type type = nir_instr_as_jump(instr)->type;
+               if (type == nir_jump_break || type == nir_jump_continue)
+                  *jump = true;
+               else
+                  t->external = true;
+            }
+         }
+         break;
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(node);
+         const bool selected = region ||
+            nir_def_instr(nif->condition.ssa)->pass_flags;
+         m5_rq_taint_list(&nif->then_list, selected, jump, t);
+         m5_rq_taint_list(&nif->else_list, selected, jump, t);
+         if (selected) {
+            nir_block *after = nir_cf_node_as_block(nir_cf_node_next(node));
+            nir_foreach_phi(phi, after)
+               m5_rq_taint_instr(&phi->instr, t);
+         }
+         break;
+      }
+      case nir_cf_node_loop: {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         bool own_jump = false;
+         const bool selected = region || _mesa_set_search(t->loops, loop);
+         m5_rq_taint_list(&loop->body, selected, &own_jump, t);
+         if (nir_loop_has_continue_construct(loop))
+            m5_rq_taint_list(&loop->continue_list, selected, &own_jump, t);
+         if (!selected && own_jump) {
+            /* The trip count depends on the query: every value produced in
+             * the loop does too. */
+            _mesa_set_add(t->loops, loop);
+            t->progress = true;
+         }
+         /* So does which break left it: the phis after the loop merge one
+          * value per break, and those values may be defined before it. */
+         if (_mesa_set_search(t->loops, loop)) {
+            nir_block *after = nir_cf_node_as_block(nir_cf_node_next(node));
+            nir_foreach_phi(phi, after)
+               m5_rq_taint_instr(&phi->instr, t);
+         }
+         break;
+      }
+      default:
+         t->external = true;
+         break;
+      }
+   }
+}
+
+static void
+m5_rq_collect_ifs(struct exec_list *list, struct set *ifs)
+{
+   foreach_list_typed(nir_cf_node, node, node, list) {
+      if (node->type == nir_cf_node_if) {
+         nir_if *nif = nir_cf_node_as_if(node);
+         if (nir_def_instr(nif->condition.ssa)->pass_flags)
+            _mesa_set_add(ifs, nif);
+         m5_rq_collect_ifs(&nif->then_list, ifs);
+         m5_rq_collect_ifs(&nif->else_list, ifs);
+      } else if (node->type == nir_cf_node_loop) {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         m5_rq_collect_ifs(&loop->body, ifs);
+         if (nir_loop_has_continue_construct(loop))
+            m5_rq_collect_ifs(&loop->continue_list, ifs);
+      }
+   }
+}
+
+/* Remove the outermost selected control flow. Selected instructions outside
+ * it were removed already; the selected phis after it are gone too. */
+static void
+m5_rq_remove_selected_cf(struct exec_list *list, struct set *ifs,
+                         struct set *loops)
+{
+   foreach_list_typed_safe(nir_cf_node, node, node, list) {
+      if (node->type == nir_cf_node_if) {
+         nir_if *nif = nir_cf_node_as_if(node);
+         if (_mesa_set_search(ifs, nif)) {
+            nir_cf_node_remove(node);
+         } else {
+            m5_rq_remove_selected_cf(&nif->then_list, ifs, loops);
+            m5_rq_remove_selected_cf(&nif->else_list, ifs, loops);
+         }
+      } else if (node->type == nir_cf_node_loop) {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         if (_mesa_set_search(loops, loop)) {
+            nir_cf_node_remove(node);
+         } else {
+            m5_rq_remove_selected_cf(&loop->body, ifs, loops);
+            if (nir_loop_has_continue_construct(loop))
+               m5_rq_remove_selected_cf(&loop->continue_list, ifs, loops);
+         }
+      }
+   }
+}
+
+bool
+agx_nir_m5_remove_dead_ray_queries(nir_shader *shader)
+{
+   bool any = false;
+   nir_foreach_function_impl(impl, shader) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block)
+            any |= m5_rq_native_query_op(instr);
+      }
+   }
+   if (!any || !nir_shader_get_entrypoint(shader))
+      return false;
+   /* Calls would hide both query operations and side effects. */
+   unsigned impls = 0;
+   nir_foreach_function_impl(impl, shader)
+      impls++;
+   if (impls != 1)
+      return false;
+
+   /* The lowering's merges often carry one value from every path; fold them
+    * first so they are not mistaken for query-selected values. */
+   bool progress = false;
+   NIR_PASS(progress, shader, nir_opt_remove_phis);
+   NIR_PASS(progress, shader, nir_opt_copy_prop);
+   NIR_PASS(progress, shader, nir_opt_dce);
+
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+   nir_shader_clear_pass_flags(shader);
+   struct m5_rq_taint t = {
+      .loops = _mesa_pointer_set_create(NULL),
+   };
+   do {
+      bool jump = false;
+      t.progress = false;
+      m5_rq_taint_list(&impl->body, false, &jump, &t);
+      /* A selected jump outside every loop would end the invocation. */
+      t.external |= jump;
+   } while (t.progress && !t.external);
+
+   /* Every surviving side effect must be independent of the query. Only
+    * native query operations may be selected; reorderable loads are pure. */
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         if (!instr->pass_flags || m5_rq_native_query_op(instr))
+            continue;
+         if (instr->type == nir_instr_type_call ||
+             (instr->type == nir_instr_type_intrinsic &&
+              !(nir_intrinsic_infos[nir_instr_as_intrinsic(instr)->intrinsic]
+                   .flags & NIR_INTRINSIC_CAN_ELIMINATE))) {
+            if (!t.external &&
+                debug_get_bool_option("ASAHI_M5_RQ_DEAD_DIAG", false)) {
+               fprintf(stderr, "M5-RQ live query consumer: ");
+               nir_print_instr(instr, stderr);
+               fprintf(stderr, "\n");
+               if (debug_get_bool_option("ASAHI_M5_RQ_DEAD_DIAG_SHADER", false))
+                  nir_print_shader(shader, stderr);
+            }
+            t.external = true;
+         }
+      }
+   }
+   if (t.external) {
+      _mesa_set_destroy(t.loops, NULL);
+      nir_shader_clear_pass_flags(shader);
+      return progress;
+   }
+
+   /* Selected values are only used by selected instructions. Detach them,
+    * then remove selected instructions outside selected control flow (and
+    * the phis that merge it), and finally that control flow itself. */
+   struct set *ifs = _mesa_pointer_set_create(NULL);
+   m5_rq_collect_ifs(&impl->body, ifs);
+   struct util_dynarray defs = UTIL_DYNARRAY_INIT;
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         nir_def *def = instr->pass_flags ? nir_instr_def(instr) : NULL;
+         if (def && !nir_def_is_unused(def))
+            util_dynarray_append(&defs, def);
+      }
+   }
+   nir_builder b = nir_builder_at(nir_before_impl(impl));
+   util_dynarray_foreach(&defs, nir_def *, def) {
+      /* New instructions are zero-allocated: pass_flags stays clear. */
+      nir_def_rewrite_uses(*def, nir_undef(&b, (*def)->num_components,
+                                           (*def)->bit_size));
+   }
+   util_dynarray_fini(&defs);
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr_safe(instr, block) {
+         if (instr->pass_flags && instr->type != nir_instr_type_jump)
+            nir_instr_remove(instr);
+      }
+   }
+   m5_rq_remove_selected_cf(&impl->body, ifs, t.loops);
+   _mesa_set_destroy(ifs, NULL);
+   _mesa_set_destroy(t.loops, NULL);
+   nir_shader_clear_pass_flags(shader);
+   nir_progress(true, impl, nir_metadata_none);
+   NIR_PASS(_, shader, nir_opt_dce);
+   return true;
+}
diff --git a/src/asahi/compiler/agx_nir_m5_ray_query.h b/src/asahi/compiler/agx_nir_m5_ray_query.h
new file mode 100644
index 000000000000..f8ea53a14a6a
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_m5_ray_query.h
@@ -0,0 +1,108 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+#include "compiler/nir/nir.h"
+struct nir_builder;
+
+/* Candidate layout witnessed in the G17G triangle/instancing query program.
+ * This is the candidate-query driver profile, not the ordinary intersector's
+ * 72-byte profile. The caller owns the profile, allocation and live queue.
+ * Merely reading these records does not advance or commit traversal. */
+struct agx_m5_ray_candidate {
+   nir_def *compressed_address;
+   nir_def *flags;
+   nir_def *barycentrics;
+   nir_def *distance;
+};
+
+/* Native triangle import boundary: initial tmax is inclusive; committed ties
+ * keep the existing hit. Used by lowering and exact constant-fold regressions. */
+nir_def *agx_nir_m5_should_import_triangle(struct nir_builder *b,
+                                          nir_def *native_type, nir_def *prior_type,
+                                          nir_def *distance, nir_def *best_t,
+                                          nir_def *minimum);
+
+nir_def *agx_nir_m5_candidate_count(struct nir_builder *b, nir_def *driver,
+                                   nir_def *lane);
+struct agx_m5_ray_candidate
+agx_nir_m5_load_candidate(struct nir_builder *b, nir_def *driver,
+                          nir_def *lane, nir_def *index);
+
+/* Execute an already initialized M5 opaque-triangle query. The caller owns
+ * the SCS packet and pointer table, BVH state, and resident intersection
+ * programs. These addresses are SCS ABI values, not device virtual addresses.
+ *
+ * This is a compiler building block, not a public ray-query lowering pass.
+ * Procedural/non-opaque candidates require their own accept/ignore protocol.
+ * launch_words explicitly supplies staging halves 0/2/4; the caller must
+ * derive these from its native query layout. They are not inferred from GPRs.
+ * Optional counters are for validation and disappear when not requested.
+ */
+void agx_nir_m5_trace_opaque(struct nir_builder *b, nir_def *request,
+                             nir_def *bvh_state, nir_def *payload_base,
+                             nir_def *payload_index, nir_def *status_base,
+                             nir_def *launch_words, nir_def **initial_pending,
+                             nir_def **continuations);
+
+/* Initialize the captured 64-byte, untransformed triangle-query packet.
+ * query_flags is the caller's native configuration word (the retained opaque
+ * reference uses 0x122). Address/continuation tables are initialized separately.
+ * Inputs must be finite; negative t_min and invalid direction handling belong
+ * to the caller. Bounds describe the acceleration structure's root AABB.
+ */
+void agx_nir_m5_init_triangle_packet(struct nir_builder *b,
+                                     nir_def *payload_base,
+                                     nir_def *payload_index, nir_def *origin,
+                                     nir_def *direction, nir_def *t_min,
+                                     nir_def *t_max, nir_def *bounds_min,
+                                     nir_def *bounds_max, nir_def *mask,
+                                     nir_def *query_flags, nir_def *opaque,
+                                     nir_def *first_hit, nir_def *triangle_cull);
+/* Explicit per-invocation launch ABI for the M5 native query pipeline.
+ * These are half-word uniform indices, not GPU addresses. The caller must bind
+ * the same native AS to bvh_uniform and the rq_initialize AS operand, provide
+ * the resident IFT, and allocate the per-invocation SCS packet/status region.
+ * This interface does not advertise a Vulkan feature or create that state.
+ */
+struct agx_m5_ray_query_layout {
+   unsigned payload_uniform;
+   unsigned bvh_uniform;
+   unsigned status_uniform;
+   uint32_t launch_header;
+   /* Optional per-dispatch native context and AS pointer table. Zero keeps
+    * the caller-initialized ABI used by standalone compiler probes. */
+   unsigned context_uniform;
+   unsigned as_table_uniform;
+   bool instance_traversal;
+   bool compute_kernel;
+};
+
+/* Lower standard NIR rq_* operations for opaque triangle queries and arrays.
+ * Each query retains its own parameters and committed packet; interleaved
+ * queries may reuse the physical traversal packet after a proceed completes.
+ * Unsupported operations/flags are rejected before changing the shader.
+ * Candidate enumeration, instances, and other query layouts still
+ * need separate lowering; callers must not silently substitute this pass.
+ */
+/* Split ray-query arrays that are only accessed with constant indices into
+ * separate ray-query variables, so each element can be range-merged and
+ * mapped to its own native query context. */
+/* Queue-exhaustion predicate shared with the production lowering and host
+ * regression: caller has already consumed this completed batch. */
+nir_def *agx_nir_m5_should_resume_batch(struct nir_builder *b, nir_def *status,
+                                         nir_def *control, nir_def *count);
+
+bool agx_nir_m5_split_ray_query_arrays(nir_shader *shader);
+
+bool agx_nir_lower_m5_ray_query(nir_shader *shader,
+                                const struct agx_m5_ray_query_layout *layout);
+
+/* Remove an already lowered native query whose results cannot reach any
+ * remaining memory, output or vertex-emission side effect. Geometry-shader
+ * lowering clones the query into kernels from which it strips the API
+ * stores; the query there is dead work. Native query operations are removed
+ * only when every one of them is provably dead; otherwise only the
+ * value-preserving phi/copy cleanup the analysis needs is applied. Returns
+ * progress. */
+bool agx_nir_m5_remove_dead_ray_queries(nir_shader *shader);
diff --git a/src/asahi/compiler/agx_nir_opt_preamble.c b/src/asahi/compiler/agx_nir_opt_preamble.c
index 7d5d9e9c49f7..5db1e3717fc4 100644
--- a/src/asahi/compiler/agx_nir_opt_preamble.c
+++ b/src/asahi/compiler/agx_nir_opt_preamble.c
@@ -61,6 +61,23 @@ def_size_g17p(nir_def *def, unsigned *size, unsigned *align,
    *size = *align = 4;
 }
 
+/* The AGX3C constant-program export is a 32-bit uniform COPY: a lone half
+ * is published as one pair-aligned copy whose upper half is padding (Apple
+ * widens scalar half preamble results the same way), and no half-width
+ * copy-to-uniform form is witnessed. Give a scalar 16-bit value its own
+ * aligned pair so the export never lands on an odd half. */
+static void
+def_size_agx3c(nir_def *def, unsigned *size, unsigned *align,
+               nir_preamble_class *class)
+{
+   def_size_g17p(def, size, align, class);
+   if (*class == nir_preamble_class_general && def->num_components == 1 &&
+       def->bit_size <= 16) {
+      *size = 2;
+      *align = 2;
+   }
+}
+
 static bool
 all_uses_float(nir_def *def)
 {
@@ -361,6 +378,17 @@ rewrite_cost_g17p(nir_def *def, const void *data)
    return rewrite_cost_impl(def, true);
 }
 
+/* AGX3C publishes constant-program results with 32-bit uniform copies only
+ * (agx3_pack_binary), so a narrow vector has no export form there: keep it
+ * out of the preamble and let its scalar or 32-bit uses be hoisted instead. */
+static float
+rewrite_cost_agx3c(nir_def *def, const void *data)
+{
+   if (def->bit_size < 32 && def->num_components > 1)
+      return 1e30f;
+   return rewrite_cost_impl(def, true);
+}
+
 static bool
 avoid_instr(const nir_instr *instr, const void *data)
 {
@@ -622,14 +650,21 @@ agx_nir_export_select_flags_as_halves(nir_shader *nir)
 }
 
 bool
-agx_nir_opt_preamble(nir_shader *nir, unsigned *sizes, bool g17p)
+agx_nir_opt_preamble(nir_shader *nir, unsigned *sizes, bool g17p,
+                     bool agx3c)
 {
    bool progress = false;
    nir_opt_preamble_options options = preamble_options;
+   /* Native M5 compiler specimens leave texture operations in main even
+    * for uniform coordinates. Keep their dependent expressions there too;
+    * invariant ALU and buffer loads remain eligible for the CProg. */
+   options.no_textures = agx3c;
    if (g17p) {
-      options.def_size = def_size_g17p;
-      options.instr_cost_cb = instr_cost_g17p;
-      options.rewrite_cost_cb = rewrite_cost_g17p;
+      /* The G17P uniform fabs cost is not an M5 witness; keep the
+       * generic instruction cost there. */
+      options.def_size = agx3c ? def_size_agx3c : def_size_g17p;
+      options.instr_cost_cb = agx3c ? instr_cost : instr_cost_g17p;
+      options.rewrite_cost_cb = agx3c ? rewrite_cost_agx3c : rewrite_cost_g17p;
    }
    NIR_PASS(progress, nir, nir_opt_preamble, &options, sizes);
    if (progress && g17p)
diff --git a/src/asahi/compiler/agx_nir_opt_sink_loop_sample_uses.c b/src/asahi/compiler/agx_nir_opt_sink_loop_sample_uses.c
new file mode 100644
index 000000000000..62bccd401800
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_opt_sink_loop_sample_uses.c
@@ -0,0 +1,435 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_nir.h"
+#include "compiler/nir/nir.h"
+
+/*
+ * Sink the ALU consumers of a loop body's texture results to the latest
+ * top-level block of that body that still dominates every use.
+ *
+ * Mesa's order consumes each texture result right after its sample, so a
+ * loop whose body samples several times completes those samples one at a
+ * time: twelve serial texture round trips per iteration in the robustness2
+ * sampled_image 3D vertex loop. Apple's compiler for the same straight-line
+ * loop (native G17G witness) issues all twelve
+ * SMP in source order, the three conditional ones inside their own
+ * cndst/cndend, and places every compare and accumulate of the twelve
+ * results after the last SMP, so the iteration waits for the samples once.
+ *
+ * nir_opt_sink leaves general ALU alone and nir_opt_move only reorders
+ * within a block; the backend pressure and ILP schedules are per block too,
+ * and a sample under a condition (c != 0 here) splits the body into several
+ * blocks. Moving a pure ALU instruction to a block its definition dominates
+ * and that dominates all of its uses changes neither its value nor the SSA
+ * dominance of the program; it may only run less often (it no longer runs on
+ * paths that leave the body early). Texture instructions themselves never
+ * move, so no sample is speculated.
+ *
+ * The sunk results stay live until the end of the body, so the flagged
+ * sample results are bounded per loop by AGX_NIR_LOOP_SAMPLE_SINK_HALVES
+ * (twelve 32-bit vec4 results, the witness loop); consumers of later samples
+ * keep their place.
+ */
+
+#define AGX_NIR_LOOP_SAMPLE_SINK_HALVES 96
+
+/* pass_flags bit: the instruction is, or depends on, a sample of this loop
+ * whose consumers may sink. */
+#define SAMPLE_DEPENDENT 1
+
+static unsigned
+def_halves(const nir_def *def)
+{
+   return def->num_components * DIV_ROUND_UP(def->bit_size, 16);
+}
+
+/* nir_foreach_src callback: keep going while the source is independent, so
+ * the walk returns false exactly when some source is sample dependent. */
+static bool
+src_independent(nir_src *src, void *data)
+{
+   return (nir_def_instr(src->ssa)->pass_flags & SAMPLE_DEPENDENT) == 0;
+}
+
+/* The top-level block of the loop body containing (or preceding the nested
+ * control flow containing) block b, or NULL when b is outside the loop. */
+static nir_block *
+loop_top_level_block(nir_loop *loop, nir_block *b)
+{
+   nir_cf_node *node = &b->cf_node;
+   while (node->parent != &loop->cf_node) {
+      node = node->parent;
+      if (!node)
+         return NULL;
+   }
+   if (node->type == nir_cf_node_block)
+      return nir_cf_node_as_block(node);
+   /* Nested if or loop: the top-level block right before it. */
+   nir_cf_node *prev = nir_cf_node_prev(node);
+   assert(prev && prev->type == nir_cf_node_block);
+   return nir_cf_node_as_block(prev);
+}
+
+static bool
+sink_loop(nir_loop *loop)
+{
+   if (nir_loop_has_continue_construct(loop))
+      return false;
+
+   /* Mark the body's samples, within the result budget, and everything
+    * computed from them. Loop-carried values reach their consumers through
+    * header phis, so iterate to a fixed point. */
+   unsigned budget = AGX_NIR_LOOP_SAMPLE_SINK_HALVES;
+   nir_foreach_block_in_cf_node(block, &loop->cf_node) {
+      nir_foreach_instr(instr, block)
+         instr->pass_flags = 0;
+   }
+   bool any_sample = false;
+   nir_foreach_block_in_cf_node(block, &loop->cf_node) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_tex)
+            continue;
+         nir_tex_instr *tex = nir_instr_as_tex(instr);
+         const unsigned halves = def_halves(&tex->def);
+         if (halves > budget)
+            continue;
+         budget -= halves;
+         instr->pass_flags = SAMPLE_DEPENDENT;
+         any_sample = true;
+      }
+   }
+   if (!any_sample)
+      return false;
+
+   bool changed;
+   do {
+      changed = false;
+      nir_foreach_block_in_cf_node(block, &loop->cf_node) {
+         nir_foreach_instr(instr, block) {
+            if (instr->pass_flags & SAMPLE_DEPENDENT)
+               continue;
+            if (instr->type != nir_instr_type_alu &&
+                instr->type != nir_instr_type_phi)
+               continue;
+            if (!nir_foreach_src(instr, src_independent, NULL)) {
+               instr->pass_flags = SAMPLE_DEPENDENT;
+               changed = true;
+            }
+         }
+      }
+   } while (changed);
+
+   /* Consumers first: visiting in reverse lets a producer follow the uses
+    * that already moved. */
+   bool progress = false;
+   nir_foreach_block_in_cf_node_reverse_safe(block, &loop->cf_node) {
+      if (block->cf_node.parent != &loop->cf_node)
+         continue;
+      nir_foreach_instr_reverse_safe(instr, block) {
+         if (instr->type != nir_instr_type_alu ||
+             !(instr->pass_flags & SAMPLE_DEPENDENT))
+            continue;
+         nir_alu_instr *alu = nir_instr_as_alu(instr);
+
+         nir_block *lca = NULL;
+         nir_foreach_use_including_if(use, &alu->def)
+            lca = nir_dominance_lca(lca, nir_src_get_block(use));
+         if (!lca)
+            continue;
+
+         nir_block *target = loop_top_level_block(loop, lca);
+         if (!target || target->index <= block->index)
+            continue;
+         assert(nir_block_dominates(block, target));
+
+         nir_instr_remove(instr);
+         nir_instr_insert(nir_after_phis(target), instr);
+         progress = true;
+      }
+   }
+   return progress;
+}
+
+static bool
+sink_cf_list(struct exec_list *list)
+{
+   bool progress = false;
+   foreach_list_typed(nir_cf_node, node, node, list) {
+      if (node->type == nir_cf_node_if) {
+         nir_if *nif = nir_cf_node_as_if(node);
+         progress |= sink_cf_list(&nif->then_list);
+         progress |= sink_cf_list(&nif->else_list);
+      } else if (node->type == nir_cf_node_loop) {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         progress |= sink_cf_list(&loop->body);
+         progress |= sink_loop(loop);
+      }
+   }
+   return progress;
+}
+
+bool
+agx_nir_opt_sink_loop_sample_uses(nir_shader *shader)
+{
+   bool progress = false;
+   nir_foreach_function_impl(impl, shader) {
+      nir_metadata_require(impl, nir_metadata_block_index |
+                                    nir_metadata_dominance |
+                                    nir_metadata_dominance_lca);
+      const bool impl_progress = sink_cf_list(&impl->body);
+      progress |= nir_progress(impl_progress, impl, nir_metadata_control_flow);
+   }
+   return progress;
+}
+
+/*
+ * Hoist a loop body's unconditional samples above the conditional blocks
+ * before them.
+ *
+ * The M5 backend retires every outstanding scoreboard slot before a
+ * control-flow instruction (agx3_try_control_flow), so each conditional
+ * block of a loop body closes the window in which samples stay in flight.
+ * The robustness2 vertex loops have a conditional texel fetch after each
+ * axis's first fetch; even with every consumer sunk to the end of the body
+ * their eleven or twelve samples completed in seven serial round trips per
+ * iteration. The native compiler issues the unconditional samples of such a
+ * body back to back.
+ *
+ * A sample in a later top-level block of the body runs exactly when the
+ * top-level blocks before it run, as long as the conditional blocks in
+ * between hold no jump, no loop and nothing with side effects. Moving it,
+ * together with the pure computation of its sources from those later
+ * blocks, to the end of the earliest such block changes neither which
+ * samples run nor their values, and no sample is speculated. Conditional
+ * blocks left adjacent with the same condition are then merged by
+ * nir_opt_if_merge, after which the merged block's duplicate fetches CSE.
+ *
+ * Only samples whose consumers agx_nir_opt_sink_loop_sample_uses sinks
+ * (the same per-loop budget) move, so the live sample results stay bounded
+ * by AGX_NIR_LOOP_SAMPLE_SINK_HALVES.
+ */
+
+/* pass_flags bits for the hoist. */
+#define HOIST_ELIGIBLE 1
+#define HOIST_MOVE     2
+
+/* Bound on the source chain walked for one sample. */
+#define HOIST_MAX_DEPTH 32
+
+/* An instruction without side effects whose value does not depend on its
+ * position among the body's samples. */
+static bool
+instr_reorderable(nir_instr *instr)
+{
+   switch (instr->type) {
+   case nir_instr_type_alu:
+   case nir_instr_type_load_const:
+   case nir_instr_type_undef:
+   case nir_instr_type_tex:
+      return true;
+   case nir_instr_type_intrinsic:
+      return nir_intrinsic_can_reorder(nir_instr_as_intrinsic(instr));
+   default:
+      return false;
+   }
+}
+
+/* Whether a sample may move across this control flow: only if-statements
+ * whose arms hold no jump, loop or side effect. */
+static bool
+cf_list_reorderable(struct exec_list *list)
+{
+   foreach_list_typed(nir_cf_node, node, node, list) {
+      switch (node->type) {
+      case nir_cf_node_block:
+         nir_foreach_instr(instr, nir_cf_node_as_block(node)) {
+            if (instr->type != nir_instr_type_phi && !instr_reorderable(instr))
+               return false;
+         }
+         break;
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(node);
+         if (!cf_list_reorderable(&nif->then_list) ||
+             !cf_list_reorderable(&nif->else_list))
+            return false;
+         break;
+      }
+      default:
+         return false;
+      }
+   }
+   return true;
+}
+
+static bool
+cf_node_reorderable(nir_cf_node *node)
+{
+   if (!node || node->type != nir_cf_node_if)
+      return false;
+   nir_if *nif = nir_cf_node_as_if(node);
+   return cf_list_reorderable(&nif->then_list) &&
+          cf_list_reorderable(&nif->else_list);
+}
+
+struct hoist_state {
+   nir_loop *loop;
+   nir_block *target;
+   unsigned depth;
+};
+
+static bool hoist_src(nir_src *src, void *data);
+
+/* Whether def is available at the end of target, or is a pure value of a
+ * later top-level block of the body whose sources are, flagging the
+ * instructions that have to move. Samples never join a chain: a sample
+ * feeding another only counts once it has been hoisted itself. */
+static bool
+hoist_def(nir_def *def, struct hoist_state *st)
+{
+   nir_instr *instr = nir_def_instr(def);
+   if (nir_block_dominates(instr->block, st->target) ||
+       (instr->pass_flags & HOIST_MOVE))
+      return true;
+   if (instr->block->cf_node.parent != &st->loop->cf_node ||
+       instr->type == nir_instr_type_phi ||
+       instr->type == nir_instr_type_tex || !instr_reorderable(instr) ||
+       st->depth >= HOIST_MAX_DEPTH)
+      return false;
+   instr->pass_flags |= HOIST_MOVE;
+   st->depth++;
+   const bool ok = nir_foreach_src(instr, hoist_src, st);
+   st->depth--;
+   return ok;
+}
+
+static bool
+hoist_src(nir_src *src, void *data)
+{
+   return hoist_def(src->ssa, data);
+}
+
+/* Clear the move flags of the top-level blocks after target up to and
+ * including tex's block, the only instructions hoist_def flags. */
+static void
+clear_moves(nir_block *target, nir_instr *tex)
+{
+   for (nir_cf_node *node = nir_cf_node_next(&target->cf_node);
+        node != nir_cf_node_next(&tex->block->cf_node);
+        node = nir_cf_node_next(node)) {
+      if (node->type != nir_cf_node_block)
+         continue;
+      nir_foreach_instr(instr, nir_cf_node_as_block(node))
+         instr->pass_flags &= ~HOIST_MOVE;
+   }
+}
+
+static bool
+hoist_sample(nir_loop *loop, nir_instr *tex)
+{
+   /* Walk back over reorderable conditional blocks while the sample's
+    * sources stay available. */
+   nir_block *target = NULL;
+   nir_cf_node *prev = nir_cf_node_prev(&tex->block->cf_node);
+   while (cf_node_reorderable(prev)) {
+      nir_block *candidate = nir_cf_node_as_block(nir_cf_node_prev(prev));
+      struct hoist_state st = { .loop = loop, .target = candidate };
+      const bool ok = nir_foreach_src(tex, hoist_src, &st);
+      clear_moves(candidate, tex);
+      if (!ok)
+         break;
+      target = candidate;
+      prev = nir_cf_node_prev(&candidate->cf_node);
+   }
+   if (!target)
+      return false;
+
+   struct hoist_state st = { .loop = loop, .target = target };
+   ASSERTED bool ok = nir_foreach_src(tex, hoist_src, &st);
+   assert(ok);
+   tex->pass_flags |= HOIST_MOVE;
+
+   /* Append the flagged instructions to the target in their original order,
+    * which keeps every definition ahead of its uses. */
+   nir_cf_node *end = nir_cf_node_next(&tex->block->cf_node);
+   for (nir_cf_node *node = nir_cf_node_next(&target->cf_node); node != end;
+        node = nir_cf_node_next(node)) {
+      if (node->type != nir_cf_node_block)
+         continue;
+      nir_foreach_instr_safe(instr, nir_cf_node_as_block(node)) {
+         if (!(instr->pass_flags & HOIST_MOVE))
+            continue;
+         instr->pass_flags &= ~HOIST_MOVE;
+         nir_instr_remove(instr);
+         nir_instr_insert(nir_after_block(target), instr);
+      }
+   }
+   return true;
+}
+
+static bool
+hoist_loop(nir_loop *loop)
+{
+   if (nir_loop_has_continue_construct(loop))
+      return false;
+
+   /* The samples agx_nir_opt_sink_loop_sample_uses sinks consumers of. */
+   unsigned budget = AGX_NIR_LOOP_SAMPLE_SINK_HALVES;
+   nir_foreach_block_in_cf_node(block, &loop->cf_node) {
+      nir_foreach_instr(instr, block) {
+         instr->pass_flags = 0;
+         if (instr->type != nir_instr_type_tex)
+            continue;
+         const unsigned halves = def_halves(&nir_instr_as_tex(instr)->def);
+         if (halves > budget)
+            continue;
+         budget -= halves;
+         instr->pass_flags = HOIST_ELIGIBLE;
+      }
+   }
+
+   bool progress = false;
+   foreach_list_typed(nir_cf_node, node, node, &loop->body) {
+      if (node->type != nir_cf_node_block)
+         continue;
+      nir_foreach_instr_safe(instr, nir_cf_node_as_block(node)) {
+         if (instr->type == nir_instr_type_tex &&
+             (instr->pass_flags & HOIST_ELIGIBLE))
+            progress |= hoist_sample(loop, instr);
+      }
+   }
+   return progress;
+}
+
+static bool
+hoist_cf_list(struct exec_list *list)
+{
+   bool progress = false;
+   foreach_list_typed(nir_cf_node, node, node, list) {
+      if (node->type == nir_cf_node_if) {
+         nir_if *nif = nir_cf_node_as_if(node);
+         progress |= hoist_cf_list(&nif->then_list);
+         progress |= hoist_cf_list(&nif->else_list);
+      } else if (node->type == nir_cf_node_loop) {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         progress |= hoist_cf_list(&loop->body);
+         progress |= hoist_loop(loop);
+      }
+   }
+   return progress;
+}
+
+bool
+agx_nir_opt_hoist_loop_samples(nir_shader *shader)
+{
+   bool progress = false;
+   nir_foreach_function_impl(impl, shader) {
+      nir_metadata_require(impl, nir_metadata_block_index |
+                                    nir_metadata_dominance);
+      const bool impl_progress = hoist_cf_list(&impl->body);
+      progress |= nir_progress(impl_progress, impl, nir_metadata_control_flow);
+   }
+   return progress;
+}
diff --git a/src/asahi/compiler/agx_nir_share_bounds.c b/src/asahi/compiler/agx_nir_share_bounds.c
new file mode 100644
index 000000000000..679ec6153cd6
--- /dev/null
+++ b/src/asahi/compiler/agx_nir_share_bounds.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "util/hash_table.h"
+#include "util/u_dynarray.h"
+#include "agx_nir.h"
+#include "nir.h"
+
+/*
+ * Share the end computation of robust buffer reads at constant distances.
+ *
+ * A robust read keeps its value only when its exclusive end lies within the
+ * descriptor's byte size:
+ *
+ *    bounds_agx(data, end, size) = end <= size ? data : 0
+ *
+ * The reads of one structure in a loop (a light record, a constant buffer
+ * row) differ only by a constant field offset, and each computes its own end:
+ *
+ *    end = uadd_sat(iadd(X, c), n)
+ *
+ * an add and a saturating add per read, on top of the compare-select. For the
+ * reads of one block with the same X and size, take the smallest c and
+ *
+ *    R = usub_sat(size, iadd(X, c_min))
+ *
+ * once, after which each read checks the immediate c - c_min + n against R:
+ *
+ *    X + c + n <= size  <=>  c - c_min + n <= size - (X + c_min)
+ *
+ * when size >= X + c_min, and both sides are false otherwise (R = 0 while the
+ * immediate is at least n > 0). This is exact, except where X + c wraps a
+ * 32-bit offset: such a read was kept when its wrapped end fell inside the
+ * buffer, and is now zeroed, as its real offset lies past the end.
+ */
+
+/* bounds_agx takes its end as an 8-bit immediate. */
+#define SHARE_BOUNDS_MAX_IMM 255
+
+struct bound_read {
+   nir_alu_instr *alu;
+   uint32_t c, n;
+};
+
+struct bound_group {
+   nir_scalar x, size;
+   struct util_dynarray reads;
+};
+
+static bool
+scalar_const(nir_scalar s, uint32_t *value)
+{
+   if (!nir_scalar_is_const(s))
+      return false;
+   *value = nir_scalar_as_uint(s);
+   return true;
+}
+
+/* Match end = uadd_sat(iadd(X, c), n) or uadd_sat(X, n), constants in either
+ * position. */
+static bool
+match_end(nir_scalar end, nir_scalar *x, uint32_t *c, uint32_t *n)
+{
+   if (!nir_scalar_is_alu(end) || nir_scalar_alu_op(end) != nir_op_uadd_sat ||
+       end.def->bit_size != 32)
+      return false;
+
+   nir_scalar a = nir_scalar_chase_alu_src(end, 0);
+   nir_scalar b = nir_scalar_chase_alu_src(end, 1);
+   nir_scalar inner;
+   if (scalar_const(b, n))
+      inner = a;
+   else if (scalar_const(a, n))
+      inner = b;
+   else
+      return false;
+
+   if (*n == 0 || *n > SHARE_BOUNDS_MAX_IMM)
+      return false;
+
+   *c = 0;
+   *x = inner;
+   if (nir_scalar_is_alu(inner) && nir_scalar_alu_op(inner) == nir_op_iadd) {
+      nir_scalar p = nir_scalar_chase_alu_src(inner, 0);
+      nir_scalar q = nir_scalar_chase_alu_src(inner, 1);
+      if (scalar_const(q, c))
+         *x = p;
+      else if (scalar_const(p, c))
+         *x = q;
+   }
+
+   return !nir_scalar_is_const(*x);
+}
+
+static uint32_t
+group_hash(const void *key)
+{
+   const struct bound_group *g = key;
+   uint32_t h = _mesa_hash_pointer(g->x.def) ^ (g->x.comp * 31);
+   return h ^ (_mesa_hash_pointer(g->size.def) * 7) ^ g->size.comp;
+}
+
+static bool
+group_equal(const void *a, const void *b)
+{
+   const struct bound_group *x = a, *y = b;
+   return nir_scalar_equal(x->x, y->x) && nir_scalar_equal(x->size, y->size);
+}
+
+static bool
+share_group(nir_builder *b, struct bound_group *g)
+{
+   unsigned count = util_dynarray_num_elements(&g->reads, struct bound_read);
+   if (count < 2)
+      return false;
+
+   uint32_t c_min = UINT32_MAX;
+   util_dynarray_foreach(&g->reads, struct bound_read, r)
+      c_min = MIN2(c_min, r->c);
+
+   /* Reads whose immediate would not fit keep their own end. */
+   unsigned fit = 0;
+   util_dynarray_foreach(&g->reads, struct bound_read, r) {
+      if ((uint64_t)r->c - c_min + r->n <= SHARE_BOUNDS_MAX_IMM)
+         fit++;
+   }
+   if (fit < 2)
+      return false;
+
+   struct bound_read *first =
+      util_dynarray_element(&g->reads, struct bound_read, 0);
+   b->cursor = nir_before_instr(&first->alu->instr);
+
+   nir_def *x = nir_mov_scalar(b, g->x);
+   nir_def *base = c_min ? nir_iadd_imm(b, x, c_min) : x;
+   nir_def *rest = nir_usub_sat(b, nir_mov_scalar(b, g->size), base);
+
+   util_dynarray_foreach(&g->reads, struct bound_read, r) {
+      uint64_t imm = (uint64_t)r->c - c_min + r->n;
+      if (imm > SHARE_BOUNDS_MAX_IMM)
+         continue;
+
+      b->cursor = nir_before_instr(&r->alu->instr);
+      nir_src_rewrite(&r->alu->src[1].src, nir_imm_int(b, imm));
+      r->alu->src[1].swizzle[0] = 0;
+      nir_src_rewrite(&r->alu->src[2].src, rest);
+      r->alu->src[2].swizzle[0] = 0;
+   }
+
+   return true;
+}
+
+static bool
+share_block(nir_builder *b, nir_block *block, void *mem_ctx)
+{
+   struct set *groups = _mesa_set_create(mem_ctx, group_hash, group_equal);
+   struct util_dynarray order;
+   util_dynarray_init(&order, mem_ctx);
+
+   nir_foreach_instr(instr, block) {
+      if (instr->type != nir_instr_type_alu)
+         continue;
+      nir_alu_instr *alu = nir_instr_as_alu(instr);
+      if (alu->op != nir_op_bounds_agx || alu->def.bit_size != 32 ||
+          alu->def.num_components != 1)
+         continue;
+
+      nir_scalar end = nir_scalar_resolved(alu->src[1].src.ssa,
+                                           alu->src[1].swizzle[0]);
+      nir_scalar size = nir_scalar_resolved(alu->src[2].src.ssa,
+                                            alu->src[2].swizzle[0]);
+      if (size.def->bit_size != 32)
+         continue;
+
+      struct bound_read read = {.alu = alu};
+      nir_scalar x;
+      if (!match_end(end, &x, &read.c, &read.n))
+         continue;
+
+      struct bound_group key = {.x = x, .size = size};
+      bool found = false;
+      struct set_entry *e = _mesa_set_search_or_add(groups, &key, &found);
+      struct bound_group *g;
+      if (!found) {
+         g = ralloc(mem_ctx, struct bound_group);
+         *g = key;
+         util_dynarray_init(&g->reads, mem_ctx);
+         e->key = g;
+         util_dynarray_append(&order, g);
+      } else {
+         g = (struct bound_group *)e->key;
+      }
+      util_dynarray_append(&g->reads, read);
+   }
+
+   bool progress = false;
+   util_dynarray_foreach(&order, struct bound_group *, g)
+      progress |= share_group(b, *g);
+
+   return progress;
+}
+
+bool
+agx_nir_share_bounds(nir_shader *s)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(s);
+   if (!impl)
+      return false;
+
+   void *mem_ctx = ralloc_context(NULL);
+   nir_builder b = nir_builder_create(impl);
+   bool progress = false;
+
+   nir_foreach_block(block, impl)
+      progress |= share_block(&b, block, mem_ctx);
+
+   ralloc_free(mem_ctx);
+   return nir_progress(progress, impl, nir_metadata_control_flow);
+}
diff --git a/src/asahi/compiler/agx_opcodes.py b/src/asahi/compiler/agx_opcodes.py
index 3c5631e6d999..f22976ff381f 100644
--- a/src/asahi/compiler/agx_opcodes.py
+++ b/src/asahi/compiler/agx_opcodes.py
@@ -66,6 +66,7 @@ def enum(name, value_dict):
 _ = None
 
 FORMAT = immediate("format", "enum agx_format")
+MATRIX_TYPE = immediate("matrix_type")
 IMM = immediate("imm", "uint64_t")
 WRITEOUT = immediate("writeout")
 INDEX = immediate("index")
@@ -111,6 +112,7 @@ def enum(name, value_dict):
 SCS_ZERO_EXTEND = immediate("scs_zero_extend", "bool")
 SCS_C_STATUS_LOAD = immediate("scs_c_status_load", "bool")
 SCS_C_LOAD_MARKER = immediate("scs_c_load_marker", "bool")
+SCS_RT_COMPLETION = immediate("scs_rt_completion", "bool")
 SCS_ADDRESS_SELECTOR = immediate("scs_address_selector", "uint8_t")
 SCOREBOARD = immediate("scoreboard")
 ICOND = immediate("icond", "enum agx_icond")
@@ -159,7 +161,15 @@ def enum(name, value_dict):
    147: 'helper_arg_h',
    # G17P builtin background reload: native half-register layer input.
    195: 'background_layer_id',
+   # Native M5 ray shader reads normalized SR59 (hardware byte0xea).
    234: 'scs_thread_index',
+   # Native object_amplify and mesh_amplified payload pointer witness.
+   235: 'mesh_payload_lo',
+   236: 'mesh_payload_hi',
+   # Native primitive_color/primitive_id mesh witnesses read the per-workgroup
+   # per-primitive record base from normalized SR64 (W32) and SR65 (W16).
+   237: 'mesh_primitive_storage_lo',
+   238: 'mesh_primitive_storage_hi',
    # AGX3 hardware-vertex inputs. Apple emits these as explicit 32-bit rld
    # instructions instead of the AGX2 preloaded-input ABI.
    216: 'instance_id',
@@ -247,6 +257,8 @@ def iunop(name, opcode):
 op("hadd", encoding = (0x26, 4, 6), srcs = 2, is_float = True)
 op("hfma", encoding = (0x36, 6, 8), srcs = 3, is_float = True)
 op("hmul", encoding = (0x16, 4, 6), srcs = 2, is_float = True)
+op("matmad8_agx3", None, srcs = 3, imms = [MATRIX_TYPE], can_reorder = False)
+op("matmad16_agx3", None, srcs = 3, imms = [MATRIX_TYPE], can_reorder = False)
 # Apple's AGX3 backend calls the same semantic operation `ldimm`.  This is the
 # first source-grounded AGX2 IR -> AGX3 mnemonic mapping: the native immediate
 # field reproduces all 14 probe constants exactly on G15, G16 and G17, while
@@ -467,6 +479,8 @@ def subgroup_op(name, opc):
 op("iterproj", (0x21, 8, _), srcs = 3, imms = [CHANNELS, INTERPOLATION])
 
 op("ldcf", (0xA1, 8, _), srcs = 1, imms = [CHANNELS])
+# Native mesh output allocation and two-word UVB descriptor publication.
+# Keep their source dependencies explicit through scheduling and allocation.
 op("allocate_uvb_m5", _, dests=0, srcs=2, can_eliminate=False,
    can_reorder=False, schedule_class="store", agx3_mnemonic="allocate_uvb")
 op("store_uvb_m5", _, dests=0, srcs=1, imms=[IMM], can_eliminate=False,
@@ -520,7 +534,7 @@ def subgroup_op(name, opc):
    can_eliminate = False, schedule_class = "barrier",
    agx3_mnemonic = "barrier")
 
-# Direct-absolute scalar coverage-stack access. These internal pseudos are the
+# Direct-absolute coverage-stack access (word vectors or narrow scalars). These internal pseudos are the
 # bounded landing point for future ray-query lowering; no generic address-space
 # operation is silently reinterpreted as SCS.
 op("scs_load_agx3", _, dests = 1, srcs = 0, imms = [IMM, FORMAT, MASK],
@@ -528,6 +542,11 @@ def subgroup_op(name, opc):
 op("scs_store_agx3", _, dests = 0, srcs = 1, imms = [IMM, FORMAT, MASK],
    can_eliminate = False, can_reorder = False, schedule_class = "store", agx3_mnemonic = "stscs")
 
+# Uniform-base plus scalar-GPR-index SCS access.  The source order keeps the
+# address provenance explicit instead of recovering it from generic pointer
+# arithmetic after the fact: load(base, index), store(value, base, index).
+# The native address selector is explicit and independent of FORMAT; the
+# backend refuses other address forms.
 op("scs_load_uniform_indexed_agx3", _, dests = 1, srcs = 2,
    imms = [IMM, FORMAT, MASK, SCS_ADDRESS_SELECTOR, SCS_ZERO_EXTEND, SCS_C_LOAD_MARKER],
    can_reorder = False, schedule_class = "load", agx3_mnemonic = "ldscs")
@@ -535,13 +554,18 @@ def subgroup_op(name, opc):
    imms = [IMM, FORMAT, MASK, SCS_ADDRESS_SELECTOR], can_eliminate = False, can_reorder = False, schedule_class = "store",
    agx3_mnemonic = "stscs")
 
+# Explicit two-GPR M5 SCS addressing, including the instanced traversal form.
+# Source order matches the uniform-indexed pseudos: (base, index) and
+# (value, base, index). The native address selector is preserved explicitly.
 op("scs_load_register_indexed_agx3", _, dests = 1, srcs = 2,
-   imms = [IMM, FORMAT, MASK, SCS_ADDRESS_SELECTOR, SCS_C_LOAD_MARKER],
+   imms = [IMM, FORMAT, MASK, SCS_ADDRESS_SELECTOR, SCS_C_LOAD_MARKER, SCS_RT_COMPLETION],
    can_reorder = False, schedule_class = "load", agx3_mnemonic = "ldscs")
 op("scs_store_register_indexed_agx3", _, dests = 0, srcs = 3,
    imms = [IMM, FORMAT, MASK, SCS_ADDRESS_SELECTOR],
    can_eliminate = False, can_reorder = False, schedule_class = "store", agx3_mnemonic = "stscs")
 
+# Register-base form has no index operand. Its special status load cannot be
+# represented as a normal byte load: both native control fields are explicit.
 op("scs_load_register_base_agx3", _, dests = 1, srcs = 1,
    imms = [IMM, FORMAT, MASK, SCS_ADDRESS_SELECTOR, SCS_C_LOAD_MARKER,
            SCS_C_STATUS_LOAD],
@@ -558,6 +582,19 @@ def subgroup_op(name, opc):
 op("continue_ray_gen1", _, dests=0, srcs=1, imms=[IMM],
    can_eliminate=False, can_reorder=False, schedule_class="barrier",
    agx3_mnemonic="continue_ray")
+# M5 native traversal entry. Two native operands are 64-bit; the BVH operand
+# may remain uniform. Three explicit 32-bit launch words are published to
+# staging halves 0/2/4 immediately before the instruction. Completion is
+# observed by the SCS polling protocol, not a fabricated GPR result.
+op("intersect_ray_m5", _, dests = 0, srcs = 5,
+   can_eliminate = False, can_reorder = False, schedule_class = "barrier",
+   agx3_mnemonic = "intersect_ray")
+op("enter_query_m5", _, dests = 1, srcs = 0, imms = [IMM],
+   can_eliminate = False, can_reorder = False, schedule_class = "barrier",
+   agx3_mnemonic = "incStk")
+op("continue_ray_m5", _, dests = 0, srcs = 1,
+   can_eliminate = False, can_reorder = False, schedule_class = "barrier",
+   agx3_mnemonic = "continue_ray")
 
 def memory_barrier(name, a, b, c):
     op(name, (0xF5 | (a << 10) | (b << 8) | (c << 12), 2, _), dests = 0, srcs = 0,
@@ -642,6 +679,9 @@ def memory_barrier(name, a, b, c):
 # Native normalized RGB10A2 conversion, four floats to/from one packed word.
 op("pack_rgb10a2_agx3", _, dests=1, srcs=4, is_float=True, agx3_mnemonic="pack")
 op("unpack_rgb10a2_agx3", _, dests=1, srcs=1, agx3_mnemonic="unpack")
+# G17G packs reusable half-float LOD/bias values into normal register words.
+# Pure SSA producers can be CSE'd before each sample publishes its own COPY.
+op("pack_sampler_lod_agx3", _, dests=1, srcs=1, agx3_mnemonic="pack_smpir")
 # G17P's native coverage word is prepared before RA, from SR2 at one sample.
 # Keep the general sample-mask target explicit while selecting its native zero
 # no-op.
diff --git a/src/asahi/compiler/agx_opt_compact_constants.c b/src/asahi/compiler/agx_opt_compact_constants.c
index 7fcccf07356d..574e41bc1661 100644
--- a/src/asahi/compiler/agx_opt_compact_constants.c
+++ b/src/asahi/compiler/agx_opt_compact_constants.c
@@ -24,10 +24,20 @@ agx_opt_compact_constants(agx_context *ctx)
    BITSET_WORD *src_other = calloc(ctx->alloc, sizeof(BITSET_WORD));
    BITSET_WORD *replaced = calloc(ctx->alloc, sizeof(BITSET_WORD));
 
+   /* M5's f32 add takes at most one half register, so a constant added to a
+    * half in an f32 add must stay 32-bit. */
+   const bool agx3c = ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C;
+
    /* Analyze the types that we read constants as */
    agx_foreach_instr_global(ctx, I) {
       agx_foreach_ssa_src(I, s) {
-         if (agx_is_float_src(I, s))
+         const bool half_pair =
+            agx3c && I->op == AGX_OPCODE_FADD && I->nr_srcs == 2 &&
+            I->dest[0].size == AGX_SIZE_32 && s < 2 &&
+            I->src[1 - s].type == AGX_INDEX_NORMAL &&
+            I->src[1 - s].size == AGX_SIZE_16;
+
+         if (agx_is_float_src(I, s) && !half_pair)
             BITSET_SET(src_float, I->src[s].value);
          else
             BITSET_SET(src_other, I->src[s].value);
diff --git a/src/asahi/compiler/agx_opt_cse.c b/src/asahi/compiler/agx_opt_cse.c
index 5b80ee2149ae..7e312d1894dc 100644
--- a/src/asahi/compiler/agx_opt_cse.c
+++ b/src/asahi/compiler/agx_opt_cse.c
@@ -80,6 +80,8 @@ instrs_equal(const void *_i1, const void *_i2)
       return false;
    if (i1->agx3_cube_scalar_coords != i2->agx3_cube_scalar_coords)
       return false;
+   if (i1->agx3_split_coords != i2->agx3_split_coords)
+      return false;
    if (i1->offset != i2->offset)
       return false;
    /* Two samples that differ only in their coordinate offset are different
@@ -103,6 +105,8 @@ instrs_equal(const void *_i1, const void *_i2)
       return false;
    if (i1->memory_index_is_bytes != i2->memory_index_is_bytes)
       return false;
+   if (i1->no_signed_zero != i2->no_signed_zero)
+      return false;
    if (i1->saturate != i2->saturate)
       return false;
    if (i1->mask != i2->mask)
diff --git a/src/asahi/compiler/agx_opt_jmp_none.c b/src/asahi/compiler/agx_opt_jmp_none.c
index cd9027126735..b3ffc7e1387d 100644
--- a/src/asahi/compiler/agx_opt_jmp_none.c
+++ b/src/asahi/compiler/agx_opt_jmp_none.c
@@ -41,6 +41,21 @@ cost_instr(agx_instr *I)
    }
 }
 
+/*
+ * A coverage update that runs the depth/stencil tests (AGX3 MOVMSK with
+ * flag 2) must not execute with an empty EXEC. agx_nir_split_g17p_sample_tests
+ * puts it in its own arm on every AGX3 encoding (G17P and M5), and a SIMD
+ * group whose live lanes all take the kill arms instead otherwise still runs
+ * the arm's pixel wait, update and release with no active lane, which stops
+ * the 3D stage.
+ */
+static bool
+testing_coverage_update(const agx_context *ctx, const agx_instr *I)
+{
+   return agx_isa_is_agx3(ctx->key->dev.encoding) &&
+          I->op == AGX_OPCODE_SAMPLE_MASK_AGX3 && (I->imm & 2);
+}
+
 /*
  * Estimate the cost between the instruction and the branch target. This is an
  * input for our heuristic. The branch target is guaranteed to be a forward
@@ -60,8 +75,7 @@ cost_between(agx_context *ctx, agx_block *from, agx_instr *from_I,
              J == agx_last_instr(target))
             break;
 
-         *must_skip |= ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-                       J->op == AGX_OPCODE_SAMPLE_MASK_AGX3 && (J->imm & 2);
+         *must_skip |= testing_coverage_update(ctx, J);
          cost += cost_instr(J);
       }
    }
@@ -81,8 +95,7 @@ cost_between(agx_context *ctx, agx_block *from, agx_instr *from_I,
          if (block == target && I == agx_last_instr(target))
             break;
 
-         *must_skip |= ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
-                       I->op == AGX_OPCODE_SAMPLE_MASK_AGX3 && (I->imm & 2);
+         *must_skip |= testing_coverage_update(ctx, I);
          cost += cost_instr(I);
       }
 
diff --git a/src/asahi/compiler/agx_opt_promote_constants.c b/src/asahi/compiler/agx_opt_promote_constants.c
index 731696cee025..9f8a50b60ea4 100644
--- a/src/asahi/compiler/agx_opt_promote_constants.c
+++ b/src/asahi/compiler/agx_opt_promote_constants.c
@@ -90,7 +90,13 @@ constant_use_accepts_uniform(agx_context *ctx, const agx_instr *I, unsigned s,
    if (agx_instr_accepts_uniform(ctx->key->dev.encoding, I->op, s, uniform, size))
       return true;
 
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B || I->nr_srcs > 3 ||
+   /* M5 floating-point operations read an aligned f32 uniform exactly where
+    * the native uniform-operand encoder admits it. */
+   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       agx3_m5_float_accepts_uniform(ctx, I, s, uniform, size))
+      return true;
+
+   if (I->nr_srcs > 3 ||
        uniform >= AGX_NUM_UNIFORMS)
       return false;
 
@@ -100,11 +106,56 @@ constant_use_accepts_uniform(agx_context *ctx, const agx_instr *I, unsigned s,
    candidate.src = sources;
    sources[s] = agx_replace_index(sources[s], agx_uniform(uniform, size));
    struct agx3_uniform_integer_desc desc;
-   return agx3_uniform_integer_instruction(&candidate, false, &desc);
+   return agx3_uniform_integer_for_encoding(ctx->key->dev.encoding,
+                                            &candidate, false, &desc);
+}
+
+/* The native R/U/U IMADD endpoint can promote two constants together.
+ * Probing C alone would falsely reject it before the multiplier is promoted.
+ * Do not admit the unwitnessed R/R/U endpoint to make that probe pass. */
+static bool
+m5_constant_mad_pair(agx_context *ctx, const agx_instr *I,
+                     const BITSET_WORD *is_immediate, unsigned uniform,
+                     unsigned *multiplier)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       I->op != AGX_OPCODE_IMAD || I->nr_srcs != 3 || uniform > 508 ||
+       I->src[0].type != AGX_INDEX_NORMAL ||
+       I->src[1].type != AGX_INDEX_NORMAL ||
+       I->src[2].type != AGX_INDEX_NORMAL ||
+       !BITSET_TEST(is_immediate, I->src[2].value))
+      return false;
+   const bool a = BITSET_TEST(is_immediate, I->src[0].value);
+   const bool b = BITSET_TEST(is_immediate, I->src[1].value);
+   if (a == b)
+      return false;
+   *multiplier = a ? 0 : 1;
+   agx_instr candidate = *I;
+   agx_index sources[3];
+   memcpy(sources, I->src, sizeof(sources));
+   candidate.src = sources;
+   sources[*multiplier] = agx_replace_index(
+      sources[*multiplier], agx_uniform(uniform, I->src[*multiplier].size));
+   sources[2] = agx_replace_index(sources[2],
+                                agx_uniform(uniform + 2, I->src[2].size));
+   struct agx3_uniform_integer_desc desc;
+   return agx3_uniform_integer_for_encoding(ctx->key->dev.encoding,
+                                            &candidate, false, &desc);
+}
+
+/* The late M5 round promotes only float constants that M5 constant
+ * materialization left in GPRs for native uniform-operand forms. */
+static bool
+late_m5_float_use(agx_context *ctx, const agx_instr *I, unsigned s,
+                  unsigned uniform, enum agx_size size)
+{
+   return (I->op == AGX_OPCODE_FADD || I->op == AGX_OPCODE_FMUL ||
+           I->op == AGX_OPCODE_FFMA) &&
+          agx3_m5_float_accepts_uniform(ctx, I, s, uniform, size);
 }
 
 static void
-pass(agx_context *ctx, void *memctx)
+pass(agx_context *ctx, void *memctx, bool late_m5_float)
 {
    /* Map from SSA indices to struct constant_info */
    struct hash_table_u64 *constants = _mesa_hash_table_u64_create(memctx);
@@ -132,7 +183,23 @@ pass(agx_context *ctx, void *memctx)
          assert(I->dest[0].type == AGX_INDEX_NORMAL);
          BITSET_SET(is_immediate, I->dest[0].value);
          values[I->dest[0].value] = I->imm;
+      } else if (late_m5_float) {
+         agx_foreach_ssa_src(I, s) {
+            if (BITSET_TEST(is_immediate, I->src[s].value) &&
+                late_m5_float_use(ctx, I, s, probe_uniform, I->src[s].size))
+               record_use(memctx, constants, values[I->src[s].value],
+                          I->src[s].size);
+         }
       } else {
+         unsigned multiplier;
+         if (m5_constant_mad_pair(ctx, I, is_immediate, probe_uniform,
+                                 &multiplier)) {
+            record_use(memctx, constants, values[I->src[multiplier].value],
+                       I->src[multiplier].size);
+            record_use(memctx, constants, values[I->src[2].value],
+                       I->src[2].size);
+            continue;
+         }
          agx_foreach_ssa_src(I, s) {
             if (BITSET_TEST(is_immediate, I->src[s].value) &&
                 constant_use_accepts_uniform(ctx, I, s, probe_uniform,
@@ -189,7 +256,7 @@ pass(agx_context *ctx, void *memctx)
        * zero if we needed to pad, compared to doing this before the loop. Same
        * shader code, but slightly more efficient USC record.
        */
-      if (i == 0) {
+      if (i == 0 && !(late_m5_float && ctx->out->rodata.size_16)) {
          ctx->out->rodata.base_uniform = uniform;
       }
 
@@ -253,6 +320,34 @@ pass(agx_context *ctx, void *memctx)
 
    /* Promote in the IR */
    agx_foreach_instr_global(ctx, I) {
+      if (late_m5_float) {
+         agx_foreach_ssa_src(I, s) {
+            if (!BITSET_TEST(is_immediate, I->src[s].value))
+               continue;
+            struct constant_info *info =
+               _mesa_hash_table_u64_search(constants, values[I->src[s].value]);
+            if (info && info->promoted &&
+                late_m5_float_use(ctx, I, s, info->uniform, I->src[s].size))
+               agx_replace_src(I, s,
+                               agx_uniform(info->uniform, I->src[s].size));
+         }
+         continue;
+      }
+      unsigned multiplier;
+      if (m5_constant_mad_pair(ctx, I, is_immediate, probe_uniform,
+                              &multiplier)) {
+         struct constant_info *m = _mesa_hash_table_u64_search(
+            constants, values[I->src[multiplier].value]);
+         struct constant_info *c = _mesa_hash_table_u64_search(
+            constants, values[I->src[2].value]);
+         if (m && c && m->promoted && c->promoted) {
+            /* Both values own aligned uniforms before either source changes. */
+            agx_replace_src(I, multiplier,
+                            agx_uniform(m->uniform, I->src[multiplier].size));
+            agx_replace_src(I, 2, agx_uniform(c->uniform, I->src[2].size));
+            continue;
+         }
+      }
       agx_foreach_ssa_src(I, s) {
          if (!BITSET_TEST(is_immediate, I->src[s].value))
             continue;
@@ -280,6 +375,30 @@ agx_opt_promote_constants(agx_context *ctx)
       return;
 
    void *memctx = ralloc_context(NULL);
-   pass(ctx, memctx);
+   pass(ctx, memctx, false);
+   ralloc_free(memctx);
+}
+
+/*
+ * M5 constant materialization (agx_materialize_agx3_alu_constants) runs after
+ * the ordinary round and moves immediates that no M5 form can read into GPRs,
+ * e.g. the 0.25 multiplicand of Minecraft's RGSS fma(x, 0.25, -y). Apple keeps
+ * such constants in uniforms. Promote those feeding a native M5 float
+ * uniform-operand form, appending to the ordinary round's constant range so
+ * the program keeps one contiguous rodata upload.
+ */
+void
+agx_opt_promote_m5_float_constants(agx_context *ctx)
+{
+   const struct agx_rodata *rodata = &ctx->out->rodata;
+   if (ctx->is_preamble ||
+       ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       (ctx->key->g17p_rodata_hint && ctx->key->g17p_rodata_hint_size_16) ||
+       (rodata->size_16 &&
+        ctx->out->push_count != rodata->base_uniform + rodata->size_16))
+      return;
+
+   void *memctx = ralloc_context(NULL);
+   pass(ctx, memctx, true);
    ralloc_free(memctx);
 }
diff --git a/src/asahi/compiler/agx_opt_register_cache.c b/src/asahi/compiler/agx_opt_register_cache.c
index 2d3de5827c2f..e77fa78af119 100644
--- a/src/asahi/compiler/agx_opt_register_cache.c
+++ b/src/asahi/compiler/agx_opt_register_cache.c
@@ -6,11 +6,102 @@
 
 #include <stdlib.h>
 #include "util/bitset.h"
+#include "util/lut.h"
 #include "util/u_debug.h"
 #include "agx_compiler.h"
 #include "agx_debug.h"
 #include "shader_enums.h"
 
+DEBUG_GET_ONCE_BOOL_OPTION(m5_redundant_copies,
+                           "ASAHI_M5_REDUNDANT_COPIES", true)
+
+/* Remove bit-identical overwrites only after allocation, without extending
+ * allocation live ranges. The cache pass below recomputes all use hints. */
+static void
+agx3_remove_redundant_physical_copies(agx_context *ctx)
+{
+   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       ctx->stage != MESA_SHADER_FRAGMENT || ctx->is_preamble ||
+       !debug_get_option_m5_redundant_copies())
+      return;
+
+   uint32_t serial = 0;
+   agx_foreach_block(ctx, block) {
+      uint32_t value[AGX_NUM_REGS] = {0};
+      agx_foreach_instr_in_block_safe(block, I) {
+         const struct agx_opcode_info *info = &agx_opcodes_info[I->op];
+         if (!info->can_eliminate || !info->can_reorder ||
+             info->schedule_class != AGX_SCHEDULE_CLASS_NONE ||
+             !I->nr_dests) {
+            memset(value, 0, sizeof(value));
+            continue;
+         }
+
+         uint32_t copied[4] = {0}, old_value[4] = {0};
+         bool copy = false;
+         const agx_index dst = I->nr_dests == 1 ? I->dest[0] : agx_null();
+         const unsigned count = agx_is_null(dst) ? 0 : agx_index_size_16(dst);
+         const bool plain_dst = dst.type == AGX_INDEX_REGISTER && !dst.memory &&
+            !dst.abs && !dst.neg && !I->saturate && !I->shift &&
+            !I->invert_cond && !I->memory_is_packed &&
+            count >= 1 && count <= 4 && dst.value + count <= AGX_NUM_REGS;
+         if (plain_dst) {
+            for (unsigned c = 0; c < count; ++c)
+               old_value[c] = value[dst.value + c];
+         }
+         if (plain_dst && I->op == AGX_OPCODE_MOV_IMM && count <= 2) {
+            for (unsigned c = 0; c < count; ++c)
+               copied[c] = UINT32_C(0x80000000) |
+                           ((I->imm >> (16 * c)) & 0xffff);
+            copy = true;
+         } else if (plain_dst && I->op == AGX_OPCODE_BITOP &&
+                    I->truth_table == UTIL_LUT2(a) && I->nr_srcs == 2) {
+            const agx_index src = I->src[0];
+            if (!src.memory && !src.abs && !src.neg && src.size == dst.size &&
+                agx_index_size_16(src) == count) {
+               if (src.type == AGX_INDEX_REGISTER &&
+                   src.value + count <= AGX_NUM_REGS) {
+                  for (unsigned c = 0; c < count; ++c) {
+                     if (!value[src.value + c])
+                        value[src.value + c] = ++serial;
+                     copied[c] = value[src.value + c];
+                  }
+                  copy = true;
+               } else if (src.type == AGX_INDEX_UNIFORM &&
+                          src.value + count <= 512) {
+                  for (unsigned c = 0; c < count; ++c)
+                     copied[c] = UINT32_C(0x40000000) | (src.value + c);
+                  copy = true;
+               }
+            }
+         }
+         if (copy) {
+            bool redundant = true;
+            for (unsigned c = 0; c < count; ++c)
+               redundant &= old_value[c] == copied[c];
+            if (redundant) {
+               agx_remove_instruction(I);
+               continue;
+            }
+            for (unsigned c = 0; c < count; ++c)
+               value[dst.value + c] = copied[c];
+         } else {
+            agx_foreach_dest(I, d) {
+               const agx_index dest = I->dest[d];
+               const unsigned n = agx_index_size_16(dest);
+               if (dest.type != AGX_INDEX_REGISTER || dest.memory ||
+                   dest.value + n > AGX_NUM_REGS) {
+                  memset(value, 0, sizeof(value));
+                  break;
+               }
+               for (unsigned c = 0; c < n; ++c)
+                  value[dest.value + c] = ++serial;
+            }
+         }
+      }
+   }
+}
+
 static void
 postra_liveness_ins(BITSET_WORD *live, agx_instr *I)
 {
@@ -97,6 +188,8 @@ writes_reg(const agx_instr *I, unsigned reg)
 void
 agx_opt_register_cache(agx_context *ctx)
 {
+   agx3_remove_redundant_physical_copies(ctx);
+
    /* Analyze the shader globally */
    postra_liveness(ctx);
 
@@ -162,7 +255,7 @@ agx_opt_register_cache(agx_context *ctx)
     * reads are at the top of the loop, across the back edge. Cache a loop
     * compare's register source when the loop header reads it with an ALU
     * instruction before writing it, unless this read is its last use. */
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_block(ctx, block) {
diff --git a/src/asahi/compiler/agx_optimizer.c b/src/asahi/compiler/agx_optimizer.c
index 49118e8e9841..1df093b72249 100644
--- a/src/asahi/compiler/agx_optimizer.c
+++ b/src/asahi/compiler/agx_optimizer.c
@@ -9,6 +9,7 @@
 #include "agx3_select_core.h"
 #include "agx_compiler.h"
 #include "agx_debug.h"
+#include "util/u_debug.h"
 #include "agx_opcodes.h"
 
 /* AGX peephole optimizer responsible for instruction combining. It operates in
@@ -87,7 +88,10 @@ agx_compose_float_src(agx_index to, agx_index from)
 static bool
 agx_preserve_g17p_tied_fcmpsel(agx_context *ctx, agx_instr *I, unsigned src)
 {
-   return ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   return ctx->key &&
+          (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+            (I->fcond == AGX_FCOND_LTN || I->fcond == AGX_FCOND_GTN))) &&
           I->op == AGX_OPCODE_FCMPSEL && src < 4 &&
           agx_is_equiv(I->src[src], I->src[src ^ 2]);
 }
@@ -248,10 +252,25 @@ agx_optimizer_fmov(agx_context *ctx, agx_instr **defs, agx_instr *ins)
             agx_can_fold_half_uniform(ins, defs)))
          continue;
 
+      /* M5's f32 add takes at most one half register (f32 + f16, or a half
+       * plus a minifloat immediate); two half registers into an f32 result
+       * have no encoding. Keep the second widening explicit. */
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          ins->op == AGX_OPCODE_FADD && ins->nr_srcs == 2 &&
+          ins->dest[0].size == AGX_SIZE_32 && src.size == AGX_SIZE_32 &&
+          def->src[0].type == AGX_INDEX_NORMAL &&
+          def->src[0].size == AGX_SIZE_16) {
+         const agx_index other = ins->src[1 - s];
+         if (other.type == AGX_INDEX_NORMAL && other.size == AGX_SIZE_16)
+            continue;
+      }
+
       /* FLTNORM and RGB10A2 packing have only captured f32 sources. Keep
        * the explicit widening used by linked half-width fragment outputs
        * before normalized packing. */
-      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (ctx->key &&
+          (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
           (ins->op == AGX_OPCODE_FLTNORM_AGX3 ||
            ins->op == AGX_OPCODE_PACK_RGB10A2_AGX3) &&
           src.size == AGX_SIZE_32 &&
@@ -308,6 +327,9 @@ image_write_source_can_be_immediate(agx_instr *I, unsigned s)
 static void
 agx_optimizer_inline_imm(agx_context *ctx, agx_instr **defs, agx_instr *I)
 {
+   if (I->op == AGX_OPCODE_MATMAD8_AGX3 ||
+       I->op == AGX_OPCODE_MATMAD16_AGX3)
+      return;
    agx_foreach_ssa_src(I, s) {
       agx_index src = I->src[s];
       if (I->op == AGX_OPCODE_INTERSECT_RAY_GEN1 ||
@@ -333,6 +355,32 @@ agx_optimizer_inline_imm(agx_context *ctx, agx_instr **defs, agx_instr *I)
           I->op == AGX_OPCODE_STORE_OBJECT_CHILDCOUNT_M5)
          continue;
 
+      /* M5 reductions read a register even when every active lane contributes
+       * the same constant, as for the active invocation count. */
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_SIMD_REDUCE)
+         continue;
+
+      /* Coverage-stack address/data operands, traversal addresses, and UVB
+       * allocation/descriptor and child-count operands have register forms
+       * only. Keep constant materialization intact. */
+      if (I->op == AGX_OPCODE_SCS_STORE_AGX3 ||
+          I->op == AGX_OPCODE_SCS_LOAD_REGISTER_INDEXED_AGX3 ||
+          I->op == AGX_OPCODE_SCS_STORE_REGISTER_INDEXED_AGX3 ||
+          I->op == AGX_OPCODE_SCS_LOAD_REGISTER_BASE_AGX3 ||
+          I->op == AGX_OPCODE_SCS_STORE_REGISTER_BASE_AGX3 ||
+          I->op == AGX_OPCODE_SCS_LOAD_UNIFORM_INDEXED_AGX3 ||
+          I->op == AGX_OPCODE_SCS_STORE_UNIFORM_INDEXED_AGX3 ||
+          I->op == AGX_OPCODE_INTERSECT_RAY_M5 ||
+          I->op == AGX_OPCODE_ALLOCATE_UVB_M5 ||
+          I->op == AGX_OPCODE_STORE_UVB_M5 ||
+          I->op == AGX_OPCODE_STORE_OBJECT_CHILDCOUNT_M5)
+         continue;
+
+      /* PACK_SMPIR-to-GPR takes a half register, not an integer literal. */
+      if (I->op == AGX_OPCODE_PACK_SAMPLER_LOD_AGX3)
+         continue;
+
       /* G17P's native subtraction takes two registers. Instruction selection
        * deliberately materializes an immediate minuend for `c - x`; folding
        * it back here would leave immediate-plus-negated-register, whose iadd
@@ -359,7 +407,9 @@ agx_optimizer_inline_imm(agx_context *ctx, agx_instr **defs, agx_instr *I)
        * at all -- agx3_select_core.c's AGX3_SEL_OP_ASR requires two plain
        * 32-bit registers -- and the packer then refuses the whole shader.
        * Keep the MOV so selection reproduces Apple's ldimm+sra pair. */
-      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (ctx->key &&
+          (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
           I->op == AGX_OPCODE_ASR && s == 0)
          continue;
 
@@ -515,6 +565,15 @@ agx_optimizer_fmov_rev(agx_context *ctx, agx_instr *I, agx_instr *use)
        use->saturate)
       return false;
 
+   /* M5 (AGX3C) encodes reciprocal and reciprocal square root without a
+    * saturating form, so a saturate folded into one made the shader
+    * unencodable (Skyrim SE's fragment shaders clamp a reciprocal). Keep the
+    * separate saturating move there. */
+   if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       use->saturate &&
+       (I->op == AGX_OPCODE_RCP || I->op == AGX_OPCODE_RSQRT))
+      return false;
+
    /* We can fold f2f16 into 32-bit instructions, but we can't fold f2f32 into
     * 16-bit instructions, since the latter would implicitly promote to a 32-bit
     * instruction which is not exact.
@@ -535,6 +594,19 @@ agx_optimizer_fmov_rev(agx_context *ctx, agx_instr *I, agx_instr *use)
        I->op != AGX_OPCODE_FMUL)
       return false;
 
+   /* M5 (AGX3C): an f16-destination ADD/FMA fed by an immediate has no
+    * grounded encoding -- the packer refuses `ffma d16, r32, #imm16, r32`
+    * (RayTracedGL1's IndirectInit raygen). Keep the explicit narrowing move in
+    * that case only; register-only producers still fold. */
+   if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       I->dest[0].size == AGX_SIZE_32 && use->dest[0].size == AGX_SIZE_16 &&
+       I->op != AGX_OPCODE_FMUL) {
+      agx_foreach_src(I, s) {
+         if (I->src[s].type == AGX_INDEX_IMMEDIATE)
+            return false;
+      }
+   }
+
    /* ...and that narrowing FMUL takes plain registers only: a multiply that
     * already carries |x| or -x on a source keeps its explicit f32-to-f16
     * move (Skia's conic/quad edge shaders narrow abs(f) * rsqrt(g)). */
@@ -704,6 +776,32 @@ agx_optimizer_copyprop(agx_context *ctx, agx_instr **defs, agx_instr *I)
 
       if (def == NULL)
          continue; /* happens for phis in loops */
+
+      /* Preserve a pointer assembled from adjacent uniform words instead of
+       * materializing its two words in registers. M5 loads can address the
+       * complete uniform file with an aligned 64-bit base. */
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_DEVICE_LOAD && s == 0 &&
+          src.size == AGX_SIZE_64 && agx_channels(src) == 1 &&
+          !src.abs && !src.neg && !src.memory &&
+          def->op == AGX_OPCODE_COLLECT && def->nr_srcs == 2 &&
+          def->nr_dests == 1 && def->dest[0].size == AGX_SIZE_64 &&
+          agx_channels(def->dest[0]) == 1 && !def->dest[0].memory &&
+          !def->saturate &&
+          debug_get_bool_option("ASAHI_M5_HIGH_UNIFORM_LOAD", true)) {
+         const agx_index lo = def->src[0], hi = def->src[1];
+         if (lo.type == AGX_INDEX_UNIFORM && hi.type == AGX_INDEX_UNIFORM &&
+             lo.size == AGX_SIZE_32 && hi.size == AGX_SIZE_32 &&
+             agx_channels(lo) == 1 && agx_channels(hi) == 1 &&
+             !lo.abs && !lo.neg && !lo.memory &&
+             !hi.abs && !hi.neg && !hi.memory &&
+             !(lo.value & 3) && hi.value == lo.value + 2 &&
+             agx_instr_accepts_uniform(ctx->key->dev.encoding, I->op, s,
+                                       lo.value, AGX_SIZE_64)) {
+            agx_replace_src(I, s, agx_uniform(lo.value, AGX_SIZE_64));
+            continue;
+         }
+      }
       if (def->op != AGX_OPCODE_MOV)
          continue;
 
@@ -715,6 +813,23 @@ agx_optimizer_copyprop(agx_context *ctx, agx_instr **defs, agx_instr *I)
           !(def->src[0].size < src.size && agx_supports_zext(I, s)))
          continue;
 
+      /* M5's mixed-width compare path does not preserve a NIR unsigned
+       * half-to-word conversion. A native SCS u16 load correctly returned
+       * 0x0000c56d, but comparison against a folded half operand rejected it;
+       * the same compiled test passed below bit15. Keep the explicit widening
+       * before comparing. This is the value conversion required by NIR, not
+       * an alteration of the SCS load format or its tested extension bit. */
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          def->src[0].size == AGX_SIZE_16 && src.size == AGX_SIZE_32 && s < 2 &&
+          (I->op == AGX_OPCODE_ICMP || I->op == AGX_OPCODE_ICMPSEL))
+         continue;
+
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_BITOP && I->dest[0].size == AGX_SIZE_32 &&
+          def->src[0].size == AGX_SIZE_16 && src.size == AGX_SIZE_32 &&
+          def->src[0].type != AGX_INDEX_IMMEDIATE)
+         continue;
+
       /* AGX2 can carry the zero-extension modifier on integer arithmetic and
        * compare-select, but the grounded G17P forms are width-matched.
        * Preserve the explicit 16-to-32 move so the AGX3 packer emits its
@@ -731,6 +846,26 @@ agx_optimizer_copyprop(agx_context *ctx, agx_instr **defs, agx_instr *I)
            I->op == AGX_OPCODE_ICMP_QUAD_BALLOT))
          continue;
 
+      /* M5's funnel extract takes two word registers. Only a zero-extended
+       * low half shifted out entirely (an immediate shift of 16 or more,
+       * which legalization turns into a bitfield insert) may stay narrow;
+       * keep the explicit extension move for every other half source. */
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          def->src[0].size == AGX_SIZE_16 && src.size == AGX_SIZE_32 &&
+          I->op == AGX_OPCODE_EXTR && s < 2 &&
+          !(s == 0 && I->src[2].type == AGX_INDEX_IMMEDIATE &&
+            I->src[2].value >= 16 && I->src[2].value <= 31))
+         continue;
+
+      /* The AGX3C saturating add has only a witnessed 32-bit register form
+       * (iadd/isub with f53, agx3_pack_binary.c); a zero-extended 16-bit
+       * source is refused by the packer. Keep the explicit extension move,
+       * as the AGX3B rule above does for all of these ops. */
+      if (ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          def->src[0].size == AGX_SIZE_16 && src.size == AGX_SIZE_32 &&
+          I->op == AGX_OPCODE_IADD && I->saturate)
+         continue;
+
       /* Optimize split(64-bit uniform) so we can get better copyprop of the
        * 32-bit uniform parts. This helps reduce moves with 64-bit uniforms.
        */
@@ -757,16 +892,38 @@ agx_optimizer_copyprop(agx_context *ctx, agx_instr **defs, agx_instr *I)
       if (def->src[0].type == AGX_INDEX_IMMEDIATE)
          continue;
 
+      /* Native G17S texel fetches can read a static descriptor directly
+       * from CProg uniforms. Keep it through copy propagation, then let
+       * uniform-source validation enforce the witnessed range/alignment. */
+      const bool m5_texture_uniform =
+         ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+         I->op == AGX_OPCODE_TEXTURE_LOAD && s == 3 &&
+         def->src[0].type == AGX_INDEX_UNIFORM &&
+         debug_get_bool_option("ASAHI_M5_UNIFORM_TEXTURE_OPERANDS", true) &&
+         agx_instr_accepts_uniform(ctx->key->dev.encoding, I->op, s,
+                                  def->src[0].value, def->src[0].size);
+
+      /* An M5 fragment SMP reads its texture and sampler handles from a
+       * pair of aligned 64-bit uniforms (native G17G compiler
+       * argument-buffer samples); forwarded below only as a whole pair. */
+      const bool m5_sample_uniform =
+         ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+         ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_preamble &&
+         I->op == AGX_OPCODE_TEXTURE_SAMPLE && (s == 3 || s == 4) &&
+         def->src[0].type == AGX_INDEX_UNIFORM;
+
       /* ALU instructions cannot take 64-bit. G17P SMP's texture/sampler index
        * operands can (agx_lower_uniform_sources
        * copies back any placement the native form cannot encode). */
-      if (def->src[0].size == AGX_SIZE_64 &&
+      if (def->src[0].size == AGX_SIZE_64 && !m5_texture_uniform &&
+          !m5_sample_uniform &&
           !(I->op == AGX_OPCODE_DEVICE_LOAD && s == 0) &&
           !(I->op == AGX_OPCODE_DEVICE_STORE && s == 1) &&
           !(I->op == AGX_OPCODE_ATOMIC && s == 1) &&
           !(I->op == AGX_OPCODE_INTERSECT_RAY_GEN1 && s < 2) &&
-          !(I->op == AGX_OPCODE_SCS_LOAD_UNIFORM_INDEXED_AGX3 && s == 0) &&
-          !(I->op == AGX_OPCODE_SCS_STORE_UNIFORM_INDEXED_AGX3 && s == 1) &&
+          !((I->op == AGX_OPCODE_SCS_LOAD_UNIFORM_INDEXED_AGX3 && s == 0) ||
+            (I->op == AGX_OPCODE_SCS_STORE_UNIFORM_INDEXED_AGX3 && s == 1) ||
+            (I->op == AGX_OPCODE_INTERSECT_RAY_M5 && s == 1)) &&
           !(I->op == AGX_OPCODE_TEXTURE_SAMPLE && (s == 3 || s == 4) &&
             ctx->key && ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
             def->src[0].type == AGX_INDEX_UNIFORM))
@@ -789,6 +946,16 @@ agx_optimizer_copyprop(agx_context *ctx, agx_instr **defs, agx_instr *I)
          }
          if (agx3_tex_uniform_pair_unwitnessed(ctx, I, pair[0], pair[1]))
             continue;
+         if (m5_sample_uniform) {
+            bool both = true;
+            for (unsigned j = 0; j < 2; ++j) {
+               both &= pair[j].type == AGX_INDEX_UNIFORM &&
+                       pair[j].size == AGX_SIZE_64 && !(pair[j].value & 3) &&
+                       pair[j].value <= 252 && !pair[j].abs && !pair[j].neg;
+            }
+            if (!both)
+               continue;
+         }
       }
 
       agx_replace_src(I, s, def->src[0]);
diff --git a/src/asahi/compiler/agx_pressure_schedule.c b/src/asahi/compiler/agx_pressure_schedule.c
index 56654b2f39f0..359c43941e78 100644
--- a/src/asahi/compiler/agx_pressure_schedule.c
+++ b/src/asahi/compiler/agx_pressure_schedule.c
@@ -12,6 +12,7 @@
 #include "util/u_debug.h"
 #include "agx_debug.h"
 #include "util/sparse_bitset.h"
+#include "agx_builder.h"
 #include "agx_compiler.h"
 #include "agx3_fixed_dependencies.h"
 #include "agx_opcodes.h"
@@ -128,7 +129,8 @@ struct sched_dag_state {
 
 static struct dag *
 create_dag(agx_context *ctx, agx_block *block, void *memctx, bool native_uniforms,
-           struct util_dynarray *all_nodes, struct sched_dag_state *state)
+           struct util_dynarray *all_nodes, struct sched_dag_state *state,
+           bool reorder_loads)
 {
    struct dag *dag = dag_create(ctx);
    struct util_dynarray nodes;
@@ -211,6 +213,15 @@ create_dag(agx_context *ctx, agx_block *block, void *memctx, bool native_uniform
       enum agx_schedule_class dep = agx_opcodes_info[I->op].schedule_class;
       assert(dep != AGX_SCHEDULE_CLASS_INVALID && "invalid instruction seen");
 
+      /* No store of the shader can write a reorderable load's memory
+       * (memory_can_reorder), so the copy schedule may drop its memory-order
+       * edges, exactly like texture_load's. Other schedules keep them: the
+       * bottom-up pressure order would otherwise sink such loads next to
+       * their uses, shortening the latency they already hide. */
+      if (reorder_loads && I->op == AGX_OPCODE_IMAGE_LOAD &&
+          I->memory_can_reorder)
+         dep = AGX_SCHEDULE_CLASS_NONE;
+
       bool barrier = dep == AGX_SCHEDULE_CLASS_BARRIER;
       bool discards =
          I->op == AGX_OPCODE_SAMPLE_MASK || I->op == AGX_OPCODE_ZS_EMIT ||
@@ -400,7 +411,7 @@ agx_pressure_schedule(agx_context *ctx)
    agx_foreach_block(ctx, block) {
       struct sched_ctx sctx = {
          .dag = create_dag(ctx, block, memctx, native_uniforms, NULL,
-                           &dag_state),
+                           &dag_state, false),
       };
       /* Preserve the original order if a complete dependency graph cannot be
        * built (invalid range or workspace allocation failure). */
@@ -455,6 +466,18 @@ agx_pressure_schedule(agx_context *ctx)
 /* Samples a fragment block may issue ahead of the first unissued one. */
 #define AGX_ILP_FS_HOIST_SAMPLES 6
 
+/* A sample for the sample-issue schedule. M5 encodes a storage-image read as
+ * the same native SMP as a texture fetch (Apple's texture.read), so it has
+ * the same latency to hide and is batched the same way. */
+static bool
+ilp_is_sample(const agx_context *ctx, const agx_instr *I)
+{
+   return I->op == AGX_OPCODE_TEXTURE_SAMPLE ||
+          I->op == AGX_OPCODE_TEXTURE_LOAD ||
+          (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+           I->op == AGX_OPCODE_IMAGE_LOAD);
+}
+
 static unsigned
 ilp_latency(const agx_instr *I, unsigned alu, unsigned load)
 {
@@ -742,12 +765,13 @@ ilp_schedule_block(agx_context *ctx, agx_block *block, void *memctx,
                    unsigned load_latency, signed slack, signed cap,
                    bool place_exit_compare, bool order_edges_free,
                    signed fwd_cap, unsigned hoist_samples,
+                   bool hoist_past_writes,
                    struct sched_dag_state *dag_state)
 {
    struct util_dynarray all;
    util_dynarray_init(&all, memctx);
    struct dag *dag = create_dag(ctx, block, memctx, native_uniforms, &all,
-                                dag_state);
+                                dag_state, hoist_past_writes);
    if (!dag)
       return false;
 
@@ -799,15 +823,18 @@ ilp_schedule_block(agx_context *ctx, agx_block *block, void *memctx,
     * that node i feeds (itself for a sample). */
    unsigned *first_sample = NULL;
    unsigned issued_samples = 0;
+   /* With hoist_past_writes the window instead starts at the number of
+    * memory writes already issued: a copy loop then keeps hoist_samples
+    * reads in flight ahead of the write that consumes the oldest one. */
+   unsigned issued_writes = 0;
    if (hoist_samples) {
       first_sample = rzalloc_array(memctx, unsigned, n);
       if (!first_sample)
          return false;
       unsigned ordinal = 0;
       for (unsigned i = 0; i < n; ++i) {
-         const enum agx_opcode op = nodes[i]->instr->op;
-         first_sample[i] = (op == AGX_OPCODE_TEXTURE_SAMPLE ||
-                            op == AGX_OPCODE_TEXTURE_LOAD) ? ordinal++ : ~0u;
+         first_sample[i] =
+            ilp_is_sample(ctx, nodes[i]->instr) ? ordinal++ : ~0u;
       }
       for (unsigned i = n; i-- > 0;) {
          util_dynarray_foreach(&dependents[i], unsigned, d)
@@ -873,7 +900,9 @@ ilp_schedule_block(agx_context *ctx, agx_block *block, void *memctx,
          else if (!fits && grow != best_grow)
             better = grow < best_grow;
          else if (hoist_samples) {
-            const unsigned window = issued_samples + hoist_samples;
+            const unsigned window =
+               (hoist_past_writes ? issued_writes : issued_samples) +
+               hoist_samples;
             const bool hoist_i = first_sample[i] < window;
             const bool hoist_b = first_sample[best] < window;
             better = hoist_i != hoist_b ? hoist_i : i < (unsigned)best;
@@ -906,8 +935,9 @@ ilp_schedule_block(agx_context *ctx, agx_block *block, void *memctx,
 
       done[best] = true;
       order[step] = nodes[best];
-      issued_samples += nodes[best]->instr->op == AGX_OPCODE_TEXTURE_SAMPLE ||
-                        nodes[best]->instr->op == AGX_OPCODE_TEXTURE_LOAD;
+      issued_samples += ilp_is_sample(ctx, nodes[best]->instr);
+      issued_writes += nodes[best]->instr->op == AGX_OPCODE_IMAGE_WRITE ||
+                       nodes[best]->instr->op == AGX_OPCODE_DEVICE_STORE;
       changed |= (unsigned)best != step;
       cycle = best_t + 1;
       const unsigned lat = ilp_latency(nodes[best]->instr, alu_latency, load_latency);
@@ -927,7 +957,7 @@ ilp_schedule_block(agx_context *ctx, agx_block *block, void *memctx,
       for (unsigned i = 0; i < n; ++i)
          changed |= order[i]->index != i;
    }
-   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B) {
+   if (agx_isa_is_agx3(ctx->key->dev.encoding)) {
       ilp_sink_pixel_waits(order, n, dependents);
       changed = false;
       for (unsigned i = 0; i < n; ++i)
@@ -1005,15 +1035,102 @@ ilp_block_has_memory(const agx_block *block)
    return false;
 }
 
+/* A block that writes images or device memory. On M5 such a block whose
+ * reads are reorderable samples is a copy loop: Mesa's pressure order puts
+ * each write right after its read, so the program waits on one read at a
+ * time, while Apple's compiler issues several reads ahead of their writes
+ * (native725 vs_copy: four SMP, then four IMGWCI per iteration). */
+static bool
+ilp_block_writes_memory(const agx_block *block)
+{
+   agx_foreach_instr_in_block(block, I) {
+      if (I->op == AGX_OPCODE_IMAGE_WRITE || I->op == AGX_OPCODE_DEVICE_STORE)
+         return true;
+   }
+   return false;
+}
+
 static unsigned
-ilp_block_samples(const agx_block *block)
+ilp_block_samples(const agx_context *ctx, const agx_block *block)
 {
    unsigned samples = 0;
+   agx_foreach_instr_in_block(block, I)
+      samples += ilp_is_sample(ctx, I);
+   return samples;
+}
+
+/* Register halves the block's sample results occupy together. */
+static unsigned
+ilp_block_sample_halves(const agx_context *ctx, const agx_block *block)
+{
+   unsigned halves = 0;
    agx_foreach_instr_in_block(block, I) {
-      samples += I->op == AGX_OPCODE_TEXTURE_SAMPLE ||
-                 I->op == AGX_OPCODE_TEXTURE_LOAD;
+      if (!ilp_is_sample(ctx, I))
+         continue;
+      agx_foreach_ssa_dest(I, d)
+         halves += agx_index_size_16(I->dest[d]);
    }
-   return samples;
+   return halves;
+}
+
+/* Reads a copy block keeps in flight ahead of the write consuming the oldest
+ * one: Apple's vs_copy issues four SMP before its four IMGWCI. */
+#define AGX_ILP_M5_COPY_SAMPLES 4
+
+/* Samples a geometry-side M5 loop block needs for the sample-issue schedule.
+ * Its pressure order otherwise places each sample right before its first use,
+ * so the block waits for them one at a time; Apple issues a loop body's
+ * samples ahead of the consumers (native robust_loop witness). */
+#define AGX_ILP_M5_LOOP_MIN_SAMPLES 2
+
+static bool
+ilp_m5_copy_block(const agx_context *ctx, const agx_block *block)
+{
+   return ilp_block_samples(ctx, block) >= AGX_ILP_FS_MIN_SAMPLES &&
+          ilp_block_writes_memory(block);
+}
+
+/* A fully unrolled copy loop addresses each texel with its own constant
+ * coordinate, defined once ahead of the loop and so live across the whole
+ * body: 64 constants held 128 of the 150 halves of the 64x64 vs_copy loop,
+ * leaving no registers within its occupancy bucket for reads in flight.
+ * Rematerialize such constants in the copy block, ahead of their first use
+ * there (Apple computes these coordinates in the loop, with IADD). */
+static bool
+ilp_m5_sink_constants(agx_context *ctx, agx_block *block,
+                      agx_instr **defs, agx_block **def_blocks, void *memctx)
+{
+   unsigned *remap = rzalloc_array(memctx, unsigned, MAX2(ctx->alloc, 1));
+   if (!remap)
+      return false;
+
+   const unsigned old_alloc = ctx->alloc;
+   bool progress = false;
+   agx_foreach_instr_in_block(block, I) {
+      if (instr_after_logical_end(I))
+         break;
+      if (I->op == AGX_OPCODE_PHI)
+         continue;
+      agx_foreach_ssa_src(I, s) {
+         const unsigned v = I->src[s].value;
+         if (v >= old_alloc || !defs[v] || def_blocks[v] == block ||
+             defs[v]->op != AGX_OPCODE_MOV_IMM ||
+             (defs[v]->dest[0].size != AGX_SIZE_16 &&
+              defs[v]->dest[0].size != AGX_SIZE_32))
+            continue;
+         if (!remap[v]) {
+            agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+            agx_index t = agx_temp_like(ctx, defs[v]->dest[0]);
+            agx_mov_imm_to(&b, t, defs[v]->imm);
+            remap[v] = t.value;
+         }
+         agx_index replacement = I->src[s];
+         replacement.value = remap[v];
+         I->src[s] = replacement;
+         progress = true;
+      }
+   }
+   return progress;
 }
 
 /* Schedule only a small, explicit arithmetic/tile domain. Schedule-class
@@ -1023,7 +1140,7 @@ ilp_block_samples(const agx_block *block)
 static bool
 ilp_single_tile_blend(const agx_context *ctx, const agx_block *block)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B ||
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding) ||
        ctx->stage != MESA_SHADER_FRAGMENT)
       return false;
 
@@ -1073,6 +1190,32 @@ ilp_single_tile_blend(const agx_context *ctx, const agx_block *block)
    return loads == 1 && stores == 1 && load_target == store_target;
 }
 
+/* M5 compute sample-issue budget: register halves kept free below an
+ * occupancy bucket for allocation alignment, and the register file size. */
+#define AGX_ILP_M5_OCCUPANCY_MARGIN 8
+#define AGX_ILP_M5_MAX_HALVES 256
+
+/* Absolute peak register pressure of a block in source order, in halves:
+ * its live-out values plus the relative peak walking back from them. */
+static signed
+ilp_block_peak(agx_context *ctx, agx_block *block, const uint8_t *size16,
+               void *memctx)
+{
+   struct u_sparse_bitset live;
+   u_sparse_bitset_init(&live, ctx->alloc, memctx);
+   u_sparse_bitset_dup(&live, &block->live_out);
+   signed out = 0;
+   U_SPARSE_BITSET_FOREACH_SET(&live, v)
+      out += size16[v];
+   signed pressure = 0, peak = 0;
+   agx_foreach_instr_in_block_rev(block, I) {
+      pressure += calculate_pressure_delta(I, &live);
+      peak = MAX2(peak, pressure);
+      agx_liveness_ins_update(&live, I);
+   }
+   return out + peak;
+}
+
 void
 agx_ilp_schedule(agx_context *ctx)
 {
@@ -1110,29 +1253,204 @@ agx_ilp_schedule(agx_context *ctx)
       }
    }
 
+   const bool m5 = ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C;
+   const bool m5_compute =
+      m5 && ctx->stage == MESA_SHADER_COMPUTE && !ctx->is_preamble;
+   const bool m5_sample_probe = m5_compute &&
+      debug_get_bool_option("ASAHI_M5_SAMPLE_ILP_PROBE", false);
+   /* ASAHI_M5_ILP_PROBE keeps its whole-policy experiment. Otherwise M5
+    * compute uses only the sample-issue schedule below, as Apple's compute
+    * kernels issue independent samples back to back (svgfVariance: nine SMP
+    * in a row, then the accumulation); ASAHI_M5_SAMPLE_ILP=0 disables it. */
+   const bool m5_full = m5_compute &&
+      debug_get_bool_option("ASAHI_M5_ILP_PROBE", false);
+   const bool m5_samples = m5_compute &&
+      debug_get_bool_option("ASAHI_M5_SAMPLE_ILP", true);
+   /* Other M5 geometry-side stages (vertex and its lowered relatives) take
+    * the same occupancy-capped sample-issue schedule, but only for blocks
+    * that also write memory (copy loops); everything else keeps the shared
+    * AGX3 policy below. */
+   const bool m5_copy = m5 && !ctx->is_preamble &&
+      ctx->stage != MESA_SHADER_COMPUTE &&
+      ctx->stage != MESA_SHADER_FRAGMENT &&
+      debug_get_bool_option("ASAHI_M5_SAMPLE_ILP", true);
+   /* Copy blocks the M5 sample-issue schedule newly covers: every such block
+    * of those other stages, and compute loop bodies. */
+   bool *copy_mode = rzalloc_array(memctx, bool, MAX2(nr_blocks, 1));
+   if (!copy_mode) {
+      ralloc_free(memctx);
+      return;
+   }
+   bool any_copy = false;
+   if (m5_copy || (m5_samples && !m5_full)) {
+      agx_foreach_block(ctx, block) {
+         if (block->index < nr_blocks &&
+             (m5_copy || in_loop[block->index]) &&
+             ilp_m5_copy_block(ctx, block)) {
+            copy_mode[block->index] = true;
+            any_copy = true;
+         }
+      }
+   }
+   if (any_copy) {
+      agx_instr **defs = rzalloc_array(memctx, agx_instr *, MAX2(ctx->alloc, 1));
+      agx_block **def_blocks =
+         rzalloc_array(memctx, agx_block *, MAX2(ctx->alloc, 1));
+      if (!defs || !def_blocks) {
+         ralloc_free(memctx);
+         return;
+      }
+      agx_foreach_block(ctx, block) {
+         agx_foreach_instr_in_block(block, I) {
+            agx_foreach_ssa_dest(I, d) {
+               defs[I->dest[d].value] = I;
+               def_blocks[I->dest[d].value] = block;
+            }
+         }
+      }
+      bool sunk = false;
+      agx_foreach_block(ctx, block) {
+         if (block->index < nr_blocks && copy_mode[block->index])
+            sunk |= ilp_m5_sink_constants(ctx, block, defs, def_blocks, memctx);
+      }
+      if (sunk) {
+         agx_dce(ctx, true);
+         agx_compute_liveness(ctx);
+      }
+   }
+
+   /* Geometry-side M5 loop blocks with several samples and no memory writes:
+    * after agx_nir_opt_sink_loop_sample_uses the robustness2 sampled_image
+    * 3D vertex loop ends in a block holding two samples and the consumers of
+    * all twelve; the pressure order placed those two samples late, behind
+    * the consumers of the earlier ten, each completing on its own. */
+   bool *loop_sample_mode = rzalloc_array(memctx, bool, MAX2(nr_blocks, 1));
+   if (!loop_sample_mode) {
+      ralloc_free(memctx);
+      return;
+   }
+   if (m5_copy) {
+      agx_foreach_block(ctx, block) {
+         if (block->index < nr_blocks && in_loop[block->index] &&
+             !copy_mode[block->index] &&
+             ilp_block_samples(ctx, block) >= AGX_ILP_M5_LOOP_MIN_SAMPLES)
+            loop_sample_mode[block->index] = true;
+      }
+   }
+
+   /* M5 fragments: the same occupancy-capped growth as M5 compute where it
+    * exceeds the G17P fragment slack below. */
+   const bool m5_fragment = m5 && !ctx->is_preamble &&
+      ctx->stage == MESA_SHADER_FRAGMENT &&
+      debug_get_bool_option("ASAHI_M5_SAMPLE_ILP", true);
+
+   uint8_t *size16 = NULL;
+   signed m5_occupancy_cap = 0;
+   if ((m5_samples && !m5_full) || m5_copy || m5_fragment) {
+      size16 = rzalloc_array(memctx, uint8_t, MAX2(ctx->alloc, 1));
+      if (!size16) {
+         ralloc_free(memctx);
+         return;
+      }
+      agx_foreach_instr_global(ctx, I) {
+         agx_foreach_ssa_dest(I, d)
+            size16[I->dest[d].value] = agx_index_size_16(I->dest[d]);
+      }
+      signed program_peak = 0;
+      agx_foreach_block(ctx, block)
+         program_peak = MAX2(program_peak,
+                             ilp_block_peak(ctx, block, size16, memctx));
+      /* Keep a margin for register-allocation alignment. */
+      if (program_peak + AGX_ILP_M5_OCCUPANCY_MARGIN <= AGX_ILP_M5_MAX_HALVES) {
+         m5_occupancy_cap = agx_occupancy_for_register_count(
+            program_peak + AGX_ILP_M5_OCCUPANCY_MARGIN).max_registers -
+            AGX_ILP_M5_OCCUPANCY_MARGIN;
+      }
+   }
+
    agx_foreach_block(ctx, block) {
       if (block->index >= nr_blocks)
          continue;
-      if (in_loop[block->index] && !ilp_block_has_memory(block)) {
+      /* M5 compute: only the occupancy-capped sample-issue schedule (the
+       * whole policy under ASAHI_M5_ILP_PROBE). Other M5 stages take the
+       * shared AGX3 policy below, as agx_isa_is_agx3() gates it. */
+      const bool copy = copy_mode[block->index];
+      const bool loop_samples = loop_sample_mode[block->index];
+      if ((m5_compute && !m5_full) || copy || loop_samples) {
+         if ((m5_samples || copy || loop_samples) &&
+             (!in_loop[block->index] || m5_sample_probe || copy ||
+              loop_samples) &&
+             ilp_block_samples(ctx, block) >=
+                (loop_samples ? AGX_ILP_M5_LOOP_MIN_SAMPLES
+                              : AGX_ILP_FS_MIN_SAMPLES)) {
+            /* Let the block's peak grow only up to the largest register
+             * count with the occupancy the whole program already has, so
+             * batching never costs threads; a program at or past the last
+             * bucket (the spilling raygens) may not grow at all. */
+            const signed peak = ilp_block_peak(ctx, block, size16, memctx);
+            signed growth = m5_occupancy_cap > peak
+                               ? m5_occupancy_cap - peak : 0;
+            /* A loop block's samples otherwise complete one after another
+             * on every iteration. Let the block hold all of its sample
+             * results in flight, plus the fragment sample-issue slack for
+             * their operands, even past the occupancy bucket (the
+             * robustness2 3D vertex loop's last block: two vec4 results,
+             * 21 halves of growth). */
+            if (loop_samples) {
+               growth = MAX2(growth, ilp_block_sample_halves(ctx, block) +
+                                        AGX_ILP_FS_SAMPLE_SLACK);
+            }
+            ilp_schedule_block(ctx, block, memctx, native_uniforms,
+                               alu_latency, 16 * alu_latency, growth, 0,
+                               false, true, AGX_ILP_FS_SAMPLE_HALVES,
+                               copy ? AGX_ILP_M5_COPY_SAMPLES
+                                    : AGX_ILP_FS_HOIST_SAMPLES,
+                               copy, &dag_state);
+         }
+         continue;
+      }
+      /* The old ALU-only classifier predates native SCS traversal. On M5,
+       * treat every ordered memory operation and completion barrier as
+       * memory, keeping RT polling out of the ALU loop scheduling policy. */
+      bool m5_ordered_memory = false;
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+         agx_foreach_instr_in_block(block, I) {
+            const enum agx_schedule_class cls =
+               agx_opcodes_info[I->op].schedule_class;
+            m5_ordered_memory |= cls == AGX_SCHEDULE_CLASS_LOAD ||
+                                  cls == AGX_SCHEDULE_CLASS_STORE ||
+                                  cls == AGX_SCHEDULE_CLASS_ATOMIC ||
+                                  cls == AGX_SCHEDULE_CLASS_BARRIER;
+         }
+      }
+      if (in_loop[block->index] && !ilp_block_has_memory(block) &&
+          !m5_ordered_memory) {
          ilp_schedule_block(ctx, block, memctx, native_uniforms, alu_latency,
-                            4 * alu_latency, AGX_ILP_SLACK, 0, true, false, 0, 0,
+                            4 * alu_latency, AGX_ILP_SLACK, 0, true, false, 0, 0, false,
                             &dag_state);
       } else if (!in_loop[block->index] && ilp_single_tile_blend(ctx, block)) {
          /* Hardware-qualified color-only budget; leave loop pressure alone. */
          ilp_schedule_block(ctx, block, memctx, native_uniforms, alu_latency,
-                            4 * alu_latency, 8, 0, true, false, 0, 0, &dag_state);
-      } else if (!in_loop[block->index] && !ctx->is_preamble &&
-                 ctx->stage == MESA_SHADER_FRAGMENT &&
-                 ilp_block_samples(block) >= AGX_ILP_FS_MIN_SAMPLES) {
+                            4 * alu_latency, 8, 0, true, false, 0, 0, false,
+                            &dag_state);
+      } else if ((!in_loop[block->index] || m5_sample_probe) &&
+                 !ctx->is_preamble &&
+                 (ctx->stage == MESA_SHADER_FRAGMENT || m5_sample_probe) &&
+                 ilp_block_samples(ctx, block) >= AGX_ILP_FS_MIN_SAMPLES) {
          /* The pressure schedule places each sample's consumers right after
           * it, so the program waits for one sample before issuing the next.
           * Apple's fragment programs issue independent samples back to back
           * and consume the results afterwards: move each sample and what it
           * depends on up, and keep the pressure order for everything else. */
+         signed slack = AGX_ILP_FS_SAMPLE_SLACK;
+         if (m5_fragment) {
+            const signed peak = ilp_block_peak(ctx, block, size16, memctx);
+            slack = MAX2(slack, m5_occupancy_cap - peak);
+         }
          ilp_schedule_block(ctx, block, memctx, native_uniforms, alu_latency,
-                            16 * alu_latency, AGX_ILP_FS_SAMPLE_SLACK, 0, false,
+                            16 * alu_latency, slack, 0, false,
                             true, AGX_ILP_FS_SAMPLE_HALVES,
-                            AGX_ILP_FS_HOIST_SAMPLES, &dag_state);
+                            AGX_ILP_FS_HOIST_SAMPLES, false, &dag_state);
       } else if (!in_loop[block->index] && ctx->is_preamble) {
          /* A constant program's device loads are mostly independent, but the
           * pressure schedule places each right before its uniform store, so
@@ -1145,7 +1463,7 @@ agx_ilp_schedule(agx_context *ctx)
          ilp_schedule_block(ctx, block, memctx, native_uniforms, alu_latency,
                             16 * alu_latency, AGX_ILP_SLACK,
                             AGX_ILP_CPROG_HALVES, false, true,
-                            AGX_ILP_CPROG_HALVES - 16, 0, &dag_state);
+                            AGX_ILP_CPROG_HALVES - 16, 0, false, &dag_state);
       }
    }
 
diff --git a/src/asahi/compiler/agx_register_allocate.c b/src/asahi/compiler/agx_register_allocate.c
index b4d553b9b0b4..11af5c589c18 100644
--- a/src/asahi/compiler/agx_register_allocate.c
+++ b/src/asahi/compiler/agx_register_allocate.c
@@ -13,6 +13,10 @@
 #include "util/u_math.h"
 #include "util/u_memory.h"
 #include "util/u_qsort.h"
+#include "util/u_debug.h"
+
+DEBUG_GET_ONCE_BOOL_OPTION(m5_wide_scalar_sources,
+                           "ASAHI_M5_WIDE_SCALAR_SOURCES", true)
 #include "agx3_copy_core.h"
 #include "agx_builder.h"
 #include "agx_compile.h"
@@ -92,6 +96,9 @@ struct ra_ctx {
    BITSET_WORD *visited;
    BITSET_WORD *used_regs[RA_CLASSES];
 
+   /* SSA values read by some instruction; see AGX_OPCODE_ENTER_QUERY_M5. */
+   const BITSET_WORD *dest_read;
+
    /* Were any sources killed early this instruction? We assert this is not true
     * when shuffling.
     */
@@ -212,6 +219,20 @@ agx_split_width(const agx_instr *I)
    return width;
 }
 
+/* SSA values read by any instruction (including phis). */
+static BITSET_WORD *
+agx_values_read(agx_context *ctx)
+{
+   BITSET_WORD *read = calloc(BITSET_WORDS(ctx->alloc), sizeof(BITSET_WORD));
+
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s)
+         BITSET_SET(read, I->src[s].value);
+   }
+
+   return read;
+}
+
 /*
  * Calculate register demand in 16-bit registers, while gathering widths and
  * classes. Becuase we allocate in SSA, this calculation is exact in
@@ -229,6 +250,7 @@ agx_calc_register_demand(agx_context *ctx, bool remat)
 
    uint8_t *widths = calloc(ctx->alloc, sizeof(uint8_t));
    enum ra_class *classes = calloc(ctx->alloc, sizeof(enum ra_class));
+   BITSET_WORD *dest_read = agx_values_read(ctx);
 
    agx_foreach_instr_global(ctx, I) {
       if (remat && (I->op == AGX_OPCODE_MOV_IMM ||
@@ -333,6 +355,10 @@ agx_calc_register_demand(agx_context *ctx, bool remat)
 
             demand += pot_width;
             late_kill_count += (pot_width - real_width);
+
+            /* An unread M5 query-entry result dies right after its write. */
+            if (I->op == AGX_OPCODE_ENTER_QUERY_M5 && !BITSET_TEST(dest_read, I->dest[d].value))
+               late_kill_count += real_width;
          }
 
          max_demand = MAX2(demand, max_demand);
@@ -341,6 +367,7 @@ agx_calc_register_demand(agx_context *ctx, bool remat)
       demand -= late_kill_count;
    }
 
+   free(dest_read);
    free(widths);
    free(classes);
    return max_demand;
@@ -369,7 +396,9 @@ dest_limit(const struct ra_ctx *rctx, const agx_instr *I, unsigned d)
    unsigned limit = rctx->bound[cls];
    /* G17P TEX encodes a seven-bit half-register destination, for image
     * reads as for texture reads. Coordinate and other staged sources are
-    * bounded separately through texture_gpr below. */
+    * bounded separately through texture_gpr below. AGX3C's destination is
+    * eight bits: G17G samples and reads land at halves 148 and 174 under
+    * register pressure (port/rawit k_tex_hi, k_read_hi). */
    if (rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
        (I->op == AGX_OPCODE_TEXTURE_SAMPLE ||
         I->op == AGX_OPCODE_TEXTURE_LOAD || I->op == AGX_OPCODE_IMAGE_LOAD))
@@ -382,7 +411,8 @@ dest_limit(const struct ra_ctx *rctx, const agx_instr *I, unsigned d)
    if (rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
        BITSET_TEST(rctx->low_gpr, I->dest[d].value))
       limit = MIN2(limit, rctx->tile_gpr_limit);
-   if (rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if ((rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+        rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
        ra_class_for_index(I->dest[d]) == RA_GPR &&
        BITSET_TEST(rctx->texture_gpr, I->dest[d].value))
       limit = MIN2(limit, 128);
@@ -402,7 +432,7 @@ dest_limit(const struct ra_ctx *rctx, const agx_instr *I, unsigned d)
 static unsigned
 dest_minimum(const struct ra_ctx *rctx, const agx_instr *I, unsigned d)
 {
-   if (rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(rctx->shader->key->dev.encoding) &&
        ra_class_for_index(I->dest[d]) == RA_GPR &&
        BITSET_TEST(rctx->high_gpr, I->dest[d].value))
       return 16;
@@ -433,9 +463,46 @@ ssa_has_bank_constraint(const struct ra_ctx *rctx, unsigned ssa)
           BITSET_TEST(rctx->high_gpr, ssa);
 }
 
+/* The interval a bank-constrained value may live in: the same bounds
+ * dest_limit() and dest_minimum() gave it when it was defined. */
+static void
+ssa_bank_range(const struct ra_ctx *rctx, unsigned ssa, unsigned *floor,
+               unsigned *limit)
+{
+   const bool agx3b =
+      rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B;
+   *floor = BITSET_TEST(rctx->high_gpr, ssa) ? 16 : 0;
+   *limit = ~0u;
+   if (agx3b && BITSET_TEST(rctx->low_gpr, ssa))
+      *limit = MIN2(*limit, rctx->tile_gpr_limit);
+   if (BITSET_TEST(rctx->texture_gpr, ssa))
+      *limit = MIN2(*limit, 128);
+   if (agx3b && BITSET_TEST(rctx->log2_gpr, ssa))
+      *limit = MIN2(*limit, 192);
+}
+
+/* Whether a value's bank limit narrows the current register file, mirroring
+ * dest_limit(): the low-bank and log2 bounds are G17P (AGX3B) encodings, and
+ * the seven-bit texture/COPY bound means nothing in a file of 128 halves or
+ * fewer, which is what compute shaders get once the workgroup bounds their
+ * registers. */
+static bool
+ssa_has_binding_bank_constraint(const struct ra_ctx *rctx, unsigned ssa)
+{
+   const bool agx3b =
+      rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B;
+   const unsigned bound = rctx->bound[RA_GPR];
+
+   return (agx3b && BITSET_TEST(rctx->low_gpr, ssa) &&
+           bound > rctx->tile_gpr_limit) ||
+          (BITSET_TEST(rctx->texture_gpr, ssa) && bound > 128) ||
+          (agx3b && BITSET_TEST(rctx->log2_gpr, ssa) && bound > 192) ||
+          BITSET_TEST(rctx->high_gpr, ssa);
+}
+
 static bool
 region_has_bank_constrained_value(const struct ra_ctx *rctx, unsigned base,
-                                  unsigned size)
+                                  unsigned size, bool binding_only)
 {
    for (unsigned reg = reserved_size(rctx->shader);
         reg < rctx->bound[RA_GPR];) {
@@ -448,7 +515,8 @@ region_has_bank_constrained_value(const struct ra_ctx *rctx, unsigned base,
       unsigned width = rctx->ncomps[ssa];
       assert(width && rctx->ssa_to_reg[ssa] == reg);
       if (reg < base + size && reg + width > base &&
-          ssa_has_bank_constraint(rctx, ssa))
+          (binding_only ? ssa_has_binding_bank_constraint(rctx, ssa)
+                        : ssa_has_bank_constraint(rctx, ssa)))
          return true;
 
       reg += width;
@@ -471,8 +539,9 @@ region_has_bank_constrained_value(const struct ra_ctx *rctx, unsigned base,
  * Postcondition: at least one register in the returned region is already free.
  */
 static unsigned
-find_best_region_to_evict(struct ra_ctx *rctx, unsigned size,
-                          BITSET_WORD *already_evicted, unsigned cap)
+find_best_region_to_evict_impl(struct ra_ctx *rctx, unsigned size,
+                               BITSET_WORD *already_evicted, unsigned cap,
+                               unsigned floor, bool honor_banks, bool strict)
 {
    assert(util_is_power_of_two_or_zero(size) && "precondition");
    assert((rctx->bound[RA_GPR] % size) == 0 &&
@@ -500,7 +569,7 @@ find_best_region_to_evict(struct ra_ctx *rctx, unsigned size,
        * descending. So, we do not need extra registers to handle "single
        * region" unevictability.
        */
-      if (base < reserved_size(rctx->shader))
+      if (base < reserved_size(rctx->shader) || base < floor)
          continue;
 
       /* Do not evict the same register multiple times. It's not necessary since
@@ -512,7 +581,7 @@ find_best_region_to_evict(struct ra_ctx *rctx, unsigned size,
       /* Keep constrained values in their assigned bank across live-range
        * splitting. Otherwise a later destination can relocate a valid TEX
        * coordinate above the seven-bit COPY source range. */
-      if (region_has_bank_constrained_value(rctx, base, size))
+      if (honor_banks && region_has_bank_constrained_value(rctx, base, size, false))
          continue;
 
       /* Estimate the number of moves required if we pick this region */
@@ -543,14 +612,50 @@ find_best_region_to_evict(struct ra_ctx *rctx, unsigned size,
     * narrows the file so a half victim's relocation stays inside the seven-bit
     * COPY destination.  If nothing fits down there, fall back to the whole
     * file rather than failing allocation outright. */
-   if (best_base == ~0u && bound < rctx->bound[RA_GPR])
-      return find_best_region_to_evict(rctx, size, already_evicted, ~0u);
+   if (best_base == ~0u && bound < rctx->bound[RA_GPR] && !strict)
+      return find_best_region_to_evict_impl(rctx, size, already_evicted, ~0u,
+                                            floor, honor_banks, strict);
 
-   assert(best_base < rctx->bound[RA_GPR] &&
-          "not enough registers (should have spilled already)");
    return best_base;
 }
 
+/*
+ * `cap` and `floor` bound where the region may start; with `strict` they are
+ * requirements (a bank-constrained value's interval), otherwise a preference.
+ */
+static unsigned
+find_best_region_to_evict(struct ra_ctx *rctx, unsigned size,
+                          BITSET_WORD *already_evicted, unsigned cap,
+                          unsigned floor, bool strict)
+{
+   /* Prefer not to disturb bank-constrained values. */
+   unsigned base = find_best_region_to_evict_impl(
+      rctx, size, already_evicted, cap, floor, true, strict);
+
+   /* They are not immovable, though: the free halves around them can all be
+    * unreachable even though demand fits the file (CS2 fills G17P's low bank
+    * with alternating pinned 32-bit scalars and the high bank solid). Evict
+    * them too; the caller re-places each one inside its own bank. */
+   if (base == ~0u) {
+      base = find_best_region_to_evict_impl(rctx, size, already_evicted, cap,
+                                            floor, false, strict);
+   }
+
+   /* Still nothing legal: finish allocation consistently anywhere and refuse
+    * the result, so agx_compile_function_nir() recompiles with spill headroom.
+    * Returning ~0 here used to index the used-register bitset half a gigabyte
+    * out of bounds in release builds. */
+   if (base == ~0u) {
+      rctx->shader->ra_failed = true;
+      base = find_best_region_to_evict_impl(rctx, size, already_evicted, ~0u, 0,
+                                            false, false);
+   }
+
+   assert(base < rctx->bound[RA_GPR] &&
+          "not enough registers (should have spilled already)");
+   return base;
+}
+
 static void
 set_ssa_to_reg(struct ra_ctx *rctx, unsigned ssa, unsigned reg)
 {
@@ -679,7 +784,7 @@ assign_regs_by_copying(struct ra_ctx *rctx, agx_index dest, const agx_instr *I,
    if (minimum > 0 || limit < rctx->bound[RA_GPR]) {
       unsigned count = rctx->ncomps[dest.value], span, base;
       bool agx3b =
-         I != NULL && rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B;
+         I != NULL && agx_isa_is_agx3(rctx->shader->key->dev.encoding);
 
       /* Prefer a region with no half victim: relocating one is bounded by the
        * seven-bit COPY destination.  When the constrained window holds nothing
@@ -778,10 +883,30 @@ assign_regs_by_copying(struct ra_ctx *rctx, agx_index dest, const agx_instr *I,
        * below AGX3_COPY_REG16_DEST_HALVES on a file that reaches that far. */
       unsigned cap = ~0u;
       if (ssa != dest.value && rctx->sizes[ssa] == AGX_SIZE_16 &&
-          rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
+          agx_isa_is_agx3(rctx->shader->key->dev.encoding))
          cap = AGX3_COPY_REG16_DEST_HALVES;
 
-      unsigned new_reg = find_best_region_to_evict(rctx, nr, clobbered, cap);
+      /* An evicted bank-constrained value must land back inside its bank. */
+      unsigned floor = 0;
+      bool strict = ssa != dest.value && ssa_has_bank_constraint(rctx, ssa);
+      if (strict) {
+         unsigned bank_limit;
+         ssa_bank_range(rctx, ssa, &floor, &bank_limit);
+         cap = MIN2(cap, bank_limit);
+      }
+
+      unsigned new_reg =
+         find_best_region_to_evict(rctx, nr, clobbered, cap, floor, strict);
+
+      /* No region can be freed for this value. Never index the register file
+       * with the failure value: refuse the shader through
+       * agx_compile_function_nir()'s normal error path instead, and hand back
+       * an in-range register so allocation can finish. */
+      if (new_reg >= rctx->bound[RA_GPR] ||
+          new_reg + nr > rctx->bound[RA_GPR]) {
+         rctx->shader->ra_failed = true;
+         return reserved_size(rctx->shader);
+      }
 
       /* Blocked registers need to get reassigned. Add them to the worklist. */
       for (unsigned i = 0; i < nr; ++i) {
@@ -964,11 +1089,12 @@ pick_transient_texture_region(struct ra_ctx *rctx, agx_instr *I, unsigned d,
 /*
  * G17P texture and image results, texture coordinates and moves from the
  * uniform file name seven-bit half-register fields, so they live in halves
- * 0..127. Lowest first, every other value lands there too, and in a shader at
- * the limit of the register file each constrained definition then evicts an
- * unconstrained value from the low bank. The moved values sit in different
- * registers on the two sides of the next merge, which costs a copy per value
- * at the join.
+ * 0..127, as do M5 texture coordinates, sin/cos operands and its other
+ * bank-constrained values. Lowest first, every other value lands there too,
+ * and in a shader at the limit of the register file each constrained
+ * definition then evicts an unconstrained value from the low bank. The moved
+ * values sit in different registers on the two sides of the next merge, which
+ * costs a copy per value at the join.
  *
  * Once a shader spills, every register up to the bound is in use somewhere,
  * so where a value goes no longer changes the register count. Put values
@@ -981,11 +1107,13 @@ static bool
 prefer_high_bank(const struct ra_ctx *rctx, const agx_instr *I, unsigned d,
                  enum ra_class cls, unsigned minimum, unsigned limit)
 {
-   if (rctx->shader->key->dev.chip != AGX_CHIP_G17P ||
+   const enum agx_chip chip = rctx->shader->key->dev.chip;
+   if ((chip != AGX_CHIP_G17P && !agx_chip_g17_m5(chip)) ||
        !rctx->shader->has_spill_pcopy_reserved || cls != RA_GPR ||
        minimum != 0 || limit != rctx->bound[cls] ||
        limit <= G17P_LOW_BANK_HALVES ||
-       ssa_has_bank_constraint(rctx, I->dest[d].value))
+       (chip == AGX_CHIP_G17P &&
+        ssa_has_bank_constraint(rctx, I->dest[d].value)))
       return false;
 
    const agx_instr *use = rctx->src_to_collect_phi[I->dest[d].value];
@@ -1308,7 +1436,7 @@ static bool
 pick_bank_affinity(struct ra_ctx *rctx, agx_instr *I, unsigned d,
                    unsigned count, unsigned align, unsigned *out)
 {
-   if (rctx->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(rctx->shader->key->dev.encoding))
       return false;
 
    const enum ra_class cls = ra_class_for_index(I->dest[d]);
@@ -1354,7 +1482,7 @@ pick_phi_edge_affinity(struct ra_ctx *rctx, agx_instr *I, unsigned d,
                        unsigned count, unsigned align, unsigned *out)
 {
    if (I->op != AGX_OPCODE_PHI ||
-       rctx->shader->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+       !agx_isa_is_agx3(rctx->shader->key->dev.encoding))
       return false;
    enum ra_class cls = ra_class_for_index(I->dest[d]);
    unsigned first = ~0u;
@@ -1409,7 +1537,8 @@ static bool
 pick_phi_source_agreement(struct ra_ctx *rctx, agx_instr *I, unsigned *out)
 {
    if (I->op != AGX_OPCODE_PHI ||
-       rctx->shader->key->dev.chip != AGX_CHIP_G17P)
+       (rctx->shader->key->dev.chip != AGX_CHIP_G17P &&
+        !agx_chip_g17_m5(rctx->shader->key->dev.chip)))
       return false;
 
    enum ra_class cls = ra_class_for_index(I->dest[0]);
@@ -1446,7 +1575,8 @@ pick_phi_unclaimed(struct ra_ctx *rctx, agx_instr *I, unsigned d,
                    unsigned count, unsigned align, unsigned *out)
 {
    if (I->op != AGX_OPCODE_PHI ||
-       rctx->shader->key->dev.chip != AGX_CHIP_G17P)
+       (rctx->shader->key->dev.chip != AGX_CHIP_G17P &&
+        !agx_chip_g17_m5(rctx->shader->key->dev.chip)))
       return false;
 
    enum ra_class cls = ra_class_for_index(I->dest[d]);
@@ -1494,6 +1624,14 @@ pick_regs(struct ra_ctx *rctx, agx_instr *I, unsigned d)
 
    unsigned align = count;
 
+   /* Scalar byte loads address the low half of a word. Keep this physical
+    * requirement through allocation, including byte-copy helper loops. */
+   if (I->op == AGX_OPCODE_DEVICE_LOAD && I->format == AGX_FORMAT_I8 &&
+       idx.size == AGX_SIZE_16 && I->mask == 1 &&
+       (rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+        rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3C))
+      return find_regs(rctx, I, d, count, MAX2(align, 2));
+
    /* AGX3b atomicdev is physically two-address: field16 supplies the update
     * value and receives the returned old value.  The legalization pass gives
     * every returning scalar atomic a last-use value temporary.  Its region was
@@ -1830,7 +1968,8 @@ texture_tail_cost(struct ra_ctx *rctx, agx_instr *I, agx_instr *split,
          continue;
       }
       if (proxy.op == AGX_OPCODE_PRELOAD || proxy.op == AGX_OPCODE_PHI ||
-          proxy.agx3_cube_scalar_coords || proxy.op == AGX_OPCODE_FFS ||
+          proxy.agx3_cube_scalar_coords || proxy.agx3_split_coords ||
+          proxy.op == AGX_OPCODE_FFS ||
           proxy.op == AGX_OPCODE_BFEIL || proxy.op == AGX_OPCODE_BITOP ||
           proxy.op == AGX_OPCODE_IADD)
          goto done;
@@ -2081,6 +2220,10 @@ agx_ra_assign_local(struct ra_ctx *rctx)
          BITSET_SET_COUNT(used_regs_gpr, 0, 4);
    }
 
+   /* Match the CEM prefix counted by demand, spilling and shuffling. */
+   if (rctx->shader->agx3_cem_low_scratch)
+      BITSET_SET_COUNT(used_regs_gpr, 0, reserved_size(rctx->shader));
+
    /* Reserve bottom registers as temporaries for parallel copy lowering */
    if (rctx->shader->has_spill_pcopy_reserved) {
       BITSET_SET_COUNT(used_regs_gpr, 0, 8);
@@ -2198,7 +2341,7 @@ agx_ra_assign_local(struct ra_ctx *rctx)
        * occupies the first channel.  Scalar returned atomics were given a
        * private value above, so their source is guaranteed to die here.
        */
-      if (rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(rctx->shader->key->dev.encoding) &&
           I->op == AGX_OPCODE_ATOMIC && I->nr_dests == 1 &&
           !agx_is_null(I->dest[0]) && I->src[0].type == AGX_INDEX_NORMAL &&
           I->src[0].kill) {
@@ -2217,6 +2360,26 @@ agx_ra_assign_local(struct ra_ctx *rctx)
          continue;
       }
 
+      if (I->op == AGX_OPCODE_MATMAD16_AGX3) {
+         if (I->nr_dests != 1 || I->nr_srcs != 3 ||
+             I->src[2].type != AGX_INDEX_NORMAL || I->src[2].memory ||
+             !I->src[2].kill || rctx->ncomps[I->src[2].value] != 16 ||
+             rctx->ncomps[I->dest[0].value] != 16) {
+            rctx->shader->ra_failed = true;
+            return;
+         }
+         const unsigned reg = ssa_to_reg[I->src[2].value];
+         kill_source(rctx, I, 2);
+         assign_regs(rctx, I->dest[0], reg);
+         for (unsigned s = 0; s < 2; ++s) {
+            if (I->src[s].type == AGX_INDEX_NORMAL && I->src[s].kill)
+               kill_source(rctx, I, s);
+         }
+         agx_set_sources(rctx, I);
+         agx_set_dests(rctx, I);
+         continue;
+      }
+
       /* Search for regions of contiguous killed sources to early-kill. */
       rctx->early_killed = false;
 
@@ -2224,7 +2387,7 @@ agx_ra_assign_local(struct ra_ctx *rctx)
        * destination registers. Keep its last-use source occupied until after
        * destination allocation so generic early-kill cannot coalesce them. */
       const bool g17p_ft1 =
-         rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(rctx->shader->key->dev.encoding) &&
          I->op == AGX_OPCODE_FFS;
 
       /* A G17P register-boundary SRL may overwrite its value when that value
@@ -2232,7 +2395,7 @@ agx_ra_assign_local(struct ra_ctx *rctx)
        * the boundary operand itself. Keep a killed boundary allocated until
        * the destination has been placed. */
       const bool g17p_srl_boundary =
-         rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(rctx->shader->key->dev.encoding) &&
          I->op == AGX_OPCODE_BFEIL && I->nr_srcs == 3 &&
          I->src[2].type == AGX_INDEX_NORMAL && I->src[2].kill;
 
@@ -2240,7 +2403,14 @@ agx_ra_assign_local(struct ra_ctx *rctx)
        * shaders keep the destination distinct from their inputs. The compact
        * tied shape has not been independently qualified, so allocate the
        * result before freeing either source for the register-register shape
-       * the packer maps to ISUB. */
+       * the packer maps to ISUB.
+       *
+       * AGX3B only. Native G17G emits isub32 both tied and with a distinct
+       * destination (port/cwit4 k_sub_*), and widening this rule to AGX3C
+       * made tessellation.invariance.{inner,outer}_triangle_set.*_fractional_
+       * even_spacing fail on M5 (Pass with the rule off; bisected to this
+       * hunk on hardware). Which allocation in the 48 changed precompiled
+       * kernels or the runtime shaders causes it is not yet known. */
       const bool g17p_reg_isub =
          rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
          I->op == AGX_OPCODE_IADD && I->nr_srcs == 2 && I->shift == 0 &&
@@ -2253,7 +2423,7 @@ agx_ra_assign_local(struct ra_ctx *rctx)
        * both the fully staged shader (distinct from both) and the keepalive
        * discriminator (tied only to src0) are exact on hardware. */
       const bool g17p_reg_and =
-         rctx->shader->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+         agx_isa_is_agx3(rctx->shader->key->dev.encoding) &&
          I->op == AGX_OPCODE_BITOP && I->nr_srcs == 2 &&
          (I->truth_table & 0xfu) == 0x8u &&
          I->src[0].type == AGX_INDEX_NORMAL &&
@@ -2293,6 +2463,18 @@ agx_ra_assign_local(struct ra_ctx *rctx)
          assign_regs(rctx, I->dest[d], pick_regs(rctx, I, d));
       }
 
+      /* The M5 query entry writes its result register even when nothing reads
+       * it. It still needs a real register (so nothing live is clobbered),
+       * which is free again immediately after the instruction.
+       */
+      if (I->op == AGX_OPCODE_ENTER_QUERY_M5 && I->nr_dests == 1 &&
+          !BITSET_TEST(rctx->dest_read, I->dest[0].value)) {
+         enum ra_class cls = ra_class_for_index(I->dest[0]);
+         BITSET_CLEAR_COUNT(rctx->used_regs[cls],
+                            rctx->ssa_to_reg[I->dest[0].value],
+                            rctx->ncomps[I->dest[0].value]);
+      }
+
       /* Free late-killed sources */
       agx_foreach_ssa_src(I, s) {
          if (I->src[s].kill) {
@@ -2414,7 +2596,40 @@ agx_insert_parallel_copies(agx_context *ctx, agx_block *block)
          }
       }
 
-      agx_emit_parallel_copies(&b, copies, nr_phi);
+      /* A register pair holding no value live out of this block and touched
+       * by no copy is free on this edge: the successor's live-ins are exactly
+       * the phi destinations plus this block's live-outs.  Stay inside the
+       * allocated file so the register count is unchanged. */
+      agx_index temp = agx_null();
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          block->reg_to_ssa_out[RA_GPR]) {
+         const uint32_t *out = block->reg_to_ssa_out[RA_GPR];
+         const unsigned top = ctx->max_reg + 1;
+         for (unsigned reg = ALIGN_POT(reserved_size(ctx), 2);
+              reg + 2 <= top; reg += 2) {
+            if (out[reg] != ~0u || out[reg + 1] != ~0u)
+               continue;
+
+            bool touched = false;
+            for (unsigned j = 0; j < nr_phi && !touched; ++j) {
+               const unsigned w = agx_size_align_16(copies[j].src.size);
+               if (!copies[j].dest_mem && copies[j].dest < reg + 2 &&
+                   copies[j].dest + w > reg)
+                  touched = true;
+               if (copies[j].src.type == AGX_INDEX_REGISTER &&
+                   !copies[j].src.memory && copies[j].src.value < reg + 2 &&
+                   copies[j].src.value + w > reg)
+                  touched = true;
+            }
+            if (touched)
+               continue;
+
+            temp = agx_register(reg, AGX_SIZE_32);
+            break;
+         }
+      }
+
+      agx_emit_parallel_copies_temp(&b, copies, nr_phi, temp);
 
       free(copies);
    }
@@ -2462,7 +2677,7 @@ lower_exports(agx_context *ctx)
 static void
 agx_lower_agx3b_atomic_returns(agx_context *ctx)
 {
-   if (ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3B)
+   if (!agx_isa_is_agx3(ctx->key->dev.encoding))
       return;
 
    agx_foreach_instr_global_safe(ctx, I) {
@@ -2489,7 +2704,7 @@ agx3_tile_store_bank(agx_context *ctx)
    unsigned payload_halves = 0, payload_align = 1;
    bool tile_loads = false;
    bool quad_tile =
-      ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      agx_isa_is_agx3(ctx->key->dev.encoding) &&
       ctx->stage == MESA_SHADER_FRAGMENT &&
       (ctx->key->secondary || ctx->key->fs.ignore_tib_dependencies) &&
       list_length(&ctx->blocks) == 1;
@@ -2558,14 +2773,14 @@ agx3_tile_store_bank(agx_context *ctx)
    }
 
    const bool mrt_bank =
-      ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      agx_isa_is_agx3(ctx->key->dev.encoding) &&
       ctx->stage == MESA_SHADER_FRAGMENT &&
       (!tile_loads || targets >= 3) &&
       targets >= (ctx->key->g17p_mrt_bg ? 2 : 3) &&
       targets <= AGX_MAX_RTS &&
       (ctx->key->secondary || ctx->key->g17p_mrt_bg);
    const bool sample_bank =
-      ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      agx_isa_is_agx3(ctx->key->dev.encoding) &&
       ctx->stage == MESA_SHADER_FRAGMENT && ctx->key->secondary &&
       ctx->agx3_mrt_tile_ops_ordered && ctx->key->fs.nr_samples > 1 &&
       targets >= 2;
@@ -2662,12 +2877,27 @@ pair_peer_publication_consumed_early(agx_context *ctx, unsigned value)
    return found;
 }
 
-void
-agx_ra(agx_context *ctx)
+static void
+agx_ra_impl(agx_context *ctx, bool disable_remat)
 {
    ctx->uniforms_written = agx_has_uniform_store(ctx);
    agx_lower_agx3b_atomic_returns(ctx);
 
+   /* CEM has narrower destination fields than the M5 general register file.
+    * Own its four words (magnitude/face and two coordinates) in the low bank,
+    * reusing an axis word for packed array/LOD after the finalizers.
+    * Do not cap unrelated live values at 192:
+    * they remain eligible for all 252 usable halves. The bank starts beyond
+    * the eight halves used by CF, shuffle and spill parallel-copy lowering.
+    * Register allocation and the rematerializing spiller both reserve it. */
+   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+      agx_foreach_instr_global(ctx, I) {
+         if (I->op == AGX_OPCODE_TEXTURE_SAMPLE &&
+             (I->dim == AGX_DIM_CUBE || I->dim == AGX_DIM_CUBE_ARRAY))
+            ctx->agx3_cem_low_scratch = true;
+      }
+   }
+
    bool force_spilling =
       (agx_compiler_debug & AGX_DBG_SPILL) && ctx->key->has_scratch;
 
@@ -2705,6 +2935,36 @@ agx_ra(agx_context *ctx)
    if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
       max_possible_regs = MIN2(max_possible_regs, AGX3B_NUM_REGS);
 
+   /* Native G17G pressure shaders spill at 126 dwords. Direct M5 tests of
+    * IADD/EXTR followed by STDEV agree: pair 250 works, pair 252 does not.
+    * Align the usable file before spilling as live-range splitting requires
+    * whole vector slots; rounding only at final assignment is too late.
+    */
+   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+      unsigned alignment = 8;
+      agx_foreach_instr_global(ctx, I) {
+         agx_foreach_ssa_dest(I, d)
+            alignment = MAX2(alignment,
+                             util_next_power_of_two(agx_index_size_16(I->dest[d])));
+      }
+      max_possible_regs = MIN2(max_possible_regs,
+                               ROUND_DOWN_TO(AGX3C_NUM_REGS, alignment));
+   }
+
+   /* Large native-query kernels benefit from leaving registers available for
+    * more resident work, even when this introduces ordinary scratch spills.
+    * The 160-half budget improved the full-resolution M5 RT workload; 128 and
+    * 192 were slower. Ordinary compute, helpers, preambles and AGX3B retain
+    * their existing budgets. Zero explicitly restores the uncapped path.
+    * Disable the shader cache when comparing an override. */
+   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+       ctx->stage == MESA_SHADER_COMPUTE && !ctx->is_preamble &&
+       !ctx->key->is_helper && ctx->out->uses_ray_traversal) {
+      unsigned budget = debug_get_num_option("ASAHI_M5_RT_REGISTER_BUDGET", 160);
+      if (budget >= 128 && budget <= 240 && !(budget & 15))
+         max_possible_regs = MIN2(max_possible_regs, budget);
+   }
+
    /* Match the CEM scratch extent reserved by agx3_pack_binary. */
    if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)
       max_possible_regs = MIN2(max_possible_regs,
@@ -2726,7 +2986,7 @@ agx_ra(agx_context *ctx)
     * that used halves 184..191 now spills them instead; one that did not, and
     * every fragment without such a read, keeps the whole file.
     */
-   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
        ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_preamble) {
       agx_foreach_instr_global(ctx, I) {
          if ((I->op == AGX_OPCODE_TEXTURE_LOAD ||
@@ -2773,6 +3033,16 @@ agx_ra(agx_context *ctx)
       max_possible_regs = 32;
    }
 
+   /* A retry after a failed allocation spills down to a smaller file, leaving
+    * the rest free for live-range splitting around bank-constrained values. */
+   const unsigned uncut_regs = max_possible_regs;
+   const bool budget_cut =
+      ctx->ra_budget_cut && ctx->key->has_scratch && !ctx->key->is_helper;
+   if (budget_cut)
+      max_possible_regs =
+         ROUND_DOWN_TO(max_possible_regs - MIN2(ctx->ra_budget_cut,
+                                                max_possible_regs / 2), 16);
+
    /* Calculate the demand. We'll use it to determine if we need to spill and to
     * bound register assignment.
     */
@@ -2785,7 +3055,7 @@ agx_ra(agx_context *ctx)
    /* If we need multiple waves, see if we can rematerialize constants to save
     * waves. If we only have a single wave regardless, this is pointless.
     */
-   if (effective_demand > agx_round_registers(1) && !spilling) {
+   if (!disable_remat && effective_demand > agx_round_registers(1) && !spilling) {
       unsigned effective_demand_remat = agx_calc_register_demand(ctx, true);
 
       /* Worst-case assume we need 6 16-bit registers for constants, for a
@@ -2836,20 +3106,30 @@ agx_ra(agx_context *ctx)
       assert(effective_demand <= max_possible_regs && "spiller post-condition");
    }
 
+   /* The cut only lowers the spill target; assignment gets the whole file. */
+   if (budget_cut)
+      max_possible_regs = uncut_regs;
+
    unsigned original_hints =
       util_dynarray_num_elements(&ctx->collect_split_hints, agx_instr *);
    if (ssa_renumbered || ctx->scratch_size_B) {
       agx_foreach_instr_global(ctx, I) {
-         if (!I->agx3_cube_scalar_coords || I->nr_srcs != 8)
+         /* Split sample coordinates take no pair hint (see
+          * agx3_split_texture_coordinates()). */
+         if ((!I->agx3_cube_scalar_coords && !I->agx3_split_coords) ||
+             I->nr_srcs != 8 ||
+             (I->agx3_split_coords && I->op != AGX_OPCODE_TEXTURE_LOAD))
             continue;
          if (!ctx->collect_split_hints.data)
             util_dynarray_init(&ctx->collect_split_hints, ctx);
+         const unsigned n = agx_is_null(I->src[7]) ? 2 : 3;
          agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
-         agx_instr *hint = agx_alloc_instr(&b, AGX_OPCODE_COLLECT, 1, 3);
-         hint->dest[0] = agx_vec_temp(ctx, AGX_SIZE_32, 3);
+         agx_instr *hint = agx_alloc_instr(&b, AGX_OPCODE_COLLECT, 1, n);
+         hint->dest[0] = agx_vec_temp(ctx, AGX_SIZE_32, n);
          hint->src[0] = I->src[0];
          hint->src[1] = I->src[6];
-         hint->src[2] = I->src[7];
+         if (n == 3)
+            hint->src[2] = I->src[7];
          util_dynarray_append(&ctx->collect_split_hints, hint);
       }
    }
@@ -2902,12 +3182,14 @@ agx_ra(agx_context *ctx)
       /* The G17P funop encoder and fused quadrant selects use the bounded
        * seven-bit operand forms. Constrain only their SSA webs; unrelated
        * values retain the complete register file. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if ((ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
           (I->op == AGX_OPCODE_SINC_AGX3 ||
            I->op == AGX_OPCODE_BITTEST_SELECT_AGX3 ||
            (I->op == AGX_OPCODE_TRUNC && I->nr_dests == 1 &&
             I->dest[0].size == AGX_SIZE_32) ||
-           I->op == AGX_OPCODE_DFDX || I->op == AGX_OPCODE_DFDY)) {
+           I->op == AGX_OPCODE_DFDX || I->op == AGX_OPCODE_DFDY)) ||
+          (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+           I->op == AGX_OPCODE_SINC_AGX3)) {
          agx_foreach_ssa_src(I, s)
             BITSET_SET(texture_gpr, I->src[s].value);
          agx_foreach_ssa_dest(I, d)
@@ -2929,26 +3211,41 @@ agx_ra(agx_context *ctx)
             BITSET_SET(low_gpr, split_source[I->src[0].value] - 1);
       }
 
-      /* Explicit scalar publications are the coordinate GPR consumers now.
-       * Their native COPY source still has the same seven-bit domain, even
-       * though the following sample no longer exposes a GPR operand to RA.
+      /* Explicit publications own the coordinate GPR constraints even when
+       * the following sample no longer exposes those operands to RA.
        */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if ((ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
           agx_is_texture_publication(I)) {
-         agx_foreach_ssa_src(I, s)
-            BITSET_SET(texture_gpr, I->src[s].value);
+         /* Scalar 3D publication can use the full M5 source register field.
+          * Keep every other publication within the original low-bank range.
+          */
+         bool wide_scalar = ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+            ctx->stage == MESA_SHADER_FRAGMENT &&
+            I->op == AGX_OPCODE_PUBLISH_TEXTURE_SCALAR_AGX3 &&
+            I->nr_srcs == 3 && !agx_is_null(I->src[2]) &&
+            debug_get_option_m5_wide_scalar_sources();
+         for (unsigned s = 0; s < I->nr_srcs && wide_scalar; ++s)
+            wide_scalar &= !agx_is_null(I->src[s]) &&
+                           I->src[s].size == AGX_SIZE_32;
+         if (!wide_scalar) {
+            agx_foreach_ssa_src(I, s)
+               BITSET_SET(texture_gpr, I->src[s].value);
+         }
       }
 
-      /* G17P publishes sampled coordinates through COPY, whose source field
-       * has seven bits and no high-bit extension. CEM has the same source
-       * domain. Keep every SSA vector that reaches either path below half
-       * 128; the packer independently rejects precoloured high-bank inputs.
-       * Gradient scalars and shadow references use the same COPY producer. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      /* Other sampled coordinates and CEM retain their low-bank source
+       * constraints. Gradient scalars and shadow references also remain
+       * within the existing qualified range. */
+      if ((ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
           (I->op == AGX_OPCODE_TEXTURE_SAMPLE ||
            I->op == AGX_OPCODE_TEXTURE_LOAD ||
            I->op == AGX_OPCODE_IMAGE_LOAD)) {
-         if (I->src[0].type == AGX_INDEX_NORMAL)
+         /* Split M5 fetch and sample coordinates stage x and y through the
+          * wide scalar COPY, which reads the whole register file; the sample
+          * index or layer keeps PACK_SMPIR's seven-bit source. */
+         if (I->src[0].type == AGX_INDEX_NORMAL && !I->agx3_split_coords)
             BITSET_SET(texture_gpr, I->src[0].value);
          if (I->agx3_cube_scalar_coords) {
             for (unsigned s = 6; s < 8; ++s) {
@@ -2956,21 +3253,35 @@ agx_ra(agx_context *ctx)
                   BITSET_SET(texture_gpr, I->src[s].value);
             }
          }
+         if (I->agx3_split_coords && I->src[7].type == AGX_INDEX_NORMAL)
+            BITSET_SET(texture_gpr, I->src[7].value);
 
          if (I->op == AGX_OPCODE_TEXTURE_SAMPLE &&
              I->lod_mode == AGX_LOD_MODE_LOD_GRAD &&
              I->src[1].type == AGX_INDEX_NORMAL)
             BITSET_SET(texture_gpr, I->src[1].value);
 
+         /* M5 PACK_SMPIR level/bias operands retain a seven-bit source,
+          * even though SMP can return into the complete register file. */
+         if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+             I->src[1].type == AGX_INDEX_NORMAL)
+            BITSET_SET(texture_gpr, I->src[1].value);
+
          if (I->op == AGX_OPCODE_TEXTURE_SAMPLE && I->shadow &&
              I->src[5].type == AGX_INDEX_NORMAL)
             BITSET_SET(texture_gpr, I->src[5].value);
       }
 
+      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+          I->op == AGX_OPCODE_PACK_SAMPLER_LOD_AGX3 &&
+          I->src[0].type == AGX_INDEX_NORMAL)
+         BITSET_SET(texture_gpr, I->src[0].value);
+
       /* imgwci stages its data and coordinates through the same seven-bit
        * COPY sources and names its PBE index register in a seven-bit field.
        */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if ((ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B ||
+           ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) &&
           I->op == AGX_OPCODE_IMAGE_WRITE) {
          for (unsigned s = 0; s < I->nr_srcs; ++s) {
             if ((s == 0 || s == 1 || s == 4) &&
@@ -2983,13 +3294,13 @@ agx_ra(agx_context *ctx)
        * halves 0..191 by the recovered native form. dest_limit handles the
        * instruction result directly; mark the defining SSA of its source so
        * high register pressure cannot produce a post-RA refusal. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_LOG2 && I->src[0].type == AGX_INDEX_NORMAL)
          BITSET_SET(log2_gpr, I->src[0].value);
 
       /* agx3_tex_stage_cube_array_lod encodes the layer register (coords + 6
        * halves) in seven bits; keep such coordinate vectors below half 128. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           I->op == AGX_OPCODE_TEXTURE_SAMPLE && I->dim == AGX_DIM_CUBE_ARRAY &&
           I->src[0].type == AGX_INDEX_NORMAL && I->nr_srcs > 1 &&
           I->src[1].type == AGX_INDEX_NORMAL &&
@@ -3080,7 +3391,7 @@ agx_ra(agx_context *ctx)
     * Place those coordinates above it from the outset, within their existing
     * seven-bit COPY domain. This does not relax the tile or texture encodings.
     */
-   if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+   if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
        ctx->agx3_mrt_tile_ops_ordered) {
       unsigned tile_halfs = 0;
       for (unsigned ssa = 0; ssa < ctx->alloc; ++ssa) {
@@ -3135,10 +3446,13 @@ agx_ra(agx_context *ctx)
    /* Assign registers in dominance-order. This coincides with source-order due
     * to a NIR invariant, so we do not need special handling for this.
     */
+   BITSET_WORD *dest_read = agx_values_read(ctx);
+
    agx_foreach_block(ctx, block) {
       agx_ra_assign_local(&(struct ra_ctx){
          .shader = ctx,
          .block = block,
+         .dest_read = dest_read,
          .src_to_collect_phi = src_to_collect_phi,
          .early_scalar_export = early_scalar_export,
          .phi_web = phi_web,
@@ -3160,6 +3474,8 @@ agx_ra(agx_context *ctx)
       });
    }
 
+   free(dest_read);
+
    ctx->max_reg = reg_count ? (reg_count - 1) : 0;
    ctx->spill_base_B = ctx->scratch_size_B;
    ctx->scratch_size_B += mem_slot_count * 2;
@@ -3193,7 +3509,7 @@ agx_ra(agx_context *ctx)
        * RA has made the operand private and coalesced it with the destination
        * above. Set the physical hint only after SSA lowering, which
        * intentionally strips SSA liveness flags. */
-      if (ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
+      if (agx_isa_is_agx3(ctx->key->dev.encoding) &&
           ins->op == AGX_OPCODE_ATOMIC && ins->nr_dests == 1 &&
           !agx_is_null(ins->dest[0]) &&
           ins->atomic_opc != AGX_ATOMIC_OPC_UMIN &&
@@ -3327,3 +3643,16 @@ agx_ra(agx_context *ctx)
    free(classes);
    free(visited);
 }
+
+void
+agx_ra(agx_context *ctx)
+{
+   agx_ra_impl(ctx, false);
+}
+
+/* Retry from a fresh compiler context, retaining the original full budget. */
+void
+agx_ra_without_remat(agx_context *ctx)
+{
+   agx_ra_impl(ctx, true);
+}
diff --git a/src/asahi/compiler/agx_repair_ssa.c b/src/asahi/compiler/agx_repair_ssa.c
index 7aed03b7d998..76da50625fbf 100644
--- a/src/asahi/compiler/agx_repair_ssa.c
+++ b/src/asahi/compiler/agx_repair_ssa.c
@@ -172,12 +172,14 @@ seal_loop_headers(struct repair_ctx *ctx, struct agx_block *exit)
 static void
 agx_opt_trivial_phi(agx_context *ctx)
 {
-   /* A phi whose source was itself remapped in this round is skipped (the
-    * cycle case below). Other chips still skip every phi whose source was
-    * not remapped, which removes nothing, so their output stays unchanged.
+   /* On G17P and M5, a phi whose source was itself remapped in this round
+    * is skipped (the cycle case below). Other chips still skip every phi
+    * whose source was not remapped, which removes nothing, so their output
+    * stays unchanged.
     */
    const bool skip_remapped =
-      ctx->key && ctx->key->dev.chip == AGX_CHIP_G17P;
+      ctx->key && (ctx->key->dev.chip == AGX_CHIP_G17P ||
+                   agx_chip_g17_m5(ctx->key->dev.chip));
    agx_index *remap = calloc(ctx->alloc, sizeof(*remap));
    for (;;) {
       bool progress = false;
diff --git a/src/asahi/compiler/agx_spill.c b/src/asahi/compiler/agx_spill.c
index c6c662aa80f2..d77866726781 100644
--- a/src/asahi/compiler/agx_spill.c
+++ b/src/asahi/compiler/agx_spill.c
@@ -823,6 +823,13 @@ min_algorithm(struct spill_ctx *ctx)
          insert_W(ctx, I->dest[d].value);
       }
 
+      /* The M5 query entry's result may be unread; it then dies immediately
+       * and must not occupy W or be spilled.
+       */
+      if (I->op == AGX_OPCODE_ENTER_QUERY_M5 && I->nr_dests == 1 &&
+          ctx->next_uses[I->dest[0].value] == DIST_INFINITY)
+         remove_W_if_present(ctx, I->dest[0].value);
+
       /* Update next-use distances for this instruction. Unlike the paper, we
        * prune dead values from W as we go. This doesn't affect correctness, but
        * it speeds up limit() on average.
@@ -982,10 +989,10 @@ loop_room_for_live_through(struct spill_ctx *ctx, BITSET_WORD *used)
 
 /*
  * Hack's thesis sorts the live-in set (+ destinations of phis) by next-use
- * distance and takes as much as fits. On G17P, values read in the loop are
- * taken first that way, and values live through the loop then fill only the
- * room the loop leaves them (section 4.2 of the paper). A remat-only pass must
- * not spill, so it keeps the thesis heuristic.
+ * distance and takes as much as fits. On G17P and M5, values read in the loop
+ * are taken first that way, and values live through the loop then fill only
+ * the room the loop leaves them (section 4.2 of the paper). A remat-only pass
+ * must not spill, so it keeps the thesis heuristic.
  */
 static ATTRIBUTE_NOINLINE void
 compute_w_entry_loop_header(struct spill_ctx *ctx)
@@ -1009,7 +1016,9 @@ compute_w_entry_loop_header(struct spill_ctx *ctx)
 
    BITSET_WORD *used = NULL;
    unsigned room = 0;
-   if (ctx->shader->key->dev.chip == AGX_CHIP_G17P && !ctx->remat_only) {
+   if ((ctx->shader->key->dev.chip == AGX_CHIP_G17P ||
+        agx_chip_g17_m5(ctx->shader->key->dev.chip)) &&
+       !ctx->remat_only) {
       used = BITSET_CALLOC(ctx->n);
       room = loop_room_for_live_through(ctx, used);
    }
@@ -1326,6 +1335,182 @@ validate_next_use_info(UNUSED agx_context *ctx,
 #endif
 }
 
+/* SSA repair after rematerialization may merge different register copies of
+ * one literal. Prove the entire scalar PHI/MOV web, including loop backedges,
+ * before replacing a PHI. No partially-known input is assumed constant. */
+struct constant_phi_edge {
+   unsigned parent;
+   int next;
+};
+
+static bool
+constant_phi_scalar(agx_index x)
+{
+   return x.type == AGX_INDEX_NORMAL && !x.memory && !x.has_reg &&
+          !x.abs && !x.neg && agx_channels(x) == 1 &&
+          (x.size == AGX_SIZE_16 || x.size == AGX_SIZE_32);
+}
+
+static bool
+constant_phi_transparent(const agx_instr *I)
+{
+   if (!I || I->nr_dests != 1 || !constant_phi_scalar(I->dest[0]) ||
+       I->saturate || I->invert_cond)
+      return false;
+
+   if (I->op == AGX_OPCODE_MOV_IMM)
+      return I->nr_srcs == 0;
+
+   if ((I->op != AGX_OPCODE_MOV || I->nr_srcs != 1) &&
+       (I->op != AGX_OPCODE_PHI || I->nr_srcs == 0))
+      return false;
+
+   agx_foreach_src(I, s) {
+      if (!constant_phi_scalar(I->src[s]) ||
+          I->src[s].size != I->dest[0].size)
+         return false;
+   }
+
+   return true;
+}
+
+static bool
+constant_phi_prove(agx_instr **defs, const int *parents,
+                   const struct constant_phi_edge *edges, unsigned n,
+                   unsigned start, BITSET_WORD *visited, BITSET_WORD *grounded,
+                   unsigned *nodes, unsigned *work, uint64_t *literal)
+{
+   memset(visited, 0, BITSET_BYTES(n));
+   memset(grounded, 0, BITSET_BYTES(n));
+   unsigned count = 1, read = 0, roots = 0;
+   bool have_literal = false;
+   enum agx_size size = defs[start]->dest[0].size;
+   nodes[0] = start;
+   BITSET_SET(visited, start);
+
+   while (read < count) {
+      unsigned v = nodes[read++];
+      const agx_instr *I = defs[v];
+      if (!constant_phi_transparent(I) || I->dest[0].size != size)
+         return false;
+
+      if (I->op == AGX_OPCODE_MOV_IMM) {
+         uint64_t value = I->imm &
+            (size == AGX_SIZE_16 ? UINT64_C(0xffff) : UINT64_C(0xffffffff));
+         if (have_literal && value != *literal)
+            return false;
+         have_literal = true;
+         *literal = value;
+         BITSET_SET(grounded, v);
+         work[roots++] = v;
+         continue;
+      }
+
+      agx_foreach_src(I, s) {
+         unsigned src = I->src[s].value;
+         if (src >= n || !defs[src])
+            return false;
+         if (!BITSET_TEST(visited, src)) {
+            BITSET_SET(visited, src);
+            nodes[count++] = src;
+         }
+      }
+   }
+
+   if (!have_literal)
+      return false;
+
+   /* Every node must have a path to a literal. Otherwise a root-free cycle
+    * could have contributed an unknown input beside a known literal. Traverse
+    * reverse edges from the actual literal definitions, rather than guessing
+    * that an unresolved PHI input agrees with an already-known one. */
+   unsigned proven = roots;
+   read = 0;
+   while (read < proven) {
+      unsigned v = work[read++];
+      for (int e = parents[v]; e >= 0; e = edges[e].next) {
+         unsigned parent = edges[e].parent;
+         if (BITSET_TEST(visited, parent) &&
+             !BITSET_TEST(grounded, parent)) {
+            BITSET_SET(grounded, parent);
+            work[proven++] = parent;
+         }
+      }
+   }
+
+   return proven == count;
+}
+
+bool
+agx_fold_m5_spill_constant_phis(agx_context *ctx)
+{
+   if (!ctx->key || ctx->key->dev.encoding != AGX_ISA_ENCODING_AGX3C ||
+       ctx->is_preamble || !ctx->alloc)
+      return false;
+
+   void *memctx = ralloc_context(NULL);
+   unsigned n = ctx->alloc;
+   agx_instr **defs = rzalloc_array(memctx, agx_instr *, n);
+   int *parents = ralloc_array(memctx, int, n);
+   memset(parents, 0xff, n * sizeof(*parents));
+   struct util_dynarray edges;
+   util_dynarray_init(&edges, memctx);
+
+   agx_foreach_instr_global(ctx, I) {
+      if (I->nr_dests == 1 && I->dest[0].type == AGX_INDEX_NORMAL &&
+          !I->dest[0].memory && !I->dest[0].has_reg)
+         defs[I->dest[0].value] = I;
+
+      if (!constant_phi_transparent(I) || I->op == AGX_OPCODE_MOV_IMM)
+         continue;
+
+      agx_foreach_src(I, s) {
+         unsigned src = I->src[s].value;
+         if (src >= n)
+            continue;
+         struct constant_phi_edge edge = {
+            .parent = I->dest[0].value, .next = parents[src],
+         };
+         parents[src] = util_dynarray_num_elements(&edges, struct constant_phi_edge);
+         util_dynarray_append(&edges, edge);
+      }
+   }
+
+   BITSET_WORD *visited = rzalloc_array(memctx, BITSET_WORD, BITSET_WORDS(n));
+   BITSET_WORD *grounded = rzalloc_array(memctx, BITSET_WORD, BITSET_WORDS(n));
+   unsigned *nodes = ralloc_array(memctx, unsigned, n);
+   unsigned *work = ralloc_array(memctx, unsigned, n);
+   bool progress = false;
+
+   agx_foreach_block(ctx, block) {
+      /* Preserve the ELSE/PHI prefix: literals execute after all surviving
+       * parallel PHIs, never between them. */
+      agx_cursor cursor = agx_after_block(block);
+      agx_foreach_instr_in_block(block, I) {
+         if (I->op != AGX_OPCODE_PHI && I->op != AGX_OPCODE_ELSE_ICMP &&
+             I->op != AGX_OPCODE_ELSE_FCMP) {
+            cursor = agx_before_instr(I);
+            break;
+         }
+      }
+      agx_builder b = agx_init_builder(ctx, cursor);
+      agx_foreach_phi_in_block_safe(block, phi) {
+         uint64_t literal = 0;
+         if (constant_phi_transparent(phi) &&
+             constant_phi_prove(defs, parents, util_dynarray_begin(&edges), n,
+                                phi->dest[0].value, visited, grounded,
+                                nodes, work, &literal)) {
+            agx_mov_imm_to(&b, phi->dest[0], literal);
+            agx_remove_instruction(phi);
+            progress = true;
+         }
+      }
+   }
+
+   ralloc_free(memctx);
+   return progress;
+}
+
 void
 agx_spill(agx_context *ctx, unsigned k, bool remat_only)
 {
@@ -1336,7 +1521,8 @@ agx_spill(agx_context *ctx, unsigned k, bool remat_only)
     * inserts none, but must still leave room for the registers RA reserves at
     * the bottom of the file, which is four for an AGX3 divergent shuffle.
     */
-   k -= remat_only ? MAX2(agx_reserved_registers(ctx), 2) : 8;
+   k -= remat_only ? MAX2(agx_reserved_registers(ctx), 2)
+                   : MAX2(agx_reserved_registers(ctx), 8);
 
    uint8_t *channels = rzalloc_array(memctx, uint8_t, ctx->alloc);
    dist_t *next_uses = rzalloc_array(memctx, dist_t, ctx->alloc);
@@ -1425,6 +1611,7 @@ agx_spill(agx_context *ctx, unsigned k, bool remat_only)
 
    /* Spilling breaks SSA, so we need to repair before validating */
    agx_repair_ssa(ctx);
+   agx_fold_m5_spill_constant_phis(ctx);
    agx_validate(ctx, "Spilling");
 
    /* Remat can introduce dead code */
diff --git a/src/asahi/compiler/agx_validate.c b/src/asahi/compiler/agx_validate.c
index cda8e63f7785..7e39d8ac9037 100644
--- a/src/asahi/compiler/agx_validate.c
+++ b/src/asahi/compiler/agx_validate.c
@@ -168,6 +168,10 @@ agx_write_registers(const agx_context *ctx, const agx_instr *I, unsigned d)
    case AGX_OPCODE_SCS_LOAD_REGISTER_BASE_AGX3:
       return size * util_last_bit(I->mask ? I->mask : 1);
 
+   case AGX_OPCODE_MATMAD16_AGX3:
+      return 16;
+   case AGX_OPCODE_MATMAD8_AGX3:
+      return 4;
    case AGX_OPCODE_MOV:
    case AGX_OPCODE_PHI:
       /* Tautological */
@@ -189,7 +193,7 @@ agx_write_registers(const agx_context *ctx, const agx_instr *I, unsigned d)
 
    case AGX_OPCODE_LOAD_CPROG_AGX3:
       assert(ctx->is_preamble && ctx->agx3_cprog_inputs_lowered &&
-             ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+             agx_isa_is_agx3(ctx->key->dev.encoding));
       assert(I->nr_dests == 1 && I->nr_srcs == 0 && !I->saturate);
       assert(I->dest[d].size == AGX_SIZE_16 || I->dest[d].size == AGX_SIZE_32);
       assert(agx_channels(I->dest[d]) == 1 && I->imm < 512 && size <= 512 - I->imm);
@@ -200,12 +204,12 @@ agx_write_registers(const agx_context *ctx, const agx_instr *I, unsigned d)
    case AGX_OPCODE_STACK_LOAD:
    case AGX_OPCODE_LD_TILE:
       if (I->op == AGX_OPCODE_LOCAL_LOAD && I->memory_is_packed) {
-         assert(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+         assert(agx_isa_is_agx3(ctx->key->dev.encoding));
          assert(I->format == AGX_FORMAT_I8 && I->dest[d].size == AGX_SIZE_16);
          return DIV_ROUND_UP(util_bitcount(I->mask), 2);
       }
       if (I->op == AGX_OPCODE_DEVICE_LOAD && I->memory_is_packed) {
-         assert(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+         assert(agx_isa_is_agx3(ctx->key->dev.encoding));
          assert(I->dest[d].size == AGX_SIZE_32);
          const unsigned bytes = I->format == AGX_FORMAT_I8 ? 1 :
                                 I->format == AGX_FORMAT_I16 ? 2 : 4;
@@ -220,7 +224,7 @@ agx_write_registers(const agx_context *ctx, const agx_instr *I, unsigned d)
           I->dest[d].size == AGX_SIZE_16 &&
           (I->format == AGX_FORMAT_U8NORM || I->format == AGX_FORMAT_S8NORM ||
            (I->format == AGX_FORMAT_I8 &&
-            ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B)))
+            agx_isa_is_agx3(ctx->key->dev.encoding))))
          return DIV_ROUND_UP(util_bitcount(I->mask), 2);
 
       /* Can write 16-bit or 32-bit. Anything logically 64-bit is already
@@ -305,6 +309,17 @@ agx_read_registers(const agx_context *ctx, const agx_instr *I, unsigned s)
       return 2;
    }
 
+   if (I->agx3_split_coords && (s == 0 || s == 6 || s == 7)) {
+      assert(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C &&
+             I->nr_srcs == 8 &&
+             ((I->op == AGX_OPCODE_TEXTURE_LOAD &&
+               (I->dim == AGX_DIM_2D || I->dim == AGX_DIM_2D_MS ||
+                I->dim == AGX_DIM_2D_ARRAY)) ||
+              (I->op == AGX_OPCODE_TEXTURE_SAMPLE &&
+               I->dim == AGX_DIM_2D)));
+      return 2;
+   }
+
    /* This is a sampler-private value, not a dimensional GPR coordinate. */
    if (agx_texture_resource(I) && s == 0)
       return agx_index_size_16(I->src[s]);
@@ -319,6 +334,10 @@ agx_read_registers(const agx_context *ctx, const agx_instr *I, unsigned s)
    case AGX_OPCODE_STORE_UVB_M5:
       return 2 * size;
 
+   case AGX_OPCODE_MATMAD16_AGX3:
+      return s == 2 ? 16 : 8;
+   case AGX_OPCODE_MATMAD8_AGX3:
+      return s == 2 || (I->matrix_type & 15) == 2 ? 4 : 2;
    case AGX_OPCODE_MOV:
    case AGX_OPCODE_EXPORT:
       /* Tautological */
@@ -343,7 +362,7 @@ agx_read_registers(const agx_context *ctx, const agx_instr *I, unsigned s)
    case AGX_OPCODE_UNIFORM_FMUL_AGX3:
    case AGX_OPCODE_UNIFORM_FFMA_AGX3:
       assert(ctx->is_preamble &&
-             ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+             agx_isa_is_agx3(ctx->key->dev.encoding));
       assert(I->nr_dests == 0 && I->imm < 128 && !(I->imm & 1));
       assert(I->nr_srcs == (I->op == AGX_OPCODE_UNIFORM_FFMA_AGX3 ? 3 : 2));
       assert(!I->saturate && !I->src[s].memory &&
@@ -365,7 +384,7 @@ agx_read_registers(const agx_context *ctx, const agx_instr *I, unsigned s)
       if (s == 0) {
          if ((I->op == AGX_OPCODE_DEVICE_STORE ||
               I->op == AGX_OPCODE_LOCAL_STORE) && I->memory_is_packed) {
-            assert(ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B);
+            assert(agx_isa_is_agx3(ctx->key->dev.encoding));
             assert(I->format == AGX_FORMAT_I8);
             return DIV_ROUND_UP(util_bitcount(I->mask), 2);
          }
@@ -402,7 +421,8 @@ agx_read_registers(const agx_context *ctx, const agx_instr *I, unsigned s)
       }
 
    case AGX_OPCODE_ZS_EMIT_AGX3:
-      assert(s == 0 && I->zs >= 1 && I->zs <= 3);
+      /* Bit 2 marks one word of an M5 covered-sample loop. */
+      assert(s == 0 && (I->zs & 3) && I->zs <= 7);
       return (I->zs & 1) ? 4 : 2;
 
    case AGX_OPCODE_IMAGE_WRITE:
@@ -420,6 +440,12 @@ agx_read_registers(const agx_context *ctx, const agx_instr *I, unsigned s)
          return agx_coordinate_registers(I, I->src[0].size);
       } else if (s == 1) {
          /* LOD */
+         if (I->op == AGX_OPCODE_TEXTURE_SAMPLE && I->memory_is_packed &&
+             ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3C) {
+            assert(I->src[s].size == AGX_SIZE_32 &&
+                   agx_channels(I->src[s]) == 1);
+            return 2;
+         }
          if ((I->op == AGX_OPCODE_TEXTURE_LOAD ||
               I->op == AGX_OPCODE_IMAGE_LOAD) &&
              ctx->key->dev.encoding == AGX_ISA_ENCODING_AGX3B &&
diff --git a/src/asahi/compiler/meson.build b/src/asahi/compiler/meson.build
index 1e51310f5f99..2b766a4f14d3 100644
--- a/src/asahi/compiler/meson.build
+++ b/src/asahi/compiler/meson.build
@@ -4,6 +4,8 @@
 
 libasahi_agx_files = files(
   'agx_compile.c',
+  'agx_nir_m5_ray_query.c',
+  'agx_nir_lower_cmat.c',
   'agx_dce.c',
   'agx_liveness.c',
   'agx_insert_waits.c',
@@ -13,8 +15,11 @@ libasahi_agx_files = files(
   'agx_nir_lower_divmod64.c',
   'agx_nir_lower_atomics.c',
   'agx_nir_opt_select_load.c',
+  'agx_nir_opt_sink_loop_sample_uses.c',
   'agx_nir_lower_cull_distance.c',
   'agx_nir_lower_fminmax.c',
+  'agx_nir_lower_word_bounds.c',
+  'agx_nir_share_bounds.c',
   'agx_nir_lower_frag_sidefx.c',
   'agx_nir_lower_sample_mask.c',
   'agx_nir_lower_discard_zs_emit.c',
@@ -27,6 +32,7 @@ libasahi_agx_files = files(
   'agx_nir_lower_compute_derivatives.c',
   'agx_nir_lower_texture.c',
   'agx_nir_m5_mesh.c',
+  'agx_nir_gen1_mesh.c',
   'agx3_mesh_core.c',
   'agx_nir_opt_preamble.c',
   'agx_lower_64bit.c',
@@ -54,7 +60,9 @@ libasahi_agx_files = files(
   'agx3_mem_core.c',
   'agx3_local_tps_core.c',
   'agx3_scs_core.c',
+  'agx3_rt_m5_core.c',
   'agx3_subgroup_core.c',
+  'agx3_matrix_core.c',
   'agx3_static_spill_core.c',
   'agx3_ioto_load_packer.c',
   'agx3_ioto_load_packer.h',
@@ -63,6 +71,7 @@ libasahi_agx_files = files(
   'agx3_compute_program.c',
   'agx3_compute_program.h',
   'agx3_graphics_program.c',
+  'agx3_m5_graphics_program.c',
   'agx3_graphics_program.h',
   'agx3_tex_core.c',
   'agx3_falu_mod_core.c',
@@ -197,7 +206,9 @@ if with_tests
          'test/test-preamble-image-payload.cpp',
          'test/test-preamble-share-values.cpp',
          'test/test-preamble-modifiers.cpp',
+         'test/test-preamble-texture-policy.cpp',
          'test/test-cprog-cleanup.cpp',
+         'test/test-m5-packed-lod.cpp',
          'test/test-preamble-half-exports.cpp',
          'test/test-demote-early-kill.cpp',
          'test/test-signed-zero-fma.cpp',
@@ -211,12 +222,16 @@ if with_tests
          'test/test-agx3-scs.cpp',
          'test/test-agx3-register-copy.cpp',
          'test/test-agx3-uniform-alu.cpp',
+         'test/test-position-order.cpp',
          'test/test-mixed-uniform-alu.cpp',
          'test/test-agx3-integer-fusion.cpp',
          'test/test-agx3-integer-lowering.cpp',
          'test/test-agx3-unary-lowering.cpp',
          'test/test-agx3-fsel-uniform.cpp',
          'test/test-fminmax-constant-denorm.cpp',
+         'test/test-shared-byte-offsets.cpp',
+         'test/test-device-load-batches.cpp',
+         'test/test-m5-copy-schedule.cpp',
       ),
       c_args : [c_msvc_compat_args, no_override_init_args],
       gnu_symbol_visibility : 'hidden',
@@ -316,6 +331,18 @@ if with_tests
     ),
     suite : ['asahi'],
   )
+  test(
+    'agx_nir_sink_loop_sample_uses',
+    executable(
+      'agx_nir_sink_loop_sample_uses_test',
+      files('test/test-sink-loop-sample-uses.c'),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      dependencies : [idep_nir, idep_mesautil],
+      link_with : [libasahi_compiler],
+    ),
+    suite : ['asahi'],
+  )
 
   test(
     'agx3_gen1_hit_commit',
diff --git a/src/asahi/compiler/test/agx_test.h b/src/asahi/compiler/test/agx_test.h
index 0c2a48c19bb5..ac249fe91132 100644
--- a/src/asahi/compiler/test/agx_test.h
+++ b/src/asahi/compiler/test/agx_test.h
@@ -128,6 +128,10 @@ agx_shader_equal(agx_context *A, agx_context *B)
       ASSERT_SHADER_EQUAL(A->shader, B->shader);                               \
    } while (0)
 
+#ifdef __cplusplus
+/* The helpers below use C++ default arguments; C tests (for example
+ * test-agx3-m5-instructions.c) only need the builder helpers above. */
+
 /* nir_builder's named-argument macros expand to an initializer that mixes a
  * positional element with designators, which C++ rejects as a C99 extension.
  * Build the intrinsic first and set its indices through the accessors. */
@@ -213,4 +217,6 @@ agx_test_store_local_pixel_agx(nir_builder *b, nir_def *value, nir_def *mask,
    return intr;
 }
 
+#endif /* __cplusplus */
+
 #endif
diff --git a/src/asahi/compiler/test/test-agx3-integer-lowering.cpp b/src/asahi/compiler/test/test-agx3-integer-lowering.cpp
index d35396b5feda..91846edd90f1 100644
--- a/src/asahi/compiler/test/test-agx3-integer-lowering.cpp
+++ b/src/asahi/compiler/test/test-agx3-integer-lowering.cpp
@@ -82,3 +82,143 @@ TEST(AGX3UniformInteger, LoweringCopiesUnsupportedDomains)
    }
    ralloc_free(mem);
 }
+
+
+TEST(AGX3UniformInteger, M5MadNativeBytesAndDomains)
+{
+   struct Witness {
+      const char *hex;
+      unsigned dest, a, u, c, wait;
+      bool c_uniform, a_cache, a_discard, c_discard;
+   } cases[] = {
+      {"3782041a2100a112988442000500", 10, 2, 18, 20, 0,
+       true, true, false, false},
+      {"2f00041a2100a102a88442000f00", 0, 0, 18, 14, 128,
+       true, false, true, false},
+      {"2708047a2500a1020880e0000400", 0, 0, 12, 4, 2,
+       false, false, true, true},
+   };
+   for (const auto &w : cases) {
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      agx_index r = agx_temp(b->shader, AGX_SIZE_32);
+      agx_index u = agx_uniform(w.u, AGX_SIZE_32);
+      agx_index c = w.c_uniform ? agx_uniform(w.c, AGX_SIZE_32)
+                               : agx_temp(b->shader, AGX_SIZE_32);
+      agx_instr *I = agx_imad_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                 w.c_uniform ? r : u,
+                                 w.c_uniform ? u : r, c, 0);
+      agx3_uniform_integer_desc d;
+      ASSERT_TRUE(agx3_uniform_integer_for_encoding(key.dev.encoding, I,
+                                                    false, &d));
+      agx_lower_uniform_sources(b->shader);
+      EXPECT_EQ(I->src[w.c_uniform ? 1 : 0].type, AGX_INDEX_UNIFORM);
+      I->dest[0] = agx_register(w.dest, AGX_SIZE_32);
+      unsigned slot = w.c_uniform ? 0 : 1;
+      I->src[slot] = agx_register(w.a, AGX_SIZE_32);
+      I->src[slot].cache = w.a_cache;
+      I->src[slot].discard = w.a_discard;
+      if (!w.c_uniform) {
+         I->src[2] = agx_register(w.c, AGX_SIZE_32);
+         I->src[2].discard = w.c_discard;
+      }
+      ASSERT_TRUE(agx3_uniform_integer_for_encoding(key.dev.encoding, I,
+                                                    true, &d));
+      d.wait_mask = w.wait;
+      uint8_t bytes[16], length;
+      ASSERT_TRUE(agx3_uniform_integer_encode(key.dev.encoding, &d, bytes,
+                                              &length));
+      ASSERT_EQ(length, 14);
+      for (unsigned j = 0; j < length; j++) {
+         unsigned expected;
+         ASSERT_EQ(sscanf(w.hex + 2*j, "%2x", &expected), 1);
+         EXPECT_EQ(bytes[j], expected) << "byte " << j << " witness " << w.hex;
+      }
+      auto invalid = d;
+      invalid.half = true;
+      EXPECT_FALSE(agx3_uniform_integer_encode(key.dev.encoding, &invalid,
+                                               bytes, &length));
+      invalid = d;
+      invalid.src[2] = {AGX3_UI_ZERO, 0};
+      EXPECT_FALSE(agx3_uniform_integer_encode(key.dev.encoding, &invalid,
+                                               bytes, &length));
+      invalid = d;
+      invalid.shift = 3;
+      EXPECT_FALSE(agx3_uniform_integer_encode(key.dev.encoding, &invalid,
+                                               bytes, &length));
+      ralloc_free(mem);
+   }
+}
+
+TEST(AGX3UniformInteger, M5MadPromotesBothConstantsTogether)
+{
+   for (bool constant_first : {false, true}) {
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->out = rzalloc(mem, struct agx_shader_info);
+      b->shader->out->push_count = 16;
+      agx_index r = agx_temp(b->shader, AGX_SIZE_32);
+      agx_index m = agx_mov_imm(b, 32, 1664525);
+      agx_index c = agx_mov_imm(b, 32, 1013904223);
+      agx_instr *I = agx_imad_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                 constant_first ? m : r,
+                                 constant_first ? r : m, c, 0);
+      agx_opt_promote_constants(b->shader);
+      unsigned ms = constant_first ? 0 : 1;
+      ASSERT_EQ(I->src[ms].type, AGX_INDEX_UNIFORM);
+      ASSERT_EQ(I->src[2].type, AGX_INDEX_UNIFORM);
+      uint32_t value;
+      memcpy(&value, &b->shader->rodata[I->src[ms].value -
+                  b->shader->out->rodata.base_uniform], sizeof(value));
+      EXPECT_EQ(value, 1664525u);
+      memcpy(&value, &b->shader->rodata[I->src[2].value -
+                  b->shader->out->rodata.base_uniform], sizeof(value));
+      EXPECT_EQ(value, 1013904223u);
+      agx3_uniform_integer_desc d;
+      EXPECT_TRUE(agx3_uniform_integer_for_encoding(key.dev.encoding, I,
+                                                   false, &d));
+      ralloc_free(mem);
+   }
+}
+
+
+TEST(AGX3UniformInteger, M5MadPromotionRefusesUnwitnessedDomains)
+{
+   for (unsigned variant = 0; variant < 6; variant++) {
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->out = rzalloc(mem, struct agx_shader_info);
+      b->shader->out->push_count = variant == 5 ? 510 : 16;
+      auto size = variant == 0 ? AGX_SIZE_16 : AGX_SIZE_32;
+      agx_index r = agx_temp(b->shader, size);
+      agx_index m = agx_mov_imm(b, size == AGX_SIZE_16 ? 16 : 32, 1664525);
+      agx_index c = agx_mov_imm(b, size == AGX_SIZE_16 ? 16 : 32, 1013904223);
+      if (variant == 1)
+         m.neg = true;
+      if (variant == 2)
+         c.abs = true;
+      agx_instr *I = agx_imad_to(b, agx_temp(b->shader, size), r, m, c,
+                                 variant == 3 ? 3 : 0);
+      I->saturate = variant == 4;
+      agx_opt_promote_constants(b->shader);
+      EXPECT_NE(I->src[2].type, AGX_INDEX_UNIFORM) << variant;
+      if (variant == 5) {
+         EXPECT_EQ(I->src[1].type, AGX_INDEX_UNIFORM);
+         agx3_uniform_integer_desc d;
+         EXPECT_TRUE(agx3_uniform_integer_for_encoding(key.dev.encoding, I,
+                                                      false, &d));
+      } else {
+         EXPECT_NE(I->src[1].type, AGX_INDEX_UNIFORM) << variant;
+      }
+      ralloc_free(mem);
+   }
+}
diff --git a/src/asahi/compiler/test/test-agx3-m5-instructions.c b/src/asahi/compiler/test/test-agx3-m5-instructions.c
new file mode 100644
index 000000000000..15e2dd18f279
--- /dev/null
+++ b/src/asahi/compiler/test/test-agx3-m5-instructions.c
@@ -0,0 +1,2065 @@
+/* SPDX-License-Identifier: MIT */
+
+#include "agx_compile.h"
+#include "agx3_static_spill_core.h"
+#include "agx_test.h"
+#include "agx3_copy_core.h"
+#include "agx3_tile_core.h"
+#include "agx3_alu_core.h"
+#include "agx3_atomic_core.h"
+#include "agx3_tex_core.h"
+#include "agx3_iadd_core.h"
+#include "agx3_uniform_integer.h"
+#include "agx3_mem_core.h"
+#include "agx3_scs_core.h"
+#include "agx3_rt_m5_core.h"
+#include "agx3_local_tps_core.h"
+#include "agx3_isel_core.h"
+#include "agx3_cf_core.h"
+#include "agx3_subgroup_core.h"
+#include "agx3_falu_mod_core.h"
+#include "agx3_stage_core.h"
+#include "agx3_pixel_core.h"
+#include "agx3_fltnorm_core.h"
+#include "agx3_pack.h"
+#include "asahi/lib/agx_bg_eot_key.h"
+#include "compiler/glsl_types.h"
+#include "compiler/nir/nir_builder.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Apple's M5 ray intersector, probe.gpubin SHA256
+ * 8aef372f7c4f43939c071f90d49e4429374b894fd3544dd89dc29459cee6aac4.
+ * Check ordinary data loads separately from destination-less epilogue forms.
+ */
+static bool
+test_native_ray_scs(void)
+{
+   static const struct { const char *hex; bool load, epilogue; } cases[] = {
+      {"af0001223204a91a000003400284", false, false}, /* native +0x018e */
+      {"370401223204a912000409000284", false, false}, /* native +0x01b6 */
+      {"b70001223204a902100401000284", false, false}, /* native +0x0200 */
+      {"a70801223204a922108409000284", false, false}, /* native +0x020e */
+      {"370101023600a9120208e2460184", false, false}, /* native +0x021c */
+      {"270101023600a93a2208e9060184", false, false}, /* native +0x022a */
+      {"270101023600a92a1208e3460184", false, false}, /* native +0x0238 */
+      {"a70001223204a91a100003400284", false, false}, /* native +0x0246 */
+      {"270001223204a92a200003400284", false, false}, /* native +0x028a */
+      {"a70001223204a91a300003400284", false, false}, /* native +0x0298 */
+      {"270001225204a93a208409000284", false, false}, /* native +0x02b2 */
+      {"a70101023600a9024208e8060184", false, false}, /* native +0x02c0 */
+      {"a700018632a0a962020883430184", false, false}, /* native +0x032c */
+      {"3700018612a7a85212c8880301840000", true, false}, /* native +0x033a */
+      {"a700018632a0a962120889430184", false, false}, /* native +0x0558 */
+      {"b700018632a0a95a12c888030184", false, false}, /* native +0x0566 */
+      {"a700018632a0a942224882430184", false, false}, /* native +0x0574 */
+      {"2700018632a0a95a324888430184", false, false}, /* native +0x0582 */
+      {"3700018632a0a952224882430184", false, false}, /* native +0x05d0 */
+      {"b700018632a0a95a12c889030184", false, false}, /* native +0x05ea */
+      {"2f00018c32a7a8423238800301840000", true, false}, /* native +0x060c */
+      {"17000108020788020070000001040000", true, false}, /* native +0x0642 */
+      {"370001082200a902007001000104", false, false}, /* native +0x0666 */
+      {"3f0001082207a8020070000001040000", true, false}, /* native +0x0682 */
+      {"2700018632a7a84222c8800301840000", true, false}, /* native +0x06b0 */
+      {"2700018632a7a85a1288800301840000", true, false}, /* native +0x06d6 */
+      {"a700018612a7a8427248800301840000", true, false}, /* native +0x06e6 */
+      {"270021122200a80200082f0001040100", true, true}, /* native +0x0778 */
+      {"270021122200a80200082f0001040102", true, true}, /* native +0x0788 */
+      {"270021122200a80200082f0001040104", true, true}, /* native +0x0798 */
+      {"27002112a200a80200082f0001040106", true, true}, /* native +0x07a8 */
+      {"270021122200a80200082f0001040100", true, true}, /* native +0x07c6 */
+      {"270021122200a80200082f0001040102", true, true}, /* native +0x07d6 */
+      {"270021128600a8024008210001840104", true, true}, /* native +0x07e6 */
+      /* Native M5 instanced intersector; see operand-probes/build.json. */
+      {"270001040243a8520268a00301840000", true, false},
+      {"27000104a240a9524268a1030184", false, false},
+      {"b7000106020aa81a0208800301040000", true, false},
+      {"270001062208a90a020889030104", false, false},
+      {"a70001062208a93a0208c9030104", false, false},
+      {"3f00010c020fa8020278c00301040000", true, false},
+      {"37000106220fa8020208c00301040000", true, false},
+      {"27000106820fa8024208800301040002", true, false},
+      {"270021122200a80200082f0001040100", true, true},
+      {"270021122200a80200082f0001040102", true, true},
+      {"270021122200a80200082f0001040104", true, true},
+      {"27002112a200a80200082f0001040106", true, true},
+      {"270021122200a80200082f0001040100", true, true},
+      {"270021122200a80200082f0001040102", true, true},
+      {"270021122200a80200082f0001040104", true, true},
+      {"270021122200a80200082f0001040106", true, true},
+      {"270021128600a8024008210001840108", true, true},
+   };
+   for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) {
+      uint8_t bytes[16], packed[16], length = 0;
+      const size_t size = strlen(cases[i].hex) / 2;
+      for (unsigned j = 0; j < size; ++j) {
+         unsigned value;
+         if (sscanf(cases[i].hex + 2 * j, "%2x", &value) != 1)
+            return false;
+         bytes[j] = value;
+      }
+      struct agx3_scs_desc desc;
+      struct agx3_scs_epilogue_desc epilogue;
+      bool decoded = cases[i].epilogue
+         ? agx3_scs_decode_epilogue(AGX_ISA_ENCODING_AGX3C, bytes, size,
+                                    &epilogue)
+         : cases[i].load
+            ? agx3_scs_decode_load(AGX_ISA_ENCODING_AGX3C, bytes, size, &desc)
+            : agx3_scs_decode_store(AGX_ISA_ENCODING_AGX3C, bytes, size, &desc);
+      if (!decoded)
+         return false;
+      /* A destination-less operation must never be accepted as a GPR load,
+       * and an ordinary operation must never decode as an epilogue. */
+      if (cases[i].epilogue
+             ? agx3_scs_decode_load(AGX_ISA_ENCODING_AGX3C, bytes, size, &desc)
+             : agx3_scs_decode_epilogue(AGX_ISA_ENCODING_AGX3C, bytes, size,
+                                         &epilogue))
+         return false;
+      bool encoded = cases[i].epilogue
+         ? agx3_scs_encode_epilogue(AGX_ISA_ENCODING_AGX3C, &epilogue, packed,
+                                    &length)
+         : cases[i].load
+            ? agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, packed, &length)
+            : agx3_scs_encode_store(AGX_ISA_ENCODING_AGX3C, &desc, packed, &length);
+      if (!encoded || length != size || memcmp(bytes, packed, size))
+         return false;
+   }
+   uint8_t bytes[16], length;
+   struct agx3_scs_desc desc = {
+      .data = 250, .width = AGX3_MEM_W32, .component_mask = 15, .slot = 8,
+   };
+   /* A vector may not wrap the register file, even if its first word fits. */
+   if (agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+   desc.data = 248;
+   if (!agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+   desc.index_kill = true;
+   if (agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+   desc.index_kill = false;
+   desc.c_store_marker = true;
+   if (agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+   desc = (struct agx3_scs_desc) {
+      .data = 2, .width = AGX3_MEM_W8, .component_mask = 1, .slot = 8,
+      .address_kind = AGX3_SCS_ADDRESS_REGISTER_BASE, .base_register = 2,
+      .has_address_selector = true, .address_selector = 4,
+      .byte_offset = 7, .c_load_marker = true, .c_status_load = true,
+   };
+   if (!agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length) ||
+       agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3B, &desc, bytes, &length) ||
+       agx3_scs_encode_store(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+   desc.c_load_marker = false;
+   if (agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+   desc.c_load_marker = true;
+   desc.width = AGX3_MEM_W32;
+   if (agx3_scs_encode_load(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length))
+      return false;
+
+   struct agx3_scs_epilogue_desc epilogue = {
+      .index = 2, .base_register = 2, .component_mask = 15,
+      .address_selector = 4,
+   };
+   if (!agx3_scs_encode_epilogue(AGX_ISA_ENCODING_AGX3C, &epilogue,
+                                 bytes, &length) ||
+       agx3_scs_encode_epilogue(AGX_ISA_ENCODING_AGX3B, &epilogue,
+                                 bytes, &length))
+      return false;
+   for (unsigned invalid = 0; invalid < 5; ++invalid) {
+      struct agx3_scs_epilogue_desc bad = epilogue;
+      switch (invalid) {
+      case 0: bad.index = 256; break;
+      case 1: bad.base_register = 256; break;
+      case 2: bad.byte_offset = 1; break;
+      case 3: bad.component_mask = 0; break;
+      case 4: bad.address_selector = 3; break;
+      }
+      memset(bytes, 0xff, sizeof(bytes));
+      length = 0xff;
+      if (agx3_scs_encode_epilogue(AGX_ISA_ENCODING_AGX3C, &bad,
+                                    bytes, &length) || length)
+         return false;
+      for (unsigned j = 0; j < sizeof(bytes); ++j)
+         if (bytes[j])
+            return false;
+   }
+   return true;
+}
+
+static bool
+test_native_rt_m5(void)
+{
+   const struct {
+      struct agx3_rt_m5_intersect_desc desc;
+      uint8_t bytes[14];
+   } cases[] = {
+      {{.address_register = 0, .bvh_base = 96, .bvh_uniform = true},
+       {0x27,0x04,0x29,0x01,0x22,0xc1,0xae,0x02,0x01,0x98,0x00,0x00,0x00,0xa0}},
+      {{.address_register = 0, .bvh_base = 46, .bvh_kill = true},
+       {0x27,0x04,0x29,0x01,0xa2,0xc3,0xae,0x02,0x80,0x8a,0x00,0x00,0x00,0xa0}},
+      {{.address_register = 34, .bvh_base = 96, .bvh_uniform = true},
+       {0x27,0x04,0x29,0x01,0x22,0xc1,0xae,0x02,0x01,0x98,0x21,0x00,0x00,0xa0}},
+   };
+   uint8_t bytes[16], length;
+   for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) {
+      if (!agx3_rt_m5_encode_intersect(AGX_ISA_ENCODING_AGX3C,
+                                      &cases[i].desc, bytes, &length) ||
+          length != sizeof(cases[i].bytes) ||
+          memcmp(bytes, cases[i].bytes, length))
+         return false;
+   }
+   const uint8_t continuation[] = {0x27,0x04,0x09,0x00,0x32,0xc0,0xaf,0x02};
+   if (!agx3_rt_m5_encode_continue(AGX_ISA_ENCODING_AGX3C, bytes, &length) ||
+       length != sizeof(continuation) || memcmp(bytes, continuation, length))
+      return false;
+   struct agx3_rt_m5_intersect_desc bad = cases[0].desc;
+   if (agx3_rt_m5_encode_intersect(AGX_ISA_ENCODING_AGX3B, &bad, bytes, &length))
+      return false;
+   bad.address_register = 253;
+   if (agx3_rt_m5_encode_intersect(AGX_ISA_ENCODING_AGX3C, &bad, bytes, &length))
+      return false;
+   bad = cases[0].desc;
+   bad.bvh_kill = true;
+   if (agx3_rt_m5_encode_intersect(AGX_ISA_ENCODING_AGX3C, &bad, bytes, &length))
+      return false;
+   bad = cases[1].desc;
+   bad.bvh_base = 256;
+   return !agx3_rt_m5_encode_intersect(AGX_ISA_ENCODING_AGX3C, &bad, bytes, &length);
+}
+
+static bool
+test_native_m5_ift_entries(void)
+{
+   {
+      static const uint8_t native[] = {
+         0x4c, 0x80, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0xcc, 0xc0, 0x02, 0x00,
+         0x16, 0x00, 0x00, 0x00, 0xd4, 0x10, 0xa7, 0x00, 0x01, 0x02, 0x26, 0x81,
+         0xa8, 0x0a, 0x10, 0x04, 0x08, 0x00, 0x01, 0x84, 0x00, 0x00, 0x0f, 0x18,
+         0x43, 0x00, 0xa8, 0x0c, 0x00, 0x42, 0xcf, 0x18, 0x03, 0x00, 0xe8, 0x0c,
+         0x00, 0x40, 0x07, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x07, 0x04,
+         0x58, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x07, 0x00, 0x00, 0x0d,
+         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x80, 0x02, 0x00, 0x00, 0x00,
+         0x00, 0x00, 0x34, 0x81, 0x01, 0x00, 0x20, 0x00, 0xa0, 0x02, 0x0c, 0x80,
+         0x02, 0x00, 0x10, 0x00, 0x00, 0x00, 0x27, 0x04, 0x04, 0x1a, 0x21, 0x00,
+         0xa3, 0x02, 0x88, 0x80, 0x00, 0x00, 0x1c, 0x80, 0x02, 0x00, 0x04, 0x00,
+         0x00, 0x00, 0x27, 0x04, 0x04, 0x00, 0x30, 0x00, 0xa4, 0x12, 0x0b, 0x14,
+         0x09, 0x44, 0x1b, 0x16, 0x09, 0x04, 0x6b, 0x20, 0x09, 0x04, 0x7b, 0x22,
+         0x09, 0x04, 0x2b, 0x18, 0x09, 0x04, 0x3b, 0x1a, 0x09, 0x04, 0x4b, 0x1c,
+         0x09, 0x04, 0x5b, 0x1e, 0x09, 0x04, 0x23, 0x00, 0x07, 0x02, 0x42, 0x20,
+         0xa0, 0x02, 0x0e, 0x00, 0x00, 0x00,
+      };
+      static const struct agx3_rt_m5_ift_uniform uniforms[] = {{0, 20}, {2, 22}, {12, 32}, {14, 34}, {4, 24}, {6, 26}, {8, 28}, {10, 30}};
+      struct agx3_rt_m5_ift_entry_desc desc = {.loader = {.constants_va = UINT64_C(68719478208), .shader_va = UINT64_C(68719477760), .constant_dwords = 6, .fetch_span = 3, .context48 = false}, .uniforms = uniforms, .uniform_count = 8, .tpr20 = false, .final_copy16 = true};
+      uint8_t bytes[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE];
+      size_t length = 0;
+      if (!agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) ||
+          length != sizeof(native) || memcmp(bytes, native, sizeof(native)))
+         return false;
+      memset(bytes, 0xa5, sizeof(bytes));
+      desc.loader.shader_va |= 1;
+      length = 123;
+      if (agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) || length != 123)
+         return false;
+      for (unsigned i = 0; i < sizeof(bytes); ++i)
+         if (bytes[i] != 0xa5) return false;
+   }
+   {
+      static const uint8_t native[] = {
+         0x4c, 0x80, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0xcc, 0x80, 0x02, 0x00,
+         0x12, 0x04, 0x1e, 0x02, 0xd4, 0x10, 0xa7, 0x00, 0x01, 0x02, 0x26, 0x81,
+         0xa8, 0x0a, 0x10, 0x04, 0x08, 0x00, 0x01, 0x84, 0x00, 0x00, 0xcf, 0x18,
+         0x83, 0x00, 0xa8, 0x0c, 0x00, 0x50, 0x8f, 0x18, 0x83, 0x00, 0xe8, 0x0c,
+         0x00, 0x4e, 0x4f, 0x18, 0x83, 0x00, 0xe8, 0x0c, 0x00, 0x4c, 0x0f, 0x18,
+         0x83, 0x00, 0xe8, 0x0c, 0x00, 0x4a, 0xcf, 0x18, 0x43, 0x00, 0xe8, 0x0c,
+         0x00, 0x48, 0x8f, 0x18, 0x43, 0x00, 0xe8, 0x0c, 0x00, 0x46, 0x4f, 0x18,
+         0x43, 0x00, 0xe8, 0x0c, 0x00, 0x44, 0x0f, 0x18, 0x43, 0x00, 0xe8, 0x0c,
+         0x00, 0x42, 0xcf, 0x18, 0x03, 0x00, 0xe8, 0x0c, 0x00, 0x40, 0x07, 0x00,
+         0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0xc7, 0xef, 0x58, 0x0e, 0x00, 0x00,
+         0x42, 0x00, 0x10, 0x00, 0x07, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00,
+         0x00, 0x00, 0x1c, 0x80, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x81,
+         0x01, 0x00, 0x20, 0x00, 0xa0, 0x02, 0x0c, 0x80, 0x02, 0x00, 0x10, 0x00,
+         0x00, 0x00, 0x27, 0x04, 0x04, 0x1a, 0x21, 0x00, 0xa3, 0x02, 0x88, 0x80,
+         0x00, 0x00, 0x1c, 0x80, 0x02, 0x00, 0x04, 0x00, 0x00, 0x00, 0x27, 0x04,
+         0x04, 0x00, 0x30, 0x00, 0xa4, 0x12, 0x0b, 0x14, 0x09, 0x44, 0x1b, 0x16,
+         0x09, 0x04, 0x2b, 0x58, 0x89, 0x04, 0x3b, 0x5a, 0x89, 0x04, 0xeb, 0x50,
+         0x49, 0x04, 0xfb, 0x52, 0x49, 0x04, 0x0b, 0x54, 0x89, 0x04, 0x1b, 0x56,
+         0x89, 0x04, 0xab, 0x48, 0x49, 0x04, 0xbb, 0x4a, 0x49, 0x04, 0xcb, 0x4c,
+         0x49, 0x04, 0xdb, 0x4e, 0x49, 0x04, 0x6b, 0x40, 0x49, 0x04, 0x7b, 0x42,
+         0x49, 0x04, 0x8b, 0x44, 0x49, 0x04, 0x9b, 0x46, 0x49, 0x04, 0x2b, 0x38,
+         0x49, 0x04, 0x3b, 0x3a, 0x49, 0x04, 0x4b, 0x3c, 0x49, 0x04, 0x5b, 0x3e,
+         0x49, 0x04, 0xeb, 0x30, 0x09, 0x04, 0xfb, 0x32, 0x09, 0x04, 0x0b, 0x34,
+         0x49, 0x04, 0x1b, 0x36, 0x49, 0x04, 0xab, 0x28, 0x09, 0x04, 0xbb, 0x2a,
+         0x09, 0x04, 0xcb, 0x2c, 0x09, 0x04, 0xdb, 0x2e, 0x09, 0x04, 0x6b, 0x20,
+         0x09, 0x04, 0x7b, 0x22, 0x09, 0x04, 0x8b, 0x24, 0x09, 0x04, 0x9b, 0x26,
+         0x09, 0x04, 0x2b, 0x18, 0x09, 0x04, 0x3b, 0x1a, 0x09, 0x04, 0x4b, 0x1c,
+         0x09, 0x04, 0x5b, 0x1e, 0x09, 0x04, 0x23, 0x00, 0x07, 0x02, 0x42, 0x20,
+         0xa0, 0x02, 0x0e, 0x00, 0x00, 0x00,
+      };
+      static const struct agx3_rt_m5_ift_uniform uniforms[] = {{0, 20}, {2, 22}, {68, 88}, {70, 90}, {60, 80}, {62, 82}, {64, 84}, {66, 86}, {52, 72}, {54, 74}, {56, 76}, {58, 78}, {44, 64}, {46, 66}, {48, 68}, {50, 70}, {36, 56}, {38, 58}, {40, 60}, {42, 62}, {28, 48}, {30, 50}, {32, 52}, {34, 54}, {20, 40}, {22, 42}, {24, 44}, {26, 46}, {12, 32}, {14, 34}, {16, 36}, {18, 38}, {4, 24}, {6, 26}, {8, 28}, {10, 30}};
+      struct agx3_rt_m5_ift_entry_desc desc = {.loader = {.constants_va = UINT64_C(68723920000), .shader_va = UINT64_C(68723863488), .constant_dwords = 34, .fetch_span = 3, .context48 = false}, .uniforms = uniforms, .uniform_count = 36, .tpr20 = false, .final_copy16 = true};
+      uint8_t bytes[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE];
+      size_t length = 0;
+      if (!agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) ||
+          length != sizeof(native) || memcmp(bytes, native, sizeof(native)))
+         return false;
+      memset(bytes, 0xa5, sizeof(bytes));
+      desc.loader.shader_va |= 1;
+      length = 123;
+      if (agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) || length != 123)
+         return false;
+      for (unsigned i = 0; i < sizeof(bytes); ++i)
+         if (bytes[i] != 0xa5) return false;
+   }
+   {
+      static const uint8_t native[] = {
+         0x4c, 0x80, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0xec, 0xc0, 0x02, 0x00,
+         0x0c, 0x00, 0x4f, 0x02, 0xf4, 0x10, 0xa7, 0x00, 0x01, 0x02, 0x26, 0x81,
+         0xa8, 0x12, 0x10, 0x04, 0x08, 0x00, 0x01, 0x84, 0x00, 0x00, 0xa7, 0x00,
+         0x01, 0x02, 0x26, 0x81, 0xa8, 0x0a, 0x30, 0x00, 0x08, 0x00, 0x01, 0x84,
+         0x00, 0x00, 0xef, 0x1c, 0x43, 0x00, 0xe8, 0x0c, 0x00, 0x48, 0xaf, 0x1c,
+         0x43, 0x00, 0xe8, 0x0c, 0x00, 0x46, 0x6f, 0x1c, 0x43, 0x00, 0xe8, 0x0c,
+         0x00, 0x44, 0x2f, 0x1c, 0x43, 0x00, 0xe8, 0x0c, 0x00, 0x42, 0xef, 0x1c,
+         0x03, 0x00, 0xe8, 0x0c, 0x00, 0x40, 0x07, 0x00, 0x00, 0x0c, 0x00, 0x00,
+         0x00, 0x00, 0xc7, 0xd6, 0x58, 0x0e, 0x00, 0x00, 0x49, 0x00, 0x10, 0x00,
+         0x07, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x80,
+         0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x81, 0x01, 0x00, 0x20, 0x00,
+         0xa0, 0x02, 0x0c, 0x80, 0x02, 0x00, 0x10, 0x00, 0x00, 0x00, 0x27, 0x04,
+         0x04, 0x1a, 0x21, 0x00, 0xa3, 0x02, 0x88, 0x80, 0x00, 0x00, 0x1c, 0x80,
+         0x02, 0x00, 0x04, 0x00, 0x00, 0x00, 0x27, 0x04, 0x04, 0x00, 0x30, 0x00,
+         0xa4, 0x12, 0x2b, 0x18, 0x09, 0x44, 0x3b, 0x1a, 0x09, 0x04, 0x0b, 0x14,
+         0x09, 0x04, 0x1b, 0x16, 0x09, 0x04, 0x4b, 0x3c, 0x49, 0x04, 0x5b, 0x3e,
+         0x49, 0x04, 0x6b, 0x40, 0x49, 0x04, 0x7b, 0x42, 0x49, 0x04, 0x0b, 0x34,
+         0x49, 0x04, 0x1b, 0x36, 0x49, 0x04, 0x2b, 0x38, 0x49, 0x04, 0x3b, 0x3a,
+         0x49, 0x04, 0xcb, 0x2c, 0x09, 0x04, 0xdb, 0x2e, 0x09, 0x04, 0xeb, 0x30,
+         0x09, 0x04, 0xfb, 0x32, 0x09, 0x04, 0x8b, 0x24, 0x09, 0x04, 0x9b, 0x26,
+         0x09, 0x04, 0xab, 0x28, 0x09, 0x04, 0xbb, 0x2a, 0x09, 0x04, 0x4b, 0x1c,
+         0x09, 0x04, 0x5b, 0x1e, 0x09, 0x04, 0x6b, 0x20, 0x09, 0x04, 0x7b, 0x22,
+         0x09, 0x04, 0x23, 0x00, 0x07, 0x02, 0x42, 0x20, 0xa0, 0x02, 0x0e, 0x00,
+         0x00, 0x00,
+      };
+      static const struct agx3_rt_m5_ift_uniform uniforms[] = {{4, 24}, {6, 26}, {0, 20}, {2, 22}, {40, 60}, {42, 62}, {44, 64}, {46, 66}, {32, 52}, {34, 54}, {36, 56}, {38, 58}, {24, 44}, {26, 46}, {28, 48}, {30, 50}, {16, 36}, {18, 38}, {20, 40}, {22, 42}, {8, 28}, {10, 30}, {12, 32}, {14, 34}};
+      struct agx3_rt_m5_ift_entry_desc desc = {.loader = {.constants_va = UINT64_C(68724319040), .shader_va = UINT64_C(68724315840), .constant_dwords = 20, .fetch_span = 3, .context48 = true}, .uniforms = uniforms, .uniform_count = 24, .tpr20 = false, .final_copy16 = true};
+      uint8_t bytes[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE];
+      size_t length = 0;
+      if (!agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) ||
+          length != sizeof(native) || memcmp(bytes, native, sizeof(native)))
+         return false;
+      memset(bytes, 0xa5, sizeof(bytes));
+      desc.loader.shader_va |= 1;
+      length = 123;
+      if (agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) || length != 123)
+         return false;
+      for (unsigned i = 0; i < sizeof(bytes); ++i)
+         if (bytes[i] != 0xa5) return false;
+   }
+   {
+      static const uint8_t native[] = {
+         0x4c, 0x80, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0xec, 0x80, 0x02, 0x00,
+         0x10, 0x08, 0x54, 0x02, 0xf4, 0x10, 0xa7, 0x00, 0x01, 0x02, 0x26, 0x81,
+         0xa8, 0x12, 0x10, 0x04, 0x08, 0x00, 0x01, 0x84, 0x00, 0x00, 0xa7, 0x00,
+         0x01, 0x02, 0x26, 0x81, 0xa8, 0x0a, 0x30, 0x00, 0x08, 0x00, 0x01, 0x84,
+         0x00, 0x00, 0x2f, 0x1c, 0x83, 0x00, 0xe8, 0x0c, 0x00, 0x4a, 0xef, 0x1c,
+         0x43, 0x00, 0xe8, 0x0c, 0x00, 0x48, 0xaf, 0x1c, 0x43, 0x00, 0xe8, 0x0c,
+         0x00, 0x46, 0x6f, 0x1c, 0x43, 0x00, 0xe8, 0x0c, 0x00, 0x44, 0x2f, 0x1c,
+         0x43, 0x00, 0xe8, 0x0c, 0x00, 0x42, 0xef, 0x1c, 0x03, 0x00, 0xe8, 0x0c,
+         0x00, 0x40, 0x07, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x07, 0x80,
+         0x58, 0x0e, 0x00, 0x00, 0x4a, 0x00, 0x10, 0x00, 0x07, 0x00, 0x00, 0x0d,
+         0x00, 0x00, 0x14, 0x00, 0x14, 0x00, 0x1c, 0x80, 0x02, 0x00, 0x00, 0x00,
+         0x00, 0x00, 0x34, 0x81, 0x01, 0x00, 0x20, 0x00, 0xa0, 0x02, 0x0c, 0x80,
+         0x02, 0x00, 0x10, 0x00, 0x00, 0x00, 0x27, 0x04, 0x04, 0x1a, 0x21, 0x00,
+         0xa3, 0x02, 0x88, 0x80, 0x00, 0x00, 0x1c, 0x80, 0x02, 0x00, 0x04, 0x00,
+         0x00, 0x00, 0x27, 0x04, 0x04, 0x00, 0x30, 0x00, 0xa4, 0x12, 0x2b, 0x18,
+         0x09, 0x44, 0x3b, 0x1a, 0x09, 0x04, 0x0b, 0x14, 0x09, 0x04, 0x1b, 0x16,
+         0x09, 0x04, 0x8b, 0x44, 0x49, 0x04, 0x9b, 0x46, 0x49, 0x04, 0xab, 0x48,
+         0x49, 0x04, 0xbb, 0x4a, 0x49, 0x04, 0x4b, 0x3c, 0x49, 0x04, 0x5b, 0x3e,
+         0x49, 0x04, 0x6b, 0x40, 0x49, 0x04, 0x7b, 0x42, 0x49, 0x04, 0x0b, 0x34,
+         0x49, 0x04, 0x1b, 0x36, 0x49, 0x04, 0x2b, 0x38, 0x49, 0x04, 0x3b, 0x3a,
+         0x49, 0x04, 0xcb, 0x2c, 0x09, 0x04, 0xdb, 0x2e, 0x09, 0x04, 0xeb, 0x30,
+         0x09, 0x04, 0xfb, 0x32, 0x09, 0x04, 0x8b, 0x24, 0x09, 0x04, 0x9b, 0x26,
+         0x09, 0x04, 0xab, 0x28, 0x09, 0x04, 0xbb, 0x2a, 0x09, 0x04, 0x4b, 0x1c,
+         0x09, 0x04, 0x5b, 0x1e, 0x09, 0x04, 0x6b, 0x20, 0x09, 0x04, 0x7b, 0x22,
+         0x09, 0x04, 0x23, 0x00, 0x07, 0x02, 0x42, 0x20, 0xa0, 0x02, 0x0e, 0x00,
+         0x00, 0x00,
+      };
+      static const struct agx3_rt_m5_ift_uniform uniforms[] = {{4, 24}, {6, 26}, {0, 20}, {2, 22}, {48, 68}, {50, 70}, {52, 72}, {54, 74}, {40, 60}, {42, 62}, {44, 64}, {46, 66}, {32, 52}, {34, 54}, {36, 56}, {38, 58}, {24, 44}, {26, 46}, {28, 48}, {30, 50}, {16, 36}, {18, 38}, {20, 40}, {22, 42}, {8, 28}, {10, 30}, {12, 32}, {14, 34}};
+      struct agx3_rt_m5_ift_entry_desc desc = {.loader = {.constants_va = UINT64_C(68724364288), .shader_va = UINT64_C(68724359168), .constant_dwords = 24, .fetch_span = 3, .context48 = true}, .uniforms = uniforms, .uniform_count = 28, .tpr20 = true, .final_copy16 = true};
+      uint8_t bytes[AGX3_RT_M5_IFT_ENTRY_MAX_SIZE];
+      size_t length = 0;
+      if (!agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) ||
+          length != sizeof(native) || memcmp(bytes, native, sizeof(native)))
+         return false;
+      memset(bytes, 0xa5, sizeof(bytes));
+      desc.loader.shader_va |= 1;
+      length = 123;
+      if (agx3_rt_m5_ift_entry_generate(&desc, bytes, &length) || length != 123)
+         return false;
+      for (unsigned i = 0; i < sizeof(bytes); ++i)
+         if (bytes[i] != 0xa5) return false;
+   }
+   return true;
+}
+
+static bool
+test_native_m5_hit_commit(void)
+{
+   /* Native IFT type1, candidate8 address0x10000012c0. */
+   static const uint8_t expected[] = {
+      0x04,0xe9,0x10,0x06,0x33,0x04,0x07,0x00,0x38,0x20,0xa1,0x0a,0xc4,0x01,0x37,0x02,
+      0x04,0x3a,0x21,0x00,0xa1,0x12,0x00,0x10,0x0a,0x00,0x01,0x02,0x37,0x00,0x01,0x02,
+      0x12,0x47,0xa8,0x02,0x32,0x08,0x20,0x01,0x01,0x84,0x00,0x00,0x24,0xe8,0x10,0x06,
+      0x24,0x82,0x01,0x38,0x28,0x01,0xa0,0x12,0x0c,0x80,0x02,0x40,0x00,0x00,0x00,0x00,
+      0x27,0x04,0x04,0x3a,0x21,0x00,0xa1,0x0a,0xa0,0x01,0x02,0x00,0x01,0x02,0x3b,0x02,
+      0x07,0x80,0x3a,0x38,0xa1,0x02,0x84,0x82,0x43,0x00,0x00,0x01,0x37,0x02,0x04,0x38,
+      0x21,0x00,0xa3,0x0a,0x20,0x02,0x03,0x00,0x33,0x00,0x00,0x01,0x27,0x00,0x01,0x02,
+      0x32,0x40,0xa9,0x02,0x62,0xc8,0x29,0x01,0x01,0x84,0x37,0x08,0x01,0x02,0xc6,0x40,
+      0xa9,0x0a,0x42,0x08,0x01,0x01,0x01,0x84,0x24,0xe8,0x10,0x06,0x0c,0xc0,0x22,0x00,
+      0x02,0x00,0x00,0x00,0x1c,0x80,0x22,0x00,0x08,0x00,0x00,0x00,0x27,0x04,0x04,0x3a,
+      0x21,0x00,0xa1,0x02,0xa0,0x80,0xe0,0x00,0x00,0x00,0x27,0x00,0x21,0x12,0x82,0x00,
+      0xa8,0x02,0x00,0x00,0x03,0x00,0x01,0x84,0x01,0x00,0x17,0x09,0x00,0x06,0x00,0x00,
+      0x0e,0x00,0x00,0x00,
+   };
+   struct agx3_rt_m5_hit_commit_desc d = {4, 6, 8};
+   uint8_t bytes[AGX3_RT_M5_HIT_COMMIT_MAX_SIZE];
+   size_t length = 0;
+   if (!agx3_rt_m5_hit_commit_generate(&d, bytes, &length) ||
+       length != sizeof(expected) || memcmp(bytes, expected, length))
+      return false;
+   d.payload_uniform = 511;
+   memset(bytes, 0xa5, sizeof(bytes));
+   length = 123;
+   if (agx3_rt_m5_hit_commit_generate(&d, bytes, &length) || length != 123)
+      return false;
+   for (unsigned i = 0; i < sizeof(bytes); ++i)
+      if (bytes[i] != 0xa5) return false;
+   return true;
+}
+
+int main(void)
+{
+   if (!test_native_ray_scs() || !test_native_rt_m5() ||
+       !test_native_m5_ift_entries() || !test_native_m5_hit_commit())
+      return 140;
+   /* Preserve register-base SCS and both native BVH operand classes through
+    * the actual IR packer. The SCS load overlaps the ray address operand,
+    * exercising input retirement before the field-free ray instruction. */
+   {
+      void *memctx = ralloc_context(NULL);
+      agx_builder *b = agx_test_builder(memctx);
+      struct agx_shader_key key = {0};
+      struct agx_shader_info info = {0};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      b->shader->key = &key;
+      b->shader->out = &info;
+      b->shader->nir = rzalloc(memctx, nir_shader);
+      b->shader->nir->info.stage = MESA_SHADER_COMPUTE;
+      b->shader->stage = info.stage = MESA_SHADER_COMPUTE;
+      b->shader->max_reg = 49;
+      agx_index value = agx_register(4, AGX_SIZE_32);
+      value.channels_m1 = 1;
+      agx_scs_store_register_indexed_agx3(b, value,
+         agx_register(32, AGX_SIZE_32), agx_register(28, AGX_SIZE_32),
+         0, AGX_FORMAT_I32, 3, 3);
+      agx_scs_load_register_indexed_agx3_to(b, agx_register(2, AGX_SIZE_32),
+         agx_register(32, AGX_SIZE_32), agx_register(28, AGX_SIZE_32),
+         512, AGX_FORMAT_I32, 1, 3, true, false);
+      agx_intersect_ray_m5(b, agx_register(0, AGX_SIZE_64),
+                          agx_uniform(96, AGX_SIZE_64), agx_register(10, AGX_SIZE_32),
+                          agx_register(12, AGX_SIZE_32), agx_register(14, AGX_SIZE_32));
+      agx_index bvh = agx_register(46, AGX_SIZE_64);
+      bvh.kill = true;
+      agx_intersect_ray_m5(b, agx_register(0, AGX_SIZE_64), bvh,
+                          agx_register(10, AGX_SIZE_32), agx_register(12, AGX_SIZE_32),
+                          agx_register(14, AGX_SIZE_32));
+      agx_continue_ray_m5(b, agx_register(14, AGX_SIZE_32));
+      agx_stop(b);
+      struct util_dynarray binary = UTIL_DYNARRAY_INIT;
+      if (!agx3_pack_binary(b->shader, &binary))
+         return 141;
+      unsigned stores = 0, loads = 0, rays = 0, continuations = 0;
+      uint8_t staged[3 * AGX3_STAGE_MAX_LENGTH + 6];
+      unsigned staged_len = 0;
+      for (unsigned w = 0; w < 3; ++w) {
+         uint8_t len;
+         if (!agx3_stage_copy_liveness(AGX_ISA_ENCODING_AGX3C, 2 * w,
+                10 + 2 * w, false, false, 1, 0, staged + staged_len, &len))
+            return 143;
+         staged_len += len;
+      }
+      memcpy(staged + staged_len, "\x17\x11\x00\x06\x00\x00", 6);
+      staged_len += 6;
+      const uint8_t native_rays[2][14] = {
+         {0x27,0x04,0x29,0x01,0x22,0xc1,0xae,0x02,0x01,0x98,0x00,0x00,0x00,0xa0},
+         {0x27,0x04,0x29,0x01,0xa2,0xc3,0xae,0x02,0x80,0x8a,0x00,0x00,0x00,0xa0},
+      };
+      const uint8_t native_continue[] = {0x27,0x04,0x09,0x00,0x32,0xc0,0xaf,0x02};
+      for (unsigned off = 0; off < binary.size; off += 2) {
+         struct agx3_scs_desc d;
+         const uint8_t *bytes = (const uint8_t *)binary.data + off;
+         for (unsigned r = 0; r < 2; ++r) {
+            if (off + sizeof(native_rays[r]) <= binary.size &&
+                !memcmp(bytes, native_rays[r], sizeof(native_rays[r]))) {
+               if (off < staged_len ||
+                   memcmp(bytes - staged_len, staged, staged_len))
+                  return 144;
+               rays++;
+            }
+         }
+         if (off + sizeof(native_continue) <= binary.size &&
+             !memcmp(bytes, native_continue, sizeof(native_continue))) {
+            uint8_t prefix[AGX3_STAGE_MAX_LENGTH + 6], size;
+            if (!agx3_stage_copy_liveness(AGX_ISA_ENCODING_AGX3C, 8, 14,
+                   false, false, 1, 0, prefix, &size))
+               return 145;
+            memcpy(prefix + size, "\x17\x09\x00\x06\x00\x00", 6);
+            if (off < size + 6 || memcmp(bytes - size - 6, prefix, size + 6))
+               return 146;
+            continuations++;
+         }
+         if (off + 14 <= binary.size &&
+             agx3_scs_decode_store(AGX_ISA_ENCODING_AGX3C, bytes, 14, &d) &&
+             d.data == 4 && d.address_kind == AGX3_SCS_ADDRESS_REGISTER_INDEXED &&
+             d.base_register == 32 && d.index == 28 && d.component_mask == 3 &&
+             d.address_selector == 3 && d.byte_offset == 0)
+            stores++;
+         if (off + 16 <= binary.size &&
+             agx3_scs_decode_load(AGX_ISA_ENCODING_AGX3C, bytes, 16, &d) &&
+             d.data == 2 && d.address_kind == AGX3_SCS_ADDRESS_REGISTER_INDEXED &&
+             d.base_register == 32 && d.index == 28 && d.component_mask == 1 &&
+             d.address_selector == 3 && d.byte_offset == 512 &&
+             d.c_load_marker && d.slot != 0)
+            loads++;
+      }
+      util_dynarray_fini(&binary);
+      ralloc_free(memctx);
+      if (stores != 1 || loads != 1 || rays != 2 || continuations != 1)
+         return 142;
+      puts("M5_NATIVE_RAY_SCS_IR_PASS");
+   }
+   /* M5 STTPS has an implicit slot-1 completion. The following static
+    * reload must consume it, as in Metal552 and the array CTS reproducer. */
+   {
+      void *memctx = ralloc_context(NULL);
+      agx_builder *b = agx_test_builder(memctx);
+      struct agx_shader_key key = {0};
+      struct agx_shader_info info = {0};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      b->shader->key = &key;
+      b->shader->out = &info;
+      b->shader->nir = rzalloc(memctx, nir_shader);
+      b->shader->nir->info.stage = MESA_SHADER_COMPUTE;
+      b->shader->stage = info.stage = MESA_SHADER_COMPUTE;
+      b->shader->max_reg = 9;
+      b->shader->scratch_size_B = 64;
+      agx_stack_store(b, agx_register(0, AGX_SIZE_32),
+                      agx_register(4, AGX_SIZE_32), AGX_FORMAT_I32, 1);
+      agx_stack_load_to(b, agx_register(8, AGX_SIZE_32),
+                        agx_immediate(32), AGX_FORMAT_I32, 1);
+      agx_stop(b);
+      struct util_dynarray binary = UTIL_DYNARRAY_INIT;
+      if (!agx3_pack_binary(b->shader, &binary))
+         return 130;
+      unsigned found = 0;
+      for (unsigned off = 0; off < binary.size; off += 2) {
+         for (unsigned len = 4; len <= 8; len += 4) {
+            if (off + len > binary.size)
+               continue;
+            struct agx3_static_spill_load_desc load;
+            if (agx3_static_spill_decode_load(
+                   AGX_ISA_ENCODING_AGX3C,
+                   (const uint8_t *)binary.data + off, len, &load) &&
+                load.tps_half == 16 && load.dest == 8) {
+               if (!(load.wait_mask & 1))
+                  return 131;
+               found++;
+            }
+         }
+      }
+      if (found != 1)
+         return 132;
+      util_dynarray_fini(&binary);
+      ralloc_free(memctx);
+   }
+   /* Metal532: int/short variable ASR, then all four mixed-width signs. */
+   {
+      const uint8_t shifts[][12] = {
+         {0x27,0x04,0x00,0x1a,0x26,0x00,0xa3,0x02,0x08,0x01,0x60,0x00},
+         {0x37,0x04,0x00,0x38,0x26,0x00,0xa3,0x02,0x80,0x80,0x60,0x00},
+      };
+      const enum agx3_alu_hint hints[] = {AGX3_ALU_HINT_DISCARD, AGX3_ALU_HINT_DISCARD};
+      for (unsigned h=0; h<2; h++) {
+         uint8_t bytes[AGX3_ALU_MAX_LENGTH], len;
+         if (!agx3_alu_asr_reg_m5(h ? 2 : 0, h ? 4 : 0, h ? 2 : 4,
+                h, h, hints, false, 1, bytes, &len) ||
+             len != 12 || memcmp(bytes, shifts[h], 12))
+            return 121;
+      }
+      for (unsigned signs=0; signs<4; signs++) {
+         struct agx3_imad_desc d = {
+            .dest_64=true, .src_a=0, .src_a_kill=true,
+            .src_a_sign_extend=!!(signs&1),
+            .multiplier_is_register=true, .multiplier_reg=2,
+            .multiplier_16=true, .multiplier_kill=true,
+            .multiplier_sign_extend=!!(signs&2),
+            .addend_is_immediate=true, .wait_mask=128,
+         };
+         uint8_t expected[] = {0x2f,0,4,2,0x21,0,0xa9,2,0xa8,0,0x20,0x30};
+         expected[5] |= signs&2 ? 8 : 0;
+         expected[11] |= signs&1 ? 0x80 : 0;
+         uint8_t bytes[AGX3_IADD_MAX_LENGTH], len;
+         if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C,&d,bytes,&len) ||
+             len != sizeof(expected) || memcmp(bytes,expected,len))
+            return 122;
+      }
+      puts("M5_DYNAMIC_ASR_AND_MIXED_WIDE_IMAD_METAL_BYTES_PASS");
+   }
+
+   /* Byte-exact Metal ushort2 -> uint3 AND/OR/XOR, including odd half 5. */
+   {
+      const enum agx3_alu_op ops[] = {AGX3_ALU_BAND, AGX3_ALU_BOR, AGX3_ALU_BXOR};
+      const uint8_t expected[3][10] = {
+         {0x2b,0x00,0x07,0x00,0x30,0x70,0xa2,0x02,0x42,0x01},
+         {0x33,0x00,0x07,0x00,0x32,0x78,0xa2,0x02,0x42,0x01},
+         {0x23,0x00,0x06,0x00,0x32,0x78,0xa4,0x0a,0x44,0x01},
+      };
+      const uint16_t src[] = {4, 5};
+      for (unsigned i = 0; i < 3; ++i) {
+         const enum agx3_alu_hint hints[] = {
+            i == 2 ? AGX3_ALU_HINT_DISCARD : AGX3_ALU_HINT_CACHE,
+            i == 2 ? AGX3_ALU_HINT_DISCARD : AGX3_ALU_HINT_CACHE,
+         };
+         uint8_t bytes[AGX3_ALU_MAX_LENGTH], len = 0;
+         if (!agx3_alu_bit_half_to_word_m5(ops[i], i * 2, src, hints,
+                 false, i == 0 ? 128 : 0, bytes, &len) ||
+             len != 10 || memcmp(bytes, expected[i], len))
+            return 120;
+      }
+      puts("M5_HALF_TO_WORD_BITWISE_METAL_BYTES_PASS");
+   }
+
+   /* Fresh G17G saturated scale, ABS, signed-immediate and narrow witnesses. */
+   const uint8_t saturated_scale[][14] = {
+      {0x29,0,5,0x30,0x22,0x80,0xac,2,0x0a,0,0x20,2,0,0x10},
+      {0x29,0,5,0x30,0x22,0x80,0xac,2,0x0a,0,0x20,0x0a,0,0x10},
+      {0x29,0,5,0x30,0x22,0x80,0xad,2,0x0a,0,0x20,2,0,0x10},
+      {0x39,0,5,0x38,0x22,0x80,0xac,2,0x8a,0,0x20,2,0,0x10},
+   };
+   for (unsigned i = 0; i < 4; ++i) {
+      uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
+      const bool narrow = i == 3;
+      if (!agx3_alu_fmul_minifloat_narrow(AGX_ISA_ENCODING_AGX3C,
+             narrow ? 2 : 0, narrow ? 2 : 0, i == 2 ? 132 : 4,
+             false, i == 1, narrow, true, AGX3_ALU_HINT_DISCARD, 128,
+             bytes, &length) || length != 14 ||
+          memcmp(bytes, saturated_scale[i], length)) {
+         puts("M5_SATURATED_SCALE_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   /* Apple G17G vertex-native394 normalized16 and packed fetch witnesses. */
+   for (unsigned snorm = 0; snorm < 2; ++snorm) {
+      uint8_t bytes[AGX3_FLTNORM_MAX_LENGTH], length;
+      uint8_t expected[] = {0x2f,0,0,0x1a,0x2d,0,0xa8,2,0x98,0};
+      expected[8] = snorm ? 0x58 : 0x98;
+      const struct agx3_normflt_desc d = {
+         .dest_hint = true, .two_lanes = true, .is_16bit = true,
+         .snorm = snorm, .wait_mask = 128,
+      };
+      if (!agx3_normflt_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length) ||
+          length != sizeof(expected) || memcmp(bytes, expected, length)) {
+         puts("M5_NORMALIZED16_VERTEX_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   for (unsigned layout = 0; layout <= 2; layout += 2) {
+      uint8_t bytes[16], length;
+      uint8_t expected[] = {0x2f,0,0,2,0x2e,0x80,0xac,2,0x48,0};
+      if (layout) { expected[8] = 8; expected[9] = 2; }
+      const struct agx3_rgb10a2_unpack_desc d = {
+         .dest_hint = true, .wait_mask = 128, .discard = true,
+         .layout = layout,
+      };
+      if (!agx3_rgb10a2_unpack_encode(AGX_ISA_ENCODING_AGX3C,
+                                                 &d, bytes, &length) ||
+          length != sizeof(expected) || memcmp(bytes, expected, length)) {
+         puts("M5_PACKED_VERTEX_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   /* Apple G17G arithmetic captures from metal-browser379/380. */
+   {
+      uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], size;
+      const struct agx3_falu_mods mods = {.saturate = true};
+      const struct agx3_falu_runtime runtime = {
+         .discard = {true, true, false}, .wait_mask = 128,
+      };
+      const uint8_t add[] = {0x39,0,4,0x18,0x22,0x30,0xa0,2,4,0x18,0x1a,0};
+      const uint8_t mul[] = {0x39,0,5,0x38,0x22,0x80,0xa0,2,0x9c,0,0x20,2,0,0x10};
+      if (!agx3_falu_add_narrow_m5(2,2,4,&mods,&runtime,bytes,&size) ||
+          size != sizeof(add) || memcmp(bytes,add,size) ||
+          !agx3_falu_mul_narrow_m5(2,2,4,&mods,&runtime,bytes,&size) ||
+          size != sizeof(mul) || memcmp(bytes,mul,size)) {
+         puts("M5_SATURATED_NARROW_ARITHMETIC_NATIVE_MISMATCH");
+         return 1;
+      }
+      const uint16_t src[] = {5,4,2};
+      const enum agx3_alu_hint hints[] = {
+         AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD,
+      };
+      const uint8_t hfma[] = {0x38,1,6,0x0a,0x23,0x20,0xa4,2,5,1,0x80,0x22};
+      if (!agx3_alu_hfma_saturate_m5(2,src,hints,true,false,false,128,bytes,&size) ||
+          size != sizeof(hfma) || memcmp(bytes,hfma,size)) {
+         puts("M5_SATURATED_HFMA_NATIVE_MISMATCH");
+         return 1;
+      }
+      const uint16_t cmp_src[] = {0,2};
+      const bool half[] = {false,false}, absolute[] = {true,false};
+      const struct agx3_cf_source_hints cmp_hints[] = {
+         {.discard=true}, {.discard=true},
+      };
+      const uint8_t cmp[] = {0x2a,4,0x2b,0,0x23,0x82,0xa0,0x12,0x80,0x20};
+      if (!agx3_cf_fcmp_width_m5(AGX3_CF_FCOND_EQ,false,0,cmp_src,half,
+                               absolute,cmp_hints,1,bytes,&size) ||
+          size != sizeof(cmp) || memcmp(bytes,cmp,size)) {
+         puts("M5_ABSOLUTE_FCMP_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   /* Native G17G R8/RG8 framebuffer-fetch captures (metal-browser381). */
+   for (unsigned pair = 0; pair < 2; ++pair) {
+      uint8_t expected[] = {
+         0x27,0x01,0x05,0x08,0x01,0x87,0xbc,0x02,0,0,0x40,0x18,0x20,0,
+      };
+      if (pair)
+         expected[11] = 0x98;
+      uint8_t bytes[AGX3_TILE_MAX_LENGTH], size;
+      if (!agx3_tile_load(AGX_ISA_ENCODING_AGX3C, 0, AGX3_TILE_W8, 8,
+                         pair ? 3 : 1, 0, AGX3_TILE_SAMPLE_1X, 0,
+                         true, true, bytes, &size) ||
+          size != sizeof(expected) || memcmp(bytes, expected, size)) {
+         puts("M5_SCALAR_PAIR_TILE_LOAD_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   {
+      const uint8_t expected[] = {
+         0x2f,0x00,0x04,0x1a,0x25,0x00,0xa3,0x02,0x20,0x00,0xa0,0x00,
+      };
+      const struct agx3_iadd_desc desc = {
+         .src_16 = true, .sign_extend = true, .shift = 4,
+         .src_kill = true, .wait_mask = 128,
+      };
+      uint8_t bytes[AGX3_IADD_MAX_LENGTH], size;
+      if (!agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &size) ||
+          size != sizeof(expected) || memcmp(bytes, expected, size)) {
+         puts("M5_SIGNED_HALF_SHIFT_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   {
+      uint8_t bytes[AGX3_ALU_MAX_LENGTH], size;
+      const uint8_t expected[] = {0x29,0x00,0x05,0x90,0x22,0x80,0xac,
+         0x02,0x0b,0x00,0x20,0x08,0x00,0x10};
+      if (!agx3_alu_fmul_minifloat_narrow(AGX_ISA_ENCODING_AGX3C, 0, 0,
+             0x50, false, true, false, false, AGX3_ALU_HINT_DISCARD, 128,
+             bytes, &size) || size != sizeof(expected) ||
+          memcmp(bytes, expected, size)) {
+         puts("M5_ABS_MUL_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   {
+      uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], size;
+      const struct agx3_falu_mods mods = {0};
+      const struct agx3_falu_runtime runtime = {
+         .discard = {true, true, true}, .wait_mask = 128,
+      };
+      /* Native device-half result from three device-float operands. */
+      const uint8_t expected[] = {0x39,0x00,0x07,0x08,0x20,0x30,0xa3,
+         0x02,0xb8,0x00,0xa0,0x00,0x04,0x08,0x00,0x80};
+      if (!agx3_falu_fma_narrow_m5(2, 2, 0, 4, &mods, &runtime, bytes, &size) ||
+          size != sizeof(expected) || memcmp(bytes, expected, size)) {
+         puts("M5_NARROW_FMA_NATIVE_MISMATCH");
+         return 1;
+      }
+   }
+   /* Fresh G17G RGBA32Float fragments: retain Apple's native width pairs,
+    * fused arithmetic and modifiers, checked byte-for-byte. */
+   {
+      uint8_t b[AGX3_ALU_MAX_LENGTH], n;
+      const uint16_t same[] = {0, 0}, pair[] = {0, 2}, add_pair[] = {0, 4};
+      const enum agx3_alu_hint dead[] = {AGX3_ALU_HINT_DISCARD,
+                                         AGX3_ALU_HINT_DISCARD};
+      const bool abs[] = {true, false}, neg[] = {false, true};
+      const uint8_t rcp[] = {0x27, 0x00, 0x04, 0x12, 0x2a,
+                             0x20, 0xa1, 0x02, 0x28, 0x00};
+      const uint8_t rsqrt[] = {0x27, 0x00, 0x04, 0x12, 0x2a,
+                               0x20, 0xa3, 0x02, 0x28, 0x00};
+      const uint8_t sqrt[] = {0x27, 0x82, 0x04, 0x18, 0x2a,
+                              0x20, 0xa2, 0x02, 0x08, 0x00};
+      const uint8_t fma[] = {0x00, 0x00, 0x3e, 0x00, 0x21, 0xb0, 0x00, 0x00};
+      const uint8_t add[] = {0x20, 0x00, 0x24, 0x02, 0x2a,
+                             0x20, 0xa0, 0x02, 0x06, 0x28};
+      const uint8_t mixed_add[] = {0x21, 0x00, 0x24, 0xca, 0x2a, 0x20,
+                                   0xac, 0x02, 0x01, 0x08, 0x40, 0x40};
+      const uint8_t mul[] = {0x01, 0x01, 0x39, 0x03};
+#define CHECK_NATIVE(call, expected)                                           \
+   do {                                                                        \
+      if (!(call) || n != sizeof(expected) || memcmp(b, expected, n)) {        \
+         printf("M5_HALF_NATIVE_MISMATCH %s len=%u\n", #expected, n);          \
+         for (unsigned i = 0; i < n; i++)                                      \
+            printf("%02x", b[i]);                                              \
+         puts("");                                                             \
+         return 1;                                                             \
+      }                                                                        \
+   } while (0)
+      CHECK_NATIVE(agx3_alu_rcp(AGX_ISA_ENCODING_AGX3C, 0, 0, true, true, false,
+                                false, AGX3_ALU_HINT_DISCARD, true, 0, b, &n),
+                   rcp);
+      CHECK_NATIVE(agx3_alu_hrsqrt(AGX_ISA_ENCODING_AGX3C, 0, 0,
+                                   AGX3_ALU_HINT_DISCARD, true, 0, b, &n),
+                   rsqrt);
+      CHECK_NATIVE(agx3_alu_sqrt_step_half(AGX_ISA_ENCODING_AGX3C, 0, 2,
+                                           AGX3_ALU_HINT_CACHE, true, 0, b, &n),
+                   sqrt);
+      CHECK_NATIVE(
+         agx3_alu_hfma_addend_minifloat(AGX_ISA_ENCODING_AGX3C, 0, same, 48,
+                                        dead, true, 0, b, &n),
+         fma);
+      CHECK_NATIVE(agx3_alu_hadd_mod_m5(0, add_pair, dead, abs, neg, true,
+                                        false, 0, b, &n),
+                   add);
+      CHECK_NATIVE(
+         agx3_alu_half_add_imm_to_f32_m5(0, 0, 48, true, true,
+                                         AGX3_ALU_HINT_DISCARD, true, 0, b, &n),
+         mixed_add);
+      CHECK_NATIVE(agx3_alu_fmul_narrow(AGX_ISA_ENCODING_AGX3C, 0, pair, dead,
+                                        true, 0, b, &n),
+                   mul);
+      /* The buffer inputs force real half operands. Apple evaluates these
+       * in the constant program; the operand widths are not inferred from
+       * the half render-target format. */
+      const uint8_t half_abs[] = {0x20, 0x04, 0x04, 0x02, 0x2a,
+                                  0x20, 0xa0, 0x02, 0x84, 0x26};
+      const uint8_t half_neg[] = {0x00, 0x00, 0x3c, 0x80,
+                                  0x01, 0x22, 0x02, 0x00};
+      const uint8_t half_negabs[] = {0x20, 0x04, 0x04, 0x02, 0x2a,
+                                     0x20, 0xa0, 0x02, 0x84, 0xa6};
+      const uint8_t mixed_narrow[] = {0x01, 0x01, 0x3d, 0x04, 0x00, 0x20};
+      const uint8_t float_uint[] = {0x27, 0x04, 0x00, 0x1a, 0x2a,
+                                    0x00, 0xae, 0x02, 0xb0, 0x03};
+      CHECK_NATIVE(agx3_alu_half_add_negzero_m5(
+                      0, 0, true, false, false, AGX3_ALU_HINT_DISCARD, true, 1, b, &n),
+                   half_abs);
+      CHECK_NATIVE(agx3_alu_half_add_negzero_m5(
+                      0, 0, false, true, false, AGX3_ALU_HINT_DISCARD, true, 1, b, &n),
+                   half_neg);
+      CHECK_NATIVE(agx3_alu_half_add_negzero_m5(
+                      0, 0, true, true, false, AGX3_ALU_HINT_DISCARD, true, 1, b, &n),
+                   half_negabs);
+      CHECK_NATIVE(agx3_alu_fmul_mixed_half(
+                      AGX_ISA_ENCODING_AGX3C, 0, 0, 4, AGX3_ALU_HINT_DISCARD,
+                      AGX3_ALU_HINT_DISCARD, true, 1, b, &n),
+                   mixed_narrow);
+      CHECK_NATIVE(
+         agx3_alu_float_to_int32(AGX_ISA_ENCODING_AGX3C, 0, 0, false,
+                                 AGX3_ALU_HINT_DISCARD, true, 1, b, &n),
+         float_uint);
+      const uint16_t half_pair[] = {1, 0};
+      const bool both_abs[] = {true, true};
+      const uint8_t abs_product[] = {
+         0x20,0x05,0x05,0x02,0x2a,0x20,0xa4,0x02,0x24,0x80};
+      const uint8_t signed_half[] = {
+         0x27,0x00,0x04,0x1a,0x29,0x00,0xa3,0x02,0x20,0x00,0x4b,0x00};
+      CHECK_NATIVE(agx3_alu_hmul_abs(AGX_ISA_ENCODING_AGX3C,0,half_pair,
+                   dead,both_abs,true,1,b,&n), abs_product);
+      const struct agx3_isub_imm_desc sign_extend = {
+         .dest = 0, .src = 0, .src_16 = true, .sign_extend = true,
+         .dest_cache = true, .src_kill = true,
+      };
+      CHECK_NATIVE(agx3_iadd_immediate_encode(AGX_ISA_ENCODING_AGX3C,
+                   &sign_extend,b,&n), signed_half);
+#undef CHECK_NATIVE
+      puts("M5_HALF_MATH_NATIVE_PASS");
+   }
+
+   {
+      const struct agx3_falu_mods m = {.src0_16 = true};
+      const struct agx3_falu_runtime runtime = {
+         .dest_cache = true, .discard = {true, true},
+      };
+      const uint8_t expected_mul[] = {0x09,0x0f,0x3d,0x00,0x10,0x00};
+      uint8_t bytes[AGX3_FALU_MOD_MAX_LENGTH], length;
+      if (!agx3_falu_mul_mod_runtime(AGX_ISA_ENCODING_AGX3C, 0, 1, 14,
+                                    &m, &runtime, bytes, &length) ||
+          length != sizeof(expected_mul) ||
+          memcmp(bytes, expected_mul, length)) return 1;
+      puts("M5_MIXED_MUL_NATIVE_PASS");
+   }
+   {
+      const uint8_t native[] = {0x37,0x02,0x04,0x38,0x2a,0x20,0xa2,0x12,0x10,0x00};
+      uint8_t bytes[AGX3_ALU_MAX_LENGTH], length;
+      if (!agx3_alu_sqrt_step_source_modifiers(AGX_ISA_ENCODING_AGX3C,
+              10, 6, AGX3_ALU_HINT_NONE, true, 0, false, false, bytes, &length) ||
+          length != sizeof(native) || memcmp(bytes, native, length)) return 1;
+      puts("M5_SQRT_STEP_NATIVE_PASS");
+   }
+   /* Native level-zero samples isolate each mask bit, then compose XZ,
+    * XW, XZW and XYZW without changing the operand ABI. */
+   {
+      static const unsigned masks[] = {1, 2, 4, 8, 5, 9, 13, 15};
+      static const unsigned char mask_bytes[][4] = {
+         {0x00,0x21,0x01,0x00}, {0x20,0x21,0x01,0x00},
+         {0x40,0x21,0x01,0x00}, {0x60,0x21,0x01,0x00},
+         {0x00,0x21,0x03,0x00}, {0x00,0x29,0x01,0x00},
+         {0x00,0x29,0x03,0x00}, {0x60,0x29,0x01,0x00},
+      };
+      for (unsigned i = 0; i < 8; ++i) {
+         const struct agx3_tex_sample_desc d = {
+            .width = AGX3_TEX_W32, .mask = masks[i],
+            .lod = AGX3_TEX_LOD_LEVEL, .slot = 1, .wait_mask = 1,
+            .coord_kill = true, .final = true,
+         };
+         unsigned char bytes[AGX3_TEX_MAX_LENGTH]; uint8_t length;
+         unsigned char native[] = {
+            0x0f,0x04,0x16,0x00,0x41,0x80,0x00,0x21,0x01,0x00,0x01,
+            0x00,0x00,0x00,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x00,
+         };
+         memcpy(native + 6, mask_bytes[i], 4);
+         if (!agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, &d, bytes,
+                                    &length) || length != sizeof(native) ||
+             memcmp(bytes, native, length)) return 1;
+         /* metal625 sparse_sample/gather/read all carry f142 in byte15[7]
+          * of the 22-byte form. The driver's operand profile stays intact. */
+         struct agx3_tex_sample_desc sparse = d;
+         sparse.sparse = true;
+         native[15] |= 0x80;
+         if (!agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, &sparse, bytes,
+                                     &length) || length != sizeof(native) ||
+             memcmp(bytes, native, length)) return 1;
+      }
+      puts("M5_SAMPLE_COMPONENT_MASK_NATIVE_PASS");
+   }
+   {
+      const unsigned char native[] = {0x27, 0, 0, 0x1a, 0x2e, 0, 0xa9, 2, 8, 0};
+      unsigned char bytes[AGX3_ALU_MAX_LENGTH];
+      uint8_t length;
+      if (!agx3_alu_bitrev_m5(0, 0, false, AGX3_ALU_HINT_DISCARD, true, 0,
+                              bytes, &length) ||
+          length != sizeof(native) || memcmp(bytes, native, length))
+         return 1;
+      puts("M5_BIT_REVERSE_NATIVE_PASS");
+   }
+   /* metal-fence189: a memory-only device fence differs from a rendezvous. */
+   const unsigned char native_fence[] = {0x0f,0x2b,0x00,0x06,0x00,0x00};
+   struct agx3_encoded_scoped_barrier fence;
+   if (!agx3_pack_scoped_barrier(AGX_ISA_ENCODING_AGX3C,
+                                AGX3_BARRIER_SCOPE_DEVICE_FENCE, &fence) ||
+       fence.length != sizeof(native_fence) ||
+       memcmp(fence.bytes, native_fence, sizeof(native_fence))) return 1;
+   puts("M5_DEVICE_FENCE_PASS");
+   memset(&fence, 0xa5, sizeof(fence));
+   if (!agx3_pack_scoped_barrier(AGX_ISA_ENCODING_AGX3C,
+                                AGX3_BARRIER_SCOPE_SUBGROUP_SHARED, &fence) ||
+       fence.length != 0 ||
+       !agx3_pack_scoped_barrier(AGX_ISA_ENCODING_AGX3B,
+                                AGX3_BARRIER_SCOPE_SUBGROUP_SHARED, &fence) ||
+       fence.length != 0)
+      return 1;
+   puts("M5_SUBGROUP_SHARED_NO_RENDEZVOUS_PASS");
+
+
+   static const unsigned char expected[] = {
+      0x1c,0x9c,0x10,0x06,0x24,0xa8,0x14,0x06,
+      0x0c,0x9d,0x10,0x26,0x24,0x81,0x01,0x40,
+      0x28,0x01,0xa0,0x0a,0x27,0x06,0x04,0x0a,
+      0x21,0x00,0xa1,0x02,0x68,0x01,0x20,0x00,
+      0x01,0x02,0x37,0x08,0x04,0x0a,0x21,0x00,
+      0xa1,0x02,0x28,0x01,0x20,0x00,0x01,0x02,
+      0x27,0x00,0x20,0x2a,0x21,0x00,0xab,0x0a,
+      0x00,0x00,0x4a,0x30,0x18,0x00,0x25,0x04,
+      0x15,0x00,0x24,0x00,0xa0,0x22,0x00,0xe0,
+      0x00,0x55,0x0e,0x00,0x00,0x00,
+   };
+   const struct {unsigned d,s,w; bool discard; unsigned char bytes[4];} copies[] = {
+      {16,1,1,false,{0x83,0x01,0x21,0x20}},
+      {0,1,0,true,{0x03,0x01,0x29,0x00}},
+      {1,2,0,true,{0x03,0x02,0x29,0x01}},
+   };
+   for (unsigned i=0;i<3;i++) {
+      unsigned char out[AGX3_COPY_MAX_LENGTH]; uint8_t len=0;
+      if (!agx3_copy_encode_reg16(AGX_ISA_ENCODING_AGX3C,copies[i].d,
+             copies[i].s,true,false,copies[i].discard,copies[i].w,out,&len) ||
+          len!=4 || memcmp(out,copies[i].bytes,4)) return 2;
+   }
+   unsigned char out[MAX2(AGX3_STAGE_MAX_LENGTH, AGX3_ALU_MAX_LENGTH)];
+   uint8_t len=0;
+   const unsigned char reciprocal[] = {
+      0x37,0x06,0x04,0x18,0x2a,0x20,0xa1,0x02,0x30,0x00,
+   };
+   if (!agx3_alu_rcp(AGX_ISA_ENCODING_AGX3C,2,2,false,false,false,false,
+                     AGX3_ALU_HINT_DISCARD,true,1,out,&len) ||
+       len!=sizeof(reciprocal) || memcmp(out,reciprocal,len)) return 1;
+   const unsigned char uniform_half[] = {0x13,0x08,0x01,0x08};
+   if (!agx3_copy_encode_uniform_to_reg16(AGX_ISA_ENCODING_AGX3C,2,8,
+                                          false,out,&len) ||
+       len!=sizeof(uniform_half) || memcmp(out,uniform_half,len)) return 1;
+   const unsigned char fragment_entry[] = {0x07,0x30,0x00,0x05,0x00,0x00};
+   if (!agx3_pixel_colour_entry(AGX_ISA_ENCODING_AGX3C,out,&len) ||
+       len!=sizeof(fragment_entry) || memcmp(out,fragment_entry,len)) return 1;
+   /* Native zs_depth_4x / zs_stencil_4x entry gate (f34), AGX3c only. */
+   const unsigned char zs_unreleased_entry[] = {0x07,0x70,0x00,0x05,0x00,0x00};
+   if (!agx3_pixel_colour_entry_zs_unreleased(AGX_ISA_ENCODING_AGX3C,out,&len) ||
+       len!=sizeof(zs_unreleased_entry) ||
+       memcmp(out,zs_unreleased_entry,len)) return 1;
+   if (agx3_pixel_colour_entry_zs_unreleased(AGX_ISA_ENCODING_AGX3B,out,&len))
+      return 1;
+   const unsigned char zs_release[] = {0x0f,0x00,0x00,0x04,0x00,0x00};
+   if (!agx3_pixel_release(AGX_ISA_ENCODING_AGX3C,1,false,0,out,&len) ||
+       len!=sizeof(zs_release) || memcmp(out,zs_release,len)) return 1;
+   const unsigned char clear[]={0x13,0,0,0};
+   if (!agx3_clr(AGX_ISA_ENCODING_AGX3C,2,true,false,out,&len) ||
+       len!=4 || memcmp(out,clear,4)) return 3;
+   puts("M5_COPY_CLEAR_PASS");
+   const struct agx3_imad_desc mad = {
+      .dest=2, .src_a=8, .multiplier=32, .src_c=2,
+      .src_a_kill=true, .src_c_kill=true, .wait_mask=9,
+   };
+   const unsigned char mad_bytes[] = {
+      0x37,0x24,0x04,0x5a,0x21,0x00,0xa1,0x02,0x28,0x08,0xca,0x00,0x02,0x00,
+   };
+   unsigned char mad_out[AGX3_IADD_MAX_LENGTH]; len=0;
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C,&mad,mad_out,&len) ||
+       len!=sizeof(mad_bytes) || memcmp(mad_out,mad_bytes,sizeof(mad_bytes))) return 4;
+   puts("M5_IMAD_PASS");
+   const struct agx3_iadd_desc add = {
+      .dest=24, .src_a_is_register=true, .src_a_reg=2, .src_a_cache=true,
+      .src=16, .src_kill=true, .wait_mask=16,
+   };
+   const unsigned char add_bytes[] = {
+      0xa7,0xc2,0x04,0x1a,0x21,0x00,0xa3,0x12,0x08,0x84,0x21,0x00,
+   };
+   len=0;
+   if (!agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C,&add,mad_out,&len) ||
+       len!=sizeof(add_bytes) || memcmp(mad_out,add_bytes,sizeof(add_bytes))) return 5;
+   struct agx3_mem_desc load = {
+      .data=16, .width=AGX3_MEM_W32, .mask=7, .base_is_uniform=true,
+      .base=8, .idx_kind=AGX3_MEM_IDX_IMM, .byte_off=776, .slot=5,
+   };
+   const unsigned char load_bytes[] = {
+      0x8f,0x08,0x03,0x00,0x40,0x0c,0x10,0xa1,0x01,0,0,0,0,3,
+   };
+   unsigned char memory[AGX3_MEM_MAX_LENGTH];len=0;
+   if (!agx3_mem_encode_load(AGX_ISA_ENCODING_AGX3C,&load,memory,&len) ||
+       len!=sizeof(load_bytes) || memcmp(memory,load_bytes,sizeof(load_bytes))) return 6;
+   const struct agx3_isel_desc sel = {
+      .dest=1,.dest_16=true,.dest_cache=true,.cmp0=2,.cmp1=12,
+      .cond=AGX3_ISEL_ICOND_ULT,.true_kind=AGX3_ISEL_IMMEDIATE,
+      .false_kind=AGX3_ISEL_IMMEDIATE,.sel_true=1,.sel_false=0,
+      .discard={true,true,false,false},.wait_mask=64,
+   };
+   const unsigned char sel_bytes[] = {
+      0x22,0x02,0x45,0x0b,0x2f,0x00,0xac,0x02,0x00,0x83,0x64,0x98,0x00,0x28,
+   };
+   unsigned char select[AGX3_ISEL_MAX_LENGTH];len=0;
+   if (!agx3_isel_encode(AGX_ISA_ENCODING_AGX3C,&sel,select,&len) ||
+       len!=sizeof(sel_bytes) || memcmp(select,sel_bytes,sizeof(sel_bytes))) return 7;
+   puts("M5_MEMORY_SELECT_LIVENESS_PASS");
+   const unsigned char triple_bytes[] = {
+      0x27,0x04,0x04,0x1a,0x29,0x00,0xa3,0x02,0x20,0x00,0x24,0x00,
+   };
+   const struct agx3_iadd_desc triple = {
+      .dest=0, .dest_cache=true, .src_a_is_register=true,
+      .src_a_reg=0, .src_a_16=true, .src_a_kill=true,
+      .src=0, .src_16=true, .src_kill=true, .shift=1, .wait_mask=1,
+   };
+   len=0;
+   if (!agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C,&triple,mad_out,&len) ||
+       len!=sizeof(triple_bytes) || memcmp(mad_out,triple_bytes,len)) return 8;
+   const unsigned char add_scaled_bytes[] = {
+      0x27,0x08,0x04,0x3a,0x21,0x00,0xa3,0x02,0x20,0x80,0x24,0x00,
+   };
+   const struct agx3_iadd_desc add_scaled = {
+      .dest=0, .src_a_is_register=true, .src_a_reg=4,
+      .src_a_16=true, .src_a_kill=true, .src=0, .src_kill=true,
+      .shift=1, .wait_mask=2,
+   };
+   len=0;
+   if (!agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C,&add_scaled,mad_out,&len) ||
+       len!=sizeof(add_scaled_bytes) || memcmp(mad_out,add_scaled_bytes,len)) return 9;
+   puts("M5_NARROW_SCALED_ADD_PASS");
+   const uint16_t fma_src[] = {2, 3, 4};
+   const enum agx3_alu_hint fma_hint[] = {
+      AGX3_ALU_HINT_DISCARD, AGX3_ALU_HINT_DISCARD, AGX3_ALU_HINT_DISCARD,
+   };
+   const unsigned char fma_expected[] = {
+      0x38,0x02,0x06,0x02,0x23,0x20,0xa4,0x02,0xc6,0x00,0x80,0x00,
+   };
+   unsigned char native[AGX3_TEX_MAX_LENGTH];
+   if (!agx3_alu_hfma(AGX_ISA_ENCODING_AGX3C, 2, fma_src, fma_hint,
+                       false, false, false, 128, native, &len) ||
+       len != sizeof(fma_expected) || memcmp(native, fma_expected, len)) return 10;
+   const struct agx3_iadd_desc sat_sub = {
+      .dest=0, .src_a_is_register=true, .src_a_reg=0, .src_a_kill=true,
+      .src=4, .src_kill=true, .subtract=true, .saturate=true, .wait_mask=128,
+   };
+   const unsigned char sat_expected[] = {
+      0x2f,0x00,0x04,0x1a,0x25,0x00,0xa2,0x02,0x18,0x81,0x20,0x00,
+   };
+   if (!agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C, &sat_sub, native, &len) ||
+       len != sizeof(sat_expected) || memcmp(native, sat_expected, len)) return 11;
+   const struct agx3_local_atomic_desc atom = {
+      .op=AGX3_ATOMIC_ADD, .value=0, .value_kill=true,
+      .address_kind=AGX3_ATOMIC_INDEX_IMMEDIATE,
+   };
+   const unsigned char atom_expected[] = {
+      0x0f,0x00,0x03,0x00,0x02,0x05,0x04,0xa0,0x90,0x00,0x40,0x02,
+   };
+   if (!agx3_atomic_encode_local(AGX_ISA_ENCODING_AGX3C, &atom, native, &len) ||
+       len != sizeof(atom_expected) || memcmp(native, atom_expected, len)) return 12;
+   puts("M5_FMA_SAT_ATOMIC_NATIVE_BYTES_PASS");
+   const unsigned char sub16_expected[] = {
+      0x3f,0x02,0x04,0x18,0x25,0x00,0xa2,0x02,0x40,0x00,0x0b,0x00,
+   };
+   if (!agx3_alu_isub_imm16_m5(2, 2, 1, AGX3_ALU_HINT_DISCARD, false,
+                              128, native, &len) ||
+       len != sizeof(sub16_expected) || memcmp(native, sub16_expected, len)) return 13;
+   const unsigned char and_expected[] = {
+      0x2b,0x00,0x07,0x80,0x30,0x30,0xa4,0x02,0x04,0x01,
+   };
+   if (!agx3_alu_and_mixed_m5(0, 0, 4, AGX3_ALU_HINT_DISCARD,
+                             AGX3_ALU_HINT_DISCARD, false, 128, native, &len) ||
+       len != sizeof(and_expected) || memcmp(native, and_expected, len)) return 14;
+   const unsigned char scale_expected[] = {
+      0x29,0x00,0x05,0x10,0x22,0x80,0xac,0x02,0x0b,0x00,0x20,0x00,0x00,0x10,
+   };
+   if (!agx3_alu_fmul_minifloat_narrow(AGX_ISA_ENCODING_AGX3C, 0, 0, 64,
+          false, false, false, false, AGX3_ALU_HINT_DISCARD, 128, native, &len) ||
+       len != sizeof(scale_expected) || memcmp(native, scale_expected, len)) return 15;
+   puts("M5_MIXED_AND_HALF_SUB_SCALE_NATIVE_BYTES_PASS");
+   const unsigned char add64_expected[] = {
+      0x3f,0x00,0x04,0x18,0x25,0x00,0xa3,0x02,0xa1,0x00,0x21,0x00,
+   };
+   if (!agx3_alu_iadd16_gpu1(2, false, true, 64, AGX3_ALU_HINT_NONE,
+                            2, AGX3_ALU_HINT_DISCARD, 0, 128, native, &len) ||
+       len != sizeof(add64_expected) || memcmp(native, add64_expected, len)) return 16;
+   const uint16_t andc_src[] = {0,4};
+   const enum agx3_alu_hint andc_hints[] = {AGX3_ALU_HINT_DISCARD, AGX3_ALU_HINT_DISCARD};
+   const unsigned char andc_expected[] = {
+      0x2b,0x00,0x06,0x80,0x32,0x30,0xa4,0x02,0x04,0x81,
+   };
+   if (!agx3_alu_bit_andc(AGX_ISA_ENCODING_AGX3C, 0, andc_src, andc_hints,
+                         false, 128, native, &len) ||
+       len != sizeof(andc_expected) || memcmp(native, andc_expected, len)) return 17;
+   puts("M5_ADD64_ANDC_NATIVE_BYTES_PASS");
+   struct agx3_imad_desc mixed_mad = {
+      .dest=0, .src_a=0, .src_a_kill=true,
+      .multiplier_is_register=true, .multiplier_reg=8,
+      .multiplier_16=true, .multiplier_kill=true,
+      .src_c=4, .src_c_kill=true, .wait_mask=128,
+   };
+   const unsigned char mixed_mad_expected[] = {
+      0x2f,0x00,0x04,0x1a,0x21,0x00,0xa1,0x02,0x28,0x02,0xe0,0x00,0x04,0x00,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C, &mixed_mad, native, &len) ||
+       len != sizeof(mixed_mad_expected) ||
+       memcmp(native, mixed_mad_expected, len)) return 18;
+   mixed_mad.multiplier_sign_extend=true;
+   unsigned char signed_mad_expected[sizeof(mixed_mad_expected)];
+   memcpy(signed_mad_expected, mixed_mad_expected, sizeof(mixed_mad_expected));
+   signed_mad_expected[11] = 0x80;
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C, &mixed_mad, native, &len) ||
+       len != sizeof(signed_mad_expected) ||
+       memcmp(native, signed_mad_expected, len)) return 19;
+   const struct agx3_imad_desc wide_mad = {
+      .dest=0, .dest_64=true, .src_a=0, .src_a_kill=true,
+      .multiplier=96, .addend_is_immediate=true, .wait_mask=128,
+   };
+   const unsigned char wide_mad_expected[] = {
+      0x2f,0x00,0x04,0x02,0x21,0x80,0xa9,0x02,0x28,0x18,0x02,0x30,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C, &wide_mad, native, &len) ||
+       len != sizeof(wide_mad_expected) ||
+       memcmp(native, wide_mad_expected, len)) return 20;
+   mixed_mad.src_a=4;
+   mixed_mad.src_a_16=true;
+   mixed_mad.multiplier_reg=5;
+   mixed_mad.multiplier_sign_extend=false;
+   mixed_mad.src_c=0;
+   const unsigned char half_pair_expected[] = {
+      0x2f,0x00,0x04,0x3a,0x21,0x00,0xa1,0x02,0x60,0x01,0xe0,0x00,0x00,0x00,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C, &mixed_mad, native, &len) ||
+       len != sizeof(half_pair_expected) ||
+       memcmp(native, half_pair_expected, len)) return 21;
+   puts("M5_MIXED_IMAD_WIDE_IMMEDIATE_NATIVE_BYTES_PASS");
+   const unsigned char asr_expected[] = {
+      0x3f,0x02,0x00,0x18,0x26,0x00,0xa3,0x02,0x30,0x82,0x40,0x00,
+   };
+   if (!agx3_alu_asr_imm_m5(2,2,true,true,8,AGX3_ALU_HINT_DISCARD,
+          false,128,native,&len) || len != sizeof(asr_expected) ||
+       memcmp(native,asr_expected,len)) { fprintf(stderr,"ASR len=%u ",len); for(unsigned i=0;i<len;i++) fprintf(stderr,"%02x",native[i]); fputc(10,stderr); return 30; }
+   const struct agx3_imad_desc half_addend = {
+      .dest=0,.src_a=1,.src_a_16=true,.src_a_kill=true,
+      .multiplier=129,.src_c=0,.src_c_16=true,.src_c_kill=true,.wait_mask=128,
+   };
+   const unsigned char half_addend_expected[] = {
+      0x2f,0x01,0x04,0x1a,0x21,0x00,0xa1,0x02,0x60,0x20,0x8a,0x00,0x00,0x00,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C,&half_addend,native,&len) ||
+       len != sizeof(half_addend_expected) ||
+       memcmp(native,half_addend_expected,len)) return 31;
+   const struct agx3_falu_mods abs_add = {.src0_abs=true,.src1_abs=true};
+   const struct agx3_falu_runtime abs_runtime = {
+      .discard={true,true,false},.wait_mask=128,
+   };
+   const unsigned char abs_expected[] = {
+      0x29,0x00,0x04,0x1a,0x22,0x20,0xa0,0x02,0x04,0x88,0x12,0x40,
+   };
+   if (!agx3_falu_add_mod_runtime(AGX_ISA_ENCODING_AGX3C,0,0,4,&abs_add,
+          &abs_runtime,native,&len) || len != sizeof(abs_expected) ||
+       memcmp(native,abs_expected,len)) return 32;
+   const struct agx3_isel_desc mixed_select = {
+      .dest=2,.dest_16=true,.cmp0=2,.cmp1=4,.cmp_mixed_width=true,
+      .cond=AGX3_ISEL_ICOND_UGT,.true_kind=AGX3_ISEL_IMMEDIATE,
+      .false_kind=AGX3_ISEL_IMMEDIATE,.sel_true=1,.sel_false=0,
+      .discard={true,true,false,false},.wait_mask=128,
+   };
+   const unsigned char mixed_select_expected[] = {
+      0x32,0x02,0x05,0x0a,0x27,0x00,0xac,0x02,0x00,0x01,0x64,0x1c,0x00,0x38,
+   };
+   if (!agx3_isel_encode(AGX_ISA_ENCODING_AGX3C,&mixed_select,native,&len) ||
+       len != sizeof(mixed_select_expected) ||
+       memcmp(native,mixed_select_expected,len)) return 33;
+   puts("M5_ASR_HALF_ADDEND_ABS_MIXED_SELECT_NATIVE_BYTES_PASS");
+   const uint16_t srl_src[] = {4,0};
+   const bool srl_half[] = {true,false};
+   const enum agx3_alu_hint srl_hints[] = {AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD};
+   const unsigned char srl_expected[] = {
+      0x27,0x04,0x00,0x3a,0x26,0x00,0xa1,0x02,0x00,0x80,0x20,0x34,0x00,0x00,
+   };
+   if (!agx3_alu_srl_reg(AGX_ISA_ENCODING_AGX3C,0,false,false,srl_src,
+          srl_half,srl_hints,1,native,&len) || len != sizeof(srl_expected) ||
+       memcmp(native,srl_expected,len)) return 34;
+   const uint16_t narrow_bit_src[] = {3,2};
+   const enum agx3_alu_hint narrow_bit_hints[] = {AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD};
+   const enum agx3_alu_op narrow_bit_op[] = {AGX3_ALU_BAND,AGX3_ALU_BOR,AGX3_ALU_BXOR};
+   const unsigned char narrow_bit_expected[][10] = {
+      {0x3b,0x03,0x07,0x00,0x30,0x20,0xa4,0x02,0x84,0x00},
+      {0x3b,0x03,0x07,0x00,0x32,0x28,0xa4,0x02,0x84,0x00},
+      {0x3b,0x03,0x06,0x00,0x32,0x28,0xa4,0x02,0x84,0x00},
+   };
+   for (unsigned i=0;i<3;i++) {
+      if (!agx3_alu_bit_reg16(AGX_ISA_ENCODING_AGX3C,narrow_bit_op[i],2,
+             narrow_bit_src,narrow_bit_hints,128,native,&len) || len != 10 ||
+          memcmp(native,narrow_bit_expected[i],len)) return 35+i;
+   }
+   puts("M5_NARROW_BITOPS_SRL_NATIVE_BYTES_PASS");
+   const unsigned char half_int_expected[][10] = {
+      {0x2f,0x00,0x00,0x1a,0x22,0x00,0xae,0x02,0xa8,0x03},
+      {0x2f,0x00,0x00,0x1a,0x22,0x00,0xae,0x02,0x68,0x01},
+   };
+   for (unsigned sign=0;sign<2;sign++) {
+      if (!agx3_alu_f16_to_int32(AGX_ISA_ENCODING_AGX3C,0,0,sign,
+             AGX3_ALU_HINT_DISCARD,false,128,native,&len) || len != 10 ||
+          memcmp(native,half_int_expected[sign],len)) return 38+sign;
+   }
+   const struct agx3_imad_desc wide_half_mad = {
+      .dest=0,.dest_64=true,.src_a=0,.src_a_kill=true,.src_a_sign_extend=true,
+      .multiplier_is_register=true,.multiplier_reg=2,.multiplier_16=true,
+      .multiplier_kill=true,.multiplier_sign_extend=true,
+      .addend_is_immediate=true,.wait_mask=128,
+   };
+   const unsigned char wide_half_expected[] = {
+      0x2f,0x00,0x04,0x02,0x21,0x08,0xa9,0x02,0xa8,0x00,0x20,0xb0,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C,&wide_half_mad,native,&len) ||
+       len != sizeof(wide_half_expected) ||
+       memcmp(native,wide_half_expected,len)) return 40;
+   puts("M5_HALF_CONVERSION_WIDE_MULTIPLY_NATIVE_BYTES_PASS");
+   const struct agx3_fsel32_gpu1_desc fsel_half_reg = {
+      .dest=2,.dest_cache=true,.cmp0=2,.cmp1=4,
+      .cmp0_hint=AGX3_ALU_HINT_DISCARD,.cmp1_hint=AGX3_ALU_HINT_DISCARD,
+      .sel={6,0},.sel_immediate={false,true},
+      .sel_hint={AGX3_ALU_HINT_CACHE,AGX3_ALU_HINT_NONE},
+      .condition=0,.wait_mask=128,
+   };
+   const unsigned char fsel_half_reg_expected[] = {
+      0x32,0x02,0x05,0x0a,0x2b,0x00,0xa0,0x02,0x16,0x01,0x21,0x18,0x03,0x00,0x80,0x00,
+   };
+   if (!agx3_alu_fcmp_select16_gpu1(&fsel_half_reg,native,&len) ||
+       len != sizeof(fsel_half_reg_expected) ||
+       memcmp(native,fsel_half_reg_expected,len)) return 41;
+   puts("M5_FLOAT_REGISTER_SELECT_NATIVE_BYTES_PASS");
+   const struct agx3_iadd_desc mixed_sub = {
+      .dest=0,.src_a_is_register=true,.src_a_reg=4,.src_a_16=true,
+      .src_a_kill=true,.src=0,.src_kill=true,.subtract=true,.wait_mask=128,
+   };
+   const unsigned char mixed_sub_expected[] = {
+      0x2f,0x00,0x04,0x3a,0x25,0x00,0xa2,0x02,0x00,0x80,0x21,0x00,
+   };
+   if (!agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C,&mixed_sub,native,&len) ||
+       len != sizeof(mixed_sub_expected) ||
+       memcmp(native,mixed_sub_expected,len)) return 42;
+   puts("M5_MIXED_SUBTRACT_NATIVE_BYTES_PASS");
+   const unsigned char active_mask_expected[] = {
+      0x27,0x01,0x20,0x1a,0x25,0x00,0xae,0x02,0x30,0x00,0x08,0x40,
+   };
+   if (!agx3_subgroup_encode_active_mask(0,false,0,native,&len) || len != 12 ||
+       memcmp(native,active_mask_expected,len)) return 43;
+   const struct agx3_subgroup_ballot_desc ballot = {
+      .dest=0,.src=0,.type=AGX3_SUBGROUP_BALLOT_INTEGER,
+      .compare_immediate=true,.condition=4,.invert=true,.wait_mask=128,
+   };
+   const unsigned char ballot_expected[] = {
+      0x2f,0x00,0x00,0x1a,0x25,0x00,0xae,0x02,0x38,0x00,0x08,0x04,
+   };
+   if (!agx3_subgroup_encode_ballot(AGX_ISA_ENCODING_AGX3C,&ballot,native,&len) ||
+       len != 12 || memcmp(native,ballot_expected,len)) return 44;
+   const struct agx3_subgroup_shuffle_desc quad = {
+      .quad=true,.op=AGX3_SUBGROUP_SHUFFLE_BROADCAST,
+      .lane_kind=AGX3_SUBGROUP_LANE_IMMEDIATE,
+      .dest=0,.src=0,.lane=0,.wait_mask=128,
+   };
+   const unsigned char quad_expected[] = {
+      0x2f,0x00,0x21,0x1a,0x20,0x00,0xa0,0x02,0x38,0x00,
+   };
+   if (!agx3_subgroup_encode_shuffle(AGX_ISA_ENCODING_AGX3C,&quad,native,&len) ||
+       len != 10 || memcmp(native,quad_expected,len)) return 45;
+   const unsigned char int8_float_expected[] = {
+      0x2f,0x00,0x00,0x02,0x22,0x80,0xaf,0x02,0x70,0x20,
+   };
+   if (!agx3_alu_narrow_int_to_float(AGX_ISA_ENCODING_AGX3C,0,false,0,true,
+          true,AGX3_ALU_HINT_DISCARD,false,128,native,&len) || len != 10 ||
+       memcmp(native,int8_float_expected,len)) return 46;
+   puts("M5_BALLOT_QUAD_INT8_FLOAT_NATIVE_BYTES_PASS");
+   const unsigned char mixed_xor_expected[] = {
+      0x2b,0x00,0x06,0x80,0x32,0x38,0xa4,0x02,0x04,0x01,
+   };
+   if (!agx3_alu_bit_mixed_m5(AGX3_ALU_BXOR,0,0,4,AGX3_ALU_HINT_DISCARD,
+          AGX3_ALU_HINT_DISCARD,false,128,native,&len) || len != 10 ||
+       memcmp(native,mixed_xor_expected,len)) return 47;
+   const struct agx3_subgroup_shuffle_desc half_bcast = {
+      .op=AGX3_SUBGROUP_SHUFFLE_BROADCAST,.lane_kind=AGX3_SUBGROUP_LANE_IMMEDIATE,
+      .dest=2,.src=2,.half=true,.wait_mask=128,
+   };
+   const unsigned char half_bcast_expected[] = {
+      0x3f,0x02,0x21,0x18,0x20,0x00,0xa8,0x02,0x30,0x00,
+   };
+   if (!agx3_subgroup_encode_shuffle(AGX_ISA_ENCODING_AGX3C,&half_bcast,native,&len) ||
+       len != 10 || memcmp(native,half_bcast_expected,len)) return 48;
+   const struct agx3_subgroup_shuffle_desc reg_bcast = {
+      .op=AGX3_SUBGROUP_SHUFFLE_BROADCAST,.lane_kind=AGX3_SUBGROUP_LANE_REGISTER,
+      .dest=4,.src=0,.lane=4,.dest_hint=true,.src_cache=true,
+      .lane_discard=true,.wait_mask=1,
+   };
+   const unsigned char reg_bcast_expected[] = {
+      0x27,0x84,0x01,0x1a,0x28,0x00,0xa8,0x0a,0x08,0x81,
+   };
+   if (!agx3_subgroup_encode_shuffle(AGX_ISA_ENCODING_AGX3C,&reg_bcast,native,&len) ||
+       len != 10 || memcmp(native,reg_bcast_expected,len)) return 49;
+   puts("M5_MIXED_XOR_HALF_AND_REGISTER_BROADCAST_NATIVE_BYTES_PASS");
+   const unsigned char ft1_expected[] = {
+      0x2f,0x80,0x00,0x1a,0x2a,0x00,0xab,0x0a,0x08,0x00,
+   };
+   if (!agx3_alu_ft1_u32(AGX_ISA_ENCODING_AGX3C,4,0,AGX3_ALU_HINT_CACHE,
+          true,128,native,&len) || len != 10 || memcmp(native,ft1_expected,len)) return 50;
+   const unsigned char pop_expected[] = {
+      0x2f,0x00,0x00,0x80,0x32,0x30,0xaa,0x02,0x04,0x00,
+   };
+   if (!agx3_alu_popcount(AGX_ISA_ENCODING_AGX3C,0,0,false,
+          AGX3_ALU_HINT_DISCARD,false,128,native,&len) || len != 10 ||
+       memcmp(native,pop_expected,len)) return 51;
+   /* Fresh G17G fma_loop: fma(r1, 0.75, 1.0), no dependency. The source
+    * discard is independent of the absent slot-8 wait. */
+   const struct agx3_falu_runtime fma2_no_wait = {.discard={true}};
+   const unsigned char fma2_short[] = {0x19,0x03,0x0e,0xa9,0x21,0xb0,0x02,0x02};
+   if (!agx3_falu_fma_two_imm(AGX_ISA_ENCODING_AGX3C,2,2,40,48,
+          &fma2_no_wait,native,&len) || len != sizeof(fma2_short) ||
+       memcmp(native,fma2_short,len)) return 102;
+
+   /* Native G17G Minecraft terrain fragments: fma.sat(a, b, 0.5)
+    * keeps its minifloat addend (f51 = 32, f14 = 1) beside two discarded
+    * GPR multipliers and a cached result. */
+   {
+      const struct agx3_falu_mods sat = {.saturate = true};
+      const struct agx3_falu_runtime sat_runtime = {
+         .discard = {true, true}, .dest_cache = true,
+      };
+      const struct { uint16_t dest, src0, src1; unsigned char bytes[10]; } sat_cases[] = {
+         {88, 96, 88, {0xc9,0x59,0xbe,0x61,0x22,0xa0,0x02,0x00,0x00,0x02}},
+         {90, 92, 90, {0xd9,0x5b,0xbe,0x5d,0x22,0xa0,0x02,0x00,0x00,0x02}},
+         {18, 22, 18, {0x99,0x13,0x3e,0x17,0x22,0xa0,0x02,0x00,0x00,0x02}},
+      };
+      for (unsigned i = 0; i < 3; ++i) {
+         if (!agx3_falu_fma_addend_imm_width_mod(AGX_ISA_ENCODING_AGX3C,
+                sat_cases[i].dest, sat_cases[i].src0, sat_cases[i].src1, 32,
+                &sat, &sat_runtime, native, &len) ||
+             len != 10 || memcmp(native, sat_cases[i].bytes, len)) {
+            fprintf(stderr, "fma.sat addend witness %u: len %u\n", i, len);
+            for (unsigned b = 0; b < len; ++b) fprintf(stderr, "%02x", native[b]);
+            fprintf(stderr, "\n");
+            return 140;
+         }
+      }
+   }
+
+   /* With the native G17G compiler, a fragment sampling an argument-buffer
+    * texture with an argument-buffer sampler at an explicit register LOD:
+    * both handles are aligned 64-bit uniforms (f97/f103 = u20/3,
+    * f126/f132 = u16/3), otherwise the driver-ABI register sample. */
+   {
+      const struct agx3_tex_sample_desc uniform_handles = {
+         .dest = 16, .coord = 0, .width = AGX3_TEX_W32, .mask = 15,
+         .lod = AGX3_TEX_LOD_REGISTER, .slot = 2, .wait_mask = 2,
+         .coord_kill = true, .sample_group_nonterminal = true,
+         .texture_kind = AGX3_TEX_OPERAND_UNIFORM, .texture_value = 20,
+         .sampler_kind = AGX3_TEX_OPERAND_UNIFORM, .sampler_value = 16,
+         .sample_abi = AGX3_TEX_SAMPLE_ABI_DRIVER,
+      };
+      const unsigned char expected[] = {0x8f,0x07,0x12,0x00,0x8d,0x20,0x60,0x48,0x89,0x18,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x00};
+      if (!agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, &uniform_handles,
+                                  native, &len) ||
+          len != sizeof(expected) || memcmp(native, expected, len)) {
+         fprintf(stderr, "uniform-handle smp: len %u ", len);
+         for (unsigned b = 0; b < len; ++b) fprintf(stderr, "%02x", native[b]);
+         fprintf(stderr, "\n");
+         return 141;
+      }
+   }
+
+   /* Native G17G Minecraft fragments: a select whose first
+    * comparand is a 32-bit uniform (f19, f25 = 3) beside a register,
+    * choosing a minifloat or a register (fself32). Every distinct one. */
+   {
+      static const struct {
+         uint16_t dest; bool dest_cache; uint16_t uniform, cmp1;
+         enum agx3_alu_hint cmp1_hint; uint16_t imm, sel1;
+         enum agx3_alu_hint sel1_hint; unsigned condition;
+         unsigned char bytes[16]; unsigned length;
+      } sel_cases[] = {
+      {8, true, 10, 6, AGX3_ALU_HINT_CACHE, 0, 8, AGX3_ALU_HINT_DISCARD, 6, {0x42,0x0b,0x23,0x87,0x25,0x80,0x83,0x00}, 8},
+      {20, true, 42, 16, AGX3_ALU_HINT_CACHE, 48, 20, AGX3_ALU_HINT_DISCARD, 6, {0xa2,0x2b,0x23,0x91,0x25,0xb0,0x83,0x00}, 8},
+      {20, false, 40, 16, AGX3_ALU_HINT_DISCARD, 0, 20, AGX3_ALU_HINT_DISCARD, 5, {0xa2,0x29,0x13,0x11,0x25,0x80,0x82,0x00}, 8},
+      {22, true, 46, 18, AGX3_ALU_HINT_CACHE, 48, 22, AGX3_ALU_HINT_DISCARD, 6, {0xb2,0x2f,0x23,0x93,0x25,0xb0,0x83,0x00}, 8},
+      {18, true, 44, 18, AGX3_ALU_HINT_DISCARD, 0, 22, AGX3_ALU_HINT_DISCARD, 5, {0x92,0x2d,0x33,0x13,0x26,0x80,0x82,0x00,0x80,0x16}, 10},
+      {12, true, 18, 10, AGX3_ALU_HINT_CACHE, 48, 8, AGX3_ALU_HINT_DISCARD, 6, {0x62,0x13,0x23,0x8b,0x26,0xb0,0x83,0x00,0x80,0x08}, 10},
+      {30, false, 16, 10, AGX3_ALU_HINT_DISCARD, 0, 12, AGX3_ALU_HINT_DISCARD, 5, {0xf2,0x11,0x13,0x0b,0x26,0x80,0x82,0x00,0x80,0x0c}, 10},
+      {16, true, 22, 8, AGX3_ALU_HINT_CACHE, 48, 10, AGX3_ALU_HINT_DISCARD, 6, {0x82,0x17,0x23,0x89,0x26,0xb0,0x83,0x00,0x80,0x0a}, 10},
+      {8, false, 20, 8, AGX3_ALU_HINT_DISCARD, 0, 16, AGX3_ALU_HINT_DISCARD, 5, {0x42,0x15,0x13,0x09,0x26,0x80,0x82,0x00,0x80,0x10}, 10},
+      {20, true, 42, 18, AGX3_ALU_HINT_CACHE, 48, 14, AGX3_ALU_HINT_DISCARD, 6, {0xa2,0x2b,0x23,0x93,0x26,0xb0,0x83,0x00,0x80,0x0e}, 10},
+      {20, false, 40, 18, AGX3_ALU_HINT_DISCARD, 0, 20, AGX3_ALU_HINT_DISCARD, 5, {0xa2,0x29,0x13,0x13,0x25,0x80,0x82,0x00}, 8},
+      {22, false, 46, 12, AGX3_ALU_HINT_NONE, 48, 16, AGX3_ALU_HINT_DISCARD, 6, {0xb2,0x2f,0x03,0x0d,0x26,0xb0,0x83,0x00,0x80,0x10}, 10},
+      {12, true, 44, 12, AGX3_ALU_HINT_DISCARD, 0, 22, AGX3_ALU_HINT_DISCARD, 5, {0x62,0x2d,0x33,0x0d,0x26,0x80,0x82,0x00,0x80,0x16}, 10},
+      {6, true, 20, 2, AGX3_ALU_HINT_CACHE, 48, 6, AGX3_ALU_HINT_DISCARD, 6, {0x32,0x15,0x23,0x83,0x25,0xb0,0x83,0x00}, 8},
+      {4, true, 20, 0, AGX3_ALU_HINT_CACHE, 48, 4, AGX3_ALU_HINT_DISCARD, 6, {0x22,0x15,0x23,0x81,0x25,0xb0,0x83,0x00}, 8},
+      {0, true, 20, 0, AGX3_ALU_HINT_DISCARD, 0, 4, AGX3_ALU_HINT_DISCARD, 5, {0x02,0x15,0x33,0x01,0x26,0x80,0x82,0x00,0x80,0x04}, 10},
+      };
+      for (unsigned i = 0; i < sizeof(sel_cases) / sizeof(sel_cases[0]); ++i) {
+         const struct agx3_fsel32_gpu1_desc d = {
+            .dest = sel_cases[i].dest, .dest_cache = sel_cases[i].dest_cache,
+            .cmp0 = sel_cases[i].uniform, .cmp0_uniform = true,
+            .cmp1 = sel_cases[i].cmp1, .cmp1_hint = sel_cases[i].cmp1_hint,
+            .sel_immediate = {true, false},
+            .sel = {sel_cases[i].imm, sel_cases[i].sel1},
+            .sel_hint = {AGX3_ALU_HINT_NONE, sel_cases[i].sel1_hint},
+            .condition = sel_cases[i].condition,
+         };
+         if (!agx3_alu_fcmp_select32_gpu1(&d, native, &len) ||
+             len != sel_cases[i].length ||
+             memcmp(native, sel_cases[i].bytes, len)) {
+            fprintf(stderr, "uniform select witness %u: len %u ", i, len);
+            for (unsigned b = 0; b < len; ++b) fprintf(stderr, "%02x", native[b]);
+            fprintf(stderr, "\n");
+            return 142;
+         }
+      }
+   }
+
+   const struct agx3_falu_runtime fma2_runtime = {.discard={true},.wait_mask=128};
+   const unsigned char fma2_expected[] = {
+      0x29,0x00,0x07,0x0a,0x20,0x20,0xa3,0x02,0x30,0x8d,0x59,0x00,0x14,0x06,0x00,0x80,
+   };
+   if (!agx3_falu_fma_two_imm(AGX_ISA_ENCODING_AGX3C,0,0,52,84,
+          &fma2_runtime,native,&len) || len != sizeof(fma2_expected) ||
+       memcmp(native,fma2_expected,len)) return 52;
+   const struct agx3_imad_desc mul_half = {
+      .dest=2,.dest_16=true,.src_a=3,.src_a_16=true,.src_a_kill=true,
+      .multiplier_is_register=true,.multiplier_reg=2,.multiplier_16=true,
+      .multiplier_kill=true,.no_addend=true,.wait_mask=128,
+   };
+   const unsigned char mul_half_expected[] = {
+      0x3f,0x03,0x04,0x18,0x21,0x00,0xa1,0x02,0xa0,0x00,0x20,0x00,0x01,0x02,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C,&mul_half,native,&len) ||
+       len != sizeof(mul_half_expected) || memcmp(native,mul_half_expected,len)) return 53;
+   const enum agx3_alu_hint extract_hint[] = {AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD};
+   const unsigned char extract_expected[] = {
+      0x27,0x04,0x00,0x1a,0x26,0x00,0xa1,0x02,0x08,0x01,0x20,0x34,0x00,0x10,
+   };
+   if (!agx3_alu_bfeil_m5(0,0,false,4,true,true,1,extract_hint,1,native,&len) ||
+       len != sizeof(extract_expected) || memcmp(native,extract_expected,len)) return 54;
+   const struct agx3_subgroup_reduce_desc red_half = {
+      .op=AGX3_SUBGROUP_REDUCE_IADD,.dest=2,.src=2,.half=true,.wait_mask=128,
+   };
+   const unsigned char red_half_expected[] = {
+      0x3f,0x02,0x04,0x08,0x27,0x80,0xa3,0x02,0x00,0x02,
+   };
+   if (!agx3_subgroup_encode_reduce(AGX_ISA_ENCODING_AGX3C,&red_half,native,&len) ||
+       len != sizeof(red_half_expected) || memcmp(native,red_half_expected,len)) return 55;
+   puts("M5_FT1_POPCOUNT_FMA2_NARROW_MUL_EXTRACT_REDUCE_NATIVE_BYTES_PASS");
+   const unsigned char ft1_alias_expected[] = {
+      0x27,0x00,0x00,0x1a,0x2a,0x00,0xab,0x02,0x28,0x00,
+   };
+   if (!agx3_alu_ft1_u32(AGX_ISA_ENCODING_AGX3C,0,0,AGX3_ALU_HINT_DISCARD,
+          true,0,native,&len) || len != 10 || memcmp(native,ft1_alias_expected,len)) return 56;
+   const unsigned char convert_short_expected[][10] = {
+      {0x3f,0x02,0x00,0x18,0x22,0x00,0xae,0x02,0x30,0x03},
+      {0x3f,0x02,0x00,0x18,0x22,0x00,0xae,0x02,0x28,0x03},
+   };
+   for (unsigned half=0;half<2;half++) {
+      if (!agx3_alu_float_to_int16(AGX_ISA_ENCODING_AGX3C,2,2,half,true,
+             AGX3_ALU_HINT_DISCARD,false,128,native,&len) || len != 10 ||
+          memcmp(native,convert_short_expected[half],len)) return 57+half;
+   }
+   puts("M5_INPLACE_FT1_FLOAT_TO_SHORT_NATIVE_BYTES_PASS");
+   const struct agx3_subgroup_ballot_desc ballot_pair = {
+      .src_half=true,.compare_register=true,.compare_value=1,.compare_half=true,
+      .compare_discard=true,.condition=4,.invert=true,.wait_mask=128,
+   };
+   const unsigned char ballot_pair_expected[] = {
+      0x2f,0x00,0x00,0x1a,0x25,0x00,0xae,0x02,0x40,0x00,0x28,0x04,
+   };
+   if (!agx3_subgroup_encode_ballot(AGX_ISA_ENCODING_AGX3C,&ballot_pair,native,&len) ||
+       len != 12 || memcmp(native,ballot_pair_expected,len)) return 59;
+   const enum agx3_subgroup_reduce_op quad_ops[] = {
+      AGX3_SUBGROUP_REDUCE_IADD,AGX3_SUBGROUP_REDUCE_SMIN,AGX3_SUBGROUP_REDUCE_SMAX,
+   };
+   const unsigned char quad_int_expected[][10] = {
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xa3,0x02,0x00,0x02},
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xa4,0x02,0x40,0x02},
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xa5,0x02,0x40,0x02},
+   };
+   for (unsigned op=0;op<3;op++) {
+      const struct agx3_subgroup_reduce_desc d = {
+         .quad=true,.op=quad_ops[op],.dest=2,.src=2,.half=true,.wait_mask=128,
+      };
+      if (!agx3_subgroup_encode_reduce(AGX_ISA_ENCODING_AGX3C,&d,native,&len) ||
+          len != 10 || memcmp(native,quad_int_expected[op],len)) return 60+op;
+   }
+   puts("M5_REGISTER_BALLOT_QUAD_INTEGER_NATIVE_BYTES_PASS");
+   const unsigned char short_srl_expected[] = {
+      0x3f,0x02,0x00,0x18,0x26,0x00,0xa1,0x02,0x30,0x82,0x00,0x34,0x00,0x00,
+   };
+   if (!agx3_alu_srl_imm16(AGX_ISA_ENCODING_AGX3C,2,2,8,
+          AGX3_ALU_HINT_DISCARD,false,128,native,&len) ||
+       len != 14 || memcmp(native,short_srl_expected,len)) return 63;
+   const uint16_t andc_mixed_src[] = {4,0};
+   const bool andc_half[] = {true,false};
+   const enum agx3_alu_hint andc_mixed_hints[] = {
+      AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD,
+   };
+   const unsigned char andc_half_expected[] = {
+      0x2b,0x00,0x06,0x00,0x32,0x70,0xa4,0x02,0x04,0x80,
+   };
+   if (!agx3_alu_bit_andc_width_m5(0,andc_mixed_src,andc_half,andc_mixed_hints,
+          false,128,native,&len) || len != 10 ||
+       memcmp(native,andc_half_expected,len)) return 64;
+   const enum agx3_subgroup_reduce_op quad_more_ops[] = {
+      AGX3_SUBGROUP_REDUCE_UMIN,AGX3_SUBGROUP_REDUCE_UMAX,
+      AGX3_SUBGROUP_REDUCE_FADD,AGX3_SUBGROUP_REDUCE_FMIN,AGX3_SUBGROUP_REDUCE_FMAX,
+   };
+   const unsigned char quad_more_expected[][10] = {
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xa6,0x02,0x00,0x02},
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xa7,0x02,0x00,0x02},
+      {0x3f,0x02,0x00,0x0a,0x27,0x00,0xa8,0x02,0x00,0x02},
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xab,0x02,0x00,0x02},
+      {0x3f,0x02,0x00,0x08,0x27,0x80,0xaf,0x02,0x00,0x02},
+   };
+   for (unsigned op=0;op<5;op++) {
+      const struct agx3_subgroup_reduce_desc d = {
+         .quad=true,.op=quad_more_ops[op],.dest=2,.src=2,.half=true,.wait_mask=128,
+      };
+      if (!agx3_subgroup_encode_reduce(AGX_ISA_ENCODING_AGX3C,&d,native,&len) ||
+          len != 10 || memcmp(native,quad_more_expected[op],len)) return 65+op;
+   }
+   puts("M5_NARROW_SHIFT_ANDC_QUAD_FLOAT_NATIVE_BYTES_PASS");
+   const unsigned char ushort_float_expected[] = {
+      0x2f,0x00,0x00,0x02,0x22,0x80,0xaf,0x02,0x20,0x00,
+   };
+   if (!agx3_alu_narrow_int_to_float(AGX_ISA_ENCODING_AGX3C,0,false,0,false,
+          false,AGX3_ALU_HINT_DISCARD,false,128,native,&len) ||
+       len != 10 || memcmp(native,ushort_float_expected,len)) return 70;
+   puts("M5_USHORT_FLOAT_NATIVE_BYTES_PASS");
+   const unsigned char float_ballot_expected[][12] = {
+      {0x2f,0x00,0x00,0x1a,0x25,0x00,0xaf,0x02,0x10,0x01,0x29,0x00},
+      {0x2f,0x00,0x00,0x1a,0x25,0x00,0xaf,0x02,0x48,0x80,0x28,0x00},
+   };
+   for (unsigned half=0;half<2;half++) {
+      const struct agx3_subgroup_ballot_desc d = {
+         .type=AGX3_SUBGROUP_BALLOT_FLOAT,.src_half=half,
+         .compare_register=true,.compare_value=half ? 1 : 4,.compare_half=half,
+         .compare_discard=true,.condition=0,.invert=true,.wait_mask=128,
+      };
+      if (!agx3_subgroup_encode_ballot(AGX_ISA_ENCODING_AGX3C,&d,native,&len) ||
+          len != 12 || memcmp(native,float_ballot_expected[half],len)) return 71+half;
+   }
+   puts("M5_FLOAT_REGISTER_BALLOT_NATIVE_BYTES_PASS");
+   const unsigned char selected_float_expected[][16] = {
+      {0x32,0x02,0x03,0x0a,0x23,0x00,0xa0,0x02,0x94,0x02,0x21,0x08,0x00,0x06,0x80,0x00},
+      {0x32,0x00,0x03,0x4a,0x23,0x00,0xa0,0x02,0x4c,0x82,0x20,0x08,0x00,0x06,0x80,0x00},
+   };
+   for (unsigned half=0;half<2;half++) {
+      const struct agx3_fsel32_gpu1_desc d = {
+         .dest=2,.cmp0=half ? 8 : 2,.cmp1=half ? 9 : 10,
+         .cmp0_16=half,.cmp1_16=half,
+         .cmp0_hint=AGX3_ALU_HINT_DISCARD,.cmp1_hint=AGX3_ALU_HINT_DISCARD,
+         .sel={0,48},.sel_immediate={false,true},.wait_mask=128,
+      };
+      if (!agx3_alu_fcmp_select32_gpu1(&d,native,&len) || len != 16 ||
+          memcmp(native,selected_float_expected[half],len)) {
+         printf("FSEL_MISMATCH half=%u len=%u ",half,len);
+         for (unsigned byte=0;byte<len;byte++) printf("%02x",native[byte]);
+         puts("");
+         return 73+half;
+      }
+   }
+   puts("M5_FLOAT_SELECTED_CONSTANT_NATIVE_BYTES_PASS");
+   const uint16_t half_bin_src[] = {2,3};
+   const enum agx3_alu_hint half_bin_hints[] = {
+      AGX3_ALU_HINT_DISCARD,AGX3_ALU_HINT_DISCARD,
+   };
+   const bool half_sub_neg[] = {false,true};
+   const unsigned char half_bin_expected[][10] = {
+      {0x38,0x02,0x04,0x02,0x22,0xa0,0xa0,0x02,0x05,0x08},
+      {0x38,0x02,0x05,0x02,0x22,0x20,0xa4,0x02,0xc4,0x00},
+      {0x38,0x02,0x24,0x02,0x22,0xa0,0xa0,0x02,0x05,0x08},
+   };
+   for (unsigned op=0;op<3;op++) {
+      bool ok = op == 0 ? agx3_alu_hadd(AGX_ISA_ENCODING_AGX3C,2,
+                      half_bin_src,half_bin_hints,false,128,native,&len)
+                : op == 1 ? agx3_alu_hmul(AGX_ISA_ENCODING_AGX3C,2,
+                      half_bin_src,half_bin_hints,false,128,native,&len)
+                : agx3_alu_hadd_signed(AGX_ISA_ENCODING_AGX3C,2,
+                      half_bin_src,half_bin_hints,half_sub_neg,false,128,native,&len);
+      if (!ok || len != 10 || memcmp(native,half_bin_expected[op],len)) return 75+op;
+   }
+   puts("M5_HALF_ARITHMETIC_NATIVE_BYTES_PASS");
+   const struct agx3_imad_desc all_half_mad = {
+      .src_a=1,.src_a_16=true,.src_a_kill=true,
+      .multiplier_is_register=true,.multiplier_reg=4,.multiplier_16=true,
+      .multiplier_kill=true,.src_c_16=true,.src_c_kill=true,.wait_mask=128,
+   };
+   const unsigned char all_half_mad_expected[] = {
+      0x2f,0x01,0x04,0x1a,0x21,0x00,0xa1,0x02,0x20,0x01,0xa0,0x00,0x00,0x00,
+   };
+   if (!agx3_imad_encode(AGX_ISA_ENCODING_AGX3C,&all_half_mad,native,&len) ||
+       len != 14 || memcmp(native,all_half_mad_expected,len)) return 78;
+   const struct agx3_fsel32_gpu1_desc select_two = {
+      .cmp0_16=true,.cmp1=1,.cmp1_16=true,
+      .cmp0_hint=AGX3_ALU_HINT_DISCARD,.cmp1_hint=AGX3_ALU_HINT_DISCARD,
+      .sel={2,0},.sel_immediate={true,true},.sel_integer=true,.wait_mask=128,
+   };
+   const unsigned char select_two_expected[] = {
+      0x22,0x00,0x07,0x0a,0x23,0x00,0xa0,0x02,0x4c,0x80,0x60,0x08,0x02,0x40,
+   };
+   if (!agx3_alu_fcmp_select32_gpu1(&select_two,native,&len) ||
+       len != 14 || memcmp(native,select_two_expected,len)) return 79;
+   puts("M5_HALF_MULTIPLY_ADD_INTEGER_SELECT_NATIVE_BYTES_PASS");
+   const uint16_t shift_src[] = {4, 0};
+   const bool shift_half[] = {true, false};
+   const enum agx3_alu_hint shift_hint[] = {AGX3_ALU_HINT_DISCARD, AGX3_ALU_HINT_DISCARD};
+   const unsigned char shift_expected[] = {
+      0x27,0x04,0x00,0x3a,0x26,0x00,0xa0,0x02,0x00,0x80,0x20,0x34,0x00,0x00,
+   };
+   if (!agx3_alu_sll_reg(AGX_ISA_ENCODING_AGX3C,0,false,false,shift_src,
+          shift_half,shift_hint,1,native,&len) || len != sizeof(shift_expected) ||
+       memcmp(native,shift_expected,len)) return 22;
+   const unsigned char or_expected[] = {
+      0x2b,0x00,0x07,0x80,0x32,0x38,0xa4,0x02,0x04,0x01,
+   };
+   if (!agx3_alu_bit_mixed_m5(AGX3_ALU_BOR,0,0,4,AGX3_ALU_HINT_DISCARD,
+          AGX3_ALU_HINT_DISCARD,false,128,native,&len) ||
+       len != sizeof(or_expected) || memcmp(native,or_expected,len)) return 23;
+   const unsigned char cmp_expected[] = {
+      0x2a,0x02,0x3b,0x04,0x23,0x80,0xa5,0x02,0x88,0x00,
+   };
+   if (!agx3_cf_icmp_imm16_runtime(AGX_ISA_ENCODING_AGX3C,AGX3_CF_ICOND_ULT,
+          0,2,2,(struct agx3_cf_source_hints){false,true},128,native,&len) ||
+       len != sizeof(cmp_expected) || memcmp(native,cmp_expected,len)) return 24;
+   const unsigned char fcmp_expected[] = {
+      0x2a,0x02,0x2b,0x00,0x23,0x00,0xa3,0x12,0x00,0x81,
+   };
+   if (!agx3_cf_fcmp_zero_imm(AGX_ISA_ENCODING_AGX3C,AGX3_CF_FCOND_EQ,
+          false,0,2,false,0,(struct agx3_cf_source_hints){false,true},128,
+          native,&len) || len != sizeof(fcmp_expected) ||
+       memcmp(native,fcmp_expected,len)) return 25;
+   const struct agx3_fsel32_gpu1_desc fbool = {
+      .dest=2, .cmp0=2, .cmp0_hint=AGX3_ALU_HINT_DISCARD,
+      .cmp1_immediate=true, .sel_immediate={true,true}, .sel={1,0},
+      .sel_integer=true, .wait_mask=128,
+   };
+   const unsigned char fbool_expected[] = {
+      0x32,0x02,0x05,0x0a,0x23,0x00,0xa0,0x02,0x14,0x80,0x4e,0x18,0x01,0x00,0x80,0x00,
+   };
+   if (!agx3_alu_fcmp_select16_gpu1(&fbool,native,&len) ||
+       len != sizeof(fbool_expected) || memcmp(native,fbool_expected,len)) return 26;
+   const struct agx3_subgroup_ballot_desc quad_vote = {
+      .type=AGX3_SUBGROUP_BALLOT_INTEGER, .quad=true,
+      .dest=2, .dest_half=true, .dest_cache=true, .src=2,
+      .src_live=false, .condition=4, .invert=true, .wait_mask=128,
+   };
+   const unsigned char quad_vote_expected[] = {
+      0x3f,0x02,0x00,0x18,0x2d,0x00,0xa6,0x02,0x38,0x00,0x08,0x04,
+   };
+   if (!agx3_subgroup_encode_ballot(AGX_ISA_ENCODING_AGX3C,&quad_vote,native,&len) ||
+       len != sizeof(quad_vote_expected) || memcmp(native,quad_vote_expected,len)) return 76;
+   const uint16_t hcompare_src[] = {2, 3};
+   const struct agx3_cf_source_hints hcompare_hints[] = {{false,true},{false,true}};
+   const unsigned char hcompare_expected[] = {0x2a,0x02,0x2b,0x00,0x23,0x02,0xa0,0x0a,0xc0,0x81};
+   if (!agx3_cf_fcmp_reg16(AGX_ISA_ENCODING_AGX3C,AGX3_CF_FCOND_EQ,
+          false,0,hcompare_src,hcompare_hints,128,native,&len) ||
+       len != sizeof(hcompare_expected) || memcmp(native,hcompare_expected,len)) return 77;
+   const unsigned char hfma_expected[] = {0x38,0x02,0x06,0x02,0x23,0x20,0xa0,0x02,0x96,0x02,0x78,0x00};
+   if (!agx3_alu_hfma_immediates_m5(2,2,10,20,AGX3_ALU_HINT_DISCARD,
+          false,128,native,&len) || len != sizeof(hfma_expected) ||
+       memcmp(native,hfma_expected,len)) return 78;
+   for (unsigned sign = 0; sign < 2; sign++) {
+      unsigned char expected_half[] = {0x3f,0x02,0x00,0x00,0x22,0x80,0xaf,0x02,0x28,0x00};
+      if (sign) expected_half[8] = 0x68;
+      if (!agx3_alu_i32_to_f16(AGX_ISA_ENCODING_AGX3C, 2, 2, sign,
+                               AGX3_ALU_HINT_DISCARD, false, 128, native, &len) ||
+          len != sizeof(expected_half) || memcmp(native, expected_half, len)) return 80;
+   }
+   const struct agx3_tps_desc byte_tps = {
+      .data=10, .width=AGX3_MEM_W8, .mask=7, .byte_offset=4,
+      .index_immediate=true, .frame_region=6, .value_kill=true,
+      .wait_mask=1,
+   };
+   const unsigned char byte_tps_expected[] = {0x57,0x00,0x0b,0x00,0xc1,0x2c,0x92,0xa0,0x00,0x04};
+   if (!agx3_tps_encode_store(AGX_ISA_ENCODING_AGX3C,&byte_tps,native,&len) ||
+       len != sizeof(byte_tps_expected) || memcmp(native,byte_tps_expected,len)) return 79;
+   puts("M5_SHIFT_COMPARE_OR_NATIVE_BYTES_PASS");
+   /* A vertex ID converted to float may still index a later attribute load.
+    * The old compact M5 packer always discarded that live integer source. */
+   const unsigned char int_to_float_hints[3][10] = {
+      {0x37,0x00,0x00,0x02,0x2a,0x80,0xaf,0x02,0x48,0x00},
+      {0x37,0x00,0x00,0x02,0x2a,0x80,0xaf,0x02,0x68,0x00},
+      {0x37,0x80,0x00,0x02,0x2a,0x80,0xaf,0x02,0x48,0x00},
+   };
+   for (unsigned hint = 0; hint < 3; ++hint) {
+      if (!agx3_alu_int_to_float(AGX_ISA_ENCODING_AGX3C, 2, 0, true,
+                                hint, true, 0, native, &len) ||
+          len != 10 || memcmp(native, int_to_float_hints[hint], 10)) return 80;
+   }
+   if (!agx3_alu_int_to_float(AGX_ISA_ENCODING_AGX3C, 254, 252, false,
+                             AGX3_ALU_HINT_CACHE, true, 128, native, &len) ||
+       agx3_alu_int_to_float(AGX_ISA_ENCODING_AGX3C, 256, 0, false,
+                            AGX3_ALU_HINT_NONE, false, 0, native, &len)) return 81;
+   puts("M5_INT_TO_FLOAT_SOURCE_LIFETIME_PASS");
+   const struct agx3_falu_runtime saturate_runtime = {
+      .dest_cache = true, .discard = {true}, .wait_mask = 1,
+   };
+   const unsigned char saturate_expected[] = {
+      0x09, 0x01, 0x3c, 0x81, 0x01, 0x22, 0x00, 0x02,
+   };
+   if (!agx3_falu_add_imm_m5(0, 0, 0x80, false, false, true,
+                             &saturate_runtime, native, &len) ||
+       len != sizeof(saturate_expected) ||
+       memcmp(native, saturate_expected, len)) return 50;
+   puts("M5_SATURATING_ADD_NATIVE_BYTES_PASS");
+   glsl_type_singleton_init_or_ref();
+   bool ok = true;
+   /* A dynamic ray-query array exposed the old G17P limit on an M5 add
+    * allocated to half232. Check both chip limits and transactional refusal. */
+   const unsigned integer_regs[] = {190, 192, 232, 250, 252};
+   for (unsigned chip = 0; chip < 2; ++chip) {
+      const enum agx_isa_encoding enc = chip ? AGX_ISA_ENCODING_AGX3C :
+                                             AGX_ISA_ENCODING_AGX3B;
+      for (unsigned i = 0; i < ARRAY_SIZE(integer_regs); ++i) {
+         struct agx3_uniform_integer_desc d = {
+            .op = AGX3_UI_ADD, .dest = integer_regs[i],
+            .src = {{.kind = AGX3_UI_UNIFORM, .value = 134},
+                    {.kind = AGX3_UI_REGISTER, .value = 16}},
+         };
+         uint8_t bytes[16], size = 0;
+         memset(bytes, 0xa5, sizeof(bytes));
+         const bool expected = integer_regs[i] < (chip ? 252 : 192);
+         const bool encoded = agx3_uniform_integer_encode(enc, &d, bytes, &size);
+         ok &= encoded == expected;
+         if (encoded) {
+            ok &= size > 0 && size <= sizeof(bytes);
+         } else {
+            for (unsigned j = 0; j < sizeof(bytes); ++j)
+               ok &= bytes[j] == 0xa5;
+         }
+      }
+   }
+   puts(ok ? "M5_UNIFORM_INTEGER_REGISTER_LIMIT_PASS" :
+             "M5_UNIFORM_INTEGER_REGISTER_LIMIT_FAIL");
+   /* Native G17G 4x sparse MRT: metal577 RT0+RT2 and RT1+RT2. */
+   const uint32_t sparse_classes[] = {204, 240};
+   const uint8_t sparse_wait[][6] = {
+      {0x67, 0x46, 0x00, 0x05, 0x00, 0x00},
+      {0x87, 0x47, 0x00, 0x05, 0x00, 0x00},
+   };
+   const uint8_t sparse_release[][6] = {
+      {0x67, 0x16, 0x00, 0x04, 0x00, 0x00},
+      {0x87, 0x17, 0x00, 0x04, 0x00, 0x00},
+   };
+   for (unsigned i = 0; i < ARRAY_SIZE(sparse_classes); i++) {
+      uint8_t bytes[AGX3_PIXEL_MAX_LENGTH], length = 0;
+      ok &= agx3_pixel_wait(AGX_ISA_ENCODING_AGX3C, sparse_classes[i], 0,
+                           bytes, &length) && length == 6 &&
+            memcmp(bytes, sparse_wait[i], 6) == 0;
+      ok &= agx3_pixel_release(AGX_ISA_ENCODING_AGX3C, sparse_classes[i],
+                              true, 0, bytes, &length) && length == 6 &&
+            memcmp(bytes, sparse_release[i], 6) == 0;
+   }
+
+   /* The custom MRT generator's one-target limit must reproduce the
+    * independently compiled, hardware-qualified RT0 path, including layers
+    * and MS arrays. All target counts retain bounded register allocation. */
+   for (unsigned layered = 0; layered < 2; ++layered) {
+      for (unsigned ms = 0; ms < 2; ++ms) {
+         const uint8_t selectors[8] = {
+            0x75, 0x15, 0x35, 0x55, 0x95, 0x75, 0x15, 0x35,
+         };
+         for (unsigned targets = 1; targets <= 8; ++targets) {
+            uint8_t generated[AGX3_M5_MRT_EOT_MAX_LENGTH];
+            size_t length = 0;
+            if (!agx3_m5_mrt_eot_generate_layered(targets, selectors, ms,
+                   layered, generated, sizeof(generated), &length) ||
+                agx3_m5_mrt_eot_nr_gprs(targets) != 8 ||
+                agx3_m5_mrt_eot_staging_halfs(targets) != targets * 8)
+               return 133;
+            if (targets != 1)
+               continue;
+            nir_builder b = nir_builder_init_simple_shader(
+               MESA_SHADER_COMPUTE, &agx_nir_options, "M5 array MRT limit");
+            nir_def *xy = nir_trim_vector(
+               &b, nir_load_global_invocation_id(&b, 32), 2);
+            nir_def *coords = layered
+               ? nir_vec3(&b, nir_channel(&b, xy, 0),
+                          nir_channel(&b, xy, 1),
+                          nir_u2u32(&b, nir_load_layer_id(&b)))
+               : xy;
+            nir_image_store_block_agx(&b, nir_imm_intN_t(&b, 0, 16),
+               nir_imm_int(&b, 0), coords, .format=PIPE_FORMAT_R8_UNORM,
+               .image_dim=GLSL_SAMPLER_DIM_2D,
+               .image_array=layered, .explicit_coord=true);
+            agx_preprocess_nir(b.shader);
+            struct agx_shader_key key = {
+               .dev.chip=AGX_CHIP_G17S, .dev.encoding=AGX_ISA_ENCODING_AGX3C,
+               .no_preamble=true, .m5_rt0_eot=true,
+               .g17p_rt0_eot_multisample=ms,
+            };
+            agx_g17p_configure_rt0_eot_key(PIPE_FORMAT_R8_UNORM, &key);
+            struct agx_shader_part part = {0};
+            bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+            bool match = compiled && part.info.binary_size == length &&
+               memcmp(part.binary, generated, length) == 0;
+            printf("M5_MRT_RT0_ARRAY_%u_MS_%u_%s size=%zu\n",
+                   layered, ms, match ? "PASS" : "FAIL", length);
+            ok &= match;
+            free(part.binary);
+            ralloc_free(b.shader);
+         }
+      }
+   }
+
+   for (unsigned ms = 0; ms < 2; ++ms) {
+      const enum pipe_format formats[] = {
+         PIPE_FORMAT_R10G10B10A2_UNORM, PIPE_FORMAT_B10G10R10A2_UNORM,
+      };
+      for (unsigned f = 0; f < ARRAY_SIZE(formats); ++f) {
+         nir_builder b = nir_builder_init_simple_shader(
+            MESA_SHADER_COMPUTE, &agx_nir_options, "M5 layered RGB10 EOT");
+         nir_def *xy = nir_trim_vector(
+            &b, nir_load_global_invocation_id(&b, 32), 2);
+         nir_def *coords = nir_vec3(&b, nir_channel(&b, xy, 0),
+            nir_channel(&b, xy, 1), nir_u2u32(&b, nir_load_layer_id(&b)));
+         nir_image_store_block_agx(&b, nir_imm_intN_t(&b, 0, 16),
+            nir_imm_int(&b, 0), coords, .format=PIPE_FORMAT_R10G10B10A2_UNORM,
+            .image_dim=GLSL_SAMPLER_DIM_2D, .image_array=true,
+            .explicit_coord=true);
+         agx_preprocess_nir(b.shader);
+         struct agx_shader_key key = {
+            .dev.chip=AGX_CHIP_G17S, .dev.encoding=AGX_ISA_ENCODING_AGX3C,
+            .no_preamble=true, .m5_rt0_eot=true,
+            .g17p_rt0_eot_multisample=ms,
+         };
+         agx_g17p_configure_rt0_eot_key(formats[f], &key);
+         struct agx_shader_part part = {0};
+         uint8_t generated[AGX3_M5_MRT_EOT_MAX_LENGTH];
+         size_t length = 0;
+         const uint8_t selector = 0x75;
+         bool generated_ok = agx3_m5_mrt_eot_generate_layered(
+            1, &selector, ms, true, generated, sizeof(generated), &length);
+         bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+         bool match = generated_ok && compiled &&
+            part.info.binary_size == length &&
+            memcmp(part.binary, generated, length) == 0;
+         printf("M5_LAYERED_RGB10_%u_MS_%u_%s\n", f, ms,
+                match ? "PASS" : "FAIL");
+         ok &= match;
+         free(part.binary);
+         ralloc_free(b.shader);
+      }
+   }
+
+   /* A narrow zero-extended low word vanishes from this funnel shift.
+    * These local-ID address calculations formerly failed after narrowing. */
+   for (unsigned shift = 16; shift <= 31; shift++) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_COMPUTE, &agx_nir_options, "M5 narrow funnel shift");
+      nir_def *index = nir_channel(&b, nir_load_local_invocation_id(&b), 0);
+      nir_def *wide = nir_ishl_imm(&b, nir_i2i64(&b, index), 32 - shift);
+      nir_store_global(&b, nir_unpack_64_2x32_split_y(&b, wide),
+                       nir_imm_int64(&b, 0x100000000ull), .align_mul = 4);
+      agx_preprocess_nir_m5(b.shader);
+      struct agx_shader_key key = {
+         .dev.chip = AGX_CHIP_G17S, .dev.encoding = AGX_ISA_ENCODING_AGX3C,
+         .no_preamble = true,
+      };
+      struct agx_shader_part part = {0};
+      const bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+      printf("M5_NARROW_FUNNEL_SHIFT_%u_%s\n", shift,
+             compiled ? "PASS" : "FAIL");
+      ok &= compiled && part.info.binary_size > 0;
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+   /* The same shared-memory operations need a rendezvous only when the
+    * execution scope is the entire workgroup. In particular, a subgroup
+    * fence may occur under a subgroup-uniform branch. */
+   for (unsigned workgroup = 0; workgroup < 2; ++workgroup) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_COMPUTE, &agx_nir_options, "M5 shared barrier scope");
+      b.shader->info.workgroup_size[0] = 256;
+      b.shader->info.workgroup_size[1] = 1;
+      b.shader->info.workgroup_size[2] = 1;
+      b.shader->info.shared_size = 1024;
+      nir_def *id = nir_load_local_invocation_index(&b);
+      nir_def *offset = nir_ishl_imm(&b, id, 2);
+      nir_store_shared(&b, id, offset, .align_mul = 4);
+      nir_barrier(&b,
+         .execution_scope = workgroup ? SCOPE_WORKGROUP : SCOPE_NONE,
+         .memory_scope = workgroup ? SCOPE_WORKGROUP : SCOPE_SUBGROUP,
+         .memory_semantics = NIR_MEMORY_ACQUIRE | NIR_MEMORY_RELEASE,
+         .memory_modes = nir_var_mem_shared);
+      nir_def *other = nir_ishl_imm(&b, nir_ixor(&b, id, nir_imm_int(&b, 1)), 2);
+      nir_def *value = nir_load_shared(&b, 1, 32, other, .align_mul = 4);
+      nir_store_global(&b, value,
+         nir_iadd(&b, nir_imm_int64(&b, 0x100000000ull), nir_u2u64(&b, offset)),
+         .align_mul = 4);
+      agx_preprocess_nir_m5(b.shader);
+      struct agx_shader_key key = {
+         .dev.chip = AGX_CHIP_G17S, .dev.encoding = AGX_ISA_ENCODING_AGX3C,
+         .no_preamble = true,
+      };
+      struct agx_shader_part part = {0};
+      bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+      unsigned barriers = 0;
+      const uint8_t rendezvous[] = {0x27, 0x51, 0, 6, 0, 0};
+      for (unsigned i = part.info.main_offset;
+           compiled && i + sizeof(rendezvous) <=
+              part.info.main_offset + part.info.main_size; i += 2)
+         barriers += !memcmp(part.binary + i, rendezvous, sizeof(rendezvous));
+      bool correct = compiled && barriers == workgroup;
+      printf("M5_SHARED_BARRIER_SCOPE_%u_%s\n", workgroup,
+             correct ? "PASS" : "FAIL");
+      ok &= correct;
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+   const enum pipe_format eot_formats[] = {PIPE_FORMAT_R8_UNORM,
+                                           PIPE_FORMAT_R8_UINT,
+                                           PIPE_FORMAT_R8_SINT};
+   for (unsigned i = 0; i < 3; ++i) {
+      nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,
+                                                    &agx_nir_options, "M5 EOT");
+      nir_image_store_block_agx(&b, nir_imm_intN_t(&b,0,16), nir_imm_int(&b,0),
+         nir_trim_vector(&b, nir_load_global_invocation_id(&b,32), 2),
+         /* IMGWBLK names the raw tile class. Signedness belongs to the
+          * PBE descriptor, so both integer8 logical formats use I8. */
+         .format=i ? PIPE_FORMAT_R8_UINT : PIPE_FORMAT_R8_UNORM,
+         .image_dim=GLSL_SAMPLER_DIM_2D,
+         .image_array=false, .explicit_coord=true);
+      agx_preprocess_nir(b.shader);
+      struct agx_shader_key key = {
+         .dev.chip=AGX_CHIP_G17S, .dev.encoding=AGX_ISA_ENCODING_AGX3C,
+         .no_preamble=true, .m5_rt0_eot=true,
+      };
+      agx_g17p_configure_rt0_eot_key(eot_formats[i], &key);
+      unsigned char expected_format[sizeof(expected)];
+      memcpy(expected_format, expected, sizeof(expected));
+      if (i)
+         expected_format[71] = 0xa0;
+      struct agx_shader_part part={0};
+      bool compiled=agx_compile_shader_nir(b.shader,&key,&part);
+      bool format_ok=compiled && part.info.binary_size==sizeof(expected) &&
+         part.info.main_offset==0 && part.info.main_size==sizeof(expected) &&
+         part.info.nr_gprs==8 && part.info.agx3_staging_known &&
+         part.info.agx3_staging_halfs==8 &&
+         !memcmp(part.binary,expected_format,sizeof(expected));
+      printf("M5_EOT_FORMAT_%u_%s compiled=%u size=%u gprs=%u staging=%u\n",
+         i,format_ok ? "PASS":"FAIL",compiled,part.info.binary_size,
+         part.info.nr_gprs,part.info.agx3_staging_halfs);
+      ok &= format_ok;
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+   {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_COMPUTE, &agx_nir_options, "M5 robust UBO end offset");
+      nir_def *offset = nir_load_global(&b, 1, 32,
+         nir_imm_int64(&b, 0x100000000ull), .align_mul = 4);
+      nir_def *end = nir_uadd_sat(&b, offset, nir_imm_int(&b, 4));
+      nir_store_global(&b, end, nir_imm_int64(&b, 0x100000010ull),
+                       .align_mul = 4);
+      agx_preprocess_nir_m5(b.shader);
+      struct agx_shader_key key = {
+         .dev.chip = AGX_CHIP_G17S, .dev.encoding = AGX_ISA_ENCODING_AGX3C,
+         .no_preamble = true,
+      };
+      struct agx_shader_part part = {0};
+      const bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+      printf("M5_ROBUST_UBO_UADD_SAT_IMMEDIATE_%s compiled=%u size=%u\n",
+             compiled ? "PASS" : "FAIL", compiled, part.info.binary_size);
+      ok &= compiled && part.info.binary_size > 0;
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+   /* Qt's curve coverage shader clamps a fused float product plus a
+    * minifloat constant. Keep the all-f32 form on the native FMA path. */
+   for (unsigned i = 0; i < 3; ++i) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_COMPUTE, &agx_nir_options, "M5 saturated FMA immediate");
+      nir_def *a = nir_load_global(&b, 1, 32,
+         nir_imm_int64(&b, 0x100000000ull), .align_mul = 4);
+      nir_def *c = nir_load_global(&b, 1, 32,
+         nir_imm_int64(&b, 0x100000004ull), .align_mul = 4);
+      nir_def *value = nir_fsat(&b, nir_ffma(&b, a, c,
+                                           nir_imm_float(&b, i * 0.5f)));
+      nir_store_global(&b, value, nir_imm_int64(&b, 0x100000010ull),
+                       .align_mul = 4);
+      agx_preprocess_nir_m5(b.shader);
+      struct agx_shader_key key = {
+         .dev.chip = AGX_CHIP_G17S, .dev.encoding = AGX_ISA_ENCODING_AGX3C,
+         .no_preamble = true,
+      };
+      struct agx_shader_part part = {0};
+      const bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+      printf("M5_SATURATED_FMA_IMMEDIATE_%u_%s\n", i,
+             compiled ? "PASS" : "FAIL");
+      ok &= compiled && part.info.binary_size > 0;
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+   glsl_type_singleton_decref();
+   return !ok;
+}
diff --git a/src/asahi/compiler/test/test-agx3-m5-program.c b/src/asahi/compiler/test/test-agx3-m5-program.c
new file mode 100644
index 000000000000..73cb413a8eb4
--- /dev/null
+++ b/src/asahi/compiler/test/test-agx3-m5-program.c
@@ -0,0 +1,764 @@
+/* SPDX-License-Identifier: MIT */
+
+
+#include "agx3_graphics_program.h"
+#include "agx3_copy_core.h"
+#include "agx3_binder.h"
+#include "asahi/lib/agx_ioto_program_core.h"
+#include "agx3_ioto_load_packer.h"
+#include "agx3_cprog_input.h"
+#include "asahi/lib/agx_ioto_preshader_core.h"
+#include "asahi/lib/agx_ppp_hal300_graphics_core.h"
+#include "asahi/lib/agx3_compute_program_core.h"
+#include "util/macros.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#define CHECK(x)                                                               \
+   do {                                                                        \
+      if (!(x)) {                                                              \
+         fprintf(stderr, "CHECK failed at %s:%d: %s\n", __FILE__, __LINE__,  \
+                 #x);                                                          \
+         return 1;                                                             \
+      }                                                                        \
+   } while (0)
+
+static int
+test_g17g_programs(enum agx_chip chip)
+{
+
+   static const uint8_t expected_tail[2][28] = {
+      {0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+       0x87,0x04,0x48,0x0e,0x00,0x00,0x00,0x00,0x00,0x01,
+       0x87,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00},
+      {0x17,0x00,0x00,0x2c,0x00,0x00,0x18,0x08,
+       0xc7,0x03,0x40,0x0e,0x00,0x00,0x00,0x00,0x00,0x01,
+       0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00},
+   };
+   for (unsigned fragment = 0; fragment < 2; ++fragment) {
+      struct agx3_g17p_graphics_program_desc desc = {
+         .program_va = UINT64_C(0x10000100000),
+         .shader_va = fragment ? UINT64_C(0x100000003c0)
+                              : UINT64_C(0x10000000480),
+         .shader_binary_size = fragment ? 60 : 128,
+         .nr_gprs = 16,
+         .compiler_d_valid = true,
+         .compiler_d = fragment ? 0 : 32,
+         .fragment = fragment,
+         .imageblock = { .sample_size_B = 8, .nr_samples = 1, .tile_size_px = 1024 },
+      };
+      struct agx3_g17p_graphics_program output;
+      size_t empty_bound;
+      CHECK(agx3_g17g_graphics_program_bound(chip, NULL, 0,
+                                              &empty_bound));
+      CHECK(empty_bound == 90);
+      CHECK(agx3_g17g_graphics_program_generate(chip, &desc,
+                                                 &output));
+      CHECK(output.length == (fragment ? 90 : 84));
+      CHECK(memcmp(output.bytes, expected_tail[fragment], 28) == 0);
+      CHECK(output.bytes[48] == (fragment ? 4 : 8) *
+            (chip == AGX_CHIP_G17S ? 2 : 1));
+
+
+      {
+         const unsigned d[] = {256, 288, 320, 512};
+         const uint8_t expected_spectpr[][10] = {
+            {0x07,0x04,0x00,0x0d,0,0,0,0,0,0},
+            {0x87,0x04,0x00,0x0d,0,0,0,0,0,0},
+            {0x07,0x05,0x00,0x0d,0,0,0,0,0,0},
+            {0x07,0x08,0x00,0x0d,0,0,0,0,0,0},
+         };
+         for (unsigned i = 0; i < ARRAY_SIZE(d); ++i) {
+            struct agx3_g17p_graphics_program_desc wide = desc;
+            struct agx3_g17p_graphics_program wide_output;
+            wide.compiler_d = d[i];
+            CHECK(agx3_g17g_graphics_program_generate(chip, &wide, &wide_output));
+            CHECK(wide_output.length == output.length);
+            CHECK(memcmp(wide_output.bytes + 18, expected_spectpr[i], 10) == 0);
+         }
+      }
+      const uint32_t scratch_bytes[] = {4, 272, 400, 528, 1040, 262140};
+      const uint8_t scratch_fields[][4] = {
+         {1, 0, 1, 0}, {68, 0, 68, 0}, {100, 0, 100, 0},
+         {132, 0, 132, 0}, {4, 1, 4, 1}, {255, 255, 255, 255},
+      };
+      for (unsigned i = 0; i < ARRAY_SIZE(scratch_bytes); ++i) {
+         struct agx3_g17p_graphics_program_desc spill = desc;
+         struct agx3_g17p_graphics_program spill_output;
+         spill.scratch_size = scratch_bytes[i];
+         CHECK(agx3_g17g_graphics_program_generate(chip, &spill, &spill_output));
+         CHECK(spill_output.length == output.length);
+         CHECK(memcmp(spill_output.bytes + 24, scratch_fields[i], 4) == 0);
+         CHECK(memcmp(spill_output.bytes, output.bytes, 24) == 0);
+      }
+      struct agx3_g17p_graphics_program_desc too_wide = desc;
+      too_wide.compiler_d = 513;
+      CHECK(!agx3_g17g_graphics_program_generate(chip, &too_wide, &output));
+
+
+      if (fragment) {
+         struct agx3_g17p_graphics_program_desc mrt = desc;
+         mrt.imageblock.sample_size_B = 32;
+         struct agx3_g17p_graphics_program mrt_output;
+         CHECK(agx3_g17g_graphics_program_generate(chip, &mrt,
+                                                    &mrt_output));
+         uint8_t expected_mrt[28];
+         memcpy(expected_mrt, expected_tail[1], sizeof(expected_mrt));
+         expected_mrt[7] = 0x20;
+         CHECK(mrt_output.length == output.length);
+         CHECK(memcmp(mrt_output.bytes, expected_mrt, 28) == 0);
+
+
+         struct agx3_g17p_graphics_program_desc bare = desc;
+         bare.imageblock.sample_size_B = 0;
+         struct agx3_g17p_graphics_program bare_output;
+         CHECK(agx3_g17g_graphics_program_generate(chip, &bare,
+                                                    &bare_output));
+         uint8_t expected_bare[28];
+         memcpy(expected_bare, expected_tail[1], sizeof(expected_bare));
+         memcpy(expected_bare, expected_tail[0], 8);
+         CHECK(memcmp(bare_output.bytes, expected_bare, sizeof(expected_bare)) == 0);
+
+         for (unsigned stride = 8; stride <= 64; stride += 8) {
+            mrt.imageblock.sample_size_B = stride;
+            CHECK(agx3_g17g_graphics_program_generate(chip, &mrt, &mrt_output));
+            expected_mrt[7] = stride;
+            CHECK(memcmp(mrt_output.bytes, expected_mrt, sizeof(expected_mrt)) == 0);
+         }
+
+         const uint8_t ungrounded[] = {4, 12, 65};
+         for (unsigned i = 0; i < sizeof(ungrounded); ++i) {
+            struct agx3_g17p_graphics_program_desc bad = desc;
+            bad.imageblock.sample_size_B = ungrounded[i];
+            CHECK(!agx3_g17g_graphics_program_generate(chip, &bad,
+                                                        &mrt_output));
+         }
+      }
+
+      const uint32_t original_size = desc.shader_binary_size;
+      desc.shader_binary_size = 512;
+      CHECK(agx3_g17g_graphics_program_generate(chip, &desc, &output));
+      const struct agx3_g17p_graphics_program saturated = output;
+      const uint32_t larger_sizes[] = {513, 790, 1024, 65536, UINT32_MAX};
+      for (unsigned i = 0; i < sizeof(larger_sizes)/sizeof(larger_sizes[0]); ++i) {
+         desc.shader_binary_size = larger_sizes[i];
+         CHECK(agx3_g17g_graphics_program_generate(chip, &desc, &output));
+         CHECK(output.length == saturated.length &&
+               !memcmp(output.bytes, saturated.bytes, output.length));
+      }
+      desc.shader_binary_size = 0;
+      CHECK(!agx3_g17g_graphics_program_generate(chip, &desc, &output));
+      desc.shader_binary_size = original_size;
+
+      if (fragment) {
+         const struct agx3_ioto_uniform_range pointer = {
+            .start_halfs = 0, .size_halfs = 6, .address = 0x4000001000,
+         };
+         const uint8_t copies[] = {
+            0x0b,0x04,0x09,0x24, 0x1b,0x06,0x09,0x04,
+            0x2b,0x08,0x09,0x04, 0x0e,0x00,0x00,0x00,
+         };
+         struct agx3_g17p_graphics_program_desc pointer_desc = desc;
+         pointer_desc.uniform_ranges = &pointer;
+         pointer_desc.uniform_range_count = 1;
+         CHECK(agx3_g17g_tile_program_generate(chip, &pointer_desc, true, &output));
+         CHECK(output.length >= sizeof(copies));
+         CHECK(!memcmp(output.bytes + output.length - sizeof(copies),
+                       copies, sizeof(copies)));
+      }
+
+      struct agx3_ioto_uniform_range ranges[] = {
+         {.start_halfs = 0, .size_halfs = 4, .address = 0x4000001000},
+         {.start_halfs = 4, .size_halfs = 2, .address = 0x4000002000},
+      };
+      desc.uniform_ranges = ranges;
+      desc.uniform_range_count = 2;
+      size_t bound;
+      CHECK(agx3_g17g_graphics_program_bound(chip, ranges, 2, &bound));
+      CHECK(agx3_g17g_graphics_program_generate(chip, &desc,
+                                                 &output));
+      CHECK(output.length > 90 && output.length <= bound);
+      struct agx3_g17p_graphics_program baseline = output;
+      for (unsigned i = 0; i < 2; ++i) {
+         ranges[i].address += 64;
+         CHECK(agx3_g17g_graphics_program_generate(chip, &desc,
+                                                    &output));
+         CHECK(output.length == baseline.length &&
+               memcmp(output.bytes, baseline.bytes, output.length) != 0);
+         ranges[i].address -= 64;
+      }
+
+      baseline = output;
+      desc.scratch_size = UINT16_MAX * 4u + 1;
+      CHECK(!agx3_g17g_graphics_program_generate(chip, &desc, &output));
+      desc.scratch_size = 0;
+      desc.compiler_d_valid = false;
+      CHECK(!agx3_g17g_graphics_program_generate(chip, &desc, &output));
+      desc.compiler_d_valid = true;
+      desc.shader_va++;
+      CHECK(!agx3_g17g_graphics_program_generate(chip, &desc, &output));
+      desc.shader_va--;
+      CHECK(!agx3_g17g_graphics_program_generate(AGX_CHIP_G17P, &desc, &output));
+      CHECK(memcmp(&output, &baseline, sizeof(output)) == 0);
+      if (fragment) {
+         static const uint8_t bg_end[] = {7, 0x10, 0, 4, 0, 0, 14, 0, 0, 0};
+         static const uint8_t eot_end[] = {14, 0, 0, 0};
+         CHECK(agx3_g17g_tile_program_generate(chip, &desc, false,
+                                               &output));
+         CHECK(output.length == baseline.length);
+         CHECK(memcmp(output.bytes + output.length - sizeof(bg_end),
+                      bg_end, sizeof(bg_end)) == 0);
+         CHECK(agx3_g17g_tile_program_generate(chip, &desc, true,
+                                               &output));
+         CHECK(output.length + 6 == baseline.length);
+         CHECK(memcmp(output.bytes + output.length - sizeof(eot_end),
+                      eot_end, sizeof(eot_end)) == 0);
+      } else {
+         CHECK(!agx3_g17g_tile_program_generate(chip, &desc, false,
+                                                &output));
+
+         desc.uniform_ranges = NULL;
+         desc.uniform_range_count = 0;
+         desc.shader_va = UINT64_C(0x100000003c0);
+         desc.shader_binary_size = 64;
+         desc.compiler_d = 0;
+         CHECK(agx3_g17g_compute_program_generate(chip, &desc, &output));
+         CHECK(output.length == 84);
+         CHECK(!memcmp(output.bytes, expected_tail[0], 8));
+         CHECK(!memcmp(output.bytes + 8, expected_tail[1] + 8, 20));
+         CHECK(output.bytes[28 + 20] == (chip == AGX_CHIP_G17S ? 8 : 4));
+         static const uint8_t end[] = {14, 0, 0, 0};
+         CHECK(!memcmp(output.bytes + 80, end, sizeof(end)));
+
+         const struct {
+            unsigned bytes;
+            uint8_t spectpr[10];
+         } staging[] = {
+            {256, {0x07,0x04,0x00,0x0d,0,0,0,0,0,0}},
+            {384, {0x07,0x06,0x00,0x0d,0,0,0,0,0,0}},
+            {512, {0x07,0x08,0x00,0x0d,0,0,0,0,0,0}},
+         };
+         for (unsigned i = 0; i < ARRAY_SIZE(staging); ++i) {
+            desc.compiler_d = staging[i].bytes;
+            CHECK(agx3_g17g_compute_program_generate(chip, &desc, &output));
+            CHECK(output.length == 84);
+            CHECK(!memcmp(output.bytes + 18, staging[i].spectpr, 10));
+         }
+         desc.compiler_d = 513;
+         CHECK(!agx3_g17g_compute_program_generate(chip, &desc, &output));
+         desc.compiler_d = 0;
+
+         const struct {
+            uint32_t size;
+            uint8_t speclm[8];
+         } shared[] = {
+            {256,   {0x0f,0x20,0x00,0x0c,0,0,0,0}},
+            {512,   {0x0f,0x40,0x00,0x0c,0,0,0,0}},
+            {1024,  {0x0f,0x80,0x00,0x0c,0,0,0,0}},
+            {4096,  {0x0f,0x00,0x10,0x0c,0,0,0,0}},
+            {16384, {0x0f,0x00,0x40,0x0c,0,0,0,0}},
+            {32768, {0x0f,0x00,0x80,0x0c,0,0,0,0}},
+         };
+         for (unsigned i = 0; i < ARRAY_SIZE(shared); ++i) {
+            desc.threadgroup_memory_size_B = shared[i].size;
+            CHECK(agx3_g17g_compute_program_generate(chip, &desc, &output));
+            CHECK(output.length == 84);
+            CHECK(!memcmp(output.bytes, shared[i].speclm, 8));
+            CHECK(!agx3_g17g_graphics_program_generate(chip, &desc, &output));
+         }
+         const uint32_t invalid_shared[] = {1, 255, 257, 32769, UINT32_MAX};
+         for (unsigned i = 0; i < ARRAY_SIZE(invalid_shared); ++i) {
+            desc.threadgroup_memory_size_B = invalid_shared[i];
+            CHECK(!agx3_g17g_compute_program_generate(chip, &desc, &output));
+         }
+         desc.threadgroup_memory_size_B = 0;
+         desc.fragment = true;
+         CHECK(!agx3_g17g_compute_program_generate(chip, &desc, &output));
+      }
+   }
+   return 0;
+}
+
+
+static int
+test_m5_dispatch(enum agx_chip chip)
+{
+   struct agx3_g17p_native_compute_program_desc desc = {
+      .program_va = UINT64_C(0x10000090000),
+      .shader_va = UINT64_C(0x100000003c0),
+   };
+   CHECK(agx3_cdm_direct_template_sources_compute(false, false, false,
+                                                  &desc.launch_sources));
+   struct agx3_compute_program program;
+   CHECK(agx3_m5_native_compute_program_bind(chip, &desc, &program));
+   struct agx3_compute_direct_grid grid = {
+      .source = AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,
+      .size = {1024,1,1}, .local_size = {64,1,1}, .dimensions = 1,
+   };
+   const uint32_t expected[] = {
+      0, 0x01000000, 0x2400, 0x40000001,
+      1024, 1, 1, 64, 1, 1, 0x60000160,
+   };
+   uint8_t out[sizeof(expected)];
+   struct agx3_compute_dispatch_sink sink = {.data = out, .capacity = sizeof(out)};
+   CHECK(agx3_m5_native_compute_dispatch_append(&program, &grid, 0, &sink));
+   CHECK(sink.length == sizeof(out));
+   CHECK(!memcmp(out, expected, sizeof(out)));
+   const uint32_t local[] = {64,2,1};
+   const uint32_t indirect_expected[] = {
+      0x08080000, 0x01000000, 0x2400, 0x40000001,
+      0x23, 0x45678000, 64, 2, 1, 0x6000157c,
+   };
+   sink.length = 0;
+   CHECK(agx3_m5_native_indirect_dispatch_append(
+      &program, UINT64_C(0x2345678000), local, 5, &sink));
+   CHECK(sink.length == sizeof(indirect_expected));
+   CHECK(!memcmp(out, indirect_expected, sizeof(indirect_expected)));
+   uint8_t saved[sizeof(out)];
+   memcpy(saved, out, sizeof(out));
+   CHECK(!agx3_m5_native_indirect_dispatch_append(
+      &program, UINT64_C(0x2345678001), local, 5, &sink));
+   CHECK(!memcmp(saved, out, sizeof(out)));
+   const uint32_t indirect_local[] = {
+      0x10080000, 0x01000000, 0x2400, 0x40000001,
+      0x23, 0x45678000, 0x60000160,
+   };
+   sink.length = 0;
+   CHECK(agx3_m5_native_indirect_dispatch_append(
+      &program, UINT64_C(0x2345678000), NULL, 0, &sink));
+   CHECK(sink.length == sizeof(indirect_local));
+   CHECK(!memcmp(out, indirect_local, sizeof(indirect_local)));
+   return 0;
+}
+
+/* Host model of libagx_trace_rays_select_launch (libagx/draws.cl): the words
+ * it writes into the single-kick slot from a kernel's template. */
+static void
+select_launch_model(const uint32_t tmpl[11], bool run, uint32_t groups_x,
+                    uint32_t height, uint32_t depth, unsigned log2,
+                    uint32_t out[11])
+{
+   uint32_t word0 = tmpl[0];
+   if (!run || (height < 2 && depth < 2))
+      word0 &= 0xfff1ffffu;
+   out[0] = word0;
+   out[1] = tmpl[1];
+   out[2] = tmpl[2];
+   out[3] = tmpl[3];
+   out[4] = run ? groups_x << log2 : 0;
+   out[5] = run ? height : 0;
+   out[6] = run ? depth : 0;
+   out[7] = run ? tmpl[7] : 0;
+   out[8] = run ? tmpl[8] : 0;
+   out[9] = run ? tmpl[9] : 0;
+   out[10] = tmpl[10];
+}
+
+/* The GPU-written single-kick launch must be byte-identical to the record the
+ * host encodes for the same kernel and grid (hk_dispatch_m5_native). */
+static int
+test_m5_single_kick_launch(enum agx_chip chip)
+{
+   for (unsigned sources = 0; sources < 2; ++sources) {
+      struct agx3_g17p_native_compute_program_desc desc = {
+         .program_va = UINT64_C(0x10000090000),
+         .shader_va = UINT64_C(0x100000003c0),
+      };
+      CHECK(agx3_cdm_direct_template_sources_compute(sources, sources, sources,
+                                                     &desc.launch_sources));
+      struct agx3_compute_program program;
+      CHECK(agx3_m5_native_compute_program_bind(chip, &desc, &program));
+
+      /* The template hk captures: a 3D grid of the kernel's local size. */
+      const struct agx3_compute_direct_grid tmpl_grid = {
+         .source = AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,
+         .size = {32, 2, 2}, .local_size = {32, 1, 1}, .dimensions = 3,
+      };
+      uint32_t tmpl[11];
+      struct agx3_compute_dispatch_sink sink = {
+         .data = (uint8_t *)tmpl, .capacity = sizeof(tmpl)};
+      CHECK(agx3_m5_native_compute_dispatch_append(&program, &tmpl_grid, 0,
+                                                    &sink));
+      CHECK(sink.length == sizeof(tmpl));
+
+      const uint32_t dims[][3] = {
+         {1, 1, 1}, {31, 1, 1}, {32, 1, 1}, {33, 1, 1}, {1920, 1080, 1},
+         {64, 1, 7}, {5, 3, 2}, {2976, 1872, 1},
+      };
+      for (unsigned i = 0; i < ARRAY_SIZE(dims); ++i) {
+         const uint32_t w = dims[i][0], h = dims[i][1], d = dims[i][2];
+         const uint32_t groups_x = (w >> 5) + ((w & 31) ? 1 : 0);
+         uint32_t model[11];
+         select_launch_model(tmpl, true, groups_x, h, d, 5, model);
+
+         /* hk_dispatch_m5_native's direct record for the same launch. */
+         const struct agx3_compute_direct_grid grid = {
+            .source = AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,
+            .size = {groups_x * 32, h, d}, .local_size = {32, 1, 1},
+            .dimensions = d != 1 ? 3 : h != 1 ? 2 : 1,
+         };
+         uint32_t host[11];
+         sink = (struct agx3_compute_dispatch_sink){
+            .data = (uint8_t *)host, .capacity = sizeof(host)};
+         CHECK(agx3_m5_native_compute_dispatch_append(&program, &grid, 0,
+                                                       &sink));
+         CHECK(!memcmp(model, host, sizeof(host)));
+      }
+
+      /* No matching kernel: kernel 0's header, all six sizes zero, 1D bits. */
+      uint32_t empty[11];
+      select_launch_model(tmpl, false, 3, 4, 5, 5, empty);
+      CHECK(empty[4] == 0 && empty[5] == 0 && empty[6] == 0 &&
+            empty[7] == 0 && empty[8] == 0 && empty[9] == 0);
+      CHECK(empty[10] == tmpl[10] && (empty[0] & 0x000e0000u) == 0);
+   }
+   return 0;
+}
+
+static int
+test_color_disabled(void)
+{
+   struct agx_ppp_hal300_graphics_fields fields = {
+      .fragment_program = 0x1000,
+      .line_width = 15,
+   };
+   struct agx_ppp_hal300_graphics_core with_color, without_color;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S,&fields,&with_color));
+   fields.no_color_writes=true;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S,&fields,&without_color));
+   const unsigned offset = with_color.offsets[5] + 16;
+   const uint8_t enabled[] = {0xff,0xff,0x01,0x00};
+   CHECK(memcmp(with_color.bytes+offset,enabled,4)==0);
+   memset(with_color.bytes+offset,0,4);
+   CHECK(memcmp(with_color.bytes,without_color.bytes,sizeof(with_color.bytes))==0);
+   return 0;
+}
+
+
+/* The native M5 driver's visibility-result-mode setter (macOS 26A428)
+ * ORs the mode into bits 15:14 of the fragment control word and inserts
+ * offset / 8 into bits 31:17 of the word 0x14 bytes after it: the last word
+ * of the 0x020000e0 record. Disabled visibility leaves the bytes unchanged. */
+static int
+test_visibility(void)
+{
+   struct agx_ppp_hal300_graphics_fields fields = {
+      .fragment_program = 0x1000,
+   };
+   struct agx_ppp_hal300_graphics_core none, counting, boolean;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &none));
+   fields.visibility_mode = 2;
+   fields.visibility_index = 5;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &counting));
+   fields.visibility_mode = 3;
+   fields.visibility_index = 0x7fff;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &boolean));
+   const unsigned control = none.offsets[4] + 4, query = control + 0x14;
+   uint32_t w;
+   memcpy(&w, counting.bytes + control, 4);
+   CHECK(w == (*(uint32_t *)(none.bytes + control) | 0x8000));
+   memcpy(&w, counting.bytes + query, 4);
+   CHECK(w == 5u << 17);
+   memcpy(&w, boolean.bytes + control, 4);
+   CHECK(w == (*(uint32_t *)(none.bytes + control) | 0xc000));
+   memcpy(&w, boolean.bytes + query, 4);
+   CHECK(w == 0x7fffu << 17);
+   memcpy(counting.bytes + control, none.bytes + control, 4);
+   memset(counting.bytes + query, 0, 4);
+   CHECK(!memcmp(counting.bytes, none.bytes, sizeof(none.bytes)));
+   fields.visibility_mode = 1;
+   CHECK(!agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &none));
+   return 0;
+}
+
+static int
+test_line_width(void)
+{
+   struct agx_ppp_hal300_graphics_fields fields = {
+      .fragment_program = 0x1000,
+   };
+   struct agx_ppp_hal300_graphics_core state;
+   const unsigned widths[] = {0, 15, 31, 127, 255};
+   for (unsigned i = 0; i < ARRAY_SIZE(widths); ++i) {
+      fields.line_width = widths[i];
+      CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+      /* Each face carries the encoded width in bits 8..15. */
+      CHECK(state.bytes[state.offsets[4] + 9] == widths[i]);
+      CHECK(state.bytes[state.offsets[4] + 13] == widths[i]);
+   }
+   struct agx_ppp_hal300_graphics_core saved = state;
+   fields.line_width = 256;
+   CHECK(!agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   CHECK(!memcmp(&saved, &state, sizeof(state)));
+   return 0;
+}
+
+
+static int
+test_polygon_and_primitive_msaa(void)
+{
+   struct agx_ppp_hal300_graphics_fields fields = {
+      .fragment_program = 0x1000,
+      .line_width = 15,
+   };
+   struct agx_ppp_hal300_graphics_core state;
+   for (unsigned mode = 0; mode <= 2; ++mode) {
+      fields.polygon_mode = mode;
+      for (unsigned msaa = 0; msaa <= 1; ++msaa) {
+         fields.primitive_msaa = msaa;
+         CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+         CHECK(((state.bytes[state.offsets[4] + 10] >> 2) & 3) == mode);
+         CHECK(((state.bytes[state.offsets[4] + 14] >> 2) & 3) == mode);
+         CHECK(((state.bytes[state.offsets[7] + 5] >> 7) & 1) == msaa);
+      }
+   }
+   struct agx_ppp_hal300_graphics_core saved = state;
+   fields.polygon_mode = 3;
+   CHECK(!agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   CHECK(!memcmp(&saved, &state, sizeof(state)));
+   return 0;
+}
+
+
+static int
+test_multiround_uniforms(enum agx_chip chip)
+{
+   struct agx3_ioto_uniform_range ranges[] = {
+      { .start_halfs = 0, .size_halfs = 8, .address = 0x4000001000 },
+      { .start_halfs = 8, .size_halfs = 180, .address = 0x5000010000 },
+   };
+   struct agx3_g17p_graphics_program_desc desc = {
+      .program_va = 0x10000100000, .shader_va = 0x10000000480,
+      .shader_binary_size = 128, .nr_gprs = 16,
+      .compiler_d_valid = true, .uniform_ranges = ranges,
+      .uniform_range_count = ARRAY_SIZE(ranges), .patchable = true,
+   };
+   struct agx3_g17p_graphics_program_template first, moved;
+   CHECK(agx3_g17g_compute_program_template_generate(chip, &desc, &first));
+   uint8_t body[AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE];
+   struct agx_ioto_inflight_load loads[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+   struct agx_ioto_inflight_load sort[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+   struct agx_ioto_load_cache_entry cache[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+   const struct agx3_ioto_uniform_program_desc load_desc = {
+      .program_va = desc.program_va, .ranges = ranges,
+      .range_count = ARRAY_SIZE(ranges), .patchable = true,
+   };
+   const struct agx3_ioto_uniform_program_workspace workspace = {
+      .body = body, .body_capacity = sizeof(body),
+      .loads = loads, .load_capacity = ARRAY_SIZE(loads),
+      .sort = sort, .sort_capacity = ARRAY_SIZE(sort),
+      .cache = cache, .cache_capacity = ARRAY_SIZE(cache),
+   };
+   size_t body_size, last_copy, first_copy;
+   CHECK(agx3_ioto_uniform_program_body_generate(chip, &load_desc, workspace,
+                                                 &body_size, &last_copy));
+   CHECK(agx_ioto_program_first_copy_offset(chip, body, body_size, &first_copy));
+   CHECK(first_copy < last_copy);
+
+   struct agx3_g17p_graphics_program_desc empty_desc = desc;
+   empty_desc.uniform_ranges = NULL;
+   empty_desc.uniform_range_count = 0;
+   struct agx3_g17p_graphics_program empty;
+   CHECK(agx3_g17g_compute_program_generate(chip, &empty_desc, &empty));
+   const size_t prefix = AGX_IOTO_STATE_TAIL_TYPE0_SIZE;
+   CHECK(first.program.length == body_size + empty.length);
+   CHECK(!memcmp(first.program.bytes, body, first_copy));
+   CHECK(!memcmp(first.program.bytes + first_copy, empty.bytes, prefix));
+   CHECK(!memcmp(first.program.bytes + first_copy + prefix,
+                 body + first_copy, body_size - first_copy));
+   /* Keep every upload, including later allocation rounds, before the
+    * helper: a native instruction probe detects zero uniforms otherwise. */
+   CHECK(!memcmp(first.program.bytes + body_size + prefix,
+                 empty.bytes + prefix, empty.length - prefix));
+
+   /* Template addresses after the early COPY round must still patch at
+    * their actual positions in the composed instruction stream. */
+   uint8_t patched[AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE];
+   memcpy(patched, first.program.bytes, first.program.length);
+   bool later_relocation = false;
+   for (unsigned i = 0; i < first.relocation_count; ++i) {
+      const struct agx_ioto_load_relocation *rel = &first.relocations[i];
+      uint8_t low[8], high[8];
+      const uint8_t ln = 8, hn = 8;
+      agx_ioto_load_pack_ldimm_long(rel->dest_half, rel->address, low);
+      agx_ioto_load_pack_ldimm_long(rel->dest_half + 2, rel->address >> 32, high);
+      CHECK(ln == rel->low_length && hn == rel->high_length);
+      CHECK(rel->offset + ln + hn <= first.program.length);
+      CHECK(!memcmp(patched + rel->offset, low, ln));
+      CHECK(!memcmp(patched + rel->offset + ln, high, hn));
+      later_relocation |= rel->offset > first_copy;
+      uint64_t address = rel->address + 0x100000;
+      agx_ioto_load_pack_ldimm_long(rel->dest_half, address, low);
+      agx_ioto_load_pack_ldimm_long(rel->dest_half + 2, address >> 32, high);
+      memcpy(patched + rel->offset, low, ln);
+      memcpy(patched + rel->offset + ln, high, hn);
+   }
+   CHECK(later_relocation);
+   for (unsigned i = 0; i < ARRAY_SIZE(ranges); ++i)
+      ranges[i].address += 0x100000;
+   CHECK(agx3_g17g_compute_program_template_generate(chip, &desc, &moved));
+   CHECK(first.program.length == moved.program.length);
+   CHECK(!memcmp(patched, moved.program.bytes, moved.program.length));
+   return 0;
+}
+
+/* Compute loaders that publish a few hundred constant-program inputs (UBO
+ * contents read by a preamble, as in ubo.*_compute) must fit the M5 program
+ * buffer, and the body bound must still cover what the loader writes for
+ * aligned pairs, lone halves and pairs at odd compact positions. */
+static int
+test_cprog_input_bound(enum agx_chip chip)
+{
+   const struct agx3_ioto_uniform_range ranges[] = {
+      { .start_halfs = 0, .size_halfs = 8, .address = 0x4000001000 },
+      { .start_halfs = 8, .size_halfs = 128, .address = 0x5000010000 },
+      { .start_halfs = 136, .size_halfs = 128, .address = 0x5000010240 },
+      { .start_halfs = 264, .size_halfs = 128, .address = 0x5000010480 },
+      { .start_halfs = 392, .size_halfs = 24, .address = 0x5000020000 },
+      { .start_halfs = 416, .size_halfs = 6, .address = 0x4000001098 },
+   };
+   for (unsigned pattern = 0; pattern < 3; ++pattern) {
+      uint64_t halves[AGX3_CPROG_INPUT_WORDS] = {0};
+      for (unsigned h = 8; h < 422; ++h) {
+         const bool selected = pattern == 0 ? true
+                             : pattern == 1 ? (h & 1)
+                             : (h == 9 || h >= 10);
+         if (selected)
+            halves[h / 64] |= UINT64_C(1) << (h % 64);
+      }
+      struct agx3_cprog_inputs inputs;
+      CHECK(agx3_cprog_inputs_init(halves, &inputs));
+      const struct agx3_ioto_uniform_program_desc load_desc = {
+         .program_va = 0x10000100000, .ranges = ranges,
+         .range_count = ARRAY_SIZE(ranges), .patchable = true,
+         .cprog_inputs = &inputs,
+      };
+      size_t bound, chunks;
+      CHECK(agx3_ioto_uniform_program_desc_body_bound(chip, &load_desc,
+                                                      &bound, &chunks));
+      if (pattern == 0)
+         CHECK(bound <= AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE);
+
+      uint8_t body[2 * AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE];
+      struct agx_ioto_inflight_load loads[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+      struct agx_ioto_inflight_load sort[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+      struct agx_ioto_load_cache_entry cache[AGX3_G17P_GRAPHICS_MAX_LOAD_CHUNKS];
+      const struct agx3_ioto_uniform_program_workspace workspace = {
+         .body = body, .body_capacity = sizeof(body),
+         .loads = loads, .load_capacity = ARRAY_SIZE(loads),
+         .sort = sort, .sort_capacity = ARRAY_SIZE(sort),
+         .cache = cache, .cache_capacity = ARRAY_SIZE(cache),
+      };
+      size_t body_size, copy_offset;
+      CHECK(agx3_ioto_uniform_program_body_generate(chip, &load_desc, workspace,
+                                                    &body_size, &copy_offset));
+      CHECK(body_size <= bound);
+   }
+   return 0;
+}
+
+static int
+test_load_completion_wait(enum agx_chip chip)
+{
+   for (unsigned words = 1; words <= 4; ++words) {
+      uint8_t body[512];
+      struct agx_ioto_inflight_load loads[4], sorted[4];
+      struct agx_ioto_load_cache_entry cache[4];
+      struct agx_ioto_load_core core;
+      CHECK(agx_ioto_load_core_init(&core, chip,
+         (struct agx_ioto_load_sink){body, sizeof(body), 0},
+         loads, ARRAY_SIZE(loads), cache, ARRAY_SIZE(cache), 0, false, false));
+      struct agx_ioto_load_absolute load = {
+         .native_slot = 8, .count_dwords = words,
+         .address = 0x1234567800ull,
+         .lane_mask = (1u << (2 * words)) - 1,
+      };
+      struct agx_ioto_load_allocation allocation;
+      struct agx_ioto_load_workspace workspace = {sorted, ARRAY_SIZE(sorted)};
+      CHECK(agx3_ioto_load_absolute_native(&core, &load, workspace, &allocation));
+      CHECK(agx3_ioto_load_finish_round_native(&core, true, workspace));
+      /* Physical load slot 1 is wait bit zero, independently of length. */
+      const struct agx3_field_kv fields[] = {
+         {9, 3}, {10, 2}, {23, 2}, {24, 2}, {27, 1}, {28, 1}, {29, 35},
+         {3, 16}, {16, 4}, {30, 1},
+      };
+      uint8_t expected[AGX_IOTO_MAX_RECORD_SIZE], length;
+      CHECK(agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "copy", 0,
+         fields, ARRAY_SIZE(fields), NULL, 0, expected, &length) ||
+            agx3_bind_compose(AGX_ISA_ENCODING_AGX3C, "copy", 1,
+         fields, ARRAY_SIZE(fields), NULL, 0, expected, &length));
+      CHECK(core.copy_suffix_offset + length <= core.sink.length);
+      CHECK(!memcmp(body + core.copy_suffix_offset, expected, length));
+   }
+   return 0;
+}
+
+static int
+test_unused_vertex_outputs(void)
+{
+   struct agx_ppp_hal300_graphics_fields fields = {
+      .fragment_program = 0x1000,
+      .line_width = 15,
+      .vertex_varying_components = 9,
+      .clip_distance_words = 8,
+      .layer_output = true,
+   };
+   struct agx_ppp_hal300_graphics_core state;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   const uint8_t words22[] = {22, 0, 0, 0};
+   CHECK(!memcmp(state.bytes + state.offsets[3] + 16, words22, 4));
+   fields.vertex_varying_components = 115;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   const uint8_t words128[] = {128, 0, 0, 0};
+   CHECK(!memcmp(state.bytes + state.offsets[3] + 16, words128, 4));
+   fields.vertex_varying_components = 116;
+   CHECK(!agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   fields.vertex_varying_components = 9;
+   fields.fragment_varying_components = 1;
+   fields.cf_binding_count = 1;
+   fields.cf_register_count = 1;
+   fields.varying_table = 0x2000;
+   CHECK(!agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   fields.vertex_varying_components = 8;
+   CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   return 0;
+}
+
+/* Native mesh draws select their provoking vertex in the PPP cull word's
+ * flat-shading field ((vertex + 1) << 7); vertex-stage draws keep 0 there
+ * and select it through the VDM vertex program template. */
+static int
+test_flat_shading_vertex(void)
+{
+   struct agx_ppp_hal300_graphics_fields fields = {
+      .fragment_program = 0x1000,
+      .line_width = 15,
+   };
+   struct agx_ppp_hal300_graphics_core state;
+   for (unsigned vertex = 0; vertex <= 2; ++vertex) {
+      fields.flat_shading_vertex = vertex;
+      CHECK(agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+      uint32_t cull;
+      memcpy(&cull, state.bytes + state.offsets[7] + 4, 4);
+      CHECK(((cull >> 7) & 3) == vertex + 1);
+   }
+   fields.flat_shading_vertex = 3;
+   CHECK(!agx_ppp_hal300_graphics_core_pack(AGX_CHIP_G17S, &fields, &state));
+   return 0;
+}
+
+int main(void) {
+   int r = test_unused_vertex_outputs() || test_flat_shading_vertex() || test_load_completion_wait(AGX_CHIP_G17G) ||
+           test_load_completion_wait(AGX_CHIP_G17S) ||
+           test_multiround_uniforms(AGX_CHIP_G17G) ||
+           test_multiround_uniforms(AGX_CHIP_G17S) ||
+           test_cprog_input_bound(AGX_CHIP_G17G) ||
+           test_cprog_input_bound(AGX_CHIP_G17S) ||
+           test_color_disabled() || test_visibility() || test_line_width() || test_polygon_and_primitive_msaa() || test_g17g_programs(AGX_CHIP_G17G) || test_g17g_programs(AGX_CHIP_G17S) ||
+           test_m5_dispatch(AGX_CHIP_G17G) || test_m5_dispatch(AGX_CHIP_G17S) ||
+           test_m5_single_kick_launch(AGX_CHIP_G17G) ||
+           test_m5_single_kick_launch(AGX_CHIP_G17S);
+   if (!r) puts("M5_PROGRAM_DISPATCH_PASS");
+   return r;
+}
diff --git a/src/asahi/compiler/test/test-agx3-scs.cpp b/src/asahi/compiler/test/test-agx3-scs.cpp
index f9cde6ddba33..34e530569da7 100644
--- a/src/asahi/compiler/test/test-agx3-scs.cpp
+++ b/src/asahi/compiler/test/test-agx3-scs.cpp
@@ -26,7 +26,7 @@ TEST(SCSRegisterIR, NeoScalarAndVector)
       result.channels_m1 = util_last_bit(mask) - 1;
       agx_scs_load_register_indexed_agx3_to(
          b, result, agx_register(32, AGX_SIZE_32),
-         agx_register(28, AGX_SIZE_32), 512, AGX_FORMAT_I32, mask, 3, false);
+         agx_register(28, AGX_SIZE_32), 512, AGX_FORMAT_I32, mask, 3, false, false);
       agx_stop(b);
       util_dynarray binary = UTIL_DYNARRAY_INIT;
       ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
diff --git a/src/asahi/compiler/test/test-agx3-sr.cpp b/src/asahi/compiler/test/test-agx3-sr.cpp
index 21712bf3c4bc..82590c13b100 100644
--- a/src/asahi/compiler/test/test-agx3-sr.cpp
+++ b/src/asahi/compiler/test/test-agx3-sr.cpp
@@ -105,10 +105,19 @@ TEST(AGX3SrCore, CoverageEntryMarker)
    ASSERT_EQ(len, sizeof(later));
    EXPECT_EQ(0, memcmp(out, later, len));
 
-   /* The marker is witnessed on nothing else. */
+   /* Native G17G 2x/4x/8x constant-colour fragments all read marked SR39
+    * into h2 at offset 64. Its extended RLD encoding differs from G17P. */
+   static const uint8_t m5_h2[] = {0x34, 0x82, 0x01, 0x00,
+                                   0x20, 0x00, 0xb0, 0x02};
+   d.dest = 2;
    d.first = true;
-   EXPECT_FALSE(agx3_sr_encode(AGX_ISA_ENCODING_AGX3C, &d, out, &len));
+   ASSERT_TRUE(agx3_sr_encode(AGX_ISA_ENCODING_AGX3C, &d, out, &len));
+   ASSERT_EQ(len, sizeof(m5_h2));
+   EXPECT_EQ(0, memcmp(out, m5_h2, len));
+
+   /* Other system registers still reject the coverage entry marker. */
    d.sr = 41;
+   EXPECT_FALSE(agx3_sr_encode(AGX_ISA_ENCODING_AGX3C, &d, out, &len));
    EXPECT_FALSE(agx3_sr_encode(AGX_ISA_ENCODING_AGX3B, &d, out, &len));
 }
 
diff --git a/src/asahi/compiler/test/test-agx3-uniform-alu.cpp b/src/asahi/compiler/test/test-agx3-uniform-alu.cpp
index 564233f9135d..6ecc58f95672 100644
--- a/src/asahi/compiler/test/test-agx3-uniform-alu.cpp
+++ b/src/asahi/compiler/test/test-agx3-uniform-alu.cpp
@@ -1,19 +1,577 @@
 /* SPDX-License-Identifier: MIT */
 #include <array>
 #include <cstring>
+#include <cstdlib>
 #include <set>
 #include <string>
+#include <vector>
 #include <gtest/gtest.h>
 #include "agx3_binder.h"
+#include "agx3_alu_core.h"
+#include "agx3_cf_core.h"
 #include "agx3_uniform_alu.h"
+#include "agx3_stage_core.h"
+#include "agx3_isel_core.h"
 #include "agx3_tex_core.h"
 #include "agx_test.h"
 #include "util/u_debug.h"
 
+TEST(AGX3UniformALU, PackedUniformLoadPointerBoundaries)
+{
+   const bool enabled =
+      debug_get_bool_option("ASAHI_M5_HIGH_UNIFORM_LOAD", true);
+   struct test_case {
+      enum agx_isa_encoding encoding;
+      unsigned low, high;
+      bool modified, admitted;
+   };
+   const test_case cases[] = {
+      {AGX_ISA_ENCODING_AGX3C, 252, 254, false, true},
+      {AGX_ISA_ENCODING_AGX3C, 256, 258, false, true},
+      {AGX_ISA_ENCODING_AGX3C, 448, 450, false, true},
+      {AGX_ISA_ENCODING_AGX3C, 476, 478, false, true},
+      {AGX_ISA_ENCODING_AGX3C, 508, 510, false, true},
+      {AGX_ISA_ENCODING_AGX3C, 512, 514, false, false},
+      {AGX_ISA_ENCODING_AGX3C, 250, 252, false, false},
+      {AGX_ISA_ENCODING_AGX3C, 448, 452, false, false},
+      {AGX_ISA_ENCODING_AGX3C, 450, 448, false, false},
+      {AGX_ISA_ENCODING_AGX3C, 448, 450, true, false},
+      {AGX_ISA_ENCODING_AGX3B, 448, 450, false, false},
+   };
+   for (const auto &t : cases) {
+      void *mem = ralloc_context(NULL);
+      agx_builder *b = agx_test_builder(mem);
+      agx_shader_key key = {};
+      key.dev.encoding = t.encoding;
+      b->shader->key = &key;
+      b->shader->stage = MESA_SHADER_FRAGMENT;
+      agx_index pointer = agx_temp(b->shader, AGX_SIZE_64);
+      agx_instr *collect = agx_collect_to(b, pointer, 2);
+      /* Include out-of-domain indices to test rejection by the optimizer.
+       * The checked production constructor correctly rejects those inputs. */
+      collect->src[0] = agx_uniform(0, AGX_SIZE_32);
+      collect->src[0].value = t.low;
+      collect->src[0].neg = t.modified;
+      collect->src[1] = agx_uniform(0, AGX_SIZE_32);
+      collect->src[1].value = t.high;
+      agx_instr *load = agx_device_load_to(
+         b, agx_temp(b->shader, AGX_SIZE_32), pointer, agx_immediate(0),
+         AGX_FORMAT_I32, 1, 0, false);
+      agx_optimizer_forward(b->shader);
+      if (enabled && t.admitted) {
+         EXPECT_EQ(load->src[0].type, AGX_INDEX_UNIFORM);
+         EXPECT_EQ(load->src[0].value, t.low);
+         EXPECT_EQ(load->src[0].size, AGX_SIZE_64);
+      } else {
+         EXPECT_EQ(load->src[0].type, AGX_INDEX_NORMAL);
+         EXPECT_EQ(load->src[0].value, pointer.value);
+      }
+      ralloc_free(mem);
+   }
+   EXPECT_FALSE(agx_instr_accepts_uniform(AGX_ISA_ENCODING_AGX3C,
+      AGX_OPCODE_DEVICE_STORE, 1, 448, AGX_SIZE_64));
+   EXPECT_FALSE(agx_instr_accepts_uniform(AGX_ISA_ENCODING_AGX3C,
+      AGX_OPCODE_ATOMIC, 1, 448, AGX_SIZE_64));
+}
+
+static uint32_t
+uniform_difference_zero_bits(bool relax)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_FRAGMENT;
+   agx_index input = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index multiplier = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index difference = agx_temp(b->shader, AGX_SIZE_32);
+   agx_instr *sub = agx_fadd_to(b, difference, agx_uniform(12, AGX_SIZE_32),
+                               agx_neg(input));
+   sub->no_signed_zero = relax;
+   agx_index product = agx_fmul(b, difference, multiplier);
+   agx_lower_uniform_sources(b->shader);
+
+   std::vector<float> values(b->shader->alloc, 0.0f);
+   values[input.value] = values[multiplier.value] = 1.0f;
+   auto read = [&](agx_index x) {
+      float f = x.type == AGX_INDEX_UNIFORM ? 1.0f : values[x.value];
+      uint32_t bits;
+      std::memcpy(&bits, &f, sizeof(bits));
+      if (x.abs) bits &= 0x7fffffff;
+      if (x.neg) bits ^= 0x80000000;
+      std::memcpy(&f, &bits, sizeof(f));
+      return f;
+   };
+   agx_foreach_instr_global(b->shader, I) {
+      volatile float a = read(I->src[0]);
+      volatile float result;
+      if (I->op == AGX_OPCODE_MOV) {
+         result = a;
+      } else {
+         volatile float c = read(I->src[1]);
+         assert(I->op == AGX_OPCODE_FADD || I->op == AGX_OPCODE_FMUL);
+         result = I->op == AGX_OPCODE_FADD ? a + c : a * c;
+      }
+      values[I->dest[0].value] = result;
+   }
+   uint32_t bits;
+   std::memcpy(&bits, &values[product.value], sizeof(bits));
+   ralloc_free(mem);
+   return bits;
+}
+
+/* u0 - u1 on M5: copying u0 leaves the native R + -U addition, so u1 stays a
+ * uniform operand instead of being copied as well. */
+TEST(AGX3UniformALU, M5UniformDifferenceCopiesOneUniform)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_FRAGMENT;
+   agx_instr *sub = agx_fadd_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                agx_uniform(12, AGX_SIZE_32),
+                                agx_neg(agx_uniform(14, AGX_SIZE_32)));
+   agx_unit_test(b, sub->dest[0]);
+   agx_lower_uniform_sources(b->shader);
+
+   unsigned copies = 0;
+   agx_foreach_instr_global(b->shader, I) {
+      if (I->op == AGX_OPCODE_MOV && I->src[0].type == AGX_INDEX_UNIFORM) {
+         EXPECT_EQ(I->src[0].value, 12u);
+         ++copies;
+      }
+   }
+   EXPECT_EQ(copies, 1u);
+   EXPECT_EQ(sub->src[0].type, AGX_INDEX_NORMAL);
+   EXPECT_FALSE(sub->src[0].neg);
+   EXPECT_EQ(sub->src[1].type, AGX_INDEX_UNIFORM);
+   EXPECT_EQ(sub->src[1].value, 14u);
+   EXPECT_TRUE(sub->src[1].neg);
+   ralloc_free(mem);
+}
+
+TEST(AGX3UniformALU, UniformDifferencePreservesPositiveZero)
+{
+   EXPECT_EQ(uniform_difference_zero_bits(false), 0u);
+}
+
+TEST(AGX3UniformALU, UniformDifferenceMayNegateRelaxedZero)
+{
+   EXPECT_EQ(uniform_difference_zero_bits(true), 0x80000000u);
+}
+
+TEST(AGX3UniformALU, CSEKeepsDifferentSignedZeroContracts)
+{
+   void *mem = ralloc_context(NULL);
+   agx_builder *b = agx_test_builder(mem);
+   agx_index input = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index strict = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index relaxed = agx_temp(b->shader, AGX_SIZE_32);
+   agx_fadd_to(b, strict, agx_uniform(12, AGX_SIZE_32), agx_neg(input));
+   agx_fadd_to(b, relaxed, agx_uniform(12, AGX_SIZE_32),
+               agx_neg(input))->no_signed_zero = true;
+   agx_instr *use = agx_fmul_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                               strict, relaxed);
+   agx_opt_cse(b->shader);
+   EXPECT_NE(use->src[0].value, use->src[1].value);
+   ralloc_free(mem);
+}
+
+TEST(AGX3FetchBanks, MultisampleRetirementPreservesPendingR0)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   b->shader->max_reg = 95;
+   const unsigned destinations[] = {8, 0, 16, 24, 32};
+   for (unsigned i = 0; i < 5; ++i) {
+      agx_index coords = agx_register(64 + 6 * i, AGX_SIZE_32);
+      coords.channels_m1 = 2;
+      agx_index dest = agx_register(destinations[i], AGX_SIZE_32);
+      dest.channels_m1 = 3;
+      agx_texture_load_to(b, dest, coords, agx_immediate(0), agx_immediate(0),
+                         agx_immediate(i), agx_immediate(0), agx_null(),
+                         AGX_DIM_2D_MS, AGX_LOD_MODE_AUTO_LOD, 15, false, false);
+   }
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   /* Non-array multisample reads take eight operand banks by default; the
+    * fifth read recycles a bank only with the four-bank override. */
+   setenv("ASAHI_M5_MS_FETCH_BANKS", "4", 1);
+   const bool packed = agx3_pack_binary(b->shader, &binary);
+   unsetenv("ASAHI_M5_MS_FETCH_BANKS");
+   ASSERT_TRUE(packed);
+   const uint8_t *bytes = static_cast<const uint8_t *>(binary.data);
+   for (unsigned carrier : {0u, 88u}) {
+      uint8_t expected[AGX3_ALU_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_alu_band_identity_wait(AGX_ISA_ENCODING_AGX3C,
+                                            carrier, 1, expected, &length));
+      unsigned matches = 0;
+      for (unsigned i = 0; i + length <= binary.size; ++i)
+         matches += std::memcmp(bytes + i, expected, length) == 0;
+      EXPECT_EQ(matches, carrier == 88 ? 1u : 0u);
+   }
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+}
+
+static agx_instr *
+split_test_fetch(agx_builder *b, agx_index dest, agx_index coords,
+                 enum agx_dim dim, agx_index lod = agx_immediate(0))
+{
+   const bool ms = dim == AGX_DIM_2D_MS;
+   return agx_texture_load_to(
+      b, dest, coords, lod, agx_immediate(0), agx_immediate(0),
+      agx_immediate(0), agx_null(), dim,
+      ms ? AGX_LOD_MODE_AUTO_LOD : AGX_LOD_MODE_LOD_MIN, 15, false, false);
+}
+
+/* Rebuild a fetch with split coordinates, as agx3_split_texture_coordinates()
+ * does, from physical registers. */
+static void
+split_test_registers(agx_builder *b, agx_instr *I, unsigned x, unsigned y,
+                     unsigned sample)
+{
+   agx_instr *fetch = agx_alloc_instr(b, I->op, 1, 8);
+   memcpy(&fetch->op, &I->op, sizeof(*I) - offsetof(agx_instr, op));
+   fetch->nr_srcs = 8;
+   fetch->dest[0] = I->dest[0];
+   memcpy(fetch->src, I->src, 6 * sizeof(agx_index));
+   fetch->src[0] = agx_register(x, AGX_SIZE_32);
+   fetch->src[6] = agx_register(y, AGX_SIZE_32);
+   fetch->src[7] = I->dim != AGX_DIM_2D ? agx_register(sample, AGX_SIZE_32)
+                                        : agx_null();
+   fetch->agx3_split_coords = true;
+   agx_cursor cursor = agx_before_instr(I);
+   agx_builder_insert(&cursor, fetch);
+   agx_remove_instruction(I);
+   b->cursor = agx_after_instr(fetch);
+}
+
+static std::vector<uint8_t>
+split_test_pack(enum agx_dim dim, int split_x, unsigned y, unsigned sample,
+                bool register_lod = false)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   b->shader->max_reg = 95;
+   agx_index coords = agx_register(20, AGX_SIZE_32);
+   coords.channels_m1 = dim == AGX_DIM_2D ? 1 : 2;
+   agx_index dest = agx_register(0, AGX_SIZE_32);
+   dest.channels_m1 = 3;
+   agx_instr *I = split_test_fetch(
+      b, dest, coords, dim,
+      register_lod ? agx_register(100, AGX_SIZE_32) : agx_immediate(0));
+   if (split_x >= 0)
+      split_test_registers(b, I, split_x, y, sample);
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   std::vector<uint8_t> out;
+   if (agx3_pack_binary(b->shader, &binary)) {
+      const uint8_t *data = static_cast<const uint8_t *>(binary.data);
+      out.assign(data, data + binary.size);
+   }
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+   return out;
+}
+
+/* Split coordinates in the registers a contiguous vector would occupy stage
+ * exactly like the vector, and scattered ones still encode. */
+TEST(AGX3FetchBanks, SplitCoordinatesStageLikeVector)
+{
+   for (enum agx_dim dim : {AGX_DIM_2D, AGX_DIM_2D_MS, AGX_DIM_2D_ARRAY}) {
+      SCOPED_TRACE(dim);
+      const std::vector<uint8_t> vector = split_test_pack(dim, -1, 0, 0);
+      ASSERT_FALSE(vector.empty());
+      EXPECT_EQ(split_test_pack(dim, 20, 22, 24), vector);
+      const std::vector<uint8_t> scattered = split_test_pack(dim, 40, 20, 60);
+      ASSERT_FALSE(scattered.empty());
+      EXPECT_EQ(scattered.size(), vector.size());
+      EXPECT_NE(scattered, vector);
+      /* The same register for two components still encodes. */
+      EXPECT_FALSE(split_test_pack(dim, 40, 40, 40).empty());
+      /* x and y may live above the seven-bit COPY range; the sample index
+       * or layer may not. */
+      EXPECT_FALSE(split_test_pack(dim, 200, 150, 60).empty());
+      if (dim != AGX_DIM_2D) {
+         EXPECT_TRUE(split_test_pack(dim, 20, 22, 200).empty());
+      }
+   }
+   /* A register level stages beside the layer (2D array) or after the
+    * coordinates (2D). */
+   for (enum agx_dim dim : {AGX_DIM_2D, AGX_DIM_2D_ARRAY}) {
+      SCOPED_TRACE(dim);
+      const std::vector<uint8_t> vector = split_test_pack(dim, -1, 0, 0, true);
+      ASSERT_FALSE(vector.empty());
+      EXPECT_EQ(split_test_pack(dim, 20, 22, 24, true), vector);
+      EXPECT_FALSE(split_test_pack(dim, 40, 20, 60, true).empty());
+      EXPECT_FALSE(split_test_pack(dim, 60, 20, 60, true).empty());
+   }
+}
+
+TEST(AGX3FetchBanks, SplitCoordinatesReplaceCollects)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   agx_index x = agx_mov_imm(b, 32, 5), y = agx_mov_imm(b, 32, 7);
+   agx_index samples[2] = {agx_mov_imm(b, 32, 0), agx_mov_imm(b, 32, 1)};
+   for (unsigned i = 0; i < 2; ++i) {
+      agx_index coords = agx_vec_temp(b->shader, AGX_SIZE_32, 3);
+      agx_instr *collect = agx_collect_to(b, coords, 3);
+      collect->src[0] = x;
+      collect->src[1] = y;
+      collect->src[2] = samples[i];
+      split_test_fetch(b, agx_vec_temp(b->shader, AGX_SIZE_32, 4), coords,
+                       AGX_DIM_2D_MS);
+   }
+   /* A uniform component cannot be staged in place: keep the vector. */
+   agx_index uniform_coords = agx_vec_temp(b->shader, AGX_SIZE_32, 2);
+   agx_instr *collect = agx_collect_to(b, uniform_coords, 2);
+   collect->src[0] = x;
+   collect->src[1] = agx_uniform(8, AGX_SIZE_32);
+   split_test_fetch(b, agx_vec_temp(b->shader, AGX_SIZE_32, 4), uniform_coords,
+                    AGX_DIM_2D);
+
+   EXPECT_TRUE(agx3_split_texture_coordinates(b->shader));
+   unsigned split = 0, vector = 0;
+   agx_foreach_instr_global(b->shader, I) {
+      if (I->op != AGX_OPCODE_TEXTURE_LOAD)
+         continue;
+      if (!I->agx3_split_coords) {
+         EXPECT_EQ(I->nr_srcs, 6);
+         EXPECT_EQ(I->src[0].value, uniform_coords.value);
+         ++vector;
+         continue;
+      }
+      ASSERT_EQ(I->nr_srcs, 8);
+      EXPECT_EQ(I->src[0].value, x.value);
+      EXPECT_EQ(I->src[6].value, y.value);
+      EXPECT_EQ(I->src[7].value, samples[split].value);
+      EXPECT_EQ(I->src[3].type, AGX_INDEX_IMMEDIATE);
+      ++split;
+   }
+   EXPECT_EQ(split, 2u);
+   EXPECT_EQ(vector, 1u);
+
+   /* Other encodings keep their coordinate vectors. */
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3B;
+   EXPECT_FALSE(agx3_split_texture_coordinates(b->shader));
+   ralloc_free(mem);
+}
+
+static agx_instr *
+split_test_sample(agx_builder *b, agx_index dest, agx_index coords,
+                  agx_index reference)
+{
+   const bool shadow = !agx_is_null(reference);
+   return agx_texture_sample_to(
+      b, dest, coords, agx_immediate(0), agx_immediate(0), agx_immediate(0),
+      agx_immediate(0), reference, AGX_DIM_2D, AGX_LOD_MODE_LOD_MIN,
+      shadow ? 1 : 15, false, shadow, false, AGX_GATHER_NONE, false);
+}
+
+/* A compute 2D sample at level zero, with vector coordinates in r10..r11 or
+ * split ones in r<x>/r<y> (word numbers), and a comparison reference in
+ * r<reference> when that is not negative. */
+static std::vector<uint8_t>
+split_test_pack_sample(int split_x, unsigned y, int reference,
+                       bool discard = false)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   b->shader->max_reg = 127;
+   agx_index coords = agx_register(20, AGX_SIZE_32);
+   coords.channels_m1 = 1;
+   agx_index dest = agx_register(0, AGX_SIZE_32);
+   dest.channels_m1 = 3;
+   agx_instr *I = split_test_sample(
+      b, dest, coords,
+      reference >= 0 ? agx_register(2 * reference, AGX_SIZE_32) : agx_null());
+   if (split_x >= 0) {
+      split_test_registers(b, I, 2 * split_x, 2 * y, 0);
+      agx_foreach_instr_global(b->shader, split) {
+         if (split->agx3_split_coords)
+            split->src[0].discard = split->src[6].discard = discard;
+      }
+   }
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   std::vector<uint8_t> out;
+   if (agx3_pack_binary(b->shader, &binary)) {
+      const uint8_t *data = static_cast<const uint8_t *>(binary.data);
+      out.assign(data, data + binary.size);
+   }
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+   return out;
+}
+
+/* Split 2D sample coordinates stage byte for byte like the vector in the same
+ * registers, with or without a comparison reference; scattered and high
+ * coordinates still encode. */
+TEST(AGX3FetchBanks, SplitSampleCoordinatesStageLikeVector)
+{
+   for (int reference : {-1, 30}) {
+      SCOPED_TRACE(reference);
+      const std::vector<uint8_t> vector =
+         split_test_pack_sample(-1, 0, reference);
+      ASSERT_FALSE(vector.empty());
+      EXPECT_EQ(split_test_pack_sample(10, 11, reference), vector);
+      const std::vector<uint8_t> scattered =
+         split_test_pack_sample(21, 9, reference);
+      ASSERT_FALSE(scattered.empty());
+      EXPECT_EQ(scattered.size(), vector.size());
+      EXPECT_NE(scattered, vector);
+      EXPECT_FALSE(split_test_pack_sample(21, 21, reference).empty());
+      /* x and y may live above the seven-bit COPY range. */
+      EXPECT_FALSE(split_test_pack_sample(100, 75, reference).empty());
+   }
+   /* A coordinate in the reference's register is refused, not staged after
+    * the reference may have discarded it. */
+   EXPECT_TRUE(split_test_pack_sample(30, 11, 30).empty());
+}
+
+/* x and y in one register: only the second COPY may discard it. */
+TEST(AGX3FetchBanks, SplitSampleSharedRegisterDiscardsOnce)
+{
+   const std::vector<uint8_t> binary = split_test_pack_sample(21, 21, -1, true);
+   ASSERT_FALSE(binary.empty());
+   bool found = false;
+   for (unsigned slot = 1; slot <= 8 && !found; ++slot) {
+      for (unsigned bank = 0; bank < 64 && !found; bank += 4) {
+         uint8_t x[AGX3_TEX_COPY_MAX_LENGTH], y[AGX3_TEX_COPY_MAX_LENGTH];
+         uint8_t x_len = 0, y_len = 0;
+         ASSERT_TRUE(agx3_tex_stage_copy_scalar_m5(bank, 42, slot, 0, true,
+                                                   false, x, &x_len));
+         ASSERT_TRUE(agx3_tex_stage_copy_scalar_m5(bank + 2, 42, slot, 0,
+                                                   false, true, y, &y_len));
+         for (unsigned i = 0; i + x_len + y_len <= binary.size(); ++i) {
+            found |= std::memcmp(binary.data() + i, x, x_len) == 0 &&
+                     std::memcmp(binary.data() + i + x_len, y, y_len) == 0;
+         }
+      }
+   }
+   EXPECT_TRUE(found);
+}
+
+TEST(AGX3FetchBanks, SplitSampleCoordinatesReplaceCollects)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_FRAGMENT;
+   agx_index x = agx_mov_imm(b, 32, 5), y = agx_mov_imm(b, 32, 7);
+   agx_index reference = agx_mov_imm(b, 32, 9);
+   agx_index coords = agx_vec_temp(b->shader, AGX_SIZE_32, 2);
+   agx_instr *collect = agx_collect_to(b, coords, 2);
+   collect->src[0] = x;
+   collect->src[1] = y;
+   /* One coordinate pair, three filtered taps: a plain, a comparison and a
+    * comparison whose reference is also its x coordinate. */
+   split_test_sample(b, agx_vec_temp(b->shader, AGX_SIZE_32, 4), coords,
+                     agx_null());
+   split_test_sample(b, agx_vec_temp(b->shader, AGX_SIZE_32, 4), coords,
+                     reference);
+   split_test_sample(b, agx_vec_temp(b->shader, AGX_SIZE_32, 4), coords, x);
+
+   EXPECT_TRUE(agx3_split_texture_coordinates(b->shader));
+   unsigned split = 0, vector = 0;
+   agx_foreach_instr_global(b->shader, I) {
+      if (I->op != AGX_OPCODE_TEXTURE_SAMPLE)
+         continue;
+      if (!I->agx3_split_coords) {
+         EXPECT_EQ(I->nr_srcs, 6);
+         EXPECT_EQ(I->src[0].value, coords.value);
+         EXPECT_EQ(I->src[5].value, x.value);
+         ++vector;
+         continue;
+      }
+      ASSERT_EQ(I->nr_srcs, 8);
+      EXPECT_EQ(I->src[0].value, x.value);
+      EXPECT_EQ(I->src[6].value, y.value);
+      EXPECT_TRUE(agx_is_null(I->src[7]));
+      ++split;
+   }
+   EXPECT_EQ(split, 2u);
+   EXPECT_EQ(vector, 1u);
+   ralloc_free(mem);
+}
+
+/* The M5 3D publication stages its coordinates with the wide scalar COPY, so
+ * it reads them where they live instead of from fresh copies. */
+TEST(AGX3FetchBanks, ScalarPublicationReadsCoordinatesInPlace)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_FRAGMENT;
+   agx_index c[3] = {agx_mov_imm(b, 32, 5), agx_mov_imm(b, 32, 7),
+                     agx_mov_imm(b, 32, 9)};
+   agx_index coords = agx_vec_temp(b->shader, AGX_SIZE_32, 3);
+   agx_instr *collect = agx_collect_to(b, coords, 3);
+   for (unsigned i = 0; i < 3; ++i)
+      collect->src[i] = c[i];
+   agx_index result = agx_vec_temp(b->shader, AGX_SIZE_32, 4);
+   agx_texture_sample_to(b, result, coords, agx_immediate(0), agx_immediate(0),
+                         agx_uniform(24, AGX_SIZE_64),
+                         agx_uniform(28, AGX_SIZE_64), agx_null(), AGX_DIM_3D,
+                         AGX_LOD_MODE_AUTO_LOD, 15, false, false, false,
+                         AGX_GATHER_NONE, false);
+   agx_unit_test(b, result);
+
+   ASSERT_TRUE(agx3_lower_scalar_texture_publications(b->shader));
+   unsigned publications = 0, moves = 0;
+   agx_foreach_instr_global(b->shader, I) {
+      moves += I->op == AGX_OPCODE_MOV;
+      if (I->op != AGX_OPCODE_PUBLISH_TEXTURE_SCALAR_AGX3)
+         continue;
+      ++publications;
+      for (unsigned i = 0; i < 3; ++i)
+         EXPECT_EQ(I->src[i].value, c[i].value);
+   }
+   EXPECT_EQ(publications, 1u);
+   EXPECT_EQ(moves, 0u);
+   ralloc_free(mem);
+}
+
 TEST(AGX3UniformALU, VertexTextureUniformQualification)
 {
    /* Both operands name the same aligned uniform, a witnessed
-    * uniform/uniform pair, so every stage keeps them in the uniform file. */
+    * uniform/uniform pair, so every G17P stage keeps them in the uniform
+    * file; M5 keeps a 64-bit pair in fragment main programs (the witnessed
+    * argument-buffer sample). */
    for (auto encoding : {AGX_ISA_ENCODING_AGX3B, AGX_ISA_ENCODING_AGX3C}) {
       for (auto stage : {MESA_SHADER_VERTEX, MESA_SHADER_FRAGMENT}) {
          for (bool preamble : {false, true}) {
@@ -38,8 +596,11 @@ TEST(AGX3UniformALU, VertexTextureUniformQualification)
                   agx_optimizer_forward(b->shader);
                   agx_lower_uniform_sources(b->shader);
                   const bool keep =
-                     encoding == AGX_ISA_ENCODING_AGX3B &&
-                     value <= 252 && !(value & 3) && size != AGX_SIZE_16;
+                     value <= 252 && !(value & 3) &&
+                     (encoding == AGX_ISA_ENCODING_AGX3B
+                         ? size != AGX_SIZE_16
+                         : size == AGX_SIZE_64 &&
+                              stage == MESA_SHADER_FRAGMENT && !preamble);
                   EXPECT_EQ(I->src[3].type == AGX_INDEX_UNIFORM, keep);
                   EXPECT_EQ(I->src[4].type == AGX_INDEX_UNIFORM, keep);
                   EXPECT_NE(I->src[1].type, AGX_INDEX_UNIFORM);
@@ -329,6 +890,17 @@ TEST(AGX3UniformALU, RefusesInvalidInputsWithoutWriting)
                                           AGX3_UNIFORM_FADD, 0, src, false,
                                           false, 0, bytes, &length));
    src[1].discard = false;
+   /* M5 ADD takes one uniform and one register; only the uniform may be
+    * negated, and the result is never saturated. */
+   src[0].neg = true;
+   EXPECT_FALSE(agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C,
+                                          AGX3_UNIFORM_FADD, 0, src, false,
+                                          false, 0, bytes, &length));
+   src[0].neg = false;
+   EXPECT_FALSE(agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C,
+                                          AGX3_UNIFORM_FADD, 0, src, true,
+                                          false, 0, bytes, &length));
+   src[0].kind = AGX3_FLOAT_UNIFORM;
    EXPECT_FALSE(agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C,
                                           AGX3_UNIFORM_FADD, 0, src, false,
                                           false, 0, bytes, &length));
@@ -443,6 +1015,129 @@ TEST(AGX3ArrayStaging, ReusedCompletionSlotRetainsCoordinateConsumer)
    ralloc_free(mem);
 }
 
+static void
+check_fetch_group(unsigned mode)
+{
+   struct option_guard {
+      bool had = getenv("ASAHI_M5_FETCH_REUSE") != nullptr;
+      std::string old = had ? getenv("ASAHI_M5_FETCH_REUSE") : "";
+      option_guard() { setenv("ASAHI_M5_FETCH_REUSE", "1", 1); }
+      ~option_guard() {
+         if (had) setenv("ASAHI_M5_FETCH_REUSE", old.c_str(), 1);
+         else unsetenv("ASAHI_M5_FETCH_REUSE");
+      }
+   } guard;
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   b->shader->max_reg = 40;
+   for (unsigned i = 0; i < 3; ++i) {
+      if (mode == 3 && i == 1)
+         agx_mov_imm_to(b, agx_register(24, AGX_SIZE_32), 0);
+      agx_index coords = agx_register(mode == 1 && i ? 4 : 0, AGX_SIZE_32);
+      coords.channels_m1 = 1;
+      agx_index dest = agx_register(mode == 2 && !i ? 0 : 32 + i * 16,
+                                    AGX_SIZE_32);
+      dest.channels_m1 = 3;
+      agx_texture_load_to(b, dest, coords, agx_immediate(0), agx_immediate(0),
+                           agx_immediate(i), agx_immediate(0), agx_null(),
+                           AGX_DIM_2D, AGX_LOD_MODE_LOD_MIN, 15, false, false);
+   }
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+   const uint8_t *bytes = static_cast<const uint8_t *>(binary.data);
+   std::array<agx3_tex_fetch_desc, 3> fetches = {};
+   unsigned found = 0;
+   for (unsigned i = 0; i + 8 <= binary.size; ++i) {
+      agx3_tex_fetch_desc d;
+      if (agx3_tex_decode_fetch(AGX_ISA_ENCODING_AGX3C, bytes + i, 8, &d)) {
+         ASSERT_LT(found, fetches.size());
+         fetches[found++] = d;
+         i += 7;
+      }
+   }
+   ASSERT_EQ(found, 3u);
+   EXPECT_EQ(fetches[0].coord_kill, mode != 0);
+   EXPECT_FALSE(fetches[1].coord_kill);
+   EXPECT_TRUE(fetches[2].coord_kill);
+   EXPECT_EQ(fetches[1].coord, fetches[2].coord);
+   EXPECT_EQ(fetches[2].wait_mask, 0u);
+   if (mode == 0) {
+      EXPECT_EQ(fetches[0].coord, fetches[1].coord);
+      EXPECT_EQ(fetches[1].wait_mask, 0u);
+   }
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+}
+
+TEST(AGX3FetchGroups, SameCoordinates) { check_fetch_group(0); }
+TEST(AGX3FetchGroups, ChangedCoordinates) { check_fetch_group(1); }
+TEST(AGX3FetchGroups, OverwrittenCoordinates) { check_fetch_group(2); }
+TEST(AGX3FetchGroups, InterveningInstruction) { check_fetch_group(3); }
+
+TEST(AGX3FetchBanks, EightIndependentCoordinates)
+{
+   struct option_guard {
+      const char *name;
+      bool had;
+      std::string old;
+      option_guard(const char *n, const char *value) : name(n), had(getenv(n)),
+         old(had ? getenv(n) : "") { setenv(n, value, 1); }
+      ~option_guard() {
+         if (had) setenv(name, old.c_str(), 1);
+         else unsetenv(name);
+      }
+   } reuse("ASAHI_M5_FETCH_REUSE", "0"), banks("ASAHI_M5_FETCH_BANKS", "8");
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   b->shader->max_reg = 63;
+   for (unsigned i = 0; i < 8; ++i) {
+      agx_index coords = agx_register(i * 4, AGX_SIZE_32);
+      coords.channels_m1 = 1;
+      agx_index dest = agx_register(64 + i * 8, AGX_SIZE_32);
+      dest.channels_m1 = 3;
+      agx_texture_load_to(b, dest, coords, agx_immediate(0), agx_immediate(0),
+                           agx_immediate(i), agx_immediate(0), agx_null(),
+                           AGX_DIM_2D, AGX_LOD_MODE_LOD_MIN, 15, false, false);
+   }
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+   const uint8_t *bytes = static_cast<const uint8_t *>(binary.data);
+   for (unsigned i = 0; i < 8; ++i) {
+      agx3_tex_fetch_desc d = {};
+      d.dest = 64 + i * 8;
+      d.coord = i * 8;
+      d.width = AGX3_TEX_W32;
+      d.mask = 15;
+      d.texture = i;
+      d.slot = i + 1;
+      d.wait_mask = 1u << i;
+      d.coord_kill = true;
+      d.final = i == 7;
+      uint8_t expected[AGX3_TEX_MAX_LENGTH], len = 0;
+      ASSERT_TRUE(agx3_tex_encode_fetch(AGX_ISA_ENCODING_AGX3C, &d, expected, &len));
+      bool found = false;
+      for (unsigned j = 0; j + len <= binary.size; ++j)
+         found |= std::memcmp(bytes + j, expected, len) == 0;
+      EXPECT_TRUE(found) << "bank " << i;
+   }
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+}
+
 TEST(AGX3PixelTests, EmptyExecutionSkipsShortTestingBlocks)
 {
    for (auto encoding : {AGX_ISA_ENCODING_AGX3B, AGX_ISA_ENCODING_AGX3C}) {
@@ -450,7 +1145,7 @@ TEST(AGX3PixelTests, EmptyExecutionSkipsShortTestingBlocks)
          void *mem = ralloc_context(NULL);
          agx_shader_key key = {};
          key.dev.encoding = encoding;
-         key.dev.chip = encoding == AGX_ISA_ENCODING_AGX3B ? AGX_CHIP_G17P : AGX_CHIP_G16G;
+         key.dev.chip = encoding == AGX_ISA_ENCODING_AGX3B ? AGX_CHIP_G17P : AGX_CHIP_G17S;
          agx_builder *b = agx_test_builder(mem);
          b->shader->key = &key;
          b->shader->stage = MESA_SHADER_FRAGMENT;
@@ -466,7 +1161,9 @@ TEST(AGX3PixelTests, EmptyExecutionSkipsShortTestingBlocks)
          agx_stop(&last);
          agx_opt_jmp_none(b->shader);
          agx_instr *jump = agx_last_instr(entry);
-         bool required = encoding == AGX_ISA_ENCODING_AGX3B && mode == 2;
+         /* Every AGX3 encoding: M5 (AGX3C) stops its 3D stage on an
+          * empty-EXEC testing update just as G17P does. */
+         bool required = mode == 2;
          EXPECT_EQ(jump->op == AGX_OPCODE_JMP_EXEC_NONE, required);
          if (required) {
             EXPECT_EQ(jump->target, end);
@@ -477,23 +1174,807 @@ TEST(AGX3PixelTests, EmptyExecutionSkipsShortTestingBlocks)
 }
 
 TEST(AGX3PixelTests, EmptyExecutionSkipsShortTestingElse)
+{
+   for (auto encoding : {AGX_ISA_ENCODING_AGX3B, AGX_ISA_ENCODING_AGX3C}) {
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = encoding;
+      key.dev.chip = encoding == AGX_ISA_ENCODING_AGX3B ? AGX_CHIP_G17P : AGX_CHIP_G17S;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->stage = MESA_SHADER_FRAGMENT;
+      agx_block *body = list_first_entry(&b->shader->blocks, agx_block, link);
+      agx_instr *else_ = agx_else_fcmp(b, agx_zero(), agx_zero(), 1,
+                                     AGX_FCOND_EQ, false, body);
+      agx_sample_mask_agx3(b, agx_register(0, AGX_SIZE_16),
+                           agx_register(2, AGX_SIZE_32), 2);
+      agx_pop_exec(b, 1);
+      agx_opt_jmp_none(b->shader);
+      agx_instr *jump = list_entry(else_->link.next, agx_instr, link);
+      EXPECT_EQ(jump->op, AGX_OPCODE_JMP_EXEC_NONE_AFTER);
+      EXPECT_EQ(jump->target, body);
+      ralloc_free(mem);
+   }
+}
+
+TEST(AGX3UniformALU, M5NativeMultiplyAndFmaUniformWitnesses)
+{
+   const struct {
+      agx3_uniform_float_op op; uint16_t dest; bool sat, cache; uint8_t wait;
+      agx3_uniform_float_src src[3]; uint8_t length; uint8_t expected[16];
+   } cases[] = {
+      {AGX3_UNIFORM_FMA, 14, false, true, 2, {{AGX3_FLOAT_REGISTER, 20, false, false, true, false},{AGX3_FLOAT_UNIFORM, 22, false, true, false, false},{AGX3_FLOAT_REGISTER, 12, false, false, false, true}}, 8, {0x79,0x95,0x26,0x17,0x81,0x0c,0x02,0x4a}},
+      {AGX3_UNIFORM_FMA, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false},{AGX3_FLOAT_UNIFORM, 20, false, true, false, false},{AGX3_FLOAT_REGISTER, 10, false, false, false, true}}, 8, {0x59,0x93,0x26,0x15,0x81,0x0a,0x02,0x0a}},
+      {AGX3_UNIFORM_FMUL, 18, true, true, 0, {{AGX3_FLOAT_UNIFORM, 28, false, false, false, false},{AGX3_FLOAT_REGISTER, 18, false, false, false, true},{}}, 8, {0x99,0x1d,0x35,0x13,0x81,0x00,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true},{AGX3_FLOAT_UNIFORM, 36, false, false, false, false},{}}, 6, {0x59,0x01,0x2d,0x25,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true},{AGX3_FLOAT_UNIFORM, 38, false, false, false, false},{AGX3_FLOAT_REGISTER, 10, false, false, false, true}}, 6, {0x19,0x03,0x2e,0x27,0xa0,0x0a}},
+      {AGX3_UNIFORM_FMA, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 4, false, false, false, true},{AGX3_FLOAT_UNIFORM, 34, false, false, false, false},{AGX3_FLOAT_REGISTER, 2, false, false, false, true}}, 6, {0x29,0x05,0x2e,0x23,0xa0,0x02}},
+      {AGX3_UNIFORM_FMUL, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 10, false, false, false, true},{AGX3_FLOAT_UNIFORM, 22, false, false, false, false},{}}, 6, {0x59,0x0b,0x2d,0x17,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 4, false, false, false, true},{AGX3_FLOAT_UNIFORM, 22, false, false, false, false},{AGX3_FLOAT_REGISTER, 6, false, false, false, true}}, 6, {0x29,0x05,0x2e,0x17,0xa0,0x06}},
+      {AGX3_UNIFORM_FMUL, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true},{AGX3_FLOAT_UNIFORM, 150, false, false, false, false},{}}, 6, {0x09,0x01,0x3d,0x17,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true},{AGX3_FLOAT_UNIFORM, 158, false, false, false, false},{}}, 6, {0x19,0x03,0x3d,0x1f,0x00,0x02}},
+   };
+   for (const auto &c : cases) {
+      uint8_t bytes[16], length = 0;
+      ASSERT_TRUE(agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C, c.op,
+                  c.dest, c.src, c.sat, c.cache, c.wait, bytes, &length));
+      ASSERT_EQ(length, c.length);
+      EXPECT_EQ(std::memcmp(bytes, c.expected, length), 0);
+   }
+}
+
+/* Native G17G Minecraft 26.3 terrain, entity, cloud and
+ * sky pipelines: every distinct f32 ADD/MUL/FMA with a uniform operand or a
+ * uniform-file result in their mains and constant programs whose fields the
+ * encoder names, re-encoded from the decoded operands; plus the R*R+U FMAs of
+ * the agx3-diff-corpus native G17G r_heavy_loop fragment. */
+TEST(AGX3UniformALU, M5NativeUniformOperandAndResultShapes)
+{
+   const struct {
+      agx3_uniform_float_op op; bool uniform_dest; uint16_t dest;
+      bool sat, cache; uint8_t wait; agx3_uniform_float_src src[3];
+      uint8_t length; uint8_t expected[16];
+   } cases[] = {
+      {AGX3_UNIFORM_FMA, false, 8, false, true, 0, {{AGX3_FLOAT_REGISTER, 6, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 4, false, true, false, false, false}, {AGX3_FLOAT_IMMEDIATE, 48, false, false, false, false, false}}, 8, {0x49,0x87,0x26,0x05,0x21,0xb0,0x02,0x0a}},
+      {AGX3_UNIFORM_FMUL, true, 32, false, false, 0, {{AGX3_FLOAT_IMMEDIATE, 8, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 8, false, false, true, false, false}, {}}, 6, {0x09,0x89,0x45,0x89,0xc0,0x00}},
+      {AGX3_UNIFORM_FMUL, true, 36, false, false, 0, {{AGX3_FLOAT_IMMEDIATE, 24, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 8, false, false, false, true, false}, {}}, 6, {0x29,0x99,0x55,0x09,0xc0,0x00}},
+      {AGX3_UNIFORM_FMUL, true, 34, false, false, 0, {{AGX3_FLOAT_IMMEDIATE, 24, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 10, false, false, true, false, false}, {}}, 6, {0x19,0x99,0x45,0x8b,0xc0,0x00}},
+      {AGX3_UNIFORM_FMUL, true, 38, false, false, 0, {{AGX3_FLOAT_IMMEDIATE, 8, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 10, false, false, false, true, false}, {}}, 6, {0x39,0x89,0x55,0x0b,0xc0,0x00}},
+      {AGX3_UNIFORM_FMUL, false, 12, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 30, false, false, false, false, false}, {}}, 6, {0x69,0x0d,0x2d,0x1f,0x00,0x02}},
+      {AGX3_UNIFORM_FADD, false, 18, false, true, 0, {{AGX3_FLOAT_UNIFORM, 32, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {}}, 6, {0x99,0x21,0x24,0x83,0x80,0x00}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_UNIFORM, 34, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {}}, 6, {0x79,0x23,0x24,0x81,0x80,0x00}},
+      {AGX3_UNIFORM_FADD, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 32, false, true, false, false, false}, {}}, 6, {0x99,0x83,0x24,0x21,0x00,0x0a}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 34, false, true, false, false, false}, {}}, 6, {0x79,0x81,0x24,0x23,0x00,0x0a}},
+      {AGX3_UNIFORM_FADD, false, 18, false, true, 0, {{AGX3_FLOAT_UNIFORM, 36, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {}}, 6, {0x99,0x25,0x24,0x83,0x80,0x00}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 38, false, true, false, false, false}, {}}, 6, {0x79,0x81,0x24,0x27,0x00,0x0a}},
+      {AGX3_UNIFORM_FADD, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 36, false, true, false, false, false}, {}}, 6, {0x99,0x83,0x24,0x25,0x00,0x0a}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_UNIFORM, 38, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {}}, 6, {0x79,0x27,0x24,0x81,0x80,0x00}},
+      {AGX3_UNIFORM_FMUL, false, 90, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 20, false, false, false, false, false}, {}}, 6, {0xd9,0x03,0xad,0x15,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 88, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 22, false, false, false, false, false}, {}}, 6, {0xc9,0x01,0xad,0x17,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 88, false, true, 0, {{AGX3_FLOAT_REGISTER, 88, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}, {}}, 6, {0xc9,0x59,0xad,0x1b,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 90, false, true, 0, {{AGX3_FLOAT_REGISTER, 90, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}, {}}, 6, {0xd9,0x5b,0xad,0x19,0x00,0x02}},
+      {AGX3_UNIFORM_FADD, false, 52, false, true, 0, {{AGX3_FLOAT_REGISTER, 62, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 28, false, true, false, false, false}, {}}, 6, {0xa9,0x3f,0x6c,0x1d,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 56, true, true, 0, {{AGX3_FLOAT_REGISTER, 52, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 30, false, false, false, false, false}, {}}, 8, {0xc9,0x35,0x6d,0x1f,0x01,0x02,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 20, false, false, false, false, false}, {}}, 6, {0x99,0x83,0x25,0x15,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 22, false, false, false, false, false}, {}}, 6, {0x89,0x81,0x25,0x17,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}, {}}, 6, {0x99,0x13,0x2d,0x19,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}, {}}, 6, {0x89,0x11,0x2d,0x1b,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 16, false, true, 0, {{AGX3_FLOAT_UNIFORM, 62, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {}}, 6, {0x89,0x3f,0x25,0x8f,0x80,0x00}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 40, false, true, false, false, false}, {}}, 6, {0x79,0x91,0x24,0x29,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 20, false, true, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 48, false, false, false, false, false}, {}}, 6, {0xa9,0x0f,0x2d,0x31,0x00,0x02}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 44, false, true, false, false, false}, {}}, 6, {0x79,0x93,0x24,0x2d,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 22, false, true, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 50, false, false, false, false, false}, {}}, 6, {0xb9,0x0f,0x2d,0x33,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 26, false, true, 0, {{AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 12, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}}, 8, {0xd9,0x39,0x26,0x8d,0x21,0x38,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 20, false, true, 0, {{AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 12, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}}, 8, {0xa9,0x3b,0x26,0x8d,0x21,0x3a,0x82,0x08}},
+      {AGX3_UNIFORM_FMUL, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 62, false, false, false, false, false}, {}}, 6, {0x19,0x03,0x2d,0x3f,0x00,0x02}},
+      /* Native G17G compile of fma(x, u, -y) in a fragment: R*U-R. */
+      {AGX3_UNIFORM_FMA, false, 0, false, false, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 32, false, true, true, false, false}}, 8, {0x09,0x93,0x06,0x05,0x11,0xa0,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}}, 6, {0x09,0x83,0x26,0x39,0xa0,0x00}},
+      {AGX3_UNIFORM_FMA, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}}, 6, {0x89,0x83,0x26,0x3b,0xa0,0x10}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 12, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}}, 8, {0x79,0x3d,0x26,0x8d,0x21,0x3c,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}}, 6, {0x19,0x03,0x2e,0x3d,0xa0,0x04}},
+      {AGX3_UNIFORM_FADD, false, 8, false, true, 0, {{AGX3_FLOAT_REGISTER, 10, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 16, false, true, false, false, false}, {}}, 6, {0x49,0x8b,0x24,0x11,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 8, false, true, 0, {{AGX3_FLOAT_REGISTER, 8, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}, {}}, 6, {0x49,0x09,0x2d,0x19,0x00,0x02}},
+      {AGX3_UNIFORM_FADD, false, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 8, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 20, false, true, false, false, false}, {}}, 6, {0x59,0x89,0x24,0x15,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 10, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}, {}}, 6, {0x59,0x0b,0x2d,0x1b,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 0, false, true, 0, {{AGX3_FLOAT_UNIFORM, 36, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {}}, 6, {0x09,0x25,0x35,0x01,0x80,0x00}},
+      {AGX3_UNIFORM_FMUL, false, 2, false, true, 0, {{AGX3_FLOAT_UNIFORM, 38, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {}}, 6, {0x19,0x27,0x35,0x03,0x80,0x00}},
+      {AGX3_UNIFORM_FMUL, false, 8, false, false, 0, {{AGX3_FLOAT_REGISTER, 8, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 34, false, false, false, false, false}, {}}, 6, {0x49,0x09,0x0d,0x23,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 8, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 28, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}}, 6, {0x09,0x89,0x26,0x1d,0xa0,0x00}},
+      {AGX3_UNIFORM_FMA, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 8, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 30, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}}, 6, {0x19,0x89,0x26,0x1f,0xa0,0x0c}},
+      {AGX3_UNIFORM_FMUL, false, 2, false, true, 0, {{AGX3_FLOAT_UNIFORM, 40, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}, {}}, 6, {0x19,0x29,0x35,0x05,0x80,0x00}},
+      {AGX3_UNIFORM_FMUL, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 42, false, false, false, false, false}, {}}, 6, {0x19,0x03,0x2d,0x2b,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 8, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 32, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}}, 6, {0x29,0x09,0x2e,0x21,0xa0,0x04}},
+      {AGX3_UNIFORM_FADD, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 40, false, true, false, false, false}, {}}, 6, {0x79,0x93,0x24,0x29,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 48, false, false, false, false, false}, {}}, 6, {0x79,0x0f,0x2d,0x31,0x00,0x02}},
+      {AGX3_UNIFORM_FADD, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 44, false, true, false, false, false}, {}}, 6, {0x89,0x8d,0x24,0x2d,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 50, false, false, false, false, false}, {}}, 6, {0x89,0x11,0x2d,0x33,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 12, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 62, false, false, false, false, false}, {}}, 6, {0x69,0x0d,0x2d,0x3f,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 24, false, true, 0, {{AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}}, 8, {0xc9,0x39,0x26,0x83,0x21,0x38,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 22, false, true, 0, {{AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}}, 8, {0xb9,0x3b,0x26,0x83,0x21,0x3a,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}}, 6, {0x09,0x8d,0x26,0x39,0xa0,0x10}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}}, 6, {0x79,0x8d,0x26,0x3b,0xa0,0x0e}},
+      {AGX3_UNIFORM_FMA, false, 18, false, true, 0, {{AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}}, 8, {0x99,0x3d,0x26,0x83,0x21,0x3c,0x82,0x08}},
+      {AGX3_UNIFORM_FMUL, false, 10, false, true, 0, {{AGX3_FLOAT_UNIFORM, 62, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 6, false, false, true, false, false}, {}}, 6, {0x59,0x3f,0x25,0x87,0x80,0x00}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}}, 6, {0x29,0x0d,0x2e,0x3d,0xa0,0x04}},
+      {AGX3_UNIFORM_FADD, false, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 20, false, true, false, false, false}, {}}, 6, {0x29,0x81,0x24,0x15,0x00,0x0a}},
+      {AGX3_UNIFORM_FMUL, false, 6, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 8, false, false, false, false, false}, {}}, 6, {0x39,0x83,0x25,0x09,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 18, false, false, false, false, false}, {}}, 6, {0x19,0x01,0x2d,0x13,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}}, 8, {0x09,0x19,0x26,0x83,0x21,0x18,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}}, 8, {0x29,0x1b,0x26,0x83,0x21,0x1a,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 12, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}}, 6, {0x09,0x83,0x26,0x0d,0xa0,0x00}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 14, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}}, 6, {0x29,0x83,0x26,0x0f,0xa0,0x04}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_UNIFORM, 28, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 2, false, true, true, false, false}, {AGX3_FLOAT_UNIFORM, 28, false, false, false, false, false}}, 8, {0x29,0x1d,0x26,0x83,0x21,0x1c,0x82,0x08}},
+      {AGX3_UNIFORM_FMA, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 16, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}}, 6, {0x19,0x03,0x2e,0x11,0xa0,0x04}},
+      {AGX3_UNIFORM_FMA, true, 24, false, false, 8, {{AGX3_FLOAT_REGISTER, 64, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 68, false, false, false, true, false}}, 8, {0xc9,0xc1,0x06,0x81,0x81,0x44,0x42,0x80}},
+      {AGX3_UNIFORM_FMA, true, 26, false, false, 0, {{AGX3_FLOAT_REGISTER, 64, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 66, false, false, false, true, false}}, 8, {0xd9,0xc1,0x06,0x83,0x81,0x42,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 28, false, false, 0, {{AGX3_FLOAT_REGISTER, 64, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 48, false, false, false, true, false}}, 8, {0xe9,0xc1,0x06,0x05,0x81,0x30,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 30, false, false, 0, {{AGX3_FLOAT_REGISTER, 64, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 6, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 62, false, false, false, true, false}}, 8, {0xf9,0x41,0x0e,0x07,0x81,0x3e,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 32, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 64, false, false, false, true, false}}, 8, {0x09,0xb9,0x46,0x81,0x81,0x40,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 34, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 58, false, false, false, true, false}}, 8, {0x19,0xb9,0x46,0x83,0x81,0x3a,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 36, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 48, false, false, false, true, false}}, 8, {0x29,0xb9,0x46,0x05,0x81,0x30,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 38, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 6, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 52, false, false, false, true, false}}, 8, {0x39,0x39,0x4e,0x07,0x81,0x34,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 40, false, false, 0, {{AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 50, false, false, false, true, false}}, 8, {0x49,0xaf,0x46,0x81,0x81,0x32,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 42, false, false, 0, {{AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 48, false, false, false, true, false}}, 8, {0x59,0xaf,0x46,0x83,0x81,0x30,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 44, false, false, 0, {{AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 42, false, false, false, true, false}}, 8, {0x69,0xaf,0x46,0x05,0x81,0x2a,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 46, false, false, 0, {{AGX3_FLOAT_REGISTER, 46, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 6, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 44, false, false, false, true, false}}, 8, {0x79,0x2f,0x4e,0x07,0x81,0x2c,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 48, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 40, false, false, false, true, false}}, 8, {0x89,0x8f,0x56,0x01,0x81,0x28,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 50, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}}, 8, {0x99,0x8f,0x56,0x03,0x81,0x10,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 52, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}}, 8, {0xa9,0x8f,0x56,0x05,0x81,0x00,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 70, false, false, false, false, true}, {AGX3_FLOAT_IMMEDIATE, 2, false, false, false, false, false}}, 8, {0x29,0x05,0x2e,0x46,0x21,0x82,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 70, false, false, false, false, true}, {AGX3_FLOAT_IMMEDIATE, 2, false, false, false, false, false}}, 8, {0x19,0x03,0x2e,0x46,0x21,0x82,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, false, 40, false, true, 0, {{AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 48, false, false, false, false, false}}, 8, {0x49,0x19,0x66,0x91,0x21,0x30,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 22, false, true, 0, {{AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 50, false, false, false, false, false}}, 8, {0xb9,0x1b,0x26,0x91,0x21,0x32,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 40, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 32, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 40, false, false, false, true, false}}, 6, {0x49,0x93,0x66,0x21,0xa0,0x28}},
+      {AGX3_UNIFORM_FMA, false, 22, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 34, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 22, false, false, false, true, false}}, 6, {0xb9,0x93,0x26,0x23,0xa0,0x16}},
+      {AGX3_UNIFORM_FMA, false, 40, false, true, 0, {{AGX3_FLOAT_UNIFORM, 28, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 52, false, false, false, false, false}}, 8, {0x49,0x1d,0x66,0x91,0x21,0x34,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 46, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 36, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 40, false, false, false, true, false}}, 6, {0x79,0x93,0x66,0x25,0xa0,0x28}},
+      {AGX3_UNIFORM_FMA, false, 16, false, true, 0, {{AGX3_FLOAT_UNIFORM, 30, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 54, false, false, false, false, false}}, 8, {0x89,0x1f,0x36,0x11,0x21,0x36,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 38, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}}, 6, {0x89,0x93,0x26,0x27,0xa0,0x10}},
+      {AGX3_UNIFORM_FMUL, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 36, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 64, false, false, false, false, false}, {}}, 6, {0x99,0xa5,0x25,0x41,0x00,0x02}},
+      {AGX3_UNIFORM_FMUL, false, 22, false, true, 0, {{AGX3_FLOAT_REGISTER, 36, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 56, false, false, false, false, false}, {}}, 6, {0xb9,0x25,0x2d,0x39,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 38, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 66, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}}, 6, {0x99,0xa7,0x26,0x43,0xa0,0x12}},
+      {AGX3_UNIFORM_FMA, false, 22, false, true, 0, {{AGX3_FLOAT_REGISTER, 38, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 58, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 22, false, false, false, true, false}}, 6, {0xb9,0x27,0x2e,0x3b,0xa0,0x16}},
+      {AGX3_UNIFORM_FMA, false, 22, false, true, 0, {{AGX3_FLOAT_REGISTER, 20, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 60, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 22, false, false, false, true, false}}, 6, {0xb9,0x95,0x26,0x3d,0xa0,0x16}},
+      {AGX3_UNIFORM_FMA, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 20, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 68, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}}, 6, {0x99,0x15,0x2e,0x45,0xa0,0x12}},
+      {AGX3_UNIFORM_FMA, false, 20, false, true, 0, {{AGX3_FLOAT_REGISTER, 20, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 74, false, false, false, false, false}, {AGX3_FLOAT_UNIFORM, 72, false, false, false, false, false}}, 8, {0xa9,0x15,0x2e,0x4b,0x21,0x48,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 74, false, false, false, false, false}, {AGX3_FLOAT_UNIFORM, 72, false, false, false, false, false}}, 8, {0x99,0x13,0x2e,0x4b,0x21,0x48,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, true, 20, false, false, 8, {{AGX3_FLOAT_REGISTER, 64, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 68, false, false, false, true, false}}, 8, {0xa9,0xc1,0x06,0x01,0x81,0x44,0x42,0x80}},
+      {AGX3_UNIFORM_FMA, true, 22, false, false, 0, {{AGX3_FLOAT_REGISTER, 64, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 66, false, false, false, true, false}}, 8, {0xb9,0xc1,0x06,0x83,0x81,0x42,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 24, false, false, 0, {{AGX3_FLOAT_REGISTER, 64, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 48, false, false, false, true, false}}, 8, {0xc9,0xc1,0x06,0x05,0x81,0x30,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 26, false, false, 0, {{AGX3_FLOAT_REGISTER, 64, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 6, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 62, false, false, false, true, false}}, 8, {0xd9,0x41,0x0e,0x07,0x81,0x3e,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 28, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 64, false, false, false, true, false}}, 8, {0xe9,0xb9,0x06,0x01,0x81,0x40,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 30, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 58, false, false, false, true, false}}, 8, {0xf9,0xb9,0x06,0x83,0x81,0x3a,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 32, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 48, false, false, false, true, false}}, 8, {0x09,0xb9,0x46,0x05,0x81,0x30,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 34, false, false, 0, {{AGX3_FLOAT_REGISTER, 56, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 6, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 52, false, false, false, true, false}}, 8, {0x19,0x39,0x4e,0x07,0x81,0x34,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 36, false, false, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 50, false, false, false, true, false}}, 8, {0x29,0x01,0x46,0xaf,0x81,0x32,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 38, false, false, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 48, false, false, false, true, false}}, 8, {0x39,0x83,0x46,0xaf,0x81,0x30,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 40, false, false, 0, {{AGX3_FLOAT_REGISTER, 4, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 42, false, false, false, true, false}}, 8, {0x49,0x05,0x46,0xaf,0x81,0x2a,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 42, false, false, 0, {{AGX3_FLOAT_REGISTER, 6, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 46, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 44, false, false, false, true, false}}, 8, {0x59,0x07,0x56,0x2f,0x81,0x2c,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 44, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 34, false, false, false, true, false}}, 8, {0x69,0x8f,0x56,0x01,0x81,0x22,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 46, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}}, 8, {0x79,0x8f,0x56,0x03,0x81,0x10,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 48, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}}, 8, {0x89,0x8f,0x56,0x05,0x81,0x12,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, false, 16, false, true, 0, {{AGX3_FLOAT_UNIFORM, 74, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 66, false, false, false, false, false}}, 8, {0x89,0x4b,0x36,0x13,0x21,0x42,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 12, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 72, false, false, false, false, false}, {AGX3_FLOAT_UNIFORM, 64, false, false, false, false, false}}, 8, {0x69,0x0d,0x2e,0x49,0x21,0x40,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, false, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 10, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 76, false, false, false, false, false}, {AGX3_FLOAT_UNIFORM, 68, false, false, false, false, false}}, 8, {0x59,0x0b,0x2e,0x4d,0x21,0x44,0x02,0x02}},
+      {AGX3_UNIFORM_FMA, false, 20, false, true, 0, {{AGX3_FLOAT_UNIFORM, 28, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 44, false, false, false, false, false}}, 8, {0xa9,0x1d,0x26,0x91,0x21,0x2c,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_UNIFORM, 30, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 46, false, false, false, false, false}}, 8, {0x79,0x1f,0x26,0x91,0x21,0x2e,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 36, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 20, false, false, false, true, false}}, 6, {0x99,0x8d,0x26,0x25,0xa0,0x14}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 38, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}}, 6, {0x79,0x8d,0x26,0x27,0xa0,0x0e}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_UNIFORM, 32, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 48, false, false, false, false, false}}, 8, {0x79,0x21,0x26,0x91,0x21,0x30,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 16, false, true, 0, {{AGX3_FLOAT_UNIFORM, 34, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 50, false, false, false, false, false}}, 8, {0x89,0x23,0x36,0x11,0x21,0x32,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 40, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}}, 6, {0x79,0x8d,0x26,0x29,0xa0,0x0e}},
+      {AGX3_UNIFORM_FMA, false, 12, false, true, 0, {{AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 42, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}}, 6, {0x69,0x0d,0x2e,0x2b,0xa0,0x10}},
+      {AGX3_UNIFORM_FMA, true, 40, false, false, 0, {{AGX3_FLOAT_REGISTER, 46, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 50, false, false, false, true, false}}, 8, {0x49,0xaf,0x46,0x01,0x81,0x32,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 48, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 34, false, false, false, true, false}}, 8, {0x89,0x8f,0x56,0x01,0x81,0x22,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, true, 52, false, false, 0, {{AGX3_FLOAT_REGISTER, 14, false, false, true, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}}, 8, {0xa9,0x8f,0x56,0x05,0x81,0x12,0x42,0x00}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 70, false, false, false, false, true}, {AGX3_FLOAT_IMMEDIATE, 2, false, false, false, false, false}}, 8, {0x09,0x01,0x2e,0x46,0x21,0x82,0x02,0x02}},
+      {AGX3_UNIFORM_FADD, false, 18, false, true, 0, {{AGX3_FLOAT_REGISTER, 16, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 64, false, false, false, false, false}, {}}, 6, {0x99,0x11,0x2c,0x41,0x00,0x02}},
+      {AGX3_UNIFORM_FADD, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 66, false, false, false, false, false}, {}}, 6, {0x09,0x01,0x2c,0x43,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 20, false, true, 0, {{AGX3_FLOAT_UNIFORM, 24, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 48, false, false, false, false, false}}, 8, {0xa9,0x19,0x26,0x93,0x21,0x30,0x82,0x00}},
+      {AGX3_UNIFORM_FADD, false, 16, false, true, 0, {{AGX3_FLOAT_REGISTER, 22, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 68, false, false, false, false, false}, {}}, 6, {0x89,0x17,0x2c,0x45,0x00,0x02}},
+      {AGX3_UNIFORM_FMA, false, 32, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 32, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 20, false, false, false, true, false}}, 6, {0x09,0x81,0x66,0x21,0xa0,0x14}},
+      {AGX3_UNIFORM_FMA, false, 20, false, true, 0, {{AGX3_FLOAT_UNIFORM, 28, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 52, false, false, false, false, false}}, 8, {0xa9,0x1d,0x26,0x93,0x21,0x34,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 20, false, false, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 36, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 20, false, false, false, true, false}}, 6, {0xa9,0x81,0x06,0x25,0xa0,0x14}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_UNIFORM, 26, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 50, false, false, false, false, false}}, 8, {0x29,0x1b,0x26,0x93,0x21,0x32,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 12, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 34, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}}, 6, {0x69,0x81,0x26,0x23,0xa0,0x04}},
+      {AGX3_UNIFORM_FMA, false, 14, false, true, 0, {{AGX3_FLOAT_UNIFORM, 30, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 18, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 54, false, false, false, false, false}}, 8, {0x79,0x1f,0x36,0x13,0x21,0x36,0x82,0x00}},
+      {AGX3_UNIFORM_FMA, false, 12, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, false, false}, {AGX3_FLOAT_UNIFORM, 38, false, false, false, false, false}, {AGX3_FLOAT_REGISTER, 14, false, false, false, true, false}}, 6, {0x69,0x01,0x26,0x27,0xa0,0x0e}},
+
+      /* agx3-diff-corpus native G17G r_heavy_loop fragment: R*R+U. */
+      {AGX3_UNIFORM_FMA, false, 2, false, true, 0, {{AGX3_FLOAT_REGISTER, 2, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 4, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 6, false, false, false, false, false}}, 8, {0x19,0x03,0x2e,0x85,0x21,0x06,0x02,0x00}},
+      {AGX3_UNIFORM_FMA, false, 4, false, true, 0, {{AGX3_FLOAT_REGISTER, 4, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 0, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 6, false, false, false, false, false}}, 8, {0x29,0x05,0x2e,0x81,0x21,0x06,0x02,0x00}},
+      {AGX3_UNIFORM_FMA, false, 0, false, true, 0, {{AGX3_FLOAT_REGISTER, 0, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 10, false, false, true, false, false}, {AGX3_FLOAT_UNIFORM, 6, false, false, false, false, false}}, 8, {0x09,0x01,0x2e,0x8b,0x21,0x06,0x02,0x00}},
+      {AGX3_UNIFORM_FMA, false, 10, false, true, 0, {{AGX3_FLOAT_REGISTER, 10, false, false, false, true, false}, {AGX3_FLOAT_REGISTER, 12, false, false, false, true, false}, {AGX3_FLOAT_UNIFORM, 6, false, false, false, false, false}}, 8, {0x59,0x0b,0x3e,0x0d,0x21,0x06,0x02,0x00}},
+   };
+   unsigned failed = 0;
+   for (const auto &c : cases) {
+      uint8_t bytes[16] = {0}, length = 0;
+      const bool ok = c.uniform_dest
+         ? agx3_uniform_float_store_encode(AGX_ISA_ENCODING_AGX3C, c.op,
+                                           c.dest, c.src, c.wait, bytes,
+                                           &length)
+         : agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C, c.op, c.dest,
+                                     c.src, c.sat, c.cache, c.wait, bytes,
+                                     &length);
+      if (!ok || length != c.length || std::memcmp(bytes, c.expected, length)) {
+         ++failed;
+         fprintf(stderr, "mismatch op=%u ud=%u dest=%u ok=%u len=%u/%u got=",
+                 (unsigned)c.op, c.uniform_dest, c.dest, ok, length, c.length);
+         for (unsigned i = 0; i < length; ++i)
+            fprintf(stderr, "%02x", bytes[i]);
+         fprintf(stderr, " want=");
+         for (unsigned i = 0; i < c.length; ++i)
+            fprintf(stderr, "%02x", c.expected[i]);
+         fprintf(stderr, "\n");
+      }
+   }
+   EXPECT_EQ(failed, 0u);
+}
+
+/* The same native M5 vertex mains write their outputs' final FMUL/FMA
+ * straight into the staging file (f9=5) that UVSW reads. Every distinct
+ * staged R*R MUL, U*R+R FMA and R*R+R FMA whose fields the encoder names,
+ * with sources in agx3_stage_vertex_alu order (src[0] is the second
+ * hardware operand, the uniform of U*R+R). */
+TEST(AGX3UniformALU, M5NativeStagedVertexOutputALU)
+{
+   const struct {
+      bool fma; uint16_t dest; uint8_t slot; uint16_t src[3]; bool uniform0;
+      bool live[3], discard[3]; uint8_t wait; uint8_t length;
+      uint8_t expected[16];
+   } cases[] = {
+      {true, 8, 1, {40, 20, 40}, true, {false, true, false}, {false, false, true}, 0, 8, {0x49,0x95,0x26,0x29,0x81,0x28,0x42,0x02}},
+      {true, 10, 2, {42, 20, 22}, true, {false, true, false}, {false, false, true}, 0, 10, {0x59,0x95,0x26,0x2b,0x82,0x16,0x42,0x02,0x00,0x04}},
+      {true, 12, 2, {44, 20, 46}, true, {false, true, false}, {false, false, true}, 0, 10, {0x69,0x95,0x26,0x2d,0x82,0x2e,0x42,0x02,0x00,0x04}},
+      {true, 14, 2, {46, 20, 16}, true, {false, true, false}, {false, false, true}, 0, 10, {0x79,0x95,0x26,0x2f,0x82,0x10,0x42,0x02,0x00,0x04}},
+      {false, 16, 8, {40, 38, 0}, false, {false, false, false}, {true, true, false}, 0, 14, {0xa1,0x02,0x05,0x30,0x22,0x70,0xa0,0x02,0x1c,0x0a,0x20,0x00,0x02,0x10}},
+      {false, 18, 8, {32, 36, 0}, false, {true, true, false}, {false, false, false}, 0, 14, {0xb1,0x80,0x05,0x30,0x22,0x70,0xa0,0x02,0x18,0x08,0x10,0x00,0x02,0x10}},
+      {false, 20, 8, {34, 36, 0}, false, {true, true, false}, {false, false, false}, 0, 14, {0xa1,0x80,0x05,0x30,0x22,0x70,0xa0,0x0a,0x98,0x08,0x10,0x00,0x02,0x10}},
+      {false, 22, 6, {32, 18, 0}, false, {false, true, false}, {true, false, false}, 0, 8, {0xb9,0x93,0x35,0x21,0x41,0x00,0x00,0x14}},
+      {false, 24, 6, {34, 18, 0}, false, {false, true, false}, {true, false, false}, 0, 8, {0xc9,0x93,0x35,0x23,0x41,0x00,0x00,0x14}},
+      {false, 26, 6, {28, 36, 0}, false, {true, false, false}, {false, true, false}, 0, 8, {0xd9,0x25,0x2d,0x9d,0x41,0x00,0x00,0x14}},
+      {false, 28, 3, {28, 18, 0}, false, {false, false, false}, {true, true, false}, 0, 8, {0xe9,0x13,0x3d,0x1d,0x41,0x00,0x00,0x08}},
+      {false, 0, 2, {14, 18, 0}, false, {false, false, false}, {true, true, false}, 0, 8, {0x09,0x13,0x3d,0x0f,0x41,0x00,0x00,0x04}},
+      {true, 2, 3, {20, 10, 18}, true, {false, true, false}, {false, false, true}, 0, 10, {0x19,0x8b,0x26,0x15,0x82,0x12,0x42,0x02,0x00,0x08}},
+      {true, 4, 4, {22, 10, 14}, true, {false, true, false}, {false, false, true}, 0, 10, {0x29,0x8b,0x26,0x17,0x82,0x0e,0x42,0x02,0x00,0x0c}},
+      {true, 6, 5, {24, 10, 14}, true, {false, true, false}, {false, false, true}, 0, 10, {0x39,0x8b,0x26,0x19,0x82,0x0e,0x42,0x02,0x00,0x10}},
+      {true, 8, 6, {26, 10, 12}, true, {false, false, false}, {false, true, true}, 0, 10, {0x49,0x0b,0x2e,0x1b,0x82,0x0c,0x42,0x02,0x00,0x14}},
+      {true, 4, 2, {40, 16, 32}, true, {false, true, false}, {false, false, true}, 0, 10, {0x29,0x91,0x26,0x29,0x82,0x20,0x42,0x02,0x00,0x04}},
+      {false, 0, 3, {28, 8, 0}, false, {false, false, false}, {true, true, false}, 1, 8, {0x09,0x09,0x3d,0x1d,0x41,0x20,0x00,0x08}},
+      {false, 2, 3, {30, 10, 0}, false, {false, false, false}, {true, true, false}, 0, 8, {0x19,0x0b,0x3d,0x1f,0x41,0x00,0x00,0x08}},
+      {false, 6, 3, {2, 12, 0}, false, {false, false, false}, {true, true, false}, 0, 8, {0x39,0x0d,0x3d,0x03,0x41,0x00,0x00,0x08}},
+      {false, 8, 3, {4, 14, 0}, false, {false, false, false}, {true, true, false}, 0, 8, {0x49,0x0f,0x3d,0x05,0x41,0x00,0x00,0x08}},
+      {true, 10, 7, {42, 16, 12}, true, {false, true, false}, {false, false, true}, 0, 10, {0x59,0x91,0x26,0x2b,0x82,0x0c,0x42,0x02,0x00,0x18}},
+      {true, 12, 8, {44, 16, 20}, true, {false, true, false}, {false, false, true}, 0, 16, {0x21,0x80,0x07,0x8a,0x20,0x07,0xa3,0x1a,0x18,0x0b,0x88,0x00,0x14,0x08,0x00,0x80}},
+      {true, 14, 1, {46, 16, 12}, true, {false, false, false}, {false, true, true}, 0, 8, {0x79,0x11,0x2e,0x2f,0x81,0x0c,0x42,0x02}},
+      {true, 16, 4, {4, 6, 8}, false, {false, false, false}, {true, true, true}, 0, 10, {0x89,0x07,0x3e,0x05,0x82,0x08,0x42,0x00,0x00,0x0c}},
+
+   };
+   unsigned failed = 0;
+   for (const auto &c : cases) {
+      uint8_t bytes[16] = {0}, length = 0;
+      const bool ok = agx3_stage_vertex_alu(AGX_ISA_ENCODING_AGX3C, c.fma,
+                                            c.dest, c.slot, c.src, c.uniform0,
+                                            c.live, c.discard, c.wait, bytes,
+                                            &length);
+      if (!ok || length != c.length || std::memcmp(bytes, c.expected, length)) {
+         ++failed;
+         fprintf(stderr, "staged mismatch fma=%u dest=%u slot=%u ok=%u len=%u/%u got=",
+                 c.fma, c.dest, c.slot, ok, length, c.length);
+         for (unsigned i = 0; i < length; ++i)
+            fprintf(stderr, "%02x", bytes[i]);
+         fprintf(stderr, " want=");
+         for (unsigned i = 0; i < c.length; ++i)
+            fprintf(stderr, "%02x", c.expected[i]);
+         fprintf(stderr, "\n");
+      }
+   }
+   EXPECT_EQ(failed, 0u);
+}
+
+TEST(AGX3UniformALU, M5LoweringRetainsWitnessedFloatUniforms)
 {
    void *mem = ralloc_context(NULL);
    agx_shader_key key = {};
-   key.dev.encoding = AGX_ISA_ENCODING_AGX3B;
-   key.dev.chip = AGX_CHIP_G17P;
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
    agx_builder *b = agx_test_builder(mem);
    b->shader->key = &key;
-   b->shader->stage = MESA_SHADER_FRAGMENT;
-   agx_block *body = list_first_entry(&b->shader->blocks, agx_block, link);
-   agx_instr *else_ = agx_else_fcmp(b, agx_zero(), agx_zero(), 1,
-                                  AGX_FCOND_EQ, false, body);
-   agx_sample_mask_agx3(b, agx_register(0, AGX_SIZE_16),
-                        agx_register(2, AGX_SIZE_32), 2);
-   agx_pop_exec(b, 1);
-   agx_opt_jmp_none(b->shader);
-   agx_instr *jump = list_entry(else_->link.next, agx_instr, link);
-   EXPECT_EQ(jump->op, AGX_OPCODE_JMP_EXEC_NONE_AFTER);
-   EXPECT_EQ(jump->target, body);
+   agx_index value = agx_mov(b, agx_register(0, AGX_SIZE_32));
+   agx_instr *mul = agx_fmul_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                              value, agx_uniform(158, AGX_SIZE_32));
+   agx_instr *add = agx_fadd_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                              value, agx_uniform(158, AGX_SIZE_32));
+   /* U*R+U and R*U+imm (native terrain/sky witnesses) keep both uniforms. */
+   agx_instr *fma = agx_ffma_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                agx_uniform(56, AGX_SIZE_32), value,
+                                agx_uniform(56, AGX_SIZE_32));
+   agx_instr *fma_imm = agx_ffma_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                    value, agx_uniform(70, AGX_SIZE_16),
+                                    agx_immediate_f(1.0));
+   /* An absolute uniform is not a witnessed operand. */
+   agx_instr *add_abs = agx_fadd_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                    value,
+                                    agx_abs(agx_uniform(158, AGX_SIZE_32)));
+   agx_lower_uniform_sources(b->shader);
+   EXPECT_EQ(mul->src[1].type, AGX_INDEX_UNIFORM);
+   EXPECT_EQ(add->src[1].type, AGX_INDEX_UNIFORM);
+   EXPECT_EQ(fma->src[0].type, AGX_INDEX_UNIFORM);
+   EXPECT_EQ(fma->src[2].type, AGX_INDEX_UNIFORM);
+   EXPECT_EQ(fma_imm->src[1].type, AGX_INDEX_UNIFORM);
+   EXPECT_NE(add_abs->src[1].type, AGX_INDEX_UNIFORM);
    ralloc_free(mem);
 }
+
+TEST(AGX3UniformALU, M5SelectKeepsWitnessedUniformComparand)
+{
+   for (unsigned cond : {(unsigned)AGX_FCOND_GE, (unsigned)AGX_FCOND_LTN,
+                         (unsigned)AGX_FCOND_EQ}) {
+      for (bool zero : {true, false}) {
+         void *mem = ralloc_context(NULL);
+         agx_shader_key key = {};
+         key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+         agx_builder *b = agx_test_builder(mem);
+         b->shader->key = &key;
+         b->shader->stage = MESA_SHADER_FRAGMENT;
+         agx_index x = agx_mov(b, agx_register(0, AGX_SIZE_32));
+         agx_index y = agx_mov(b, agx_register(2, AGX_SIZE_32));
+         /* LTN normalizes to GE with the selected operands exchanged, so
+          * its zero belongs in the other slot. */
+         const bool exchanged = cond == AGX_FCOND_LTN;
+         agx_index imm = agx_immediate(zero ? 0 : 1);
+         agx_instr *I = agx_fcmpsel_to(b, agx_temp(b->shader, AGX_SIZE_32),
+                                       agx_uniform(40, AGX_SIZE_32), x,
+                                       exchanged ? y : imm,
+                                       exchanged ? imm : y,
+                                       (enum agx_fcond)cond);
+         agx_lower_uniform_sources(b->shader);
+         EXPECT_EQ(I->src[0].type == AGX_INDEX_UNIFORM,
+                   zero && cond != AGX_FCOND_EQ);
+         ralloc_free(mem);
+      }
+   }
+}
+
+TEST(AGX3UniformALU, M5FloatUniformEightCompletionSlots)
+{
+   const agx3_uniform_float_src src[3] = {
+      {AGX3_FLOAT_REGISTER, 26}, {AGX3_FLOAT_UNIFORM, 294},
+      {AGX3_FLOAT_REGISTER, 40},
+   };
+   /* ADD has endpoints 0..3 only; a multi-producer wait takes its 12-byte
+    * ep3, as Minecraft's terrain vertex shaders need. */
+   for (auto op : {AGX3_UNIFORM_FADD, AGX3_UNIFORM_FMUL, AGX3_UNIFORM_FMA}) {
+      std::set<std::string> unique;
+      for (unsigned wait = 0; wait < 256; ++wait) {
+         uint8_t bytes[16], length = 0;
+         ASSERT_TRUE(agx3_uniform_float_encode(AGX_ISA_ENCODING_AGX3C, op,
+                      26, src, false, true, wait, bytes, &length));
+         unique.emplace(reinterpret_cast<char *>(bytes), length);
+      }
+      EXPECT_EQ(unique.size(), 256u);
+   }
+}
+
+TEST(AGX3UniformALU, M5NativeIntegerUniformSelectWitnesses)
+{
+   {
+      agx3_isel_desc d = {};
+      d.dest = 6;
+      d.dest_cache = true;
+      d.cmp0 = 28;
+      d.cmp1 = 30;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)1;
+      d.explicit_hints = true;
+      d.sel_true = 28;
+      d.sel_false = 30;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x32,0x1d,0x2e,0x1f,0x05,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 0;
+      d.dest_cache = true;
+      d.cmp0 = 26;
+      d.cmp1 = 30;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)1;
+      d.explicit_hints = true;
+      d.sel_true = 26;
+      d.sel_false = 30;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x02,0x1b,0x2e,0x1f,0x05,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 12;
+      d.dest_cache = true;
+      d.cmp0 = 4;
+      d.cmp1 = 16;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)5;
+      d.explicit_hints = true;
+      d.sel_true = 4;
+      d.sel_false = 16;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x62,0x05,0x2e,0x11,0x07,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 10;
+      d.dest_cache = true;
+      d.cmp0 = 6;
+      d.cmp1 = 16;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)5;
+      d.explicit_hints = true;
+      d.sel_true = 6;
+      d.sel_false = 16;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x52,0x07,0x2e,0x11,0x07,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 6;
+      d.dest_cache = true;
+      d.cmp0 = 4;
+      d.cmp1 = 16;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)5;
+      d.explicit_hints = true;
+      d.sel_true = 4;
+      d.sel_false = 16;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x32,0x05,0x2e,0x11,0x07,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 4;
+      d.dest_cache = true;
+      d.cmp0 = 4;
+      d.cmp1 = 18;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)5;
+      d.explicit_hints = true;
+      d.sel_true = 4;
+      d.sel_false = 18;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x22,0x05,0x2e,0x13,0x07,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 16;
+      d.dest_cache = true;
+      d.cmp0 = 14;
+      d.cmp1 = 18;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)5;
+      d.explicit_hints = true;
+      d.sel_true = 14;
+      d.sel_false = 18;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x82,0x0f,0x2e,0x13,0x07,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 14;
+      d.dest_cache = true;
+      d.cmp0 = 14;
+      d.cmp1 = 18;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)5;
+      d.explicit_hints = true;
+      d.sel_true = 14;
+      d.sel_false = 18;
+      d.true_kind = AGX3_ISEL_REGISTER;
+      d.false_kind = AGX3_ISEL_UNIFORM;
+      d.wait_mask = 0;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = true;
+      d.discard[3] = false;
+      const uint8_t expected[] = {0x72,0x0f,0x2e,0x13,0x07,0x02};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 10;
+      d.dest_cache = true;
+      d.cmp0 = 2;
+      d.cmp1 = 154;
+      d.cmp1_immediate = false;
+      d.cmp1_uniform = true;
+      d.cond = (agx3_isel_icond)1;
+      d.explicit_hints = true;
+      d.sel_true = 156;
+      d.sel_false = 0;
+      d.true_kind = AGX3_ISEL_UNIFORM;
+      d.false_kind = AGX3_ISEL_REGISTER;
+      d.wait_mask = 1;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = false;
+      d.discard[3] = true;
+      const uint8_t expected[] = {0x52,0x03,0x3f,0x1b,0xa2,0x1c,0x05,0x22,0x80,0x00};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 4;
+      d.dest_cache = false;
+      d.cmp0 = 40;
+      d.cmp1 = 0;
+      d.cmp1_immediate = true;
+      d.cmp1_uniform = false;
+      d.cond = (agx3_isel_icond)4;
+      d.explicit_hints = true;
+      d.sel_true = 144;
+      d.sel_false = 36;
+      d.true_kind = AGX3_ISEL_UNIFORM;
+      d.false_kind = AGX3_ISEL_REGISTER;
+      d.wait_mask = 0;
+      d.cache[0] = true;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = false;
+      d.discard[1] = false;
+      d.discard[2] = false;
+      d.discard[3] = true;
+      const uint8_t expected[] = {0x22,0xa9,0x07,0x80,0xa6,0x10,0x07,0x02,0x80,0x24};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+   {
+      agx3_isel_desc d = {};
+      d.dest = 6;
+      d.dest_cache = false;
+      d.cmp0 = 40;
+      d.cmp1 = 0;
+      d.cmp1_immediate = true;
+      d.cmp1_uniform = false;
+      d.cond = (agx3_isel_icond)4;
+      d.explicit_hints = true;
+      d.sel_true = 144;
+      d.sel_false = 14;
+      d.true_kind = AGX3_ISEL_UNIFORM;
+      d.false_kind = AGX3_ISEL_REGISTER;
+      d.wait_mask = 1;
+      d.cache[0] = false;
+      d.cache[1] = false;
+      d.cache[2] = false;
+      d.cache[3] = false;
+      d.discard[0] = true;
+      d.discard[1] = false;
+      d.discard[2] = false;
+      d.discard[3] = true;
+      const uint8_t expected[] = {0x32,0x29,0x0f,0x80,0xa6,0x10,0x07,0x22,0x80,0x0e};
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+      ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, sizeof(expected));
+      EXPECT_EQ(std::memcmp(bytes, expected, length), 0);
+   }
+}
+
+TEST(AGX3UniformALU, M5IntegerUniformSelectLowering)
+{
+   for (unsigned mode = 0; mode < 4; ++mode) {
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->is_preamble = mode == 1;
+      agx_index x = agx_temp(b->shader, AGX_SIZE_32);
+      agx_index bound = agx_uniform(mode == 3 ? 379 : 378, AGX_SIZE_32);
+      agx_instr *I = agx_icmpsel_to(b,
+         agx_temp(b->shader, mode == 2 ? AGX_SIZE_16 : AGX_SIZE_32),
+         x, bound, agx_uniform(374, AGX_SIZE_32), agx_immediate(0), AGX_ICOND_ULT);
+      agx_lower_uniform_sources(b->shader);
+      EXPECT_EQ(I->src[1].type == AGX_INDEX_UNIFORM, mode == 0);
+      EXPECT_EQ(I->src[2].type == AGX_INDEX_UNIFORM, mode == 0);
+      EXPECT_EQ(I->icond, AGX_ICOND_ULT);
+      ralloc_free(mem);
+   }
+}
+
+TEST(AGX3UniformALU, M5IntegerUniformSelectEightCompletionSlots)
+{
+   agx3_isel_desc d = {};
+   d.dest = 40; d.cmp0 = 36; d.cmp1 = 378; d.cmp1_uniform = true;
+   d.cond = AGX3_ISEL_ICOND_ULT; d.explicit_hints = true;
+   d.true_kind = AGX3_ISEL_UNIFORM; d.sel_true = 374;
+   d.false_kind = AGX3_ISEL_IMMEDIATE; d.sel_false = 0;
+   for (unsigned mode = 0; mode < 3; ++mode) {
+      d.cmp1 = mode == 0 ? 378 : mode == 1 ? 38 : 7;
+      d.cmp1_uniform = mode == 0;
+      d.cmp1_immediate = mode == 2;
+      d.false_kind = mode == 0 ? AGX3_ISEL_IMMEDIATE :
+                     mode == 1 ? AGX3_ISEL_REGISTER : AGX3_ISEL_UNIFORM;
+      d.sel_false = mode == 0 ? 0 : mode == 1 ? 42 : 376;
+      std::set<std::string> unique;
+      for (unsigned wait = 0; wait < 256; ++wait) {
+         d.wait_mask = wait;
+         uint8_t bytes[AGX3_ISEL_MAX_LENGTH], length = 0;
+         ASSERT_TRUE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+         unique.emplace(reinterpret_cast<char *>(bytes), length);
+      }
+      EXPECT_EQ(unique.size(), 256u);
+   }
+}
+
+TEST(AGX3UniformALU, M5IntegerUniformSelectInvalidFormsAreTransactional)
+{
+   agx3_isel_desc good = {};
+   good.dest = 40; good.cmp0 = 36; good.cmp1 = 378;
+   good.cmp1_uniform = true; good.cond = AGX3_ISEL_ICOND_ULT;
+   good.explicit_hints = true;
+   good.true_kind = AGX3_ISEL_UNIFORM; good.sel_true = 374;
+   good.false_kind = AGX3_ISEL_IMMEDIATE; good.sel_false = 0;
+   for (unsigned mode = 0; mode < 12; ++mode) {
+      agx3_isel_desc d = good;
+      switch (mode) {
+      case 0: d.cmp0_uniform = true; break;
+      case 1: d.dest_16 = true; break;
+      case 2: d.cmp_16 = true; break;
+      case 3: d.cmp_mixed_width = true; break;
+      case 4: d.cmp1 = 379; break;
+      case 5: d.cmp1 = 512; break;
+      case 6: d.cmp1_immediate = true; break;
+      case 7: d.cache[1] = true; break;
+      case 8: d.discard[1] = true; break;
+      case 9: d.sel_true = 375; break;
+      case 10: d.sel_true = 512; break;
+      case 11: d.discard[2] = true; break;
+      }
+      uint8_t bytes[AGX3_ISEL_MAX_LENGTH], expected[AGX3_ISEL_MAX_LENGTH];
+      std::memset(bytes, 0xa5, sizeof(bytes));
+      std::memset(expected, 0xa5, sizeof(expected));
+      uint8_t length = 0xa5;
+      EXPECT_FALSE(agx3_isel_encode(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      EXPECT_EQ(std::memcmp(bytes, expected, sizeof(bytes)), 0);
+      EXPECT_EQ(length, 0xa5);
+   }
+}
+
+TEST(AGX3UniformALU, M5NativeUniformBranchWitnesses)
+{
+   uint8_t out[AGX3_CF_MAX_LENGTH], length = 0;
+   const uint8_t reg_uniform[] = {0x0a,0x05,0x23,0x09,0x06,0x00,0x07,0x02,0x00,0x00};
+   ASSERT_TRUE(agx3_cf_icmp_reg_uniform32(AGX_ISA_ENCODING_AGX3C,
+      AGX3_CF_ICOND_EQ, 0, 4, 8, {false,false}, out, &length));
+   ASSERT_EQ(length, sizeof(reg_uniform));
+   EXPECT_EQ(std::memcmp(out, reg_uniform, length), 0);
+   const uint8_t uniform_zero[] = {0x0a,0x09,0x23,0x80,0x06,0x00,0x87,0x02,0x00,0x00};
+   ASSERT_TRUE(agx3_cf_icmp_uniform_imm(AGX_ISA_ENCODING_AGX3C,
+      AGX3_CF_ICOND_EQ, 0, 8, 0, out, &length));
+   ASSERT_EQ(length, sizeof(uniform_zero));
+   EXPECT_EQ(std::memcmp(out, uniform_zero, length), 0);
+   EXPECT_FALSE(agx3_cf_icmp_uniform_imm(AGX_ISA_ENCODING_AGX3C,
+      AGX3_CF_ICOND_EQ, 0, 9, 0, out, &length));
+   EXPECT_FALSE(agx3_cf_icmp_reg_uniform32(AGX_ISA_ENCODING_AGX3C,
+      AGX3_CF_ICOND_EQ, 0, 4, 256, {false,false}, out, &length));
+}
diff --git a/src/asahi/compiler/test/test-demote-early-kill.cpp b/src/asahi/compiler/test/test-demote-early-kill.cpp
index e1ca05029448..fa59e3a956d5 100644
--- a/src/asahi/compiler/test/test-demote-early-kill.cpp
+++ b/src/asahi/compiler/test/test-demote-early-kill.cpp
@@ -27,6 +27,9 @@ enum shape {
    SUBGROUP_AFTER,
    NESTED_GUARDED,
    QUAD_BEFORE_SECOND_DEMOTE,
+   ACTIVE_COUNT_AFTER,
+   ACTIVE_INDEX_AFTER,
+   LOWERED_QUAD_BALLOT_AFTER,
 };
 
 static nir_shader *
@@ -156,6 +159,15 @@ build(enum shape s)
    case SUBGROUP_AFTER:
       out = nir_u2f32(&b, nir_channel(&b, nir_ballot(&b, 1, 32, d), 0));
       break;
+   case ACTIVE_COUNT_AFTER:
+      out = nir_u2f32(&b, nir_load_active_subgroup_count_agx(&b, 32));
+      break;
+   case ACTIVE_INDEX_AFTER:
+      out = nir_u2f32(&b, nir_load_active_subgroup_invocation_agx(&b, 32));
+      break;
+   case LOWERED_QUAD_BALLOT_AFTER:
+      out = nir_u2f32(&b, nir_quad_ballot_agx(&b, 16, d));
+      break;
    case QUAD_ONLY_BEFORE:
       out = nir_fadd(&b, out, x);
       break;
@@ -281,3 +293,76 @@ TEST_F(DemoteEarlyKill, EarlierDemoteReachesQuadBeforeLaterDemote)
 {
    EXPECT_FALSE(lower(QUAD_BEFORE_SECOND_DEMOTE));
 }
+
+TEST_F(DemoteEarlyKill, LoweredLaneOperationsKeepDeferredKill)
+{
+   EXPECT_FALSE(lower(ACTIVE_COUNT_AFTER));
+   EXPECT_FALSE(lower(ACTIVE_INDEX_AFTER));
+   EXPECT_FALSE(lower(LOWERED_QUAD_BALLOT_AFTER));
+
+   for (enum shape s : {ACTIVE_COUNT_AFTER, ACTIVE_INDEX_AFTER}) {
+      nir_shader *nir = build(s);
+      agx_nir_lower_discard_zs_emit(nir);
+      agx_nir_lower_sample_mask_deferred(nir);
+      nir_validate_shader(nir, "lowered subgroup operations");
+      EXPECT_EQ(count_quad_ballots(nir), 0u);
+      ralloc_free(nir);
+   }
+}
+
+/* M5 final coverage update: the testing form (all samples targeted, the
+ * surviving set live) runs in the then-arm of an if on the shader part's
+ * tests_zs uniform, and the pure kill in its else-arm. No coverage update
+ * takes its target from tests_zs | killed, which the per-lane test split
+ * would otherwise turn into a divergent testing arm. */
+static void
+expect_m5_final_update(nir_shader *s)
+{
+   unsigned testing = 0, fused = 0;
+   nir_foreach_block(block, nir_shader_get_entrypoint(s)) {
+      nir_cf_node *parent = block->cf_node.parent;
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         if (intr->intrinsic != nir_intrinsic_sample_mask_agx)
+            continue;
+         const bool live_zero = nir_src_is_const(intr->src[1]) &&
+                                nir_src_as_uint(intr->src[1]) == 0;
+         if (!nir_src_is_const(intr->src[0])) {
+            nir_instr *t = nir_def_instr(intr->src[0].ssa);
+            if (t->type == nir_instr_type_alu &&
+                nir_instr_as_alu(t)->op == nir_op_ior)
+               fused++;
+         }
+         if (live_zero)
+            continue;
+         testing++;
+         EXPECT_TRUE(nir_src_is_const(intr->src[0]) &&
+                     (nir_src_as_uint(intr->src[0]) & 0xff) == 0xff);
+         ASSERT_EQ(parent->type, nir_cf_node_if);
+         nir_if *nif = nir_cf_node_as_if(parent);
+         nir_def *cond = nif->condition.ssa;
+         nir_instr *ci = nir_def_instr(cond);
+         ASSERT_EQ(ci->type, nir_instr_type_alu);
+         nir_instr *src = nir_def_instr(nir_instr_as_alu(ci)->src[0].src.ssa);
+         ASSERT_EQ(src->type, nir_instr_type_intrinsic);
+         EXPECT_EQ(nir_instr_as_intrinsic(src)->intrinsic,
+                   nir_intrinsic_load_shader_part_tests_zs_agx);
+      }
+   }
+   EXPECT_EQ(testing, 1u);
+   EXPECT_EQ(fused, 0u);
+}
+
+TEST_F(DemoteEarlyKill, M5FinalCoverageUpdateIsUniform)
+{
+   for (enum shape s : {UNGUARDED, GUARDED_NOT, QUAD_IN_LOOP}) {
+      nir_shader *nir = build(s);
+      agx_nir_lower_discard_zs_emit(nir);
+      agx_nir_lower_sample_mask_deferred_m5(nir);
+      nir_validate_shader(nir, "M5 coverage update");
+      expect_m5_final_update(nir);
+      ralloc_free(nir);
+   }
+}
diff --git a/src/asahi/compiler/test/test-device-load-batches.cpp b/src/asahi/compiler/test/test-device-load-batches.cpp
new file mode 100644
index 000000000000..f4c4011f5433
--- /dev/null
+++ b/src/asahi/compiler/test/test-device-load-batches.cpp
@@ -0,0 +1,88 @@
+/* Copyright 2026 Aurora Silicon; SPDX-License-Identifier: MIT */
+#include <gtest/gtest.h>
+#include <array>
+extern "C" {
+#include "agx3_slot_core.h"
+}
+
+static agx3_slot_instr
+load(unsigned reg, bool join = true, bool prefer = false)
+{
+   agx3_slot_instr i{};
+   i.is_async = true;
+   i.nr_dsts = 1;
+   i.dst[0] = reg;
+   i.join_class = join ? 1 : 0;
+   i.join_limit = join ? 8 : 0;
+   i.prefer_join = prefer;
+   return i;
+}
+
+TEST(DeviceLoadBatches, IndependentReadsShareQueues)
+{
+   std::array<agx3_slot_instr, 32> i;
+   std::array<agx3_slot_result, 32> r;
+   for (unsigned n = 0; n < i.size(); ++n) i[n] = load(n);
+   ASSERT_TRUE(agx3_slot_assign(i.data(), i.size(), 8, r.data()));
+   for (unsigned n = 0; n < i.size(); ++n) {
+      EXPECT_EQ(r[n].wait_mask, 0);
+      EXPECT_EQ(r[n].slot, n % 8 + 1);
+   }
+}
+
+TEST(DeviceLoadBatches, RetiredAliasesDoNotConsumeNewGeneration)
+{
+   std::array<agx3_slot_instr, 37> i{};
+   std::array<agx3_slot_result, 37> r;
+   for (unsigned n = 0; n < 32; ++n) i[n] = load(n);
+   i[32].nr_srcs = i[33].nr_srcs = i[35].nr_srcs = i[36].nr_srcs = 1;
+   i[32].src[0] = 0; i[33].src[0] = 8;
+   i[34] = load(40); i[34].preferred_slot = 1;
+   i[35].src[0] = 16; i[36].src[0] = 40;
+   ASSERT_TRUE(agx3_slot_assign(i.data(), i.size(), 8, r.data()));
+   EXPECT_EQ(r[32].wait_mask, 1);
+   EXPECT_EQ(r[33].wait_mask, 0);
+   EXPECT_EQ(r[35].wait_mask, 0);
+   EXPECT_EQ(r[36].wait_mask, 1);
+}
+
+TEST(DeviceLoadBatches, BoundAndMixedEngineRetirement)
+{
+   std::array<agx3_slot_instr, 9> i;
+   std::array<agx3_slot_result, 9> r;
+   for (unsigned n = 0; n < i.size(); ++n) i[n] = load(n);
+   ASSERT_TRUE(agx3_slot_assign(i.data(), i.size(), 1, r.data()));
+   for (unsigned n = 0; n < 8; ++n) EXPECT_EQ(r[n].wait_mask, 0);
+   EXPECT_EQ(r[8].wait_mask, 1);
+   i[2] = load(15, false); i[2].preferred_slot = 1;
+   ASSERT_TRUE(agx3_slot_assign(i.data(), 3, 1, r.data()));
+   EXPECT_EQ(r[2].wait_mask, 1);
+   i[2] = {}; i[2].nr_dsts = 1; i[2].dst[0] = 0;
+   ASSERT_TRUE(agx3_slot_assign(i.data(), 3, 1, r.data()));
+   EXPECT_EQ(r[2].wait_mask, 1);
+}
+
+TEST(DeviceLoadBatches, GroupedPolicyAndLegacyPolicy)
+{
+   std::array<agx3_slot_instr, 32> i;
+   std::array<agx3_slot_result, 32> r;
+   for (unsigned n = 0; n < i.size(); ++n) i[n] = load(n, true, true);
+   ASSERT_TRUE(agx3_slot_assign(i.data(), i.size(), 8, r.data()));
+   for (unsigned n = 0; n < i.size(); ++n) {
+      EXPECT_EQ(r[n].wait_mask, 0);
+      EXPECT_EQ(r[n].slot, n / 8 + 1);
+   }
+   for (unsigned n = 0; n < i.size(); ++n) i[n] = load(n, false);
+   ASSERT_TRUE(agx3_slot_assign(i.data(), 9, 8, r.data()));
+   EXPECT_NE(r[8].wait_mask, 0);
+}
+
+TEST(DeviceLoadBatches, InvalidContractsFailClosed)
+{
+   agx3_slot_instr i = load(0);
+   agx3_slot_result r;
+   i.join_limit = 0; EXPECT_FALSE(agx3_slot_assign(&i, 1, 8, &r));
+   i.join_limit = 9; EXPECT_FALSE(agx3_slot_assign(&i, 1, 8, &r));
+   i.join_limit = 8; i.is_async = false;
+   EXPECT_FALSE(agx3_slot_assign(&i, 1, 8, &r));
+}
diff --git a/src/asahi/compiler/test/test-fminmax-constant-denorm.cpp b/src/asahi/compiler/test/test-fminmax-constant-denorm.cpp
index 6bd4e233fb1d..6bdc011d8498 100644
--- a/src/asahi/compiler/test/test-fminmax-constant-denorm.cpp
+++ b/src/asahi/compiler/test/test-fminmax-constant-denorm.cpp
@@ -224,3 +224,141 @@ TEST(FminmaxConstantDenorm, FlushOnlyInFlushToZeroShaders)
       }
    }
 }
+
+static nir_def *
+unknown_word(nir_builder *b)
+{
+   return nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .range = 4);
+}
+
+static void
+check_shared(uint32_t bits, bool arithmetic, bool mixed, bool narrow,
+             unsigned alignment, bool preserve, bool expect_removed)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_COMPUTE, &agx_nir_options, "shared-minmax");
+   b.shader->info.workgroup_size[0] = 32;
+   b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+   b.shader->info.shared_size = 16;
+   if (preserve)
+      b.shader->info.float_controls_execution_mode |=
+         FLOAT_CONTROLS_DENORM_PRESERVE_FP32;
+   nir_def *stored = arithmetic
+                        ? nir_fmul(&b, unknown_word(&b), nir_imm_float(&b, 2))
+                        : nir_imm_int(&b, bits);
+   nir_store_shared(&b, stored, nir_imm_int(&b, 0), .align_mul = 4);
+   if (mixed)
+      nir_store_shared(&b, nir_imm_int(&b, 1), nir_imm_int(&b, 4),
+                       .align_mul = 4);
+   if (narrow)
+      nir_store_shared(&b, nir_imm_intN_t(&b, 1, 16), nir_imm_int(&b, 8),
+                       .align_mul = 2);
+   nir_def *load =
+      nir_load_shared(&b, 1, 32, nir_imm_int(&b, 0), .align_mul = alignment);
+   nir_def *out = nir_fmin(&b, load, nir_imm_float(&b, 1));
+   nir_store_ssbo(&b, out, nir_imm_int(&b, 0), nir_imm_int(&b, 0),
+                  .write_mask = 1, .align_mul = 4);
+   ASSERT_TRUE(agx_nir_lower_fminmax(b.shader, false));
+   unsigned before = fadds(b.shader);
+   ASSERT_EQ(before, 1u);
+   EXPECT_EQ(agx_nir_opt_canonical_minmax(b.shader), expect_removed);
+   EXPECT_EQ(fadds(b.shader), before - unsigned(expect_removed));
+   nir_validate_shader(b.shader, "shared-minmax result");
+   ralloc_free(b.shader);
+}
+
+TEST(FminmaxProvenance, SharedArithmetic)
+{
+   check_shared(0, true, false, false, 4, false, true);
+}
+TEST(FminmaxProvenance, SharedNormalConstant)
+{
+   check_shared(0x3f800000, false, false, false, 4, false, true);
+}
+TEST(FminmaxProvenance, SharedSubnormalConstant)
+{
+   check_shared(1, false, false, false, 4, false, false);
+}
+TEST(FminmaxProvenance, MixedUnknownWord)
+{
+   check_shared(0, true, true, false, 4, false, false);
+}
+TEST(FminmaxProvenance, NarrowWrite)
+{
+   check_shared(0, true, false, true, 4, false, false);
+}
+TEST(FminmaxProvenance, UnalignedRead)
+{
+   check_shared(0, true, false, false, 2, false, false);
+}
+TEST(FminmaxProvenance, DenormPreserve)
+{
+   check_shared(0, true, false, false, 4, true, false);
+}
+TEST(FminmaxProvenance, SharedSignallingNan)
+{
+   check_shared(0x7f800001, false, false, false, 4, false, false);
+}
+
+static void
+check_select(bool unknown)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_COMPUTE, &agx_nir_options, "select-minmax");
+   nir_def *x = unknown_word(&b);
+   nir_def *a = nir_fmul(&b, x, nir_imm_float(&b, 2));
+   nir_def *value = nir_bcsel(&b, nir_ine_imm(&b, x, 0), a,
+                              unknown ? x : nir_imm_float(&b, 0));
+   nir_def *out = nir_fmax(&b, value, nir_imm_float(&b, -1));
+   nir_store_ssbo(&b, out, nir_imm_int(&b, 0), nir_imm_int(&b, 0),
+                  .write_mask = 1, .align_mul = 4);
+   ASSERT_TRUE(agx_nir_lower_fminmax(b.shader, false));
+   /* The lowering may already prove a canonical select; either pass must
+    * leave exactly the unknown input's flush. */
+   const unsigned before = fadds(b.shader);
+   ASSERT_LE(before, 1u);
+   EXPECT_EQ(agx_nir_opt_canonical_minmax(b.shader), before > unsigned(unknown));
+   EXPECT_EQ(fadds(b.shader), unsigned(unknown));
+   nir_validate_shader(b.shader, "select-minmax result");
+   ralloc_free(b.shader);
+}
+TEST(FminmaxProvenance, CanonicalSelect)
+{
+   check_select(false);
+}
+TEST(FminmaxProvenance, UnknownSelect)
+{
+   check_select(true);
+}
+
+static void
+check_phi(bool unknown)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_COMPUTE, &agx_nir_options, "phi-minmax");
+   nir_def *x = unknown_word(&b);
+   nir_push_if(&b, nir_ine_imm(&b, x, 0));
+   nir_def *a = nir_fmul(&b, x, nir_imm_float(&b, 2));
+   nir_push_else(&b, nullptr);
+   nir_def *other = unknown ? x : nir_imm_float(&b, 0);
+   nir_pop_if(&b, nullptr);
+   nir_def *value = nir_if_phi(&b, a, other);
+   nir_def *out = nir_fmax(&b, value, nir_imm_float(&b, -1));
+   nir_store_ssbo(&b, out, nir_imm_int(&b, 0), nir_imm_int(&b, 0),
+                  .write_mask = 1, .align_mul = 4);
+   ASSERT_TRUE(agx_nir_lower_fminmax(b.shader, false));
+   const unsigned before = fadds(b.shader);
+   ASSERT_LE(before, 1u);
+   EXPECT_EQ(agx_nir_opt_canonical_minmax(b.shader), before > unsigned(unknown));
+   EXPECT_EQ(fadds(b.shader), unsigned(unknown));
+   nir_validate_shader(b.shader, "phi-minmax result");
+   ralloc_free(b.shader);
+}
+TEST(FminmaxProvenance, CanonicalPhi)
+{
+   check_phi(false);
+}
+TEST(FminmaxProvenance, UnknownPhi)
+{
+   check_phi(true);
+}
diff --git a/src/asahi/compiler/test/test-inline-mrt.cpp b/src/asahi/compiler/test/test-inline-mrt.cpp
index 57665926fcbd..6f51e57130eb 100644
--- a/src/asahi/compiler/test/test-inline-mrt.cpp
+++ b/src/asahi/compiler/test/test-inline-mrt.cpp
@@ -8,6 +8,7 @@ extern "C" {
 #include "agx_nir.h"
 }
 #include <vector>
+#include <algorithm>
 
 static std::vector<uint8_t>
 compile_mrt(bool secondary, unsigned targets, bool *ok, bool no_stop = false)
@@ -60,3 +61,40 @@ TEST(InlineMRT, IncompleteMainPartStillRejected)
    auto bytes = compile_mrt(false, 2, &ok, true);
    EXPECT_FALSE(ok);
 }
+
+TEST(Background, M5MultisampleTileTransaction)
+{
+   for (unsigned samples : {2u, 4u, 8u}) {
+      SCOPED_TRACE(samples);
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_FRAGMENT, &agx_nir_options, "M5 background transaction");
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.dev.chip = AGX_CHIP_G17S;
+      key.no_preamble = true;
+      key.reserved_preamble = 8;
+      key.fs.background = true;
+      key.fs.nr_samples = samples;
+      nir_def *value = nir_load_preamble(&b, 4, 32, .base = 4);
+      nir_store_local_pixel_agx(
+         &b, value, nir_u2u16(&b, nir_load_sample_mask(&b)),
+         nir_undef(&b, 2, 16), .write_mask = 0xf,
+         .format = PIPE_FORMAT_R8_UNORM);
+      b.shader->info.outputs_written = BITFIELD64_BIT(FRAG_RESULT_DATA0);
+      agx_preprocess_nir(b.shader);
+      agx_shader_part part = {};
+      ASSERT_TRUE(agx_compile_shader_nir(b.shader, &key, &part));
+      const uint8_t acquire[] = {0x67, 0x40, 0, 5, 0, 0};
+      const uint8_t release[] = {0x67, 0x10, 0, 4, 0, 0};
+      const uint8_t *begin = (uint8_t *)part.binary + part.info.main_offset;
+      const uint8_t *end = begin + part.info.main_size;
+      const uint8_t *wait = std::search(begin, end, std::begin(acquire),
+                                       std::end(acquire));
+      const uint8_t *signal = std::search(begin, end, std::begin(release),
+                                         std::end(release));
+      EXPECT_NE(wait, end);
+      EXPECT_NE(signal, end);
+      EXPECT_LT(wait, signal);
+      free(part.binary);
+   }
+}
diff --git a/src/asahi/compiler/test/test-lower-divmod64.cpp b/src/asahi/compiler/test/test-lower-divmod64.cpp
index 002726428594..f2dbf550addc 100644
--- a/src/asahi/compiler/test/test-lower-divmod64.cpp
+++ b/src/asahi/compiler/test/test-lower-divmod64.cpp
@@ -131,26 +131,30 @@ TEST(LowerDivmod64, MatchesReference)
    const nir_shader_compiler_options options = unroll_options();
    const nir_op ops[] = {nir_op_udiv, nir_op_umod, nir_op_idiv, nir_op_irem,
                          nir_op_imod};
-   for (nir_op op : ops) {
-      const bool is_signed = op == nir_op_idiv || op == nir_op_irem ||
-                             op == nir_op_imod;
-      for (uint64_t n : values) {
-         for (uint64_t d : values) {
-            if (d == 0)
-               continue;
-            /* INT64_MIN / -1 overflows and is undefined. */
-            if (is_signed && n == 0x8000000000000000ull &&
-                d == ~UINT64_C(0))
-               continue;
-            nir_shader *shader = make_program(&options, op, 1);
-            ASSERT_TRUE(agx_nir_lower_divmod64(shader));
-            nir_validate_shader(shader, "after divmod64 lowering");
-            uint64_t got = 0;
-            ASSERT_TRUE(evaluate(shader, &n, &d, 1, &got))
-               << nir_op_infos[op].name << " " << n << " " << d;
-            EXPECT_EQ(got, reference(op, n, d))
-               << nir_op_infos[op].name << " " << std::hex << n << " / " << d;
-            ralloc_free(shader);
+   /* Both forms of the loop's 64-bit add and subtract (G17P and M5). */
+   for (bool native_add64 : {false, true}) {
+      for (nir_op op : ops) {
+         const bool is_signed = op == nir_op_idiv || op == nir_op_irem ||
+                                op == nir_op_imod;
+         for (uint64_t n : values) {
+            for (uint64_t d : values) {
+               if (d == 0)
+                  continue;
+               /* INT64_MIN / -1 overflows and is undefined. */
+               if (is_signed && n == 0x8000000000000000ull &&
+                   d == ~UINT64_C(0))
+                  continue;
+               nir_shader *shader = make_program(&options, op, 1);
+               ASSERT_TRUE(agx_nir_lower_divmod64(shader, native_add64));
+               nir_validate_shader(shader, "after divmod64 lowering");
+               uint64_t got = 0;
+               ASSERT_TRUE(evaluate(shader, &n, &d, 1, &got))
+                  << nir_op_infos[op].name << " " << n << " " << d;
+               EXPECT_EQ(got, reference(op, n, d))
+                  << nir_op_infos[op].name << " " << std::hex << n << " / "
+                  << d;
+               ralloc_free(shader);
+            }
          }
       }
    }
@@ -162,7 +166,7 @@ TEST(LowerDivmod64, VectorComponentsAreIndependent)
    const uint64_t n[4] = {100, 0xffff859a3bf78592ull, 7, 0x123456789abcdefull};
    const uint64_t d[4] = {7, 3, 0x100000000ull, 0xffffffffull};
    nir_shader *shader = make_program(&options, nir_op_idiv, 4);
-   ASSERT_TRUE(agx_nir_lower_divmod64(shader));
+   ASSERT_TRUE(agx_nir_lower_divmod64(shader, false));
    uint64_t got[4] = {0};
    ASSERT_TRUE(evaluate(shader, n, d, 4, got));
    for (unsigned c = 0; c < 4; ++c)
@@ -178,6 +182,6 @@ TEST(LowerDivmod64, LeavesNarrowDivisionAlone)
    nir_def *d = agx_test_load_preamble(&b, 1, 32, 2);
    nir_store_global(&b, nir_udiv(&b, n, d), nir_imm_int64(&b, 0),
                     .align_mul = 4);
-   EXPECT_FALSE(agx_nir_lower_divmod64(b.shader));
+   EXPECT_FALSE(agx_nir_lower_divmod64(b.shader, false));
    ralloc_free(b.shader);
 }
diff --git a/src/asahi/compiler/test/test-lower-spill.cpp b/src/asahi/compiler/test/test-lower-spill.cpp
index b560ccea6b36..831299601235 100644
--- a/src/asahi/compiler/test/test-lower-spill.cpp
+++ b/src/asahi/compiler/test/test-lower-spill.cpp
@@ -193,3 +193,158 @@ TEST_F(LowerSpill, AGX3SplitsVectorStackAccess)
    agx3_split_vector_stack_access(A->shader);
    ASSERT_SHADER_EQUAL(A->shader, B->shader);
 }
+
+class M5SpillConstantPhi : public testing::Test {
+protected:
+   void *mem = ralloc_context(NULL);
+   agx_builder *b = agx_test_builder(mem);
+   agx_shader_key key = {};
+   M5SpillConstantPhi() {
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      b->shader->key = &key;
+   }
+   ~M5SpillConstantPhi() { ralloc_free(mem); }
+   agx_index literal(uint64_t bits, enum agx_size size = AGX_SIZE_32) {
+      agx_index x = agx_temp(b->shader, size);
+      agx_mov_imm_to(b, x, bits);
+      return x;
+   }
+   void join() { b->cursor = agx_after_block(agx_test_block(b->shader)); }
+   agx_instr *phi(agx_index x, agx_index y, agx_index dest) {
+      agx_instr *I = agx_phi_to(b, dest, 2);
+      I->src[0] = x; I->src[1] = y;
+      return I;
+   }
+   agx_instr *definition(agx_index x) {
+      agx_foreach_instr_global(b->shader, I) {
+         if (I->nr_dests == 1 && agx_is_equiv(I->dest[0], x)) return I;
+      }
+      return nullptr;
+   }
+};
+
+TEST_F(M5SpillConstantPhi, SameLiteralCopiesJoin)
+{
+   agx_index x = literal(0x3f800000), y = literal(0x3f800000);
+   agx_index copy = agx_mov(b, x);
+   join();
+   agx_index out = agx_temp(b->shader, AGX_SIZE_32);
+   phi(copy, y, out);
+   ASSERT_TRUE(agx_fold_m5_spill_constant_phis(b->shader));
+   ASSERT_EQ(definition(out)->op, AGX_OPCODE_MOV_IMM);
+   EXPECT_EQ(definition(out)->imm, 0x3f800000u);
+}
+
+TEST_F(M5SpillConstantPhi, GroundedLoopWebPreservesLiteralPayload)
+{
+   agx_index root = literal(0x7fc01234);
+   join();
+   agx_index a = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index c = agx_temp(b->shader, AGX_SIZE_32);
+   phi(root, c, a); phi(root, a, c);
+   ASSERT_TRUE(agx_fold_m5_spill_constant_phis(b->shader));
+   EXPECT_EQ(definition(a)->op, AGX_OPCODE_MOV_IMM);
+   EXPECT_EQ(definition(c)->op, AGX_OPCODE_MOV_IMM);
+   EXPECT_EQ(definition(a)->imm, 0x7fc01234u);
+   EXPECT_EQ(definition(c)->imm, 0x7fc01234u);
+}
+
+TEST_F(M5SpillConstantPhi, ChangingLoopInputRemainsPhi)
+{
+   agx_index zero = literal(0), one = literal(1);
+   join();
+   agx_index a = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index c = agx_temp(b->shader, AGX_SIZE_32);
+   phi(zero, c, a); phi(one, a, c);
+   EXPECT_FALSE(agx_fold_m5_spill_constant_phis(b->shader));
+   EXPECT_EQ(definition(a)->op, AGX_OPCODE_PHI);
+   EXPECT_EQ(definition(c)->op, AGX_OPCODE_PHI);
+}
+
+TEST_F(M5SpillConstantPhi, RootFreeCycleCannotBorrowAnotherRootsValue)
+{
+   agx_index root = literal(1);
+   join();
+   agx_index a = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index c = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index out = agx_temp(b->shader, AGX_SIZE_32);
+   phi(c, c, a); phi(a, a, c); phi(root, a, out);
+   EXPECT_FALSE(agx_fold_m5_spill_constant_phis(b->shader));
+   EXPECT_EQ(definition(out)->op, AGX_OPCODE_PHI);
+}
+
+TEST_F(M5SpillConstantPhi, RejectsUnprovenOrModifiedSources)
+{
+   for (unsigned mode = 0; mode < 9; ++mode) {
+      agx_builder *old = b;
+      b = agx_test_builder(mem); b->shader->key = &key;
+      agx_index root = literal(1), other = root;
+      switch (mode) {
+      case 0: other.abs = true; break;
+      case 1: other.neg = true; break;
+      case 2: other.memory = true; break;
+      case 3: other.size = AGX_SIZE_16; break;
+      case 4: other.channels_m1 = 1; break;
+      case 5: other = agx_uniform(2, AGX_SIZE_32); break;
+      case 6: other = agx_register(2, AGX_SIZE_32); break;
+      case 7: other = agx_immediate(1); break;
+      case 8: other = agx_undef(AGX_SIZE_32); break;
+      }
+      join();
+      agx_index out = agx_temp(b->shader, AGX_SIZE_32);
+      phi(root, other, out);
+      EXPECT_FALSE(agx_fold_m5_spill_constant_phis(b->shader));
+      EXPECT_EQ(definition(out)->op, AGX_OPCODE_PHI);
+      b = old;
+   }
+}
+
+TEST_F(M5SpillConstantPhi, LiteralWidthsAndSignedZeroBits)
+{
+   agx_index zero = literal(0), negzero = literal(0x80000000);
+   agx_index short1 = literal(0x10001, AGX_SIZE_16);
+   agx_index short2 = literal(1, AGX_SIZE_16);
+   join();
+   agx_index wide = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index narrow = agx_temp(b->shader, AGX_SIZE_16);
+   phi(zero, negzero, wide); phi(short1, short2, narrow);
+   ASSERT_TRUE(agx_fold_m5_spill_constant_phis(b->shader));
+   EXPECT_EQ(definition(wide)->op, AGX_OPCODE_PHI);
+   EXPECT_EQ(definition(narrow)->op, AGX_OPCODE_MOV_IMM);
+   EXPECT_EQ(definition(narrow)->imm, 1u);
+}
+
+TEST_F(M5SpillConstantPhi, PreservesParallelPhiPrefix)
+{
+   agx_index zero = literal(0), one = literal(1);
+   join();
+   agx_index folded = agx_temp(b->shader, AGX_SIZE_32);
+   agx_index varying = agx_temp(b->shader, AGX_SIZE_32);
+   phi(zero, zero, folded); phi(zero, one, varying);
+   agx_index result = agx_iadd(b, folded, varying, 0);
+   ASSERT_TRUE(agx_fold_m5_spill_constant_phis(b->shader));
+   EXPECT_EQ(definition(varying)->op, AGX_OPCODE_PHI);
+   EXPECT_EQ(definition(folded)->op, AGX_OPCODE_MOV_IMM);
+   EXPECT_EQ(definition(result)->op, AGX_OPCODE_IADD);
+   bool body = false;
+   agx_foreach_block(b->shader, block) {
+      body = false;
+      agx_foreach_instr_in_block(block, I) {
+         if (I->op != AGX_OPCODE_PHI) body = true;
+         else EXPECT_FALSE(body);
+      }
+   }
+}
+
+TEST_F(M5SpillConstantPhi, G17PAndConstantProgramsRemainUnchanged)
+{
+   agx_index root = literal(1); join();
+   agx_index out = agx_temp(b->shader, AGX_SIZE_32);
+   phi(root, root, out);
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3B;
+   EXPECT_FALSE(agx_fold_m5_spill_constant_phis(b->shader));
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   b->shader->is_preamble = true;
+   EXPECT_FALSE(agx_fold_m5_spill_constant_phis(b->shader));
+   EXPECT_EQ(definition(out)->op, AGX_OPCODE_PHI);
+}
diff --git a/src/asahi/compiler/test/test-m5-copy-schedule.cpp b/src/asahi/compiler/test/test-m5-copy-schedule.cpp
new file mode 100644
index 000000000000..0be67ec4c1ef
--- /dev/null
+++ b/src/asahi/compiler/test/test-m5-copy-schedule.cpp
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+/* M5 copy loops: reorderable image reads issue ahead of the writes that
+ * consume them, as Apple's compiler does for a vertex copy loop (four SMP,
+ * then four IMGWCI). */
+
+namespace {
+
+struct copy_shader {
+   void *mem;
+   agx_shader_key key;
+   agx_builder *b;
+   agx_block *body;
+};
+
+static void
+copy_shader_init(copy_shader *s, mesa_shader_stage stage, bool loop)
+{
+   s->mem = ralloc_context(NULL);
+   s->key = {};
+   s->key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   s->key.dev.chip = AGX_CHIP_G17S;
+   s->b = agx_test_builder(s->mem);
+   s->b->shader->key = &s->key;
+   s->b->shader->stage = stage;
+   s->b->shader->nir = rzalloc(s->mem, nir_shader);
+   s->b->shader->nir->info.stage = stage;
+   s->body = agx_start_block(s->b->shader);
+   if (loop) {
+      agx_block *entry = s->body;
+      s->body = agx_test_block(s->b->shader);
+      agx_block *exit = agx_test_block(s->b->shader);
+      s->body->loop_header = true;
+      agx_block_add_successor(entry, s->body);
+      agx_block_add_successor(s->body, s->body);
+      agx_block_add_successor(s->body, exit);
+   }
+}
+
+/* Copy n texels at constant x coordinates defined in the entry block. */
+static void
+emit_copies(copy_shader *s, unsigned n, bool reorderable)
+{
+   agx_builder *b = s->b;
+   agx_index y = agx_preload(b, agx_register(2, AGX_SIZE_32));
+   agx_index z = agx_preload(b, agx_register(4, AGX_SIZE_32));
+   agx_index x[16];
+   assert(n <= 16);
+   for (unsigned i = 0; i < n; ++i)
+      x[i] = agx_mov_imm(b, 32, i);
+
+   b->cursor = agx_after_block(s->body);
+   for (unsigned i = 0; i < n; ++i) {
+      agx_index coords = agx_vec_temp(b->shader, AGX_SIZE_32, 3);
+      agx_instr *collect = agx_collect_to(b, coords, 3);
+      collect->src[0] = x[i];
+      collect->src[1] = y;
+      collect->src[2] = z;
+
+      agx_index texel = agx_vec_temp(b->shader, AGX_SIZE_32, 4);
+      agx_instr *load = agx_image_load_to(
+         b, texel, coords, agx_zero(), agx_zero(), agx_immediate(1),
+         agx_immediate(0), agx_null(), AGX_DIM_3D, AGX_LOD_MODE_AUTO_LOD, 0,
+         false, false, false);
+      load->mask = 1;
+      load->memory_can_reorder = reorderable;
+      agx_image_write(b, texel, coords, agx_zero(),
+                      agx_uniform(8, AGX_SIZE_64),
+                      agx_register(28, AGX_SIZE_32), AGX_DIM_3D, false);
+   }
+}
+
+/* Largest number of reads issued ahead of the first unissued write. */
+static unsigned
+max_reads_in_flight(agx_block *block, unsigned *first_write_at)
+{
+   unsigned loads = 0, writes = 0, max = 0, pos = 0;
+   *first_write_at = ~0u;
+   agx_foreach_instr_in_block(block, I) {
+      if (I->op == AGX_OPCODE_IMAGE_LOAD) {
+         ++loads;
+         max = MAX2(max, loads - writes);
+      } else if (I->op == AGX_OPCODE_IMAGE_WRITE) {
+         if (writes == 0)
+            *first_write_at = pos;
+         ++writes;
+      }
+      ++pos;
+   }
+   return max;
+}
+
+static unsigned
+loads_before_first_write(agx_block *block)
+{
+   unsigned loads = 0;
+   agx_foreach_instr_in_block(block, I) {
+      if (I->op == AGX_OPCODE_IMAGE_WRITE)
+         break;
+      loads += I->op == AGX_OPCODE_IMAGE_LOAD;
+   }
+   return loads;
+}
+
+} /* namespace */
+
+TEST(M5CopySchedule, VertexReadsIssueAheadOfWrites)
+{
+   copy_shader s;
+   copy_shader_init(&s, MESA_SHADER_VERTEX, false);
+   emit_copies(&s, 8, true);
+   agx_ilp_schedule(s.b->shader);
+
+   unsigned first_write;
+   EXPECT_EQ(loads_before_first_write(s.body), 4u);
+   EXPECT_EQ(max_reads_in_flight(s.body, &first_write), 4u);
+   ralloc_free(s.mem);
+}
+
+TEST(M5CopySchedule, OrderedReadsStayBehindWrites)
+{
+   /* A read that may observe an earlier write keeps memory order. */
+   copy_shader s;
+   copy_shader_init(&s, MESA_SHADER_VERTEX, false);
+   emit_copies(&s, 8, false);
+   agx_ilp_schedule(s.b->shader);
+
+   unsigned first_write;
+   EXPECT_EQ(loads_before_first_write(s.body), 1u);
+   EXPECT_EQ(max_reads_in_flight(s.body, &first_write), 1u);
+   ralloc_free(s.mem);
+}
+
+TEST(M5CopySchedule, LoopConstantsRematerializedInBody)
+{
+   copy_shader s;
+   copy_shader_init(&s, MESA_SHADER_VERTEX, true);
+   emit_copies(&s, 8, true);
+   agx_ilp_schedule(s.b->shader);
+
+   /* Every coordinate constant is now defined in the loop body. */
+   unsigned body_constants = 0;
+   agx_foreach_instr_in_block(s.body, I)
+      body_constants += I->op == AGX_OPCODE_MOV_IMM;
+   EXPECT_EQ(body_constants, 8u);
+   agx_foreach_instr_in_block(agx_start_block(s.b->shader), I)
+      EXPECT_NE(I->op, AGX_OPCODE_MOV_IMM);
+
+   unsigned first_write;
+   EXPECT_EQ(loads_before_first_write(s.body), 4u);
+   EXPECT_EQ(max_reads_in_flight(s.body, &first_write), 4u);
+   ralloc_free(s.mem);
+}
+
+TEST(M5CopySchedule, OtherGenerationsUnchanged)
+{
+   copy_shader s;
+   copy_shader_init(&s, MESA_SHADER_VERTEX, false);
+   s.key.dev.encoding = AGX_ISA_ENCODING_AGX3B;
+   s.key.dev.chip = AGX_CHIP_G17P;
+   emit_copies(&s, 8, true);
+   agx_ilp_schedule(s.b->shader);
+
+   unsigned first_write;
+   EXPECT_EQ(loads_before_first_write(s.body), 1u);
+   EXPECT_EQ(max_reads_in_flight(s.body, &first_write), 1u);
+   ralloc_free(s.mem);
+}
diff --git a/src/asahi/compiler/test/test-m5-packed-lod.cpp b/src/asahi/compiler/test/test-m5-packed-lod.cpp
new file mode 100644
index 000000000000..65e57346a02c
--- /dev/null
+++ b/src/asahi/compiler/test/test-m5-packed-lod.cpp
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: MIT */
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <cstdlib>
+#include <string>
+#include <gtest/gtest.h>
+#include "agx3_pack.h"
+#include "agx3_tex_core.h"
+#include "agx_test.h"
+
+TEST(M5PackedLOD, NativeSharedAndLiveLoadWords)
+{
+   uint8_t out[AGX3_TEX_MAX_LENGTH], len;
+   const uint8_t shared[] = {
+      0x27,0x02,0x00,0x3a,0x2d,0x00,0xaa,0x02,0x00,0x00,0x40,0x30,0x10,0x00,
+   };
+   ASSERT_TRUE(agx3_tex_pack_float_lod_m5(6,0,false,true,0,out,&len));
+   ASSERT_EQ(len, sizeof(shared));
+   EXPECT_EQ(memcmp(out, shared, len), 0);
+
+   /* The device-half value is read again after sampling. Native preserves
+    * it and waits its slot8 load directly, without a carrier or fence. */
+   const uint8_t live_load[] = {
+      0x2f,0x00,0x00,0x9a,0x29,0x00,0xaa,0x02,0x00,0x00,0x40,0x30,0x10,0x00,
+   };
+   ASSERT_TRUE(agx3_tex_pack_float_lod_m5(16,0,false,false,128,out,&len));
+   ASSERT_EQ(len, sizeof(live_load));
+   EXPECT_EQ(memcmp(out, live_load, len), 0);
+   for (unsigned waits=0; waits<256; ++waits)
+      EXPECT_TRUE(agx3_tex_pack_float_lod_m5(16,0,false,false,waits,out,&len));
+   EXPECT_FALSE(agx3_tex_pack_float_lod_m5(128,0,false,false,0,out,&len));
+   EXPECT_FALSE(agx3_tex_pack_float_lod_m5(16,1,false,false,0,out,&len));
+}
+
+TEST(M5PackedLOD, PureProducersShareThroughCSE)
+{
+   void *mem = ralloc_context(NULL);
+   agx_builder *b = agx_test_builder(mem);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_FRAGMENT;
+   agx_index half = agx_temp(b->shader, AGX_SIZE_16);
+   agx_preload_to(b, half, agx_register(6, AGX_SIZE_16));
+   agx_index word[2];
+   agx_instr *samples[2];
+   for (unsigned i=0; i<2; ++i) {
+      word[i] = agx_temp(b->shader, AGX_SIZE_32);
+      agx_pack_sampler_lod_agx3_to(b, word[i], half);
+      agx_index coords = agx_register(8, AGX_SIZE_32);
+      coords.channels_m1 = 1;
+      agx_index result = agx_vec_temp(b->shader, AGX_SIZE_32, 4);
+      samples[i] = agx_texture_sample_to(
+         b, result, coords, word[i], agx_zero(), agx_immediate(i),
+         agx_zero(), agx_null(), AGX_DIM_2D, AGX_LOD_MODE_LOD_MIN, 0,
+         false, false, false, AGX_GATHER_NONE, false);
+      samples[i]->memory_is_packed = true;
+      samples[i]->mask = 15;
+      agx_unit_test(b, result);
+   }
+   agx_opt_cse(b->shader);
+   agx_dce(b->shader, true);
+   unsigned packs = 0;
+   agx_foreach_instr_global(b->shader, I)
+      packs += I->op == AGX_OPCODE_PACK_SAMPLER_LOD_AGX3;
+   EXPECT_EQ(packs, 1u);
+   EXPECT_TRUE(agx_is_equiv(samples[0]->src[1], samples[1]->src[1]));
+   EXPECT_EQ(samples[0]->src[1].size, AGX_SIZE_32);
+   EXPECT_EQ(samples[1]->src[1].size, AGX_SIZE_32);
+   ralloc_free(mem);
+}
+
+TEST(M5PackedLOD, ReusedWordPublishesWithoutDiscardingFirstUse)
+{
+   void *mem = ralloc_context(NULL);
+   agx_builder *b = agx_test_builder(mem);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.fs.ignore_tib_dependencies = true;
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_FRAGMENT;
+   agx_index input = agx_register(6, AGX_SIZE_16);
+   input.discard = true;
+   agx_pack_sampler_lod_agx3_to(b, agx_register(0, AGX_SIZE_32), input);
+   for (unsigned i=0; i<2; ++i) {
+      agx_index word = agx_register(0, AGX_SIZE_32);
+      word.discard = i == 1;
+      agx_index coords = agx_register(8 + i*4, AGX_SIZE_32);
+      coords.channels_m1 = 1;
+      coords.discard = true;
+      agx_index result = agx_register(16 + i*8, AGX_SIZE_32);
+      result.channels_m1 = 3;
+      agx_instr *I = agx_texture_sample_to(
+         b, result, coords, word, agx_zero(), agx_immediate(i),
+         agx_zero(), agx_null(), AGX_DIM_2D, AGX_LOD_MODE_LOD_MIN, 0,
+         false, false, false, AGX_GATHER_NONE, false);
+      I->memory_is_packed = true;
+      I->mask = 15;
+   }
+   agx_stop(b);
+   util_dynarray bin;
+   util_dynarray_init(&bin, mem);
+   ASSERT_TRUE(agx3_pack_binary(b->shader, &bin));
+   /* Two independent eight-half groups retain both samples in flight. */
+   EXPECT_EQ(b->shader->agx3_staging_halfs, 16u);
+   const uint8_t first[] = {0x3b,0x80,0x21,0x04};
+   const uint8_t *start = (const uint8_t *)bin.data;
+   EXPECT_NE(std::search(start, start+bin.size, std::begin(first),
+                        std::end(first)), start+bin.size);
+   ralloc_free(mem);
+}
+
+TEST(M5PackedLOD, VertexImageReadPublishesCoordinates)
+{
+   void *mem = ralloc_context(NULL);
+   agx_builder *b = agx_test_builder(mem);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_VERTEX;
+   agx_instr *I = agx_alloc_instr(b, AGX_OPCODE_IMAGE_LOAD, 1, 6);
+   I->dest[0] = agx_register(16, AGX_SIZE_32);
+   I->dest[0].channels_m1 = 3;
+   I->src[0] = agx_register(4, AGX_SIZE_32);
+   I->src[0].channels_m1 = 1;
+   I->src[1] = agx_immediate(0);
+   I->src[2] = agx_uniform(4, AGX_SIZE_64);
+   I->src[3] = agx_register(24, AGX_SIZE_64);
+   I->src[4] = agx_immediate(0);
+   I->src[5] = agx_null();
+   I->dim = AGX_DIM_2D;
+   I->lod_mode = AGX_LOD_MODE_AUTO_LOD;
+   I->mask = 15;
+   agx_builder_insert(&b->cursor, I);
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+   EXPECT_TRUE(b->shader->agx3_staging_known);
+   EXPECT_EQ(b->shader->agx3_staging_halfs, 4u);
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+}
+
+TEST(M5PackedLOD, Bank8MinimumOwnsCompleteStagingRange)
+{
+   for (bool shadow : {false, true}) {
+      void *mem = ralloc_context(NULL);
+      agx_builder *b = agx_test_builder(mem);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.fs.ignore_tib_dependencies = true;
+      b->shader->key = &key;
+      b->shader->stage = MESA_SHADER_FRAGMENT;
+      agx_index coords = agx_register(8, AGX_SIZE_32);
+      coords.channels_m1 = 2;
+      agx_index result = agx_register(24, AGX_SIZE_32);
+      result.channels_m1 = 3;
+      agx_index bias_min = agx_register(0, AGX_SIZE_32);
+      agx_index offset = agx_register(16, AGX_SIZE_32);
+      offset.channels_m1 = shadow ? 1 : 0;
+      agx_instr *I = agx_texture_sample_to(
+         b, result, coords, bias_min, agx_zero(), agx_zero(),
+         agx_zero(), offset, shadow ? AGX_DIM_2D_ARRAY : AGX_DIM_3D,
+         AGX_LOD_MODE_AUTO_LOD_BIAS_MIN, 0, true, shadow, false,
+         AGX_GATHER_NONE, false);
+      I->mask = shadow ? 1 : 15;
+      agx_stop(b);
+      util_dynarray binary;
+      util_dynarray_init(&binary, mem);
+      EXPECT_TRUE(agx3_pack_binary(b->shader, &binary));
+      EXPECT_TRUE(b->shader->agx3_staging_known);
+      EXPECT_EQ(b->shader->agx3_staging_halfs, 10u);
+      util_dynarray_fini(&binary);
+      ralloc_free(mem);
+   }
+
+}
diff --git a/src/asahi/compiler/test/test-native-form-regressions.cpp b/src/asahi/compiler/test/test-native-form-regressions.cpp
index cc0bc8129b9c..5e69bd6b6792 100644
--- a/src/asahi/compiler/test/test-native-form-regressions.cpp
+++ b/src/asahi/compiler/test/test-native-form-regressions.cpp
@@ -1,12 +1,24 @@
 #include "agx3_alu_core.h"
+#include "agx3_falu_mod_core.h"
+#include "agx3_iadd_core.h"
 #include "agx3_pack.h"
 #include "agx3_stage_core.h"
 #include "agx3_tex_core.h"
 #include "agx_test.h"
+#include "agx3_tile_core.h"
+#include "agx3_sr_core.h"
+#include "agx3_subgroup_core.h"
+#include "agx3_matrix_core.h"
+#include "agx3_vary_core.h"
 extern "C" {
 #include "agx_nir.h"
 #include "asahi/layout/layout.h"
 }
+#include "agx_compile.h"
+#include "agx_debug.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_nir_m5_mesh.h"
+#include "agx3_mesh_core.h"
 #include <gtest/gtest.h>
 #include <vector>
 #include <csignal>
@@ -845,6 +857,109 @@ TEST(NativeForms, TransientTextureResultPreservesEarlierAsyncWriteSpan)
       ralloc_free(mem);
    }
 }
+#include <algorithm>
+
+TEST(NativeForms, M5Matrix8MetalCompilerWitnesses)
+{
+   const uint8_t zero[] = {
+      0xa7, 0x02, 0x05, 0x2a, 0x20, 0x20, 0xaf,
+      0x12, 0x28, 0x06, 0x44, 0x00, 0x02, 0x01,
+   };
+   const uint8_t accumulator[] = {
+      0x2f, 0x02, 0x05, 0x6b, 0x20, 0x20, 0xaf,
+      0x02, 0x28, 0x08, 0x14, 0x00, 0x00, 0x01,
+   };
+   agx3_matrix_mma8_desc d = {};
+   d.dest = 24;
+   d.a = 2;
+   d.b = 20;
+   d.c_zero = true;
+   d.discard[0] = d.discard[1] = true;
+   uint8_t bytes[14], length;
+   ASSERT_TRUE(agx3_matrix_encode_mma8(AGX_ISA_ENCODING_AGX3C, &d,
+                                     bytes, &length));
+   ASSERT_EQ(length, sizeof(zero));
+   EXPECT_EQ(memcmp(bytes, zero, sizeof(zero)), 0);
+
+   d.dest = d.c = 0;
+   d.a = 10;
+   d.b = 6;
+   d.c_zero = false;
+   d.discard[2] = true;
+   d.wait_mask = 0x80;
+   ASSERT_TRUE(agx3_matrix_encode_mma8(AGX_ISA_ENCODING_AGX3C, &d,
+                                     bytes, &length));
+   ASSERT_EQ(length, sizeof(accumulator));
+   EXPECT_EQ(memcmp(bytes, accumulator, sizeof(accumulator)), 0);
+}
+
+static unsigned
+m5_absolute_comparison_instrs(bool repeat_absolute)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_FRAGMENT, &agx_nir_options, "M5 absolute comparisons");
+   nir_def *input = agx_test_load_preamble(&b, 4, 32, 4);
+   nir_def *x = nir_channel(&b, input, 0);
+   nir_def *abs_x = nir_fabs(&b, x);
+   nir_def *first = nir_bcsel(&b, nir_flt(&b, abs_x, nir_channel(&b, input, 1)),
+                              nir_channel(&b, input, 2),
+                              nir_channel(&b, input, 3));
+   nir_def *second =
+      nir_bcsel(&b,
+                nir_flt(&b, repeat_absolute ? abs_x : x,
+                        nir_channel(&b, input, 2)),
+                nir_channel(&b, input, 1), nir_channel(&b, input, 3));
+   agx_test_store_local_pixel_agx(
+      &b, nir_vec4(&b, first, second, first, second),
+      nir_imm_intN_t(&b, 0xff, 16), nir_undef(&b, 2, 16), 0, 0xf,
+      (enum pipe_format)AIL_ISA_FORMAT_I32);
+   b.shader->info.outputs_written = BITFIELD64_BIT(FRAG_RESULT_DATA0);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   key.no_preamble = true;
+   key.reserved_preamble = 16;
+   key.fs.nr_samples = 1;
+   agx_preprocess_nir(b.shader);
+   agx_shader_part part = {};
+   EXPECT_TRUE(agx_compile_shader_nir(b.shader, &key, &part));
+   unsigned instrs = part.info.stats.instrs;
+   free(part.binary);
+   ralloc_free(b.shader);
+   return instrs;
+}
+
+/* M5 compares |x| through a plain copy. Comparing the same |x| twice in a
+ * block costs no more than comparing |x| and x. */
+TEST(NativeForms, M5RepeatedAbsoluteComparandSharesItsCopy)
+{
+   unsigned once = m5_absolute_comparison_instrs(false);
+   EXPECT_GT(once, 0u);
+   EXPECT_EQ(m5_absolute_comparison_instrs(true), once);
+}
+
+TEST(NativeForms, M5SignedByteToHalfReachesPacker)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   agx_instr *convert = agx_alloc_instr(b, AGX_OPCODE_CONVERT, 1, 2);
+   convert->dest[0] = agx_register(10, AGX_SIZE_16);
+   convert->src[0] = agx_immediate(AGX_CONVERT_S8_TO_F);
+   convert->src[1] = agx_register(10, AGX_SIZE_16);
+   convert->round = AGX_ROUND_RTE;
+   agx_builder_insert(&b->cursor, convert);
+   agx_stop(b);
+   util_dynarray bytes;
+   util_dynarray_init(&bytes, mem);
+   EXPECT_TRUE(agx3_pack_binary(b->shader, &bytes));
+   EXPECT_GT(bytes.size, 4u);
+   ralloc_free(mem);
+}
 
 TEST(NativeForms, BoundedUnaryRegistersUnderPressure)
 {
@@ -1031,8 +1146,17 @@ TEST(NativeForms, ImageStoreUsesDataTypeAcrossChipsAndFormats)
       {nir_type_float32, PIPE_FORMAT_R32G32B32A32_FLOAT},
       {nir_type_invalid, PIPE_FORMAT_R32_UINT},
    };
+   agx3_encoded_end end;
+   ASSERT_TRUE(agx3_pack_end(AGX_ISA_ENCODING_AGX3B, &end));
+   /* The store is the program's last instruction before its end. Code around
+    * it may differ by chip (G17P runs its own uniform-copy CSE, which changes
+    * register allocation), so compare only the store. Integer and float data
+    * each select one encoding whatever the format; a typeless store takes its
+    * type from the format. */
+   std::vector<uint8_t> reference[2];
    for (const auto &c : cases) {
-      std::vector<uint8_t> reference;
+      const bool is_float =
+         nir_alu_type_get_base_type(c.type) == nir_type_float;
       for (agx_chip chip : {AGX_CHIP_G17P, AGX_CHIP_G16G}) {
          SCOPED_TRACE(::testing::Message() << chip << " " << c.type << " "
                                           << c.format);
@@ -1056,31 +1180,21 @@ TEST(NativeForms, ImageStoreUsesDataTypeAcrossChipsAndFormats)
          ASSERT_TRUE(agx_compile_shader_nir(b.shader, &key, &part));
          ASSERT_GT(part.info.binary_size, 0u);
          const uint8_t *code = static_cast<const uint8_t *>(part.binary);
-         /* Compare the image-write packet, not unrelated register copies
-          * and scheduling in the two chip profiles. imgwci.ep0 is 16 bytes;
-          * its fixed opcode bytes are pinned by agx3_tex_encode_image_write_2d.
-          * This small shader must contain exactly one such instruction. */
-         std::vector<uint8_t> write;
-         for (unsigned i = 0; i + AGX3_IMAGE_WRITE_LENGTH <= part.info.binary_size;
-              i += 2) {
-            if (code[i] == 0xd7 && code[i + 2] == 0x54) {
-               ASSERT_TRUE(write.empty());
-               write.assign(code + i, code + i + AGX3_IMAGE_WRITE_LENGTH);
-            }
-         }
-         ASSERT_EQ(write.size(), AGX3_IMAGE_WRITE_LENGTH);
-         /* f70 is the float-versus-integer data bit, encoded at byte 13 bit 5.
-          * Check the expected type independently of cross-chip equivalence. */
-         EXPECT_EQ((write[13] >> 5) & 1,
-                   nir_alu_type_get_base_type(c.type) == nir_type_float);
-         if (reference.empty())
-            reference = write;
+         const unsigned size = part.info.binary_size;
+         ASSERT_GE(size, AGX3_IMAGE_WRITE_LENGTH + end.length);
+         EXPECT_EQ(memcmp(code + size - end.length, end.bytes, end.length), 0);
+         const uint8_t *write =
+            code + size - end.length - AGX3_IMAGE_WRITE_LENGTH;
+         std::vector<uint8_t> encoded(write, write + AGX3_IMAGE_WRITE_LENGTH);
+         if (reference[is_float].empty())
+            reference[is_float] = encoded;
          else
-            EXPECT_EQ(write, reference);
+            EXPECT_EQ(encoded, reference[is_float]);
          free(part.binary);
          ralloc_free(b.shader);
       }
    }
+   EXPECT_NE(reference[0], reference[1]);
 }
 
 TEST(NativeForms, VertexImageLoadRegisterLod)
@@ -1160,6 +1274,88 @@ TEST(NativeForms, TexturePublicationWaitPreservesUnrelatedResult)
    ralloc_free(mem);
 }
 
+/* A vertex output whose staging COPY reuses the operand staging of a sample
+ * still in flight retires that sample first. The r0 carrier used for that
+ * rewrites r0, so it must not run while a load is still writing r0 (the
+ * texturelodoffset_pcoffset *shadow_vertex position load): the retirement
+ * must also wait for the stored value's own producer. */
+TEST(NativeForms, M5VertexOutputStagingWaitCoversStoredLoad)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_VERTEX;
+   b->shader->max_reg = 63;
+
+   /* Earlier loads occupy the first slots, so the sample's slot is not the
+    * one the first output's staging COPY takes. */
+   for (unsigned i = 0; i < 3; ++i) {
+      agx_index value = agx_register(80 + 8 * i, AGX_SIZE_32);
+      value.channels_m1 = 3;
+      agx_device_load_to(b, value, agx_register(8, AGX_SIZE_64),
+                         agx_immediate(i), AGX_FORMAT_I32, 15, 0, false);
+   }
+
+   /* The CTS vertex shader's sample, as register allocation left it: the
+    * reference and the packed level/offset word in r14..r15. */
+   agx_index coords = agx_register(20, AGX_SIZE_32);
+   coords.channels_m1 = 1;
+   coords.discard = true;
+   agx_index level = agx_register(3, AGX_SIZE_16);
+   level.discard = true;
+   agx_index texture = agx_register(12, AGX_SIZE_64);
+   texture.discard = true;
+   agx_index sampler = agx_register(24, AGX_SIZE_64);
+   sampler.discard = true;
+   agx_index compare = agx_register(28, AGX_SIZE_32);
+   compare.channels_m1 = 1;
+   compare.discard = true;
+   agx_index result = agx_register(32, AGX_SIZE_32);
+   result.channels_m1 = 3;
+   agx_texture_sample_to(b, result, coords, level, agx_zero(), texture,
+                         sampler, compare, AGX_DIM_2D,
+                         AGX_LOD_MODE_LOD_MIN, 1, true, true, false,
+                         AGX_GATHER_NONE, false);
+   /* The position load into r0..r3 while the sample is in flight. */
+   agx_index position = agx_register(0, AGX_SIZE_32);
+   position.channels_m1 = 3;
+   agx_device_load_to(b, position, agx_register(8, AGX_SIZE_64),
+                      agx_register(6, AGX_SIZE_32), AGX_FORMAT_I32, 15, 0,
+                      false);
+   for (unsigned i = 0; i < 4; ++i)
+      agx_st_vary(b, agx_immediate(i), agx_register(2 * i, AGX_SIZE_32));
+   agx_st_vary(b, agx_immediate(4), agx_register(32, AGX_SIZE_32));
+   for (unsigned i = 0; i < 3; ++i)
+      agx_st_vary(b, agx_immediate(5 + i), agx_register(80 + 8 * i, AGX_SIZE_32));
+   agx_stop(b);
+
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+
+   /* The one identity carrier, found by its wait mask, waits for both the
+    * sample (staging) and the load (r0). */
+   const uint8_t *bytes = (const uint8_t *)binary.data;
+   unsigned carriers = 0;
+   for (unsigned mask = 1; mask < 0x80; ++mask) {
+      uint8_t carrier[AGX3_ALU_MAX_LENGTH], len = 0;
+      ASSERT_TRUE(agx3_alu_band_identity_wait(AGX_ISA_ENCODING_AGX3C, 0,
+                                              mask, carrier, &len));
+      for (unsigned i = 0; i + len <= binary.size; ++i) {
+         if (memcmp(bytes + i, carrier, len) != 0)
+            continue;
+         ++carriers;
+         EXPECT_EQ(util_bitcount(mask), 2u) << "mask " << mask;
+      }
+   }
+   EXPECT_EQ(carriers, 1u);
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+}
+
 TEST(NativeForms, MinmaxReusesUniformCopiedForComparison)
 {
    for (agx_fcond cond : {AGX_FCOND_LTN, AGX_FCOND_GTN}) {
@@ -3350,3 +3546,2356 @@ TEST(LegacyFragments, SideEffectsRetainUpstreamLateDepth)
    EXPECT_EQ(masks, 0u);
    ralloc_free(b.shader);
 }
+
+/* AGX3C bitwise operations with an 8-bit immediate operand, against native
+ * G17G witnesses (the native G17G compiler; port/cwit k_or1, k_and255,
+ * k_xor1, k_or1h). Each follows a load whose scoreboard wait is slot 7. */
+TEST(NativeForms, M5BitwiseImmediateWitnesses)
+{
+   struct {
+      enum agx3_alu_op op;
+      bool narrow;
+      uint16_t dest, src;
+      uint8_t imm;
+      std::vector<uint8_t> expect;
+   } rows[] = {
+      {AGX3_ALU_BOR, false, 0, 0, 1,
+       {0x2b, 0x00, 0x07, 0x80, 0x32, 0x38, 0xa3, 0x02, 0x44, 0x80}},
+      {AGX3_ALU_BAND, false, 0, 0, 255,
+       {0x2b, 0x00, 0x07, 0x80, 0x30, 0x30, 0xa3, 0x02, 0xc4, 0xbf}},
+      {AGX3_ALU_BXOR, false, 0, 0, 1,
+       {0x2b, 0x00, 0x06, 0x80, 0x32, 0x38, 0xa3, 0x02, 0x44, 0x80}},
+      {AGX3_ALU_BOR, true, 2, 2, 1,
+       {0x3b, 0x02, 0x07, 0x00, 0x32, 0x28, 0xa3, 0x02, 0x44, 0x80}},
+   };
+   for (const auto &row : rows) {
+      uint8_t out[AGX3_ALU_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_alu_bit_imm(AGX_ISA_ENCODING_AGX3C, row.op, row.narrow,
+                                   row.dest, row.src, row.imm,
+                                   AGX3_ALU_HINT_DISCARD, 0x80, out, &len))
+         << row.op;
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), row.expect) << row.op;
+   }
+}
+
+
+/* AGX3C runtime texture offsets: PACK_SMPIR into bank 6, against native G17G
+ * witnesses (port/texwit3 d_auto/d_gath: X = h2, Y = h4; d_cmp: h4/h5). */
+TEST(NativeForms, M5RuntimeTextureOffsetWitnesses)
+{
+   uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+   ASSERT_TRUE(agx3_tex_stage_offset_2d(AGX_ISA_ENCODING_AGX3C, 2, 4, 1, 0,
+                                        false, true, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x37, 0x02, 0x00, 0x0a, 0x25, 0x00, 0xaa,
+                                   0x0a, 0x00, 0x01, 0x60, 0x30, 0x00, 0x00}));
+   ASSERT_TRUE(agx3_tex_stage_gather_offset(AGX_ISA_ENCODING_AGX3C, 4, false,
+                                            0, 1, false, true, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x37, 0x00, 0x00, 0x2a, 0x25, 0x00, 0xaa,
+                                   0x0a, 0x40, 0x01, 0x60, 0x30, 0x00, 0x00}));
+}
+
+/* Native G17G (texwit3) SMPs that read a runtime offset staged in bank 6:
+ * d_auto (implicit LOD), d_lvlr (register LOD in the same word, f139 = 4) and
+ * d_arr (array layer in the same word, f77 = 2). Each waits on the staging
+ * slot. The word itself has G17P's layout: layer in bits 0..11, X/Y nibbles
+ * at 12/28, LOD x64 in 16..27 (the PACK_SMPIR f74/f75 pairs of d_lvlr/d_arr). */
+TEST(NativeForms, M5StagedOffsetSampleWitnesses)
+{
+   const struct agx3_tex_sample_desc d_auto = {
+      .width = AGX3_TEX_W32, .mask = 15, .texture = 0, .sampler = 0,
+      .lod = AGX3_TEX_LOD_AUTO, .slot = 1, .wait_mask = 1,
+      .coord_kill = true, .final = true,
+      .texture_kind = AGX3_TEX_OPERAND_IMMEDIATE,
+      .sampler_kind = AGX3_TEX_OPERAND_IMMEDIATE,
+      .offset = true, .offset_staged = true,
+      .sample_abi = AGX3_TEX_SAMPLE_ABI_DRIVER,
+   };
+   struct agx3_tex_sample_desc d_lvlr = d_auto;
+   d_lvlr.lod = AGX3_TEX_LOD_REGISTER;
+   struct agx3_tex_sample_desc d_arr = d_auto;
+   d_arr.array = true;
+   d_arr.offset_staged_array = true;
+   const struct {
+      const struct agx3_tex_sample_desc *d;
+      std::vector<uint8_t> native;
+   } cases[] = {
+      {&d_auto, {0x0f, 0x04, 0x16, 0x00, 0x41, 0x80, 0x63, 0x29, 0x01, 0x10, 0x07,
+                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}},
+      {&d_lvlr, {0x0f, 0x05, 0x16, 0x00, 0x45, 0x80, 0x63, 0x29, 0x01, 0x10, 0x07,
+                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}},
+      {&d_arr,  {0x0f, 0x05, 0x16, 0x00, 0x41, 0x80, 0x62, 0x29, 0x01, 0x18, 0x07,
+                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}},
+   };
+   for (const auto &c : cases) {
+      uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, c.d, out, &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native);
+   }
+
+   /* texwit6 g_arr / g_arr_y: an array gather reads the same layer word
+    * (f77 = 2) with the gather selector f144 (and f145 for component y). */
+   struct agx3_tex_sample_desc g_arr = d_arr;
+   g_arr.gather = AGX3_TEX_GATHER_R;
+   struct agx3_tex_sample_desc g_arr_y = d_arr;
+   g_arr_y.gather = AGX3_TEX_GATHER_G;
+   /* Comparisons (f143) with the reference staged beside the same word. */
+   struct agx3_tex_sample_desc g_cmp_arr = g_arr;
+   g_cmp_arr.shadow = true;
+   struct agx3_tex_sample_desc c_arr = d_arr;
+   c_arr.shadow = true;
+   c_arr.mask = 1;
+   struct agx3_tex_sample_desc c_lvlr = d_lvlr;
+   c_lvlr.shadow = true;
+   c_lvlr.mask = 1;
+   const struct {
+      const struct agx3_tex_sample_desc *d;
+      std::vector<uint8_t> native;
+   } gathers[] = {
+      {&g_arr, {0x0f, 0x05, 0x16, 0x00, 0x41, 0x80, 0x02, 0x21, 0x01, 0x18, 0x16,
+                0x00, 0x00, 0x00}},
+      {&g_arr_y, {0x0f, 0x05, 0x16, 0x00, 0x41, 0x80, 0x22, 0x21, 0x01, 0x18, 0x16,
+                  0x00, 0x00, 0x00}},
+      {&g_cmp_arr, {0x0f, 0x06, 0x16, 0x00, 0x41, 0x80, 0x02, 0x21, 0x01, 0x18, 0x16,
+                    0x00, 0x00, 0x00}},
+      {&c_arr, {0x0f, 0x06, 0x16, 0x00, 0x41, 0x80, 0x02, 0x21, 0x01, 0x18, 0x06,
+                0x00, 0x00, 0x00}},
+      {&c_lvlr, {0x0f, 0x06, 0x16, 0x00, 0x45, 0x80, 0x03, 0x21, 0x01, 0x10, 0x06,
+                 0x00, 0x00, 0x00}},
+   };
+   for (const auto &c : gathers) {
+      uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, c.d, out, &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native);
+   }
+}
+
+/* Native G17G (port/cwit3) ALU forms behind the spirv_assembly.type 8/16-bit
+ * refusals: `ushort(1) << uint(n)` is one 32-bit-count sll with the
+ * immediate one. */
+TEST(NativeForms, M5ShiftOneWitness)
+{
+   uint8_t out[AGX3_ALU_MAX_LENGTH] = {0}, len = 0;
+   ASSERT_TRUE(agx3_alu_sll_one_m5(0, false, 0, AGX3_ALU_HINT_DISCARD, false,
+                                   0, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x27, 0x01, 0x20, 0x1a, 0x26, 0x00, 0xa0,
+                                   0x02, 0x00, 0x80, 0x20, 0x34, 0x00, 0x00}));
+   ASSERT_TRUE(agx3_alu_sll_one_m5(3, true, 4, AGX3_ALU_HINT_NONE, false, 0,
+                                   out, &len));
+   EXPECT_EQ(len, 14);
+}
+
+/* Native G17G (port/cwit3) addsat/subsat, popcount(ushort) and ~x. */
+TEST(NativeForms, M5SpirvIntegerWitnesses)
+{
+   const auto eq = [](const uint8_t *out, uint8_t len,
+                      std::vector<uint8_t> native) {
+      return std::vector<uint8_t>(out, out + len) == native;
+   };
+   uint8_t out[AGX3_ALU_MAX_LENGTH] = {0}, len = 0;
+
+   /* 32-bit: iadd32/isub32 register point with saturation; each native
+    * instance waits on slot 8 (f53), as its loads publish there. */
+   for (unsigned v = 0; v < 3; ++v) {
+      const struct agx3_iadd_desc d = {
+         .src_a_is_register = true, .src_a_reg = 0,
+         .src_a_sign_extend = v == 1, .src_a_kill = true, .src_kill = true,
+         .dest = 0, .src = 4, .sign_extend = v == 1,
+         .wait_mask = 0x80, .subtract = v == 2, .saturate = true,
+      };
+      uint8_t b[AGX3_IADD_MAX_LENGTH] = {0}, n = 0;
+      ASSERT_TRUE(agx3_iadd_encode(AGX_ISA_ENCODING_AGX3C, &d, b, &n)) << v;
+      const std::vector<uint8_t> native[] = {
+         {0x2f, 0x00, 0x04, 0x1a, 0x21, 0x00, 0xa3, 0x02, 0x38, 0x81, 0x20, 0x00},
+         {0x2f, 0x00, 0x04, 0x1a, 0x21, 0x00, 0xa3, 0x02, 0x38, 0x81, 0xe0, 0x00},
+         {0x2f, 0x00, 0x04, 0x1a, 0x25, 0x00, 0xa2, 0x02, 0x18, 0x81, 0x20, 0x00},
+      };
+      EXPECT_EQ(std::vector<uint8_t>(b, b + n), native[v]) << v;
+   }
+
+   /* 16-bit: half result and operands. */
+   const uint16_t src[2] = {4, 2};
+   const enum agx3_alu_hint discard[2] = {AGX3_ALU_HINT_DISCARD,
+                                          AGX3_ALU_HINT_DISCARD};
+   ASSERT_TRUE(agx3_alu_addsub_sat16_m5(2, src, discard, false, false, false,
+                                        0x80, out, &len));
+   EXPECT_TRUE(eq(out, len, {0x3f, 0x00, 0x04, 0x38, 0x21, 0x00, 0xa3, 0x02,
+                             0xb0, 0x00, 0x20, 0x00}));
+   ASSERT_TRUE(agx3_alu_addsub_sat16_m5(2, src, discard, true, false, false,
+                                        0x80, out, &len));
+   EXPECT_TRUE(eq(out, len, {0x3f, 0x00, 0x04, 0x38, 0x21, 0x00, 0xa3, 0x02,
+                             0xb0, 0x00, 0xe0, 0x00}));
+   ASSERT_TRUE(agx3_alu_addsub_sat16_m5(2, src, discard, true, true, false,
+                                        0x80, out, &len));
+   EXPECT_TRUE(eq(out, len, {0x3f, 0x00, 0x04, 0x38, 0x25, 0x08, 0xa2, 0x02,
+                             0x90, 0x00, 0x60, 0x00}));
+
+   /* popcount(ushort), waiting on slot 8. */
+   ASSERT_TRUE(agx3_alu_popcount(AGX_ISA_ENCODING_AGX3C, 2, 2, true,
+                                 AGX3_ALU_HINT_DISCARD, false, 0x80, out,
+                                 &len));
+   EXPECT_TRUE(eq(out, len, {0x3f, 0x02, 0x00, 0x00, 0x32, 0x20, 0xaa, 0x02,
+                             0x04, 0x00}));
+
+   /* ~ushort and ~uint, waiting on slot 8. */
+   ASSERT_TRUE(agx3_alu_bit_not_m5(2, 2, false, AGX3_ALU_HINT_DISCARD, false,
+                                   0x80, out, &len));
+   EXPECT_TRUE(eq(out, len, {0x3b, 0x02, 0x26, 0x18, 0x21, 0x08, 0xa0, 0x02}));
+   ASSERT_TRUE(agx3_alu_bit_not_m5(0, 0, true, AGX3_ALU_HINT_DISCARD, false,
+                                   0x80, out, &len));
+   EXPECT_TRUE(eq(out, len, {0x2b, 0x00, 0x26, 0x1a, 0x21, 0x08, 0xa8, 0x02}));
+}
+
+/* texwit6 minimum-LOD samples with a runtime offset in bank 6: the bias
+ * (f139 = 3) or gradient (f139 = 5) and the minimum (f140 = 2) come from the
+ * staged word / banks, as on G17P. */
+TEST(NativeForms, M5MinimumLodOffsetSampleWitnesses)
+{
+   struct agx3_tex_sample_desc base = {
+      .width = AGX3_TEX_W32, .mask = 15, .texture = 0, .sampler = 0,
+      .slot = 1, .wait_mask = 1, .coord_kill = true, .final = true,
+      .texture_kind = AGX3_TEX_OPERAND_IMMEDIATE,
+      .sampler_kind = AGX3_TEX_OPERAND_IMMEDIATE,
+      .offset = true, .offset_staged = true,
+      .sample_abi = AGX3_TEX_SAMPLE_ABI_DRIVER,
+   };
+   struct agx3_tex_sample_desc m_bias = base;
+   m_bias.lod = AGX3_TEX_LOD_BIAS_REGISTER;
+   m_bias.min_lod = true;
+   struct agx3_tex_sample_desc m_grad = base;
+   m_grad.lod = AGX3_TEX_LOD_GRADIENT;
+   m_grad.min_lod = true;
+   const struct {
+      const char *name;
+      const struct agx3_tex_sample_desc *d;
+      std::vector<uint8_t> native;
+   } cases[] = {
+      {"m_bias", &m_bias, {0x0f, 0x06, 0x16, 0x00, 0x45, 0x80, 0x60, 0x29, 0x01, 0x18, 0x07,
+                           0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}},
+      {"m_grad", &m_grad, {0x0f, 0x06, 0x12, 0x00, 0x41, 0x80, 0x60, 0x29, 0x01, 0x18, 0x07,
+                           0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}},
+   };
+   /* texwit7: the same with a comparison (f143, mask 1, 14-byte form). */
+   struct agx3_tex_sample_desc c_bias_min = m_bias;
+   c_bias_min.shadow = true;
+   c_bias_min.mask = 1;
+   struct agx3_tex_sample_desc c_arr_grad_min = m_grad;
+   c_arr_grad_min.shadow = true;
+   c_arr_grad_min.mask = 1;
+   c_arr_grad_min.array = true;
+   c_arr_grad_min.offset_staged_array = true;
+   const struct {
+      const char *name;
+      const struct agx3_tex_sample_desc *d;
+      std::vector<uint8_t> native;
+   } cmp[] = {
+      {"c_bias_min", &c_bias_min, {0x0f, 0x07, 0x16, 0x00, 0x45, 0x80, 0x00, 0x21, 0x01, 0x18, 0x06,
+                                   0x00, 0x00, 0x00}},
+      {"c_arr_grad_min", &c_arr_grad_min, {0x0f, 0x04, 0x12, 0x00, 0x45, 0x80, 0x03, 0x21, 0x01, 0x18, 0x06,
+                                           0x00, 0x00, 0x00}},
+   };
+   for (const auto &c : cmp) {
+      uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+      const bool ok = agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, c.d, out, &len);
+      EXPECT_TRUE(ok) << c.name;
+      if (ok) {
+         EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native) << c.name;
+      }
+   }
+   for (const auto &c : cases) {
+      uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+      const bool ok = agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, c.d, out, &len);
+      EXPECT_TRUE(ok) << c.name;
+      if (ok) {
+         EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native) << c.name;
+      }
+   }
+}
+
+TEST(NativeForms, M5MinimumBank8Witness)
+{
+   uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+   ASSERT_TRUE(agx3_tex_stage_minimum_bank8_m5(6, 1, 0, false, true, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x27, 0x02, 0x00, 0x2a, 0x25, 0x00, 0xaa,
+                                   0x12, 0x00, 0x00, 0x4a, 0x30, 0x18, 0x00}));
+
+   /* The SMPs that read that minimum: texwit8 ca_br_min_d (2D-array
+    * comparison, register bias, runtime offset) and m3_br_min_d (3D). */
+   struct agx3_tex_sample_desc ca = {
+      .width = AGX3_TEX_W32, .mask = 1, .lod = AGX3_TEX_LOD_BIAS_REGISTER,
+      .min_lod = true, .slot = 1, .wait_mask = 1, .coord_kill = true,
+      .final = true, .array = true,
+      .texture_kind = AGX3_TEX_OPERAND_IMMEDIATE,
+      .sampler_kind = AGX3_TEX_OPERAND_IMMEDIATE, .shadow = true,
+      .offset = true, .offset_staged = true, .offset_staged_array = true,
+      .sample_abi = AGX3_TEX_SAMPLE_ABI_DRIVER,
+   };
+   ASSERT_TRUE(agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, &ca, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x0f, 0x04, 0x16, 0x00, 0x41, 0x80, 0x00,
+                                   0x21, 0x01, 0x00, 0x00, 0x00, 0x00, 0x80}));
+   struct agx3_tex_sample_desc cc = ca;
+   cc.array = false;
+   cc.cube = true;
+   cc.offset = false;
+   cc.offset_staged = false;
+   cc.offset_staged_array = false;
+   ASSERT_TRUE(agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, &cc, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x0f, 0x06, 0x16, 0x00, 0x41, 0x80, 0x03,
+                                   0x21, 0x01, 0x00, 0x02, 0x00, 0x00, 0x00}));
+
+   struct agx3_tex_sample_desc m3 = ca;
+   m3.shadow = false;
+   m3.mask = 15;
+   m3.array = false;
+   m3.offset_staged_array = false;
+   m3.three_d = true;
+   ASSERT_TRUE(agx3_tex_encode_sample(AGX_ISA_ENCODING_AGX3C, &m3, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x0f, 0x04, 0x16, 0x00, 0x41, 0x80, 0x62,
+                                   0x29, 0x01, 0x18, 0x07, 0x00, 0x00, 0x00,
+                                   0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
+                                   0x00}));
+}
+
+
+/* Native G17G at-sample interpolation (lane-mesh/cwit-interp ias_k1,
+ * ias_k1_np, sp@4): ITRCOORDS f38=2 on the sample index, then f32interp
+ * token leaves (f39=2, f32 = token, f41 live / f43 final) with the projective
+ * RHW leaf (f54) first. */
+TEST(NativeForms, M5SampleInterpolationWitnesses)
+{
+   const auto bytes = [](const uint8_t *out, uint8_t len) {
+      return std::vector<uint8_t>(out, out + len);
+   };
+   uint8_t out[AGX3_VARY_MAX_LENGTH] = {0}, len = 0;
+
+   const struct {
+      uint16_t dest, src;
+      std::vector<uint8_t> native;
+   } coords[] = {
+      {6, 0, {0x37, 0x00, 0x24, 0x1a, 0x2a, 0x00, 0xab, 0x0a, 0x00, 0x02}},
+      {10, 4, {0x37, 0x00, 0x24, 0x3a, 0x2a, 0x00, 0xab, 0x12, 0x00, 0x02}},
+      {14, 1, {0x37, 0x01, 0x24, 0x1a, 0x2a, 0x00, 0xab, 0x1a, 0x00, 0x02}},
+   };
+   for (const auto &c : coords) {
+      ASSERT_TRUE(agx3_vary_sample_coords(AGX_ISA_ENCODING_AGX3C, c.dest,
+                                          c.src, true, 0, out, &len));
+      EXPECT_EQ(bytes(out, len), c.native) << c.dest;
+   }
+
+   ASSERT_TRUE(agx3_vary_projective_rhw_sample(AGX_ISA_ENCODING_AGX3C, 0, 0,
+                                               6, false, out, &len));
+   EXPECT_EQ(bytes(out, len),
+             (std::vector<uint8_t>{0x27, 0x00, 0x04, 0x0a, 0x2a, 0x20, 0xaa,
+                                   0x02, 0x06, 0x88}));
+   ASSERT_TRUE(agx3_vary_projective_rhw_sample(AGX_ISA_ENCODING_AGX3C, 0, 0,
+                                               10, false, out, &len));
+   EXPECT_EQ(bytes(out, len),
+             (std::vector<uint8_t>{0x27, 0x00, 0x04, 0x0a, 0x2a, 0x20, 0xaa,
+                                   0x02, 0x0a, 0x88}));
+
+   const struct {
+      uint16_t dest, coef;
+      bool perspective;
+      uint16_t coords;
+      bool kill;
+      std::vector<uint8_t> native;
+   } leaves[] = {
+      {0, 1, true, 6, false,
+       {0x27, 0x01, 0x04, 0x0a, 0x2a, 0x20, 0xaa, 0x02, 0x06, 0x80}},
+      {8, 2, true, 6, false,
+       {0x27, 0x02, 0x04, 0x0a, 0x2a, 0x20, 0xaa, 0x12, 0x06, 0x80}},
+      {2, 3, true, 6, false,
+       {0x37, 0x03, 0x04, 0x0a, 0x2a, 0x20, 0xaa, 0x02, 0x06, 0x80}},
+      {6, 4, true, 6, true,
+       {0x37, 0x00, 0x04, 0x1a, 0x2a, 0x20, 0xaa, 0x0a, 0x06, 0x04}},
+      {0, 0, false, 4, false,
+       {0x27, 0x00, 0x04, 0x0a, 0x2a, 0x20, 0xaa, 0x02, 0x04, 0x80}},
+      {4, 3, false, 4, true,
+       {0x27, 0x03, 0x04, 0x0a, 0x2a, 0x20, 0xaa, 0x0a, 0x04, 0x04}},
+   };
+   for (const auto &l : leaves) {
+      ASSERT_TRUE(agx3_vary_interp_sample(AGX_ISA_ENCODING_AGX3C, l.dest,
+                                          l.coef, l.perspective, l.coords,
+                                          l.kill, out, &len));
+      EXPECT_EQ(bytes(out, len), l.native) << l.dest << " " << l.coef;
+   }
+}
+
+/* Native G17G quad ballots of a boolean immediate (lane-mesh/cwit-quad
+ * q_term/q_vote, cwit-quad2 q_any_mem): quad_any(false) and the quad
+ * active-lane ballot(1 != 0). */
+TEST(NativeForms, M5QuadImmediateBallotWitnesses)
+{
+   const struct {
+      uint16_t dest, value;
+      std::vector<uint8_t> native;
+   } rows[] = {
+      {14, 0, {0x37, 0x00, 0x20, 0x18, 0x2d, 0x00, 0xa6, 0x1a, 0x30, 0x00, 0x08, 0x40}},
+      {4, 1, {0x27, 0x01, 0x20, 0x18, 0x2d, 0x00, 0xa6, 0x0a, 0x30, 0x00, 0x08, 0x40}},
+      {2, 1, {0x37, 0x01, 0x20, 0x18, 0x2d, 0x00, 0xa6, 0x02, 0x30, 0x00, 0x08, 0x40}},
+   };
+   for (const auto &r : rows) {
+      uint8_t out[AGX3_SUBGROUP_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_subgroup_encode_quad_ballot_imm(r.dest, true, r.value, 0,
+                                                       out, &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), r.native) << r.dest;
+   }
+}
+
+/* Native G17G centroid inputs (lane-mesh/cwit-interp iac, cwit-interp2
+ * cp/cnp at 1 and 4 samples): the fragment opens with the marked SR39 read,
+ * whose slot 1 ITRCOORDS f38=1 waits on; the token then feeds the same
+ * f32interp leaves as at-sample interpolation. */
+TEST(NativeForms, M5CentroidInterpolationWitnesses)
+{
+   struct agx3_sr_desc sr = {};
+   sr.width = AGX3_SR_W16;
+   sr.sr = 39;
+   sr.slot = 1;
+   sr.first = true;
+   uint8_t srb[AGX3_SR_MAX_LENGTH] = {0}, srlen = 0;
+   ASSERT_TRUE(agx3_sr_encode(AGX_ISA_ENCODING_AGX3C, &sr, srb, &srlen));
+   EXPECT_EQ(std::vector<uint8_t>(srb, srb + srlen),
+             (std::vector<uint8_t>{0x24, 0x82, 0x01, 0x00, 0x20, 0x00, 0xb0,
+                                   0x02}));
+
+   const struct {
+      uint16_t dest;
+      std::vector<uint8_t> native;
+   } coords[] = {
+      {6, {0x37, 0x04, 0x24, 0x1a, 0x2a, 0x00, 0xab, 0x0a, 0x00, 0x00}},
+      {4, {0x27, 0x04, 0x24, 0x1a, 0x2a, 0x00, 0xab, 0x0a, 0x00, 0x00}},
+   };
+   for (const auto &c : coords) {
+      uint8_t out[AGX3_VARY_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_vary_centroid_coords(AGX_ISA_ENCODING_AGX3C, c.dest, 0,
+                                            true, 0x1, out, &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native) << c.dest;
+   }
+}
+
+/* AGX3C texture barriers wait in their own first word on the result slots of
+ * the image writes in flight. Native G17G bytes from port/fencewit:
+ * k_fence (slot2), k_fence2 (slots 2+3), k_simdbar_tex (slot1), k_tgbar_tex,
+ * k_atf_tg. */
+TEST(NativeForms, M5TextureFenceWitnesses)
+{
+   const struct {
+      agx3_barrier_scope scope;
+      uint16_t wait;
+      std::vector<uint8_t> native;
+   } rows[] = {
+      {AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT, 0x2,
+       {0x87, 0x26, 0x00, 0x06, 0x00, 0x08, 0x87, 0x1e, 0x00, 0x06, 0x00, 0x00}},
+      {AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT, 0x6,
+       {0x87, 0x26, 0x00, 0x06, 0x00, 0x18, 0x87, 0x1e, 0x00, 0x06, 0x00, 0x00}},
+      {AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT, 0x1,
+       {0x87, 0x26, 0x00, 0x06, 0x00, 0x04, 0x87, 0x1e, 0x00, 0x06, 0x00, 0x00}},
+      {AGX3_BARRIER_SCOPE_TEXTURE, 0x2,
+       {0x87, 0x67, 0x00, 0x06, 0x00, 0x08, 0x87, 0x5f, 0x00, 0x06, 0x00, 0x00}},
+      {AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP, 0x2,
+       {0x87, 0x27, 0x00, 0x06, 0x00, 0x08, 0x87, 0x1f, 0x00, 0x06, 0x00, 0x00}},
+      {AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT, 0,
+       {0x87, 0x26, 0x00, 0x06, 0x00, 0x00, 0x87, 0x1e, 0x00, 0x06, 0x00, 0x00}},
+   };
+   for (const auto &r : rows) {
+      agx3_encoded_scoped_barrier enc = {};
+      ASSERT_TRUE(agx3_pack_scoped_barrier_wait(AGX_ISA_ENCODING_AGX3C,
+                                                r.scope, r.wait, &enc))
+         << r.scope << " " << r.wait;
+      EXPECT_EQ(std::vector<uint8_t>(enc.bytes, enc.bytes + enc.length),
+                r.native) << r.scope << " " << r.wait;
+   }
+
+   agx3_encoded_scoped_barrier enc = {};
+   EXPECT_FALSE(agx3_pack_scoped_barrier(
+      AGX_ISA_ENCODING_AGX3B, AGX3_BARRIER_SCOPE_TEXTURE_FENCE_THREADGROUP,
+      &enc));
+   EXPECT_FALSE(agx3_pack_scoped_barrier(
+      AGX_ISA_ENCODING_AGX3C, AGX3_BARRIER_SCOPE_TEXTURE_FENCE, &enc));
+   EXPECT_FALSE(agx3_pack_scoped_barrier_wait(
+      AGX_ISA_ENCODING_AGX3B, AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT, 0x2,
+      &enc));
+   EXPECT_FALSE(agx3_pack_scoped_barrier_wait(
+      AGX_ISA_ENCODING_AGX3C, AGX3_BARRIER_SCOPE_DEVICE, 0x2, &enc));
+}
+
+/* Each AGX3C image write's fence waits on that write's own result slot in
+ * the barrier word, with no separate retirement, as k_fence_twice does. */
+TEST(NativeForms, M5ImageFenceWaitsWriteSlotInBarrier)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_COMPUTE;
+   agx_index data = agx_register(16, AGX_SIZE_32);
+   data.channels_m1 = 3;
+   agx_index coord = agx_register(24, AGX_SIZE_32);
+   coord.channels_m1 = 1;
+   for (unsigned i = 0; i < 3; ++i) {
+      agx_image_write(b, data, coord, agx_zero(), agx_uniform(8, AGX_SIZE_64),
+                      agx_register(28, AGX_SIZE_32), AGX_DIM_2D, true);
+      agx_scoped_barrier_agx3(b, AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT);
+   }
+   agx_stop(b);
+   util_dynarray binary;
+   util_dynarray_init(&binary, mem);
+   ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+
+   const uint8_t *bytes = (const uint8_t *)binary.data;
+   unsigned fences = 0;
+   for (unsigned i = 0; i + 12 <= binary.size; ++i) {
+      if (bytes[i] != 0x87 || bytes[i + 1] != 0x26 || bytes[i + 6] != 0x87 ||
+          bytes[i + 7] != 0x1e)
+         continue;
+      /* One wait bit, on a slot an imgwci before it signals. */
+      const uint16_t wait = (bytes[i + 5] >> 2) | (((bytes[i + 3] >> 5) & 3) << 6);
+      EXPECT_EQ(util_bitcount(wait), 1u) << fences;
+      agx3_encoded_scoped_barrier enc = {};
+      ASSERT_TRUE(agx3_pack_scoped_barrier_wait(
+         AGX_ISA_ENCODING_AGX3C, AGX3_BARRIER_SCOPE_TEXTURE_FENCE_WAIT, wait,
+         &enc));
+      EXPECT_EQ(memcmp(bytes + i, enc.bytes, 12), 0) << fences;
+      ++fences;
+      i += 11;
+   }
+   EXPECT_EQ(fences, 3u);
+   util_dynarray_fini(&binary);
+   ralloc_free(mem);
+}
+
+/* G17G 2D-MS texel fetches staged through PACK_SMPIR: the vertex form
+ * (port/vmswit v_ms_f) is the fragment form (f_ms_f) with sampler index 0. */
+TEST(NativeForms, M5VertexMultisampleFetchWitness)
+{
+   const struct {
+      uint16_t sampler;
+      std::vector<uint8_t> native;
+   } rows[] = {
+      {0, {0x0f, 0x04, 0x1a, 0x00, 0x45, 0x80, 0x61, 0x29, 0x01, 0x08, 0x02,
+           0x00, 0x00, 0x00}},
+      {1, {0x0f, 0x04, 0x1a, 0x00, 0x45, 0x81, 0x61, 0x29, 0x01, 0x08, 0x02,
+           0x00, 0x00, 0x00}},
+   };
+   for (const auto &r : rows) {
+      agx3_tex_fetch_desc d = {};
+      d.width = AGX3_TEX_W32;
+      d.mask = 0xf;
+      d.slot = 1;
+      d.wait_mask = 1;
+      d.coord_kill = true;
+      d.final = true;
+      d.multisample = true;
+      d.staged_sample = true;
+      d.sampler_value = r.sampler;
+      uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_tex_encode_fetch(AGX_ISA_ENCODING_AGX3C, &d, out, &len))
+         << r.sampler;
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), r.native) << r.sampler;
+   }
+}
+
+/* AGX3C TEX destinations are eight-bit half indices: G17G places a read at
+ * half 174 under register pressure (port/rawit k_read_hi). */
+TEST(NativeForms, M5HighTextureDestinationWitness)
+{
+   agx3_tex_fetch_desc d = {};
+   d.width = AGX3_TEX_W32;
+   d.dest = 174;
+   d.mask = 0xf;
+   d.slot = 2;
+   d.wait_mask = 1;
+   d.coord_kill = true;
+   d.final = true;
+   uint8_t out[AGX3_TEX_MAX_LENGTH] = {0}, len = 0;
+   ASSERT_TRUE(agx3_tex_encode_fetch(AGX_ISA_ENCODING_AGX3C, &d, out, &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x7f, 0x06, 0x5a, 0x00, 0xc8, 0x80, 0x60,
+                                   0x39}));
+   d.dest = 128;
+   EXPECT_FALSE(agx3_tex_encode_fetch(AGX_ISA_ENCODING_AGX3B, &d, out, &len));
+}
+
+/* G17G saturate(half): the f16add -0.0h form with f14, plain and with the
+ * negate/abs modifiers (port/hsatwit k_hsat, k_hsat_neg, k_hsat_abs,
+ * k_hsat_live). */
+TEST(NativeForms, M5HalfSaturateWitness)
+{
+   const struct {
+      uint16_t dest, src;
+      bool abs, neg;
+      enum agx3_alu_hint hint;
+      std::vector<uint8_t> native;
+   } cases[] = {
+      {2, 2, false, false, AGX3_ALU_HINT_DISCARD,
+       {0x38, 0x02, 0x04, 0x02, 0x22, 0x20, 0xa0, 0x02, 0x84, 0x16}},
+      {2, 2, false, true, AGX3_ALU_HINT_DISCARD,
+       {0x38, 0x02, 0x04, 0x02, 0x22, 0x20, 0xa0, 0x02, 0x84, 0x96}},
+      {2, 2, true, false, AGX3_ALU_HINT_DISCARD,
+       {0x38, 0x02, 0x04, 0x02, 0x22, 0x20, 0xa0, 0x02, 0x84, 0x36}},
+      {5, 4, false, false, AGX3_ALU_HINT_NONE,
+       {0x28, 0x00, 0x04, 0x0b, 0x22, 0x20, 0xa0, 0x0a, 0x80, 0x16}},
+   };
+   for (const auto &c : cases) {
+      uint8_t out[AGX3_ALU_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_alu_half_add_negzero_m5(c.dest, c.src, c.abs, c.neg,
+                                               true, c.hint, false, 0x80, out,
+                                               &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native);
+   }
+}
+
+/* G17G float(h) * minifloat into a float (port/hmulwit). */
+TEST(NativeForms, M5HalfTimesMinifloatWitness)
+{
+   const struct {
+      uint16_t dest, src, imm;
+      bool abs, sat;
+      enum agx3_alu_hint hint;
+      uint16_t wait;
+      std::vector<uint8_t> native;
+   } cases[] = {
+      {0, 0, 32, false, false, AGX3_ALU_HINT_DISCARD, 0x80,
+       {0x29, 0x00, 0x05, 0x10, 0x22, 0x80, 0xac, 0x02, 0x02, 0x80, 0x20,
+        0x00, 0x02, 0x10}},
+      {0, 0, 32, false, true, AGX3_ALU_HINT_DISCARD, 0x80,
+       {0x29, 0x00, 0x05, 0x10, 0x22, 0x80, 0xac, 0x02, 0x02, 0x80, 0x20,
+        0x02, 0x02, 0x10}},
+      {0, 0, 32, true, false, AGX3_ALU_HINT_DISCARD, 0x80,
+       {0x29, 0x00, 0x05, 0x10, 0x22, 0x80, 0xac, 0x02, 0x02, 0x80, 0x20,
+        0x08, 0x02, 0x10}},
+      {0, 4, 32, false, false, AGX3_ALU_HINT_NONE, 0x80,
+       {0x29, 0x00, 0x05, 0x10, 0x22, 0x80, 0xac, 0x02, 0x02, 0x81, 0x00,
+        0x00, 0x02, 0x10}},
+      {0, 6, 32, false, false, AGX3_ALU_HINT_CACHE, 0x80,
+       {0x29, 0x00, 0x05, 0x10, 0x22, 0x80, 0xac, 0x02, 0x82, 0x81, 0x10,
+        0x00, 0x02, 0x10}},
+      {2, 6, 64, false, false, AGX3_ALU_HINT_CACHE, 0,
+       {0x19, 0xc1, 0x05, 0x86, 0x80, 0x00}},
+      {6, 6, 80, false, false, AGX3_ALU_HINT_DISCARD, 0,
+       {0x39, 0xd1, 0x15, 0x06, 0x80, 0x00}},
+   };
+   for (const auto &c : cases) {
+      uint8_t out[AGX3_ALU_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_alu_fmul_half_minifloat_m5(c.dest, c.src, c.imm, false,
+                                                  c.abs, c.sat, c.hint,
+                                                  c.wait, out, &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native);
+   }
+}
+
+/* G17G half(fma(a, k, b)): the immediate-multiplier FMA with f10=1
+ * (port/hfmawit). */
+TEST(NativeForms, M5NarrowingFmaImmediateWitness)
+{
+   const struct {
+      uint16_t dest, src1, add;
+      uint8_t k;
+      bool src1_16, sat, src_cache, src_discard, add_cache, add_discard;
+      std::vector<uint8_t> native;
+   } cases[] = {
+      {2, 2, 4, 32, false, false, false, true, false, true,
+       {0x39, 0x02, 0x07, 0x08, 0x20, 0x30, 0xa3, 0x02, 0x30, 0x88, 0x98,
+        0x00, 0x04, 0x08, 0x00, 0x80}},
+      {2, 2, 4, 160, false, false, false, true, false, true,
+       {0x39, 0x02, 0x07, 0x08, 0x20, 0x30, 0xa3, 0x02, 0x30, 0xa8, 0x98,
+        0x00, 0x04, 0x08, 0x00, 0x80}},
+      {2, 2, 4, 32, false, true, false, true, false, true,
+       {0x39, 0x02, 0x07, 0x08, 0x20, 0x30, 0xa3, 0x02, 0x30, 0x88, 0x98,
+        0x02, 0x04, 0x08, 0x00, 0x80}},
+      {10, 4, 6, 32, false, false, true, false, true, false,
+       {0x39, 0x80, 0x07, 0x18, 0x20, 0x30, 0xa3, 0x12, 0x10, 0x88, 0x98,
+        0x00, 0x06, 0x04, 0x00, 0x80}},
+      {2, 4, 2, 32, true, false, false, true, false, true,
+       {0x39, 0x00, 0x07, 0x18, 0x20, 0x30, 0xa7, 0x02, 0x20, 0x88, 0x98,
+        0x00, 0x02, 0x08, 0x00, 0x80}},
+   };
+   for (const auto &c : cases) {
+      const struct agx3_falu_mods m = {.src1_16 = c.src1_16,
+                                       .saturate = c.sat};
+      const struct agx3_falu_runtime rt = {
+         .cache = {false, c.src_cache, c.add_cache},
+         .discard = {false, c.src_discard, c.add_discard},
+         .dest_cache = false,
+         .wait_mask = 0x80,
+      };
+      uint8_t out[AGX3_FALU_MOD_MAX_LENGTH] = {0}, len = 0;
+      ASSERT_TRUE(agx3_falu_fma_mul_imm_narrow_m5(c.dest, c.k, c.src1, c.add,
+                                                  &m, &rt, out, &len));
+      EXPECT_EQ(std::vector<uint8_t>(out, out + len), c.native);
+   }
+}
+
+/* G17G B5G6R5Unorm framebuffer fetch: three 16-bit lanes (port/fbwit). */
+TEST(NativeForms, M5Rgb565TileLoadWitness)
+{
+   uint8_t out[AGX3_TILE_MAX_LENGTH] = {0}, len = 0;
+   ASSERT_TRUE(agx3_tile_load(AGX_ISA_ENCODING_AGX3C, 0, AGX3_TILE_W16, 8, 7,
+                              0, AGX3_TILE_SAMPLE_1X, 0, true, true, out,
+                              &len));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + len),
+             (std::vector<uint8_t>{0x27, 0x01, 0x05, 0x00, 0x01, 0x87, 0xbc,
+                                   0x02, 0x00, 0x00, 0x40, 0xd8, 0x20, 0x00}));
+}
+TEST(NativeForms, M5MeshUVBNativeOperands)
+{
+   uint8_t bytes[8], length = 0;
+   const uint8_t allocation[] = {0x35,0x01,0x0f,0x00,0x20,0x00,0xa2,0x0a};
+   const uint8_t primitive[] = {0x25,0x00,0x0b,0x18,0x20,0x00,0xa2,0x0a};
+   const uint8_t position[] = {0x35,0x02,0x0b,0x0a,0x20,0x00,0xa2,0x0a};
+   ASSERT_TRUE(agx3_mesh_allocate_uvb(AGX_ISA_ENCODING_AGX3C,1,10,true,bytes,&length));
+   EXPECT_EQ(length,8u); EXPECT_EQ(memcmp(bytes,allocation,8),0);
+   ASSERT_TRUE(agx3_mesh_uvst(AGX_ISA_ENCODING_AGX3C,12,true,true,bytes,&length));
+   EXPECT_EQ(memcmp(bytes,primitive,8),0);
+   ASSERT_TRUE(agx3_mesh_uvst(AGX_ISA_ENCODING_AGX3C,22,false,true,bytes,&length));
+   EXPECT_EQ(memcmp(bytes,position,8),0);
+   EXPECT_TRUE(agx3_mesh_allocate_uvb(AGX_ISA_ENCODING_AGX3B,1,10,true,bytes,&length));
+   EXPECT_FALSE(agx3_mesh_uvst(AGX_ISA_ENCODING_AGX3C,1,false,true,bytes,&length));
+   EXPECT_FALSE(agx3_mesh_uvst(AGX_ISA_ENCODING_AGX3C,254,false,true,bytes,&length));
+}
+
+TEST(NativeForms, M5MeshUVBReachesNativePacker)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,
+                          &agx_nir_options, "M5 native UVB descriptor");
+   b.shader->info.workgroup_size[0] = 3;
+   b.shader->info.workgroup_size[1] = 1;
+   b.shader->info.workgroup_size[2] = 1;
+   nir_def *index = nir_load_local_invocation_index(&b);
+   /* Dynamic values deliberately exercise SSA -> scheduling -> register
+    * allocation. The backend must see both words of the hidden descriptor. */
+   nir_allocate_mesh_uvb_agx(&b,nir_u2u16(&b,index),
+                              nir_u2u16(&b,nir_iadd_imm(&b,index,3)));
+   nir_def *descriptor = nir_vec2(&b,index,nir_iadd_imm(&b,index,32));
+   nir_store_mesh_uvb_agx(&b,descriptor,.flags=0);
+   nir_store_mesh_uvb_agx(&b,descriptor,.flags=1);
+   nir_validate_shader(b.shader,"M5 native UVB");
+   agx_preprocess_nir(b.shader);
+   agx_shader_key key = {};
+   key.dev.chip = AGX_CHIP_G17S;
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.no_preamble = true;
+   agx_shader_part part = {};
+   bool compiled = agx_compile_shader_nir(b.shader,&key,&part);
+   EXPECT_TRUE(compiled) << agx_last_compile_refusal();
+   if (compiled) { EXPECT_GT(part.info.binary_size,24u); free(part.binary); }
+   ralloc_free(b.shader);
+}
+
+
+TEST(NativeForms, M5ObjectChildcountNativeLeaf)
+{
+   uint8_t bytes[8], size;
+   const uint8_t native[] = {0x35,0x04,0x33,0x00,0x24,0x00,0xa8,0x02};
+   ASSERT_TRUE(agx3_mesh_object_childcount(AGX_ISA_ENCODING_AGX3C,bytes,&size));
+   ASSERT_EQ(size,sizeof(native));
+   EXPECT_EQ(memcmp(bytes,native,size),0);
+   EXPECT_TRUE(agx3_mesh_object_childcount(AGX_ISA_ENCODING_AGX3B,bytes,&size));
+}
+
+TEST(NativeForms, M5ObjectChildcountSSACompiles)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_TASK,
+                     &agx_nir_options,"M5 native object childcount SSA");
+   b.shader->info.workgroup_size[0]=3;
+   b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.io_lowered=true;
+   nir_def *pointer=nir_pack_64_2x32(&b,nir_load_preamble(&b,2,32,.base=0));
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_def *value=nir_load_global_constant(&b,1,32,
+      nir_iadd(&b,pointer,nir_u2u64(&b,nir_imul_imm(&b,lane,4))),.align_mul=4);
+   nir_store_object_childcount_agx(&b,value);
+   nir_validate_shader(b.shader,"object childcount");
+   agx_preprocess_nir_m5(b.shader);
+   agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;
+   key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+   agx_shader_part part={};
+   bool compiled=agx_compile_shader_nir(b.shader,&key,&part);
+   EXPECT_TRUE(compiled)<<agx_last_compile_refusal();
+   if(compiled){EXPECT_EQ(part.info.scratch_size,0u);free(part.binary);}
+   ralloc_free(b.shader);
+}
+
+/* The record task's child grid takes 16 bytes past the body's threadgroup
+ * memory; a body using all 32 KiB aliases word 0 after a rendezvous. */
+TEST(NativeForms, M5TaskGridFitsFullThreadgroupMemory)
+{
+   for (unsigned shared : {64u, 32768u}) {
+      for (bool payload : {false, true}) {
+         nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_TASK,&agx_nir_options,"task grid");
+         b.shader->info.workgroup_size[0]=32;
+         b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+         b.shader->info.shared_size=shared;b.shader->info.task_payload_size=payload ? 4 : 0;
+         if (payload)
+            nir_store_task_payload(&b,nir_imm_int(&b,7),nir_imm_int(&b,0),.align_mul=4);
+         nir_launch_mesh_workgroups(&b,nir_imm_ivec3(&b,2,1,1));
+         ASSERT_TRUE(agx_nir_lower_m5_task(b.shader,payload ? 16384 : 0,payload));
+         nir_validate_shader(b.shader,"task grid");
+         EXPECT_LE(b.shader->info.shared_size,32768u);
+         if (payload) {
+            EXPECT_EQ(b.shader->info.shared_size,shared==64 ? 128u : 32768u);
+         }
+         ralloc_free(b.shader);
+      }
+   }
+}
+
+TEST(NativeForms, M5MeshUserSharedBarrierCompile)
+{
+   for (unsigned shared : {0u,4u,64u,128u}) {
+      nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,
+                            &agx_nir_options,"native mesh shared output readback");
+      b.shader->info.workgroup_size[0]=3;
+      b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+      b.shader->info.mesh.max_vertices_out=3;
+      b.shader->info.mesh.max_primitives_out=1;
+      b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;
+      b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES;
+      b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;
+      b.shader->info.io_lowered=true;b.shader->info.shared_size=shared;
+      nir_def *lane=nir_load_local_invocation_index(&b);
+      nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,3),nir_imm_int(&b,1),
+                                         nir_imm_int(&b,0),.stream_id=0);
+      nir_def *pos=nir_vec4(&b,nir_u2f32(&b,lane),nir_imm_float(&b,1),
+                            nir_imm_float(&b,.25),nir_imm_float(&b,1));
+      nir_store_per_vertex_output(&b,pos,lane,nir_imm_int(&b,0),
+         .write_mask=15,.src_type=nir_type_float32,
+         .io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+      nir_push_if(&b,nir_ieq_imm(&b,lane,0));
+      nir_store_per_primitive_output(&b,nir_imm_ivec3(&b,0,1,2),
+         nir_imm_int(&b,0),nir_imm_int(&b,0),.write_mask=7,.src_type=nir_type_uint32,
+         .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+      if(shared)nir_store_shared(&b,nir_imm_int(&b,0x531a),nir_imm_int(&b,0),
+                                .write_mask=1,.align_mul=4);
+      nir_pop_if(&b,NULL);
+      nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+         .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
+      nir_def *token=shared?nir_load_shared(&b,1,32,nir_imm_int(&b,0),.align_mul=4):nir_imm_int(&b,0);
+      nir_store_global(&b,token,nir_iadd(&b,nir_imm_int64(&b,0x2000000),
+            nir_u2u64(&b,nir_imul_imm(&b,lane,4))),.write_mask=1,.align_mul=4);
+      ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+      EXPECT_EQ(b.shader->info.shared_size,((shared+63)&~63u)+128);
+      unsigned output_reads=0;bool user_address=false;
+      nir_foreach_block(block,b.impl)nir_foreach_instr(in,block) {
+         if(in->type!=nir_instr_type_intrinsic)continue;
+         nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+         EXPECT_NE(i->intrinsic,nir_intrinsic_load_per_vertex_output);
+         EXPECT_NE(i->intrinsic,nir_intrinsic_load_per_primitive_output);
+         if(i->intrinsic==nir_intrinsic_load_shared) {
+            output_reads++;
+            if(nir_src_is_const(i->src[0])&&nir_src_as_uint(i->src[0])==0&&i->def.bit_size==32)
+               user_address=true;
+         }
+      }
+      EXPECT_GE(output_reads,2u);EXPECT_EQ(user_address,shared!=0);
+      nir_validate_shader(b.shader,"mesh output load arena");
+      agx_preprocess_nir(b.shader);agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;
+      key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+      agx_shader_part part={};
+      ASSERT_TRUE(agx_compile_shader_nir(b.shader,&key,&part))<<agx_last_compile_refusal();
+      free(part.binary);ralloc_free(b.shader);
+   }
+}
+
+/* A padded mesh workgroup keeps its API body under the exact invocation
+ * mask; a top-level execution barrier splits that body and is passed by every
+ * physical lane. */
+TEST(NativeForms, M5MeshPaddedTopLevelBarrierStaysUnmasked)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"padded top-level barrier");
+   b.shader->info.workgroup_size[0]=b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=3;b.shader->info.mesh.max_primitives_out=1;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;b.shader->info.shared_size=4;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES;
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;
+   nir_def *index=nir_load_local_invocation_index(&b);
+   nir_store_shared(&b,index,nir_imm_int(&b,0),.align_mul=4);
+   nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+      .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
+   nir_set_vertex_and_primitive_count(&b,nir_iadd_imm(&b,index,3),nir_imm_int(&b,1),nir_imm_int(&b,0));
+   ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+   EXPECT_GT(b.shader->info.workgroup_size[0],1u);
+   nir_validate_shader(b.shader,"padded top-level barrier");
+   unsigned user_barriers=0;
+   nir_foreach_block(block,nir_shader_get_entrypoint(b.shader)) {
+      nir_foreach_instr(in,block) {
+         if(in->type!=nir_instr_type_intrinsic)continue;
+         nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+         if(i->intrinsic==nir_intrinsic_barrier &&
+            nir_intrinsic_execution_scope(i)!=SCOPE_NONE) {
+            EXPECT_EQ(block->cf_node.parent->type,nir_cf_node_function);
+            user_barriers++;
+         }
+      }
+   }
+   EXPECT_GE(user_barriers,1u);
+   ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5MeshPaddedNestedBarrierRefusesWithoutMutation)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"padded nested barrier refusal");
+   b.shader->info.workgroup_size[0]=b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=3;b.shader->info.mesh.max_primitives_out=1;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;b.shader->info.shared_size=4;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES;
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;
+   nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,3),nir_imm_int(&b,1),nir_imm_int(&b,0));
+   nir_push_if(&b,nir_ieq_imm(&b,nir_load_local_invocation_index(&b),0));
+   nir_barrier(&b,.execution_scope=SCOPE_WORKGROUP,.memory_scope=SCOPE_WORKGROUP,
+      .memory_semantics=NIR_MEMORY_ACQ_REL,.memory_modes=nir_var_mem_shared);
+   nir_pop_if(&b,NULL);
+   EXPECT_FALSE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+   EXPECT_EQ(b.shader->info.shared_size,4u);EXPECT_EQ(b.shader->info.workgroup_size[0],1u);
+   ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5MeshPrimitiveDeclarationsRespectWrittenMask)
+{
+   for (bool written : {false, true}) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_MESH, &agx_nir_options, "mesh primitive declaration");
+      b.shader->info.io_lowered = true;
+      b.shader->info.workgroup_size[0] = 1;
+      b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+      b.shader->info.mesh.max_vertices_out = 256;
+      b.shader->info.mesh.max_primitives_out = 256;
+      b.shader->info.mesh.primitive_type = MESA_PRIM_TRIANGLES;
+      b.shader->info.outputs_written = VARYING_BIT_POS |
+         VARYING_BIT_PRIMITIVE_INDICES | (written ? VARYING_BIT_VAR(0) : 0);
+      b.shader->info.per_primitive_outputs =
+         VARYING_BIT_PRIMITIVE_INDICES | VARYING_BIT_VAR(0);
+      nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, 3),
+                                       nir_imm_int(&b, 1), nir_imm_int(&b, 0));
+      for (unsigned vertex = 0; vertex < 3; ++vertex) {
+         nir_store_per_vertex_output(&b, nir_imm_vec4(&b, vertex, 0, 0, 1),
+            nir_imm_int(&b, vertex), nir_imm_int(&b, 0), .write_mask = 15,
+            .src_type = nir_type_float32,
+            .io_semantics = {.location = VARYING_SLOT_POS, .num_slots = 1});
+      }
+      nir_store_per_primitive_output(&b, nir_imm_ivec3(&b, 0, 1, 2),
+         nir_imm_int(&b, 0), nir_imm_int(&b, 0), .write_mask = 7,
+         .src_type = nir_type_uint32,
+         .io_semantics = {.location = VARYING_SLOT_PRIMITIVE_INDICES, .num_slots = 1});
+      if (written) {
+         nir_store_per_primitive_output(&b, nir_imm_vec4(&b, 0, 1, 0, 1),
+            nir_imm_int(&b, 0), nir_imm_int(&b, 0), .write_mask = 15,
+            .src_type = nir_type_float32,
+            .io_semantics = {.location = VARYING_SLOT_VAR0, .num_slots = 1});
+      }
+      EXPECT_EQ(agx_nir_lower_m5_mesh_triangle_probe(b.shader), !written);
+      if (written) {
+         EXPECT_EQ(b.shader->info.shared_size, 0u);
+         EXPECT_EQ(b.shader->info.workgroup_size[0], 1u);
+      } else {
+         EXPECT_EQ(b.shader->info.shared_size, 4928u);
+         nir_validate_shader(b.shader, "mesh unused primitive declaration");
+      }
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5MeshVertexColorUVSTReachesRA)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"native mesh color UVST");
+   b.shader->info.workgroup_size[0]=3;b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=3;b.shader->info.mesh.max_primitives_out=1;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES|VARYING_BIT_VAR(0);
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;b.shader->info.io_lowered=true;
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,3),nir_imm_int(&b,1),nir_imm_int(&b,0));
+   nir_store_per_vertex_output(&b,nir_imm_vec4(&b,0,1,0,1),lane,nir_imm_int(&b,0),
+      .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_VAR0,.num_slots=1});
+   nir_store_per_vertex_output(&b,nir_imm_vec4(&b,0,0,.25,1),lane,nir_imm_int(&b,0),
+      .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+   nir_push_if(&b,nir_ieq_imm(&b,lane,0));
+   nir_store_per_primitive_output(&b,nir_imm_ivec3(&b,0,1,2),nir_imm_int(&b,0),
+      nir_imm_int(&b,0),.write_mask=7,.src_type=nir_type_uint32,
+      .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+   nir_pop_if(&b,NULL);
+   ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));EXPECT_EQ(b.shader->info.shared_size,192u);
+   unsigned uvst=0;nir_foreach_block(block,b.impl)nir_foreach_instr(in,block)
+      if(in->type==nir_instr_type_intrinsic&&nir_instr_as_intrinsic(in)->intrinsic==nir_intrinsic_store_mesh_uvb_agx)uvst++;
+   EXPECT_EQ(uvst,9u);nir_validate_shader(b.shader,"native color planes");agx_preprocess_nir(b.shader);
+   agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+   agx_shader_part part={};ASSERT_TRUE(agx_compile_shader_nir(b.shader,&key,&part))<<agx_last_compile_refusal();
+   free(part.binary);ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5MeshNativeLargeOutputBatchesCompile)
+{
+   for (unsigned vertices : {3u,4u,5u,8u,9u,16u,32u,33u,64u,256u}) {
+      const unsigned primitives=vertices==64?126:vertices==256?256:1;
+      nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,
+         &agx_nir_options,"native mesh larger UVB rounds");
+      b.shader->info.workgroup_size[0]=32;
+      b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+      b.shader->info.mesh.max_vertices_out=vertices;
+      b.shader->info.mesh.max_primitives_out=primitives;
+      b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;
+      b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES|VARYING_BIT_VAR(0);
+      b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;
+      b.shader->info.io_lowered=true;
+      nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,vertices),nir_imm_int(&b,primitives),nir_imm_int(&b,0));
+      /* The UVS layout comes from the per-vertex stores themselves. */
+      nir_store_per_vertex_output(&b,nir_imm_vec4(&b,0,1,0,1),nir_load_local_invocation_index(&b),
+         nir_imm_int(&b,0),.write_mask=15,.src_type=nir_type_float32,
+         .io_semantics={.location=VARYING_SLOT_VAR0,.num_slots=1});
+      ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+      unsigned varying=0,primitive=0,round2=0;
+      const unsigned position=((3*primitives+15)&~15u)+16;
+      const unsigned stride=((vertices+3)&~3u)*4;
+      nir_foreach_block(block,b.impl)nir_foreach_instr(in,block) {
+         if(in->type!=nir_instr_type_intrinsic)continue;
+         nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+         if(i->intrinsic!=nir_intrinsic_store_mesh_uvb_agx)continue;
+         ASSERT_TRUE(nir_src_is_const(i->src[0]));
+         unsigned target=nir_src_comp_as_uint(i->src[0],0), address=nir_src_comp_as_uint(i->src[0],1);
+         if(nir_intrinsic_flags(i)) {primitive++;EXPECT_EQ(address,(target>>16)*4);}
+         else {varying++;EXPECT_EQ(address,position+(target&65535)*stride+4*(target>>16));
+            if((target>>16)==32)round2++;}
+      }
+      EXPECT_EQ(varying,8*((vertices+31)/32));
+      EXPECT_EQ(primitive,((3*primitives+3)/4+31)/32);
+      EXPECT_EQ(round2,vertices>32?8u:0u);
+      EXPECT_EQ(b.shader->info.shared_size,(position+8*stride+63)&~63u);
+      nir_validate_shader(b.shader,"native larger mesh batch descriptors");
+      agx_preprocess_nir(b.shader);agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;
+      key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+      agx_shader_part part={};ASSERT_TRUE(agx_compile_shader_nir(b.shader,&key,&part))<<agx_last_compile_refusal();
+      free(part.binary);ralloc_free(b.shader);
+   }
+}
+
+/* Mesh shader with three per-vertex user slots of different interpolation
+ * classes: VAR0 vec4 smooth, VAR1 ivec2 flat (stored as 16-bit), VAR2 float
+ * noperspective, plus position. */
+static nir_shader *
+m5_mesh_multi_varying_shader(void)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"native mesh multi varyings");
+   b.shader->info.workgroup_size[0]=4;b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=4;b.shader->info.mesh.max_primitives_out=2;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES|
+      VARYING_BIT_VAR(0)|VARYING_BIT_VAR(1)|VARYING_BIT_VAR(2);
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;b.shader->info.io_lowered=true;
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,4),nir_imm_int(&b,2),nir_imm_int(&b,0));
+   nir_store_per_vertex_output(&b,nir_imm_vec4(&b,0,0,.25,1),lane,nir_imm_int(&b,0),
+      .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+   nir_store_per_vertex_output(&b,nir_vec4(&b,nir_u2f32(&b,lane),nir_imm_float(&b,1),
+      nir_imm_float(&b,2),nir_imm_float(&b,3)),lane,nir_imm_int(&b,0),
+      .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_VAR0,.num_slots=1});
+   nir_store_per_vertex_output(&b,nir_u2u16(&b,nir_vec2(&b,lane,nir_imm_int(&b,7))),lane,nir_imm_int(&b,0),
+      .write_mask=3,.src_type=nir_type_int16,.io_semantics={.location=VARYING_SLOT_VAR1,.num_slots=1});
+   nir_store_per_vertex_output(&b,nir_imm_float(&b,5),lane,nir_imm_int(&b,0),
+      .write_mask=1,.component=1,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_VAR2,.num_slots=1});
+   nir_push_if(&b,nir_ieq_imm(&b,lane,0));
+   nir_store_per_primitive_output(&b,nir_imm_ivec3(&b,0,1,2),nir_imm_int(&b,0),
+      nir_imm_int(&b,0),.write_mask=7,.src_type=nir_type_uint32,
+      .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+   nir_pop_if(&b,NULL);
+   return b.shader;
+}
+
+/* Point and line meshes with user and per-primitive words: the point size
+ * word follows the user words (agx_nir_lower_uvs group order) and every
+ * primitive gets one private vertex per corner for its flat words. */
+TEST(NativeForms, M5PointLineMeshPerPrimitiveWords)
+{
+   for (auto primitive : {MESA_PRIM_POINTS, MESA_PRIM_LINES}) {
+      const unsigned corners = primitive == MESA_PRIM_POINTS ? 1 : 2;
+      nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"point/line per-primitive");
+      b.shader->info.workgroup_size[0]=2;b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+      b.shader->info.mesh.max_vertices_out=2;b.shader->info.mesh.max_primitives_out=2;
+      b.shader->info.mesh.primitive_type=primitive;b.shader->info.io_lowered=true;
+      b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES|VARYING_BIT_VAR(0)|
+         (primitive==MESA_PRIM_POINTS ? VARYING_BIT_PSIZ : 0);
+      b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES|VARYING_BIT_VAR(0);
+      nir_def *lane=nir_load_local_invocation_index(&b);
+      nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,2),nir_imm_int(&b,2),nir_imm_int(&b,0));
+      nir_store_per_vertex_output(&b,nir_imm_vec4(&b,0,0,.25,1),lane,nir_imm_int(&b,0),
+         .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+      if (primitive==MESA_PRIM_POINTS)
+         nir_store_per_vertex_output(&b,nir_imm_float(&b,1),lane,nir_imm_int(&b,0),
+            .write_mask=1,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_PSIZ,.num_slots=1});
+      nir_store_per_primitive_output(&b,nir_imm_vec4(&b,0,0,1,1),lane,nir_imm_int(&b,0),
+         .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_VAR0,.num_slots=1});
+      nir_store_per_primitive_output(&b,primitive==MESA_PRIM_POINTS ? lane : nir_vec2(&b,lane,lane),
+         lane,nir_imm_int(&b,0),.write_mask=BITFIELD_MASK(corners),.src_type=nir_type_uint32,
+         .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+      struct agx_m5_mesh_varyings v;
+      ASSERT_TRUE(agx_m5_mesh_gather_varyings(b.shader,&v));
+      EXPECT_EQ(v.first[VARYING_SLOT_VAR0],4u);EXPECT_FALSE(v.dynamic);
+      if (primitive==MESA_PRIM_POINTS) {
+         EXPECT_EQ(v.first[VARYING_SLOT_PSIZ],8u);EXPECT_EQ(v.size,9u);
+      } else {
+         EXPECT_EQ(v.size,8u);
+      }
+      ASSERT_TRUE(agx_nir_lower_m5_mesh(b.shader,&v));
+      EXPECT_EQ(b.shader->info.mesh.max_vertices_out,2u*corners);
+      nir_validate_shader(b.shader,"point/line per-primitive");
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5MeshMultiVaryingPlanesFollowLinkedUVS)
+{
+   nir_shader *s=m5_mesh_multi_varying_shader();
+   struct agx_m5_mesh_varyings v;
+   ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));
+   EXPECT_EQ(v.components[VARYING_SLOT_VAR0],4u);EXPECT_EQ(v.components[VARYING_SLOT_VAR1],2u);
+   EXPECT_EQ(v.components[VARYING_SLOT_VAR2],2u);EXPECT_EQ(v.size,12u);EXPECT_TRUE(v.dynamic);
+   /* Linked order: smooth VAR0, linear VAR2 last, flat VAR1 between. */
+   v.first[VARYING_SLOT_VAR0]=4;v.first[VARYING_SLOT_VAR1]=8;v.first[VARYING_SLOT_VAR2]=10;v.dynamic=false;
+   ASSERT_TRUE(agx_nir_lower_m5_mesh(s,&v));
+   const unsigned stride=16,position=16+16;
+   EXPECT_EQ(s->info.shared_size,(position+12*stride+63)&~63u);
+   unsigned planes=0;bool seen[12]={};
+   nir_foreach_block(block,nir_shader_get_entrypoint(s))nir_foreach_instr(in,block) {
+      if(in->type!=nir_instr_type_intrinsic)continue;
+      nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+      EXPECT_NE(i->intrinsic,nir_intrinsic_load_uvs_index_agx);
+      if(i->intrinsic!=nir_intrinsic_store_mesh_uvb_agx||nir_intrinsic_flags(i))continue;
+      ASSERT_TRUE(nir_src_is_const(i->src[0]));
+      unsigned target=nir_src_comp_as_uint(i->src[0],0),address=nir_src_comp_as_uint(i->src[0],1);
+      ASSERT_LT(target&65535,12u);seen[target&65535]=true;planes++;
+      EXPECT_EQ(address,position+(target&65535)*stride);
+   }
+   EXPECT_EQ(planes,12u);for(unsigned k=0;k<12;k++)EXPECT_TRUE(seen[k])<<k;
+   /* Stores land in the linked planes: VAR2.y is plane 11. */
+   bool var2y=false;unsigned narrow=0;
+   nir_foreach_block(block,nir_shader_get_entrypoint(s))nir_foreach_instr(in,block) {
+      if(in->type!=nir_instr_type_intrinsic)continue;
+      nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+      if(i->intrinsic==nir_intrinsic_store_shared) {
+         /* Only the vertex/primitive counts are 16-bit; INT16 VAR1 is widened. */
+         narrow+=i->src[0].ssa->bit_size==16;
+         nir_scalar a=nir_scalar_resolved(i->src[1].ssa,0);
+         if(nir_scalar_is_alu(a)&&nir_scalar_alu_op(a)==nir_op_iadd) {
+            for(unsigned k=0;k<2;k++){nir_scalar c=nir_scalar_chase_alu_src(a,k);
+               if(nir_scalar_is_const(c)&&nir_scalar_as_uint(c)==position+11*stride)var2y=true;}
+         }
+      }
+   }
+   EXPECT_TRUE(var2y);EXPECT_EQ(narrow,2u);
+   nir_validate_shader(s,"native multi varying planes");agx_preprocess_nir(s);
+   agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+   agx_shader_part part={};ASSERT_TRUE(agx_compile_shader_nir(s,&key,&part))<<agx_last_compile_refusal();
+   free(part.binary);ralloc_free(s);
+}
+
+TEST(NativeForms, M5MeshDynamicVaryingSelectorsReadRootIndex)
+{
+   nir_shader *s=m5_mesh_multi_varying_shader();
+   struct agx_m5_mesh_varyings v;
+   ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));ASSERT_TRUE(v.dynamic);
+   ASSERT_TRUE(agx_nir_lower_m5_mesh(s,&v));
+   unsigned index_loads=0,dynamic=0,fixed=0;
+   nir_foreach_block(block,nir_shader_get_entrypoint(s))nir_foreach_instr(in,block) {
+      if(in->type!=nir_instr_type_intrinsic)continue;
+      nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+      if(i->intrinsic==nir_intrinsic_load_uvs_index_agx)index_loads++;
+      if(i->intrinsic!=nir_intrinsic_store_mesh_uvb_agx||nir_intrinsic_flags(i))continue;
+      if(nir_src_is_const(i->src[0])){fixed++;EXPECT_LT(nir_src_comp_as_uint(i->src[0],0)&65535,4u);}
+      else dynamic++;
+   }
+   EXPECT_EQ(index_loads,3u);EXPECT_EQ(fixed,4u);EXPECT_EQ(dynamic,8u);
+   nir_validate_shader(s,"native dynamic varying selectors");ralloc_free(s);
+}
+
+TEST(NativeForms, M5MeshClipDistancesPublishVaryingAndClippingPlanes)
+{
+   for (unsigned count : {2u, 8u}) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_MESH, &agx_nir_options, "native mesh clipping planes");
+      b.shader->info.io_lowered = true;
+      b.shader->info.workgroup_size[0] = 3;
+      b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+      b.shader->info.mesh.max_vertices_out = 3;
+      b.shader->info.mesh.max_primitives_out = 1;
+      b.shader->info.mesh.primitive_type = MESA_PRIM_TRIANGLES;
+      b.shader->info.clip_distance_array_size = count;
+      b.shader->info.outputs_written = VARYING_BIT_POS |
+         VARYING_BIT_PRIMITIVE_INDICES | VARYING_BIT_CLIP_DIST0 |
+         (count > 4 ? VARYING_BIT_CLIP_DIST1 : 0);
+      b.shader->info.per_primitive_outputs = VARYING_BIT_PRIMITIVE_INDICES;
+      nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, 3),
+                                       nir_imm_int(&b, 1), nir_imm_int(&b, 0));
+      nir_store_per_vertex_output(&b, nir_imm_vec4(&b, 0, 0, 0, 1),
+         nir_imm_int(&b, 0), nir_imm_int(&b, 0), .write_mask = 15,
+         .src_type = nir_type_float32,
+         .io_semantics = {.location = VARYING_SLOT_POS, .num_slots = 1});
+      for (unsigned c = 0; c < count; ++c) {
+         nir_store_per_vertex_output(&b, nir_imm_float(&b, c + 0.25f),
+            nir_imm_int(&b, 0), nir_imm_int(&b, 0), .write_mask = 1,
+            .component = c % 4, .src_type = nir_type_float32,
+            .io_semantics = {.location = VARYING_SLOT_CLIP_DIST0 + c / 4,
+                             .num_slots = 1});
+      }
+      nir_store_per_primitive_output(&b, nir_imm_ivec3(&b, 0, 1, 2),
+         nir_imm_int(&b, 0), nir_imm_int(&b, 0), .write_mask = 7,
+         .src_type = nir_type_uint32,
+         .io_semantics = {.location = VARYING_SLOT_PRIMITIVE_INDICES,
+                          .num_slots = 1});
+      struct agx_m5_mesh_varyings v;
+      ASSERT_TRUE(agx_m5_mesh_gather_varyings(b.shader, &v));
+      EXPECT_EQ(v.clip_distance_count, count);
+      EXPECT_EQ(v.clip_distance_first, 4u + count);
+      EXPECT_EQ(v.size, 4u + 2 * count);
+      EXPECT_EQ(v.dynamic, count > 4);
+      v.dynamic = false;
+      ASSERT_TRUE(agx_nir_lower_m5_mesh(b.shader, &v));
+      NIR_PASS(_, b.shader, nir_opt_constant_folding);
+      unsigned copies[8] = {0};
+      nir_foreach_block(block, b.impl) nir_foreach_instr(in, block) {
+         if (in->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+         if (i->intrinsic != nir_intrinsic_store_shared ||
+             !nir_src_is_const(i->src[1]))
+            continue;
+         unsigned address = nir_src_as_uint(i->src[1]);
+         if (address < 96)
+            continue;
+         ASSERT_LT(address, 32u + 16 * v.size);
+         unsigned component = ((address - 96) / 16) % count;
+         ASSERT_TRUE(nir_src_is_const(i->src[0]));
+         EXPECT_FLOAT_EQ(nir_src_as_float(i->src[0]), component + 0.25f);
+         copies[component]++;
+      }
+      for (unsigned c = 0; c < count; ++c)
+         EXPECT_EQ(copies[c], 2u);
+      nir_validate_shader(b.shader, "native mesh clip-distance publication");
+      agx_preprocess_nir_m5(b.shader);
+      agx_shader_key key = {};
+      key.dev.chip = AGX_CHIP_G17S;
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.no_preamble = true;
+      agx_shader_part part = {};
+      ASSERT_TRUE(agx_compile_shader_nir(b.shader, &key, &part))
+         << agx_last_compile_refusal();
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5MeshVaryingRefusalsLeaveShaderUntouched)
+{
+   /* A placement that does not cover every written component is refused. */
+   nir_shader *s=m5_mesh_multi_varying_shader();
+   struct agx_m5_mesh_varyings v;
+   ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));
+   v.first[VARYING_SLOT_VAR2]=v.first[VARYING_SLOT_VAR1];
+   EXPECT_FALSE(agx_nir_lower_m5_mesh(s,&v));
+   EXPECT_EQ(s->info.shared_size,0u);EXPECT_EQ(s->info.workgroup_size[0],4u);
+   ralloc_free(s);
+}
+
+TEST(NativeForms, M5MeshPaddedGlobalInvocationIdUsesApiSize)
+{
+   /* API workgroup 8 threads padded to 24 physical lanes (max_vertices 24):
+    * the hardware thread position in grid counts physical lanes, so the
+    * lowering must rebuild gl_GlobalInvocationID from the API size. */
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"padded global id");
+   b.shader->info.workgroup_size[0]=8;b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=24;b.shader->info.mesh.max_primitives_out=8;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES;
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;b.shader->info.io_lowered=true;
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,24),nir_imm_int(&b,8),nir_imm_int(&b,0));
+   nir_def *gid=nir_channel(&b,nir_load_global_invocation_id(&b,32),0);
+   nir_store_per_vertex_output(&b,nir_vec4(&b,nir_u2f32(&b,gid),nir_imm_float(&b,0),
+      nir_imm_float(&b,0),nir_imm_float(&b,1)),lane,nir_imm_int(&b,0),
+      .write_mask=15,.src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+   nir_store_per_primitive_output(&b,nir_imm_ivec3(&b,0,1,2),lane,nir_imm_int(&b,0),
+      .write_mask=7,.src_type=nir_type_uint32,
+      .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+   ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+   EXPECT_EQ(b.shader->info.workgroup_size[0],24u);
+   unsigned global=0,workgroup=0;
+   nir_foreach_block(block,b.impl)nir_foreach_instr(in,block) {
+      if(in->type!=nir_instr_type_intrinsic)continue;
+      nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+      global+=i->intrinsic==nir_intrinsic_load_global_invocation_id;
+      workgroup+=i->intrinsic==nir_intrinsic_load_workgroup_id;
+   }
+   EXPECT_EQ(global,0u);EXPECT_GE(workgroup,1u);
+   nir_validate_shader(b.shader,"padded global id");
+   agx_preprocess_nir(b.shader);agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;
+   key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+   agx_shader_part part={};ASSERT_TRUE(agx_compile_shader_nir(b.shader,&key,&part))<<agx_last_compile_refusal();
+   free(part.binary);ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5SaturatingAddOfHalfIndexCompiles)
+{
+   /* nir_opt_idiv_const turns an unsigned divide by 7 into
+    * umul_high(uadd_sat(x, 1), magic); x here is the 16-bit local index
+    * special register, zero-extended. The AGX3C saturating add only has a
+    * witnessed 32-bit register form, so the extension must stay a move. */
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,&agx_nir_options,"udiv7 of local index");
+   b.shader->info.workgroup_size[0]=64;b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_def *q=nir_udiv_imm(&b,lane,7);
+   nir_store_global(&b,q,nir_iadd(&b,nir_imm_int64(&b,0x2000000),
+      nir_u2u64(&b,nir_imul_imm(&b,lane,4))),.write_mask=1,.align_mul=4);
+   agx_preprocess_nir(b.shader);
+   agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+   agx_shader_part part={};
+   ASSERT_TRUE(agx_compile_shader_nir(b.shader,&key,&part))<<agx_last_compile_refusal();
+   free(part.binary);ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5NativeTriangleMeshProbeCompiles)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,
+                     &agx_nir_options,"M5 native triangle mesh probe");
+   b.shader->info.workgroup_size[0]=3;b.shader->info.workgroup_size[1]=1;
+   b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=3;b.shader->info.mesh.max_primitives_out=1;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES;
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES;
+   b.shader->info.io_lowered=true;
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,3),nir_imm_int(&b,1),
+                                       nir_imm_int(&b,0),.stream_id=0);
+   nir_def *x=nir_bcsel(&b,nir_ieq_imm(&b,lane,2),nir_imm_float(&b,0),
+                nir_bcsel(&b,nir_ieq_imm(&b,lane,0),nir_imm_float(&b,-1),
+                                                     nir_imm_float(&b,1)));
+   nir_def *y=nir_bcsel(&b,nir_ieq_imm(&b,lane,2),nir_imm_float(&b,1),
+                                                     nir_imm_float(&b,-1));
+   nir_def *pos=nir_vec4(&b,x,y,nir_imm_float(&b,0.25),nir_imm_float(&b,1));
+   nir_store_per_vertex_output(&b,pos,lane,nir_imm_int(&b,0),
+     .write_mask=15,.src_type=nir_type_float32,
+     .io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+   nir_push_if(&b,nir_ieq_imm(&b,lane,0));
+   nir_store_per_primitive_output(&b,nir_imm_ivec3(&b,0,1,2),
+     nir_imm_int(&b,0),nir_imm_int(&b,0),.write_mask=7,.src_type=nir_type_uint32,
+     .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+   nir_pop_if(&b,NULL);
+   ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+   EXPECT_EQ(b.shader->info.shared_size,128u);
+   nir_validate_shader(b.shader,"native triangle mesh probe");
+   agx_preprocess_nir(b.shader);
+   agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;
+   key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+   agx_shader_part part={};
+   bool compiled=agx_compile_shader_nir(b.shader,&key,&part);
+   EXPECT_TRUE(compiled)<<agx_last_compile_refusal();
+   if(compiled){
+      printf("native-triangle-mesh binary=%u gprs=%u shared=%u scratch=%u\n",
+        part.info.binary_size,part.info.nr_gprs,part.info.local_size,part.info.scratch_size);
+      FILE *f=fopen("native-triangle-mesh.bin","wb");
+      if(f){fwrite(part.binary,1,part.info.binary_size,f);fclose(f);}
+      free(part.binary);
+   }
+   ralloc_free(b.shader);
+}
+
+/* cts-coverage/fp-controls/native/nanmin.gpubin, G17G with -fno-fast-math:
+ * IEEE minNum/maxNum retain condition 3/7 and the compare/select ties. */
+TEST(NativeForms, M5MinNumMaxNumNaNWitnesses)
+{
+   for (bool maximum : {false, true}) {
+      agx3_fsel32_gpu1_desc d = {};
+      d.dest = 2;
+      d.cmp0 = 2; d.cmp1 = 3;
+      d.cmp0_16 = d.cmp1_16 = true;
+      d.cmp0_hint = d.cmp1_hint = AGX3_ALU_HINT_DISCARD;
+      d.sel[0] = 2; d.sel[1] = 3;
+      d.sel_hint[0] = d.sel_hint[1] = AGX3_ALU_HINT_DISCARD;
+      d.sel_integer = true;
+      d.condition = maximum ? 7 : 3;
+      d.wait_mask = 0x80;
+      uint8_t out[AGX3_ALU_MAX_LENGTH] = {}, length = 0;
+      ASSERT_TRUE(agx3_alu_fcmp_select16_gpu1(&d, out, &length));
+      const std::vector<uint8_t> native = maximum
+         ? std::vector<uint8_t>{0x32,0x02,0x05,0x0a,0x23,0x00,0xa0,0x02,
+                               0xcc,0x80,0xa0,0x64,0x26,0x00,0x80,0x00}
+         : std::vector<uint8_t>{0x32,0x02,0x45,0x0a,0x23,0x00,0xa0,0x02,
+                               0xcc,0x80,0xa0,0x60,0x26,0x00,0x80,0x00};
+      EXPECT_EQ(std::vector<uint8_t>(out, out + length), native);
+      d.sel[1] = 5;
+      EXPECT_FALSE(agx3_alu_fcmp_select16_gpu1(&d, out, &length));
+   }
+   agx3_fsel32_gpu1_desc d = {};
+   d.dest = 4; d.dest_cache = true;
+   d.cmp0 = 4; d.cmp1 = 6;
+   d.cmp0_hint = d.cmp1_hint = AGX3_ALU_HINT_DISCARD;
+   d.sel[0] = 4; d.sel[1] = 6;
+   d.sel_hint[0] = d.sel_hint[1] = AGX3_ALU_HINT_DISCARD;
+   d.sel_integer = true; d.condition = 7; d.wait_mask = 0x80;
+   uint8_t out[AGX3_ALU_MAX_LENGTH] = {}, length = 0;
+   ASSERT_TRUE(agx3_alu_fcmp_select32_gpu1(&d, out, &length));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + length),
+      (std::vector<uint8_t>{0x22,0x00,0x07,0x2b,0x2b,0x00,0xa0,
+                           0x0a,0x96,0x01,0xa1,0x64,0x20,0x40}));
+   d.dest = 2; d.dest_cache = false; d.cmp1 = d.sel[1] = 2;
+   d.condition = 3; d.wait_mask = 1;
+   ASSERT_TRUE(agx3_alu_fcmp_select32_gpu1(&d, out, &length));
+   EXPECT_EQ(std::vector<uint8_t>(out, out + length),
+             (std::vector<uint8_t>{0x12,0x05,0x1e,0x03,0x01,0x20}));
+   d.sel[0] = 8;
+   EXPECT_FALSE(agx3_alu_fcmp_select32_gpu1(&d, out, &length));
+}
+
+TEST(NativeForms, M5MappedBufferPointerOffsets)
+{
+   nir_builder builder = nir_builder_init_simple_shader(
+      MESA_SHADER_COMPUTE, &agx_nir_options, "M5 mapped buffer pointers");
+   nir_builder *b = &builder;
+   std::vector<nir_intrinsic_instr *> stores;
+   auto store_test_val = [&](nir_def *value) {
+      nir_intrinsic_instr *store = nir_intrinsic_instr_create(
+         b->shader, nir_intrinsic_store_global);
+      store->num_components = 1;
+      store->src[0] = nir_src_for_ssa(value);
+      store->src[1] = nir_src_for_ssa(nir_imm_int64(b, 0x1000 + 4 * stores.size()));
+      nir_intrinsic_set_align(store, 4, 0);
+      nir_intrinsic_set_write_mask(store, 1);
+      nir_builder_instr_insert(b, &store->instr);
+      stores.push_back(store);
+   };
+   auto test_val = [&](unsigned i) { return stores[i]->src[0].ssa; };
+   for (nir_address_format format : {
+        nir_address_format_64bit_global_32bit_offset_map,
+        nir_address_format_64bit_bounded_global_map}) {
+      nir_def *parts[] = {nir_imm_int(b, 0x1000), nir_imm_int(b, 2),
+         nir_imm_int(b, 4096), nir_imm_int(b, 12), nir_imm_int(b, 0x1234)};
+      nir_def *ptr = nir_vec(b, parts, 5);
+      ptr = nir_build_addr_iadd(b, ptr, format, nir_var_mem_ssbo,
+                               nir_imm_int(b, 28));
+      ptr = nir_bcsel(b, nir_imm_false(b), nir_build_imm(b, 5, 32, nir_address_format_null_value(format)), ptr);
+      unsigned start = stores.size();
+      for (unsigned c = 0; c < 5; ++c)
+         store_test_val(nir_channel(b, ptr, c));
+      nir_def *other = nir_vector_insert_imm(b, ptr, nir_imm_int(b, 0x4321), 4);
+      store_test_val(nir_b2i32(b, nir_build_addr_ieq(b, ptr, other, format)));
+      nir_def *changed = nir_build_addr_iadd(b, ptr, format, nir_var_mem_ssbo,
+                                           nir_imm_int(b, 4));
+      store_test_val(nir_b2i32(b, nir_build_addr_ieq(b, ptr, changed, format)));
+      NIR_PASS(_, b->shader, nir_opt_constant_folding);
+      const uint32_t expected[] = {0x1000, 2, 4096, 40, 0x1234, 1, 0};
+      for (unsigned c = 0; c < 7; ++c)
+         EXPECT_EQ(nir_src_as_uint(nir_src_for_ssa(test_val(start + c))), expected[c]);
+   }
+   nir_validate_shader(b->shader, "mapped buffer pointers");
+   ralloc_free(b->shader);
+}
+
+TEST(NativeForms, M5GemmNativeCapturedInstructions)
+{
+   struct Case { agx3_matrix_gemm16_desc desc; uint8_t bytes[10]; };
+   const Case cases[] = {
+      {{16, 80, 72, AGX3_MATRIX_INPUT_BF16, true,false,false, 1}, {0xaf,0x00,0x25,0x82,0x2a,0x02,0xa0,0x02,0x00,0x12}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_BF16, false,false,true, 0}, {0xa7,0x04,0x25,0x80,0x22,0x02,0xa0,0x02,0x00,0x0e}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_BF16, true,false,false, 1}, {0xaf,0x00,0x25,0x80,0x2a,0x02,0xa0,0x02,0x00,0x0e}},
+      {{16, 48, 0, AGX3_MATRIX_INPUT_BF16, false,false,true, 0}, {0xa7,0x04,0x25,0x03,0x22,0x02,0xa0,0x02,0x00,0x00}},
+      {{16, 72, 64, AGX3_MATRIX_INPUT_BF16, true,false,false, 1}, {0xaf,0x00,0x25,0x90,0x2a,0x02,0xa0,0x02,0x00,0x10}},
+      {{16, 56, 48, AGX3_MATRIX_INPUT_BF16, false,false,true, 0}, {0xa7,0x04,0x25,0x13,0x22,0x02,0xa0,0x02,0x00,0x0c}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_BF16, true,false,false, 1}, {0xaf,0x00,0x25,0x80,0x2a,0x02,0xa0,0x02,0x00,0x0e}},
+      {{16, 48, 0, AGX3_MATRIX_INPUT_BF16, false,false,true, 0}, {0xa7,0x04,0x25,0x03,0x22,0x02,0xa0,0x02,0x00,0x00}},
+      {{16, 80, 72, AGX3_MATRIX_INPUT_F16, true,false,false, 1}, {0xaf,0x00,0x25,0x82,0x2a,0x02,0xa4,0x42,0x00,0x12}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_F16, false,false,true, 0}, {0xa7,0x04,0x25,0x80,0x22,0x02,0xa4,0x42,0x00,0x0e}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_F16, true,false,false, 1}, {0xaf,0x00,0x25,0x80,0x2a,0x02,0xa4,0x42,0x00,0x0e}},
+      {{16, 48, 0, AGX3_MATRIX_INPUT_F16, false,false,true, 0}, {0xa7,0x04,0x25,0x03,0x22,0x02,0xa4,0x42,0x00,0x00}},
+      {{16, 72, 64, AGX3_MATRIX_INPUT_F16, true,false,false, 1}, {0xaf,0x00,0x25,0x90,0x2a,0x02,0xa4,0x42,0x00,0x10}},
+      {{16, 56, 48, AGX3_MATRIX_INPUT_F16, false,false,true, 0}, {0xa7,0x04,0x25,0x13,0x22,0x02,0xa4,0x42,0x00,0x0c}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_F16, true,false,false, 1}, {0xaf,0x00,0x25,0x80,0x2a,0x02,0xa4,0x42,0x00,0x0e}},
+      {{16, 48, 0, AGX3_MATRIX_INPUT_F16, false,false,true, 0}, {0xa7,0x04,0x25,0x03,0x22,0x02,0xa4,0x42,0x00,0x00}},
+      {{16, 80, 72, AGX3_MATRIX_INPUT_F16, true,true,false, 1}, {0xaf,0x00,0x25,0x82,0x2a,0x02,0xa4,0x42,0x08,0x12}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_F16, false,true,true, 0}, {0xa7,0x04,0x25,0x80,0x22,0x02,0xa4,0x42,0x08,0x0e}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_F16, true,true,false, 1}, {0xaf,0x00,0x25,0x80,0x2a,0x02,0xa4,0x42,0x08,0x0e}},
+      {{16, 48, 0, AGX3_MATRIX_INPUT_F16, false,true,true, 0}, {0xa7,0x04,0x25,0x03,0x22,0x02,0xa4,0x42,0x08,0x00}},
+      {{16, 72, 64, AGX3_MATRIX_INPUT_F16, true,true,false, 1}, {0xaf,0x00,0x25,0x90,0x2a,0x02,0xa4,0x42,0x08,0x10}},
+      {{16, 56, 48, AGX3_MATRIX_INPUT_F16, true,true,true, 0}, {0xa7,0x04,0x25,0x13,0x2a,0x02,0xa4,0x42,0x08,0x0c}},
+      {{16, 64, 56, AGX3_MATRIX_INPUT_F16, true,true,false, 1}, {0xaf,0x00,0x25,0x80,0x2a,0x02,0xa4,0x42,0x08,0x0e}},
+      {{16, 48, 0, AGX3_MATRIX_INPUT_F16, false,true,true, 0}, {0xa7,0x04,0x25,0x03,0x22,0x02,0xa4,0x42,0x08,0x00}},
+   };
+   for (const auto &c : cases) {
+      uint8_t out[10] = {}, length = 0;
+      ASSERT_TRUE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3C,
+                                           &c.desc, out, &length));
+      ASSERT_EQ(length, 10u);
+      EXPECT_EQ(std::vector<uint8_t>(out, out + length),
+                std::vector<uint8_t>(c.bytes, c.bytes + 10));
+      EXPECT_FALSE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3B,
+                                            &c.desc, out, &length));
+   }
+   auto invalid = cases[0].desc;
+   uint8_t out[10] = {}, length = 0;
+   invalid.dest = 17;
+   EXPECT_FALSE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3C,
+                                         &invalid, out, &length));
+   invalid = cases[0].desc;
+   invalid.a = 81;
+   EXPECT_FALSE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3C,
+                                         &invalid, out, &length));
+   invalid = cases[0].desc;
+   invalid.input_type = static_cast<agx3_matrix_input_type>(2);
+   EXPECT_FALSE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3C,
+                                         &invalid, out, &length));
+}
+
+TEST(NativeForms, M5MeshCullDistancesKeepVertexAndClipDomains)
+{
+   nir_shader *s = m5_mesh_multi_varying_shader();
+   s->info.clip_distance_array_size = 1;
+   s->info.cull_distance_array_size = 5;
+   s->info.outputs_written |= VARYING_BIT_CLIP_DIST0 | VARYING_BIT_CLIP_DIST1;
+   nir_builder b = nir_builder_at(nir_after_cf_list(&nir_shader_get_entrypoint(s)->body));
+   const float distances[] = {-7.0f, -1.0f, 0.0f, 2.0f, -2.0f, 1.0f};
+   for (unsigned c = 0; c < ARRAY_SIZE(distances); ++c) {
+      nir_store_per_vertex_output(&b, nir_imm_float(&b, distances[c]),
+         nir_imm_int(&b, 2), nir_imm_int(&b, c / 4),
+         .write_mask = 1, .component = c % 4, .src_type = nir_type_float32,
+         .io_semantics = {.location = VARYING_SLOT_CLIP_DIST0, .num_slots = 2});
+   }
+   ASSERT_TRUE(agx_nir_lower_cull_distance_vs(s));
+   NIR_PASS(_, s, nir_opt_constant_folding);
+   unsigned masks = 0;
+   bool found[5] = {};
+   nir_foreach_block(block, nir_shader_get_entrypoint(s)) nir_foreach_instr(in, block) {
+      if (in->type != nir_instr_type_intrinsic) continue;
+      nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+      if (i->intrinsic != nir_intrinsic_store_per_vertex_output)
+         continue;
+      const unsigned slot = nir_intrinsic_io_semantics(i).location;
+      if (slot < VARYING_SLOT_CULL_PRIMITIVE ||
+          slot > VARYING_SLOT_CULL_PRIMITIVE + 1)
+         continue;
+      unsigned c = 4 * (slot - VARYING_SLOT_CULL_PRIMITIVE +
+                        nir_src_as_uint(i->src[2])) + nir_intrinsic_component(i);
+      ASSERT_LT(c, 5u);
+      EXPECT_FALSE(found[c]); found[c] = true;
+      EXPECT_EQ(nir_src_as_uint(i->src[1]), 2u);
+      ASSERT_TRUE(nir_src_is_const(i->src[0]));
+      EXPECT_FLOAT_EQ(nir_src_as_float(i->src[0]), distances[c + 1] >= 0.0f ? 1.0f : 0.0f);
+      masks++;
+   }
+   EXPECT_EQ(masks, 5u);
+   struct agx_m5_mesh_varyings v;
+   ASSERT_TRUE(agx_m5_mesh_gather_varyings(s, &v));
+   EXPECT_EQ(v.components[VARYING_SLOT_CULL_PRIMITIVE], 4u);
+   EXPECT_EQ(v.components[VARYING_SLOT_CULL_PRIMITIVE + 1], 1u);
+   EXPECT_EQ(v.clip_distance_count, 1u);
+   v.dynamic = false;
+   ASSERT_TRUE(agx_nir_lower_m5_mesh(s, &v));
+   nir_validate_shader(s, "mesh cull and clip distance domains");
+   agx_preprocess_nir_m5(s);
+   agx_shader_key key = {};
+   key.dev.chip = AGX_CHIP_G17S; key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.no_preamble = true;
+   agx_shader_part part = {};
+   ASSERT_TRUE(agx_compile_shader_nir(s, &key, &part)) << agx_last_compile_refusal();
+   free(part.binary); ralloc_free(s);
+}
+
+TEST(NativeForms, M5IntegerRegisterBallotPredicatesReachPacker)
+{
+   for (agx_icond condition : {AGX_ICOND_UEQ, AGX_ICOND_SEQ, AGX_ICOND_ULT,
+                              AGX_ICOND_UGT, AGX_ICOND_SLT, AGX_ICOND_SGT}) {
+      for (bool invert : {false, true}) {
+         void *mem = ralloc_context(NULL);
+         agx_shader_key key = {};
+         key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+         key.dev.chip = AGX_CHIP_G17S;
+         agx_builder *b = agx_test_builder(mem);
+         b->shader->key = &key;
+         b->shader->stage = MESA_SHADER_COMPUTE;
+         agx_instr *ballot = agx_alloc_instr(b, AGX_OPCODE_ICMP_BALLOT, 1, 2);
+         ballot->dest[0] = agx_register(22, AGX_SIZE_32);
+         ballot->src[0] = agx_register(20, AGX_SIZE_32);
+         ballot->src[1] = agx_register(16, AGX_SIZE_32);
+         ballot->icond = condition;
+         ballot->invert_cond = invert;
+         agx_builder_insert(&b->cursor, ballot);
+         agx_stop(b);
+         util_dynarray bytes;
+         util_dynarray_init(&bytes, mem);
+         EXPECT_TRUE(agx3_pack_binary(b->shader, &bytes)) << condition << " invert=" << invert;
+         EXPECT_GT(bytes.size, 4u);
+         ralloc_free(mem);
+      }
+   }
+}
+
+TEST(NativeForms, M5GemmIndependentIntrinsicWitnesses)
+{
+   struct Case { unsigned a, b; bool zero, transpose, preserve; uint8_t bytes[10]; };
+   const Case cases[] = {
+      {8, 0, true, false, true, {0x2f,0x00,0x05,0x10,0x22,0x00,0xa4,0x42,0x02,0x00}},
+      {8, 0, true, true, true, {0x2f,0x00,0x05,0x10,0x22,0x00,0xa4,0x42,0x0e,0x00}},
+      {24,16,false,false,false,{0x2f,0x00,0x25,0x12,0x22,0x02,0xa4,0x42,0x00,0x04}},
+   };
+   for (const auto &c : cases) {
+      agx3_matrix_gemm16_desc desc = {};
+      desc.a = c.a; desc.b = c.b; desc.input_type = AGX3_MATRIX_INPUT_F16;
+      desc.wait_mask = 128; desc.zero_accumulator = c.zero;
+      desc.transpose_a = c.transpose; desc.transpose_b = c.transpose;
+      desc.preserve_a = c.preserve; desc.preserve_b = c.preserve;
+      uint8_t bytes[10], length;
+      ASSERT_TRUE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3C, &desc, bytes, &length));
+      ASSERT_EQ(length, 10u);
+      EXPECT_EQ(std::vector<uint8_t>(bytes, bytes + length), std::vector<uint8_t>(c.bytes, c.bytes + 10));
+   }
+}
+
+TEST(NativeForms, M5MmaNativeOperandTypeWitnesses)
+{
+   struct Case { agx3_matrix_mma8_desc desc; uint8_t bytes[14]; };
+   const Case cases[] = {
+      {{14,78,52,14,false,true,{false,false,false},{true,true,true},0,AGX3_MATRIX_INPUT_F32},
+       {0x37,0x02,0x05,0x7a,0x28,0x20,0xaf,0x1a,0x31,0x08,0x18,0x00,0x1a,0x19}},
+      {{102,102,94,98,false,true,{false,true,true},{true,false,false},128,AGX3_MATRIX_INPUT_BF16},
+       {0x3f,0x02,0x85,0xbb,0x28,0x20,0xaf,0x2a,0x61,0x04,0x19,0x00,0x86,0x04}},
+   };
+   for (const auto &c : cases) {
+      uint8_t bytes[14], length;
+      ASSERT_TRUE(agx3_matrix_encode_mma8(AGX_ISA_ENCODING_AGX3C, &c.desc, bytes, &length));
+      ASSERT_EQ(length, 14u);
+      EXPECT_EQ(std::vector<uint8_t>(bytes, bytes + length), std::vector<uint8_t>(c.bytes, c.bytes + 14));
+   }
+}
+
+TEST(NativeForms, M5GemmMixedOperandExecutionWitnesses)
+{
+   agx3_matrix_gemm16_desc d = {};
+   d.a = 24; d.b = 16; d.wait_mask = 128;
+   for (unsigned swap = 0; swap < 2; ++swap) {
+      d.input_type = swap ? AGX3_MATRIX_INPUT_F16 : AGX3_MATRIX_INPUT_BF16;
+      d.b_input_type = swap ? AGX3_MATRIX_INPUT_BF16 : AGX3_MATRIX_INPUT_F16;
+      const uint8_t expected[2][10] = {
+         {0x2f,0x00,0x25,0x12,0x22,0x02,0xa0,0x42,0x00,0x04},
+         {0x2f,0x00,0x25,0x12,0x22,0x02,0xa4,0x02,0x00,0x04},
+      };
+      uint8_t bytes[10], length;
+      ASSERT_TRUE(agx3_matrix_encode_gemm16(AGX_ISA_ENCODING_AGX3C, &d, bytes, &length));
+      ASSERT_EQ(length, 10u);
+      EXPECT_EQ(std::vector<uint8_t>(bytes, bytes + length), std::vector<uint8_t>(expected[swap], expected[swap] + 10));
+   }
+}
+
+TEST(NativeForms, M5VectorAccumulatorCopiesEveryComponent)
+{
+   for (unsigned channels : {2u, 4u, 8u}) {
+      for (unsigned dest : {8u, 34u, 64u}) {
+         void *mem = ralloc_context(NULL);
+         agx_shader_key key = {};
+         key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+         agx_builder *b = agx_test_builder(mem);
+         b->shader->key = &key;
+         agx_index src = agx_register(32, AGX_SIZE_32);
+         agx_index dst = agx_register(dest, AGX_SIZE_32);
+         src.channels_m1 = dst.channels_m1 = channels - 1;
+         agx_mov_to(b, dst, src);
+         agx_stop(b);
+         agx_lower_pseudo(b->shader);
+         uint32_t registers[128];
+         for (unsigned i = 0; i < ARRAY_SIZE(registers); ++i)
+            registers[i] = 0x10000000 + i;
+         unsigned copies = 0;
+         agx_foreach_instr_global(b->shader, I) {
+            if (I->op == AGX_OPCODE_STOP)
+               continue;
+            ASSERT_EQ(I->op, AGX_OPCODE_BITOP);
+            ASSERT_EQ(agx_channels(I->dest[0]), 1u);
+            ASSERT_EQ(agx_channels(I->src[0]), 1u);
+            registers[I->dest[0].value] = registers[I->src[0].value];
+            ++copies;
+         }
+         EXPECT_EQ(copies, channels);
+         for (unsigned c = 0; c < channels; ++c)
+            EXPECT_EQ(registers[dest + 2 * c], 0x10000000u + 32 + 2 * c);
+         util_dynarray bytes;
+         util_dynarray_init(&bytes, mem);
+         EXPECT_TRUE(agx3_pack_binary(b->shader, &bytes));
+         ralloc_free(mem);
+      }
+   }
+}
+
+TEST(NativeForms, M5QuadHelpersKeepFragmentEntryUnmarked)
+{
+   for (unsigned helpers : {0u, 1u, 2u}) {
+      SCOPED_TRACE(helpers);
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.dev.chip = AGX_CHIP_G17S;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->stage = MESA_SHADER_FRAGMENT;
+      b->shader->nir = rzalloc(mem, nir_shader);
+      b->shader->nir->info.stage = MESA_SHADER_FRAGMENT;
+      b->shader->nir->info.fs.needs_coarse_quad_helper_invocations = helpers == 1;
+      b->shader->nir->info.fs.needs_full_quad_helper_invocations = helpers == 2;
+      agx_interp_agx3_to(b, agx_register(0, AGX_SIZE_32), agx_immediate(1),
+                         agx_immediate(0), AGX_INTERPOLATION_CENTER, false);
+      agx_stop(b);
+      util_dynarray binary;
+      util_dynarray_init(&binary, mem);
+      ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+      uint8_t expected[AGX3_VARY_MAX_LENGTH], len = 0;
+      ASSERT_TRUE(agx3_vary_interp_at_start(AGX_ISA_ENCODING_AGX3C,
+                     0, 1, false, AGX3_VARY_CENTER, helpers == 0, expected, &len));
+      ASSERT_GE(binary.size, len);
+      EXPECT_EQ(memcmp(binary.data, expected, len), 0);
+      ralloc_free(mem);
+   }
+}
+
+
+TEST(NativeForms, M5MultisampleTileDataRemainsLiveUntilLastStore)
+{
+   for (unsigned targets : {1u, 4u}) {
+      SCOPED_TRACE(targets);
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.dev.chip = AGX_CHIP_G17S;
+      key.secondary = true;
+      key.no_preamble = true;
+      key.fs.nr_samples = 4;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->stage = MESA_SHADER_FRAGMENT;
+      b->shader->agx3_mrt_tile_ops_ordered = true;
+      for (unsigned rt = 0; rt < targets; ++rt)
+         BITSET_SET(key.g17p_tile_partitions, 4 * rt);
+      agx_wait_pix(b, 12);
+      std::vector<std::vector<uint8_t>> expected;
+      for (unsigned sample = 0; sample < 4; ++sample) {
+         for (unsigned target = targets; target-- > 0;) {
+            const bool last = sample == 3 && target == 0;
+            agx_index data = agx_register(0, AGX_SIZE_16);
+            data.channels_m1 = 1;
+            data.discard = last;
+            const agx_index coverage = agx_register(16 + sample, AGX_SIZE_16);
+            agx_st_tile(b, data, coverage, agx_null(), AGX_FORMAT_U8NORM,
+                        15, target * 4, false, target);
+            uint8_t bytes[AGX3_TILE_MAX_LENGTH], length;
+            ASSERT_TRUE(agx3_tile_store_f13(AGX_ISA_ENCODING_AGX3C, 0,
+               AGX3_TILE_W8, 15, target * 4,
+               agx3_tile_sample_reg(16 + sample, false), 0, last,
+               bytes, &length));
+            expected.emplace_back(bytes, bytes + length);
+         }
+      }
+      agx_stop(b);
+      util_dynarray binary;
+      util_dynarray_init(&binary, mem);
+      ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+      const uint8_t *start = (const uint8_t *)binary.data;
+      const uint8_t *end = start + binary.size;
+      for (const auto &bytes : expected) {
+         auto found = std::search(start, end, bytes.begin(), bytes.end());
+         ASSERT_NE(found, end);
+         start = found + bytes.size();
+      }
+      ralloc_free(mem);
+   }
+}
+
+TEST(NativeForms, M5EmptyMeshTopology)
+{
+   for (auto primitive : {MESA_PRIM_POINTS, MESA_PRIM_LINES}) {
+      nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+         &agx_nir_options, "empty mesh with memory work");
+      b.shader->info.workgroup_size[0] = 32;
+      b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+      b.shader->info.mesh.primitive_type = primitive;
+      b.shader->info.mesh.max_vertices_out = 1;
+      b.shader->info.mesh.max_primitives_out = 1;
+      b.shader->info.io_lowered = true;
+      nir_def *lane = nir_load_local_invocation_index(&b);
+      nir_store_global(&b, lane, nir_iadd(&b, nir_imm_int64(&b, 0x2000000),
+                       nir_u2u64(&b, nir_imul_imm(&b, lane, 4))),
+                       .write_mask=1, .align_mul=4);
+      nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, 0),
+         nir_imm_int(&b, 0), nir_undef(&b, 1, 32));
+      ASSERT_TRUE(agx_nir_m5_canonicalize_empty_mesh(b.shader));
+      EXPECT_EQ(b.shader->info.mesh.primitive_type, MESA_PRIM_TRIANGLES);
+      ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+      agx_preprocess_nir(b.shader);
+      agx_shader_key key = {};
+      key.dev.chip = AGX_CHIP_G17S;
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.no_preamble = true;
+      agx_shader_part part = {};
+      ASSERT_TRUE(agx_compile_shader_nir(b.shader, &key, &part))
+         << agx_last_compile_refusal();
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5EmptyMeshRejectsOutput)
+{
+   for (unsigned nonzero = 0; nonzero < 3; ++nonzero) {
+      nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+         &agx_nir_options, "nonempty mesh");
+      b.shader->info.mesh.primitive_type = MESA_PRIM_POINTS;
+      if (nonzero == 2)
+         b.shader->info.outputs_written = VARYING_BIT_POS;
+      nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, nonzero == 0),
+         nir_imm_int(&b, nonzero == 1), nir_undef(&b, 1, 32));
+      EXPECT_FALSE(agx_nir_m5_canonicalize_empty_mesh(b.shader));
+      EXPECT_EQ(b.shader->info.mesh.primitive_type, MESA_PRIM_POINTS);
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5EmptyMeshRejectsConditionalCount)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+      &agx_nir_options, "conditional mesh count");
+   b.shader->info.mesh.primitive_type = MESA_PRIM_POINTS;
+   EXPECT_FALSE(agx_nir_m5_canonicalize_empty_mesh(b.shader));
+   nir_push_if(&b, nir_ieq_imm(&b, nir_load_local_invocation_index(&b), 0));
+   nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, 0),
+      nir_imm_int(&b, 0), nir_undef(&b, 1, 32));
+   nir_pop_if(&b, NULL);
+   EXPECT_FALSE(agx_nir_m5_canonicalize_empty_mesh(b.shader));
+   EXPECT_EQ(b.shader->info.mesh.primitive_type, MESA_PRIM_POINTS);
+   ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5MeshActiveSubgroupIndexReachesPacker)
+{
+   for (unsigned half : {0u, 1u, 14u, 63u, 254u}) {
+      void *mem = ralloc_context(NULL);
+      agx_shader_key key = {};
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.dev.chip = AGX_CHIP_G17S;
+      agx_builder *b = agx_test_builder(mem);
+      b->shader->key = &key;
+      b->shader->stage = MESA_SHADER_MESH;
+      agx_get_sr_coverage_to(b, agx_register(half, AGX_SIZE_16),
+                             AGX_SR_ACTIVE_THREAD_INDEX_IN_SUBGROUP);
+      agx_stop(b);
+      util_dynarray bytes;
+      util_dynarray_init(&bytes, mem);
+      EXPECT_TRUE(agx3_pack_binary(b->shader, &bytes));
+      EXPECT_GT(bytes.size, 4u);
+      ralloc_free(mem);
+   }
+}
+
+TEST(NativeForms, M5MeshLargeWorkgroupsCompile)
+{
+   const unsigned sizes[][3] = {
+      {129,1,1}, {256,1,1}, {512,1,1}, {1024,1,1},
+      {1,1024,1}, {1,1,1024}, {3,5,7}, {1025,1,1},
+   };
+   for (const auto &size : sizes) {
+      SCOPED_TRACE(size[0] * size[1] * size[2]);
+      nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+         &agx_nir_options, "mesh workgroup bounds");
+      for (unsigned i = 0; i < 3; ++i)
+         b.shader->info.workgroup_size[i] = size[i];
+      b.shader->info.mesh.primitive_type = MESA_PRIM_TRIANGLES;
+      b.shader->info.mesh.max_vertices_out = 1;
+      b.shader->info.mesh.max_primitives_out = 1;
+      b.shader->info.io_lowered = true;
+      nir_def *lane = nir_load_local_invocation_index(&b);
+      nir_def *id = nir_load_local_invocation_id(&b);
+      nir_def *value = nir_iadd(&b, lane, nir_channel(&b, id, 2));
+      nir_store_global(&b, value, nir_iadd(&b, nir_imm_int64(&b, 0x2000000),
+                       nir_u2u64(&b, nir_imul_imm(&b, lane, 4))),
+                       .write_mask=1, .align_mul=4);
+      nir_barrier(&b, .execution_scope=SCOPE_WORKGROUP,
+         .memory_scope=SCOPE_WORKGROUP, .memory_semantics=NIR_MEMORY_ACQ_REL,
+         .memory_modes=nir_var_mem_global);
+      nir_set_vertex_and_primitive_count(&b, nir_imm_int(&b, 0),
+         nir_imm_int(&b, 0), nir_undef(&b, 1, 32));
+      bool valid = size[0] * size[1] * size[2] <= 1024;
+      ASSERT_EQ(agx_nir_lower_m5_mesh_triangle_probe(b.shader), valid);
+      if (!valid) {
+         EXPECT_EQ(b.shader->info.workgroup_size[0], size[0]);
+         ralloc_free(b.shader);
+         continue;
+      }
+      EXPECT_EQ(b.shader->info.workgroup_size[2], size[2] > 1023 ? 1u : size[2]);
+      agx_preprocess_nir_m5(b.shader);
+      agx_shader_key key = {};
+      key.dev.chip = AGX_CHIP_G17S;
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.no_preamble = true;
+      agx_shader_part part = {};
+      ASSERT_TRUE(agx_compile_shader_nir(b.shader, &key, &part))
+         << agx_last_compile_refusal();
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5MeshImageReadStagesCoordinates)
+{
+   void *mem = ralloc_context(NULL);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   agx_builder *b = agx_test_builder(mem);
+   b->shader->key = &key;
+   b->shader->stage = MESA_SHADER_MESH;
+   agx_instr *read = agx_alloc_instr(b, AGX_OPCODE_IMAGE_LOAD, 1, 6);
+   read->dest[0] = agx_register(24, AGX_SIZE_32);
+   read->dest[0].channels_m1 = 3;
+   read->src[0] = agx_register(12, AGX_SIZE_32);
+   read->src[0].channels_m1 = 1;
+   read->src[1] = agx_immediate(0);
+   read->src[2] = agx_immediate(0);
+   read->src[3] = agx_register(18, AGX_SIZE_64);
+   read->src[4] = agx_immediate(0);
+   read->src[5] = agx_null();
+   read->dim = AGX_DIM_2D;
+   read->lod_mode = AGX_LOD_MODE_AUTO_LOD;
+   read->mask = 1;
+   agx_builder_insert(&b->cursor, read);
+   agx_stop(b);
+   util_dynarray bytes;
+   util_dynarray_init(&bytes, mem);
+   EXPECT_TRUE(agx3_pack_binary(b->shader, &bytes));
+   EXPECT_GT(b->shader->agx3_staging_halfs, 0u);
+   ralloc_free(mem);
+}
+
+TEST(NativeForms, M5MeshImageReadUsesNativeHeapIndex)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+      &agx_nir_options, "mesh native image descriptor");
+   nir_def *handle = agx_test_bindless_image_agx(&b, nir_imm_int(&b, 64), 2);
+   nir_def *coord = nir_imm_ivec4(&b, 0, 0, 0, 0);
+   nir_def *value = agx_test_bindless_image_load(&b, 4, 32, handle, coord,
+      nir_imm_int(&b, 0), nir_imm_int(&b, 0), GLSL_SAMPLER_DIM_2D,
+      PIPE_FORMAT_R32_UINT, ACCESS_COHERENT, nir_type_uint32);
+   nir_store_global(&b, value, nir_imm_int64(&b, 0x2000000),
+                    .write_mask=15, .align_mul=4);
+   ASSERT_TRUE(agx_nir_lower_g17g_indexed_reads(b.shader));
+   unsigned reads = 0;
+   nir_foreach_block(block, b.impl) nir_foreach_instr(in, block) {
+      if (in->type != nir_instr_type_intrinsic) continue;
+      nir_intrinsic_instr *i = nir_instr_as_intrinsic(in);
+      if (i->intrinsic != nir_intrinsic_bindless_image_load) continue;
+      auto image = nir_src_as_intrinsic(i->src[0]);
+      ASSERT_NE(image, nullptr);
+      EXPECT_EQ(image->intrinsic, nir_intrinsic_bindless_image_agx);
+      EXPECT_EQ(image->src[0].ssa->bit_size, 64u);
+      EXPECT_EQ(nir_intrinsic_desc_set(image), 0u);
+      reads++;
+   }
+   EXPECT_EQ(reads, 1u);
+   nir_validate_shader(b.shader, "native mesh image index");
+   ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5Independent3DSamplesUseSeparateStaging)
+{
+   for (auto stage : {MESA_SHADER_VERTEX, MESA_SHADER_COMPUTE}) {
+      for (bool gradient : {false, true}) {
+         const unsigned span = gradient ? (stage == MESA_SHADER_VERTEX ? 40u : 32u) : 8u;
+         for (unsigned count : {2u, std::min(8u, 256u / span)}) {
+         SCOPED_TRACE(count);
+         SCOPED_TRACE(stage);
+         SCOPED_TRACE(gradient);
+         void *mem = ralloc_context(NULL);
+         agx_shader_key key = {};
+         key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+         key.dev.chip = AGX_CHIP_G17S;
+         agx_builder *b = agx_test_builder(mem);
+         b->shader->key = &key;
+         b->shader->stage = stage;
+         b->shader->max_reg = 63;
+         agx_index coords = agx_register(64, AGX_SIZE_32);
+         coords.channels_m1 = 2;
+         agx_index lod = agx_immediate(0);
+         if (gradient) {
+            lod = agx_register(72, AGX_SIZE_32);
+            lod.channels_m1 = 5;
+         }
+         for (unsigned n = 0; n < count; ++n) {
+            unsigned dest = 8 * n;
+            agx_index value = agx_register(dest, AGX_SIZE_32);
+            value.channels_m1 = 3;
+            agx_texture_sample_to(b, value, coords, lod,
+               agx_immediate(0), agx_immediate(0), agx_immediate(0),
+               agx_null(), AGX_DIM_3D,
+               gradient ? AGX_LOD_MODE_LOD_GRAD : AGX_LOD_MODE_LOD_MIN,
+               15, false, false, false, AGX_GATHER_NONE, false);
+         }
+         agx_fadd_to(b, agx_register(96, AGX_SIZE_32),
+                     agx_register(0, AGX_SIZE_32),
+                     agx_register(8, AGX_SIZE_32));
+         for (unsigned n = 2; n < count; ++n)
+            agx_fadd_to(b, agx_register(96, AGX_SIZE_32),
+                        agx_register(96, AGX_SIZE_32),
+                        agx_register(8 * n, AGX_SIZE_32));
+         agx_stop(b);
+         util_dynarray binary;
+         util_dynarray_init(&binary, mem);
+         ASSERT_TRUE(agx3_pack_binary(b->shader, &binary));
+         EXPECT_GE(b->shader->agx3_staging_halfs, count * span);
+         uint8_t expected[AGX3_TEX_COPY_MAX_LENGTH], len;
+         ASSERT_TRUE(agx3_tex_stage_copy_hints(AGX_ISA_ENCODING_AGX3C,
+            (count - 1) * span + (gradient ? 8 : 0),
+            gradient ? 72 : 64, count, 0,
+            true, false, expected, &len));
+         const auto *data = static_cast<const uint8_t *>(binary.data);
+         bool found = false;
+         for (unsigned i = 0; i + len <= binary.size; ++i)
+            found |= memcmp(data + i, expected, len) == 0;
+         EXPECT_TRUE(found);
+         util_dynarray_fini(&binary);
+         ralloc_free(mem);
+         }
+      }
+   }
+}
+
+TEST(NativeForms, M5MeshWithoutOutputsKeepsMemoryWrites)
+{
+   for (unsigned threads : {1u, 32u}) {
+      nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+         &agx_nir_options, "memory-only mesh");
+      b.shader->info.workgroup_size[0] = threads;
+      b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+      b.shader->info.mesh.primitive_type = MESA_PRIM_TRIANGLES;
+      b.shader->info.mesh.max_vertices_out = 3;
+      b.shader->info.mesh.max_primitives_out = 1;
+      b.shader->info.io_lowered = true;
+      nir_store_global(&b, nir_load_local_invocation_index(&b),
+                       nir_imm_int64(&b, 0x2000000),
+                       .write_mask=1, .align_mul=4);
+      ASSERT_TRUE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+      unsigned memory_writes = 0;
+      nir_foreach_function_impl(impl, b.shader) {
+         nir_foreach_block(block, impl) {
+            nir_foreach_instr(instr, block) {
+               if (instr->type == nir_instr_type_intrinsic &&
+                   nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_store_global)
+                  memory_writes++;
+            }
+         }
+      }
+      EXPECT_EQ(memory_writes, 1u);
+      ralloc_free(b.shader);
+   }
+}
+
+TEST(NativeForms, M5MeshOutputStillRequiresCount)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_MESH,
+      &agx_nir_options, "mesh output without count");
+   b.shader->info.workgroup_size[0] = 32;
+   b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+   b.shader->info.mesh.primitive_type = MESA_PRIM_TRIANGLES;
+   b.shader->info.mesh.max_vertices_out = 3;
+   b.shader->info.mesh.max_primitives_out = 1;
+   b.shader->info.io_lowered = true;
+   b.shader->info.outputs_written = VARYING_BIT_POS;
+   EXPECT_FALSE(agx_nir_lower_m5_mesh_triangle_probe(b.shader));
+   ralloc_free(b.shader);
+}
+
+/* Two triangles sharing vertices 1 and 2, a smooth per-vertex VAR1 and
+ * per-primitive VAR0/viewport (and optionally layer) words. */
+static nir_shader *
+m5_mesh_per_primitive_shader(bool layer, unsigned max_primitives,
+                             gl_varying_slot extra_primitive_slot)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_MESH,&agx_nir_options,"mesh per-primitive words");
+   b.shader->info.workgroup_size[0]=4;b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+   b.shader->info.mesh.max_vertices_out=4;b.shader->info.mesh.max_primitives_out=max_primitives;
+   b.shader->info.mesh.primitive_type=MESA_PRIM_TRIANGLES;b.shader->info.io_lowered=true;
+   b.shader->info.outputs_written=VARYING_BIT_POS|VARYING_BIT_PRIMITIVE_INDICES|VARYING_BIT_VAR(0)|
+      VARYING_BIT_VAR(1)|VARYING_BIT_VIEWPORT|(layer?VARYING_BIT_LAYER:0);
+   b.shader->info.per_primitive_outputs=VARYING_BIT_PRIMITIVE_INDICES|VARYING_BIT_VAR(0)|
+      VARYING_BIT_VIEWPORT|(layer?VARYING_BIT_LAYER:0);
+   nir_def *lane=nir_load_local_invocation_index(&b);
+   nir_set_vertex_and_primitive_count(&b,nir_imm_int(&b,4),nir_imm_int(&b,2),nir_imm_int(&b,0));
+   nir_store_per_vertex_output(&b,nir_vec4(&b,nir_u2f32(&b,lane),nir_imm_float(&b,0),nir_imm_float(&b,0),
+      nir_imm_float(&b,1)),lane,nir_imm_int(&b,0),.write_mask=15,.src_type=nir_type_float32,
+      .io_semantics={.location=VARYING_SLOT_POS,.num_slots=1});
+   nir_store_per_vertex_output(&b,nir_imm_vec2(&b,.5,.25),lane,nir_imm_int(&b,0),.write_mask=3,
+      .src_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_VAR1,.num_slots=1});
+   nir_push_if(&b,nir_ult_imm(&b,lane,2));
+   nir_def *first=nir_imul_imm(&b,lane,2);
+   nir_store_per_primitive_output(&b,nir_vec3(&b,first,nir_iadd_imm(&b,first,1),nir_iadd_imm(&b,first,2)),
+      lane,nir_imm_int(&b,0),.write_mask=7,.src_type=nir_type_uint32,
+      .io_semantics={.location=VARYING_SLOT_PRIMITIVE_INDICES,.num_slots=1});
+   nir_store_per_primitive_output(&b,nir_vec4(&b,nir_u2f32(&b,lane),nir_imm_float(&b,1),nir_imm_float(&b,0),
+      nir_imm_float(&b,1)),lane,nir_imm_int(&b,0),.write_mask=15,.src_type=nir_type_float32,
+      .io_semantics={.location=VARYING_SLOT_VAR0,.num_slots=1});
+   nir_store_per_primitive_output(&b,lane,lane,nir_imm_int(&b,0),.write_mask=1,.src_type=nir_type_int32,
+      .io_semantics={.location=VARYING_SLOT_VIEWPORT,.num_slots=1});
+   if (layer)
+      nir_store_per_primitive_output(&b,nir_iadd_imm(&b,lane,3),lane,nir_imm_int(&b,0),.write_mask=1,
+         .src_type=nir_type_int32,.io_semantics={.location=VARYING_SLOT_LAYER,.num_slots=1});
+   if (extra_primitive_slot != VARYING_SLOT_POS)
+      nir_store_per_primitive_output(&b,lane,lane,nir_imm_int(&b,0),.write_mask=1,.src_type=nir_type_int32,
+         .io_semantics={.location=extra_primitive_slot,.num_slots=1});
+   nir_pop_if(&b,NULL);
+   return b.shader;
+}
+
+TEST(NativeForms, M5MeshPerPrimitiveWordsUsePrivateVertices)
+{
+   for (bool layer : {false, true}) {
+      nir_shader *s=m5_mesh_per_primitive_shader(layer,2,VARYING_SLOT_POS);
+      struct agx_m5_mesh_varyings v;
+      ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));
+      /* Slot order: layer, viewport, VAR0, VAR1; then the packed
+       * layer/viewport word, as the vertex stage's UVS layout. */
+      EXPECT_TRUE(v.lv);EXPECT_TRUE(v.dynamic);
+      EXPECT_EQ(v.components[VARYING_SLOT_LAYER],1u);EXPECT_EQ(v.first[VARYING_SLOT_LAYER],4u);
+      EXPECT_EQ(v.primitive_comps[VARYING_SLOT_LAYER],layer?1u:0u);
+      EXPECT_EQ(v.components[VARYING_SLOT_VIEWPORT],1u);EXPECT_EQ(v.first[VARYING_SLOT_VIEWPORT],5u);
+      EXPECT_EQ(v.components[VARYING_SLOT_VAR0],4u);EXPECT_EQ(v.primitive_comps[VARYING_SLOT_VAR0],4u);
+      EXPECT_EQ(v.first[VARYING_SLOT_VAR0],6u);
+      EXPECT_EQ(v.components[VARYING_SLOT_VAR1],2u);EXPECT_EQ(v.primitive_comps[VARYING_SLOT_VAR1],0u);
+      EXPECT_EQ(v.first[VARYING_SLOT_VAR1],10u);
+      EXPECT_EQ(v.lv_first,12u);EXPECT_EQ(v.size,13u);EXPECT_EQ(v.primitive_stride_B,0u);
+      /* Linked (declaration) order, so every plane has a fixed UVS index. */
+      v.dynamic=false;
+      ASSERT_TRUE(agx_nir_lower_m5_mesh(s,&v));
+      /* Three private vertices per primitive. */
+      EXPECT_EQ(s->info.mesh.max_vertices_out,6u);
+      /* Index bytes 16, counts 16, export 13 planes x 32, vertex sources
+       * 6 x 16, primitive sources (VAR0 4 + viewport 1 + layer) x 16. */
+      const unsigned primitive_planes=5+(layer?1:0);
+      EXPECT_EQ(s->info.shared_size,(32+13*32+6*16+primitive_planes*16+63)&~63u);
+      unsigned index_words=0,vertex_planes=0,index_loads=0,u8_stores=0,pack=0,rendezvous=0,loops=0;
+      nir_foreach_block(block,nir_shader_get_entrypoint(s))nir_foreach_instr(in,block) {
+         if(in->type==nir_instr_type_alu&&nir_instr_as_alu(in)->op==nir_op_pack_32_2x16_split)pack++;
+         if(in->type==nir_instr_type_intrinsic&&nir_instr_as_intrinsic(in)->intrinsic==nir_intrinsic_barrier&&
+            nir_intrinsic_execution_scope(nir_instr_as_intrinsic(in))==SCOPE_WORKGROUP)rendezvous++;
+         if(in->type!=nir_instr_type_intrinsic)continue;
+         nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+         EXPECT_NE(i->intrinsic,nir_intrinsic_store_per_primitive_output);
+         EXPECT_NE(i->intrinsic,nir_intrinsic_load_mesh_primitive_storage_agx);
+         if(i->intrinsic==nir_intrinsic_load_shared&&i->def.bit_size==8)index_loads++;
+         if(i->intrinsic==nir_intrinsic_store_shared&&i->src[0].ssa->bit_size==8)u8_stores++;
+         if(i->intrinsic!=nir_intrinsic_store_mesh_uvb_agx)continue;
+         if(nir_intrinsic_flags(i)){index_words++;continue;}
+         vertex_planes++;
+         ASSERT_TRUE(nir_src_is_const(i->src[0]));
+         EXPECT_EQ(nir_src_comp_as_uint(i->src[0],1),32+32*(nir_src_comp_as_uint(i->src[0],0)&65535));
+      }
+      /* One 32-vertex batch of 13 planes and one index batch. SIMD group 0
+       * copies between two rendezvous, without a loop. */
+      foreach_list_typed(nir_cf_node,node,node,&nir_shader_get_entrypoint(s)->body)
+         loops+=node->type==nir_cf_node_loop;
+      EXPECT_EQ(index_words,1u);EXPECT_EQ(vertex_planes,13u);
+      EXPECT_EQ(rendezvous,2u);EXPECT_EQ(loops,0u);
+      /* The copy reads every index byte once and rewrites it. */
+      EXPECT_EQ(index_loads,1u);EXPECT_GE(u8_stores,2u);EXPECT_EQ(pack,1u);
+      nir_validate_shader(s,"mesh per-primitive private vertices");agx_preprocess_nir_m5(s);
+      agx_shader_key key={};key.dev.chip=AGX_CHIP_G17S;key.dev.encoding=AGX_ISA_ENCODING_AGX3C;key.no_preamble=true;
+      agx_shader_part part={};ASSERT_TRUE(agx_compile_shader_nir(s,&key,&part))<<agx_last_compile_refusal();
+      free(part.binary);ralloc_free(s);
+   }
+}
+
+/* gl_CullPrimitiveEXT: one byte per primitive, cleared before the body and
+ * compacted away before the export. */
+TEST(NativeForms, M5MeshCullPrimitiveCompacts)
+{
+   for (unsigned primitives : {2u, 85u}) {
+      nir_shader *s=m5_mesh_per_primitive_shader(false,primitives,VARYING_SLOT_CULL_PRIMITIVE);
+      s->info.outputs_written|=VARYING_BIT_CULL_PRIMITIVE;
+      s->info.per_primitive_outputs|=VARYING_BIT_CULL_PRIMITIVE;
+      struct agx_m5_mesh_varyings v;
+      ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));
+      EXPECT_TRUE(v.cull_primitive);
+      EXPECT_EQ(v.components[VARYING_SLOT_CULL_PRIMITIVE],0u);
+      ASSERT_TRUE(agx_nir_lower_m5_mesh(s,&v));
+      nir_validate_shader(s,"cull primitive");
+      EXPECT_LE(s->info.shared_size,32768u);
+      ralloc_free(s);
+   }
+}
+
+TEST(NativeForms, M5MeshPerPrimitiveRefusalsLeaveShaderUntouched)
+{
+   /* No private-vertex form beyond the native 256 output vertices. */
+   const struct { unsigned primitives; gl_varying_slot extra; bool gather; } rows[] = {
+      {86, VARYING_SLOT_POS, true},
+   };
+   for (const auto &r : rows) {
+      nir_shader *s=m5_mesh_per_primitive_shader(false,r.primitives,r.extra);
+      struct agx_m5_mesh_varyings v;
+      EXPECT_EQ(agx_m5_mesh_gather_varyings(s,&v),r.gather);
+      if (r.gather) {
+         EXPECT_FALSE(agx_nir_lower_m5_mesh(s,&v));
+      }
+      EXPECT_EQ(s->info.shared_size,0u);EXPECT_EQ(s->info.mesh.max_vertices_out,4u);
+      ralloc_free(s);
+   }
+   /* 85 primitives fit: 255 private vertices. */
+   nir_shader *s=m5_mesh_per_primitive_shader(false,85,VARYING_SLOT_POS);
+   struct agx_m5_mesh_varyings v;
+   ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));
+   ASSERT_TRUE(agx_nir_lower_m5_mesh(s,&v));
+   EXPECT_EQ(s->info.mesh.max_vertices_out,255u);
+   nir_validate_shader(s,"mesh 85 primitives");ralloc_free(s);
+}
+
+TEST(NativeForms, M5MeshPrimitiveIdIsAPrimitiveWord)
+{
+   /* A mesh-written gl_PrimitiveID is a flat user word of the primitive's
+    * private vertices, which the fragment shader's gl_PrimitiveID links to. */
+   nir_shader *s=m5_mesh_per_primitive_shader(false,2,VARYING_SLOT_PRIMITIVE_ID);
+   s->info.outputs_written|=VARYING_BIT_PRIMITIVE_ID;
+   s->info.per_primitive_outputs|=VARYING_BIT_PRIMITIVE_ID;
+   struct agx_m5_mesh_varyings v;
+   ASSERT_TRUE(agx_m5_mesh_gather_varyings(s,&v));
+   EXPECT_EQ(v.primitive_comps[VARYING_SLOT_PRIMITIVE_ID],1u);
+   EXPECT_EQ(v.components[VARYING_SLOT_PRIMITIVE_ID],1u);
+   ASSERT_TRUE(agx_nir_lower_m5_mesh(s,&v));
+   EXPECT_EQ(s->info.mesh.max_vertices_out,6u);
+   nir_validate_shader(s,"mesh primitive id");ralloc_free(s);
+}
+
+TEST(NativeForms, M5MeshFragmentReadsPrimitiveWordsFlat)
+{
+   nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,&agx_nir_options,"fs per-primitive inputs");
+   b.shader->info.io_lowered=true;
+   b.shader->info.per_primitive_inputs=VARYING_BIT_VAR(0)|VARYING_BIT_VAR(1)|VARYING_BIT_VIEWPORT|
+                                       VARYING_BIT_PRIMITIVE_ID;
+   nir_def *color=nir_load_per_primitive_input(&b,4,32,nir_imm_int(&b,0),.component=0,
+      .dest_type=nir_type_float32,.io_semantics={.location=VARYING_SLOT_VAR0,.num_slots=1});
+   nir_def *half=nir_load_per_primitive_input(&b,2,16,nir_imm_int(&b,0),.component=1,
+      .dest_type=nir_type_float16,.io_semantics={.location=VARYING_SLOT_VAR1,.num_slots=1});
+   nir_def *viewport=nir_load_per_primitive_input(&b,1,32,nir_imm_int(&b,0),
+      .dest_type=nir_type_int32,.io_semantics={.location=VARYING_SLOT_VIEWPORT,.num_slots=1});
+   nir_def *id=nir_load_per_primitive_input(&b,1,32,nir_imm_int(&b,0),
+      .dest_type=nir_type_int32,.io_semantics={.location=VARYING_SLOT_PRIMITIVE_ID,.num_slots=1});
+   nir_store_global(&b,nir_vec4(&b,nir_channel(&b,color,0),nir_f2f32(&b,nir_channel(&b,half,1)),
+      nir_i2f32(&b,viewport),nir_i2f32(&b,id)),nir_imm_int64(&b,0x1000),.write_mask=15,.align_mul=16);
+   ASSERT_TRUE(agx_nir_lower_m5_mesh_fragment(b.shader));
+   unsigned flat=0,primitive=0,narrowed=0;
+   nir_foreach_block(block,b.impl)nir_foreach_instr(in,block) {
+      if(in->type==nir_instr_type_alu&&nir_instr_as_alu(in)->op==nir_op_f2f16)narrowed++;
+      if(in->type!=nir_instr_type_intrinsic)continue;
+      nir_intrinsic_instr *i=nir_instr_as_intrinsic(in);
+      if(i->intrinsic==nir_intrinsic_load_per_primitive_input)
+         primitive++;
+      if(i->intrinsic==nir_intrinsic_load_input) {
+         flat++;EXPECT_EQ(i->def.bit_size,32u);
+         EXPECT_EQ(nir_alu_type_get_type_size(nir_intrinsic_dest_type(i)),32u);
+         if(nir_intrinsic_io_semantics(i).location==VARYING_SLOT_VAR1) {
+            EXPECT_EQ(nir_intrinsic_component(i),1u);
+         }
+      }
+   }
+   EXPECT_EQ(flat,4u);EXPECT_EQ(primitive,0u);EXPECT_EQ(narrowed,1u);
+   EXPECT_EQ(b.shader->info.per_primitive_inputs,0u);
+   nir_validate_shader(b.shader,"fs per-primitive inputs");
+   EXPECT_FALSE(agx_nir_lower_m5_mesh_fragment(b.shader));
+   ralloc_free(b.shader);
+}
+
+TEST(NativeForms, M5UniformDifferenceHonorsNIRSignedZero)
+{
+   std::vector<uint8_t> programs[2];
+   for (unsigned strict = 0; strict < 2; ++strict) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_COMPUTE, &agx_nir_options, "uniform difference zero");
+      b.shader->info.workgroup_size[0] = 32;
+      b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+      b.fp_math_ctrl = nir_fp_exact |
+         (strict ? nir_fp_preserve_signed_zero : 0);
+      nir_def *u = agx_test_load_preamble(&b, 1, 32, 12);
+      nir_def *r = nir_u2f32(&b, nir_load_local_invocation_index(&b));
+      nir_def *difference = nir_fsub(&b, u, r);
+      nir_def *scale = agx_test_load_preamble(&b, 1, 32, 14);
+      nir_def *product = nir_fmul(&b, difference, scale);
+      nir_store_global(&b, product, nir_imm_int64(&b, 0x2000000),
+                       .write_mask = 1, .align_mul = 4);
+      agx_preprocess_nir_m5(b.shader);
+      agx_shader_key key = {};
+      key.dev.chip = AGX_CHIP_G17S;
+      key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+      key.no_preamble = true;
+      key.reserved_preamble = 16;
+      agx_shader_part part = {};
+      ASSERT_TRUE(agx_compile_shader_nir(b.shader, &key, &part))
+         << agx_last_compile_refusal();
+      const uint8_t *bytes = static_cast<const uint8_t *>(part.binary);
+      programs[strict].assign(bytes, bytes + part.info.main_size);
+      free(part.binary);
+      ralloc_free(b.shader);
+   }
+   EXPECT_NE(programs[1], programs[0]);
+}
+
+TEST(NativeForms, M5OrSelectTracksVectorConversionComponent)
+{
+   for (unsigned bits : {16u, 32u}) {
+      for (unsigned conditions = 0; conditions < 8; ++conditions) {
+         for (unsigned component = 0; component < 2; ++component) {
+            for (unsigned selected = 0; selected < 2; ++selected) {
+               for (bool swapped : {false, true}) {
+                  for (bool inverted : {false, true}) {
+                     SCOPED_TRACE(::testing::Message() << bits << ":" << conditions
+                        << ":" << component << ":" << selected << ":"
+                        << swapped << ":" << inverted);
+                     nir_builder b = nir_builder_init_simple_shader(
+                        MESA_SHADER_COMPUTE, &agx_nir_options, "vector b2f component");
+                     nir_def *a = nir_ieq(&b,
+                        nir_imm_ivec2(&b, conditions & 1, (conditions >> 1) & 1),
+                        nir_imm_int(&b, 1));
+                     nir_def *other = nir_ieq(&b, nir_imm_int(&b, conditions >> 2),
+                                              nir_imm_int(&b, 1));
+                     nir_def *condition = nir_ior(&b, nir_channel(&b, a, component), other);
+                     nir_alu_instr *ior = nir_def_as_alu(condition);
+                     nir_src_rewrite(&ior->src[0].src, a);
+                     ior->src[0].swizzle[0] = component;
+                     nir_def *boolean = inverted ? nir_inot(&b, a) : a;
+                     nir_def *converted = bits == 16 ? nir_b2f16(&b, boolean)
+                                                     : nir_b2f32(&b, boolean);
+                     nir_alu_instr *b2f = nir_def_as_alu(converted);
+                     b2f->src[0].swizzle[0] = swapped ? 1 : 0;
+                     b2f->src[0].swizzle[1] = swapped ? 0 : 1;
+                     nir_def *value = nir_bcsel(&b, condition,
+                        nir_channel(&b, converted, selected), nir_imm_floatN_t(&b, 3.0, bits));
+                     nir_alu_instr *sel = nir_def_as_alu(value);
+                     nir_src_rewrite(&sel->src[1].src, converted);
+                     sel->src[1].swizzle[0] = selected;
+                     nir_intrinsic_instr *store = nir_store_global(&b,
+                        bits == 16 ? nir_f2f32(&b, value) : value,
+                        nir_imm_int64(&b, 0x2000000), .write_mask = 1, .align_mul = 4);
+                     nir_validate_shader(b.shader, "vector conversion before OR select");
+                     unsigned source_component = selected ^ unsigned(swapped);
+                     EXPECT_EQ(agx_nir_m5_split_or_selects(b.shader),
+                               source_component == component);
+                     nir_validate_shader(b.shader, "vector conversion after OR select");
+                     while (nir_opt_constant_folding(b.shader)) {}
+                     ASSERT_TRUE(nir_src_is_const(store->src[0]));
+                     bool selected_bool = ((conditions >> source_component) & 1) ^ inverted;
+                     bool taken = ((conditions >> component) & 1) || (conditions & 4);
+                     EXPECT_FLOAT_EQ(nir_src_as_float(store->src[0]),
+                                     taken ? float(selected_bool) : 3.0f);
+                     ralloc_free(b.shader);
+                  }
+               }
+            }
+         }
+      }
+   }
+}
diff --git a/src/asahi/compiler/test/test-native-mesh-program.c b/src/asahi/compiler/test/test-native-mesh-program.c
new file mode 100644
index 000000000000..9e2a8908f9de
--- /dev/null
+++ b/src/asahi/compiler/test/test-native-mesh-program.c
@@ -0,0 +1,38 @@
+#undef NDEBUG
+#include "agx3_graphics_program.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+int main(void)
+{
+   const struct {
+      bool object; unsigned shared, binary_size, scratch, compiler_d;
+      uint64_t shader; uint8_t tail[28];
+   } cases[] = {
+      {true,0,32,0,1,UINT64_C(0x1000000280),
+       {7,0,0,12,0,0,0,0,135,2,64,14,0,0,0,0,16,0,
+        39,0,0,13,0,0,0,0,0,0}},
+      {false,192,660,16,0,UINT64_C(0x10000002a40),
+       {15,24,0,12,0,0,0,0,71,42,88,14,0,0,0,0,0,1,
+        7,0,0,13,0,0,4,0,4,0}},
+      {true,64,334,16,1,UINT64_C(0x10000002e40),
+       {15,8,0,12,0,0,0,0,71,46,88,14,0,0,0,0,0,1,
+        39,0,0,13,0,0,4,0,4,0}},
+   };
+   for (unsigned i=0;i<sizeof(cases)/sizeof(cases[0]);i++) {
+      const __typeof__(cases[0]) c=cases[i];
+      struct agx3_g17p_graphics_program_desc d = {0};
+      d.program_va = UINT64_C(0x10000128000);d.shader_va=c.shader;
+      d.shader_binary_size=c.binary_size;d.scratch_size=c.scratch;
+      d.nr_gprs=32;d.compiler_d_valid=true;d.compiler_d=c.compiler_d;
+      d.threadgroup_memory_size_B=c.shared;
+      struct agx3_g17p_graphics_program p = {0};
+      assert(agx3_g17g_mesh_program_generate(AGX_CHIP_G17S,&d,c.object,&p));
+      assert(p.length>=28);assert(memcmp(p.bytes,c.tail,28)==0);
+      assert(!agx3_g17g_mesh_program_generate(AGX_CHIP_G17P,&d,c.object,&p));
+      d.fragment=true;
+      assert(!agx3_g17g_mesh_program_generate(AGX_CHIP_G17S,&d,c.object,&p));
+   }
+   puts("Native mesh/object IOTO tails: 3/3 exact captured matches");
+   return 0;
+}
diff --git a/src/asahi/compiler/test/test-optimizer.cpp b/src/asahi/compiler/test/test-optimizer.cpp
index 1c3ea0f662a7..0bd63a55091b 100644
--- a/src/asahi/compiler/test/test-optimizer.cpp
+++ b/src/asahi/compiler/test/test-optimizer.cpp
@@ -520,12 +520,14 @@ TEST_F(Optimizer, G17PDoesNotFoldTheSelectSlotOfAnUntiedFcmpsel)
    ASSERT_SHADER_EQUAL(actual->shader, expected->shader);
 }
 
-TEST_F(Optimizer, G17PPreservesWideningBeforeWidthMatchedIntegerALU)
+TEST_F(Optimizer, NativePreservesWideningBeforeWidthMatchedIntegerALU)
 {
+   for (enum agx_isa_encoding encoding :
+        {AGX_ISA_ENCODING_AGX3B, AGX_ISA_ENCODING_AGX3C}) {
    agx_builder *actual = agx_test_builder(mem_ctx);
    agx_builder *expected = agx_test_builder(mem_ctx);
    struct agx_shader_key key = {};
-   key.dev.encoding = AGX_ISA_ENCODING_AGX3B;
+   key.dev.encoding = encoding;
    actual->shader->key = &key;
    expected->shader->key = &key;
 
@@ -534,6 +536,9 @@ TEST_F(Optimizer, G17PPreservesWideningBeforeWidthMatchedIntegerALU)
    agx_index out = agx_temp(actual->shader, AGX_SIZE_32);
    agx_xor_to(actual, out, widened, wy);
    agx_unit_test(actual, out);
+   agx_index right_out = agx_temp(actual->shader, AGX_SIZE_32);
+   agx_or_to(actual, right_out, wy, widened);
+   agx_unit_test(actual, right_out);
    agx_index selected = agx_temp(actual->shader, AGX_SIZE_32);
    agx_icmpsel_to(actual, selected, widened, wz, wx, wy, AGX_ICOND_ULT);
    agx_unit_test(actual, selected);
@@ -543,12 +548,16 @@ TEST_F(Optimizer, G17PPreservesWideningBeforeWidthMatchedIntegerALU)
    out = agx_temp(expected->shader, AGX_SIZE_32);
    agx_xor_to(expected, out, widened, wy);
    agx_unit_test(expected, out);
+   right_out = agx_temp(expected->shader, AGX_SIZE_32);
+   agx_or_to(expected, right_out, wy, widened);
+   agx_unit_test(expected, right_out);
    selected = agx_temp(expected->shader, AGX_SIZE_32);
    agx_icmpsel_to(expected, selected, widened, wz, wx, wy, AGX_ICOND_ULT);
    agx_unit_test(expected, selected);
 
    agx_optimize_and_dce(actual->shader);
    ASSERT_SHADER_EQUAL(actual->shader, expected->shader);
+   }
 }
 
 TEST_F(Optimizer, FusedFABSNEG)
@@ -1427,6 +1436,35 @@ TEST_F(Optimizer, G17POuterInputProvesCompoundSelectGuard)
    }
 }
 
+TEST_F(Optimizer, G17PSelectGuardImmediateBoundary)
+{
+   for (enum agx_size size : {AGX_SIZE_16, AGX_SIZE_32}) {
+      for (unsigned threshold : {254u, 255u}) {
+         agx_builder *b = agx_test_builder(mem_ctx);
+         struct agx_shader_key key = {};
+         key.dev.encoding = AGX_ISA_ENCODING_AGX3B;
+         b->shader->key = &key;
+         agx_index variable = agx_preload(b, agx_register(0, size));
+         agx_index predicate = agx_icmp(
+            b, variable, agx_immediate(threshold), AGX_ICOND_UGT, false);
+         agx_index out = agx_temp(b->shader, AGX_SIZE_16);
+         agx_icmpsel_to(b, out, variable, agx_immediate(threshold), predicate,
+                        agx_zero(), AGX_ICOND_UGT);
+         agx_unit_test(b, out);
+         agx_optimize_and_dce(b->shader);
+         agx_foreach_instr_global(b->shader, I) {
+            agx_foreach_src(I, s) {
+               if (I->src[s].type == AGX_INDEX_IMMEDIATE) {
+                  EXPECT_LT(I->src[s].value, 0x100u);
+               }
+            }
+         }
+         for (uint32_t input = threshold - 1; input <= threshold + 2; ++input)
+            EXPECT_EQ(select_test_run(b->shader, input), input > threshold);
+      }
+   }
+}
+
 TEST_F(Optimizer, G17PCompoundInnerGuardSemantics)
 {
    const enum agx_icond conditions[] = {
diff --git a/src/asahi/compiler/test/test-position-order.cpp b/src/asahi/compiler/test/test-position-order.cpp
new file mode 100644
index 000000000000..4c0bf13bf36b
--- /dev/null
+++ b/src/asahi/compiler/test/test-position-order.cpp
@@ -0,0 +1,146 @@
+/* SPDX-License-Identifier: MIT */
+#include <array>
+#include <cmath>
+#include <cstring>
+#include <gtest/gtest.h>
+#include "agx_test.h"
+extern "C" {
+#include "agx_nir.h"
+}
+
+static float
+position_value(nir_scalar s, nir_def *world, const std::array<nir_def *, 3> &u,
+               float x, const std::array<float, 3> &v)
+{
+   s = nir_scalar_chase_movs(s);
+   if (s.def == world) return x;
+   for (unsigned i = 0; i < u.size(); i++) if (s.def == u[i]) return v[i];
+   if (nir_scalar_is_const(s)) {
+      uint32_t bits = nir_scalar_as_uint(s); float f;
+      std::memcpy(&f, &bits, 4); return f;
+   }
+   if (!nir_scalar_is_alu(s)) { ADD_FAILURE(); return 0; }
+   auto src = [&](unsigned i) { return position_value(
+      nir_scalar_chase_alu_src(s, i), world, u, x, v); };
+   switch (nir_scalar_alu_op(s)) {
+   case nir_op_fadd: return src(0) + src(1);
+   case nir_op_fmul: return src(0) * src(1);
+   case nir_op_ffma: return std::fma(src(0), src(1), src(2));
+   case nir_op_fneg: return -src(0);
+   default: ADD_FAILURE() << nir_op_infos[nir_scalar_alu_op(s)].name; return 0;
+   }
+}
+
+static void
+paired_position(bool keep_varying, bool protect, std::array<float, 8> *out)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_VERTEX, &agx_nir_options, "paired position order");
+   nir_def *world = nir_load_input(&b, 1, 32, nir_imm_int(&b, 0),
+      .base = 0, .dest_type = nir_type_float32,
+      .io_semantics = {.location = VERT_ATTRIB_GENERIC0});
+   std::array<nir_def *, 3> u = {
+      agx_test_load_preamble(&b, 1, 32, 0),
+      agx_test_load_preamble(&b, 1, 32, 2),
+      agx_test_load_preamble(&b, 1, 32, 4)};
+   nir_def *intermediate = nir_fadd(&b, nir_fadd(&b, world, u[0]), u[1]);
+   nir_def *clip = nir_fadd(&b, intermediate, u[2]);
+   auto *store = nir_store_output(&b, clip, nir_imm_int(&b, 0),
+      .base = 0, .write_mask = 1, .component = 2,
+      .src_type = nir_type_float32, .io_semantics = {.location = VARYING_SLOT_POS});
+   if (keep_varying) nir_store_output(&b, intermediate, nir_imm_int(&b, 0),
+      .base = 1, .write_mask = 1, .src_type = nir_type_float32,
+      .io_semantics = {.location = VARYING_SLOT_VAR0});
+   nir_def *unrelated = nir_fadd(&b, u[0], u[1]);
+   nir_store_output(&b, unrelated, nir_imm_int(&b, 0),
+      .base = 2, .write_mask = 1, .src_type = nir_type_float32,
+      .io_semantics = {.location = VARYING_SLOT_VAR1});
+   if (protect) {
+      ASSERT_TRUE(agx_nir_preserve_position_order(b.shader));
+      EXPECT_FALSE(agx_nir_preserve_position_order(b.shader));
+      EXPECT_TRUE(nir_alu_instr_no_reassoc(nir_def_as_alu(clip)));
+      EXPECT_TRUE(nir_alu_instr_no_transform(nir_def_as_alu(clip)));
+      EXPECT_FALSE(nir_alu_instr_no_contract(nir_def_as_alu(clip)));
+      EXPECT_FALSE(nir_alu_instr_no_reassoc(nir_def_as_alu(unrelated)));
+   }
+   nir_divergence_analysis(b.shader);
+   nir_opt_reassociate_loop(b.shader, (nir_reassociate_options)(
+      nir_reassociate_scalar_math | nir_reassociate_cse_heuristic |
+      nir_reassociate_loop_invariant));
+   const std::array<float, 3> values = {1.23e-5f, -6.7e-6f, 64.43448638916016f};
+   for (unsigned i = 0; i < out->size(); i++) {
+      float x = -63.93449f + float(i) * 0.01879f;
+      (*out)[i] = position_value(nir_get_scalar(store->src[0].ssa, 0), world, u, x, values);
+      if (protect) {
+         EXPECT_EQ((*out)[i], ((x + values[0]) + values[1]) + values[2]);
+      }
+   }
+   nir_validate_shader(b.shader, "position order");
+   ralloc_free(b.shader);
+}
+
+TEST(PositionOrder, SamePositionWithLiveVaryingKeepsIdenticalRounding)
+{
+   std::array<float, 8> depth, color, protected_depth, protected_color;
+   paired_position(false, false, &depth);
+   paired_position(true, false, &color);
+   paired_position(false, true, &protected_depth);
+   paired_position(true, true, &protected_color);
+   EXPECT_NE(depth, color);
+   EXPECT_EQ(protected_depth, protected_color);
+}
+
+TEST(PositionOrder, KeepsMultiplyAddContractionAvailable)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_VERTEX, &agx_nir_options, "position contraction");
+   nir_def *x = agx_test_load_preamble(&b, 1, 32, 0);
+   nir_def *y = agx_test_load_preamble(&b, 1, 32, 2);
+   nir_def *z = agx_test_load_preamble(&b, 1, 32, 4);
+   nir_def *product = nir_fmul(&b, x, y);
+   nir_def *value = nir_fadd(&b, product, z);
+   auto *store = nir_store_output(&b, value, nir_imm_int(&b, 0),
+      .base = 0, .write_mask = 1, .src_type = nir_type_float32,
+      .io_semantics = {.location = VARYING_SLOT_POS});
+   ASSERT_TRUE(agx_nir_preserve_position_order(b.shader));
+   EXPECT_FALSE(nir_alu_instr_no_contract(nir_def_as_alu(product)));
+   EXPECT_FALSE(nir_alu_instr_no_contract(nir_def_as_alu(value)));
+   nir_opt_algebraic_late(b.shader);
+   EXPECT_EQ(nir_scalar_alu_op(nir_scalar_chase_movs(
+      nir_get_scalar(store->src[0].ssa, 0))), nir_op_ffma);
+   ralloc_free(b.shader);
+}
+
+TEST(PositionOrder, IncludesPhiControlDependencies)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_VERTEX, &agx_nir_options, "position control dependencies");
+   nir_def *x = agx_test_load_preamble(&b, 1, 32, 0);
+   nir_def *y = agx_test_load_preamble(&b, 1, 32, 2);
+   nir_def *z = agx_test_load_preamble(&b, 1, 32, 4);
+   nir_def *predicate_sum = nir_fadd(&b, x, y);
+   nir_def *condition = nir_flt(&b, predicate_sum, z);
+   nir_push_if(&b, condition);
+   nir_def *then_value = nir_fadd(&b, x, z);
+   nir_push_else(&b, NULL);
+   nir_def *else_value = nir_fadd(&b, y, z);
+   nir_pop_if(&b, NULL);
+   nir_def *position = nir_if_phi(&b, then_value, else_value);
+   nir_def *unrelated = nir_fadd(&b, x, y);
+   nir_store_output(&b, position, nir_imm_int(&b, 0),
+      .base = 0, .write_mask = 1, .src_type = nir_type_float32,
+      .io_semantics = {.location = VARYING_SLOT_POS});
+   nir_store_output(&b, unrelated, nir_imm_int(&b, 0),
+      .base = 1, .write_mask = 1, .src_type = nir_type_float32,
+      .io_semantics = {.location = VARYING_SLOT_VAR0});
+
+   ASSERT_TRUE(agx_nir_preserve_position_order(b.shader));
+   EXPECT_TRUE(nir_alu_instr_no_reassoc(nir_def_as_alu(predicate_sum)));
+   EXPECT_TRUE(nir_alu_instr_no_reassoc(nir_def_as_alu(then_value)));
+   EXPECT_TRUE(nir_alu_instr_no_reassoc(nir_def_as_alu(else_value)));
+   EXPECT_FALSE(nir_alu_instr_no_reassoc(nir_def_as_alu(unrelated)));
+   EXPECT_FALSE(nir_alu_instr_no_contract(nir_def_as_alu(predicate_sum)));
+   EXPECT_FALSE(agx_nir_preserve_position_order(b.shader));
+   nir_validate_shader(b.shader, "position phi control dependencies");
+   ralloc_free(b.shader);
+}
diff --git a/src/asahi/compiler/test/test-preamble-image-payload.cpp b/src/asahi/compiler/test/test-preamble-image-payload.cpp
index da0389c29a00..fb2f75c1fc6d 100644
--- a/src/asahi/compiler/test/test-preamble-image-payload.cpp
+++ b/src/asahi/compiler/test/test-preamble-image-payload.cpp
@@ -38,7 +38,7 @@ check_payload(unsigned bits, unsigned heap = 24, bool g17p = true)
       agx_test_load_preamble(&b, 1, 32, 28, nir_preamble_class_image);
    nir_validate_shader(b.shader, "before preamble");
    unsigned sizes[] = {28, 1000, 1000};
-   ASSERT_TRUE(agx_nir_opt_preamble(b.shader, sizes, g17p));
+   ASSERT_TRUE(agx_nir_opt_preamble(b.shader, sizes, g17p, false));
    bool occupied[512] = {};
    unsigned payload_stores = 0, payload_loads = 0, stores = 0;
    nir_foreach_function_impl(impl, b.shader) {
@@ -117,7 +117,7 @@ TEST(AGXPreamblePayload, LegacyImageClassSlots)
       nir_store_uvs_agx(&b, nir_channel(&b, value, 0), nir_imm_int(&b, i));
    }
    unsigned sizes[] = {28, 0, 0};
-   ASSERT_TRUE(agx_nir_opt_preamble(b.shader, sizes, false));
+   ASSERT_TRUE(agx_nir_opt_preamble(b.shader, sizes, false, false));
    EXPECT_EQ(sizes[1], 4u);
    unsigned seen = 0;
    nir_foreach_function_impl(impl, b.shader) {
diff --git a/src/asahi/compiler/test/test-preamble-modifiers.cpp b/src/asahi/compiler/test/test-preamble-modifiers.cpp
index c6d418c900ac..7193b0844954 100644
--- a/src/asahi/compiler/test/test-preamble-modifiers.cpp
+++ b/src/asahi/compiler/test/test-preamble-modifiers.cpp
@@ -18,7 +18,7 @@ check_modifier(bool g17p, bool widen)
                          nir_imm_int(&b, 0), nir_type_float32,
                          FRAG_RESULT_DATA0);
    unsigned sizes[3] = {16, 0, 0};
-   bool progress = agx_nir_opt_preamble(b.shader, sizes, g17p);
+   bool progress = agx_nir_opt_preamble(b.shader, sizes, g17p, false);
    const bool hoisted = g17p && !widen;
    if (hoisted) {
       EXPECT_TRUE(progress);
diff --git a/src/asahi/compiler/test/test-preamble-share-values.cpp b/src/asahi/compiler/test/test-preamble-share-values.cpp
index 5f373f905183..b30ca236ec7d 100644
--- a/src/asahi/compiler/test/test-preamble-share-values.cpp
+++ b/src/asahi/compiler/test/test-preamble-share-values.cpp
@@ -59,7 +59,7 @@ run_share(bool uniform_if, unsigned arms_sharing)
 
    nir_validate_shader(b.shader, "before preamble");
    unsigned sizes[] = {4, 0, 0};
-   EXPECT_TRUE(agx_nir_opt_preamble(b.shader, sizes, true));
+   EXPECT_TRUE(agx_nir_opt_preamble(b.shader, sizes, true, false));
    nir_validate_shader(b.shader, "after preamble");
 
    share_result r = {};
diff --git a/src/asahi/compiler/test/test-preamble-texture-policy.cpp b/src/asahi/compiler/test/test-preamble-texture-policy.cpp
new file mode 100644
index 000000000000..e685a9a479c6
--- /dev/null
+++ b/src/asahi/compiler/test/test-preamble-texture-policy.cpp
@@ -0,0 +1,71 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#include <gtest/gtest.h>
+#include "agx_test.h"
+extern "C" {
+#include "agx_nir.h"
+}
+
+TEST(PreambleTexturePolicy, M5KeepsTextureAndConsumersInMain)
+{
+   for (bool m5 : {false, true}) {
+      for (bool dependent : {false, true}) {
+         SCOPED_TRACE(m5);
+         SCOPED_TRACE(dependent);
+         nir_builder b = nir_builder_init_simple_shader(
+            MESA_SHADER_COMPUTE, &agx_nir_options, "texture preamble policy");
+         b.shader->info.num_textures = 1;
+         nir_def *address = agx_test_load_preamble(&b, 1, 64, 0);
+         nir_def *other = agx_test_load_global_constant(
+            &b, 1, 32, nir_iadd_imm(&b, address, 16), 4,
+            (gl_access_qualifier)0);
+
+         nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2);
+         tex->op = nir_texop_txf;
+         tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
+         tex->dest_type = nir_type_float32;
+         tex->coord_components = 2;
+         tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord,
+                                          nir_imm_ivec2(&b, 0, 0));
+         tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(&b, 0));
+         nir_def_init(&tex->instr, &tex->def, 4, 32);
+         nir_builder_instr_insert(&b, &tex->instr);
+         nir_def *value = dependent ? nir_fmul_imm(&b, &tex->def, 2.0)
+                                    : &tex->def;
+         nir_intrinsic_instr *store = nir_store_global(&b, value, address);
+         nir_intrinsic_set_align(store, 4, 0);
+         store = nir_store_global(&b, other, nir_iadd_imm(&b, address, 64));
+         nir_intrinsic_set_align(store, 4, 0);
+         nir_validate_shader(b.shader, "before texture preamble policy");
+
+         unsigned sizes[] = {4, 1000, 1000};
+         EXPECT_TRUE(agx_nir_opt_preamble(b.shader, sizes, true, m5));
+         nir_opt_dce(b.shader);
+         nir_validate_shader(b.shader, "after texture preamble policy");
+
+         unsigned main_tex = 0, preamble_tex = 0, preamble_loads = 0;
+         nir_foreach_function_impl(impl, b.shader) {
+            nir_foreach_block(block, impl) {
+               nir_foreach_instr(instr, block) {
+                  if (instr->type == nir_instr_type_tex) {
+                     if (impl->function->is_preamble)
+                        preamble_tex++;
+                     else
+                        main_tex++;
+                  }
+                  if (impl->function->is_preamble &&
+                      instr->type == nir_instr_type_intrinsic &&
+                      nir_instr_as_intrinsic(instr)->intrinsic ==
+                         nir_intrinsic_load_global_constant)
+                     preamble_loads++;
+               }
+            }
+         }
+         EXPECT_EQ(main_tex, m5 ? 1u : 0u);
+         EXPECT_EQ(preamble_tex, m5 ? 0u : 1u);
+         EXPECT_EQ(preamble_loads, 1u);
+         ralloc_free(b.shader);
+      }
+   }
+}
diff --git a/src/asahi/compiler/test/test-repair-ssa.cpp b/src/asahi/compiler/test/test-repair-ssa.cpp
index 1d320052e469..660ced5c838d 100644
--- a/src/asahi/compiler/test/test-repair-ssa.cpp
+++ b/src/asahi/compiler/test/test-repair-ssa.cpp
@@ -142,14 +142,17 @@ TEST_F(RepairSSA, UseModifiersDoNotLeakIntoPhi)
 }
 
 /* A value defined before a join and read after it gets a phi of itself from
- * the repair. G17P removes that trivial phi; other chips keep their output. */
+ * the repair. G17P and M5 remove that trivial phi; other chips keep their
+ * output. */
 TEST_F(RepairSSA, G17PRemovesTrivialJoinPhi)
 {
-   for (bool g17p : {false, true}) {
+   for (enum agx_chip chip : {AGX_CHIP_G13G, AGX_CHIP_G17P, AGX_CHIP_G17S}) {
+      const bool g17p = chip != AGX_CHIP_G13G;
       agx_shader_key key = {};
-      key.dev.encoding =
-         g17p ? AGX_ISA_ENCODING_AGX3B : AGX_ISA_ENCODING_AGX2;
-      key.dev.chip = g17p ? AGX_CHIP_G17P : AGX_CHIP_G13G;
+      key.dev.encoding = chip == AGX_CHIP_G17S ? AGX_ISA_ENCODING_AGX3C
+                         : g17p                ? AGX_ISA_ENCODING_AGX3B
+                                               : AGX_ISA_ENCODING_AGX2;
+      key.dev.chip = chip;
 
       CASE({
          b->shader->key = &key;
diff --git a/src/asahi/compiler/test/test-rte-fma.cpp b/src/asahi/compiler/test/test-rte-fma.cpp
index f8e2c3184cfa..92fed4319dba 100644
--- a/src/asahi/compiler/test/test-rte-fma.cpp
+++ b/src/asahi/compiler/test/test-rte-fma.cpp
@@ -92,8 +92,12 @@ check_fma_case(fma_case t)
       agx_nir_fma_rte_fixup(&b, nir_imm_int(&b, t.a), nir_imm_int(&b, t.c),
                             nir_imm_int(&b, t.z), nir_imm_int(&b, t.native));
    nir_store_global(&b, fixed, nir_imm_int64(&b, 0), .align_mul = 4);
-   while (nir_opt_constant_folding(b.shader)) {
-   }
+   bool progress;
+   do {
+      progress = nir_opt_constant_folding(b.shader);
+      progress |= nir_opt_dead_cf(b.shader);
+      progress |= nir_opt_remove_phis(b.shader);
+   } while (progress);
    nir_intrinsic_instr *store =
       nir_instr_as_intrinsic(nir_block_last_instr(nir_start_block(b.impl)));
    ASSERT_TRUE(nir_src_is_const(store->src[0]));
@@ -121,6 +125,42 @@ TEST(NativeFMA, NormalAddendAndCancellation)
       check_fma_case(t);
 }
 
+TEST(NativeFMA, NonzeroResultSkipsWideRepair)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_COMPUTE, &agx_nir_options, "FMA normal-result fast path");
+   nir_def *a = agx_test_load_preamble(&b, 1, 32, 4);
+   nir_def *c = agx_test_load_preamble(&b, 1, 32, 6);
+   nir_def *z = agx_test_load_preamble(&b, 1, 32, 8);
+   nir_def *native = agx_test_load_preamble(&b, 1, 32, 10);
+   nir_def *fixed = agx_nir_fma_rte_fixup(&b, a, c, z, native);
+   nir_store_global(&b, fixed, nir_imm_int64(&b, 0), .align_mul = 4);
+   nir_validate_shader(b.shader, "guarded FMA tininess repair");
+
+   nir_block *entry = nir_start_block(b.impl);
+   nir_cf_node *next = nir_cf_node_next(&entry->cf_node);
+   ASSERT_NE(next, nullptr);
+   ASSERT_EQ(next->type, nir_cf_node_if);
+   nir_if *guard = nir_cf_node_as_if(next);
+   nir_alu_instr *eq = nir_def_as_alu(guard->condition.ssa);
+   ASSERT_EQ(eq->op, nir_op_ieq);
+   nir_alu_instr *abs_bits = nir_src_as_alu(eq->src[0].src);
+   ASSERT_EQ(abs_bits->op, nir_op_iand);
+   EXPECT_EQ(abs_bits->src[0].src.ssa, native);
+   EXPECT_EQ(nir_alu_src_as_uint(abs_bits->src[1]), 0x7fffffffu);
+
+   unsigned wide_products = 0;
+   nir_foreach_instr(instr, nir_if_first_then_block(guard)) {
+      if (instr->type == nir_instr_type_alu &&
+          nir_instr_as_alu(instr)->op == nir_op_imul &&
+          nir_instr_as_alu(instr)->def.bit_size == 64)
+         wide_products++;
+   }
+   EXPECT_EQ(wide_products, 1u);
+   EXPECT_TRUE(exec_list_is_empty(&nir_if_first_else_block(guard)->instr_list));
+   ralloc_free(b.shader);
+}
+
 TEST(NativeFMA, ExactDyadicOracle)
 {
    /* Optional extended oracle generated with exact rationals, independent of
diff --git a/src/asahi/compiler/test/test-scs-nir.c b/src/asahi/compiler/test/test-scs-nir.c
new file mode 100644
index 000000000000..d47e851a5fdf
--- /dev/null
+++ b/src/asahi/compiler/test/test-scs-nir.c
@@ -0,0 +1,145 @@
+/* Explicit NIR SCS compiler qualification, independent of API ray queries. */
+#include "agx_compile.h"
+#include "agx_debug.h"
+#include "agx3_graphics_program.h"
+#include "agx3_scs_core.h"
+#include "agx3_rt_m5_core.h"
+#include "asahi/lib/agx3_compute_program_core.h"
+#include "agx_nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "compiler/glsl_types.h"
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static bool compile(unsigned mode, unsigned variant, unsigned width,
+                    int offset, enum agx_isa_encoding encoding,
+                    const char *out)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,
+                                                 &agx_nir_options, "M5 NIR SCS");
+   b.shader->info.workgroup_size[0] = 64;
+   b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+   nir_def *id = nir_channel(&b, nir_load_global_invocation_id(&b, 32), 0);
+   nir_def *in_addr = nir_iadd_imm(&b, nir_u2u64(&b, nir_ishl_imm(&b,id,2)), 0x1000001cd00ull);
+   nir_def *input = nir_load_global(&b, 1, 32, in_addr, .align_mul=4);
+   nir_def *value = nir_iadd_imm(&b, nir_imul_imm(&b,input,3),5);
+   nir_def *result;
+   if (mode == 0) {
+      nir_def *v = nir_u2uN(&b, value, width);
+      nir_store_scs_agx(&b, v, .base=offset);
+      result = nir_u2u32(&b, nir_load_scs_agx(&b,1,width,.base=offset));
+   } else if (mode == 1 || mode == 3) {
+      nir_def *v[4];
+      for(unsigned i=0;i<variant;i++) v[i]=nir_iadd_imm(&b,value,i*17);
+      nir_def *base = nir_ishl_imm(&b,id,4);
+      /* Keep both address operands live in registers. */
+      nir_def *index = nir_iand_imm(&b,input,0);
+      nir_store_scs_indexed_agx(&b,nir_vec(&b,v,variant),base,index,
+                                .base=offset,.component=4);
+      nir_def *loaded = nir_load_scs_indexed_agx(&b,variant, 32,base,index,
+                                               .base=offset,.component=4);
+      nir_def *valid=nir_imm_true(&b);
+      for(unsigned i=0;i<variant;i++)
+         valid=nir_iand(&b,valid,nir_ieq(&b,nir_channel(&b,loaded,i),v[i]));
+      if (mode == 3) {
+         for(unsigned i=0;i<variant;i++) v[i]=nir_iadd_imm(&b,v[i],33);
+         nir_store_scs_indexed_agx(&b,nir_vec(&b,v,variant),base,index,
+                                   .base=offset,.component=4);
+         nir_def *again=nir_load_scs_indexed_agx(&b,variant, 32,base,index,
+                                               .base=offset,.component=4);
+         for(unsigned i=0;i<variant;i++)
+            valid=nir_iand(&b,valid,nir_ieq(&b,nir_channel(&b,again,i),v[i]));
+      }
+      result=nir_bcsel(&b,valid,value,nir_imm_int(&b,0xbad0bad0));
+   } else {
+      nir_enter_query_agx(&b);
+      nir_intersect_ray_agx(&b,nir_imm_int64(&b,0x100),nir_imm_int64(&b,0x200),
+                            nir_imm_int(&b,0x10000000),nir_imm_int(&b,0),id);
+      nir_continue_ray_agx(&b, id);
+      result=value;
+   }
+   nir_def *out_addr = nir_iadd_imm(&b,nir_u2u64(&b,nir_ishl_imm(&b,id,2)),0x1000001dd00ull);
+   nir_store_global(&b,result,out_addr,.align_mul=4,.write_mask=1);
+   agx_preprocess_nir(b.shader);
+   struct agx_shader_key key={.dev={.chip=encoding == AGX_ISA_ENCODING_AGX3B ? AGX_CHIP_G17P : AGX_CHIP_G17S,.encoding=encoding},
+                              .no_preamble=true};
+   struct agx_shader_part part={0};
+   bool ok=agx_compile_shader_nir(b.shader,&key,&part);
+   printf("mode=%u vec=%u width=%u offset=%d enc=%u %s %s\n",mode,variant,width,offset,encoding,ok?"PASS":"REFUSED",ok?"":agx_last_compile_refusal());
+   if(ok) {
+      assert(part.info.uses_ray_traversal);
+      assert(!part.info.ray_traversal_depth_nonzero);
+      unsigned loads=0,stores=0,entries=0;
+      /* The query entry (incStk, allocation index 4) writes its query pointer
+       * to a register-allocated f2 destination, as native does (witnesses
+       * 7f04400000000000 = f2 46, 3f04000000000000 = f2 6); it no longer
+       * writes a fixed r0 (0f04000000000000). Accept any allocated half. */
+      for(size_t i=part.info.main_offset;i<part.info.main_offset+part.info.main_size;i++) {
+         for(unsigned dest=0;dest<=252 && i+8<=part.info.main_offset+part.info.main_size;dest+=2) {
+            uint8_t entry[16],entry_len=0;
+            if(agx3_rt_m5_encode_enter_query(AGX_ISA_ENCODING_AGX3C,dest,4,entry,&entry_len) &&
+               entry_len==8 && !memcmp(part.binary+i,entry,8)) { entries++; break; }
+         }
+         for(unsigned n=2;n<=AGX3_SCS_MAX_LENGTH && i+n<=part.info.main_offset+part.info.main_size;n+=2) {
+            struct agx3_scs_desc d;
+            bool load=agx3_scs_decode_load(encoding,part.binary+i,n,&d);
+            bool store=!load && agx3_scs_decode_store(encoding,part.binary+i,n,&d);
+            if(!load && !store)continue;
+            assert(d.byte_offset==offset);
+            if(mode==1 || mode==3) {
+               assert(d.address_kind==AGX3_SCS_ADDRESS_REGISTER_INDEXED);
+               assert(d.component_mask==((1u<<variant)-1));
+               assert(d.address_selector==4);
+            }
+            loads+=load;stores+=store;
+         }
+      }
+      assert(loads==(mode==2?0:mode==3?2:1));
+      assert(stores==loads);
+      assert(entries==(mode==2 ? 1 : 0));
+      if(out){FILE *f=fopen(out,"wb");assert(f);assert(fwrite(part.binary+part.info.main_offset,1,part.info.main_size,f)==part.info.main_size);assert(!fclose(f));printf("bytes=%u gprs=%u\n",part.info.main_size,part.info.nr_gprs);
+         struct agx3_g17p_graphics_program_desc desc={
+            .program_va=0x10000000000ull, .shader_va=0x10000003000ull,
+            .shader_binary_size=part.info.main_size,.nr_gprs=part.info.nr_gprs,
+            .scratch_size=MAX2(part.info.scratch_size, part.info.agx3_spectpr_scratch_size),
+            .compiler_d_valid=part.info.agx3_staging_known,
+            .compiler_d=(uint32_t)part.info.agx3_staging_halfs*2,
+         };
+         struct agx3_g17p_graphics_program program;
+         assert(agx3_g17g_compute_program_generate(AGX_CHIP_G17S,&desc,&program));
+         char file[512];snprintf(file,sizeof(file),"%s.program",out);
+         f=fopen(file,"wb");assert(f);assert(fwrite(program.bytes,1,program.length,f)==program.length);assert(!fclose(f));
+         struct agx3_g17p_native_compute_program_desc launch_desc={
+            .program_va=desc.program_va,.shader_va=desc.shader_va,
+         };
+         assert(agx3_cdm_direct_template_sources_compute(false,true,false,&launch_desc.launch_sources));
+         struct agx3_compute_program launch;
+         assert(agx3_m5_native_compute_program_bind(AGX_CHIP_G17S,&launch_desc,&launch));
+         uint8_t cdm[48]={0};struct agx3_compute_dispatch_sink sink={.data=cdm,.capacity=44};
+         const struct agx3_compute_direct_grid grid={.source=AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,.size={1024,1,1},.local_size={64,1,1},.dimensions=1};
+         assert(agx3_m5_native_compute_dispatch_append(&launch,&grid,0,&sink));
+         assert(sink.length==44);cdm[47]=0x40;
+         snprintf(file,sizeof(file),"%s.cdm",out);f=fopen(file,"wb");assert(f);assert(fwrite(cdm,1,48,f)==48);assert(!fclose(f));
+}
+   } else assert(!part.binary);
+   free(part.binary);ralloc_free(b.shader);return ok;
+}
+int main(int argc,char **argv) {
+   setbuf(stdout,NULL);
+   glsl_type_singleton_init_or_ref();
+   for(unsigned m=0;m<4;m++)for(unsigned n=1;n<=4;n*=2) {
+      if(m==2)continue;
+      if(!m&&n!=1)continue;
+      char path[256];snprintf(path,sizeof(path),"%s/scs-%u-%u.bin",argc>1?argv[1]:".",m,n);
+      assert(compile(m,n,32,0x6000,AGX_ISA_ENCODING_AGX3C,argc>1?path:NULL));
+   }
+   assert(compile(0,1,16,-128,AGX_ISA_ENCODING_AGX3C,NULL));
+   assert(compile(2,1,32,0,AGX_ISA_ENCODING_AGX3C,NULL));
+   assert(compile(1,3,32,0,AGX_ISA_ENCODING_AGX3C,NULL));
+   assert(!compile(1,4,32,32768,AGX_ISA_ENCODING_AGX3C,NULL));
+   assert(compile(1,4,32,0,AGX_ISA_ENCODING_AGX3B,NULL));
+   assert(!compile(1,4,32,0,AGX_ISA_ENCODING_AGX2,NULL));
+   glsl_type_singleton_decref();return 0;
+}
diff --git a/src/asahi/compiler/test/test-scs-query.c b/src/asahi/compiler/test/test-scs-query.c
new file mode 100644
index 000000000000..9d1a2cc9b6f4
--- /dev/null
+++ b/src/asahi/compiler/test/test-scs-query.c
@@ -0,0 +1,643 @@
+/* Compile the SCS forms used by the retained native M5 query runtime. */
+#include "agx_compile.h"
+#include "agx_debug.h"
+#include "agx_nir.h"
+#include "agx_nir_m5_ray_query.h"
+#include "agx3_scs_core.h"
+#include "agx3_sr_core.h"
+#include "agx3_pixel_core.h"
+#include "agx3_graphics_program.h"
+#include "asahi/lib/agx3_compute_program_core.h"
+#include "compiler/nir/nir_builder.h"
+#include "compiler/glsl_types.h"
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static void write_file(const char *name,const void *data,size_t size)
+{
+ FILE *f=fopen(name,"wb");assert(f);assert(fwrite(data,1,size,f)==size);assert(!fclose(f));
+}
+
+/* mode0 direct, mode1 uniform indexed, mode2 register base, mode3 status. */
+static void compile(unsigned mode,unsigned count,unsigned width,unsigned flags,
+                    const char *dir)
+{
+ nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,&agx_nir_options,"M5 query SCS forms");
+ b.shader->info.workgroup_size[0]=64;
+ b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+ nir_def *id=nir_channel(&b,nir_load_global_invocation_id(&b,32),0);
+ nir_def *address=nir_iadd_imm(&b,nir_u2u64(&b,nir_ishl_imm(&b,id,2)),0x1000001cd00ull);
+ nir_def *input=nir_load_global(&b,1,32,address,.align_mul=4);
+ nir_def *expected=nir_iadd_imm(&b,nir_imul_imm(&b,input,3),5);
+ nir_def *base=mode==1?nir_load_preamble(&b,1,64,.base=122):nir_ishl_imm(&b,id,4);
+ enum pipe_format format=width==8?PIPE_FORMAT_R8_UINT:width==16?PIPE_FORMAT_R16_UINT:PIPE_FORMAT_R32_UINT;
+ const unsigned result_bits=(width==32||(flags&2))?32:16;
+ nir_def *components[4];
+ for(unsigned c=0;c<count;c++) {
+  nir_def *v=nir_iadd_imm(&b,expected,17*c);
+  components[c]=width==32?v:nir_u2u16(&b,v);
+ }
+ nir_def *data=nir_vec(&b,components,count);
+ if(mode==0)nir_push_if(&b,nir_ieq_imm(&b,id,0));
+ nir_def *values;
+ if(mode==0) {
+  nir_store_scs_agx(&b,data,.base=0x6000);
+  values=nir_load_scs_agx(&b,count,result_bits,.base=0x6000);
+ } else if(mode==1) {
+  nir_def *index=nir_ishl_imm(&b,id,4);
+  nir_store_scs_uniform_agx(&b,data,base,index,.base=0x6000,.format=format,.component=0);
+  values=nir_load_scs_uniform_agx(&b,count,result_bits,base,index,.base=0x6000,.format=format,.component=0,.flags=flags);
+ } else {
+  nir_store_scs_base_agx(&b,data,base,.base=0x6000,.format=format,.component=4);
+  values=nir_load_scs_base_agx(&b,count,result_bits,base,.base=0x6000,.format=format,.component=4,.flags=flags);
+ }
+ if(getenv("M5_QUERY_RAW") && mode==1 && (flags&2)) {
+  nir_def *raw_address=nir_iadd_imm(&b,nir_u2u64(&b,nir_ishl_imm(&b,id,2)),0x1000001ed00ull);
+  nir_store_global(&b,nir_u2u32(&b,values),raw_address,.align_mul=4,.write_mask=1);
+ }
+ nir_def *valid=nir_imm_true(&b);
+ for(unsigned c=0;c<count;c++) {
+  nir_def *v=nir_u2u32(&b,nir_channel(&b,values,c));
+  nir_def *e=nir_u2u32(&b,components[c]);
+  if(width<32) {
+   unsigned mask=(1u<<width)-1;
+   e=nir_iand_imm(&b,e,mask);
+   if(!(flags&2))v=nir_iand_imm(&b,v,mask);
+  }
+  valid=nir_iand(&b,valid,nir_ieq(&b,v,e));
+ }
+ nir_def *result=nir_bcsel(&b,valid,expected,nir_imm_int(&b,0xbad0bad0));
+ if(mode==0) {
+  nir_push_else(&b,NULL);
+  nir_pop_if(&b,NULL);
+  result=nir_if_phi(&b,result,expected);
+ }
+ address=nir_iadd_imm(&b,nir_u2u64(&b,nir_ishl_imm(&b,id,2)),0x1000001dd00ull);
+ nir_store_global(&b,result,address,.align_mul=4,.write_mask=1);
+ agx_preprocess_nir(b.shader);
+ struct agx_shader_key key={.dev={.chip=AGX_CHIP_G17S,.encoding=AGX_ISA_ENCODING_AGX3C},.no_preamble=true};
+ struct agx_shader_part part={0};
+ bool ok=agx_compile_shader_nir(b.shader,&key,&part);
+ printf("mode=%u count=%u width=%u flags=%u %s %s\n",mode,count,width,flags,ok?"PASS":"FAIL",ok?"":agx_last_compile_refusal());
+ assert(ok&&part.info.uses_ray_traversal&&!part.info.ray_traversal_depth_nonzero);
+ unsigned stores=0,loads=0;
+ for(size_t off=part.info.main_offset;off<part.info.main_offset+part.info.main_size;off++)
+  for(unsigned n=2;n<=16&&off+n<=part.info.main_offset+part.info.main_size;n+=2) {
+   struct agx3_scs_desc d;bool load=agx3_scs_decode_load(AGX_ISA_ENCODING_AGX3C,part.binary+off,n,&d);
+   bool store=!load&&agx3_scs_decode_store(AGX_ISA_ENCODING_AGX3C,part.binary+off,n,&d);
+   if(!load&&!store)continue;
+   assert(d.address_kind==(mode==0?AGX3_SCS_ADDRESS_DIRECT:mode==1?AGX3_SCS_ADDRESS_UNIFORM_INDEXED:AGX3_SCS_ADDRESS_REGISTER_BASE));
+   assert(d.component_mask==((1u<<count)-1)&&d.byte_offset==0x6000);
+   assert(d.width==(width==8?AGX3_MEM_W8:width==16?AGX3_MEM_W16:AGX3_MEM_W32));
+   if(load)assert(d.zero_extend==!!(flags&2)&&d.c_load_marker==!!(flags&1)&&d.c_status_load==!!(flags&4));
+   stores+=store;loads+=load;
+  }
+ assert(stores==1&&loads==1);
+ if(dir) {
+  char name[512];snprintf(name,sizeof(name),"%s/form-%u-%u-%u-%u.bin",dir,mode,count,width,flags);
+  write_file(name,part.binary+part.info.main_offset,part.info.main_size);
+  struct agx3_ioto_uniform_range range={.start_halfs=122,.size_halfs=4,.address=0x10000090000ull};
+  struct agx3_g17p_graphics_program_desc desc={.program_va=0x10000000000ull,.shader_va=0x10000003000ull,
+   .uniform_ranges=mode==1?&range:NULL,.uniform_range_count=mode==1,
+   .shader_binary_size=part.info.main_size,.nr_gprs=part.info.nr_gprs,
+   .scratch_size=MAX2(part.info.scratch_size,part.info.agx3_spectpr_scratch_size),
+   .compiler_d_valid=part.info.agx3_staging_known,.compiler_d=(uint32_t)part.info.agx3_staging_halfs*2};
+  struct agx3_g17p_graphics_program program;assert(agx3_g17g_compute_program_generate(AGX_CHIP_G17S,&desc,&program));
+  snprintf(name,sizeof(name),"%s/form-%u-%u-%u-%u.program",dir,mode,count,width,flags);write_file(name,program.bytes,program.length);
+  struct agx3_g17p_native_compute_program_desc ld={.program_va=desc.program_va,.shader_va=desc.shader_va};
+  assert(agx3_cdm_direct_template_sources_compute(false,true,false,&ld.launch_sources));
+  struct agx3_compute_program launch;assert(agx3_m5_native_compute_program_bind(AGX_CHIP_G17S,&ld,&launch));
+  uint8_t cdm[48]={0};struct agx3_compute_dispatch_sink sink={.data=cdm,.capacity=44};
+  const struct agx3_compute_direct_grid grid={.source=AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,.size={1024,1,1},.local_size={64,1,1},.dimensions=1};
+  assert(agx3_m5_native_compute_dispatch_append(&launch,&grid,0,&sink));assert(sink.length==44);cdm[47]=0x40;
+  snprintf(name,sizeof(name),"%s/form-%u-%u-%u-%u.cdm",dir,mode,count,width,flags);write_file(name,cdm,48);
+  printf("  bytes=%u gprs=%u loader=%u\n",part.info.main_size,part.info.nr_gprs,program.length);
+ }
+ free(part.binary);ralloc_free(b.shader);
+}
+/* Check the native candidate record layout across all ten records per lane.
+ * This validates SCS transport, not hardware candidate production. */
+static void candidate_packet(const char *dir)
+{
+ nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,&agx_nir_options,"M5 candidate record transport");
+ b.shader->info.workgroup_size[0]=64;
+ b.shader->info.workgroup_size[1]=b.shader->info.workgroup_size[2]=1;
+ nir_def *id=nir_channel(&b,nir_load_global_invocation_id(&b,32),0);
+ nir_def *resident=nir_load_scs_thread_index_agx(&b);
+ nir_def *lane=nir_iand_imm(&b,resident,31);
+ nir_def *driver=nir_iadd_imm(&b,nir_imul_imm(&b,nir_iand_imm(&b,resident,~31u),272),0x5d8000);
+ nir_def *input=nir_load_global(&b,1,32,nir_iadd_imm(&b,nir_u2u64(&b,nir_ishl_imm(&b,id,2)),0x1000001cd00ull),.align_mul=4);
+ nir_def *expected=nir_iadd_imm(&b,nir_imul_imm(&b,input,3),5);
+ nir_def *address=nir_iadd_imm(&b,nir_iadd(&b,driver,nir_imul_imm(&b,lane,200)),1856);
+ nir_def *queue=nir_iadd_imm(&b,nir_iadd(&b,driver,nir_ishl_imm(&b,lane,2)),1728);
+ nir_store_scs_base_agx(&b,nir_imm_int(&b,(10u<<21)|0x1fffff),queue,.base=0,.format=PIPE_FORMAT_R32_UINT,.component=4);
+ for(unsigned i=0;i<10;i++) {
+  nir_def *v=nir_iadd_imm(&b,nir_imul_imm(&b,id,17),i);
+  nir_def *flags=nir_iand_imm(&b,nir_ixor(&b,nir_iadd_imm(&b,id,i),nir_imm_int(&b,0x5a00)),0xffff);
+  nir_def *header=nir_vec2(&b,v,nir_ior_imm(&b,nir_ishl_imm(&b,flags,16),0xa123));
+  nir_store_scs_base_agx(&b,header,address,.base=20*i,.format=PIPE_FORMAT_R32_UINT,.component=4);
+  nir_store_scs_base_agx(&b,nir_vec2(&b,nir_iadd_imm(&b,v,100),nir_iadd_imm(&b,v,200)),address,.base=20*i+8,.format=PIPE_FORMAT_R32_UINT,.component=4);
+  nir_store_scs_base_agx(&b,nir_iadd_imm(&b,v,300),address,.base=20*i+16,.format=PIPE_FORMAT_R32_UINT,.component=4);
+ }
+ nir_def *valid=nir_ieq_imm(&b,agx_nir_m5_candidate_count(&b,driver,lane),10);
+ for(unsigned i=0;i<10;i++) {
+  struct agx_m5_ray_candidate c=agx_nir_m5_load_candidate(&b,driver,lane,nir_imm_int(&b,i));
+  nir_def *v=nir_iadd_imm(&b,nir_imul_imm(&b,id,17),i);
+  nir_def *pointer=nir_pack_64_2x32_split(&b,v,nir_imm_int(&b,0xa123));
+  nir_def *flags=nir_iand_imm(&b,nir_ixor(&b,nir_iadd_imm(&b,id,i),nir_imm_int(&b,0x5a00)),0xffff);
+  valid=nir_iand(&b,valid,nir_ieq(&b,c.compressed_address,pointer));
+  valid=nir_iand(&b,valid,nir_ieq(&b,c.flags,flags));
+  valid=nir_iand(&b,valid,nir_ieq(&b,nir_channel(&b,c.barycentrics,0),nir_iadd_imm(&b,v,100)));
+  valid=nir_iand(&b,valid,nir_ieq(&b,nir_channel(&b,c.barycentrics,1),nir_iadd_imm(&b,v,200)));
+  valid=nir_iand(&b,valid,nir_ieq(&b,c.distance,nir_iadd_imm(&b,v,300)));
+ }
+ nir_def *result=nir_bcsel(&b,valid,expected,nir_imm_int(&b,0xbad0bad0));
+ nir_store_global(&b,result,nir_iadd_imm(&b,nir_u2u64(&b,nir_ishl_imm(&b,id,2)),0x1000001dd00ull),.align_mul=4,.write_mask=1);
+ agx_preprocess_nir(b.shader);
+ struct agx_shader_key key={.dev={.chip=AGX_CHIP_G17S,.encoding=AGX_ISA_ENCODING_AGX3C},.no_preamble=true};
+ struct agx_shader_part part={0};
+ bool ok=agx_compile_shader_nir(b.shader,&key,&part);
+ printf("M5_CANDIDATE_PACKET %s %s\n",ok?"PASS":"FAIL",ok?"":agx_last_compile_refusal());
+ assert(ok&&part.info.uses_ray_traversal&&!part.info.ray_traversal_depth_nonzero);
+ if(dir) {
+  char name[512];snprintf(name,sizeof(name),"%s/candidate-packet.bin",dir);
+  write_file(name,part.binary+part.info.main_offset,part.info.main_size);
+  struct agx3_g17p_graphics_program_desc desc={.program_va=0x10000000000ull,.shader_va=0x10000002000ull,
+   .uniform_ranges=NULL,.uniform_range_count=0,
+   .shader_binary_size=part.info.main_size,.nr_gprs=part.info.nr_gprs,
+   .scratch_size=MAX2(part.info.scratch_size,part.info.agx3_spectpr_scratch_size),
+   .compiler_d_valid=part.info.agx3_staging_known,.compiler_d=(uint32_t)part.info.agx3_staging_halfs*2};
+  struct agx3_g17p_graphics_program program;assert(agx3_g17g_compute_program_generate(AGX_CHIP_G17S,&desc,&program));
+  snprintf(name,sizeof(name),"%s/candidate-packet.program",dir);write_file(name,program.bytes,program.length);
+  struct agx3_g17p_native_compute_program_desc ld={.program_va=desc.program_va,.shader_va=desc.shader_va};
+  assert(agx3_cdm_direct_template_sources_compute(false,true,false,&ld.launch_sources));
+  struct agx3_compute_program launch;assert(agx3_m5_native_compute_program_bind(AGX_CHIP_G17S,&ld,&launch));
+  uint8_t cdm[48]={0};struct agx3_compute_dispatch_sink sink={.data=cdm,.capacity=44};
+  const struct agx3_compute_direct_grid grid={.source=AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,.size={1024,1,1},.local_size={64,1,1},.dimensions=1};
+  assert(agx3_m5_native_compute_dispatch_append(&launch,&grid,0,&sink));assert(sink.length==44);cdm[47]=0x40;
+  snprintf(name,sizeof(name),"%s/candidate-packet.cdm",dir);write_file(name,cdm,48);
+  printf("  bytes=%u gprs=%u loader=%u\n",part.info.main_size,part.info.nr_gprs,program.length);
+ }
+ free(part.binary);ralloc_free(b.shader);
+}
+static void refused(unsigned format,unsigned flags,unsigned selector,int offset)
+{
+ nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,&agx_nir_options,"invalid SCS form");
+ nir_def *v=nir_load_scs_base_agx(&b,1,16,nir_imm_int(&b,0),.base=offset,
+                                .format=format,.flags=flags,.component=selector);
+ nir_store_global(&b,v,nir_imm_int64(&b,0x10000000000ull),.align_mul=4,.write_mask=1);
+ agx_preprocess_nir(b.shader);
+ struct agx_shader_key key={.dev={.chip=AGX_CHIP_G17S,.encoding=AGX_ISA_ENCODING_AGX3C},.no_preamble=true};
+ struct agx_shader_part part={0};
+ assert(!agx_compile_shader_nir(b.shader,&key,&part)&&!part.binary);
+ ralloc_free(b.shader);
+}
+
+static void
+standard_ray_query(unsigned count, bool array, mesa_shader_stage stage)
+{
+   const bool kernel = stage != MESA_SHADER_COMPUTE;
+   const char *value = getenv("ASAHI_M5_RQ_CANDIDATE_API");
+   char *saved = value ? strdup(value) : NULL;
+   if (kernel)
+      setenv("ASAHI_M5_RQ_CANDIDATE_API", "1", 1);
+   nir_builder b = nir_builder_init_simple_shader(
+      stage, &agx_nir_options, "M5 standard ray query");
+   b.shader->info.workgroup_size[0] = 64;
+   b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+   b.shader->info.ray_queries = count;
+   nir_variable *queries[16];
+   nir_def *rqs[16];
+   assert(count && count <= 16);
+   nir_variable *query_array = array ? nir_local_variable_create(
+      b.impl, glsl_array_type(glsl_uint64_t_type(), count, 0), "queries") : NULL;
+   if (query_array)
+      query_array->data.ray_query = true;
+   for (unsigned i = 0; i < count; ++i) {
+      nir_variable *query = array ? query_array :
+         nir_local_variable_create(b.impl, glsl_uint64_t_type(), "query");
+      query->data.ray_query = true;
+      queries[i] = query;
+      nir_deref_instr *deref = nir_build_deref_var(&b, query);
+      if (array) {
+         nir_def *index = nir_iand_imm(&b,
+            nir_iadd_imm(&b, nir_load_local_invocation_index(&b), i), count - 1);
+         deref = nir_build_deref_array(&b, deref, index);
+      }
+      nir_def *rq = &deref->def;
+      rqs[i] = rq;
+      nir_rq_initialize(&b, rq, nir_imm_int64(&b, 0x10002000000ull),
+                        nir_imm_int(&b, 1), nir_imm_int(&b, 255),
+                        nir_imm_vec3(&b, 0, 0, -1), nir_imm_float(&b, 0),
+                        nir_imm_vec3(&b, 0, 0, 1), nir_imm_float(&b, 10));
+   }
+   for (unsigned i = count; i-- > 0;) {
+      nir_push_loop(&b);
+      nir_break_if(&b, nir_inot(&b, nir_rq_proceed(&b, 1, rqs[i])));
+      nir_pop_loop(&b, NULL);
+   }
+   nir_def *rq = rqs[0];
+   nir_def *type = nir_rq_load(&b, 1, 32, rq,
+      .ray_query_value = nir_ray_query_value_intersection_type, .committed = true);
+   nir_def *distance = nir_rq_load(&b, 1, 32, rq,
+      .ray_query_value = nir_ray_query_value_intersection_t, .committed = true);
+   nir_store_global(&b, nir_vec2(&b, type, distance),
+                     nir_imm_int64(&b, 0x10003010000ull),
+                     .align_mul = 8, .write_mask = 3);
+   const struct agx_m5_ray_query_layout layout = {
+      .payload_uniform = kernel ? 134 : 122,
+      .bvh_uniform = kernel ? 100 : 96, .status_uniform = 134,
+      .launch_header = 0x10000000,
+      .context_uniform = kernel ? 160 : 0,
+      .as_table_uniform = kernel ? 164 : 0,
+      .instance_traversal = kernel,
+   };
+   struct agx_m5_ray_query_layout bad_layout = layout;
+   bad_layout.payload_uniform++;
+   assert(!agx_nir_lower_m5_ray_query(b.shader, &bad_layout));
+   assert(queries[0]->data.ray_query && b.shader->info.ray_queries == count);
+   struct agx_m5_ray_query_layout kernel_layout = layout;
+   if (stage != MESA_SHADER_COMPUTE) {
+      assert(!agx_nir_lower_m5_ray_query(b.shader, &layout));
+      assert(queries[0]->data.ray_query && b.shader->info.ray_queries == count);
+      kernel_layout.compute_kernel = true;
+   }
+   assert(agx_nir_lower_m5_ray_query(b.shader, &kernel_layout));
+   assert(!b.shader->info.ray_queries);
+   nir_foreach_block(block, b.impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type == nir_instr_type_intrinsic)
+            assert(!nir_intrinsic_is_ray_query(nir_instr_as_intrinsic(instr)->intrinsic));
+      }
+   }
+   b.shader->info.stage = stage == MESA_SHADER_FRAGMENT ? stage : MESA_SHADER_COMPUTE;
+   agx_preprocess_nir(b.shader);
+   struct agx_shader_key key = {
+      .dev = {.chip = AGX_CHIP_G17S, .encoding = AGX_ISA_ENCODING_AGX3C},
+      .no_preamble = true,
+      .has_scratch = true,
+   };
+   struct agx_shader_part part = {0};
+   bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+   if (!compiled) fprintf(stderr, "Query compile refusal: %s\n", agx_last_compile_refusal());
+   assert(compiled);
+   assert(part.info.uses_ray_traversal && part.info.main_size);
+   if (stage == MESA_SHADER_FRAGMENT) {
+      const uint8_t acquire[] = {0x27, 0x00, 0x00, 0x05, 0x00, 0x00};
+      unsigned acquires = 0;
+      for (size_t i = part.info.main_offset;
+           i + sizeof(acquire) <= part.info.main_offset + part.info.main_size; ++i)
+         acquires += !memcmp(part.binary + i, acquire, sizeof(acquire));
+      assert(acquires == 1);
+      assert(!part.info.agx3_colour_link.rt_query_write_acquired);
+      const uint8_t release[] = {0x27, 0x00, 0x00, 0x04, 0x00, 0x00};
+      unsigned releases = 0;
+      for (size_t i = part.info.main_offset;
+           i + sizeof(release) <= part.info.main_offset + part.info.main_size; ++i)
+         releases += !memcmp(part.binary + i, release, sizeof(release));
+      assert(releases == 1);
+   }
+   free(part.binary);
+   ralloc_free(b.shader);
+   if (saved) {
+      setenv("ASAHI_M5_RQ_CANDIDATE_API", saved, 1);
+      free(saved);
+   } else {
+      unsetenv("ASAHI_M5_RQ_CANDIDATE_API");
+   }
+   printf("M5_STANDARD_RAY_QUERY_PASS count=%u array=%u stage=%u\n", count, array, stage);
+}
+
+static unsigned
+native_query_ops(nir_shader *shader)
+{
+   unsigned n = 0;
+   nir_foreach_function_impl(impl, shader) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            switch (nir_instr_as_intrinsic(instr)->intrinsic) {
+            case nir_intrinsic_enter_query_agx:
+            case nir_intrinsic_intersect_ray_agx:
+            case nir_intrinsic_continue_ray_agx:
+            case nir_intrinsic_load_scs_indexed_agx:
+            case nir_intrinsic_store_scs_indexed_agx:
+            case nir_intrinsic_load_scs_uniform_agx:
+            case nir_intrinsic_store_scs_uniform_agx:
+            case nir_intrinsic_load_scs_base_agx:
+            case nir_intrinsic_store_scs_base_agx:
+            case nir_intrinsic_load_scs_register_agx:
+            case nir_intrinsic_store_scs_agx:
+               n++;
+               break;
+            default:
+               break;
+            }
+         }
+      }
+   }
+   return n;
+}
+
+/* Geometry lowering clones a lowered query into kernels whose API stores it
+ * strips. Such a query is removed; one whose result is stored is kept. */
+static void
+dead_ray_query(bool live)
+{
+   const char *value = getenv("ASAHI_M5_RQ_CANDIDATE_API");
+   char *saved = value ? strdup(value) : NULL;
+   setenv("ASAHI_M5_RQ_CANDIDATE_API", "1", 1);
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_GEOMETRY, &agx_nir_options, "M5 dead ray query");
+   b.shader->info.workgroup_size[0] = 64;
+   b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+   b.shader->info.ray_queries = 1;
+   nir_variable *query =
+      nir_local_variable_create(b.impl, glsl_uint64_t_type(), "query");
+   query->data.ray_query = true;
+   nir_def *rq = &nir_build_deref_var(&b, query)->def;
+   /* A live ID-dependent value before the query, as poly's index write. */
+   nir_def *id = nir_channel(&b, nir_load_global_invocation_id(&b, 32), 0);
+   nir_rq_initialize(&b, rq, nir_imm_int64(&b, 0x10002000000ull),
+                     nir_imm_int(&b, 1), nir_imm_int(&b, 255),
+                     nir_imm_vec3(&b, 0, 0, -1), nir_imm_float(&b, 0),
+                     nir_imm_vec3(&b, 0, 0, 1), nir_imm_float(&b, 10));
+   nir_push_loop(&b);
+   nir_break_if(&b, nir_inot(&b, nir_rq_proceed(&b, 1, rq)));
+   nir_pop_loop(&b, NULL);
+   nir_def *type = nir_rq_load(&b, 1, 32, rq,
+      .ray_query_value = nir_ray_query_value_intersection_type, .committed = true);
+   nir_intrinsic_instr *result = nir_store_global(&b, type,
+      nir_imm_int64(&b, 0x10003010000ull), .align_mul = 4, .write_mask = 1);
+   nir_store_global(&b, id, nir_imm_int64(&b, 0x10003020000ull),
+                    .align_mul = 4, .write_mask = 1);
+   const struct agx_m5_ray_query_layout layout = {
+      .payload_uniform = 134, .bvh_uniform = 100, .status_uniform = 134,
+      .launch_header = 0x10000000, .context_uniform = 160,
+      .as_table_uniform = 164, .instance_traversal = true,
+      .compute_kernel = true,
+   };
+   assert(agx_nir_lower_m5_ray_query(b.shader, &layout));
+   assert(native_query_ops(b.shader));
+   if (!live)
+      nir_instr_remove(&result->instr);
+   const unsigned before = native_query_ops(b.shader);
+   const bool progress = agx_nir_m5_remove_dead_ray_queries(b.shader);
+   assert(progress || live);
+   assert(live ? native_query_ops(b.shader) == before
+               : !native_query_ops(b.shader));
+   unsigned stores = 0;
+   nir_foreach_block(block, b.impl) {
+      nir_foreach_instr(instr, block) {
+         stores += instr->type == nir_instr_type_intrinsic &&
+            nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_store_global;
+      }
+   }
+   assert(stores == (live ? 2 : 1));
+   b.shader->info.stage = MESA_SHADER_COMPUTE;
+   agx_preprocess_nir(b.shader);
+   struct agx_shader_key key = {
+      .dev = {.chip = AGX_CHIP_G17S, .encoding = AGX_ISA_ENCODING_AGX3C},
+      .no_preamble = true,
+   };
+   struct agx_shader_part part = {0};
+   bool compiled = agx_compile_shader_nir(b.shader, &key, &part);
+   if (!compiled) fprintf(stderr, "Dead query compile refusal: %s\n", agx_last_compile_refusal());
+   assert(compiled);
+   assert(part.info.uses_ray_traversal == live && part.info.main_size);
+   free(part.binary);
+   ralloc_free(b.shader);
+   if (saved) {
+      setenv("ASAHI_M5_RQ_CANDIDATE_API", saved, 1);
+      free(saved);
+   } else {
+      unsetenv("ASAHI_M5_RQ_CANDIDATE_API");
+   }
+   printf("M5_DEAD_RAY_QUERY_PASS live=%u\n", live);
+}
+
+/* Which break left a query-selected loop is a query result: the phis after
+ * the loop merge one value per break, here constants. A store of such a phi
+ * keeps the query, as rayQueryProceedEXT's "found" flag after the traversal
+ * loop does in ray_query.builtin.getworldrayorigin.geom. */
+static void
+dead_ray_query_loop_exit(void)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_GEOMETRY, &agx_nir_options, "M5 query loop exit");
+   nir_variable *found =
+      nir_local_variable_create(b.impl, glsl_bool_type(), "found");
+   /* Defined before the loop, as the lowering's hoisted constants are. */
+   nir_def *yes = nir_imm_true(&b), *no = nir_imm_false(&b);
+   nir_push_loop(&b);
+   {
+      nir_def *q = nir_load_scs_thread_index_agx(&b);
+      nir_push_if(&b, nir_ieq_imm(&b, q, 0));
+      {
+         nir_store_var(&b, found, yes, 1);
+         nir_jump(&b, nir_jump_break);
+      }
+      nir_pop_if(&b, NULL);
+      nir_store_var(&b, found, no, 1);
+      nir_jump(&b, nir_jump_break);
+   }
+   nir_pop_loop(&b, NULL);
+   nir_store_global(&b, nir_b2i32(&b, nir_load_var(&b, found)),
+                    nir_imm_int64(&b, 0x10003010000ull), .align_mul = 4,
+                    .write_mask = 1);
+   nir_lower_vars_to_ssa(b.shader);
+   agx_nir_m5_remove_dead_ray_queries(b.shader);
+   unsigned queries = 0;
+   nir_foreach_block(block, b.impl) {
+      nir_foreach_instr(instr, block) {
+         queries += instr->type == nir_instr_type_intrinsic &&
+            nir_instr_as_intrinsic(instr)->intrinsic ==
+               nir_intrinsic_load_scs_thread_index_agx;
+      }
+   }
+   ralloc_free(b.shader);
+   if (queries != 1) {
+      fprintf(stderr, "M5 query feeding a loop-exit phi was removed\n");
+      exit(1);
+   }
+   printf("M5_DEAD_RAY_QUERY_LOOP_EXIT_PASS\n");
+}
+
+/* Native instance completion is a byte read, not a word read and shift. */
+static void
+instance_completion_byte(void)
+{
+ nir_builder b=nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,&agx_nir_options,"instance completion byte");
+ nir_def *base=nir_load_global(&b,1,32,nir_imm_int64(&b,0x10000),.align_mul=4);
+ nir_def *lane=nir_load_global(&b,1,32,nir_imm_int64(&b,0x10004),.align_mul=4);
+ nir_def *v=nir_load_scs_register_agx(&b,1,16,base,lane,
+    .base=263,.format=PIPE_FORMAT_R8_UINT,.component=3,.flags=9);
+ nir_store_global(&b,nir_u2u32(&b,v),nir_imm_int64(&b,0x20000),.align_mul=4,.write_mask=1);
+ agx_preprocess_nir(b.shader);
+ struct agx_shader_key key={.dev={.chip=AGX_CHIP_G17S,.encoding=AGX_ISA_ENCODING_AGX3C},.no_preamble=true};
+ struct agx_shader_part part={0};assert(agx_compile_shader_nir(b.shader,&key,&part));
+ unsigned found=0;
+ for(size_t off=part.info.main_offset;off<part.info.main_offset+part.info.main_size;off++)
+  for(unsigned n=2;n<=16&&off+n<=part.info.main_offset+part.info.main_size;n+=2){
+   struct agx3_scs_desc d;if(!agx3_scs_decode_load(AGX_ISA_ENCODING_AGX3C,part.binary+off,n,&d))continue;
+   assert(d.address_kind==AGX3_SCS_ADDRESS_REGISTER_INDEXED&&d.width==AGX3_MEM_W8&&
+          d.byte_offset==263&&d.address_selector==3&&d.component_mask==1&&d.c_load_marker&&(d.wait_mask&0x80));
+   found++;
+  }
+ assert(found==1);free(part.binary);ralloc_free(b.shader);
+ puts("M5_INSTANCE_COMPLETION_BYTE_PASS");
+}
+
+/* Directly reproduce the native entry's SR59/W16 read (rld h28). */
+static void
+scs_thread_identity(void)
+{
+ struct agx3_sr_desc d={.dest=28,.sr=59,.width=AGX3_SR_W16,.slot=3};
+ const uint8_t native[]={0xe4,0xea,0x10,0x46};uint8_t out[16],len=0;
+ assert(agx3_sr_encode(AGX_ISA_ENCODING_AGX3C,&d,out,&len));
+ assert(len==sizeof(native)&&!memcmp(out,native,len));
+ assert(agx3_sr_encode(AGX_ISA_ENCODING_AGX3B,&d,out,&len));
+ puts("M5_NATIVE_SCS_THREAD_ID_BYTES_PASS");
+}
+
+/* A completed singleton batch owns its fields until continuation. Check the
+ * lowering's actual NIR guards, resets and completion operations, rather than
+ * using fabricated SCS state. The hardware controls supply real candidates. */
+static bool
+inside_batch_completion_guard(nir_instr *instr)
+{
+   for (nir_cf_node *node = instr->block->cf_node.parent; node; node = node->parent) {
+      /* The query activation guard outside the batch loop is not a latch. */
+      if (node->type == nir_cf_node_loop)
+         return false;
+      if (node->type != nir_cf_node_if)
+         continue;
+      nir_def *cond = nir_cf_node_as_if(node)->condition.ssa;
+      if (nir_def_instr(cond)->type != nir_instr_type_alu)
+         continue;
+      nir_alu_instr *alu = nir_instr_as_alu(nir_def_instr(cond));
+      if (alu->op == nir_op_inot && alu->src[0].src.ssa->bit_size == 1) {
+         nir_def *input = alu->src[0].src.ssa;
+         while (nir_def_instr(input)->type == nir_instr_type_alu &&
+                nir_instr_as_alu(nir_def_instr(input))->op == nir_op_mov)
+            input = nir_instr_as_alu(nir_def_instr(input))->src[0].src.ssa;
+         if (nir_def_instr(input)->type == nir_instr_type_phi)
+            return true;
+      }
+   }
+   return false;
+}
+
+static void
+batch_completion_latch(unsigned count)
+{
+   const char *option = getenv("ASAHI_M5_RQ_CANDIDATE_API");
+   char *saved = option ? strdup(option) : NULL;
+   setenv("ASAHI_M5_RQ_CANDIDATE_API", "1", 1);
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,
+      &agx_nir_options, "M5 completed batch ownership");
+   b.shader->info.ray_queries = count;
+   b.shader->info.workgroup_size[0] = 32;
+   b.shader->info.workgroup_size[1] = b.shader->info.workgroup_size[2] = 1;
+   nir_variable *queries[2];
+   for (unsigned q = 0; q < count; ++q) {
+      queries[q] = nir_local_variable_create(b.impl, glsl_uint64_t_type(), "query");
+      queries[q]->data.ray_query = true;
+      for (unsigned epoch = 0; epoch < 2; ++epoch) {
+         nir_def *rq = &nir_build_deref_var(&b, queries[q])->def;
+         nir_rq_initialize(&b, rq, nir_imm_int64(&b, 0x10002000000ull),
+            nir_imm_int(&b, epoch ? 2 : 1), nir_imm_int(&b, 255),
+            nir_imm_vec3(&b, 0, 0, -1), nir_imm_float(&b, 0.001),
+            nir_imm_vec3(&b, 0, 0, 1), nir_imm_float(&b, 10));
+         nir_push_loop(&b);
+         nir_break_if(&b, nir_inot(&b, nir_rq_proceed(&b, 1, rq)));
+         nir_rq_confirm_intersection(&b, rq);
+         nir_pop_loop(&b, NULL);
+      }
+   }
+   const struct agx_m5_ray_query_layout layout = {
+      .payload_uniform = 152, .bvh_uniform = 96, .status_uniform = 134,
+      .launch_header = 0x10000000, .instance_traversal = true,
+   };
+   assert(agx_nir_lower_m5_ray_query(b.shader, &layout));
+   unsigned waited = 0, guarded_waits = 0, raw_status = 0;
+   nir_foreach_block(block, b.impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         if (intr->intrinsic == nir_intrinsic_load_scs_base_agx &&
+             nir_intrinsic_base(intr) == 7) {
+            if (nir_intrinsic_flags(intr) == 8) {
+               guarded_waits += inside_batch_completion_guard(instr);
+               waited++;
+            } else if (nir_intrinsic_flags(intr) == 0) raw_status++;
+         }
+      }
+   }
+   fprintf(stderr, "BATCH_COUNTS count=%u waited=%u guarded=%u raw=%u\n", count, waited, guarded_waits, raw_status);
+   assert((count == 1 ? waited == 2 : waited >= 4 * count) &&
+          guarded_waits == (count == 1 ? 2 : 0) &&
+          raw_status == (count == 1 ? 2 : 0));
+   agx_preprocess_nir(b.shader);
+   struct agx_shader_key key = {
+      .dev = {.chip = AGX_CHIP_G17S, .encoding = AGX_ISA_ENCODING_AGX3C},
+      .no_preamble = true,
+      .has_scratch = true,
+   };
+   struct agx_shader_part part = {0};
+   assert(agx_compile_shader_nir(b.shader, &key, &part));
+   assert(part.info.uses_ray_traversal && part.info.main_size);
+   free(part.binary);
+   ralloc_free(b.shader);
+   if (saved) { setenv("ASAHI_M5_RQ_CANDIDATE_API", saved, 1); free(saved); }
+   else unsetenv("ASAHI_M5_RQ_CANDIDATE_API");
+   printf("M5_BATCH_COMPLETION_LATCH_PASS count=%u\n", count);
+}
+
+int main(int argc,char **argv)
+{
+ setbuf(stdout,NULL);glsl_type_singleton_init_or_ref();const char *out=argc>1?argv[1]:NULL;
+ candidate_packet(out);
+ scs_thread_identity();
+ instance_completion_byte();
+ batch_completion_latch(1);
+ batch_completion_latch(2);
+ standard_ray_query(1, false, MESA_SHADER_COMPUTE);
+ standard_ray_query(1, false, MESA_SHADER_TESS_CTRL);
+ dead_ray_query(false);
+ dead_ray_query(true);
+ dead_ray_query_loop_exit();
+ standard_ray_query(1, false, MESA_SHADER_TESS_EVAL);
+ standard_ray_query(1, false, MESA_SHADER_GEOMETRY);
+ standard_ray_query(1, false, MESA_SHADER_VERTEX);
+ standard_ray_query(1, false, MESA_SHADER_FRAGMENT);
+ standard_ray_query(2, false, MESA_SHADER_COMPUTE);
+ standard_ray_query(8, false, MESA_SHADER_COMPUTE);
+ standard_ray_query(4, true, MESA_SHADER_COMPUTE);
+ standard_ray_query(8, true, MESA_SHADER_COMPUTE);
+ /* Candidate-API path (kernel stages): more logical queries than native
+  * slots, and dynamically selected query-array elements. */
+ standard_ray_query(9, false, MESA_SHADER_TESS_CTRL);
+ standard_ray_query(12, false, MESA_SHADER_TESS_CTRL);
+ standard_ray_query(2, true, MESA_SHADER_TESS_CTRL);
+ standard_ray_query(8, true, MESA_SHADER_TESS_CTRL);
+ standard_ray_query(16, true, MESA_SHADER_TESS_CTRL);
+ for(unsigned mode=0;mode<3;mode++)for(unsigned n=1;n<=4;n++)compile(mode,n,32,0,out);
+ compile(0,1,16,0,out);
+ for(unsigned mode=1;mode<=2;mode++)for(unsigned width=8;width<=16;width+=8)compile(mode,1,width,0,out);
+ for(unsigned width=8;width<=16;width+=8)compile(1,1,width,2,out);
+ /* The status conversion is unknown: exact encoding only, never run using
+  * fabricated query state. */
+ compile(3,1,8,5,NULL);
+ refused(PIPE_FORMAT_R8_UINT,4,4,0);  /* status requires marker */
+ refused(PIPE_FORMAT_R16_UINT,5,4,0); /* status is byte-only */
+ refused(PIPE_FORMAT_R16_UINT,0,5,0); /* invalid selector */
+ refused(PIPE_FORMAT_R16_UINT,0,4,1); /* unaligned access */
+ refused(PIPE_FORMAT_R32_FLOAT,0,4,0); /* unsupported memory format */
+ printf("Invalid SCS forms rejected transactionally\n");
+ glsl_type_singleton_decref();return 0;
+}
diff --git a/src/asahi/compiler/test/test-shared-byte-offsets.cpp b/src/asahi/compiler/test/test-shared-byte-offsets.cpp
new file mode 100644
index 000000000000..db80342d7f6f
--- /dev/null
+++ b/src/asahi/compiler/test/test-shared-byte-offsets.cpp
@@ -0,0 +1,136 @@
+/* Copyright 2026 Aurora Silicon; SPDX-License-Identifier: MIT */
+#include <gtest/gtest.h>
+#include "agx_test.h"
+#include "compiler/nir/nir_builder.h"
+extern "C" {
+#include "agx_nir.h"
+}
+
+static void
+check_offset(unsigned bits, unsigned add, int base, uint32_t mask,
+             bool store, bool expected)
+{
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_COMPUTE, &agx_nir_options, "shared-byte-offset");
+   nir_def *v = nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .range = 4);
+   v = nir_iand_imm(&b, v, mask);
+   nir_def *sum = nir_iadd_imm(&b, v, add);
+   nir_def *index = nir_u2u16(&b, nir_ushr_imm(&b, sum, util_logbase2(bits / 8)));
+   nir_intrinsic_instr *intr;
+   if (store) {
+      intr = nir_store_shared(&b, nir_imm_intN_t(&b, 0, bits), index,
+                              .base = base, .align_mul = bits / 8);
+   } else {
+      nir_def *value = nir_load_shared(&b, 1, bits, index, .base = base,
+                                        .align_mul = bits / 8);
+      intr = nir_def_as_intrinsic(value);
+   }
+   EXPECT_EQ(agx_nir_opt_shared_byte_offsets(b.shader), expected);
+   EXPECT_EQ(nir_intrinsic_base(intr), base + (expected ? add : 0));
+   EXPECT_EQ(nir_get_io_offset_src(intr)->ssa == index, !expected);
+   nir_validate_shader(b.shader, "shared-byte-offset result");
+   ralloc_free(b.shader);
+}
+
+TEST(SharedByteOffsets, Load32) { check_offset(32, 12, 0, 4095, false, true); }
+TEST(SharedByteOffsets, Store32) { check_offset(32, 56, 0, 4095, true, true); }
+TEST(SharedByteOffsets, Load16) { check_offset(16, 14, 0, 4095, false, true); }
+TEST(SharedByteOffsets, ExistingBase) { check_offset(32, 12, 16, 4095, false, true); }
+TEST(SharedByteOffsets, ByteOffsetTooLarge) { check_offset(32, 256, 0, 4095, false, false); }
+TEST(SharedByteOffsets, BaseOverflow) { check_offset(32, 16, 240, 4095, false, false); }
+TEST(SharedByteOffsets, Misaligned) { check_offset(32, 3, 0, 4095, false, false); }
+TEST(SharedByteOffsets, AdditionWrap) { check_offset(32, 12, 0, UINT32_MAX, false, false); }
+TEST(SharedByteOffsets, NarrowingWrap) { check_offset(32, 12, 0, 262143, false, false); }
+TEST(SharedByteOffsets, ExactBoundary) { check_offset(32, 12, 0, 262131, false, true); }
+
+static nir_builder
+workgroup_shader(const char *name)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE,
+                                                  &agx_nir_options, "%s", name);
+   b.shader->info.workgroup_size[0] = 8;
+   b.shader->info.workgroup_size[1] = 8;
+   b.shader->info.workgroup_size[2] = 1;
+   return b;
+}
+
+static unsigned
+count_alu(nir_shader *shader, nir_op op)
+{
+   unsigned n = 0;
+   nir_foreach_function_impl(impl, shader) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            n += instr->type == nir_instr_type_alu &&
+                 nir_instr_as_alu(instr)->op == op;
+         }
+      }
+   }
+   return n;
+}
+
+/* A byte offset that scales thread indices by multiples of the element size,
+ * including through address multiplies, becomes an element index without a
+ * shift per access. */
+TEST(SharedByteOffsets, ExactElementIndex)
+{
+   nir_builder b = workgroup_shader("shared-exact-index");
+   nir_def *id = nir_load_local_invocation_id(&b);
+   nir_def *x = nir_ishl_imm(&b, nir_channel(&b, id, 0), 1);
+   nir_def *y = nir_ishl_imm(&b, nir_channel(&b, id, 1), 1);
+   nir_def *offset = nir_iadd(&b, nir_amul(&b, x, nir_imm_int(&b, 72)),
+                              nir_amul(&b, y, nir_imm_int(&b, 4)));
+   nir_store_shared(&b, nir_imm_int(&b, 0), offset, .align_mul = 4);
+   EXPECT_TRUE(agx_nir_lower_shared_bitsize(b.shader, true));
+   EXPECT_EQ(count_alu(b.shader, nir_op_ushr), 0u);
+   nir_validate_shader(b.shader, "shared-exact-index result");
+   ralloc_free(b.shader);
+}
+
+/* An odd scale still needs the shift. */
+TEST(SharedByteOffsets, InexactElementIndex)
+{
+   nir_builder b = workgroup_shader("shared-inexact-index");
+   nir_def *id = nir_load_local_invocation_id(&b);
+   nir_def *offset = nir_amul(&b, nir_channel(&b, id, 0), nir_imm_int(&b, 6));
+   nir_store_shared(&b, nir_imm_int(&b, 0), offset, .align_mul = 4);
+   EXPECT_TRUE(agx_nir_lower_shared_bitsize(b.shader, true));
+   EXPECT_EQ(count_alu(b.shader, nir_op_ushr), 1u);
+   ralloc_free(b.shader);
+}
+
+static void
+check_element_addend(bool bounded, unsigned addend, unsigned expected_base)
+{
+   nir_builder b = workgroup_shader("shared-element-addend");
+   nir_def *id = nir_load_local_invocation_id(&b);
+   nir_def *x = nir_channel(&b, id, 0);
+   if (!bounded)
+      x = nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .range = 4);
+   nir_def *row = nir_imul_imm(&b, x, 36);
+   nir_def *col = nir_imadshl_agx(&b, nir_imm_int(&b, 0), nir_imm_int(&b, 1),
+                                  nir_channel(&b, id, 1), nir_imm_int(&b, 1));
+   nir_def *index =
+      nir_u2u16(&b, nir_iadd(&b, nir_iadd_imm(&b, row, addend), col));
+   nir_def *value = nir_load_shared(&b, 1, 32, index, .align_mul = 4);
+   nir_intrinsic_instr *intr = nir_def_as_intrinsic(value);
+   EXPECT_EQ(agx_nir_opt_shared_byte_offsets(b.shader), expected_base != 0);
+   EXPECT_EQ(nir_intrinsic_base(intr), expected_base);
+   if (expected_base) {
+      /* The constant left the index: u2u16(row + col). */
+      nir_scalar narrow = nir_scalar_resolved(intr->src[0].ssa, 0);
+      ASSERT_TRUE(nir_scalar_is_alu(narrow));
+      ASSERT_EQ(nir_scalar_alu_op(narrow), nir_op_u2u16);
+      nir_scalar sum = nir_scalar_chase_alu_src(narrow, 0);
+      ASSERT_TRUE(nir_scalar_is_alu(sum));
+      ASSERT_EQ(nir_scalar_alu_op(sum), nir_op_iadd);
+      for (unsigned s = 0; s < 2; ++s)
+         EXPECT_FALSE(nir_scalar_is_const(nir_scalar_chase_alu_src(sum, s)));
+   }
+   nir_validate_shader(b.shader, "shared-element-addend result");
+   ralloc_free(b.shader);
+}
+
+TEST(SharedByteOffsets, NestedElementAddend) { check_element_addend(true, 18, 72); }
+TEST(SharedByteOffsets, NestedElementAddendTooLarge) { check_element_addend(true, 64, 0); }
+TEST(SharedByteOffsets, NestedElementAddendUnbounded) { check_element_addend(false, 18, 0); }
diff --git a/src/asahi/compiler/test/test-sink-loop-sample-uses.c b/src/asahi/compiler/test/test-sink-loop-sample-uses.c
new file mode 100644
index 000000000000..c900b43d8ce5
--- /dev/null
+++ b/src/asahi/compiler/test/test-sink-loop-sample-uses.c
@@ -0,0 +1,361 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * agx_nir_opt_sink_loop_sample_uses: a loop body's sample consumers move to
+ * the latest top-level block dominating their uses; samples never move;
+ * results beyond the per-loop budget keep their consumers in place.
+ *
+ * agx_nir_opt_hoist_loop_samples: unconditional samples move ahead of the
+ * side-effect-free conditional blocks before them, which nir_opt_if_merge
+ * then merges.
+ */
+#include "agx_nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "compiler/glsl_types.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+static unsigned failures;
+
+#define CHECK(cond)                                                          \
+   do {                                                                      \
+      if (!(cond)) {                                                         \
+         fprintf(stderr, "%s:%d: CHECK(%s) failed\n", __FILE__, __LINE__,   \
+                 #cond);                                                     \
+         failures++;                                                         \
+      }                                                                      \
+   } while (0)
+
+static nir_def *
+txf3d(nir_builder *b, nir_def *c, unsigned bit_size)
+{
+   nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
+   tex->op = nir_texop_txf;
+   tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
+   tex->dest_type = bit_size == 16 ? nir_type_int16 : nir_type_int32;
+   tex->coord_components = 3;
+   tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord,
+                                     nir_vec3(b, c, nir_imm_int(b, 0),
+                                              nir_imm_int(b, 0)));
+   tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(b, 0));
+   nir_def_init(&tex->instr, &tex->def, 4, bit_size);
+   nir_builder_instr_insert(b, &tex->instr);
+   return &tex->def;
+}
+
+static nir_block *
+def_block(nir_def *def)
+{
+   return nir_def_block(def);
+}
+
+struct loop_shader {
+   nir_builder b;
+   nir_loop *loop;
+   nir_phi_instr *c_phi, *acc_phi;
+   nir_def *c, *acc;
+   bool early_break;
+};
+
+/* for (c = 100, acc = 0;; --c) { body; if (c - 1 < 0) break; } */
+static void
+loop_begin(struct loop_shader *s)
+{
+   s->early_break = false;
+   s->b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "loop");
+   nir_builder *b = &s->b;
+   nir_def *c0 = nir_imm_int(b, 100);
+   nir_def *acc0 = nir_imm_int(b, 0);
+   s->loop = nir_push_loop(b);
+   s->c_phi = nir_phi_instr_create(b->shader);
+   s->acc_phi = nir_phi_instr_create(b->shader);
+   nir_def_init(&s->c_phi->instr, &s->c_phi->def, 1, 32);
+   nir_def_init(&s->acc_phi->instr, &s->acc_phi->def, 1, 32);
+   nir_block *preheader = nir_cf_node_as_block(nir_cf_node_prev(&s->loop->cf_node));
+   nir_phi_instr_add_src(s->c_phi, preheader, c0);
+   nir_phi_instr_add_src(s->acc_phi, preheader, acc0);
+   nir_builder_instr_insert(b, &s->c_phi->instr);
+   nir_builder_instr_insert(b, &s->acc_phi->instr);
+   s->c = &s->c_phi->def;
+   s->acc = &s->acc_phi->def;
+}
+
+static void
+loop_end(struct loop_shader *s, nir_def *acc)
+{
+   nir_builder *b = &s->b;
+   nir_def *next = nir_iadd_imm(b, s->c, -1);
+   nir_push_if(b, nir_ilt_imm(b, next, 0));
+   nir_jump(b, nir_jump_break);
+   nir_pop_if(b, NULL);
+   /* Phi sources only join their definitions' use lists on insertion. */
+   nir_block *latch = nir_loop_last_block(s->loop);
+   nir_block *header = nir_loop_first_block(s->loop);
+   nir_instr_remove(&s->acc_phi->instr);
+   nir_instr_remove(&s->c_phi->instr);
+   nir_phi_instr_add_src(s->c_phi, latch, next);
+   nir_phi_instr_add_src(s->acc_phi, latch, acc);
+   nir_instr_insert(nir_before_block(header), &s->acc_phi->instr);
+   nir_instr_insert(nir_before_block(header), &s->c_phi->instr);
+   nir_pop_loop(b, s->loop);
+   /* Only the header phi dominates an early break's exit as well. */
+   nir_store_global(b, s->early_break ? s->acc : acc,
+                    nir_imm_int64(b, 0x1000), .write_mask = 1,
+                    .align_mul = 4);
+   nir_validate_shader(b->shader, "before");
+}
+
+/* The witness shape: sample, consume, conditional sample, consume, sample,
+ * consume. After the pass every consumer sits in the last top-level block
+ * (the one holding the final sample), and no sample moved. */
+static void
+test_witness_shape(void)
+{
+   struct loop_shader s;
+   loop_begin(&s);
+   nir_builder *b = &s.b;
+
+   nir_def *t0 = txf3d(b, s.c, 32);
+   nir_def *n0 = nir_ineg(b, nir_channel(b, t0, 0));
+   nir_def *a1 = nir_iadd(b, s.acc, n0);
+
+   nir_def *zero = nir_imm_int(b, 0);
+   nir_push_if(b, nir_ine_imm(b, s.c, 0));
+   nir_def *t1 = txf3d(b, s.c, 32);
+   nir_def *m1 = nir_imax(b, nir_channel(b, t1, 1), zero);
+   nir_push_else(b, NULL);
+   nir_pop_if(b, NULL);
+   nir_def *p1 = nir_if_phi(b, m1, zero);
+   nir_def *a2 = nir_iadd(b, a1, p1);
+   nir_def *keep = nir_ieq_imm(b, s.c, 7); /* independent of any sample */
+
+   nir_def *t2 = txf3d(b, s.c, 32);
+   nir_def *n2 = nir_ineg(b, nir_channel(b, t2, 2));
+   nir_def *a3 = nir_iadd(b, a2, n2);
+   nir_def *a4 = nir_bcsel(b, keep, a3, a2);
+   nir_block *b_t0 = def_block(t0), *b_t1 = def_block(t1), *b_t2 = def_block(t2);
+   nir_block *b_m1 = def_block(m1), *b_keep = def_block(keep);
+   CHECK(b_t0 != b_t2);
+   loop_end(&s, a4);
+
+   CHECK(agx_nir_opt_sink_loop_sample_uses(b->shader));
+   nir_validate_shader(b->shader, "after");
+
+   CHECK(def_block(t0) == b_t0);
+   CHECK(def_block(t1) == b_t1);
+   CHECK(def_block(t2) == b_t2);
+   CHECK(def_block(n0) == b_t2);
+   CHECK(def_block(a1) == b_t2);
+   CHECK(def_block(a2) == b_t2);
+   CHECK(def_block(a3) == b_t2);
+   CHECK(def_block(a4) == b_t2);
+   /* A consumer inside the conditional stays: its use is the phi. */
+   CHECK(def_block(m1) == b_m1);
+   /* Sample-independent ALU keeps its place. */
+   CHECK(def_block(keep) == b_keep);
+   /* Producers precede their consumers in the target block. */
+   CHECK(nir_block_dominates(def_block(n0), def_block(a1)));
+   bool seen_n0 = false, ordered = true;
+   nir_foreach_instr(instr, b_t2) {
+      if (instr == nir_def_instr(n0))
+         seen_n0 = true;
+      if (instr == nir_def_instr(a1) && !seen_n0)
+         ordered = false;
+   }
+   CHECK(ordered);
+   CHECK(!agx_nir_opt_sink_loop_sample_uses(b->shader));
+   ralloc_free(b->shader);
+}
+
+/* Thirteen 32-bit vec4 samples: the first twelve (96 halves) sink their
+ * consumers; the thirteenth's own consumer stays where it was. */
+static void
+test_budget(void)
+{
+   struct loop_shader s;
+   loop_begin(&s);
+   nir_builder *b = &s.b;
+   nir_def *neg[13];
+   nir_def *acc = s.acc;
+   for (unsigned i = 0; i < 13; ++i) {
+      nir_def *t = txf3d(b, s.c, 32);
+      neg[i] = nir_ineg(b, nir_channel(b, t, 0));
+      acc = nir_iadd(b, acc, neg[i]);
+      nir_push_if(b, nir_ine_imm(b, s.c, (int)i + 1));
+      nir_pop_if(b, NULL);
+   }
+   nir_block *before[13];
+   for (unsigned i = 0; i < 13; ++i)
+      before[i] = def_block(neg[i]);
+   nir_def *last = txf3d(b, s.c, 32);
+   acc = nir_iadd(b, acc, nir_channel(b, last, 3));
+   nir_block *tail = def_block(last);
+   loop_end(&s, acc);
+
+   CHECK(agx_nir_opt_sink_loop_sample_uses(b->shader));
+   nir_validate_shader(b->shader, "after");
+   for (unsigned i = 0; i < 12; ++i)
+      CHECK(def_block(neg[i]) == tail);
+   CHECK(def_block(neg[12]) == before[12]);
+   ralloc_free(b->shader);
+}
+
+/* No sample in the loop: nothing moves. */
+static void
+test_no_samples(void)
+{
+   struct loop_shader s;
+   loop_begin(&s);
+   nir_builder *b = &s.b;
+   nir_def *a1 = nir_iadd(b, s.acc, s.c);
+   nir_push_if(b, nir_ine_imm(b, s.c, 0));
+   nir_pop_if(b, NULL);
+   nir_def *a2 = nir_imul(b, a1, s.c);
+   nir_block *b1 = def_block(a1);
+   loop_end(&s, a2);
+   CHECK(!agx_nir_opt_sink_loop_sample_uses(b->shader));
+   CHECK(def_block(a1) == b1);
+   ralloc_free(b->shader);
+}
+
+/* Count the if-statements directly in the loop body. */
+static unsigned
+loop_body_ifs(nir_loop *loop)
+{
+   unsigned n = 0;
+   foreach_list_typed(nir_cf_node, node, node, &loop->body)
+      n += node->type == nir_cf_node_if;
+   return n;
+}
+
+static unsigned
+count_tex(nir_shader *shader)
+{
+   unsigned n = 0;
+   nir_foreach_function_impl(impl, shader) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block)
+            n += instr->type == nir_instr_type_tex;
+      }
+   }
+   return n;
+}
+
+/* The robustness2 shape: per axis an unconditional fetch at a coordinate
+ * derived from c, then a conditional fetch of texel 0 at level c. After the
+ * sink, every unconditional fetch and its coordinate math moves to the first
+ * block, the two conditional blocks become adjacent and merge, and their
+ * identical fetches CSE. */
+static void
+test_hoist_witness_shape(void)
+{
+   struct loop_shader s;
+   loop_begin(&s);
+   nir_builder *b = &s.b;
+   nir_def *cond = nir_ine_imm(b, s.c, 0);
+   nir_def *zero = nir_imm_int(b, 0);
+
+   nir_def *t0 = txf3d(b, s.c, 32);
+   nir_def *acc = nir_iadd(b, s.acc, nir_channel(b, t0, 0));
+   nir_def *p[2], *t[2], *coord[2];
+   for (unsigned i = 0; i < 2; ++i) {
+      nir_push_if(b, cond);
+      nir_def *f = txf3d(b, zero, 32);
+      nir_def *m = nir_imax(b, nir_channel(b, f, 1), zero);
+      nir_push_else(b, NULL);
+      nir_pop_if(b, NULL);
+      p[i] = nir_if_phi(b, m, zero);
+      acc = nir_iadd(b, acc, p[i]);
+      coord[i] = nir_iadd_imm(b, s.c, (int)i + 1);
+      t[i] = txf3d(b, coord[i], 32);
+      acc = nir_iadd(b, acc, nir_channel(b, t[i], 2));
+   }
+   nir_block *first = def_block(t0);
+   CHECK(def_block(t[0]) != first && def_block(t[1]) != first);
+   loop_end(&s, acc);
+   CHECK(loop_body_ifs(s.loop) == 3);
+   CHECK(count_tex(b->shader) == 5);
+
+   CHECK(agx_nir_opt_sink_loop_sample_uses(b->shader));
+   CHECK(agx_nir_opt_hoist_loop_samples(b->shader));
+   nir_validate_shader(b->shader, "after hoist");
+   CHECK(def_block(t[0]) == first);
+   CHECK(def_block(t[1]) == first);
+   CHECK(def_block(coord[0]) == first);
+   CHECK(def_block(coord[1]) == first);
+   /* The samples keep their source order. */
+   bool seen_t0 = false, seen_t1 = false, ordered = true;
+   nir_foreach_instr(instr, first) {
+      if (instr == nir_def_instr(t0))
+         seen_t0 = true;
+      if (instr == nir_def_instr(t[0])) {
+         ordered &= seen_t0;
+         seen_t1 = true;
+      }
+      if (instr == nir_def_instr(t[1]))
+         ordered &= seen_t1;
+   }
+   CHECK(ordered);
+
+   CHECK(nir_opt_if_merge(b->shader));
+   nir_validate_shader(b->shader, "after merge");
+   CHECK(loop_body_ifs(s.loop) == 2); /* the merged block and the break */
+   nir_opt_cse(b->shader);
+   nir_opt_dce(b->shader);
+   nir_validate_shader(b->shader, "after cse");
+   CHECK(count_tex(b->shader) == 4);
+
+   CHECK(!agx_nir_opt_hoist_loop_samples(b->shader));
+   ralloc_free(b->shader);
+}
+
+/* A conditional block with a side effect, or one leaving the loop, keeps the
+ * samples after it in place. */
+static void
+test_hoist_barriers(void)
+{
+   for (unsigned store = 0; store < 2; ++store) {
+      struct loop_shader s;
+      loop_begin(&s);
+      s.early_break = !store;
+      nir_builder *b = &s.b;
+      nir_def *t0 = txf3d(b, s.c, 32);
+      nir_push_if(b, nir_ine_imm(b, s.c, 3));
+      if (store) {
+         nir_store_global(b, s.c, nir_imm_int64(b, 0x2000), .write_mask = 1,
+                          .align_mul = 4);
+      } else {
+         nir_jump(b, nir_jump_break);
+      }
+      nir_pop_if(b, NULL);
+      nir_def *t1 = txf3d(b, nir_iadd_imm(b, s.c, 1), 32);
+      nir_def *acc = nir_iadd(b, s.acc, nir_channel(b, t0, 0));
+      acc = nir_iadd(b, acc, nir_channel(b, t1, 0));
+      nir_block *b_t1 = def_block(t1);
+      loop_end(&s, acc);
+
+      agx_nir_opt_sink_loop_sample_uses(b->shader);
+      CHECK(!agx_nir_opt_hoist_loop_samples(b->shader));
+      CHECK(def_block(t1) == b_t1);
+      ralloc_free(b->shader);
+   }
+}
+
+int
+main(void)
+{
+   glsl_type_singleton_init_or_ref();
+   test_witness_shape();
+   test_budget();
+   test_no_samples();
+   test_hoist_witness_shape();
+   test_hoist_barriers();
+   glsl_type_singleton_decref();
+   if (failures) {
+      fprintf(stderr, "%u check(s) failed\n", failures);
+      return EXIT_FAILURE;
+   }
+   printf("agx_nir_opt_sink_loop_sample_uses: all checks passed\n");
+   return EXIT_SUCCESS;
+}
diff --git a/src/asahi/drm-shim/asahi_noop.c b/src/asahi/drm-shim/asahi_noop.c
index ec957799c63a..2beb6d6fdf17 100644
--- a/src/asahi/drm-shim/asahi_noop.c
+++ b/src/asahi/drm-shim/asahi_noop.c
@@ -59,6 +59,127 @@ static const struct drm_asahi_params_global params_g13g = {
    .gpu_hal_generation = DRM_ASAHI_GPU_HAL_LEGACY,
 };
 
+/* Compile-only M1 Pro, M2 and M4 Pro profiles. Timing and submission
+ * ceilings deliberately use the existing fixtures, not hardware measurements.
+ * The product identity selects the real driver/compiler contract; T6040's
+ * two ten-core clusters exercise its larger helper tables (Mesa PR #55).
+ * No RT capabilities are invented for the M4 firmware runtime.
+ */
+static const struct drm_asahi_params_global params_g13x = {
+   .features = 0,
+   .gpu_generation = 13,
+   .gpu_variant = 'S',
+   .gpu_revision = 0,
+   .chip_id = 0x6000,
+   .num_dies = 1,
+   .num_clusters_total = 2,
+   .num_cores_per_cluster = 8,
+   .core_masks = {0xff, 0xff},
+   .vm_start = 0x4000,
+   .vm_end = 0x7fffff8000ull,
+   .vm_kernel_min_size = 0x20000000,
+   .max_commands_per_submission = 1,
+   .max_attachments = 16,
+   .max_frequency_khz = 1278000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 2,
+   .gpu_hal_generation = DRM_ASAHI_GPU_HAL_LEGACY,
+};
+
+static const struct drm_asahi_params_global params_g14g = {
+   .features = 0,
+   .gpu_generation = 14,
+   .gpu_variant = 'G',
+   .gpu_revision = 0,
+   .chip_id = 0x8112,
+   .num_dies = 1,
+   .num_clusters_total = 1,
+   .num_cores_per_cluster = 10,
+   .core_masks = {0x3ff},
+   .vm_start = 0x4000,
+   .vm_end = 0x7fffff8000ull,
+   .vm_kernel_min_size = 0x20000000,
+   .max_commands_per_submission = 1,
+   .max_attachments = 16,
+   .max_frequency_khz = 1278000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 2,
+   .gpu_hal_generation = DRM_ASAHI_GPU_HAL_LEGACY,
+};
+
+static const struct drm_asahi_params_global params_g16s = {
+   .features = DRM_ASAHI_FEATURE_SOFT_FAULTS,
+   .gpu_generation = 16,
+   .gpu_variant = 'S',
+   .gpu_revision = 0x11,
+   .chip_id = 0x6040,
+   .num_dies = 1,
+   .num_clusters_total = 2,
+   .num_cores_per_cluster = 10,
+   .core_masks = {0x3ff, 0x3ff},
+   .vm_start = 0x4000,
+   .vm_end = (1ull << 42) - 0x8000ull,
+   .vm_kernel_min_size = 0x20000000,
+   .max_commands_per_submission = 64,
+   .max_attachments = 16,
+   .max_frequency_khz = 1500000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 3,
+   .gpu_hal_generation = DRM_ASAHI_GPU_HAL_200,
+};
+
+/* An explicit capability-bearing fixture tests RT exposure and eight queues.
+ * The plain g16s fixture remains a negative control for the kernel RT gate.
+ * Neither fixture is a claim that those commands executed on hardware. */
+static const struct drm_asahi_params_global params_g16s_rt = {
+   .features = DRM_ASAHI_FEATURE_SOFT_FAULTS |
+               DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS |
+               DRM_ASAHI_FEATURE_RENDER_RAY_QUERY |
+               DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES,
+   .gpu_generation = 16,
+   .gpu_variant = 'S',
+   .gpu_revision = 0x11,
+   .chip_id = 0x6040,
+   .num_dies = 1,
+   .num_clusters_total = 2,
+   .num_cores_per_cluster = 10,
+   .core_masks = {0x3ff, 0x3ff},
+   .vm_start = 0x4000,
+   .vm_end = (1ull << 42) - 0x8000ull,
+   .vm_kernel_min_size = 0x20000000,
+   .max_commands_per_submission = 64,
+   .max_attachments = 16,
+   .max_frequency_khz = 1500000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 3,
+   .gpu_hal_generation = DRM_ASAHI_GPU_HAL_200,
+};
+
+/* The AS-only contract deliberately lacks the traversal capability. */
+static const struct drm_asahi_params_global params_g16s_as = {
+   .features = DRM_ASAHI_FEATURE_SOFT_FAULTS |
+               DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS |
+               DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD |
+               DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES,
+   .gpu_generation = 16,
+   .gpu_variant = 'S',
+   .gpu_revision = 0x11,
+   .chip_id = 0x6040,
+   .num_dies = 1,
+   .num_clusters_total = 2,
+   .num_cores_per_cluster = 10,
+   .core_masks = {0x3ff, 0x3ff},
+   .vm_start = 0x4000,
+   .vm_end = (1ull << 42) - 0x8000ull,
+   .vm_kernel_min_size = 0x20000000,
+   .max_commands_per_submission = 64,
+   .max_attachments = 16,
+   .max_frequency_khz = 1500000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 3,
+   .gpu_hal_generation = DRM_ASAHI_GPU_HAL_200,
+};
+
 /*
  * T8140 / A18 Pro, exactly as our G17P kernel adapter reports it
  * (drivers/gpu/drm/asahi/g17_drm.rs:24-27,118-140 and file.rs:295-324): a
@@ -219,12 +340,55 @@ static const struct drm_asahi_queue_limits queue_limits_g15s = {
  * and refuses user binds there. */
 #define SHIM_G15S_KERNEL_VA_START UINT64_C(0x7000000000)
 #define SHIM_G15S_KERNEL_VA_END   UINT64_C(0x7100000000)
+/* Explicit offline M5 Pro identity for command-generation tests. */
+static const struct drm_asahi_params_global params_g17s = {
+   .features = DRM_ASAHI_FEATURE_SOFT_FAULTS | DRM_ASAHI_FEATURE_RENDER_RESOURCES |
+               DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2,
+   .gpu_generation = 17, .gpu_variant = 'S', .gpu_revision = 0,
+   .chip_id = 0x6050, .num_dies = 1, .num_clusters_total = 2,
+   .num_cores_per_cluster = 10, .core_masks = {0x3ff, 0x3ff},
+   .vm_start = 0x4000, .vm_end = (1ull << 42) - 0x8000ull,
+   .vm_kernel_min_size = 0x20000000, .max_commands_per_submission = 64,
+   .max_attachments = 16, .max_frequency_khz = 1500000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 3, .gpu_hal_generation = DRM_ASAHI_GPU_HAL_300,
+};
+
+/* M5 Pro with the task (object->mesh) mesh rendering buffer bound
+ * (m5_mesh_storage_chunks != 0): DRM_ASAHI_FEATURE_TASK_MESH_STORAGE. */
+static const struct drm_asahi_params_global params_g17s_task = {
+   .features = DRM_ASAHI_FEATURE_SOFT_FAULTS | DRM_ASAHI_FEATURE_RENDER_RESOURCES |
+               DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2 |
+               DRM_ASAHI_FEATURE_COMPUTE_RESOURCES |
+               DRM_ASAHI_FEATURE_TASK_MESH_STORAGE,
+   .gpu_generation = 17, .gpu_variant = 'S', .gpu_revision = 0,
+   .chip_id = 0x6050, .num_dies = 1, .num_clusters_total = 2,
+   .num_cores_per_cluster = 10, .core_masks = {0x3ff, 0x3ff},
+   .vm_start = 0x4000, .vm_end = (1ull << 42) - 0x8000ull,
+   .vm_kernel_min_size = 0x20000000, .max_commands_per_submission = 64,
+   .max_attachments = 16, .max_frequency_khz = 1500000,
+   .command_timestamp_frequency_hz = 1000000000,
+   .usc_generation = 3, .gpu_hal_generation = DRM_ASAHI_GPU_HAL_300,
+};
 
 static const struct drm_asahi_params_global *
 shim_params(void)
 {
    const char *chip = getenv("ASAHI_SHIM_CHIP");
 
+   if (chip && !strcmp(chip, "g17s-task"))
+      return &params_g17s_task;
+
+   if (chip && !strcmp(chip, "g13x"))
+      return &params_g13x;
+   if (chip && !strcmp(chip, "g14g"))
+      return &params_g14g;
+   if (chip && !strcmp(chip, "g16s-as"))
+      return &params_g16s_as;
+   if (chip && !strcmp(chip, "g16s-rt"))
+      return &params_g16s_rt;
+   if (chip && !strcmp(chip, "g16s"))
+      return &params_g16s;
    if (chip && !strcmp(chip, "g17p"))
       return &params_g17p;
    if (chip && !strcmp(chip, "g15s"))
@@ -235,6 +399,8 @@ shim_params(void)
       return &params_g15c;
    if (chip && !strcmp(chip, "g16g"))
       return &params_g16g;
+   if (chip && !strcmp(chip, "g17s"))
+      return &params_g17s;
 
    return &params_g13g;
 }
@@ -911,14 +1077,16 @@ dump_submit(int fd, const struct drm_asahi_submit *submit, unsigned index)
          }
          h = dump_hash_bytes(h, chunk, len);
       }
-      fprintf(f, "MAP %#llx size=%#llx single=%u hash=%016llx\n",
+      fprintf(f, "MAP %#llx size=%#llx single=%u hash=%016llx backing=%#llx offset=%#llx\n",
               (unsigned long long)m->address, (unsigned long long)m->size,
-              m->single_page, (unsigned long long)h);
+              m->single_page, (unsigned long long)h,
+              (unsigned long long)m->bo->mem_addr,
+              (unsigned long long)m->object_offset);
 
       /* ASAHI_SHIM_DUMP_BYTES=<hex address>: also write that mapping's raw
        * bytes, one file per submit, next to the dump. */
       const char *want = getenv("ASAHI_SHIM_DUMP_BYTES");
-      if (want && strtoull(want, NULL, 16) == m->address) {
+      if (want && (!strcmp(want, "all") || strtoull(want, NULL, 16) == m->address)) {
          char path[4096];
          snprintf(path, sizeof(path), "%s/bytes-%llx-%u.bin",
                   getenv("ASAHI_SHIM_DUMP_SUBMIT"),
@@ -1182,6 +1350,19 @@ asahi_ioctl_get_param(int fd, unsigned long request, void *arg)
       return asahi_ioctl_vm_status_mirror(fd, gp);
 
    case DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS:
+      if (shim_params() == &params_g16s_rt ||
+          shim_params() == &params_g16s_as) {
+         const struct drm_asahi_queue_limits limits = {
+            .max_queues = 8, .max_in_flight_per_queue = 2,
+         };
+         if (gp->size < sizeof(limits)) {
+            errno = EINVAL;
+            return -1;
+         }
+         memcpy((void *)(uintptr_t)gp->pointer, &limits, sizeof(limits));
+         gp->size = sizeof(limits);
+         return 0;
+      }
       if (!shim_g15s() || gp->size < sizeof(queue_limits_g15s)) {
          errno = EINVAL;
          return -1;
@@ -1801,12 +1982,29 @@ drm_shim_driver_init(void)
                           render_node_minor);
 
    const char *chip = getenv("ASAHI_SHIM_CHIP");
+   const bool g13x = chip && !strcmp(chip, "g13x");
+   const bool g14g = chip && !strcmp(chip, "g14g");
+   const bool g16s = chip &&
+      (!strcmp(chip, "g16s") || !strcmp(chip, "g16s-rt") ||
+       !strcmp(chip, "g16s-as"));
    const bool g17p = chip && !strcmp(chip, "g17p");
    const bool g15s = chip && !strcmp(chip, "g15s");
    const bool g15c = chip && !strcmp(chip, "g15c");
    const bool g16g = chip && !strcmp(chip, "g16g");
 
-   drm_shim_override_file(g17p   ? "DRIVER=asahi\n"
+   drm_shim_override_file(g13x   ? "DRIVER=asahi\n"
+                                   "OF_FULLNAME=/soc/agx\n"
+                                   "OF_COMPATIBLE_0=apple,agx-g13s\n"
+                                   "OF_COMPATIBLE_N=1\n"
+                          : g14g ? "DRIVER=asahi\n"
+                                   "OF_FULLNAME=/soc/agx\n"
+                                   "OF_COMPATIBLE_0=apple,agx-g14g\n"
+                                   "OF_COMPATIBLE_N=1\n"
+                          : g16s ? "DRIVER=asahi\n"
+                                   "OF_FULLNAME=/soc/agx\n"
+                                   "OF_COMPATIBLE_0=apple,gpu-g16s\n"
+                                   "OF_COMPATIBLE_N=1\n"
+                          : g17p ? "DRIVER=asahi\n"
                                    "OF_FULLNAME=/soc/agx\n"
                                    "OF_COMPATIBLE_0=apple,gpu-g17p\n"
                                    "OF_COMPATIBLE_N=1\n"
diff --git a/src/asahi/drm-shim/meson.build b/src/asahi/drm-shim/meson.build
index 9f5bbbbb25cc..80821e6c18d0 100644
--- a/src/asahi/drm-shim/meson.build
+++ b/src/asahi/drm-shim/meson.build
@@ -15,6 +15,7 @@ libasahi_noop_drm_shim = shared_library(
     'agx_vdm_decode_core.c',
     'agx_vdm_hal200_bind_tail_core.c',
     'agx_vdm_index_list_core.c',
+    'agx_vdm_mesh_core.c',
     'agx_vdm_ppp_state_core.c',
     'agx_vdm_state_core.c',
     'agx_vdm_state_update_core.c',
diff --git a/src/asahi/genxml/agx3_cmdstream.h b/src/asahi/genxml/agx3_cmdstream.h
index 5a889a233850..f6e00946e455 100644
--- a/src/asahi/genxml/agx3_cmdstream.h
+++ b/src/asahi/genxml/agx3_cmdstream.h
@@ -94,6 +94,9 @@ struct agx3_cdm_barrier_info {
     * 0x600215fe for a nonzero visibility: the ordinary word with 0x2009e
     * set. */
    bool wide_visibility;
+   /* HAL300 memory visibility also requires the low cache-scope bits.
+    * The ordering-only record leaves these clear. */
+   bool flush_memory;
 };
 
 struct agx3_cdm_substream_barrier_info {
@@ -625,10 +628,15 @@ agx3_cdm_barrier_pack(const struct agx3_cdm_barrier_info *info,
         !agx3_cdm_barrier_wide_visibility_grounded(info->chip)))
       return false;
 
+   const bool gen7 = info->chip == AGX_CHIP_G17G ||
+                     info->chip == AGX_CHIP_G17S || info->chip == AGX_CHIP_G17C;
+   if (info->flush_memory && (!gen7 || info->flags != 5))
+      return false;
    agx3_cmdstream_store_le32(
       packed, agx3_cdm_barrier_word(info->flags) |
                  (info->wide_visibility ? AGX3_CDM_BARRIER_WIDE_VISIBILITY
-                                        : 0));
+                                        : 0) |
+                 (info->flush_memory ? 0x1c : 0));
    return true;
 }
 
@@ -654,7 +662,13 @@ agx3_cdm_barrier_unpack(enum agx_chip chip, const void *packed,
       word &= ~AGX3_CDM_BARRIER_WIDE_VISIBILITY;
    }
 
-   const uint32_t optional_mask = UINT32_C(0x00001c02);
+   const bool flush_memory = (word & 0x1c) == 0x1c;
+   const bool gen7 = chip == AGX_CHIP_G17G || chip == AGX_CHIP_G17S ||
+                     chip == AGX_CHIP_G17C;
+   if ((word & 0x1c) && (!gen7 || !flush_memory))
+      return false;
+   const uint32_t optional_mask = UINT32_C(0x00001c02) |
+                                 (flush_memory ? 0x1c : 0);
    if ((word & ~optional_mask) != UINT32_C(0x60000160))
       return false;
 
@@ -665,13 +679,16 @@ agx3_cdm_barrier_unpack(enum agx_chip chip, const void *packed,
    if (!agx3_cdm_barrier_flags_grounded(chip, flags))
       return false;
 
+   if (flush_memory && flags != 5)
+      return false;
    info->chip = chip;
    info->flags = flags;
    info->wide_visibility = wide_visibility;
+   info->flush_memory = flush_memory;
    return true;
 }
 
-/* HAL200's encodeSSTTFlushInvalidate emits this standalone record after the
+/* HAL200 and HAL300 encodeSSTTFlushInvalidate emit this standalone record after the
  * CPU changes a sparse texture translation table. It is not an ordinary CDM
  * barrier: the low control byte and SSTT selector at bit 14 are distinct. */
 static inline bool
@@ -683,7 +700,7 @@ agx3_cdm_sstt_flush_invalidate_pack(
       return false;
 
    memset(packed, 0, AGX3_CDM_SSTT_FLUSH_INVALIDATE_SIZE);
-   if (chip != AGX_CHIP_G17P)
+   if (chip != AGX_CHIP_G17P && !agx_chip_g17_m5(chip))
       return false;
 
    agx3_cmdstream_store_le32(packed, UINT32_C(0x60024102));
@@ -695,7 +712,7 @@ agx3_cdm_sstt_flush_invalidate_validate(enum agx_chip chip,
                                         const void *packed,
                                         size_t available)
 {
-   return chip == AGX_CHIP_G17P && packed &&
+   return (chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip)) && packed &&
           available >= AGX3_CDM_SSTT_FLUSH_INVALIDATE_SIZE &&
           agx3_cmdstream_load_le32((const uint8_t *)packed) ==
              UINT32_C(0x60024102);
diff --git a/src/asahi/genxml/agx_pack_header.h b/src/asahi/genxml/agx_pack_header.h
index 3700ede2a985..914b61405839 100644
--- a/src/asahi/genxml/agx_pack_header.h
+++ b/src/asahi/genxml/agx_pack_header.h
@@ -47,11 +47,19 @@ __gen_pack_lod(float f, uint32_t start, uint32_t end)
    return util_bitpack_uint(fixed, start, end);
 }
 
-/* HAL300 raises the sampler maximum LOD clamp from 14.0 to 15.0. */
+/* HAL300 raises the sampler maximum LOD clamp from 14.0 to 15.0.
+ *
+ * The hardware selects the minification filter only for a clamped lambda
+ * above zero, so a positive clamp must not truncate to 0: minLod = 0.01
+ * packed as 0 turned an explicit-LOD-0 lookup into a magnified one, where
+ * Vulkan clamps lambda to 0.01 first and minifies. Clamps below one 1/64
+ * step become one step; every other value packs as before. */
 static inline uint32_t
 __gen_pack_lod15(float f, uint32_t start, uint32_t end)
 {
    uint32_t fixed = CLAMP(f * (1 << 6), 0 /* 0.0 */, 0x3c0 /* 15.0 */);
+   if (fixed == 0 && f > 0.0f)
+      fixed = 1;
    return util_bitpack_uint(fixed, start, end);
 }
 
diff --git a/src/asahi/genxml/tests/test-agx3-packing.c b/src/asahi/genxml/tests/test-agx3-packing.c
index 22d5c2aece90..05acb2a88feb 100644
--- a/src/asahi/genxml/tests/test-agx3-packing.c
+++ b/src/asahi/genxml/tests/test-agx3-packing.c
@@ -5,6 +5,7 @@
 
 #include "asahi/genxml/agx3_cmdstream.h"
 #include "asahi/libagx/libagx_dgc.h"
+#include "asahi/lib/agx_m5_va.h"
 #include "agx_pack.h"
 
 #include <stdio.h>
@@ -571,6 +572,23 @@ test_agx3_live_vdm_draw(void)
    static const uint32_t empty_jump_without_state[] = {
       UINT32_C(0x8000005a), UINT32_C(0x12345678),
    };
+   for (enum agx_chip chip = AGX_CHIP_G17G; chip <= AGX_CHIP_G17C; ++chip) {
+      uint64_t compact;
+      CHECK_EQ(agx_m5_vdm_compact(UINT64_C(0x5a12345678), &compact), true);
+      CHECK_EQ(compact, UINT64_C(0x4a12345678));
+      for (unsigned count = 0; count < 2; ++count) {
+         struct agx_draw empty = robust_draw;
+         empty.b.count[count] = 0;
+         memset(robust_packed, 0xcd, sizeof(robust_packed));
+         robust_end = agx_vdm_draw_robust_index(
+            robust_packed, chip, empty, (enum agx_primitive)9,
+            UINT32_MAX, UINT32_C(0x48000), true, compact);
+         CHECK_EQ(robust_end - robust_packed, 2);
+         CHECK_EQ(robust_packed[0], UINT32_C(0x8000004a));
+         CHECK_EQ(robust_packed[1], UINT32_C(0x12345678));
+         CHECK_EQ(robust_packed[2], UINT32_C(0xcdcdcdcd));
+      }
+   }
    memset(robust_packed, 0xcd, sizeof(robust_packed));
    robust_end = agx_vdm_draw_robust_index(
       robust_packed, AGX_CHIP_G17P, empty_draw,
@@ -631,11 +649,11 @@ test_agx3_live_vdm_draw(void)
    CHECK_EQ(
       agx_vdm_draw_supported(AGX_CHIP_G17S, rejected, (enum agx_primitive)6),
       false);
-   rejected = agx_draw_indexed(1, 1, 0, 0, 0, UINT64_C(0x1000), 4,
+   struct agx_draw byte_indexed = agx_draw_indexed(1, 1, 0, 0, 0, UINT64_C(0x1000), 4,
                                AGX_INDEX_SIZE_U8, false);
    CHECK_EQ(
-      agx_vdm_draw_supported(AGX_CHIP_G17S, rejected, (enum agx_primitive)6),
-      false);
+      agx_vdm_draw_supported(AGX_CHIP_G17S, byte_indexed, (enum agx_primitive)6),
+      true);
 }
 
 static void
@@ -866,6 +884,22 @@ test_agx3_cdm_barrier(void)
       }
    }
 
+   /* Native HAL300 full-memory record. Ordering flags alone do not make
+    * writes visible through another virtual mapping of the same memory. */
+   for (unsigned i = 0; i < ARRAY_SIZE(gen7); ++i) {
+      struct agx3_cdm_barrier_info full = {
+         .chip = gen7[i], .flags = 5, .flush_memory = true,
+      }, decoded;
+      uint8_t packed[AGX3_CDM_BARRIER_SIZE];
+      CHECK_EQ(agx3_cdm_barrier_pack(&full, packed), true);
+      CHECK_EQ(agx3_cmdstream_load_le32(packed), 0x6000157c);
+      CHECK_EQ(agx3_cdm_barrier_unpack(full.chip, packed, sizeof(packed), &decoded), true);
+      CHECK_EQ(decoded.flush_memory, true);
+      CHECK_EQ(decoded.flags, 5);
+      full.chip = AGX_CHIP_G17P;
+      CHECK_EQ(agx3_cdm_barrier_pack(&full, packed), false);
+   }
+
    const enum agx_chip unsupported[] = {
       AGX_CHIP_G13G, AGX_CHIP_G15P, AGX_CHIP_G16P, AGX_CHIP_G17A,
       AGX_CHIP_UNKNOWN,
diff --git a/src/asahi/layout/layout.c b/src/asahi/layout/layout.c
index 6daf904a5868..27c4b06c8337 100644
--- a/src/asahi/layout/layout.c
+++ b/src/asahi/layout/layout.c
@@ -176,7 +176,15 @@ ail_initialize_gpu_tiled(struct ail_layout *layout)
       layout->layer_stride_B =
          ail_m5_layer_stride_B(offset_B, layout->depth_px, layout->compressed);
       layout->page_aligned_layers =
-         layout->depth_px > 1 && offset_B > AIL_PAGESIZE;
+         layout->depth_px > 1 && offset_B >= AIL_PAGESIZE;
+      /* ZLS layer strides use page units even when color/texture arrays can
+       * use compact strides. A sub-page depth/stencil layer cannot be
+       * represented by the render submission's ZLS descriptor. */
+      if (layout->depth_px > 1 && layout->renderable &&
+          util_format_is_depth_or_stencil(layout->format)) {
+         layout->page_aligned_layers = true;
+         layout->layer_stride_B = ALIGN_POT(offset_B, AIL_PAGESIZE);
+      }
    } else {
       /* Align layer size if we have mipmaps and one miptree is larger than one
        * page */
@@ -200,6 +208,13 @@ ail_initialize_gpu_tiled(struct ail_layout *layout)
          layout->layer_stride_B = offset_B;
    }
 
+   /* Each sparse layer has its own page-map range. Include the partial
+    * final page so mip-tail entries cannot be mistaken for the next layer. */
+   if (layout->sparse_residency) {
+      layout->page_aligned_layers = true;
+      layout->layer_stride_B = ALIGN_POT(layout->layer_stride_B, AIL_PAGESIZE);
+   }
+
    layout->size_B = (uint64_t)layout->layer_stride_B * layout->depth_px;
 }
 
diff --git a/src/asahi/layout/layout.h b/src/asahi/layout/layout.h
index fa5f581e3c5a..bc242a580414 100644
--- a/src/asahi/layout/layout.h
+++ b/src/asahi/layout/layout.h
@@ -71,6 +71,9 @@ struct ail_layout {
     */
    bool mipmapped_z;
 
+   /** Sparse residency maps address whole pages within each layer. */
+   bool sparse_residency;
+
    /** Tiling mode used */
    enum ail_tiling tiling;
 
@@ -340,17 +343,22 @@ ail_get_tiled_element_B(const struct ail_layout *layout, unsigned level,
              ail_get_blocksize_B(layout);
 }
 
+/* The samples one compression metadata entry covers. */
+static inline struct ail_tile
+ail_metadata_tile(const struct ail_layout *layout)
+{
+   return layout->policy == AIL_LAYOUT_POLICY_M5 && !layout->mipmapped_z
+             ? ail_m5_compression_tile_size(
+                  util_format_get_blocksize(layout->format))
+             : (struct ail_tile){16, 16};
+}
+
 static inline unsigned
 ail_metadata_width_tl(struct ail_layout *layout, unsigned level)
 {
    unsigned px = u_minify(layout->width_px, level);
    uint32_t sa = ail_effective_width_sa(px, layout->sample_count_sa);
-   const struct ail_tile tile =
-      layout->policy == AIL_LAYOUT_POLICY_M5 && !layout->mipmapped_z
-         ? ail_m5_compression_tile_size(
-              util_format_get_blocksize(layout->format))
-         : (struct ail_tile){16, 16};
-   return DIV_ROUND_UP(sa, tile.width_el);
+   return DIV_ROUND_UP(sa, ail_metadata_tile(layout).width_el);
 }
 
 static inline unsigned
@@ -358,12 +366,7 @@ ail_metadata_height_tl(struct ail_layout *layout, unsigned level)
 {
    unsigned px = u_minify(layout->height_px, level);
    uint32_t sa = ail_effective_height_sa(px, layout->sample_count_sa);
-   const struct ail_tile tile =
-      layout->policy == AIL_LAYOUT_POLICY_M5 && !layout->mipmapped_z
-         ? ail_m5_compression_tile_size(
-              util_format_get_blocksize(layout->format))
-         : (struct ail_tile){16, 16};
-   return DIV_ROUND_UP(sa, tile.height_el);
+   return DIV_ROUND_UP(sa, ail_metadata_tile(layout).height_el);
 }
 
 /*
@@ -582,7 +585,7 @@ ail_formats_compatible(enum pipe_format a, enum pipe_format b)
 }
 
 static inline bool
-ail_is_view_compatible(struct ail_layout *layout, enum pipe_format view)
+ail_is_view_compatible(const struct ail_layout *layout, enum pipe_format view)
 {
    return !layout->compressed || ail_formats_compatible(layout->format, view);
 }
diff --git a/src/asahi/layout/layout_core.c b/src/asahi/layout/layout_core.c
index 6f9dc9788559..6e8248f9f40f 100644
--- a/src/asahi/layout/layout_core.c
+++ b/src/asahi/layout/layout_core.c
@@ -33,7 +33,7 @@ ail_m5_layer_stride_B(uint64_t unaligned_size_B, unsigned layers,
    if (layers <= 1)
       return unaligned_size_B;
 
-   if (unaligned_size_B > 0x4000)
+   if (unaligned_size_B >= 0x4000)
       return ALIGN_POT(unaligned_size_B, 0x4000);
 
    if (has_compression_metadata)
diff --git a/src/asahi/layout/tests/test-layout.cpp b/src/asahi/layout/tests/test-layout.cpp
index c7aa9e5300ce..6c546a9ce378 100644
--- a/src/asahi/layout/tests/test-layout.cpp
+++ b/src/asahi/layout/tests/test-layout.cpp
@@ -6,6 +6,54 @@
 #include <gtest/gtest.h>
 #include "layout.h"
 
+TEST(Layout, M5DepthStencilArrayPageStride)
+{
+   for (auto format : {PIPE_FORMAT_Z16_UNORM, PIPE_FORMAT_Z32_FLOAT,
+                       PIPE_FORMAT_S8_UINT}) {
+      struct ail_layout layout = {};
+      layout.policy = AIL_LAYOUT_POLICY_M5;
+      layout.tiling = AIL_TILING_TWIDDLED;
+      layout.format = format;
+      layout.width_px = 16;
+      layout.height_px = 16;
+      layout.depth_px = 4;
+      layout.levels = 1;
+      layout.sample_count_sa = 1;
+      layout.renderable = true;
+      ail_make_miptree(&layout);
+      EXPECT_TRUE(layout.page_aligned_layers);
+      EXPECT_EQ(layout.layer_stride_B, AIL_PAGESIZE);
+      EXPECT_EQ(layout.size_B, 4u * AIL_PAGESIZE);
+   }
+}
+
+TEST(Layout, M5SparseMipTailPageStride)
+{
+   struct ail_layout ordinary = {};
+   ordinary.policy = AIL_LAYOUT_POLICY_M5;
+   ordinary.tiling = AIL_TILING_GPU;
+   ordinary.format = PIPE_FORMAT_R32_UINT;
+   ordinary.width_px = 128;
+   ordinary.height_px = 64;
+   ordinary.depth_px = 1;
+   ordinary.levels = 8;
+   ordinary.sample_count_sa = 1;
+   struct ail_layout sparse = ordinary;
+   sparse.sparse_residency = true;
+   ail_make_miptree(&ordinary);
+   ail_make_miptree(&sparse);
+
+   EXPECT_NE(ordinary.layer_stride_B % AIL_PAGESIZE, 0u);
+   EXPECT_EQ(sparse.layer_stride_B % AIL_PAGESIZE, 0u);
+   EXPECT_TRUE(sparse.page_aligned_layers);
+   const unsigned tail = sparse.mip_tail_first_lod;
+   EXPECT_LT(sparse.level_offsets_B[tail] / AIL_PAGESIZE,
+             sparse.layer_stride_B / AIL_PAGESIZE);
+   EXPECT_EQ(sparse.size_B, sparse.layer_stride_B);
+   for (unsigned l = 0; l <= sparse.levels; ++l)
+      EXPECT_EQ(ordinary.level_offsets_B[l], sparse.level_offsets_B[l]);
+}
+
 TEST(Layout, M5EightSampleGeometry)
 {
    EXPECT_EQ(ail_effective_width_sa(37, 8), 74u);
diff --git a/src/asahi/lib/agx3_cdm_decode_core.c b/src/asahi/lib/agx3_cdm_decode_core.c
index bf0d3fe83848..c7cff88d9cc0 100644
--- a/src/asahi/lib/agx3_cdm_decode_core.c
+++ b/src/asahi/lib/agx3_cdm_decode_core.c
@@ -153,6 +153,7 @@ agx3_cdm_decode_one(enum agx_chip chip, const void *bytes, size_t available,
       record->length = AGX3_CDM_BARRIER_SIZE;
       record->chip = chip;
       record->data.barrier.flags = barrier.flags;
+      record->data.barrier.flush_memory = barrier.flush_memory;
       return true;
 
    case AGX_CDM_BLOCK_TYPE_UNKNOWN_5: {
diff --git a/src/asahi/lib/agx3_cdm_decode_core.h b/src/asahi/lib/agx3_cdm_decode_core.h
index 37e88485f65d..9dec4d30c761 100644
--- a/src/asahi/lib/agx3_cdm_decode_core.h
+++ b/src/asahi/lib/agx3_cdm_decode_core.h
@@ -64,6 +64,7 @@ struct agx3_cdm_decoded_record {
       } link_padding;
       struct {
          uint8_t flags;
+         bool flush_memory;
       } barrier;
       struct {
          uint8_t flags;
diff --git a/src/asahi/lib/agx3_code_container_core.c b/src/asahi/lib/agx3_code_container_core.c
index 6dc09d0b3a3f..b2ab8db4855c 100644
--- a/src/asahi/lib/agx3_code_container_core.c
+++ b/src/asahi/lib/agx3_code_container_core.c
@@ -7,6 +7,47 @@
 
 #include <string.h>
 
+bool
+agx3_build_m5_container_lead(void *image, size_t capacity, unsigned clusters)
+{
+   if (!clusters || clusters > (UINT32_MAX - 0x40u) / 0x300u)
+      return false;
+   const size_t size = (size_t)clusters * 0x300u + 0x40u;
+   if (!image || capacity < size)
+      return false;
+
+   /* Both native M5 tables contain ten helper records, then two identical
+    * terminals. The GPU0 table has only six helpers and different opcodes. */
+   static const uint32_t helpers[] = {
+      0x0e33005e, 0x0e31405e, 0x0e31005e, 0x0e1b405e, 0x0e1b005e,
+      0x0e19405e, 0x0e19005e, 0x0e13405e, 0x0e13005e, 0x0e11405e,
+   };
+   static const uint8_t terminal[] = {
+      0x07,0x00,0x10,0x0f,0x00,0x00,0x27,0x00,
+      0x04,0x00,0x20,0x00,0xa5,0x0a,0x06,0x00,
+   };
+   uint8_t *bytes = image;
+   memset(bytes, 0, size);
+   bytes[0] = size;
+   bytes[1] = size >> 8;
+   for (size_t i = 0x40; i < size; i += 2)
+      bytes[i] = 0x06;
+   /* One helper table per cluster for each of the two stage groups. */
+   for (unsigned table = clusters; table < 3 * clusters; ++table) {
+      uint8_t *base = bytes + table * 0x100;
+      for (unsigned i = 0; i < sizeof(helpers) / sizeof(helpers[0]); ++i) {
+         uint8_t *record = base + i * 16;
+         memset(record, 0, 10);
+         for (unsigned j = 0; j < 4; ++j)
+            record[j] = helpers[i] >> (8 * j);
+      }
+      memcpy(base + 0xa0, terminal, sizeof(terminal));
+      memcpy(base + 0xb0, terminal, sizeof(terminal));
+   }
+   return true;
+}
+
+
 size_t
 agx3_code_block_size(size_t main_span)
 {
@@ -67,10 +108,11 @@ agx3_build_container_constant_program(
                 : sampled_fragment ? sizeof(sampled_fragment_program)
                                    : sizeof(fragment_stop);
 
-   memcpy(image, program, program_size);
-   for (size_t offset = program_size;
-        offset < AGX3_CONTAINER_CONSTANT_PROGRAM_SIZE; offset += 2) {
-      image[offset] = 0x06;
-      image[offset + 1] = 0x00;
+   /* Pad in instruction pairs, keeping both byte stores within the image. */
+   for (size_t pair = program_size / 2;
+        pair < AGX3_CONTAINER_CONSTANT_PROGRAM_SIZE / 2; ++pair) {
+      image[2 * pair] = 0x06;
+      image[2 * pair + 1] = 0x00;
    }
+   memcpy(image, program, program_size);
 }
diff --git a/src/asahi/lib/agx3_code_container_core.h b/src/asahi/lib/agx3_code_container_core.h
index f8c43483faa9..d2ab295c90ee 100644
--- a/src/asahi/lib/agx3_code_container_core.h
+++ b/src/asahi/lib/agx3_code_container_core.h
@@ -23,6 +23,9 @@ void agx3_build_container_constant_program(
  * deliberately not part of this helper. */
 #define AGX3_CODE_BLOCK_HEADER_SIZE 0x40u
 #define AGX3_CODE_BLOCK_ENTRY_OFFSET 0x80u
+bool agx3_build_m5_container_lead(void *image, size_t capacity,
+                                  unsigned clusters);
+
 size_t agx3_code_block_size(size_t main_span);
 bool agx3_build_code_block(void *image, size_t capacity, const void *main,
                           size_t main_size, size_t main_span,
diff --git a/src/asahi/lib/agx3_compute_program_core.c b/src/asahi/lib/agx3_compute_program_core.c
index 28a6f2a2f5a3..2522973b1474 100644
--- a/src/asahi/lib/agx3_compute_program_core.c
+++ b/src/asahi/lib/agx3_compute_program_core.c
@@ -56,6 +56,29 @@ agx3_g17p_native_compute_program_bind(
    return true;
 }
 
+bool
+agx3_m5_native_compute_program_bind(
+   enum agx_chip chip,
+   const struct agx3_g17p_native_compute_program_desc *desc,
+   struct agx3_compute_program *program)
+{
+   if (!agx_chip_g17_m5(chip) || !program || !desc)
+      return false;
+
+   /* The shared binder validates the AGX3B launch record it builds; the M5
+    * program takes its own chip afterwards. */
+   struct agx3_g17p_native_compute_program_desc record = *desc;
+   record.chip = AGX_CHIP_G17P;
+   struct agx3_compute_program staged;
+   if (!agx3_g17p_native_compute_program_bind(&record, &staged))
+      return false;
+   staged.chip = chip;
+   staged.g17p_native_compute_live_bound = false;
+   staged.m5_native_compute_live_bound = true;
+   *program = staged;
+   return true;
+}
+
 static bool
 agx3_g17p_native_compute_launch_pack(
    const struct agx3_compute_program *program,
@@ -66,7 +89,9 @@ agx3_g17p_native_compute_launch_pack(
       return false;
 
    memset(packed, 0, AGX3_CDM_DIRECT_LAUNCH_SIZE);
-   if (!program || !grid || !agx_chip_native_agx3(program->chip) ||
+   if (!program || !grid ||
+       (!agx_chip_native_agx3(program->chip) &&
+        !agx_chip_g17_m5(program->chip)) ||
        program->program_va == 0 ||
        program->program_va > UINT64_C(0xffffffffffff) ||
        (program->program_va & (AGX_IOTO_PROGRAM_ALIGNMENT - 1)) != 0 ||
@@ -115,7 +140,8 @@ agx3_g17p_native_compute_launch_pack(
     * carries the upper pointer bits and must be derived from the VA, not
     * hard-coded -- see the note on the store below. */
    uint32_t word0 = program->launch_template.word0;
-   if (global_size[1] < 2 && global_size[2] < 2)
+   if ((program->chip == AGX_CHIP_G17P || program->m5_native_compute_live_bound) &&
+       global_size[1] < 2 && global_size[2] < 2)
       word0 &= UINT32_C(0xfff1ffff);
    agx3_cmdstream_store_le32(packed, word0);
    /* Word 1 bits 22..31 carry program_va[47:38]; the hardware reconstructs
@@ -203,6 +229,30 @@ agx3_g17p_native_compute_dispatch_append(
       program, grid, 0, sink);
 }
 
+bool
+agx3_m5_native_compute_dispatch_append(
+   const struct agx3_compute_program *program,
+   const struct agx3_compute_direct_grid *grid, uint8_t barrier_flags,
+   struct agx3_compute_dispatch_sink *sink)
+{
+   if (!program || !agx_chip_g17_m5(program->chip) ||
+       !program->m5_native_compute_live_bound || !grid || !sink ||
+       !sink->data || sink->length > sink->capacity ||
+       AGX3_G17P_NATIVE_COMPUTE_DISPATCH_SIZE > sink->capacity - sink->length)
+      return false;
+   uint8_t staged[AGX3_G17P_NATIVE_COMPUTE_DISPATCH_SIZE];
+   const struct agx3_cdm_barrier_info barrier = {
+      .chip = program->chip, .flags = barrier_flags,
+      .flush_memory = agx_chip_g17_m5(program->chip) && barrier_flags == 5,
+   };
+   if (!agx3_g17p_native_compute_launch_pack(program, grid, staged) ||
+       !agx3_cdm_barrier_pack(&barrier, staged + AGX3_CDM_DIRECT_LAUNCH_SIZE))
+      return false;
+   memcpy(sink->data + sink->length, staged, sizeof(staged));
+   sink->length += sizeof(staged);
+   return true;
+}
+
 static void
 agx3_g17p_store_native_program_pointer(uint8_t *packed, uint64_t program_va)
 {
@@ -217,6 +267,52 @@ agx3_g17p_store_native_program_pointer(uint8_t *packed, uint64_t program_va)
       UINT32_C(0x40000000) | (uint32_t)((program_va >> 40) & 0xff));
 }
 
+/* M5 indirect groups retain the direct program header and append local
+ * sizes after the grid address. Indirect-local reads both counts and local
+ * sizes from memory instead. */
+bool
+agx3_m5_native_indirect_dispatch_append(
+   const struct agx3_compute_program *program, uint64_t grid_va,
+   const uint32_t local_size[3], uint8_t barrier_flags,
+   struct agx3_compute_dispatch_sink *sink)
+{
+   const unsigned launch_size = local_size ? 36 : 24;
+   uint8_t bytes[36 + AGX3_CDM_BARRIER_SIZE] = {0};
+   if (!program || !agx_chip_g17_m5(program->chip) ||
+       !program->m5_native_compute_live_bound || !program->program_va ||
+       program->program_va > UINT64_C(0xffffffffffff) ||
+       (program->program_va & (AGX_IOTO_PROGRAM_ALIGNMENT - 1)) ||
+       !grid_va || grid_va >= (UINT64_C(1) << 48) || (grid_va & 3) ||
+       program->launch_template.word0 >= (UINT32_C(1) << 27) ||
+       program->launch_template.word1 >= (UINT32_C(1) << 22) ||
+       !sink || !sink->data || sink->length > sink->capacity ||
+       launch_size + AGX3_CDM_BARRIER_SIZE > sink->capacity - sink->length)
+      return false;
+
+   agx3_cmdstream_store_le32(bytes, program->launch_template.word0 |
+      (local_size ? UINT32_C(0x08000000) : UINT32_C(0x10000000)));
+   agx3_cmdstream_store_le32(bytes + 4,
+      (uint32_t)(((program->program_va >> 38) & 0x3ff) << 22) |
+      program->launch_template.word1);
+   agx3_g17p_store_native_program_pointer(bytes + 8, program->program_va);
+   agx3_cmdstream_store_le32(bytes + 16, grid_va >> 32);
+   agx3_cmdstream_store_le32(bytes + 20, grid_va);
+   for (unsigned i = 0; local_size && i < 3; ++i) {
+      if (!local_size[i] || local_size[i] > 0x7ff)
+         return false;
+      agx3_cmdstream_store_le32(bytes + 24 + 4*i, local_size[i]);
+   }
+   const struct agx3_cdm_barrier_info barrier = {
+      .chip=program->chip, .flags=barrier_flags,
+      .flush_memory = agx_chip_g17_m5(program->chip) && barrier_flags == 5,
+   };
+   if (!agx3_cdm_barrier_pack(&barrier, bytes + launch_size))
+      return false;
+   memcpy(sink->data + sink->length, bytes, launch_size + AGX3_CDM_BARRIER_SIZE);
+   sink->length += launch_size + AGX3_CDM_BARRIER_SIZE;
+   return true;
+}
+
 /* The six-word indirect launch.  Words 0..3 are the same program header the
  * direct record uses; mode 2 then replaces the direct record's own three
  * global and three local size words with words 4..5, the 48-bit address of a
diff --git a/src/asahi/lib/agx3_compute_program_core.h b/src/asahi/lib/agx3_compute_program_core.h
index 2a51ceabe577..9a41d662833f 100644
--- a/src/asahi/lib/agx3_compute_program_core.h
+++ b/src/asahi/lib/agx3_compute_program_core.h
@@ -62,6 +62,7 @@ struct agx3_compute_program {
    /* Set only after a runtime G17P Ioto program has been composed in the
     * allocation named by program_va. */
    bool g17p_native_compute_live_bound;
+   bool m5_native_compute_live_bound;
 
    /* Only the live G17S minimal-resource preparation path sets this.  Keep
     * research composers and the zero-relocation witness out of a command
@@ -140,6 +141,22 @@ bool agx3_g17p_native_compute_program_bind(
    const struct agx3_g17p_native_compute_program_desc *desc,
    struct agx3_compute_program *program);
 
+bool agx3_m5_native_compute_program_bind(
+   enum agx_chip chip,
+   const struct agx3_g17p_native_compute_program_desc *desc,
+   struct agx3_compute_program *program);
+
+bool agx3_m5_native_compute_dispatch_append(
+   const struct agx3_compute_program *program,
+   const struct agx3_compute_direct_grid *grid, uint8_t barrier_flags,
+   struct agx3_compute_dispatch_sink *sink);
+
+/* local_size is NULL for an indirect-local grid and non-NULL for group counts. */
+bool agx3_m5_native_indirect_dispatch_append(
+   const struct agx3_compute_program *program, uint64_t grid_va,
+   const uint32_t local_size[3], uint8_t barrier_flags,
+   struct agx3_compute_dispatch_sink *sink);
+
 /* Append one direct G17P launch and the flags=0 barrier used by the
  * output-positive native compute stream.  Direct global or threadgroup counts
  * are accepted; indirect dispatch uses a different helper-program contract. */
diff --git a/src/asahi/lib/agx_bg_eot.c b/src/asahi/lib/agx_bg_eot.c
index b65ecfcd279f..2cf20e49a1cc 100644
--- a/src/asahi/lib/agx_bg_eot.c
+++ b/src/asahi/lib/agx_bg_eot.c
@@ -9,6 +9,7 @@
 #include "util/u_debug.h"
 #include "agx_compile.h"
 #include "agx3_code_container_core.h"
+#include "agx3_tile_core.h"
 #include "agx_device.h"
 #include "agx_linker.h"
 #include "agx_nir.h"
@@ -80,7 +81,9 @@ agx_g17p_upload_bg_eot(struct agx_bg_eot_cache *cache,
 {
    /* Match the standalone allocation lead already used for G17P main
     * shaders. Native packed CODE blocks do not each own such a lead. */
-   const size_t lead_size = 0x340;
+   const bool m5 = agx_chip_g17_m5(cache->dev->chip);
+   const size_t lead_size = m5
+      ? (size_t)cache->dev->params.num_clusters_total * 0x300u + 0x40u : 0x340;
    /* LDSHDR's existing auxiliary fetch-span metadata is based on compiler
     * text size. Reserve at least that span after the main entry too, so the
     * EOT text128/main64 case cannot prefetch outside this CODE block. */
@@ -108,8 +111,15 @@ agx_g17p_upload_bg_eot(struct agx_bg_eot_cache *cache,
    if (!image)
       return false;
 
-   agx3_build_container_lead(image, cache->dev->chip,
-                             cache->dev->params.num_cores_per_cluster);
+   if (m5) {
+      if (!agx3_build_m5_container_lead(image, lead_size,
+                                       cache->dev->params.num_clusters_total)) {
+         free(image);
+         return false;
+      }
+   } else {
+      agx3_build_container_lead(image, cache->dev->chip, cache->dev->params.num_cores_per_cluster);
+   }
    const uint8_t *text = bin->binary;
    const bool built = agx3_build_code_block(
       image + lead_size, block_size, text + main_offset, bin->info.main_size,
@@ -191,13 +201,15 @@ agx_compile_bg_eot_shader(struct agx_bg_eot_cache *cache, nir_shader *shader,
 
       unsigned bindless_base = 0;
       NIR_PASS(_, shader, agx_nir_lower_tilebuffer, tib, NULL, &bindless_base,
-               NULL, NULL, false, agx_chip_native_agx3(cache->dev->chip));
+               NULL, NULL, false, agx_chip_native_agx3(cache->dev->chip) ||
+                                 agx_chip_g17_m5(cache->dev->chip));
       NIR_PASS(_, shader, agx_nir_lower_monolithic_msaa, tib->nr_samples);
       NIR_PASS(_, shader, agx_nir_lower_multisampled_image_store);
       NIR_PASS(_, shader, agx_nir_lower_texture);
       NIR_PASS(_, shader, nir_shader_intrinsics_pass, lower_tex_handle_to_u0,
                nir_metadata_control_flow, &descriptor_size);
-      if (agx_chip_native_agx3(cache->dev->chip)) {
+      if (agx_chip_native_agx3(cache->dev->chip) ||
+          (agx_chip_g17_m5(cache->dev->chip) && tib->nr_samples > 1)) {
          /* Monolithic MSAA introduces a small sample loop after the common
           * preprocessing pass has already run. Fold that loop here so each
           * local-pixel access has the static one-hot sample mask required by
@@ -238,7 +250,7 @@ agx_compile_bg_eot_shader(struct agx_bg_eot_cache *cache, nir_shader *shader,
    }
 
    res->info = bin.info;
-   if (agx_chip_native_agx3(cache->dev->chip)) {
+   if (agx_chip_native_agx3(cache->dev->chip) || agx_chip_g17_m5(cache->dev->chip)) {
       if (!agx_g17p_upload_bg_eot(cache, &bin, res)) {
          free(bin.binary);
          ralloc_free(shader);
@@ -258,7 +270,7 @@ agx_compile_bg_eot_shader(struct agx_bg_eot_cache *cache, nir_shader *shader,
 static nir_def *
 build_background_op(nir_builder *b, enum agx_bg_eot_op op, unsigned rt,
                     unsigned nr, bool source_msaa, bool layered,
-                    bool wide_coords)
+                    bool wide_coords, unsigned clear_base)
 {
    if (op == AGX_BG_LOAD) {
       nir_def *coord = nir_load_pixel_coord(b);
@@ -294,7 +306,7 @@ build_background_op(nir_builder *b, enum agx_bg_eot_op op, unsigned rt,
    } else {
       assert(op == AGX_BG_CLEAR);
 
-      return nir_load_preamble(b, nr, 32, 4 + (rt * 8));
+      return nir_load_preamble(b, nr, 32, clear_base + (rt * 8));
    }
 }
 
@@ -302,18 +314,58 @@ static struct agx_bg_eot_shader *
 agx_build_background_shader(struct agx_bg_eot_cache *cache,
                             struct agx_bg_eot_key *key)
 {
+   if (agx_chip_g17_m5(cache->dev->chip) && !key->tib.sample_size_B &&
+       key->tib.nr_samples <= 1 && !key->tib.layered &&
+       !key->reserved_preamble && !agx_tilebuffer_spills(&key->tib)) {
+      bool empty = true;
+      for (unsigned rt = 0; rt < ARRAY_SIZE(key->op); ++rt)
+         empty &= key->op[rt] == AGX_BG_EOT_NONE;
+      if (empty) {
+         /* Fresh G17G's no-op compute object is exactly END, with common
+          * field0=1 GPR dword and absent field38/42 (zero staging bytes).
+          * BG and EOT loaders use the same empty executable; their release
+          * policy remains in the native stage-specific Ioto program. */
+         uint8_t text[] = {0x0e, 0, 0, 0};
+         const struct agx_shader_part bin = {
+            .binary = text,
+            .info = {
+               .stage = MESA_SHADER_COMPUTE,
+               .binary_size = sizeof(text),
+               .main_size = sizeof(text),
+               .nr_gprs = 2,
+               .agx3_staging_known = true,
+               .agx3_staging_halfs = 0,
+            },
+         };
+         struct agx_bg_eot_shader *res =
+            rzalloc(cache->ht, struct agx_bg_eot_shader);
+         if (!res || !agx_g17p_upload_bg_eot(cache, &bin, res)) {
+            ralloc_free(res);
+            return NULL;
+         }
+         res->info = bin.info;
+         return res;
+      }
+   }
+
    nir_builder b = nir_builder_init_simple_shader(
       MESA_SHADER_FRAGMENT, &agx_nir_options, "agx_background");
    b.shader->info.fs.untyped_color_outputs = true;
 
    struct agx_shader_key compiler_key = {
       .dev = agx_gather_device_key(cache->dev),
-      .fs.ignore_tib_dependencies = true,
-      /* G17P multisample backgrounds store through the coverage register. */
+      /* M5 multisample backgrounds retain the colour tile transaction. */
+      .fs.ignore_tib_dependencies =
+         !agx_chip_g17_m5(cache->dev->chip) || key->tib.nr_samples == 1,
+      .fs.background = true,
+      /* Native multisample backgrounds store through the coverage register. */
       .fs.nr_samples =
-         agx_chip_native_agx3(cache->dev->chip) ? key->tib.nr_samples : 0,
+         agx_chip_native_agx3(cache->dev->chip) ||
+               agx_chip_g17_m5(cache->dev->chip)
+            ? key->tib.nr_samples : 0,
       .reserved_preamble = key->reserved_preamble,
-      .no_preamble = agx_chip_native_agx3(cache->dev->chip),
+      .no_preamble = agx_chip_native_agx3(cache->dev->chip) ||
+                     agx_chip_g17_m5(cache->dev->chip),
    };
 
    /* The multi-target background protocol is keyed, not inferred: without
@@ -325,6 +377,14 @@ agx_build_background_shader(struct agx_bg_eot_cache *cache,
    for (unsigned rt = 0; rt < ARRAY_SIZE(key->op); ++rt)
       background_targets += key->op[rt] != AGX_BG_EOT_NONE;
    compiler_key.g17p_mrt_bg = key->mixed_mrt || background_targets > 1;
+   unsigned clear_base = 4;
+   if (agx_chip_g17_m5(cache->dev->chip) &&
+       (key->mixed_mrt || background_targets > 1)) {
+      for (unsigned rt = 0; rt < ARRAY_SIZE(key->op); ++rt) {
+         if (key->op[rt] == AGX_BG_LOAD)
+            clear_base = 8;
+      }
+   }
 
    for (unsigned rt = 0; rt < ARRAY_SIZE(key->op); ++rt) {
       if (key->op[rt] == AGX_BG_EOT_NONE)
@@ -353,7 +413,8 @@ agx_build_background_shader(struct agx_bg_eot_cache *cache,
 
       nir_def *color = build_background_op(
          &b, key->op[rt], rt, nr, source_msaa, layered,
-         agx_chip_native_agx3(cache->dev->chip));
+         agx_chip_native_agx3(cache->dev->chip) ||
+            agx_chip_g17_m5(cache->dev->chip), clear_base);
       if (key->preserve_resolve_outside) {
          assert(rt == 0 && key->op[rt] == AGX_BG_LOAD);
          nir_def *xy = nir_u2u32(&b, nir_load_pixel_coord(&b));
@@ -362,7 +423,7 @@ agx_build_background_shader(struct agx_bg_eot_cache *cache,
             nir_ball(&b, nir_uge(&b, xy, nir_channels(&b, rect, 0x3))),
             nir_ball(&b, nir_ult(&b, xy, nir_channels(&b, rect, 0xc))));
          nir_def *saved = build_background_op(
-            &b, AGX_BG_LOAD, 1, nr, false, layered, true);
+            &b, AGX_BG_LOAD, 1, nr, false, layered, true, clear_base);
          color = nir_bcsel(&b, inside, color, saved);
       }
 
@@ -383,6 +444,62 @@ static struct agx_bg_eot_shader *
 agx_build_end_of_tile_shader(struct agx_bg_eot_cache *cache,
                              struct agx_bg_eot_key *key)
 {
+   unsigned m5_stores = 0;
+   for (unsigned rt = 0; rt < ARRAY_SIZE(key->op); ++rt)
+      m5_stores += key->op[rt] == AGX_EOT_STORE;
+   const bool m5_layered_pair = agx_chip_g17_m5(cache->dev->chip) &&
+      key->tib.layered && key->m5_store_and_resolve;
+   if (agx_chip_g17_m5(cache->dev->chip) && m5_stores &&
+       (m5_stores > 1 || key->op[0] != AGX_EOT_STORE || m5_layered_pair)) {
+      if (agx_tilebuffer_spills(&key->tib))
+         return NULL;
+      uint8_t selectors[8];
+      unsigned active = 0;
+      for (unsigned rt = 0; rt < ARRAY_SIZE(key->op); ++rt) {
+         if (key->op[rt] != AGX_EOT_STORE)
+            continue;
+         const unsigned selector =
+            agx_g17p_mrt_eot_selector(key->tib.logical_format[rt]);
+         if (selector != 0x15 && selector != 0x35 && selector != 0x55 &&
+             selector != 0x75 && selector != 0x95)
+            return NULL;
+         selectors[active++] = selector;
+      }
+      uint8_t text[AGX3_M5_MRT_EOT_MAX_LENGTH];
+      size_t size;
+      if (m5_layered_pair) {
+         if (m5_stores != 1 || !key->multisample_store)
+            return NULL;
+         selectors[1] = selectors[0];
+         m5_stores = 2;
+      }
+      const unsigned sample_mask = m5_layered_pair ? 1 :
+         key->multisample_store ? (1u << m5_stores) - 1 : 0;
+      if (!agx3_m5_mrt_eot_generate_sample_mask(m5_stores, selectors,
+             sample_mask, key->tib.layered, text, sizeof(text), &size))
+         return NULL;
+      const struct agx_shader_part bin = {
+         .binary = text,
+         .info = {
+            .stage = MESA_SHADER_COMPUTE,
+            .binary_size = size,
+            .main_size = size,
+            /* Native ordinary register extent fits the qualified RT0
+             * allocation; each per-target wire needs eight staging halves. */
+            .nr_gprs = agx3_m5_mrt_eot_nr_gprs(m5_stores),
+            .agx3_staging_known = true,
+            .agx3_staging_halfs = agx3_m5_mrt_eot_staging_halfs(m5_stores),
+         },
+      };
+      struct agx_bg_eot_shader *res = rzalloc(cache->ht, struct agx_bg_eot_shader);
+      if (!res || !agx_g17p_upload_bg_eot(cache, &bin, res)) {
+         ralloc_free(res);
+         return NULL;
+      }
+      res->info = bin.info;
+      return res;
+   }
+
    const unsigned rgba8_mask = agx_g17p_rgba8_mrt_mask(&key->tib);
    const unsigned active_count = util_bitcount(rgba8_mask);
    if (agx_chip_native_agx3(cache->dev->chip) && rgba8_mask) {
@@ -430,6 +547,7 @@ agx_build_end_of_tile_shader(struct agx_bg_eot_cache *cache,
     * target really is multisample.
     */
    const bool g17p = agx_chip_native_agx3(cache->dev->chip);
+   const bool native_eot = g17p || agx_chip_g17_m5(cache->dev->chip);
    if (key->store_and_resolve) {
       /* This bounded path has two memory outputs from one physical tile. */
       assert(g17p && key->multisample_store && key->tib.nr_samples > 1 &&
@@ -439,17 +557,17 @@ agx_build_end_of_tile_shader(struct agx_bg_eot_cache *cache,
          assert(key->op[rt] == AGX_BG_EOT_NONE);
    }
    const bool store_multisample =
-      key->tib.nr_samples > 1 && (!g17p || key->multisample_store);
+      key->tib.nr_samples > 1 && (!native_eot || key->multisample_store);
 
    /* AGX2 carries the sample in the store's coordinate vector, so its
     * multisample block store is a three-component GLSL_SAMPLER_DIM_MS one.
-    * Apple's G17P builtins do not: both the resolving and the unresolved
+    * Apple's native G17P/G17G builtins do not: both the 2D and multisample
     * end-of-tile programs read exactly the same two grid coordinates and
     * differ only in the IMGWBLK dimension class the packer deposits. Keep
-    * the two-coordinate 2D shape on G17P and let the compiler key select
+    * the two-coordinate 2D shape on these targets and let the compiler key select
     * which of the two Apple programs is meant, the same way every other
     * G17P end-of-tile selector is chosen. */
-   enum glsl_sampler_dim dim = (store_multisample && !g17p)
+   enum glsl_sampler_dim dim = (store_multisample && !native_eot)
                                   ? GLSL_SAMPLER_DIM_MS
                                   : GLSL_SAMPLER_DIM_2D;
 
@@ -473,7 +591,8 @@ agx_build_end_of_tile_shader(struct agx_bg_eot_cache *cache,
 
       bool explicit = false;
       nir_def *coords = layer;
-      if (agx_chip_native_agx3(cache->dev->chip)) {
+      if (agx_chip_native_agx3(cache->dev->chip) ||
+          agx_chip_g17_m5(cache->dev->chip)) {
          /* AGX3's EOT imgwblk has no implicit pixel-coordinate form.  Apple's
           * G17P compiler reads the hardware grid X/Y SRs explicitly.  Array
           * stores append the hardware layer ID; dropping it here made every
@@ -491,13 +610,17 @@ agx_build_end_of_tile_shader(struct agx_bg_eot_cache *cache,
 
       enum pipe_format physical =
          agx_tilebuffer_physical_format(&key->tib, rt);
-      if (agx_chip_native_agx3(cache->dev->chip) &&
+      if ((agx_chip_native_agx3(cache->dev->chip) ||
+           agx_chip_g17_m5(cache->dev->chip)) &&
           (key->tib.logical_format[rt] == PIPE_FORMAT_R11G11B10_FLOAT ||
            key->tib.logical_format[rt] == PIPE_FORMAT_R9G9B9E5_FLOAT)) {
          /* G17P keeps packed-float tiles as one raw 32-bit word. The matching
-          * PBE descriptor also uses R32_UINT to preserve those exact bits. */
+          * PBE descriptor also uses R32_UINT to preserve those exact bits.
+          * G17G stores RG11B10Float and RGB9E5Float tiles the same way: one
+          * 32-bit lane (port/fbwit lstcol/lldcol W32 mask1). */
          physical = PIPE_FORMAT_R32_UINT;
-      } else if (agx_chip_native_agx3(cache->dev->chip) &&
+      } else if ((agx_chip_native_agx3(cache->dev->chip) ||
+                  agx_chip_g17_m5(cache->dev->chip)) &&
           util_format_is_srgb(key->tib.logical_format[rt])) {
          /* G17P applies the sRGB transfer in the fragment's own packs
           * (fltnorm/normflt sRGB modes) and stores the resulting bytes in a
@@ -529,14 +652,18 @@ agx_build_end_of_tile_shader(struct agx_bg_eot_cache *cache,
    struct agx_shader_key compiler_key = {
       .dev = agx_gather_device_key(cache->dev),
       .reserved_preamble = key->reserved_preamble,
-      .no_preamble = agx_chip_native_agx3(cache->dev->chip),
+      .no_preamble = agx_chip_native_agx3(cache->dev->chip) ||
+                     agx_chip_g17_m5(cache->dev->chip),
+      .m5_rt0_eot = agx_chip_g17_m5(cache->dev->chip),
+      .m5_rt0_eot_store_and_resolve = key->m5_store_and_resolve,
    };
 
-   if (agx_chip_native_agx3(cache->dev->chip)) {
-      agx_g17p_configure_rt0_eot_key(key->tib.logical_format[0],
-                                    &compiler_key);
+   if (native_eot) {
+      agx_g17p_configure_rt0_eot_key(key->tib.logical_format[0], &compiler_key);
       compiler_key.g17p_rt0_eot_multisample = store_multisample;
+   }
 
+   if (agx_chip_native_agx3(cache->dev->chip)) {
       /* A mixed layout storing several targets uses the multi-target
        * builtin, selecting each target's IMGWBLK tail by its format. A
        * single stored target keeps its format's own program. */
@@ -661,8 +788,12 @@ agx_get_precompiled_locked(struct agx_bg_eot_cache *cache, unsigned program)
       agx_workgroup(info->workgroup_size[0], info->workgroup_size[1],
                     info->workgroup_size[2]);
 
-   if (agx_chip_native_agx3(cache->dev->chip)) {
-      const size_t lead_size = 0x340;
+   if (agx_chip_native_agx3(cache->dev->chip) ||
+       agx_chip_g17_m5(cache->dev->chip)) {
+      const bool m5 = agx_chip_g17_m5(cache->dev->chip);
+      const size_t lead_size = m5
+         ? (size_t)cache->dev->params.num_clusters_total * 0x300u + 0x40u
+         : 0x340;
       const size_t main_span = MAX2(info->main_size, info->binary_size);
       const size_t block_size = agx3_code_block_size(main_span);
       /* Promoted constants live in the binary and are published as a uniform
@@ -694,12 +825,14 @@ agx_get_precompiled_locked(struct agx_bg_eot_cache *cache, unsigned program)
          ralloc_free(p);
          return NULL;
       }
-      agx3_build_container_lead(image, cache->dev->chip,
-                             cache->dev->params.num_cores_per_cluster);
-      const bool built = agx3_build_code_block(
+      const bool lead_ok = m5
+         ? agx3_build_m5_container_lead(image, lead_size,
+                                        cache->dev->params.num_clusters_total)
+         : (agx3_build_container_lead(image, cache->dev->chip, cache->dev->params.num_cores_per_cluster), true);
+      const bool built = lead_ok && agx3_build_code_block(
          image + lead_size, block_size,
          (const uint8_t *)binary + info->main_offset, info->main_size,
-         main_span, NULL, false);
+         main_span, NULL, m5);
       if (rodata_size) {
          memcpy(image + lead_size + block_size,
                 (const uint8_t *)binary + info->rodata.offset, rodata_size);
@@ -721,7 +854,8 @@ agx_get_precompiled_locked(struct agx_bg_eot_cache *cache, unsigned program)
       return NULL;
    }
 
-   if (agx_chip_native_agx3(cache->dev->chip)) {
+   if (agx_chip_native_agx3(cache->dev->chip) ||
+       agx_chip_g17_m5(cache->dev->chip)) {
       /* HAL200 consumes the CODE entry through a dispatch-local Ioto program;
        * none of the legacy USC launch words below are part of that ABI. */
       p_atomic_set(&cache->precomp[program], p);
diff --git a/src/asahi/lib/agx_bg_eot.h b/src/asahi/lib/agx_bg_eot.h
index 858b6a31ef68..b74dc89863e2 100644
--- a/src/asahi/lib/agx_bg_eot.h
+++ b/src/asahi/lib/agx_bg_eot.h
@@ -70,6 +70,10 @@ struct agx_bg_eot_key {
    /* Store RT0 samples and also resolve that same tile through PBE1.
     * The two EOT outputs share an imageblock offset, not storage. */
    bool store_and_resolve;
+   /* M5 final EOT stores samples and resolves the same tile in one launch.
+    * Kept separate from the G17P store_and_resolve path above; the two
+    * selectors drive different EOT programs. */
+   bool m5_store_and_resolve;
    /* The framebuffer is an AGX_G17P_MRT_MIXED layout. The tile layout keeps
     * only the targets this program touches, at their framebuffer offsets. */
    bool mixed_mrt;
@@ -100,6 +104,9 @@ agx_g17p_packed_clear_ok(const struct agx_tilebuffer_layout *tib)
 }
 
 struct agx_bg_eot_shader {
+   /* Driver-owned immutable launch templates, ralloc children of this
+    * shader. The cache lock serializes creation and list access. */
+   void *m5_tile_templates;
    struct agx_bg_eot_key key;
    struct agx_shader_info info;
    struct agx_bo *bo;
diff --git a/src/asahi/lib/agx_bo.c b/src/asahi/lib/agx_bo.c
index daea142c1fd3..f015629dbb85 100644
--- a/src/asahi/lib/agx_bo.c
+++ b/src/asahi/lib/agx_bo.c
@@ -53,6 +53,11 @@ static void
 agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo)
 {
    simple_mtx_assert_locked(&dev->bo_cache.lock);
+   if (dev->bo_cache.next_unadvised == bo->handle) {
+      struct list_head *next = bo->lru_link.next;
+      dev->bo_cache.next_unadvised = next == &dev->bo_cache.lru ? 0 :
+         list_entry(next, struct agx_bo, lru_link)->handle;
+   }
    list_del(&bo->bucket_link);
    list_del(&bo->lru_link);
    dev->bo_cache.size -= bo->size;
@@ -100,11 +105,33 @@ agx_bo_cache_advise_aged(struct agx_device *dev, int64_t now_ns)
    if (dev->bo_cache.madvise < 0 || !dev->ops.bo_madvise)
       return;
 
-   list_for_each_entry(struct agx_bo, entry, &dev->bo_cache.lru, lru_link) {
+   const uint32_t handle = dev->bo_cache.next_unadvised;
+   if (!handle)
+      return;
+
+   struct agx_bo *entry = util_sparse_array_get(&dev->bo_map, handle);
+   assert(entry->handle == handle);
+   bool advance = true;
+   for (struct list_head *link = &entry->lru_link;
+        link != &dev->bo_cache.lru; link = link->next) {
+      entry = list_entry(link, struct agx_bo, lru_link);
       if (now_ns - entry->cached_ns < AGX_BO_CACHE_ADVISE_NS)
          break;
       if (!entry->purgeable)
          agx_bo_cache_advise_dontneed(dev, entry);
+
+      /* Keep failed advice eligible for retry, but do not rescan a prefix
+       * whose objects are already purgeable or never accept advice. */
+      const bool complete = entry->purgeable ||
+         (entry->flags & (AGX_BO_SHARED | AGX_BO_SHAREABLE)) ||
+         dev->bo_cache.madvise < 0;
+      if (advance && complete) {
+         struct list_head *next = link->next;
+         dev->bo_cache.next_unadvised = next == &dev->bo_cache.lru ? 0 :
+            list_entry(next, struct agx_bo, lru_link)->handle;
+      } else {
+         advance = false;
+      }
    }
 }
 
@@ -287,6 +314,10 @@ agx_bo_cache_put_locked(struct agx_device *dev, struct agx_bo *bo)
    /* Add us to the bucket */
    list_addtail(&bo->bucket_link, bucket);
 
+   /* An empty LRU initializes the cursor, including on its first insertion. */
+   if (list_is_empty(&dev->bo_cache.lru) || !dev->bo_cache.next_unadvised)
+      dev->bo_cache.next_unadvised = bo->handle;
+
    /* Add us to the LRU list and update the last_used field. */
    list_addtail(&bo->lru_link, &dev->bo_cache.lru);
    bo->last_used = os_time_get_nano() / INT64_C(1000000000);
diff --git a/src/asahi/lib/agx_chip.c b/src/asahi/lib/agx_chip.c
index 1cf859a0a99f..da139589e67c 100644
--- a/src/asahi/lib/agx_chip.c
+++ b/src/asahi/lib/agx_chip.c
@@ -371,6 +371,21 @@ agx_chip_exact_product_name(enum agx_chip chip)
    }
 }
 
+bool
+agx_params_m5_bringup_supported(const struct drm_asahi_params_global *params)
+{
+   if (!params || params->num_dies != 1 ||
+       params->usc_generation != AGX_ISA_AGX3 ||
+       params->gpu_hal_generation != DRM_ASAHI_GPU_HAL_300)
+      return false;
+
+   const enum agx_chip chip = agx_chip_from_params(params);
+   return (chip == AGX_CHIP_G17G && params->chip_id == 0x8142 &&
+           params->num_clusters_total == 1) ||
+          (chip == AGX_CHIP_G17S && params->chip_id == 0x6050);
+
+}
+
 bool
 agx_chip_uses_legacy_precompiled_programs(enum agx_chip chip)
 {
diff --git a/src/asahi/lib/agx_chip.h b/src/asahi/lib/agx_chip.h
index ed8698de3446..d22b2b0be4c5 100644
--- a/src/asahi/lib/agx_chip.h
+++ b/src/asahi/lib/agx_chip.h
@@ -194,6 +194,15 @@ agx_chip_record_family(enum agx_chip chip)
 
 #include "drm-uapi/asahi_drm.h"
 
+/* The AGX3 ISA family: G17P (AGX3B) and G17S (AGX3C) share instruction forms
+ * and the AGX3 backend. Encoder/packer capability stays per encoding; the
+ * shared compiler policy gates on the family. */
+static inline bool
+agx_isa_is_agx3(enum agx_isa_encoding enc)
+{
+   return enc == AGX_ISA_ENCODING_AGX3B || enc == AGX_ISA_ENCODING_AGX3C;
+}
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -545,8 +554,18 @@ const char *agx_chip_exact_product_name(enum agx_chip chip);
 /* libagx_g13{g,x} contains AGX2 machine code and USC records. It must never be
  * selected for an AGX3 device, even when no current draw is expected to call a
  * precompiled helper. */
+bool agx_params_m5_bringup_supported(
+   const struct drm_asahi_params_global *params);
+
 bool agx_chip_uses_legacy_precompiled_programs(enum agx_chip chip);
 
+/* M5 variants sharing the AGX3C instruction and HAL300 command formats. */
+static inline bool
+agx_chip_g17_m5(enum agx_chip chip)
+{
+   return chip == AGX_CHIP_G17G || chip == AGX_CHIP_G17S;
+}
+
 /* Stable name for diagnostics, e.g. "G15X". Never NULL. */
 const char *agx_chip_name(enum agx_chip chip);
 
diff --git a/src/asahi/lib/agx_device.c b/src/asahi/lib/agx_device.c
index 230b3488a459..b24942a8ebae 100644
--- a/src/asahi/lib/agx_device.c
+++ b/src/asahi/lib/agx_device.c
@@ -8,6 +8,8 @@
 #include "agx_device.h"
 #include "agx_g17p_bind0.h"
 #include "agx_linker.h"
+#include "agx_m5_va.h"
+#include "agx3_code_container_core.h"
 #include "agx_va_runtime_hole.h"
 #include <errno.h>
 #include <stdlib.h>
@@ -142,8 +144,18 @@ agx_bo_free(struct agx_device *dev, struct agx_bo *bo)
 
    /* Free the VA. No need to unmap the BO or unbind the VA, as the kernel will
     * take care of that when we close it.
+    *
+    * Under virtio the host closes the GEM handle, and with it the mapping,
+    * only when it gets to the resource unref. Another thread can recycle the
+    * VA before that, and the host then refuses the new BO's bind: its blob is
+    * never created and every later map of it fails with EINVAL. Unbind first;
+    * vdrm_bo_create() flushes this ahead of any blob that reuses the VA. Keep
+    * (leak) the VA if the unbind cannot be queued.
     */
-   agx_va_free(dev, bo->va, false);
+   if (!dev->is_virtio || !bo->va ||
+       !agx_bo_bind(dev, NULL, bo->va->addr, bo->size, 0,
+                    DRM_ASAHI_BIND_UNBIND))
+      agx_va_free(dev, bo->va, false);
 
    if (bo->prime_fd != -1)
       close(bo->prime_fd);
@@ -901,15 +913,19 @@ agx_open_device(void *memctx, struct agx_device *dev)
       drmFreeVersion(version);
    }
 
+   memset(&dev->params, 0, sizeof(dev->params));
    params_size = dev->ops.get_params(dev, &dev->params, sizeof(dev->params));
    if (params_size <= 0) {
       fprintf(stderr, "Failed to query AGX device parameters: %m\n");
       goto fail;
    }
-   if ((size_t)params_size < sizeof(dev->params)) {
+   /* Capacity limits are an optional tail; keep accepting the legacy reply. */
+   const size_t required_params_size =
+      offsetof(struct drm_asahi_params_global, memory_limit);
+   if ((size_t)params_size < required_params_size) {
       fprintf(stderr,
               "AGX parameter reply is too short (%zd, expected at least %zu)\n",
-              params_size, sizeof(dev->params));
+              params_size, required_params_size);
       goto fail;
    }
 
@@ -929,7 +945,12 @@ agx_open_device(void *memctx, struct agx_device *dev)
     * is implemented; otherwise a newly-recognised chip can silently reach an
     * encoder from the wrong generation.
     */
-   if (!agx_params_driver_supported(&dev->params)) {
+   const bool m5_bringup =
+      debug_get_bool_option("ASAHI_M5_BRINGUP", false) &&
+      agx_params_m5_bringup_supported(&dev->params);
+   if (m5_bringup)
+      fprintf(stderr, "Experimental M5 HAL300 integration enabled; general rendering is unqualified\n");
+   if (!agx_params_driver_supported(&dev->params) && !m5_bringup) {
       if (dev->chip == AGX_CHIP_UNKNOWN) {
          fprintf(stderr,
                  "Unsupported Apple GPU architecture G%u (variant 0x%x, %u "
@@ -1083,6 +1104,8 @@ agx_open_device(void *memctx, struct agx_device *dev)
     * a practical limit. */
    if (agx_chip_native_agx3(dev->chip))
       dev->shader_base = AGX_G17P_USC_WINDOW_BASE;
+   else if (agx_chip_g17_m5(dev->chip))
+      dev->shader_base = AGX_M5_SHADER_BASE;
 
    if (dev->shader_base < reservation) {
       /* Our robustness implementation requires the bottom unmapped */
@@ -1119,7 +1142,8 @@ agx_open_device(void *memctx, struct agx_device *dev)
     * robustness carveout untouched while making every ordinary VDM payload
     * directly representable, with no per-run alias option.
     */
-   const bool heap_below_usc = agx_chip_native_agx3(dev->chip);
+   const bool heap_below_usc = agx_chip_native_agx3(dev->chip) ||
+                               agx_chip_g17_m5(dev->chip);
 
    /*
     * Start the low heap as high as it can go rather than at the reservation.
@@ -1174,8 +1198,21 @@ agx_open_device(void *memctx, struct agx_device *dev)
     * absolute USC texture/sampler table pointer, both of which sit below the
     * USC window, and T8122 additionally by its runtime's reservation. Other
     * chips retain the kernel-reported user limit. */
-   const uint64_t user_limit =
-      agx_va_user_limit(dev->chip, heap_below_usc, vm_create.kernel_start);
+   if (agx_chip_g17_m5(dev->chip)) {
+      struct agx_m5_va_layout layout;
+      if (!agx_m5_va_layout(dev->params.vm_start, dev->params.vm_end,
+                            dev->params.vm_kernel_min_size, dev->shader_base,
+                            &layout)) {
+         fprintf(stderr, "M5 address windows do not fit the reported GPU VM\n");
+         goto fail;
+      }
+      vm_create.kernel_start = layout.driver_start;
+      vm_create.kernel_end = layout.driver_end;
+      user_start = layout.data_start;
+   }
+   const uint64_t user_limit = agx_chip_g17_m5(dev->chip)
+      ? AGX_M5_DATA_END
+      : agx_va_user_limit(dev->chip, heap_below_usc, vm_create.kernel_start);
    uint64_t user_size = user_limit - user_start;
 
    int ret = dev->ops.simple_ioctl(dev, DRM_IOCTL_ASAHI_VM_CREATE, &vm_create);
@@ -1284,8 +1321,8 @@ agx_open_device(void *memctx, struct agx_device *dev)
 
    /* Keep captured addresses unavailable to ordinary allocations between
     * capture and replay. Reserve after all fixed firmware/alias holes, and
-    * only for G17P using the common allocator; other winsyses own their VA. */
-   if (agx_chip_native_agx3(dev->chip)) {
+    * for native AGX3 and M5 using the common allocator; other winsyses own their VA. */
+   if (agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip)) {
       dev->replay_heap_initialized = agx_capture_replay_heap_init(
          &dev->main_heap, &dev->replay_heap, user_size);
       if (!dev->replay_heap_initialized) {
@@ -1315,6 +1352,8 @@ agx_open_device(void *memctx, struct agx_device *dev)
       dev->libagx_programs = libagx_g15s;
    } else if (dev->chip == AGX_CHIP_G16G) {
       dev->libagx_programs = libagx_g16g;
+   } else if (agx_chip_g17_m5(dev->chip)) {
+      dev->libagx_programs = libagx_g17g;
    } else {
       dev->libagx_programs = NULL;
    }
@@ -1385,8 +1424,10 @@ agx_open_device(void *memctx, struct agx_device *dev)
       initialized |= AGX_INIT_CONTEXT_SLOTS;
    }
 
+   const bool dynamic_printf = agx_chip_native_agx3(dev->chip) ||
+                               agx_chip_g17_m5(dev->chip);
    struct agx_bo *bo =
-      agx_chip_native_agx3(dev->chip)
+      dynamic_printf
          ? agx_bo_create(dev, LIBAGX_PRINTF_BUFFER_SIZE, 0, AGX_BO_WRITEBACK,
                          "Printf/abort")
          : agx_create_fixed_bo(
@@ -1398,7 +1439,7 @@ agx_open_device(void *memctx, struct agx_device *dev)
       goto fail;
    }
 
-   if (agx_chip_native_agx3(dev->chip) &&
+   if (dynamic_printf &&
        (bo->va == NULL || dev->main_heap_size < LIBAGX_PRINTF_BUFFER_SIZE ||
         bo->va->addr < dev->main_heap_base ||
         bo->va->addr - dev->main_heap_base >
@@ -1408,9 +1449,9 @@ agx_open_device(void *memctx, struct agx_device *dev)
       goto fail;
    }
 
-   dev->printf_buffer_address = agx_chip_native_agx3(dev->chip)
-                                   ? bo->va->addr
-                                   : LIBAGX_PRINTF_BUFFER_ADDRESS;
+   dev->printf_buffer_address =
+      dynamic_printf ? bo->va->addr
+                                 : LIBAGX_PRINTF_BUFFER_ADDRESS;
 
    dev->printf.bo = bo;
    uint32_t *printf_map = agx_bo_map(bo);
@@ -1419,23 +1460,8 @@ agx_open_device(void *memctx, struct agx_device *dev)
    u_printf_init(&dev->printf, bo, printf_map);
    initialized |= AGX_INIT_PRINTF;
 
-   /* G17P: make the USC exec window's base page resident.
-    *
-    * The kernel programs dev->shader_base into USC_EXEC_BASE_CP for every
-    * command on the queue, so that register always names a real address to
-    * the hardware.  In every m1n1 witness that address is the base of the
-    * code container and is mapped; in Honeykrisp it was the bottom of a
-    * top-down heap and was mapped by nothing, which the kernel audit confirms
-    * on hardware ("usc_exec_base 0x1100000000+0x4000 covered=false phys=0x0").
-    * That is the last structural difference left between a passing add3 kick
-    * and a failing Honeykrisp one.
-    *
-    * The page carries the container's fixed leading block -- the same bytes
-    * both captured Apple code images start with -- and its own block-size
-    * dword terminates the chain, so it is a well-formed empty container
-    * rather than a blank page.
-    */
-   if (agx_chip_native_agx3(dev->chip)) {
+   /* Map the stage helper tables at the USC execution-window base. */
+   if (agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip)) {
       const size_t usc_base_size = AIL_PAGESIZE;
 
       simple_mtx_lock(&dev->vma_lock);
@@ -1463,8 +1489,14 @@ agx_open_device(void *memctx, struct agx_device *dev)
          goto fail;
       }
 
-      agx3_build_usc_base_page(map, usc_base_size, dev->chip,
-                               dev->params.num_cores_per_cluster);
+      if (agx_chip_g17_m5(dev->chip)) {
+         memset(map, 0, usc_base_size);
+         if (!agx3_build_m5_container_lead(map, usc_base_size,
+                                          dev->params.num_clusters_total))
+            goto fail;
+      } else {
+         agx3_build_usc_base_page(map, usc_base_size, dev->chip, dev->params.num_cores_per_cluster);
+      }
       agx_bo_assert_range(usc_base, 0, usc_base_size);
 
    }
diff --git a/src/asahi/lib/agx_device.h b/src/asahi/lib/agx_device.h
index 346ff65a8682..d53e5eeddcc0 100644
--- a/src/asahi/lib/agx_device.h
+++ b/src/asahi/lib/agx_device.h
@@ -253,6 +253,9 @@ struct agx_device {
       struct u_cnd_monotonic idle_cond;
       thrd_t idle_thread;
       bool idle_started, idle_stop, idle_pending;
+
+      /* First cached object whose purge advice still needs checking. */
+      uint32_t next_unadvised;
    } bo_cache;
 
    struct agxdecode_ctx *agxdecode;
@@ -282,6 +285,16 @@ agx_bo_assert_range(ASSERTED struct agx_bo *bo, ASSERTED size_t offset,
    assert(offset <= bo->size && size <= bo->size - offset);
 }
 
+/* Make CPU writes to a BO range visible to the GPU. Linux DRM mappings are
+ * coherent, so this only checks the range; the M5 sparse/physical paths call
+ * it where a non-coherent winsys would need a flush. Returns 0 on success. */
+static inline int
+agx_bo_sync_for_device(struct agx_bo *bo, size_t offset, size_t size)
+{
+   agx_bo_assert_range(bo, offset, size);
+   return 0;
+}
+
 /*
  * Determine if an address is in the read-only section. See the documentation
  * for sparse_ro_offset.
@@ -296,6 +309,21 @@ agx_addr_is_ro(struct agx_device *dev, uint64_t addr)
  * Convert a read-write address to its read-only shadow address. See the
  * documentation for sparse_ro_offset.
  */
+/* Flags for filling a range with the scratch or zero page. When the kernel
+ * offers DRM_ASAHI_FEATURE_SPARSE_TILES it fills the range from its own
+ * physically contiguous tiles (zero for read-only, a per-VM sink for
+ * writable binds), so each provider operation maps up to 64 pages instead of
+ * one; the BO then only anchors the mapping.
+ */
+static inline uint32_t
+agx_fill_bind_flags(const struct agx_device *dev)
+{
+   return DRM_ASAHI_BIND_SINGLE_PAGE |
+          ((dev->params.features & DRM_ASAHI_FEATURE_SPARSE_TILES)
+              ? DRM_ASAHI_BIND_SPARSE_TILE
+              : 0);
+}
+
 static inline uint64_t
 agx_rw_addr_to_ro(struct agx_device *dev, uint64_t addr)
 {
@@ -492,6 +520,15 @@ agx_native_graphics(const struct agx_device *dev)
    return agx_chip_native_agx3(dev->chip);
 }
 
+/* T6040 uses the G17P architecture profile, while capability rollout may
+ * still be product-specific. Device admission has already checked USC/HAL. */
+static inline bool
+agx_is_m4_pro(const struct agx_device *dev)
+{
+   return dev->chip == AGX_CHIP_G17P && dev->params.chip_id == 0x6040 &&
+          dev->params.gpu_generation == 16 && dev->params.gpu_variant == 'S';
+}
+
 struct agx_va *agx_va_alloc(struct agx_device *dev, uint64_t size_B,
                             uint64_t align_B, enum agx_va_flags flags,
                             uint64_t fixed_va);
diff --git a/src/asahi/lib/agx_helpers.h b/src/asahi/lib/agx_helpers.h
index a2c5b9fca497..feda32374861 100644
--- a/src/asahi/lib/agx_helpers.h
+++ b/src/asahi/lib/agx_helpers.h
@@ -151,6 +151,8 @@ agx_translate_sample_count(unsigned samples)
       return AGX_SAMPLE_COUNT_2;
    case 4:
       return AGX_SAMPLE_COUNT_4;
+   case 8:
+      return AGX_SAMPLE_COUNT_8;
    default:
       UNREACHABLE("Invalid sample count");
    }
diff --git a/src/asahi/lib/agx_ioto_load_core.c b/src/asahi/lib/agx_ioto_load_core.c
index 1e56c5a71d97..841e06cf83c7 100644
--- a/src/asahi/lib/agx_ioto_load_core.c
+++ b/src/asahi/lib/agx_ioto_load_core.c
@@ -619,8 +619,9 @@ agx_ioto_load_absolute(struct agx_ioto_load_core *core,
       return false;
 
    if (load->cprog_input_mask &&
-       (!agx_chip_native_preamble(core->chip) || !ops ||
-        !ops->pack_cprog_copy ||
+       ((!agx_chip_native_preamble(core->chip) &&
+         !agx_chip_g17_m5(core->chip)) ||
+        !ops || !ops->pack_cprog_copy ||
         (load->cprog_input_mask & ~load->lane_mask) ||
         (load->cprog_input_mask >> (2 * load->count_dwords)) ||
         load->cprog_input_tps_half < 8 || load->cprog_input_tps_half >= 520))
diff --git a/src/asahi/lib/agx_ioto_preshader_core.c b/src/asahi/lib/agx_ioto_preshader_core.c
index a200c62068a7..6591b70b423a 100644
--- a/src/asahi/lib/agx_ioto_preshader_core.c
+++ b/src/asahi/lib/agx_ioto_preshader_core.c
@@ -237,7 +237,7 @@ agx_ioto_constant_program_emit(
          return false;
 
       if (program->absolute_branch) {
-         if (!agx_chip_native_preamble(chip))
+         if (!agx_chip_native_preamble(chip) && !agx_chip_g17_m5(chip))
             return false;
          desc.external_va = program->code_va;
       } else {
@@ -264,7 +264,11 @@ agx_ioto_constant_program_emit(
        * cloneProgramInfo changes the code VA without subtracting a B PC.
        * Witnesses: 0x218780f48/f4c, 0x218737b8c, 0x2188416d8/e0/e4 in
        * AGXMetalG17P UUID 4A1B0ACE-B7E3-3B5B-A594-84289B937288. */
-      staged.bytes[staged.branch_offset + 3] |= 1;
+      if (!agx_chip_g17_m5(chip))
+         staged.bytes[staged.branch_offset + 3] |= 1;
+      else
+         /* Captured G17S BLAS loader+512: B_EP1 f11 is bit61. */
+         staged.bytes[staged.branch_offset + 7] |= 0x20;
    }
 
    *output = staged;
diff --git a/src/asahi/lib/agx_ioto_program_core.c b/src/asahi/lib/agx_ioto_program_core.c
index ff8037f2534e..f2cd8215eb97 100644
--- a/src/asahi/lib/agx_ioto_program_core.c
+++ b/src/asahi/lib/agx_ioto_program_core.c
@@ -12,6 +12,7 @@
 #include "util/u_debug.h"
 
 #include <string.h>
+#include <stdlib.h>
 
 /* Exact fixed regions surrounding the runtime state tail in the SPDX-MIT
  * m1n1 NATIVE_ADD3_SHADER witness. */
@@ -862,3 +863,239 @@ agx_ioto_g17s_minimal_compute_program_compose_resources(
    return agx_ioto_g17s_minimal_compute_program_compose_internal(
       program_va, shader_va, resources, output, capacity, written);
 }
+
+
+bool
+agx_ioto_program_copy_suffix_offset(enum agx_chip chip, const void *body,
+                                      size_t body_size, size_t *offset)
+{
+   if (!offset || (body_size && !body))
+      return false;
+
+   const uint8_t *bytes = body;
+   size_t cursor = 0, suffix = 0;
+   while (cursor < body_size) {
+      if (agx_ioto_program_gpu0_inline_lddev(chip, bytes + cursor,
+                                             body_size - cursor)) {
+         cursor += AGX_IOTO_GPU0_INLINE_LDDEV_SIZE;
+         suffix = cursor;
+         continue;
+      }
+      struct agx_ioto_record record;
+      size_t consumed = 0;
+      if (!agx_ioto_decode_one(chip, bytes + cursor, body_size - cursor,
+                                 &record, &consumed) || consumed == 0 ||
+          !agx_ioto_straight_line_body_kind(record.kind))
+         return false;
+      cursor += consumed;
+      if (record.kind != AGX_IOTO_KIND_COPY_EP0 &&
+          record.kind != AGX_IOTO_KIND_COPY_EP1)
+         suffix = cursor;
+   }
+   *offset = suffix;
+   return true;
+}
+
+/* Loader allocation can finish an early round before more LDIMM/LDDEVs.
+ * State setup must precede those early uniform writes as well as the final
+ * contiguous COPY suffix. A one-round body keeps its original ordering. */
+bool
+agx_ioto_program_first_copy_offset(enum agx_chip chip, const void *body,
+                                    size_t body_size, size_t *offset)
+{
+   if (!offset || (body_size && !body))
+      return false;
+   const uint8_t *bytes = body;
+   size_t cursor = 0;
+   *offset = body_size;
+   while (cursor < body_size) {
+      struct agx_ioto_record record;
+      size_t consumed = 0;
+      if (!agx_ioto_decode_one(chip, bytes + cursor, body_size - cursor,
+                               &record, &consumed) || !consumed ||
+          !agx_ioto_straight_line_body_kind(record.kind))
+         return false;
+      if (*offset == body_size &&
+          (record.kind == AGX_IOTO_KIND_COPY_EP0 ||
+           record.kind == AGX_IOTO_KIND_COPY_EP1))
+         *offset = cursor;
+      cursor += consumed;
+   }
+   return true;
+}
+
+static bool
+type0_tail_records_valid(enum agx_chip chip,
+                         const struct agx_ioto_state_tail_output *packed_tail)
+{
+   if (!packed_tail || packed_tail->length != AGX_IOTO_STATE_TAIL_TYPE0_SIZE)
+      return false;
+
+   static const enum agx_ioto_kind kinds[] = {
+      AGX_IOTO_KIND_SPECLM_EP0,
+      AGX_IOTO_KIND_LDSHDR_EP0,
+      AGX_IOTO_KIND_SPECTPR_EP0,
+   };
+   static const size_t sizes[] = {
+      AGX_IOTO_SPECLM_SIZE,
+      AGX_IOTO_LDSHDR_SIZE,
+      AGX_IOTO_SPECTPR_SIZE,
+   };
+   size_t offset = 0;
+   for (size_t i = 0; i < sizeof(kinds) / sizeof(kinds[0]); ++i) {
+      struct agx_ioto_record record;
+      size_t consumed = 0;
+      if (!agx_ioto_validate_kind(
+             chip, kinds[i], packed_tail->bytes + offset,
+             packed_tail->length - offset, &record, &consumed) ||
+          consumed != sizes[i])
+         return false;
+      offset += consumed;
+   }
+
+   return offset == packed_tail->length;
+}
+
+static bool
+m5_no_resource_program_compose(
+   enum agx_chip chip, uint64_t program_va, const struct agx_ioto_state_tail_output *packed_tail,
+   bool fragment, void *output, size_t capacity, size_t *written)
+{
+   const size_t size = fragment
+      ? AGX_IOTO_G17G_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE
+      : AGX_IOTO_G17G_VERTEX_NO_RESOURCE_PROGRAM_SIZE;
+   if (!output || !written ||
+       !type0_tail_records_valid(AGX_CHIP_G17G, packed_tail) ||
+       program_va == 0 || program_va > UINT64_C(0x1000000000000) - size ||
+       (program_va & (AGX_IOTO_PROGRAM_ALIGNMENT - 1)) != 0 || capacity < size)
+      return false;
+
+   const uint8_t *ldshdr = packed_tail->bytes + AGX_IOTO_SPECLM_SIZE;
+   if (!(agx_load_le16(ldshdr) & UINT16_C(0xffc0)) &&
+       !agx_load_le16(ldshdr + 6) && !agx_load_le16(ldshdr + 8))
+      return false;
+
+   uint8_t staged[AGX_IOTO_G17G_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE] = {0};
+   memcpy(staged, packed_tail->bytes, packed_tail->length);
+   size_t offset = packed_tail->length;
+
+   if (!agx_ioto_g17s_minimal_profile_loader_pack(staged + offset,
+                                                sizeof(staged) - offset))
+      return false;
+
+   staged[offset + 20] = (fragment ? 4 : 8) *
+                         (chip == AGX_CHIP_G17S ? 2 : 1);
+   offset += AGX_IOTO_G17S_MINIMAL_PROFILE_LOADER_SIZE;
+   if (fragment) {
+      static const uint8_t pixrel[] = {0x9f, 0x17, 0x00, 0x04, 0x00, 0x00};
+      memcpy(staged + offset, pixrel, sizeof(pixrel));
+      offset += sizeof(pixrel);
+   }
+   staged[offset] = 0x0e;
+   offset += AGX_IOTO_PROGRAM_TERMINATE_SIZE;
+   if (offset != size)
+      return false;
+
+   memcpy(output, staged, size);
+   *written = size;
+   return true;
+}
+
+bool
+agx_ioto_m5_resource_program_compose_classes(
+   enum agx_chip chip, uint64_t program_va, const void *body, size_t body_size,
+   const struct agx_ioto_state_tail_output *packed_tail,
+   enum agx_ioto_g17g_program_stage stage, unsigned colour_classes,
+   void *output, size_t capacity, size_t *written)
+{
+   if (!agx_chip_g17_m5(chip) || !output || !written ||
+       (colour_classes == 0 || (colour_classes & ~7u)) ||
+       (stage != AGX_IOTO_G17G_PROGRAM_VERTEX &&
+        stage != AGX_IOTO_G17G_PROGRAM_FRAGMENT &&
+        stage != AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT &&
+        stage != AGX_IOTO_G17G_PROGRAM_EOT &&
+        stage != AGX_IOTO_G17G_PROGRAM_BACKGROUND &&
+        stage != AGX_IOTO_G17G_PROGRAM_COMPUTE &&
+        stage != AGX_IOTO_G17G_PROGRAM_MESH &&
+        stage != AGX_IOTO_G17G_PROGRAM_OBJECT))
+      return false;
+
+   size_t copy_offset;
+   if (!agx_ioto_program_first_copy_offset(AGX_CHIP_G17G, body, body_size,
+                                            &copy_offset))
+      return false;
+
+   uint8_t base[AGX_IOTO_G17G_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE];
+   size_t base_size;
+   if (!m5_no_resource_program_compose(
+          chip, program_va, packed_tail, stage != AGX_IOTO_G17G_PROGRAM_VERTEX,
+          base, sizeof(base), &base_size))
+      return false;
+
+   const size_t prefix_size = AGX_IOTO_STATE_TAIL_TYPE0_SIZE +
+                             AGX_IOTO_G17S_MINIMAL_PROFILE_LOADER_SIZE;
+   const bool pixel_release = stage == AGX_IOTO_G17G_PROGRAM_FRAGMENT ||
+                              stage == AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT ||
+                              stage == AGX_IOTO_G17G_PROGRAM_BACKGROUND;
+   if (stage == AGX_IOTO_G17G_PROGRAM_FRAGMENT ||
+       stage == AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT) {
+      /* HAL300 PIXREL::emit for 0x2ff with the shader's colour classes
+       * removed. Sparse targets retain their own class pair. */
+      static const uint8_t release[8][6] = {
+         {0},
+         {0x9f, 0x17, 0, 4, 0, 0},
+         {0x7f, 0x16, 0, 4, 0, 0},
+         {0x1f, 0x16, 0, 4, 0, 0},
+         {0xff, 0x11, 0, 4, 0, 0},
+         {0x9f, 0x11, 0, 4, 0, 0},
+         {0x7f, 0x10, 0, 4, 0, 0},
+         {0x1f, 0x10, 0, 4, 0, 0},
+      };
+      memcpy(base + prefix_size, release[colour_classes], sizeof(release[0]));
+      /* Depth/coverage consumes class 0 as well. HAL300 clears descriptor
+       * bit 0, encoded as bit 3 of the first PIXREL byte. */
+      if (stage == AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT)
+         base[prefix_size] &= ~8u;
+   }
+   if (stage == AGX_IOTO_G17G_PROGRAM_BACKGROUND) {
+
+      static const uint8_t background_pixrel[] = {7, 0x10, 0, 4, 0, 0};
+      memcpy(base + prefix_size, background_pixrel, sizeof(background_pixrel));
+   }
+   const size_t suffix_size = pixel_release
+                                ? base_size - prefix_size
+                                : AGX_IOTO_PROGRAM_TERMINATE_SIZE;
+   const size_t fixed_size = prefix_size + suffix_size;
+   if (body_size > SIZE_MAX - fixed_size)
+      return false;
+   const size_t size = body_size + fixed_size;
+   if (capacity < size || size > UINT64_C(0x1000000000000) ||
+       program_va > UINT64_C(0x1000000000000) - size)
+      return false;
+
+   uint8_t *staged = malloc(size);
+   if (!staged)
+      return false;
+   /* The compute profile helper must run after the uniform uploads. On M5,
+    * calling it before COPY leaves the probe's uniforms zero, even when
+    * those copies follow in the Ioto stream. Keep shader-state setup
+    * before the first copy, but defer the helper until all uploads finish.
+    */
+   const size_t before_copy = stage == AGX_IOTO_G17G_PROGRAM_COMPUTE
+                                ? AGX_IOTO_STATE_TAIL_TYPE0_SIZE : prefix_size;
+   if (copy_offset)
+      memcpy(staged, body, copy_offset);
+   memcpy(staged + copy_offset, base, before_copy);
+   if (body_size != copy_offset)
+      memcpy(staged + copy_offset + before_copy,
+             (const uint8_t *)body + copy_offset, body_size - copy_offset);
+   if (before_copy != prefix_size)
+      memcpy(staged + body_size + before_copy, base + before_copy,
+             prefix_size - before_copy);
+   memcpy(staged + body_size + prefix_size,
+          base + base_size - suffix_size, suffix_size);
+   memcpy(output, staged, size);
+   free(staged);
+   *written = size;
+   return true;
+}
diff --git a/src/asahi/lib/agx_ioto_program_core.h b/src/asahi/lib/agx_ioto_program_core.h
index a828b98982f4..8d09014c7a82 100644
--- a/src/asahi/lib/agx_ioto_program_core.h
+++ b/src/asahi/lib/agx_ioto_program_core.h
@@ -32,6 +32,8 @@ extern "C" {
 #define AGX_IOTO_G17P_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE 86u
 /* Two root LDIMMs, two high-uniform carrier COPYs, state/profile, and the
  * external request prelude plus long branch currently occupy 124 bytes. */
+#define AGX_IOTO_G17G_VERTEX_NO_RESOURCE_PROGRAM_SIZE 84u
+#define AGX_IOTO_G17G_FRAGMENT_NO_RESOURCE_PROGRAM_SIZE 90u
 #define AGX_IOTO_G17P_EXTERNAL_LAUNCHER_MAX_SIZE   128u
 #define AGX_IOTO_G17P_RT0_CLEAR_PROGRAM_SIZE       140u
 #define AGX_IOTO_G17P_WBOIT_MRT_CLEAR_PROGRAM_SIZE 158u
@@ -187,6 +189,29 @@ bool agx_ioto_g17s_minimal_compute_program_compose_resources(
    const struct agx_ioto_g17s_minimal_compute_resources *resources,
    void *output, size_t capacity, size_t *written);
 
+enum agx_ioto_g17g_program_stage {
+   AGX_IOTO_G17G_PROGRAM_VERTEX,
+   AGX_IOTO_G17G_PROGRAM_FRAGMENT,
+   AGX_IOTO_G17G_PROGRAM_COVERAGE_FRAGMENT,
+   AGX_IOTO_G17G_PROGRAM_EOT,
+   AGX_IOTO_G17G_PROGRAM_BACKGROUND,
+   AGX_IOTO_G17G_PROGRAM_COMPUTE,
+   AGX_IOTO_G17G_PROGRAM_MESH,
+   AGX_IOTO_G17G_PROGRAM_OBJECT,
+};
+
+bool agx_ioto_program_copy_suffix_offset(enum agx_chip chip, const void *body,
+                                        size_t body_size, size_t *offset);
+/* M5 state must precede copies from every loader allocation round. */
+bool agx_ioto_program_first_copy_offset(enum agx_chip chip, const void *body,
+                                       size_t body_size, size_t *offset);
+
+bool agx_ioto_m5_resource_program_compose_classes(
+   enum agx_chip chip, uint64_t program_va, const void *body, size_t body_size,
+   const struct agx_ioto_state_tail_output *packed_tail,
+   enum agx_ioto_g17g_program_stage stage, unsigned colour_classes,
+   void *output, size_t capacity, size_t *written);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/asahi/lib/agx_ioto_state_core.c b/src/asahi/lib/agx_ioto_state_core.c
index 402c898a98d6..313f62f1c2e9 100644
--- a/src/asahi/lib/agx_ioto_state_core.c
+++ b/src/asahi/lib/agx_ioto_state_core.c
@@ -347,7 +347,7 @@ agx_ioto_state_pack_record(enum agx_chip chip,
 }
 
 /* Each packer is transactional: output is unchanged on every failure. */
-static bool
+bool
 agx_ioto_state_pack_speclm(enum agx_chip chip,
                            const struct agx_ioto_state *state, void *output,
                            size_t capacity)
@@ -357,7 +357,7 @@ agx_ioto_state_pack_speclm(enum agx_chip chip,
       AGX_IOTO_SPECLM_SIZE, agx_ioto_state_encode_speclm);
 }
 
-static bool
+bool
 agx_ioto_state_pack_ldshdr(enum agx_chip chip,
                            const struct agx_ioto_state *state, void *output,
                            size_t capacity)
diff --git a/src/asahi/lib/agx_ioto_state_core.h b/src/asahi/lib/agx_ioto_state_core.h
index 09158294fe59..76b72886f6e5 100644
--- a/src/asahi/lib/agx_ioto_state_core.h
+++ b/src/asahi/lib/agx_ioto_state_core.h
@@ -31,6 +31,13 @@ struct agx_ioto_state {
    uint64_t f[AGX_IOTO_STATE_FIELD_COUNT];
 };
 
+bool agx_ioto_state_pack_speclm(enum agx_chip chip,
+                              const struct agx_ioto_state *state,
+                              void *output, size_t capacity);
+bool agx_ioto_state_pack_ldshdr(enum agx_chip chip,
+                              const struct agx_ioto_state *state,
+                              void *output, size_t capacity);
+
 bool agx_ioto_state_pack_spectpr(enum agx_chip chip,
                                  const struct agx_ioto_state *state,
                                  void *output, size_t capacity);
diff --git a/src/asahi/lib/agx_linker.c b/src/asahi/lib/agx_linker.c
index 13423e905005..fabd5e6dc089 100644
--- a/src/asahi/lib/agx_linker.c
+++ b/src/asahi/lib/agx_linker.c
@@ -47,7 +47,8 @@ agx_fs_epilog_apply_device_policy(enum agx_chip chip,
 
    const bool g17p = agx_chip_native_agx3(chip);
    key->preserve_zero_mask = g17p;
-   key->g17p_tile_formats = g17p;
+   key->m5_coverage = agx_chip_g17_m5(chip);
+   key->g17p_tile_formats = g17p || agx_chip_g17_m5(chip);
 }
 
 bool
@@ -180,9 +181,17 @@ agx3_fill_helper_terminals(uint8_t *image, size_t from, size_t to)
 
 /* Total container bytes for `code_size` bytes of executable. */
 static size_t
-agx3_container_size(size_t code_size)
+agx3_container_lead_size(const struct agx_device *dev)
 {
-   return AGX3_CONTAINER_LEAD + AGX3_CONTAINER_BLOCK_HDR +
+   return agx_chip_g17_m5(dev->chip)
+      ? (size_t)dev->params.num_clusters_total * 0x300u + 0x40u
+      : AGX3_CONTAINER_LEAD;
+}
+
+static size_t
+agx3_container_size(const struct agx_device *dev, size_t code_size)
+{
+   return agx3_container_lead_size(dev) + AGX3_CONTAINER_BLOCK_HDR +
           AGX3_CONTAINER_CONST_PROG + ALIGN_POT(code_size, 64);
 }
 
@@ -228,27 +237,33 @@ agx3_build_usc_base_page(uint8_t *image, size_t size, enum agx_chip chip,
    }
 }
 
-static void
-agx3_build_container(uint8_t *image, enum agx_chip chip,
-                     unsigned cores_per_cluster, size_t code_size,
-                     bool fragment, bool sampled_fragment)
+static bool
+agx3_build_container(const struct agx_device *dev, uint8_t *image,
+                     size_t code_size, bool fragment, bool sampled_fragment)
 {
-   memset(image, 0, agx3_container_size(code_size));
-   agx3_build_container_lead(image, chip, cores_per_cluster);
-
-   agx_store_le32(image + AGX3_CONTAINER_LEAD,
+   const bool m5 = agx_chip_g17_m5(dev->chip);
+   const size_t lead_size = agx3_container_lead_size(dev);
+   memset(image, 0, agx3_container_size(dev, code_size));
+   if (m5) {
+      if (!agx3_build_m5_container_lead(image, lead_size,
+                                       dev->params.num_clusters_total))
+         return false;
+   } else {
+      agx3_build_container_lead(image, dev->chip, dev->params.num_cores_per_cluster);
+   }
+   agx_store_le32(image + lead_size,
                  AGX3_CONTAINER_BLOCK_HDR + AGX3_CONTAINER_CONST_PROG +
                     ALIGN_POT(code_size, 64));
-
-   uint8_t *constant = image + AGX3_CONTAINER_LEAD + AGX3_CONTAINER_BLOCK_HDR;
-   agx3_build_container_constant_program(constant, fragment,
-                                         sampled_fragment);
+   uint8_t *constant = image + lead_size + AGX3_CONTAINER_BLOCK_HDR;
+   agx3_build_container_constant_program(constant, fragment || m5,
+                                         sampled_fragment && !m5);
+   return true;
 }
 
 static bool
 agx3_uses_code_container(const struct agx_device *dev)
 {
-   return agx_chip_native_agx3(dev->chip);
+   return agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip);
 }
 
 bool
@@ -260,7 +275,8 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
    /* These literal loop fragments are AGX2 machine code. G17P exposes only
     * one sample and its Vulkan path folds sample shading before split-FS
     * lowering; fail closed if a caller ever bypasses that canonicalization. */
-   if (agx_chip_native_agx3(dev->chip) && nr_samples_shaded) {
+   if ((agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip)) &&
+       nr_samples_shaded) {
       fprintf(stderr,
               "agx_fast_link: external sample loop is not AGX3 machine code\n");
       return false;
@@ -276,12 +292,17 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
     * shader's second entry resets the coverage state immediately before its
     * forced-early wait.  Keep the first gate and remove every later one.  A
     * gate at offset zero is outside all of that part's branches; otherwise
-    * retain the older branch-free proof that no branch spans the bytes. */
+    * require the compiler's proof that no relative branch spans the bytes. */
    struct agx_shader_part *parts[] = {prolog, main, epilog};
    bool drop_entry[ARRAY_SIZE(parts)] = {false};
+   /* A later entry that a relative branch spans (the empty-EXEC skip over a
+    * testing coverage update can) is overwritten with NOPs instead: the
+    * part keeps its size, so every branch offset stays valid. */
+   bool nop_entry[ARRAY_SIZE(parts)] = {false};
    bool entry_seen = false;
-   if (agx_chip_native_agx3(dev->chip) && fragment && !nr_samples_shaded &&
-       !colour_fused) {
+   int kept_entry = -1;
+   if ((agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip)) &&
+       fragment && !nr_samples_shaded && !colour_fused) {
       for (unsigned i = 0; i < ARRAY_SIZE(parts); ++i) {
          struct agx_shader_part *part = parts[i];
          if (!part || !part->info.agx3_colour_link.entry_emitted)
@@ -289,29 +310,37 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
 
          const uint32_t off = part->info.agx3_colour_link.entry_offset;
          if (entry_seen) {
-            if (off != 0 && !part->info.agx3_colour_link.branch_free)
-               return false;
-            drop_entry[i] = true;
+            if (off != 0 && !part->info.agx3_colour_link.entry_unspanned)
+               nop_entry[i] = true;
+            else
+               drop_entry[i] = true;
          } else {
             entry_seen = true;
+            kept_entry = i;
          }
       }
 
       for (unsigned i = 0; i < ARRAY_SIZE(parts); ++i) {
-         if (!drop_entry[i])
+         if (!drop_entry[i] && !nop_entry[i])
             continue;
          struct agx_shader_part *part = parts[i];
          const uint32_t off = part->info.agx3_colour_link.entry_offset;
          uint8_t gate[AGX3_PIXEL_MAX_LENGTH], gate_size;
          if ((off & 1) || off > part->info.main_size ||
              part->info.main_size - off < AGX3_PIXEL_MAX_LENGTH ||
-             !agx3_pixel_colour_entry(AGX_ISA_ENCODING_AGX3B, gate,
-                                      &gate_size) ||
+             !agx3_pixel_colour_entry(agx_chip_g17_m5(dev->chip)
+                                         ? AGX_ISA_ENCODING_AGX3C
+                                         : AGX_ISA_ENCODING_AGX3B,
+                                      gate, &gate_size) ||
              gate_size != AGX3_PIXEL_MAX_LENGTH ||
              memcmp((const uint8_t *)part->binary + part->info.main_offset +
                        off,
                     gate, gate_size))
+         {
+            fprintf(stderr, "agx_fast_link: part %u entry mismatch at %u of %u bytes\n",
+                    i, off, part->info.main_size);
             return false;
+         }
       }
    }
    size_t size = 0;
@@ -370,7 +399,7 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
    /* The staging accounting is per instruction, so a linked software vertex
     * shader (a compute-stage main behind the vertex prolog) is described the
     * same way as a hardware vertex or fragment program. */
-   if (agx_chip_native_agx3(dev->chip) &&
+   if ((agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip)) &&
        (main->info.stage == MESA_SHADER_VERTEX ||
         main->info.stage == MESA_SHADER_FRAGMENT ||
         main->info.stage == MESA_SHADER_COMPUTE ||
@@ -382,7 +411,7 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
    }
 
    const bool container = agx3_uses_code_container(dev);
-   const size_t alloc_size = container ? agx3_container_size(size) : size;
+   const size_t alloc_size = container ? agx3_container_size(dev, size) : size;
 
    uint32_t code_offset = 0;
    linked->bo = dev->code_alloc
@@ -392,7 +421,8 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
                                    "Linked executable");
    linked->size = size;
    linked->entry_offset =
-      code_offset + (container ? AGX3_CONTAINER_ENTRY : 0);
+      code_offset + (container ? agx3_container_lead_size(dev) +
+                                  AGX3_CONTAINER_BLOCK_HDR + AGX3_CONTAINER_CONST_PROG : 0);
    if (!linked->bo) {
       fprintf(stderr,
               "agx_fast_link: executable low-VA allocation of %zu bytes "
@@ -418,10 +448,9 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
       return false;
    }
 
-   if (container)
-      agx3_build_container(container_map, dev->chip,
-                           dev->params.num_cores_per_cluster, size, fragment,
-                           fragment && sampled_fragment);
+   if (container && !agx3_build_container(dev, container_map, size, fragment,
+                                          fragment && sampled_fragment))
+      goto invalid_start_marker;
 
    size_t offset = 0;
    if (colour_fused) {
@@ -429,8 +458,12 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
       offset = sizeof(fused_colour);
    }
 
+   /* Where each part starts in the linked program. */
+   size_t part_offset[ARRAY_SIZE(parts)] = {0};
+
    /* FS prolog happens per-pixel, outside the sample loop */
    if (prolog) {
+      part_offset[0] = offset;
       size_t sz = prolog->info.main_size;
       memcpy(linked_map + offset, prolog->binary, sz);
       if (agx_chip_native_agx3(dev->chip) && fragment &&
@@ -454,6 +487,7 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
       if (!part)
          continue;
 
+      part_offset[i] = offset;
       size_t sz = part->info.main_size;
       const uint8_t *part_code = (const uint8_t *)part->binary + part->info.main_offset;
       if (drop_entry[i]) {
@@ -464,6 +498,14 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
          sz -= AGX3_PIXEL_MAX_LENGTH;
       } else {
          memcpy(linked_map + offset, part_code, sz);
+         if (nop_entry[i]) {
+            uint8_t *gate = linked_map + offset +
+                            part->info.agx3_colour_link.entry_offset;
+            for (unsigned h = 0; h < AGX3_PIXEL_MAX_LENGTH; h += 2) {
+               gate[h] = 0x06;
+               gate[h + 1] = 0x00;
+            }
+         }
       }
       if (i == 2 && main && main->info.agx3_colour_link.rt_query_write_acquired) {
          const struct agx3_colour_link_info *link = &part->info.agx3_colour_link;
@@ -471,12 +513,20 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
             goto invalid_start_marker;
          uint32_t at = link->rt_query_wait_offset;
          if (drop_entry[i] && at > link->entry_offset) at -= AGX3_PIXEL_MAX_LENGTH;
+         /* The main shader already holds the colour write class (pixwait 4
+          * ahead of its first traversal), so the epilog's read+write wait
+          * becomes a read wait. Native M5 colour fragments with a ray query
+          * show the same pair: pixwait 4 before intersect_ray, pixwait 8
+          * before lstcol. Encode both in the device's ISA. */
+         const enum agx_isa_encoding enc = agx_chip_g17_m5(dev->chip)
+                                              ? AGX_ISA_ENCODING_AGX3C
+                                              : AGX_ISA_ENCODING_AGX3B;
          uint8_t before[AGX3_PIXEL_MAX_LENGTH], after[AGX3_PIXEL_MAX_LENGTH];
          uint8_t before_len, after_len;
          if (at > sz || sz - at < AGX3_PIXEL_MAX_LENGTH ||
-             !agx3_pixel_wait(AGX_ISA_ENCODING_AGX3B, 12, link->rt_query_wait_mask,
+             !agx3_pixel_wait(enc, 12, link->rt_query_wait_mask,
                              before, &before_len) ||
-             !agx3_pixel_wait(AGX_ISA_ENCODING_AGX3B, 8, link->rt_query_wait_mask,
+             !agx3_pixel_wait(enc, 8, link->rt_query_wait_mask,
                              after, &after_len) || before_len != after_len ||
              before_len != AGX3_PIXEL_MAX_LENGTH ||
              memcmp(linked_map + offset + at, before, before_len))
@@ -520,6 +570,52 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
 
    assert(offset == size);
 
+   /* Native M5 fragments writing depth or stencil alone at several samples
+    * (native G17G compiler zs_depth_{2,4,8}x, zs_stencil_{2,4}x, and
+    * zs_depth_discard_4x after its kill) open with the f34 entry gate and
+    * never release the depth/coverage class explicitly. Where a part ends
+    * such a covered-sample loop, take that form for the program: mark its
+    * entry gate and drop that release. If either instruction is not the plain
+    * one recorded, keep both. */
+   if (agx_chip_g17_m5(dev->chip) && fragment && kept_entry >= 0 &&
+       !colour_fused) {
+      const struct agx_shader_part *kept = parts[kept_entry];
+      for (unsigned i = 0; i < ARRAY_SIZE(parts); ++i) {
+         const struct agx_shader_part *part = parts[i];
+         if (!part || !part->info.agx3_colour_link.zs_loop_release ||
+             (int)i < kept_entry)
+            continue;
+         size_t release_at = part->info.agx3_colour_link.zs_loop_release_offset;
+         if (drop_entry[i] &&
+             release_at > part->info.agx3_colour_link.entry_offset)
+            release_at -= AGX3_PIXEL_MAX_LENGTH;
+         release_at += part_offset[i];
+         const size_t gate_at =
+            part_offset[kept_entry] + kept->info.agx3_colour_link.entry_offset;
+         uint8_t gate[AGX3_PIXEL_MAX_LENGTH], unreleased[AGX3_PIXEL_MAX_LENGTH];
+         uint8_t release[AGX3_PIXEL_MAX_LENGTH];
+         uint8_t gate_len, unreleased_len, release_len;
+         if (release_at > gate_at && release_at + AGX3_PIXEL_MAX_LENGTH <= size &&
+             agx3_pixel_colour_entry(AGX_ISA_ENCODING_AGX3C, gate, &gate_len) &&
+             agx3_pixel_colour_entry_zs_unreleased(AGX_ISA_ENCODING_AGX3C,
+                                                   unreleased, &unreleased_len) &&
+             agx3_pixel_release(AGX_ISA_ENCODING_AGX3C, 1, false, 0, release,
+                                &release_len) &&
+             gate_len == AGX3_PIXEL_MAX_LENGTH &&
+             unreleased_len == AGX3_PIXEL_MAX_LENGTH &&
+             release_len == AGX3_PIXEL_MAX_LENGTH &&
+             !memcmp(linked_map + gate_at, gate, gate_len) &&
+             !memcmp(linked_map + release_at, release, release_len)) {
+            memcpy(linked_map + gate_at, unreleased, unreleased_len);
+            for (unsigned h = 0; h < AGX3_PIXEL_MAX_LENGTH; h += 2) {
+               linked_map[release_at + h] = 0x06;
+               linked_map[release_at + h + 1] = 0x00;
+            }
+         }
+         break;
+      }
+   }
+
    linked->scratch_size = scratch_size;
    linked->agx3_spectpr_scratch_size = agx3_spectpr_scratch_size;
    linked->nr_gprs = nr_gprs;
@@ -600,6 +696,15 @@ agx_fast_link(struct agx_linked_shader *linked, struct agx_device *dev,
       }
    }
 
+   /* The linked program is what runs; its parts are dumped at compile. */
+   if ((agx_get_compiler_debug() & AGX_DBG_SHADERS) &&
+       (dev->chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->chip))) {
+      fprintf(stderr, "AGX3 linked: fragment=%u size=%zu hex=", fragment, size);
+      for (size_t i = 0; i < size; ++i)
+         fprintf(stderr, "%02x", linked_map[i]);
+      fprintf(stderr, "\n");
+   }
+
    return true;
 
 invalid_start_marker:
diff --git a/src/asahi/lib/agx_linker.h b/src/asahi/lib/agx_linker.h
index 77b29c47555b..e2993ee182a0 100644
--- a/src/asahi/lib/agx_linker.h
+++ b/src/asahi/lib/agx_linker.h
@@ -335,10 +335,16 @@ struct agx_fs_epilog_key {
    uint8_t nr_samples;
    /* agx_tilebuffer_layout::max_sample_size_B of the render pass. */
    uint8_t max_sample_size_B;
-   uint16_t max_bytes_per_tile;
+   /* agx_tilebuffer_layout::resident_mask of the render pass. */
+   uint8_t resident_mask;
+   bool native_imageblock;
+   uint32_t max_bytes_per_tile;
    uint16_t max_samples_per_tile;
    bool force_small_tile;
    bool preserve_zero_mask;
+   /* M5: the epilog's coverage update is the single native testing form
+    * (agx_nir_lower_sample_mask_m5). */
+   bool m5_coverage;
    /* G17P tile storage: sRGB as U8NORM bytes converted by the native packs,
     * R11G11B10/RGB9E5 as one raw 32-bit word. */
    bool g17p_tile_formats;
@@ -394,6 +400,9 @@ void agx_nir_fs_epilog_agx2(struct nir_builder *b, const void *key_);
 void agx_nir_fs_epilog_g17p(struct nir_builder *b, const void *key);
 bool agx_nir_lower_g17p_sample_shading(
    nir_shader *nir, struct agx_fs_epilog_link_info *link, unsigned samples);
+bool agx_nir_lower_g17p_sample_shading_file(
+   nir_shader *nir, struct agx_fs_epilog_link_info *link, unsigned samples,
+   unsigned registers, bool eight_samples);
 void agx_nir_fs_prolog(struct nir_builder *b, const void *key_);
 
 bool agx_nir_gather_vs_inputs(nir_shader *s,
diff --git a/src/asahi/lib/agx_m5_va.h b/src/asahi/lib/agx_m5_va.h
new file mode 100644
index 000000000000..186a83989236
--- /dev/null
+++ b/src/asahi/lib/agx_m5_va.h
@@ -0,0 +1,73 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef AGX_M5_VA_H
+#define AGX_M5_VA_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/* M5 uses compact TA/deflake pointers relative to this architectural base.
+ * Their 32-bit fields require driver resources in the first 4 GiB. Reserve
+ * a larger driver interval there, away from both data/sparse heaps and USC.
+ * Keep this contract in sync with the kernel's m5_vm.rs.
+ */
+#define AGX_M5_DRIVER_BASE UINT64_C(0x1000000000)
+#define AGX_M5_DRIVER_SIZE UINT64_C(0x800000000)
+#define AGX_M5_COMPACT_SIZE UINT64_C(0x100000000)
+#define AGX_M5_DATA_START UINT64_C(0x4000000000)
+#define AGX_M5_DATA_END UINT64_C(0x8000000000)
+#define AGX_M5_SHADER_BASE UINT64_C(0x10000000000)
+#define AGX_M5_SHADER_SIZE UINT64_C(0x100000000)
+
+struct agx_m5_va_layout {
+   uint64_t driver_start, driver_end;
+   uint64_t data_start, data_end;
+   uint64_t shader_start, shader_end;
+};
+
+/* VDM PPP references are relative to the architectural GART base. */
+static inline bool
+agx_m5_vdm_compact(uint64_t address, uint64_t *out)
+{
+   if (!out || address < AGX_M5_DRIVER_BASE || (address & 3) ||
+       address - AGX_M5_DRIVER_BASE >= (UINT64_C(1) << 39))
+      return false;
+   *out = address - AGX_M5_DRIVER_BASE;
+   return true;
+}
+
+static inline bool
+agx_m5_va_layout(uint64_t vm_start, uint64_t vm_end,
+                 uint64_t kernel_min_size, uint64_t shader_base,
+                 struct agx_m5_va_layout *out)
+{
+   if (!out || vm_start >= vm_end || vm_start > AGX_M5_DRIVER_BASE ||
+       shader_base < vm_start || (shader_base & (AGX_M5_SHADER_SIZE - 1)) ||
+       shader_base > UINT64_MAX - AGX_M5_SHADER_SIZE ||
+       shader_base + AGX_M5_SHADER_SIZE > vm_end || AGX_M5_DATA_END > vm_end)
+      return false;
+
+   uint64_t size = kernel_min_size > AGX_M5_DRIVER_SIZE
+                     ? kernel_min_size : AGX_M5_DRIVER_SIZE;
+   if (size > UINT64_MAX - 0x3fff)
+      return false;
+   size = (size + 0x3fff) & ~UINT64_C(0x3fff);
+   if (size > AGX_M5_DATA_START - AGX_M5_DRIVER_BASE)
+      return false;
+
+   const uint64_t driver_end = AGX_M5_DRIVER_BASE + size;
+   const uint64_t shader_end = shader_base + AGX_M5_SHADER_SIZE;
+   if ((shader_base < driver_end && shader_end > AGX_M5_DRIVER_BASE) ||
+       (shader_base < AGX_M5_DATA_END && shader_end > AGX_M5_DATA_START))
+      return false;
+
+   *out = (struct agx_m5_va_layout) {
+      .driver_start = AGX_M5_DRIVER_BASE, .driver_end = driver_end,
+      .data_start = AGX_M5_DATA_START, .data_end = AGX_M5_DATA_END,
+      .shader_start = shader_base, .shader_end = shader_end,
+   };
+   return true;
+}
+
+#endif
diff --git a/src/asahi/lib/agx_nir_lower_msaa.c b/src/asahi/lib/agx_nir_lower_msaa.c
index c00fad82e144..5e6c975b1ae3 100644
--- a/src/asahi/lib/agx_nir_lower_msaa.c
+++ b/src/asahi/lib/agx_nir_lower_msaa.c
@@ -54,11 +54,47 @@ specialize_sample_count(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 bool
 agx_nir_specialize_sample_count(nir_shader *shader, unsigned samples)
 {
-   assert(samples == 1 || samples == 2 || samples == 4);
+   assert(samples == 1 || samples == 2 || samples == 4 || samples == 8);
    return nir_shader_intrinsics_pass(shader, specialize_sample_count,
                                      nir_metadata_control_flow, &samples);
 }
 
+static bool
+apply_api_mask_to_force_early(nir_builder *b, nir_intrinsic_instr *intr,
+                              void *data)
+{
+   bool *done = data;
+   if (*done || intr->intrinsic != nir_intrinsic_sample_mask_agx)
+      return false;
+
+   /* Fold the API mask into the one forced-early coverage update. A separate
+    * prolog MOVMSK followed by the main shader's forced-early MOVMSK loses the
+    * surviving samples on G17P; Apple's native program performs one combined
+    * update before reading the post-test coverage for fragment side effects.
+    * M5 also suppresses the separate prolog mask for forced-early shaders, so
+    * its per-pixel program must include the API mask in this update. */
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_def *api = nir_load_api_sample_mask_agx(b);
+   nir_src_rewrite(&intr->src[1],
+                   nir_iand(b, intr->src[1].ssa,
+                            nir_u2uN(b, api, nir_src_bit_size(intr->src[1]))));
+   *done = true;
+   return true;
+}
+
+bool
+agx_nir_apply_api_mask_to_force_early(nir_shader *shader, enum agx_chip chip)
+{
+   const bool supported = agx_chip_native_agx3(chip) ||
+      (agx_chip_g17_m5(chip) && !shader->info.fs.uses_sample_shading);
+   if (!supported || !shader->info.fs.early_fragment_tests)
+      return false;
+
+   bool done = false;
+   return nir_shader_intrinsics_pass(shader, apply_api_mask_to_force_early,
+                                     nir_metadata_control_flow, &done);
+}
+
 static bool
 lower_to_per_sample(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 {
@@ -269,8 +305,8 @@ specialize_g17p_sample_copy(nir_cf_list *copy,
  * it. Conditional depth/stencil writes need phis here: moving their exports
  * directly to the end would move uses outside the defining branch.
  */
-static void
-lower_sample_exports_to_end(nir_shader *shader)
+void
+agx_nir_lower_sample_exports_to_end(nir_shader *shader)
 {
    nir_function_impl *impl = nir_shader_get_entrypoint(shader);
    nir_block *end = nir_cursor_current_block(nir_after_impl(impl));
@@ -349,7 +385,7 @@ agx_nir_unroll_g17p_samples_layout(nir_shader *shader, unsigned nr_samples,
                                    const struct agx_g17p_colour_layout *layout)
 {
    assert(shader->info.stage == MESA_SHADER_FRAGMENT);
-   assert(nr_samples == 2 || nr_samples == 4);
+   assert(nr_samples == 2 || nr_samples == 4 || nr_samples == 8);
 
    nir_function_impl *impl = nir_shader_get_entrypoint(shader);
 
@@ -375,7 +411,7 @@ agx_nir_unroll_g17p_samples_layout(nir_shader *shader, unsigned nr_samples,
       }
    }
 
-   lower_sample_exports_to_end(shader);
+   agx_nir_lower_sample_exports_to_end(shader);
 
    nir_cf_list body;
    nir_cf_extract(&body, nir_before_impl(impl), nir_after_impl(impl));
@@ -411,7 +447,7 @@ agx_nir_unroll_g17p_samples_layout(nir_shader *shader, unsigned nr_samples,
     * end of the function, so gather them all there. */
    struct util_dynarray exports;
    util_dynarray_init(&exports, NULL);
-   nir_def *write_samples[4] = {NULL};
+   nir_def *write_samples[8] = {NULL};
    unsigned write_sample_count = 0;
    nir_foreach_block(block, impl) {
       nir_foreach_instr_safe(instr, block) {
@@ -692,18 +728,253 @@ agx_nir_hoist_g17p_zs_writes(nir_shader *shader)
    return nir_progress(true, impl, nir_metadata_control_flow);
 }
 
+/*
+ * M5 pixel-rate depth-only or stencil-only writes at several samples.
+ *
+ * Native G17G fragments (native G17G compiler zs_depth_{2,4,8}x,
+ * zs_cdepth_4x, zs_depth_only_4x, zs_stencil_{2,4}x) take the depth/coverage
+ * class once, then loop over the covered samples with a single DEPTHF, each
+ * iteration naming one sample. The code is identical at every sample count.
+ * Rewrite a whole-pixel store_zs_agx of that shape, executed once at the top
+ * level, into the same loop; the backend stages one word per iteration
+ * (AGX_ZS_COVERED_SAMPLE). The loop runs at least once, so a lane whose
+ * coverage is already empty still balances the transaction with an empty
+ * word, as the native loop's bottom test does. Whole-pixel depth and stencil
+ * together stay one word, and masked writes stay one word per sample.
+ */
+bool
+agx_nir_lower_m5_zs_sample_loop(nir_shader *shader, unsigned nr_samples)
+{
+   assert(shader->info.stage == MESA_SHADER_FRAGMENT);
+   if (nr_samples < 2)
+      return false;
+
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+   nir_intrinsic_instr *store = NULL;
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         if (intr->intrinsic != nir_intrinsic_store_zs_agx)
+            continue;
+         /* Exactly one write, of a supported shape, or none rewritten. */
+         if (store)
+            return false;
+         store = intr;
+      }
+   }
+
+   if (!store || !agx_block_runs_once(impl, store->instr.block))
+      return false;
+
+   const unsigned base = nir_intrinsic_base(store);
+   const unsigned all = BITFIELD_MASK(nr_samples);
+   if ((base != 1 && base != 2) || !nir_src_is_const(store->src[0]) ||
+       (nir_src_as_uint(store->src[0]) & all) != all)
+      return false;
+
+   nir_builder b = nir_builder_at(nir_before_instr(&store->instr));
+
+   /* Native clamps the depth once, ahead of the loop. The backend does not
+    * clamp a covered-sample word again. */
+   nir_def *depth = store->src[1].ssa;
+   if (base == 1)
+      depth = nir_fmin(&b, nir_fmax(&b, depth, nir_imm_float(&b, 0.0f)),
+                       nir_imm_float(&b, 1.0f));
+   nir_zs_transaction_agx(&b, .flags = 0);
+
+   nir_variable *left =
+      nir_local_variable_create(impl, glsl_uint16_t_type(), "zs_samples");
+   nir_store_var(&b, left,
+                 nir_iand_imm(&b, nir_u2u16(&b, nir_load_sample_mask(&b)), all),
+                 1);
+
+   nir_loop *loop = nir_push_loop(&b);
+   {
+      nir_def *samples = nir_load_var(&b, left);
+      nir_def *sample = nir_iand(&b, samples, nir_ineg(&b, samples));
+      nir_store_zs_agx(&b, sample, depth, store->src[2].ssa,
+                       .base = base | AGX_ZS_COVERED_SAMPLE);
+      nir_def *rest = nir_iand(&b, samples, nir_inot(&b, sample));
+      nir_store_var(&b, left, rest, 1);
+      nir_break_if(&b, nir_ieq_imm(&b, rest, 0));
+   }
+   nir_pop_loop(&b, loop);
+
+   nir_zs_transaction_agx(&b, .flags = 1);
+   nir_instr_remove(&store->instr);
+
+   nir_progress(true, impl, nir_metadata_none);
+   nir_lower_vars_to_ssa(shader);
+   return true;
+}
+
+/* Whether an instruction may run once per pixel ahead of the sample loop:
+ * the coverage update, the depth/stencil write and the pixel-rate values they
+ * are computed from. */
+static bool
+m5_pixel_rate_instr(const nir_instr *instr)
+{
+   if (instr->type != nir_instr_type_intrinsic)
+      return instr->type == nir_instr_type_alu ||
+             instr->type == nir_instr_type_load_const ||
+             instr->type == nir_instr_type_undef;
+
+   switch (nir_instr_as_intrinsic(instr)->intrinsic) {
+   case nir_intrinsic_sample_mask_agx:
+   case nir_intrinsic_store_zs_agx:
+   case nir_intrinsic_load_exported_agx:
+   case nir_intrinsic_load_shader_part_tests_zs_agx:
+   /* The coverage dither is per pixel. */
+   case nir_intrinsic_load_frag_coord:
+   case nir_intrinsic_load_pixel_coord:
+      return true;
+   default:
+      return false;
+   }
+}
+
+static bool
+m5_coverage_or_zs(const nir_instr *instr)
+{
+   if (instr->type != nir_instr_type_intrinsic)
+      return false;
+   nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;
+   return op == nir_intrinsic_sample_mask_agx ||
+          op == nir_intrinsic_store_zs_agx;
+}
+
+/* The one top-level depth/stencil write, moved with its pixel-rate sources
+ * to just after the last coverage update, ahead of any tile access. Epilogs
+ * emit it last, after the blend. */
+static void
+m5_hoist_zs_write(nir_function_impl *impl)
+{
+   nir_intrinsic_instr *store = NULL;
+   nir_cf_node *last_update = NULL;
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         if (intr->intrinsic == nir_intrinsic_store_zs_agx) {
+            if (store || !agx_block_runs_once(impl, block))
+               return;
+            store = intr;
+         } else if (intr->intrinsic == nir_intrinsic_sample_mask_agx) {
+            /* A kill must reach the write first. */
+            if (store)
+               return;
+            nir_cf_node *node = &block->cf_node;
+            while (node->parent != &impl->cf_node)
+               node = node->parent;
+            last_update = node;
+         }
+      }
+   }
+   if (!store)
+      return;
+   for (unsigned i = 0; i < nir_intrinsic_infos[store->intrinsic].num_srcs; ++i) {
+      if (!movable_zs_source(store->src[i].ssa))
+         return;
+   }
+   /* Only where the whole part ahead of it stays at pixel rate. */
+   if (last_update) {
+      foreach_list_typed(nir_cf_node, node, node, &impl->body) {
+         nir_foreach_block_in_cf_node(block, node) {
+            nir_foreach_instr(instr, block) {
+               if (!m5_pixel_rate_instr(instr))
+                  return;
+            }
+         }
+         if (node == last_update)
+            break;
+      }
+   }
+
+   struct move_zs_state state = {
+      .cursor = last_update ? nir_after_cf_node(last_update)
+                            : nir_before_impl(impl),
+      .moved = _mesa_pointer_set_create(NULL),
+   };
+   nir_foreach_src(&store->instr, move_zs_source, &state);
+   nir_instr_move(state.cursor, &store->instr);
+   _mesa_set_destroy(state.moved, NULL);
+}
+
+/*
+ * The end of the leading pixel-rate part of a pixel-shaded M5 epilog that
+ * holds all of its coverage updates and depth/stencil writes. False when
+ * there are none, or when other work precedes the last of them.
+ *
+ * Native G17G blended alpha-to-coverage fragments (native G17G compiler
+ * a2cb_pure_{2,4,8}x, with and without a depth attachment) issue one testing
+ * MOVMSK at pixel rate, then blend each covered sample. Wrapping the update
+ * in the sample loop instead issues one per sample, each naming its own
+ * sample. A testing word tests every covered sample, not only the ones it
+ * names: that is how a2c at alpha 0 wrote depth before 911fc231107. So the
+ * first live sample's update would also test, and write depth for, samples
+ * that a later sample's update kills (expected from that result).
+ *
+ * Blended fragments that write depth likewise finish depth before blending:
+ * one DEPTHF per sample with the alpha coverage folded in (a2cb_depth_{2,4,8}x)
+ * or, without alpha-to-coverage, the covered-sample DEPTHF loop
+ * (blend_depth_{2,4,8}x). Leaving the write in the sample loop interleaves a
+ * DEPTHF with each sample's colour instead.
+ */
+static bool
+m5_pixel_rate_prefix(nir_shader *shader, nir_cursor *end)
+{
+   if (shader->info.fs.early_fragment_tests)
+      return false;
+
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+   m5_hoist_zs_write(impl);
+
+   unsigned left = 0;
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block)
+         left += m5_coverage_or_zs(instr);
+   }
+   if (!left)
+      return false;
+
+   foreach_list_typed(nir_cf_node, node, node, &impl->body) {
+      if (node->type == nir_cf_node_loop)
+         return false;
+      nir_foreach_block_in_cf_node(block, node) {
+         nir_foreach_instr(instr, block) {
+            if (!m5_pixel_rate_instr(instr))
+               return false;
+            if (!m5_coverage_or_zs(instr) || --left)
+               continue;
+            /* The last one: end after it, or after the control flow
+             * around it. */
+            *end = node->type == nir_cf_node_block ? nir_after_instr(instr)
+                                                   : nir_after_cf_node(node);
+            return true;
+         }
+      }
+   }
+   UNREACHABLE("every update was counted");
+}
+
 /*
  * Lower a fragment shader into a monolithic pixel shader, with static sample
  * count, blend state, and tilebuffer formats in the shader key. For dynamic,
  * epilogs must be used, which have separate lowerings.
  */
-bool
-agx_nir_lower_monolithic_msaa(nir_shader *shader, uint8_t nr_samples)
+static bool
+lower_monolithic_msaa(nir_shader *shader, uint8_t nr_samples, bool m5)
 {
    assert(shader->info.stage == MESA_SHADER_FRAGMENT);
-   assert(nr_samples == 1 || nr_samples == 2 || nr_samples == 4);
+   assert(nr_samples == 1 || nr_samples == 2 || nr_samples == 4 || nr_samples == 8);
 
-   agx_nir_lower_sample_mask(shader);
+   if (m5)
+      agx_nir_lower_sample_mask_m5(shader);
+   else
+      agx_nir_lower_sample_mask(shader);
 
    /* In single sampled programs, interpolateAtSample needs to return the
     * center pixel.
@@ -714,8 +985,23 @@ agx_nir_lower_monolithic_msaa(nir_shader *shader, uint8_t nr_samples)
       };
       nir_lower_single_sampled(shader, &lss_opts);
    } else if (shader->info.fs.uses_sample_shading) {
+      /* M5 keeps a pixel-rate coverage update and depth/stencil write ahead
+       * of the sample loop: set them aside while the rest becomes
+       * per-sample, then put them back in front of the loop. */
+      nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+      nir_cursor prefix_end;
+      const bool split = m5 && m5_pixel_rate_prefix(shader, &prefix_end);
+      nir_cf_list prefix;
+      if (split)
+         nir_cf_extract(&prefix, nir_before_impl(impl), prefix_end);
+
       agx_nir_lower_to_per_sample(shader);
       agx_nir_wrap_per_sample_loop(shader, nr_samples);
+
+      if (split) {
+         nir_cf_reinsert(&prefix, nir_before_impl(impl));
+         nir_progress(true, impl, nir_metadata_none);
+      }
    }
 
    return true;
@@ -749,3 +1035,15 @@ agx_nir_lower_monolithic_msaa_agx2(nir_shader *shader, uint8_t nr_samples)
 
    return true;
 }
+
+bool
+agx_nir_lower_monolithic_msaa(nir_shader *shader, uint8_t nr_samples)
+{
+   return lower_monolithic_msaa(shader, nr_samples, false);
+}
+
+bool
+agx_nir_lower_monolithic_msaa_m5(nir_shader *shader, uint8_t nr_samples)
+{
+   return lower_monolithic_msaa(shader, nr_samples, true);
+}
diff --git a/src/asahi/lib/agx_nir_lower_sample_intrinsics.c b/src/asahi/lib/agx_nir_lower_sample_intrinsics.c
index 739f61beb982..a19e563ac9d0 100644
--- a/src/asahi/lib/agx_nir_lower_sample_intrinsics.c
+++ b/src/asahi/lib/agx_nir_lower_sample_intrinsics.c
@@ -19,9 +19,15 @@ select_if_msaa_else_0(nir_builder *b, nir_def *x)
    return nir_bcsel(b, msaa, x, nir_imm_intN_t(b, 0, x->bit_size));
 }
 
+struct sample_lowering {
+   bool ignore_sample_mask_without_msaa;
+   bool wide_positions;
+};
+
 static bool
 lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 {
+   const struct sample_lowering *args = data;
    b->cursor = nir_before_instr(&intr->instr);
 
    switch (intr->intrinsic) {
@@ -45,6 +51,9 @@ lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
        *    }
        */
       nir_def *packed = nir_load_sample_positions_agx(b);
+      if (args->wide_positions)
+         packed = nir_pack_64_2x32(b, nir_vec2(
+            b, packed, nir_load_sample_positions_hi_agx(b)));
 
       /* The n'th sample is the in the n'th byte of the register */
       nir_def *shifted = nir_ushr(
@@ -55,6 +64,8 @@ lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
          /* Get the appropriate nibble */
          nir_def *nibble =
             nir_iand_imm(b, nir_ushr_imm(b, shifted, i * 4), 0xF);
+         if (args->wide_positions)
+            nibble = nir_u2u32(b, nibble);
 
          /* Convert it from fixed point to float */
          xy[i] = nir_fmul_imm(b, nir_u2f16(b, nibble), 1.0 / 16.0);
@@ -140,7 +151,8 @@ lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
       nir_src *src = &intr->src[0];
       nir_def *sample = src->ssa;
 
-      if (nir_src_is_const(*src) && nir_src_as_uint(*src) >= 4) {
+      if (nir_src_is_const(*src) &&
+          nir_src_as_uint(*src) >= (args->wide_positions ? 8 : 4)) {
          sample = nir_imm_int(b, 0);
       } else {
          sample = select_if_msaa_else_0(b, sample);
@@ -170,9 +182,7 @@ lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
          return false;
 
       nir_def *mask = nir_inot(b, nir_u2u16(b, intr->src[0].ssa));
-      bool *ignore_sample_mask_without_msaa = data;
-
-      if (*ignore_sample_mask_without_msaa)
+      if (args->ignore_sample_mask_without_msaa)
          mask = select_if_msaa_else_0(b, mask);
 
       nir_demote_samples(b, mask);
@@ -204,6 +214,21 @@ bool
 agx_nir_lower_sample_intrinsics(nir_shader *shader,
                                 bool ignore_sample_mask_without_msaa)
 {
+   const struct sample_lowering args = {
+      .ignore_sample_mask_without_msaa = ignore_sample_mask_without_msaa,
+   };
+   return nir_shader_intrinsics_pass(shader, lower, nir_metadata_control_flow,
+                                     (void *)&args);
+}
+
+bool
+agx_nir_lower_sample_intrinsics_wide(nir_shader *shader,
+                                     bool ignore_sample_mask_without_msaa)
+{
+   const struct sample_lowering args = {
+      .ignore_sample_mask_without_msaa = ignore_sample_mask_without_msaa,
+      .wide_positions = true,
+   };
    return nir_shader_intrinsics_pass(shader, lower, nir_metadata_control_flow,
-                                     &ignore_sample_mask_without_msaa);
+                                     (void *)&args);
 }
diff --git a/src/asahi/lib/agx_nir_lower_uvs.c b/src/asahi/lib/agx_nir_lower_uvs.c
index 773d7ae51155..ebfd9c715ff8 100644
--- a/src/asahi/lib/agx_nir_lower_uvs.c
+++ b/src/asahi/lib/agx_nir_lower_uvs.c
@@ -415,9 +415,9 @@ agx_link_varyings_vs_fs(void *out, struct agx_varyings_vs *vs,
          } else if (b.slot == VARYING_SLOT_PRIMITIVE_ID &&
                     !vs->slots[VARYING_SLOT_PRIMITIVE_ID]) {
             cfg.source = AGX_COEFFICIENT_SOURCE_PRIMITIVE_ID;
-            
-
-
+            /* Opt-in M5 mesh diagnostic: sweep the 3-bit VIT source of the
+             * PrimitiveID row to locate the native per-primitive offset
+             * coefficient (ASAHI_MESH_PP_VIT_SOURCE=0..7). */
             const char *probe_source = getenv("ASAHI_MESH_PP_VIT_SOURCE");
             if (probe_source)
                cfg.source = atoi(probe_source) & 7;
diff --git a/src/asahi/lib/agx_nir_prolog_epilog.c b/src/asahi/lib/agx_nir_prolog_epilog.c
index b34e4ccacda0..d1bfa069d340 100644
--- a/src/asahi/lib/agx_nir_prolog_epilog.c
+++ b/src/asahi/lib/agx_nir_prolog_epilog.c
@@ -662,9 +662,16 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)
    b->shader->info.stage = MESA_SHADER_FRAGMENT;
    b->shader->info.name = "FS epilog";
 
-   struct agx_tilebuffer_layout tib = agx_build_tilebuffer_layout_with_limits(
-      key->rt_formats, ARRAY_SIZE(key->rt_formats), key->nr_samples, true,
-      key->max_sample_size_B, key->max_bytes_per_tile, key->max_samples_per_tile);
+   struct agx_tilebuffer_layout tib =
+      key->native_imageblock
+         ? agx_build_tilebuffer_layout_resident(
+              key->rt_formats, ARRAY_SIZE(key->rt_formats), key->nr_samples,
+              true, key->max_sample_size_B, key->max_bytes_per_tile,
+              key->max_samples_per_tile, key->resident_mask)
+         : agx_build_tilebuffer_layout_with_limits(
+              key->rt_formats, ARRAY_SIZE(key->rt_formats), key->nr_samples,
+              true, key->max_sample_size_B, key->max_bytes_per_tile,
+              key->max_samples_per_tile);
 
    unsigned rt_order[AGX_MAX_RENDER_TARGETS];
    for (unsigned rt = 0; rt < ARRAY_SIZE(rt_order); ++rt)
@@ -845,6 +852,7 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)
    }
 
    /* Alpha-to-coverage must be lowered before alpha-to-one */
+   const bool discards_before_a2c = b->shader->info.fs.uses_discard;
    if (key->blend.alpha_to_coverage)
       NIR_PASS(_, b->shader, nir_lower_alpha_to_coverage, false,
                NULL);
@@ -871,6 +879,34 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)
       (key->link.write_z || key->link.write_s) && key->nr_samples > 1 &&
       !key->defer_monolithic_msaa;
 
+   /* M5 carries alpha coverage the same way. Native G17G multisample
+    * fragments that write depth (native G17G compiler zs_depth_4x_a2c,
+    * zs_cdepth_4x_a2c and zs_depth_2x_a2c) evaluate it into each sample's
+    * DEPTHF coverage word and issue no MOVMSK; the colour store follows
+    * unmasked. A separate kill ahead of the words would be a second coverage
+    * update. Sample-rate epilogs keep the kill: they are lowered per sample
+    * below. */
+   nir_def *zs_alpha_coverage = NULL;
+   if (key->m5_coverage && zs_carries_write_samples &&
+       key->blend.alpha_to_coverage && !key->link.sample_shading) {
+      nir_foreach_block(block, b->impl) {
+         nir_foreach_instr_safe(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+            if (intr->intrinsic != nir_intrinsic_demote_samples)
+               continue;
+            nir_builder mask_b = nir_builder_at(nir_before_instr(instr));
+            nir_def *covered =
+               nir_u2u16(&mask_b, nir_inot(&mask_b, intr->src[0].ssa));
+            zs_alpha_coverage = zs_alpha_coverage
+               ? nir_iand(&mask_b, zs_alpha_coverage, covered) : covered;
+            nir_instr_remove(instr);
+         }
+      }
+      b->shader->info.fs.uses_discard = discards_before_a2c;
+   }
+
    /* already_ran_zs says the main shader has already triggered the
     * depth/stencil tests, so the epilog must not trigger them again.  It does
     * not say the main shader's coverage has been applied: a forced-early
@@ -942,6 +978,8 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)
          write_samples ? nir_iand(b, write_samples,
                                   nir_load_active_samples_agx(b))
                        : nir_imm_intN_t(b, 0xFF, 16);
+      if (zs_alpha_coverage)
+         zs_samples = nir_iand(b, zs_samples, zs_alpha_coverage);
       nir_def *depth =
          nir_load_exported_agx(b, 1, 32, .base = AGX_ABI_FOUT_Z);
 
@@ -997,7 +1035,10 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)
        * to avoid duplicating tests.
        */
       if (b->shader->info.fs.uses_discard) {
-         NIR_PASS(_, b->shader, agx_nir_lower_sample_mask);
+         if (key->m5_coverage)
+            NIR_PASS(_, b->shader, agx_nir_lower_sample_mask_m5);
+         else
+            NIR_PASS(_, b->shader, agx_nir_lower_sample_mask);
       }
 
       NIR_PASS(_, b->shader, agx_nir_lower_to_per_sample);
@@ -1010,16 +1051,31 @@ agx_nir_fs_epilog_native(nir_builder *b, const void *key_)
       b->cursor = nir_after_impl(b->impl);
       nir_export_agx(b, sample_id, .base = AGX_ABI_FIN_SAMPLE_MASK);
    } else if (!key->defer_monolithic_msaa) {
-      NIR_PASS(_, b->shader, agx_nir_lower_monolithic_msaa, key->nr_samples);
+      if (key->m5_coverage)
+         NIR_PASS(_, b->shader, agx_nir_lower_monolithic_msaa_m5, key->nr_samples);
+      else
+         NIR_PASS(_, b->shader, agx_nir_lower_monolithic_msaa, key->nr_samples);
    } else if (b->shader->info.fs.uses_discard) {
       /* G17P can keep a pixel-shaded multisample blend as one tile
        * load/store pair carrying the live coverage register. Lower the
        * alpha discard while the epilog's test intrinsics are still available;
        * doing it after this function leaves a newly introduced test query
        * past divergence analysis. */
-      NIR_PASS(_, b->shader, agx_nir_lower_sample_mask);
+      if (key->m5_coverage)
+         NIR_PASS(_, b->shader, agx_nir_lower_sample_mask_m5);
+      else
+         NIR_PASS(_, b->shader, agx_nir_lower_sample_mask);
    }
 
+   /* M5 writes whole-pixel depth or stencil alone in one covered-sample
+    * DEPTHF loop, as native G17G does. A sample-rate epilog writes its own
+    * sample instead. A blended epilog's write qualifies once the MSAA
+    * lowering has kept it ahead of the sample loop; one left inside that
+    * loop does not run once and stays as it is. */
+   if (key->m5_coverage && !key->link.sample_shading &&
+       !key->defer_monolithic_msaa)
+      NIR_PASS(_, b->shader, agx_nir_lower_m5_zs_sample_loop, key->nr_samples);
+
    if (native_spills) {
       /* The generated loop has exactly two or four iterations. G17P's MRT
        * transaction is specialized for those sample groups; do not let the
@@ -1249,6 +1305,8 @@ struct lower_epilog_ctx {
    struct agx_fs_epilog_link_info *info;
    nir_variable *masked_samples;
    bool preserve_early_demote;
+   bool preserve_memory_demote;
+   bool preserve_alpha_one;
 };
 
 static bool
@@ -1276,11 +1334,11 @@ lower_output_to_epilog(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 
    if (intr->intrinsic == nir_intrinsic_demote_samples &&
        (b->shader->info.fs.early_fragment_tests ||
+        (ctx->preserve_memory_demote && b->shader->info.writes_memory) ||
         (ctx->preserve_early_demote &&
-         (b->shader->info.writes_memory ||
-          (!b->shader->info.fs.uses_sample_shading &&
-           (b->shader->info.outputs_written &
-            BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK))))))) {
+         !b->shader->info.fs.uses_sample_shading &&
+         (b->shader->info.outputs_written &
+          BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK))))) {
 
       /* G17P carries the discard mask across the split for both early and
        * late tests. LSTCOL consumes it directly, and late DEPTHF must use it
@@ -1366,7 +1424,7 @@ lower_output_to_epilog(nir_builder *b, nir_intrinsic_instr *intr, void *data)
       /* G17P's mixed-width MRT epilog cannot materialize the elided alpha
        * through AGX2's 16-bit immediate move. Keep it in the ordinary export
        * ABI, which is also what native multi-target shaders use. */
-      if (!ctx->preserve_early_demote && loc == 0 && c == 3 &&
+      if (!ctx->preserve_alpha_one && loc == 0 && c == 3 &&
           nir_scalar_is_const(s) &&
           nir_scalar_as_uint(s) == one_f) {
 
@@ -1384,11 +1442,14 @@ lower_output_to_epilog(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 
 static bool
 lower_fs_output_to_epilog(nir_shader *s, struct agx_fs_epilog_link_info *out,
-                          bool preserve_early_demote)
+                          bool preserve_early_demote, bool preserve_memory_demote,
+                          bool preserve_alpha_one)
 {
    struct lower_epilog_ctx ctx = {
       .info = out,
       .preserve_early_demote = preserve_early_demote,
+      .preserve_memory_demote = preserve_memory_demote,
+      .preserve_alpha_one = preserve_alpha_one,
    };
 
    nir_shader_intrinsics_pass(s, lower_output_to_epilog,
@@ -1418,14 +1479,16 @@ bool
 agx_nir_lower_fs_output_to_epilog(nir_shader *s,
                                   struct agx_fs_epilog_link_info *out)
 {
-   return lower_fs_output_to_epilog(s, out, false);
+   return lower_fs_output_to_epilog(s, out, false, false, false);
 }
 
 bool
 agx_nir_lower_fs_output_to_epilog_for_device(
    nir_shader *s, enum agx_chip chip, struct agx_fs_epilog_link_info *out)
 {
-   return lower_fs_output_to_epilog(s, out, agx_chip_native_agx3(chip));
+   return lower_fs_output_to_epilog(
+      s, out, agx_chip_native_agx3(chip) || agx_chip_g17_m5(chip),
+      agx_chip_native_agx3(chip), agx_chip_native_agx3(chip));
 }
 
 static bool
@@ -1650,7 +1713,7 @@ lower_g17p_tile_sample(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 bool
 agx_nir_lower_g17p_tile_samples(nir_shader *s, unsigned nr_samples)
 {
-   if (nr_samples != 1 && nr_samples != 2 && nr_samples != 4)
+   if (nr_samples != 1 && nr_samples != 2 && nr_samples != 4 && nr_samples != 8)
       return false;
 
    struct g17p_tile_sample_state state = {.samples = nr_samples};
@@ -1795,6 +1858,11 @@ agx_nir_fs_epilog_g17p(nir_builder *b, const void *key_)
          opt_progress = false;
          NIR_PASS(opt_progress, b->shader, nir_opt_constant_folding);
          NIR_PASS(opt_progress, b->shader, nir_opt_algebraic);
+         /* M5 folds dead branches so its sample-rate coverage updates
+          * coalesce ahead of the colour stores. Other parts keep their
+          * qualified epilog shape. */
+         if (key->m5_coverage)
+            NIR_PASS(opt_progress, b->shader, nir_opt_dead_cf);
       } while (opt_progress);
       NIR_PASS(_, b->shader, agx_nir_hoist_g17p_zs_writes);
    } else {
@@ -1816,22 +1884,23 @@ agx_nir_fs_epilog_g17p(nir_builder *b, const void *key_)
  * and its epilog. Keep layout selection and the register-file limit together.
  */
 bool
-agx_nir_lower_g17p_sample_shading(nir_shader *nir,
-                                  struct agx_fs_epilog_link_info *link,
-                                  unsigned samples)
+agx_nir_lower_g17p_sample_shading_file(nir_shader *nir,
+                                       struct agx_fs_epilog_link_info *link,
+                                       unsigned samples, unsigned registers,
+                                       bool eight_samples)
 {
-   if (samples != 2 && samples != 4)
+   if (samples != 2 && samples != 4 && !(eight_samples && samples == 8))
       return false;
 
    struct agx_g17p_colour_layout layout;
    agx_g17p_colour_layout_init(&layout, link->loc_written, link->size_32,
                                link->colour_comps, false);
-   if (agx_g17p_sample_exports_end(&layout, samples) > AGX3B_NUM_REGS) {
+   if (agx_g17p_sample_exports_end(&layout, samples) > registers) {
       NIR_PASS(_, nir, agx_nir_pack_g17p_colour_exports, link);
       agx_g17p_colour_layout_init(&layout, link->loc_written, link->size_32,
                                   link->colour_comps, true);
    }
-   if (agx_g17p_sample_exports_end(&layout, samples) > AGX3B_NUM_REGS)
+   if (agx_g17p_sample_exports_end(&layout, samples) > registers)
       return false;
 
    NIR_PASS(_, nir, agx_nir_lower_to_per_sample);
@@ -1839,3 +1908,13 @@ agx_nir_lower_g17p_sample_shading(nir_shader *nir,
    nir->info.fs.uses_sample_shading = false;
    return true;
 }
+
+/* G17P: two or four samples within its 192-half register file. */
+bool
+agx_nir_lower_g17p_sample_shading(nir_shader *nir,
+                                  struct agx_fs_epilog_link_info *link,
+                                  unsigned samples)
+{
+   return agx_nir_lower_g17p_sample_shading_file(nir, link, samples,
+                                                 AGX3B_NUM_REGS, false);
+}
diff --git a/src/asahi/lib/agx_pbe_agx3_core.c b/src/asahi/lib/agx_pbe_agx3_core.c
index 18065174c7aa..c002aa69ae49 100644
--- a/src/asahi/lib/agx_pbe_agx3_core.c
+++ b/src/asahi/lib/agx_pbe_agx3_core.c
@@ -83,20 +83,25 @@ fields_valid(uint32_t generation,
    const bool strided = linear || interchange;
    const bool stride_valid =
       strided ? fields->stride_B >= 16 && !(fields->stride_B & 0xf) &&
-                   fields->stride_B - 16 < (UINT32_C(1) << 21)
+                   fields->stride_B - 16 <
+                      (UINT32_C(1) <<
+                       (generation == AGX_PBE_AGX3_GENERATION_10 ? 20 : 21))
               : fields->stride_B == 0;
    const bool address_valid =
       fields->buffer && !(fields->buffer & 0xf) &&
       !(fields->buffer >> (strided ? 44 : 48));
    const bool compression_valid = fields->compressed
-      ? generation == AGX_PBE_AGX3_GENERATION_9 &&
-           (fields->layout == AGX_PBE_LAYOUT_GPU || interchange) &&
+      ? (generation == AGX_PBE_AGX3_GENERATION_9 ||
+         generation == AGX_PBE_AGX3_GENERATION_10) &&
+           (fields->layout == AGX_PBE_LAYOUT_GPU ||
+            (interchange && generation == AGX_PBE_AGX3_GENERATION_9)) &&
+           (generation != AGX_PBE_AGX3_GENERATION_10 ||
+            (!(fields->buffer & 0x3f) && !fields->page_aligned_layers)) &&
            fields->acceleration_buffer &&
            !(fields->acceleration_buffer & 0xf) &&
            !(fields->acceleration_buffer >> 48)
       : !fields->acceleration_buffer;
-   return (!strided || generation == AGX_PBE_AGX3_GENERATION_9) &&
-          (!fields->srgb || generation == AGX_PBE_AGX3_GENERATION_9) &&
+   return (!interchange || generation == AGX_PBE_AGX3_GENERATION_9) &&
           (strided || fields->layout == AGX_PBE_LAYOUT_TWIDDLED ||
            fields->layout == AGX_PBE_LAYOUT_GPU) &&
           channel_is_uncompressed(fields->channels) && fields->type <= 5 &&
@@ -132,8 +137,12 @@ agx_pbe_agx3_2d_pack(uint32_t generation,
    word0 |= (uint64_t)fields->swizzle_a << 22;
    word0 |= (uint64_t)(fields->width - 1u) << 24;
    word0 |= (uint64_t)(fields->height - 1u) << height_shift;
+   /* The native HAL300 PBE encoder puts compression at bit61,
+    * whereas HAL200 uses bit59. Both extend word1 and write aux>>4 in
+    * word2. Full single-level packets are checked against both oracles. */
    if (fields->compressed)
-      word0 |= UINT64_C(1) << 59;
+      word0 |= UINT64_C(1) <<
+               (generation == AGX_PBE_AGX3_GENERATION_10 ? 61 : 59);
 
    /* Gen9 StateFields (0x34f188..0x34f1f8) only writes word1 bit60 for
     * array/type-flagged or kind3/4 images. This codec admits ordinary kind2,
@@ -156,6 +165,10 @@ agx_pbe_agx3_2d_pack(uint32_t generation,
     * that select bit 62 as well (+0x22c..+0x2dc). TextureGen4's
     * encodeImageReadState turns the same field into the texture descriptor's
     * sRGB bits, and AGX2's PBE keeps its sRGB store conversion at this bit. */
+   /* Gen10 (M5) converts stores to sRGB at the same word1 bit 61: an
+    * R8G8B8A8_SRGB storage image and storage texel buffer stored
+    * vec4(0.5, 0.2, 0.735, 0.5) as bc 7c df 80 with it and as the linear
+    * 80 33 bb 80 without it (port/srgbsweep, vmfix6c). */
    word1 |= (uint64_t)fields->srgb << 61;
    if (fields->compressed)
       word1 |= UINT64_C(1) << 63;
@@ -180,13 +193,14 @@ agx_pbe_agx3_2d_pack(uint32_t generation,
  * layered targets take the same extension.
  */
 static void
-mark_mip_chain(struct agx_pbe_agx3_descriptor *out, uint32_t level,
+mark_mip_chain(uint32_t generation, struct agx_pbe_agx3_descriptor *out, uint32_t level,
                uint32_t levels)
 {
    if (!level && levels == 1)
       return;
 
-   agx_store_le64(&out->bytes[0], agx_load_le64(&out->bytes[0]) | (UINT64_C(1) << 58));
+   agx_store_le64(&out->bytes[0], agx_load_le64(&out->bytes[0]) |
+                  (UINT64_C(1) << (generation == AGX_PBE_AGX3_GENERATION_10 ? 60 : 58)));
    agx_store_le64(&out->bytes[8], agx_load_le64(&out->bytes[8]) | (UINT64_C(1) << 63));
    agx_store_le64(&out->bytes[16],
               (agx_load_le64(&out->bytes[16]) & ((UINT64_C(1) << 44) - 1)) |
@@ -209,11 +223,25 @@ pack_layered(uint32_t generation, const struct agx_pbe_agx3_2d_fields *fields,
    if (out)
       memset(out, 0, sizeof(*out));
 
-   /* The exact HAL200 producer keeps layered targets tiled in this packet.
-    * Linear stride and Gen10's shifted mode fields remain independently
-    * gated. */
-   if (generation != AGX_PBE_AGX3_GENERATION_9 || !fields ||
-       fields->compressed ||
+   /* The pinned native G17X Gen10 PBE encoder selects array
+    * dimension 3, deposits layers-1 at word1[59:44], and
+    * controls page alignment at word1[60]. Full 2/4/16-layer
+    * packets agree over Gen10's own 2D width/height geometry.
+    * The 3D branch selects dimension 5, then takes its
+    * depth from w2 and deposits depth-1 in the same field.
+    * Gen10 mip chains retain the shared level fields but move the extension
+    * flag from word0 bit58 to bit60. */
+   const bool m5_layered = generation == AGX_PBE_AGX3_GENERATION_10 &&
+      (dimension == AGX_PBE_DIMENSION_2D_ARRAY ||
+       dimension == AGX_PBE_DIMENSION_2D_ARRAY_MULTISAMPLE ||
+       dimension == AGX_PBE_DIMENSION_3D);
+   if ((generation != AGX_PBE_AGX3_GENERATION_9 && !m5_layered) || !fields ||
+       (fields->compressed &&
+        !((generation == AGX_PBE_AGX3_GENERATION_9 &&
+           fields->layout == AGX_PBE_LAYOUT_GPU && level == 0 && levels == 1) ||
+          (generation == AGX_PBE_AGX3_GENERATION_10 &&
+           dimension == AGX_PBE_DIMENSION_2D_ARRAY && layers == 1 &&
+           level == 0 && levels == 1))) ||
        fields->layout == AGX_PBE_LAYOUT_LINEAR || layers < 1 ||
        layers > UINT16_MAX + 1u || !levels || levels > 16 ||
        level >= levels ||
@@ -227,7 +255,7 @@ pack_layered(uint32_t generation, const struct agx_pbe_agx3_2d_fields *fields,
    word1 |= (uint64_t)fields->page_aligned_layers << 60;
    agx_store_le64(&out->bytes[0], word0);
    agx_store_le64(&out->bytes[8], word1);
-   mark_mip_chain(out, level, levels);
+   mark_mip_chain(generation, out, level, levels);
    return true;
 }
 
@@ -270,13 +298,42 @@ agx_pbe_agx3_2d_multisample_pack(
 
    /* A multisample colour target is never linear and never mipmapped: the
     * end-of-tile store addresses it through the sample-expanded twiddled
-    * layout. Refuse 8x and every other count: the encoder's selector covers
-    * them, but no tilebuffer geometry does. Gen10 has its own encoder and
-    * its own witness to recover, so it is refused here too.
+    * layout. The pinned native PBE encoder deposits its sample selector at
+    * word0[59:58], two bits above Gen9's. Native 2x/4x
+    * complete packets ground both generations here, and the native 8x packet
+    * grounds Gen10's selector 2. Tile profiles are qualified independently.
     */
-   if (generation != AGX_PBE_AGX3_GENERATION_9 || !fields ||
-       fields->compressed ||
-       (samples != 2 && samples != 4) ||
+   /* Preserve raw D32 copies alongside the private RGBA16 and RGBA8 color
+    * cases. A compressed multisample packet is the compressed single-sample
+    * packet with the multisample dimension and selector; the metadata follows
+    * the sample-expanded surface, so it depends only on the texel size, which
+    * RGBA8 shares with the witnessed R32_UINT shape. Gen10 keeps the sRGB
+    * store conversion at word1 bit 61, as in the single-sample packet. Image
+    * allocation policy remains separate from packet encoding. */
+   const bool compressed_r32_uint = fields &&
+      generation == AGX_PBE_AGX3_GENERATION_10 &&
+      fields->layout == AGX_PBE_LAYOUT_GPU &&
+      fields->channels == 0x21 /* R32 */ && fields->type == 2 /* UINT */ &&
+      !fields->srgb && (samples == 2 || samples == 4);
+   const bool compressed_rgba16 = fields &&
+      generation == AGX_PBE_AGX3_GENERATION_10 &&
+      fields->layout == AGX_PBE_LAYOUT_GPU &&
+      fields->channels == 0x32 /* R16G16B16A16 */ &&
+      (fields->type == 4 /* FLOAT */ || fields->type == 2 /* UINT */) &&
+      !fields->srgb && (samples == 2 || samples == 4);
+   const bool compressed_rgba8 = fields &&
+      generation == AGX_PBE_AGX3_GENERATION_10 &&
+      fields->layout == AGX_PBE_LAYOUT_GPU &&
+      fields->channels == 0x28 /* R8G8B8A8 */ && fields->type == 0 /* UNORM */ &&
+      (samples == 2 || samples == 4);
+   if ((generation != AGX_PBE_AGX3_GENERATION_9 &&
+        generation != AGX_PBE_AGX3_GENERATION_10) || !fields ||
+       (fields->compressed && !compressed_r32_uint && !compressed_rgba16 &&
+        !compressed_rgba8 &&
+        !(generation == AGX_PBE_AGX3_GENERATION_9 &&
+          fields->layout == AGX_PBE_LAYOUT_GPU)) ||
+       (samples != 2 && samples != 4 &&
+        !(generation == AGX_PBE_AGX3_GENERATION_10 && samples == 8)) ||
        fields->layout == AGX_PBE_LAYOUT_LINEAR ||
        !agx_pbe_agx3_2d_pack(generation, fields, out))
       return false;
@@ -288,7 +345,8 @@ agx_pbe_agx3_2d_multisample_pack(
    const uint64_t selector = (uint64_t)((samples >> 2) & 3u);
    uint64_t word0 = agx_load_le64(&out->bytes[0]);
    word0 = (word0 & ~UINT64_C(0xf)) | AGX_PBE_DIMENSION_2D_MULTISAMPLE;
-   word0 |= selector << AGX_PBE_SAMPLE_SELECTOR_SHIFT;
+   word0 |= selector << (generation == AGX_PBE_AGX3_GENERATION_10
+                            ? 58 : AGX_PBE_SAMPLE_SELECTOR_SHIFT);
    agx_store_le64(&out->bytes[0], word0);
    return true;
 }
@@ -304,7 +362,8 @@ agx_pbe_agx3_2d_array_multisample_pack(
 
    /* HAL200's array+multisample path selects dimension 8 while retaining the
     * layered word1 fields and the ordinary multisample selector. */
-   if ((samples != 2 && samples != 4) ||
+   if ((samples != 2 && samples != 4 &&
+        !(generation == AGX_PBE_AGX3_GENERATION_10 && samples == 8)) ||
        !pack_layered(generation, fields,
                      AGX_PBE_DIMENSION_2D_ARRAY_MULTISAMPLE, layers, 0, 1,
                      out))
@@ -312,7 +371,8 @@ agx_pbe_agx3_2d_array_multisample_pack(
 
    uint64_t word0 = agx_load_le64(&out->bytes[0]);
    word0 |= (uint64_t)((samples >> 2) & 3u)
-            << AGX_PBE_SAMPLE_SELECTOR_SHIFT;
+            << (generation == AGX_PBE_AGX3_GENERATION_10
+                   ? 58 : AGX_PBE_SAMPLE_SELECTOR_SHIFT);
    agx_store_le64(&out->bytes[0], word0);
    return true;
 }
@@ -324,13 +384,23 @@ agx_pbe_agx3_2d_mip_pack(
 {
    if (out)
       memset(out, 0, sizeof(*out));
-   if (generation != AGX_PBE_AGX3_GENERATION_9 || !levels || levels > 16 ||
+   /* A compressed Gen10 level keeps the compressed single-level packet
+    * (word0 bit61, word1 bit63, word2[43:0] = metadata >> 4) and takes the
+    * same chain fields as an uncompressed one: the pinned native PBE encoder
+    * (level and level count passed as arguments) gives
+    * 0x30001f803fe40a22 / 0x8000000001234500 / 0x0004200004567800 for an
+    * RGBA8 UNORM 64x64 level 2 of 5 with metadata 0x45678000. The metadata
+    * address stays the level-0 one: the native framebuffer PBE setup asks
+    * for the GPU address of level 0 of both the surface and the metadata
+    * (two calls on the same path). */
+   if ((generation != AGX_PBE_AGX3_GENERATION_9 &&
+        generation != AGX_PBE_AGX3_GENERATION_10) || !levels || levels > 16 ||
        (fields && fields->layout == AGX_PBE_LAYOUT_LINEAR &&
         (level || levels != 1)) ||
        level >= levels || !agx_pbe_agx3_2d_pack(generation, fields, out))
       return false;
 
-   mark_mip_chain(out, level, levels);
+   mark_mip_chain(generation, out, level, levels);
    return true;
 }
 
@@ -357,10 +427,10 @@ agx_pbe_agx3_2d_unpack(uint32_t generation, const void *bytes, size_t size,
          : (UINT64_C(1) << 54) - 1;
 
    const uint8_t layout = (word0 >> 4) & 0x3;
-   const bool compressed = generation == AGX_PBE_AGX3_GENERATION_9 &&
-                           (word0 & (UINT64_C(1) << 59));
-   const uint64_t srgb_bit = generation == AGX_PBE_AGX3_GENERATION_9
-      ? UINT64_C(1) << 61 : 0;
+   const unsigned compression_bit =
+      generation == AGX_PBE_AGX3_GENERATION_10 ? 61 : 59;
+   const bool compressed = (word0 & (UINT64_C(1) << compression_bit)) != 0;
+   const uint64_t srgb_bit = UINT64_C(1) << 61;
    const uint64_t word1_mask = layout == AGX_PBE_LAYOUT_LINEAR
       ? ((UINT64_C(1) << 61) - 1) | srgb_bit
       : layout == AGX_PBE_LAYOUT_INTERCHANGE
@@ -370,7 +440,7 @@ agx_pbe_agx3_2d_unpack(uint32_t generation, const void *bytes, size_t size,
            (compressed ? UINT64_C(1) << 63 : 0) |
            (generation == AGX_PBE_AGX3_GENERATION_10 ? UINT64_C(1) << 60 : 0);
    const uint64_t permitted_word0 =
-      word0_mask | (compressed ? UINT64_C(1) << 59 : 0);
+      word0_mask | (compressed ? UINT64_C(1) << compression_bit : 0);
    if ((word0 & 0xf) != AGX_PBE_DIMENSION_2D ||
        (word0 & ~permitted_word0) || (word1 & ~word1_mask) ||
        (compressed ? (word2 & ~((UINT64_C(1) << 44) - 1)) : word2) ||
diff --git a/src/asahi/lib/agx_pbe_agx3_core.h b/src/asahi/lib/agx_pbe_agx3_core.h
index 2d6ac1517a52..9c8bc298d71d 100644
--- a/src/asahi/lib/agx_pbe_agx3_core.h
+++ b/src/asahi/lib/agx_pbe_agx3_core.h
@@ -20,8 +20,8 @@ extern "C" {
 #define AGX_PBE_AGX3_GENERATION_10     10u
 
 /* This is intentionally the subset whose complete byte geometry is recovered:
- * ordinary non-sparse 2D images in Gen9 linear or Gen9/10 twiddled/GPU
- * memory, Gen9 GPU-tiled compressed 2D images, and Gen9's exact uncompressed
+ * ordinary non-sparse 2D images in Gen9/10 linear or twiddled/GPU
+ * memory, Gen9/10 GPU-tiled compressed 2D images, and Gen9's exact uncompressed
  * tiled 2D-array extension.
  * Gen9's ordinary single-layer packet has no layer-mode bit regardless of
  * allocation padding; the inherited page-aligned-layer input is ignored on
@@ -40,11 +40,11 @@ struct agx_pbe_agx3_2d_fields {
    uint16_t width;
    uint16_t height;
    uint64_t buffer;
-   /* Gen9 GPU-tiled 2D lossless compression metadata base. Exactly zero when
+   /* Gen9/10 GPU-tiled 2D lossless compression metadata base. Exactly zero when
     * compressed is false; 16-byte aligned and below bit 48 otherwise. */
    uint64_t acceleration_buffer;
    bool compressed;
-   /* Gen9 linear row pitch in bytes. Zero for tiled layouts. */
+   /* Linear row pitch in bytes. Zero for tiled layouts. */
    uint32_t stride_B;
    /* Legacy layout metadata: encoded only for Gen10, not ordinary Gen9 2D. */
    bool page_aligned_layers;
diff --git a/src/asahi/lib/agx_ppp_hal200_graphics_core.c b/src/asahi/lib/agx_ppp_hal200_graphics_core.c
index 810359baa716..44705198e9c3 100644
--- a/src/asahi/lib/agx_ppp_hal200_graphics_core.c
+++ b/src/asahi/lib/agx_ppp_hal200_graphics_core.c
@@ -420,7 +420,8 @@ agx_ppp_hal200_fragment_face_depth_pack(
     * inserts this mode into both SECONDARY face words. Isolated native
     * witnesses give ANY=0, GREATER=1, LESS=2, UNCHANGED=3; adding a late
     * atomic to either conservative depth shader changes its mode to ANY. */
-   unsigned mode = early_tests ? 3 : writes_memory ? 0 : conservative_depth;
+   unsigned mode = agx_ppp_fragment_depth_mode(conservative_depth, early_tests,
+                                              writes_memory);
    *out = (original & ~UINT32_C(0x00c00000)) | (mode << 22);
    return true;
 }
diff --git a/src/asahi/lib/agx_ppp_hal200_graphics_core.h b/src/asahi/lib/agx_ppp_hal200_graphics_core.h
index ac04079de34f..0cddfbbec6f0 100644
--- a/src/asahi/lib/agx_ppp_hal200_graphics_core.h
+++ b/src/asahi/lib/agx_ppp_hal200_graphics_core.h
@@ -65,6 +65,13 @@ bool agx_ppp_hal200_fragment_face_object_pack(
 /* SECONDARY front/back face bits22:23 carry conservative depth. Native
  * late-memory shaders select ANY even with a conservative depth qualifier;
  * explicit early tests select UNCHANGED and ignore shader depth writes. */
+static inline unsigned
+agx_ppp_fragment_depth_mode(unsigned conservative_depth, bool early_tests,
+                           bool writes_memory)
+{
+   return early_tests ? 3 : writes_memory ? 0 : conservative_depth;
+}
+
 bool agx_ppp_hal200_fragment_face_depth_pack(
    enum agx_chip chip, unsigned conservative_depth, bool early_tests,
    bool writes_memory, uint32_t original, uint32_t *out);
diff --git a/src/asahi/lib/agx_ppp_hal300_graphics_core.c b/src/asahi/lib/agx_ppp_hal300_graphics_core.c
new file mode 100644
index 000000000000..24995965f9f6
--- /dev/null
+++ b/src/asahi/lib/agx_ppp_hal300_graphics_core.c
@@ -0,0 +1,225 @@
+/* SPDX-License-Identifier: MIT */
+#include "agx_ppp_hal300_graphics_core.h"
+#include "agx_vdm_state_core.h"
+#include <string.h>
+
+bool
+agx_ppp_hal300_graphics_core_pack(
+   enum agx_chip chip, const struct agx_ppp_hal300_graphics_fields *fields,
+   struct agx_ppp_hal300_graphics_core *out)
+{
+
+   if (!agx_chip_g17_m5(chip) || !fields || !out ||
+       !fields->fragment_program || fields->cull_mode > 3 ||
+       fields->clip_distance_words > 8 || fields->viewport_count > 16 ||
+       fields->fragment_pass > 3 || fields->flat_shading_vertex > 2)
+      return false;
+
+   const uint64_t ti_spill_quads =
+      ((uint64_t)fields->fragment_ti_spill_B + 3) >> 2;
+   const uint64_t ti_staging_quads =
+      ((((uint64_t)fields->fragment_ti_staging_B + 3) >> 2) + 1) &
+      ~UINT64_C(1);
+   if (ti_spill_quads > 0x3ffff || ti_staging_quads > 0xffe ||
+       ((ti_spill_quads || ti_staging_quads) && !agx_chip_g17_m5(chip)))
+      return false;
+
+   if (fields->general_varyings) {
+      /* HAL300's native VSStateConfig and VIT producer retain the shared
+       * all-f32 UVS/CF ABI: 128 total UVS words and 127 coefficients. */
+      const unsigned smooth = fields->varying_counts_32 & 0xff;
+      const unsigned flat = (fields->varying_counts_32 >> 8) & 0xff;
+      const unsigned linear = (fields->varying_counts_32 >> 16) & 0xff;
+      const uint32_t expected_output_select =
+         (fields->point_size_output ? 0x40000 : 0) |
+         (fields->layer_output ? 0x180000 : 0) |
+         ((fields->vertex_varying_components || fields->cf_binding_count)
+             ? 0x10000 : 0) |
+         (fields->fragment_reads_depth ? 0x200000 : 0) |
+         ((UINT32_C(1) << fields->clip_distance_words) - 1);
+      if (fields->vertex_varying_components > 124 ||
+          fields->fragment_varying_components >
+             fields->vertex_varying_components ||
+          fields->varying_counts_16 || (fields->varying_counts_32 >> 24) ||
+          smooth + flat + linear != fields->vertex_varying_components ||
+          fields->uvs_words > 128 ||
+          fields->uvs_words != 4 + fields->vertex_varying_components +
+             fields->point_size_output + fields->layer_output +
+             fields->clip_distance_words ||
+          fields->output_select != expected_output_select ||
+          !fields->cf_binding_count || fields->cf_binding_count > 127 ||
+          fields->cf_register_count < fields->cf_binding_count ||
+          fields->cf_register_count > 127 ||
+          !fields->varying_table || (fields->varying_table & 63) ||
+          fields->varying_table >= (UINT64_C(1) << 48))
+         return false;
+   } else {
+      if (fields->vertex_varying_components >
+             ((fields->rasterizer_discard ||
+               !fields->fragment_varying_components) ? 124 : 8) ||
+          4 + fields->point_size_output + fields->layer_output +
+             fields->clip_distance_words + fields->vertex_varying_components > 128 ||
+          fields->fragment_varying_components >
+             fields->vertex_varying_components || fields->needs_primitive_id)
+         return false;
+
+      if (fields->fragment_varying_components || fields->fragment_reads_depth ||
+          fields->fragment_reads_reciprocal_w) {
+         if (!fields->varying_table ||
+             (fields->varying_table & 63) ||
+             fields->varying_table >= (UINT64_C(1) << 48) ||
+             !fields->cf_binding_count || fields->cf_binding_count > 3 ||
+             (fields->interpolation != AGX_PPP_HAL300_SMOOTH &&
+              fields->cf_binding_count > 2 &&
+              !fields->fragment_reads_reciprocal_w))
+            return false;
+      } else if (fields->varying_table || fields->cf_binding_count ||
+                 fields->interpolation != AGX_PPP_HAL300_SMOOTH) {
+         return false;
+      }
+      if (fields->interpolation > AGX_PPP_HAL300_LINEAR)
+         return false;
+
+      if (fields->no_color_attachment &&
+          fields->interpolation == AGX_PPP_HAL300_FLAT &&
+          fields->fragment_varying_components)
+         return false;
+   }
+   const bool psiz = fields->general_varyings
+                        ? (fields->output_select & 0x40000) != 0
+                        : fields->point_size_output;
+
+   if (fields->object_type > AGX_PPP_HAL300_LINES ||
+       (fields->object_type == AGX_PPP_HAL300_POINTS && !psiz &&
+        !fields->rasterizer_discard))
+      return false;
+
+   if ((fields->visibility_mode != 0 && fields->visibility_mode != 2 &&
+        fields->visibility_mode != 3) ||
+       fields->visibility_index >= (1u << 15))
+      return false;
+
+   if (fields->line_width > 0xff || fields->polygon_mode > 2 ||
+       fields->fragment_depth_mode > 3 || fields->depth_compare > 7 ||
+       (!fields->depth_active && (fields->depth_compare || fields->depth_write)))
+      return false;
+
+   const uint8_t compare = fields->depth_active ? fields->depth_compare : 7;
+   const bool write = fields->depth_active && fields->depth_write;
+   const bool depth_inactive = !fields->depth_active;
+   const bool points = fields->object_type == AGX_PPP_HAL300_POINTS;
+   const bool lines = fields->object_type == AGX_PPP_HAL300_LINES;
+   const uint32_t control = 0x00000200 | (points || lines ? 0x04000000 : 0) |
+                            (fields->scissor_enable ? 0x00010000 : 0) |
+                            (fields->depth_bias_enable ? 0x00020000 : 0) |
+                            ((depth_inactive || fields->stencil_active) ? 0x00040000 : 0) |
+                            (fields->stencil_active ? 0x00080000 : 0) |
+                            /* The native visibility-result-mode setter
+                             * sets bits 15:14 of this word (0, 2 counting,
+                             * 3 boolean) and offset / 8 into bits 31:17 of
+                             * the word 0x14 bytes after it. */
+                            ((uint32_t)fields->visibility_mode << 14);
+   const uint32_t visibility_query =
+      fields->visibility_mode ? (uint32_t)fields->visibility_index << 17 : 0;
+   const uint32_t face = ((uint32_t)compare << 24) |
+                         (write ? 0 : 0x00200000) | (fields->line_width << 8) |
+                         (fields->polygon_mode << 18);
+
+   /* HAL300 setFragmentVariant(nullptr) disables tag writes, clears its
+    * fragment output mask and selects depth mode 3. Rasterizer discard
+    * selects that inert fragment state while the VDM still runs the VS. */
+   const uint32_t early_test =
+      0x07200000 |
+      ((fields->rasterizer_discard ? 3 : fields->fragment_depth_mode) << 22) |
+      (points ? 0x40000000 : 0) | (lines ? 0x10000000 : 0);
+
+   struct agx_vdm_state_vertex_pipeline_pair program;
+
+   const uint32_t program_template = agx_chip_g17_m5(chip)
+      ? (uint32_t)(ti_spill_quads << 3)
+      : fields->fragment_writes_depth ? 0x40 : 0;
+   if (!agx_vdm_state_vertex_pipeline_pair_pack(
+          chip, fields->fragment_program, program_template, &program))
+      return false;
+
+   const uint32_t counts_32 =
+      fields->general_varyings
+         ? fields->varying_counts_32
+         : fields->fragment_varying_components <<
+              (fields->no_color_attachment
+                  ? 16
+                  : fields->interpolation == AGX_PPP_HAL300_FLAT ? 8 : 0);
+   const uint32_t output_select =
+      fields->general_varyings
+         ? fields->output_select
+         : (fields->point_size_output ? 0x40000 : 0) |
+              (fields->layer_output ? 0x180000 : 0) |
+              /* System Z/1-W bindings also require coefficient production.
+               * Keep the shared linker's OUTPUT_SELECT.varyings contract. */
+              ((fields->vertex_varying_components || fields->cf_binding_count)
+                  ? 0x10000 : 0) |
+              (fields->fragment_reads_depth ? 0x200000 : 0) |
+              ((UINT32_C(1) << fields->clip_distance_words) - 1);
+   const uint32_t uvs_words =
+      fields->general_varyings
+         ? fields->uvs_words
+         : 4 + (fields->point_size_output ? 1 : 0) +
+              (fields->layer_output ? 1 : 0) +
+              fields->clip_distance_words + fields->vertex_varying_components;
+   const uint32_t records[][7] = {
+      {0x10100000, counts_32,
+       fields->general_varyings ? fields->varying_counts_16 : 0, 0, 0},
+      {0x01000000, (uint32_t)(fields->varying_table >> 32) |
+                      (fields->cf_binding_count << 16),
+       (uint32_t)fields->varying_table, (uint32_t)ti_staging_quads, 0, 0, 0},
+      {0x00000001, 0x000007aa, 0xe2e00000, 0xf0c00000,
+       0x00023020, 0x0001ffff},
+      {0x0c280000, output_select, 0,
+       fields->viewport_count ? fields->viewport_count - 1 : 0, uvs_words},
+      {0x020000e0, control,
+       face | fields->front_stencil_reference,
+       face | fields->back_stencil_reference,
+       fields->stencil_active ? fields->front_stencil : 0x0e000000,
+       fields->stencil_active ? fields->back_stencil : 0x0e000000,
+       visibility_query},
+
+      /* Bit 21 is the tag-write disable that setFragmentVariant(nullptr)
+       * selects; a linked fragment that writes no tile needs it as well. */
+      {0x02000062, 0x00000200 |
+                      (fields->rasterizer_discard ? 0x00200000
+                                                : fields->fragment_pass << 29) |
+                      (fields->tag_write_disable ? 0x00200000 : 0) |
+                      (fields->disable_tri_merging || fields->polygon_mode || points || lines
+                          ? 0x04000000
+                          : 0),
+       early_test, early_test,
+       (fields->no_color_attachment || fields->no_color_writes ||
+        fields->rasterizer_discard)
+          ? 0 : 0x0001ffff},
+
+      {0x00840002, 0,
+       (fields->rasterizer_discard ? 0x00020080 : 0x80) |
+          (fields->needs_primitive_id ? 0x1000 : 0)},
+
+      {0x00800000, (fields->depth_clip_disable ? 0x800 : 0x400) |
+                      ((fields->flat_shading_vertex + 1) << 7) |
+                      fields->cull_mode |
+                      ((uint32_t)fields->front_face_ccw << 16) |
+                      (fields->primitive_msaa ? 0x8000 : 0)},
+      {0x00000400, 0, 0},
+   };
+   const uint8_t lengths[] = {20, 28, 24, 20, 28, 20, 12, 8, 12};
+   struct agx_ppp_hal300_graphics_core staged = {0};
+   unsigned offset = 0;
+   for (unsigned i = 0; i < AGX_PPP_HAL300_GRAPHICS_RECORDS; ++i) {
+      staged.offsets[i] = offset;
+      staged.lengths[i] = lengths[i];
+      for (unsigned j = 0; j < lengths[i]; ++j)
+         staged.bytes[offset + j] = records[i][j / 4] >> (8 * (j % 4));
+      offset += lengths[i];
+   }
+
+   memcpy(staged.bytes + staged.offsets[1] + 16, program.bytes, 8);
+   *out = staged;
+   return true;
+}
diff --git a/src/asahi/lib/agx_ppp_hal300_graphics_core.h b/src/asahi/lib/agx_ppp_hal300_graphics_core.h
new file mode 100644
index 000000000000..72ec8bf61253
--- /dev/null
+++ b/src/asahi/lib/agx_ppp_hal300_graphics_core.h
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: MIT */
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "agx_chip.h"
+
+#define AGX_PPP_HAL300_GRAPHICS_RECORDS 9u
+#define AGX_PPP_HAL300_GRAPHICS_BYTES 172u
+
+struct agx_ppp_hal300_graphics_fields {
+   uint64_t fragment_program;
+   uint64_t varying_table;
+   unsigned cf_binding_count;
+
+   uint32_t fragment_ti_spill_B;
+   uint32_t fragment_ti_staging_B;
+   unsigned cull_mode;
+
+   bool scissor_enable;
+   bool depth_bias_enable;
+   bool depth_clip_disable;
+   unsigned polygon_mode;
+   bool primitive_msaa;
+   /* Encoded 4:4 fixed-point line width, with the hardware off-by-one. */
+   unsigned line_width;
+
+   bool no_color_attachment;
+   bool no_color_writes;
+
+   unsigned fragment_pass;
+   bool disable_tri_merging;
+
+   bool fragment_writes_depth;
+   /* Native conservative-depth mode after early-test/side-effect policy. */
+   unsigned fragment_depth_mode;
+   bool fragment_reads_depth;
+   /* 1/W uses the existing position coefficient, with no user UVS word. */
+   bool fragment_reads_reciprocal_w;
+   bool front_face_ccw;
+   bool point_size_output;
+   bool layer_output;
+
+   unsigned clip_distance_words;
+
+   enum agx_ppp_hal300_object_type {
+      AGX_PPP_HAL300_TRIANGLES = 0,
+      AGX_PPP_HAL300_POINTS = 1,
+      AGX_PPP_HAL300_LINES = 2,
+   } object_type;
+
+   unsigned vertex_varying_components;
+   unsigned fragment_varying_components;
+
+   enum agx_ppp_hal300_interpolation {
+      AGX_PPP_HAL300_SMOOTH = 0,
+      AGX_PPP_HAL300_FLAT = 1,
+      AGX_PPP_HAL300_LINEAR = 2,
+   } interpolation;
+
+   bool depth_active;
+   uint8_t depth_compare;
+   bool depth_write;
+   bool stencil_active;
+
+   uint8_t front_stencil_reference;
+   uint8_t back_stencil_reference;
+   uint32_t front_stencil;
+   uint32_t back_stencil;
+
+   bool rasterizer_discard;
+   /* FRAGMENT_CONTROL bit 21 of the linked fragment: no part writes the
+    * tile, so the fragment must not take the visibility tag. */
+   bool tag_write_disable;
+
+   /* Checked all-f32 shared VIT/UVS layout, including mixed interpolation. */
+   bool general_varyings;
+   unsigned cf_register_count;
+   uint32_t varying_counts_32;
+   uint32_t varying_counts_16;
+   uint32_t output_select;
+   uint32_t uvs_words;
+
+   bool needs_primitive_id;
+
+   /* Visibility (occlusion) result: AGX_VISIBILITY_MODE_* (0, 2 counting, 3
+    * boolean) and the 64-bit slot index in the render's result buffer. */
+   uint8_t visibility_mode;
+   uint16_t visibility_index;
+
+   unsigned flat_shading_vertex;
+
+   /* Zero retains the single-viewport default for standalone builders. */
+   unsigned viewport_count;
+};
+
+#define AGX_PPP_HAL300_STENCIL_KEEP_ALWAYS UINT32_C(0x0e000000)
+#define AGX_PPP_HAL300_STENCIL_REPLACE_ALWAYS UINT32_C(0x0e020000)
+#define AGX_PPP_HAL300_STENCIL_KEEP_EQUAL UINT32_C(0x04000000)
+
+struct agx_ppp_hal300_graphics_core {
+   uint8_t bytes[AGX_PPP_HAL300_GRAPHICS_BYTES];
+   uint16_t offsets[AGX_PPP_HAL300_GRAPHICS_RECORDS];
+   uint8_t lengths[AGX_PPP_HAL300_GRAPHICS_RECORDS];
+};
+
+bool agx_ppp_hal300_graphics_core_pack(
+   enum agx_chip chip, const struct agx_ppp_hal300_graphics_fields *fields,
+   struct agx_ppp_hal300_graphics_core *out);
diff --git a/src/asahi/lib/agx_rt_bvh.h b/src/asahi/lib/agx_rt_bvh.h
index d238513b47ef..37c2b1b644fb 100644
--- a/src/asahi/lib/agx_rt_bvh.h
+++ b/src/asahi/lib/agx_rt_bvh.h
@@ -995,6 +995,8 @@ agx_rt_bvh_gen1_header_set_child_layout(
    unsigned leaf_boxes, uint32_t child_space_slot, unsigned leaf_type,
    unsigned leaf_slot_capacity, bool indexed_leaf_counts)
 {
+   /* Indexed counts do not need a fixed slot capacity. Native M5 instance
+    * roots use this form with capacity zero and two-slot instance records. */
    if (!header || inner_count > 8 || leaf_boxes > 8 ||
        inner_count + leaf_boxes == 0 || inner_count + leaf_boxes > 8 ||
        leaf_type > 31 || leaf_slot_capacity > 15 ||
diff --git a/src/asahi/lib/agx_tilebuffer.c b/src/asahi/lib/agx_tilebuffer.c
index 09f22123ae94..26879ab4de3d 100644
--- a/src/asahi/lib/agx_tilebuffer.c
+++ b/src/asahi/lib/agx_tilebuffer.c
@@ -5,6 +5,7 @@
 
 #include "agx_tilebuffer.h"
 #include <assert.h>
+#include <string.h>
 #include "util/bitset.h"
 #include "util/format/u_format.h"
 #include "util/u_debug.h"
@@ -88,36 +89,35 @@ agx_build_tilebuffer_layout_bounded(const enum pipe_format *formats,
       formats, nr_cbufs, nr_samples, layered, max_sample_size_B, 0, 0);
 }
 
-struct agx_tilebuffer_layout
-agx_build_tilebuffer_layout_with_limits(const enum pipe_format *formats,
-                                      uint8_t nr_cbufs, uint8_t nr_samples,
-                                      bool layered, uint8_t max_sample_size_B,
-                                      uint16_t max_bytes_per_tile,
-                                      uint16_t max_samples_per_tile)
+/* Whether render target a is allocated before b: the targets in first_mask
+ * ahead of the others, and within each group in descending order of
+ * alignment, which leaves no padding inside a group. */
+static bool
+agx_tilebuffer_allocate_before(const enum pipe_format *formats,
+                               uint8_t first_mask, unsigned a, unsigned b)
 {
-   struct agx_tilebuffer_layout tib = {
-      .nr_samples = nr_samples,
-      .layered = layered,
-      .max_sample_size_B = max_sample_size_B,
-      .max_bytes_per_tile = max_bytes_per_tile,
-      .max_samples_per_tile = max_samples_per_tile,
-   };
-   const unsigned max_bytes_per_sample =
-      MIN2(max_sample_size_B ? max_sample_size_B : MAX_BYTES_PER_SAMPLE,
-           MAX_ENCODABLE_BYTES_PER_SAMPLE);
-   const unsigned tile_capacity =
-      max_bytes_per_tile ? max_bytes_per_tile : MAX_BYTES_PER_TILE;
+   const bool first_a = first_mask & BITFIELD_BIT(a);
+   const bool first_b = first_mask & BITFIELD_BIT(b);
+   if (first_a != first_b)
+      return first_a;
+   return format_align_B(formats[a]) > format_align_B(formats[b]);
+}
 
+/* Allocate each render target a tilebuffer offset in the order above, or
+ * spill it once it no longer fits. Returns the bytes used per sample. */
+static uint32_t
+agx_tilebuffer_allocate(struct agx_tilebuffer_layout *tib,
+                        const enum pipe_format *formats, uint8_t nr_cbufs,
+                        uint8_t nr_samples, unsigned max_bytes_per_sample,
+                        unsigned tile_capacity, uint8_t first_mask)
+{
    uint32_t offset_B = 0;
    uint8_t order[] = {0, 1, 2, 3, 4, 5, 6, 7};
 
-   /* Sort render targets in descending order of alignment, eliminating padding
-    * and giving the optimal order of render targets. We use insertion sort
-    * because it is simple, stable, fast for small n, and free for n=1.
-    */
+   /* Insertion sort: simple, stable, fast for small n, and free for n=1. */
    for (int i = 1; i < nr_cbufs; ++i) {
-      for (int j = i; j > 0 && format_align_B(formats[order[j - 1]]) <
-                                  format_align_B(formats[order[j]]);
+      for (int j = i; j > 0 && agx_tilebuffer_allocate_before(
+                                  formats, first_mask, order[j], order[j - 1]);
            --j) {
          SWAP(order[j], order[j - 1]);
       }
@@ -126,10 +126,16 @@ agx_build_tilebuffer_layout_with_limits(const enum pipe_format *formats,
    for (unsigned i = 0; i < nr_cbufs; ++i) {
       unsigned rt = order[i];
       enum pipe_format format = formats[rt];
-      tib.logical_format[rt] = format;
-
-      assert(util_is_aligned(offset_B, MAX2(format_align_B(formats[rt]), 1)) &&
+      tib->logical_format[rt] = format;
+      tib->spilled[rt] = false;
+      tib->_offset_B[rt] = 0;
+
+      /* Padding only arises between the two groups. */
+      const uint32_t aligned_B =
+         align(offset_B, MAX2(format_align_B(format), 1));
+      assert((first_mask || aligned_B == offset_B) &&
              "loop invariant ensured by the sort");
+      offset_B = aligned_B;
 
       unsigned size_B = format_size_B(format);
 
@@ -141,13 +147,60 @@ agx_build_tilebuffer_layout_with_limits(const enum pipe_format *formats,
                   (ALIGN_POT(new_offset_B, 8) * MIN_TILE_SIZE_PX *
                    nr_samples) <= tile_capacity;
       if (fits) {
-         tib._offset_B[rt] = offset_B;
+         tib->_offset_B[rt] = offset_B;
          offset_B = new_offset_B;
       } else {
-         tib.spilled[rt] = true;
+         tib->spilled[rt] = true;
       }
    }
 
+   return offset_B;
+}
+
+static struct agx_tilebuffer_layout
+agx_build_tilebuffer_layout_internal(const enum pipe_format *formats,
+                                      uint8_t nr_cbufs, uint8_t nr_samples,
+                                      bool layered, uint8_t max_sample_size_B,
+                                      uint32_t max_bytes_per_tile,
+                                      uint16_t max_samples_per_tile,
+                                      bool native_imageblock,
+                                      uint8_t resident_mask)
+{
+   struct agx_tilebuffer_layout tib = {
+      .nr_samples = nr_samples,
+      .layered = layered,
+      .native_imageblock = native_imageblock,
+      .max_sample_size_B = max_sample_size_B,
+      .max_bytes_per_tile = max_bytes_per_tile,
+      .max_samples_per_tile = max_samples_per_tile,
+   };
+   const unsigned max_bytes_per_sample =
+      native_imageblock ? max_sample_size_B :
+      MIN2(max_sample_size_B ? max_sample_size_B : MAX_BYTES_PER_SAMPLE,
+           MAX_ENCODABLE_BYTES_PER_SAMPLE);
+   const unsigned tile_capacity =
+      max_bytes_per_tile ? max_bytes_per_tile : MAX_BYTES_PER_TILE;
+
+   uint32_t offset_B =
+      agx_tilebuffer_allocate(&tib, formats, nr_cbufs, nr_samples,
+                              max_bytes_per_sample, tile_capacity, 0);
+
+   /* Keep the requested render targets in the tilebuffer if the plain order
+    * would spill one of them (M5 compressed multisample attachments: their
+    * in-pass memory reads do not observe the pass's own stores). */
+   uint8_t spilled_mask = 0;
+   for (unsigned rt = 0; rt < nr_cbufs; ++rt) {
+      if (tib.spilled[rt])
+         spilled_mask |= BITFIELD_BIT(rt);
+   }
+   resident_mask &= BITFIELD_MASK(nr_cbufs);
+   if (resident_mask & spilled_mask) {
+      offset_B = agx_tilebuffer_allocate(&tib, formats, nr_cbufs, nr_samples,
+                                         max_bytes_per_sample, tile_capacity,
+                                         resident_mask);
+      tib.resident_mask = resident_mask;
+   }
+
    assert(offset_B <= max_bytes_per_sample && "loop invariant");
 
    /* Multisampling needs a nonempty allocation.
@@ -156,6 +209,13 @@ agx_build_tilebuffer_layout_with_limits(const enum pipe_format *formats,
    if (nr_samples > 1)
       offset_B = MAX2(offset_B, 1);
 
+   /* M5 likewise reserves one eight-byte sample for a layered pass with no
+    * colour attachment: native layered attachmentless and depth-only passes
+    * carry imageblock sample size 8 (command +0x1fb0) at one and four samples
+    * (native-window/item1-layered). */
+   if (native_imageblock && layered)
+      offset_B = MAX2(offset_B, 1);
+
    tib.sample_size_B = align(offset_B, 8);
    tib.tile_size = agx_select_tile_size(
       tib.sample_size_B, nr_samples,
@@ -166,6 +226,41 @@ agx_build_tilebuffer_layout_with_limits(const enum pipe_format *formats,
    return tib;
 }
 
+struct agx_tilebuffer_layout
+agx_build_tilebuffer_layout_with_limits(
+   const enum pipe_format *formats, uint8_t nr_cbufs, uint8_t nr_samples,
+   bool layered, uint8_t max_sample_size_B, uint32_t max_bytes_per_tile,
+   uint16_t max_samples_per_tile)
+{
+   return agx_build_tilebuffer_layout_internal(
+      formats, nr_cbufs, nr_samples, layered, max_sample_size_B,
+      max_bytes_per_tile, max_samples_per_tile, false, 0);
+}
+
+struct agx_tilebuffer_layout
+agx_build_tilebuffer_layout_for_imageblock(
+   const enum pipe_format *formats, uint8_t nr_cbufs, uint8_t nr_samples,
+   bool layered, uint8_t max_sample_size_B, uint32_t max_bytes_per_tile,
+   uint16_t max_samples_per_tile)
+{
+   return agx_build_tilebuffer_layout_resident(
+      formats, nr_cbufs, nr_samples, layered, max_sample_size_B,
+      max_bytes_per_tile, max_samples_per_tile, 0);
+}
+
+struct agx_tilebuffer_layout
+agx_build_tilebuffer_layout_resident(
+   const enum pipe_format *formats, uint8_t nr_cbufs, uint8_t nr_samples,
+   bool layered, uint8_t max_sample_size_B, uint32_t max_bytes_per_tile,
+   uint16_t max_samples_per_tile, uint8_t resident_mask)
+{
+   assert(max_sample_size_B && !(max_sample_size_B & 7));
+   assert(max_bytes_per_tile);
+   return agx_build_tilebuffer_layout_internal(
+      formats, nr_cbufs, nr_samples, layered, max_sample_size_B,
+      max_bytes_per_tile, max_samples_per_tile, true, resident_mask);
+}
+
 /*
  * With attachmentless rendering in Vulkan, the sample count may not known until
  * draw-time. It's convenient to construct an agx_tilebuffer_layout anyway when
@@ -180,7 +275,8 @@ agx_tilebuffer_set_samples(struct agx_tilebuffer_layout *tib,
 {
    assert(tib->nr_samples == 0 && "must not be initialized");
 
-   *tib = agx_build_tilebuffer_layout_with_limits(
+   *tib = (tib->native_imageblock ? agx_build_tilebuffer_layout_for_imageblock
+                                  : agx_build_tilebuffer_layout_with_limits)(
       tib->logical_format, ARRAY_SIZE(tib->logical_format), nr_samples,
       tib->layered, tib->max_sample_size_B, tib->max_bytes_per_tile,
       tib->max_samples_per_tile);
@@ -215,6 +311,11 @@ agx_tilebuffer_total_size(struct agx_tilebuffer_layout *tib)
 void
 agx_tilebuffer_pack_usc(struct agx_tilebuffer_layout *tib)
 {
+   if (tib->native_imageblock) {
+      memset(&tib->usc, 0, sizeof(tib->usc));
+      return;
+   }
+
    agx_pack(&tib->usc, USC_SHARED, cfg) {
       if (tib->nr_samples > 0) {
          cfg.uses_shared_memory = true;
@@ -542,7 +643,7 @@ agx_g17p_classify_mrt(const struct agx_tilebuffer_layout *tib)
 
    /* Multisample multi-target layouts take the general mixed-format
     * programs, whose end-of-tile builtin has a multisample form. */
-   if (tib->nr_samples == 2 || tib->nr_samples == 4)
+   if (tib->nr_samples == 2 || tib->nr_samples == 4 || tib->nr_samples == 8)
       return agx_g17p_classify_mixed_mrt(tib, active_count);
    if (tib->nr_samples != 1)
       return AGX_G17P_MRT_NONE;
diff --git a/src/asahi/lib/agx_tilebuffer.h b/src/asahi/lib/agx_tilebuffer.h
index af7dd1e7f11a..6b1ee9baf151 100644
--- a/src/asahi/lib/agx_tilebuffer.h
+++ b/src/asahi/lib/agx_tilebuffer.h
@@ -53,7 +53,10 @@ struct agx_tilebuffer_layout {
    uint8_t nr_samples;
 
    /* If layered rendering is used */
-   bool layered;
+   bool layered : 1;
+
+   /* Native imageblock state replaces the AGX2 USC shared record. */
+   bool native_imageblock : 1;
 
    /* Most bytes a sample may hold before render targets spill, or zero for
     * the 64-byte AGX2 bound. The tile size bounds it separately. */
@@ -61,12 +64,21 @@ struct agx_tilebuffer_layout {
 
    /* Independent storage and raster-sample budgets. Zero retains the legacy
     * byte capacity or imposes no additional sample budget, respectively. */
-   uint16_t max_bytes_per_tile;
+   uint32_t max_bytes_per_tile;
    uint16_t max_samples_per_tile;
 
    /* Selected tile size in pixels. Either 32*32, 32*16, or 16*16. */
    uint16_t tile_size;
 
+   /* Render targets allocated ahead of the others because the descending
+    * alignment order would spill one of them; zero for that order. See
+    * agx_build_tilebuffer_layout_resident(). */
+   uint8_t resident_mask;
+
+   /* Named so that layouts hashed or compared byte-wise have no
+    * indeterminate padding. Always zero. */
+   uint8_t reserved[3];
+
    /* USC word corresponding to this configuration of the tilebuffer */
    struct agx_usc_shared_packed usc;
 };
@@ -118,9 +130,27 @@ struct agx_tilebuffer_layout
 agx_build_tilebuffer_layout_with_limits(const enum pipe_format *formats,
                                       uint8_t nr_cbufs, uint8_t nr_samples,
                                       bool layered, uint8_t max_sample_size_B,
-                                      uint16_t max_bytes_per_tile,
+                                      uint32_t max_bytes_per_tile,
                                       uint16_t max_samples_per_tile);
 
+/* Native imageblock strides are not limited by the AGX2 USC bitfield. */
+struct agx_tilebuffer_layout
+agx_build_tilebuffer_layout_for_imageblock(
+   const enum pipe_format *formats, uint8_t nr_cbufs, uint8_t nr_samples,
+   bool layered, uint8_t max_sample_size_B, uint32_t max_bytes_per_tile,
+   uint16_t max_samples_per_tile);
+
+/* As agx_build_tilebuffer_layout_for_imageblock(), but when that layout
+ * spills one of the render targets in resident_mask, those are allocated
+ * first so that they stay in the tilebuffer and the others spill instead.
+ * The applied mask is recorded in the layout; building again with it
+ * reproduces the layout. */
+struct agx_tilebuffer_layout
+agx_build_tilebuffer_layout_resident(
+   const enum pipe_format *formats, uint8_t nr_cbufs, uint8_t nr_samples,
+   bool layered, uint8_t max_sample_size_B, uint32_t max_bytes_per_tile,
+   uint16_t max_samples_per_tile, uint8_t resident_mask);
+
 /* Compact attachment slots for internal programs without moving tile data.
  * map[dense slot] identifies the original logical attachment. */
 unsigned agx_tilebuffer_compact(const struct agx_tilebuffer_layout *src,
@@ -138,9 +168,15 @@ bool agx_nir_lower_tilebuffer(struct nir_shader *shader,
                               bool lower_g17p_formats);
 
 bool agx_nir_lower_to_per_sample(struct nir_shader *shader);
+bool agx_nir_apply_api_mask_to_force_early(struct nir_shader *shader,
+                                          enum agx_chip chip);
 
 bool agx_nir_lower_monolithic_msaa(struct nir_shader *shader,
                                    uint8_t nr_samples);
+bool agx_nir_lower_monolithic_msaa_m5(struct nir_shader *shader,
+                                      uint8_t nr_samples);
+bool agx_nir_lower_m5_zs_sample_loop(struct nir_shader *shader,
+                                     unsigned nr_samples);
 
 /* Export registers of one sample's colours in a G17P fragment unrolled per
  * sample: eight halves for each location up to the last one written. */
@@ -250,6 +286,7 @@ agx_g17p_sample_exports_end(const struct agx_g17p_colour_layout *l,
    return agx_g17p_sample_zs_base_layout(l, nr_samples, nr_samples);
 }
 
+void agx_nir_lower_sample_exports_to_end(struct nir_shader *shader);
 bool agx_nir_needs_sample_count(const struct nir_shader *shader);
 bool agx_nir_specialize_sample_count(struct nir_shader *shader, unsigned samples);
 bool agx_nir_unroll_g17p_samples(struct nir_shader *shader,
@@ -263,6 +300,9 @@ bool agx_nir_hoist_g17p_zs_writes(struct nir_shader *shader);
 bool agx_nir_lower_sample_intrinsics(struct nir_shader *shader,
                                      bool ignore_sample_mask_without_msaa);
 
+bool agx_nir_lower_sample_intrinsics_wide(struct nir_shader *shader,
+                                          bool ignore_sample_mask_without_msaa);
+
 uint32_t agx_tilebuffer_total_size(struct agx_tilebuffer_layout *tib);
 
 enum pipe_format
diff --git a/src/asahi/lib/agx_va.c b/src/asahi/lib/agx_va.c
index b4f5820db498..1554c54787fc 100644
--- a/src/asahi/lib/agx_va.c
+++ b/src/asahi/lib/agx_va.c
@@ -75,9 +75,9 @@ agx_va_free(struct agx_device *dev, struct agx_va *va, bool unbind)
       return;
 
    if (unbind) {
-      /* Sparse/capture-replay VAs are always installed as a lower read-write
-       * mapping plus the corresponding upper read-only shadow.  Remove both
-       * in one VM_BIND transaction so neither half can survive VA reuse. */
+      /* Sparse/capture-replay VAs may contain a lower read-write mapping and
+       * its upper read-only shadow. Clear both, including unbound holes, in
+       * one VM_BIND transaction so neither half can survive VA reuse. */
       assert(!(va->flags & AGX_VA_USC));
       assert(va->size_B >= dev->guard_size);
       const uint64_t bound_size_B = va->size_B - dev->guard_size;
diff --git a/src/asahi/lib/agx_vdm_decode_core.c b/src/asahi/lib/agx_vdm_decode_core.c
index 761dee30b0a6..e0ac7e4b477d 100644
--- a/src/asahi/lib/agx_vdm_decode_core.c
+++ b/src/asahi/lib/agx_vdm_decode_core.c
@@ -34,6 +34,29 @@ agx_vdm_decode_one(enum agx_chip chip, const void *bytes, size_t available,
    const uint32_t word0 = agx_load_le32(bytes);
    const uint8_t block_type = word0 >> 29;
 
+   /* Native M5 mesh forms share block types with ordinary state/index
+    * records. Match their exact captured headers before those grammars. */
+   if (word0 == AGX_VDM_MESH_STATE_HEADER) {
+      if (available < AGX_VDM_MESH_STATE_LENGTH ||
+          !agx_vdm_mesh_state_unpack(chip, bytes, AGX_VDM_MESH_STATE_LENGTH,
+                                    &record->data.mesh_state))
+         return false;
+      record->kind = AGX_VDM_RECORD_MESH_STATE;
+      record->length = AGX_VDM_MESH_STATE_LENGTH;
+      record->chip = chip;
+      return true;
+   }
+   if ((word0 & ~UINT32_C(0xff00)) == AGX_VDM_MESH_DISPATCH_BASE) {
+      if (available < AGX_VDM_MESH_DISPATCH_LENGTH ||
+          !agx_vdm_mesh_dispatch_unpack(chip, bytes, AGX_VDM_MESH_DISPATCH_LENGTH,
+                                       &record->data.mesh_dispatch))
+         return false;
+      record->kind = AGX_VDM_RECORD_MESH_DISPATCH;
+      record->length = AGX_VDM_MESH_DISPATCH_LENGTH;
+      record->chip = chip;
+      return true;
+   }
+
    switch (block_type) {
    case 0: {
       uint64_t pointer;
diff --git a/src/asahi/lib/agx_vdm_decode_core.h b/src/asahi/lib/agx_vdm_decode_core.h
index 1b3c3ec92d1c..5a14d66a917d 100644
--- a/src/asahi/lib/agx_vdm_decode_core.h
+++ b/src/asahi/lib/agx_vdm_decode_core.h
@@ -15,6 +15,7 @@
 #include "agx_vdm_index_list_core.h"
 #include "agx_vdm_state_update_core.h"
 #include "agx_vdm_tessellate_core.h"
+#include "agx_vdm_mesh_core.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -34,6 +35,8 @@ enum agx_vdm_record_kind {
    AGX_VDM_RECORD_TESSELLATE_DIRECT,
    AGX_VDM_RECORD_TESSELLATE_INDIRECT,
    AGX_VDM_RECORD_TESSELLATION_STATE,
+   AGX_VDM_RECORD_MESH_STATE,
+   AGX_VDM_RECORD_MESH_DISPATCH,
 };
 
 struct agx_vdm_decoded_record {
@@ -55,6 +58,8 @@ struct agx_vdm_decoded_record {
       struct agx_vdm_tessellate_direct_info tessellate_direct;
       struct agx_vdm_tessellate_indirect_info tessellate_indirect;
       struct agx_vdm_tessellation_state_info tessellation_state;
+      struct agx_vdm_mesh_state_info mesh_state;
+      struct agx_vdm_mesh_dispatch_info mesh_dispatch;
       struct {
          uint64_t target;
       } stream_link;
diff --git a/src/asahi/lib/agx_vdm_index_list_core.h b/src/asahi/lib/agx_vdm_index_list_core.h
index cf0cb8b44208..131d166b1cbb 100644
--- a/src/asahi/lib/agx_vdm_index_list_core.h
+++ b/src/asahi/lib/agx_vdm_index_list_core.h
@@ -148,12 +148,10 @@ static inline bool
 agx_vdm_index_list_index_size_supported_inline(enum agx_chip chip,
                                                 uint8_t index_size)
 {
-   /* U8 is grounded by the native G17P/HAL200 golden.  The HAL300 captures
-    * cover U16/U32 only, so keep that distinct encoding family fail-closed. */
-   return index_size == AGX_VDM_INDEX_LIST_INDEX_SIZE_U16 ||
-          index_size == AGX_VDM_INDEX_LIST_INDEX_SIZE_U32 ||
-          (index_size == AGX_VDM_INDEX_LIST_INDEX_SIZE_U8 &&
-           !agx_vdm_index_list_direct_is_hal300_inline(chip));
+   (void)chip;
+   return index_size == AGX_VDM_INDEX_LIST_INDEX_SIZE_U8 ||
+          index_size == AGX_VDM_INDEX_LIST_INDEX_SIZE_U16 ||
+          index_size == AGX_VDM_INDEX_LIST_INDEX_SIZE_U32;
 }
 
 static inline bool
diff --git a/src/asahi/lib/agx_vdm_mesh_core.c b/src/asahi/lib/agx_vdm_mesh_core.c
index 2f100f75a9dc..a210baf7c086 100644
--- a/src/asahi/lib/agx_vdm_mesh_core.c
+++ b/src/asahi/lib/agx_vdm_mesh_core.c
@@ -33,6 +33,27 @@ agx_vdm_mesh_threadgroup_unpack(uint32_t word, uint32_t size[3])
    return true;
 }
 
+bool
+agx_vdm_m5_mesh_set_output_limits(struct agx_vdm_mesh_state_info *info,
+                                  uint32_t max_vertices,
+                                  uint32_t max_primitives,
+                                  uint32_t uvs_words)
+{
+   if (!info || !max_vertices || max_vertices > 256 ||
+       !max_primitives || max_primitives > 256 ||
+       uvs_words < 4 || uvs_words > 255)
+      return false;
+
+   const uint32_t last_primitive = max_primitives - 1;
+   const uint32_t aligned_vertices = (max_vertices + 3) & ~3u;
+   /* Native mesh-cell (6V/2P/4 words), primitive-ID (6V/2P/5 words)
+    * and overlap (192V/64P/4 words) packets establish these fields. */
+   info->raw_prefix[0] = uvs_words | ((uvs_words + 1) << 8);
+   info->raw_tail[1] = last_primitive | ((last_primitive >> 4) << 16);
+   info->raw_tail[4] = 1 + aligned_vertices * uvs_words / 4;
+   return true;
+}
+
 bool
 agx_vdm_mesh_state_pack(enum agx_chip chip,
                         const struct agx_vdm_mesh_state_info *info,
@@ -103,9 +124,10 @@ agx_vdm_mesh_dispatch_pack(enum agx_chip chip,
 {
    if (out)
       memset(out, 0, AGX_VDM_MESH_DISPATCH_LENGTH);
-   if (!out || !supported(chip) || !valid_grid(info))
+   if (!out || !supported(chip) || !valid_grid(info) ||
+       (info->primitive != 0 && info->primitive != 1 && info->primitive != 6))
       return false;
-   agx_store_le32(out, AGX_VDM_MESH_DISPATCH_HEADER);
+   agx_store_le32(out, AGX_VDM_MESH_DISPATCH_BASE | (info->primitive << 8));
    for (unsigned i = 0; i < 3; i++) {
       agx_store_le32(out + 4 + 4 * i, info->threads[i]);
       agx_store_le32(out + 16 + 4 * i, info->local_size[i]);
@@ -123,7 +145,10 @@ agx_vdm_mesh_dispatch_unpack(enum agx_chip chip, const void *bytes, size_t size,
       return false;
    const uint8_t *raw = bytes;
    struct agx_vdm_mesh_dispatch_info tmp = {0};
-   if (agx_load_le32(raw) != AGX_VDM_MESH_DISPATCH_HEADER)
+   const uint32_t header = agx_load_le32(raw);
+   tmp.primitive = (header >> 8) & 0xff;
+   if ((header & ~UINT32_C(0xff00)) != AGX_VDM_MESH_DISPATCH_BASE ||
+       (tmp.primitive != 0 && tmp.primitive != 1 && tmp.primitive != 6))
       return false;
    for (unsigned i = 0; i < 3; i++) {
       tmp.threads[i] = agx_load_le32(raw + 4 + 4 * i);
diff --git a/src/asahi/lib/agx_vdm_mesh_core.h b/src/asahi/lib/agx_vdm_mesh_core.h
index 4dc0bf7329b9..5c14ff9dc3bb 100644
--- a/src/asahi/lib/agx_vdm_mesh_core.h
+++ b/src/asahi/lib/agx_vdm_mesh_core.h
@@ -11,7 +11,10 @@ extern "C" {
 #define AGX_VDM_MESH_STATE_LENGTH    56u
 #define AGX_VDM_MESH_DISPATCH_LENGTH 28u
 #define AGX_VDM_MESH_STATE_HEADER    UINT32_C(0x40000d2c)
+/* Index List block with the mesh bit; bits 8..15 are the Index List
+ * "Primitive" field (cmdbuf.xml): native triangle mesh records carry 6. */
 #define AGX_VDM_MESH_DISPATCH_HEADER UINT32_C(0x70000600)
+#define AGX_VDM_MESH_DISPATCH_BASE   UINT32_C(0x70000000)
 
 
 struct agx_vdm_mesh_state_info {
@@ -22,10 +25,19 @@ struct agx_vdm_mesh_state_info {
 };
 struct agx_vdm_mesh_dispatch_info {
    uint32_t threads[3], local_size[3];
+   /* Index List "Primitive" of the mesh output: 0 points, 1 lines,
+    * 6 triangles. */
+   uint8_t primitive;
 };
 
 bool agx_vdm_mesh_threadgroup_pack(const uint32_t size[3], uint32_t *word);
 bool agx_vdm_mesh_threadgroup_unpack(uint32_t word, uint32_t size[3]);
+/* Captured HAL300 limits: max primitive index and vertex output arena. */
+bool agx_vdm_m5_mesh_set_output_limits(struct agx_vdm_mesh_state_info *info,
+                                     uint32_t max_vertices,
+                                     uint32_t max_primitives,
+                                     uint32_t uvs_words);
+
 bool agx_vdm_mesh_state_pack(enum agx_chip chip,
                              const struct agx_vdm_mesh_state_info *info,
                              uint8_t out[AGX_VDM_MESH_STATE_LENGTH]);
diff --git a/src/asahi/lib/agx_vdm_state_update_core.c b/src/asahi/lib/agx_vdm_state_update_core.c
index c733cf808bcd..bce73688bd0b 100644
--- a/src/asahi/lib/agx_vdm_state_update_core.c
+++ b/src/asahi/lib/agx_vdm_state_update_core.c
@@ -231,7 +231,7 @@ agx_vdm_state_update_template_pack(
 /*
  * Publish a vertex constant program's thread-invariant (TI) resources in the
  * two state-load counts, from the compiler's TI spill-buffer bytes and TI
- * largest-IPR bytes. G17P only.
+ * largest-IPR bytes. G17P and HAL300.
  *
  * Pinned AGXMetalG17P (HAL200):
  *
@@ -251,6 +251,13 @@ agx_vdm_state_update_template_pack(
  * (agx_ppp_hal200_fragment_ti_resources). A vertex shader without a constant
  * program leaves both zero. Fails closed, leaving `info` unchanged, outside
  * the encodable range.
+ *
+ * The native HAL300 vertex program finalization computes the same
+ * two quad counts (ESL+0x158/0x15c) and, after
+ * the state-load encoder finishes, packs each template's vertex-shader word
+ * 0 as IPR << 8 (ubfiz #8, #12) and its pipeline template as spill << 3
+ * (ubfiz #3, #18); the HAL300 materializer only
+ * omits bit 31.
  */
 static bool
 agx_vdm_state_update_vertex_ti_resources(
@@ -263,7 +270,8 @@ agx_vdm_state_update_vertex_ti_resources(
 
    /* Same bounds as the HAL200 fragment record: an 18-bit spill field and
     * the 12-bit IPR field, whose even rounding caps it at 0xffe. */
-   if (!agx_chip_native_agx3(chip) || !info || spill_quads > UINT32_C(0x3ffff) ||
+   if ((!agx_chip_native_agx3(chip) && !agx_vdm_state_update_is_hal300(chip)) ||
+       !info || spill_quads > UINT32_C(0x3ffff) ||
        ipr_quads > UINT32_C(0xffe))
       return false;
 
@@ -333,14 +341,16 @@ agx_g17p_vertex_program_info(
    const bool preamble_ok =
       compiler_info &&
       (compiler_info->has_preamble
-          ? compiler_info->constant_program_allowed &&
+          ? (agx_chip_native_agx3(chip) || agx_chip_g17_m5(chip)) &&
+               compiler_info->constant_program_allowed &&
                compiler_info->preamble_staging_known &&
                compiler_info->nr_preamble_gprs != 0 &&
                compiler_info->nr_preamble_gprs <= 256
           : compiler_info->nr_preamble_gprs == 0 &&
                compiler_info->preamble_scratch_size == 0);
 
-   if (!agx_chip_native_agx3(chip) || !compiler_info || !program ||
+   if ((!agx_chip_native_agx3(chip) && !agx_chip_g17_m5(chip)) ||
+       !compiler_info || !program ||
        !compiler_info->hardware_vertex || !preamble_ok ||
        compiler_info->position_off != 0 || compiler_info->varyings_off != 4 ||
        compiler_info->psiz_off != 4 + compiler_info->user_size ||
@@ -383,7 +393,8 @@ agx_g17p_vertex_program_info(
             .raw_vs_output_count_1_extra = false,
             .raw_vs_output_selector = 0,
             .raw_vs_output_flag_6 = false,
-            .pre_hal300_flag_group_nonzero = true,
+            .pre_hal300_flag_group_nonzero =
+               !agx_vdm_state_update_is_hal300(chip),
             .pre_hal300_flag_bit_4 = false,
             .variant = AGX_VDM_STATE_UPDATE_TEMPLATE_00,
             .raw_active_context_selector = UINT32_MAX,
diff --git a/src/asahi/lib/meson.build b/src/asahi/lib/meson.build
index a461fe7c3bac..c3245df94f01 100644
--- a/src/asahi/lib/meson.build
+++ b/src/asahi/lib/meson.build
@@ -55,6 +55,8 @@ libasahi_lib_files = files(
   'agx_ppp_hal200_raster_core.h',
   'agx_ppp_hal300_depth_bias_scissor_core.c',
   'agx_ppp_hal300_depth_bias_scissor_core.h',
+  'agx_ppp_hal300_graphics_core.c',
+  'agx_ppp_hal300_graphics_core.h',
   'agx_ppp_hal300_depth_stencil_core.c',
   'agx_ppp_hal300_depth_stencil_core.h',
   'agx_ppp_decode_core.c',
@@ -65,6 +67,8 @@ libasahi_lib_files = files(
   'agx_ppp_w_clamp_cull2_core.h',
   'agx_pbe_agx3_core.c',
   'agx_pbe_agx3_core.h',
+  'agx_scs_layout.c',
+  'agx_scs_layout.h',
   'agx_scratch.c',
   'agx_texture_side_entry.c',
   'agx_texture_side_entry.h',
@@ -110,6 +114,8 @@ libasahi_decode_files = files(
   'agx_ppp_decode_core.c',
   'agx_ppp_hal200_graphics_core.c',
   'agx_ppp_hal300_depth_bias_scissor_core.c',
+  'agx_ppp_hal300_graphics_core.c',
+  'agx_ppp_hal300_graphics_core.h',
   'agx_ppp_hal300_depth_stencil_core.c',
   'agx_ppp_viewport_payload_core.c',
   'agx_ppp_w_clamp_cull2_core.c',
@@ -245,6 +251,9 @@ if with_tests
         'tests/test-packing.cpp',
         'tests/test-tilebuffer.cpp',
         'tests/test-g17p-msaa-epilog.cpp',
+        'tests/test-m5-depth-export.cpp',
+        'tests/test-primitive-varyings.cpp',
+        '../vulkan/hk_nir_widen_16bit_varyings.c',
       ),
       c_args : [c_msvc_compat_args, no_override_init_args],
       gnu_symbol_visibility : 'hidden',
@@ -252,7 +261,8 @@ if with_tests
         idep_gtest, idep_agx_pack, idep_mesautil, idep_nir,
       ],
       include_directories: [inc_asahi],
-      link_with : [libasahi_lib, libasahi_compiler, libasahi_layout],
+      link_with : [libasahi_lib, libasahi_compiler, libasahi_layout,
+                   libagx3c_disasm],
     ),
     suite : ['asahi'],
     protocol : 'gtest',
@@ -287,12 +297,72 @@ if with_tests
     c_args : ['-ffp-contract=off'],
     include_directories : include_directories('.'),
     dependencies : [dep_m]), suite : ['asahi'])
-  test('asahi-rt-blas-gen3', executable('asahi-rt-blas-gen3',
-    'tests/test-rt-blas-gen3.c',
-    c_args : ['-ffp-contract=off'],
-    include_directories : include_directories('.')), suite : ['asahi'])
-  test('asahi-rt-envelope', executable('asahi-rt-envelope',
+  test('agx3_m5_programs', executable('agx3_m5_programs',
+    '../compiler/test/test-agx3-m5-program.c',
+    include_directories: [inc_include, inc_src, inc_asahi],
+    dependencies: [idep_mesautil, idep_nir],
+    link_with: [libasahi_compiler, libasahi_lib]), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx_rt_envelope', executable('agx_rt_envelope',
     'tests/test-rt-envelope.c',
-    c_args : ['-ffp-contract=off'],
-    include_directories : include_directories('.')), suite : ['asahi'])
+    c_args: ['-ffp-contract=off'],
+    include_directories: include_directories('.')), suite: ['asahi'])
+  test('agx_rt_blas_gen3', executable('agx_rt_blas_gen3',
+    'tests/test-rt-blas-gen3.c',
+    include_directories: include_directories('.')), suite: ['asahi'])
+  test('agx_rt_bvh', executable('agx_rt_bvh',
+    'tests/test-rt-bvh.c',
+    include_directories: include_directories('.')), suite: ['asahi'])
+  test('agx_m5_layout', executable('agx_m5_layout',
+    ['tests/test-agx-m5-layout.c', 'agx3_code_container_core.c'],
+    include_directories: include_directories('.')), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx3_m5_instructions', executable('agx3_m5_instructions',
+    '../compiler/test/test-agx3-m5-instructions.c',
+    include_directories: [inc_include, inc_src, inc_asahi],
+    dependencies: [idep_mesautil, idep_nir, idep_agx_opcodes_h],
+    link_with: [libasahi_compiler, libasahi_lib]), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx_scs_nir', executable('agx_scs_nir',
+    '../compiler/test/test-scs-nir.c',
+    c_args: ['-UNDEBUG'],
+    include_directories: [inc_include, inc_src, inc_asahi],
+    dependencies: [idep_mesautil, idep_nir, idep_agx_opcodes_h],
+    link_with: [libasahi_compiler, libasahi_lib]), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx3-m5-mesh-program', executable('agx3-m5-mesh-program',
+    '../compiler/test/test-native-mesh-program.c',
+    include_directories: [inc_include, inc_src, inc_asahi],
+    dependencies: [idep_mesautil, idep_nir, idep_agx_opcodes_h],
+    link_with: [libasahi_compiler, libasahi_lib]), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx_scs_query', executable('agx_scs_query',
+    '../compiler/test/test-scs-query.c',
+    c_args: ['-UNDEBUG'],
+    include_directories: [inc_include, inc_src, inc_asahi],
+    dependencies: [idep_mesautil, idep_nir, idep_agx_opcodes_h],
+    link_with: [libasahi_compiler, libasahi_lib]), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx_scs_layout', executable('agx_scs_layout',
+    ['tests/test-scs-layout.c', 'agx_scs_layout.c'],
+    c_args: ['-UNDEBUG'],
+    include_directories: include_directories('.')), suite: ['asahi'])
+endif
+
+if with_tests
+  test('agx_rt_ift', executable('agx_rt_ift', 'tests/test-rt-ift.c',
+    c_args: ['-UNDEBUG'],
+    include_directories: include_directories('.')), suite: ['asahi'])
 endif
diff --git a/src/asahi/lib/tests/test-agx-m5-layout.c b/src/asahi/lib/tests/test-agx-m5-layout.c
new file mode 100644
index 000000000000..e5e7ad32a0b4
--- /dev/null
+++ b/src/asahi/lib/tests/test-agx-m5-layout.c
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: MIT */
+#include "agx_m5_va.h"
+#include "agx3_code_container_core.h"
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#define CHECK(x) do { if (!(x)) { fprintf(stderr, "line %d: %s\n", __LINE__, #x); return 1; } } while (0)
+int main(void)
+{
+   static const uint8_t prefixes[][14] = {
+      {0x03, 0, 0x07, 0, 0x02, 0, 0, 0, 0x60, 0, 0x0e, 0, 0, 0},
+      {0x0e, 0, 0, 0},
+      {0x03, 0, 0x07, 0, 0x02, 0, 0, 0, 0x20, 0, 0x0e, 0, 0, 0},
+   };
+   for (unsigned variant = 0; variant < 4; ++variant) {
+      uint8_t guarded[AGX3_CONTAINER_CONSTANT_PROGRAM_SIZE + 2];
+      memset(guarded, 0xa5, sizeof(guarded));
+      const bool fragment = variant & 1, sampled = variant & 2;
+      const unsigned program = !fragment ? 0 : sampled ? 2 : 1;
+      const unsigned size = program == 1 ? 4 : 14;
+      agx3_build_container_constant_program(guarded + 1, fragment, sampled);
+      CHECK(guarded[0] == 0xa5 && guarded[sizeof(guarded) - 1] == 0xa5);
+      CHECK(!memcmp(guarded + 1, prefixes[program], size));
+      for (unsigned offset = size; offset < AGX3_CONTAINER_CONSTANT_PROGRAM_SIZE;
+           offset += 2) {
+         CHECK(guarded[1 + offset] == 0x06);
+         CHECK(guarded[2 + offset] == 0x00);
+      }
+   }
+   uint8_t bytes[0x641], before[sizeof(bytes)];
+   memset(bytes, 0xa5, sizeof(bytes));
+   CHECK(agx3_build_m5_container_lead(bytes, sizeof(bytes), 2));
+   uint64_t hash = UINT64_C(0xcbf29ce484222325);
+   for (unsigned i = 0; i < 0x640; ++i)
+      hash = (hash ^ bytes[i]) * UINT64_C(0x100000001b3);
+   CHECK(hash == UINT64_C(0x6b9ff646a3a29f63));
+   CHECK(bytes[0x640] == 0xa5);
+   CHECK(agx3_build_m5_container_lead(bytes, sizeof(bytes), 1));
+   CHECK(!memcmp(bytes, (uint8_t[]){0x40,3,0,0}, 4));
+   CHECK(!memcmp(bytes + 0x100, bytes + 0x200, 0x100));
+   CHECK(!memcmp(bytes + 0x100, (uint8_t[]){0x5e,0,0x33,0x0e,0,0,0,0,0,0,6,0,6,0,6,0}, 16));
+   memcpy(before, bytes, sizeof(bytes));
+   CHECK(!agx3_build_m5_container_lead(bytes, 0x63f, 2));
+   CHECK(!agx3_build_m5_container_lead(bytes, sizeof(bytes), 0));
+   CHECK(!agx3_build_m5_container_lead(bytes, sizeof(bytes), UINT_MAX));
+   CHECK(!memcmp(before, bytes, sizeof(bytes)));
+   struct agx_m5_va_layout layout;
+   CHECK(agx_m5_va_layout(UINT64_C(0x100000000), UINT64_C(0x20000000000),
+                          0, AGX_M5_SHADER_BASE, &layout));
+   CHECK(layout.driver_start == AGX_M5_DRIVER_BASE);
+   CHECK(layout.driver_end <= layout.data_start);
+   CHECK(layout.data_end <= layout.shader_start);
+   CHECK(agx_m5_va_layout(UINT64_C(0x100000000), UINT64_C(0x20000000000),
+                          UINT64_C(0x900000000), AGX_M5_SHADER_BASE, &layout));
+   CHECK(layout.driver_end == UINT64_C(0x1900000000));
+   CHECK(!agx_m5_va_layout(AGX_M5_DRIVER_BASE + 1, UINT64_C(0x20000000000),
+                           0, AGX_M5_SHADER_BASE, &layout));
+   CHECK(!agx_m5_va_layout(0, AGX_M5_SHADER_BASE, 0, AGX_M5_SHADER_BASE, &layout));
+   CHECK(!agx_m5_va_layout(0, UINT64_C(0x20000000000), UINT64_MAX,
+                           AGX_M5_SHADER_BASE, &layout));
+   puts("M5_LAYOUT_PASS");
+   return 0;
+}
diff --git a/src/asahi/lib/tests/test-g17p-msaa-epilog.cpp b/src/asahi/lib/tests/test-g17p-msaa-epilog.cpp
index 407a0f939b53..afb4cbb84fa9 100644
--- a/src/asahi/lib/tests/test-g17p-msaa-epilog.cpp
+++ b/src/asahi/lib/tests/test-g17p-msaa-epilog.cpp
@@ -18,6 +18,111 @@ class G17PMSAAEpilog : public ::testing::Test {
    ~G17PMSAAEpilog() { glsl_type_singleton_decref(); }
 };
 
+TEST_F(G17PMSAAEpilog, M5SampleAlphaCoveragePrecedesAllColourStores)
+{
+   for (unsigned samples : {2u, 4u, 8u}) {
+      SCOPED_TRACE(samples);
+      agx_fs_epilog_key fs = {};
+      fs.nr_samples = samples;
+      fs.max_sample_size_B = 248;
+      fs.max_bytes_per_tile = 65536;
+      fs.max_samples_per_tile = 2048;
+      fs.link.loc_written = 1;
+      fs.link.size_32 = 1;
+      fs.link.colour_comps = 15;
+      fs.link.sample_shading = true;
+      fs.rt_formats[0] = PIPE_FORMAT_R8G8B8A8_UNORM;
+      fs.remap[0] = 0;
+      fs.blend.rt[0].colormask = 15;
+      fs.blend.rt[0].mode = agx_pack_blend_standard(
+         PIPE_BLEND_ADD, PIPE_BLENDFACTOR_ONE, PIPE_BLENDFACTOR_ZERO,
+         PIPE_BLEND_ADD, PIPE_BLENDFACTOR_ONE, PIPE_BLENDFACTOR_ZERO);
+      fs.blend.alpha_to_coverage = true;
+      agx_fs_epilog_apply_device_policy(AGX_CHIP_G17S, &fs);
+
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_FRAGMENT, &agx_nir_options, "M5 sample alpha coverage");
+      agx_nir_fs_epilog_g17p(&b, &fs);
+      agx_preprocess_nir(b.shader);
+      nir_validate_shader(b.shader, "M5 sample alpha coverage ordering");
+
+      unsigned updates = 0, stores = 0;
+      nir_foreach_block(block, b.impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr);
+            if (in->intrinsic == nir_intrinsic_sample_mask_agx) {
+               EXPECT_EQ(stores, 0u);
+               EXPECT_EQ(block->cf_node.parent, &b.impl->cf_node);
+               updates++;
+            }
+            if (in->intrinsic == nir_intrinsic_store_local_pixel_agx)
+               stores++;
+         }
+      }
+      EXPECT_EQ(updates, 1u);
+      EXPECT_EQ(stores, samples);
+      ralloc_free(b.shader);
+   }
+}
+
+TEST_F(G17PMSAAEpilog, ForceEarlyAppliesApiMaskBeforeSampleCounting)
+{
+   for (enum agx_chip chip : {AGX_CHIP_G13G, AGX_CHIP_G17P, AGX_CHIP_G17S}) {
+      for (unsigned samples : {2u, 4u, 8u}) {
+         for (bool early : {false, true}) {
+            for (bool sample_shading : {false, true}) {
+               for (unsigned mask : {0u, 0xaau, 0xffu}) {
+                  SCOPED_TRACE(::testing::Message() << chip << ", " << samples
+                     << " samples, early=" << early << ", shading="
+                     << sample_shading << ", mask=" << mask);
+                  nir_builder b = nir_builder_init_simple_shader(
+                     MESA_SHADER_FRAGMENT, &agx_nir_options, "forced early API mask");
+                  b.shader->info.fs.early_fragment_tests = early;
+                  b.shader->info.fs.uses_sample_shading = sample_shading;
+                  b.shader->info.fs.uses_discard = true;
+                  nir_intrinsic_instr *first = nir_sample_mask_agx(
+                     &b, nir_imm_intN_t(&b, 0xff, 16),
+                     nir_imm_intN_t(&b, 0xff, 16));
+                  nir_intrinsic_instr *last = nir_sample_mask_agx(
+                     &b, nir_imm_intN_t(&b, 0xff, 16),
+                     nir_imm_intN_t(&b, 0, 16));
+                  const bool changed = early &&
+                     (chip == AGX_CHIP_G17P ||
+                      (chip == AGX_CHIP_G17S && !sample_shading));
+                  EXPECT_EQ(agx_nir_apply_api_mask_to_force_early(b.shader, chip),
+                            changed);
+                  unsigned api_loads = 0;
+                  nir_foreach_block(block, b.impl) {
+                     nir_foreach_instr_safe(instr, block) {
+                        if (instr->type != nir_instr_type_intrinsic)
+                           continue;
+                        nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr);
+                        if (in->intrinsic != nir_intrinsic_load_api_sample_mask_agx)
+                           continue;
+                        api_loads++;
+                        b.cursor = nir_before_instr(instr);
+                        nir_def_replace(&in->def, nir_imm_intN_t(
+                           &b, mask & BITFIELD_MASK(samples), in->def.bit_size));
+                     }
+                  }
+                  EXPECT_EQ(api_loads, changed ? 1u : 0u);
+                  nir_opt_constant_folding(b.shader);
+                  EXPECT_EQ(nir_src_as_uint(first->src[0]), 0xffu);
+                  EXPECT_EQ(nir_src_as_uint(first->src[1]),
+                            changed ? (mask & BITFIELD_MASK(samples)) : 0xffu);
+                  EXPECT_EQ(nir_src_as_uint(last->src[0]), 0xffu);
+                  EXPECT_EQ(nir_src_as_uint(last->src[1]), 0u);
+                  nir_validate_shader(b.shader, "forced early API mask");
+                  ralloc_free(b.shader);
+               }
+            }
+         }
+      }
+   }
+}
+
 TEST_F(G17PMSAAEpilog, PerSampleColourImportsCompile)
 {
    for (unsigned samples : {2u, 4u}) {
@@ -281,6 +386,45 @@ TEST_F(G17PMSAAEpilog, BroadcastNoopPreservesBoundTargets)
    }
 }
 
+TEST(G17pMsaaEpilog, NativeLateSampleMaskCrossesEpilogBoundary)
+{
+   for (enum agx_chip chip : {AGX_CHIP_G13G, AGX_CHIP_G17P, AGX_CHIP_G17S}) {
+      for (bool sample_mask : {false, true}) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_FRAGMENT, &agx_nir_options, "late coverage boundary");
+      b.shader->info.io_lowered = true;
+      b.shader->info.fs.uses_discard = true;
+      b.shader->info.writes_memory = !sample_mask;
+      b.shader->info.outputs_written =
+         BITFIELD64_BIT(FRAG_RESULT_DATA0) |
+         (sample_mask ? BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK) : 0);
+      nir_demote_samples(&b, nir_imm_intN_t(&b, 0x5555, 16));
+      nir_store_output(&b, nir_imm_vec4(&b, 1, 1, 0, 1),
+         nir_imm_int(&b, 0), .write_mask = 15, .src_type = nir_type_float32,
+         .io_semantics = {.location = FRAG_RESULT_DATA0, .num_slots = 1});
+      struct agx_fs_epilog_link_info info = {};
+      ASSERT_TRUE(agx_nir_lower_fs_output_to_epilog_for_device(b.shader, chip, &info));
+      unsigned demotes = 0, masks = 0;
+      nir_foreach_block(block, b.impl) nir_foreach_instr(in, block) {
+         if (in->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(in);
+         demotes += intr->intrinsic == nir_intrinsic_demote_samples;
+         masks += intr->intrinsic == nir_intrinsic_export_agx &&
+            nir_intrinsic_base(intr) == AGX_ABI_FOUT_WRITE_SAMPLES;
+      }
+      const bool native = chip == AGX_CHIP_G17P ||
+         (chip == AGX_CHIP_G17S && sample_mask);
+      EXPECT_EQ(info.sample_mask_after_force_early, native);
+      EXPECT_EQ(info.loc0_w_1, chip != AGX_CHIP_G17P);
+      EXPECT_EQ(demotes, native ? 0u : 1u);
+      EXPECT_EQ(masks, native ? 1u : 0u);
+      nir_validate_shader(b.shader, "late coverage transport");
+      ralloc_free(b.shader);
+      }
+   }
+}
+
 TEST(LegacyFragments, EpilogUsesUpstreamDepthCoverage)
 {
    nir_builder b = nir_builder_init_simple_shader(
diff --git a/src/asahi/lib/tests/test-m5-depth-export.cpp b/src/asahi/lib/tests/test-m5-depth-export.cpp
new file mode 100644
index 000000000000..6a60c0c737bc
--- /dev/null
+++ b/src/asahi/lib/tests/test-m5-depth-export.cpp
@@ -0,0 +1,406 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * M5 fragment epilog depth/stencil export forms, against native
+ * Native G17G compiler fragments:
+ *  - depth or stencil alone, whole pixel: one DEPTHF in a loop over the
+ *    covered samples, with class 1 taken once ahead of it;
+ *  - depth and stencil together, whole pixel: one DEPTHF word;
+ *  - alpha-to-coverage: one DEPTHF per sample with the alpha coverage folded
+ *    into its word, and no separate MOVMSK;
+ *  - blended alpha-to-coverage without a depth write: one MOVMSK per pixel,
+ *    ahead of the per-sample blend;
+ *  - blended with a depth write: all of the depth words above, ahead of the
+ *    per-sample blend.
+ */
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include <string>
+#include <vector>
+#include "asahi/compiler/agx_compile.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_tilebuffer.h"
+#include "agx_linker.h"
+extern "C" {
+#include "asahi/compiler/agx_nir.h"
+#include "asahi/compiler/agx_debug.h"
+#include "asahi/isa/disasm.h"
+}
+
+namespace {
+
+class M5DepthExport : public ::testing::Test {
+ protected:
+   M5DepthExport() { glsl_type_singleton_init_or_ref(); }
+   ~M5DepthExport() { glsl_type_singleton_decref(); }
+};
+
+struct epilog_shape {
+   enum agx_chip chip;
+   unsigned samples;
+   bool z, s, a2c;
+   bool blend = false, sample_shading = false;
+};
+
+static nir_shader *
+build_epilog(const epilog_shape &shape)
+{
+   agx_fs_epilog_key fs = {};
+   fs.nr_samples = shape.samples;
+   fs.max_sample_size_B = AGX_G17P_MAX_BYTES_PER_SAMPLE;
+   fs.max_bytes_per_tile = 32768;
+   fs.max_samples_per_tile = 2048;
+   fs.link.loc_written = 1;
+   fs.link.size_32 = 1;
+   fs.link.colour_comps = BITFIELD_MASK(4);
+   fs.link.write_z = shape.z;
+   fs.link.write_s = shape.s;
+   fs.link.sample_shading = shape.sample_shading;
+   fs.blend.alpha_to_coverage = shape.a2c;
+   agx_fs_epilog_apply_device_policy(shape.chip, &fs);
+   fs.rt_formats[0] = PIPE_FORMAT_R8G8B8A8_UNORM;
+   fs.remap[0] = 0;
+   for (unsigned rt = 1; rt < ARRAY_SIZE(fs.remap); ++rt)
+      fs.remap[rt] = -1;
+   fs.blend.rt[0].colormask = 0xf;
+   fs.blend.rt[0].mode =
+      shape.blend
+         ? agx_pack_blend_standard(PIPE_BLEND_ADD, PIPE_BLENDFACTOR_SRC_ALPHA,
+                                   PIPE_BLENDFACTOR_INV_SRC_ALPHA,
+                                   PIPE_BLEND_ADD, PIPE_BLENDFACTOR_ONE,
+                                   PIPE_BLENDFACTOR_INV_SRC_ALPHA)
+         : agx_pack_blend_standard(PIPE_BLEND_ADD, PIPE_BLENDFACTOR_ONE,
+                                   PIPE_BLENDFACTOR_ZERO, PIPE_BLEND_ADD,
+                                   PIPE_BLENDFACTOR_ONE, PIPE_BLENDFACTOR_ZERO);
+
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_FRAGMENT, &agx_nir_options, "M5 depth export epilog");
+   /* As hk selects the builder for M5. */
+   if (shape.sample_shading)
+      agx_nir_fs_epilog_g17p(&b, &fs);
+   else
+      agx_nir_fs_epilog(&b, &fs);
+   return b.shader;
+}
+
+struct counts {
+   unsigned store_zs = 0, covered_sample = 0, transactions = 0, demotes = 0;
+   unsigned coverage_updates = 0;
+   unsigned stores_in_loop = 0;
+};
+
+static counts
+count_nir(nir_shader *s)
+{
+   counts c;
+   nir_foreach_block(block, nir_shader_get_entrypoint(s)) {
+      bool in_loop = false;
+      for (nir_cf_node *n = block->cf_node.parent; n; n = n->parent)
+         in_loop |= n->type == nir_cf_node_loop;
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         if (intr->intrinsic == nir_intrinsic_store_zs_agx) {
+            c.store_zs++;
+            c.covered_sample +=
+               !!(nir_intrinsic_base(intr) & AGX_ZS_COVERED_SAMPLE);
+            c.stores_in_loop += in_loop;
+         }
+         c.transactions += intr->intrinsic == nir_intrinsic_zs_transaction_agx;
+         c.demotes += intr->intrinsic == nir_intrinsic_demote_samples;
+         c.coverage_updates += intr->intrinsic == nir_intrinsic_sample_mask_agx;
+      }
+   }
+   return c;
+}
+
+/* Mnemonics of an AGX3c binary, in order. */
+static std::vector<std::string>
+mnemonics(const agx_shader_part &part)
+{
+   std::vector<std::string> out;
+   const uint8_t *code = (const uint8_t *)part.binary + part.info.main_offset;
+   size_t len = part.info.main_size, i = 0;
+   while (i < len) {
+      char *text = NULL;
+      size_t text_len = 0;
+      FILE *fp = open_memstream(&text, &text_len);
+      uint8_t buf[32] = {0};
+      memcpy(buf, code + i, MIN2(sizeof(buf), len - i));
+      signed ret = agx3c_disassemble_instr((BITSET_WORD *)buf, fp, i,
+                                           len - i, false);
+      fclose(fp);
+      std::string line = text ? text : "";
+      free(text);
+      unsigned length = abs(ret);
+      if (length == 0)
+         break;
+      /* Keep the mnemonic: the first word after the decoder's
+       * "# XXX: Unknown bit set N" notes. */
+      size_t at = 0;
+      while (at < line.size()) {
+         size_t start = line.find_first_not_of(" \t\n", at);
+         if (start == std::string::npos)
+            break;
+         size_t end = line.find_first_of(" \t\n", start);
+         std::string word = line.substr(start, end - start);
+         at = end;
+         if (word == "#" || word == "XXX:" || word == "Unknown" ||
+             word == "bit" || word == "set" ||
+             word.find_first_not_of("0123456789") == std::string::npos)
+            continue;
+         out.push_back(word);
+         break;
+      }
+      i += length;
+   }
+   return out;
+}
+
+static unsigned
+count_of(const std::vector<std::string> &m, const char *name)
+{
+   unsigned n = 0;
+   for (const auto &s : m)
+      n += s == name;
+   return n;
+}
+
+static int
+index_of(const std::vector<std::string> &m, const char *name)
+{
+   for (unsigned i = 0; i < m.size(); ++i) {
+      if (m[i] == name)
+         return i;
+   }
+   return -1;
+}
+
+static bool
+compile_m5(nir_shader *s, unsigned samples, agx_shader_part *part,
+           bool preprocessed = false)
+{
+   if (!preprocessed)
+      agx_preprocess_nir(s);
+   if (samples > 1)
+      agx_nir_lower_g17p_tile_samples(s, samples);
+   agx_shader_key key = {};
+   key.dev.encoding = AGX_ISA_ENCODING_AGX3C;
+   key.dev.chip = AGX_CHIP_G17S;
+   key.no_preamble = true;
+   key.secondary = true;
+   key.fs.nr_samples = samples;
+   BITSET_SET(key.g17p_tile_partitions, 0);
+   return agx_compile_shader_nir(s, &key, part);
+}
+
+} /* namespace */
+
+/* Native zs_depth_{2,4,8}x, zs_cdepth_4x and zs_stencil_{2,4}x. */
+TEST_F(M5DepthExport, SingleAttachmentLoopsOverCoveredSamples)
+{
+   for (unsigned samples : {2u, 4u, 8u}) {
+      for (bool stencil : {false, true}) {
+         SCOPED_TRACE(::testing::Message() << samples << " samples, "
+                      << (stencil ? "stencil" : "depth"));
+         nir_shader *s = build_epilog({AGX_CHIP_G17S, samples, !stencil,
+                                       stencil, false});
+         counts c = count_nir(s);
+         EXPECT_EQ(c.store_zs, 1u);
+         EXPECT_EQ(c.covered_sample, 1u);
+         EXPECT_EQ(c.stores_in_loop, 1u);
+         EXPECT_EQ(c.transactions, 2u);
+
+         agx_shader_part part = {};
+         ASSERT_TRUE(compile_m5(s, samples, &part)) << agx_last_compile_refusal();
+         std::vector<std::string> m = mnemonics(part);
+         EXPECT_EQ(count_of(m, "depthf"), 1u);
+         EXPECT_EQ(count_of(m, "cndlt"), 1u);
+         /* One class-1 acquire, ahead of the loop and its DEPTHF. */
+         int depthf = index_of(m, "depthf"), loop = index_of(m, "cndlt");
+         int last_wait = -1;
+         unsigned waits_before_depthf = 0;
+         for (int i = 0; i < depthf; ++i) {
+            if (m[i] == "pixwait") {
+               last_wait = i;
+               waits_before_depthf++;
+            }
+         }
+         EXPECT_EQ(waits_before_depthf, 2u); /* entry gate, then class 1 */
+         EXPECT_LT(last_wait, depthf);
+         EXPECT_LT(depthf, loop);
+         bool cf_between = false;
+         for (int i = last_wait + 1; i < depthf; ++i)
+            cf_between |= m[i] == "cndst";
+         EXPECT_TRUE(cf_between) << "class-1 wait must precede the loop";
+         free(part.binary);
+         ralloc_free(s);
+      }
+   }
+}
+
+/* Native zs_depth_stencil_{2,4}x. */
+TEST_F(M5DepthExport, DepthAndStencilIsOneWord)
+{
+   for (unsigned samples : {2u, 4u}) {
+      SCOPED_TRACE(::testing::Message() << samples << " samples");
+      nir_shader *s = build_epilog({AGX_CHIP_G17S, samples, true, true, false});
+      counts c = count_nir(s);
+      EXPECT_EQ(c.store_zs, 1u);
+      EXPECT_EQ(c.covered_sample, 0u);
+      EXPECT_EQ(c.transactions, 0u);
+      agx_shader_part part = {};
+      ASSERT_TRUE(compile_m5(s, samples, &part)) << agx_last_compile_refusal();
+      std::vector<std::string> m = mnemonics(part);
+      EXPECT_EQ(count_of(m, "depthf"), 1u);
+      EXPECT_EQ(count_of(m, "cndlt"), 0u);
+      EXPECT_EQ(count_of(m, "movmsk"), 0u);
+      free(part.binary);
+      ralloc_free(s);
+   }
+}
+
+/* Native zs_depth_{2,4}x_a2c, zs_cdepth_4x_a2c, zs_stencil_4x_a2c and
+ * zs_depth_stencil_4x_a2c. */
+TEST_F(M5DepthExport, AlphaToCoverageFoldsIntoSampleWords)
+{
+   for (unsigned samples : {2u, 4u}) {
+      for (unsigned zs : {1u, 2u, 3u}) {
+         SCOPED_TRACE(::testing::Message() << samples << " samples, zs=" << zs);
+         nir_shader *s = build_epilog({AGX_CHIP_G17S, samples, !!(zs & 1),
+                                       !!(zs & 2), true});
+         counts c = count_nir(s);
+         EXPECT_EQ(c.demotes, 0u);
+         EXPECT_EQ(c.coverage_updates, 0u);
+         EXPECT_EQ(c.covered_sample, 0u);
+         EXPECT_FALSE(s->info.fs.uses_discard);
+         agx_shader_part part = {};
+         ASSERT_TRUE(compile_m5(s, samples, &part)) << agx_last_compile_refusal();
+         std::vector<std::string> m = mnemonics(part);
+         EXPECT_EQ(count_of(m, "depthf"), samples);
+         EXPECT_EQ(count_of(m, "movmsk"), 0u);
+         EXPECT_EQ(count_of(m, "cndlt"), 0u);
+         free(part.binary);
+         ralloc_free(s);
+      }
+   }
+}
+
+/* Without a depth or stencil write, alpha-to-coverage keeps its kill, and a
+ * single sample keeps its one word. */
+TEST_F(M5DepthExport, OtherShapesUnchanged)
+{
+   nir_shader *s = build_epilog({AGX_CHIP_G17S, 4, false, false, true});
+   counts c = count_nir(s);
+   EXPECT_EQ(c.store_zs, 0u);
+   EXPECT_EQ(c.transactions, 0u);
+   EXPECT_TRUE(s->info.fs.uses_discard);
+   ralloc_free(s);
+
+   s = build_epilog({AGX_CHIP_G17S, 1, true, false, false});
+   c = count_nir(s);
+   EXPECT_EQ(c.store_zs, 1u);
+   EXPECT_EQ(c.covered_sample, 0u);
+   EXPECT_EQ(c.transactions, 0u);
+   ralloc_free(s);
+
+   /* Not M5: no covered-sample loop, and alpha coverage stays a separate
+    * coverage update. */
+   s = build_epilog({AGX_CHIP_G17P, 4, true, false, true});
+   c = count_nir(s);
+   EXPECT_EQ(c.covered_sample, 0u);
+   EXPECT_EQ(c.transactions, 0u);
+   EXPECT_GT(c.demotes + c.coverage_updates, 0u);
+   ralloc_free(s);
+}
+
+TEST_F(M5DepthExport, SampleRateCoverageTestsBeforeColour)
+{
+   for (bool sample_shading : {false, true}) {
+      for (unsigned samples : {2u, 4u}) {
+         SCOPED_TRACE(::testing::Message() << samples << " samples"
+                      << (sample_shading ? ", sample shaded" : ""));
+         epilog_shape shape = {AGX_CHIP_G17S, samples, false, false, true};
+         shape.sample_shading = sample_shading;
+         nir_shader *s = build_epilog(shape);
+         agx_preprocess_nir(s);
+         counts c = count_nir(s);
+         EXPECT_EQ(c.coverage_updates, 1u);
+
+         agx_shader_part part = {};
+         ASSERT_TRUE(compile_m5(s, samples, &part, true))
+            << agx_last_compile_refusal();
+         std::vector<std::string> m = mnemonics(part);
+         EXPECT_EQ(count_of(m, "movmsk"), 1u);
+         EXPECT_LT(index_of(m, "movmsk"), index_of(m, "lstcol"));
+         free(part.binary);
+         ralloc_free(s);
+      }
+   }
+}
+
+TEST_F(M5DepthExport, BlendedAlphaCoverageTestsOncePerPixel)
+{
+   for (bool sample_shading : {false, true}) {
+      for (unsigned samples : {2u, 4u, 8u}) {
+         if (sample_shading && samples == 8)
+            continue;
+         SCOPED_TRACE(::testing::Message() << samples << " samples"
+                      << (sample_shading ? ", sample shaded" : ""));
+         epilog_shape shape = {AGX_CHIP_G17S, samples, false, false, true};
+         shape.blend = true;
+         shape.sample_shading = sample_shading;
+         nir_shader *s = build_epilog(shape);
+         agx_preprocess_nir(s);
+         counts c = count_nir(s);
+         EXPECT_EQ(c.coverage_updates, 1u);
+
+         agx_shader_part part = {};
+         ASSERT_TRUE(compile_m5(s, samples, &part, true))
+            << agx_last_compile_refusal();
+         std::vector<std::string> m = mnemonics(part);
+         EXPECT_EQ(count_of(m, "lldcol"), samples);
+         EXPECT_EQ(count_of(m, "movmsk"), 1u);
+         EXPECT_LT(index_of(m, "movmsk"), index_of(m, "lldcol"));
+         free(part.binary);
+         ralloc_free(s);
+      }
+   }
+}
+
+/* Native blend_depth_{2,4,8}x and a2cb_depth_{2,4,8}x: a blended fragment
+ * that writes depth finishes its depth words before blending any sample. */
+TEST_F(M5DepthExport, BlendedDepthWritesPrecedeTheBlend)
+{
+   for (bool a2c : {false, true}) {
+      for (unsigned samples : {2u, 4u, 8u}) {
+         SCOPED_TRACE(::testing::Message() << samples << " samples"
+                      << (a2c ? ", alpha to coverage" : ""));
+         epilog_shape shape = {AGX_CHIP_G17S, samples, true, false, a2c};
+         shape.blend = true;
+         nir_shader *s = build_epilog(shape);
+         agx_preprocess_nir(s);
+         counts c = count_nir(s);
+         EXPECT_EQ(c.store_zs, 1u);
+         EXPECT_EQ(c.covered_sample, a2c ? 0u : 1u);
+         EXPECT_EQ(c.coverage_updates, 0u);
+
+         agx_shader_part part = {};
+         ASSERT_TRUE(compile_m5(s, samples, &part, true))
+            << agx_last_compile_refusal();
+         std::vector<std::string> m = mnemonics(part);
+         EXPECT_EQ(count_of(m, "lldcol"), samples);
+         EXPECT_EQ(count_of(m, "depthf"), a2c ? samples : 1u);
+         EXPECT_EQ(count_of(m, "movmsk"), 0u);
+         int last_depthf = -1;
+         for (unsigned i = 0; i < m.size(); ++i) {
+            if (m[i] == "depthf")
+               last_depthf = i;
+         }
+         EXPECT_LT(last_depthf, index_of(m, "lldcol"));
+         free(part.binary);
+         ralloc_free(s);
+      }
+   }
+}
diff --git a/src/asahi/lib/tests/test-packing.cpp b/src/asahi/lib/tests/test-packing.cpp
index e3273d2d7b89..f167e86da82f 100644
--- a/src/asahi/lib/tests/test-packing.cpp
+++ b/src/asahi/lib/tests/test-packing.cpp
@@ -4,6 +4,9 @@
  */
 
 #include "agx_pack.h"
+#include "agx_vdm_mesh_core.h"
+#include <cstring>
+#include <array>
 
 #include <gtest/gtest.h>
 
@@ -83,3 +86,60 @@ TEST(Groups, Decode)
          expected);
    }
 }
+
+
+TEST(M5MeshLimits, MatchesNativePackets)
+{
+   struct { uint32_t vertices, primitives, words, prefix, prim_limit, vert_limit; } cases[] = {
+      {6, 2, 4, 0x504, 0x1, 0x9},
+      {6, 2, 5, 0x605, 0x1, 0xb},
+      {192, 64, 4, 0x504, 0x3003f, 0xc1},
+   };
+   for (const auto &c : cases) {
+      agx_vdm_mesh_state_info info = {};
+      ASSERT_TRUE(agx_vdm_m5_mesh_set_output_limits(&info, c.vertices, c.primitives, c.words));
+      EXPECT_EQ(info.raw_prefix[0], c.prefix);
+      EXPECT_EQ(info.raw_tail[1], c.prim_limit);
+      EXPECT_EQ(info.raw_tail[4], c.vert_limit);
+   }
+}
+
+/* The mesh dispatch record is an Index List block whose Primitive field
+ * (bits 8..15) names the mesh output topology; native triangle meshes
+ * carry 6 (0x70000600). */
+TEST(M5MeshDispatch, PrimitiveFollowsTopology)
+{
+   for (const auto &c : {std::array<uint32_t, 2>{0, 0x70000000}, {1, 0x70000100},
+                        {6, 0x70000600}}) {
+      agx_vdm_mesh_dispatch_info info = {};
+      for (unsigned i = 0; i < 3; ++i)
+         info.threads[i] = info.local_size[i] = i ? 1 : 3;
+      info.primitive = c[0];
+      uint8_t bytes[AGX_VDM_MESH_DISPATCH_LENGTH];
+      ASSERT_TRUE(agx_vdm_mesh_dispatch_pack(AGX_CHIP_G17S, &info, bytes));
+      uint32_t header;
+      memcpy(&header, bytes, 4);
+      EXPECT_EQ(header, c[1]);
+      agx_vdm_mesh_dispatch_info back;
+      ASSERT_TRUE(agx_vdm_mesh_dispatch_unpack(AGX_CHIP_G17S, bytes, sizeof(bytes), &back));
+      EXPECT_EQ(back.primitive, c[0]);
+   }
+   agx_vdm_mesh_dispatch_info info = {};
+   for (unsigned i = 0; i < 3; ++i)
+      info.threads[i] = info.local_size[i] = 1;
+   info.primitive = 3; /* line strip: no mesh topology */
+   uint8_t bytes[AGX_VDM_MESH_DISPATCH_LENGTH];
+   EXPECT_FALSE(agx_vdm_mesh_dispatch_pack(AGX_CHIP_G17S, &info, bytes));
+}
+
+TEST(M5MeshLimits, RejectsInvalidLimitsWithoutMutation)
+{
+   for (const auto &c : {std::array<uint32_t, 3>{0, 1, 4}, {1, 0, 4},
+                        {257, 1, 4}, {1, 257, 4}, {1, 1, 3}, {1, 1, 256}}) {
+      agx_vdm_mesh_state_info info;
+      memset(&info, 0x5a, sizeof(info));
+      const auto before = info;
+      EXPECT_FALSE(agx_vdm_m5_mesh_set_output_limits(&info, c[0], c[1], c[2]));
+      EXPECT_EQ(memcmp(&info, &before, sizeof(info)), 0);
+   }
+}
diff --git a/src/asahi/lib/tests/test-primitive-varyings.cpp b/src/asahi/lib/tests/test-primitive-varyings.cpp
new file mode 100644
index 000000000000..f3ac7f9efc9b
--- /dev/null
+++ b/src/asahi/lib/tests/test-primitive-varyings.cpp
@@ -0,0 +1,70 @@
+#include <gtest/gtest.h>
+#include "asahi/compiler/agx_compile.h"
+#include "compiler/nir/nir_builder.h"
+extern "C" {
+#include "asahi/vulkan/hk_nir_widen_16bit_varyings.h"
+}
+
+class PrimitiveVaryings : public ::testing::Test {
+protected:
+   PrimitiveVaryings() { glsl_type_singleton_init_or_ref(); }
+   ~PrimitiveVaryings() { glsl_type_singleton_decref(); }
+};
+
+TEST_F(PrimitiveVaryings, MeshStoresWidenBeforeInterstageOptimization)
+{
+   for (nir_alu_type type : {nir_type_int16, nir_type_uint16, nir_type_float16}) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_MESH, &agx_nir_options, "primitive store widening");
+      nir_def *value = nir_imm_intN_t(&b, 0xfc01, 16);
+      nir_store_per_primitive_output(&b, value, nir_imm_int(&b, 0),
+         nir_imm_int(&b, 0), .write_mask = 1, .src_type = type,
+         .io_semantics = {.location = VARYING_SLOT_VAR0, .num_slots = 1});
+      ASSERT_TRUE(hk_nir_widen_16bit_varyings(b.shader, HK_NIR_WIDEN_16BIT_OUTPUTS));
+      nir_intrinsic_instr *store = nir_instr_as_intrinsic(nir_block_last_instr(nir_cursor_current_block(b.cursor)));
+      EXPECT_EQ(store->src[0].ssa->bit_size, 32u);
+      EXPECT_EQ(nir_intrinsic_src_type(store), nir_alu_type_get_base_type(type) | 32);
+      nir_alu_instr *convert = nir_instr_as_alu(nir_def_instr(store->src[0].ssa));
+      EXPECT_EQ(convert->op, type == nir_type_float16 ? nir_op_f2f32 :
+                            type == nir_type_int16 ? nir_op_i2i32 : nir_op_u2u32);
+      EXPECT_EQ(convert->src[0].src.ssa, value);
+      nir_opt_constant_folding(b.shader);
+      if (type == nir_type_int16) {
+         EXPECT_EQ(nir_src_as_uint(store->src[0]), 0xfffffc01u);
+      }
+      if (type == nir_type_uint16) {
+         EXPECT_EQ(nir_src_as_uint(store->src[0]), 0xfc01u);
+      }
+      ralloc_free(b.shader);
+   }
+}
+
+TEST_F(PrimitiveVaryings, PrimitiveLoadsWidenAndNarrowNumerically)
+{
+   for (bool output : {false, true}) {
+      for (nir_alu_type type : {nir_type_int16, nir_type_uint16, nir_type_float16}) {
+         nir_builder b = nir_builder_init_simple_shader(
+            output ? MESA_SHADER_MESH : MESA_SHADER_FRAGMENT,
+            &agx_nir_options, "primitive load widening");
+         nir_def *value = output
+            ? nir_load_per_primitive_output(&b, 1, 16, nir_imm_int(&b, 0),
+                 nir_imm_int(&b, 0), .dest_type = type,
+                 .io_semantics = {.location = VARYING_SLOT_VAR0, .num_slots = 1})
+            : nir_load_per_primitive_input(&b, 1, 16, nir_imm_int(&b, 0),
+                 .dest_type = type,
+                 .io_semantics = {.location = VARYING_SLOT_VAR0, .num_slots = 1});
+         nir_def *use = nir_iadd_imm(&b, value, 1);
+         auto mode = output ? HK_NIR_WIDEN_16BIT_OUTPUTS : HK_NIR_WIDEN_16BIT_INPUTS;
+         ASSERT_TRUE(hk_nir_widen_16bit_varyings(b.shader, mode));
+         EXPECT_EQ(value->bit_size, 32u);
+         auto *load = nir_instr_as_intrinsic(nir_def_instr(value));
+         EXPECT_EQ(nir_intrinsic_dest_type(load), nir_alu_type_get_base_type(type) | 32);
+         auto *user = nir_instr_as_alu(nir_def_instr(use));
+         auto *convert = nir_instr_as_alu(nir_def_instr(user->src[0].src.ssa));
+         EXPECT_EQ(convert->op, type == nir_type_float16 ? nir_op_f2f16 : nir_op_u2u16);
+         EXPECT_EQ(convert->def.bit_size, 16u);
+         EXPECT_EQ(convert->src[0].src.ssa, value);
+         ralloc_free(b.shader);
+      }
+   }
+}
diff --git a/src/asahi/lib/tests/test-rt-bvh.c b/src/asahi/lib/tests/test-rt-bvh.c
index 851e7b7fc1fc..cf007422101c 100644
--- a/src/asahi/lib/tests/test-rt-bvh.c
+++ b/src/asahi/lib/tests/test-rt-bvh.c
@@ -233,6 +233,11 @@ test_gen1_encoding(void)
    }
 
    struct agx_rt_bvh_gen1_header header = {0};
+   require(agx_rt_bvh_gen1_header_set_child_layout(
+              &header, 0, 2, 3, 26, 0, true) &&
+              header.words[0] == 1 && header.words[6] == 3 &&
+              header.words[7] == 0x1a10,
+           "native M5 instance root uses indexed counts with zero capacity");
    require(agx_rt_bvh_gen1_header_set_child_layout(
               &header, 2, 3, 0x12345678, 17, 3, true),
            "mixed child topology encodes");
diff --git a/src/asahi/lib/tests/test-tilebuffer.cpp b/src/asahi/lib/tests/test-tilebuffer.cpp
index bb53e042e924..4c944a3c4050 100644
--- a/src/asahi/lib/tests/test-tilebuffer.cpp
+++ b/src/asahi/lib/tests/test-tilebuffer.cpp
@@ -705,6 +705,44 @@ TEST(Tilebuffer, IndependentNativeStorageAndSampleLimits)
    EXPECT_FALSE(spilling.spilled[1]);
 }
 
+TEST(Tilebuffer, ResidentAttachmentsStayInTheTilebuffer)
+{
+   /* M5 at 4x: 64 bytes per sample. Four RGBA32F targets fill it, so the
+    * plain descending-alignment order spills the RGBA8 target. */
+   enum pipe_format formats[AGX_MAX_RENDER_TARGETS] = {
+      PIPE_FORMAT_R8G8B8A8_UNORM,     PIPE_FORMAT_R32G32B32A32_FLOAT,
+      PIPE_FORMAT_R32G32B32A32_FLOAT, PIPE_FORMAT_R32G32B32A32_FLOAT,
+      PIPE_FORMAT_R32G32B32A32_FLOAT,
+   };
+   const auto plain = agx_build_tilebuffer_layout_resident(
+      formats, 5, 4, false, AGX_MAX_RENDER_TARGETS * 16, 65536, 2048, 0);
+   EXPECT_TRUE(plain.spilled[0]);
+   EXPECT_EQ(plain.resident_mask, 0u);
+
+   /* Asking for RT0 to stay resident moves it first; a filler spills. */
+   const auto resident = agx_build_tilebuffer_layout_resident(
+      formats, 5, 4, false, AGX_MAX_RENDER_TARGETS * 16, 65536, 2048, 0x1);
+   EXPECT_FALSE(resident.spilled[0]);
+   EXPECT_EQ(resident._offset_B[0], 0u);
+   EXPECT_EQ(resident._offset_B[1], 4u);
+   EXPECT_TRUE(resident.spilled[4]);
+   EXPECT_EQ(resident.resident_mask, 0x1u);
+
+   /* Rebuilding with the recorded mask reproduces the layout. */
+   const auto again = agx_build_tilebuffer_layout_resident(
+      formats, 5, 4, false, AGX_MAX_RENDER_TARGETS * 16, 65536, 2048,
+      resident.resident_mask);
+   EXPECT_EQ(memcmp(&again, &resident, sizeof(again)), 0);
+
+   /* A mask is ignored when the plain order keeps those targets resident. */
+   const auto fits = agx_build_tilebuffer_layout_resident(
+      formats, 2, 4, false, AGX_MAX_RENDER_TARGETS * 16, 65536, 2048, 0x1);
+   EXPECT_FALSE(agx_tilebuffer_spills(&fits));
+   EXPECT_EQ(fits.resident_mask, 0u);
+   EXPECT_EQ(fits._offset_B[1], 0u);
+   EXPECT_EQ(fits._offset_B[0], 16u);
+}
+
 class SampleDepthOrder : public ::testing::Test {
  protected:
    SampleDepthOrder()
diff --git a/src/asahi/libagx/as_properties.cl b/src/asahi/libagx/as_properties.cl
index 8a73c8be3bf6..c5615e7ce9fb 100644
--- a/src/asahi/libagx/as_properties.cl
+++ b/src/asahi/libagx/as_properties.cl
@@ -280,9 +280,11 @@ KERNEL(1)
 libagx_gen1_as_empty(global uint64_t *image, global uint64_t *metadata,
                     uint32_t resource_id, uint32_t build_flags, uint32_t geometry_type)
 {
-   global uint32_t *header = (global uint32_t *)image + 16;
-   for (uint32_t i = 0; i < 28; ++i)
-      header[i] = i >= 4 && i < 10 ? 0x7fc00000u : 0;
+   global uint32_t *header = (global uint32_t *)image;
+   for (uint32_t i = 0; i < LIBAGX_GEN1_AS_EMPTY_BYTES / 4; ++i)
+      header[i] = i >= 20 && i < 26 ? 0x7fc00000u : 0;
+   image[LIBAGX_GEN1_AS_BASE_WORD] = (uint64_t)image;
+   image[LIBAGX_GEN1_AS_ID_WORD] = resource_id;
    struct libagx_gen1_as_metadata live = {
       .native_base = (uint64_t)image,
       .image_bytes = LIBAGX_GEN1_AS_EMPTY_BYTES,
diff --git a/src/asahi/libagx/compression.cl b/src/asahi/libagx/compression.cl
index 1097dd19ffdc..2af776a15a8e 100644
--- a/src/asahi/libagx/compression.cl
+++ b/src/asahi/libagx/compression.cl
@@ -155,12 +155,82 @@ stretched_sa_to_px(ushort2 px, uint samples)
    /* clang-format on */
 }
 
+/* A half promoted exactly to float needs no rounding to recover its bits. */
+static uint
+libagx_m5_recover_half(uint value)
+{
+   const uint sign = (value >> 16) & 0x8000u;
+   const uint exponent = (value >> 23) & 0xffu;
+   const uint fraction = value & 0x7fffffu;
+   if (exponent == 255)
+      return sign | 0x7c00u | (fraction >> 13);
+   if (exponent < 103)
+      return sign;
+   if (exponent < 113)
+      return sign | ((fraction | 0x800000u) >> (126u - exponent));
+   return sign | ((exponent - 112u) << 10) | (fraction >> 13);
+}
+
+/* Native RGBA16F spill conversion. The texture heap index names the existing
+ * compressed view; the native fetch promotes its half lanes exactly to float.
+ * Each workgroup reads a complete metadata tile before overwriting its body.
+ */
+KERNEL(32)
+libagx_m5_decompress_rgba16f(global void *texture_heap,
+                            global void *sampler_heap,
+                            global uint64_t *metadata, global uint2 *body,
+                            uint read_index, uint metadata_width_tl,
+                            uint metadata_height_tl, uint aligned_width_px,
+                            ushort tile_width_px, ushort tile_height_px,
+                            uint samples_log2__2)
+{
+   const uint samples = 2u << samples_log2__2;
+   const uint index_tl = index_metadata(cl_group_id, metadata_width_tl,
+                                        metadata_height_tl, 0);
+   if (metadata[index_tl] == ~(uint64_t)0)
+      return;
+
+   const uint handle = nir_bindless_image_agx(read_index, 0);
+   const uint2 origin = cl_group_id.xy * 16u;
+   const uint2 lane = (uint2)((cl_local_id.x & 1u) * 8u,
+                              cl_local_id.x >> 1);
+   uint2 values[8];
+   for (uint i = 0; i < 8; i++) {
+      const int4 sample_coord = (int4)(origin.x + lane.x + i,
+                                       origin.y + lane.y, 0, 0);
+      const uint4 v = nir_bindless_image_load(
+         handle, decompose_px(sample_coord, samples),
+         sample_id(sample_coord, samples), 0, GLSL_SAMPLER_DIM_MS, false,
+         0, ACCESS_IN_BOUNDS, nir_type_uint32);
+      values[i] = (uint2)(libagx_m5_recover_half(v.x) |
+                            (libagx_m5_recover_half(v.y) << 16),
+                          libagx_m5_recover_half(v.z) |
+                            (libagx_m5_recover_half(v.w) << 16));
+   }
+
+   sub_group_barrier(CLK_LOCAL_MEM_FENCE);
+   for (uint i = 0; i < 8; i++) {
+      const int4 sample_coord = (int4)(origin.x + lane.x + i,
+                                       origin.y + lane.y, 0, 0);
+      const int4 pixel = decompose_px(sample_coord, samples);
+      const uint offset = libagx_twiddle_coordinates(
+         convert_ushort2(pixel.xy), tile_width_px, tile_height_px,
+         aligned_width_px);
+      body[offset * samples + sample_id(sample_coord, samples)] =
+         values[i];
+   }
+   if (cl_local_id.x == 0)
+      metadata[index_tl] = ~(uint64_t)0;
+}
+
 /*
  * Compute kernel to clear a compressed image to a solid colour by filling the
  * compression metadata to all solid and writing the colour to each tile.
  *
- * Invoked as <width in 16x16 tiles, height in 16x16 tiles, layers>. Each
- * invocation clears a single 16x16 compression tile.
+ * Invoked as <width in compression tiles, height in compression tiles,
+ * layers>. Each invocation clears a single compression tile: 16x16 samples,
+ * or 16x8 for M5 16-byte formats (ail_metadata_tile, given here as the log2
+ * of its height).
  *
  * Despite the ALU complexity, this is twice as fast as clearing with the
  * 3D pipe for a few reasons.
@@ -190,7 +260,8 @@ KERNEL(32)
 libagx_fast_clear(global uint2 *meta, global uint4 *body, uint width_tl,
                   uint height_tl, uint tile_w_el, uint tile_h_el,
                   uint meta_layer_stride_tl, uint body_layer_stride_uint4,
-                  uint blocksize_B_log2__5, uint samples_log2__3, uint a,
+                  uint meta_tile_h_log2, uint blocksize_B_log2__5,
+                  uint samples_log2__3, uint a,
                   uint b, uint c, uint d, uint solid_mode)
 {
    uint samples = 1 << samples_log2__3;
@@ -200,12 +271,11 @@ libagx_fast_clear(global uint2 *meta, global uint4 *body, uint width_tl,
    meta[index_metadata(cl_global_id, width_tl, height_tl, meta_layer_stride_tl)] =
       solid_mode;
 
-   /* SWAR vectorized multiply by 16, we know this won't overflow */
-   ushort2 coord_el =
-      as_ushort2(as_uint(convert_ushort2(cl_global_id.xy)) << 4);
+   ushort2 coord_el = (ushort2)(cl_global_id.x << 4,
+                                cl_global_id.y << meta_tile_h_log2);
 
    /* Calculate the offset of the tile in the body buffer */
-   ushort2 dim_sa = (ushort2)(width_tl, height_tl) * (ushort)16;
+   ushort2 dim_sa = (ushort2)(width_tl << 4, height_tl << meta_tile_h_log2);
    uint aligned_width_px =
       align(stretched_sa_to_px(dim_sa, samples).x, tile_w_el);
 
diff --git a/src/asahi/libagx/copy.cl b/src/asahi/libagx/copy.cl
index ffe296cd9fc5..e492fbdd6b1a 100644
--- a/src/asahi/libagx/copy.cl
+++ b/src/asahi/libagx/copy.cl
@@ -301,3 +301,164 @@ libagx_fill_uint4(global uint4 *address, uint layer_stride_uint4, uint a,
    uint offs = cl_global_id.x + cl_global_id.y * layer_stride_uint4;
    address[offs] = (uint4)(a, b, c, d);
 }
+
+/*
+ * Native M5 acceleration structures (vulkan/bvh/agx3_envelope.comp). An
+ * image starts at a 1KiB-aligned base. Slot 0 holds the root and the header
+ * starts at byte 64: byte 72 is the image size from the base, 112 the
+ * descriptor offset, 128 the instance pointer-table offset, 144 the absolute
+ * base address and 176 the device-wide structure ID. Driver-only empty
+ * metadata has a zero size and occupies bytes 0..191. Every other offset is
+ * relative to the base and instance records reference bottom-level images by
+ * address, so a word copy that rewrites the base and ID words relocates an
+ * image.
+ *
+ * The serialized form is the Vulkan header (driver UUID twice, serialized
+ * size, deserialized size, handle count), one bottom-level handle per
+ * instance in instance-index order, then the image.
+ */
+#define AGX3_AS_EMPTY_BYTES      192
+#define AGX3_AS_SERIAL_WORDS     7
+#define AGX3_AS_SLACK            1023
+#define AGX3_AS_INSTANCE_RECORD  26
+
+static uint64_t
+agx3_as_bytes(global uint64_t *image)
+{
+   return image[9] ? image[9] : AGX3_AS_EMPTY_BYTES;
+}
+
+static uint32_t
+agx3_as_instance_slots(global uint64_t *image)
+{
+   global uint32_t *words = (global uint32_t *)image;
+   if (!image[9] || (words[16] >> 24) != 0x83)
+      return 0;
+   return (image[14] - image[16]) / 16;
+}
+
+/* Instance pointer-table entries are written only for encoded instances, so
+ * an entry counts only when it names a record carrying its own index. */
+static global uint64_t *
+agx3_as_instance_record(global uint64_t *image, uint32_t index)
+{
+   global uint64_t *table =
+      (global uint64_t *)((global uint8_t *)image + image[16]);
+   uint64_t offset = table[2 * index];
+   uint64_t bytes = agx3_as_bytes(image);
+   if (table[2 * index + 1] != 1 || offset < 4 || offset - 4 > bytes ||
+       bytes - (offset - 4) < 128 || ((offset - 4) & 63))
+      return NULL;
+
+   global uint64_t *record =
+      (global uint64_t *)((global uint8_t *)image + offset - 4);
+   global uint32_t *words = (global uint32_t *)record;
+   if (words[0] != AGX3_AS_INSTANCE_RECORD || words[25] != index)
+      return NULL;
+   return record;
+}
+
+static void
+agx3_as_copy_words(global uint64_t *dst, global uint64_t *src, uint32_t chunk,
+                   uint64_t bytes, uint32_t dst_id, bool relocate)
+{
+   for (uint32_t w = chunk * 2; w < chunk * 2 + 2; ++w) {
+      if ((uint64_t)w * 8 >= bytes)
+         return;
+      uint64_t value = src[w];
+      if (relocate && w == 18)
+         value = (uint64_t)dst;
+      else if (relocate && w == 22)
+         value = (value & 0xffffffff00000000ul) | dst_id;
+      dst[w] = value;
+   }
+}
+
+KERNEL(32)
+libagx_agx3_as_copy(global uint64_t *dst, global uint64_t *src,
+                    uint32_t dst_id)
+{
+   agx3_as_copy_words(dst, src, cl_global_id.x, agx3_as_bytes(src), dst_id,
+                      src[9] != 0);
+}
+
+KERNEL(32)
+libagx_agx3_as_serialize(global uint64_t *dst, global uint64_t *src,
+                         uint32_t uuid0, uint32_t uuid1, uint32_t uuid2,
+                         uint32_t uuid3)
+{
+   uint32_t i = cl_global_id.x;
+   uint64_t bytes = agx3_as_bytes(src);
+   uint32_t handles = agx3_as_instance_slots(src);
+
+   if (i == 0) {
+      dst[0] = dst[2] = uuid0 | ((uint64_t)uuid1 << 32);
+      dst[1] = dst[3] = uuid2 | ((uint64_t)uuid3 << 32);
+      dst[4] = AGX3_AS_SERIAL_WORDS * 8 + (uint64_t)handles * 8 + bytes;
+      dst[5] = bytes + AGX3_AS_SLACK;
+      dst[6] = handles;
+   }
+
+   if (i < handles) {
+      global uint64_t *record = agx3_as_instance_record(src, i);
+      dst[AGX3_AS_SERIAL_WORDS + i] = record ? record[11] : 0;
+   }
+
+   agx3_as_copy_words(dst + AGX3_AS_SERIAL_WORDS + handles, src, i, bytes, 0,
+                      false);
+}
+
+KERNEL(32)
+libagx_agx3_as_deserialize(global uint64_t *dst, global uint64_t *src,
+                           uint32_t dst_id)
+{
+   uint64_t handles = src[6];
+   global uint64_t *image = src + AGX3_AS_SERIAL_WORDS + handles;
+   uint64_t bytes = src[4] - AGX3_AS_SERIAL_WORDS * 8 - handles * 8;
+   agx3_as_copy_words(dst, image, cl_global_id.x, bytes, dst_id,
+                      image[9] != 0);
+}
+
+/* After deserialization, point each instance at the bottom-level handle the
+ * application left in the serialized header. IDs come from the driver's
+ * created-object table: replacement BLAS need not have been built yet. Bounds
+ * remain those of the serialized geometry, which must be compatible. */
+KERNEL(32)
+libagx_agx3_as_relink(global uint64_t *dst, global uint64_t *src,
+                      global uint64_t *table, uint32_t table_count)
+{
+   uint32_t i = cl_global_id.x;
+   if (i >= src[6] || i >= agx3_as_instance_slots(dst))
+      return;
+
+   global uint64_t *record = agx3_as_instance_record(dst, i);
+   uint64_t blas = src[AGX3_AS_SERIAL_WORDS + i];
+   if (!record)
+      return;
+
+   record[11] = blas;
+   if (blas) {
+      global uint32_t *words = (global uint32_t *)record;
+      for (uint32_t id = 1; id <= table_count; ++id) {
+         if (table[2 * id] == blas) {
+            words[1] = id;
+            break;
+         }
+      }
+   }
+}
+
+/* kind: 0 compacted/structure size, 1 serialization size, 2 pointer count. */
+KERNEL(1)
+libagx_agx3_as_property(global uint64_t *image, global uint64_t *report,
+                        uint32_t kind)
+{
+   uint64_t bytes = agx3_as_bytes(image);
+   uint64_t handles = agx3_as_instance_slots(image);
+   if (kind == 2)
+      *report = handles;
+   else if (kind == 1)
+      *report = AGX3_AS_SERIAL_WORDS * 8 + handles * 8 + bytes;
+   else
+      *report = bytes + AGX3_AS_SLACK;
+}
diff --git a/src/asahi/libagx/draws.cl b/src/asahi/libagx/draws.cl
index 139581200cc9..23dff2679612 100644
--- a/src/asahi/libagx/draws.cl
+++ b/src/asahi/libagx/draws.cl
@@ -57,6 +57,32 @@ libagx_draw_indirect_byte_count(global VkDrawIndirectCommand *out,
  * requires a 32-bit load (D3D12 uses 64-bit predication), permits inversion,
  * and compute dispatch records contain three rather than four/five words.
  */
+/*
+ * M5 task (object) draws whose grid, draw count or predicate live in GPU
+ * memory. The VDM mesh dispatch record of such a draw is recorded with one
+ * object workgroup; patch its object thread counts before the render runs and
+ * publish whether the draw executes, which the task shader checks on entry
+ * (a disabled draw keeps one workgroup, which launches no mesh children).
+ */
+KERNEL(1)
+libagx_m5_task_dispatch(global uint32_t *record, global uint32_t *flag,
+                        constant uint32_t *grid, constant uint32_t *draw_count,
+                        constant uint32_t *predicate, uint32_t draw_index,
+                        uint32_t inverted, uint32_t local_x, uint32_t local_y,
+                        uint32_t local_z)
+{
+   bool enabled = grid[0] != 0 && grid[1] != 0 && grid[2] != 0;
+   if (draw_count)
+      enabled = enabled && draw_index < *draw_count;
+   if (predicate)
+      enabled = enabled && ((*predicate != 0) != (inverted != 0));
+
+   record[0] = enabled ? grid[0] * local_x : local_x;
+   record[1] = enabled ? grid[1] * local_y : local_y;
+   record[2] = enabled ? grid[2] * local_z : local_z;
+   *flag = enabled ? 1 : 0;
+}
+
 KERNEL(1)
 libagx_conditional_render(global uint32_t *out, constant uint32_t *in,
                           constant uint32_t *predicate, uint32_t words,
@@ -356,6 +382,137 @@ libagx_draw_robust_index_batch(constant struct libagx_robust_index_draw *draws,
  * only the dimensions), or zero when the application supplied the whole record
  * itself (vkCmdTraceRaysIndirect2KHR).
  */
+/*
+ * Per-raygen kernel selection for vkCmdTraceRays*.  A ray tracing pipeline
+ * compiles one compute kernel per ray-generation group, and the group that a
+ * trace runs is named only by the shader binding table record the application
+ * points at -- memory whose contents the specification requires to be valid
+ * at execution, not at record time (RayTracedGL1, for one, fills its SBT with
+ * a GPU copy recorded in the same command buffer).  So the selection happens
+ * here: read the raygen record's group handle, and publish one
+ * VkDispatchIndirectCommand per kernel: the launch's threadgroup count for the
+ * kernel whose raygen handle matches, zero for every other kernel.  The host
+ * then records one indirect dispatch per kernel.
+ *
+ * `record` is the VkTraceRaysIndirectCommand2KHR every trace form publishes
+ * (hk_cmd_trace_rays.c); its first quadword is raygenShaderRecordAddress and
+ * the group handle's index is the first dword of that record
+ * (hk_ray_tracing_core.h: hk_ray_tracing_group_handle).  `in_dims` is where the
+ * three launch dimensions live right now; `out_record_dims` is where they must
+ * be copied to for the kernels to read, or zero when the record already holds
+ * them.  `handles` lists each kernel's raygen handle index.
+ */
+KERNEL(1)
+libagx_trace_rays_select_setup(constant uint64_t *record,
+                               global uint32_t *out_record_dims,
+                               global VkDispatchIndirectCommand *out_groups,
+                               constant uint32_t *in_dims,
+                               constant uint32_t *handles,
+                               uint32_t kernel_count,
+                               uint32_t local_size_x_log2)
+{
+   uint32_t width = in_dims[0];
+   uint32_t height = in_dims[1];
+   uint32_t depth = in_dims[2];
+
+   if (out_record_dims) {
+      out_record_dims[0] = width;
+      out_record_dims[1] = height;
+      out_record_dims[2] = depth;
+   }
+
+   /* Same non-wrapping round-up as libagx_trace_rays_indirect_setup and
+    * hk_rt_launch_group_count. */
+   const uint32_t mask = (1u << local_size_x_log2) - 1u;
+   const uint32_t groups_x =
+      (width >> local_size_x_log2) + ((width & mask) ? 1u : 0u);
+
+   constant uint32_t *raygen_handle = (constant uint32_t *)(uintptr_t)record[0];
+   const uint32_t selected = *raygen_handle;
+
+   for (uint32_t k = 0; k < kernel_count; ++k) {
+      const bool run = handles[k] == selected;
+      out_groups[k] = (VkDispatchIndirectCommand){
+         .x = run ? groups_x : 0u,
+         .y = run ? height : 0u,
+         .z = run ? depth : 0u,
+      };
+   }
+}
+
+/*
+ * Single-kick variant of libagx_trace_rays_select_setup (opt-in
+ * ASAHI_M5_RT_SINGLE_KICK on M5). Instead of K indirect dispatches, the host
+ * records one Stream Link with return into `slot`, and this invocation writes
+ * the selected kernel's launch there, as Metal's indirect command buffers do
+ * (a GPU-written 40-byte direct launch reached through a barrier and a link).
+ *
+ * `templates` holds, per kernel, the 11 words the host encoded for that
+ * kernel: the 40-byte direct launch (sizes are overwritten here) and its
+ * trailing barrier. `slot` is {0x60000000, launch[10], barrier, return}; the
+ * host wrote words 0 and 12. With no matching kernel the launch keeps kernel
+ * 0's header with every size zero, the encoding Metal uses for an empty
+ * command. `out_groups` still receives the VkDispatchIndirectCommand the
+ * kernels' root descriptor points at.
+ */
+KERNEL(1)
+libagx_trace_rays_select_launch(constant uint64_t *record,
+                                global uint32_t *out_record_dims,
+                                global VkDispatchIndirectCommand *out_groups,
+                                constant uint32_t *in_dims,
+                                constant uint32_t *handles,
+                                constant uint32_t *templates,
+                                global uint32_t *slot,
+                                uint32_t kernel_count,
+                                uint32_t local_size_x_log2)
+{
+   uint32_t width = in_dims[0];
+   uint32_t height = in_dims[1];
+   uint32_t depth = in_dims[2];
+
+   if (out_record_dims) {
+      out_record_dims[0] = width;
+      out_record_dims[1] = height;
+      out_record_dims[2] = depth;
+   }
+
+   const uint32_t mask = (1u << local_size_x_log2) - 1u;
+   const uint32_t groups_x =
+      (width >> local_size_x_log2) + ((width & mask) ? 1u : 0u);
+
+   constant uint32_t *raygen_handle = (constant uint32_t *)(uintptr_t)record[0];
+   const uint32_t selected = *raygen_handle;
+
+   uint32_t chosen = kernel_count;
+   for (uint32_t k = 0; k < kernel_count; ++k) {
+      if (chosen == kernel_count && handles[k] == selected)
+         chosen = k;
+   }
+
+   const bool run = chosen < kernel_count && groups_x && height && depth;
+   *out_groups = (VkDispatchIndirectCommand){
+      .x = run ? groups_x : 0u,
+      .y = run ? height : 0u,
+      .z = run ? depth : 0u,
+   };
+
+   constant uint32_t *t = templates + 11u * (run ? chosen : 0u);
+   uint32_t word0 = t[0];
+   if (!run || (height < 2u && depth < 2u))
+      word0 &= 0xfff1ffffu;
+   slot[1] = word0;
+   slot[2] = t[1];
+   slot[3] = t[2];
+   slot[4] = t[3];
+   slot[5] = run ? (groups_x << local_size_x_log2) : 0u;
+   slot[6] = run ? height : 0u;
+   slot[7] = run ? depth : 0u;
+   slot[8] = run ? t[7] : 0u;
+   slot[9] = run ? t[8] : 0u;
+   slot[10] = run ? t[9] : 0u;
+   slot[11] = t[10];
+}
+
 KERNEL(1)
 libagx_trace_rays_indirect_setup(global uint32_t *out_record_dims,
                                  global VkDispatchIndirectCommand *out_groups,
diff --git a/src/asahi/libagx/gen1_as.h b/src/asahi/libagx/gen1_as.h
index d406490ede89..a65e2ab1b806 100644
--- a/src/asahi/libagx/gen1_as.h
+++ b/src/asahi/libagx/gen1_as.h
@@ -88,9 +88,11 @@ libagx_gen1_as_relocate_word(uint64_t value, uint64_t word,
                            uint64_t header_size, uint64_t destination,
                            uint32_t destination_id)
 {
-   if (header_size && word == LIBAGX_GEN1_AS_BASE_WORD)
+   /* Empty structures also retain their identity when used as instances. */
+   (void)header_size;
+   if (word == LIBAGX_GEN1_AS_BASE_WORD)
       return destination;
-   if (header_size && word == LIBAGX_GEN1_AS_ID_WORD)
+   if (word == LIBAGX_GEN1_AS_ID_WORD)
       return (value & UINT64_C(0xffffffff00000000)) | destination_id;
    return value;
 }
diff --git a/src/asahi/libagx/geometry.cl b/src/asahi/libagx/geometry.cl
index dc73a0483a2d..73c7c43c8f8f 100644
--- a/src/asahi/libagx/geometry.cl
+++ b/src/asahi/libagx/geometry.cl
@@ -176,7 +176,8 @@ libagx_prefix_sum_geom(constant struct poly_geometry_params *p)
 
 KERNEL(1024)
 libagx_prefix_sum_tess(global struct poly_tess_params *p, global uint *c_prims,
-                       global uint *c_invs, uint increment_stats__2)
+                       global uint *c_invs, global uint *prims_generated,
+                       uint increment_stats__2)
 {
    local uint scratch[32];
    poly_prefix_sum(scratch, p->counts, p->nr_patches, 1 /* words */,
@@ -217,6 +218,6 @@ libagx_prefix_sum_tess(global struct poly_tess_params *p, global uint *c_prims,
                    : p->isolines  ? (total / 2)
                                   : (total / 3);
 
-      poly_increment_counters(c_prims, c_invs, NULL, prims);
+      poly_increment_counters(c_prims, c_invs, prims_generated, prims);
    }
 }
diff --git a/src/asahi/libagx/libagx_dgc.h b/src/asahi/libagx/libagx_dgc.h
index cb6e01064bf0..5fb80aab0085 100644
--- a/src/asahi/libagx/libagx_dgc.h
+++ b/src/asahi/libagx/libagx_dgc.h
@@ -912,9 +912,10 @@ agx_vdm_draw_robust_index(GLOBAL uint32_t *out, enum agx_chip chip,
     * executes and the following CPU-authored command remains the only parser
     * target.
     */
-   if (chip == AGX_CHIP_G17P &&
+   if ((chip == AGX_CHIP_G17P ||
+        agx_vdm_index_list_direct_is_hal300_inline(chip)) &&
        (draw.b.count[0] == 0 || draw.b.count[1] == 0)) {
-      if (render_state_emitted)
+      if (chip == AGX_CHIP_G17P && render_state_emitted)
          *out++ = draw_state_context_offset;
 
       *out++ = 0x80000000u | (uint32_t)(empty_draw_jump >> 32);
diff --git a/src/asahi/libagx/query.cl b/src/asahi/libagx/query.cl
index 2b08e4d2fa9e..9fde3965adc6 100644
--- a/src/asahi/libagx/query.cl
+++ b/src/asahi/libagx/query.cl
@@ -17,6 +17,8 @@ query_report(global uint64_t *results, global uint16_t *oq_index,
     * need to remap indices according to the query pool's allocation.
     */
    uint result_index = oq_index ? oq_index[query] : query;
+   if (oq_index && result_index == LIBAGX_QUERY_INDEX_UNUSED)
+      return 0;
 
    return results + (result_index * reports_per_query);
 }
@@ -42,7 +44,7 @@ libagx_copy_query(global uint32_t *availability, global uint64_t *results,
 
    if (available || (flags & VK_QUERY_RESULT_PARTIAL_BIT)) {
       for (unsigned i = 0; i < reports_per_query; ++i) {
-         vk_write_query(dst, i, flags, report[i]);
+         vk_write_query(dst, i, flags, report ? report[i] : 0);
       }
    }
 
@@ -57,27 +59,34 @@ KERNEL(32)
 libagx_copy_query_g17p(global uint32_t *availability, global uint64_t *results,
                        global uint16_t *oq_index, uint64_t dst_addr,
                        uint64_t dst_stride, uint32_t first_query,
-                       VkQueryResultFlagBits flags, uint16_t reports_per_query)
+                       VkQueryResultFlagBits flags, uint16_t reports_per_query,
+                       uint16_t availability_shift)
 {
    libagx_copy_query_g17p_result(availability, results, oq_index, dst_addr,
                                 dst_stride, first_query, flags,
-                                reports_per_query, cl_global_id.x);
+                                reports_per_query, availability_shift,
+                                cl_global_id.x);
 }
 
 /**
  * Goes through a series of consecutive query indices in the given pool,
- * setting all element values to 0 and emitting them as available.
+ * setting all element values to 0 and emitting them as available. With
+ * availability_shift 1, each availability slot is 64 bits wide.
  */
 KERNEL(32)
 libagx_reset_query(global uint32_t *availability, global uint64_t *results,
                    global uint16_t *oq_index, uint32_t first_query,
-                   uint16_t reports_per_query, int set_available)
+                   uint16_t reports_per_query, int set_available,
+                   uint16_t availability_shift)
 {
    uint32_t query = first_query + cl_global_id.x;
 
    uint64_t value = 0;
    if (availability) {
-      availability[query] = set_available;
+      uint32_t word = query << availability_shift;
+      availability[word] = set_available;
+      if (availability_shift)
+         availability[word + 1] = 0;
    } else {
       value = set_available ? 0 : LIBAGX_QUERY_UNAVAILABLE;
    }
@@ -86,8 +95,9 @@ libagx_reset_query(global uint32_t *availability, global uint64_t *results,
       query_report(results, oq_index, reports_per_query, query);
 
    /* XXX: is this supposed to happen on the begin? */
-   for (unsigned j = 0; j < reports_per_query; ++j) {
-      report[j] = value;
+   if (report) {
+      for (unsigned j = 0; j < reports_per_query; ++j)
+         report[j] = value;
    }
 }
 
diff --git a/src/asahi/libagx/query.h b/src/asahi/libagx/query.h
index 9b9774162c88..4f29bf438c4b 100644
--- a/src/asahi/libagx/query.h
+++ b/src/asahi/libagx/query.h
@@ -19,3 +19,5 @@ struct libagx_imm_write {
 };
 
 #define LIBAGX_QUERY_UNAVAILABLE (uint64_t)((int64_t)-1)
+
+#define LIBAGX_QUERY_INDEX_UNUSED ((uint16_t)0xffff)
diff --git a/src/asahi/libagx/query_copy.h b/src/asahi/libagx/query_copy.h
index 160e16d0adaa..e1d237ab5691 100644
--- a/src/asahi/libagx/query_copy.h
+++ b/src/asahi/libagx/query_copy.h
@@ -17,8 +17,23 @@
 #include "vulkan/vulkan_core.h"
 #endif
 
+/* Availability of one query: a 32-bit word, or with availability_shift 1 a
+ * 64-bit slot that the kernel may fill with the retirement timestamp of the
+ * render that ended the query. Any nonzero bit of the slot means available.
+ */
+static inline bool
+libagx_query_available(GLOBAL volatile uint32_t *availability,
+                       uint32_t availability_shift, uint32_t query)
+{
+   uint32_t word = query << availability_shift;
+   uint32_t value = availability[word];
+   if (availability_shift)
+      value |= availability[word + 1];
+   return value != 0;
+}
+
 /* Shared with the host tests. One invocation owns a complete query tuple,
- * including its single availability word. Keep loads volatile: WAIT must
+ * including its single availability slot. Keep loads volatile: WAIT must
  * reload availability, and results must be read after observing availability.
  * The caller must order the producer before launching a waiting consumer.
  */
@@ -28,7 +43,7 @@ libagx_copy_query_g17p_result(GLOBAL volatile uint32_t *availability,
                               GLOBAL uint16_t *oq_index, uint64_t dst_addr,
                               uint64_t dst_stride, uint32_t first_query,
                               uint32_t flags, uint16_t reports_per_query,
-                              uint32_t invocation)
+                              uint32_t availability_shift, uint32_t invocation)
 {
    uint32_t query = first_query + invocation;
    uint32_t index = oq_index ? oq_index[query] : query;
@@ -38,8 +53,10 @@ libagx_copy_query_g17p_result(GLOBAL volatile uint32_t *availability,
 
    bool available;
    do {
-      available =
-         availability ? availability[query] != 0 : report[0] != UINT64_MAX;
+      available = availability
+                     ? libagx_query_available(availability,
+                                              availability_shift, query)
+                     : report[0] != UINT64_MAX;
    } while (!available && (flags & VK_QUERY_RESULT_WAIT_BIT));
 
    if (available || (flags & VK_QUERY_RESULT_PARTIAL_BIT)) {
diff --git a/src/asahi/libagx/texture.cl b/src/asahi/libagx/texture.cl
index 8ccf6d521f4b..815a510480e7 100644
--- a/src/asahi/libagx/texture.cl
+++ b/src/asahi/libagx/texture.cl
@@ -244,7 +244,7 @@ libagx_image_texel_address_gen9(
    constant const struct agx_pbe_gen9_packed *ptr,
    constant const uint64_t *sw_ptr, uint4 coord, uint sample_idx,
    uint bytes_per_sample_B, bool is_1d, bool is_msaa, bool is_layered,
-   bool return_index)
+   bool return_index, bool gen10)
 {
    /* Decode only the native fields used for address arithmetic.  Calling the
     * generated unpacker here would also inline its diagnostic mask checks;
@@ -254,7 +254,8 @@ libagx_image_texel_address_gen9(
       (uint64_t)ptr->opaque[0] | ((uint64_t)ptr->opaque[1] << 32);
    const uint64_t native23 =
       (uint64_t)ptr->opaque[2] | ((uint64_t)ptr->opaque[3] << 32);
-   const uint width_px_lod0 = (uint)((native01 >> 24) & 0x3fffu) + 1;
+   const uint width_mask = gen10 ? 0x7fffu : 0x3fffu;
+   const uint width_px_lod0 = (uint)((native01 >> 24) & width_mask) + 1;
    /* A linear PBE shortens the address to 40 bits and stores its row pitch
     * minus 16 bytes at bit 40, see agx_pbe_agx3_2d_pack(). */
    const bool linear = ((native01 >> 4) & 0x3u) == AGX_LAYOUT_LINEAR;
@@ -435,6 +436,84 @@ libagx_texture_samples_gen9(constant struct agx_texture_gen9_packed *ptr)
    return 2u << d.samples;
 }
 
+/* Gen10 widens the dimensions and moves the sample field. Decode the
+ * matching descriptor while retaining the shared view/LOD query semantics. */
+uint3
+libagx_txs_gen10(constant struct agx_texture_gen10_packed *ptr, uint16_t lod,
+           unsigned nr_comps, bool is_1d, bool is_2d,
+           bool is_cube, bool is_array)
+{
+   agx_unpack(NULL, ptr, TEXTURE_GEN10, d);
+
+   /* From the Vulkan spec:
+    *
+    *    OpImageQuery*...  return 0 if the bound descriptor is a null descriptor
+    */
+   if (d.address == AGX_ZERO_PAGE_ADDRESS)
+      return 0;
+
+   /* Load standard dimensions */
+   uint3 size = (uint3)(d.width, d.height, d.depth);
+   lod += d.first_level;
+
+   /* Linear 2D arrays are special.
+    *
+    * TODO: Optimize this, since linear 2D arrays aren't needed for APIs and
+    * this just gets used internally for blits.
+    */
+   if (is_2d && is_array && d.layout == AGX_LAYOUT_LINEAR)
+      size.z = d.depth_linear;
+
+   /* 1D Arrays have their second component as the layer count */
+   if (is_1d && is_array)
+      size.y = size.z;
+
+   /* Adjust for LOD, do not adjust array size */
+   size.x = max(size.x >> lod, 1u);
+
+   if (nr_comps - (uint)is_array >= 2)
+      size.y = max(size.y >> lod, 1u);
+
+   if (nr_comps - (uint)is_array >= 3)
+      size.z = max(size.z >> lod, 1u);
+
+   /* Cube maps have equal width and height, we save some instructions by only
+    * reading one. Dead code elimination will remove the redundant instructions.
+    */
+   if (is_cube)
+      size.y = size.x;
+
+   return size;
+}
+
+uint
+libagx_texture_samples_gen10(constant struct agx_texture_gen10_packed *ptr)
+{
+   agx_unpack(NULL, ptr, TEXTURE_GEN10, d);
+
+   /* As above */
+   if (d.address == AGX_ZERO_PAGE_ADDRESS)
+      return 0;
+
+   /* Single-sampled views use an ordinary 2D dimension; the sample field
+    * encodes 2/4/8 only for multisampled dimensions. */
+   const bool multisampled =
+      d.dimension == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED ||
+      d.dimension == AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED;
+   return multisampled ? (2u << d.samples) : 1u;
+}
+
+uint
+libagx_texture_levels_gen10(constant struct agx_texture_gen10_packed *ptr)
+{
+   agx_unpack(NULL, ptr, TEXTURE_GEN10, d);
+
+   if (d.address == AGX_ZERO_PAGE_ADDRESS)
+      return 0;
+   else
+      return (d.last_level - d.first_level) + 1;
+}
+
 /* robustImageAccess2 reads outside a view return (0, 0, 0, 1) for a format
  * without alpha and zero otherwise. A view's descriptor records a missing alpha
  * as a constant one alpha swizzle; the null texture swizzles every channel to
diff --git a/src/asahi/vulkan/00-hk-defaults.conf b/src/asahi/vulkan/00-hk-defaults.conf
index e773712d5f10..d37f0891d082 100644
--- a/src/asahi/vulkan/00-hk-defaults.conf
+++ b/src/asahi/vulkan/00-hk-defaults.conf
@@ -32,5 +32,17 @@
                -->
             <option name="hk_image_view_min_lod" value="true" />
         </engine>
+
+        <application name="Counter-Strike 2" application_name_match="csgo">
+            <!--
+               Its multisample render targets and many textures are mutable
+               without a format list, but every view only changes sRGB-ness.
+               Compressing them is +5% at 4x MSAA. Compressed data does not
+               decode through views of other channel layouts, so the
+               assumption is per application; hk_CreateImageView() reports a
+               violation.
+               -->
+            <option name="hk_mutable_format_srgb_pair" value="true" />
+        </application>
     </device>
 </driconf>
diff --git a/src/asahi/vulkan/bvh/agx3_batch_build_leaf.comp b/src/asahi/vulkan/bvh/agx3_batch_build_leaf.comp
new file mode 100644
index 000000000000..44487363d1ae
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_batch_build_leaf.comp
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+#version 460
+#include "vk_bvh_defines.h"
+#define VK_USED_BUILD_FLAGS VK_LEAF_BUILD_FLAGS
+#define VK_BATCH_LEAVES 1
+uvec3 batch_global_id;
+#define gl_GlobalInvocationID batch_global_id
+#define main build_leaf_body
+#include "leaf.h"
+#undef main
+#undef gl_GlobalInvocationID
+
+/* One subgroup handles a chunk of one geometry. Retain the original geometry
+ * base and primitive IDs; only argument transport and launch grouping change. */
+struct hk_leaf_task {
+   leaf_args leaf;
+   uint first;
+   uint count;
+};
+TYPE(hk_leaf_task, 8);
+layout(push_constant) uniform BATCH_CONSTS {
+   REF(hk_leaf_task) tasks;
+};
+void main()
+{
+   hk_leaf_task task = DEREF(INDEX(hk_leaf_task, tasks, gl_WorkGroupID.x));
+   args = task.leaf;
+   batch_global_id = uvec3(task.first + gl_LocalInvocationID.x, 0, 0);
+   if (gl_LocalInvocationID.x < task.count)
+      build_leaf_body();
+}
diff --git a/src/asahi/vulkan/bvh/agx3_envelope.comp b/src/asahi/vulkan/bvh/agx3_envelope.comp
index 8c071c9270d4..43e1718832d5 100644
--- a/src/asahi/vulkan/bvh/agx3_envelope.comp
+++ b/src/asahi/vulkan/bvh/agx3_envelope.comp
@@ -103,15 +103,16 @@ publish_metadata(hk_agx3_envelope_job job, uint64_t image_bytes)
    live.retained_offset = 0ul;
    live.retained_bytes = 0ul;
    if (args.keep_retained != 0u) {
-      /* In-place UPDATE did not re-save IR at the build-bound offset. */
+      /* An in-place update keeps the retained jobs where they are. */
       libagx_gen1_as_metadata previous =
          DEREF(REF(libagx_gen1_as_metadata)(args.metadata));
       live.retained_offset = previous.retained_offset;
       live.retained_bytes = previous.retained_bytes;
    } else if ((args.build_flags & 1u) != 0u) { /* Vulkan ALLOW_UPDATE */
+      /* The retained jobs (refit save, or an update's copy of them). */
       live.retained_offset = args.retained_offset;
-      live.retained_bytes = uint64_t(SIZEOF(vk_ir_header)) +
-         uint64_t(DEREF(args.ir_header).ir_internal_node_count) * uint64_t(SIZEOF(vk_ir_box_node));
+      live.retained_bytes =
+         uint64_t(HK_AGX3_RETAINED_BYTES(job.node_count, job.leaf_bound));
    }
    live.storage_bytes = max(image_bytes, live.retained_offset + live.retained_bytes);
    live.build_flags = args.build_flags;
@@ -124,7 +125,7 @@ publish_metadata(hk_agx3_envelope_job job, uint64_t image_bytes)
 }
 
 void
-clear_instance_pointers(uint job_index, uint lane)
+clear_instance_pointers(uint job_index, uint first, uint stride)
 {
    hk_agx3_envelope_job job = DEREF(INDEX(hk_agx3_envelope_job, args.jobs, job_index));
    if (job.geometry_type != VK_GEOMETRY_TYPE_INSTANCES_KHR ||
@@ -135,7 +136,7 @@ clear_instance_pointers(uint job_index, uint lane)
    uint64_t size = pointers + uint64_t(job.leaf_bound) * 16ul + 184ul;
    if (job.output_bvh >= (1ul << 48) || size > (1ul << 48) - job.output_bvh)
       return;
-   for (uint i = lane; i < job.leaf_bound; i += 64u) {
+   for (uint i = first; i < job.leaf_bound; i += stride) {
       store64(job.output_bvh + pointers + uint64_t(i)*16ul, 0ul);
       store64(job.output_bvh + pointers + uint64_t(i)*16ul + 8ul, 0ul);
    }
@@ -144,8 +145,14 @@ clear_instance_pointers(uint job_index, uint lane)
 void
 main()
 {
-   for (uint i = gl_WorkGroupID.x; i < args.job_count; i += gl_NumWorkGroups.x)
-      clear_instance_pointers(i, gl_LocalInvocationID.x);
+   /* A single job's pointers are cleared by the whole grid, which a TLAS
+    * build sizes to its instances; several jobs take a workgroup each. */
+   if (args.refit == 0u && args.job_count == 1u)
+      clear_instance_pointers(0u, gl_GlobalInvocationID.x,
+                              gl_NumWorkGroups.x * gl_WorkGroupSize.x);
+   for (uint i = gl_WorkGroupID.x; i < args.job_count && args.job_count > 1u &&
+        args.refit == 0u; i += gl_NumWorkGroups.x)
+      clear_instance_pointers(i, gl_LocalInvocationID.x, gl_WorkGroupSize.x);
    uint id = gl_GlobalInvocationID.x;
    if (id >= args.job_count)
       return;
@@ -157,9 +164,12 @@ main()
          DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_RANGE;
          return;
       }
-      for (uint i = 0; i < 28; ++i)
-         store32(job.output_bvh + 64ul + uint64_t(i) * 4ul,
-                 i >= 4 && i < 10 ? 0x7fc00000u : 0u);
+      for (uint i = 0; i < 48; ++i)
+         store32(job.output_bvh + uint64_t(i) * 4ul,
+                 i >= 20 && i < 26 ? 0x7fc00000u : 0u);
+      store64(job.output_bvh + 144ul, job.output_bvh);
+      store32(job.output_bvh + 176ul,
+              job.resource_id != 0 ? job.resource_id : 1);
       publish_metadata(job, 192ul);
       DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_SUCCESS;
       return;
diff --git a/src/asahi/vulkan/bvh/agx3_leaf.comp b/src/asahi/vulkan/bvh/agx3_leaf.comp
index 773ad2c7f2fd..c5b97e991a5c 100644
--- a/src/asahi/vulkan/bvh/agx3_leaf.comp
+++ b/src/asahi/vulkan/bvh/agx3_leaf.comp
@@ -2,13 +2,25 @@
  * SPDX-License-Identifier: MIT
  *
  * Native M5 Gen3 nodes, derived from the saved 2/4/8/64-triangle AGX
- * captures and agx_rt_blas_gen3.h. One workgroup encodes one independent
- * node: one lane per triangle, two lanes for QTBs, lane zero for the root.
- * Inner nodes read child bounds from the same common build IR.
- * No CPU readback, cross-workgroup synchronization, or global allocator.
+ * captures and agx_rt_blas_gen3.h. Eight lanes encode one independent node,
+ * and a workgroup, one 32-lane subgroup, encodes HK_AGX3_NODE_JOBS_PER_GROUP
+ * of them: one lane per child for its bounds, its root word, its record and
+ * its QTB vertex bytes. The eight lanes of a node share its bounds, each QTB
+ * block's bounds and their checks through clustered subgroup operations, so
+ * encoding needs no shared memory or barriers. Inner nodes read child bounds
+ * from the same common build IR. No CPU readback, cross-workgroup
+ * synchronization, or global allocator.
+ *
+ * Built with HK_AGX3_REFIT for updates: jobs come from the retained jobs
+ * (see hk_agx3_retained_job). Each leaf job encodes as above and leaves its
+ * bounds for its parent, or NaN bounds when it fails, so that its
+ * ancestors fail too; agx3_refit.comp then encodes the inner jobs. After a
+ * triangle update's leaf pass (agx3_refit_leaf.comp), leaf jobs read their
+ * children at their slots instead of from IR leaves.
  */
 #version 460
 #extension GL_GOOGLE_include_directive : require
+#extension GL_KHR_memory_scope_semantics : require
 #extension GL_EXT_buffer_reference : require
 #extension GL_EXT_buffer_reference2 : require
 #extension GL_EXT_scalar_block_layout : require
@@ -19,73 +31,42 @@
 #extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
 
 #include "hk_build_interface.h"
+#include "agx3_node.h"
+
+/* Variants for one geometry kind: their jobs are inner nodes or that
+ * kind's leaves only, so the other kinds' encoders, and their registers,
+ * drop out. A job of another kind fails as an invalid job. */
+#ifdef HK_AGX3_REFIT_TRIANGLES
+#define HK_AGX3_REFIT 1
+#define HK_AGX3_LEAF_ONLY HK_AGX3_NODE_LEAF
+#endif
+/* Only triangle structures pair triangles. */
+#if defined(HK_AGX3_LEAF_PAIRS) && !defined(HK_AGX3_LEAF_ONLY)
+#define HK_AGX3_LEAF_ONLY HK_AGX3_NODE_LEAF
+#endif
 
-layout(local_size_x = 8) in;
+#define JOBS HK_AGX3_NODE_JOBS_PER_GROUP
+/* Built twice: HK_AGX3_LEAF_PAIRS encodes jobs whose children may be
+ * triangle pairs (see hk_agx3_plan_args.pairs) and keeps four vertices per
+ * child; the other build serves every other job with three. */
+#ifdef HK_AGX3_LEAF_PAIRS
+#define VERTICES 4u
+#else
+#define VERTICES 3u
+#endif
+layout(local_size_x = 8 * JOBS) in;
 layout(push_constant) uniform CONSTS
 {
+#ifdef HK_AGX3_REFIT
+   hk_agx3_refit_encode_args args;
+#else
    hk_agx3_leaf_args args;
+#endif
 };
 
-shared vec3 vertices[24];
-shared vec3 lower[8], upper[8];
-shared uint primitive[8];
-shared uint geometry_id[8];
-shared uint triangle_type[8];
-shared uint bad;
-shared uint64_t root[8];
-
-bool
-finite(float x)
-{
-   return !isnan(x) && !isinf(x);
-}
-float
-origin_down(float x)
-{
-   uint bits = floatBitsToUint(x);
-   uint upper_bits = bits >> 16;
-   if ((bits & 0x80000000u) != 0 && (bits & 65535u) != 0)
-      ++upper_bits;
-   return uintBitsToFloat(upper_bits << 16);
-}
-
-float
-step(uint exponent)
-{
-   return uintBitsToFloat(exponent << 23);
-}
-
 bool
-fit_root(vec3 lo, vec3 hi, out vec3 base, out uvec3 exponent)
+fit_qtb(vec3 lo, vec3 hi, out vec3 base, out uvec3 exponent)
 {
-   for (uint a = 0; a < 3; ++a) {
-      base[a] = origin_down(lo[a]);
-      precise float extent = hi[a] - base[a];
-      if (!finite(base[a]) || !finite(extent))
-         return false;
-      precise float unit = extent / 127.0;
-      uint bits = floatBitsToUint(unit);
-      uint e = (bits >> 23) & 255u;
-      if (e != 0 && (bits & 0x7fffffu) != 0)
-         ++e;
-      e = max(e, 1u);
-      while (e < 255u && ceil(extent / step(e)) > 127.0)
-         ++e;
-      if (e == 255u)
-         return false;
-      exponent[a] = e;
-   }
-   return true;
-}
-
-bool
-fit_qtb(uint first, uint count, out vec3 base, out uvec3 exponent)
-{
-   vec3 lo = lower[first], hi = upper[first];
-   for (uint t = first + 1; t < first + count; ++t) {
-      lo = min(lo, lower[t]);
-      hi = max(hi, upper[t]);
-   }
    for (uint a = 0; a < 3; ++a) {
       base[a] = origin_down(lo[a]);
       precise float extent = hi[a] - base[a];
@@ -97,11 +78,33 @@ fit_qtb(uint first, uint count, out vec3 base, out uvec3 exponent)
    return true;
 }
 
+/* A triangle record holds four vertices and two primitive words. The
+ * hardware tests triangle A = (v0, v1, v2) and triangle B = (v2, v1, v3),
+ * which shares A's edge v1-v2 with the opposite direction. Bits 30-31 of
+ * each primitive word (above its 30-bit primitive index) give the rotation
+ * of the application's vertex order against that stored order, which the
+ * hardware applies to the barycentrics it reports: 0 for none, 2 when the
+ * original order starts at the stored second vertex, 1 when it starts at
+ * the third. An unpaired record repeats v2 as v3, so B is never hit.
+ *
+ * Store the pair (a, b) sharing edge ea of a and edge eb of b (see
+ * hk_agx3_shared_edge): rotate a so its shared edge becomes v1-v2, then b
+ * is (v2, v1, v3) rotated to start at its own vertex eb. */
+#ifdef HK_AGX3_LEAF_PAIRS
 void
-store32(uint64_t address, uint value)
+arrange_pair(inout vec3 p[4], inout uint prim0, out uint prim1, vec3 b[3],
+             uint edges, uint b_id)
 {
-   DEREF(REF(uint32_t)(address)) = value;
+   uint ea = edges & 3u, eb = edges >> 2;
+   vec3 a[3] = vec3[3](p[0], p[1], p[2]);
+   p[0] = a[(ea + 2u) % 3u];
+   p[1] = a[ea];
+   p[2] = a[(ea + 1u) % 3u];
+   p[3] = b[(eb + 2u) % 3u];
+   prim0 |= ((ea + 2u) % 3u) << 30;
+   prim1 = b_id | (eb << 30);
 }
+#endif
 
 uint
 quantize_vertex(float value, float base, uint exponent)
@@ -162,7 +165,8 @@ store64(uint64_t address, uint64_t value)
 }
 
 void
-encode_instance(hk_agx3_leaf_job job, uint lane)
+encode_instance(hk_agx3_leaf_job job, uint lane, uint64_t transforms,
+                uint64_t pointers)
 {
    REF(vk_ir_instance_node) src = REF(vk_ir_instance_node)(
       OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
@@ -170,8 +174,6 @@ encode_instance(hk_agx3_leaf_job job, uint lane)
    uint64_t record_offset = uint64_t(job.child_slot + 2u * lane) * 64ul;
    uint64_t record = job.output_bvh + record_offset;
    uint index = DEREF(src).instance_id;
-   uint64_t transforms = DEREF(REF(uint64_t)(job.output_bvh + 104ul));
-   uint64_t pointers = DEREF(REF(uint64_t)(job.output_bvh + 128ul));
    uint64_t transform = transforms + uint64_t(index) * 64ul;
    mat3x4 otw = DEREF(src).otw_matrix;
    mat3x4 wto = invert_affine(otw);
@@ -205,140 +207,243 @@ encode_instance(hk_agx3_leaf_job job, uint lane)
 void
 main()
 {
-   uint id = gl_WorkGroupID.x, lane = gl_LocalInvocationID.x;
-   if (id >= args.job_count)
-      return;
-   hk_agx3_leaf_job job = DEREF(INDEX(hk_agx3_leaf_job, args.jobs, id));
-   uint count = job.child_count;
-   bool leaf = job.node_kind == HK_AGX3_NODE_LEAF;
-   bool instances = job.node_kind == HK_AGX3_NODE_INSTANCES;
-   bool aabbs = job.node_kind == HK_AGX3_NODE_AABBS;
-   if (count < 1u || count > 8 ||
-       job.node_kind > HK_AGX3_NODE_AABBS) {
-      if (lane == 0)
-         DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_INVALID_COUNT;
-      return;
+   uint slot = gl_LocalInvocationID.x / 8u, lane = gl_LocalInvocationID.x % 8u;
+   uint id = gl_WorkGroupID.x * JOBS + slot;
+   /* Every lane reaches every subgroup operation; a job that has no node to
+    * encode, or fails a check, only stops doing work. */
+#ifdef HK_AGX3_REFIT
+   hk_agx3_refit_state refit = DEREF(REF(hk_agx3_refit_state)(args.refit));
+   uint jobs = refit.job_count;
+   uint64_t records = refit.retained + uint64_t(HK_AGX3_RETAINED_STATE_BYTES);
+   bool live = id < jobs;
+   hk_agx3_leaf_job job;
+   hk_agx3_retained_job record;
+   uint count = 0;
+   if (live) {
+      /* Inner jobs are encoded from their children's bounds. */
+      record = DEREF(INDEX(hk_agx3_retained_job, records, id));
+      job.node_kind = (record.kind_count >> 4) & 15u;
+      live = job.node_kind != HK_AGX3_NODE_INNER;
    }
-   uint capacity = min(count, 4u), blocks = (count + capacity - 1u) / capacity;
-   if (lane == 0)
-      bad = 0;
-   barrier();
-   if (lane < count) {
+   if (live) {
+      uint64_t offsets = records + uint64_t(jobs) * uint64_t(HK_AGX3_RETAINED_JOB_BYTES);
+      job.intermediate_bvh = args.intermediate_bvh;
+      job.output_bvh = args.output_bvh;
+      job.root_slot = record.root_slot;
+      job.child_slot = record.child_slot;
+      job.child_count = record.kind_count & 15u;
+      /* Triangles at their slots need no child offsets. */
+      for (uint c = 0; c < 8u; ++c)
+         job.triangle_offsets[c] = c < job.child_count && args.triangles == 0ul
+            ? DEREF(INDEX(uint32_t, offsets, record.first + c)) : 0u;
+      count = job.child_count;
+#else
+   uint jobs = args.job_count;
+   if (args.counts != 0ul)
+      jobs = min(jobs, DEREF(REF(uint32_t)(args.counts)));
+   bool live = id < jobs;
+   hk_agx3_leaf_job job;
+   uint count = 0;
+   if (live) {
+      job = DEREF(INDEX(hk_agx3_leaf_job, args.jobs, id));
+      count = job.child_count;
+#endif
+#ifdef HK_AGX3_LEAF_ONLY
+      const bool kind_ok = job.node_kind == HK_AGX3_NODE_INNER ||
+                           job.node_kind == HK_AGX3_LEAF_ONLY;
+#else
+      const bool kind_ok = job.node_kind <= HK_AGX3_NODE_AABBS;
+#endif
+      if (count < 1u || count > 8 || !kind_ok) {
+         if (lane == 0)
+            DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_INVALID_COUNT;
+         live = false;
+      }
+   }
+   const bool valid = live;
+#ifdef HK_AGX3_LEAF_ONLY
+   const bool leaf = HK_AGX3_LEAF_ONLY == HK_AGX3_NODE_LEAF &&
+                     live && job.node_kind == HK_AGX3_NODE_LEAF;
+   const bool instances = HK_AGX3_LEAF_ONLY == HK_AGX3_NODE_INSTANCES &&
+                          live && job.node_kind == HK_AGX3_NODE_INSTANCES;
+   const bool aabbs = HK_AGX3_LEAF_ONLY == HK_AGX3_NODE_AABBS &&
+                      live && job.node_kind == HK_AGX3_NODE_AABBS;
+#else
+   bool leaf = live && job.node_kind == HK_AGX3_NODE_LEAF;
+   bool instances = live && job.node_kind == HK_AGX3_NODE_INSTANCES;
+   bool aabbs = live && job.node_kind == HK_AGX3_NODE_AABBS;
+#endif
+   uint capacity = max(min(count, 4u), 1u), blocks = (count + capacity - 1u) / capacity;
+   /* This lane's child: its bounds, record vertices and words. Lanes past
+    * the last child keep empty bounds, which the reductions ignore. */
+   bool mine = live && lane < count;
+   vec3 my_lo = vec3(INFINITY), my_hi = vec3(-INFINITY);
+   vec3 my_vertices[4];
+   uint my_primitive = 0, my_primitive1 = 0, my_geometry_id = 0, my_type = 0;
+   uint my_bad = 0;
+   if (mine) {
       if (!leaf) {
          REF(vk_ir_node)
          src = REF(vk_ir_node)(
             OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
          if (aabbs) {
             REF(vk_ir_aabb_node) box = REF(vk_ir_aabb_node)(src);
-            geometry_id[lane] = DEREF(box).geometry_id_and_flags;
-            primitive[lane] = DEREF(box).primitive_id;
+            my_geometry_id = DEREF(box).geometry_id_and_flags;
+            my_primitive = DEREF(box).primitive_id;
          }
-         lower[lane] = DEREF(src).aabb.min;
-         upper[lane] = DEREF(src).aabb.max;
+         my_lo = DEREF(src).aabb.min;
+         my_hi = DEREF(src).aabb.max;
          for (uint a = 0; a < 3; ++a) {
-            if (!finite(lower[lane][a]) || !finite(upper[lane][a]))
-               atomicOr(bad, HK_AGX3_LEAF_NONFINITE);
-            if (lower[lane][a] > upper[lane][a])
-               atomicOr(bad, HK_AGX3_LEAF_RANGE);
+            if (!finite(my_lo[a]) || !finite(my_hi[a]))
+               my_bad |= HK_AGX3_LEAF_NONFINITE;
+            if (my_lo[a] > my_hi[a])
+               my_bad |= HK_AGX3_LEAF_RANGE;
          }
       } else {
-         REF(vk_ir_triangle_node)
-         src = REF(vk_ir_triangle_node)(
-            OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
-         uint geometry_flags = DEREF(src).geometry_id_and_flags;
-         geometry_id[lane] = geometry_flags & ~VK_GEOMETRY_OPAQUE;
+#ifndef HK_AGX3_LEAF_PAIRS
+         vec3 tri[3];
+         uint geometry_flags;
+#ifdef HK_AGX3_REFIT
+         if (args.triangles != 0ul) {
+            REF(hk_agx3_refit_triangle) src =
+               INDEX(hk_agx3_refit_triangle, args.triangles, record.first + lane);
+            geometry_flags = DEREF(src).geometry_id_and_flags;
+            my_primitive = DEREF(src).triangle_id;
+            for (uint v = 0; v < 3; ++v)
+               for (uint a = 0; a < 3; ++a)
+                  tri[v][a] = DEREF(src).coords[v][a];
+         } else
+#endif
+         {
+            REF(vk_ir_triangle_node)
+            src = REF(vk_ir_triangle_node)(
+               OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
+            geometry_flags = DEREF(src).geometry_id_and_flags;
+            my_primitive = DEREF(src).triangle_id;
+            for (uint v = 0; v < 3; ++v)
+               for (uint a = 0; a < 3; ++a)
+                  tri[v][a] = DEREF(src).coords[v][a];
+         }
+         my_geometry_id = geometry_flags & ~VK_GEOMETRY_OPAQUE;
          /* Native Gen1 extractor 0x218e084c4..0x218e087f4 accepts the
           * same triangle record for types8 and16, then compares type8
           * to set the extracted geometry's opaque field. */
-         triangle_type[lane] =
-            (geometry_flags & VK_GEOMETRY_OPAQUE) != 0 ? 8u : 16u;
-         primitive[lane] = DEREF(src).triangle_id;
+         my_type = (geometry_flags & VK_GEOMETRY_OPAQUE) != 0 ? 8u : 16u;
          for (uint v = 0; v < 3; ++v) {
-            vec3 p;
-            for (uint a = 0; a < 3; ++a) {
-               p[a] = DEREF(src).coords[v][a];
-               if (!finite(p[a]))
-                  atomicOr(bad, HK_AGX3_LEAF_NONFINITE);
-            }
-            vertices[lane * 3u + v] = p;
+            for (uint a = 0; a < 3; ++a)
+               if (!finite(tri[v][a]))
+                  my_bad |= HK_AGX3_LEAF_NONFINITE;
+            my_vertices[v] = tri[v];
+         }
+         my_vertices[3] = my_vertices[2];
+#else
+         /* A pair names the internal node over its two triangles. */
+         uint offset = job.triangle_offsets[lane];
+         bool paired = (offset & HK_AGX3_LEAF_PAIR_OFFSET) != 0u;
+         offset &= ~3u;
+         REF(vk_ir_triangle_node)
+         src = REF(vk_ir_triangle_node)(OFFSET(job.intermediate_bvh, offset));
+         REF(vk_ir_triangle_node) second = src;
+         if (paired) {
+            REF(vk_ir_box_node) box =
+               REF(vk_ir_box_node)(OFFSET(job.intermediate_bvh, offset));
+            src = REF(vk_ir_triangle_node)(OFFSET(job.intermediate_bvh,
+               ir_id_to_offset(DEREF(box).children[0])));
+            second = REF(vk_ir_triangle_node)(OFFSET(job.intermediate_bvh,
+               ir_id_to_offset(DEREF(box).children[1])));
+         }
+         uint geometry_flags = DEREF(src).geometry_id_and_flags;
+         my_geometry_id = geometry_flags & ~VK_GEOMETRY_OPAQUE;
+         my_type = (geometry_flags & VK_GEOMETRY_OPAQUE) != 0 ? 8u : 16u;
+         vec3 p[4];
+         for (uint v = 0; v < 3; ++v)
+            for (uint a = 0; a < 3; ++a)
+               p[v][a] = DEREF(src).coords[v][a];
+         /* An unpaired record repeats its last vertex. */
+         p[3] = p[2];
+         my_primitive = DEREF(src).triangle_id;
+         my_primitive1 = my_primitive;
+         if (paired) {
+            vec3 a[3] = vec3[3](p[0], p[1], p[2]), b[3];
+            for (uint v = 0; v < 3; ++v)
+               for (uint k = 0; k < 3; ++k)
+                  b[v][k] = DEREF(second).coords[v][k];
+            uint edges = hk_agx3_shared_edge(a, b);
+            if (edges == ~0u)
+               my_bad |= HK_AGX3_LEAF_RANGE;
+            else
+               arrange_pair(p, my_primitive, my_primitive1, b, edges,
+                            DEREF(second).triangle_id);
+         }
+         for (uint v = 0; v < 4; ++v) {
+            for (uint a = 0; a < 3; ++a)
+               if (!finite(p[v][a]))
+                  my_bad |= HK_AGX3_LEAF_NONFINITE;
+            my_vertices[v] = p[v];
          }
-         lower[lane] = min(vertices[lane * 3u], min(vertices[lane * 3u + 1u],
-                                                    vertices[lane * 3u + 2u]));
-         upper[lane] = max(vertices[lane * 3u], max(vertices[lane * 3u + 1u],
-                                                    vertices[lane * 3u + 2u]));
+#endif
+         my_lo = min(min(my_vertices[0], my_vertices[1]),
+                     min(my_vertices[2], my_vertices[3]));
+         my_hi = max(max(my_vertices[0], my_vertices[1]),
+                     max(my_vertices[2], my_vertices[3]));
       }
    }
-   barrier();
-   if (bad != 0) {
-      if (lane == 0)
-         DEREF(INDEX(uint32_t, args.status, id)) = bad;
-      return;
-   }
 
-   vec3 qbase = vec3(0);
-   uvec3 qexp = uvec3(0);
-   if (leaf && lane < blocks &&
-       !fit_qtb(lane * capacity, min(capacity, count - lane * capacity), qbase,
-                qexp))
-      atomicOr(bad, HK_AGX3_LEAF_RANGE);
-   if (lane == 0) {
-      vec3 lo = lower[0], hi = upper[0], base;
-      uvec3 exponent;
-      for (uint t = 1; t < count; ++t) {
-         lo = min(lo, lower[t]);
-         hi = max(hi, upper[t]);
-      }
-      if (!fit_root(lo, hi, base, exponent)) {
-         atomicOr(bad, HK_AGX3_LEAF_RANGE);
-      } else {
-         for (uint t = 0; t < 8; ++t)
-            root[t] = uint64_t(0);
-         root[0] =
-            uint64_t((instances ? 0xe1u : 0xa0u) |
-                     (leaf ? capacity << 1 : 0u) | (exponent.x << 8));
-         for (uint a = 0; a < 3; ++a)
-            root[a + 1] = uint64_t(floatBitsToUint(base[a]) >> 16);
-         root[4] = uint64_t((exponent.y << 8) | exponent.z);
-         uint child = job.child_slot + args.slot_bias;
-         root[5] = uint64_t(child >> 16);
-         root[6] = uint64_t(child & 65535u);
-         root[7] = instances ? uint64_t(0x1a00u | (count == 8u ? 0x3fu : count << 3))
-                   : aabbs ? uint64_t(0x18c0u | (count == 8u ? 0x3fu : count << 3))
-                   : leaf ? uint64_t(0x40c0u | (triangle_type[0] << 8) |
-                                             (count == 8u ? 0x3fu : count << 3))
-                          : uint64_t(0x1f00u | (count == 8u ? 0u : count));
-         const uint shifts[6] = uint[6](57, 50, 43, 36, 29, 22);
-         for (uint t = 0; t < count; ++t) {
-            if (leaf || instances || aabbs)
-               root[t] |= uint64_t((t << 16) | (1u << 20));
+   /* Reductions over the node's eight lanes, and over each QTB block's four
+    * (children 0-3 and 4-7; a node of four or fewer has one block). Every
+    * lane takes part, in uniform control flow. */
+   uint loaded_bad = subgroupClusteredOr(my_bad, 8u);
+   vec3 node_lo = subgroupClusteredMin(my_lo, 8u);
+   vec3 node_hi = subgroupClusteredMax(my_hi, 8u);
+   vec3 block_lo = subgroupClusteredMin(my_lo, 4u);
+   vec3 block_hi = subgroupClusteredMax(my_hi, 4u);
+   uint type0 = subgroupShuffle(my_type, gl_SubgroupInvocationID & ~7u);
+
+   /* Root word of child lane: its header bits, then its quantized bounds;
+    * and its triangle's QTB vertex bytes, each in bits of its own. */
+   vec3 base, qbase;
+   uvec3 exponent, qexp;
+   uint64_t my_root = 0ul;
+   uint packed[3] = uint[3](0u, 0u, 0u);
+   if (live && loaded_bad == 0u) {
+      bool ok = fit_root(node_lo, node_hi, base, exponent);
+      if (ok)
+         my_root = root_word(lane, count, job.node_kind, type0, job.child_slot,
+                             args.slot_bias, base, exponent, my_lo, my_hi, ok);
+      if (ok && leaf && lane < count) {
+         ok = fit_qtb(block_lo, block_hi, qbase, qexp);
+         /* The QTB holds the record's vertices in reverse order; an
+          * unpaired record repeats v2 as v3. */
+#ifdef HK_AGX3_LEAF_PAIRS
+         const uint order[4] = uint[4](3, 2, 1, 0);
+#else
+         const uint order[4] = uint[4](2, 2, 1, 0);
+#endif
+         /* A triangle's twelve vertex bytes are three whole words of its
+          * block. */
+         for (uint v = 0; v < 4 && ok; ++v)
             for (uint a = 0; a < 3; ++a) {
-               float s = step(exponent[a]);
-               uint qlo =
-                  uint(clamp(floor((lower[t][a] - base[a]) / s), 0.0, 127.0));
-               uint qhi =
-                  uint(clamp(ceil((upper[t][a] - base[a]) / s), 0.0, 127.0));
-               precise float rlo = base[a] + float(qlo) * s;
-               precise float rhi = base[a] + float(qhi) * s;
-               while (rlo > lower[t][a] && qlo > 0u) {
-                  --qlo;
-                  rlo = base[a] + float(qlo) * s;
-               }
-               while (rhi < upper[t][a] && qhi < 127u) {
-                  ++qhi;
-                  rhi = base[a] + float(qhi) * s;
-               }
-               if (rlo > lower[t][a] || rhi < upper[t][a])
-                  atomicOr(bad, HK_AGX3_LEAF_RANGE);
-               root[t] |= uint64_t(qlo) << shifts[a];
-               root[t] |= uint64_t(qhi) << shifts[a + 3];
+               /* Power-of-two scaling avoids reciprocal overflow on small
+                * axes, including the minimum normal binary32 exponent. */
+               uint q = quantize_vertex(my_vertices[order[v]][a], qbase[a], qexp[a]);
+               uint byte_offset = v * 3u + 2u - a;
+               packed[byte_offset / 4u] |= q << ((byte_offset % 4u) * 8u);
             }
-         }
       }
+      if (!ok)
+         my_bad |= HK_AGX3_LEAF_RANGE;
    }
-   barrier();
-   if (bad != 0) {
-      if (lane == 0)
-         DEREF(INDEX(uint32_t, args.status, id)) = bad;
+   uint node_bad = subgroupClusteredOr(my_bad, 8u);
+   if (!valid)
+      return;
+   if (node_bad != 0u) {
+      if (lane == 0) {
+         DEREF(INDEX(uint32_t, args.status, id)) = node_bad;
+#ifdef HK_AGX3_REFIT
+         store_aabb(uint64_t(INDEX(vk_aabb, args.bounds, record.position)),
+                    HK_AGX3_FAILED_BOUNDS, HK_AGX3_FAILED_BOUNDS);
+#endif
+      }
       return;
    }
 
@@ -349,60 +454,67 @@ main()
        * payload starts at+4: xyz min/max then geometry/primitive words.
        * Bit31 of the geometry word is opacity; each root child uses64B. */
       uint64_t geometry = children + uint64_t(lane) * 64ul;
-      for (uint i = 0; i < 16; ++i)
-         store32(geometry + uint64_t(i) * 4ul, 0);
-      store32(geometry, 24);
-      for (uint a = 0; a < 3; ++a) {
-         store32(geometry + 4ul + uint64_t(a) * 4ul, floatBitsToUint(lower[lane][a]));
-         store32(geometry + 16ul + uint64_t(a) * 4ul, floatBitsToUint(upper[lane][a]));
-      }
-      store32(geometry + 28ul, geometry_id[lane]);
-      store32(geometry + 32ul, primitive[lane]);
+      uvec3 lo_bits = floatBitsToUint(my_lo), hi_bits = floatBitsToUint(my_hi);
+      store128(geometry, uvec4(24u, lo_bits));
+      store128(geometry + 16ul, uvec4(hi_bits, my_geometry_id));
+      store128(geometry + 32ul, uvec4(my_primitive, 0u, 0u, 0u));
+      store128(geometry + 48ul, uvec4(0u));
    }
+#ifdef HK_AGX3_REFIT
+   if (instances && lane < count) {
+      /* As the envelope lays them out after the nodes. */
+      uint64_t transforms = uint64_t(refit.end_slot) * 64ul;
+      encode_instance(job, lane, transforms,
+                      transforms + 64ul * uint64_t(args.leaf_bound));
+   }
+#else
    if (instances && lane < count)
-      encode_instance(job, lane);
+      encode_instance(job, lane, DEREF(REF(uint64_t)(job.output_bvh + 104ul)),
+                      DEREF(REF(uint64_t)(job.output_bvh + 128ul)));
+#endif
    if (leaf && lane < count) {
+      /* The record: its header, the four x, y and z coordinates, then the
+       * geometry and the two primitive words, as four 16-byte stores. */
       uint64_t geometry = children + uint64_t(blocks + lane) * 64ul;
-      store32(geometry, 0x40u | triangle_type[lane]);
-      for (uint a = 0; a < 3; ++a)
-         for (uint v = 0; v < 4; ++v)
-            store32(geometry + uint64_t(4u + a * 16u + v * 4u),
-                    floatBitsToUint(vertices[lane * 3u + min(v, 2u)][a]));
-      store32(geometry + 52ul, geometry_id[lane]);
-      store32(geometry + 56ul, primitive[lane]);
-      store32(geometry + 60ul, primitive[lane]);
-   }
-   if (leaf && lane < blocks) {
-      /* Each QTB writer owns all sixteen words, avoiding byte-store races. */
-      uint words[16];
-      for (uint w = 0; w < 16; ++w)
-         words[w] = 0;
-      words[0] = 6u | (qexp.z << 8) | (floatBitsToUint(qbase.x) & 0xffff0000u);
-      words[1] = (floatBitsToUint(qbase.y) >> 16) |
-                 (floatBitsToUint(qbase.z) & 0xffff0000u);
-      words[2] = qexp.x | (qexp.y << 8);
-      uint first = lane * capacity, n = min(capacity, count - first);
-      const uint order[4] = uint[4](2, 2, 1, 0);
-      for (uint t = 0; t < n; ++t)
-         for (uint v = 0; v < 4; ++v)
-            for (uint a = 0; a < 3; ++a) {
-               /* Power-of-two scaling avoids reciprocal overflow on small
-                * axes, including the minimum normal binary32 exponent. */
-               uint q = quantize_vertex(
-                  vertices[(first + t) * 3u + order[v]][a], qbase[a], qexp[a]);
-               uint byte_offset = 16u + t * 12u + v * 3u + 2u - a;
-               words[byte_offset / 4u] |= q << ((byte_offset % 4u) * 8u);
-            }
-      for (uint w = 0; w < 16; ++w)
-         store32(children + uint64_t(lane * 64u + w * 4u), words[w]);
+      uvec3 p0 = floatBitsToUint(my_vertices[0]);
+      uvec3 p1 = floatBitsToUint(my_vertices[1]);
+      uvec3 p2 = floatBitsToUint(my_vertices[2]);
+      uvec3 p3 = floatBitsToUint(my_vertices[VERTICES - 1u]);
+#ifdef HK_AGX3_LEAF_PAIRS
+      uint second = my_primitive1;
+#else
+      uint second = my_primitive;
+#endif
+      store128(geometry, uvec4(0x40u | my_type, p0.x, p1.x, p2.x));
+      store128(geometry + 16ul, uvec4(p3.x, p0.y, p1.y, p2.y));
+      store128(geometry + 32ul, uvec4(p3.y, p0.z, p1.z, p2.z));
+      store128(geometry + 48ul, uvec4(p3.z, my_geometry_id, my_primitive, second));
+
+      /* QTB block: the first lane of a block writes its header and clears
+       * the slots of triangles it does not have; each lane writes its own
+       * triangle's three words. */
+      uint block = lane / capacity, in_block = lane % capacity;
+      uint64_t qtb = children + uint64_t(block) * 64ul;
+      if (in_block == 0u) {
+         store128(qtb, uvec4(6u | (qexp.z << 8) | (floatBitsToUint(qbase.x) & 0xffff0000u),
+                             (floatBitsToUint(qbase.y) >> 16) |
+                                (floatBitsToUint(qbase.z) & 0xffff0000u),
+                             qexp.x | (qexp.y << 8), 0u));
+         uint present = min(capacity, count - block * capacity);
+         for (uint w = 4u + present * 3u; w < 16u; ++w)
+            store32(qtb + uint64_t(w) * 4ul, 0u);
+      }
+      for (uint w = 0; w < 3u; ++w)
+         store32(qtb + 16ul + uint64_t(in_block * 12u + w * 4u), packed[w]);
    }
+   /* Every lane stores its own root word, filling one 64-byte line. */
+   store_word64(uint64_t(job.output_bvh) + uint64_t(job.root_slot) * 64ul +
+                   uint64_t(lane) * 8ul, my_root);
    if (lane == 0) {
-      for (uint w = 0; w < 8; ++w) {
-         uint64_t address = uint64_t(job.output_bvh) +
-                            uint64_t(job.root_slot) * 64ul + uint64_t(w) * 8ul;
-         store32(address, uint(root[w]));
-         store32(address + 4ul, uint(root[w] >> 32));
-      }
       DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_SUCCESS;
+#ifdef HK_AGX3_REFIT
+      /* Its parent's encoding reads these (agx3_refit.comp). */
+      store_aabb(uint64_t(INDEX(vk_aabb, args.bounds, record.position)), node_lo, node_hi);
+#endif
    }
 }
diff --git a/src/asahi/vulkan/bvh/agx3_m5_legacy_envelope.comp b/src/asahi/vulkan/bvh/agx3_m5_legacy_envelope.comp
new file mode 100644
index 000000000000..91761f834926
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_m5_legacy_envelope.comp
@@ -0,0 +1,288 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ * Native explicit-ID, single-geometry triangle AS header and descriptor.
+ * Matches agx_rt_blas_gen3_envelope and the native 2/4/8/64 captures.
+ */
+#version 460
+#extension GL_GOOGLE_include_directive : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference2 : require
+#extension GL_EXT_scalar_block_layout : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
+
+#include "hk_build_interface.h"
+
+#ifndef HK_AGX3_GEN1
+#define HK_AGX3_GEN1 0
+#endif
+
+layout(local_size_x = 64) in;
+#ifdef HK_AGX3_BATCH
+layout(push_constant) uniform CONSTS
+{
+   hk_agx3_batch_args batch;
+};
+hk_agx3_envelope_args args;
+#else
+layout(push_constant) uniform CONSTS
+{
+   hk_agx3_envelope_args args;
+};
+#endif
+
+void
+store32(uint64_t address, uint value)
+{
+   DEREF(REF(uint32_t)(address)) = value;
+}
+
+void
+store64(uint64_t address, uint64_t value)
+{
+   store32(address, uint(value));
+   store32(address + 4ul, uint(value >> 32));
+}
+
+/* Use the reciprocal estimate only to locate an integer quotient. Its exact
+ * 48-bit product/remainder gives round-to-nearest-even without shader f64 or
+ * a general 64-bit division. A float/FMA refinement alone was insufficient
+ * on the current M5 compiler path for some captured-envelope scale tests. */
+uint
+reciprocal_rte(float value)
+{
+   uint bits = floatBitsToUint(value);
+   if ((bits & 0x80000000u) != 0 || (bits & 0x7fffffffu) == 0 ||
+       (bits & 0x7f800000u) == 0x7f800000u)
+      return 0x7f800000u;
+   int exponent = int((bits >> 23) & 255u);
+   uint mantissa = bits & 0x7fffffu;
+   if (exponent != 0) {
+      mantissa |= 0x800000u;
+   } else {
+      int shift = 23 - findMSB(mantissa);
+      mantissa <<= shift;
+      exponent = 1 - shift;
+   }
+   int result_exponent = max(253 - exponent, 0);
+   if (result_exponent >= 255)
+      return 0x7f800000u;
+   int numerator_bit = result_exponent == 0 ? 299 - exponent : 47;
+   uint64_t numerator = 1ul << numerator_bit;
+   uint estimate_bits = floatBitsToUint(1.0 / float(mantissa));
+   int estimate_exponent = int((estimate_bits >> 23) & 255u);
+   uint estimate_mantissa = 0x800000u | (estimate_bits & 0x7fffffu);
+   int estimate_shift = 150 - numerator_bit - estimate_exponent;
+   /* An exact power-of-two mantissa has a reciprocal exponent one higher
+    * than the other mantissas. Scale left in that case: a negative right
+    * shift otherwise loses the estimate and makes correction take millions
+    * of iterations (or wrap), hanging the envelope dispatch. */
+   uint quotient = estimate_shift < 0
+                      ? estimate_mantissa << uint(-estimate_shift)
+                      : estimate_mantissa >> uint(estimate_shift);
+   uint64_t product = uint64_t(mantissa) * uint64_t(quotient);
+   while (product > numerator) {
+      --quotient;
+      product -= uint64_t(mantissa);
+   }
+   while (product + uint64_t(mantissa) <= numerator) {
+      ++quotient;
+      product += uint64_t(mantissa);
+   }
+   uint64_t twice_remainder = (numerator - product) * 2ul;
+   if (twice_remainder > uint64_t(mantissa) ||
+       (twice_remainder == uint64_t(mantissa) && (quotient & 1u) != 0))
+      ++quotient;
+   if (quotient == 0x1000000u) {
+      quotient >>= 1;
+      ++result_exponent;
+   }
+   return result_exponent == 0
+             ? quotient
+             : (uint(result_exponent) << 23) | (quotient & 0x7fffffu);
+}
+
+void
+publish_metadata(hk_agx3_envelope_job job, uint64_t image_bytes)
+{
+   if (args.metadata == 0ul)
+      return;
+   libagx_gen1_as_metadata live;
+   live.native_base = job.output_bvh;
+   live.image_bytes = image_bytes;
+   live.retained_offset = 0ul;
+   live.retained_bytes = 0ul;
+   if (args.keep_retained != 0u) {
+      /* In-place UPDATE did not re-save IR at the build-bound offset. */
+      libagx_gen1_as_metadata previous =
+         DEREF(REF(libagx_gen1_as_metadata)(args.metadata));
+      live.retained_offset = previous.retained_offset;
+      live.retained_bytes = previous.retained_bytes;
+   } else if ((args.build_flags & 1u) != 0u) { /* Vulkan ALLOW_UPDATE */
+      live.retained_offset = args.retained_offset;
+      live.retained_bytes = uint64_t(SIZEOF(vk_ir_header)) +
+         uint64_t(DEREF(args.ir_header).ir_internal_node_count) * uint64_t(SIZEOF(vk_ir_box_node));
+   }
+   live.storage_bytes = max(image_bytes, live.retained_offset + live.retained_bytes);
+   live.build_flags = args.build_flags;
+   live.leaf_bound = job.leaf_bound;
+   live.node_bound = args.node_bound;
+   live.geometry_type = job.geometry_type;
+   live.resource_id = job.resource_id;
+   live.version = LIBAGX_GEN1_AS_METADATA_VERSION;
+   DEREF(REF(libagx_gen1_as_metadata)(args.metadata)) = live;
+}
+
+void
+clear_instance_pointers(uint job_index, uint lane)
+{
+   hk_agx3_envelope_job job = DEREF(INDEX(hk_agx3_envelope_job, args.jobs, job_index));
+   if (job.geometry_type != VK_GEOMETRY_TYPE_INSTANCES_KHR ||
+       job.primitive_count == 0 || job.end_slot < 3 ||
+       job.output_bvh == 0 || (job.output_bvh & 63ul) != 0)
+      return;
+   uint64_t pointers = uint64_t(job.end_slot) * 64ul + uint64_t(job.leaf_bound) * 64ul;
+   uint64_t size = pointers + uint64_t(job.leaf_bound) * 16ul + 184ul;
+   if (job.output_bvh >= (1ul << 48) || size > (1ul << 48) - job.output_bvh)
+      return;
+   for (uint i = lane; i < job.leaf_bound; i += 64u) {
+      store64(job.output_bvh + pointers + uint64_t(i)*16ul, 0ul);
+      store64(job.output_bvh + pointers + uint64_t(i)*16ul + 8ul, 0ul);
+   }
+}
+
+void
+main()
+{
+   if (HK_AGX3_GEN1 != 0)
+   for (uint i = gl_WorkGroupID.x; i < args.job_count; i += gl_NumWorkGroups.x)
+      clear_instance_pointers(i, gl_LocalInvocationID.x);
+   uint id = gl_GlobalInvocationID.x;
+#ifdef HK_AGX3_BATCH
+   if (id >= batch.job_count)
+      return;
+   hk_agx3_batch_job entry = DEREF(INDEX(hk_agx3_batch_job, batch.jobs, id));
+   args.jobs = entry.plan.envelope;
+   args.status = entry.envelope_status;
+   args.job_count = 1;
+   args.reserved = 0;
+   args.metadata = 0ul;
+   id = 0;
+#endif
+   if (id >= args.job_count)
+      return;
+   hk_agx3_envelope_job job = DEREF(INDEX(hk_agx3_envelope_job, args.jobs, id));
+   if (job.primitive_count == 0 &&
+       job.opaque_build_word == HK_AGX3_PLAN_EMPTY) {
+      if (job.output_bvh == 0 || (job.output_bvh & (HK_AGX3_GEN1 != 0 ? 63ul : 1023ul)) != 0 ||
+          job.output_bvh >= (1ul << 48) - 192ul) {
+         DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_RANGE;
+         return;
+      }
+      for (uint i = 0; i < 28; ++i)
+         store32(job.output_bvh + 64ul + uint64_t(i) * 4ul,
+                 i >= 4 && i < 10 ? 0x7fc00000u : 0u);
+      publish_metadata(job, 192ul);
+      DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_SUCCESS;
+      return;
+   }
+   uint64_t tail = uint64_t(job.end_slot) * 64ul;
+   bool instances = job.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR;
+   bool aabbs = job.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR;
+   uint64_t transform_offset = tail;
+   uint64_t pointer_offset = tail + 64ul * uint64_t(job.leaf_bound);
+   uint64_t descriptor_offset = instances
+      ? pointer_offset + 16ul * uint64_t(job.leaf_bound) : tail;
+   uint64_t size = descriptor_offset + 184ul;
+   uint status = 0;
+   if (job.end_slot < 3 || job.primitive_count == 0 || job.node_count == 0 ||
+       job.depth == 0)
+      status |= HK_AGX3_LEAF_INVALID_COUNT;
+   if (job.output_bvh == 0 || (job.output_bvh & 63ul) != 0 ||
+       job.output_bvh >= (1ul << 48) || size > (1ul << 48) - job.output_bvh)
+      status |= HK_AGX3_LEAF_RANGE;
+   vec3 lo = DEREF(job.bounds).aabb.min, hi = DEREF(job.bounds).aabb.max;
+   precise vec3 extent = hi - lo;
+   for (uint a = 0; a < 3; ++a) {
+      if (isnan(lo[a]) || isinf(lo[a]) || isnan(hi[a]) || isinf(hi[a]))
+         status |= HK_AGX3_LEAF_NONFINITE;
+      if (lo[a] > hi[a])
+         status |= HK_AGX3_LEAF_RANGE;
+      if (HK_AGX3_GEN1 != 0 && extent[a] == 0.0) {
+         hi[a] = lo[a] + max(abs(lo[a]), 1.0) * 1.52587890625e-05;
+         extent[a] = hi[a] - lo[a];
+      }
+   }
+   precise float xy = extent.x * extent.y;
+   precise float xz = extent.x * extent.z;
+   precise float yz = extent.y * extent.z;
+   precise float area = (xy + xz) + yz;
+   uint inverse_bits = reciprocal_rte(area);
+   float inverse = uintBitsToFloat(inverse_bits);
+   if (!(area > 0.0) || isinf(area) || isinf(inverse))
+      status |= HK_AGX3_LEAF_RANGE;
+   if (status != 0) {
+      DEREF(INDEX(uint32_t, args.status, id)) = status;
+      return;
+   }
+   uint64_t h = job.output_bvh + 64ul;
+   for (uint i = 0; i < 32; ++i)
+      store32(h + uint64_t(i) * 4ul, 0);
+   store64(h, (instances ? 0x080000008000f032ul : 0x080000002000f032ul) |
+               (uint64_t(HK_AGX3_GEN1 != 0 && args.reserved == 2 ? 2 : 3) << 24));
+   store64(h + 8ul, size);
+   for (uint a = 0; a < 3; ++a) {
+      store32(h + 16ul + uint64_t(a) * 4ul, floatBitsToUint(lo[a]));
+      store32(h + 28ul + uint64_t(a) * 4ul, floatBitsToUint(hi[a]));
+   }
+   for (uint i = 40; i <= 88; i += 8) {
+      uint64_t offset = tail;
+      if (instances)
+         offset = (i == 48 || i == 56) ? descriptor_offset :
+                  i == 64 ? pointer_offset : transform_offset;
+      store64(h + uint64_t(i), i == 80 ? job.output_bvh : offset);
+   }
+   store32(h + 96ul, 1);
+   store32(h + 112ul, job.resource_id != 0 ? job.resource_id : 1);
+   store32(h + 120ul, instances ? 0x41d00000u : job.opaque_build_word);
+   store32(h + 124ul, 0x01000000);
+
+   uint descriptor[46] =
+      uint[46](job.primitive_count, job.primitive_count, 0, 0, job.node_count,
+               job.depth, 1, 0, 1, 0, 0, 0, 0, 0x3f800000, 0, 0, 2, 1, 0, 0, 2,
+               10, 0, 0, 0, 0, 0, inverse_bits, 0x01000000, 0x01000000,
+               0x00000101, 0x00010000, 0x00010000, 0x00ff0001, 0x02000000,
+               0x01000005, 0x03000000, 1, 0, 0x3f800000, 0, 0, 0, 0, 0, 0);
+   if (aabbs && HK_AGX3_GEN1 == 0) {
+      /* Descriptor word 16 selects the native geometry class: Apple's
+       * extractor maps 4 to BVHGeometryBoundingBox (8 is the instance class
+       * and 0/2/10 the triangle classes). Word 20 mirrors word 16 in both
+       * captured triangle and instance descriptors; the remaining words keep
+       * the triangle values, which no native bounding-box capture has yet
+       * confirmed or refuted. */
+      descriptor[16] = 4;
+      descriptor[20] = 4;
+   }
+   if (instances) {
+      const uint template_words[46] = uint[46](
+         0,0,0,0,0,0,0,0,1,0,0,0,0,0x3f800000,0,0,
+         8,0,0,0,8,18,0,6,1,1,1,0,0x01000000,0,0x100,0x10000,
+         0x10000,0x00ff0001,0x02000000,5,0x03000002,1,0,0x3f800000,
+         0,0,0,0,0,0);
+      for (uint i = 0; i < 46; ++i)
+         descriptor[i] = template_words[i];
+      descriptor[0] = job.primitive_count;
+      descriptor[3] = job.primitive_count;
+      descriptor[4] = job.node_count;
+      descriptor[5] = job.depth;
+      descriptor[7] = job.primitive_count;
+      descriptor[27] = inverse_bits;
+   }
+   for (uint i = 0; i < 46; ++i)
+      store32(job.output_bvh + descriptor_offset + uint64_t(i) * 4ul, descriptor[i]);
+   publish_metadata(job, size);
+   DEREF(INDEX(uint32_t, args.status, id)) = 0;
+}
diff --git a/src/asahi/vulkan/bvh/agx3_m5_legacy_leaf.comp b/src/asahi/vulkan/bvh/agx3_m5_legacy_leaf.comp
new file mode 100644
index 000000000000..1c83b93da8e8
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_m5_legacy_leaf.comp
@@ -0,0 +1,470 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * Native M5 Gen3 nodes, derived from the saved 2/4/8/64-triangle AGX
+ * captures and agx_rt_blas_gen3.h. One workgroup encodes one independent
+ * node: one lane per triangle, two lanes for QTBs, lane zero for the root.
+ * Inner nodes read child bounds from the same common build IR.
+ * No CPU readback, cross-workgroup synchronization, or global allocator.
+ *
+ * Procedural leaves use the 48-byte native bounding-box record (marker 0x18,
+ * minimum at 4, maximum at 16, geometry id/opaque bit at 28, primitive id at
+ * 32) captured from public Metal on M5 Pro and read at those offsets by the
+ * native runtime's agx.bounding_box_test. Records are packed at a 48-byte
+ * stride directly in the child region: Apple's Gen1 extractor derives the
+ * record stride from root word0 bits 5..7 (class 4 = 48 bytes) and dispatches
+ * on record marker low five bits (24 = bounding box).
+ */
+#version 460
+#extension GL_GOOGLE_include_directive : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference2 : require
+#extension GL_EXT_scalar_block_layout : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
+
+#include "hk_build_interface.h"
+
+#ifndef HK_AGX3_GEN1
+#define HK_AGX3_GEN1 0
+#endif
+
+layout(local_size_x = 8) in;
+#ifdef HK_AGX3_BATCH
+layout(push_constant) uniform CONSTS
+{
+   hk_agx3_batch_args batch;
+};
+hk_agx3_leaf_args args;
+#else
+layout(push_constant) uniform CONSTS
+{
+   hk_agx3_leaf_args args;
+};
+#endif
+
+shared vec3 vertices[24];
+shared vec3 lower[8], upper[8];
+shared uint primitive[8];
+shared uint geometry_id[8];
+shared uint triangle_type[8];
+shared uint bad;
+shared uint64_t root[8];
+
+bool
+finite(float x)
+{
+   return !isnan(x) && !isinf(x);
+}
+float
+origin_down(float x)
+{
+   uint bits = floatBitsToUint(x);
+   uint upper_bits = bits >> 16;
+   if ((bits & 0x80000000u) != 0 && (bits & 65535u) != 0)
+      ++upper_bits;
+   return uintBitsToFloat(upper_bits << 16);
+}
+
+float
+step(uint exponent)
+{
+   return uintBitsToFloat(exponent << 23);
+}
+
+bool
+fit_root(vec3 lo, vec3 hi, out vec3 base, out uvec3 exponent)
+{
+   for (uint a = 0; a < 3; ++a) {
+      base[a] = origin_down(lo[a]);
+      precise float extent = hi[a] - base[a];
+      if (!finite(base[a]) || !finite(extent))
+         return false;
+      precise float unit = extent / 127.0;
+      uint bits = floatBitsToUint(unit);
+      uint e = (bits >> 23) & 255u;
+      if (e != 0 && (bits & 0x7fffffu) != 0)
+         ++e;
+      e = max(e, 1u);
+      while (e < 255u && ceil(extent / step(e)) > 127.0)
+         ++e;
+      if (e == 255u)
+         return false;
+      exponent[a] = e;
+   }
+   return true;
+}
+
+bool
+fit_qtb(uint first, uint count, out vec3 base, out uvec3 exponent)
+{
+   vec3 lo = lower[first], hi = upper[first];
+   for (uint t = first + 1; t < first + count; ++t) {
+      lo = min(lo, lower[t]);
+      hi = max(hi, upper[t]);
+   }
+   for (uint a = 0; a < 3; ++a) {
+      base[a] = origin_down(lo[a]);
+      precise float extent = hi[a] - base[a];
+      if (!finite(base[a]) || !finite(extent))
+         return false;
+      uint e = (floatBitsToUint(extent) >> 23) & 255u;
+      exponent[a] = e > 8u ? e - 7u : 1u;
+   }
+   return true;
+}
+
+void
+store32(uint64_t address, uint value)
+{
+   DEREF(REF(uint32_t)(address)) = value;
+}
+
+uint
+quantize_vertex(float value, float base, uint exponent)
+{
+   /* A binary32 subtraction can round an exact coordinate just below a
+    * grid line onto that line. Retain its residual (TwoDiff) so truncation
+    * agrees with the reference's binary64 arithmetic without shaderFloat64.
+    * All operations must remain separate; contraction/reassociation would
+    * discard the very residual we are recovering. */
+   precise float delta = value - base;
+   precise float virtual_base = value - delta;
+   precise float virtual_value = delta + virtual_base;
+   precise float base_error = virtual_base - base;
+   precise float value_error = value - virtual_value;
+   precise float error = value_error + base_error;
+   precise float q = ldexp(delta, 127 - int(exponent));
+   float whole = floor(q);
+   if (q == whole && error < 0.0)
+      whole -= 1.0;
+   return uint(clamp(whole, 0.0, 255.0));
+}
+
+mat3x4
+invert_affine(mat3x4 otw)
+{
+   vec3 r0 = vec3(otw[0][0], otw[0][1], otw[0][2]);
+   vec3 r1 = vec3(otw[1][0], otw[1][1], otw[1][2]);
+   vec3 r2 = vec3(otw[2][0], otw[2][1], otw[2][2]);
+   vec3 t = vec3(otw[0][3], otw[1][3], otw[2][3]);
+
+   vec3 c0 = cross(r1, r2);
+   vec3 c1 = cross(r2, r0);
+   vec3 c2 = cross(r0, r1);
+
+   float det = dot(r0, c0);
+   /* A singular instance transform is undefined behaviour in Vulkan.  Fall
+    * back to something finite so the traversal cannot fault on a NaN
+    * address computation. */
+   float inv_det = det == 0.0 ? 0.0 : 1.0 / det;
+
+   /* Rows of R^-1 are the columns of the cofactor matrix, scaled. */
+   vec3 i0 = vec3(c0.x, c1.x, c2.x) * inv_det;
+   vec3 i1 = vec3(c0.y, c1.y, c2.y) * inv_det;
+   vec3 i2 = vec3(c0.z, c1.z, c2.z) * inv_det;
+
+   mat3x4 wto;
+   wto[0] = vec4(i0, -dot(i0, t));
+   wto[1] = vec4(i1, -dot(i1, t));
+   wto[2] = vec4(i2, -dot(i2, t));
+   return wto;
+}
+
+void
+store64(uint64_t address, uint64_t value)
+{
+   store32(address, uint(value));
+   store32(address + 4ul, uint(value >> 32));
+}
+
+void
+encode_instance(hk_agx3_leaf_job job, uint lane)
+{
+   REF(vk_ir_instance_node) src = REF(vk_ir_instance_node)(
+      OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
+   uint64_t blas = DEREF(src).base_ptr;
+   uint64_t record_offset = uint64_t(job.child_slot + 2u * lane) * 64ul;
+   uint64_t record = job.output_bvh + record_offset;
+   uint index = DEREF(src).instance_id;
+   uint64_t transforms = DEREF(REF(uint64_t)(job.output_bvh + 104ul));
+   uint64_t pointers = DEREF(REF(uint64_t)(job.output_bvh + 128ul));
+   uint64_t transform = transforms + uint64_t(index) * 64ul;
+   mat3x4 otw = DEREF(src).otw_matrix;
+   mat3x4 wto = invert_affine(otw);
+   for (uint i = 0; i < 32; ++i)
+      store32(record + uint64_t(i) * 4ul, 0);
+   store32(record, 26);
+   store32(record + 4ul, DEREF(REF(uint32_t)(blas + 176ul)));
+   store32(record + 8ul, DEREF(src).sbt_offset_and_flags >> 24);
+   store32(record + 12ul, DEREF(src).custom_instance_and_mask >> 24);
+   for (uint c = 0; c < 4; ++c)
+      for (uint r = 0; r < 3; ++r) {
+         uint offset = 4u * (c * 3u + r);
+         store32(record + 16ul + uint64_t(offset), floatBitsToUint(wto[r][c]));
+         store32(job.output_bvh + transform + uint64_t(offset),
+                 floatBitsToUint(otw[r][c]));
+      }
+   for (uint i = 0; i < 6; ++i)
+      store32(record + 64ul + uint64_t(i) * 4ul,
+              DEREF(REF(uint32_t)(blas + 80ul + uint64_t(i) * 4ul)));
+   store64(record + 88ul, blas);
+   store32(record + 96ul, DEREF(src).custom_instance_and_mask & 0xffffffu);
+   /* Native Metal user_instance_id reads this independent32-bit field. */
+   store32(record + 100ul, index);
+   /* Vulkan-only metadata lives outside the native128-byte instance record. */
+   store32(job.output_bvh + transform + 48ul, DEREF(src).sbt_offset_and_flags);
+   store64(record + 104ul, transform - record_offset);
+   store64(job.output_bvh + pointers + 16ul * uint64_t(index), record_offset + 4ul);
+   store64(job.output_bvh + pointers + 16ul * uint64_t(index) + 8ul, 1ul);
+}
+
+void
+main()
+{
+   uint id = gl_WorkGroupID.x, lane = gl_LocalInvocationID.x;
+#ifdef HK_AGX3_BATCH
+   hk_agx3_batch_map mapping = DEREF(INDEX(hk_agx3_batch_map, batch.map, id));
+   if (mapping.build >= batch.job_count)
+      return;
+   hk_agx3_batch_job entry = DEREF(INDEX(hk_agx3_batch_job, batch.jobs, mapping.build));
+   if (DEREF(entry.plan.state).error != 0)
+      return;
+   args.jobs = entry.plan.jobs;
+   args.status = entry.node_status;
+   args.job_count = DEREF(entry.plan.state).job_count;
+   args.slot_bias = entry.plan.m5_aabb_stride;
+   id = mapping.node;
+#endif
+   if (id >= args.job_count)
+      return;
+   hk_agx3_leaf_job job = DEREF(INDEX(hk_agx3_leaf_job, args.jobs, id));
+   uint count = job.child_count;
+   bool leaf = job.node_kind == HK_AGX3_NODE_LEAF;
+   bool instances = job.node_kind == HK_AGX3_NODE_INSTANCES;
+   bool aabbs = job.node_kind == HK_AGX3_NODE_AABBS;
+   /* Bounding-box record stride: 48 (native record class 4) by default; 64
+    * (class 5, one record per slot) is the ASAHI_M5_AABB_RECORD_STRIDE
+    * diagnostic. Both decode in Apple's extractor; only hardware traversal
+    * distinguishes them. */
+   uint aabb_stride = HK_AGX3_GEN1 != 0 || args.slot_bias == 64u ? 64u : 48u;
+   if (count < 1u || count > 8 ||
+       job.node_kind > HK_AGX3_NODE_AABBS) {
+      if (lane == 0)
+         DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_INVALID_COUNT;
+      return;
+   }
+   uint capacity = min(count, 4u), blocks = (count + capacity - 1u) / capacity;
+   if (lane == 0)
+      bad = 0;
+   barrier();
+   if (lane < count) {
+      if (!leaf) {
+         REF(vk_ir_node)
+         src = REF(vk_ir_node)(
+            OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
+         if (aabbs) {
+            REF(vk_ir_aabb_node) box = REF(vk_ir_aabb_node)(src);
+            geometry_id[lane] = DEREF(box).geometry_id_and_flags;
+            primitive[lane] = DEREF(box).primitive_id;
+         }
+         lower[lane] = DEREF(src).aabb.min;
+         upper[lane] = DEREF(src).aabb.max;
+         for (uint a = 0; a < 3; ++a) {
+            if (!finite(lower[lane][a]) || !finite(upper[lane][a]))
+               atomicOr(bad, HK_AGX3_LEAF_NONFINITE);
+            if (lower[lane][a] > upper[lane][a])
+               atomicOr(bad, HK_AGX3_LEAF_RANGE);
+         }
+         if (aabbs && HK_AGX3_GEN1 == 0) {
+            REF(vk_ir_aabb_node) box = REF(vk_ir_aabb_node)(
+               OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
+            /* The native record keeps the geometry id in the low 30 bits and
+             * the opaque flag in bit 31, matching VK_GEOMETRY_OPAQUE. */
+            geometry_id[lane] = DEREF(box).geometry_id_and_flags & 0xbfffffffu;
+            primitive[lane] = DEREF(box).primitive_id & 0x3fffffffu;
+         }
+      } else {
+         REF(vk_ir_triangle_node)
+         src = REF(vk_ir_triangle_node)(
+            OFFSET(job.intermediate_bvh, job.triangle_offsets[lane]));
+         uint geometry_flags = DEREF(src).geometry_id_and_flags;
+         geometry_id[lane] = geometry_flags & ~VK_GEOMETRY_OPAQUE;
+         /* Native Gen1 extractor 0x218e084c4..0x218e087f4 accepts the
+          * same triangle record for types8 and16, then compares type8
+          * to set the extracted geometry's opaque field. */
+         triangle_type[lane] =
+            (geometry_flags & VK_GEOMETRY_OPAQUE) != 0 ? 8u : 16u;
+         primitive[lane] = DEREF(src).triangle_id;
+         for (uint v = 0; v < 3; ++v) {
+            vec3 p;
+            for (uint a = 0; a < 3; ++a) {
+               p[a] = DEREF(src).coords[v][a];
+               if (!finite(p[a]))
+                  atomicOr(bad, HK_AGX3_LEAF_NONFINITE);
+            }
+            vertices[lane * 3u + v] = p;
+         }
+         lower[lane] = min(vertices[lane * 3u], min(vertices[lane * 3u + 1u],
+                                                    vertices[lane * 3u + 2u]));
+         upper[lane] = max(vertices[lane * 3u], max(vertices[lane * 3u + 1u],
+                                                    vertices[lane * 3u + 2u]));
+      }
+   }
+   barrier();
+   if (bad != 0) {
+      if (lane == 0)
+         DEREF(INDEX(uint32_t, args.status, id)) = bad;
+      return;
+   }
+
+   vec3 qbase = vec3(0);
+   uvec3 qexp = uvec3(0);
+   if (leaf && lane < blocks &&
+       !fit_qtb(lane * capacity, min(capacity, count - lane * capacity), qbase,
+                qexp))
+      atomicOr(bad, HK_AGX3_LEAF_RANGE);
+   if (lane == 0) {
+      vec3 lo = lower[0], hi = upper[0], base;
+      uvec3 exponent;
+      for (uint t = 1; t < count; ++t) {
+         lo = min(lo, lower[t]);
+         hi = max(hi, upper[t]);
+      }
+      if (!fit_root(lo, hi, base, exponent)) {
+         atomicOr(bad, HK_AGX3_LEAF_RANGE);
+      } else {
+         for (uint t = 0; t < 8; ++t)
+            root[t] = uint64_t(0);
+         /* Word0 bits 5..7 select the native leaf record stride: class 5
+          * (0xa0) is the 64-byte triangle record, class 7 (0xe1) the 128-byte
+          * instance record and class 4 (0x80) the 48-byte bounding-box record.
+          * Leaves without quantized blocks have slot capacity 0 and, like the
+          * native instance roots, use indexed lane fields (bit 0): the header
+          * codec rejects capacity 0 in count mode, and a count-mode box
+          * cluster returned every lane as record 0 on hardware (only
+          * single-AABB geometries passed). */
+         root[0] =
+            uint64_t((instances ? 0xe1u
+                      : aabbs ? (HK_AGX3_GEN1 != 0 ? 0xa0u : ((aabb_stride == 64u ? 0xa0u : 0x80u) | 1u))
+                      : 0xa0u) |
+                     (leaf ? capacity << 1 : 0u) | (exponent.x << 8));
+         for (uint a = 0; a < 3; ++a)
+            root[a + 1] = uint64_t(floatBitsToUint(base[a]) >> 16);
+         root[4] = uint64_t((exponent.y << 8) | exponent.z);
+         uint child = job.child_slot + (HK_AGX3_GEN1 != 0 ? args.slot_bias : 0);
+         root[5] = uint64_t(child >> 16);
+         root[6] = uint64_t(child & 65535u);
+         /* Leaf type 24 is the native bounding-box record marker. Bits 6..7
+          * (0xc0) follow the captured triangle roots; the QTB-only bit 14 is
+          * not set because no quantized triangle block exists. */
+         root[7] = instances ? uint64_t(0x1a00u | (count == 8u ? 0x3fu : count << 3))
+                   : aabbs ? uint64_t(0x18c0u | (count == 8u ? 0x3fu : count << 3))
+                   : leaf ? uint64_t(0x40c0u | (triangle_type[0] << 8) |
+                                             (count == 8u ? 0x3fu : count << 3))
+                          : uint64_t(0x1f00u | (count == 8u ? 0u : count));
+         const uint shifts[6] = uint[6](57, 50, 43, 36, 29, 22);
+         for (uint t = 0; t < count; ++t) {
+            if (leaf || instances || aabbs)
+               root[t] |= uint64_t((t << 16) | (1u << 20));
+            for (uint a = 0; a < 3; ++a) {
+               float s = step(exponent[a]);
+               uint qlo =
+                  uint(clamp(floor((lower[t][a] - base[a]) / s), 0.0, 127.0));
+               uint qhi =
+                  uint(clamp(ceil((upper[t][a] - base[a]) / s), 0.0, 127.0));
+               precise float rlo = base[a] + float(qlo) * s;
+               precise float rhi = base[a] + float(qhi) * s;
+               while (rlo > lower[t][a] && qlo > 0u) {
+                  --qlo;
+                  rlo = base[a] + float(qlo) * s;
+               }
+               while (rhi < upper[t][a] && qhi < 127u) {
+                  ++qhi;
+                  rhi = base[a] + float(qhi) * s;
+               }
+               if (rlo > lower[t][a] || rhi < upper[t][a])
+                  atomicOr(bad, HK_AGX3_LEAF_RANGE);
+               root[t] |= uint64_t(qlo) << shifts[a];
+               root[t] |= uint64_t(qhi) << shifts[a + 3];
+            }
+         }
+      }
+   }
+   barrier();
+   if (bad != 0) {
+      if (lane == 0)
+         DEREF(INDEX(uint32_t, args.status, id)) = bad;
+      return;
+   }
+
+   uint64_t children =
+      uint64_t(job.output_bvh) + uint64_t(job.child_slot) * 64ul;
+   if (instances && lane < count)
+      encode_instance(job, lane);
+   if (aabbs && lane < count) {
+      if (HK_AGX3_GEN1 != 0)
+         for (uint i = 0; i < 16; ++i)
+            store32(children + uint64_t(lane) * 64ul + uint64_t(i) * 4ul, 0);
+      /* 48-byte native bounding-box record at the selected record stride. */
+      uint64_t record = children + uint64_t(lane) * uint64_t(aabb_stride);
+      store32(record, 0x18u);
+      for (uint a = 0; a < 3; ++a) {
+         store32(record + 4ul + uint64_t(a) * 4ul, floatBitsToUint(lower[lane][a]));
+         store32(record + 16ul + uint64_t(a) * 4ul, floatBitsToUint(upper[lane][a]));
+      }
+      store32(record + 28ul, geometry_id[lane]);
+      store32(record + 32ul, primitive[lane]);
+      /* No optional primitive data: bit 62 of the packed id word is clear
+       * and the relative pointer stays zero. */
+      store32(record + 36ul, 0);
+      store64(record + 40ul, 0ul);
+   }
+   if (leaf && lane < count) {
+      uint64_t geometry = children + uint64_t(blocks + lane) * 64ul;
+      store32(geometry, 0x40u | triangle_type[lane]);
+      for (uint a = 0; a < 3; ++a)
+         for (uint v = 0; v < 4; ++v)
+            store32(geometry + uint64_t(4u + a * 16u + v * 4u),
+                    floatBitsToUint(vertices[lane * 3u + min(v, 2u)][a]));
+      store32(geometry + 52ul, geometry_id[lane]);
+      store32(geometry + 56ul, primitive[lane]);
+      store32(geometry + 60ul, primitive[lane]);
+   }
+   if (leaf && lane < blocks) {
+      /* Each QTB writer owns all sixteen words, avoiding byte-store races. */
+      uint words[16];
+      for (uint w = 0; w < 16; ++w)
+         words[w] = 0;
+      words[0] = 6u | (qexp.z << 8) | (floatBitsToUint(qbase.x) & 0xffff0000u);
+      words[1] = (floatBitsToUint(qbase.y) >> 16) |
+                 (floatBitsToUint(qbase.z) & 0xffff0000u);
+      words[2] = qexp.x | (qexp.y << 8);
+      uint first = lane * capacity, n = min(capacity, count - first);
+      const uint order[4] = uint[4](2, 2, 1, 0);
+      for (uint t = 0; t < n; ++t)
+         for (uint v = 0; v < 4; ++v)
+            for (uint a = 0; a < 3; ++a) {
+               /* Power-of-two scaling avoids reciprocal overflow on small
+                * axes, including the minimum normal binary32 exponent. */
+               uint q = quantize_vertex(
+                  vertices[(first + t) * 3u + order[v]][a], qbase[a], qexp[a]);
+               uint byte_offset = 16u + t * 12u + v * 3u + 2u - a;
+               words[byte_offset / 4u] |= q << ((byte_offset % 4u) * 8u);
+            }
+      for (uint w = 0; w < 16; ++w)
+         store32(children + uint64_t(lane * 64u + w * 4u), words[w]);
+   }
+   if (lane == 0) {
+      for (uint w = 0; w < 8; ++w) {
+         uint64_t address = uint64_t(job.output_bvh) +
+                            uint64_t(job.root_slot) * 64ul + uint64_t(w) * 8ul;
+         store32(address, uint(root[w]));
+         store32(address + 4ul, uint(root[w] >> 32));
+      }
+      DEREF(INDEX(uint32_t, args.status, id)) = HK_AGX3_LEAF_SUCCESS;
+   }
+}
diff --git a/src/asahi/vulkan/bvh/agx3_m5_legacy_plan.comp b/src/asahi/vulkan/bvh/agx3_m5_legacy_plan.comp
new file mode 100644
index 000000000000..72c48d9a74ae
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_m5_legacy_plan.comp
@@ -0,0 +1,499 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * Convert a common binary BVH into independent native-node encoding jobs.
+ * Collapse up to three binary levels into each wide inner node; stop early
+ * at subtrees containing at most eight triangles. Those subtrees become one
+ * leaf cluster. Preserve the common builder's spatial partition and order.
+ * A global atomic allocator compacts jobs/child regions; it never changes
+ * traversal order. The five phases are separated by dispatch barriers.
+ */
+#version 460
+#extension GL_GOOGLE_include_directive : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference2 : require
+#extension GL_EXT_scalar_block_layout : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
+#include "hk_build_interface.h"
+
+#ifndef HK_AGX3_GEN1
+#define HK_AGX3_GEN1 0
+#endif
+
+layout(local_size_x = 64) in;
+#ifdef HK_AGX3_BATCH
+layout(push_constant) uniform CONSTS
+{
+   hk_agx3_batch_args batch;
+};
+hk_agx3_plan_args args;
+#else
+layout(push_constant) uniform CONSTS
+{
+   hk_agx3_plan_args args;
+};
+#endif
+
+void
+fail(uint flag)
+{
+   atomicOr(DEREF(args.state).error, flag);
+}
+
+
+uint
+leaf_size()
+{
+   return args.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR
+             ? SIZEOF(vk_ir_instance_node)
+             : args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                  ? SIZEOF(vk_ir_aabb_node) : SIZEOF(vk_ir_triangle_node);
+}
+
+uint
+leaf_kind()
+{
+   return args.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR
+             ? vk_ir_node_instance
+             : args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                  ? vk_ir_node_aabb : vk_ir_node_triangle;
+}
+
+uint
+source_index(uint ir_id)
+{
+   if (ir_id == VK_BVH_INVALID_NODE)
+      return 0xffffffffu;
+   uint offset = ir_id_to_offset(ir_id), kind = ir_id_to_type(ir_id);
+   if (kind == leaf_kind() && offset < args.root_offset &&
+       offset % leaf_size() == 0) {
+      uint i = offset / leaf_size();
+      if (i < args.leaf_bound)
+         return i;
+   }
+   if (kind == vk_ir_node_internal && offset >= args.root_offset &&
+       (offset - args.root_offset) % SIZEOF(vk_ir_box_node) == 0) {
+      uint i = (offset - args.root_offset) / SIZEOF(vk_ir_box_node);
+      if (i < DEREF(args.header).ir_internal_node_count &&
+          i < args.node_bound - args.leaf_bound)
+         return args.leaf_bound + i;
+   }
+   fail(HK_AGX3_PLAN_INDEX);
+   return 0xffffffffu;
+}
+
+uint
+source_offset(uint index)
+{
+   return index < args.leaf_bound
+             ? index * leaf_size()
+             : args.root_offset +
+                  (index - args.leaf_bound) * SIZEOF(vk_ir_box_node);
+}
+
+uint
+root_index()
+{
+   return args.leaf_bound + DEREF(args.header).ir_internal_node_count - 1;
+}
+
+uvec2
+children(uint index)
+{
+   REF(vk_ir_box_node)
+   node =
+      REF(vk_ir_box_node)(OFFSET(args.intermediate_bvh, source_offset(index)));
+   return uvec2(DEREF(node).children[0], DEREF(node).children[1]);
+}
+
+/* Every non-empty pending subtree contributes at least one leaf. Stop at
+ * nine before pushing past this bounded stack, even for a skewed tree. */
+uint
+count_leaves(uint start)
+{
+   uint stack[9], sp = 1, count = 0, steps = 0;
+   uint opacity = 0;
+   bool have_opacity = false;
+   stack[0] = start;
+   while (sp != 0) {
+      if (++steps > args.node_bound * 2u) {
+         fail(HK_AGX3_PLAN_CYCLE);
+         return 0;
+      }
+      uint index = stack[--sp];
+      if (index < args.leaf_bound) {
+         if (args.geometry_type == VK_GEOMETRY_TYPE_TRIANGLES_KHR) {
+            REF(vk_ir_triangle_node) triangle = REF(vk_ir_triangle_node)(
+               OFFSET(args.intermediate_bvh, source_offset(index)));
+            uint current_opacity = DEREF(triangle).geometry_id_and_flags &
+                                   VK_GEOMETRY_OPAQUE;
+            /* One native leaf cluster shares its child triangle type.
+             * A mixed-opacity subtree must remain an inner node; retain
+             * normal spatial collapse for every homogeneous subtree. */
+            if (have_opacity && opacity != current_opacity)
+               return 9;
+            opacity = current_opacity;
+            have_opacity = true;
+         }
+         ++count;
+      } else {
+         uvec2 pair = children(index);
+         for (uint c = 0; c < 2; c++)
+            if (pair[c] != VK_BVH_INVALID_NODE) {
+               uint child = source_index(pair[c]);
+               if (child == 0xffffffffu)
+                  return 0;
+               if (sp == 9)
+                  return 9;
+               stack[sp++] = child;
+            }
+      }
+      if (count + sp > 8)
+         return 9;
+   }
+   return count;
+}
+
+void
+initialize(uint id)
+{
+   if (id < args.node_bound) {
+      hk_agx3_plan_node node;
+      node.parent = 0xffffffffu;
+      node.leaf_count = id < args.leaf_bound ? 1u : 0u;
+      node.depth = 0;
+      node.job = 0xffffffffu;
+      DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)) = node;
+   }
+   if (id == 0) {
+      hk_agx3_plan_state state;
+      state.job_count = 0;
+      state.groups_y = 1;
+      state.groups_z = 1;
+      state.next_slot = 3;
+      state.depth = 0;
+      state.error = 0;
+      state.saved_offsets = 0;
+      state.saved_positions = 0;
+      if (DEREF(args.header).active_leaf_count == 0)
+         state.error = HK_AGX3_PLAN_EMPTY;
+      if (args.leaf_bound > args.node_bound || args.node_bound > 0xffffffffu / 64u ||
+          (DEREF(args.header).active_leaf_count != 0 &&
+           (DEREF(args.header).ir_internal_node_count == 0 ||
+            DEREF(args.header).ir_internal_node_count >
+               args.node_bound - args.leaf_bound)) ||
+          DEREF(args.header).active_leaf_count > args.leaf_bound ||
+          args.root_offset != args.leaf_bound * leaf_size() ||
+          args.output_bvh == 0 || (args.output_bvh & 63ul) != 0 ||
+          args.output_bvh >= (1ul << 48) ||
+          uint64_t(args.slot_bound) * 64ul + 184ul > (1ul << 48) - args.output_bvh ||
+          args.slot_bound < 5)
+         state.error |= HK_AGX3_PLAN_INDEX;
+      DEREF(args.state) = state;
+   }
+}
+
+void
+link_and_count(uint id)
+{
+   if (id < args.leaf_bound ||
+       id >= args.leaf_bound + DEREF(args.header).ir_internal_node_count)
+      return;
+   uvec2 pair = children(id);
+   for (uint c = 0; c < 2; c++)
+      if (pair[c] != VK_BVH_INVALID_NODE) {
+         uint child = source_index(pair[c]);
+         if (child == 0xffffffffu)
+            return;
+         uint old = atomicCompSwap(
+            DEREF(INDEX(hk_agx3_plan_node, args.nodes, child)).parent,
+            0xffffffffu, id);
+         if (old != 0xffffffffu)
+            fail(HK_AGX3_PLAN_SHAPE);
+      }
+   DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).leaf_count =
+      count_leaves(id);
+}
+
+void
+classify(uint id)
+{
+   if (id >= args.node_bound)
+      return;
+   hk_agx3_plan_node node = DEREF(INDEX(hk_agx3_plan_node, args.nodes, id));
+   uint root = root_index();
+   if (id != root && node.parent == 0xffffffffu)
+      return; /* inactive IR leaf */
+   uint cursor = id, depth = 0;
+   while (cursor != root) {
+      if (depth++ >= args.node_bound) {
+         fail(HK_AGX3_PLAN_CYCLE);
+         return;
+      }
+      cursor = DEREF(INDEX(hk_agx3_plan_node, args.nodes, cursor)).parent;
+      if (cursor >= args.node_bound) {
+         fail(HK_AGX3_PLAN_INDEX);
+         return;
+      }
+   }
+   DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).depth = depth;
+   bool selected = id == root;
+   if (!selected)
+      selected = node.leaf_count <= 8
+                    ? DEREF(INDEX(hk_agx3_plan_node, args.nodes, node.parent))
+                            .leaf_count > 8
+                    : depth % 3 == 0;
+   if (!selected)
+      return;
+   uint job = atomicAdd(DEREF(args.state).job_count, 1);
+   if (job >= args.node_bound) {
+      fail(HK_AGX3_PLAN_ALLOCATION);
+      return;
+   }
+   DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).job = job;
+   hk_agx3_leaf_job output_job;
+   output_job.intermediate_bvh = args.intermediate_bvh;
+   output_job.output_bvh = args.output_bvh;
+   output_job.root_slot = id == root ? 0u : 0xffffffffu;
+   output_job.child_slot = 0;
+   output_job.child_count = 0;
+   output_job.node_kind =
+      node.leaf_count <= 8
+         ? (args.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR
+               ? HK_AGX3_NODE_INSTANCES
+               : args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                    ? HK_AGX3_NODE_AABBS : HK_AGX3_NODE_LEAF)
+         : HK_AGX3_NODE_INNER;
+   for (uint i = 0; i < 8; i++)
+      output_job.triangle_offsets[i] = 0;
+   DEREF(INDEX(hk_agx3_leaf_job, args.jobs, job)) = output_job;
+   atomicMax(DEREF(args.state).depth, (depth + 2u) / 3u + 1u);
+}
+
+void
+emit(uint id)
+{
+   if (id >= args.node_bound)
+      return;
+   hk_agx3_plan_node node = DEREF(INDEX(hk_agx3_plan_node, args.nodes, id));
+   if (node.job == 0xffffffffu)
+      return;
+   bool leaf = node.leaf_count <= 8;
+   uint stack[9], level[9], selected[8], sp = 1, count = 0, steps = 0;
+   stack[0] = id;
+   level[0] = 0;
+   while (sp != 0) {
+      if (++steps > args.node_bound * 2u) {
+         fail(HK_AGX3_PLAN_CYCLE);
+         return;
+      }
+      uint current = stack[--sp], d = level[sp];
+      bool end = leaf ? current < args.leaf_bound
+                      : current != id &&
+                           (DEREF(INDEX(hk_agx3_plan_node, args.nodes, current))
+                                  .leaf_count <= 8 ||
+                            d == 3);
+      if (end) {
+         if (count == 8) {
+            fail(HK_AGX3_PLAN_SHAPE);
+            return;
+         }
+         selected[count++] = current;
+      } else {
+         uvec2 pair = children(current);
+         /* Stack is LIFO; visit the common builder's left child first. */
+         for (int c = 1; c >= 0; c--)
+            if (pair[c] != VK_BVH_INVALID_NODE) {
+               uint child = source_index(pair[c]);
+               if (child == 0xffffffffu || sp == 9) {
+                  fail(HK_AGX3_PLAN_SHAPE);
+                  return;
+               }
+               stack[sp] = child;
+               level[sp++] = d + 1;
+            }
+      }
+   }
+   if (count == 0 || (leaf && count != node.leaf_count)) {
+      fail(HK_AGX3_PLAN_SHAPE);
+      return;
+   }
+   /* Instances take two slots each; triangles one record slot each plus one
+    * quantized block per four; 48-byte bounding-box records are packed
+    * contiguously (native record class 4) and rounded up to whole slots. */
+   uint slots = leaf ? (args.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR
+                           ? 2u * count
+                           : args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                                ? HK_AGX3_GEN1 != 0 ? count : (count * (args.m5_aabb_stride == 64u ? 64u : 48u) + 63u) / 64u
+                                : count + (count + 3u) / 4u) : count;
+   uint first = atomicAdd(DEREF(args.state).next_slot, slots);
+   if (slots > args.slot_bound || first > args.slot_bound - slots) {
+      fail(HK_AGX3_PLAN_ALLOCATION);
+      return;
+   }
+   REF(hk_agx3_leaf_job) job = INDEX(hk_agx3_leaf_job, args.jobs, node.job);
+   DEREF(job).child_slot = first;
+   DEREF(job).child_count = count;
+   for (uint c = 0; c < count; c++) {
+      DEREF(job).triangle_offsets[c] = source_offset(selected[c]);
+      if (!leaf) {
+         uint child_job =
+            DEREF(INDEX(hk_agx3_plan_node, args.nodes, selected[c])).job;
+         if (child_job >= DEREF(args.state).job_count) {
+            fail(HK_AGX3_PLAN_SHAPE);
+            return;
+         }
+         DEREF(INDEX(hk_agx3_leaf_job, args.jobs, child_job)).root_slot =
+            first + c;
+      }
+   }
+}
+
+void
+finish(uint id)
+{
+   if (id != 0)
+      return;
+   hk_agx3_envelope_job job;
+   uint root = DEREF(args.state).error == 0 ? root_index() : 0;
+   job.bounds =
+      REF(vk_ir_node)(OFFSET(args.intermediate_bvh, source_offset(root)));
+   job.output_bvh = args.output_bvh;
+   job.end_slot = DEREF(args.state).next_slot;
+   job.primitive_count =
+      DEREF(args.state).error == 0 ? DEREF(args.header).active_leaf_count : 0;
+   job.node_count = DEREF(args.state).job_count;
+   job.depth = DEREF(args.state).depth;
+   /* Internal job sentinel only: an all-inactive tree has no native nodes.
+    * Other planner errors remain errors; do not publish them as empty. */
+   job.opaque_build_word = DEREF(args.state).error == HK_AGX3_PLAN_EMPTY
+                             ? HK_AGX3_PLAN_EMPTY : 0;
+   job.resource_id = args.resource_id;
+   job.geometry_type = args.geometry_type;
+   job.leaf_bound = args.leaf_bound;
+   DEREF(args.envelope) = job;
+   if (DEREF(args.state).error != 0)
+      DEREF(args.state).job_count = 0;
+}
+
+/* A single active common leaf collapses to the same one-job native cluster
+ * as the general planner. No IR internal node, Morton ordering, parent walk
+ * or atomic allocation is needed. The leaf shader still decides inactivity. */
+void
+plan_single_leaf()
+{
+   hk_agx3_plan_state state;
+   state.job_count = 0;
+   state.groups_y = 1;
+   state.groups_z = 1;
+   state.next_slot = 3;
+   state.depth = 0;
+   state.error = 0;
+   state.saved_offsets = 0;
+   state.saved_positions = 0;
+   uint active_count = DEREF(args.header).active_leaf_count;
+   if (active_count == 0)
+      state.error = HK_AGX3_PLAN_EMPTY;
+   if (args.leaf_bound != 1 || args.node_bound < 2 || active_count > 1 ||
+       args.root_offset != leaf_size() ||
+       (args.geometry_type != VK_GEOMETRY_TYPE_TRIANGLES_KHR &&
+        args.geometry_type != VK_GEOMETRY_TYPE_AABBS_KHR) ||
+       args.output_bvh == 0 || (args.output_bvh & 1023ul) != 0 ||
+       args.output_bvh >= (1ul << 48) ||
+       uint64_t(args.slot_bound) * 64ul + 184ul > (1ul << 48) - args.output_bvh ||
+       args.slot_bound < 5)
+      state.error |= HK_AGX3_PLAN_INDEX;
+   if (state.error == 0) {
+      hk_agx3_leaf_job job;
+      job.intermediate_bvh = args.intermediate_bvh;
+      job.output_bvh = args.output_bvh;
+      job.root_slot = 0;
+      job.child_slot = 3;
+      job.child_count = 1;
+      job.node_kind = args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                        ? HK_AGX3_NODE_AABBS : HK_AGX3_NODE_LEAF;
+      for (uint i = 0; i < 8; ++i)
+         job.triangle_offsets[i] = 0;
+      DEREF(INDEX(hk_agx3_leaf_job, args.jobs, 0)) = job;
+      state.job_count = 1;
+      state.depth = 1;
+      state.next_slot = args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                           ? 4 : 5;
+   }
+   hk_agx3_envelope_job envelope;
+   envelope.bounds = REF(vk_ir_node)(args.intermediate_bvh);
+   envelope.output_bvh = args.output_bvh;
+   envelope.end_slot = state.next_slot;
+   envelope.primitive_count = state.error == 0 ? 1 : 0;
+   envelope.node_count = state.job_count;
+   envelope.depth = state.depth;
+   envelope.opaque_build_word = state.error == HK_AGX3_PLAN_EMPTY
+                                 ? HK_AGX3_PLAN_EMPTY : 0;
+   envelope.resource_id = args.resource_id;
+   envelope.geometry_type = args.geometry_type;
+   envelope.leaf_bound = 1;
+   DEREF(args.envelope) = envelope;
+   DEREF(args.state) = state;
+}
+
+void
+main()
+{
+   uint id = gl_GlobalInvocationID.x;
+#ifdef HK_AGX3_BATCH
+   uint build = gl_WorkGroupID.y;
+   if (build >= batch.job_count)
+      return;
+   REF(hk_agx3_batch_job) entry = INDEX(hk_agx3_batch_job, batch.jobs, build);
+   args = DEREF(entry).plan;
+   args.phase = batch.phase;
+   /* Snapshot the final number of independently writable jobs. Allocation
+    * is bounded by the host's sum of node bounds. Planner failures still
+    * produce mapped entries whose consumer observes the error and returns. */
+   if (args.phase == 3 && id == 0) {
+      uint count = DEREF(args.state).error == 0 ? DEREF(args.state).job_count : 0;
+      DEREF(entry).first_job = atomicAdd(DEREF(batch.dispatch).x, count);
+      DEREF(entry).encode_count = count;
+   }
+   if (args.phase == 4 && id < DEREF(entry).encode_count) {
+      hk_agx3_batch_map mapping;
+      mapping.build = build;
+      mapping.node = id;
+      DEREF(INDEX(hk_agx3_batch_map, batch.map,
+                  DEREF(entry).first_job + id)) = mapping;
+   }
+   if (DEREF(entry).single_leaf != 0) {
+      if (args.phase == 0 && id == 0)
+         plan_single_leaf();
+      return;
+   }
+#endif
+#if !HK_AGX3_GEN1
+   /* The single-leaf phase is dispatched by the M5 backend only. */
+   if (args.phase == 5) {
+      if (id == 0)
+         plan_single_leaf();
+      return;
+   }
+#endif
+   if (args.phase == 0) {
+      initialize(id);
+      return;
+   }
+   if (args.phase == 4) {
+      finish(id);
+      return;
+   }
+   if (DEREF(args.state).error != 0)
+      return;
+   if (args.phase == 1)
+      link_and_count(id);
+   if (args.phase == 2)
+      classify(id);
+   if (args.phase == 3)
+      emit(id);
+}
diff --git a/src/asahi/vulkan/bvh/agx3_m5_legacy_refit.comp b/src/asahi/vulkan/bvh/agx3_m5_legacy_refit.comp
new file mode 100644
index 000000000000..7e6dcafb0986
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_m5_legacy_refit.comp
@@ -0,0 +1,101 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ * Refit a retained binary topology without sorting or repartitioning.
+ */
+#version 460
+#extension GL_GOOGLE_include_directive : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference2 : require
+#extension GL_EXT_scalar_block_layout : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
+#extension GL_KHR_memory_scope_semantics : require
+#include "hk_build_interface.h"
+
+#ifndef HK_AGX3_GEN1
+#define HK_AGX3_GEN1 0
+#endif
+layout(local_size_x = 64) in;
+layout(push_constant) uniform CONSTS { hk_m5_refit_args args; };
+struct refit_node { uint parent; uint arrivals; uint reserved[2]; };
+TYPE(refit_node, 4);
+uint source_index(uint child)
+{
+   uint offset = ir_id_to_offset(child);
+   return ir_id_to_type(child) == vk_ir_node_internal
+      ? args.leaf_bound + (offset - args.root_offset) / SIZEOF(vk_ir_box_node)
+      : offset / (args.root_offset / args.leaf_bound);
+}
+void main()
+{
+   uint id = gl_GlobalInvocationID.x;
+   uint header_words = SIZEOF(vk_ir_header) / 4;
+   uint internal_bound = args.node_bound - args.leaf_bound;
+   if (args.phase <= 1) {
+      uint64_t retained_base = args.retained;
+      if (HK_AGX3_GEN1 != 0 && args.phase == 1 && args.metadata != 0ul) {
+         libagx_gen1_as_metadata live =
+            DEREF(REF(libagx_gen1_as_metadata)(args.metadata));
+         retained_base = live.native_base + live.retained_offset;
+      }
+      /* Preserve the actual binary topology, not unused internal capacity.
+       * Restore counts come from the retained GPU header after copy/rebase. */
+      uint internal_count = HK_AGX3_GEN1 == 0 ? internal_bound : args.phase == 0 ? DEREF(args.header).ir_internal_node_count
+         : DEREF(REF(vk_ir_header)(retained_base)).ir_internal_node_count;
+      uint words = header_words + internal_count * (SIZEOF(vk_ir_box_node) / 4);
+      if (id < words) {
+         uint64_t addr = id < header_words ? uint64_t(args.header) + uint64_t(id)*4ul
+            : args.intermediate_bvh + uint64_t(args.root_offset) + uint64_t(id-header_words)*4ul;
+         REF(uint32_t) retained = REF(uint32_t)(retained_base + uint64_t(id)*4ul);
+         if (args.phase == 0) DEREF(retained) = DEREF(REF(uint32_t)(addr));
+         else {
+            uint value = DEREF(retained);
+            if (id < 3) value = 0x7fffffffu;
+            else if (id < 6) value = 0x80000000u;
+            else if (id == 6) value = 0;
+            DEREF(REF(uint32_t)(addr)) = value;
+         }
+      }
+      if (args.phase == 1 && id < args.node_bound) {
+         DEREF(INDEX(refit_node, args.parents, id)).parent = 0xffffffffu;
+         DEREF(INDEX(refit_node, args.parents, id)).arrivals = 0;
+      }
+      return;
+   }
+   if (args.phase == 2) {
+      if (id >= DEREF(args.header).ir_internal_node_count) return;
+      REF(vk_ir_box_node) node = REF(vk_ir_box_node)(args.intermediate_bvh +
+         uint64_t(args.root_offset) + uint64_t(id)*uint64_t(SIZEOF(vk_ir_box_node)));
+      for (uint c = 0; c < 2; ++c) {
+         uint child = DEREF(node).children[c];
+         if (child != VK_BVH_INVALID_NODE)
+            DEREF(INDEX(refit_node, args.parents, source_index(child))).parent = args.leaf_bound + id;
+      }
+      return;
+   }
+   if (id >= args.leaf_bound) return;
+   uint parent = DEREF(INDEX(refit_node, args.parents, id)).parent;
+   for (uint steps = 0; parent != 0xffffffffu && steps < args.node_bound; ++steps) {
+      REF(vk_ir_box_node) node = REF(vk_ir_box_node)(args.intermediate_bvh +
+         uint64_t(args.root_offset) + uint64_t(parent-args.leaf_bound)*uint64_t(SIZEOF(vk_ir_box_node)));
+      uint expected = uint(DEREF(node).children[0] != VK_BVH_INVALID_NODE) +
+                      uint(DEREF(node).children[1] != VK_BVH_INVALID_NODE);
+      uint arrived = atomicAdd(DEREF(INDEX(refit_node, args.parents, parent)).arrivals,
+         1, gl_ScopeDevice, gl_StorageSemanticsBuffer,
+         gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
+      if (arrived + 1 != expected) break;
+      vk_aabb bounds; bounds.min = vec3(INFINITY); bounds.max = vec3(-INFINITY);
+      for (uint c = 0; c < 2; ++c) {
+         uint child = DEREF(node).children[c];
+         if (child == VK_BVH_INVALID_NODE) continue;
+         REF(vk_ir_node) src = REF(vk_ir_node)(args.intermediate_bvh + uint64_t(ir_id_to_offset(child)));
+         bounds.min = min(bounds.min, DEREF(src).aabb.min);
+         bounds.max = max(bounds.max, DEREF(src).aabb.max);
+      }
+      DEREF(node).base.aabb = bounds;
+      parent = DEREF(INDEX(refit_node, args.parents, parent)).parent;
+   }
+}
diff --git a/src/asahi/vulkan/bvh/agx3_node.h b/src/asahi/vulkan/bvh/agx3_node.h
new file mode 100644
index 000000000000..1996855fe17f
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_node.h
@@ -0,0 +1,139 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * Native G17P node encoding helpers shared by node encoding (agx3_leaf.comp)
+ * and refit propagation (agx3_refit.comp).
+ */
+#ifndef HK_AGX3_NODE_H
+#define HK_AGX3_NODE_H
+
+bool
+finite(float x)
+{
+   return !isnan(x) && !isinf(x);
+}
+float
+origin_down(float x)
+{
+   uint bits = floatBitsToUint(x);
+   uint upper_bits = bits >> 16;
+   if ((bits & 0x80000000u) != 0 && (bits & 65535u) != 0)
+      ++upper_bits;
+   return uintBitsToFloat(upper_bits << 16);
+}
+
+float
+step(uint exponent)
+{
+   return uintBitsToFloat(exponent << 23);
+}
+
+bool
+fit_root(vec3 lo, vec3 hi, out vec3 base, out uvec3 exponent)
+{
+   for (uint a = 0; a < 3; ++a) {
+      base[a] = origin_down(lo[a]);
+      precise float extent = hi[a] - base[a];
+      if (!finite(base[a]) || !finite(extent))
+         return false;
+      precise float unit = extent / 127.0;
+      uint bits = floatBitsToUint(unit);
+      uint e = (bits >> 23) & 255u;
+      if (e != 0 && (bits & 0x7fffffu) != 0)
+         ++e;
+      e = max(e, 1u);
+      while (e < 255u && ceil(extent / step(e)) > 127.0)
+         ++e;
+      if (e == 255u)
+         return false;
+      exponent[a] = e;
+   }
+   return true;
+}
+
+/* Stores go through these helpers: glslang's precise propagation runs out
+ * of memory on buffer-reference stores written out in main(). */
+void
+store32(uint64_t address, uint value)
+{
+   DEREF(REF(uint32_t)(address)) = value;
+}
+
+void
+store_word64(uint64_t address, uint64_t value)
+{
+   DEREF(REF(uint64_t)(address)) = value;
+}
+
+void
+store128(uint64_t address, uvec4 value)
+{
+   DEREF(REF(uvec4)(address)) = value;
+}
+
+void
+store_aabb(uint64_t address, vec3 lo, vec3 hi)
+{
+   DEREF(REF(vk_aabb)(address)).min = lo;
+   DEREF(REF(vk_aabb)(address)).max = hi;
+}
+
+/* Bounds a refit leaves for a job that failed: its ancestors fail too. */
+#define HK_AGX3_FAILED_BOUNDS vec3(uintBitsToFloat(0x7fc00000u))
+
+/* Root word t of a node with `count` children: its header bits, then the
+ * bounds of child t (lo, hi) quantized against the node's base and
+ * exponent. Clears `ok` when the bounds do not fit. */
+uint64_t
+root_word(uint t, uint count, uint kind, uint triangle_type0, uint child_slot,
+          uint slot_bias, vec3 base, uvec3 exponent, vec3 lo, vec3 hi, inout bool ok)
+{
+   bool leaf = kind == HK_AGX3_NODE_LEAF, instances = kind == HK_AGX3_NODE_INSTANCES;
+   bool aabbs = kind == HK_AGX3_NODE_AABBS;
+   uint capacity = max(min(count, 4u), 1u);
+   uint64_t word = uint64_t(0);
+   if (t == 0)
+      word = uint64_t((instances ? 0xe1u : 0xa0u) |
+                      (leaf ? capacity << 1 : 0u) | (exponent.x << 8));
+   else if (t <= 3)
+      word = uint64_t(floatBitsToUint(base[t - 1u]) >> 16);
+   else if (t == 4)
+      word = uint64_t((exponent.y << 8) | exponent.z);
+   else if (t == 5)
+      word = uint64_t((child_slot + slot_bias) >> 16);
+   else if (t == 6)
+      word = uint64_t((child_slot + slot_bias) & 65535u);
+   else
+      word = instances ? uint64_t(0x1a00u | (count == 8u ? 0x3fu : count << 3))
+             : aabbs ? uint64_t(0x18c0u | (count == 8u ? 0x3fu : count << 3))
+             : leaf ? uint64_t(0x40c0u | (triangle_type0 << 8) |
+                                       (count == 8u ? 0x3fu : count << 3))
+                    : uint64_t(0x1f00u | (count == 8u ? 0u : count));
+   if (t < count) {
+      const uint shifts[6] = uint[6](57, 50, 43, 36, 29, 22);
+      if (leaf || instances || aabbs)
+         word |= uint64_t((t << 16) | (1u << 20));
+      for (uint a = 0; a < 3; ++a) {
+         float s = step(exponent[a]);
+         uint qlo = uint(clamp(floor((lo[a] - base[a]) / s), 0.0, 127.0));
+         uint qhi = uint(clamp(ceil((hi[a] - base[a]) / s), 0.0, 127.0));
+         precise float rlo = base[a] + float(qlo) * s;
+         precise float rhi = base[a] + float(qhi) * s;
+         while (rlo > lo[a] && qlo > 0u) {
+            --qlo;
+            rlo = base[a] + float(qlo) * s;
+         }
+         while (rhi < hi[a] && qhi < 127u) {
+            ++qhi;
+            rhi = base[a] + float(qhi) * s;
+         }
+         if (rlo > lo[a] || rhi < hi[a])
+            ok = false;
+         word |= uint64_t(qlo) << shifts[a];
+         word |= uint64_t(qhi) << shifts[a + 3];
+      }
+   }
+   return word;
+}
+
+#endif /* HK_AGX3_NODE_H */
diff --git a/src/asahi/vulkan/bvh/agx3_plan.comp b/src/asahi/vulkan/bvh/agx3_plan.comp
index ceb6e383c1b4..39ec47be13f1 100644
--- a/src/asahi/vulkan/bvh/agx3_plan.comp
+++ b/src/asahi/vulkan/bvh/agx3_plan.comp
@@ -3,10 +3,15 @@
  *
  * Convert a common binary BVH into independent native-node encoding jobs.
  * Collapse up to three binary levels into each wide inner node; stop early
- * at subtrees containing at most eight triangles. Those subtrees become one
- * leaf cluster. Preserve the common builder's spatial partition and order.
+ * at subtrees containing at most eight triangle records. Those subtrees
+ * become one leaf cluster. A record holds one triangle, or two sibling
+ * triangles that share an edge (phase 0 marks their parent as a pair).
+ * Preserve the common builder's spatial partition and order. Built twice:
+ * HK_AGX3_PLAN_PAIRS forms pairs (see hk_agx3_plan_args.pairs).
  * A global atomic allocator compacts jobs/child regions; it never changes
  * traversal order. The five phases are separated by dispatch barriers.
+ * An update does not plan: it encodes from the retained jobs (see
+ * hk_agx3_retained_job).
  */
 #version 460
 #extension GL_GOOGLE_include_directive : require
@@ -26,6 +31,11 @@ layout(push_constant) uniform CONSTS
    hk_agx3_plan_args args;
 };
 
+/* The common builder's internal node count, final before planning starts:
+ * main() reads it once rather than every source_index() reloading it past
+ * this invocation's own stores and atomics. */
+uint internal_count;
+
 void
 fail(uint flag)
 {
@@ -57,7 +67,7 @@ source_index(uint ir_id)
    if (kind == vk_ir_node_internal && offset >= args.root_offset &&
        (offset - args.root_offset) % SIZEOF(vk_ir_box_node) == 0) {
       uint i = (offset - args.root_offset) / SIZEOF(vk_ir_box_node);
-      if (i < DEREF(args.header).ir_internal_node_count &&
+      if (i < internal_count &&
           i < args.node_bound - args.leaf_bound)
          return args.leaf_bound + i;
    }
@@ -77,7 +87,7 @@ source_offset(uint index)
 uint
 root_index()
 {
-   return args.leaf_bound + DEREF(args.header).ir_internal_node_count - 1;
+   return args.leaf_bound + internal_count - 1;
 }
 
 uvec2
@@ -89,8 +99,66 @@ children(uint index)
    return uvec2(DEREF(node).children[0], DEREF(node).children[1]);
 }
 
+#ifdef HK_AGX3_PLAN_PAIRS
+/* Triangle leaf index of an IR child reference, or ~0u. Unlike
+ * source_index() this never records an error, so phase 0 may use it before
+ * the plan state exists. */
+uint
+triangle_index(uint ir_id)
+{
+   if (ir_id == VK_BVH_INVALID_NODE || ir_id_to_type(ir_id) != vk_ir_node_triangle)
+      return ~0u;
+   uint offset = ir_id_to_offset(ir_id);
+   if (offset >= args.root_offset || offset % SIZEOF(vk_ir_triangle_node) != 0)
+      return ~0u;
+   uint i = offset / SIZEOF(vk_ir_triangle_node);
+   return i < args.leaf_bound ? i : ~0u;
+}
+
+/* Pair flags (HK_AGX3_PLAN_PAIR...) for internal node `index`: set when its
+ * children are two triangles of one geometry and opacity sharing an edge,
+ * which then make up one native record. */
+uint
+pair_flags(uint index)
+{
+   if (index < args.leaf_bound ||
+       index - args.leaf_bound >= internal_count)
+      return 0u;
+   uvec2 pair = children(index);
+   uint t0 = triangle_index(pair[0]), t1 = triangle_index(pair[1]);
+   if (t0 == ~0u || t1 == ~0u)
+      return 0u;
+   REF(vk_ir_triangle_node) a = REF(vk_ir_triangle_node)(
+      OFFSET(args.intermediate_bvh, source_offset(t0)));
+   REF(vk_ir_triangle_node) b = REF(vk_ir_triangle_node)(
+      OFFSET(args.intermediate_bvh, source_offset(t1)));
+   uint flags = DEREF(a).geometry_id_and_flags;
+   if (flags != DEREF(b).geometry_id_and_flags)
+      return 0u;
+   vec3 va[3], vb[3];
+   for (uint v = 0; v < 3; ++v)
+      for (uint c = 0; c < 3; ++c) {
+         va[v][c] = DEREF(a).coords[v][c];
+         vb[v][c] = DEREF(b).coords[v][c];
+      }
+   if (hk_agx3_shared_edge(va, vb) == ~0u)
+      return 0u;
+   return HK_AGX3_PLAN_PAIR |
+          ((flags & VK_GEOMETRY_OPAQUE) != 0 ? HK_AGX3_PLAN_PAIR_OPAQUE : 0u);
+}
+
+bool
+is_pair(uint index)
+{
+   return index >= args.leaf_bound &&
+          (DEREF(INDEX(hk_agx3_plan_node, args.nodes, index)).depth &
+           HK_AGX3_PLAN_PAIR) != 0u;
+}
+#endif
+
 /* Every non-empty pending subtree contributes at least one leaf. Stop at
- * nine before pushing past this bounded stack, even for a skewed tree. */
+ * nine before pushing past this bounded stack, even for a skewed tree. A
+ * triangle pair counts as one leaf. */
 uint
 count_leaves(uint start)
 {
@@ -104,12 +172,29 @@ count_leaves(uint start)
          return 0;
       }
       uint index = stack[--sp];
+#ifdef HK_AGX3_PLAN_PAIRS
+      uint flags = index < args.leaf_bound ? 0u
+         : DEREF(INDEX(hk_agx3_plan_node, args.nodes, index)).depth;
+      if (index < args.leaf_bound || (flags & HK_AGX3_PLAN_PAIR) != 0u) {
+         if (args.geometry_type == VK_GEOMETRY_TYPE_TRIANGLES_KHR) {
+            uint current_opacity;
+            if (index < args.leaf_bound) {
+               REF(vk_ir_triangle_node) triangle = REF(vk_ir_triangle_node)(
+                  OFFSET(args.intermediate_bvh, source_offset(index)));
+               current_opacity = DEREF(triangle).geometry_id_and_flags &
+                                 VK_GEOMETRY_OPAQUE;
+            } else {
+               current_opacity = (flags & HK_AGX3_PLAN_PAIR_OPAQUE) != 0u
+                                    ? VK_GEOMETRY_OPAQUE : 0u;
+            }
+#else
       if (index < args.leaf_bound) {
          if (args.geometry_type == VK_GEOMETRY_TYPE_TRIANGLES_KHR) {
             REF(vk_ir_triangle_node) triangle = REF(vk_ir_triangle_node)(
                OFFSET(args.intermediate_bvh, source_offset(index)));
             uint current_opacity = DEREF(triangle).geometry_id_and_flags &
                                    VK_GEOMETRY_OPAQUE;
+#endif
             /* One native leaf cluster shares its child triangle type.
              * A mixed-opacity subtree must remain an inner node; retain
              * normal spatial collapse for every homogeneous subtree. */
@@ -144,8 +229,15 @@ initialize(uint id)
       hk_agx3_plan_node node;
       node.parent = 0xffffffffu;
       node.leaf_count = id < args.leaf_bound ? 1u : 0u;
+#ifdef HK_AGX3_PLAN_PAIRS
+      node.depth = pair_flags(id);
+      node.job = 0xffffffffu;
+      if (node.depth != 0u)
+         node.leaf_count = 1;
+#else
       node.depth = 0;
       node.job = 0xffffffffu;
+#endif
       DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)) = node;
    }
    if (id == 0) {
@@ -156,8 +248,8 @@ initialize(uint id)
       state.next_slot = 3;
       state.depth = 0;
       state.error = 0;
-      state.reserved[0] = 0;
-      state.reserved[1] = 0;
+      state.saved_offsets = 0;
+      state.saved_positions = 0;
       if (DEREF(args.header).active_leaf_count == 0)
          state.error = HK_AGX3_PLAN_EMPTY;
       if (args.leaf_bound > args.node_bound || args.node_bound > 0xffffffffu / 64u ||
@@ -180,7 +272,7 @@ void
 link_and_count(uint id)
 {
    if (id < args.leaf_bound ||
-       id >= args.leaf_bound + DEREF(args.header).ir_internal_node_count)
+       id >= args.leaf_bound + internal_count)
       return;
    uvec2 pair = children(id);
    for (uint c = 0; c < 2; c++)
@@ -194,8 +286,11 @@ link_and_count(uint id)
          if (old != 0xffffffffu)
             fail(HK_AGX3_PLAN_SHAPE);
       }
-   DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).leaf_count =
-      count_leaves(id);
+#ifdef HK_AGX3_PLAN_PAIRS
+   if (!is_pair(id))
+#endif
+      DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).leaf_count =
+         count_leaves(id);
 }
 
 void
@@ -219,7 +314,12 @@ classify(uint id)
          return;
       }
    }
+#ifdef HK_AGX3_PLAN_PAIRS
+   DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).depth =
+      depth | (node.depth & (HK_AGX3_PLAN_PAIR | HK_AGX3_PLAN_PAIR_OPAQUE));
+#else
    DEREF(INDEX(hk_agx3_plan_node, args.nodes, id)).depth = depth;
+#endif
    bool selected = id == root;
    if (!selected)
       selected = node.leaf_count <= 8
@@ -271,8 +371,13 @@ emit(uint id)
          return;
       }
       uint current = stack[--sp], d = level[sp];
+#ifdef HK_AGX3_PLAN_PAIRS
+      bool end = leaf ? current < args.leaf_bound || is_pair(current)
+                      : current != id &&
+#else
       bool end = leaf ? current < args.leaf_bound
                       : current != id &&
+#endif
                            (DEREF(INDEX(hk_agx3_plan_node, args.nodes, current))
                                   .leaf_count <= 8 ||
                             d == 3);
@@ -314,7 +419,13 @@ emit(uint id)
    DEREF(job).child_slot = first;
    DEREF(job).child_count = count;
    for (uint c = 0; c < count; c++) {
+#ifdef HK_AGX3_PLAN_PAIRS
+      DEREF(job).triangle_offsets[c] =
+         source_offset(selected[c]) |
+         (leaf && selected[c] >= args.leaf_bound ? HK_AGX3_LEAF_PAIR_OFFSET : 0u);
+#else
       DEREF(job).triangle_offsets[c] = source_offset(selected[c]);
+#endif
       if (!leaf) {
          uint child_job =
             DEREF(INDEX(hk_agx3_plan_node, args.nodes, selected[c])).job;
@@ -349,14 +460,21 @@ finish(uint id)
    job.geometry_type = args.geometry_type;
    job.leaf_bound = args.leaf_bound;
    DEREF(args.envelope) = job;
+   uint jobs = DEREF(args.state).error != 0 ? 0u : DEREF(args.state).job_count;
    if (DEREF(args.state).error != 0)
       DEREF(args.state).job_count = 0;
+   uint64_t dispatch = uint64_t(args.state) + uint64_t(HK_AGX3_PLAN_ENCODE_DISPATCH);
+   DEREF(REF(uint32_t)(dispatch)) =
+      (jobs + HK_AGX3_NODE_JOBS_PER_GROUP - 1u) / HK_AGX3_NODE_JOBS_PER_GROUP;
+   DEREF(REF(uint32_t)(dispatch + 4ul)) = 1u;
+   DEREF(REF(uint32_t)(dispatch + 8ul)) = 1u;
 }
 
 void
 main()
 {
    uint id = gl_GlobalInvocationID.x;
+   internal_count = DEREF(args.header).ir_internal_node_count;
    if (args.phase == 0) {
       initialize(id);
       return;
diff --git a/src/asahi/vulkan/bvh/agx3_refit.comp b/src/asahi/vulkan/bvh/agx3_refit.comp
index d5737b36935b..1a7434d82cdd 100644
--- a/src/asahi/vulkan/bvh/agx3_refit.comp
+++ b/src/asahi/vulkan/bvh/agx3_refit.comp
@@ -1,6 +1,20 @@
 /* Copyright 2026 Aurora Silicon
  * SPDX-License-Identifier: MIT
- * Refit a retained binary topology without sorting or repartitioning.
+ *
+ * Retained jobs of updatable structures (see hk_agx3_retained_job).
+ * HK_AGX3_REFIT_SAVE keeps a full build's encoding jobs after its image, one
+ * thread per job, and HK_AGX3_REFIT_COUNT_LEVELS and LIST_LEVELS list its
+ * inner jobs by level; counting also gives triangle leaves their slots. HK_AGX3_REFIT_PREPARE readies an update, after its
+ * leaves: it finds the source's jobs, copies them when updating into
+ * another structure, clears the arrival counters and writes the refit
+ * state, the level dispatches and the envelope job. After refit encoding
+ * has encoded the leaf jobs, HK_AGX3_REFIT_PROPAGATE encodes the inner jobs
+ * deeper than the levels, one thread per job, and HK_AGX3_REFIT_INNER those
+ * of one level, eight lanes per job.
+ *
+ * Propagation runs apart from encoding: its device-scope arrivals wait for
+ * the thread's earlier global stores, which in node encoding are a whole
+ * node's.
  */
 #version 460
 #extension GL_GOOGLE_include_directive : require
@@ -12,86 +26,373 @@
 #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
 #extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
 #extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
-#extension GL_KHR_memory_scope_semantics : require
 #include "hk_build_interface.h"
-layout(local_size_x = 64) in;
+#include "agx3_node.h"
+layout(local_size_x = HK_AGX3_REFIT_GROUP_SIZE) in;
 layout(push_constant) uniform CONSTS { hk_agx3_refit_args args; };
-struct refit_node { uint parent; uint arrivals; uint reserved[2]; };
-TYPE(refit_node, 4);
-uint source_index(uint child)
+
+uint
+leaf_size()
 {
-   uint offset = ir_id_to_offset(child);
-   return ir_id_to_type(child) == vk_ir_node_internal
-      ? args.leaf_bound + (offset - args.root_offset) / SIZEOF(vk_ir_box_node)
-      : offset / (args.root_offset / args.leaf_bound);
+   return args.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR
+             ? SIZEOF(vk_ir_instance_node)
+             : args.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR
+                ? SIZEOF(vk_ir_aabb_node) : SIZEOF(vk_ir_triangle_node);
 }
-void main()
+
+/* The retained level table: counts, then fill cursors. */
+uint64_t
+level_table(uint64_t retained, uint jobs)
 {
-   uint id = gl_GlobalInvocationID.x;
-   uint header_words = SIZEOF(vk_ir_header) / 4;
-   uint internal_bound = args.node_bound - args.leaf_bound;
-   if (args.phase <= 1) {
-      uint64_t retained_base = args.retained;
-      if (args.phase == 1 && args.metadata != 0ul) {
-         libagx_gen1_as_metadata live =
-            DEREF(REF(libagx_gen1_as_metadata)(args.metadata));
-         retained_base = live.native_base + live.retained_offset;
+   return retained + uint64_t(HK_AGX3_RETAINED_LEVELS(jobs, args.leaf_bound));
+}
+
+/* Index into the retained inner jobs by level of the first of `level`. */
+uint
+level_start(uint64_t table, uint level)
+{
+   uint start = 0;
+   for (uint l = 0; l < level && l < HK_AGX3_REFIT_LEVELS; ++l)
+      start += DEREF(INDEX(uint32_t, table, l));
+   return start;
+}
+
+/* Plan node of the IR node at `offset` (an IR leaf or internal node). */
+uint
+node_index(uint offset)
+{
+   return offset < args.root_offset
+             ? offset / leaf_size()
+             : args.leaf_bound + (offset - args.root_offset) / SIZEOF(vk_ir_box_node);
+}
+
+void
+save(uint id)
+{
+   hk_agx3_plan_state plan = DEREF(REF(hk_agx3_plan_state)(args.plan_state));
+   uint jobs = plan.job_count;
+   REF(hk_agx3_retained_state) state = REF(hk_agx3_retained_state)(args.retained);
+   uint64_t records = args.retained + uint64_t(HK_AGX3_RETAINED_STATE_BYTES);
+   uint64_t offsets = records + uint64_t(jobs) * uint64_t(HK_AGX3_RETAINED_JOB_BYTES);
+   if (id == 0) {
+      uint root = args.leaf_bound + DEREF(args.header).ir_internal_node_count - 1u;
+      hk_agx3_retained_state saved;
+      saved.job_count = jobs;
+      saved.root_job = jobs != 0u && root < args.node_bound
+         ? DEREF(INDEX(hk_agx3_plan_node, args.plan_nodes, root)).job : 0xffffffffu;
+      saved.end_slot = plan.next_slot;
+      saved.depth = plan.depth;
+      saved.deepest_level = 0;
+      saved.active_leaves = DEREF(args.header).active_leaf_count;
+      for (uint i = 0; i < 2; ++i)
+         saved.reserved[i] = 0;
+      DEREF(state) = saved;
+      uint64_t table = level_table(args.retained, jobs);
+      for (uint i = 0; i < HK_AGX3_REFIT_LEVELS * 2u; ++i)
+         DEREF(INDEX(uint32_t, table, i)) = 0u;
+   }
+   if (id < args.leaf_bound)
+      DEREF(INDEX(uint32_t, args.retained + uint64_t(HK_AGX3_RETAINED_SLOTS(jobs, args.leaf_bound)),
+                  id)) = ~0u;
+   if (id >= jobs)
+      return;
+   hk_agx3_leaf_job job = DEREF(INDEX(hk_agx3_leaf_job, args.plan_jobs, id));
+   uint count = min(job.child_count, 8u);
+   REF(hk_agx3_retained_job) record = INDEX(hk_agx3_retained_job, records, id);
+   uint first;
+   if (job.node_kind == HK_AGX3_NODE_INNER) {
+      first = atomicAdd(DEREF(REF(hk_agx3_plan_state)(args.plan_state)).saved_positions, count);
+      for (uint c = 0; c < count; ++c) {
+         hk_agx3_plan_node node = DEREF(INDEX(hk_agx3_plan_node, args.plan_nodes,
+                                              node_index(job.triangle_offsets[c])));
+         uint child = node.job;
+         if (child >= jobs)
+            continue;
+         REF(hk_agx3_retained_job) child_record = INDEX(hk_agx3_retained_job, records, child);
+         DEREF(child_record).parent = id;
+         DEREF(child_record).position = first + c;
+         /* Inner jobs sit at depths of whole levels. */
+         DEREF(INDEX(uint32_t, args.status, child)) =
+            (node.depth & ~(HK_AGX3_PLAN_PAIR | HK_AGX3_PLAN_PAIR_OPAQUE)) / 3u;
       }
-      /* Preserve the actual binary topology, not unused internal capacity.
-       * Restore counts come from the retained GPU header after copy/rebase. */
-      uint internal_count = args.phase == 0 ? DEREF(args.header).ir_internal_node_count
-         : DEREF(REF(vk_ir_header)(retained_base)).ir_internal_node_count;
-      uint words = header_words + internal_count * (SIZEOF(vk_ir_box_node) / 4);
-      if (id < words) {
-         uint64_t addr = id < header_words ? uint64_t(args.header) + uint64_t(id)*4ul
-            : args.intermediate_bvh + uint64_t(args.root_offset) + uint64_t(id-header_words)*4ul;
-         REF(uint32_t) retained = REF(uint32_t)(retained_base + uint64_t(id)*4ul);
-         if (args.phase == 0) DEREF(retained) = DEREF(REF(uint32_t)(addr));
-         else {
-            uint value = DEREF(retained);
-            if (id < 3) value = 0x7fffffffu;
-            else if (id < 6) value = 0x80000000u;
-            else if (id == 6) value = 0;
-            DEREF(REF(uint32_t)(addr)) = value;
+   } else {
+      first = atomicAdd(DEREF(REF(hk_agx3_plan_state)(args.plan_state)).saved_offsets, count);
+      for (uint c = 0; c < count; ++c)
+         DEREF(INDEX(uint32_t, offsets, first + c)) = job.triangle_offsets[c];
+   }
+   DEREF(record).root_slot = job.root_slot;
+   DEREF(record).child_slot = job.child_slot;
+   DEREF(record).kind_count = job.child_count | (job.node_kind << 4);
+   DEREF(record).first = first;
+   if (job.root_slot == 0u) {
+      DEREF(record).parent = 0xffffffffu;
+      DEREF(record).position = jobs - 1u;
+      DEREF(INDEX(uint32_t, args.status, id)) = 0u;
+   }
+}
+
+/* Give each inner job its level and count the inner jobs of each level;
+ * give each triangle leaf its slot, the index of its child offset. */
+void
+count_levels(uint id)
+{
+   REF(hk_agx3_retained_state) state = REF(hk_agx3_retained_state)(args.retained);
+   uint jobs = DEREF(state).job_count;
+   uint64_t records = args.retained + uint64_t(HK_AGX3_RETAINED_STATE_BYTES);
+   uint level = 0;
+   bool inner = false;
+   if (id < jobs) {
+      REF(hk_agx3_retained_job) record = INDEX(hk_agx3_retained_job, records, id);
+      uint kind_count = DEREF(record).kind_count;
+      inner = (kind_count >> 4) == HK_AGX3_NODE_INNER;
+      if ((kind_count >> 4) == HK_AGX3_NODE_LEAF) {
+         uint first = DEREF(record).first;
+         uint64_t offsets = args.retained + uint64_t(HK_AGX3_RETAINED_OFFSETS(jobs));
+         uint64_t slots = args.retained + uint64_t(HK_AGX3_RETAINED_SLOTS(jobs, args.leaf_bound));
+         for (uint c = 0; c < min(kind_count & 15u, 8u); ++c) {
+            uint leaf = DEREF(INDEX(uint32_t, offsets, first + c)) / leaf_size();
+            if (leaf < args.leaf_bound)
+               DEREF(INDEX(uint32_t, slots, leaf)) = first + c;
          }
       }
-      if (args.phase == 1 && id < args.node_bound) {
-         DEREF(INDEX(refit_node, args.parents, id)).parent = 0xffffffffu;
-         DEREF(INDEX(refit_node, args.parents, id)).arrivals = 0;
+      if (inner) {
+         level = DEREF(INDEX(uint32_t, args.status, id));
+         DEREF(record).kind_count = kind_count | (level << 8);
+         if (level < HK_AGX3_REFIT_LEVELS)
+            atomicAdd(DEREF(INDEX(uint32_t, level_table(args.retained, jobs), level)), 1u);
       }
+   }
+   uint deepest = subgroupMax(level);
+   if (subgroupElect() && deepest != 0u)
+      atomicMax(DEREF(state).deepest_level, deepest);
+}
+
+/* List the inner jobs of the first HK_AGX3_REFIT_LEVELS levels by level. */
+void
+list_levels(uint id)
+{
+   uint jobs = DEREF(REF(hk_agx3_retained_state)(args.retained)).job_count;
+   if (id >= jobs)
+      return;
+   uint64_t records = args.retained + uint64_t(HK_AGX3_RETAINED_STATE_BYTES);
+   uint kind_count = DEREF(INDEX(hk_agx3_retained_job, records, id)).kind_count;
+   uint level = kind_count >> 8;
+   if (((kind_count >> 4) & 15u) != HK_AGX3_NODE_INNER || level >= HK_AGX3_REFIT_LEVELS)
       return;
+   uint64_t table = level_table(args.retained, jobs);
+   uint index = level_start(table, level) +
+      atomicAdd(DEREF(INDEX(uint32_t, table, HK_AGX3_REFIT_LEVELS + level)), 1u);
+   uint64_t list = table + uint64_t(HK_AGX3_RETAINED_LEVEL_BYTES);
+   DEREF(INDEX(uint32_t, list, index)) = id;
+}
+
+void
+prepare(uint id)
+{
+   uint64_t source = args.retained;
+   if (args.metadata != 0ul) {
+      libagx_gen1_as_metadata live = DEREF(REF(libagx_gen1_as_metadata)(args.metadata));
+      source = live.native_base + live.retained_offset;
    }
-   if (args.phase == 2) {
-      if (id >= DEREF(args.header).ir_internal_node_count) return;
-      REF(vk_ir_box_node) node = REF(vk_ir_box_node)(args.intermediate_bvh +
-         uint64_t(args.root_offset) + uint64_t(id)*uint64_t(SIZEOF(vk_ir_box_node)));
-      for (uint c = 0; c < 2; ++c) {
-         uint child = DEREF(node).children[c];
-         if (child != VK_BVH_INVALID_NODE)
-            DEREF(INDEX(refit_node, args.parents, source_index(child))).parent = args.leaf_bound + id;
-      }
+   hk_agx3_retained_state saved = DEREF(REF(hk_agx3_retained_state)(source));
+   uint jobs = saved.job_count;
+   if (args.copy != 0u) {
+      /* An update into another structure keeps it updatable: copy the state,
+       * the records and every child offset the leaf jobs may use. */
+      uint words = HK_AGX3_RETAINED_BYTES(jobs, args.leaf_bound) / 4u;
+      if (id < words)
+         DEREF(INDEX(uint32_t, args.retained, id)) = DEREF(INDEX(uint32_t, source, id));
+   }
+   if (id < jobs)
+      DEREF(INDEX(uint32_t, args.counters, id)) = 0u;
+
+   bool instances = args.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR;
+   uint64_t transforms = uint64_t(saved.end_slot) * 64ul;
+   uint64_t pointers = transforms + 64ul * uint64_t(args.leaf_bound);
+   /* Instance encoding fills the pointer of every active instance. */
+   if (instances && jobs != 0u && id < args.leaf_bound) {
+      uint64_t entry = args.output_bvh + pointers + uint64_t(id) * 16ul;
+      DEREF(REF(uint64_t)(entry)) = 0ul;
+      DEREF(REF(uint64_t)(entry + 8ul)) = 0ul;
+   }
+   if (id != 0)
       return;
+   hk_agx3_refit_state refit;
+   refit.dispatch[0] = (jobs + HK_AGX3_NODE_JOBS_PER_GROUP - 1u) / HK_AGX3_NODE_JOBS_PER_GROUP;
+   refit.dispatch[1] = 1u;
+   refit.dispatch[2] = 1u;
+   refit.job_count = jobs;
+   refit.retained = args.copy != 0u ? args.retained : source;
+   /* Propagation only for inner jobs past the levels. */
+   refit.propagate[0] = saved.deepest_level >= args.level
+      ? (jobs + HK_AGX3_REFIT_GROUP_SIZE - 1u) / HK_AGX3_REFIT_GROUP_SIZE : 0u;
+   refit.propagate[1] = 1u;
+   refit.propagate[2] = 1u;
+   refit.end_slot = saved.end_slot;
+   DEREF(REF(hk_agx3_refit_state)(args.refit)) = refit;
+   /* Workgroups of HK_AGX3_REFIT_GROUP_SIZE / 8 jobs for each level the
+    * update encodes by level, all from this thread: a small structure runs
+    * fewer threads than there are levels, and a level left unwritten would
+    * dispatch whatever the scratch held. */
+   if (args.levels != 0ul) {
+      uint64_t table = level_table(source, jobs);
+      const uint per_group = HK_AGX3_REFIT_GROUP_SIZE / 8u;
+      for (uint l = 0; l < args.level && l < HK_AGX3_REFIT_LEVELS; ++l) {
+         uint count = min(DEREF(INDEX(uint32_t, table, l)), jobs);
+         uint64_t dispatch = args.levels + uint64_t(l) * 12ul;
+         store32(dispatch, (count + per_group - 1u) / per_group);
+         store32(dispatch + 4ul, 1u);
+         store32(dispatch + 8ul, 1u);
+      }
    }
-   if (id >= args.leaf_bound) return;
-   uint parent = DEREF(INDEX(refit_node, args.parents, id)).parent;
-   for (uint steps = 0; parent != 0xffffffffu && steps < args.node_bound; ++steps) {
-      REF(vk_ir_box_node) node = REF(vk_ir_box_node)(args.intermediate_bvh +
-         uint64_t(args.root_offset) + uint64_t(parent-args.leaf_bound)*uint64_t(SIZEOF(vk_ir_box_node)));
-      uint expected = uint(DEREF(node).children[0] != VK_BVH_INVALID_NODE) +
-                      uint(DEREF(node).children[1] != VK_BVH_INVALID_NODE);
-      uint arrived = atomicAdd(DEREF(INDEX(refit_node, args.parents, parent)).arrivals,
-         1, gl_ScopeDevice, gl_StorageSemanticsBuffer,
+
+   uint active_leaves = args.no_leaves != 0u ? saved.active_leaves
+                                             : DEREF(args.header).active_leaf_count;
+   bool empty = jobs == 0u || active_leaves == 0u;
+   hk_agx3_envelope_job job;
+   job.bounds = REF(vk_ir_node)(args.bounds + uint64_t(jobs != 0u ? jobs - 1u : 0u) *
+                                uint64_t(SIZEOF(vk_aabb)));
+   job.output_bvh = args.output_bvh;
+   job.end_slot = saved.end_slot;
+   job.primitive_count = empty ? 0u : active_leaves;
+   job.node_count = jobs;
+   job.depth = saved.depth;
+   job.opaque_build_word = empty ? HK_AGX3_PLAN_EMPTY : 0u;
+   job.resource_id = args.resource_id;
+   job.geometry_type = args.geometry_type;
+   job.leaf_bound = args.leaf_bound;
+   DEREF(REF(hk_agx3_envelope_job)(args.envelope)) = job;
+}
+
+/* From leaf job `id`, whose bounds refit encoding left at its position:
+ * arrive at the parent; while this was the parent's last child, encode the
+ * parent's node from its children's bounds, leave the parent's bounds at
+ * its position and arrive at its parent in turn. The arrival orders the
+ * bounds stores of every child before the reads of the last. Parents of
+ * the levels encoded by level are left to those. */
+void
+propagate(uint id)
+{
+   hk_agx3_refit_state refit = DEREF(REF(hk_agx3_refit_state)(args.refit));
+   if (id >= refit.job_count)
+      return;
+   uint64_t records = refit.retained + uint64_t(HK_AGX3_RETAINED_STATE_BYTES);
+   hk_agx3_retained_job leaf = DEREF(INDEX(hk_agx3_retained_job, records, id));
+   if (((leaf.kind_count >> 4) & 15u) == HK_AGX3_NODE_INNER)
+      return;
+   uint parent = leaf.parent;
+   for (uint steps = 0; parent != 0xffffffffu && steps < 64u; ++steps) {
+      hk_agx3_retained_job job = DEREF(INDEX(hk_agx3_retained_job, records, parent));
+      if ((job.kind_count >> 8) < args.level)
+         return;
+      uint count = job.kind_count & 15u;
+      uint arrived = atomicAdd(DEREF(INDEX(uint32_t, args.counters, parent)), 1u,
+         gl_ScopeDevice, gl_StorageSemanticsBuffer,
          gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
-      if (arrived + 1 != expected) break;
-      vk_aabb bounds; bounds.min = vec3(INFINITY); bounds.max = vec3(-INFINITY);
-      for (uint c = 0; c < 2; ++c) {
-         uint child = DEREF(node).children[c];
-         if (child == VK_BVH_INVALID_NODE) continue;
-         REF(vk_ir_node) src = REF(vk_ir_node)(args.intermediate_bvh + uint64_t(ir_id_to_offset(child)));
-         bounds.min = min(bounds.min, DEREF(src).aabb.min);
-         bounds.max = max(bounds.max, DEREF(src).aabb.max);
+      if (arrived + 1u != count)
+         return;
+      vec3 child_lo[8], child_hi[8];
+      vec3 lo = vec3(INFINITY), hi = vec3(-INFINITY);
+      for (uint c = 0; c < count && c < 8u; ++c) {
+         child_lo[c] = DEREF(INDEX(vk_aabb, args.bounds, job.first + c)).min;
+         child_hi[c] = DEREF(INDEX(vk_aabb, args.bounds, job.first + c)).max;
+         lo = min(lo, child_lo[c]);
+         hi = max(hi, child_hi[c]);
+      }
+      vec3 base;
+      uvec3 exponent;
+      bool ok = count >= 1u && count <= 8u && fit_root(lo, hi, base, exponent);
+      uint64_t words[8];
+      for (uint t = 0; t < 8u && ok; ++t)
+         words[t] = root_word(t, count, HK_AGX3_NODE_INNER, 0u, job.child_slot,
+                              args.slot_bias, base, exponent, child_lo[t], child_hi[t], ok);
+      if (ok) {
+         uint64_t node = args.output_bvh + uint64_t(job.root_slot) * 64ul;
+         for (uint t = 0; t < 8u; ++t)
+            store_word64(node + uint64_t(t) * 8ul, words[t]);
+      } else {
+         lo = hi = HK_AGX3_FAILED_BOUNDS;
+      }
+      store_aabb(uint64_t(INDEX(vk_aabb, args.bounds, job.position)), lo, hi);
+      DEREF(INDEX(uint32_t, args.status, parent)) = ok ? HK_AGX3_LEAF_SUCCESS : HK_AGX3_LEAF_RANGE;
+      parent = job.parent;
+   }
+}
+
+/* Inner jobs of level args.level, whose children are encoded: eight lanes
+ * per job, a lane per child for its bounds and root word. Every lane
+ * reaches every subgroup operation. A job whose children's bounds do not
+ * fit, as when a child failed, fails in turn. */
+void
+inner(uint id)
+{
+   uint slot = id / 8u, lane = id % 8u;
+   hk_agx3_refit_state refit = DEREF(REF(hk_agx3_refit_state)(args.refit));
+   uint64_t records = refit.retained + uint64_t(HK_AGX3_RETAINED_STATE_BYTES);
+   uint64_t table = level_table(refit.retained, refit.job_count);
+   bool live = slot < min(DEREF(INDEX(uint32_t, table, args.level)), refit.job_count);
+   uint index = 0, count = 0;
+   hk_agx3_retained_job job;
+   vec3 my_lo = vec3(INFINITY), my_hi = vec3(-INFINITY);
+   if (live) {
+      uint64_t list = table + uint64_t(HK_AGX3_RETAINED_LEVEL_BYTES);
+      index = DEREF(INDEX(uint32_t, list, level_start(table, args.level) + slot));
+      job = DEREF(INDEX(hk_agx3_retained_job, records, index));
+      count = job.kind_count & 15u;
+      if (lane < count) {
+         my_lo = DEREF(INDEX(vk_aabb, args.bounds, job.first + lane)).min;
+         my_hi = DEREF(INDEX(vk_aabb, args.bounds, job.first + lane)).max;
+      }
+   }
+   vec3 lo = subgroupClusteredMin(my_lo, 8u);
+   vec3 hi = subgroupClusteredMax(my_hi, 8u);
+   vec3 base;
+   uvec3 exponent;
+   bool ok = live && count >= 1u && count <= 8u && fit_root(lo, hi, base, exponent);
+   uint64_t word = 0ul;
+   if (ok)
+      word = root_word(lane, count, HK_AGX3_NODE_INNER, 0u, job.child_slot, args.slot_bias,
+                       base, exponent, my_lo, my_hi, ok);
+   bool bad = subgroupClusteredOr(ok ? 0u : 1u, 8u) != 0u;
+   if (!live)
+      return;
+   if (bad) {
+      if (lane == 0u) {
+         DEREF(INDEX(uint32_t, args.status, index)) = HK_AGX3_LEAF_RANGE;
+         store_aabb(uint64_t(INDEX(vk_aabb, args.bounds, job.position)),
+                    HK_AGX3_FAILED_BOUNDS, HK_AGX3_FAILED_BOUNDS);
       }
-      DEREF(node).base.aabb = bounds;
-      parent = DEREF(INDEX(refit_node, args.parents, parent)).parent;
+      return;
+   }
+   store_word64(args.output_bvh + uint64_t(job.root_slot) * 64ul + uint64_t(lane) * 8ul, word);
+   if (lane == 0u) {
+      store_aabb(uint64_t(INDEX(vk_aabb, args.bounds, job.position)), lo, hi);
+      DEREF(INDEX(uint32_t, args.status, index)) = HK_AGX3_LEAF_SUCCESS;
    }
 }
+
+void
+main()
+{
+   uint id = gl_GlobalInvocationID.x;
+   /* Built once per update phase that runs per level or per job
+    * (HK_AGX3_REFIT_ONLY_PHASE), so those carry only their own registers,
+    * and once for the other phases. */
+#ifdef HK_AGX3_REFIT_ONLY_PHASE
+   const uint phase = HK_AGX3_REFIT_ONLY_PHASE;
+#else
+   const uint phase = args.phase;
+#endif
+   if (phase == HK_AGX3_REFIT_SAVE)
+      save(id);
+   else if (phase == HK_AGX3_REFIT_PREPARE)
+      prepare(id);
+   else if (phase == HK_AGX3_REFIT_PROPAGATE)
+      propagate(id);
+   else if (phase == HK_AGX3_REFIT_COUNT_LEVELS)
+      count_levels(id);
+   else if (phase == HK_AGX3_REFIT_LIST_LEVELS)
+      list_levels(id);
+   else
+      inner(id);
+}
diff --git a/src/asahi/vulkan/bvh/agx3_refit_leaf.comp b/src/asahi/vulkan/bvh/agx3_refit_leaf.comp
new file mode 100644
index 000000000000..9b2bc8401f1a
--- /dev/null
+++ b/src/asahi/vulkan/bvh/agx3_refit_leaf.comp
@@ -0,0 +1,72 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * An update's leaf pass for triangles, in place of generating IR leaves: a
+ * thread per primitive of one geometry reads its triangle as leaf
+ * generation does and leaves it at its leaf's slot (see
+ * hk_agx3_retained_state), so that refit encoding reads each leaf job's
+ * children together rather than gathering IR leaves. Reads follow the
+ * application's index order; only the stores scatter.
+ */
+#version 460
+#extension GL_GOOGLE_include_directive : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference2 : require
+#extension GL_EXT_scalar_block_layout : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
+#extension GL_EXT_shader_explicit_arithmetic_types_float32 : require
+#include "hk_build_interface.h"
+#include "agx3_node.h"
+
+/* Inactive triangles have NaN vertices, as for leaf generation. */
+#define SpvCapabilitySignedZeroInfNanPreserve 4466
+#define SpvExecutionModeSignedZeroInfNanPreserve 4461
+spirv_execution_mode(extensions = ["SPV_KHR_float_controls"],
+                     capabilities = [SpvCapabilitySignedZeroInfNanPreserve],
+                     SpvExecutionModeSignedZeroInfNanPreserve, 32);
+
+layout(local_size_x = 64) in;
+layout(push_constant) uniform CONSTS { hk_agx3_refit_leaf_args args; };
+
+void
+main()
+{
+   uint id = gl_GlobalInvocationID.x;
+   if (id >= args.primitive_count)
+      return;
+   hk_agx3_refit_state refit = DEREF(REF(hk_agx3_refit_state)(args.refit));
+   uint64_t slots = refit.retained +
+                    uint64_t(HK_AGX3_RETAINED_SLOTS(refit.job_count, args.leaf_bound));
+   uint slot = DEREF(INDEX(uint32_t, slots, args.geometry.first_id + id));
+   if (slot == ~0u)
+      return;
+
+   vk_bvh_geometry_data geom = args.geometry;
+   triangle_vertices vertices =
+      load_vertices(geom.data, load_indices(geom.indices, geom.index_format, id),
+                    geom.vertex_format, geom.stride);
+   if (geom.transform != 0ul) {
+      mat4 transform = mat4(1.0);
+      for (uint col = 0; col < 4; col++)
+         for (uint row = 0; row < 3; row++)
+            transform[col][row] = DEREF(INDEX(float, geom.transform, col + row * 4));
+      for (uint i = 0; i < 3; i++)
+         vertices.vertex[i] = transform * vertices.vertex[i];
+   }
+   /* An updatable structure keeps an inactive triangle as zero vertices
+    * (VK_BUILD_FLAG_ALWAYS_ACTIVE). */
+   bool inactive = false;
+   for (uint i = 0; i < 3; i++)
+      inactive = inactive || any(isnan(vertices.vertex[i]));
+   uvec3 v[3];
+   for (uint i = 0; i < 3; i++)
+      v[i] = inactive ? uvec3(0u) : floatBitsToUint(vertices.vertex[i].xyz);
+
+   uint64_t triangle = uint64_t(INDEX(hk_agx3_refit_triangle, args.triangles, slot));
+   store128(triangle, uvec4(v[0], v[1].x));
+   store128(triangle + 16ul, uvec4(v[1].yz, v[2].xy));
+   store128(triangle + 32ul, uvec4(v[2].z, geom.geometry_id, id, 0u));
+}
diff --git a/src/asahi/vulkan/bvh/hk_build_interface.h b/src/asahi/vulkan/bvh/hk_build_interface.h
index d305c4a20108..c9832381532a 100644
--- a/src/asahi/vulkan/bvh/hk_build_interface.h
+++ b/src/asahi/vulkan/bvh/hk_build_interface.h
@@ -38,9 +38,161 @@ TYPE(hk_bvh_header, 4);
 TYPE(libagx_gen1_as_metadata, 8);
 #endif
 
-/* Retained binary topology for native refits. Only internal nodes and header
- * persist; updated leaf data is read directly from application buffers. */
+/* Native refits (updates).
+ *
+ * A full build that allows updates keeps its encoding jobs after the image,
+ * at the retained offset: an hk_agx3_retained_state, one hk_agx3_retained_job
+ * per job, then the IR offsets of the children of every leaf job,
+ * concatenated. An update keeps the topology: it regenerates the IR leaves
+ * and re-encodes each leaf job from them, then each inner job from its
+ * children's bounds: refit encoding (agx3_leaf.comp built with
+ * HK_AGX3_REFIT) encodes the leaf jobs, then agx3_refit.comp the inner
+ * jobs. It reads neither IR internal nodes nor a plan.
+ *
+ * A child job hands its bounds to its parent at its position: the parent's
+ * first position plus the child's index. Positions number the children of
+ * all inner jobs, so they run to job_count - 2; the root's is job_count - 1.
+ *
+ * Inner jobs have levels, a third of the binary depth of the job's IR node:
+ * an inner job's inner children are one level deeper. After the child
+ * offsets (room for every leaf) come the inner jobs of the first
+ * HK_AGX3_REFIT_LEVELS levels by level: a count and a fill cursor per
+ * level, then the jobs of level 0, of level 1, and so on. An update encodes
+ * those a level per dispatch, deepest first, with no arrivals; refit
+ * propagation encodes any deeper ones.
+ *
+ * Last come the slots of the leaves of a triangle structure: for each IR
+ * leaf, the index of its child offset, or ~0 for a leaf in no leaf job. A
+ * triangle update may skip generating IR leaves: its leaf pass leaves each
+ * triangle (hk_agx3_refit_triangle) at its slot, so that each leaf job
+ * finds its children together.
+ */
+struct hk_agx3_retained_state {
+   uint32_t job_count;
+   uint32_t root_job;
+   uint32_t end_slot;
+   uint32_t depth;
+   uint32_t deepest_level;
+   /* The full build's active leaves: with updatable structures every
+    * primitive is active (VK_BUILD_FLAG_ALWAYS_ACTIVE), so an update has as
+    * many. */
+   uint32_t active_leaves;
+   uint32_t reserved[2];
+};
+
+struct hk_agx3_retained_job {
+   uint32_t root_slot;
+   uint32_t child_slot;
+   uint32_t kind_count; /* child count | node kind << 4 | inner level << 8 */
+   /* A leaf job's first child offset; an inner job's first position. */
+   uint32_t first;
+   uint32_t parent; /* 0xffffffff for the root */
+   uint32_t position;
+};
+
+#ifdef VULKAN
+TYPE(hk_agx3_retained_state, 4);
+TYPE(hk_agx3_retained_job, 4);
+#endif
+
+#define HK_AGX3_RETAINED_STATE_BYTES 32u
+#define HK_AGX3_RETAINED_JOB_BYTES 24u
+/* Levels of inner jobs an update encodes level by level. */
+#define HK_AGX3_REFIT_LEVELS 16u
+#define HK_AGX3_RETAINED_LEVEL_BYTES (HK_AGX3_REFIT_LEVELS * 8u)
+/* Where the retained jobs of a structure with `jobs` jobs and `leaves`
+ * leaves keep their child offsets, level table (and the inner jobs by level
+ * after it) and leaf slots, and their size. */
+#define HK_AGX3_RETAINED_OFFSETS(jobs)                                         \
+   (HK_AGX3_RETAINED_STATE_BYTES + (jobs) * HK_AGX3_RETAINED_JOB_BYTES)
+#define HK_AGX3_RETAINED_LEVELS(jobs, leaves)                                  \
+   (HK_AGX3_RETAINED_OFFSETS(jobs) + (leaves) * 4u)
+#define HK_AGX3_RETAINED_SLOTS(jobs, leaves)                                   \
+   (HK_AGX3_RETAINED_LEVELS(jobs, leaves) + HK_AGX3_RETAINED_LEVEL_BYTES + (jobs) * 4u)
+#define HK_AGX3_RETAINED_BYTES(jobs, leaves)                                   \
+   (HK_AGX3_RETAINED_SLOTS(jobs, leaves) + (leaves) * 4u)
+
+/* Refit phases. A full build that allows updates saves its jobs, then
+ * counts its inner jobs by level and lists them. An update prepares (after
+ * its leaves) to encode from them. After refit encoding has encoded the
+ * leaf jobs, propagation encodes the inner jobs of the levels past those
+ * the update encodes by level: each leaf job below them arrives at its
+ * parent, and the last child of an inner job to arrive encodes that job's
+ * node and arrives at its parent in turn. Then the update encodes the
+ * inner jobs of one level per dispatch, deepest first (inner). */
+#define HK_AGX3_REFIT_SAVE 0u
+#define HK_AGX3_REFIT_PREPARE 1u
+#define HK_AGX3_REFIT_PROPAGATE 2u
+#define HK_AGX3_REFIT_COUNT_LEVELS 3u
+#define HK_AGX3_REFIT_LIST_LEVELS 4u
+#define HK_AGX3_REFIT_INNER 5u
+
+/* Written by the prepare phase in encode scratch, read by refit encoding. */
+struct hk_agx3_refit_state {
+   /* Refit encoding dispatch (VkDispatchIndirectCommand, workgroups of
+    * HK_AGX3_NODE_JOBS_PER_GROUP jobs). */
+   uint32_t dispatch[3];
+   uint32_t job_count;
+   /* Retained jobs this update encodes from. */
+   uint64_t retained;
+   /* Refit propagation dispatch, one thread per job. */
+   uint32_t propagate[3];
+   uint32_t end_slot;
+};
+
+#define HK_AGX3_REFIT_PROPAGATE_DISPATCH 24u /* bytes into hk_agx3_refit_state */
+#define HK_AGX3_REFIT_GROUP_SIZE 64u
+
+#ifdef VULKAN
+TYPE(hk_agx3_refit_state, 8);
+#endif
+
 struct hk_agx3_refit_args {
+   /* This structure's retained jobs: what the save phase writes, and where
+    * the prepare phase copies the source's when updating into another
+    * structure. */
+   VOID_REF retained;
+   /* Prepare: the source's metadata, which finds its retained jobs. */
+   VOID_REF metadata;
+   VOID_REF intermediate_bvh;
+   REF(vk_ir_header) header;
+   /* Save: the plan in encode scratch. */
+   VOID_REF plan_nodes;
+   VOID_REF plan_jobs;
+   VOID_REF plan_state;
+   /* Prepare: refit state, arrival counters, child bounds and the envelope
+    * job, all in encode scratch. */
+   VOID_REF refit;
+   VOID_REF counters;
+   VOID_REF bounds;
+   VOID_REF envelope;
+   VOID_REF output_bvh;
+   uint32_t root_offset;
+   uint32_t leaf_bound;
+   uint32_t node_bound;
+   uint32_t phase;
+   uint32_t resource_id;
+   uint32_t geometry_type;
+   /* Prepare: the source is another structure. */
+   uint32_t copy;
+   /* Propagate and inner: as for node encoding. */
+   uint32_t slot_bias;
+   /* Propagate and inner: as for node encoding. Save: each job's level, in
+    * the node statuses, which node encoding has finished with. */
+   REF(uint32_t) status;
+   /* Prepare: the level dispatches (VkDispatchIndirectCommands, workgroups
+    * of HK_AGX3_REFIT_GROUP_SIZE / 8 jobs) to write. */
+   VOID_REF levels;
+   /* Prepare and propagate: the levels the update encodes by level; inner:
+    * the level to encode. */
+   uint32_t level;
+   /* Prepare: the update generated no IR leaves, so no active leaf count. */
+   uint32_t no_leaves;
+};
+
+/* The separate M5 backend keeps its qualified IR-topology refit protocol.
+ * GEN1 uses hk_agx3_refit_args and the current retained-job protocol. */
+struct hk_m5_refit_args {
    VOID_REF retained;
    VOID_REF metadata;
    VOID_REF intermediate_bvh;
@@ -50,6 +202,49 @@ struct hk_agx3_refit_args {
    uint32_t leaf_bound;
    uint32_t node_bound;
    uint32_t phase;
+   VOID_REF plan_nodes;
+   VOID_REF plan_jobs;
+   VOID_REF plan_state;
+};
+
+/* Push constants of refit encoding. */
+struct hk_agx3_refit_encode_args {
+   VOID_REF refit;
+   VOID_REF bounds;
+   VOID_REF intermediate_bvh;
+   VOID_REF output_bvh;
+   REF(uint32_t) status;
+   uint32_t slot_bias;
+   uint32_t leaf_bound;
+   /* Nonzero when the update's leaf pass left its triangles at their slots
+    * instead of generating IR leaves: triangle leaf jobs read their children
+    * from these hk_agx3_refit_triangles at their child offsets' indices. */
+   VOID_REF triangles;
+};
+
+/* A triangle as an update's leaf pass leaves it, at its leaf's slot: its
+ * vertices, zero for an inactive triangle as updatable structures keep it,
+ * and its geometry and primitive IDs, as an IR triangle leaf has them. */
+struct hk_agx3_refit_triangle {
+   float coords[3][3];
+   uint32_t geometry_id_and_flags;
+   uint32_t triangle_id;
+   uint32_t padding;
+};
+
+#ifdef VULKAN
+TYPE(hk_agx3_refit_triangle, 4);
+#endif
+
+/* Push constants of an update's leaf pass, one dispatch per geometry with
+ * a thread per primitive. */
+struct hk_agx3_refit_leaf_args {
+   /* The refit state, which finds the retained jobs. */
+   VOID_REF refit;
+   REF(hk_agx3_refit_triangle) triangles;
+   vk_bvh_geometry_data geometry;
+   uint32_t leaf_bound;
+   uint32_t primitive_count;
 };
 
 
@@ -77,8 +272,8 @@ struct hk_encode_args {
  * so the build planner can select/reorder leaves without copying vertices.
  * Each job owns disjoint root and child regions. Slots are relative to its
  * output AS, not the job or the root. The caller supplies the AS metadata.
- * This covers inner nodes and the opaque, geometry-zero 1..8 leaf form;
- * it is not a complete acceleration-structure build implementation. */
+ * This covers inner nodes, triangle leaf clusters, instance records and
+ * procedural bounding-box leaf clusters. */
 struct hk_agx3_leaf_job {
    VOID_REF intermediate_bvh;
    VOID_REF output_bvh;
@@ -98,10 +293,17 @@ struct hk_agx3_leaf_args {
    REF(uint32_t) status;
    uint32_t job_count;
    /* Slots from the traversal base to output_bvh, added to every child
-    * reference (see libagx/gen1_as.h). Zero when output_bvh is the base. */
+    * reference (see libagx/gen1_as.h). Zero when output_bvh is the base.
+    * M5 uses this ABI word for its diagnostic bounding-box record stride. */
    uint32_t slot_bias;
+   /* When nonzero, the number of jobs written by planning (its first word),
+    * which the indirect dispatch rounds up to whole workgroups. */
+   VOID_REF counts;
 };
 
+/* Node jobs per encoding workgroup, eight lanes each. */
+#define HK_AGX3_NODE_JOBS_PER_GROUP 4u
+
 #define HK_AGX3_LEAF_SUCCESS 0
 #define HK_AGX3_LEAF_INVALID_COUNT 1
 #define HK_AGX3_LEAF_NONFINITE 2
@@ -110,6 +312,7 @@ struct hk_agx3_leaf_args {
 #define HK_AGX3_NODE_LEAF 0
 #define HK_AGX3_NODE_INNER 1
 #define HK_AGX3_NODE_INSTANCES 2
+/* Procedural (AABB) leaf cluster: 48-byte native bounding-box records. */
 #define HK_AGX3_NODE_AABBS 3
 
 /* Finish a single opaque triangle geometry after the node jobs. Bounds name
@@ -142,7 +345,8 @@ struct hk_agx3_envelope_args {
    uint32_t build_flags;
    uint32_t node_bound;
    uint32_t keep_retained;
-   uint32_t padding;
+   /* A refit: the prepare phase cleared the instance pointers already. */
+   uint32_t refit;
 };
 
 /* Scratch records for parallel binary-to-wide planning. Phases are separate
@@ -155,14 +359,19 @@ struct hk_agx3_plan_node {
 };
 
 struct hk_agx3_plan_state {
-   /* First three words are VkDispatchIndirectCommand for node encoding. */
+   /* The node encoding dispatch (VkDispatchIndirectCommand, workgroups of
+    * HK_AGX3_NODE_JOBS_PER_GROUP jobs) follows this state, at
+    * HK_AGX3_PLAN_ENCODE_DISPATCH; plan phase 4 writes it. */
    uint32_t job_count;
    uint32_t groups_y;
    uint32_t groups_z;
    uint32_t next_slot;
    uint32_t depth;
    uint32_t error;
-   uint32_t reserved[2];
+   /* Leaf-job child offsets and positions handed out by the refit save
+    * phase (see hk_agx3_retained_job). */
+   uint32_t saved_offsets;
+   uint32_t saved_positions;
 };
 
 #ifdef VULKAN
@@ -185,17 +394,91 @@ struct hk_agx3_plan_args {
    uint32_t phase;
    uint32_t resource_id;
    uint32_t geometry_type;
-   uint32_t reserved;
+   /* GEN1: phases 0..3 were replaced by a restored plan (an update): phase
+    * 4 rebuilds the jobs from the records in nodes. M5 uses this ABI word
+    * for its diagnostic bounding-box record stride, as m5_aabb_stride. */
+   uint32_t plan_restored;
    VOID_REF metadata;
    uint64_t retained_offset;
    uint32_t build_flags;
    uint32_t keep_retained;
+   /* Nonzero to encode sibling triangle leaves that share an edge as one
+    * native record (see hk_agx3_shared_edge); triangle geometry only. */
+   uint32_t pairs;
+   uint32_t padding;
+};
+
+/* Independent ASs share dispatches, not their writable records. The planner
+ * allocates a dense job map after classification; each encoder workgroup
+ * still owns precisely the same native root/child region as the scalar path. */
+struct hk_agx3_batch_job {
+#ifdef VULKAN
+   hk_agx3_plan_args plan;
+#else
+   struct hk_agx3_plan_args plan;
+#endif
+   REF(uint32_t) node_status;
+   REF(uint32_t) envelope_status;
+   uint32_t single_leaf;
+   uint32_t first_job;
+   uint32_t encode_count;
+   uint32_t reserved;
+};
+struct hk_agx3_batch_map {
+   uint32_t build;
+   uint32_t node;
+};
+struct hk_agx3_batch_dispatch {
+   uint32_t x;
+   uint32_t y;
+   uint32_t z;
+   uint32_t reserved;
+};
+#ifdef VULKAN
+TYPE(hk_agx3_batch_job, 8);
+TYPE(hk_agx3_batch_map, 4);
+TYPE(hk_agx3_batch_dispatch, 4);
+#endif
+struct hk_agx3_batch_args {
+   REF(hk_agx3_batch_job) jobs;
+   REF(hk_agx3_batch_map) map;
+   REF(hk_agx3_batch_dispatch) dispatch;
+   uint32_t job_count;
+   uint32_t phase;
 };
 
+/* M5's name for hk_agx3_plan_args::plan_restored: 48, or 64 for the 64-byte
+ * bounding-box record diagnostic. */
+#define m5_aabb_stride plan_restored
+
+#define HK_AGX3_PLAN_ENCODE_DISPATCH 32u /* bytes after hk_agx3_plan_state */
+/* hk_agx3_plan_node.depth flags of an internal node whose two children are
+ * triangles encoded as one record, and whether that record is opaque. */
+#define HK_AGX3_PLAN_PAIR 0x80000000u
+#define HK_AGX3_PLAN_PAIR_OPAQUE 0x40000000u
+/* Set in a leaf job's triangle offset when it names such an internal node. */
+#define HK_AGX3_LEAF_PAIR_OFFSET 1u
 #define HK_AGX3_PLAN_INDEX 1
 #define HK_AGX3_PLAN_CYCLE 2
 #define HK_AGX3_PLAN_ALLOCATION 4
 #define HK_AGX3_PLAN_SHAPE 8
 #define HK_AGX3_PLAN_EMPTY 16
 
+#ifdef VULKAN
+/* A native triangle record holds two triangles that share an edge with
+ * opposite directions, as neighbours in a consistently wound mesh do. Edge
+ * e of a triangle runs from its vertex e to vertex e + 1. Returns
+ * e_a | e_b << 2 for the first such edge pair, or ~0u when there is none. */
+uint
+hk_agx3_shared_edge(vec3 a[3], vec3 b[3])
+{
+   for (uint ea = 0; ea < 3; ++ea)
+      for (uint eb = 0; eb < 3; ++eb)
+         if (all(equal(a[ea], b[(eb + 1u) % 3u])) &&
+             all(equal(a[(ea + 1u) % 3u], b[eb])))
+            return ea | (eb << 2);
+   return ~0u;
+}
+#endif
+
 #endif /* HK_BVH_BUILD_INTERFACE_H */
diff --git a/src/asahi/vulkan/bvh/meson.build b/src/asahi/vulkan/bvh/meson.build
index d44e6753b86a..33d729810201 100644
--- a/src/asahi/vulkan/bvh/meson.build
+++ b/src/asahi/vulkan/bvh/meson.build
@@ -4,6 +4,7 @@
 hk_bvh_include_dir = dir_source_root + '/src/asahi/vulkan/bvh'
 
 hk_bvh_includes = files(
+  'agx3_node.h',
   'hk_build_interface.h',
   '../hk_bvh.h',
 )
@@ -11,12 +12,33 @@ hk_bvh_includes = files(
 # Reassigning bvh_spv follows the same pattern turnip uses: src/vulkan is
 # subdir'd before src/asahi, so the common bvh_spv has already been consumed
 # by then and vk_bvh_include_dir / vk_glsl_shader_preamble are in scope.
+#
+# The native node encoders are chosen by the device's chip
+# (hk_agx3_as_format_for_chip()). GEN1 (G17P and the other native AGX3
+# chips, T6040 included) builds from agx3_{leaf,envelope,plan,refit}.comp,
+# the same sources its refit encoding uses; the hk_agx3_gen1_* programs are
+# those sources. M5 (G17G/G17S) builds from agx3_m5_legacy_*.comp. Each AS
+# format's records are written, refit, traversed and copied only by its own
+# family.
 hk_bvh_spv = []
 foreach s : [['encode.comp', 'hk_encode'], ['write.comp', 'hk_bvh_write'],
-             ['agx3_leaf.comp', 'hk_agx3_leaf'],
-             ['agx3_envelope.comp', 'hk_agx3_envelope'],
-             ['agx3_plan.comp', 'hk_agx3_plan'],
+             ['agx3_leaf.comp', 'hk_agx3_refit_encode', '-DHK_AGX3_REFIT=1'],
+             ['agx3_leaf.comp', 'hk_agx3_refit_encode_triangles', '-DHK_AGX3_REFIT_TRIANGLES=1'],
+             ['agx3_batch_build_leaf.comp', 'hk_agx3_batch_build_leaf'],
              ['agx3_refit.comp', 'hk_agx3_refit'],
+             ['agx3_refit.comp', 'hk_agx3_refit_propagate', '-DHK_AGX3_REFIT_ONLY_PHASE=2u'],
+             ['agx3_refit.comp', 'hk_agx3_refit_inner', '-DHK_AGX3_REFIT_ONLY_PHASE=5u'],
+             ['agx3_refit_leaf.comp', 'hk_agx3_refit_leaf'],
+             ['agx3_m5_legacy_leaf.comp', 'hk_agx3_m5_legacy_leaf'],
+             ['agx3_m5_legacy_envelope.comp', 'hk_agx3_m5_legacy_envelope'],
+             ['agx3_m5_legacy_plan.comp', 'hk_agx3_m5_legacy_plan'],
+             ['agx3_m5_legacy_refit.comp', 'hk_agx3_m5_legacy_refit'],
+             ['agx3_leaf.comp', 'hk_agx3_gen1_leaf'],
+             ['agx3_leaf.comp', 'hk_agx3_gen1_leaf_pairs', '-DHK_AGX3_LEAF_PAIRS=1'],
+             ['agx3_leaf.comp', 'hk_agx3_gen1_leaf_instances', '-DHK_AGX3_LEAF_ONLY=2'],
+             ['agx3_envelope.comp', 'hk_agx3_gen1_envelope'],
+             ['agx3_plan.comp', 'hk_agx3_gen1_plan'],
+             ['agx3_plan.comp', 'hk_agx3_gen1_plan_pairs', '-DHK_AGX3_PLAN_PAIRS=1'],
              ['vertex_table.comp', 'hk_vertex_table']]
   command = [
     prog_glslang, '-V',
@@ -25,6 +47,7 @@ foreach s : [['encode.comp', 'hk_encode'], ['write.comp', 'hk_bvh_write'],
     '-I' + dir_source_root + '/src/asahi/vulkan',
     '--target-env', 'spirv1.5', '-x', '-o', '@OUTPUT@', '@INPUT@',
   ]
+  command += s.length() > 2 ? [s[2]] : []
   command += glslang_quiet
   command += vk_glsl_shader_preamble
 
@@ -36,3 +59,20 @@ foreach s : [['encode.comp', 'hk_encode'], ['write.comp', 'hk_bvh_write'],
     depend_files : [vk_bvh_includes, hk_bvh_includes],
   )
 endforeach
+
+# A dense GPU-built map batches independent native AS node jobs (M5). Reuse
+# each encoder's exact node/envelope arithmetic; only the argument transport
+# differs.
+foreach s : [['agx3_m5_legacy_plan.comp', 'hk_agx3_batch_plan'],
+             ['agx3_m5_legacy_leaf.comp', 'hk_agx3_batch_leaf'],
+             ['agx3_m5_legacy_envelope.comp', 'hk_agx3_batch_envelope']]
+  command = [prog_glslang, '-V', '-DHK_AGX3_BATCH=1',
+    '-I' + vk_bvh_include_dir, '-I' + hk_bvh_include_dir,
+    '-I' + dir_source_root + '/src/asahi/vulkan',
+    '--target-env', 'spirv1.5', '-x', '-o', '@OUTPUT@', '@INPUT@']
+  command += glslang_quiet
+  command += vk_glsl_shader_preamble
+  hk_bvh_spv += custom_target(s[1] + '.spv.h', input : s[0],
+    output : s[1] + '.spv.h', command : command,
+    depend_files : [vk_bvh_includes, hk_bvh_includes])
+endforeach
diff --git a/src/asahi/vulkan/hk_acceleration_structure.c b/src/asahi/vulkan/hk_acceleration_structure.c
index a763186a0433..fc0278073c19 100644
--- a/src/asahi/vulkan/hk_acceleration_structure.c
+++ b/src/asahi/vulkan/hk_acceleration_structure.c
@@ -16,6 +16,8 @@
 
 #include "hk_acceleration_structure.h"
 #include "hk_host_as.h"
+#include "hk_agx3_as_format.h"
+#include "hk_m5_acceleration_structure.h"
 #include "asahi/compiler/agx3_rt_gen1_default.h"
 #include "asahi/compiler/agx3_rt_gen1_triangle.h"
 #include "asahi/compiler/agx3_rt_gen1_aabb.h"
@@ -71,27 +73,67 @@ struct hk_vertex_table_args {
 };
 
 static const uint32_t agx3_node_spv[] = {
-#include "bvh/hk_agx3_leaf.spv.h"
+#include "bvh/hk_agx3_gen1_leaf.spv.h"
+};
+
+static const uint32_t agx3_plan_pairs_spv[] = {
+#include "bvh/hk_agx3_gen1_plan_pairs.spv.h"
+};
+
+static const uint32_t agx3_node_pairs_spv[] = {
+#include "bvh/hk_agx3_gen1_leaf_pairs.spv.h"
 };
 
 static const uint32_t agx3_envelope_spv[] = {
-#include "bvh/hk_agx3_envelope.spv.h"
+#include "bvh/hk_agx3_gen1_envelope.spv.h"
 };
 
 static const uint32_t agx3_plan_spv[] = {
-#include "bvh/hk_agx3_plan.spv.h"
+#include "bvh/hk_agx3_gen1_plan.spv.h"
 };
 
 static const uint32_t agx3_refit_spv[] = {
 #include "bvh/hk_agx3_refit.spv.h"
 };
 
+/* The update phases that run per level or per job, each on its own. */
+static const uint32_t agx3_refit_propagate_spv[] = {
+#include "bvh/hk_agx3_refit_propagate.spv.h"
+};
+static const uint32_t agx3_refit_inner_spv[] = {
+#include "bvh/hk_agx3_refit_inner.spv.h"
+};
+
+static const uint32_t agx3_refit_encode_spv[] = {
+#include "bvh/hk_agx3_refit_encode.spv.h"
+};
+
+/* The refit encoder for triangle structures only. */
+static const uint32_t agx3_refit_encode_triangles_spv[] = {
+#include "bvh/hk_agx3_refit_encode_triangles.spv.h"
+};
+
+/* The node encoder for instance structures only. */
+static const uint32_t agx3_node_instances_spv[] = {
+#include "bvh/hk_agx3_gen1_leaf_instances.spv.h"
+};
+
+static const uint32_t agx3_refit_leaf_spv[] = {
+#include "bvh/hk_agx3_refit_leaf.spv.h"
+};
+
 static void hk_flush_buffer_write_cp(VkCommandBuffer);
 static VkResult hk_host_as_init(struct hk_device *dev);
 static void hk_as_phase_begin(VkCommandBuffer, struct vk_acceleration_structure_build_marker *);
 static void hk_as_phase_end(VkCommandBuffer, struct vk_acceleration_structure_build_marker *);
+static void hk_agx3_refit_save(VkCommandBuffer,
+                               const struct vk_acceleration_structure_build_state *);
 static void hk_agx3_refit_phase(VkCommandBuffer,
-   const struct vk_acceleration_structure_build_state *, unsigned);
+   const struct vk_acceleration_structure_build_state *, unsigned, unsigned);
+static VkResult hk_cmd_refit_agx3(VkCommandBuffer,
+   const struct vk_acceleration_structure_build_state *, bool);
+static bool hk_agx3_refit_skips_leaves(const struct hk_device *,
+   const struct vk_acceleration_structure_build_state *);
 
 /* Allow a million distinct BLASes plus a TLAS and the reserved query IDs. */
 #define HK_GEN1_AS_ID_LIMIT (1u << 21)
@@ -516,6 +558,10 @@ hk_CreateAccelerationStructureKHR(VkDevice device,
    const VkAccelerationStructureCreateInfoKHR *info,
    const VkAllocationCallbacks *alloc, VkAccelerationStructureKHR *out)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_CreateAccelerationStructureKHR(device, info, alloc, out);
+
    VK_FROM_HANDLE(hk_device, dev, device);
    VkAccelerationStructureCreateInfoKHR native_info;
    const VkDeviceAddress replay_address = dev->g17p_native_as ? info->deviceAddress : 0;
@@ -581,6 +627,12 @@ VKAPI_ATTR void VKAPI_CALL
 hk_DestroyAccelerationStructureKHR(VkDevice device, VkAccelerationStructureKHR handle,
                                    const VkAllocationCallbacks *alloc)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_DestroyAccelerationStructureKHR(device, handle, alloc);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_device, dev, device);
    VK_FROM_HANDLE(vk_acceleration_structure, as, handle);
    if (as && dev->g17p_native_as) {
@@ -644,9 +696,52 @@ hk_gen1_upload_as_table(struct hk_cmd_buffer *cmd)
 }
 
 static_assert(sizeof(struct hk_agx3_leaf_job) == 64, "native node job ABI");
-static_assert(sizeof(struct hk_agx3_leaf_args) == 24, "native node push ABI");
+static_assert(sizeof(struct hk_agx3_leaf_args) == 32, "native node push ABI");
 static_assert(sizeof(struct hk_agx3_envelope_job) == 48, "native envelope job ABI");
 static_assert(sizeof(struct hk_agx3_envelope_args) == 64, "native envelope push ABI");
+/* The GEN1 layout: the plan state, then the node encoding dispatch arguments
+ * the plan writes for the indirect encode (bvh/agx3_plan.comp), a
+ * 64-byte record slot per child including bounding boxes, 16 KiB images,
+ * and four 8-lane node jobs per encoding workgroup. */
+static_assert(HK_AGX3_PLAN_ENCODE_DISPATCH == sizeof(struct hk_agx3_plan_state) &&
+              HK_AGX3_GEN1_PLAN_TAIL_B >= HK_AGX3_PLAN_ENCODE_DISPATCH +
+                                             sizeof(VkDispatchIndirectCommand),
+              "GEN1 scratch: plan state, then the encode dispatch arguments");
+static_assert(HK_AGX3_GEN1_AABB_RECORD_B == HK_AGX3_AS_SLOT_B,
+              "GEN1 bounding-box records fill their child slot");
+static_assert(HK_AGX3_GEN1_IMAGE_ALIGN == 16384, "GEN1 image alignment");
+static_assert(HK_AGX3_GEN1_NODE_GROUP_LANES == 8 * HK_AGX3_NODE_JOBS_PER_GROUP,
+              "GEN1 encoder workgroup");
+
+const uint32_t *
+hk_gen1_agx3_node_encoder_spirv(bool pairs, size_t *size_B)
+{
+   *size_B = pairs ? sizeof(agx3_node_pairs_spv) : sizeof(agx3_node_spv);
+   return pairs ? agx3_node_pairs_spv : agx3_node_spv;
+}
+
+const uint32_t *
+hk_gen1_agx3_refit_spirv(size_t *size_B)
+{
+   *size_B = sizeof(agx3_refit_spv);
+   return agx3_refit_spv;
+}
+
+const uint32_t *
+hk_agx3_node_encoder_spirv(enum agx_chip chip, size_t *size_B)
+{
+   return hk_agx3_as_format_for_chip(chip) == HK_AGX3_AS_FORMAT_M5
+             ? hk_m5_agx3_node_encoder_spirv(false, size_B)
+             : hk_gen1_agx3_node_encoder_spirv(false, size_B);
+}
+
+const uint32_t *
+hk_agx3_refit_spirv(enum agx_chip chip, size_t *size_B)
+{
+   return hk_agx3_as_format_for_chip(chip) == HK_AGX3_AS_FORMAT_M5
+             ? hk_m5_agx3_refit_spirv(size_B)
+             : hk_gen1_agx3_refit_spirv(size_B);
+}
 
 static_assert(sizeof(struct hk_bvh_triangle_node) == 48, "node format");
 static_assert(sizeof(struct hk_bvh_aabb_node) == 32, "node format");
@@ -705,6 +800,11 @@ VkResult
 hk_cmd_encode_agx3_nodes(VkCommandBuffer command_buffer,
                         const struct hk_agx3_leaf_args *args)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, command_buffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_cmd_encode_agx3_nodes(command_buffer, args);
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    if (agx_chip_isa_encoding(dev->dev.chip) != AGX_ISA_ENCODING_AGX3C &&
@@ -714,7 +814,8 @@ hk_cmd_encode_agx3_nodes(VkCommandBuffer command_buffer,
       return VK_SUCCESS;
 
    /* The native phase uses eight invocations per independently writable
-    * node. Job planning bounds the count before it reaches this helper. */
+    * node, HK_AGX3_NODE_JOBS_PER_GROUP nodes per workgroup. Job planning
+    * bounds the count before it reaches this helper. */
    assert(args->job_count <= UINT32_MAX / 8);
    VkPipeline pipeline;
    VkPipelineLayout layout;
@@ -736,6 +837,11 @@ VkResult
 hk_cmd_finalize_agx3_as(VkCommandBuffer command_buffer,
                        const struct hk_agx3_envelope_args *args)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, command_buffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_cmd_finalize_agx3_as(command_buffer, args);
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    if (agx_chip_isa_encoding(dev->dev.chip) != AGX_ISA_ENCODING_AGX3C &&
@@ -766,25 +872,43 @@ hk_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
                         VkDeviceAddress node_status,
                         VkDeviceAddress envelope_status, uint32_t slot_bias)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, command_buffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_cmd_build_agx3_from_ir(command_buffer, args, node_status, envelope_status);
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    if (agx_chip_isa_encoding(dev->dev.chip) != AGX_ISA_ENCODING_AGX3C &&
        !dev->g17p_native_as)
       return VK_ERROR_FEATURE_NOT_PRESENT;
-   const struct {
+   struct stage {
       const char *name;
       const uint32_t *code;
       size_t size;
       unsigned push_size;
-   } stages[] = {
+   };
+   /* A TLAS encodes only inner and instance nodes. */
+   const bool instances = args->geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR;
+   const struct stage node_encode = instances
+      ? (struct stage){"hk-agx3-node-encode-instances", agx3_node_instances_spv,
+                       sizeof(agx3_node_instances_spv),
+                       sizeof(struct hk_agx3_leaf_args)}
+      : (struct stage){"hk-agx3-node-encode", agx3_node_spv,
+                       sizeof(agx3_node_spv), sizeof(struct hk_agx3_leaf_args)};
+   const struct stage stages[] = {
       {"hk-agx3-plan", agx3_plan_spv, sizeof(agx3_plan_spv), sizeof(*args)},
-      {"hk-agx3-node-encode", agx3_node_spv, sizeof(agx3_node_spv),
-       sizeof(struct hk_agx3_leaf_args)},
+      node_encode,
       {"hk-agx3-as-envelope", agx3_envelope_spv, sizeof(agx3_envelope_spv),
        sizeof(struct hk_agx3_envelope_args)},
+      /* Planning and node encoding that pair triangles. */
+      {"hk-agx3-node-encode-pairs", agx3_node_pairs_spv,
+       sizeof(agx3_node_pairs_spv), sizeof(struct hk_agx3_leaf_args)},
+      {"hk-agx3-plan-pairs", agx3_plan_pairs_spv, sizeof(agx3_plan_pairs_spv),
+       sizeof(*args)},
    };
-   VkPipeline pipeline[3];
-   VkPipelineLayout layout[3];
+   VkPipeline pipeline[5];
+   VkPipelineLayout layout[5];
    for (unsigned i = 0; i < ARRAY_SIZE(stages); ++i) {
       VkResult result = get_pipeline_spv(
          dev, stages[i].name, stages[i].code, stages[i].size,
@@ -793,8 +917,9 @@ hk_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
          return result;
    }
 
+   const unsigned plan = args->pairs ? 4 : 0;
    vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
-                            pipeline[0]);
+                            pipeline[plan]);
    const VkMemoryBarrier barrier = {
       .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
       .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
@@ -804,10 +929,12 @@ hk_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
    for (unsigned phase = 0; phase < 5; ++phase) {
       struct hk_agx3_plan_args push = *args;
       push.phase = phase;
-      vk_common_CmdPushConstants(command_buffer, layout[0],
+      vk_common_CmdPushConstants(command_buffer, layout[plan],
                                 VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push),
                                 &push);
-      hk_cmd_dispatch_unaligned(command_buffer, args->node_bound, 1, 1);
+      hk_cmd_dispatch_unaligned(command_buffer,
+                                phase == 4 ? 1 : args->node_bound,
+                                1, 1);
       vk_common_CmdPipelineBarrier(
          command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
          VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
@@ -816,7 +943,7 @@ hk_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
 
    const struct hk_agx3_leaf_args encode = {
       .jobs = args->jobs, .status = node_status, .job_count = args->node_bound,
-      .slot_bias = slot_bias,
+      .slot_bias = slot_bias, .counts = args->state,
    };
    const struct hk_agx3_envelope_args finish = {
       .jobs = args->envelope, .status = envelope_status, .job_count = 1,
@@ -826,20 +953,27 @@ hk_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
       .build_flags = args->build_flags, .node_bound = args->node_bound,
       .keep_retained = args->keep_retained,
    };
-   const bool instances = args->geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR;
    for (unsigned step = 0; step < 2; ++step) {
       bool envelope = instances ? step == 0 : step == 1;
-      unsigned stage = envelope ? 2 : 1;
+      unsigned stage = envelope ? 2
+                     : args->pairs ? 3 : 1;
       vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
                                pipeline[stage]);
       vk_common_CmdPushConstants(command_buffer, layout[stage],
                                 VK_SHADER_STAGE_COMPUTE_BIT, 0,
                                 envelope ? sizeof(finish) : sizeof(encode),
                                 envelope ? (const void *)&finish : (const void *)&encode);
+      /* A TLAS envelope also clears a pointer per instance: spread that
+       * over a thread per instance rather than one workgroup. */
       if (envelope)
-         hk_cmd_dispatch_unaligned(command_buffer, 64, 1, 1);
+         hk_cmd_dispatch_unaligned(command_buffer,
+                                   instances ? CLAMP(ALIGN_POT(args->leaf_bound, 64),
+                                                     64, 65536)
+                                             : 64,
+                                   1, 1);
       else
-         hk_cmd_dispatch_indirect_addr(command_buffer, args->state);
+         hk_cmd_dispatch_indirect_addr(command_buffer,
+                                       args->state + HK_AGX3_PLAN_ENCODE_DISPATCH);
       if (step == 0)
          vk_common_CmdPipelineBarrier(command_buffer,
             VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
@@ -896,7 +1030,7 @@ static VkDeviceSize
 hk_agx3_retained_offset(const struct vk_acceleration_structure_build_state *state)
 {
    struct hk_agx3_as_layout layout;
-   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout))
+   if (!hk_gen1_as_layout_init(state->leaf_node_count, &layout))
       return 0;
    return ALIGN_POT(layout.as_bytes +
       (vk_get_as_geometry_type(state->build_info) == VK_GEOMETRY_TYPE_INSTANCES_KHR
@@ -922,11 +1056,13 @@ static VkDeviceSize
 hk_get_native_as_size(const struct vk_acceleration_structure_build_state *state)
 {
    struct hk_agx3_as_layout layout;
-   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout))
+   if (!hk_gen1_as_layout_init(state->leaf_node_count, &layout))
       return 0;
+   /* The retained jobs, for as many jobs as there can be. */
    if (state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR)
-      return hk_agx3_retained_offset(state) + sizeof(struct vk_ir_header) +
-             (uint64_t)state->internal_node_count * sizeof(struct vk_ir_box_node);
+      return hk_agx3_retained_offset(state) +
+             HK_AGX3_RETAINED_BYTES((uint64_t)layout.node_bound,
+                                    (uint64_t)state->leaf_node_count);
    return layout.as_bytes +
       (vk_get_as_geometry_type(state->build_info) == VK_GEOMETRY_TYPE_INSTANCES_KHR
           ? (uint64_t)state->leaf_node_count * 80 : 0);
@@ -971,7 +1107,7 @@ hk_get_encode_scratch_size(VkDevice _device,
 {
    VK_FROM_HANDLE(hk_device, dev, _device);
    struct hk_agx3_as_layout layout;
-   return dev->g17p_native_as && hk_agx3_as_layout_init(state->leaf_node_count, &layout)
+   return dev->g17p_native_as && hk_gen1_as_layout_init(state->leaf_node_count, &layout)
              ? layout.encode_bytes + ALIGN_POT(sizeof(struct vk_ir_header), 8) -
                   sizeof(struct vk_ir_header) : 0;
 }
@@ -987,6 +1123,10 @@ VKAPI_ATTR VkDeviceAddress VKAPI_CALL
 hk_GetAccelerationStructureDeviceAddressKHR(
    VkDevice _device, const VkAccelerationStructureDeviceAddressInfoKHR *info)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_GetAccelerationStructureDeviceAddressKHR(_device, info);
+
    VK_FROM_HANDLE(vk_acceleration_structure, as, info->accelerationStructure);
    return vk_acceleration_structure_get_va(as);
 }
@@ -1139,30 +1279,34 @@ hk_encode(VkCommandBuffer commandBuffer, struct vk_device *device,
       vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
       for (uint32_t i = 0; i < build_count; ++i) {
          const struct vk_acceleration_structure_build_state *state = &states[i];
+         VK_FROM_HANDLE(vk_acceleration_structure, dst,
+                        state->build_info->dstAccelerationStructure);
          if (state->config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE) {
-            hk_agx3_refit_phase(commandBuffer, state, 2);
-            vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
-            VkResult result = vk_cmd_build_as_leaves(commandBuffer, device, meta,
-                                                      build_args, state);
+            /* An update keeps the topology: regenerate the leaves, or for
+             * triangles, place them where refit encoding reads them, then
+             * encode every node again from the retained jobs. */
+            const bool skip_leaves = hk_agx3_refit_skips_leaves(dev, state);
+            VkResult result = VK_SUCCESS;
+            if (!skip_leaves) {
+               result = vk_cmd_build_as_leaves(commandBuffer, device, meta,
+                                               build_args, state);
+               if (result == VK_SUCCESS)
+                  vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
+            }
+            if (result == VK_SUCCESS)
+               result = hk_cmd_refit_agx3(commandBuffer, state, skip_leaves);
             if (result != VK_SUCCESS) {
                vk_command_buffer_set_error(&cmd->vk, result);
                return;
             }
-            vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
-            hk_agx3_refit_phase(commandBuffer, state, 3);
-            vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
-         }
-         if ((state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR) &&
-             (state->config.internal_type != VK_INTERNAL_BUILD_TYPE_UPDATE ||
-              state->build_info->srcAccelerationStructure != state->build_info->dstAccelerationStructure)) {
-            hk_agx3_refit_phase(commandBuffer, state, 0);
-            vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
+            hk_vertex_table_build(commandBuffer, state, dst);
+            continue;
          }
+         const bool allow_update =
+            (state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR) != 0;
 
-         VK_FROM_HANDLE(vk_acceleration_structure, dst,
-                        state->build_info->dstAccelerationStructure);
          struct hk_agx3_as_layout native;
-         if (!hk_agx3_as_layout_init(state->leaf_node_count, &native)) {
+         if (!hk_gen1_as_layout_init(state->leaf_node_count, &native)) {
             vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
             return;
          }
@@ -1182,8 +1326,15 @@ hk_encode(VkCommandBuffer commandBuffer, struct vk_device *device,
             .geometry_type = vk_get_as_geometry_type(state->build_info),
             .retained_offset = hk_agx3_retained_offset(state),
             .build_flags = state->build_info->flags,
-            .keep_retained = state->config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE &&
-                             state->build_info->srcAccelerationStructure == state->build_info->dstAccelerationStructure,
+            /* An update re-encodes the retained jobs, which refit one
+             * triangle per record, so only triangle structures that cannot
+             * be updated pair triangles; and not those preferring a fast
+             * build, where finding pairs costs more than it saves. */
+            .pairs = !allow_update &&
+                     vk_get_as_geometry_type(state->build_info) ==
+                        VK_GEOMETRY_TYPE_TRIANGLES_KHR &&
+                     !(state->build_info->flags &
+                       VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_BUILD_BIT_KHR),
          };
          struct hk_agx3_plan_args native_plan = plan;
          native_plan.metadata = hk_gen1_as_metadata_addr_for_id(cmd, plan.resource_id);
@@ -1196,6 +1347,9 @@ hk_encode(VkCommandBuffer commandBuffer, struct vk_device *device,
             vk_command_buffer_set_error(&cmd->vk, result);
             return;
          }
+         /* Keep the jobs of a build that can be updated. */
+         if (allow_update)
+            hk_agx3_refit_save(commandBuffer, state);
          hk_vertex_table_build(commandBuffer, state, dst);
       }
       return;
@@ -1218,47 +1372,261 @@ hk_encode(VkCommandBuffer commandBuffer, struct vk_device *device,
       hk_encode_as(commandBuffer, &states[i], layout);
 }
 
+/* A triangle update runs its own leaf pass (agx3_refit_leaf.comp) instead of
+ * generating IR leaves: not for indirect builds, whose ranges are on the
+ * device, nor with a vertex table, which is built from the IR leaves. */
+static bool
+hk_agx3_refit_skips_leaves(const struct hk_device *dev,
+                           const struct vk_acceleration_structure_build_state *state)
+{
+   return vk_get_as_geometry_type(state->build_info) == VK_GEOMETRY_TYPE_TRIANGLES_KHR &&
+          (state->config.build_flags & VK_BUILD_FLAG_ALWAYS_ACTIVE) &&
+          !(state->config.build_flags & VK_BUILD_FLAG_HAS_QUADS) &&
+          !state->indirect_addr && !hk_vertex_table_wanted(dev, state);
+}
+
+/* Levels of inner jobs an update encodes a level per dispatch; propagation
+ * encodes any deeper. Each level costs a dispatch even when the tree is
+ * shallower, so take about as many as the tree is likely to have: at 4K,
+ * 64K and 1M triangles the fastest updates took 4-8, 8-12 and 12-16. */
+static unsigned
+hk_agx3_refit_levels(const struct vk_acceleration_structure_build_state *state)
+{
+   const unsigned bits = util_logbase2_ceil(MAX2(state->leaf_node_count, 1));
+   return CLAMP((int)bits - 6, 4, (int)HK_AGX3_REFIT_LEVELS);
+}
+
+/* Run refit phase `phase`; `level` is hk_agx3_refit_args.level. */
 static void
 hk_agx3_refit_phase(VkCommandBuffer commandBuffer,
                     const struct vk_acceleration_structure_build_state *state,
-                    unsigned phase)
+                    unsigned phase, unsigned level)
 {
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    struct hk_agx3_as_layout layout;
-   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout)) {
+   if (!hk_gen1_as_layout_init(state->leaf_node_count, &layout)) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
       return;
    }
    VkPipeline pipeline;
    VkPipelineLayout pipeline_layout;
-   VkResult result = get_pipeline_spv(dev, "hk-agx3-refit", agx3_refit_spv,
-      sizeof(agx3_refit_spv), sizeof(struct hk_agx3_refit_args), &pipeline, &pipeline_layout);
+   VkResult result =
+      phase == HK_AGX3_REFIT_INNER
+         ? get_pipeline_spv(dev, "hk-agx3-refit-inner", agx3_refit_inner_spv,
+                            sizeof(agx3_refit_inner_spv),
+                            sizeof(struct hk_agx3_refit_args), &pipeline,
+                            &pipeline_layout)
+      : phase == HK_AGX3_REFIT_PROPAGATE
+         ? get_pipeline_spv(dev, "hk-agx3-refit-propagate",
+                            agx3_refit_propagate_spv,
+                            sizeof(agx3_refit_propagate_spv),
+                            sizeof(struct hk_agx3_refit_args), &pipeline,
+                            &pipeline_layout)
+         : get_pipeline_spv(dev, "hk-agx3-refit", agx3_refit_spv,
+                            sizeof(agx3_refit_spv),
+                            sizeof(struct hk_agx3_refit_args), &pipeline,
+                            &pipeline_layout);
    if (result != VK_SUCCESS) {
       vk_command_buffer_set_error(&cmd->vk, result);
       return;
    }
-   VK_FROM_HANDLE(vk_acceleration_structure, retained_as,
-      phase == 1 ? state->build_info->srcAccelerationStructure : state->build_info->dstAccelerationStructure);
-   uint64_t scratch = state->build_info->scratchData.deviceAddress;
+   VK_FROM_HANDLE(vk_acceleration_structure, dst,
+                  state->build_info->dstAccelerationStructure);
+   VK_FROM_HANDLE(vk_acceleration_structure, src,
+                  state->build_info->srcAccelerationStructure);
+   const bool prepare = phase == HK_AGX3_REFIT_PREPARE;
+   const uint64_t scratch = state->build_info->scratchData.deviceAddress;
+   const uint64_t encode = ALIGN_POT(scratch + state->scratch.header_offset +
+                                     sizeof(struct vk_ir_header), UINT64_C(8));
    struct hk_agx3_refit_args args = {
-      .retained = hk_as_native_va(retained_as) + hk_agx3_retained_offset(state),
-      .metadata = phase == 1 ? hk_gen1_as_metadata_addr(cmd, retained_as) : 0,
+      .retained = hk_as_native_va(dst) + hk_agx3_retained_offset(state),
+      /* The source may be a compacted copy: its metadata finds its jobs. */
+      .metadata = prepare ? hk_gen1_as_metadata_addr(cmd, src) : 0,
       .intermediate_bvh = scratch + state->scratch.ir_offset,
       .header = scratch + state->scratch.header_offset,
-      .parents = scratch + state->scratch.size,
+      .plan_nodes = encode + layout.nodes, .plan_jobs = encode + layout.jobs,
+      .plan_state = encode + layout.state,
+      .refit = encode + layout.state, .counters = encode + layout.nodes,
+      .bounds = encode + layout.jobs, .envelope = encode + layout.envelope,
+      .output_bvh = hk_as_native_va(dst),
       .root_offset = state->scratch.internal_node_offset - state->scratch.ir_offset,
       .leaf_bound = state->leaf_node_count,
       .node_bound = layout.node_bound, .phase = phase,
+      .resource_id = hk_gen1_as_id(dev, dst),
+      .geometry_type = vk_get_as_geometry_type(state->build_info),
+      .copy = prepare && state->build_info->srcAccelerationStructure !=
+                         state->build_info->dstAccelerationStructure,
+      .slot_bias = hk_gen1_as_slot_bias(dev, dst),
+      .status = encode + layout.node_status,
+      .levels = prepare && layout.node_bound ? encode + layout.refit_levels : 0,
+      .level = level,
+      .no_leaves = prepare && hk_agx3_refit_skips_leaves(dev, state),
    };
-   unsigned count = phase <= 1
-      ? MAX2(layout.node_bound, (sizeof(struct vk_ir_header) +
-               state->internal_node_count * sizeof(struct vk_ir_box_node)) / 4)
-      : phase == 2 ? state->internal_node_count : state->leaf_node_count;
+   /* One thread per job, per instance pointer, and per retained word that
+    * an update into another structure copies. */
+   uint64_t count = MAX2(layout.node_bound, state->leaf_node_count);
+   if (args.copy)
+      count = MAX2(count, HK_AGX3_RETAINED_BYTES((uint64_t)layout.node_bound,
+                                                 (uint64_t)state->leaf_node_count) / 4);
    vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
    vk_common_CmdPushConstants(commandBuffer, pipeline_layout,
       VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(args), &args);
-   hk_cmd_dispatch_unaligned(commandBuffer, count, 1, 1);
+   if (phase == HK_AGX3_REFIT_PROPAGATE)
+      hk_cmd_dispatch_indirect_addr(commandBuffer, args.refit +
+                                    HK_AGX3_REFIT_PROPAGATE_DISPATCH);
+   else if (phase == HK_AGX3_REFIT_INNER)
+      hk_cmd_dispatch_indirect_addr(commandBuffer, encode + layout.refit_levels +
+                                    level * 12);
+   else
+      hk_cmd_dispatch_unaligned(commandBuffer, count, 1, 1);
+}
+
+/* Keep the jobs of a full build that can be updated, with its inner jobs
+ * listed by level. */
+static void
+hk_agx3_refit_save(VkCommandBuffer commandBuffer,
+                   const struct vk_acceleration_structure_build_state *state)
+{
+   vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
+   hk_agx3_refit_phase(commandBuffer, state, HK_AGX3_REFIT_SAVE, 0);
+   vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
+   hk_agx3_refit_phase(commandBuffer, state, HK_AGX3_REFIT_COUNT_LEVELS, 0);
+   vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
+   hk_agx3_refit_phase(commandBuffer, state, HK_AGX3_REFIT_LIST_LEVELS, 0);
+}
+
+/* Update a structure whose leaves are in scratch, or with `skip_leaves`, a
+ * triangle structure whose leaves are still to place: prepare from its
+ * retained jobs, place the triangles, encode the leaf jobs, propagate to
+ * the inner jobs past the levels, encode the inner jobs of each level,
+ * deepest first, and finish the envelope. */
+static VkResult
+hk_cmd_refit_agx3(VkCommandBuffer commandBuffer,
+                  const struct vk_acceleration_structure_build_state *state,
+                  bool skip_leaves)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   struct hk_agx3_as_layout layout;
+   if (!hk_gen1_as_layout_init(state->leaf_node_count, &layout))
+      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+   VkPipeline pipeline[3];
+   VkPipelineLayout pipeline_layout[3];
+   const bool triangles =
+      vk_get_as_geometry_type(state->build_info) == VK_GEOMETRY_TYPE_TRIANGLES_KHR;
+   VkResult result = triangles
+      ? get_pipeline_spv(dev, "hk-agx3-refit-encode-triangles",
+                         agx3_refit_encode_triangles_spv,
+                         sizeof(agx3_refit_encode_triangles_spv),
+                         sizeof(struct hk_agx3_refit_encode_args),
+                         &pipeline[0], &pipeline_layout[0])
+      : get_pipeline_spv(dev, "hk-agx3-refit-encode", agx3_refit_encode_spv,
+                         sizeof(agx3_refit_encode_spv),
+                         sizeof(struct hk_agx3_refit_encode_args),
+                         &pipeline[0], &pipeline_layout[0]);
+   if (result == VK_SUCCESS)
+      result = get_pipeline_spv(dev, "hk-agx3-as-envelope", agx3_envelope_spv,
+         sizeof(agx3_envelope_spv), sizeof(struct hk_agx3_envelope_args),
+         &pipeline[1], &pipeline_layout[1]);
+   if (result == VK_SUCCESS && skip_leaves)
+      result = get_pipeline_spv(dev, "hk-agx3-refit-leaf", agx3_refit_leaf_spv,
+         sizeof(agx3_refit_leaf_spv), sizeof(struct hk_agx3_refit_leaf_args),
+         &pipeline[2], &pipeline_layout[2]);
+   if (result != VK_SUCCESS)
+      return result;
+
+   VK_FROM_HANDLE(vk_acceleration_structure, dst,
+                  state->build_info->dstAccelerationStructure);
+   const uint64_t scratch = state->build_info->scratchData.deviceAddress;
+   const uint64_t encode = ALIGN_POT(scratch + state->scratch.header_offset +
+                                     sizeof(struct vk_ir_header), UINT64_C(8));
+   const uint32_t resource_id = hk_gen1_as_id(dev, dst);
+   const uint64_t metadata = hk_gen1_as_metadata_addr_for_id(cmd, resource_id);
+   if (!metadata)
+      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+   const VkMemoryBarrier barrier = {
+      .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+      .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
+      .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
+                       VK_ACCESS_INDIRECT_COMMAND_READ_BIT,
+   };
+
+   const unsigned levels = hk_agx3_refit_levels(state);
+   hk_agx3_refit_phase(commandBuffer, state, HK_AGX3_REFIT_PREPARE, levels);
+   vk_common_CmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+      VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
+      0, 1, &barrier, 0, NULL, 0, NULL);
+
+   if (skip_leaves) {
+      /* A thread per primitive of each geometry, numbered as leaf generation
+       * numbers them; the triangles take the IR leaves' place. */
+      static_assert(sizeof(struct hk_agx3_refit_triangle) <= sizeof(struct vk_ir_triangle_node),
+                    "placed triangles fit where the IR leaves would be");
+      const VkAccelerationStructureBuildGeometryInfoKHR *info = state->build_info;
+      struct hk_agx3_refit_leaf_args leaf = {
+         .refit = encode + layout.state,
+         .triangles = scratch + state->scratch.ir_offset,
+         .leaf_bound = state->leaf_node_count,
+      };
+      vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline[2]);
+      uint32_t first = 0;
+      for (uint32_t j = 0; j < info->geometryCount; ++j) {
+         const VkAccelerationStructureBuildRangeInfoKHR *range = &state->build_range_infos[j];
+         if (!range->primitiveCount)
+            continue;
+         leaf.geometry = vk_fill_geometry_data(info->type, first, j,
+            info->pGeometries ? &info->pGeometries[j] : info->ppGeometries[j], range);
+         leaf.primitive_count = range->primitiveCount;
+         vk_common_CmdPushConstants(commandBuffer, pipeline_layout[2],
+            VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(leaf), &leaf);
+         hk_cmd_dispatch_unaligned(commandBuffer, range->primitiveCount, 1, 1);
+         first += range->primitiveCount;
+      }
+      vk_common_CmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+         VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &barrier, 0, NULL, 0, NULL);
+   }
+
+   const struct hk_agx3_refit_encode_args refit = {
+      .refit = encode + layout.state, .bounds = encode + layout.jobs,
+      .intermediate_bvh = scratch + state->scratch.ir_offset,
+      .output_bvh = hk_as_native_va(dst), .status = encode + layout.node_status,
+      .slot_bias = hk_gen1_as_slot_bias(dev, dst),
+      .leaf_bound = state->leaf_node_count,
+      .triangles = skip_leaves ? scratch + state->scratch.ir_offset : 0,
+   };
+   vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline[0]);
+   vk_common_CmdPushConstants(commandBuffer, pipeline_layout[0],
+      VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(refit), &refit);
+   hk_cmd_dispatch_indirect_addr(commandBuffer, encode + layout.state);
+   vk_common_CmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+      VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &barrier, 0, NULL, 0, NULL);
+   hk_agx3_refit_phase(commandBuffer, state, HK_AGX3_REFIT_PROPAGATE, levels);
+   vk_common_CmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+      VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &barrier, 0, NULL, 0, NULL);
+   if (layout.node_bound) {
+      for (unsigned level = levels; level-- > 0;) {
+         hk_agx3_refit_phase(commandBuffer, state, HK_AGX3_REFIT_INNER, level);
+         vk_common_CmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+            VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &barrier, 0, NULL, 0, NULL);
+      }
+   }
+
+   const struct hk_agx3_envelope_args finish = {
+      .jobs = encode + layout.envelope, .status = encode + layout.envelope_status,
+      .job_count = 1, .reserved = 2, .metadata = metadata,
+      .ir_header = scratch + state->scratch.header_offset,
+      .retained_offset = hk_agx3_retained_offset(state),
+      .build_flags = state->build_info->flags, .node_bound = layout.node_bound,
+      .keep_retained = state->build_info->srcAccelerationStructure ==
+                       state->build_info->dstAccelerationStructure,
+      .refit = 1,
+   };
+   vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline[1]);
+   vk_common_CmdPushConstants(commandBuffer, pipeline_layout[1],
+      VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(finish), &finish);
+   hk_cmd_dispatch_unaligned(commandBuffer, 64, 1, 1);
+   return VK_SUCCESS;
 }
 
 static void
@@ -1282,7 +1650,7 @@ hk_update_scratch_size(VkDevice device,
    /* Leaf generation writes sort IDs, which alias native encode scratch.
     * Keep live refit parent/counter state beyond both workspaces. */
    struct hk_agx3_as_layout layout;
-   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout))
+   if (!hk_gen1_as_layout_init(state->leaf_node_count, &layout))
       return 0;
    return (VkDeviceSize)state->scratch.size + (uint64_t)layout.node_bound * 16;
 }
@@ -1292,9 +1660,21 @@ hk_init_update_scratch(VkCommandBuffer commandBuffer,
                        const struct vk_acceleration_structure_build_state *states,
                        uint32_t count)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct vk_device *device = cmd->vk.base.device;
+   /* Leaf generation counts the active leaves into a fresh header. */
+   struct vk_ir_header header = {
+      .min_bounds = {0x7fffffff, 0x7fffffff, 0x7fffffff},
+      .max_bounds = {0x80000000, 0x80000000, 0x80000000},
+      .dispatch_size_y = 1,
+      .dispatch_size_z = 1,
+   };
    for (unsigned i = 0; i < count; ++i)
       if (states[i].config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE)
-         hk_agx3_refit_phase(commandBuffer, &states[i], 1);
+         device->write_buffer_cp(commandBuffer,
+                                 states[i].build_info->scratchData.deviceAddress +
+                                    states[i].scratch.header_offset,
+                                 &header, sizeof(header));
 }
 
 static const struct vk_acceleration_structure_build_ops hk_native_as_build_ops = {
@@ -1840,6 +2220,9 @@ hk_gen1_rq_init(struct hk_device *dev,const char *dir)
 VkResult
 hk_device_init_accel_struct_state(struct hk_device *dev)
 {
+   if (hk_agx3_as_format_for_chip(dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_device_init_accel_struct_state(dev);
+
    /* The native state below is large and lives as long as the device: the
     * ray query program image with its fixed binds, the AS tables, and, through
     * gen1_native_graphics_rq/fragment_rq, a render SCS reservation in every
@@ -1847,12 +2230,17 @@ hk_device_init_accel_struct_state(struct hk_device *dev)
     * device enabling no ray-tracing feature cannot reach any of it, so it
     * keeps the state every device without native ray tracing has. Shader
     * cache keys tell the two kinds of device apart (HK_FEAT_RAY_TRACING). */
-   dev->g17p_native_as = hk_g17p_rt_enabled(&dev->dev) &&
+   dev->g17p_native_as = hk_gen1_as_enabled(&dev->dev) &&
+                         hk_rt_features_enabled(&dev->vk.enabled_features);
+   /* AS-only G16S devices must never allocate the generated intersectors,
+    * fixed ray program binds, IFT or render/compute SCS state. The kernel
+    * deliberately rejects traversal on the 25G83 firmware variant. */
+   const bool native_rq = hk_g17p_rt_enabled(&dev->dev) &&
                          hk_rt_features_enabled(&dev->vk.enabled_features);
    const char *capture=getenv("ASAHI_G17P_RQ_CAPTURE_DIR");
-   if(capture && dev->g17p_native_as && !hk_gen1_rq_init(dev,capture))
+   if(capture && native_rq && !hk_gen1_rq_init(dev,capture))
       return vk_error(dev,VK_ERROR_INITIALIZATION_FAILED);
-   if (!capture && dev->g17p_native_as && !hk_gen1_rq_init_generated(dev))
+   if (!capture && native_rq && !hk_gen1_rq_init_generated(dev))
       return vk_error(dev, VK_ERROR_INITIALIZATION_FAILED);
    dev->gen1_rq_instances = dev->gen1_native_rq;
    dev->gen1_rq_candidates = dev->gen1_rq_instances;
@@ -2000,6 +2388,12 @@ hk_GetAccelerationStructureBuildSizesKHR(
    const uint32_t *pMaxPrimitiveCounts,
    VkAccelerationStructureBuildSizesInfoKHR *pSizeInfo)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_GetAccelerationStructureBuildSizesKHR(_device, buildType, pBuildInfo, pMaxPrimitiveCounts, pSizeInfo);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_device, dev, _device);
 
    if (dev->g17p_native_as) {
@@ -2008,7 +2402,7 @@ hk_GetAccelerationStructureBuildSizesKHR(
          leaves += pMaxPrimitiveCounts[i];
       if (!leaves) {
          struct hk_agx3_as_layout layout;
-         hk_agx3_as_layout_init(0, &layout);
+         hk_gen1_as_layout_init(0, &layout);
          pSizeInfo->accelerationStructureSize = layout.as_bytes;
          pSizeInfo->buildScratchSize = 256;
          pSizeInfo->updateScratchSize = 256;
@@ -2120,17 +2514,20 @@ hk_cmd_build_acceleration_structures(
          bool triangles =
             pInfos[i].type == VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR &&
             (!geometry || geometry->geometryType == VK_GEOMETRY_TYPE_TRIANGLES_KHR);
-         bool aabbs = dev->gen1_rq_candidates && geometry &&
+         /* Native AS construction supports all three geometry kinds without
+          * intersectors. Traversal state gates query execution, not building
+          * the AABB records or the TLAS instance records it will later read. */
+         bool aabbs = geometry &&
             pInfos[i].type == VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR &&
             geometry->geometryType == VK_GEOMETRY_TYPE_AABBS_KHR;
-         bool instances = dev->gen1_rq_instances && geometry &&
+         bool instances = geometry &&
             pInfos[i].geometryCount == 1 &&
             pInfos[i].type == VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR &&
             geometry->geometryType == VK_GEOMETRY_TYPE_INSTANCES_KHR;
          if ((pInfos[i].mode != VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR &&
               pInfos[i].mode != VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR) ||
              !compatible || (!triangles && !instances && !aabbs) || primitives > UINT32_MAX ||
-             !hk_agx3_as_layout_init(primitives, &native)) {
+             !hk_gen1_as_layout_init(primitives, &native)) {
             vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
             return;
          }
@@ -2218,6 +2615,13 @@ hk_CmdBuildAccelerationStructuresKHR(
    const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
    const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, commandBuffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_CmdBuildAccelerationStructuresKHR(commandBuffer, infoCount, pInfos, ppBuildRangeInfos);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
 
@@ -2268,6 +2672,12 @@ hk_GetDeviceAccelerationStructureCompatibilityKHR(
    VkDevice _device, const VkAccelerationStructureVersionInfoKHR *pVersionInfo,
    VkAccelerationStructureCompatibilityKHR *pCompatibility)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_GetDeviceAccelerationStructureCompatibilityKHR(_device, pVersionInfo, pCompatibility);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_device, dev, _device);
    struct hk_physical_device *pdev = hk_device_physical(dev);
 
@@ -2308,6 +2718,13 @@ VKAPI_ATTR void VKAPI_CALL
 hk_CmdCopyAccelerationStructureKHR(
    VkCommandBuffer commandBuffer, const VkCopyAccelerationStructureInfoKHR *info)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, commandBuffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_CmdCopyAccelerationStructureKHR(commandBuffer, info);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    VK_FROM_HANDLE(vk_acceleration_structure, src, info->src);
    VK_FROM_HANDLE(vk_acceleration_structure, dst, info->dst);
@@ -2401,6 +2818,13 @@ VKAPI_ATTR void VKAPI_CALL
 hk_CmdCopyAccelerationStructureToMemoryKHR(VkCommandBuffer commandBuffer,
    const VkCopyAccelerationStructureToMemoryInfoKHR *info)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, commandBuffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_CmdCopyAccelerationStructureToMemoryKHR(commandBuffer, info);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    VK_FROM_HANDLE(vk_acceleration_structure, src, info->src);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
@@ -2429,6 +2853,13 @@ VKAPI_ATTR void VKAPI_CALL
 hk_CmdCopyMemoryToAccelerationStructureKHR(VkCommandBuffer commandBuffer,
    const VkCopyMemoryToAccelerationStructureInfoKHR *info)
 {
+   VK_FROM_HANDLE(hk_cmd_buffer, m5_dispatch_cmd, commandBuffer);
+   struct hk_device *m5_dispatch_dev = hk_cmd_buffer_device(m5_dispatch_cmd);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5) {
+      hk_m5_backend_CmdCopyMemoryToAccelerationStructureKHR(commandBuffer, info);
+      return;
+   }
+
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    VK_FROM_HANDLE(vk_acceleration_structure, dst, info->dst);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
@@ -2486,9 +2917,18 @@ hk_host_as_init(struct hk_device *dev)
    simple_mtx_init(&dev->host_as.lock, mtx_plain);
    dev->host_as.lock_ready = true;
 
+   /* hk_queue_init() takes the priority of queue index_in_family from
+    * pQueuePriorities unless a global priority is chained. This queue has no
+    * priority array of HK_HOST_AS_QUEUE_INDEX + 1 entries, so chain the
+    * class the 1.0 priority maps to instead. */
    const float priority = 1.0f;
+   const VkDeviceQueueGlobalPriorityCreateInfoKHR priority_info = {
+      .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_KHR,
+      .globalPriority = VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR,
+   };
    const VkDeviceQueueCreateInfo queue_info = {
       .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
+      .pNext = &priority_info,
       .queueFamilyIndex = family,
       .queueCount = HK_HOST_AS_QUEUE_INDEX + 1,
       .pQueuePriorities = &priority,
@@ -2627,6 +3067,10 @@ hk_BuildAccelerationStructuresKHR(
    const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
    const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_BuildAccelerationStructuresKHR(_device, deferredOperation, infoCount, pInfos, ppBuildRangeInfos);
+
    VK_FROM_HANDLE(hk_device, dev, _device);
    struct hk_host_as_run run;
    VkResult result = hk_host_as_begin(dev, &run);
@@ -2716,6 +3160,10 @@ hk_CopyAccelerationStructureKHR(VkDevice _device,
                                 VkDeferredOperationKHR deferredOperation,
                                 const VkCopyAccelerationStructureInfoKHR *pInfo)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_CopyAccelerationStructureKHR(_device, deferredOperation, pInfo);
+
    VK_FROM_HANDLE(hk_device, dev, _device);
    struct hk_host_as_run run;
    VkResult result = hk_host_as_begin(dev, &run);
@@ -2732,6 +3180,10 @@ hk_CopyMemoryToAccelerationStructureKHR(
    VkDevice _device, VkDeferredOperationKHR deferredOperation,
    const VkCopyMemoryToAccelerationStructureInfoKHR *pInfo)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_CopyMemoryToAccelerationStructureKHR(_device, deferredOperation, pInfo);
+
    VK_FROM_HANDLE(hk_device, dev, _device);
    uint64_t bytes;
    memcpy(&bytes, (const uint8_t *)pInfo->src.hostAddress + 32, sizeof(bytes));
@@ -2764,6 +3216,10 @@ hk_CopyAccelerationStructureToMemoryKHR(
    VkDevice _device, VkDeferredOperationKHR deferredOperation,
    const VkCopyAccelerationStructureToMemoryInfoKHR *pInfo)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_CopyAccelerationStructureToMemoryKHR(_device, deferredOperation, pInfo);
+
    VK_FROM_HANDLE(hk_device, dev, _device);
    VK_FROM_HANDLE(vk_acceleration_structure, src, pInfo->src);
    struct hk_host_as_run run;
@@ -2803,6 +3259,10 @@ hk_WriteAccelerationStructuresPropertiesKHR(
    const VkAccelerationStructureKHR *pAccelerationStructures,
    VkQueryType queryType, size_t dataSize, void *pData, size_t stride)
 {
+   VK_FROM_HANDLE(hk_device, m5_dispatch_dev, _device);
+   if (hk_agx3_as_format_for_chip(m5_dispatch_dev->dev.chip) == HK_AGX3_AS_FORMAT_M5)
+      return hk_m5_backend_WriteAccelerationStructuresPropertiesKHR(_device, accelerationStructureCount, pAccelerationStructures, queryType, dataSize, pData, stride);
+
    VK_FROM_HANDLE(hk_device, dev, _device);
    const struct vk_device_dispatch_table *disp = &dev->vk.dispatch_table;
    if (!accelerationStructureCount)
diff --git a/src/asahi/vulkan/hk_acceleration_structure.h b/src/asahi/vulkan/hk_acceleration_structure.h
index b6a1a543415c..fc6fda6446ed 100644
--- a/src/asahi/vulkan/hk_acceleration_structure.h
+++ b/src/asahi/vulkan/hk_acceleration_structure.h
@@ -6,6 +6,7 @@
 #ifndef HK_ACCELERATION_STRUCTURE_H
 #define HK_ACCELERATION_STRUCTURE_H
 
+#include "hk_agx3_as_format.h"
 #include "hk_bvh.h"
 
 #include "vk_acceleration_structure.h"
@@ -25,19 +26,34 @@ struct hk_agx3_plan_args;
 
 void hk_gen1_rq_finish(struct hk_device *dev);
 VkResult hk_device_init_accel_struct_state(struct hk_device *device);
+void hk_device_finish_accel_struct_state(struct hk_device *device);
+uint64_t hk_m5_upload_as_table(struct hk_cmd_buffer *cmd);
+void hk_m5_as_buffer_bound(struct hk_device *dev, struct vk_buffer *buffer);
+
 /* Native build phase; the caller owns job planning, validated output ranges,
- * barriers and meta state save/restore. Not wired to the public build path
- * until the complete native AS/query contract is implemented. */
+ * barriers and meta state save/restore. */
 VkResult hk_cmd_encode_agx3_nodes(VkCommandBuffer command_buffer,
                                 const struct hk_agx3_leaf_args *args);
 VkResult hk_cmd_finalize_agx3_as(VkCommandBuffer command_buffer,
                                const struct hk_agx3_envelope_args *args);
+/* Plans, encodes and finalizes one triangle AS from common build IR, with
+ * internal phase barriers and a GPU-written indirect node count. */
 VkResult hk_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
                                  const struct hk_agx3_plan_args *args,
                                  VkDeviceAddress node_status,
                                  VkDeviceAddress envelope_status,
                                  uint32_t slot_bias);
 
+/* The SPIR-V that the node encoding and refit pipelines of each native AS
+ * format are created from; hk_agx3_as_format_for_chip() selects the format.
+ * GEN1 also has a triangle-pair encoder, M5 a batched one. */
+const uint32_t *hk_agx3_node_encoder_spirv(enum agx_chip chip, size_t *size_B);
+const uint32_t *hk_agx3_refit_spirv(enum agx_chip chip, size_t *size_B);
+const uint32_t *hk_gen1_agx3_node_encoder_spirv(bool pairs, size_t *size_B);
+const uint32_t *hk_gen1_agx3_refit_spirv(size_t *size_B);
+const uint32_t *hk_m5_agx3_node_encoder_spirv(bool batch, size_t *size_B);
+const uint32_t *hk_m5_agx3_refit_spirv(size_t *size_B);
+
 uint64_t hk_gen1_ray_query_scs_layout(const struct hk_device *dev);
 uint64_t hk_gen1_query_state(struct hk_cmd_buffer *cmd,
                             const struct agx_scs_layout_gen2 *scs);
diff --git a/src/asahi/vulkan/hk_agx3_as_format.h b/src/asahi/vulkan/hk_agx3_as_format.h
new file mode 100644
index 000000000000..cf53fedb3ca0
--- /dev/null
+++ b/src/asahi/vulkan/hk_agx3_as_format.h
@@ -0,0 +1,70 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef HK_AGX3_AS_FORMAT_H
+#define HK_AGX3_AS_FORMAT_H
+
+#include "asahi/lib/agx_chip.h"
+
+/*
+ * Native acceleration-structure formats. The device's chip selects one, and
+ * that format's encoder writes the records that its own refit, traversal,
+ * serialization, relocation and copies read:
+ *
+ *  - GEN1: G17P and the other native AGX3 chips. bvh/agx3_gen1_*.comp encode
+ *    four node jobs per 32-lane workgroup through an indirect dispatch
+ *    written after the plan state, and may pair triangles sharing an edge in
+ *    one record; every child has a 64-byte record slot; images are aligned
+ *    to 16 KiB (hk_acceleration_structure.c, libagx/gen1_as.h,
+ *    compiler/agx3_rt_gen1_*).
+ *  - M5: G17G and G17S. bvh/agx3_*.comp encode one node per 8-lane
+ *    workgroup, batched through a dense job map, with 48-byte bounding-box
+ *    records; images are aligned to 1 KiB (hk_m5_acceleration_structure.c).
+ */
+enum hk_agx3_as_format {
+   HK_AGX3_AS_FORMAT_GEN1,
+   HK_AGX3_AS_FORMAT_M5,
+};
+
+static inline enum hk_agx3_as_format
+hk_agx3_as_format_for_chip(enum agx_chip chip)
+{
+   return agx_chip_g17_m5(chip) ? HK_AGX3_AS_FORMAT_M5 : HK_AGX3_AS_FORMAT_GEN1;
+}
+
+/* Bytes of a native child record slot, common to both formats. */
+#define HK_AGX3_AS_SLOT_B 64u
+
+/* Image alignment. */
+#define HK_AGX3_GEN1_IMAGE_ALIGN 16384u
+#define HK_AGX3_M5_IMAGE_ALIGN   1024u
+
+/* Encoding scratch after hk_agx3_plan_state's offset: GEN1 keeps the node
+ * encoding dispatch arguments (HK_AGX3_PLAN_ENCODE_DISPATCH) there, M5 only
+ * the plan state. */
+#define HK_AGX3_GEN1_PLAN_TAIL_B 48u
+#define HK_AGX3_M5_PLAN_TAIL_B   32u
+
+/* Bounding-box leaf record size. */
+#define HK_AGX3_GEN1_AABB_RECORD_B 64u
+#define HK_AGX3_M5_AABB_RECORD_B   48u
+
+/* Lanes of one node-encoding workgroup. */
+#define HK_AGX3_GEN1_NODE_GROUP_LANES 32u
+#define HK_AGX3_M5_NODE_GROUP_LANES   8u
+
+static inline unsigned
+hk_agx3_as_image_align(enum hk_agx3_as_format format)
+{
+   return format == HK_AGX3_AS_FORMAT_M5 ? HK_AGX3_M5_IMAGE_ALIGN
+                                         : HK_AGX3_GEN1_IMAGE_ALIGN;
+}
+
+static inline unsigned
+hk_agx3_as_plan_tail(enum hk_agx3_as_format format)
+{
+   return format == HK_AGX3_AS_FORMAT_M5 ? HK_AGX3_M5_PLAN_TAIL_B
+                                         : HK_AGX3_GEN1_PLAN_TAIL_B;
+}
+
+#endif
diff --git a/src/asahi/vulkan/hk_agx3_as_layout.h b/src/asahi/vulkan/hk_agx3_as_layout.h
index 3f281c07282a..c6e36f9587e7 100644
--- a/src/asahi/vulkan/hk_agx3_as_layout.h
+++ b/src/asahi/vulkan/hk_agx3_as_layout.h
@@ -5,22 +5,29 @@
 #define HK_AGX3_AS_LAYOUT_H
 #include <stdbool.h>
 #include <stdint.h>
+#include "hk_agx3_as_format.h"
 
 /* Encoding scratch aliases the common builder's finished sort workspace.
  * All offsets are relative to the first byte after vk_ir_header. */
 struct hk_agx3_as_layout {
    uint32_t nodes, jobs, envelope, state, node_status, envelope_status;
+   /* An update's dispatches of inner jobs, one per depth (see
+    * HK_AGX3_REFIT_LEVELS). */
+   uint32_t refit_levels;
    uint32_t encode_bytes, node_bound, slot_bound;
    uint64_t as_bytes;
 };
 
 static inline bool
-hk_agx3_as_layout_init(uint32_t leaves, struct hk_agx3_as_layout *out)
+hk_agx3_as_layout_init_format(enum hk_agx3_as_format format, uint32_t leaves,
+                              struct hk_agx3_as_layout *out)
 {
    if (!out)
       return false;
+   const uint32_t alignment = hk_agx3_as_image_align(format);
    if (!leaves) {
-      *out = (struct hk_agx3_as_layout){.slot_bound = 3, .as_bytes = 192};
+      *out = (struct hk_agx3_as_layout){.slot_bound = 3,
+         .as_bytes = 192 + (alignment == HK_AGX3_GEN1_IMAGE_ALIGN ? 0 : alignment - 1)};
       return true;
    }
    const uint64_t nodes = (uint64_t)leaves + (leaves > 1 ? leaves : 2) - 1;
@@ -32,18 +39,39 @@ hk_agx3_as_layout_init(uint32_t leaves, struct hk_agx3_as_layout *out)
    const uint64_t jobs = nodes * 16;
    const uint64_t envelope = jobs + nodes * 64;
    const uint64_t state = envelope + 48;
-   const uint64_t status = state + 32;
-   const uint64_t end = (status + nodes * 4 + 4 + 7) & ~UINT64_C(7);
+   /* GEN1 keeps the indirect encode arguments after the plan state. */
+   const uint64_t status = state + hk_agx3_as_plan_tail(format);
+   const uint64_t refit_levels = (status + nodes * 4 + 4 + 7) & ~UINT64_C(7);
+   /* GEN1's current refit uses per-level dispatches. The M5 backend keeps
+    * its independent IR-topology refit and needs no level-dispatch tail. */
+   const uint64_t end = refit_levels +
+      (format == HK_AGX3_AS_FORMAT_GEN1 ? 16 * 12 : 0);
    if (nodes > UINT32_MAX / 8 || slots > UINT32_MAX || end > UINT32_MAX)
       return false;
    *out = (struct hk_agx3_as_layout){
       .nodes = 0, .jobs = jobs, .envelope = envelope, .state = state,
       .node_status = status, .envelope_status = status + nodes * 4,
+      .refit_levels = refit_levels,
       .encode_bytes = end, .node_bound = nodes, .slot_bound = slots,
-      .as_bytes = slots * 64 + 184,
+      .as_bytes = slots * HK_AGX3_AS_SLOT_B + 184 +
+                  (alignment == HK_AGX3_GEN1_IMAGE_ALIGN ? 0 : alignment - 1),
    };
    return true;
 }
+/* The nonempty envelope publishes readiness at native AS byte0xa0. Empty
+ * metadata clears that word and publishes NaN bounds so parent builders
+ * discard references to empty BLAS. This is not a hardware empty-node ABI. */
+static inline bool
+hk_agx3_as_layout_init(uint32_t leaves, struct hk_agx3_as_layout *out)
+{
+   return hk_agx3_as_layout_init_format(HK_AGX3_AS_FORMAT_M5, leaves, out);
+}
+
+static inline bool
+hk_gen1_as_layout_init(uint32_t leaves, struct hk_agx3_as_layout *out)
+{
+   return hk_agx3_as_layout_init_format(HK_AGX3_AS_FORMAT_GEN1, leaves, out);
+}
 
 static inline void
 hk_agx3_as_empty_header(uint32_t header[28])
diff --git a/src/asahi/vulkan/hk_agx3_ioto_program_core.c b/src/asahi/vulkan/hk_agx3_ioto_program_core.c
index 983a7c50311f..e5d86aaabf1f 100644
--- a/src/asahi/vulkan/hk_agx3_ioto_program_core.c
+++ b/src/asahi/vulkan/hk_agx3_ioto_program_core.c
@@ -76,30 +76,45 @@ hk_agx3_g17p_rt0_tile_layout_supported(struct agx_tilebuffer_layout *tib)
        physical.logical_format[0] == PIPE_FORMAT_NONE)
       physical.nr_samples = 1;
 
+   /* M5's native imageblock reserves one eight-byte sample for a layered
+    * colourless pass (agx_build_tilebuffer_layout_for_imageblock). It holds
+    * no target; check the rest of the layout as the unlayered empty one. */
+   bool colourless = true;
+   for (unsigned rt = 0; rt < AGX_MAX_RENDER_TARGETS; ++rt)
+      colourless &= physical.logical_format[rt] == PIPE_FORMAT_NONE;
+   if (tib->native_imageblock && tib->layered && physical.nr_samples <= 1 &&
+       physical.sample_size_B == 8 && colourless) {
+      physical.sample_size_B = 0;
+      if (!physical.nr_samples)
+         physical.nr_samples = 1;
+   }
+
    return agx_g17p_rt0_tile_layout_supported(&physical);
 }
 
 bool
-hk_agx3_g17p_direct_color_resolve_supported(
-   struct agx_tilebuffer_layout *tib, unsigned color_attachment_count,
-   bool has_resolve_target, bool average_resolve, bool store_source)
+hk_agx3_format_needs_shader_resolve(enum pipe_format format)
 {
-   if (!tib)
-      return false;
-
    /* A 32-bit float target's lane is stored raw, and resolving it on the
     * way out averages the bit patterns: an R32 or RG32 float view resolved
     * this way reads back values like 4e22 on an A18 Pro. Keep the samples
     * and resolve after the pass instead. */
-   const enum pipe_format format = tib->logical_format[0];
    if (util_format_is_float(format) &&
        util_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 0) ==
           32)
-      return false;
+      return true;
 
    /* An RGB9E5 tile keeps each sample's packed word, which averages no
     * better. */
-   if (format == PIPE_FORMAT_R9G9B9E5_FLOAT)
+   return format == PIPE_FORMAT_R9G9B9E5_FLOAT;
+}
+
+bool
+hk_agx3_g17p_direct_color_resolve_supported(
+   struct agx_tilebuffer_layout *tib, unsigned color_attachment_count,
+   bool has_resolve_target, bool average_resolve, bool store_source)
+{
+   if (!tib || hk_agx3_format_needs_shader_resolve(tib->logical_format[0]))
       return false;
 
    return (tib->nr_samples == 2 || tib->nr_samples == 4) &&
@@ -160,7 +175,10 @@ hk_agx3_g17p_basic_graphics_program_generate(
    const struct agx3_g17p_graphics_program_desc shared_desc =
       hk_agx3_g17p_shared_graphics_desc(desc);
    struct agx3_g17p_graphics_program shared;
-   if (!agx3_g17p_graphics_program_generate(chip, &shared_desc, &shared))
+   const bool generated = agx_chip_g17_m5(chip)
+      ? agx3_g17g_graphics_program_generate(chip, &shared_desc, &shared)
+      : agx3_g17p_graphics_program_generate(chip, &shared_desc, &shared);
+   if (!generated)
       return false;
    memcpy(program->bytes, shared.bytes, shared.length);
    program->length = shared.length;
@@ -177,8 +195,9 @@ hk_agx3_g17p_graphics_program_template_generate(
       return false;
    const struct agx3_g17p_graphics_program_desc shared_desc =
       hk_agx3_g17p_shared_graphics_desc(desc);
-   return agx3_g17p_graphics_program_template_generate(chip, &shared_desc,
-                                                        program);
+   return agx_chip_g17_m5(chip)
+      ? agx3_g17g_graphics_program_template_generate(chip, &shared_desc, program)
+      : agx3_g17p_graphics_program_template_generate(chip, &shared_desc, program);
 }
 
 bool
diff --git a/src/asahi/vulkan/hk_agx3_ioto_program_core.h b/src/asahi/vulkan/hk_agx3_ioto_program_core.h
index 076d9b2b25b2..1156ac91ca35 100644
--- a/src/asahi/vulkan/hk_agx3_ioto_program_core.h
+++ b/src/asahi/vulkan/hk_agx3_ioto_program_core.h
@@ -11,6 +11,7 @@
 #include "asahi/compiler/agx3_graphics_program.h"
 #include "asahi/lib/agx3_compute_program_core.h"
 #include "asahi/lib/agx_ioto_program_core.h"
+#include "util/format/u_format.h"
 
 #include <stdbool.h>
 #include <stddef.h>
@@ -22,6 +23,8 @@ extern "C" {
 
 struct agx_tilebuffer_layout;
 
+bool hk_agx3_format_needs_shader_resolve(enum pipe_format format);
+
 /* Describe a nonempty compiler preamble uploaded with its whole binary.
  * Checks both CPU and executable extents. The caller keeps the immutable CPU
  * binary alive during composition and the BO alive through execution. This
diff --git a/src/asahi/vulkan/hk_agx3_vdm_decode.h b/src/asahi/vulkan/hk_agx3_vdm_decode.h
index 276bdb000e0e..2d37bf910628 100644
--- a/src/asahi/vulkan/hk_agx3_vdm_decode.h
+++ b/src/asahi/vulkan/hk_agx3_vdm_decode.h
@@ -24,7 +24,7 @@ extern "C" {
 /* Bump whenever the Honeykrisp shader payload below changes.  Vulkan shader
  * binaries carry this version outside the driver-private blob, so stale
  * layouts are rejected before any fields are consumed. */
-#define HK_AGX3_SHADER_BINARY_VERSION 35u
+#define HK_AGX3_SHADER_BINARY_VERSION 36u
 #define HK_CLASSIC_SHADER_BINARY_VERSION 34u
 /* AGX2 image-acquire binaries now include memory-to-texture visibility. */
 #define HK_AGX2_SHADER_BINARY_VERSION 36u
@@ -40,6 +40,8 @@ hk_shader_binary_version(enum agx_chip chip)
    case AGX_CHIP_G15P:
       return HK_AGX2_SHADER_BINARY_VERSION;
    case AGX_CHIP_G17P:
+   case AGX_CHIP_G17G:
+   case AGX_CHIP_G17S:
       return HK_AGX3_SHADER_BINARY_VERSION;
    default:
       return HK_CLASSIC_SHADER_BINARY_VERSION;
diff --git a/src/asahi/vulkan/hk_buffer.c b/src/asahi/vulkan/hk_buffer.c
index 9035b40653cc..6db531756766 100644
--- a/src/asahi/vulkan/hk_buffer.c
+++ b/src/asahi/vulkan/hk_buffer.c
@@ -7,12 +7,57 @@
 #include "hk_buffer.h"
 #include "hk_acceleration_structure.h"
 
+#include <errno.h>
+
 #include "agx_bo.h"
 #include "agx_device.h"
 #include "hk_device.h"
 #include "hk_device_memory.h"
 #include "hk_entrypoints.h"
 #include "hk_physical_device.h"
+#include "util/u_atomic.h"
+
+VkResult
+hk_update_physical_sparse_map(struct hk_device *dev, uint64_t address,
+                               uint64_t size, bool nonresident)
+{
+   struct agx_bo *bo = dev->physical_sparse_map;
+   if (!bo || !size)
+      return VK_SUCCESS;
+   const uint64_t base = dev->dev.main_heap_base;
+   const uint64_t span = dev->dev.main_heap_size;
+   if (address < base || address - base > span ||
+       size > span - (address - base) ||
+       (address | size) & (AIL_PAGESIZE - 1))
+      return vk_error(dev, VK_ERROR_UNKNOWN);
+
+   const uint64_t first = (address - base) / AIL_PAGESIZE;
+   const uint64_t last = first + size / AIL_PAGESIZE;
+   uint32_t *header = agx_bo_map(bo);
+   uint32_t *map = header + 4;
+   int32_t changed = 0;
+   for (uint64_t page = first; page < last;) {
+      const unsigned count = MIN2(last - page, 32 - (page & 31));
+      const uint32_t mask = BITFIELD_MASK(count) << (page & 31);
+      if (nonresident) {
+         uint32_t old = __atomic_fetch_or(&map[page >> 5], mask,
+                                           __ATOMIC_RELAXED);
+         changed += util_bitcount(~old & mask);
+      } else {
+         uint32_t old = __atomic_fetch_and(&map[page >> 5], ~mask,
+                                            __ATOMIC_RELAXED);
+         changed -= util_bitcount(old & mask);
+      }
+      page += count;
+   }
+   __atomic_fetch_add(&header[0], changed, __ATOMIC_RELEASE);
+   const uint64_t offset = 16 + (first >> 5) * sizeof(uint32_t);
+   const uint64_t end = 16 + DIV_ROUND_UP(last, 32) * sizeof(uint32_t);
+   if (agx_bo_sync_for_device(bo, offset, end - offset) ||
+       agx_bo_sync_for_device(bo, 0, 16))
+      return vk_error(dev, VK_ERROR_DEVICE_LOST);
+   return VK_SUCCESS;
+}
 
 static uint32_t
 hk_get_buffer_alignment(const struct hk_physical_device *pdev,
@@ -83,23 +128,52 @@ hk_bind_scratch(struct hk_device *dev, struct agx_va *va, unsigned offset_B,
                 size_t size_B)
 {
    uint64_t addr = va->addr + offset_B;
-   uint32_t flags = DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_SINGLE_PAGE;
+   uint32_t flags = DRM_ASAHI_BIND_READ | agx_fill_bind_flags(&dev->dev);
 
+   /* The kernel refuses a fill it cannot back (ENOMEM; on M5 also its
+    * hypervisor page-table budget): that is device memory exhaustion. */
    /* Map read-write scratch to the primary (bottom half) VA range */
    int ret = agx_bo_bind(&dev->dev, dev->dev.scratch_bo, addr, size_B, 0,
                          flags | DRM_ASAHI_BIND_WRITE);
    if (ret)
-      return VK_ERROR_UNKNOWN;
+      return errno == ENOMEM ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_ERROR_UNKNOWN;
 
    /* Map read-only scratch to the secondary (top half) VA range */
    ret = agx_bo_bind(&dev->dev, dev->dev.zero_bo,
                      addr + dev->dev.sparse_ro_offset, size_B, 0, flags);
    if (ret)
-      return VK_ERROR_UNKNOWN;
+      return errno == ENOMEM ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_ERROR_UNKNOWN;
 
    return VK_SUCCESS;
 }
 
+/* M5 publishes every scratch-filled sparse page through the hypervisor,
+ * which allocates one page-table frame from its own heap per 32 MiB of VA and
+ * never frees it. Keep each device within its advertised
+ * sparseAddressSpaceSize; beyond it creation fails cleanly.
+ */
+VkResult
+hk_reserve_sparse_scratch(struct hk_device *dev, uint64_t size_B)
+{
+   if (!agx_chip_g17_m5(dev->dev.chip))
+      return VK_SUCCESS;
+
+   uint64_t used = p_atomic_add_return(&dev->sparse_scratch_B, size_B);
+   if (used > HK_M5_SPARSE_ADDR_SPACE_SIZE) {
+      p_atomic_add(&dev->sparse_scratch_B, -(int64_t)size_B);
+      return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
+                       "M5 sparse address space exhausted");
+   }
+   return VK_SUCCESS;
+}
+
+void
+hk_release_sparse_scratch(struct hk_device *dev, uint64_t size_B)
+{
+   if (size_B)
+      p_atomic_add(&dev->sparse_scratch_B, -(int64_t)size_B);
+}
+
 VKAPI_ATTR VkResult VKAPI_CALL
 hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo,
                 const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer)
@@ -149,17 +223,38 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo,
       }
       buffer->vk.device_address = buffer->va->addr;
 
-      /* Bind scratch pages to make read/write across the VA valid */
-      VkResult result = hk_bind_scratch(dev, buffer->va, 0, vma_size_B);
-      if (result != VK_SUCCESS) {
-         agx_va_free(&dev->dev, buffer->va, true);
-         buffer->va = NULL;
-         vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk);
-         return result;
+      /* Binding-only sparse buffers must be fully bound before GPU use.
+       * On M5, reserve their VA without eagerly populating both views with
+       * scratch pages through SPTM. Partially resident buffers still need
+       * safe unbound accesses.
+       */
+      const bool binding_only =
+         agx_chip_g17_m5(dev->dev.chip) &&
+         (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) &&
+         !(buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT);
+      if (!binding_only) {
+         VkResult result = hk_reserve_sparse_scratch(dev, vma_size_B);
+         if (result != VK_SUCCESS) {
+            agx_va_free(&dev->dev, buffer->va, true);
+            buffer->va = NULL;
+            vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk);
+            return result;
+         }
+         buffer->sparse_scratch_B =
+            agx_chip_g17_m5(dev->dev.chip) ? vma_size_B : 0;
+         result = hk_bind_scratch(dev, buffer->va, 0, vma_size_B);
+         if (result != VK_SUCCESS) {
+            hk_release_sparse_scratch(dev, buffer->sparse_scratch_B);
+            agx_va_free(&dev->dev, buffer->va, true);
+            buffer->va = NULL;
+            vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk);
+            return result;
+         }
       }
 
-      if (dev->dev.chip == AGX_CHIP_G17P &&
-         (buffer->vk.create_flags &
+      if ((dev->dev.chip == AGX_CHIP_G17P ||
+           agx_chip_g17_m5(dev->dev.chip)) &&
+          (buffer->vk.create_flags &
            VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT)) {
          const uint64_t entries = DIV_ROUND_UP(vma_size_B, AIL_PAGESIZE);
 
@@ -167,6 +262,7 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo,
              DIV_ROUND_UP(entries, 32) >
                 (SIZE_MAX - sizeof(struct hk_sparse_buffer_map_header)) /
                    sizeof(uint32_t)) {
+            hk_release_sparse_scratch(dev, buffer->sparse_scratch_B);
             agx_va_free(&dev->dev, buffer->va, true);
             buffer->va = NULL;
             vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk);
@@ -186,6 +282,7 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo,
          if (!buffer->sparse_map || !agx_bo_map(buffer->sparse_map)) {
             agx_bo_unreference(&dev->dev, buffer->sparse_map);
             buffer->sparse_map = NULL;
+            hk_release_sparse_scratch(dev, buffer->sparse_scratch_B);
             agx_va_free(&dev->dev, buffer->va, true);
             buffer->va = NULL;
             vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk);
@@ -204,6 +301,20 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo,
       }
    }
 
+   if (buffer->sparse_map) {
+      VkResult result = hk_update_physical_sparse_map(
+         dev, buffer->va->addr, align64(buffer->vk.size, AIL_PAGESIZE), true);
+      if (result != VK_SUCCESS) {
+         hk_update_physical_sparse_map(dev, buffer->va->addr,
+            align64(buffer->vk.size, AIL_PAGESIZE), false);
+         agx_bo_unreference(&dev->dev, buffer->sparse_map);
+         hk_release_sparse_scratch(dev, buffer->sparse_scratch_B);
+         agx_va_free(&dev->dev, buffer->va, true);
+         vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk);
+         return result;
+      }
+   }
+
    *pBuffer = hk_buffer_to_handle(buffer);
 
    return VK_SUCCESS;
@@ -219,9 +330,14 @@ hk_DestroyBuffer(VkDevice device, VkBuffer _buffer,
    if (!buffer)
       return;
 
+   if (buffer->sparse_map)
+      hk_update_physical_sparse_map(dev, buffer->va->addr,
+         align64(buffer->vk.size, AIL_PAGESIZE), false);
+
    if (buffer->va) {
       agx_va_free(&dev->dev, buffer->va, true);
    }
+   hk_release_sparse_scratch(dev, buffer->sparse_scratch_B);
 
    agx_bo_unreference(&dev->dev, buffer->sparse_map);
 
@@ -346,10 +462,12 @@ hk_BindBufferMemory2(VkDevice device, uint32_t bindInfoCount,
             mem->bo->va->addr + pBindInfos[i].memoryOffset;
       }
 
-      if (buffer->vk.usage & VK_BUFFER_USAGE_2_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR) {
-         VK_FROM_HANDLE(hk_device, dev, device);
+      VK_FROM_HANDLE(hk_device, dev, device);
+      if (dev->g17p_native_as &&
+          (buffer->vk.usage & VK_BUFFER_USAGE_2_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR))
          hk_gen1_publish_buffer_as_addresses(dev, &buffer->vk);
-      }
+      if (dev->m5_native_as)
+         hk_m5_as_buffer_bound(dev, &buffer->vk);
 
       const VkBindMemoryStatusKHR *status =
          vk_find_struct_const(pBindInfos[i].pNext, BIND_MEMORY_STATUS_KHR);
diff --git a/src/asahi/vulkan/hk_buffer.h b/src/asahi/vulkan/hk_buffer.h
index fe1eb46c8e0b..8077dddf5e77 100644
--- a/src/asahi/vulkan/hk_buffer.h
+++ b/src/asahi/vulkan/hk_buffer.h
@@ -21,7 +21,10 @@ struct hk_buffer {
    /** Reserved VA for sparse buffers, NULL otherwise. */
    struct agx_va *va;
 
-   /** One residency bit per sparse-buffer page, G17P only. */
+   /** Bytes of that VA counted in hk_device::sparse_scratch_B. */
+   uint64_t sparse_scratch_B;
+
+   /** One residency bit per sparse-buffer page on Gen9/Gen10 AGX3. */
    struct agx_bo *sparse_map;
    uint32_t sparse_map_entries;
 };
@@ -92,5 +95,12 @@ hk_buffer_addr_range(const struct hk_buffer *buffer, uint64_t offset,
    };
 }
 
+VkResult hk_reserve_sparse_scratch(struct hk_device *dev, uint64_t size_B);
+void hk_release_sparse_scratch(struct hk_device *dev, uint64_t size_B);
+
 VkResult hk_bind_scratch(struct hk_device *dev, struct agx_va *va,
                          unsigned offs_B, size_t size_B);
+
+/* Publish a bound/unbound sparse buffer range to physical pointer reads. */
+VkResult hk_update_physical_sparse_map(struct hk_device *dev, uint64_t address,
+                                       uint64_t size, bool nonresident);
diff --git a/src/asahi/vulkan/hk_buffer_view.c b/src/asahi/vulkan/hk_buffer_view.c
index 91695c339339..dbd167a3c0ee 100644
--- a/src/asahi/vulkan/hk_buffer_view.c
+++ b/src/asahi/vulkan/hk_buffer_view.c
@@ -56,7 +56,8 @@ hk_get_buffer_format_features(struct hk_physical_device *pdev,
 
       if (p_format == PIPE_FORMAT_R32_UINT || p_format == PIPE_FORMAT_R32_SINT ||
           (p_format == PIPE_FORMAT_R32_FLOAT &&
-           pdev->dev.chip == AGX_CHIP_G17P))
+           (pdev->dev.chip == AGX_CHIP_G17P ||
+            agx_chip_g17_m5(pdev->dev.chip))))
          features |= VK_FORMAT_FEATURE_2_STORAGE_TEXEL_BUFFER_ATOMIC_BIT;
    }
 
@@ -79,7 +80,8 @@ hk_CreateBufferView(VkDevice _device, const VkBufferViewCreateInfo *pCreateInfo,
 
    enum pipe_format format = hk_format_to_pipe_format(view->vk.format);
    const struct util_format_description *desc = util_format_description(format);
-   const bool g17p = agx_native_graphics(&device->dev);
+   const bool g17p = agx_native_graphics(&device->dev) ||
+                      agx_chip_g17_m5(device->dev.chip);
    const bool rgb32 =
       ail_pixel_format[format].channels == AGX_CHANNELS_R32G32B32_EMULATED;
 
@@ -256,10 +258,11 @@ hk_CreateBufferView(VkDevice _device, const VkBufferViewCreateInfo *pCreateInfo,
 
    if (hk_native_pbe_supported(generation)) {
       hk_pack_native_pbe(generation, &view->pbe, &pbe);
-   } else if (agx_native_graphics(&device->dev)) {
-      /* Gen9's linear PBE is architectural and has no software tail field,
+   } else if (agx_native_graphics(&device->dev) ||
+              agx_chip_g17_m5(device->dev.chip)) {
+      /* AGX3's linear PBE is architectural and has no software tail field,
        * exactly like the texture above: the store lowering adds the tail from
-       * the buffer metadata. A format the Gen9 packer cannot represent (the
+       * the buffer metadata. A format the native packer cannot represent (the
        * emulated RGB32 layout) has no storage usage, so its view keeps the
        * null PBE.
        */
diff --git a/src/asahi/vulkan/hk_bvh.h b/src/asahi/vulkan/hk_bvh.h
index ae1ed074c94b..0a4da9cc258e 100644
--- a/src/asahi/vulkan/hk_bvh.h
+++ b/src/asahi/vulkan/hk_bvh.h
@@ -6,20 +6,14 @@
  *
  * Honeykrisp software-traversal BVH node format.
  *
- * This is a SOFTWARE format, traversed by shader ALU and global loads.  It is
- * deliberately NOT Apple's RIA BVH layout and it is deliberately not consumed
- * by the AGX3 `intersect_ray` node-test instruction:
+ * The current builder produces this software format for shader ALU traversal.
+ * It cannot be passed to M5's native intersect_ray/continue_ray path. Native
+ * M5 uses the Gen1 node layout for BVH version 3 and an SCS query protocol;
+ * the complete acceleration-structure envelope still needs integrating.
+ * The byte encoders and their regression fixtures live in agx_rt_bvh.h and
+ * tests/test-rt-bvh.c. Hardware traversal must use that native format.
  *
- *  - Apple's `intersect_ray` is a per-node test accelerator, not a traversal
- *    engine (the loop, the stack and commit are software either way), so a
- *    software traversal gives up far less than it sounds like.
- *  - The SCS slot layout that `intersect_ray` reads and writes is only
- *    partially mapped, so there is no lowering template for it.
- *  - agx_rt_bvh.h's Gen5 layout carries a "validated over 64,000,000
- *    randomized headers" claim with no harness anywhere in the tree; it must
- *    not be adopted on that strength.
- *
- * So this format is ours to choose, and it is chosen to be byte-identical to
+ * This software layout is byte-identical to
  * lavapipe's (src/gallium/frontends/lavapipe/lvp_acceleration_structure.h),
  * because the NIR traversal in hk_nir_ray_tracing.c is a port of lavapipe's
  * and keeping the layouts identical keeps that port auditable field by field.
diff --git a/src/asahi/vulkan/hk_cmd_buffer.c b/src/asahi/vulkan/hk_cmd_buffer.c
index 420f0ce9e71d..40460151ddf9 100644
--- a/src/asahi/vulkan/hk_cmd_buffer.c
+++ b/src/asahi/vulkan/hk_cmd_buffer.c
@@ -5,7 +5,12 @@
  * SPDX-License-Identifier: MIT
  */
 #include "hk_cmd_buffer.h"
+#include "util/format/u_format.h"
+#include "hk_frame_anatomy.h"
+#include "hk_ppp_state.h"
+#include "asahi/compiler/agx3_graphics_program.h"
 #include "hk_g17p_patch.h"
+#include "asahi/lib/agx_scs_layout.h"
 
 #include "agx_abi.h"
 #include "agx_bo.h"
@@ -20,6 +25,7 @@
 #include "hk_agx3_vdm_decode.h"
 #include "asahi/lib/agx_ioto_load_core.h"
 #include "hk_buffer.h"
+#include "hk_acceleration_structure.h"
 #include "hk_cmd_pool.h"
 #include "hk_descriptor_set.h"
 #include "hk_descriptor_set_layout.h"
@@ -33,6 +39,11 @@
 #include "libagx_shaders.h"
 
 #include "util/cache_ops.h"
+#include "util/os_misc.h"
+#include "util/u_atomic.h"
+#include <inttypes.h>
+#include <stdio.h>
+#include <unistd.h>
 #include "util/log.h"
 #include "util/hash_table.h"
 #include "util/set.h"
@@ -47,6 +58,7 @@
 #include "asahi/lib/agx_ppp_w_clamp_cull2_core.h"
 #include "asahi/lib/agx_vdm_barrier_core.h"
 #include "asahi/lib/agx_vdm_stream_link_core.h"
+#include "asahi/lib/agx_m5_va.h"
 #include "asahi/lib/agx_vdm_stream_terminate_core.h"
 #include "asahi/lib/agx_va_runtime_hole.h"
 #include "libagx_dgc.h"
@@ -163,9 +175,92 @@ hk_descriptor_state_fini(struct hk_cmd_buffer *cmd,
    }
 }
 
+struct hk_m5_rt_state_diag_entry {
+   uint8_t *cpu;
+   uint64_t gpu, size;
+   uint8_t *initial;
+};
+
+/* Opt-in diagnostic (ASAHI_M5_RT_STATE_DIAG=<dir>): remember the initial
+ * contents of each SCS state block this recording allocates, so the bytes the
+ * GPU changed can be reported once the application has finished with the
+ * command buffer. No effect on the recorded commands. */
+void
+hk_m5_rt_state_diag_record(struct hk_cmd_buffer *cmd, struct agx_ptr state,
+                           uint64_t size)
+{
+   struct hk_m5_rt_state_diag_entry e = {
+      .cpu = state.cpu, .gpu = state.gpu, .size = size, .initial = malloc(size),
+   };
+   if (!e.initial)
+      return;
+   memcpy(e.initial, state.cpu, size);
+   util_dynarray_append(&cmd->m5_rt_state.diag, e);
+}
+
+static void
+hk_m5_rt_state_diag_flush(struct hk_cmd_buffer *cmd)
+{
+   const char *dir = os_get_option("ASAHI_M5_RT_STATE_DIAG");
+   static unsigned seq;
+   util_dynarray_foreach(&cmd->m5_rt_state.diag,
+                         struct hk_m5_rt_state_diag_entry, e) {
+      const unsigned n = p_atomic_inc_return(&seq);
+      char path[512];
+      snprintf(path, sizeof(path), "%s/state-%d-%u.txt", dir ? dir : "/tmp",
+               (int)getpid(), n);
+      FILE *f = fopen(path, "w");
+      if (f) {
+         uint64_t changed = 0, ranges = 0, start = UINT64_MAX, last = 0;
+         fprintf(f, "gpu 0x%" PRIx64 " size 0x%" PRIx64 "\n", e->gpu, e->size);
+         for (uint64_t o = 0; o < e->size; o += 16) {
+            if (!memcmp(e->cpu + o, e->initial + o, 16))
+               continue;
+            changed += 16;
+            if (start != UINT64_MAX && o > last + 256) {
+               fprintf(f, "range 0x%" PRIx64 " 0x%" PRIx64 "\n", start, last + 16);
+               ranges++;
+               start = UINT64_MAX;
+            }
+            if (start == UINT64_MAX)
+               start = o;
+            last = o;
+         }
+         if (start != UINT64_MAX) {
+            fprintf(f, "range 0x%" PRIx64 " 0x%" PRIx64 "\n", start, last + 16);
+            ranges++;
+         }
+         fprintf(f, "changed16 %" PRIu64 " ranges %" PRIu64 "\n", changed, ranges);
+         fclose(f);
+      }
+      if (n <= 4) {
+         snprintf(path, sizeof(path), "%s/state-%d-%u.final.bin", dir ? dir : "/tmp",
+                  (int)getpid(), n);
+         f = fopen(path, "wb");
+         if (f) {
+            fwrite(e->cpu, 1, e->size, f);
+            fclose(f);
+         }
+         snprintf(path, sizeof(path), "%s/state-%d-%u.initial.bin", dir ? dir : "/tmp",
+                  (int)getpid(), n);
+         f = fopen(path, "wb");
+         if (f) {
+            fwrite(e->initial, 1, e->size, f);
+            fclose(f);
+         }
+      }
+      free(e->initial);
+   }
+   util_dynarray_fini(&cmd->m5_rt_state.diag);
+   memset(&cmd->m5_rt_state.diag, 0, sizeof(cmd->m5_rt_state.diag));
+}
+
 static void
 hk_free_resettable_cmd_buffer(struct hk_cmd_buffer *cmd)
 {
+   if (cmd->m5_rt_state.diag.size)
+      hk_m5_rt_state_diag_flush(cmd);
+
    struct hk_cmd_pool *pool = hk_cmd_buffer_pool(cmd);
    struct hk_device *dev = hk_cmd_pool_device(pool);
 
@@ -304,10 +399,27 @@ hk_reset_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer,
       _mesa_set_clear(cmd->m5_pushed_images, NULL);
    util_dynarray_clear(&cmd->oq_ends);
 
-   memset(&cmd->state, 0, sizeof(cmd->state));
-   for (unsigned i = 0; i < HK_G17P_BG_EOT_CACHE_ENTRIES; ++i)
-      cmd->g17p_bg_eot_cache[i].valid = false;
-   cmd->g17p_bg_eot_cache_next = 0;
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (dev->dev.params.gpu_generation == 13 ||
+       dev->dev.params.gpu_generation == 14) {
+      /* These native graphics caches are zero-initialized at allocation and
+       * never used on AGX2. Reset the common state on either side without
+       * clearing their 8.5 KiB again for every recording. */
+      const size_t first =
+         offsetof(struct hk_cmd_buffer, state.gfx.g17p_ioto) -
+         offsetof(struct hk_cmd_buffer, state);
+      const size_t end =
+         offsetof(struct hk_cmd_buffer, state.gfx.g17p_spilled_rt_descs) +
+         sizeof(cmd->state.gfx.g17p_spilled_rt_descs) -
+         offsetof(struct hk_cmd_buffer, state);
+      memset(&cmd->state, 0, first);
+      memset((char *)&cmd->state + end, 0, sizeof(cmd->state) - end);
+   } else {
+      memset(&cmd->state, 0, sizeof(cmd->state));
+      for (unsigned i = 0; i < HK_G17P_BG_EOT_CACHE_ENTRIES; ++i)
+         cmd->g17p_bg_eot_cache[i].valid = false;
+      cmd->g17p_bg_eot_cache_next = 0;
+   }
    /* Nothing has been recorded since this reset. */
    cmd->g17p_pristine = true;
 }
@@ -427,6 +539,9 @@ hk_pool_alloc_from_slow(struct hk_cmd_buffer *cmd,
       vk_command_buffer_set_error(&cmd->vk, result);
       return (struct agx_ptr){0};
    }
+   bo->stream_only = type == HK_CMD_BO_MAIN &&
+                     uploader == &cmd->uploader.streams;
+
    /* Pick whichever of the current upload BO and the new BO will have more
     * room left to be the BO for the next upload.  If our upload size is
     * bigger than the old offset, we're better off burning the whole new
@@ -563,6 +678,12 @@ hk_cmd_buffer_emit_cs_jump(struct hk_cmd_buffer *cmd, void *current,
 
    if (vdm && hk_g17p_vdm_low_window_enabled(&dev->dev))
       target = hk_g17p_vdm_compact(cmd, target);
+   else if (vdm && agx_chip_g17_m5(dev->dev.chip)) {
+      if (!agx_m5_vdm_compact(target, &target)) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+         return current;
+      }
+   }
 
    switch (agx_chip_command_stream_generation(dev->dev.chip)) {
    case AGX_CS_GENERATION_AGX2:
@@ -707,7 +828,9 @@ hk_BeginCommandBuffer(VkCommandBuffer commandBuffer,
  * buffer. However, it is challenging to avoid constructing such sequences due
  * to the gymnastics required to reorder compute around graphics. Merging at
  * EndCommandBuffer is cheap O(# of control streams) and lets us get away with
- * the sloppiness.
+ * the sloppiness. On M5 a stream that waits for a render the one before it
+ * does not wait for stays separate (hk_sync_merge_delays()): merged, the
+ * earlier work would wait for that render too.
  */
 static void
 merge_control_streams(struct hk_cmd_buffer *cmd)
@@ -718,7 +841,8 @@ merge_control_streams(struct hk_cmd_buffer *cmd)
       if (cs->type == HK_CS_CDM && last && last->type == HK_CS_CDM &&
           !last->timestamp.end.handle &&
           !(last->timestamp.start.handle && cs->g17p_timestamp_query) &&
-          !(last->timestamp.start.handle && cs->timestamp.start.handle)) {
+          !(last->timestamp.start.handle && cs->timestamp.start.handle) &&
+          !hk_sync_merge_delays(cmd, last, cs)) {
 
          hk_sync_merge(cmd, last, cs);
          hk_cs_merge_cdm(last, cs);
@@ -750,6 +874,35 @@ hk_cmd_buffer_publish_root_uploads(struct hk_cmd_buffer *cmd)
    }
 }
 
+/* A dedicated stream BO can reserve much more space than the stream wrote.
+ * For an unlinked stream, its current pointer and end delimit the written
+ * records plus the existing prefetch padding. Removed inline-merged streams
+ * are no longer referenced. Linked layouts retain the conservative path. */
+static bool
+hk_cmd_bo_clean_unlinked_streams(struct hk_cmd_buffer *cmd,
+                                 const struct hk_cmd_bo *bo)
+{
+   const uintptr_t base = (uintptr_t)bo->map;
+   if (bo->bo->size > UINTPTR_MAX - base)
+      return false;
+   const uintptr_t limit = base + bo->bo->size;
+
+   list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node) {
+      const uintptr_t start = (uintptr_t)cs->start;
+      if (start < base || start >= limit)
+         continue;
+      const uintptr_t current = (uintptr_t)cs->current;
+      const uintptr_t end = (uintptr_t)cs->end;
+      if (current < start || current > end || end > limit)
+         return false;
+      const size_t padding = MIN2(end - current, HK_CS_SPACE_PADDING);
+      const size_t bytes = current - start + padding;
+      if (bytes)
+         util_flush_range_no_fence((void *)start, bytes);
+   }
+   return true;
+}
+
 /* Clean everything recording wrote to CPU-cached command memory (see
  * hk_g17p_cached_cmd_memory()); hk_cmd_buffer_publish_root_uploads() and
  * queue submission fence the cleans. */
@@ -760,8 +913,13 @@ hk_cmd_buffer_publish_cached_uploads(struct hk_cmd_buffer *cmd)
    if (!hk_g17p_cached_cmd_memory(&dev->dev))
       return;
 
+   bool linked_stream = false;
+   list_for_each_entry(struct hk_cs, cs, &cmd->control_streams, node)
+      linked_stream |= cs->stream_linked;
+
    /* An uploader's current BO is written up to its offset; a BO it moved
-    * on from may have been written anywhere. */
+    * on from may have been written anywhere. Dedicated unlinked stream BOs
+    * instead have exact written extents, independent of reserved capacity. */
    const struct hk_uploader *current[] = {
       &cmd->uploader.main, &cmd->uploader.streams, &cmd->uploader.usc,
    };
@@ -769,6 +927,9 @@ hk_cmd_buffer_publish_cached_uploads(struct hk_cmd_buffer *cmd)
       struct list_head *bos =
          l == 0 ? &cmd->uploader.main.bos : &cmd->uploader.usc.bos;
       list_for_each_entry(struct hk_cmd_bo, bo, bos, link) {
+         if (bo->stream_only && !linked_stream &&
+             hk_cmd_bo_clean_unlinked_streams(cmd, bo))
+            continue;
          size_t size = bo->bo->size;
          for (unsigned u = 0; u < ARRAY_SIZE(current); ++u) {
             if (current[u]->map == bo->map)
@@ -798,6 +959,7 @@ hk_EndCommandBuffer(VkCommandBuffer commandBuffer)
    hk_sync_flush_pending_timestamp(cmd);
    hk_cmd_buffer_end_compute(cmd);
    hk_cmd_buffer_end_compute_internal(cmd, &cmd->current_cs.post_gfx);
+   hk_resolve_deferred_oq_available(cmd);
 
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    if (likely(!(dev->dev.debug & AGX_DBG_NOMERGE))) {
@@ -908,7 +1070,34 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer,
 
    perf_debug(cmd, "Pipeline barrier");
 
-   if (!agx_native_graphics(&dev->dev)) {
+   if (unlikely(dev->frame_anatomy)) {
+      const VkDependencyInfo *dep = pDependencyInfo;
+      for (uint32_t i = 0; i < dep->memoryBarrierCount; ++i) {
+         cmd->anatomy_bar.src |= dep->pMemoryBarriers[i].srcStageMask;
+         cmd->anatomy_bar.dst |= dep->pMemoryBarriers[i].dstStageMask;
+         cmd->anatomy_bar.src_access |= dep->pMemoryBarriers[i].srcAccessMask;
+         cmd->anatomy_bar.dst_access |= dep->pMemoryBarriers[i].dstAccessMask;
+      }
+      for (uint32_t i = 0; i < dep->bufferMemoryBarrierCount; ++i) {
+         cmd->anatomy_bar.src |= dep->pBufferMemoryBarriers[i].srcStageMask;
+         cmd->anatomy_bar.dst |= dep->pBufferMemoryBarriers[i].dstStageMask;
+         cmd->anatomy_bar.src_access |= dep->pBufferMemoryBarriers[i].srcAccessMask;
+         cmd->anatomy_bar.dst_access |= dep->pBufferMemoryBarriers[i].dstAccessMask;
+      }
+      for (uint32_t i = 0; i < dep->imageMemoryBarrierCount; ++i) {
+         cmd->anatomy_bar.src |= dep->pImageMemoryBarriers[i].srcStageMask;
+         cmd->anatomy_bar.dst |= dep->pImageMemoryBarriers[i].dstStageMask;
+         cmd->anatomy_bar.src_access |= dep->pImageMemoryBarriers[i].srcAccessMask;
+         cmd->anatomy_bar.dst_access |= dep->pImageMemoryBarriers[i].dstAccessMask;
+      }
+      cmd->anatomy_bar.mem += dep->memoryBarrierCount;
+      cmd->anatomy_bar.buf += dep->bufferMemoryBarrierCount;
+      cmd->anatomy_bar.img += dep->imageMemoryBarrierCount;
+      cmd->anatomy_bar.count++;
+   }
+
+   /* M5 records its barriers with the native dependency tracker. */
+   if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) {
       hk_cmd_buffer_end_compute(cmd);
       hk_cmd_buffer_end_graphics(cmd);
       /* Preserve the same conservative ordering in the shared submit graph,
@@ -972,9 +1161,9 @@ hk_CmdPipelineBarrier2(VkCommandBuffer commandBuffer,
 
    /* A barrier-only command buffer otherwise records no GPU work, so it
     * cannot make a preceding fragment store visible to the host. Emit the
-    * exact G17P strong CDM cache-maintenance record only when the API requests
+    * native strong CDM cache-maintenance record only when the API requests
     * host-read availability, in a stream ordered after the source scope. */
-   if (dev->dev.chip == AGX_CHIP_G17P &&
+   if ((dev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->dev.chip)) &&
        hk_dependency_has_host_read(pDependencyInfo)) {
       struct hk_sync_dep dep[HK_SYNC_NUM_SQ];
       hk_sync_src_dep(cmd, src_kinds, cmd->current_cs.cs, dep);
@@ -1233,13 +1422,14 @@ hk_descriptor_type_has_image_view(VkDescriptorType type)
           type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT;
 }
 
-/* M5 render pass continuation: the image views a graphics push descriptor
+/* AGX3 render pass continuation: the image views a graphics push descriptor
  * update writes (hk_m5_pass_merge_note_push()). */
 static void
 hk_m5_note_push_writes(struct hk_cmd_buffer *cmd, uint32_t count,
                        const VkWriteDescriptorSet *writes)
 {
-   if (!agx_chip_native_agx3(hk_cmd_buffer_device(cmd)->dev.chip))
+   const enum agx_chip chip = hk_cmd_buffer_device(cmd)->dev.chip;
+   if (!agx_chip_native_agx3(chip) && !agx_chip_g17_m5(chip))
       return;
 
    for (uint32_t i = 0; i < count; ++i) {
@@ -1259,7 +1449,8 @@ hk_m5_note_push_template(struct hk_cmd_buffer *cmd,
                          const struct vk_descriptor_update_template *template,
                          const void *data)
 {
-   if (!agx_chip_native_agx3(hk_cmd_buffer_device(cmd)->dev.chip))
+   const enum agx_chip chip = hk_cmd_buffer_device(cmd)->dev.chip;
+   if (!agx_chip_native_agx3(chip) && !agx_chip_g17_m5(chip))
       return;
 
    for (uint32_t i = 0; i < template->entry_count; ++i) {
@@ -1375,6 +1566,12 @@ hk_CmdPushDescriptorSetWithTemplate2KHR(
       pPushDescriptorSetWithTemplateInfo->pData);
 }
 
+DEBUG_GET_ONCE_BOOL_OPTION(m5_fragment_root_memo, "ASAHI_M5_FRAGMENT_ROOT_MEMO",
+                           true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_fragment_root_shadow,
+                           "ASAHI_M5_FRAGMENT_ROOT_SHADOW", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_verify_fragment_root_shadow,
+                           "ASAHI_M5_VERIFY_FRAGMENT_ROOT_SHADOW", false)
 
 static uint64_t
 hk_upload_native_spilled_rt_descs(struct hk_cmd_buffer *cmd)
@@ -1506,6 +1703,19 @@ hk_reserve_scratch_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
 static bool
 hk_g17p_basic_fragment_layout(struct hk_cmd_buffer *cmd)
 {
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const struct agx_tilebuffer_layout *tib = &cmd->state.gfx.render.tilebuffer;
+   if (agx_chip_g17_m5(dev->dev.chip) && tib->nr_samples > 1) {
+      return (tib->nr_samples == 2 || tib->nr_samples == 4 ||
+              tib->nr_samples == 8) &&
+             (tib->tile_size == 32 * 32 || tib->tile_size == 32 * 16 ||
+              tib->tile_size == 16 * 16) &&
+             tib->sample_size_B >= 8 && !(tib->sample_size_B & 7) &&
+             tib->sample_size_B <= tib->max_sample_size_B &&
+             tib->nr_samples * tib->tile_size <= tib->max_samples_per_tile &&
+             tib->sample_size_B * tib->nr_samples * tib->tile_size <=
+                tib->max_bytes_per_tile;
+   }
    return cmd->state.gfx.render.g17p_rt0_layout_supported;
 }
 
@@ -1603,8 +1813,16 @@ hk_g17p_smooth_varyings_supported_uncached(struct hk_cmd_buffer *cmd,
           * generate_primitive_id. Admit it as a system coefficient beside
           * FragCoord Z and 1/W -- scalar, flat, and indexing no vertex
           * output. A VS-written gl_PrimitiveID would take the ordinary user
-          * word path instead and stays refused, having no witness. */
-         if (b->slot == VARYING_SLOT_PRIMITIVE_ID) {
+          * word path instead and stays refused, having no witness. A mesh
+          * shader's per-primitive gl_PrimitiveID is a flat user word of the
+          * primitive's private vertices (agx_m5_mesh_gather_varyings) and
+          * links like any other user word below. */
+         const bool mesh_primitive_id =
+            mesh && vs->info.stage == MESA_SHADER_MESH &&
+            b->slot == VARYING_SLOT_PRIMITIVE_ID &&
+            gfx->linked_varyings.slots[VARYING_SLOT_PRIMITIVE_ID] &&
+            b->count == 1 && b->offset == 0 && !b->smooth && !b->perspective;
+         if (b->slot == VARYING_SLOT_PRIMITIVE_ID && !mesh_primitive_id) {
             if (gfx->linked_varyings.slots[VARYING_SLOT_PRIMITIVE_ID] ||
                 b->count != 1 || b->offset != 0 || b->smooth || b->perspective)
                return false;
@@ -1648,7 +1866,8 @@ hk_g17p_smooth_varyings_supported_uncached(struct hk_cmd_buffer *cmd,
             b->count == vs->info.uvs.components[AGX_MESH_PRIMITIVE_PAGE_SLOT] &&
             b->count >= 1 && b->count <= 2 &&
             b->offset == 0 && !b->smooth && !b->perspective;
-         const bool user_slot = primitive_page || (b->slot >= VARYING_SLOT_VAR0 &&
+         const bool user_slot = primitive_page || mesh_primitive_id ||
+                                (b->slot >= VARYING_SLOT_VAR0 &&
                                  b->slot <= VARYING_SLOT_VAR31) ||
                                 b->slot == VARYING_SLOT_LAYER ||
                                 b->slot == VARYING_SLOT_VIEWPORT || clip_slot ||
@@ -1911,10 +2130,9 @@ hk_g17p_dma_set_base(const struct hk_cmd_buffer *cmd,
  * (agx_shader_key::g17p_dma_static_set_mask), so its contents cannot change
  * before the launch runs. An unbound set reads as a null buffer. */
 static bool
-hk_g17p_dma_buffer(const struct hk_cmd_buffer *cmd, int set,
-                   unsigned descriptor_B, struct hk_buffer_address *buffer)
+hk_g17p_dma_buffer_in(const struct hk_descriptor_state *desc, int set,
+                      unsigned descriptor_B, struct hk_buffer_address *buffer)
 {
-   const struct hk_descriptor_state *desc = &cmd->state.gfx.descriptors;
    const void *table;
    size_t table_B;
    if (set < 0) {
@@ -1942,11 +2160,267 @@ hk_g17p_dma_buffer(const struct hk_cmd_buffer *cmd, int set,
    return true;
 }
 
-/* sources, when not NULL, receives each range's enum hk_g17p_range_source. */
+/* The buffer descriptor of a uniform DMA span from the descriptors bound
+ * at bind_point. */
+static bool
+hk_g17p_dma_buffer(const struct hk_cmd_buffer *cmd,
+                   VkPipelineBindPoint bind_point, int set,
+                   unsigned descriptor_B, struct hk_buffer_address *buffer)
+{
+   const struct hk_descriptor_state *desc =
+      bind_point == VK_PIPELINE_BIND_POINT_COMPUTE ? &cmd->state.cs.descriptors
+                                                   : &cmd->state.gfx.descriptors;
+   return hk_g17p_dma_buffer_in(desc, set, descriptor_B, buffer);
+}
+
+/* Native compute IOTO reads immutable addresses when the launch runs, as
+ * the original constant loads did. Buffer descriptors are eligible only for
+ * layouts whose contents cannot be updated after recording (compiler mask).
+ * Every dispatch resolves the currently bound compute descriptor state; no
+ * pointer-value cache or GPU prepass is introduced. */
+bool
+hk_m5_compute_uniform_dma_ranges(
+   struct hk_cmd_buffer *cmd, const struct hk_shader *s, uint64_t root_ptr,
+   struct agx3_ioto_uniform_range *ranges, size_t capacity, size_t *count)
+{
+   if (!cmd || !s || !ranges || !count || *count > capacity)
+      return false;
+   const unsigned root = s->info.agx3_root_uniform;
+   for (unsigned i = 0; i < s->b.info.agx3_uniform_dma_count; ++i) {
+      const struct agx3_uniform_dma_span *span = &s->b.info.agx3_uniform_dma[i];
+      const unsigned pointer = span->pointer;
+      const int set = pointer == root ? -1
+         : pointer >= root + 4 && (pointer - root - 4) % 4 == 0 &&
+           (pointer - root - 4) / 4 < HK_MAX_SETS
+            ? (int)(pointer - root - 4) / 4 : -2;
+      if (set == -2 || (span->dest & 1) || (span->halfs & 1) ||
+          (span->offset_B & 3) || *count >= capacity)
+         return false;
+      if (span->source == AGX3_UNIFORM_DMA_BUFFER) {
+         struct hk_buffer_address buffer;
+         if (!hk_g17p_dma_buffer(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, set,
+                                span->descriptor_B, &buffer))
+            return false;
+         const uint64_t size_B = (uint64_t)buffer.size & ~UINT64_C(3);
+         const unsigned covered = size_B > span->offset_B
+            ? MIN2(span->halfs, (size_B - span->offset_B) / 2) : 0;
+         if (covered) {
+            const uint64_t address = buffer.base_addr + span->offset_B;
+            if (address & 3)
+               return false;
+            ranges[(*count)++] = (struct agx3_ioto_uniform_range){
+               .start_halfs = span->dest, .size_halfs = covered,
+               .address = address,
+            };
+         }
+         if (covered < span->halfs) {
+            if (*count >= capacity)
+               return false;
+            ranges[(*count)++] = (struct agx3_ioto_uniform_range){
+               .start_halfs = span->dest + covered,
+               .size_halfs = span->halfs - covered,
+               .address = AGX_ZERO_PAGE_ADDRESS,
+            };
+         }
+      } else if (span->source == AGX3_UNIFORM_DMA_POINTER) {
+         const uint64_t base = set < 0 ? root_ptr
+            : cmd->state.cs.descriptors.root.sets[set];
+         if (set < 0 && !base)
+            return false;
+         const uint64_t address = base ? base + span->offset_B
+                                       : AGX_ZERO_PAGE_ADDRESS;
+         if (address & 3)
+            return false;
+         ranges[(*count)++] = (struct agx3_ioto_uniform_range){
+            .start_halfs = span->dest, .size_halfs = span->halfs,
+            .address = address,
+         };
+      } else {
+         return false;
+      }
+   }
+   return true;
+}
+
+static bool
+hk_m5_graphics_query_uniforms(struct hk_cmd_buffer *cmd, bool vertex,
+                             struct agx3_ioto_uniform_range *range)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const uint64_t packed = UINT64_C(0x01140000600100e8);
+   struct agx_scs_layout_gen2 scs;
+   if (!dev->m5_native_rq || !dev->m5_rq_instances ||
+       !agx_scs_layout_gen2_init(packed, &scs) || !cmd->current_cs.gfx)
+      return false;
+   const uint64_t size = MAX2(UINT64_C(0x800000),
+      ALIGN_POT((uint64_t)scs.end[12], 0x4000));
+   struct agx_ptr state = hk_pool_alloc(cmd, size, 0x4000);
+   struct agx_ptr uniforms = hk_pool_alloc(cmd, 512, 16);
+   uint64_t table = hk_m5_upload_as_table(cmd);
+   if (!state.cpu || !uniforms.cpu || !table)
+      return false;
+   memcpy(state.cpu, dev->m5_rq_user_image, 0x800000);
+   if (size > 0x800000)
+      memset((uint8_t *)state.cpu + 0x800000, 0, size - 0x800000);
+   agx_scs_layout_gen2_parameters(&scs, (uint8_t *)state.cpu + 0x80020);
+   memset(uniforms.cpu, 0, 512);
+   memcpy(uniforms.cpu, dev->m5_rq_uniforms, sizeof(dev->m5_rq_uniforms));
+   uint64_t sizes = (uint64_t)scs.end[6] | ((uint64_t)scs.end[11] << 32);
+   uint64_t driver = scs.end[5];
+   memcpy((uint8_t *)uniforms.cpu + 48*2, &sizes, 8);
+   memcpy((uint8_t *)uniforms.cpu + 52*2, &packed, 8);
+   memcpy((uint8_t *)uniforms.cpu + 56*2, &driver, 8);
+   uint64_t query_payload = UINT64_C(0x3fffffff00008000);
+   memcpy((uint8_t *)uniforms.cpu + 152*2, &query_payload, 8);
+   for (unsigned i = 0; i < 80; i += 2) {
+      uint64_t pointer;
+      memcpy(&pointer, (uint8_t *)uniforms.cpu + i*4, 8);
+      if (pointer >= UINT64_C(0x10000000000) && pointer < UINT64_C(0x10000800000)) {
+         pointer += state.gpu - UINT64_C(0x10000000000);
+         memcpy((uint8_t *)uniforms.cpu + i*4, &pointer, 8);
+      }
+   }
+   memcpy((uint8_t *)uniforms.cpu + 32*2, &table, 8);
+   uint32_t context[32] = {0, 0, (uint32_t)packed, (uint32_t)(packed >> 32)};
+   const uint32_t offsets[] = {0x18000,0x18200,0x18400,0x18500,0x18700,0x19b00};
+   for (unsigned i = 0; i < 6; ++i) {
+      uint64_t pointer = i == 4 ? table : state.gpu + offsets[i];
+      memcpy(context + 4 + 2*i, &pointer, 8);
+   }
+   context[20] = scs.end[5];
+   context[22] = scs.end[10] - scs.end[5];
+   const unsigned levels = (packed >> 16) & 0xff;
+   context[23] = (levels + (levels > 1)) * 8;
+   uint64_t extra = state.gpu + 0x80050;
+   memcpy(context + 26, &extra, 8);
+   extra = state.gpu + 0x18900;
+   memcpy(context + 30, &extra, 8);
+   uint64_t ctx = uniforms.gpu + 352;
+   memcpy((uint8_t *)uniforms.cpu + 352, context, sizeof(context));
+   memcpy((uint8_t *)uniforms.cpu + 320, &ctx, 8);
+   memcpy((uint8_t *)uniforms.cpu + 328, &table, 8);
+   *range = (struct agx3_ioto_uniform_range){0, 168, uniforms.gpu};
+   hk_cmd_buffer_retain_bo(cmd, dev->m5_rq_code);
+   hk_cmd_buffer_retain_bo(cmd, dev->m5_rq_ift);
+   if (vertex) cmd->current_cs.gfx->gen1_native_vertex_rq = true;
+   else cmd->current_cs.gfx->gen1_native_fragment_rq = true;
+   return true;
+}
+
+/*
+ * Publish the uniform DMA spans of `s` (agx_shader_info::agx3_uniform_dma)
+ * after the *count_io ranges already collected: root-relative spans from
+ * root_ptr, set-pointer spans from the set addresses of `desc`, buffer spans
+ * from the root's or a static set's buffer descriptor (zero page past the
+ * buffer's end). Shared by graphics stages and compute kernels; the launch
+ * reads the memory when it runs, as the loads did. sources may be NULL.
+ */
+bool
+hk_g17p_dma_ranges(const struct hk_descriptor_state *desc,
+                   const struct hk_shader *s, uint64_t root_ptr,
+                   struct agx3_ioto_uniform_range *ranges, size_t max_ranges,
+                   size_t *count_io, uint8_t *sources)
+{
+   uint8_t ignored_sources[HK_AGX3_G17P_GRAPHICS_MAX_RANGES];
+   if (!sources)
+      sources = ignored_sources;
+   if (max_ranges > HK_AGX3_G17P_GRAPHICS_MAX_RANGES)
+      return false;
+   const unsigned root_uniform = s->info.agx3_root_uniform;
+   for (unsigned i = 0; i < s->b.info.agx3_uniform_dma_count; ++i) {
+      const struct agx3_uniform_dma_span *span =
+         &s->b.info.agx3_uniform_dma[i];
+      const unsigned pointer = span->pointer;
+      const int set = pointer == root_uniform ? -1
+                      : pointer >= root_uniform + 4 &&
+                            (pointer - root_uniform - 4) % 4 == 0 &&
+                            (pointer - root_uniform - 4) / 4 < HK_MAX_SETS
+                         ? (int)(pointer - root_uniform - 4) / 4
+                         : -2;
+      if (*count_io >= max_ranges || set == -2 ||
+          (span->dest & 1) || (span->halfs & 1) || (span->offset_B & 3))
+         return false;
+
+      if (span->source == AGX3_UNIFORM_DMA_BUFFER) {
+         struct hk_buffer_address buffer;
+         if (!hk_g17p_dma_buffer_in(desc, set, span->descriptor_B, &buffer))
+            return false;
+
+         /* Dwords below the size rounded up to 4 bytes come from the buffer,
+          * later ones from the zero page (AGX3_UNIFORM_DMA_BUFFER). A buffer
+          * holding none of the span, such as a null one, is all zero page.
+          * Both addresses come from the root's or a static set's contents.
+          * A launch replays the buffer address of a span the buffer covers
+          * (hk_g17p_launch_replay()); a split or zero-page span depends on
+          * the buffer's size, so it is collected again for every draw. */
+         const uint64_t size_B = ALIGN_POT((uint64_t)buffer.size, 4);
+         const unsigned buffer_halfs =
+            size_B > span->offset_B
+               ? MIN2(span->halfs, (size_B - span->offset_B) / 2)
+               : 0;
+         if (buffer_halfs) {
+            const uint64_t address = buffer.base_addr + span->offset_B;
+            if (address & 3)
+               return false;
+            sources[*count_io] = buffer_halfs == span->halfs
+                                ? HK_G17P_RANGE_SOURCE_BUFFER
+                                : HK_G17P_RANGE_SOURCE_UPLOADED;
+            ranges[(*count_io)++] = (struct agx3_ioto_uniform_range){
+               .start_halfs = span->dest,
+               .size_halfs = buffer_halfs,
+               .address = address,
+            };
+         }
+         if (buffer_halfs < span->halfs) {
+            if (*count_io >= max_ranges)
+               return false;
+            sources[*count_io] = HK_G17P_RANGE_SOURCE_UPLOADED;
+            ranges[(*count_io)++] = (struct agx3_ioto_uniform_range){
+               .start_halfs = span->dest + buffer_halfs,
+               .size_halfs = span->halfs - buffer_halfs,
+               .address = AGX_ZERO_PAGE_ADDRESS,
+            };
+         }
+         continue;
+      }
+
+      uint64_t base;
+      if (set < 0) {
+         base = root_ptr;
+         if (!base)
+            return false;
+      } else {
+         base = desc->root.sets[set];
+      }
+      /* A set the draw never binds is only read on statically dead paths
+       * (reading it is invalid usage); give the launch a mapped zero page
+       * instead of a fault. */
+      const uint64_t address =
+         base ? base + span->offset_B : AGX_ZERO_PAGE_ADDRESS;
+      if (address & 3)
+         return false;
+
+      /* A root-relative span follows the root, a set-pointer span the set's
+       * address (hk_g17p_dma_set_base(), which every reuse of a published
+       * launcher still compares). */
+      sources[*count_io] = set < 0 ? HK_G17P_RANGE_SOURCE_ROOT
+                               : HK_G17P_RANGE_SOURCE_SET;
+      ranges[(*count_io)++] = (struct agx3_ioto_uniform_range){
+         .start_halfs = span->dest,
+         .size_halfs = span->halfs,
+         .address = address,
+      };
+   }
+
+   return true;
+}
+
+/* sources, when not NULL, receives each range's enum hk_g17p_range_source.
+ * Root ranges address root_ptr, the root this stage's program follows. */
 static bool
 hk_g17p_graphics_uniform_ranges(
    struct hk_cmd_buffer *cmd, struct hk_shader *s,
-   const struct hk_linked_shader *linked,
+   const struct hk_linked_shader *linked, uint64_t root_ptr,
    struct agx3_ioto_uniform_range ranges[HK_AGX3_G17P_GRAPHICS_MAX_RANGES],
    size_t *range_count, uint8_t *sources)
 {
@@ -1959,7 +2433,6 @@ hk_g17p_graphics_uniform_ranges(
           HK_AGX3_G17P_GRAPHICS_MAX_RANGES);
 
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
-   const uint64_t root_ptr = cmd->state.gfx.root;
    size_t count = 0;
    const unsigned root_uniform = s->info.agx3_root_uniform;
    const unsigned texture_uniform_base = s->info.agx3_gen1_fragment_rq ? 168 : 0;
@@ -1967,8 +2440,9 @@ hk_g17p_graphics_uniform_ranges(
       if (linked->g17p_prolog_uses_root ||
           agx_tilebuffer_spills(&cmd->state.gfx.render.tilebuffer) ||
           hk_cmd_uses_blend_constant(cmd) ||
-          !hk_gen1_upload_fragment_query_state(
-             cmd, s->info.stage != MESA_SHADER_FRAGMENT, &ranges[count]))
+          !(agx_chip_g17_m5(dev->dev.chip)
+             ? hk_m5_graphics_query_uniforms(cmd, s->info.stage != MESA_SHADER_FRAGMENT, &ranges[count])
+             : hk_gen1_upload_fragment_query_state(cmd, s->info.stage != MESA_SHADER_FRAGMENT, &ranges[count])))
          return hk_g17p_graphics_ranges_refused(range_count,count);
       count++;
    }
@@ -2037,9 +2511,10 @@ hk_g17p_graphics_uniform_ranges(
          };
       }
 
-   } else if (s->info.stage == MESA_SHADER_MESH) {
-      
-
+   } else if (s->info.stage == MESA_SHADER_MESH ||
+              s->info.stage == MESA_SHADER_TASK) {
+      /* Native TA mesh and object bodies have the same driver-root/descriptor
+       * ranges, without vertex attribute or base-instance ABI ranges. */
    } else if ((s->info.stage == MESA_SHADER_GEOMETRY ||
                s->info.stage == MESA_SHADER_TESS_EVAL) &&
               s->b.info.stage == MESA_SHADER_VERTEX) {
@@ -2073,7 +2548,8 @@ hk_g17p_graphics_uniform_ranges(
          sources[count] = HK_G17P_RANGE_SOURCE_ROOT;
          count++;
       }
-   } else if (s->info.stage != MESA_SHADER_MESH) {
+   } else if (s->info.stage != MESA_SHADER_MESH &&
+              s->info.stage != MESA_SHADER_TASK) {
       return hk_g17p_graphics_ranges_refused(range_count, count);
    }
 
@@ -2179,7 +2655,10 @@ hk_g17p_graphics_uniform_ranges(
 
    if (s->info.agx3_indexed_sampler_valid || native_spills) {
       if ((s->b.info.stage != MESA_SHADER_FRAGMENT &&
-           s->b.info.stage != MESA_SHADER_VERTEX) ||
+           s->b.info.stage != MESA_SHADER_VERTEX &&
+           !(agx_chip_g17_m5(dev->dev.chip) &&
+             (s->b.info.stage == MESA_SHADER_MESH ||
+              s->b.info.stage == MESA_SHADER_TASK))) ||
           !dev->sampled_images.base_va || !dev->samplers.table.bo ||
           !dev->rodata.indexed_heap_pointers)
          return hk_g17p_graphics_ranges_refused(range_count, count);
@@ -2235,90 +2714,9 @@ hk_g17p_graphics_uniform_ranges(
     * uniform spans (agx_shader_info::agx3_uniform_dma). Publish each from the
     * address this draw gives it; the launch reads the memory when it runs,
     * as the loads did. */
-   for (unsigned i = 0; i < s->b.info.agx3_uniform_dma_count; ++i) {
-      const struct agx3_uniform_dma_span *span =
-         &s->b.info.agx3_uniform_dma[i];
-      const unsigned pointer = span->pointer;
-      const int set = pointer == root_uniform ? -1
-                      : pointer >= root_uniform + 4 &&
-                            (pointer - root_uniform - 4) % 4 == 0 &&
-                            (pointer - root_uniform - 4) / 4 < HK_MAX_SETS
-                         ? (int)(pointer - root_uniform - 4) / 4
-                         : -2;
-      if (count >= HK_AGX3_G17P_GRAPHICS_MAX_RANGES || set == -2 ||
-          (span->dest & 1) || (span->halfs & 1) || (span->offset_B & 3))
-         return hk_g17p_graphics_ranges_refused(range_count, count);
-
-      if (span->source == AGX3_UNIFORM_DMA_BUFFER) {
-         struct hk_buffer_address buffer;
-         if (!hk_g17p_dma_buffer(cmd, set, span->descriptor_B, &buffer))
-            return hk_g17p_graphics_ranges_refused(range_count, count);
-
-         /* Dwords below the size rounded up to 4 bytes come from the buffer,
-          * later ones from the zero page (AGX3_UNIFORM_DMA_BUFFER). A buffer
-          * holding none of the span, such as a null one, is all zero page.
-          * Both addresses come from the root's or a static set's contents.
-          * A launch replays the buffer address of a span the buffer covers
-          * (hk_g17p_launch_replay()); a split or zero-page span depends on
-          * the buffer's size, so it is collected again for every draw. */
-         const uint64_t size_B = ALIGN_POT((uint64_t)buffer.size, 4);
-         const unsigned buffer_halfs =
-            size_B > span->offset_B
-               ? MIN2(span->halfs, (size_B - span->offset_B) / 2)
-               : 0;
-         if (buffer_halfs) {
-            const uint64_t address = buffer.base_addr + span->offset_B;
-            if (address & 3)
-               return hk_g17p_graphics_ranges_refused(range_count, count);
-            sources[count] = buffer_halfs == span->halfs
-                                ? HK_G17P_RANGE_SOURCE_BUFFER
-                                : HK_G17P_RANGE_SOURCE_UPLOADED;
-            ranges[count++] = (struct agx3_ioto_uniform_range){
-               .start_halfs = span->dest,
-               .size_halfs = buffer_halfs,
-               .address = address,
-            };
-         }
-         if (buffer_halfs < span->halfs) {
-            if (count >= HK_AGX3_G17P_GRAPHICS_MAX_RANGES)
-               return hk_g17p_graphics_ranges_refused(range_count, count);
-            sources[count] = HK_G17P_RANGE_SOURCE_UPLOADED;
-            ranges[count++] = (struct agx3_ioto_uniform_range){
-               .start_halfs = span->dest + buffer_halfs,
-               .size_halfs = span->halfs - buffer_halfs,
-               .address = AGX_ZERO_PAGE_ADDRESS,
-            };
-         }
-         continue;
-      }
-
-      uint64_t base;
-      if (set < 0) {
-         base = root_ptr;
-         if (!base)
-            return hk_g17p_graphics_ranges_refused(range_count, count);
-      } else {
-         base = cmd->state.gfx.descriptors.root.sets[set];
-      }
-      /* A set the draw never binds is only read on statically dead paths
-       * (reading it is invalid usage); give the launch a mapped zero page
-       * instead of a fault. */
-      const uint64_t address =
-         base ? base + span->offset_B : AGX_ZERO_PAGE_ADDRESS;
-      if (address & 3)
-         return hk_g17p_graphics_ranges_refused(range_count, count);
-
-      /* A root-relative span follows the root, a set-pointer span the set's
-       * address (hk_g17p_dma_set_base(), which every reuse of a published
-       * launcher still compares). */
-      sources[count] = set < 0 ? HK_G17P_RANGE_SOURCE_ROOT
-                               : HK_G17P_RANGE_SOURCE_SET;
-      ranges[count++] = (struct agx3_ioto_uniform_range){
-         .start_halfs = span->dest,
-         .size_halfs = span->halfs,
-         .address = address,
-      };
-   }
+   if (!hk_g17p_dma_ranges(&cmd->state.gfx.descriptors, s, root_ptr, ranges,
+                           HK_AGX3_G17P_GRAPHICS_MAX_RANGES, &count, sources))
+      return hk_g17p_graphics_ranges_refused(range_count, count);
 
    *range_count = count;
    return count <= HK_AGX3_G17P_GRAPHICS_MAX_RANGES;
@@ -2492,7 +2890,7 @@ hk_g17p_launch_record(struct hk_cmd_buffer *cmd, struct hk_g17p_launch *launch,
                       const struct hk_shader *s,
                       const struct hk_linked_shader *linked,
                       const struct hk_g17p_gfx_loader *l, uint8_t flags,
-                      const struct hk_shader *hw_vs,
+                      const struct hk_shader *hw_vs, uint64_t root,
                       const struct agx3_ioto_uniform_range *ranges,
                       size_t count, const uint8_t *sources)
 {
@@ -2512,7 +2910,7 @@ hk_g17p_launch_record(struct hk_cmd_buffer *cmd, struct hk_g17p_launch *launch,
          offset = ranges[i].address;
          break;
       case HK_G17P_RANGE_SOURCE_ROOT:
-         offset = ranges[i].address - gfx->root;
+         offset = ranges[i].address - root;
          break;
       case HK_G17P_RANGE_SOURCE_DRAW_PARAMS:
       case HK_G17P_RANGE_SOURCE_DRAW_ID:
@@ -2602,13 +3000,14 @@ hk_g17p_launch_matches(const struct hk_cmd_buffer *cmd,
 }
 
 /* The stage program of s from a matching recorded launch: the ranges
- * hk_g17p_graphics_uniform_ranges() would collect, published with the loader
- * of the recording. 0 when they do not fit that loader. */
+ * hk_g17p_graphics_uniform_ranges() would collect for root, published with
+ * the loader of the recording. 0 when they do not fit that loader. */
 static inline uint64_t
 hk_g17p_launch_replay(struct hk_cmd_buffer *cmd,
                       const struct hk_g17p_launch *launch,
                       struct hk_linked_shader *linked,
-                      struct hk_g17p_graphics_ioto_cache_entry *cached)
+                      struct hk_g17p_graphics_ioto_cache_entry *cached,
+                      uint64_t root)
 {
    const struct hk_graphics_state *gfx = &cmd->state.gfx;
    const unsigned count = launch->count;
@@ -2617,7 +3016,7 @@ hk_g17p_launch_replay(struct hk_cmd_buffer *cmd,
       uint64_t address;
       switch (launch->source[i]) {
       case HK_G17P_RANGE_SOURCE_ROOT:
-         address = gfx->root + launch->offset[i];
+         address = root + launch->offset[i];
          break;
       case HK_G17P_RANGE_SOURCE_DRAW_PARAMS:
          address = gfx->draw_params;
@@ -2638,7 +3037,7 @@ hk_g17p_launch_replay(struct hk_cmd_buffer *cmd,
          /* As hk_g17p_graphics_uniform_ranges(), for a buffer that still
           * covers the whole span; otherwise the span splits differently. */
          struct hk_buffer_address buffer;
-         if (!hk_g17p_dma_buffer(cmd,
+         if (!hk_g17p_dma_buffer(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS,
                                  launch->set[i] == HK_G17P_LAUNCH_ROOT_SET
                                     ? -1 : launch->set[i],
                                  launch->descriptor_B[i], &buffer) ||
@@ -2682,7 +3081,11 @@ hk_g17p_publish_gfx_loader(struct hk_cmd_buffer *cmd, struct hk_shader *s,
    const struct agx_tilebuffer_layout *tib = &cmd->state.gfx.render.tilebuffer;
    const struct agx_ioto_g17p_imageblock imageblock = {
       .sample_size_B = tib->sample_size_B,
-      .nr_samples = tib->nr_samples,
+      /* Empty M5 fragment programs use the single-sample loader form. */
+      .nr_samples = agx_chip_g17_m5(dev->dev.chip) &&
+                       s->info.stage == MESA_SHADER_FRAGMENT &&
+                       !tib->sample_size_B && !tib->nr_samples
+                       ? 1 : tib->nr_samples,
       .tile_size_px = tib->tile_size,
    };
 
@@ -2799,6 +3202,13 @@ hk_g17p_graphics_program_desc_init(
       },
    };
 
+   /* Attachmentless discard never resolves a raster sample count. Its inert
+    * fragment loader uses the native empty single-sample imageblock without
+    * changing the render state's deferred sample count. */
+   if (agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip) && fragment &&
+       desc->imageblock.nr_samples == 0 && desc->imageblock.sample_size_B == 0)
+      desc->imageblock.nr_samples = 1;
+
    if (s->b.info.has_preamble &&
        (!s->bo || !s->bo->va || s->code_size != s->b.info.binary_size ||
         !hk_agx3_g17p_constant_program_from_binary(
@@ -2826,16 +3236,28 @@ hk_g17p_graphics_program_desc_init(
 
 uint64_t
 hk_upload_native_mesh_program(struct hk_cmd_buffer *cmd, struct hk_shader *s,
-                              struct hk_linked_shader *linked, bool object)
+                              struct hk_linked_shader *linked, bool object,
+                              bool task)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
-   if (dev->dev.chip != AGX_CHIP_G17P || !linked ||
-       !linked->b.g17p_compiler_d_valid ||
-       hk_mesh_mode() != HK_MESH_MODE_NATIVE_PROBE)
+   if (!linked ||
+       (dev->dev.chip == AGX_CHIP_G17P
+          ? (!linked->b.g17p_compiler_d_valid || hk_mesh_mode() != HK_MESH_MODE_NATIVE_PROBE)
+          : (dev->dev.chip != AGX_CHIP_G17S ||
+             !(dev->dev.params.features & DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2))))
       goto refused;
    struct agx3_ioto_uniform_range ranges[HK_AGX3_G17P_GRAPHICS_MAX_RANGES];
    size_t count = 0;
-   if (object) {
+   /* An API task shader is a compiled object program with the ordinary
+    * driver-root/descriptor ranges and its own threadgroup memory. */
+   const bool task_object = object && task && dev->dev.chip == AGX_CHIP_G17S &&
+                            s->info.stage == MESA_SHADER_TASK;
+   if (task_object) {
+      if (!hk_g17p_graphics_uniform_ranges(cmd, s, linked,
+                                           cmd->state.gfx.root, ranges, &count,
+                                           NULL))
+         goto refused;
+   } else if (object) {
       const uint64_t grid = cmd->state.gfx.mesh.object_params ?: cmd->state.gfx.mesh.grid;
       if (!grid)
          goto refused;
@@ -2844,31 +3266,36 @@ hk_upload_native_mesh_program(struct hk_cmd_buffer *cmd, struct hk_shader *s,
          .address = hk_pool_upload(cmd, &grid, sizeof(grid), 8),
       };
    } else if (!hk_g17p_graphics_uniform_ranges(cmd, s, linked,
-                                               ranges, &count, NULL)) {
+                                               cmd->state.gfx.root, ranges,
+                                               &count, NULL)) {
       goto refused;
    }
-
    struct agx3_g17p_graphics_program_desc d = {
-      .threadgroup_memory_size_B = object ? 0 : s->b.info.local_size,
       .shader_va = linked->b.bo->va->addr + linked->b.entry_offset,
       .uniform_ranges = ranges, .uniform_range_count = count,
       .shader_binary_size = linked->b.size,
       .scratch_size = MAX2(linked->b.scratch_size,
-                           s->b.info.scratch_size),
+                           dev->dev.chip == AGX_CHIP_G17P
+                              ? s->b.info.scratch_size : linked->b.agx3_spectpr_scratch_size),
       .nr_gprs = linked->b.nr_gprs,
       .compiler_d_valid = linked->b.g17p_compiler_d_valid,
       .compiler_d = linked->b.g17p_compiler_d,
-
+      .threadgroup_memory_size_B = dev->dev.chip == AGX_CHIP_G17P && object
+                                    ? 0 : s->b.info.local_size,
    };
    size_t bound;
-   if (!agx3_g17p_graphics_program_bound(dev->dev.chip, ranges, count, &bound))
+   if (!(dev->dev.chip == AGX_CHIP_G17P
+          ? agx3_g17p_graphics_program_bound(dev->dev.chip, ranges, count, &bound)
+          : agx3_g17g_graphics_program_bound(dev->dev.chip, ranges, count, &bound)))
       goto refused;
    struct agx_ptr allocation = hk_pool_usc_alloc(cmd, bound, 64);
    if (!allocation.cpu)
       return 0;
    d.program_va = allocation.gpu;
    struct agx3_g17p_graphics_program program;
-   if (!agx3_g17p_graphics_program_generate(dev->dev.chip, &d, &program))
+   if (!(dev->dev.chip == AGX_CHIP_G17P
+          ? agx3_g17p_graphics_program_generate(dev->dev.chip, &d, &program)
+          : agx3_g17g_mesh_program_generate(dev->dev.chip, &d, object, &program)))
       goto refused;
    memcpy(allocation.cpu, program.bytes, program.length);
    return allocation.gpu;
@@ -2878,6 +3305,108 @@ hk_upload_native_mesh_program(struct hk_cmd_buffer *cmd, struct hk_shader *s,
    return 0;
 }
 
+/* Read the uncached mappings only for this opt-in validation oracle. Keep
+ * its temporary snapshot and checks out of the normal recording path. */
+__attribute__((cold, noinline)) static void
+hk_m5_verify_fragment_root_shadow(const struct hk_graphics_state *gfx)
+{
+   struct hk_root_fragment_snapshot reference;
+   if (gfx->root_cpu) {
+      hk_root_fragment_snapshot_store(&reference, gfx->root_cpu);
+      if (gfx->root_fragment_snapshot_cpu != gfx->root_cpu ||
+          memcmp(&reference, &gfx->root_fragment_snapshot, sizeof(reference))) {
+         mesa_loge("M5 current root snapshot mismatch");
+         abort();
+      }
+   }
+   if (gfx->fs_root_cpu) {
+      hk_root_fragment_snapshot_store(&reference, gfx->fs_root_cpu);
+      if (gfx->fs_root_fragment_snapshot_cpu != gfx->fs_root_cpu ||
+          memcmp(&reference, &gfx->fs_root_fragment_snapshot,
+                 sizeof(reference))) {
+         mesa_loge("M5 prior root snapshot mismatch");
+         abort();
+      }
+   }
+}
+
+/* The root the M5 fragment program of s follows. A root republished only
+ * for bytes the fragment shader cannot read -- a per-draw dynamic offset of a
+ * vertex-stage uniform buffer, a vertex buffer bind -- keeps the program the
+ * stage last published, and with it the whole PPP state: that root stays
+ * valid for the command buffer and holds the same bytes wherever the shader
+ * reads. */
+static uint64_t
+hk_m5_fragment_root(struct hk_cmd_buffer *cmd, const struct hk_shader *s)
+{
+   struct hk_graphics_state *gfx = &cmd->state.gfx;
+   const bool shadow = debug_get_option_m5_fragment_root_shadow();
+
+   if (unlikely(shadow && debug_get_option_m5_verify_fragment_root_shadow()))
+      hk_m5_verify_fragment_root_shadow(gfx);
+
+   return hk_m5_fragment_root_select(gfx, s->info.agx3_root_reads,
+                                     debug_get_option_m5_fragment_root_memo(),
+                                     shadow);
+}
+
+static uint64_t
+hk_m5_upload_graphics_program(struct hk_cmd_buffer *cmd, struct hk_shader *s,
+                              struct hk_linked_shader *linked)
+{
+   const unsigned stage = s->info.stage == MESA_SHADER_FRAGMENT ? 1 : 0;
+   struct hk_g17p_graphics_ioto_cache_entry *cached =
+      &cmd->state.gfx.g17p_ioto[stage];
+   const uint64_t root = stage && !s->info.agx3_gen1_fragment_rq
+                            ? hk_m5_fragment_root(cmd, s)
+                            : cmd->state.gfx.root;
+
+   /* As on HAL200: a launch recorded for the same program and draw state
+    * already passed admission, and replays the ranges the collector below
+    * would build, so the same loader publishes the same program. */
+   struct hk_graphics_state *gfx = &cmd->state.gfx;
+   const struct hk_shader *hw_vs =
+      gfx->shaders[MESA_SHADER_VERTEX] ? hk_bound_hw_vs(gfx) : NULL;
+   const uint8_t flags = hk_g17p_launch_flags(cmd, s);
+   struct hk_g17p_launch *launch = hk_g17p_launch_slot(cmd, stage, linked);
+   if (!s->info.agx3_gen1_fragment_rq &&
+       hk_g17p_launch_matches(cmd, launch, s, linked, flags, hw_vs)) {
+      const uint64_t published =
+         hk_g17p_launch_replay(cmd, launch, linked, cached, root);
+      if (published)
+         return published;
+      if (vk_command_buffer_has_error(&cmd->vk))
+         return 0;
+   }
+
+   if (!hk_g17p_basic_graphics_program_supported(cmd, s, linked))
+      goto refused;
+
+   struct agx3_ioto_uniform_range ranges[HK_AGX3_G17P_GRAPHICS_MAX_RANGES];
+   uint8_t sources[HK_AGX3_G17P_GRAPHICS_MAX_RANGES];
+   size_t count = 0;
+   if (!hk_g17p_graphics_uniform_ranges(cmd, s, linked, root, ranges, &count,
+                                        sources))
+      goto refused;
+
+   /* Linked shader-owned loader templates use the same range shape and
+    * relocation plan as HAL200. Each recording publishes immutable bytes
+    * with its own addresses; executable lifetime follows the linked shader. */
+   const uint64_t program =
+      hk_g17p_publish_gfx_loader(cmd, s, linked, cached, ranges, count);
+   if (!program && !vk_command_buffer_has_error(&cmd->vk))
+      goto refused;
+   if (program)
+      hk_g17p_launch_record(cmd, launch, s, linked, cached->loader, flags,
+                            hw_vs, root, ranges, count, sources);
+   return program;
+
+refused:
+   mesa_loge_once("M5 graphics launch refused: stage=%u", (unsigned)s->info.stage);
+   vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+   return 0;
+}
+
 static uint64_t
 hk_upload_usc_words_internal(struct hk_cmd_buffer *cmd, struct hk_shader *s,
                              struct hk_linked_shader *linked);
@@ -2926,6 +3455,9 @@ hk_upload_usc_words_internal(struct hk_cmd_buffer *cmd, struct hk_shader *s,
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
 
+   if (agx_chip_g17_m5(dev->dev.chip))
+      return hk_m5_upload_graphics_program(cmd, s, linked);
+
    if (agx_native_graphics(&dev->dev)) {
       const unsigned stage = s->info.stage == MESA_SHADER_FRAGMENT ? 1 : 0;
       struct hk_g17p_graphics_ioto_cache_entry *cached =
@@ -2945,7 +3477,7 @@ hk_upload_usc_words_internal(struct hk_cmd_buffer *cmd, struct hk_shader *s,
       if (!s->info.agx3_gen1_fragment_rq &&
           hk_g17p_launch_matches(cmd, launch, s, linked, flags, hw_vs)) {
          const uint64_t published =
-            hk_g17p_launch_replay(cmd, launch, linked, cached);
+            hk_g17p_launch_replay(cmd, launch, linked, cached, agfx->root);
          if (published)
             return published;
          if (vk_command_buffer_has_error(&cmd->vk))
@@ -2990,13 +3522,14 @@ hk_upload_usc_words_internal(struct hk_cmd_buffer *cmd, struct hk_shader *s,
          loader_ranges[HK_AGX3_G17P_GRAPHICS_MAX_RANGES];
       uint8_t sources[HK_AGX3_G17P_GRAPHICS_MAX_RANGES];
       size_t loader_range_count = 0;
-      if (hk_g17p_graphics_uniform_ranges(cmd, s, linked, loader_ranges,
-                                          &loader_range_count, sources)) {
+      if (hk_g17p_graphics_uniform_ranges(cmd, s, linked, agfx->root,
+                                          loader_ranges, &loader_range_count,
+                                          sources)) {
          const uint64_t published = hk_g17p_publish_gfx_loader(
             cmd, s, linked, cached, loader_ranges, loader_range_count);
          if (published) {
             hk_g17p_launch_record(cmd, launch, s, linked, cached->loader,
-                                  flags, hw_vs, loader_ranges,
+                                  flags, hw_vs, agfx->root, loader_ranges,
                                   loader_range_count, sources);
             return published;
          }
@@ -3110,6 +3643,10 @@ uint64_t
 hk_upload_usc_words(struct hk_cmd_buffer *cmd, struct hk_shader *s,
                     struct hk_linked_shader *linked)
 {
+   /* An inline-blend program with its own constant program launches with
+    * its own uniform layout (hk_linked_shader::g17p_layout). */
+   if (linked && linked->g17p_layout)
+      s = linked->g17p_layout;
    return hk_upload_usc_words_internal(cmd, s, linked);
 }
 
@@ -3124,6 +3661,11 @@ hk_dispatch_precomp(struct hk_cmd_buffer *cmd, struct agx_grid grid,
       return;
    }
 
+   if (agx_chip_g17_m5(dev->dev.chip)) {
+      hk_dispatch_precomp_m5(cmd, grid, barrier, idx, data, data_size);
+      return;
+   }
+
    struct agx_precompiled_shader *prog = agx_get_precompiled(&dev->bg_eot, idx);
    if (!prog) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
@@ -3278,7 +3820,7 @@ hk_cs_init_graphics(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    if (agx_native_graphics(&dev->dev))
       hk_cs_capture_fragment_attachments(dev, render, cs);
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       /* The kext preserves a zero final VDM word when this selector is the
        * UINT32_MAX sentinel.  This reproduces m1n1's direct-triangle prefix. */
       cs->agx3_active_context_selector_valid = true;
@@ -3286,6 +3828,37 @@ hk_cs_init_graphics(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    }
 
    cs->tib = render->tilebuffer;
+   /* Attachmentless rendering defers its API sample count. Publish the
+    * native one-sample no-imageblock profile in this control stream while
+    * preserving the deferred count in render state. */
+   if (agx_chip_g17_m5(dev->dev.chip) && !cs->tib.sample_size_B &&
+       !cs->tib.nr_samples && !render->color_att_count &&
+       !render->depth_att.iview && !render->stencil_att.iview)
+      cs->tib.nr_samples = 1;
+   if (agx_chip_g17_m5(dev->dev.chip)) {
+      const struct hk_bg_eot *programs[] = {
+         &render->cr.bg.main, &render->cr.bg.partial,
+         &render->cr.eot.main, &render->cr.eot.partial,
+      };
+      cs->tile_resources = (struct drm_asahi_render_program){
+         .stage = 1,
+         .imageblock_bytes = cs->tib.tile_size * cs->tib.sample_size_B *
+                             cs->tib.nr_samples,
+         .multisample_bytes = cs->tib.nr_samples == 2
+                                ? cs->tib.tile_size * cs->tib.sample_size_B / 4
+                                : cs->tib.nr_samples == 4
+                                     ? cs->tib.tile_size * cs->tib.sample_size_B
+                                     : cs->tib.nr_samples == 8
+                                          ? 4 * cs->tib.tile_size * cs->tib.sample_size_B
+                                          : 0,
+      };
+      for (unsigned i = 0; i < ARRAY_SIZE(programs); i++) {
+         cs->tile_resources.primary_bytes = MAX2(cs->tile_resources.primary_bytes,
+                                                  programs[i]->primary_storage_B);
+         cs->tile_resources.secondary_bytes = MAX2(cs->tile_resources.secondary_bytes,
+                                                    programs[i]->staging_storage_B);
+      }
+   }
 
    /* Assume this is not the first control stream of the render pass, so
     * initially use the partial background/EOT program and ZLS control.
@@ -3367,9 +3940,9 @@ hk_cs_init_graphics(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
       uint8_t records[2 * AGX_PPP_STATE_LENGTH];
       uint8_t *record = records;
       if (!depth_stencil_gpu || !gpu ||
-          !agx_ppp_emit_state(dev->dev.chip, &record, depth_stencil_gpu,
+          !hk_ppp_emit_state(dev->dev.chip, &record, depth_stencil_gpu,
                               depth_stencil.length) ||
-          !agx_ppp_emit_state(dev->dev.chip, &record, gpu, initial.length) ||
+          !hk_ppp_emit_state(dev->dev.chip, &record, gpu, initial.length) ||
           record != records + sizeof(records)) {
          mesa_loge("failed to emit HAL300 initial PPP state for %s",
                    agx_chip_name(dev->dev.chip));
@@ -3413,7 +3986,7 @@ hk_cs_init_graphics(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
       }
 
       memcpy(T.cpu, initial.bytes, initial.length);
-      if (!agx_ppp_emit_state(dev->dev.chip, &map, T.gpu, initial.length)) {
+      if (!hk_ppp_emit_state(dev->dev.chip, &map, T.gpu, initial.length)) {
          mesa_loge("failed to emit G17P initial PPP state for %s",
                    agx_chip_name(dev->dev.chip));
          vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
@@ -3574,10 +4147,11 @@ hk_optimize_empty_vdm(struct hk_cmd_buffer *cmd)
    /* An empty final stream of a pass with earlier draws is not an empty
     * render. Replacing it with the load-op clears would erase those draws. */
    const bool continuation = cmd->sync.pass_vdm != cs;
-   if (continuation && !agx_native_graphics(&dev->dev))
+   if (continuation && !agx_native_graphics(&dev->dev) &&
+       !agx_chip_g17_m5(dev->dev.chip))
       return false;
 
-   /* Two reasons to keep the graphics representation on G17P rather than
+   /* Two reasons to keep the graphics representation on AGX3 rather than
     * replace it with the image clears below.
     *
     * The first is capability: G17P has native BG/EOT programs for the
@@ -3601,18 +4175,25 @@ hk_optimize_empty_vdm(struct hk_cmd_buffer *cmd)
     * common finalizer will append the native VDM terminator without uploading
     * any ordinary vertex or fragment program.
     */
-   const bool g17p_output_only_through_eot =
-      agx_native_graphics(&dev->dev) && render->g17p_direct_color_resolve;
+   const bool native_eot = agx_native_graphics(&dev->dev) ||
+                           agx_chip_g17_m5(dev->dev.chip);
+   const bool output_only_through_eot =
+      native_eot && render->g17p_direct_color_resolve;
 
    /* A timestamp rides on this exact command; replacing the render with
     * image clears would drop it. Keep the render. */
    if (cs->timestamp.start.handle || cs->timestamp.end.handle)
       return false;
 
-   if (agx_native_graphics(&dev->dev) &&
+   /* M5: a multisample clear-only pass stays a render. As compute it is a
+    * separate job that every later render's vertex half must wait for; as a
+    * render that writes only its attachments it is not. */
+   const bool m5_ms_clear = agx_chip_g17_m5(dev->dev.chip) &&
+                            cs->cr.process_empty_tiles && cs->tib.nr_samples > 1;
+   if (native_eot &&
        (continuation ||
         (cs->cr.process_empty_tiles &&
-         (g17p_output_only_through_eot ||
+         (output_only_through_eot || m5_ms_clear ||
           !agx_get_precompiled(&dev->bg_eot, LIBAGX_FILL_UINT4))))) {
       /* Normally supplied by the first draw. A clear has no application
        * sample-location state; use the attachment sample count and leave
@@ -3737,7 +4318,9 @@ hk_defer_clear(struct hk_cmd_buffer *cmd, struct hk_image *image,
          ? VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT
          : VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
 
-   if (!agx_chip_native_agx3(dev->dev.chip) || cmd->in_meta ||
+   if ((!agx_chip_native_agx3(dev->dev.chip) &&
+        !agx_chip_g17_m5(dev->dev.chip)) ||
+       cmd->in_meta ||
        cmd->vk.pool->queue_family_index >= pdev->queue_family_count ||
        !(pdev->queue_families[cmd->vk.pool->queue_family_index].queue_flags &
          VK_QUEUE_GRAPHICS_BIT) ||
@@ -3823,7 +4406,7 @@ hk_deferred_clears_hold_buffer(struct hk_cmd_buffer *cmd, uint64_t addr,
 }
 
 void
-hk_flush_deferred_clears(struct hk_cmd_buffer *cmd)
+hk_flush_deferred_clears_why(struct hk_cmd_buffer *cmd, const char *why)
 {
    /* Whatever observes the deferred clears may also read a pending store. */
    cmd->state.pending_z_store.cs = NULL;
@@ -3835,6 +4418,14 @@ hk_flush_deferred_clears(struct hk_cmd_buffer *cmd)
    cmd->state.deferred_clear_flushing = true;
    for (unsigned i = 0; i < cmd->state.deferred_clear_count; ++i) {
       const struct hk_deferred_clear *e = &cmd->state.deferred_clear[i];
+      if (unlikely(dev->frame_anatomy)) {
+         hk_frame_anatomy_note(
+            dev, "deferred-clear fill by %s: %ux%u %s aspects=0x%x compressed=%u",
+            why, e->image->vk.extent.width, e->image->vk.extent.height,
+            util_format_short_name(
+               hk_format_to_pipe_format(e->image->vk.format)),
+            e->aspects, e->image->planes[0].layout.compressed);
+      }
       const VkImageSubresourceRange range = {
          .aspectMask = e->aspects,
          .levelCount = 1,
@@ -3961,6 +4552,12 @@ hk_deferred_clear_for(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
       return NULL;
 
    *fill = use == HK_DEFERRED_CLEAR_LOAD_AND_FILL;
+   if (unlikely(hk_cmd_buffer_device(cmd)->frame_anatomy)) {
+      hk_frame_anatomy_note(hk_cmd_buffer_device(cmd),
+                            "deferred-clear taken: %ux%u layout=%u fill=%u",
+                            image->vk.extent.width, image->vk.extent.height,
+                            att->imageLayout, *fill);
+   }
    return e;
 }
 
diff --git a/src/asahi/vulkan/hk_cmd_buffer.h b/src/asahi/vulkan/hk_cmd_buffer.h
index 666ef56aef52..d426bb29b20e 100644
--- a/src/asahi/vulkan/hk_cmd_buffer.h
+++ b/src/asahi/vulkan/hk_cmd_buffer.h
@@ -25,6 +25,7 @@
 #include "asahi/genxml/agx3_cmdstream.h"
 #include "asahi/lib/agx3_compute_program_core.h"
 #include "asahi/lib/agx_ppp_hal200_graphics_core.h"
+#include "asahi/lib/agx_ppp_hal300_graphics_core.h"
 #include "asahi/lib/agx_ppp_viewport_payload_core.h"
 #include "asahi/lib/agx_vdm_hal200_bind_tail_core.h"
 #include "asahi/lib/agx_va_runtime_hole.h"
@@ -244,6 +245,19 @@ struct hk_root_descriptor_table {
    uint64_t mesh_grid_addr;
    uint64_t mesh_prims_query;
    uint64_t sampler_heap_address;
+   uint64_t physical_sparse_map;
+   uint64_t physical_sparse_base;
+   uint64_t physical_sparse_size;
+
+   /* Upper four sample positions on M5. It follows every agx3b root field,
+    * so the draw arm, and with it every field after the union, keeps the
+    * agx3b layout. */
+   uint32_t ppp_multisamplectl_hi;
+
+   /* M5 task draws whose grid, count or predicate live in GPU memory:
+    * address of the word libagx_m5_task_dispatch writes (nonzero = execute);
+    * zero for every other draw. Appended after every agx3b field. */
+   uint64_t mesh_task_flag;
 };
 
 /* The graphics arm is the larger of the two, so everything after the union --
@@ -273,6 +287,120 @@ struct hk_upload_stats {
          (cmd)->state.upload_stats.field += (value);                            \
    } while (0)
 
+/* Root bytes the fragment stage reads only through these classes of access,
+ * recorded per shader in hk_shader_info::agx3_root_reads. */
+enum hk_root_reads {
+   /* The root's own address: the shader can then reach any root byte. */
+   HK_ROOT_READS_ADDRESS = BITFIELD_BIT(0),
+   /* Vertex attribute bases, clamps and strides, and the vertex shader's
+    * constant data. */
+   HK_ROOT_READS_VERTEX_INPUT = BITFIELD_BIT(1),
+   HK_ROOT_READS_DYNAMIC_BUFFERS = BITFIELD_BIT(2),
+};
+
+/* Every root byte a draw may republish without the fragment stage observing
+ * it unless the fragment shader reads that class (hk_m5_fragment_root()), as
+ * X(from, to, class) spans in ascending order. The vertex-input spans hold
+ * just their fields: the clamp and stride tables are adjacent and followed by
+ * vertex_outputs, and attrib_base ends the graphics arm. */
+#define HK_ROOT_FRAGMENT_HIDDEN_SPANS(X)                                       \
+   X(root_desc_addr, sets, HK_ROOT_READS_ADDRESS)                              \
+   X(draw.attrib_clamps, draw.vertex_outputs, HK_ROOT_READS_VERTEX_INPUT)      \
+   X(draw.vs_constant_data_addr, draw.fs_constant_data_addr,                   \
+     HK_ROOT_READS_VERTEX_INPUT)                                               \
+   X(draw.attrib_base, push, HK_ROOT_READS_VERTEX_INPUT)                       \
+   X(dynamic_buffers, set_dynamic_buffer_start, HK_ROOT_READS_DYNAMIC_BUFFERS)
+
+/* The HK_ROOT_READS_* classes a read of root bytes [start_B, end_B) falls
+ * in. */
+static inline uint8_t
+hk_root_reads_of(uint64_t start_B, uint64_t end_B)
+{
+   uint8_t reads = 0;
+#define HK_ROOT_CLASSIFY(from, to, class)                                      \
+   if (start_B < hk_root_descriptor_offset(to) &&                              \
+       end_B > hk_root_descriptor_offset(from))                                \
+      reads |= (class);
+   HK_ROOT_FRAGMENT_HIDDEN_SPANS(HK_ROOT_CLASSIFY)
+#undef HK_ROOT_CLASSIFY
+   return reads;
+}
+
+/* Whether two roots agree on every byte outside the hidden spans. */
+static inline bool
+hk_root_matches_for_fragment(const struct hk_root_descriptor_table *a,
+                             const struct hk_root_descriptor_table *b)
+{
+   size_t at = 0;
+#define HK_ROOT_COMPARE_BEFORE(from, to, class)                                \
+   if (memcmp((const uint8_t *)a + at, (const uint8_t *)b + at,                \
+              hk_root_descriptor_offset(from) - at))                           \
+      return false;                                                            \
+   at = hk_root_descriptor_offset(to);
+   HK_ROOT_FRAGMENT_HIDDEN_SPANS(HK_ROOT_COMPARE_BEFORE)
+#undef HK_ROOT_COMPARE_BEFORE
+   return !memcmp((const uint8_t *)a + at, (const uint8_t *)b + at,
+                  sizeof(*a) - at);
+}
+
+/* Build the same visible-byte sequence as hk_root_matches_for_fragment().
+ * Keeping this derived from the shared span list includes future root fields
+ * and padding without duplicating the root ABI. */
+#define HK_ROOT_FRAGMENT_SUBTRACT(from, to, class)                             \
+   -(hk_root_descriptor_offset(to) - hk_root_descriptor_offset(from))
+enum {
+   HK_ROOT_FRAGMENT_VISIBLE_SIZE = sizeof(struct hk_root_descriptor_table)
+      HK_ROOT_FRAGMENT_HIDDEN_SPANS(HK_ROOT_FRAGMENT_SUBTRACT),
+};
+#undef HK_ROOT_FRAGMENT_SUBTRACT
+
+struct hk_root_fragment_snapshot {
+   uint8_t bytes[HK_ROOT_FRAGMENT_VISIBLE_SIZE];
+};
+
+static inline void
+hk_root_fragment_snapshot_store(struct hk_root_fragment_snapshot *snapshot,
+                                const struct hk_root_descriptor_table *root)
+{
+   size_t at = 0, out = 0;
+#define HK_ROOT_FRAGMENT_COPY(from, to, class)                                 \
+   do {                                                                        \
+      const size_t size = hk_root_descriptor_offset(from) - at;                \
+      memcpy(snapshot->bytes + out, (const uint8_t *)root + at, size);         \
+      out += size;                                                             \
+      at = hk_root_descriptor_offset(to);                                      \
+   } while (0);
+   HK_ROOT_FRAGMENT_HIDDEN_SPANS(HK_ROOT_FRAGMENT_COPY)
+#undef HK_ROOT_FRAGMENT_COPY
+   memcpy(snapshot->bytes + out, (const uint8_t *)root + at,
+          sizeof(*root) - at);
+   assert(out + sizeof(*root) - at == sizeof(snapshot->bytes));
+}
+
+static_assert(hk_root_descriptor_offset(draw.attrib_strides) ==
+                    hk_root_descriptor_offset(draw.attrib_clamps) +
+                       sizeof(((struct hk_root_descriptor_table *)0)
+                                 ->draw.attrib_clamps) &&
+                 hk_root_descriptor_offset(draw.vertex_outputs) ==
+                    hk_root_descriptor_offset(draw.attrib_strides) +
+                       sizeof(((struct hk_root_descriptor_table *)0)
+                                 ->draw.attrib_strides),
+              "only the clamp and stride tables precede vertex_outputs");
+static_assert(hk_root_descriptor_offset(draw.fs_constant_data_addr) ==
+                 hk_root_descriptor_offset(draw.vs_constant_data_addr) +
+                    sizeof(uint64_t),
+              "the fragment constant data follows the vertex constant data");
+static_assert(hk_root_descriptor_offset(push) ==
+                 hk_root_descriptor_offset(draw.attrib_base) +
+                    sizeof(((struct hk_root_descriptor_table *)0)
+                              ->draw.attrib_base),
+              "attrib_base ends the graphics arm");
+static_assert(hk_root_descriptor_offset(set_dynamic_buffer_start) ==
+                 hk_root_descriptor_offset(dynamic_buffers) +
+                    sizeof(((struct hk_root_descriptor_table *)0)
+                              ->dynamic_buffers),
+              "the dynamic buffer table is followed by its set starts");
+
 struct hk_descriptor_state {
    bool root_dirty;
    uint64_t generation;
@@ -318,10 +446,16 @@ struct hk_attachment {
 
 struct hk_bg_eot {
    uint64_t usc;
+   uint32_t primary_storage_B;
+   uint32_t staging_storage_B;
    union {
       struct agx_counts_packed counts;
       uint64_t rsrc_spec;
    };
+   /* Frame anatomy profiler (hk_frame_anatomy.h): per-RT op summary of the
+    * program. Written only while HK_FRAME_ANATOMY is set; otherwise zero and
+    * never read. Not part of any hardware or kernel structure. */
+   uint8_t anatomy_ops[HK_MAX_RTS];
 };
 
 /* BG/EOT programs are self-addressed and may reference command-buffer pool
@@ -375,6 +509,10 @@ struct hk_render_registers {
       struct hk_bg_eot main;
       struct hk_bg_eot partial;
    } eot;
+
+   /* Frame anatomy profiler only: bit i set when colour attachment i's image
+    * layout is compressed. Zero unless HK_FRAME_ANATOMY is set. */
+   uint8_t color_compressed_mask;
 };
 
 struct hk_rendering_state {
@@ -427,6 +565,11 @@ struct hk_rendering_state {
     */
    VkImageAspectFlags g17p_zls_written;
 
+   /* M5: an attachment of the pass can alias another resource (aliased,
+    * sparse or linear image), so its stores count as memory writes for
+    * dependency tracking (HK_SYNC_GFX_MEM). */
+   bool m5_attachments_reach_memory;
+
    /* AGX3 dead depth store: the "only" ZLS word with the depth store
     * dropped, and whether the pass processes empty tiles without that store,
     * for a later pass that makes the store dead (hk_take_pending_z_store()).
@@ -521,6 +664,10 @@ struct hk_g17p_launch {
 };
 
 struct hk_g17p_graphics_ioto_cache_entry {
+   uint16_t m5_range_count;
+   uint16_t m5_tile_size_px;
+   uint8_t m5_sample_size_B;
+   uint8_t m5_nr_samples;
    /* The loader of the last draw, the program published from it and the
     * range addresses written into it (hk_g17p_publish_gfx_loader()). */
    const struct hk_g17p_gfx_loader *loader;
@@ -573,6 +720,24 @@ struct hk_graphics_state {
    uint64_t root_vertex_buffer_generation;
 
    uint64_t root;
+   /* CPU copy of the root uploaded at root (hk_cmd_buffer_upload_root()). */
+   const struct hk_root_descriptor_table *root_cpu;
+   /* The root the fragment program was last published with, and its CPU
+    * copy (hk_m5_fragment_root()). */
+   uint64_t fs_root;
+   const struct hk_root_descriptor_table *fs_root_cpu;
+
+   /* A successful equality check between the immutable root snapshots at
+    * fs_root and this address. Cleared whenever fs_root is replaced. */
+   uint64_t fs_root_checked_root;
+
+   /* CPU-cached copies of exactly the fragment-visible published bytes. The
+    * pointer keys name the immutable GPU-mapped copies, never their contents. */
+   struct hk_root_fragment_snapshot root_fragment_snapshot;
+   struct hk_root_fragment_snapshot fs_root_fragment_snapshot;
+   const struct hk_root_descriptor_table *root_fragment_snapshot_cpu;
+   const struct hk_root_descriptor_table *fs_root_fragment_snapshot_cpu;
+
    uint64_t draw_params;
    uint64_t draw_id_ptr;
    uint32_t g17p_draw_params_value[2];
@@ -617,6 +782,15 @@ struct hk_graphics_state {
       bool valid;
    } g17p_vdm_update_memo;
 
+   /* The last M5 HAL300 graphics core agx_ppp_hal300_graphics_core_pack()
+    * packed, a pure function of its fields (zero-initialised, so they compare
+    * bytewise); see hk_flush_ppp_state(). */
+   struct {
+      struct agx_ppp_hal300_graphics_fields fields;
+      struct agx_ppp_hal300_graphics_core state;
+      bool valid;
+   } m5_ppp_pack_memo;
+
    uint32_t shaders_dirty;
    struct hk_api_shader *shaders[MESA_SHADER_MESH + 1];
 
@@ -643,6 +817,8 @@ struct hk_graphics_state {
     * per-stream. Zero if no query is bound.
     */
    uint64_t xfb_query[4];
+   /* Active VK_QUERY_TYPE_PRIMITIVES_GENERATED_EXT reports, per stream. */
+   uint64_t prims_generated_query[4];
 
    struct hk_index_buffer_state index;
    enum agx_primitive topology;
@@ -660,6 +836,16 @@ struct hk_graphics_state {
        * There can only be one active at a time (hardware contraint).
        */
       uint16_t index;
+
+      /* On M5, an empty query needs only its logical availability word.
+       * Assign its physical counter when the first draw needs it. */
+      struct hk_query_pool *pending_pool;
+      uint32_t pending_query;
+
+      /* Recording order (hk_sync_stream::seq) of the render the active query
+       * began in, 0 if none: only a query that also ends there may become
+       * available with that render's retirement. */
+      uint32_t begin_render_seq;
    } occlusion;
 
    /* Fast linked shader data structures */
@@ -699,7 +885,10 @@ struct hk_graphics_state {
       bool reads_tib;
    } fast_link_fs;
 
-   /* Exact last VS/FS loader identities. These are recording-only selectors;
+   /* Keep these four native graphics caches contiguous: AGX2 resets leave
+    * their zero-initialized storage untouched.
+    *
+    * Exact last VS/FS loader identities. These are recording-only selectors;
     * the published programs and every referenced BO remain command-buffer-
     * owned. Submission never mutates or consults this cache. */
    struct hk_g17p_graphics_ioto_cache_entry g17p_ioto[2];
@@ -741,10 +930,13 @@ struct hk_graphics_state {
       enum mesa_prim prim;
    } tess;
 
-   
+   /* Native mesh draw state (VK_EXT_mesh_shader, native TA path). */
    struct {
       struct agx_ptr params;
       uint64_t grid, object_program, object_params;
+      /* Direct-draw API task grid, used for the object dispatch when an
+       * task shader is bound. */
+      uint32_t task_grid[3];
    } mesh;
 
    /* Needed by vk_command_buffer::dynamic_graphics_state */
@@ -752,6 +944,59 @@ struct hk_graphics_state {
    struct vk_sample_locations_state _dynamic_sl;
 };
 
+/* The CPU source is mutable. Snapshot its visible bytes at publication,
+ * before copying into the immutable GPU-mapped root. */
+static inline void
+hk_root_fragment_snapshot_publish(struct hk_graphics_state *gfx,
+                                  const struct hk_root_descriptor_table *cpu,
+                                  const struct hk_root_descriptor_table *source)
+{
+   hk_root_fragment_snapshot_store(&gfx->root_fragment_snapshot, source);
+   gfx->root_fragment_snapshot_cpu = cpu;
+}
+
+/* Select the same immutable root as the fresh byte comparison. Hidden reads
+ * always use the current root; snapshots and the positive identity memo only
+ * accelerate the equality check for shaders that cannot observe those bytes. */
+static inline uint64_t
+hk_m5_fragment_root_select(struct hk_graphics_state *gfx, uint8_t root_reads,
+                           bool use_memo, bool use_shadow)
+{
+   const uint8_t hidden = HK_ROOT_READS_ADDRESS | HK_ROOT_READS_VERTEX_INPUT |
+                          HK_ROOT_READS_DYNAMIC_BUFFERS;
+
+   const bool shadow = use_shadow &&
+                       gfx->root_fragment_snapshot_cpu == gfx->root_cpu &&
+                       gfx->fs_root_fragment_snapshot_cpu == gfx->fs_root_cpu;
+
+   if (gfx->fs_root && gfx->fs_root != gfx->root && gfx->root_cpu &&
+       !(root_reads & hidden) &&
+       ((use_memo && gfx->fs_root_checked_root &&
+         gfx->fs_root_checked_root == gfx->root) ||
+        (shadow
+            ? !memcmp(&gfx->fs_root_fragment_snapshot,
+                      &gfx->root_fragment_snapshot,
+                      sizeof(gfx->root_fragment_snapshot))
+            : hk_root_matches_for_fragment(gfx->fs_root_cpu, gfx->root_cpu)))) {
+      if (use_memo)
+         gfx->fs_root_checked_root = gfx->root;
+      return gfx->fs_root;
+   }
+
+   if (use_shadow && gfx->root_cpu &&
+       gfx->root_fragment_snapshot_cpu == gfx->root_cpu) {
+      gfx->fs_root_fragment_snapshot = gfx->root_fragment_snapshot;
+      gfx->fs_root_fragment_snapshot_cpu = gfx->root_cpu;
+   } else {
+      gfx->fs_root_fragment_snapshot_cpu = NULL;
+   }
+
+   gfx->fs_root_checked_root = 0;
+   gfx->fs_root = gfx->root;
+   gfx->fs_root_cpu = gfx->root_cpu;
+   return gfx->root;
+}
+
 struct hk_compute_state {
    struct hk_descriptor_state descriptors;
    struct hk_api_shader *shader;
@@ -797,6 +1042,10 @@ enum hk_cs_type {
    HK_CS_VDM,
 };
 
+/* Fragment programs a control stream records for the frame anatomy
+ * profiler (hk_frame_anatomy.h). */
+#define HK_FA_FS_MAX 4
+
 struct hk_cs {
    struct list_head node;
 
@@ -860,6 +1109,8 @@ struct hk_cs {
    bool gen1_native_rq;
    bool gen1_native_fragment_rq;
    bool gen1_native_vertex_rq;
+   bool m5_native_rq;
+   bool m5_matrix_ops;
 
    /* At least one AS-build dispatch or compute transfer is in this stream. */
    bool as_build;
@@ -901,6 +1152,31 @@ struct hk_cs {
     */
    struct util_dynarray imm_writes;
 
+   /* Native private-memory profiles for every dispatch in this stream. */
+   struct util_dynarray compute_resources;
+   /* Fields of the M5 HAL300 graphics core this stream last emitted, and the
+    * records they packed to. The PPP latches each record for the following
+    * draws of the stream, so an identical core is not emitted again and a
+    * changed one emits only its changed records; any other PPP state emitted
+    * into the stream clears m5_ppp_emitted_valid. See hk_flush_ppp_state(). */
+   struct agx_ppp_hal300_graphics_fields m5_ppp_emitted;
+   struct agx_ppp_hal300_graphics_core m5_ppp_emitted_state;
+   bool m5_ppp_emitted_valid;
+
+   struct util_dynarray render_resources;
+   /* The render_resources entry the last draw matched or appended, see
+    * hk_m5_note_render_resources(). */
+   uint8_t m5_render_resources_last[3 * sizeof(struct drm_asahi_render_program_v2)];
+   size_t m5_render_resources_last_size;
+   struct drm_asahi_render_program tile_resources;
+   /* Native AGX3: struct hk_oq_stamp for each immediate write that makes an
+    * occlusion query available and could instead ride on the retirement
+    * timestamp of the render that ended the query. While oq_avail_deferred,
+    * the writes are not emitted yet: the end of the command buffer decides
+    * (hk_resolve_deferred_oq_available()). Implies CDM. */
+   struct util_dynarray oq_stamps;
+   bool oq_avail_deferred;
+
    /* Number of successfully emitted draws whose last raster producer is
     * positively known to export only the default Layer=0, without multiview.
     * Compare with stats.cmds: untracked emitters conservatively disqualify.
@@ -933,6 +1209,59 @@ struct hk_cs {
          robust_index_batches;
    } stats;
 
+   /* Frame anatomy profiler only: the first fragment programs this stream
+    * draws with and their draw counts; later programs are only counted.
+    * Zero unless HK_FRAME_ANATOMY is set. */
+   struct {
+      const struct hk_shader *fs;
+      uint32_t draws;
+      /* Compute streams: total threads and workgroups of the direct
+       * dispatches, and how many were indirect. */
+      uint64_t threads, groups;
+      uint32_t indirect;
+   } anatomy_fs[HK_FA_FS_MAX];
+   uint32_t anatomy_fs_other;
+
+   /* Likewise for the hardware vertex programs. */
+   struct {
+      const struct hk_shader *vs;
+      uint32_t draws;
+   } anatomy_vs[HK_FA_FS_MAX];
+   uint32_t anatomy_vs_other;
+
+   /* Frame anatomy profiler only: driver-internal (precompiled) kernels this
+    * stream dispatches, with counts and total threads (UINT64_MAX once any
+    * dispatch is indirect). */
+   struct {
+      uint16_t program;
+      uint16_t dispatches;
+      uint64_t threads;
+   } anatomy_precomp[HK_FA_FS_MAX];
+   uint32_t anatomy_precomp_other;
+
+   /* Frame anatomy profiler only: the API pipeline barriers recorded between
+    * the previous stream's creation and this one's (count, and the union of
+    * their source and destination stage masks). */
+   uint32_t anatomy_api_barriers;
+   uint16_t anatomy_api_mem, anatomy_api_buf, anatomy_api_img;
+   /* Frame anatomy profiler only: draws that republished render state,
+    * vertex-program state updates and how many named a program other than
+    * the stream's previous one, and PPP state records emitted. */
+   uint32_t anatomy_republish, anatomy_vdm, anatomy_vdm_new, anatomy_ppp;
+   /* Colour attachments whose load the first draw made unnecessary. */
+   uint8_t anatomy_covered;
+   /* Frame anatomy profiler only, M5: draws by the latched fragment pass type
+    * (opaque, translucent, punch-through, translucent punch-through), and
+    * draws with triangle merging disabled, with depth writes, and with a
+    * depth mode other than UNCHANGED (shader depth or side effects). */
+   uint32_t anatomy_pass[4], anatomy_nomerge, anatomy_zwrite, anatomy_late_z;
+   /* Frame anatomy profiler only: the render area of a pass that does not
+    * cover its whole framebuffer (zero extent otherwise). */
+   VkRect2D anatomy_area;
+   uint64_t anatomy_last_program;
+   uint64_t anatomy_api_src, anatomy_api_dst;
+   uint64_t anatomy_api_src_access, anatomy_api_dst_access;
+
    /* Timestamp writes: the start of the command (compute start, render
     * vertex start) and its end (compute end, render fragment end). The
     * kernel writes both from the firmware's native pass record.
@@ -942,6 +1271,11 @@ struct hk_cs {
       struct agx_timestamp_req end;
    } timestamp;
 
+   /* Render only: occlusion availability slots written by the kernel at
+    * retirement, indexed vertex start, vertex end, fragment start, fragment
+    * end. Slots 0 and 3 are only used when timestamp.start/end are not. */
+   struct agx_timestamp_req oq_stamp[4];
+
    /* G17P query commands observe timestamp writes from earlier streams.
     * Firmware start timestamps reach query memory only at retirement, so a
     * producer cannot be merged across this consumer boundary.
@@ -975,7 +1309,7 @@ struct hk_cs {
     *
     */
    bool has_sample_locations;
-   uint32_t ppp_multisamplectl;
+   uint64_t ppp_multisamplectl;
 
    struct hk_render_registers cr;
 
@@ -1024,6 +1358,8 @@ hk_cs_merge_cdm(struct hk_cs *a, const struct hk_cs *b)
    assert(a->type == HK_CS_CDM && b->type == HK_CS_CDM);
    assert(a->cmd == b->cmd);
    assert(!a->timestamp.end.handle);
+   a->m5_native_rq |= b->m5_native_rq;
+   a->m5_matrix_ops |= b->m5_matrix_ops;
 
    const size_t b_size = (uint8_t *)b->current - (uint8_t *)b->start;
    const size_t remaining = (uint8_t *)a->end - (uint8_t *)a->current;
@@ -1109,6 +1445,9 @@ hk_cs_merge_cdm(struct hk_cs *a, const struct hk_cs *b)
    if (!a->timestamp.start.handle)
       a->timestamp.start = b->timestamp.start;
 
+   util_dynarray_append_array(&a->compute_resources, uint8_t,
+                             b->compute_resources.data, b->compute_resources.size);
+
    a->g17p_timestamp_query |= b->g17p_timestamp_query;
 
    a->stats.calls += b->stats.calls;
@@ -1180,6 +1519,14 @@ struct hk_gen1_as_table_chunk {
 struct hk_cmd_buffer {
    struct vk_command_buffer vk;
 
+   /* Frame anatomy profiler only: API pipeline barriers recorded since the
+    * last stream was created (see hk_cs::anatomy_api_barriers). */
+   struct {
+      uint32_t count;
+      uint16_t mem, buf, img;
+      uint64_t src, dst, src_access, dst_access;
+   } anatomy_bar;
+
    /* frg-codegen v4: the per-recording BG/EOT cache (13.5 KiB) lives outside
     * `state` so the full-state memset of every reset no longer clears it;
     * hk_reset_cmd_buffer() invalidates the entries instead. Entries are only
@@ -1320,6 +1667,27 @@ struct hk_cmd_buffer {
     * conditional rendering. */
    bool conditional_meta;
 
+   /* M5 native ray traversal: the per-raygen dispatches of one
+    * vkCmdTraceRays* share a single initialized SCS state block, since only
+    * the selected kernel has threadgroups. Valid only while `share` is set. */
+   struct {
+      bool share;
+      uint64_t packed;
+      struct agx_ptr state;
+      /* Traversal uniforms shared by the same dispatches. */
+      struct agx_ptr uniforms;
+      uint64_t uniforms_packed, uniforms_state;
+      /* Opt-in ASAHI_M5_RT_STATE_DIAG: initial images of the SCS state
+       * blocks this recording allocated, compared after execution. */
+      struct util_dynarray diag;
+   } m5_rt_state;
+
+   /* Single-kick vkCmdTraceRays* (ASAHI_M5_RT_SINGLE_KICK, default on): while set,
+    * hk_dispatch_m5_native writes the kernel's launch record (40-byte direct
+    * launch + barrier) here instead of into the control stream. */
+   uint8_t *m5_rt_capture;
+   bool m5_rt_captured;
+
    /* XXX: move me?
     *
     * Indirect draw generated by the indirect GS translator.
@@ -1527,8 +1895,8 @@ hk_bound_hw_vs(struct hk_graphics_state *gfx)
    struct hk_api_shader *gs = gfx->shaders[MESA_SHADER_GEOMETRY];
    struct hk_api_shader *mesh = gfx->shaders[MESA_SHADER_MESH];
 
-   
-
+   /* A mesh pipeline has no vertex, tessellation or geometry stage; its
+    * native mesh variant is the hardware vertex program. */
    if (mesh)
       return &mesh->variants[HK_MESH_VARIANT_MAIN];
    else if (gs)
@@ -1545,8 +1913,8 @@ hk_bound_sw_vs(struct hk_graphics_state *gfx)
    struct hk_api_shader *vs = gfx->shaders[MESA_SHADER_VERTEX];
    struct hk_shader *hw_vs = hk_bound_hw_vs(gfx);
 
-   
-
+   /* Mesh pipelines: the raster variant is both the hardware and the only
+    * vertex program (no attributes, no prolog remap). */
    if (!vs)
       return hw_vs;
 
@@ -1563,7 +1931,7 @@ hk_bound_sw_vs_before_gs(struct hk_graphics_state *gfx)
    struct hk_api_shader *tes = gfx->shaders[MESA_SHADER_TESS_EVAL];
    struct hk_api_shader *api = tes ?: vs;
 
-   
+   /* Mesh pipelines have no software vertex stage. */
    if (!api)
       return NULL;
 
@@ -1712,9 +2080,10 @@ bool hk_cs_init_graphics(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
 void hk_g17p_note_depth_clear(struct hk_cmd_buffer *cmd,
                               VkImageAspectFlags aspects);
 
-uint32_t hk_default_sample_positions(unsigned nr_samples);
+uint64_t hk_default_sample_positions(unsigned nr_samples);
 
-void hk_flush_deferred_clears(struct hk_cmd_buffer *cmd);
+void hk_flush_deferred_clears_why(struct hk_cmd_buffer *cmd, const char *why);
+#define hk_flush_deferred_clears(cmd) hk_flush_deferred_clears_why(cmd, __func__)
 
 /* The depth image of the pending store is overwritten as a whole before
  * anything read it: the store is dead (see hk_take_pending_z_store()). */
@@ -1786,6 +2155,8 @@ hk_cmd_buffer_get_cs_general(struct hk_cmd_buffer *cmd, struct hk_cs **ptr,
          .end = root.cpu + initial_size,
       };
 
+      util_dynarray_init(&cs->compute_resources, NULL);
+      util_dynarray_init(&cs->render_resources, NULL);
       list_inithead(&cs->node);
       hk_drm_attachment_set_init(&cs->compute_attachments);
       /* Only known meta destinations are collected so far. */
@@ -1870,17 +2241,22 @@ hk_cmd_buffer_dirty_all(struct hk_cmd_buffer *cmd)
 static inline void
 hk_cs_destroy(struct hk_cs *cs)
 {
+   util_dynarray_fini(&cs->compute_resources);
+   util_dynarray_fini(&cs->render_resources);
    if (cs->type == HK_CS_VDM) {
       util_dynarray_fini(&cs->scissor);
       util_dynarray_fini(&cs->depth_bias);
    } else {
       util_dynarray_fini(&cs->imm_writes);
+      util_dynarray_fini(&cs->oq_stamps);
    }
 
    free(cs);
 }
 
 void hk_dispatch_imm_writes(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
+bool hk_defer_oq_available(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
+void hk_resolve_deferred_oq_available(struct hk_cmd_buffer *cmd);
 
 bool hk_g17p_smooth_varyings_supported(
    struct hk_cmd_buffer *cmd, const struct hk_linked_shader *linked);
@@ -1898,9 +2274,15 @@ hk_cmd_buffer_end_compute_internal(struct hk_cmd_buffer *cmd,
          hk_flush_query_resets(cmd, cs);
 
       /* This control stream may write immediates as it ends. Queue the writes
-       * now that we're done emitting everything else.
+       * now that we're done emitting everything else. Occlusion availability
+       * alone may instead wait for the end of the command buffer, which
+       * keeps the (still empty) stream in the list.
        */
       if (cs->imm_writes.size) {
+         if (hk_defer_oq_available(cmd, cs)) {
+            *ptr = NULL;
+            return;
+         }
          hk_dispatch_imm_writes(cmd, cs);
       }
 
@@ -2007,7 +2389,7 @@ hk_stat_enabled(uint64_t addr)
    return addr != AGX_SCRATCH_PAGE_ADDRESS;
 }
 
-/* M5 render pass continuation (hk_cmd_draw.c): a pipeline barrier recorded
+/* AGX3 render pass continuation (hk_cmd_draw.c): a pipeline barrier recorded
  * while an ended instance is still continuable decides whether the next one
  * may continue in its stream. */
 void hk_m5_pass_merge_barrier(struct hk_cmd_buffer *cmd,
@@ -2096,11 +2478,12 @@ hk_reserve_scratch(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    /* Not scratch but this is the most convenient place for this... */
    cs->uses_sampler_heap |= s->b.info.uses_sampler_heap;
 
-   /* G17P publishes a shader's spill frame as thread-private storage in its
+   /* G17P and M5 publish a shader's spill frame as thread-private storage in their
     * SPECTPR. There is no helper-program stack, and the kernel rejects a
     * command that names one. */
    if ((s->b.info.scratch_size == 0 && s->b.info.preamble_scratch_size == 0) ||
-       agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev))
+       agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) ||
+       agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip))
       return;
 
    hk_reserve_scratch_slow(cmd, cs, s);
@@ -2111,17 +2494,34 @@ hk_reserve_scratch(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
 static inline bool
 hk_sync_image_class(const struct hk_cmd_buffer *cmd)
 {
-   return hk_cmd_buffer_device(cmd)->dev.chip == AGX_CHIP_G17P;
+   const enum agx_chip chip = hk_cmd_buffer_device(cmd)->dev.chip;
+   return chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip);
 }
 
 /* AGX2 returns its shader-base-relative USC offset.  G17P returns the absolute
  * 48-bit Ioto program VA consumed by HAL200 VDM state. */
+struct agx3_ioto_uniform_range;
+void hk_m5_rt_state_diag_record(struct hk_cmd_buffer *cmd, struct agx_ptr state,
+                                uint64_t size);
+
+bool hk_m5_compute_uniform_dma_ranges(
+   struct hk_cmd_buffer *cmd, const struct hk_shader *s, uint64_t root_ptr,
+   struct agx3_ioto_uniform_range *ranges, size_t capacity, size_t *count);
+
 uint64_t hk_g17p_dma_set_base(const struct hk_cmd_buffer *cmd,
                               const struct hk_shader *s);
 
+struct agx3_ioto_uniform_range;
+bool hk_g17p_dma_ranges(const struct hk_descriptor_state *desc,
+                        const struct hk_shader *s, uint64_t root_ptr,
+                        struct agx3_ioto_uniform_range *ranges,
+                        size_t max_ranges, size_t *count_io,
+                        uint8_t *sources);
+
 uint64_t hk_upload_native_mesh_program(struct hk_cmd_buffer *cmd,
                                       struct hk_shader *s,
-                                      struct hk_linked_shader *linked, bool object);
+                                      struct hk_linked_shader *linked, bool object,
+                                      bool task);
 uint64_t hk_upload_usc_words(struct hk_cmd_buffer *cmd, struct hk_shader *s,
                              struct hk_linked_shader *linked);
 
@@ -2140,6 +2540,10 @@ void hk_dispatch_with_usc(struct hk_device *dev, struct hk_cs *cs,
                           struct agx_grid grid,
                           struct agx_workgroup local_size);
 
+void hk_dispatch_m5_native(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
+                            struct hk_shader *s, struct agx_grid grid,
+                            struct agx_workgroup local_size);
+
 void hk_dispatch_g17p_native(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
                              struct hk_shader *s, struct agx_grid grid,
                              struct agx_workgroup local_size);
@@ -2205,6 +2609,10 @@ hk_dispatch_with_local_size(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
       return;
 
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (agx_chip_g17_m5(dev->dev.chip)) {
+      hk_dispatch_m5_native(cmd, cs, s, grid, local_size);
+      return;
+   }
    if (agx_native_graphics(&dev->dev)) {
       hk_reserve_scratch(cmd, cs, s);
       hk_dispatch_g17p_native(cmd, cs, s, grid, local_size);
@@ -2234,6 +2642,9 @@ hk_dispatch_with_local_size(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
 void hk_dispatch_precomp(struct hk_cmd_buffer *cmd, struct agx_grid grid,
                          enum agx_barrier barrier, enum libagx_program idx,
                          void *data, size_t data_size);
+void hk_dispatch_precomp_m5(struct hk_cmd_buffer *cmd, struct agx_grid grid,
+                            enum agx_barrier barrier, enum libagx_program idx,
+                            const void *data, size_t data_size);
 void hk_dispatch_precomp_g17p(struct hk_cmd_buffer *cmd, struct agx_grid grid,
                               enum agx_barrier barrier,
                               enum libagx_program idx, const void *data,
@@ -2250,6 +2661,7 @@ uint64_t hk_apply_conditional_rendering(struct hk_cmd_buffer *cmd,
 
 void hk_queue_write(struct hk_cmd_buffer *cmd, uint64_t address, uint32_t value,
                     bool after_gfx);
+bool hk_ensure_occlusion_query_for_draw(struct hk_cmd_buffer *cmd);
 
 /* Dispatch the native G17P raw-address fill meta shader into an explicitly
  * selected compute stream.  The target may be the ordinary or post-graphics
diff --git a/src/asahi/vulkan/hk_cmd_clear.c b/src/asahi/vulkan/hk_cmd_clear.c
index f8ec59594261..dfbd81d7f6b9 100644
--- a/src/asahi/vulkan/hk_cmd_clear.c
+++ b/src/asahi/vulkan/hk_cmd_clear.c
@@ -9,6 +9,7 @@
 #include "util/u_math.h"
 #include "vulkan/vulkan_core.h"
 #include "hk_cmd_buffer.h"
+#include "hk_frame_anatomy.h"
 #include "layout.h"
 
 #include "hk_device.h"
@@ -43,6 +44,16 @@ hk_clear_image(struct hk_cmd_buffer *cmd, struct hk_image *image,
       struct ail_layout *layout = &image->planes[plane].layout;
       perf_debug(cmd, "Image clear (%scompressed)",
                  layout->compressed ? "" : "un");
+      if (unlikely(hk_cmd_buffer_device(cmd)->frame_anatomy)) {
+         hk_frame_anatomy_note(
+            hk_cmd_buffer_device(cmd),
+            "image clear %ux%ux%u %s samples=%u levels=%u plane=%u "
+            "compressed=%u usage=0x%x empty_pass=%u",
+            image->vk.extent.width, image->vk.extent.height,
+            image->vk.array_layers, util_format_short_name(layout->format),
+            layout->sample_count_sa, level_count, plane, layout->compressed,
+            (unsigned)image->vk.usage, cmd->sync.kind_override != 0);
+      }
 
       for (uint32_t l = 0; l < level_count; l++) {
          const uint32_t level = range->baseMipLevel + l;
@@ -143,7 +154,9 @@ hk_clear_image(struct hk_cmd_buffer *cmd, struct hk_image *image,
             libagx_fast_clear(cmd, grid, AGX_BARRIER_ALL, meta_addr, address,
                               grid.count[0], grid.count[1], tilesize.width_el,
                               tilesize.height_el, meta_layer_stride_tl,
-                              layer_stride_uint4, util_logbase2(blocksize_B),
+                              layer_stride_uint4,
+                              util_logbase2(ail_metadata_tile(layout).height_el),
+                              util_logbase2(blocksize_B),
                               util_logbase2(layout->sample_count_sa), c[0],
                               c[1], c[2], c[3], word);
          } else {
@@ -201,6 +214,16 @@ hk_CmdClearColorImage(VkCommandBuffer commandBuffer, VkImage _image,
                       rangeCount, pRanges))
       return;
 
+   if (unlikely(hk_cmd_buffer_device(cmd)->frame_anatomy)) {
+      hk_frame_anatomy_note(
+         hk_cmd_buffer_device(cmd),
+         "clear not deferred: %ux%u %s ranges=%u levels=%u layers=%u usage=0x%x compressed=%u",
+         image->vk.extent.width, image->vk.extent.height,
+         util_format_short_name(hk_format_to_pipe_format(image->vk.format)),
+         rangeCount, image->vk.mip_levels, image->vk.array_layers,
+         (unsigned)image->vk.usage, image->planes[0].layout.compressed);
+   }
+
    /* Transfer stage: the streams recorded here belong to the transfer
     * class for every recorded dependency. */
    uint64_t lo, hi;
diff --git a/src/asahi/vulkan/hk_cmd_dispatch.c b/src/asahi/vulkan/hk_cmd_dispatch.c
index 3df67b7fa834..0d183179248a 100644
--- a/src/asahi/vulkan/hk_cmd_dispatch.c
+++ b/src/asahi/vulkan/hk_cmd_dispatch.c
@@ -14,6 +14,7 @@
 #include "hk_buffer.h"
 #include "hk_acceleration_structure.h"
 #include "hk_cmd_buffer.h"
+#include "hk_frame_anatomy.h"
 #include "hk_g17p_patch.h"
 #include "hk_descriptor_set.h"
 #include "hk_device.h"
@@ -21,6 +22,7 @@
 #include "hk_entrypoints.h"
 #include "hk_agx3_ioto_program_core.h"
 #include "asahi/compiler/agx3_compute_program.h"
+#include "asahi/compiler/agx3_graphics_program.h"
 #include "hk_physical_device.h"
 #include "hk_shader.h"
 #include "libagx_dgc.h"
@@ -39,6 +41,23 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+/* Diagnostic controls, read once: these are consulted per dispatch, and an
+ * uncached getenv() scans the whole environment. */
+DEBUG_GET_ONCE_BOOL_OPTION(m5_launch_debug, "ASAHI_M5_LAUNCH_DEBUG", false)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_verify_compute_loader,
+                           "AGX_M5_VERIFY_COMPUTE_LOADER", false)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_rq_candidate_probe, "ASAHI_M5_RQ_CANDIDATE_PROBE",
+                           false)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_rq_candidate_api, "ASAHI_M5_RQ_CANDIDATE_API",
+                           false)
+
+static inline bool
+hk_m5_rq_candidate_requested(void)
+{
+   return debug_get_option_m5_rq_candidate_probe() ||
+          debug_get_option_m5_rq_candidate_api();
+}
+
 /* EXP-G1b's native access::write argument is a PBE pointer followed by a
  * pointer to this control descriptor, including when it follows a sampled
  * texture in the argument table. It is not the read_write clamp scalar. */
@@ -46,7 +65,10 @@ static const uint8_t hk_g17p_native_write_control[32] = {
    0x08, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00,
 };
 
-#define HK_G17P_COMPUTE_MAX_RANGES 8u
+/* Root, texture pointers, promoted constants and the ray-query range, plus
+ * the uniform DMA spans of an application kernel (each at most a buffer part
+ * and a zero-page tail; hk_g17p_dma_ranges()). */
+#define HK_G17P_COMPUTE_MAX_RANGES (4u + 2u * AGX3_UNIFORM_DMA_MAX_SPANS)
 #define HK_G17P_LOADER_MAX_SHAPES  4u
 static_assert(HK_G17P_COMPUTE_MAX_RANGES <= HK_G17P_PATCH_MAX_RANGES,
               "patch plan capacity");
@@ -80,6 +102,122 @@ struct hk_g17p_loader {
    uint8_t bytes[];
 };
 
+struct hk_m5_compute_loader {
+   struct hk_m5_compute_loader *next;
+   struct agx3_g17p_graphics_program_desc desc;
+   struct agx3_ioto_uniform_range ranges[HK_G17P_COMPUTE_MAX_RANGES];
+   struct hk_g17p_patch_plan plan;
+   uint16_t length;
+   uint8_t bytes[];
+};
+
+static void
+hk_m5_loader_list_free(void **head)
+{
+   struct hk_m5_compute_loader *l = p_atomic_read(head);
+   while (l) {
+      struct hk_m5_compute_loader *next = l->next;
+      free(l);
+      l = next;
+   }
+   *head = NULL;
+}
+
+static bool
+hk_m5_loader_matches(const struct hk_m5_compute_loader *l,
+                      const struct agx3_g17p_graphics_program_desc *d)
+{
+   const struct agx3_g17p_graphics_program_desc *a = &l->desc;
+   if (a->shader_va != d->shader_va ||
+       a->shader_binary_size != d->shader_binary_size ||
+       a->scratch_size != d->scratch_size || a->nr_gprs != d->nr_gprs ||
+       a->compiler_d_valid != d->compiler_d_valid || a->compiler_d != d->compiler_d ||
+       a->threadgroup_memory_size_B != d->threadgroup_memory_size_B ||
+       a->uniform_range_count != d->uniform_range_count ||
+       d->fragment || d->writes_sample_mask || d->colour_classes ||
+       d->preloaded_root || d->patchable ||
+       a->constant_program.code_va != d->constant_program.code_va ||
+       a->constant_program.code_size != d->constant_program.code_size ||
+       a->constant_program.request_prelude != d->constant_program.request_prelude ||
+       a->constant_program.absolute_branch != d->constant_program.absolute_branch ||
+       a->cprog_inputs != d->cprog_inputs)
+      return false;
+   for (unsigned i = 0; i < d->uniform_range_count; ++i)
+      if (l->ranges[i].start_halfs != d->uniform_ranges[i].start_halfs ||
+          l->ranges[i].size_halfs != d->uniform_ranges[i].size_halfs)
+         return false;
+   return true;
+}
+
+static struct hk_m5_compute_loader *
+hk_m5_loader_create(struct hk_device *dev,
+                     const struct agx3_g17p_graphics_program_desc *d)
+{
+   if (d->uniform_range_count > HK_G17P_COMPUTE_MAX_RANGES)
+      return NULL;
+   struct agx3_g17p_graphics_program_template *t = calloc(1, sizeof(*t));
+   if (!t)
+      return NULL;
+   struct hk_m5_compute_loader *l = NULL;
+   if (!agx3_g17g_compute_program_template_generate(dev->dev.chip, d, t))
+      goto out;
+   l = calloc(1, sizeof(*l) + t->program.length);
+   if (!l)
+      goto out;
+   if (!hk_g17p_patch_plan_init(&l->plan, d->uniform_ranges,
+                                d->uniform_range_count, t->relocations,
+                                t->relocation_count, t->program.length, false)) {
+      free(l);
+      l = NULL;
+      goto out;
+   }
+   l->desc = *d;
+   l->desc.uniform_ranges = l->ranges;
+   memcpy(l->ranges, d->uniform_ranges,
+          d->uniform_range_count * sizeof(l->ranges[0]));
+   l->length = t->program.length;
+   memcpy(l->bytes, t->program.bytes, l->length);
+out:
+   free(t);
+   return l;
+}
+
+static const struct hk_m5_compute_loader *
+hk_m5_loader_get(struct hk_device *dev, struct hk_linked_shader *linked,
+                  const struct agx3_g17p_graphics_program_desc *d)
+{
+   unsigned shapes = 0;
+   for (struct hk_m5_compute_loader *l =
+           p_atomic_read(&linked->m5_compute_templates); l; l = l->next) {
+      if (hk_m5_loader_matches(l, d)) {
+         if (hk_g17p_patch_plan_fits(&l->plan, d->uniform_ranges))
+            return l;
+         ++shapes;
+      }
+   }
+   /* Preserve the normal load schedule. Unusual changing address layouts
+    * fall back to fresh composition rather than lengthening GPU programs. */
+   if (shapes >= HK_G17P_LOADER_MAX_SHAPES)
+      return NULL;
+   struct hk_m5_compute_loader *created = hk_m5_loader_create(dev, d);
+   if (!created)
+      return NULL;
+   while (true) {
+      struct hk_m5_compute_loader *first =
+         p_atomic_read(&linked->m5_compute_templates);
+      for (struct hk_m5_compute_loader *l = first; l; l = l->next) {
+         if (hk_m5_loader_matches(l, d) &&
+             hk_g17p_patch_plan_fits(&l->plan, d->uniform_ranges)) {
+            free(created);
+            return l;
+         }
+      }
+      created->next = first;
+      if (p_atomic_cmpxchg_ptr(&linked->m5_compute_templates, first, created) == first)
+         return created;
+   }
+}
+
 /* Hot recording needs only the published loader bytes for tracing and the
  * compact launch record. Keep the maximum-size composer object off the draw
  * stack; one such frame crosses a host page boundary and faults on the first
@@ -113,6 +251,7 @@ void
 hk_g17p_linked_compute_cache_finish(struct hk_linked_shader *linked)
 {
    hk_g17p_loader_list_free(&linked->g17p_compute_templates);
+   hk_m5_loader_list_free(&linked->m5_compute_templates);
 }
 
 static bool
@@ -432,6 +571,7 @@ hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs)
          .chip = dev->dev.chip,
          .flags = 0x5,
          .wide_visibility = agx_chip_cdm_wide_visibility(dev->dev.chip),
+         .flush_memory = agx_chip_g17_m5(dev->dev.chip),
       };
       uint8_t packed[AGX3_CDM_BARRIER_SIZE];
       if (!agx3_cdm_barrier_pack(&info, packed)) {
@@ -540,8 +680,8 @@ hk_dispatch_with_usc(struct hk_device *dev, struct hk_cs *cs,
 }
 
 static bool
-hk_g17p_compute_shader_supported(struct hk_shader *s,
-                                 struct hk_linked_shader *linked)
+hk_g17p_compute_shader_supported(enum agx_chip chip, struct hk_shader *s,
+                                 struct hk_linked_shader *linked, unsigned max_spill_B)
 {
    /* poly_nir_lower_gs preserves the API geometry stage on the main kernel,
     * even though Honeykrisp has always launched that binary through CDM. */
@@ -551,9 +691,16 @@ hk_g17p_compute_shader_supported(struct hk_shader *s,
            s->info.stage == MESA_SHADER_TESS_CTRL) &&
           linked->b.bo && linked->b.bo->va && linked->b.size != 0 &&
           linked->b.size <= UINT32_MAX && linked->b.nr_gprs != 0 &&
-          !s->b.info.has_preamble && s->b.info.preamble_scratch_size == 0 &&
-          s->b.info.scratch_size <= AGX3_G17P_MAX_COMPUTE_SPILL_B &&
-          linked->b.scratch_size <= AGX3_G17P_MAX_COMPUTE_SPILL_B &&
+          ((!s->b.info.has_preamble && s->b.info.preamble_scratch_size == 0) ||
+           /* Only an M5 compute compile forms a constant program
+            * (hk_shader.c), and it carries its TPS frame in the launch. */
+           (agx_chip_g17_m5(chip) && s->info.stage == MESA_SHADER_COMPUTE &&
+            s->b.info.agx3_preamble_staging_known &&
+            s->b.info.preamble_scratch_size <= max_spill_B &&
+            s->b.info.preamble_scratch_size >=
+               s->b.info.agx3_preamble_inputs.storage_bytes)) &&
+          s->b.info.scratch_size <= max_spill_B &&
+          linked->b.scratch_size <= max_spill_B &&
           s->b.info.agx3_staging_known &&
           s->b.info.local_size <= 32768 &&
           s->b.info.texture_state_count == 0 &&
@@ -584,8 +731,10 @@ hk_g17p_compute_shader_supported(struct hk_shader *s,
  * the uniform file before launching the compute-stage binary. */
 static bool
 hk_g17p_compute_uniform_ranges(
-   struct hk_shader *s, uint64_t root_ptr, uint64_t texture_pointers,
-   struct agx3_ioto_uniform_range ranges[3], size_t *range_count);
+   enum agx_chip chip, struct hk_cmd_buffer *cmd, struct hk_shader *s,
+   uint64_t root_ptr, uint64_t texture_pointers,
+   struct agx3_ioto_uniform_range ranges[HK_G17P_COMPUTE_MAX_RANGES],
+   size_t *range_count);
 
 static bool
 hk_g17p_internal_graphics_uniform_ranges(
@@ -596,14 +745,16 @@ hk_g17p_internal_graphics_uniform_ranges(
       ranges[HK_G17P_COMPUTE_MAX_RANGES],
    size_t *range_count)
 {
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
    /* Uniform DMA is only enabled for hardware graphics stages;
     * this internal path cannot publish its span, so fail closed. */
    if (s->b.info.agx3_uniform_dma_count)
       return false;
 
    if (s->info.stage != MESA_SHADER_VERTEX)
-      return hk_g17p_compute_uniform_ranges(s, root_ptr, texture_pointers,
-                                             ranges, range_count);
+      return hk_g17p_compute_uniform_ranges(dev->dev.chip, cmd, s, root_ptr,
+                                             texture_pointers, ranges,
+                                             range_count);
 
    if (!!texture_pointers != s->info.agx3_indexed_sampler_valid)
       return false;
@@ -682,10 +833,15 @@ hk_g17p_internal_graphics_uniform_ranges(
    return true;
 }
 
+/* The uniform ranges of a compute-stage launch. cmd is NULL when a loader
+ * is prepared ahead of recording: the span addresses are then placeholders,
+ * which only the loader's layout and address shape retain. */
 static bool
 hk_g17p_compute_uniform_ranges(
-   struct hk_shader *s, uint64_t root_ptr, uint64_t texture_pointers,
-   struct agx3_ioto_uniform_range ranges[3], size_t *range_count)
+   enum agx_chip chip, struct hk_cmd_buffer *cmd, struct hk_shader *s,
+   uint64_t root_ptr, uint64_t texture_pointers,
+   struct agx3_ioto_uniform_range ranges[HK_G17P_COMPUTE_MAX_RANGES],
+   size_t *range_count)
 {
    if (!s || !ranges || !range_count ||
        (!!texture_pointers != (s->info.agx3_txf_texture_valid ||
@@ -695,28 +851,41 @@ hk_g17p_compute_uniform_ranges(
                                s->info.agx3_indexed_sampler_valid)))
       return false;
 
+   /* Ordinary compute pipelines, including meta, publish their uniform DMA
+    * spans here, except on M5 (see below). Geometry, tessellation and other
+    * compute-stage programs of graphics never carry spans. */
+   const unsigned dma_count =
+      agx_chip_g17_m5(chip) ? 0 : s->b.info.agx3_uniform_dma_count;
+   if (dma_count && s->info.stage != MESA_SHADER_COMPUTE)
+      return false;
+
    /* A shader that reads neither the root descriptor nor any descriptor set
     * needs no preamble loads at all: agx3_uses_root records whether the
     * compiler emitted load_root_agx, and the program generator composes a
     * no-resource program when range_count == 0. */
-   /* Uniform DMA is never enabled for compute (see hk_shader.c). */
-   if (s->b.info.agx3_uniform_dma_count)
-      return false;
-
+   /* M5's caller appends its direct DMA spans after these ABI/rodata
+    * ranges (hk_m5_compute_uniform_dma_ranges()); other chips publish them
+    * here. */
    const struct agx_rodata *rodata_probe = &s->b.info.rodata;
    if (!texture_pointers && !s->info.agx3_uses_root && s->info.set_count == 0 &&
-       rodata_probe->size_16 == 0) {
+       rodata_probe->size_16 == 0 && dma_count == 0) {
       *range_count = 0;
       return true;
    }
 
-   const unsigned root_uniform = s->info.agx3_root_uniform;
+   const unsigned root_uniform = s->info.agx3_root_uniform ? s->info.agx3_root_uniform :
+      !agx_chip_g17_m5(chip) ? 0 :
+      (s->info.agx3_storage_image_valid || s->info.agx3_sampled_storage_valid ||
+       s->info.agx3_indexed_storage_valid) ? AGX_G17P_STORAGE_ROOT_HALFS :
+      texture_pointers ? AGX_G17P_TXF_ROOT_HALFS : 0;
    if (!hk_agx3_g17p_root_range(root_uniform, s->info.set_count, root_ptr, &ranges[0]))
       return false;
 
    size_t count = 1;
    if (texture_pointers) {
-      const unsigned texture_uniform = root_uniform >= 168 ? 168 : 0;
+      const unsigned texture_uniform = agx_chip_g17_m5(chip) &&
+         (root_uniform == 164 || root_uniform == 176) ? root_uniform - 8 :
+         root_uniform >= 168 ? 168 : 0;
       ranges[count++] = (struct agx3_ioto_uniform_range){
          .start_halfs = texture_uniform,
          .size_halfs = root_uniform - texture_uniform,
@@ -733,6 +902,29 @@ hk_g17p_compute_uniform_ranges(
       count++;
    }
 
+   if (dma_count) {
+      if (cmd) {
+         if (!hk_g17p_dma_ranges(&cmd->state.cs.descriptors, s, root_ptr,
+                                 ranges, HK_G17P_COMPUTE_MAX_RANGES, &count,
+                                 NULL))
+            return false;
+      } else {
+         /* Placeholder addresses with the alignment of real ones; a buffer
+          * span is assumed to be covered by its buffer. */
+         for (unsigned i = 0; i < dma_count; ++i) {
+            const struct agx3_uniform_dma_span *span =
+               &s->b.info.agx3_uniform_dma[i];
+            if (count >= HK_G17P_COMPUTE_MAX_RANGES)
+               return false;
+            ranges[count++] = (struct agx3_ioto_uniform_range){
+               .start_halfs = span->dest,
+               .size_halfs = span->halfs,
+               .address = root_ptr + 0x4000 + span->offset_B,
+            };
+         }
+      }
+   }
+
    *range_count = count;
    return true;
 }
@@ -806,12 +998,13 @@ hk_g17p_build_compute_program(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
                               uint64_t *program_va_out,
                               struct hk_g17p_compute_program_ref *out)
 {
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
    size_t range_count;
    const bool ranges_ok = internal_graphics
       ? hk_g17p_internal_graphics_uniform_ranges(
            cmd, s, linked, root_ptr, texture_pointers, ranges, &range_count)
-      : hk_g17p_compute_uniform_ranges(s, root_ptr, texture_pointers, ranges,
-                                        &range_count);
+      : hk_g17p_compute_uniform_ranges(dev->dev.chip, cmd, s, root_ptr,
+                                        texture_pointers, ranges, &range_count);
    if (!ranges_ok) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
       return false;
@@ -919,7 +1112,8 @@ hk_g17p_prewarm_linked_compute_program_kind(
    enum agx_ioto_g17p_compute_ldshdr_f1 kind)
 {
    if (!agx_native_graphics(&dev->dev) ||
-       !hk_g17p_compute_shader_supported(s, linked))
+       !hk_g17p_compute_shader_supported(dev->dev.chip, s, linked,
+                                         AGX3_G17P_MAX_COMPUTE_SPILL_B))
       return;
 
    /* Use ordinary full-width, aligned addresses so the cached template has
@@ -947,7 +1141,7 @@ hk_g17p_prewarm_linked_compute_program_kind(
       ? hk_g17p_internal_graphics_uniform_ranges(
            cmd, s, linked, dummy_root, dummy_texture_pointers, ranges,
            &range_count)
-      : hk_g17p_compute_uniform_ranges(s, dummy_root,
+      : hk_g17p_compute_uniform_ranges(dev->dev.chip, NULL, s, dummy_root,
                                         dummy_texture_pointers, ranges,
                                         &range_count);
    struct agx3_g17p_compute_program_desc desc;
@@ -1008,8 +1202,11 @@ hk_g17p_indirect_launch_supported(struct hk_device *dev, struct agx_grid grid,
    struct hk_shader *helper =
       hk_g17p_indirect_helper(dev, grid, sparse_map_addr);
    if (grid.ptr == 0 || (grid.ptr & 3) != 0 ||
-       !hk_g17p_compute_shader_supported(helper,
-                                         helper ? helper->only_linked : NULL))
+       !hk_g17p_compute_shader_supported(dev->dev.chip, helper,
+                                         helper ? helper->only_linked : NULL,
+                                         agx_chip_g17_m5(dev->dev.chip)
+                                            ? AGX3_G17P_MAX_SPILL_B
+                                            : AGX3_G17P_MAX_COMPUTE_SPILL_B))
       return false;
 
    /* The local size must be launchable at all: a zero component would make
@@ -1311,7 +1508,615 @@ hk_dispatch_precomp_g17p(struct hk_cmd_buffer *cmd, struct agx_grid grid,
    cs->stats.cmds++;
 }
 
+static void
+hk_m5_compute_resources(struct hk_cs *cs, uint32_t scratch_B,
+                        uint32_t secondary_B, uint32_t shared_B, uint32_t threads)
+{
+   const struct drm_asahi_compute_program profile = {
+      .primary_bytes = scratch_B, .secondary_bytes = secondary_B,
+      .threadgroup_bytes = shared_B, .threads = threads,
+   };
+   /* The kernel folds every listed program into the submission's demand with
+    * a per-field maximum, so the list is a set: identical profiles add
+    * nothing. Deduplicate against the whole list, not just the last entry;
+    * long streams that alternate a few dispatch shapes (software tessellation
+    * prepasses, acceleration-structure builds) otherwise overflow the 16-bit
+    * command size and lose the device. */
+   util_dynarray_foreach(&cs->compute_resources,
+                         struct drm_asahi_compute_program, entry) {
+      if (!memcmp(entry, &profile, sizeof(profile)))
+         return;
+   }
+   util_dynarray_append(&cs->compute_resources, profile);
+}
+
+void
+hk_dispatch_precomp_m5(struct hk_cmd_buffer *cmd, struct agx_grid grid,
+                         enum agx_barrier barrier, enum libagx_program idx,
+                         const void *data, size_t data_size)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!agx_chip_g17_m5(dev->dev.chip) ||
+       (unsigned)idx >= LIBAGX_NUM_PROGRAMS) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   struct agx_precompiled_shader *prog = agx_get_precompiled(&dev->bg_eot, idx);
+   if (!prog || !prog->bo || !prog->info.main_size || !prog->info.nr_gprs ||
+       prog->info.nr_preamble_gprs ||
+       (prog->info.agx3_tps_size && !(dev->dev.params.features & DRM_ASAHI_FEATURE_COMPUTE_RESOURCES)) ||
+       prog->info.local_size >
+          dev->vk.physical->properties.maxComputeSharedMemorySize || !prog->info.agx3_staging_known ||
+       (prog->info.rodata.size_16 && !prog->rodata_offset) ||
+       !data || !data_size || prog->entry_offset > UINT64_MAX - prog->ptr ||
+       !prog->b.workgroup.x || !prog->b.workgroup.y || !prog->b.workgroup.z ||
+       (uint64_t)prog->b.workgroup.x * prog->b.workgroup.y *
+          prog->b.workgroup.z > dev->vk.physical->properties.maxComputeWorkGroupInvocations) {
+      mesa_loge("M5 precompiled helper %u: unsupported resources or grid", idx);
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   if (!agx_is_indirect(grid) &&
+       (!grid.count[0] || !grid.count[1] || !grid.count[2]))
+      return;
+
+   size_t uniform_data_size;
+   uint32_t uploaded_size;
+   uint16_t uniform_size_halfs;
+   if (!hk_g17p_precompiled_uniform_data_size(prog, data_size,
+                                               &uniform_data_size) ||
+       !hk_agx3_g17p_precompiled_data_layout(
+          uniform_data_size, &uploaded_size, &uniform_size_halfs)) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   struct hk_cs **target = (barrier & AGX_POSTGFX) ? &cmd->current_cs.post_gfx
+                         : (barrier & AGX_PREGFX) ? &cmd->current_cs.pre_gfx
+                                                 : &cmd->current_cs.cs;
+   struct hk_cs *cs = hk_cmd_buffer_get_cs_general(cmd, target, true);
+   if (!cs)
+      return;
+
+   struct agx_ptr uploaded = hk_pool_alloc(cmd, uploaded_size, 4);
+   if (!uploaded.cpu) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+      return;
+   }
+   memcpy(uploaded.cpu, data, data_size);
+   memset((uint8_t *)uploaded.cpu + data_size, 0, uploaded_size - data_size);
+   if (prog->info.printf_uniform) {
+      const size_t offset =
+         (prog->info.printf_uniform - AGX_ABI_AGX3_PRECOMP_ARG_HALFS) * 2;
+      memcpy((uint8_t *)uploaded.cpu + offset, &dev->dev.printf_buffer_address,
+             sizeof(uint64_t));
+   }
+   struct agx3_ioto_uniform_range ranges[2] = {{
+      .start_halfs = AGX_ABI_AGX3_PRECOMP_ARG_HALFS,
+      .size_halfs = uniform_size_halfs, .address = uploaded.gpu,
+   }};
+   unsigned range_count = 1;
+   if (prog->info.rodata.size_16) {
+      ranges[range_count++] = (struct agx3_ioto_uniform_range){
+         .start_halfs = prog->info.rodata.base_uniform,
+         .size_halfs = prog->info.rodata.size_16,
+         .address = prog->ptr + prog->rodata_offset,
+      };
+   }
+   size_t capacity;
+   if (!agx3_g17g_graphics_program_bound(dev->dev.chip, ranges, range_count,
+                                         &capacity)) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   struct hk_ioto_reservation reservation;
+   if (!hk_cmd_reserve_ioto_program(cmd, capacity, "M5 precompiled", &reservation))
+      return;
+   const uint64_t shader_va = prog->ptr + prog->entry_offset;
+   const struct agx3_g17p_graphics_program_desc desc = {
+      .program_va = reservation.va, .shader_va = shader_va,
+      .uniform_ranges = ranges, .uniform_range_count = range_count,
+      .shader_binary_size = prog->info.main_size,
+      .scratch_size = prog->info.agx3_tps_size,
+      .nr_gprs = prog->info.nr_gprs,
+      .compiler_d_valid = prog->info.agx3_staging_known,
+      .compiler_d = (uint32_t)prog->info.agx3_staging_halfs * 2,
+      .threadgroup_memory_size_B = prog->info.local_size
+         ? MAX2(256u, ALIGN_POT(prog->info.local_size, 64u)) : 0,
+   };
+   struct agx3_g17p_graphics_program program;
+   struct agx3_g17p_native_compute_program_desc launch_desc = {
+      .program_va = reservation.va, .shader_va = shader_va,
+   };
+   struct agx3_compute_program launch;
+   if (!agx3_g17g_compute_program_generate(dev->dev.chip, &desc, &program) ||
+       !agx3_cdm_direct_template_sources_compute(
+          false, false, false, &launch_desc.launch_sources) ||
+       !agx3_m5_native_compute_program_bind(dev->dev.chip, &launch_desc, &launch)) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   memcpy(reservation.map, program.bytes, program.length);
+   if (!hk_cmd_publish_ioto_program(cmd, &reservation, program.length))
+      return;
+   hk_cmd_buffer_retain_bo(cmd, prog->bo);
+   hk_ensure_cs_has_space(cmd, cs, 0x2000);
+   struct agx3_compute_dispatch_sink sink = {
+      .data = (uint8_t *)cs->current,
+      .capacity = (uint8_t *)cs->end - (uint8_t *)cs->current,
+   };
+   const uint8_t flags = (barrier & AGX_BARRIER_ALL) ? 5 : 0;
+   const uint32_t local[] = {
+      prog->b.workgroup.x, prog->b.workgroup.y, prog->b.workgroup.z,
+   };
+   const struct agx3_compute_direct_grid direct = {
+      .source = AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,
+      .size = {grid.count[0], grid.count[1], grid.count[2]},
+      .local_size = {local[0], local[1], local[2]},
+      .dimensions = grid.count[2] != 1 || local[2] != 1 ? 3
+                  : grid.count[1] != 1 || local[1] != 1 ? 2 : 1,
+   };
+   const bool appended = agx_is_indirect(grid)
+      ? agx3_m5_native_indirect_dispatch_append(
+           &launch, grid.ptr,
+           grid.mode == AGX_CDM_MODE_INDIRECT_GLOBAL ? local : NULL, flags, &sink)
+      : agx3_m5_native_compute_dispatch_append(&launch, &direct, flags, &sink);
+   if (!appended) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   cs->current = (uint32_t *)((uint8_t *)cs->current + sink.length);
+   if (flags == 5) {
+      cs->strong_flush_end = cs->current;
+      cs->weak_barrier_end = NULL;
+   } else {
+      cs->weak_barrier_end = cs->current;
+   }
+   hk_m5_compute_resources(cs, desc.scratch_size, 0, desc.threadgroup_memory_size_B,
+                           local[0] * local[1] * local[2]);
+   if (unlikely(dev->frame_anatomy))
+      hk_frame_anatomy_note_precomp(cs, idx, grid);
+   cs->stats.cmds++;
+}
+
+
+/*
+ * The SCS state block the native query runtime reads (the captured user
+ * image, a zero tail up to the layout's section 12, and for instance layouts
+ * the ScsLayoutGen2 parameter record at +0x80020) depends only on the packed
+ * layout, which is fixed for a device, and the GPU never writes it: with the
+ * block bound GPU-read-only the RT gate runs without a fault, and a
+ * before/after comparison over 469 traversal dispatches finds no changed byte.
+ * So it is initialized once per device and shared by every traversal
+ * dispatch, instead of allocating and copying 8 MiB per dispatch or per
+ * vkCmdTraceRays*. ASAHI_M5_RT_PERSISTENT_STATE=0 restores the per-dispatch
+ * block for A/B diagnosis.
+ */
+static bool
+hk_m5_rt_persistent_state_enabled(void)
+{
+   static int enabled = -1;
+   if (enabled < 0)
+      enabled = debug_get_bool_option("ASAHI_M5_RT_PERSISTENT_STATE", true);
+   return enabled;
+}
+
+static struct agx_ptr
+hk_m5_persistent_rt_state(struct hk_device *dev, uint64_t packed, uint64_t size,
+                          const struct agx_scs_layout_gen2 *scs)
+{
+   struct agx_ptr state = {0};
+   simple_mtx_lock(&dev->m5_as_lock);
+   if (!dev->m5_rt_state_bo) {
+      /* Diagnostic: bind the block GPU-read-only, so that any GPU write,
+       * including a transient one, faults instead of passing silently. */
+      const enum agx_bo_flags flags =
+         debug_get_bool_option("ASAHI_M5_RT_STATE_READONLY_PROBE", false)
+            ? AGX_BO_READONLY : 0;
+      struct agx_bo *bo =
+         agx_bo_create(&dev->dev, size, 0x4000, flags, "M5 SCS state");
+      if (bo) {
+         uint8_t *map = agx_bo_map(bo);
+         memcpy(map, dev->m5_rq_user_image, 0x800000);
+         if (size > 0x800000)
+            memset(map + 0x800000, 0, size - 0x800000);
+         if (dev->m5_rq_instances)
+            agx_scs_layout_gen2_parameters(scs, map + 0x80020);
+         dev->m5_rt_state_bo = bo;
+         dev->m5_rt_state_packed = packed;
+         dev->m5_rt_state_size = size;
+      }
+   }
+   if (dev->m5_rt_state_bo && dev->m5_rt_state_packed == packed &&
+       dev->m5_rt_state_size >= size) {
+      state.cpu = agx_bo_map(dev->m5_rt_state_bo);
+      state.gpu = dev->m5_rt_state_bo->va->addr;
+   }
+   simple_mtx_unlock(&dev->m5_as_lock);
+   return state;
+}
+
 __attribute__((hot)) void
+hk_dispatch_m5_native(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
+                      struct hk_shader *s, struct agx_grid grid,
+                      struct agx_workgroup local_size)
+{
+   const char *reason = "shader profile";
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   struct hk_linked_shader *linked = s ? s->only_linked : NULL;
+   if (s && !linked && s->info.stage == MESA_SHADER_VERTEX)
+      linked = cmd->state.gfx.linked[MESA_SHADER_VERTEX];
+   if (!agx_chip_g17_m5(dev->dev.chip) || cs->type != HK_CS_CDM ||
+       /* Static TPS addresses are 15-bit halfword offsets. */
+       !hk_g17p_compute_shader_supported(dev->dev.chip, s, linked, 1u << 16) ||
+       ((s->b.info.scratch_size || linked->b.scratch_size) &&
+        !(dev->dev.params.features & DRM_ASAHI_FEATURE_COMPUTE_RESOURCES)) ||
+       (s->b.info.uses_ray_traversal && !dev->m5_native_rq) || s->info.agx3_txf_texture_valid ||
+       s->info.agx3_storage_image_valid || s->info.agx3_sampled_storage_valid ||
+       s->info.agx3_indexed_storage_valid ||
+       !local_size.x || !local_size.y || !local_size.z ||
+       (uint64_t)local_size.x * local_size.y * local_size.z >
+          dev->vk.physical->properties.maxComputeWorkGroupInvocations)
+      goto refused;
+   if (!agx_is_indirect(grid) &&
+       (!grid.count[0] || !grid.count[1] || !grid.count[2]))
+      return;
+
+   if (s->b.info.uses_matrix16) {
+      reason = "matrix kernel feature";
+      if (!(dev->dev.params.features & DRM_ASAHI_FEATURE_COMPUTE_MATRIX))
+         goto refused;
+      cs->m5_matrix_ops = true;
+   }
+   const bool internal = s->info.stage != MESA_SHADER_COMPUTE;
+   if (internal && s->info.stage != MESA_SHADER_VERTEX && s->data_size)
+      goto refused;
+   if (!internal) {
+      cmd->state.cs.descriptors.root.cs.constant_data_addr = s->data_size
+         ? hk_pool_upload(cmd, s->data_ptr, s->data_size, HK_MIN_UBO_ALIGNMENT) : 0;
+      if (s->data_size && !cmd->state.cs.descriptors.root.cs.constant_data_addr)
+         return;
+   }
+   const uint64_t root = internal ? cmd->state.gfx.root :
+      hk_cmd_buffer_upload_root(cmd, VK_PIPELINE_BIND_POINT_COMPUTE);
+   if (!root)
+      return;
+   const uint64_t texture_pointers = s->info.agx3_indexed_sampler_valid
+      ? dev->rodata.indexed_heap_pointers : 0;
+   if (s->info.agx3_indexed_sampler_valid && !texture_pointers)
+      goto refused;
+   struct agx3_ioto_uniform_range ranges[HK_G17P_COMPUTE_MAX_RANGES];
+   size_t count = 0;
+   const bool collected = internal
+      ? hk_g17p_internal_graphics_uniform_ranges(cmd, s, linked, root, texture_pointers, ranges, &count)
+      : hk_g17p_compute_uniform_ranges(dev->dev.chip, cmd, s, root, texture_pointers, ranges, &count);
+   reason = "uniform ranges";
+   if (!collected)
+      goto refused;
+   if (!internal && s->b.info.agx3_uniform_dma_count &&
+       !hk_m5_compute_uniform_dma_ranges(cmd, s, root, ranges,
+                                         ARRAY_SIZE(ranges), &count))
+      goto refused;
+   if (s->b.info.uses_ray_traversal) {
+      reason = "native query dispatch shape";
+      /* Internal query kernels use the compute query ABI. A software vertex
+       * variant keeps the vertex ABI (hk_g17p_internal_graphics_uniform_ranges)
+       * and therefore stays refused. */
+      if (internal && s->info.stage != MESA_SHADER_TESS_CTRL &&
+          s->info.stage != MESA_SHADER_TESS_EVAL &&
+          s->info.stage != MESA_SHADER_GEOMETRY)
+         goto refused;
+      /* Keep the shader's captured payload profile, but derive every allocation
+       * and parameter offset from the native ScsLayoutGen2 equations. In
+       * particular, the driver's payload and the saved instance levels are
+       * different sections; neither is the shader's ordinary spill buffer. */
+      struct agx_scs_layout_gen2 scs;
+      const uint64_t packed = dev->m5_rq_instances
+         ? (hk_m5_rq_candidate_requested()
+                     ? UINT64_C(0x01140000600100e8) /* 276-entry IFT50 tail */
+                     : UINT64_C(0x00480000600100e8)) : UINT64_C(0x0028000060000070);
+      if (!agx_scs_layout_gen2_init(packed, &scs))
+         goto refused;
+      const uint64_t state_size = MAX2(UINT64_C(0x800000),
+                                       ALIGN_POT((uint64_t)scs.end[12], 0x4000));
+      /* One device-lifetime state block (hk_m5_persistent_rt_state). The
+       * per-dispatch block remains for ASAHI_M5_RT_PERSISTENT_STATE=0 and as
+       * the fallback if the device block cannot be created; the per-raygen
+       * dispatches of one vkCmdTraceRays* share it. */
+      struct agx_ptr state = hk_m5_rt_persistent_state_enabled()
+         ? hk_m5_persistent_rt_state(dev, packed, state_size, &scs)
+         : (struct agx_ptr){0};
+      const bool persistent_state = state.cpu != NULL;
+      if (persistent_state)
+         hk_cmd_buffer_retain_bo(cmd, dev->m5_rt_state_bo);
+      const bool reuse_state = !persistent_state && cmd->m5_rt_state.share &&
+                               cmd->m5_rt_state.state.cpu &&
+                               cmd->m5_rt_state.packed == packed;
+      const bool fresh_state = !persistent_state && !reuse_state;
+      if (!persistent_state)
+         state = reuse_state ? cmd->m5_rt_state.state
+                             : hk_pool_alloc(cmd, state_size, 0x4000);
+      /* The per-raygen dispatches of one vkCmdTraceRays* (see
+       * hk_trace_rays_select_and_dispatch) read identical traversal uniforms:
+       * the same state block, AS table and context. Build them once. */
+      const bool shared_uniforms = cmd->m5_rt_state.share &&
+                                   cmd->m5_rt_state.uniforms.cpu &&
+                                   cmd->m5_rt_state.uniforms_packed == packed &&
+                                   cmd->m5_rt_state.uniforms_state == state.gpu;
+      if (shared_uniforms) {
+         if (!state.cpu) return;
+         ranges[count++] = (struct agx3_ioto_uniform_range){
+            .start_halfs=0, .size_halfs=dev->m5_rq_instances ? 168 : 156,
+            .address=cmd->m5_rt_state.uniforms.gpu,
+         };
+      } else {
+         struct agx_ptr uniforms = hk_pool_alloc(cmd, 512, 16);
+         if (!state.cpu || !uniforms.cpu) return;
+         uint64_t as_table;
+         if (dev->m5_rq_instances) {
+            as_table = hk_m5_upload_as_table(cmd);
+         } else {
+            /* The native profile has96 resident cliques (3072 SCS slots).
+             * A slot is a hardware resident identity, not an invocation index.
+             * Slot0 is reserved by the native AS selector encoding. */
+            struct agx_ptr table = hk_pool_alloc(cmd, (96 * 32 + 1) * 16, 16);
+            if (!table.cpu) return;
+            memset(table.cpu, 0, (96 * 32 + 1) * 16);
+            as_table = table.gpu;
+         }
+         if (!as_table) return;
+         if (fresh_state) {
+            memcpy(state.cpu, dev->m5_rq_user_image, 0x800000);
+            if (state_size > 0x800000)
+               memset((uint8_t *)state.cpu + 0x800000, 0, state_size - 0x800000);
+         }
+         uint32_t context[32] = {0, 0, packed, packed >> 32};
+         const uint32_t offsets[] = {0x18000,0x18200,0x18400,0x18500,0x18700,0x19b00};
+         for (unsigned i = 0; i < 6; ++i) {
+            uint64_t pointer = state.gpu + offsets[i];
+            if (i == 4)
+               pointer = as_table;
+            memcpy(context + 4 + 2*i, &pointer, 8);
+         }
+         context[20] = scs.end[5];
+         context[22] = scs.end[10] - scs.end[5];
+         const unsigned levels = (packed >> 16) & 0xff;
+         context[23] = (levels + (levels > 1)) * 8;
+         if (dev->m5_rq_instances) {
+            uint64_t extra = state.gpu + 0x80050;
+            memcpy(context + 26, &extra, 8);
+            extra = state.gpu + 0x18900;
+            memcpy(context + 30, &extra, 8);
+            if (fresh_state)
+               agx_scs_layout_gen2_parameters(&scs, (uint8_t *)state.cpu + 0x80020);
+         }
+         if (cmd->m5_rt_state.share && fresh_state) {
+            cmd->m5_rt_state.state = state;
+            cmd->m5_rt_state.packed = packed;
+         }
+         if (!reuse_state && os_get_option("ASAHI_M5_RT_STATE_DIAG"))
+            hk_m5_rt_state_diag_record(cmd, state, state_size);
+         memset(uniforms.cpu, 0, 512);
+         memcpy(uniforms.cpu, dev->m5_rq_uniforms, sizeof(dev->m5_rq_uniforms));
+         if (dev->m5_rq_instances) {
+            uint64_t sizes = (uint64_t)scs.end[6] | ((uint64_t)scs.end[11] << 32);
+            uint64_t layout = packed;
+            uint64_t driver = scs.end[5];
+            memcpy((uint8_t *)uniforms.cpu + 48*2, &sizes, 8);
+            memcpy((uint8_t *)uniforms.cpu + 52*2, &layout, 8);
+            memcpy((uint8_t *)uniforms.cpu + 56*2, &driver, 8);
+         }
+         if (hk_m5_rq_candidate_requested()) {
+            /* Fresh query-once native u110..113, separately bound because the
+             * captured ordinary profile uses u140 for its per-clique state. */
+            uint64_t query_payload = UINT64_C(0x3fffffff00008000);
+            memcpy((uint8_t *)uniforms.cpu + 152*2, &query_payload, 8);
+         }
+         for (unsigned i = 0; i < (dev->m5_rq_instances ? 80 : 74); i += 2) {
+            uint64_t pointer; memcpy(&pointer, (uint8_t *)uniforms.cpu + i*4, 8);
+            if (pointer >= UINT64_C(0x10000000000) && pointer < UINT64_C(0x10000800000)) {
+               pointer += state.gpu - UINT64_C(0x10000000000);
+               memcpy((uint8_t *)uniforms.cpu + i*4, &pointer, 8);
+            }
+         }
+         if (dev->m5_rq_instances)
+            memcpy((uint8_t *)uniforms.cpu + 32*2, &as_table, 8);
+         unsigned context_offset = dev->m5_rq_instances ? 352 : 320;
+         memcpy((uint8_t *)uniforms.cpu + context_offset, context, sizeof(context));
+         uint64_t ctx = uniforms.gpu + context_offset, table = as_table;
+         memcpy((uint8_t *)uniforms.cpu + (dev->m5_rq_instances ? 320 : 296), &ctx, 8);
+         memcpy((uint8_t *)uniforms.cpu + (dev->m5_rq_instances ? 328 : 304), &table, 8);
+         ranges[count++] = (struct agx3_ioto_uniform_range){
+            .start_halfs=0, .size_halfs=dev->m5_rq_instances ? 168 : 156, .address=uniforms.gpu,
+         };
+         if (cmd->m5_rt_state.share) {
+            cmd->m5_rt_state.uniforms = uniforms;
+            cmd->m5_rt_state.uniforms_packed = packed;
+            cmd->m5_rt_state.uniforms_state = state.gpu;
+         }
+      }
+      cs->m5_native_rq = true;
+      hk_cmd_buffer_retain_bo(cmd, dev->m5_rq_code);
+      hk_cmd_buffer_retain_bo(cmd, dev->m5_rq_ift);
+      if (dev->m5_rq_diag)
+         hk_cmd_buffer_retain_bo(cmd, dev->m5_rq_diag);
+   }
+   reason = "program bound";
+   size_t capacity;
+   if (!agx3_g17g_graphics_program_bound(dev->dev.chip, ranges, count, &capacity))
+      goto refused;
+   if (s->b.info.has_preamble)
+      capacity = AGX3_G17P_GRAPHICS_PROGRAM_MAX_SIZE;
+   struct hk_ioto_reservation reservation;
+   if (!hk_cmd_reserve_ioto_program(cmd, capacity, "M5 compute", &reservation))
+      return;
+   const uint64_t shader_va = linked->b.bo->va->addr + linked->b.entry_offset;
+   struct agx3_g17p_graphics_program_desc desc = {
+      .program_va = reservation.va, .shader_va = shader_va,
+      .uniform_ranges = ranges, .uniform_range_count = count,
+      .shader_binary_size = linked->b.size,
+      .nr_gprs = MAX2(linked->b.nr_gprs, s->b.info.nr_preamble_gprs),
+      /* The constant program's TPS frame is the launch's thread-invariant
+       * spill buffer (below), not the main program's per-thread frame. */
+      .scratch_size = MAX3(s->b.info.scratch_size, linked->b.scratch_size,
+                           linked->b.agx3_spectpr_scratch_size),
+      .compiler_d_valid = s->b.info.agx3_staging_known,
+      .compiler_d = MAX2((uint32_t)s->b.info.agx3_staging_halfs * 2,
+                         (uint32_t)s->b.info.agx3_preamble_staging_halfs * 2),
+      .threadgroup_memory_size_B = s->b.info.local_size
+         ? MAX2(256u, ALIGN_POT(s->b.info.local_size, 64u)) : 0,
+   };
+   if (s->b.info.has_preamble) {
+      reason = "native constant-program extent";
+      if (internal || !s->bo || !s->bo->va ||
+          !hk_agx3_g17p_constant_program_from_binary(
+             s->code_ptr, s->code_size, s->b.info.preamble_offset,
+             s->b.info.preamble_size, hk_shader_code_va(s),
+             s->b.info.binary_size, &desc.constant_program))
+         goto refused;
+      desc.constant_program.force_external = true;
+      desc.constant_program.absolute_branch = true;
+      desc.constant_program.request_prelude = s->b.info.agx3_preamble_uniform_inputs;
+      if (s->b.info.agx3_preamble_tps_inputs)
+         desc.cprog_inputs = &s->b.info.agx3_preamble_inputs;
+   }
+   struct agx3_g17p_graphics_program program;
+   const struct hk_m5_compute_loader *loader =
+      hk_m5_loader_get(dev, linked, &desc);
+   struct agx3_g17p_native_compute_program_desc launch_desc = {
+      .program_va = reservation.va, .shader_va = shader_va,
+   };
+   struct agx3_compute_program launch;
+   reason = "program generation";
+   if (loader) {
+      program.length = loader->length;
+      memcpy(program.bytes, loader->bytes, loader->length);
+      hk_g17p_patch_plan_apply(&loader->plan, ranges, program.bytes);
+      if (debug_get_option_m5_verify_compute_loader()) {
+         struct agx3_g17p_graphics_program fresh;
+         if (!agx3_g17g_compute_program_generate(dev->dev.chip, &desc, &fresh) ||
+             fresh.length != program.length ||
+             memcmp(fresh.bytes, program.bytes, program.length)) {
+            reason = "cached program byte mismatch";
+            goto refused;
+         }
+      }
+   } else if (!agx3_g17g_compute_program_generate(dev->dev.chip, &desc, &program)) {
+      goto refused;
+   }
+   /* Native instance launch words are 0x00800000/0x01000080:
+    * the nested traversal path uses the 64-byte TI spill class and ID32 bit. */
+   const bool native_instances = dev->m5_rq_instances && s->b.info.uses_ray_traversal;
+   if (!agx3_cdm_direct_template_sources_compute(s->b.info.uses_resource_info_query || native_instances,
+                                                 s->b.info.uses_ray_traversal, native_instances, &launch_desc.launch_sources))
+      goto refused;
+   /* A constant program's TPS frame is the launch's thread-invariant spill
+    * buffer (word 1 bits 3..20): the loader publishes the CProg's inputs to
+    * TPS from half 8 and the CProg's RLDs read them back from that frame.
+    * With a zero-sized frame the RLDs never write their destinations. Native
+    * M5 launches of TPS-importing CProgs size it as the 16-byte-aligned input
+    * frame (32 B for 4 and 8 input halves, 48 B for 16 halves), which is
+    * agx3_cprog_inputs::storage_bytes; the CProg's own TPS spills share the
+    * frame, so cover preamble_scratch_size too. */
+   if (s->b.info.has_preamble) {
+      const unsigned cprog_frame =
+         ALIGN_POT(MAX2((unsigned)s->b.info.agx3_preamble_inputs.storage_bytes,
+                        s->b.info.preamble_scratch_size), 16);
+      launch_desc.launch_sources.ti_spill_bytes =
+         MAX2(launch_desc.launch_sources.ti_spill_bytes, cprog_frame);
+   }
+   if (!agx3_m5_native_compute_program_bind(dev->dev.chip, &launch_desc, &launch))
+      goto refused;
+   memcpy(reservation.map, program.bytes, program.length);
+   if (!hk_cmd_publish_ioto_program(cmd, &reservation, program.length))
+      return;
+   hk_cmd_buffer_retain_bo(cmd, linked->b.bo);
+   if (s->b.info.rodata.size_16 || s->b.info.has_preamble)
+      hk_cmd_buffer_retain_bo(cmd, s->bo);
+
+   hk_ensure_cs_has_space(cmd, cs, 0x2000);
+   struct agx3_compute_dispatch_sink sink = {
+      .data = (uint8_t *)cs->current,
+      .capacity = (uint8_t *)cs->end - (uint8_t *)cs->current,
+   };
+   const struct agx3_compute_direct_grid direct = {
+      .source = AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,
+      .size = {grid.count[0], grid.count[1], grid.count[2]},
+      .local_size = {local_size.x, local_size.y, local_size.z},
+      .dimensions = grid.count[2] != 1 || local_size.z != 1 ? 3
+                  : grid.count[1] != 1 || local_size.y != 1 ? 2 : 1,
+   };
+   reason = "dispatch record";
+   const uint32_t local[3] = {local_size.x, local_size.y, local_size.z};
+   if (cmd->m5_rt_capture) {
+      /* Single-kick vkCmdTraceRays* (hk_cmd_trace_rays.c): produce this
+       * kernel's direct launch plus its trailing barrier as a template the
+       * select kernel copies into the stream slot, instead of appending it
+       * here. Template it as a multi-dimensional grid so word 0 keeps the
+       * 2D/3D bits; the select kernel clears them for a 1D launch exactly
+       * as agx3_g17p_native_compute_launch_pack does, and writes the sizes. */
+      const struct agx3_compute_direct_grid tmpl = {
+         .source = AGX3_CDM_DIRECT_GRID_GLOBAL_THREADS,
+         .size = {local[0], 2 * local[1], 2 * local[2]},
+         .local_size = {local[0], local[1], local[2]},
+         .dimensions = 3,
+      };
+      struct agx3_compute_dispatch_sink capture = {
+         .data = cmd->m5_rt_capture,
+         .capacity = AGX3_G17P_NATIVE_COMPUTE_DISPATCH_SIZE,
+      };
+      if (!agx3_m5_native_compute_dispatch_append(&launch, &tmpl, 0, &capture) ||
+          capture.length != AGX3_G17P_NATIVE_COMPUTE_DISPATCH_SIZE)
+         goto refused;
+      cmd->m5_rt_captured = true;
+      hk_m5_compute_resources(cs, desc.scratch_size,
+                              s->b.info.uses_ray_traversal ? 128 : 0,
+                              desc.threadgroup_memory_size_B,
+                              local[0] * local[1] * local[2]);
+      cs->stats.cmds++;
+      return;
+   }
+   const bool appended = agx_is_indirect(grid)
+      ? agx3_m5_native_indirect_dispatch_append(
+           &launch, grid.ptr,
+           grid.mode == AGX_CDM_MODE_INDIRECT_GLOBAL ? local : NULL, 0, &sink)
+      : agx3_m5_native_compute_dispatch_append(&launch, &direct, 0, &sink);
+   if (!appended)
+      goto refused;
+   cs->current = (uint32_t *)((uint8_t *)cs->current + sink.length);
+   cs->weak_barrier_end = cs->current;
+   hk_m5_compute_resources(cs, desc.scratch_size,
+                           s->b.info.uses_ray_traversal ? 128 : 0,
+                           desc.threadgroup_memory_size_B,
+                           local[0] * local[1] * local[2]);
+   cs->stats.cmds++;
+   return;
+refused:
+   if (debug_get_option_m5_launch_debug()) {
+      fprintf(stderr, "M5 compute refused: %s stage=%u scratch=%u/%u "
+              "gprs=%u staging=%u:%u local=%u preamble=%u "
+              "textures=%u samplers=%u heap=%u txf=%u sets=%u "
+              "resource_flags=%u%u%u%u%u indirect=%u\n",
+              reason, s ? s->info.stage : 0,
+              s ? s->b.info.scratch_size : 0, linked ? linked->b.scratch_size : 0,
+              linked ? linked->b.nr_gprs : 0,
+              s ? s->b.info.agx3_staging_known : 0,
+              s ? s->b.info.agx3_staging_halfs : 0,
+              s ? s->b.info.local_size : 0, s ? s->b.info.has_preamble : 0,
+              s ? s->b.info.texture_state_count : 0,
+              s ? s->b.info.sampler_state_count : 0,
+              s ? s->b.info.uses_sampler_heap : 0,
+              linked ? linked->b.uses_txf : 0, s ? s->info.set_count : 0,
+              s ? s->info.agx3_txf_texture_valid : 0,
+              s ? s->info.agx3_storage_image_valid : 0,
+              s ? s->info.agx3_sampled_storage_valid : 0,
+              s ? s->info.agx3_indexed_storage_valid : 0,
+              s ? s->info.agx3_indexed_sampler_valid : 0, agx_is_indirect(grid));
+   }
+   mesa_loge_once("M5 compute launch exceeds the current resource profile");
+   vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+}
+
+void
 hk_dispatch_g17p_native(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
                         struct hk_shader *s, struct agx_grid grid,
                         struct agx_workgroup local_size)
@@ -1324,7 +2129,8 @@ hk_dispatch_g17p_native(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
                  "G17P root range must cover every Honeykrisp set");
 
    if (!agx_native_graphics(&dev->dev) || cs->type != HK_CS_CDM ||
-       !hk_g17p_compute_shader_supported(s, linked)) {
+       !hk_g17p_compute_shader_supported(dev->dev.chip, s, linked,
+                                         AGX3_G17P_MAX_COMPUTE_SPILL_B)) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
       return;
    }
@@ -1883,6 +2689,14 @@ dispatch_grid(struct hk_cmd_buffer *cmd, struct agx_grid grid, bool in_threads)
    }
 
    hk_dispatch_with_local_size(cmd, cs, s, grid, local_size);
+   if (unlikely(hk_cmd_buffer_device(cmd)->frame_anatomy)) {
+      hk_frame_anatomy_note_fs(cs, s);
+      const uint64_t wg = agx_workgroup_threads(local_size);
+      const uint64_t threads = agx_is_indirect(grid) ? 0 :
+         (uint64_t)grid.count[0] * grid.count[1] * grid.count[2];
+      hk_frame_anatomy_note_grid(cs, s, threads, wg ? threads / wg : 0,
+                                 agx_is_indirect(grid));
+   }
    cs->stats.calls++;
 }
 
@@ -1999,6 +2813,8 @@ hk_cmd_dispatch_rays(struct hk_cmd_buffer *cmd, struct hk_api_shader *shader,
    desc->root.cs.trace_rays_addr = record_addr;
    desc->root_dirty = true;
 
+   /* Conditional rendering predicates draws, dispatches and attachment
+    * clears only; vkCmdTraceRays* must execute regardless of it. */
    const bool conditional_active = cmd->state.conditional.active;
    cmd->state.conditional.active = false;
 
diff --git a/src/asahi/vulkan/hk_cmd_draw.c b/src/asahi/vulkan/hk_cmd_draw.c
index 0f05f8d4df54..ac136901c8e9 100644
--- a/src/asahi/vulkan/hk_cmd_draw.c
+++ b/src/asahi/vulkan/hk_cmd_draw.c
@@ -10,12 +10,15 @@
 #include "agx_bo.h"
 #include "agx_compile.h"
 #include "agx_compiler.h"
+#include "asahi/compiler/agx3_graphics_program.h"
 #include "agx_device.h"
 #include "agx_helpers.h"
 #include "agx_linker.h"
 #include "agx_nir_lower_vbo.h"
 #include "agx_ppp.h"
 #include "agx_ppp_hal200_graphics_core.h"
+#include "agx_ppp_hal300_graphics_core.h"
+#include "hk_ppp_state.h"
 #include "agx_vdm_state_core.h"
 #include "agx_ppp_hal200_raster_core.h"
 #include "agx_ppp_hal300_depth_bias_scissor_core.h"
@@ -30,6 +33,7 @@
 #include "hk_buffer.h"
 #include "hk_cmd_buffer.h"
 #include "hk_native_vertex_template.h"
+#include "hk_frame_anatomy.h"
 #include "asahi/lib/agx_vdm_mesh_core.h"
 #include "hk_descriptor_set.h"
 #include "hk_descriptor_set_layout.h"
@@ -45,7 +49,10 @@
 #include "hk_agx3_vdm_decode.h"
 #include "hk_physical_device.h"
 #include "hk_private.h"
+#include "asahi/lib/agx_m5_va.h"
 #include "hk_shader.h"
+#include "hk_native_write_class.h"
+#include "hk_g17p_patch.h"
 #include "util/disk_cache.h"
 #include "agx_debug.h"
 #include "asahi/lib/agx3_g17p_grid_helper_core.h"
@@ -88,6 +95,21 @@
 #include "vk_util.h"
 #include "util/set.h"
 
+/* Diagnostic controls, read once: these are consulted per draw or
+ * render pass, and an uncached getenv() scans the whole environment. */
+DEBUG_GET_ONCE_BOOL_OPTION(m5_launch_debug, "ASAHI_M5_LAUNCH_DEBUG", false)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_msaa_pass_merge, "ASAHI_M5_MSAA_PASS_MERGE", true)
+DEBUG_GET_ONCE_BOOL_OPTION(m5_verify_tile_loader, "AGX_M5_VERIFY_TILE_LOADER",
+                           false)
+DEBUG_GET_ONCE_BOOL_OPTION(mesh_native_primitive_address,
+                           "ASAHI_MESH_NATIVE_PRIMITIVE_ADDRESS_DIAGNOSTIC",
+                           false)
+DEBUG_GET_ONCE_BOOL_OPTION(mesh_native_m5_wclamp,
+                           "ASAHI_MESH_NATIVE_M5_WCLAMP_DIAGNOSTIC", false)
+DEBUG_GET_ONCE_BOOL_OPTION(mesh_native_raster_enable,
+                           "ASAHI_MESH_NATIVE_RASTER_ENABLE_DIAGNOSTIC", false)
+DEBUG_GET_ONCE_BOOL_OPTION(mesh_pp_tail0, "ASAHI_MESH_PP_TAIL0", false)
+
 #define IS_DIRTY(bit) BITSET_TEST(dyn->dirty, MESA_VK_DYNAMIC_##bit)
 
 #define IS_SHADER_DIRTY(bit)                                                   \
@@ -164,6 +186,9 @@ hk_rast_discard(struct hk_cmd_buffer *cmd)
 {
    struct vk_dynamic_graphics_state *dyn = &cmd->vk.dynamic_graphics_state;
 
+   if (dyn->rs.rasterizer_discard_enable)
+      return true;
+
    /* A non-zero rasterization stream acts as a rasterizer discard unless
     * there's a multistream geometry shader bound.
     */
@@ -172,7 +197,25 @@ hk_rast_discard(struct hk_cmd_buffer *cmd)
       return !gs || !gs->variants[HK_GS_VARIANT_COUNT].info.gs.multistream;
    }
 
-   return dyn->rs.rasterizer_discard_enable;
+   return false;
+}
+
+/* Report address of an active VK_QUERY_TYPE_PRIMITIVES_GENERATED_EXT query
+ * on @stream, or 0. Meta draws are not counted. */
+static uint64_t
+hk_prims_generated_addr(struct hk_cmd_buffer *cmd, unsigned stream)
+{
+   return cmd->in_meta ? 0 : cmd->state.gfx.prims_generated_query[stream];
+}
+
+static bool
+hk_prims_generated_active(struct hk_cmd_buffer *cmd)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(cmd->state.gfx.prims_generated_query); ++i) {
+      if (hk_prims_generated_addr(cmd, i))
+         return true;
+   }
+   return false;
 }
 
 static bool
@@ -182,7 +225,8 @@ hk_g17p_xfb_vertex_side_effects(struct hk_cmd_buffer *cmd,
    struct hk_graphics_state *gfx = &cmd->state.gfx;
    struct hk_api_shader *gs = gfx->shaders[MESA_SHADER_GEOMETRY];
 
-   return agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) &&
+   const struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   return (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
           gfx->xfb_enabled && gs && gs->is_passthrough &&
           hw_vs == &gs->variants[HK_GS_VARIANT_RAST];
 }
@@ -283,13 +327,42 @@ hk_cmd_invalidate_graphics_state(struct hk_cmd_buffer *cmd)
     *    buffers."
     *
     * We need to reset everything EXCEPT the render pass state.
+    *
+    * Active queries are not bound state either: a query begun in the primary
+    * stays active across vkCmdExecuteCommands and must also count the
+    * primary's later commands, so keep every active query binding.
     */
-   struct hk_rendering_state render_save = cmd->state.gfx.render;
+   struct hk_graphics_state *gfx = &cmd->state.gfx;
+   struct hk_rendering_state render_save = gfx->render;
    struct hk_push_descriptor_set *push[HK_MAX_SETS];
-   memcpy(push, cmd->state.gfx.descriptors.push, sizeof(push));
-   memset(&cmd->state.gfx, 0, sizeof(cmd->state.gfx));
-   cmd->state.gfx.render = render_save;
-   memcpy(cmd->state.gfx.descriptors.push, push, sizeof(push));
+   memcpy(push, gfx->descriptors.push, sizeof(push));
+   const uint64_t pipeline_stats = gfx->descriptors.root.draw.pipeline_stats;
+   const VkQueryPipelineStatisticFlags pipeline_stats_flags =
+      gfx->descriptors.root.draw.pipeline_stats_flags;
+   const uint64_t mesh_prims_query = gfx->descriptors.root.mesh_prims_query;
+   uint64_t xfb_query[ARRAY_SIZE(gfx->xfb_query)];
+   uint64_t prims_generated_query[ARRAY_SIZE(gfx->prims_generated_query)];
+   memcpy(xfb_query, gfx->xfb_query, sizeof(xfb_query));
+   memcpy(prims_generated_query, gfx->prims_generated_query,
+          sizeof(prims_generated_query));
+   const enum agx_visibility_mode occlusion_mode = gfx->occlusion.mode;
+   const uint16_t occlusion_index = gfx->occlusion.index;
+   struct hk_query_pool *occlusion_pending_pool = gfx->occlusion.pending_pool;
+   const uint32_t occlusion_pending_query = gfx->occlusion.pending_query;
+
+   memset(gfx, 0, sizeof(*gfx));
+   gfx->render = render_save;
+   memcpy(gfx->descriptors.push, push, sizeof(push));
+   gfx->descriptors.root.draw.pipeline_stats = pipeline_stats;
+   gfx->descriptors.root.draw.pipeline_stats_flags = pipeline_stats_flags;
+   gfx->descriptors.root.mesh_prims_query = mesh_prims_query;
+   memcpy(gfx->xfb_query, xfb_query, sizeof(xfb_query));
+   memcpy(gfx->prims_generated_query, prims_generated_query,
+          sizeof(prims_generated_query));
+   gfx->occlusion.mode = occlusion_mode;
+   gfx->occlusion.index = occlusion_index;
+   gfx->occlusion.pending_pool = occlusion_pending_pool;
+   gfx->occlusion.pending_query = occlusion_pending_query;
 
    /* Every bound stage is gone, so vk_graphics_pipeline_cmd_bind() must not
     * elide the application's rebind of the pipeline it last saw (often the
@@ -707,7 +780,6 @@ struct hk_g17p_bg_eot_dev_key {
    struct hk_native_pbe_descriptor eot, layered_eot;
    struct hk_native_texture_descriptor background, layered_background;
    uint32_t pbe_valid;
-   uint32_t pad;
 };
 
 struct hk_g17p_bg_eot_dev_entry {
@@ -1542,7 +1614,10 @@ hk_build_g17p_bg_eot(struct hk_cmd_buffer *cmd,
 }
 
 struct hk_g17p_bg_eot_cache_key {
-   struct agx_tilebuffer_layout tib;
+   /* 8-byte aligned so the size is a whole number of words on 32-bit x86
+    * too, where uint64_t members only need 4-byte alignment.
+    */
+   _Alignas(8) struct agx_tilebuffer_layout tib;
    struct {
       uint64_t image_view;
       uint64_t resolve_image_view;
@@ -1605,6 +1680,7 @@ hk_g17p_bg_eot_cache_get(struct hk_cmd_buffer *cmd,
          cache_key.attachments[i].resolve_image_view =
             (uintptr_t)att->resolveImageView;
          cache_key.attachments[i].resolve_mode = att->resolveMode;
+         cache_key.attachments[i].flags = vk_get_rendering_attachment_flags(att);
       }
    }
 
@@ -1662,10 +1738,915 @@ hk_g17p_bg_eot_cache_get(struct hk_cmd_buffer *cmd,
    return cached;
 }
 
+static unsigned
+hk_m5_vertex_secondary_storage(const struct agx_linked_shader *linked)
+{
+   const unsigned staging_B = linked->g17p_compiler_d_valid
+                                ? linked->g17p_compiler_d
+                                : linked->nr_gprs;
+
+   /* Native setVertexVariant adds ShaderInfo42 and ShaderInfo38 times the
+    * descriptor's maxVertexCallStackDepth. Ordinary pipelines default to one;
+    * the linked monolithic shader publishes the same staging extent in both
+    * fields. This is the staging footprint used by SPECTPR, not UVS output.
+    */
+   return staging_B * 2;
+}
+
+
+/* The object program of a native mesh draw: the compiled API task shader when
+ * one is bound (opt-in, G17S), else the mesh shader's passthrough or
+ * predicated object. */
+static inline struct hk_shader *
+hk_native_mesh_object(struct hk_graphics_state *gfx, bool *task)
+{
+   if (gfx->shaders[MESA_SHADER_TASK]) {
+      *task = true;
+      return &gfx->shaders[MESA_SHADER_TASK]->variants[0];
+   }
+   *task = false;
+   return &gfx->shaders[MESA_SHADER_MESH]->variants[gfx->mesh.object_params
+      ? HK_MESH_VARIANT_PREDICATED_OBJECT : HK_MESH_VARIANT_RAST];
+}
+
+static void
+hk_m5_note_render_resources(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!agx_chip_g17_m5(dev->dev.chip))
+      return;
+   struct hk_graphics_state *gfx = &cmd->state.gfx;
+   struct hk_shader *vs = hk_bound_hw_vs(gfx);
+   struct hk_linked_shader *linked_vs = vs && vs->only_linked
+      ? vs->only_linked : gfx->linked[MESA_SHADER_VERTEX];
+   struct hk_linked_shader *linked_fs = gfx->linked[MESA_SHADER_FRAGMENT];
+   const bool mesh = gfx->shaders[MESA_SHADER_MESH] != NULL;
+   struct hk_shader *fs = hk_bound_fs(cmd);
+   hk_m5_accumulate_shader_writes(&cs->m5_shader_writes_memory,
+                                  vs && vs->info.writes_external_memory);
+   hk_m5_accumulate_shader_writes(&cs->m5_shader_writes_memory,
+                                  fs && fs->info.writes_external_memory);
+   if (mesh) {
+      bool task;
+      struct hk_shader *object = hk_native_mesh_object(gfx, &task);
+      hk_m5_accumulate_shader_writes(&cs->m5_shader_writes_memory,
+                                     object->info.writes_external_memory);
+   }
+   if (cs->m5_shader_writes_memory || gfx->xfb_enabled ||
+       gfx->render.m5_attachments_reach_memory)
+      hk_sync_note_memory_write(cmd, cs);
+   const bool v2 = dev->dev.params.features & DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2;
+   struct drm_asahi_render_program_v2 programs[3] = {
+      [1] = {
+         .legacy = {
+            .stage = 1,
+            .threadgroup_bytes = cs->tile_resources.threadgroup_bytes,
+            .imageblock_bytes = cs->tile_resources.imageblock_bytes,
+            .multisample_bytes = cs->tile_resources.multisample_bytes,
+         },
+      },
+   };
+   if (linked_vs) {
+      programs[0].legacy.primary_bytes = MAX2(linked_vs->b.scratch_size,
+                                               linked_vs->b.agx3_spectpr_scratch_size);
+      programs[0].legacy.secondary_bytes = hk_m5_vertex_secondary_storage(&linked_vs->b);
+   }
+   if (linked_fs)
+      programs[1].legacy.primary_bytes = MAX2(linked_fs->b.scratch_size,
+                                               linked_fs->b.agx3_spectpr_scratch_size);
+   if (mesh) {
+      if (!v2) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+         return;
+      }
+      programs[0].legacy.threadgroup_bytes = vs->b.info.local_size;
+      programs[0].threads = vs->b.info.workgroup_size[0] *
+         vs->b.info.workgroup_size[1] * vs->b.info.workgroup_size[2];
+      programs[0].usage_flags = DRM_ASAHI_RENDER_PROGRAM_MESH;
+      bool task;
+      struct hk_shader *object = hk_native_mesh_object(gfx, &task);
+      programs[2].legacy.threadgroup_bytes = object->b.info.local_size;
+      programs[2].legacy.primary_bytes =
+         MAX2(object->only_linked->b.scratch_size,
+              object->only_linked->b.agx3_spectpr_scratch_size);
+      programs[2].legacy.secondary_bytes =
+         hk_m5_vertex_secondary_storage(&object->only_linked->b);
+      programs[2].threads = task ? object->b.info.workgroup_size[0] *
+                                   object->b.info.workgroup_size[1] *
+                                   object->b.info.workgroup_size[2]
+                                 : 3;
+      programs[2].usage_flags = DRM_ASAHI_RENDER_PROGRAM_OBJECT;
+      /* A task stage compiled for the native object->mesh record asks the
+       * kernel for the TA mesh rendering buffer that backs it. */
+      if (task && hk_m5_task_record(dev->dev.params.features))
+         programs[2].usage_flags |= DRM_ASAHI_RENDER_PROGRAM_TASK;
+   }
+   for (unsigned stage = 0; stage < 2; ++stage) {
+      if (!(stage ? cs->gen1_native_fragment_rq : cs->gen1_native_vertex_rq))
+         continue;
+      if (!v2) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+         return;
+      }
+      programs[stage].usage_flags |= DRM_ASAHI_RENDER_PROGRAM_SCS;
+      programs[stage].scs_layout = UINT64_C(0x01140000600100e8);
+      programs[stage].auxiliary_allocation = 14;
+      programs[stage].intersection_pool = dev->m5_rq_ift->va->addr;
+      programs[stage].intersection_pool_bytes = 0x8000;
+      programs[stage].intersection_first = 1;
+      programs[stage].intersection_count = 112;
+      programs[stage].intersection_default_program = dev->m5_rq_default_program;
+      programs[stage].intersection_default_control = dev->m5_rq_default_control;
+   }
+   const struct drm_asahi_render_program legacy[2] = {programs[0].legacy, programs[1].legacy};
+   const size_t count = mesh ? 3 : 2;
+   const void *values = v2 ? (const void *)programs : (const void *)legacy;
+   const size_t bytes = v2 ? count * sizeof(programs[0]) : sizeof(legacy);
+   static_assert(sizeof(cs->m5_render_resources_last) >= sizeof(programs),
+                 "the last entry fits");
+   /* Consecutive draws mostly repeat the previous entry. */
+   if (cs->m5_render_resources_last_size == bytes &&
+       !memcmp(cs->m5_render_resources_last, values, bytes))
+      return;
+   memcpy(cs->m5_render_resources_last, values, bytes);
+   cs->m5_render_resources_last_size = bytes;
+   for (size_t off = 0; off + bytes <= cs->render_resources.size; off += v2 ? sizeof(programs[0]) : sizeof(legacy[0])) {
+      if (!memcmp((char *)cs->render_resources.data + off, values, bytes))
+         return;
+   }
+   util_dynarray_append_array(&cs->render_resources, uint8_t, values, bytes);
+}
+
+static bool
+hk_m5_has_color(const struct agx_tilebuffer_layout *tib)
+{
+   for (unsigned rt = 0; rt < HK_MAX_RTS; ++rt) {
+      if (tib->logical_format[rt] != PIPE_FORMAT_NONE)
+         return true;
+   }
+   return false;
+}
+
+static bool
+hk_m5_empty_tile_layout_supported(const struct agx_tilebuffer_layout *tib)
+{
+   if (hk_m5_has_color(tib) || agx_tilebuffer_spills(tib))
+      return false;
+   /* A layered pass reserves one eight-byte sample, as native M5 does. */
+   if (tib->nr_samples <= 1)
+      return tib->sample_size_B == (tib->layered ? 8 : 0) &&
+             tib->tile_size == 32 * 32;
+   /* Shared tile allocation reserves one eight-byte sample even when only
+    * depth/stencil is attached. Keep its allocation and sample geometry. */
+   return tib->sample_size_B == 8 &&
+          ((tib->nr_samples == 2 && tib->tile_size == 32 * 32) ||
+           (tib->nr_samples == 4 && tib->tile_size == 32 * 16) ||
+           (tib->nr_samples == 8 && tib->tile_size == 16 * 16));
+}
+
+/* Colour tile allocations an image-operation-free tile program can carry:
+ * the same SpecLM imageblock geometry every M5 colour program encodes. */
+static bool
+hk_m5_color_imageblock_supported(const struct agx_tilebuffer_layout *tib)
+{
+   if (!hk_m5_has_color(tib) || tib->layered || agx_tilebuffer_spills(tib) ||
+       !tib->sample_size_B || (tib->sample_size_B & 7) ||
+       tib->sample_size_B > 248)
+      return false;
+   return (tib->nr_samples == 1 && tib->tile_size == 32 * 32) ||
+          (tib->nr_samples == 2 && tib->tile_size == 32 * 32) ||
+          (tib->nr_samples == 4 && tib->tile_size == 32 * 16) ||
+          (tib->nr_samples == 8 && tib->tile_size == 16 * 16);
+}
+
+static bool
+hk_m5_mrt_tile_layout(const struct hk_rendering_state *render,
+                      struct agx_tilebuffer_layout *tile)
+{
+   *tile = render->tilebuffer;
+   for (unsigned rt = 0; rt < HK_MAX_RTS; ++rt) {
+      if (tile->spilled[rt]) {
+         tile->spilled[rt] = false;
+         tile->logical_format[rt] = PIPE_FORMAT_NONE;
+      }
+   }
+   return true;
+}
+
+static bool
+hk_m5_mrt_layout_supported(const struct hk_rendering_state *render)
+{
+   const unsigned count = render->color_att_count;
+   struct agx_tilebuffer_layout tile;
+   if (!hk_m5_mrt_tile_layout(render, &tile))
+      return false;
+   if (count <= HK_MAX_RTS && render->layer_count == 1 && !render->view_mask &&
+       hk_m5_empty_tile_layout_supported(&render->tilebuffer))
+      return true;
+   if (count > HK_MAX_RTS ||
+       (render->tilebuffer.nr_samples != 1 &&
+        render->tilebuffer.nr_samples != 2 &&
+        render->tilebuffer.nr_samples != 4 &&
+        render->tilebuffer.nr_samples != 8) ||
+       (render->tilebuffer.tile_size != 32 * 32 &&
+        render->tilebuffer.tile_size != 32 * 16 &&
+        render->tilebuffer.tile_size != 16 * 16) ||
+       agx_tilebuffer_spills(&tile))
+      return false;
+   unsigned active = 0;
+   for (unsigned rt = 0; rt < count; ++rt) {
+      const struct hk_image_view *view = render->color_att[rt].iview;
+      if (!view)
+         continue;
+      if (render->tilebuffer.spilled[rt]) continue;
+      active++;
+      const unsigned selector =
+         agx_g17p_mrt_eot_selector(render->tilebuffer.logical_format[rt]);
+      if (view->vk.image->samples != render->tilebuffer.nr_samples ||
+          (selector != 0x15 && selector != 0x35 && selector != 0x55 &&
+           selector != 0x75 && selector != 0x95))
+         return false;
+      const struct hk_image_view *resolve = render->color_att[rt].resolve_iview;
+      if (resolve && (view->vk.image->samples <= 1 ||
+                      resolve->vk.image->samples != 1 ||
+                      resolve->vk.format != view->vk.format ||
+                      !resolve->planes[0].eot_pbe_valid))
+         return false;
+   }
+   if (active > 1 &&
+       agx_g17p_classify_mrt(&tile) == AGX_G17P_MRT_NONE)
+      return false;
+   return true;
+}
+
+/* A tile shader is device-owned. Cache only its immutable launch bytes;
+ * every pass still allocates and publishes its own program and descriptors.
+ * Preserve the normal load schedule and fall back for non-fitting addresses. */
+struct hk_m5_tile_template {
+   struct hk_m5_tile_template *next;
+   struct agx3_g17p_graphics_program_desc desc;
+   struct agx3_ioto_uniform_range ranges[HK_G17P_PATCH_MAX_RANGES];
+   struct hk_g17p_patch_plan plan;
+   bool store;
+   uint16_t length;
+   uint8_t bytes[];
+};
+
+static bool
+hk_m5_tile_matches(const struct hk_m5_tile_template *t,
+                   const struct agx3_g17p_graphics_program_desc *d, bool store)
+{
+   const struct agx3_g17p_graphics_program_desc *a = &t->desc;
+   if (t->store != store || a->shader_va != d->shader_va ||
+       a->shader_binary_size != d->shader_binary_size ||
+       a->scratch_size != d->scratch_size || a->nr_gprs != d->nr_gprs ||
+       a->compiler_d_valid != d->compiler_d_valid || a->compiler_d != d->compiler_d ||
+       a->imageblock.sample_size_B != d->imageblock.sample_size_B ||
+       a->imageblock.nr_samples != d->imageblock.nr_samples ||
+       a->imageblock.tile_size_px != d->imageblock.tile_size_px ||
+       a->uniform_range_count != d->uniform_range_count)
+      return false;
+   for (unsigned i = 0; i < d->uniform_range_count; ++i)
+      if (t->ranges[i].start_halfs != d->uniform_ranges[i].start_halfs ||
+          t->ranges[i].size_halfs != d->uniform_ranges[i].size_halfs)
+         return false;
+   return hk_g17p_patch_plan_fits(&t->plan, d->uniform_ranges);
+}
+
+static bool
+hk_m5_tile_program(struct hk_device *dev, struct agx_bg_eot_shader *shader,
+                   const struct agx3_g17p_graphics_program_desc *d, bool store,
+                   struct agx3_g17p_graphics_program *program)
+{
+   if (!d->fragment || d->writes_sample_mask || d->colour_classes ||
+       d->preloaded_root || d->patchable || d->threadgroup_memory_size_B ||
+       d->constant_program.code_size || (d->cprog_inputs && d->cprog_inputs->count) ||
+       d->uniform_range_count > HK_G17P_PATCH_MAX_RANGES)
+      return agx3_g17g_tile_program_generate(dev->dev.chip, d, store, program);
+
+   simple_mtx_lock(&dev->bg_eot.lock);
+   struct hk_m5_tile_template *selected = NULL;
+   unsigned count = 0;
+   for (struct hk_m5_tile_template *t = shader->m5_tile_templates; t; t = t->next) {
+      ++count;
+      if (hk_m5_tile_matches(t, d, store)) {
+         selected = t;
+         break;
+      }
+   }
+   /* Bound memory even for layouts whose addresses change incompatibly. */
+   if (!selected && count < 8) {
+      struct agx3_g17p_graphics_program_template *fresh = calloc(1, sizeof(*fresh));
+      if (fresh && agx3_g17g_tile_program_template_generate(dev->dev.chip, d, store, fresh)) {
+         struct hk_m5_tile_template *t =
+            rzalloc_size(shader, sizeof(*t) + fresh->program.length);
+         if (t && hk_g17p_patch_plan_init(&t->plan, d->uniform_ranges,
+                  d->uniform_range_count, fresh->relocations,
+                  fresh->relocation_count, fresh->program.length, false)) {
+            t->desc = *d;
+            t->desc.uniform_ranges = t->ranges;
+            t->desc.cprog_inputs = NULL;
+            if (d->uniform_range_count)
+               memcpy(t->ranges, d->uniform_ranges,
+                      d->uniform_range_count * sizeof(t->ranges[0]));
+            t->store = store;
+            t->length = fresh->program.length;
+            memcpy(t->bytes, fresh->program.bytes, t->length);
+            t->next = shader->m5_tile_templates;
+            shader->m5_tile_templates = selected = t;
+         } else {
+            ralloc_free(t);
+         }
+      }
+      free(fresh);
+   }
+   simple_mtx_unlock(&dev->bg_eot.lock);
+   if (!selected)
+      return agx3_g17g_tile_program_generate(dev->dev.chip, d, store, program);
+
+   program->length = selected->length;
+   memcpy(program->bytes, selected->bytes, selected->length);
+   hk_g17p_patch_plan_apply(&selected->plan, d->uniform_ranges, program->bytes);
+   if (debug_get_option_m5_verify_tile_loader()) {
+      struct agx3_g17p_graphics_program fresh;
+      if (!agx3_g17g_tile_program_generate(dev->dev.chip, d, store, &fresh) ||
+          fresh.length != program->length ||
+          memcmp(fresh.bytes, program->bytes, program->length)) {
+         mesa_loge("M5 cached tile launch differs from fresh composition");
+         return false;
+      }
+   }
+   return true;
+}
+
 static struct hk_bg_eot
-hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
-                bool store, bool partial_render, bool incomplete_render_area,
-                struct hk_g17p_bg_eot_cache_entry *cache)
+hk_build_m5_mrt_bg_eot(struct hk_cmd_buffer *cmd, struct agx_bg_eot_key *key,
+                       const VkRenderingInfo *info, bool store,
+                       bool partial_render)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const struct hk_rendering_state *render = &cmd->state.gfx.render;
+   const unsigned count = info->colorAttachmentCount;
+   unsigned active[HK_MAX_RTS], active_count = 0;
+   bool loads = false, clears = false;
+   bool resolving = false;
+   struct agx_bg_eot_key tile_key = *key;
+   struct agx_bg_eot_shader *shader = NULL;
+   if (!hk_m5_mrt_tile_layout(render, &tile_key.tib))
+      goto unsupported;
+   for (unsigned rt = 0; rt < count; ++rt) {
+      if (!render->tilebuffer.spilled[rt]) continue;
+      tile_key.op[rt] = AGX_BG_EOT_NONE;
+   }
+   key = &tile_key;
+   if (store && !partial_render) {
+      bool shader_resolve = false;
+      for (unsigned rt = 0; rt < count; ++rt)
+         resolving |= render->color_att[rt].resolve_iview != NULL;
+      for (unsigned rt = 0; rt < count; ++rt) {
+         /* The EOT resolve averages the samples of a tile. Only AVERAGE may
+          * use it: SAMPLE_ZERO (every integer resolve, Vulkan 1.4 table
+          * "resolve modes") averaged produces values that are neither
+          * sample, so it resolves from the stored samples after the pass. */
+         shader_resolve |= render->color_att[rt].resolve_iview &&
+            (hk_agx3_format_needs_shader_resolve(key->tib.logical_format[rt]) ||
+             info->pColorAttachments[rt].storeOp == VK_ATTACHMENT_STORE_OP_STORE ||
+             info->pColorAttachments[rt].resolveMode !=
+                VK_RESOLVE_MODE_AVERAGE_BIT);
+      }
+      /* Raw tile words need floating-point arithmetic after the store.
+       * Keep all attachments together on the shared shader resolve path. */
+      resolving &= !shader_resolve && !agx_tilebuffer_spills(&render->tilebuffer);
+   }
+   if (debug_get_option_m5_launch_debug()) {
+      for (unsigned rt = 0; rt < count; ++rt) {
+         const struct hk_image_view *view = render->color_att[rt].iview;
+         fprintf(stderr, "M5 MRT attachment %u: format=%u selector=%#x "
+                         "samples=%u resolve=%u op=%u store=%u load=%u "
+                         "offset=%u layers=%u views=%#x\n", rt,
+                 key->tib.logical_format[rt],
+                 agx_g17p_mrt_eot_selector(key->tib.logical_format[rt]),
+                 view ? view->vk.image->samples : 0,
+                 render->color_att[rt].resolve_iview != NULL, key->op[rt],
+                 info->pColorAttachments[rt].storeOp,
+                 info->pColorAttachments[rt].loadOp,
+                 key->tib._offset_B[rt], render->layer_count, render->view_mask);
+      }
+   }
+   if (count != render->color_att_count || !hk_m5_mrt_layout_supported(render))
+      goto unsupported;
+   for (unsigned rt = 0; rt < count; ++rt) {
+      const VkRenderingAttachmentInfo *att = &info->pColorAttachments[rt];
+      if (!render->color_att[rt].iview) {
+         if (key->op[rt] != AGX_BG_EOT_NONE)
+            goto unsupported;
+         continue;
+      }
+      const bool has_resolve = att->resolveMode != VK_RESOLVE_MODE_NONE;
+      /* An attachment the EOT does not store (a final EOT of a DONT_CARE or
+       * NONE attachment without resolve) is simply absent from the store:
+       * the M5 multi-target EOT numbers its PBEs and offset arguments by
+       * stored target only. A partial EOT stores every attachment
+       * (key->op), whatever its API store operation. */
+      if (store && key->op[rt] == AGX_BG_EOT_NONE)
+         continue;
+      if (render->tilebuffer.spilled[rt]) continue;
+      active[active_count++] = rt;
+      /* A final resolve-only EOT writes the single-sample descriptors.
+       * Partial EOTs always preserve the multisample source for reload. */
+      if ((has_resolve && att->resolveMode != VK_RESOLVE_MODE_AVERAGE_BIT &&
+           att->resolveMode != VK_RESOLVE_MODE_SAMPLE_ZERO_BIT) ||
+          (resolving && (!has_resolve ||
+                         att->storeOp == VK_ATTACHMENT_STORE_OP_STORE)))
+         goto unsupported;
+      if (store) {
+         if (key->op[rt] != AGX_EOT_STORE)
+            goto unsupported;
+      } else {
+         if (key->op[rt] == AGX_BG_EOT_NONE &&
+             att->loadOp == VK_ATTACHMENT_LOAD_OP_DONT_CARE)
+            key->op[rt] = AGX_BG_CLEAR;
+         if (key->op[rt] != AGX_BG_LOAD && key->op[rt] != AGX_BG_CLEAR)
+            goto unsupported;
+         loads |= key->op[rt] == AGX_BG_LOAD;
+         clears |= key->op[rt] == AGX_BG_CLEAR;
+      }
+   }
+   for (unsigned rt = count; rt < HK_MAX_RTS; ++rt) {
+      if (key->op[rt] != AGX_BG_EOT_NONE)
+         goto unsupported;
+   }
+   key->mixed_mrt = true;
+   key->multisample_store = store && !resolving && key->tib.nr_samples > 1;
+   const unsigned clear_base = loads ? 8 : 4;
+   if (clears)
+      key->reserved_preamble = clear_base + 8 * count;
+   shader = agx_get_bg_eot_shader(&dev->bg_eot, key);
+   if (!shader || !shader->info.nr_gprs || !shader->info.agx3_staging_known ||
+       shader->info.scratch_size || shader->info.has_preamble ||
+       shader->info.push_count != key->reserved_preamble ||
+       shader->info.uses_txf != loads)
+      goto unsupported;
+
+   struct agx3_ioto_uniform_range ranges[3];
+   unsigned range_count = 0;
+   if (store) {
+      struct hk_native_pbe_descriptor descriptors[HK_MAX_RTS];
+      uint64_t arguments[1 + HK_MAX_RTS] = {0};
+      for (unsigned i = 0; i < active_count; ++i) {
+         const unsigned rt = active[i];
+         const struct hk_image_view *view = resolving
+            ? render->color_att[rt].resolve_iview : render->color_att[rt].iview;
+         if (!view || !(key->tib.layered
+                           ? view->planes[0].layered_eot_pbe_valid
+                           : view->planes[0].eot_pbe_valid))
+            goto unsupported;
+         descriptors[i] = key->tib.layered ? view->planes[0].layered_eot
+                                            : view->planes[0].eot;
+         if (resolving && util_format_is_srgb(key->tib.logical_format[rt]) &&
+             !(render->color_att[rt].flags &
+               VK_RENDERING_ATTACHMENT_RESOLVE_SKIP_TRANSFER_FUNCTION_BIT_KHR))
+            ((uint8_t *)&descriptors[i])[15] |= 0x20;
+      }
+      uint64_t pbes = hk_pool_upload(cmd, descriptors,
+                                      active_count * sizeof(descriptors[0]), 32);
+      if (!pbes)
+         return (struct hk_bg_eot){0};
+      /* Genuine builtin ABI: U0 is the contiguous PBE table pointer;
+       * U4+4rt carries each tile byte offset. IMGWBLK selects descriptorrt. */
+      arguments[0] = pbes;
+      for (unsigned i = 0; i < active_count; ++i) {
+         const unsigned rt = active[i];
+         arguments[1 + i] = agx_tilebuffer_offset_B(&key->tib, rt);
+         if (debug_get_option_m5_launch_debug()) {
+            fprintf(stderr, "M5 MRT EOT argument rt=%u pbe=%#" PRIx64
+                            " offset=%" PRIu64 " descriptor=", rt,
+                    pbes + i * sizeof(descriptors[0]), arguments[1 + i]);
+            const uint8_t *bytes = (const uint8_t *)&descriptors[i];
+            for (unsigned b = 0; b < sizeof(descriptors[0]); ++b)
+               fprintf(stderr, "%02x", bytes[b]);
+            fprintf(stderr, "\n");
+         }
+      }
+      uint64_t table = hk_pool_upload(cmd, arguments, 8 * (1 + active_count), 16);
+      if (!table)
+         return (struct hk_bg_eot){0};
+      ranges[range_count++] = (struct agx3_ioto_uniform_range){0, 4 * (1 + active_count), table};
+   } else {
+      if (loads) {
+         if (!dev->rodata.bo || !dev->rodata.bo->va)
+            goto unsupported;
+         struct hk_native_texture_descriptor descriptors[2 * HK_MAX_RTS] = {0};
+         for (unsigned rt = 0; rt < count; ++rt) {
+            if (key->op[rt] == AGX_BG_LOAD)
+               descriptors[2 * rt] = key->tib.layered
+                  ? render->color_att[rt].iview->planes[0].layered_background
+                  : render->color_att[rt].iview->planes[0].background;
+         }
+         uint64_t textures = hk_pool_upload(cmd, descriptors,
+            2 * count * sizeof(descriptors[0]), 32);
+         if (!textures)
+            return (struct hk_bg_eot){0};
+         const uint64_t pointers[] = {textures, dev->rodata.bo->va->addr};
+         uint64_t table = hk_pool_upload(cmd, pointers, sizeof(pointers), 16);
+         if (!table)
+            return (struct hk_bg_eot){0};
+         ranges[range_count++] = (struct agx3_ioto_uniform_range){0, 4, table};
+         ranges[range_count++] = (struct agx3_ioto_uniform_range){4, 4, table + 8};
+      }
+      if (clears) {
+         VkClearColorValue colors[HK_MAX_RTS] = {0};
+         for (unsigned rt = 0; rt < count; ++rt) {
+            if (key->op[rt] == AGX_BG_CLEAR &&
+                info->pColorAttachments[rt].loadOp != VK_ATTACHMENT_LOAD_OP_DONT_CARE)
+               colors[rt] = info->pColorAttachments[rt].clearValue.color;
+         }
+         uint64_t table = hk_pool_upload(cmd, colors, 16 * count, 16);
+         if (!table)
+            return (struct hk_bg_eot){0};
+         ranges[range_count++] = (struct agx3_ioto_uniform_range){clear_base, 8 * count, table};
+      }
+   }
+   size_t capacity;
+   if (!agx3_g17g_graphics_program_bound(dev->dev.chip, ranges, range_count, &capacity))
+      goto unsupported;
+   struct hk_ioto_reservation reservation;
+   if (!hk_cmd_reserve_ioto_program(cmd, capacity, "M5 MRT tile program", &reservation))
+      return (struct hk_bg_eot){0};
+   const struct agx3_g17p_graphics_program_desc desc = {
+      .program_va = reservation.va,
+      .shader_va = shader->ptr + shader->info.main_offset,
+      .shader_binary_size = shader->info.binary_size,
+      .scratch_size = shader->info.scratch_size,
+      .nr_gprs = shader->info.nr_gprs,
+      .compiler_d_valid = shader->info.agx3_staging_known,
+      .compiler_d = (uint32_t)shader->info.agx3_staging_halfs * 2,
+      .fragment = true,
+      .imageblock = {key->tib.sample_size_B, key->tib.nr_samples, key->tib.tile_size},
+      .uniform_ranges = ranges,
+      .uniform_range_count = range_count,
+   };
+   struct agx3_g17p_graphics_program program;
+   if (!hk_m5_tile_program(dev, shader, &desc, store, &program) ||
+       program.length > capacity)
+      goto unsupported;
+   if (debug_get_option_m5_launch_debug()) {
+      fprintf(stderr, "M5 MRT %s count=%u sampleB=%u tile=%u gprs=%u D=%u "
+                      "shader=%#" PRIx64 " mainSize=%u ioto=",
+              store ? "EOT" : "BG", count, key->tib.sample_size_B,
+              key->tib.tile_size, desc.nr_gprs, desc.compiler_d,
+              desc.shader_va, shader->info.main_size);
+      for (size_t i = 0; i < program.length; ++i)
+         fprintf(stderr, "%02x", program.bytes[i]);
+      fprintf(stderr, "\n");
+      for (unsigned i = 0; i < range_count; ++i)
+         fprintf(stderr, "M5 MRT %s range=%u start=%u size=%u address=%#" PRIx64
+                         "\n", store ? "EOT" : "BG", i,
+                 ranges[i].start_halfs, ranges[i].size_halfs, ranges[i].address);
+      if (shader->bo && shader->bo->va &&
+          desc.shader_va >= shader->bo->va->addr) {
+         const uint64_t offset = desc.shader_va - shader->bo->va->addr;
+         if (offset <= shader->bo->size &&
+             shader->info.main_size <= shader->bo->size - offset) {
+            const uint8_t *bytes = agx_bo_map(shader->bo);
+            if (bytes) {
+               fprintf(stderr, "M5 MRT %s shaderText=", store ? "EOT" : "BG");
+               for (unsigned i = 0; i < shader->info.main_size; ++i)
+                  fprintf(stderr, "%02x", bytes[offset + i]);
+               fprintf(stderr, "\n");
+            }
+         }
+      }
+   }
+   memcpy(reservation.map, program.bytes, program.length);
+   hk_cmd_buffer_retain_bo(cmd, shader->bo);
+   uint64_t published = hk_cmd_publish_ioto_program(cmd, &reservation, program.length);
+   if (published && resolving)
+      cmd->state.gfx.render.g17p_direct_color_resolve = true;
+   return (struct hk_bg_eot){
+      .usc = published,
+      .primary_storage_B = desc.scratch_size,
+      .staging_storage_B = desc.compiler_d,
+      .rsrc_spec = store ? 0 : loads && key->tib.nr_samples > 1
+         ? UINT64_C(0xffff800000000000) : UINT64_C(0xffff800000000040),
+   };
+unsupported:
+   if (debug_get_option_m5_launch_debug())
+      mesa_loge("M5 MRT admission: store=%u shader=%u gprs=%u staging=%u "
+                "scratch=%u preamble=%u push=%u expected=%u txf=%u loads=%u "
+                "attachments=%u samples=%u sampleB=%u tile=%u",
+                store, shader != NULL, shader ? shader->info.nr_gprs : 0,
+                shader ? shader->info.agx3_staging_known : 0,
+                shader ? shader->info.scratch_size : 0,
+                shader ? shader->info.has_preamble : 0,
+                shader ? shader->info.push_count : 0, key->reserved_preamble,
+                shader ? shader->info.uses_txf : 0, loads, count,
+                key->tib.nr_samples, key->tib.sample_size_B, key->tib.tile_size);
+   mesa_loge("M5 MRT BG/EOT exceeds supported tile-program profile");
+   vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+   return (struct hk_bg_eot){0};
+}
+
+static struct hk_bg_eot
+hk_build_m5_empty_bg_eot(struct hk_cmd_buffer *cmd,
+                          struct agx_bg_eot_key *key, bool store)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const struct hk_rendering_state *render = &cmd->state.gfx.render;
+   /* A colour tile whose final EOT stores nothing (every attachment
+    * DONT_CARE/NONE) ends with the same image-operation-free program; only
+    * its launch's imageblock is the colour allocation, as G17P's empty
+    * (no-resource) tile program carries key->tib. */
+   const bool color = hk_m5_has_color(&key->tib);
+   /* The STOP program performs no image operation; a layered pass differs
+    * only in its imageblock and in render-command layer state. */
+   if (color ? !store || !hk_m5_color_imageblock_supported(&key->tib)
+             : !hk_m5_empty_tile_layout_supported(&render->tilebuffer))
+      goto unsupported;
+   for (unsigned rt = 0; rt < HK_MAX_RTS; ++rt) {
+      if (key->op[rt] != AGX_BG_EOT_NONE)
+         goto unsupported;
+   }
+   /* This program has no image operations. Reuse the same STOP shader for
+    * every sample count; its launch still carries the real tile allocation. */
+   struct agx_bg_eot_key shader_key = *key;
+   shader_key.tib.nr_samples = 1;
+   shader_key.tib.sample_size_B = 0;
+   shader_key.tib.tile_size = 32 * 32;
+   shader_key.tib.layered = false;
+   struct agx_bg_eot_shader *shader =
+      agx_get_bg_eot_shader(&dev->bg_eot, &shader_key);
+   if (!shader || shader->info.binary_size != 4 ||
+       shader->info.nr_gprs != 2 || !shader->info.agx3_staging_known ||
+       shader->info.agx3_staging_halfs || shader->info.scratch_size ||
+       shader->info.has_preamble || shader->info.push_count ||
+       shader->info.uses_txf)
+      goto unsupported;
+
+   size_t capacity;
+   if (!agx3_g17g_graphics_program_bound(dev->dev.chip, NULL, 0, &capacity))
+      goto unsupported;
+   struct hk_ioto_reservation reservation;
+   if (!hk_cmd_reserve_ioto_program(cmd, capacity,
+                                   "M5 empty tile program", &reservation))
+      return (struct hk_bg_eot){0};
+   const struct agx3_g17p_graphics_program_desc desc = {
+      .program_va = reservation.va,
+      .shader_va = shader->ptr,
+      .shader_binary_size = shader->info.binary_size,
+      .nr_gprs = shader->info.nr_gprs,
+      .compiler_d_valid = true,
+      .fragment = true,
+      .imageblock = {key->tib.sample_size_B, MAX2(key->tib.nr_samples, 1),
+                     key->tib.tile_size},
+   };
+   struct agx3_g17p_graphics_program program;
+   if (!hk_m5_tile_program(dev, shader, &desc, store, &program) ||
+       program.length > capacity)
+      goto unsupported;
+   memcpy(reservation.map, program.bytes, program.length);
+   hk_cmd_buffer_retain_bo(cmd, shader->bo);
+   return (struct hk_bg_eot){
+      .usc = hk_cmd_publish_ioto_program(cmd, &reservation, program.length),
+      .rsrc_spec = store ? 0 : UINT64_C(0xffff800000000040),
+   };
+
+unsupported:
+   if (debug_get_option_m5_launch_debug())
+      mesa_loge("M5 empty BG/EOT admission: store=%u color=%u sampleB=%u "
+                "samples=%u tile=%u layered=%u layers=%u views=%#x spills=%u "
+                "ops=%u%u%u%u%u%u%u%u",
+                store, hk_m5_has_color(&key->tib), key->tib.sample_size_B,
+                key->tib.nr_samples, key->tib.tile_size, key->tib.layered,
+                cmd->state.gfx.render.layer_count,
+                cmd->state.gfx.render.view_mask,
+                agx_tilebuffer_spills(&key->tib), key->op[0], key->op[1],
+                key->op[2], key->op[3], key->op[4], key->op[5], key->op[6],
+                key->op[7]);
+   mesa_loge("M5 empty BG/EOT exceeds native no-imageblock profile");
+   vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+   return (struct hk_bg_eot){0};
+}
+
+static struct hk_bg_eot
+hk_build_m5_bg_eot(struct hk_cmd_buffer *cmd, struct agx_bg_eot_key *key,
+                   const VkRenderingInfo *info, bool store, bool partial_render)
+{
+   const struct hk_rendering_state *render = &cmd->state.gfx.render;
+   if (!hk_m5_has_color(&key->tib))
+      return hk_build_m5_empty_bg_eot(cmd, key, store);
+   if (store) {
+      bool stores = false;
+      for (unsigned rt = 0; rt < HK_MAX_RTS; ++rt)
+         stores |= key->op[rt] != AGX_BG_EOT_NONE;
+      if (!stores)
+         return hk_build_m5_empty_bg_eot(cmd, key, store);
+   }
+   /* BG/EOT descriptor slots are dense, independently of fragment output
+    * locations. Keep the render state in Vulkan order and compact only this
+    * internal program's key and attachment selection. */
+   struct agx_bg_eot_key compact_key = *key;
+   uint8_t attachment_map[HK_MAX_RTS];
+   const unsigned active_count = agx_tilebuffer_compact(
+      &key->tib, &compact_key.tib, attachment_map);
+   /* Wide multisample colors use the same typed tile program with or without
+    * a final resolve. The narrow RT0 loader only covers eight-byte samples. */
+   if (active_count > 1 ||
+       (key->tib.nr_samples > 1 && key->tib.sample_size_B > 8) ||
+       (active_count == 1 && render->color_att[attachment_map[0]].resolve_iview &&
+        (agx_g17p_mrt_eot_selector(key->tib.logical_format[attachment_map[0]]) != 0x75 ||
+         hk_agx3_format_needs_shader_resolve(
+            key->tib.logical_format[attachment_map[0]]))))
+      return hk_build_m5_mrt_bg_eot(cmd, key, info, store, partial_render);
+   const unsigned logical = attachment_map[0];
+   memset(compact_key.op, 0, sizeof(compact_key.op));
+   compact_key.op[0] = key->op[logical];
+   VkRenderingInfo compact_info = *info;
+   compact_info.colorAttachmentCount = 1;
+   compact_info.pColorAttachments = &info->pColorAttachments[logical];
+   key = &compact_key;
+   info = &compact_info;
+
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   struct hk_image_view *view = cmd->state.gfx.render.color_att[logical].iview;
+   struct agx_bg_eot_shader *shader = NULL;
+   const bool clear_dont_care = !store && key->op[0] == AGX_BG_EOT_NONE &&
+      info->colorAttachmentCount == 1 &&
+      info->pColorAttachments[0].loadOp == VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+   enum agx_bg_eot_op op = clear_dont_care ? AGX_BG_CLEAR : key->op[0];
+   const bool multisample =
+      ((key->tib.nr_samples == 2 && key->tib.tile_size == 32 * 32) ||
+       (key->tib.nr_samples == 4 && key->tib.tile_size == 32 * 16) ||
+       (key->tib.nr_samples == 8 && key->tib.tile_size == 16 * 16));
+   const bool single_sample = key->tib.nr_samples == 1 &&
+                              key->tib.tile_size == 32 * 32;
+   /* Four raw 32-bit color lanes use the native 16-byte tile stride. */
+   const bool sample_stride = key->tib.sample_size_B == 8 ||
+      (single_sample && key->tib.sample_size_B == 16);
+   const VkRenderingAttachmentInfo *att = info->colorAttachmentCount == 1
+      ? &info->pColorAttachments[0] : NULL;
+   struct hk_image_view *resolve =
+      cmd->state.gfx.render.color_att[logical].resolve_iview;
+   const bool has_resolve = att && att->resolveMode != VK_RESOLVE_MODE_NONE;
+   const bool final_resolve = store && !partial_render && has_resolve;
+   const bool store_source = att && att->storeOp == VK_ATTACHMENT_STORE_OP_STORE;
+   if (!view || info->colorAttachmentCount != 1 || !sample_stride ||
+       (has_resolve &&
+        (!multisample || att->resolveMode != VK_RESOLVE_MODE_AVERAGE_BIT ||
+         agx_g17p_mrt_eot_selector(key->tib.logical_format[0]) != 0x75 || !resolve ||
+         resolve->vk.image->samples != 1 || resolve->vk.format != view->vk.format ||
+         !(key->tib.layered ? resolve->planes[0].layered_eot_pbe_valid
+                            : resolve->planes[0].eot_pbe_valid))) ||
+       (store ? op != AGX_EOT_STORE : op != AGX_BG_CLEAR && op != AGX_BG_LOAD) ||
+       (!single_sample && !multisample) || agx_tilebuffer_spills(&key->tib) ||
+       agx_tilebuffer_offset_B(&key->tib, 0))
+      goto unsupported;
+   for (unsigned rt = 1; rt < HK_MAX_RTS; ++rt) {
+      if (key->op[rt] != AGX_BG_EOT_NONE)
+         goto unsupported;
+   }
+   key->op[0] = op;
+   /* Partial EOTs preserve samples for reload. A final EOT can resolve the
+    * tile directly, retaining the source as well only when STORE asks for it. */
+   if (multisample)
+      key->multisample_store = store && (!final_resolve || store_source);
+   key->m5_store_and_resolve = final_resolve && store_source;
+   if (op == AGX_BG_CLEAR)
+      key->reserved_preamble = AGX3_G17P_RT0_CLEAR_PUSH_HALFS;
+   shader = agx_get_bg_eot_shader(&dev->bg_eot, key);
+   if (!shader || !shader->info.nr_gprs || !shader->info.agx3_staging_known ||
+       shader->info.scratch_size || shader->info.has_preamble)
+      goto unsupported;
+
+   struct agx3_ioto_uniform_range ranges[2];
+   unsigned range_count = 1;
+   if (store) {
+      struct hk_image_view *target = final_resolve && !store_source
+         ? resolve : view;
+      if (view->vk.image->samples != key->tib.nr_samples ||
+          !(key->tib.layered ? target->planes[0].layered_eot_pbe_valid
+                            : target->planes[0].eot_pbe_valid))
+         goto unsupported;
+      const struct hk_native_pbe_descriptor *descriptor = key->tib.layered
+         ? &target->planes[0].layered_eot : &target->planes[0].eot;
+      struct hk_native_pbe_descriptor descriptors[2] = {*descriptor};
+      if (key->m5_store_and_resolve)
+         descriptors[1] = key->tib.layered ? resolve->planes[0].layered_eot
+                                            : resolve->planes[0].eot;
+      if (final_resolve && util_format_is_srgb(key->tib.logical_format[0]) &&
+          !(render->color_att[logical].flags &
+            VK_RENDERING_ATTACHMENT_RESOLVE_SKIP_TRANSFER_FUNCTION_BIT_KHR))
+         ((uint8_t *)&descriptors[key->m5_store_and_resolve ? 1 : 0])[15] |= 0x20;
+      const unsigned descriptor_count = key->m5_store_and_resolve ? 2 : 1;
+      uint64_t pbe = hk_pool_upload(cmd, descriptors,
+         descriptor_count * sizeof(*descriptor), 32);
+      if (!pbe)
+         return (struct hk_bg_eot){0};
+      const uint64_t arguments[] = {
+         pbe, 0, key->tib.layered ? 0 : pbe + sizeof(*descriptor),
+      };
+      uint64_t table = hk_pool_upload(cmd, arguments,
+         (key->m5_store_and_resolve ? 3 : 2) * sizeof(uint64_t), 16);
+      if (!table)
+         return (struct hk_bg_eot){0};
+      ranges[0] = (struct agx3_ioto_uniform_range){
+         0, key->m5_store_and_resolve ? 12 : 6, table};
+   } else if (op == AGX_BG_CLEAR) {
+      if (shader->info.push_count != AGX3_G17P_RT0_CLEAR_PUSH_HALFS ||
+          shader->info.uses_txf)
+         goto unsupported;
+      const VkClearColorValue zero = {0};
+      uint64_t clear = hk_pool_upload(cmd,
+         clear_dont_care ? &zero : &info->pColorAttachments[0].clearValue.color,
+         sizeof(zero), 16);
+      if (!clear)
+         return (struct hk_bg_eot){0};
+      ranges[0] = (struct agx3_ioto_uniform_range){4, 8, clear};
+   } else {
+      if (shader->info.push_count || !shader->info.uses_txf ||
+          !dev->rodata.bo || !dev->rodata.bo->va)
+         goto unsupported;
+      const struct hk_native_texture_descriptor *descriptor = key->tib.layered
+         ? &view->planes[0].layered_background : &view->planes[0].background;
+      uint64_t texture = hk_pool_upload(cmd, descriptor, sizeof(*descriptor), 32);
+      if (!texture)
+         return (struct hk_bg_eot){0};
+      const uint64_t pointers[] = {texture, dev->rodata.bo->va->addr};
+      uint64_t table = hk_pool_upload(cmd, pointers, sizeof(pointers), 8);
+      if (!table)
+         return (struct hk_bg_eot){0};
+      ranges[0] = (struct agx3_ioto_uniform_range){0, 4, table};
+      ranges[1] = (struct agx3_ioto_uniform_range){4, 4, table + 8};
+      range_count = 2;
+   }
+   size_t capacity;
+   if (!agx3_g17g_graphics_program_bound(dev->dev.chip, ranges, range_count,
+                                        &capacity))
+      goto unsupported;
+   struct hk_ioto_reservation reservation;
+   if (!hk_cmd_reserve_ioto_program(cmd, capacity, "M5 tile program", &reservation))
+      return (struct hk_bg_eot){0};
+   const struct agx3_g17p_graphics_program_desc desc = {
+      .program_va = reservation.va,
+      .shader_va = shader->ptr + shader->info.main_offset,
+      .shader_binary_size = shader->info.binary_size,
+      .scratch_size = shader->info.scratch_size,
+      .nr_gprs = shader->info.nr_gprs,
+      .compiler_d_valid = shader->info.agx3_staging_known,
+      .compiler_d = (uint32_t)shader->info.agx3_staging_halfs * 2,
+      .fragment = true,
+      .imageblock = {key->tib.sample_size_B, key->tib.nr_samples, key->tib.tile_size},
+      .uniform_ranges = ranges,
+      .uniform_range_count = range_count,
+   };
+   struct agx3_g17p_graphics_program program;
+   if (!hk_m5_tile_program(dev, shader, &desc, store, &program) ||
+       program.length > capacity)
+      goto unsupported;
+   memcpy(reservation.map, program.bytes, program.length);
+   hk_cmd_buffer_retain_bo(cmd, shader->bo);
+   uint64_t published = hk_cmd_publish_ioto_program(cmd, &reservation, program.length);
+   if (published && final_resolve)
+      cmd->state.gfx.render.g17p_direct_color_resolve = true;
+   return (struct hk_bg_eot){
+      .usc = published,
+      .primary_storage_B = desc.scratch_size,
+      .staging_storage_B = desc.compiler_d,
+      .rsrc_spec = store ? 0 : multisample && op == AGX_BG_LOAD
+         ? UINT64_C(0xffff800000000000) : UINT64_C(0xffff800000000040),
+   };
+unsupported:
+   if (debug_get_option_m5_launch_debug()) {
+      mesa_loge("M5 BG/EOT admission: store=%u partial=%u op=%u attachments=%u "
+                "sampleB=%u samples=%u tile=%u layered=%u spills=%u offsetB=%u "
+                "format=%u viewformat=%u pbe=%u shader=%u gprs=%u "
+                "stagingKnown=%u scratch=%u preamble=%u push=%u txf=%u",
+                store, partial_render, op, info->colorAttachmentCount,
+                key->tib.sample_size_B, key->tib.nr_samples, key->tib.tile_size,
+                key->tib.layered, agx_tilebuffer_spills(&key->tib),
+                agx_tilebuffer_offset_B(&key->tib, 0), key->tib.logical_format[0],
+                view ? view->vk.format : VK_FORMAT_UNDEFINED,
+                view && (key->tib.layered ? view->planes[0].layered_eot_pbe_valid
+                                          : view->planes[0].eot_pbe_valid),
+                shader != NULL, shader ? shader->info.nr_gprs : 0,
+                shader ? shader->info.agx3_staging_known : 0,
+                shader ? shader->info.scratch_size : 0,
+                shader ? shader->info.has_preamble : 0,
+                shader ? shader->info.push_count : 0,
+                shader ? shader->info.uses_txf : 0);
+   }
+   vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+   return (struct hk_bg_eot){0};
+}
+
+/* anatomy_ops: NULL, or HK_MAX_RTS bytes receiving the frame anatomy
+ * profiler's per-RT summary of the key this program is built from. */
+static struct hk_bg_eot
+hk_build_bg_eot_program(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
+                        bool store, bool partial_render,
+                        bool incomplete_render_area,
+                        struct hk_g17p_bg_eot_cache_entry *cache,
+                        uint8_t *anatomy_ops)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    struct hk_rendering_state *render = &cmd->state.gfx.render;
@@ -1711,7 +2692,8 @@ hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
        * there's nothing to do.
        */
       if (key.tib.spilled[i] &&
-          (partial_render || store || agx_native_graphics(&dev->dev)))
+          (partial_render || store || agx_native_graphics(&dev->dev) ||
+           agx_chip_g17_m5(dev->dev.chip)))
          continue;
 
       if (store) {
@@ -1777,6 +2759,20 @@ hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
       }
    }
 
+   if (unlikely(anatomy_ops)) {
+      /* Profiler only. The key ops below are what every chip path builds
+       * from; the M5 single-RT path additionally treats a DONT_CARE load
+       * with no op as a clear, visible in the log as hw '-' with API 'D'. */
+      for (unsigned i = 0; i < HK_MAX_RTS; ++i) {
+         const VkRenderingAttachmentInfo *att =
+            i < info->colorAttachmentCount ? &info->pColorAttachments[i] : NULL;
+         const bool present = att && att->imageView != VK_NULL_HANDLE;
+         anatomy_ops[i] = hk_frame_anatomy_encode_op(
+            present, key.op[i], present ? att->loadOp : 0,
+            present ? att->storeOp : 0);
+      }
+   }
+
    if (agx_native_graphics(&dev->dev)) {
       const unsigned variant = (store ? 2 : 0) | (partial_render ? 1 : 0);
       if (cache && (cache->valid_variants & BITFIELD_BIT(variant))) {
@@ -1831,6 +2827,9 @@ hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
       return program;
    }
 
+   if (agx_chip_g17_m5(dev->dev.chip))
+      return hk_build_m5_bg_eot(cmd, &key, info, store, partial_render);
+
    const unsigned descriptor_generation =
       agx_chip_descriptor_generation(dev->dev.chip);
    if (!hk_native_pbe_hardware_supported(descriptor_generation)) {
@@ -1940,6 +2939,10 @@ hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
    key.reserved_preamble = uniforms;
    /* XXX: locking? */
    struct agx_bg_eot_shader *shader = agx_get_bg_eot_shader(&dev->bg_eot, &key);
+   if (!shader) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return (struct hk_bg_eot){0};
+   }
 
    agx_usc_pack(&b, SHADER, cfg) {
       cfg.code = agx_usc_addr(&dev->dev, shader->ptr);
@@ -1972,6 +2975,27 @@ hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
    return ret;
 }
 
+static struct hk_bg_eot
+hk_build_bg_eot(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info,
+                bool store, bool partial_render, bool incomplete_render_area,
+                struct hk_g17p_bg_eot_cache_entry *cache)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+
+   if (likely(!dev->frame_anatomy)) {
+      return hk_build_bg_eot_program(cmd, info, store, partial_render,
+                                     incomplete_render_area, cache, NULL);
+   }
+
+   /* Frame anatomy profiler: carry the per-RT op summary with the program,
+    * so a stream reports the main or partial variant it actually uses. */
+   uint8_t ops[HK_MAX_RTS] = {0};
+   struct hk_bg_eot program = hk_build_bg_eot_program(
+      cmd, info, store, partial_render, incomplete_render_area, cache, ops);
+   memcpy(program.anatomy_ops, ops, sizeof(ops));
+   return program;
+}
+
 static void
 hk_merge_render_iview(struct hk_rendering_state *render,
                       struct hk_image_view *iview, bool zls)
@@ -2218,27 +3242,33 @@ hk_g17p_zls_elide_store(struct hk_cmd_buffer *cmd, VkImageAspectFlags aspects)
    }
 }
 
-/* G17P: a colour attachment that a single-stream, single-sample pass loads
- * and stores, but that no draw or clear wrote and no fragment shader read,
- * keeps its memory contents as they are. Its end-of-tile stores, partial-
- * render ones included, are dropped, and the main background program clears
- * its tile instead of loading it: the end-of-tile program then carries no
- * operation for it, as for storeOp DONT_CARE, and the background program a
- * load-op clear, as for loadOp CLEAR. The partial-render background program
- * still reloads it, from the memory no store touched. The rebuild is exact
- * only for attachments without clear, resolve or flags, so other passes keep
- * their programs. A continuation of the pass restores the full programs
- * (hk_m5_pass_merge_begin()). */
+/* G17P and M5: a colour attachment that a single-stream pass (single-sample
+ * on G17P) loads and stores, but that no draw or clear wrote and no fragment
+ * shader read, keeps its memory contents as they are. Its end-of-tile
+ * stores, partial-render ones included, are dropped, and the main background
+ * program clears its tile instead of loading it: the end-of-tile program
+ * then carries no operation for it, as for storeOp DONT_CARE, and the
+ * background program a load-op clear, as for loadOp CLEAR. The partial-render
+ * background program still reloads it, from the memory no store touched.
+ * The rebuild is exact only for attachments without clear, resolve or flags,
+ * so other passes keep their programs. A continuation of the pass restores
+ * the full programs (hk_m5_pass_merge_begin()). */
 static void
 hk_g17p_elide_untouched_color(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    struct hk_rendering_state *render = &cmd->state.gfx.render;
-   if (dev->dev.chip != AGX_CHIP_G17P || cs != render->zls_first_cs ||
-       cmd->in_meta || render->color_read || render->view_mask ||
-       render->layer_count > 1 || render->tilebuffer.nr_samples > 1 ||
-       agx_tilebuffer_spills(&render->tilebuffer) ||
-       render->g17p_direct_color_resolve ||
+   const bool m5 = agx_chip_g17_m5(dev->dev.chip);
+   /* The rebuilt programs may drop every colour store; M5 has an empty
+    * end-of-tile program only for single-layer, non-spilling colour tiles
+    * (hk_build_m5_empty_bg_eot()), multisampled ones included. */
+   if ((dev->dev.chip != AGX_CHIP_G17P && !m5) ||
+       cs != render->zls_first_cs || cmd->in_meta || render->color_read ||
+       render->view_mask || render->layer_count > 1 ||
+       (m5 ? !hk_m5_color_imageblock_supported(&render->tilebuffer)
+           : (render->tilebuffer.nr_samples > 1 ||
+              agx_tilebuffer_spills(&render->tilebuffer) ||
+              render->g17p_direct_color_resolve)) ||
        (render->flags & (VK_RENDERING_SUSPENDING_BIT |
                          VK_RENDERING_RESUMING_BIT |
                          VK_RENDERING_CONTENTS_SECONDARY_COMMAND_BUFFERS_BIT)))
@@ -2305,6 +3335,7 @@ hk_g17p_elide_untouched_color(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    cs->cr.bg.main = bg_main;
    cs->cr.eot.main = eot_main;
    cs->cr.eot.partial = eot_partial;
+   cs->anatomy_covered = covered;
    perf_debug(cmd, "Dropped untouched colour attachments 0x%x, loads of "
               "covered attachments 0x%x", elide, covered);
 }
@@ -2331,7 +3362,8 @@ hk_g17p_note_first_draw(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
 
    render->first_draw_seen = true;
    render->color_covered = 0;
-   if (dev->dev.chip != AGX_CHIP_G17P || cmd->in_meta ||
+   if ((dev->dev.chip != AGX_CHIP_G17P && !agx_chip_g17_m5(dev->dev.chip)) ||
+       cmd->in_meta ||
        cs != render->zls_first_cs || cs->stats.cmds || cs->m5_continued_pass ||
        hk_conditional_render_is_active(&cmd->state.conditional, cmd->in_meta,
                                        cmd->conditional_meta) ||
@@ -2435,14 +3467,15 @@ hk_pack_zls_control_final(struct agx_zls_control_packed *packed,
 static struct agx_tilebuffer_layout
 hk_tilebuffer_layout(const struct hk_device *dev, const VkFormat *vk_formats,
                      size_t stride, unsigned count, unsigned samples,
-                     bool layered, bool g17p_wide_ms_to_memory)
+                     bool layered, bool g17p_wide_ms_to_memory,
+                     uint8_t resident_mask)
 {
    enum pipe_format formats[HK_MAX_RTS] = {0};
    for (unsigned i = 0; i < count; ++i) {
       const VkFormat format =
          *(const VkFormat *)((const uint8_t *)vk_formats + i * stride);
       formats[i] = hk_format_to_pipe_format(format);
-      if (agx_native_graphics(&dev->dev)) {
+      if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
          formats[i] = hk_padded_image_format(formats[i]);
          const enum pipe_format integer = hk_scaled_texture_format(formats[i]);
          if (integer != PIPE_FORMAT_NONE)
@@ -2454,15 +3487,47 @@ hk_tilebuffer_layout(const struct hk_device *dev, const VkFormat *vk_formats,
     * particular 16x16 at 4x may occupy the full 32 KiB, while 32x32 at 4x
     * exceeds the sample budget even for an eight-byte sample. */
    const bool imageblock = agx_chip_native_imageblock(dev->dev.chip);
+   const bool m5 = agx_chip_g17_m5(dev->dev.chip);
    const uint8_t max_sample_size_B =
+      m5 ? AGX_MAX_RENDER_TARGETS * 16 :
       !imageblock              ? 0
       : g17p_wide_ms_to_memory ? AGX_G17P_MAX_UNCOVERED_TILE_BYTES_PER_SAMPLE
                                : AGX_G17P_MAX_BYTES_PER_SAMPLE;
+   if (m5) {
+      return agx_build_tilebuffer_layout_resident(
+         formats, count, samples, layered, max_sample_size_B, 65536, 2048,
+         resident_mask);
+   }
    return agx_build_tilebuffer_layout_with_limits(
       formats, count, samples, layered, max_sample_size_B,
       imageblock ? 32768 : 0, agx_native_graphics(&dev->dev) ? 2048 : 0);
 }
 
+/* M5 compressed multisample colour attachments stay in the tilebuffer: a
+ * spilled attachment is read back during the pass through its texture view,
+ * and compressed reads do not observe the pass's own memory stores.
+ * Compressed multisample targets are at most eight bytes, so all eight fit
+ * the 4x sample budget (64 bytes) and do not spill when allocated first. */
+static uint8_t
+hk_m5_resident_attachments(const struct hk_device *dev,
+                           const struct hk_rendering_state *render)
+{
+   if (!agx_chip_g17_m5(dev->dev.chip) || render->tilebuffer.nr_samples <= 1)
+      return 0;
+
+   uint8_t mask = 0;
+   for (unsigned i = 0; i < render->color_att_count; ++i) {
+      const struct hk_image_view *view = render->color_att[i].iview;
+      if (!view)
+         continue;
+      const struct hk_image *image =
+         container_of(view->vk.image, struct hk_image, vk);
+      if (image->planes[view->planes[0].image_plane].layout.compressed)
+         mask |= BITFIELD_BIT(i);
+   }
+   return mask;
+}
+
 /*
  * G17P transfer writes (hk_sync_xfer_write in hk_sync.h): a render pass
  * begin or draw that may access a transfer destination no dependency orders
@@ -2740,7 +3805,7 @@ hk_xfer_check_draw(struct hk_cmd_buffer *cmd, uint64_t extra_lo,
 }
 
 /*
- * M5 render pass continuation.
+ * AGX3 render pass continuation.
  *
  * Two consecutive render pass instances on the same attachments, where the
  * second loads what the first stored and nothing but state binding and
@@ -2771,36 +3836,65 @@ hk_m5_pass_merge_candidate(struct hk_cmd_buffer *cmd, const struct hk_cs *cs)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    const struct hk_rendering_state *render = &cmd->state.gfx.render;
+   const unsigned samples = render->tilebuffer.nr_samples;
+   const bool msaa = samples > 1;
+   const bool msaa_enabled = msaa && dev->dev.chip == AGX_CHIP_G17S &&
+      (samples == 2 || samples == 4) &&
+      debug_get_option_m5_msaa_pass_merge();
 
    /* The instance is one ordinary render stream with draws, with no
     * pre/post-graphics streams, heap use or timestamps attached. */
-   if (!agx_chip_native_agx3(dev->dev.chip) || cmd->in_meta ||
-       cs->stats.cmds == 0 || cmd->sync.pass_vdm != cs ||
+   if ((!agx_chip_native_agx3(dev->dev.chip) &&
+        !agx_chip_g17_m5(dev->dev.chip)) ||
+       cmd->in_meta || cs->stats.cmds == 0 || cmd->sync.pass_vdm != cs ||
        cmd->current_cs.pre_gfx || cmd->current_cs.post_gfx ||
        cmd->uses_heap || cs->timestamp.start.handle ||
-       cs->timestamp.end.handle)
+       cs->timestamp.end.handle) {
+      if (msaa_enabled)
+         hk_frame_anatomy_note(dev,
+            "MSAA pass merge refused: pre=%u post=%u heap=%u draws=%u timestamp=%u",
+            cmd->current_cs.pre_gfx != NULL, cmd->current_cs.post_gfx != NULL,
+            cmd->uses_heap, cs->stats.cmds,
+            cs->timestamp.start.handle != 0 || cs->timestamp.end.handle != 0);
       return false;
+   }
 
    if (render->flags & (VK_RENDERING_SUSPENDING_BIT |
                         VK_RENDERING_RESUMING_BIT |
                         VK_RENDERING_CONTENTS_SECONDARY_COMMAND_BUFFERS_BIT))
       return false;
 
-   /* Single-sample, in-tilebuffer rendering without resolves. */
-   if (render->view_mask || render->tilebuffer.nr_samples > 1 ||
+   /* In-tilebuffer rendering without resolves. Multisample continuation
+    * keeps the existing per-sample partial store and reload programs. */
+   if (render->view_mask || (msaa && !msaa_enabled) ||
        agx_tilebuffer_spills(&render->tilebuffer) ||
        render->g17p_direct_color_resolve)
       return false;
+   /* Compressed attachments continue like uncompressed ones: the merged
+    * render keeps the first instance's load and the last instance's store,
+    * and a partial render stores and reloads them as a single instance's
+    * would. */
+   if (msaa && (render->layer_count != 1 || cs->tib.nr_samples != samples))
+      return false;
 
    unsigned attachments = 0;
    for (unsigned i = 0; i < render->color_att_count; ++i) {
       if (render->color_att[i].resolve_mode != VK_RESOLVE_MODE_NONE)
          return false;
+      if (msaa && render->color_att[i].iview &&
+          render->color_att[i].iview->vk.image->samples != samples)
+         return false;
       attachments += render->color_att[i].iview != NULL;
    }
    if (render->depth_att.resolve_mode != VK_RESOLVE_MODE_NONE ||
        render->stencil_att.resolve_mode != VK_RESOLVE_MODE_NONE)
       return false;
+   if (msaa &&
+       ((render->depth_att.iview &&
+         render->depth_att.iview->vk.image->samples != samples) ||
+        (render->stencil_att.iview &&
+         render->stencil_att.iview->vk.image->samples != samples)))
+      return false;
    attachments += render->depth_att.iview != NULL;
    attachments += render->stencil_att.iview != NULL;
    return attachments > 0;
@@ -2811,7 +3905,8 @@ hk_m5_pass_merge_candidate(struct hk_cmd_buffer *cmd, const struct hk_cs *cs)
  * loaded, stored the same way, unresolved, with the same flags. */
 static const char *
 hk_m5_attachment_continues(const VkRenderingAttachmentInfo *info,
-                           const struct hk_attachment *att)
+                           const struct hk_attachment *att,
+                           unsigned expected_samples)
 {
    const bool present = info != NULL && info->imageView != VK_NULL_HANDLE;
    if (present != (att->iview != NULL))
@@ -2822,6 +3917,12 @@ hk_m5_attachment_continues(const VkRenderingAttachmentInfo *info,
    VK_FROM_HANDLE(hk_image_view, iview, info->imageView);
    if (iview != att->iview)
       return "view";
+   if (expected_samples > 1) {
+      if (iview->vk.image->samples != expected_samples)
+         return "sample count";
+      if (info->imageLayout != att->layout)
+         return "layout";
+   }
    if (info->loadOp != VK_ATTACHMENT_LOAD_OP_LOAD)
       return "load op";
    if (info->storeOp != att->store_op)
@@ -2900,12 +4001,19 @@ hk_m5_pass_merge_classify(struct hk_cmd_buffer *cmd, const char *what,
     * image's accesses, as the barrier's scopes describe. A transition of an
     * attachment, or an ownership transfer, changes what the continued
     * instance would load. */
+   const enum agx_chip chip = hk_cmd_buffer_device(cmd)->dev.chip;
    if (transition && image &&
-       hk_cmd_buffer_device(cmd)->dev.chip == AGX_CHIP_G17P &&
+       (chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip)) &&
        !hk_m5_image_is_attachment(&m->render, image) &&
        !(image->create_flags & (VK_IMAGE_CREATE_ALIAS_BIT |
                                 VK_IMAGE_CREATE_SPARSE_BINDING_BIT)))
       transition = false;
+   if (m->render.tilebuffer.nr_samples > 1 && transition) {
+      m->valid = false;
+      hk_frame_anatomy_note(hk_cmd_buffer_device(cmd),
+                            "MSAA pass merge ended by a layout or ownership transition");
+      return;
+   }
 
    switch (hk_m5_pass_merge_dependency(src, dst, dst_access, buffer,
                                        transition)) {
@@ -2913,10 +4021,21 @@ hk_m5_pass_merge_classify(struct hk_cmd_buffer *cmd, const char *what,
       return;
    case HK_M5_MERGE_DEP_ORDERED:
       if (!m->ordered)
+         hk_frame_anatomy_note(hk_cmd_buffer_device(cmd),
+                               "pass-merge ordered: %s barrier src=0x%" PRIx64
+                               " dst=0x%" PRIx64 " dst_access=0x%" PRIx64,
+                               what, (uint64_t)src, (uint64_t)dst,
+                               (uint64_t)dst_access);
       m->ordered = true;
       return;
    case HK_M5_MERGE_DEP_END:
       m->valid = false;
+      hk_frame_anatomy_note(hk_cmd_buffer_device(cmd),
+                            "pass-merge end: %s barrier src=0x%" PRIx64
+                            " dst=0x%" PRIx64 " dst_access=0x%" PRIx64
+                            " with a layout transition or ownership transfer",
+                            what, (uint64_t)src, (uint64_t)dst,
+                            (uint64_t)dst_access);
       return;
    }
 }
@@ -3022,12 +4141,13 @@ hk_m5_layout_excludes_descriptors(VkImageLayout layout)
 /* Whether no descriptor can access an attachment of the instance while it
  * renders: every attachment is in an attachment-only layout. Both instances
  * of a continuation use the same layouts (hk_m5_ordered_attachment_refusal()),
- * so this holds for the merged pass. Taken on G17P. */
+ * so this holds for the merged pass. Taken on G17P and M5. */
 static bool
 hk_m5_attachments_exclude_descriptors(const struct hk_cmd_buffer *cmd,
                                       const VkRenderingInfo *info)
 {
-   if (hk_cmd_buffer_device(cmd)->dev.chip != AGX_CHIP_G17P)
+   const enum agx_chip chip = hk_cmd_buffer_device(cmd)->dev.chip;
+   if (chip != AGX_CHIP_G17P && !agx_chip_g17_m5(chip))
       return false;
 
    const VkRenderingAttachmentInfo *ds[] = {info->pDepthAttachment,
@@ -3204,6 +4324,7 @@ hk_m5_ordered_draw_hazard(struct hk_cmd_buffer *cmd)
 static bool
 hk_m5_pass_merge_begin(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info)
 {
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
    struct hk_pass_merge_candidate *m = &cmd->state.pass_merge;
    struct hk_cs *cs = m->cs;
    const struct hk_rendering_state *prev = &m->render;
@@ -3241,23 +4362,42 @@ hk_m5_pass_merge_begin(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info)
             info->layerCount != prev->layer_count ||
             info->viewMask != prev->view_mask ||
             info->colorAttachmentCount != prev->color_att_count) {
+      hk_frame_anatomy_note(
+         dev,
+         "pass-merge end: geometry %d,%d %ux%u layers=%u views=0x%x rts=%u -> "
+         "%d,%d %ux%u layers=%u views=0x%x rts=%u",
+         prev->area.offset.x, prev->area.offset.y, prev->area.extent.width,
+         prev->area.extent.height, prev->layer_count, prev->view_mask,
+         prev->color_att_count, info->renderArea.offset.x,
+         info->renderArea.offset.y, info->renderArea.extent.width,
+         info->renderArea.extent.height, info->layerCount, info->viewMask,
+         info->colorAttachmentCount);
       return false;
    }
 
+   const char *slot = NULL;
    if (!why) {
+      slot = "depth";
       why = hk_m5_attachment_continues(info->pDepthAttachment,
-                                       &prev->depth_att);
+                                       &prev->depth_att,
+                                       prev->tilebuffer.nr_samples);
    }
    if (!why) {
+      slot = "stencil";
       why = hk_m5_attachment_continues(info->pStencilAttachment,
-                                       &prev->stencil_att);
+                                       &prev->stencil_att,
+                                       prev->tilebuffer.nr_samples);
    }
    for (uint32_t i = 0; !why && i < info->colorAttachmentCount; ++i) {
+      slot = "color";
       why = hk_m5_attachment_continues(&info->pColorAttachments[i],
-                                       &prev->color_att[i]);
+                                       &prev->color_att[i],
+                                       prev->tilebuffer.nr_samples);
    }
 
    if (why) {
+      hk_frame_anatomy_note(dev, "pass-merge end: %s%s%s", slot ? slot : "",
+                            slot ? " " : "", why);
       return false;
    }
 
@@ -3267,6 +4407,7 @@ hk_m5_pass_merge_begin(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info)
    if (ordered) {
       why = hk_m5_ordered_refusal(cmd, info, prev, cs);
       if (why) {
+         hk_frame_anatomy_note(dev, "pass-merge end: ordered, %s", why);
          return false;
       }
    }
@@ -3316,6 +4457,9 @@ hk_m5_pass_merge_begin(struct hk_cmd_buffer *cmd, const VkRenderingInfo *info)
       cmd->state.gfx.descriptors.root_dirty = true;
    }
 
+   hk_frame_anatomy_note(dev, "pass-merge continue: %ux%u draws_before=%u%s",
+                         render->cr.width, render->cr.height, cs->stats.cmds,
+                         ordered ? " ordered" : "");
    perf_debug(cmd, "Continuing the previous render pass instance");
    return true;
 }
@@ -3359,6 +4503,19 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
    hk_attachment_init(&render->depth_att, pRenderingInfo->pDepthAttachment);
    hk_attachment_init(&render->stencil_att, pRenderingInfo->pStencilAttachment);
 
+   if (unlikely(dev->frame_anatomy)) {
+      /* Frame anatomy profiler: compression state of each colour target. */
+      for (uint32_t i = 0; i < render->color_att_count; i++) {
+         const struct hk_image_view *view = render->color_att[i].iview;
+         if (!view)
+            continue;
+         const struct hk_image *image =
+            container_of(view->vk.image, struct hk_image, vk);
+         if (image->planes[view->planes[0].image_plane].layout.compressed)
+            render->cr.color_compressed_mask |= BITFIELD_BIT(i);
+      }
+   }
+
    for (uint32_t i = 0; i < render->color_att_count; i++) {
       hk_merge_render_iview(render, render->color_att[i].iview, false);
    }
@@ -3380,7 +4537,8 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
           * their producer even for a whole-tile render area: an earlier
           * attachment clear may have been lowered to compute. */
          preserve_loaded_color[nr_written] =
-            dev->dev.chip == AGX_CHIP_G17P &&
+            (dev->dev.chip == AGX_CHIP_G17P ||
+             agx_chip_g17_m5(dev->dev.chip)) &&
             pRenderingInfo->pColorAttachments[i].loadOp ==
                VK_ATTACHMENT_LOAD_OP_LOAD;
          written[nr_written++] = render->color_att[i].iview;
@@ -3415,7 +4573,8 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
    /* On G17P, zero dimensions are the native inactive-ZLS encoding.  Keep
     * them zero for attachmentless and colour-only rendering; every control
     * stream, including a partial stream, inherits that state from render. */
-   if (!render->cr.zls_width && !agx_native_graphics(&dev->dev)) {
+   if (!render->cr.zls_width && !agx_native_graphics(&dev->dev) &&
+       !agx_chip_g17_m5(dev->dev.chip)) {
       render->cr.zls_width = render->cr.width;
       render->cr.zls_height = render->cr.height;
    }
@@ -3447,11 +4606,11 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
    render->tilebuffer = hk_tilebuffer_layout(
       dev, &render->color_att[0].vk_format, sizeof(render->color_att[0]),
       render->color_att_count, render->tilebuffer.nr_samples,
-      !agx_native_graphics(&dev->dev) ||
+      (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) ||
          !hk_agx3_g17p_single_layer_rendering(render->view_mask, layer_count),
-      g17p_wide_ms_to_memory);
+      g17p_wide_ms_to_memory, hk_m5_resident_attachments(dev, render));
 
-   if (agx_native_graphics(&dev->dev))
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
       render->g17p_rt0_layout_supported =
          hk_agx3_g17p_rt0_tile_layout_supported(&render->tilebuffer);
 
@@ -3563,7 +4722,8 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
        * The stride is unused for one layer; Apple's G17P Framebuffer writes
        * 0 for a sub-cacheline layer and cacheline units below one page, a
        * form the G17P kernel's ZLS validation does not accept. Send 0. */
-      if (agx_native_graphics(&dev->dev) && stride_pages == 0) {
+      if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+          stride_pages == 0) {
          assert(z_layout->depth_px == 1);
          render->cr.depth.stride = 0;
       }
@@ -3618,7 +4778,8 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
              "Page aligned S layers");
       unsigned stride_pages = s_layout->layer_stride_B / AIL_PAGESIZE;
       render->cr.stencil.stride = ((stride_pages - 1) << 14) | 1;
-      if (agx_native_graphics(&dev->dev) && stride_pages == 0) {
+      if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+          stride_pages == 0) {
          assert(s_layout->depth_px == 1);
          render->cr.stencil.stride = 0;
       }
@@ -3649,10 +4810,29 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
    hk_pack_zls_control(&render->cr.zls_control_partial, z_layout, s_layout,
                        attach_z, attach_s, incomplete_render_area, true);
 
-   const bool agx3 = agx_chip_native_agx3(dev->dev.chip);
+   const bool agx3 = agx_chip_native_agx3(dev->dev.chip) ||
+                     agx_chip_g17_m5(dev->dev.chip);
    render->zls_precise = agx3 && z_layout != NULL;
    render->g17p_zls_touched = 0;
    render->g17p_zls_written = 0;
+   render->m5_attachments_reach_memory = false;
+   {
+      const struct hk_image_view *views[2 * HK_MAX_RTS + 4] = {
+         render->depth_att.iview, render->stencil_att.iview,
+         render->depth_att.resolve_iview, render->stencil_att.resolve_iview};
+      for (uint32_t i = 0; i < render->color_att_count; ++i) {
+         views[4 + 2 * i] = render->color_att[i].iview;
+         views[5 + 2 * i] = render->color_att[i].resolve_iview;
+      }
+      for (unsigned i = 0; i < 4 + 2 * render->color_att_count; ++i) {
+         const struct vk_image *image = views[i] ? views[i]->vk.image : NULL;
+         if (image && ((image->create_flags &
+                        (VK_IMAGE_CREATE_ALIAS_BIT |
+                         VK_IMAGE_CREATE_SPARSE_BINDING_BIT)) ||
+                       image->tiling == VK_IMAGE_TILING_LINEAR))
+            render->m5_attachments_reach_memory = true;
+      }
+   }
    render->g17p_zls_elide_mask = 0;
    if (agx3 && !(render->flags & (VK_RENDERING_SUSPENDING_BIT |
                                  VK_RENDERING_RESUMING_BIT))) {
@@ -3744,6 +4924,12 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
       perf_debug(cmd, "eMRT render pass");
 
       for (unsigned i = 0; i < render->color_att_count; ++i) {
+         /* Resident targets retain compression. Only memory-backed targets
+          * need their compressed contents converted before shader stores.
+          */
+         if (agx_chip_g17_m5(dev->dev.chip) && !render->tilebuffer.spilled[i])
+            continue;
+
          struct hk_image_view *view = render->color_att[i].iview;
          if (view) {
             if (!view->planes[0].emrt_pbe_valid) {
@@ -3782,19 +4968,45 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
                unsigned layer = view->vk.base_array_layer;
                uint64_t base = hk_image_base_address(image, image_plane);
 
-               struct libagx_decompress_images imgs;
-               memcpy(&imgs.compressed, &view->planes[plane].emrt_texture,
-                      sizeof(imgs.compressed));
-               memcpy(&imgs.uncompressed, &view->planes[plane].emrt_pbe,
-                      sizeof(imgs.uncompressed));
-
                struct agx_grid grid =
                   agx_3d(ail_metadata_width_tl(layout, level) * 32,
                          ail_metadata_height_tl(layout, level), layer_count);
 
-               libagx_decompress(cmd, grid, AGX_BARRIER_ALL | AGX_PREGFX,
-                                 layout, layer, level, base,
-                                 hk_pool_upload(cmd, &imgs, sizeof(imgs), 64));
+               if (agx_chip_g17_m5(dev->dev.chip)) {
+                  if (layout->format != PIPE_FORMAT_R16G16B16A16_FLOAT ||
+                      (layout->sample_count_sa != 2 &&
+                       layout->sample_count_sa != 4) ||
+                      layout->depth_px != 1 || layout->levels != 1 ||
+                      !dev->sampled_images.base_va || !dev->samplers.table.bo ||
+                      view->planes[plane].emrt_read_heap_index >=
+                         dev->sampled_images.alloc) {
+                     vk_command_buffer_set_error(&cmd->vk,
+                                                 VK_ERROR_FEATURE_NOT_PRESENT);
+                     return;
+                  }
+                  const struct ail_tile tile = layout->tilesize_el[level];
+                  libagx_m5_decompress_rgba16f(
+                     cmd, grid, AGX_BARRIER_ALL | AGX_PREGFX,
+                     dev->sampled_images.base_va,
+                     dev->samplers.table.bo->va->addr,
+                     base + layout->metadata_offset_B,
+                     base + ail_get_layer_level_B(layout, layer, level),
+                     view->planes[plane].emrt_read_heap_index,
+                     ail_metadata_width_tl(layout, level),
+                     ail_metadata_height_tl(layout, level),
+                     align(layout->width_px, tile.width_el),
+                     tile.width_el, tile.height_el,
+                     util_logbase2(layout->sample_count_sa) - 1);
+               } else {
+                  struct libagx_decompress_images imgs;
+                  memcpy(&imgs.compressed, &view->planes[plane].emrt_texture,
+                         sizeof(imgs.compressed));
+                  memcpy(&imgs.uncompressed, &view->planes[plane].emrt_pbe,
+                         sizeof(imgs.uncompressed));
+                  libagx_decompress(cmd, grid, AGX_BARRIER_ALL | AGX_PREGFX,
+                                    layout, layer, level, base,
+                                    hk_pool_upload(cmd, &imgs, sizeof(imgs), 64));
+               }
             }
          }
       }
@@ -3862,7 +5074,9 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
    if (clear_att[clear_count].aspectMask != 0)
       clear_count++;
 
-   const bool clear_spills = agx_native_graphics(&dev->dev) &&
+   const bool clear_spills =
+      (agx_native_graphics(&dev->dev) ||
+       agx_chip_g17_m5(dev->dev.chip)) &&
       agx_tilebuffer_spills(&render->tilebuffer);
 
    /* A G17P multisample tilebuffer wider than eight bytes per sample cannot
@@ -3901,6 +5115,8 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
       resolved_clear || resolved_clear_z || loaded_resolve;
    render->process_empty_tiles_without_z = resolved_clear || loaded_resolve;
    cs->cr.process_empty_tiles = process_empty_tiles;
+   if (unlikely(dev->frame_anatomy) && incomplete_render_area)
+      cs->anatomy_area = render->area;
    if (clear_count > 0 &&
        (incomplete_render_area || clear_spills || clear_wide_multisample)) {
       const VkClearRect clear_rect = {
@@ -3909,6 +5125,19 @@ hk_begin_rendering(VkCommandBuffer commandBuffer,
          .layerCount = render->view_mask ? 1 : render->layer_count,
       };
 
+      /* The clear draw covers the whole render area, so every tile it has
+       * to clear is non-empty. A tile outside the area takes no draw and has
+       * nothing to clear: processing it only loads and stores it back
+       * unchanged (and an incomplete render area always loads). On M5 that
+       * was a whole-framebuffer load/store round trip, ~50 us for a 1x
+       * RGBA8 3024x1964 target, for a clear of a small HUD rectangle. Only
+       * a resolve of loaded samples still writes its destination from empty
+       * tiles. */
+      if (agx_chip_g17_m5(dev->dev.chip)) {
+         render->process_empty_tiles_without_z = loaded_resolve;
+         cs->cr.process_empty_tiles = loaded_resolve;
+      }
+
       hk_CmdClearAttachments(hk_cmd_buffer_to_handle(cmd), clear_count,
                              clear_att, 1, &clear_rect);
    } else {
@@ -4115,8 +5344,10 @@ hk_CmdEndRendering2KHR(VkCommandBuffer commandBuffer,
     * load the earlier pass's stored colour without an application barrier.
     * The native dependency tracker otherwise permits full-tile passes to
     * overlap. Retain upstream ordering on AGX2, including across command
-    * buffers/submissions through the tracker's prior requirements. */
-   if (!agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev))
+    * buffers/submissions through the tracker's prior requirements. M5
+    * schedules its render passes like native AGX3. */
+   if (!agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) &&
+       !agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip))
       hk_sync_barrier(cmd, HK_SYNC_GFX, HK_SYNC_GFX);
 
    if (continuable && !need_resolve && cs->end == end_before &&
@@ -4446,6 +5677,10 @@ hk_upload_geometry_params(struct hk_cmd_buffer *cmd, struct agx_draw draw)
          params.prims_generated_counter[i] = AGX_SCRATCH_PAGE_ADDRESS;
       }
 
+      const uint64_t generated = hk_prims_generated_addr(cmd, i);
+      params.query_prims_generated_counter[i] =
+         generated ? generated : AGX_SCRATCH_PAGE_ADDRESS;
+
       /* TODO: Optimize out? */
       params.xfb_overflow[i] = AGX_SCRATCH_PAGE_ADDRESS;
    }
@@ -4595,7 +5830,7 @@ hk_upload_tess_params(struct hk_cmd_buffer *cmd, struct poly_tess_params *out,
 
 static struct hk_api_shader *
 hk_build_meta_shader_locked(struct hk_device *dev, struct hk_internal_key *key,
-                            hk_internal_builder_t builder)
+                            hk_internal_builder_t builder, VkResult *result)
 {
    /* Try to get the cached shader */
    struct hash_entry *ent = _mesa_hash_table_search(dev->kernels.ht, key);
@@ -4622,7 +5857,8 @@ hk_build_meta_shader_locked(struct hk_device *dev, struct hk_internal_key *key,
    hk_preprocess_nir_internal(dev->vk.physical, b.shader);
 
    struct hk_api_shader *s;
-   if (hk_compile_shader(dev, &info, NULL, NULL, NULL, &s) != VK_SUCCESS)
+   *result = hk_compile_shader(dev, &info, NULL, NULL, NULL, &s);
+   if (*result != VK_SUCCESS)
       return NULL;
 
    /* ..and cache it before we return. The key is on the stack right now, so
@@ -4636,10 +5872,14 @@ hk_build_meta_shader_locked(struct hk_device *dev, struct hk_internal_key *key,
    return s;
 }
 
+/* On failure returns NULL and sets *result to the compile error, which can be
+ * an out-of-memory error under memory pressure.
+ */
 struct hk_api_shader *
 hk_meta_shader(struct hk_device *dev, hk_internal_builder_t builder, void *data,
-               size_t data_size)
+               size_t data_size, VkResult *result)
 {
+   *result = VK_SUCCESS;
    size_t total_key_size = sizeof(struct hk_internal_key) + data_size;
 
    struct hk_internal_key *key = alloca(total_key_size);
@@ -4651,7 +5891,8 @@ hk_meta_shader(struct hk_device *dev, hk_internal_builder_t builder, void *data,
       memcpy(key->key, data, data_size);
 
    simple_mtx_lock(&dev->kernels.lock);
-   struct hk_api_shader *s = hk_build_meta_shader_locked(dev, key, builder);
+   struct hk_api_shader *s =
+      hk_build_meta_shader_locked(dev, key, builder, result);
    simple_mtx_unlock(&dev->kernels.lock);
 
    return s;
@@ -4792,7 +6033,7 @@ hk_launch_gs_prerast(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
       grid_vs = agx_1d(gfx->tess.tes_direct_threads);
 
    /* Launch the vertex shader first */
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       hk_dispatch_with_local_size(cmd, cs, vs, grid_vs, wg);
    } else {
       hk_reserve_scratch(cmd, cs, vs);
@@ -4816,7 +6057,8 @@ hk_launch_gs_prerast(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
 
    struct hk_root_descriptor_table *root = &cmd->state.gfx.descriptors.root;
    bool xfb_or_queries =
-      main->info.gs.xfb || (root->draw.pipeline_stats_flags & gs_queries);
+      main->info.gs.xfb || (root->draw.pipeline_stats_flags & gs_queries) ||
+      hk_prims_generated_active(cmd);
 
    if (xfb_or_queries) {
       /* If we need counts, launch the count shader and prefix sum the results. */
@@ -4900,7 +6142,8 @@ hk_tes_direct_query_threads(struct hk_cmd_buffer *cmd, struct agx_draw draw)
 
 static __attribute__((hot)) struct agx_draw
 hk_launch_tess(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
-               struct agx_draw draw, uint64_t c_prims, uint64_t c_inv)
+               struct agx_draw draw, uint64_t c_prims, uint64_t c_inv,
+               uint64_t prims_generated)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    struct hk_graphics_state *gfx = &cmd->state.gfx;
@@ -4962,7 +6205,7 @@ hk_launch_tess(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
       }
    }
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       hk_dispatch_with_local_size(cmd, cs, vs, grid_vs,
                                   agx_workgroup(64, 1, 1));
       hk_dispatch_with_local_size(
@@ -4988,7 +6231,8 @@ hk_launch_tess(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
                      info.mode, POLY_TESS_MODE_COUNT, state);
    libagx_prefix_sum_tess(cmd, agx_1d(1024),
                           AGX_BARRIER_ALL | AGX_PREGFX, state, c_prims,
-                          c_inv, c_prims || c_inv);
+                          c_inv, prims_generated,
+                          c_prims || c_inv || prims_generated);
    libagx_tessellate(cmd, grid_tess, AGX_BARRIER_ALL | AGX_PREGFX,
                      info.mode, POLY_TESS_MODE_WITH_COUNTS, state);
 
@@ -5353,7 +6597,9 @@ hk_get_prolog_epilog_locked(struct hk_device *dev, struct hk_internal_key *key,
    if (builder == build_g17p_fs_epilog)
       backend_key.g17p_emrt =
          ((const struct agx_fs_epilog_key *)key->key)->native_spills;
-   if (builder == build_g17p_fs_epilog) {
+   if (builder == build_g17p_fs_epilog ||
+       (agx_chip_g17_m5(dev->dev.chip) &&
+        builder == agx_nir_fs_epilog && nr_samples > 1)) {
       /* Preprocessing can unroll any multisample epilog, not only eMRT.
        * Its induction masks then become literals. Materialize the native
        * coverage operands after that transformation, before binary packing. */
@@ -5454,7 +6700,8 @@ hk_get_fast_linked_locked_vs(struct hk_device *dev, struct hk_shader *shader,
 {
    struct agx_shader_part *prolog =
       hk_get_prolog_epilog(dev, &key->prolog, sizeof(key->prolog),
-                           agx_native_graphics(&dev->dev)
+                           (agx_native_graphics(&dev->dev) ||
+                            agx_chip_g17_m5(dev->dev.chip))
                               ? agx_nir_vs_prolog_raw : agx_nir_vs_prolog,
                            false, false, 0, 0, 0);
    if (!prolog)
@@ -5529,11 +6776,16 @@ build_g17p_fs_epilog(nir_builder *b, const void *key)
  */
 static const char *
 hk_g17p_mono_layout_mismatch(const struct agx_shader_part *a,
-                             const struct agx_shader_part *m)
+                             const struct agx_shader_part *m, bool own_layout)
 {
 #define HK_MONO_FIELD(x)                                                       \
    if (a->info.x != m->info.x)                                                 \
       return #x;
+   /* A program with its own constant program launches with its own uniform
+    * layout (hk_linked_shader::g17p_layout): only the resources and varyings
+    * the draw still takes from the API shader must agree. */
+   if (own_layout)
+      goto resources;
    /* The draw loads the main's whole uniform window, so the monolithic
     * program may use a prefix of it: every uniform it reads must mean the
     * same thing. Two parts of the window are placed by the compile itself:
@@ -5598,6 +6850,7 @@ hk_g17p_mono_layout_mismatch(const struct agx_shader_part *a,
                  a->info.preamble_size))
          return "preamble code";
    }
+resources:
    HK_MONO_FIELD(texture_state_count)
    HK_MONO_FIELD(sampler_state_count)
    HK_MONO_FIELD(uses_txf)
@@ -5667,6 +6920,15 @@ hk_g17p_mono_resolve_exports(nir_shader *nir)
    return true;
 }
 
+/* Whether an inline-blend program carries its own constant program, which
+ * only a program built for an API shader without one does. */
+static bool
+hk_g17p_mono_own_layout(const struct hk_shader *shader,
+                        const struct agx_shader_part *part)
+{
+   return part->info.has_preamble && !shader->b.info.has_preamble;
+}
+
 /* Every G17P fragment program that touches the tile opens the pixel
  * transaction with the entry gate and ends it with a final release
  * (PIXREL with 0x200) just before END. The packer only emits that
@@ -5688,7 +6950,10 @@ hk_g17p_mono_part_refusal(const struct hk_shader *shader,
    if (!has_gate || !final_release)
       return "pixel protocol not proven";
 
-   return hk_g17p_mono_layout_mismatch(&shader->b, part);
+   const bool own_layout = hk_g17p_mono_own_layout(shader, part);
+   if (own_layout && !hk_g17p_native_cprog_qualified(&part->info))
+      return "constant program outside the native protocol";
+   return hk_g17p_mono_layout_mismatch(&shader->b, part, own_layout);
 }
 
 /* An inline-blend program is a function of the main's NIR and compile key,
@@ -5707,7 +6972,7 @@ hk_g17p_mono_disk_key(struct hk_device *dev, const struct hk_shader *shader,
    if (partitions)
       memcpy(parts, partitions, sizeof(parts));
 
-   const char magic[8] = "hkmono2";
+   const char magic[8] = "hkmono3";
    const size_t rodata_size =
       shader->b.binary ? shader->b.info.rodata.size_16 * 2 : 0;
    const size_t size = sizeof(magic) + sizeof(shader->g17p_mono_key) +
@@ -5745,6 +7010,8 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
    /* A refusal of these inputs is final: remember it with the programs. */
 #define HK_MONO_REFUSE(why)                                                    \
    do {                                                                        \
+      if (shader->info.fs.m5_inline_sample_shading)                             \
+         mesa_loge("M5 inline sample shading: %s", why);                        \
       final_refusal = true;                                                    \
       goto fail;                                                               \
    } while (0)
@@ -5761,7 +7028,9 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
    if (!mem)
       return NULL;
    const unsigned main_samples = MAX2(shader->g17p_mono_key.fs.nr_samples, 1);
-   if (ekey->native_spills || ekey->link.sample_shading ||
+   const bool inline_samples = shader->info.fs.m5_inline_sample_shading;
+   if ((ekey->native_spills && !inline_samples) ||
+       (ekey->link.sample_shading && !inline_samples) ||
        MAX2(ekey->nr_samples, 1) != main_samples)
       HK_MONO_REFUSE("epilog shape");
 
@@ -5778,7 +7047,13 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
          if (refused)
             HK_MONO_FAIL("refused before");
          if (part) {
-            if (hk_g17p_mono_part_refusal(shader, part))
+            struct agx_shader_part expected = shader->b;
+            if (inline_samples && ekey->native_spills)
+               expected.info.push_count = MAX2(expected.info.push_count,
+                                               ekey->emrt_heap_base + 4);
+            if (inline_samples
+                   ? hk_g17p_mono_layout_mismatch(&expected, part, false)
+                   : hk_g17p_mono_part_refusal(shader, part))
                HK_MONO_FAIL("cached program");
             ralloc_free(mem);
             return part;
@@ -5802,11 +7077,43 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
       }
    }
 
+   /* A main that exports only uniform values (say, a colour from push
+    * constants) leaves every source term of the blend uniform: its
+    * clamps and factor products are the same for every pixel. Compute them
+    * once in the constant program and blend with one FMA per channel by
+    * letting this program form its own
+    * constant program when the API shader has none (a push-constant-only
+    * fragment main forms none, see hk_compile_nir()). Only chips whose
+    * native constant program is qualified form one. */
+   bool uniform_exports = agx_chip_native_preamble(dev->dev.chip) &&
+                          shader->g17p_mono_key.no_preamble;
+   if (uniform_exports) {
+      nir_divergence_analysis(nir);
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type == nir_instr_type_intrinsic &&
+                nir_instr_as_intrinsic(instr)->intrinsic ==
+                   nir_intrinsic_export_agx &&
+                nir_instr_as_intrinsic(instr)->src[0].ssa->divergent)
+               uniform_exports = false;
+         }
+      }
+   }
+
    /* The epilog, exactly as the split path builds it. */
    eb = nir_builder_init_simple_shader(0, &agx_nir_options, NULL);
-   build_g17p_fs_epilog(&eb, ekey);
+   if (inline_samples) {
+      struct agx_fs_epilog_key pixel = *ekey;
+      pixel.link.sample_shading = false;
+      pixel.defer_monolithic_msaa = true;
+      agx_nir_fs_epilog(&eb, &pixel);
+      agx_nir_lower_sample_exports_to_end(nir);
+   } else {
+      build_g17p_fs_epilog(&eb, ekey);
+   }
    agx_preprocess_nir(eb.shader);
-   NIR_PASS(_, eb.shader, agx_nir_lower_g17p_tile_samples, ekey->nr_samples);
+   if (!inline_samples)
+      NIR_PASS(_, eb.shader, agx_nir_lower_g17p_tile_samples, ekey->nr_samples);
 
    /* Epilog code reading uniforms (blend constants and the like) would
     * need uniforms the main's layout does not reserve. */
@@ -5815,6 +7122,13 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
          if (instr->type != nir_instr_type_intrinsic)
             continue;
          const nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;
+         if (inline_samples && ekey->native_spills &&
+             (op == nir_intrinsic_bindless_image_agx ||
+              op == nir_intrinsic_bindless_image_store ||
+              (op == nir_intrinsic_load_preamble &&
+               nir_intrinsic_base(nir_instr_as_intrinsic(instr)) ==
+                  ekey->emrt_heap_base)))
+            continue;
          if (op == nir_intrinsic_load_preamble ||
              op == nir_intrinsic_load_blend_const_color_rgba ||
              op == nir_intrinsic_load_blend_const_color_r_float ||
@@ -5850,11 +7164,31 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
    NIR_PASS(_, nir, nir_opt_copy_prop);
    NIR_PASS(_, nir, nir_opt_constant_folding);
    NIR_PASS(_, nir, nir_opt_dce);
+   if (inline_samples) {
+      struct agx_g17p_colour_layout layout;
+      agx_g17p_colour_layout_init(&layout, ekey->link.loc_written,
+                                  ekey->link.size_32, ekey->link.colour_comps,
+                                  ekey->link.packed_colour);
+      NIR_PASS(_, nir, agx_nir_lower_to_per_sample);
+      NIR_PASS(_, nir, agx_nir_unroll_g17p_samples_layout,
+               ekey->nr_samples, &layout);
+      bool progress;
+      do {
+         progress = false;
+         NIR_PASS(progress, nir, nir_opt_constant_folding);
+         NIR_PASS(progress, nir, nir_opt_algebraic);
+      } while (progress);
+      NIR_PASS(_, nir, agx_nir_hoist_g17p_zs_writes);
+      NIR_PASS(_, nir, agx_nir_lower_g17p_tile_samples, ekey->nr_samples);
+      nir->info.fs.uses_sample_shading = false;
+   }
 
    struct agx_shader_key key = shader->g17p_mono_key;
+   if (inline_samples)
+      key.fs.inside_sample_loop = false;
    key.no_stop = false;
    key.secondary = false;
-   key.g17p_emrt = false;
+   key.g17p_emrt = inline_samples && ekey->native_spills;
    if (shader->b.info.rodata.size_16 && shader->b.binary) {
       key.g17p_rodata_hint =
          (const uint16_t *)((const uint8_t *)shader->b.binary +
@@ -5866,17 +7200,49 @@ hk_g17p_inline_epilog_part(struct hk_device *dev, struct hk_shader *shader,
       memcpy(key.g17p_tile_partitions, partitions,
              sizeof(key.g17p_tile_partitions));
 
-   part = rzalloc(NULL, struct agx_shader_part);
-   if (!part)
-      HK_MONO_FAIL("oom");
-   if (!agx_compile_shader_nir(nir, &key, part)) {
+   /* With uniform exports, first try the program with its own constant
+    * program; if the backend or the native protocol refuses it, build the
+    * program exactly as without. */
+   nir_shader *without = NULL;
+   if (uniform_exports) {
+      without = nir_shader_clone(mem, nir);
+      if (!without)
+         HK_MONO_FAIL("oom");
+      key.no_preamble = false;
+      key.g17p_native_uniforms = true;
+   }
+
+   /* An M5 inline-samples program keeps the main's layout (its emulated
+    * render target heap may extend the push window); any other program must
+    * also follow the pixel protocol (hk_g17p_mono_part_refusal()). Uniform
+    * exports are G17P-only (agx_chip_native_preamble()), so an inline-samples
+    * program is compiled once. */
+   struct agx_shader_part expected = shader->b;
+   if (inline_samples && ekey->native_spills)
+      expected.info.push_count = MAX2(expected.info.push_count,
+                                      ekey->emrt_heap_base + 4);
+   const char *why;
+   for (;;) {
+      part = rzalloc(NULL, struct agx_shader_part);
+      if (!part)
+         HK_MONO_FAIL("oom");
       /* agx_compile_shader_nir frees nothing it did not allocate. */
-      nir = NULL;
-      HK_MONO_REFUSE("backend refused the monolithic program");
-   }
-   nir = NULL; /* consumed */
+      const bool compiled = agx_compile_shader_nir(nir, &key, part);
+      nir = NULL; /* consumed */
+      why = !compiled ? "backend refused the monolithic program"
+            : inline_samples ? hk_g17p_mono_layout_mismatch(&expected, part, false)
+                             : hk_g17p_mono_part_refusal(shader, part);
+      if (!why || !without)
+         break;
 
-   const char *why = hk_g17p_mono_part_refusal(shader, part);
+      free(part->binary);
+      ralloc_free(part);
+      part = NULL;
+      nir = without;
+      without = NULL;
+      key.no_preamble = shader->g17p_mono_key.no_preamble;
+      key.g17p_native_uniforms = shader->g17p_mono_key.g17p_native_uniforms;
+   }
    if (why)
       HK_MONO_REFUSE(why);
 
@@ -5934,12 +7300,19 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader,
    /* A G17P epilog selects each written target's pixel classes from its
     * position among all of the framebuffer's colour targets. */
    BITSET_DECLARE(g17p_tile_partitions, AGX_G17P_MAX_BYTES_PER_SAMPLE) = {0};
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       const struct agx_tilebuffer_layout tib =
-         agx_build_tilebuffer_layout_with_limits(
-            key->epilog.rt_formats, ARRAY_SIZE(key->epilog.rt_formats),
-            key->epilog.nr_samples, true, key->epilog.max_sample_size_B,
-            key->epilog.max_bytes_per_tile, key->epilog.max_samples_per_tile);
+         key->epilog.native_imageblock
+            ? agx_build_tilebuffer_layout_resident(
+                 key->epilog.rt_formats, ARRAY_SIZE(key->epilog.rt_formats),
+                 key->epilog.nr_samples, true, key->epilog.max_sample_size_B,
+                 key->epilog.max_bytes_per_tile,
+                 key->epilog.max_samples_per_tile, key->epilog.resident_mask)
+            : agx_build_tilebuffer_layout_with_limits(
+                 key->epilog.rt_formats, ARRAY_SIZE(key->epilog.rt_formats),
+                 key->epilog.nr_samples, true, key->epilog.max_sample_size_B,
+                 key->epilog.max_bytes_per_tile,
+                 key->epilog.max_samples_per_tile);
       for (unsigned rt = 0; rt < ARRAY_SIZE(tib.logical_format); ++rt) {
          if (tib.logical_format[rt] != PIPE_FORMAT_NONE && !tib.spilled[rt])
             BITSET_SET(g17p_tile_partitions, tib._offset_B[rt]);
@@ -5950,20 +7323,37 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader,
 
    /* Inline blend: try one monolithic fragment program first;
     * any refusal falls through to the split link below, unchanged. */
-   if (agx_native_graphics(&dev->dev) &&
-       (agx_get_fs_structure_flags() & AGX_FS_STRUCTURE_INLINE_BLEND) &&
+   if ((shader->info.fs.m5_inline_sample_shading ||
+        (agx_native_graphics(&dev->dev) &&
+         (agx_get_fs_structure_flags() & AGX_FS_STRUCTURE_INLINE_BLEND))) &&
        !prolog && key->nr_samples_shaded == 0 && shader->g17p_mono_nir) {
       struct agx_shader_part *mono = hk_g17p_inline_epilog_part(
          dev, shader, &key->epilog, g17p_tile_partitions);
       if (mono) {
          struct hk_fast_link_key *mono_key_clone =
             ralloc_memdup(shader->linked.ht, key, sizeof(*key));
-         struct hk_linked_shader *mono_linked =
-            mono_key_clone ? hk_fast_link_part(dev, true, shader, mono, NULL,
-                                               NULL, 0)
-                           : NULL;
+         /* A program with its own constant program brings its own uniform
+          * layout; link it as the main of a shader describing that layout. */
+         struct hk_shader *layout =
+            mono_key_clone && hk_g17p_mono_own_layout(shader, mono)
+               ? hk_g17p_layout_shader_create(dev, shader, mono)
+               : NULL;
+         struct hk_linked_shader *mono_linked = NULL;
+         if (layout)
+            mono_linked = hk_fast_link_part(dev, true, layout, NULL, NULL,
+                                            NULL, 0);
+         else if (mono_key_clone && !hk_g17p_mono_own_layout(shader, mono))
+            mono_linked = hk_fast_link_part(dev, true, shader, mono, NULL,
+                                            NULL, 0);
          free(mono->binary);
          ralloc_free(mono);
+         if (mono_linked)
+            mono_linked->g17p_layout = layout;
+         else if (layout) {
+            free(layout->b.binary);
+            agx_bo_unreference(&dev->dev, layout->bo);
+            free(layout);
+         }
          if (mono_linked) {
             mono_linked->g17p_blend_constant_base =
                key->epilog.blend_constant_base;
@@ -5977,13 +7367,22 @@ hk_get_fast_linked_locked_fs(struct hk_device *dev, struct hk_shader *shader,
       }
    }
 
+   if (shader->info.fs.m5_inline_sample_shading) {
+      mesa_loge("M5: required inline sample shading could not be linked");
+      return NULL;
+   }
+
    struct agx_shader_part *epilog =
       hk_get_prolog_epilog(dev, &key->epilog, sizeof(key->epilog),
-                           agx_native_graphics(&dev->dev)
+                           (agx_native_graphics(&dev->dev) ||
+                            (agx_chip_g17_m5(dev->dev.chip) &&
+                             (key->epilog.link.sample_shading ||
+                              key->epilog.native_spills)))
                               ? build_g17p_fs_epilog
                               : agx_nir_fs_epilog,
                            true, epilog_stop, 0, g17p_tile_partitions,
-                           agx_native_graphics(&dev->dev)
+                           (agx_native_graphics(&dev->dev) ||
+                            agx_chip_g17_m5(dev->dev.chip))
                               ? key->epilog.nr_samples
                               : 0);
    if (!epilog) {
@@ -6313,8 +7712,8 @@ hk_emit_hal300_vp_state(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
     * render-state path emits the viewport update before the base update. */
    uint8_t records[2 * AGX_PPP_STATE_LENGTH];
    uint8_t *record = records;
-   if (!agx_ppp_emit_state(chip, &record, viewport_gpu, viewport.length) ||
-       !agx_ppp_emit_state(chip, &record, bases_gpu, bases.length) ||
+   if (!hk_ppp_emit_state(chip, &record, viewport_gpu, viewport.length) ||
+       !hk_ppp_emit_state(chip, &record, bases_gpu, bases.length) ||
        record != records + sizeof(records))
       return false;
 
@@ -6356,6 +7755,9 @@ hk_flush_vp_state(struct hk_cmd_buffer *cmd, struct hk_cs *cs, uint8_t **out)
    const bool hal300 =
       agx_chip_command_stream_generation(dev->dev.chip) ==
       AGX_CS_GENERATION_HAL300;
+   /* The viewport records are PPP state too: re-emit the graphics core after
+    * them rather than rely on what the stream latched before. */
+   cs->m5_ppp_emitted_valid = false;
    const struct vk_dynamic_graphics_state *dyn =
       &cmd->vk.dynamic_graphics_state;
 
@@ -6405,6 +7807,14 @@ hk_flush_vp_state(struct hk_cmd_buffer *cmd, struct hk_cs *cs, uint8_t **out)
       maxy[i] = MIN2(maxy[i], s->offset.y + s->extent.height);
    }
 
+   if (hal300) {
+      /* A disjoint intersection is empty, not an inverted 16-bit rectangle. */
+      for (unsigned i = 0; i < count; ++i) {
+         minx[i] = MIN2(minx[i], maxx[i]);
+         miny[i] = MIN2(miny[i], maxy[i]);
+      }
+   }
+
    /* Upload a hardware scissor for each viewport, whether there's a
     * corresponding API scissor or not.
     */
@@ -6691,6 +8101,9 @@ hk_g17p_build_native_bind_tail(struct hk_cmd_buffer *cmd,
    struct hk_linked_shader *linked_fs = gfx->linked[MESA_SHADER_FRAGMENT];
    if (!fs || !linked_fs)
       return hk_g17p_bind_tail_refused(__LINE__);
+   /* The launch follows the linked program's own layout, if it has one. */
+   if (linked_fs->g17p_layout)
+      fs = linked_fs->g17p_layout;
 
    const uint64_t fragment_program = hk_upload_usc_words(cmd, fs, linked_fs);
    if (!fragment_program)
@@ -6941,26 +8354,16 @@ hk_g17p_build_native_bind_tail(struct hk_cmd_buffer *cmd,
       return hk_g17p_bind_tail_refused(__LINE__);
 
    struct agx_ppp_w_clamp_cull2_core w_cull2_record;
-   /* Only G17P mesh draws consult the environment: this runs for every
-    * draw that publishes vertex state. */
-   const bool g17p_mesh = dev->dev.chip == AGX_CHIP_G17P &&
-                          vs->info.stage == MESA_SHADER_MESH;
-   const char *native_mesh_primitive_address =
-      g17p_mesh ? getenv("ASAHI_MESH_NATIVE_PRIMITIVE_ADDRESS_DIAGNOSTIC")
-                : NULL;
    const bool native_mesh_primitive_address_enabled =
-      native_mesh_primitive_address &&
-      !strcmp(native_mesh_primitive_address, "1") &&
       dev->dev.chip == AGX_CHIP_G17P &&
       vs->info.stage == MESA_SHADER_MESH &&
       vs->info.uvs.mesh_primitive_page &&
-      vs->info.mesh.primitive_stride_B != 0;
-   const char *native_mesh_wclamp =
-      g17p_mesh ? getenv("ASAHI_MESH_NATIVE_M5_WCLAMP_DIAGNOSTIC") : NULL;
+      vs->info.mesh.primitive_stride_B != 0 &&
+      debug_get_option_mesh_native_primitive_address();
    const struct agx_ppp_w_clamp_cull2_fields w_cull2_fields = {
-      .w_clamp_raw = native_mesh_wclamp && !strcmp(native_mesh_wclamp, "1") &&
-                     dev->dev.chip == AGX_CHIP_G17P &&
-                     vs->info.stage == MESA_SHADER_MESH
+      .w_clamp_raw = dev->dev.chip == AGX_CHIP_G17P &&
+                     vs->info.stage == MESA_SHADER_MESH &&
+                     debug_get_option_mesh_native_m5_wclamp()
                         ? UINT32_C(0x00840002) : 0,
       /* The generated primitive-ID coefficient needs its rasterizer input
        * enabled as well as the matching VDM producer and output slot. */
@@ -7465,14 +8868,14 @@ hk_provoking_selector(const struct vk_dynamic_graphics_state *dyn,
 }
 
 /*
- * Return the given sample positions, packed into a 32-bit word with fixed
- * point nibbles for each x/y component of the (at most 4) samples. This is
+ * Return the given sample positions, packed into a 64-bit word with fixed
+ * point nibbles for each x/y component of the (at most 8) samples. This is
  * suitable for programming the PPP_MULTISAMPLECTL control register.
  */
-static uint32_t
+static uint64_t
 hk_pack_ppp_multisamplectrl(const struct vk_sample_locations_state *sl)
 {
-   uint32_t ctrl = 0;
+   uint64_t ctrl = 0;
 
    for (int32_t i = sl->per_pixel - 1; i >= 0; i--) {
       VkSampleLocationEXT loc = sl->locations[i];
@@ -7493,7 +8896,7 @@ hk_pack_ppp_multisamplectrl(const struct vk_sample_locations_state *sl)
 /*
  * Return the standard sample positions, prepacked as above for efficiency.
  */
-uint32_t
+uint64_t
 hk_default_sample_positions(unsigned nr_samples)
 {
    switch (nr_samples) {
@@ -7504,6 +8907,8 @@ hk_default_sample_positions(unsigned nr_samples)
       return 0x44cc;
    case 4:
       return 0xeaa26e26;
+   case 8:
+      return UINT64_C(0x1ffb71d3359db759);
    default:
       UNREACHABLE("Invalid sample count");
    }
@@ -7520,12 +8925,437 @@ hk_flush_ppp_state(struct hk_cmd_buffer *cmd, struct hk_cs *cs, uint8_t **out)
    struct hk_shader *hw_vs = hk_bound_hw_vs(gfx);
    struct hk_shader *fs = hk_bound_fs(cmd);
 
+   if (agx_chip_g17_m5(dev->dev.chip)) {
+      struct hk_linked_shader *linked = gfx->linked[MESA_SHADER_FRAGMENT];
+      /* With no fragment shader, attachments, or occlusion query, only
+       * vertex processing is observable. Keep the vertex program live and
+       * select the inert fragment state, including for transform feedback.
+       */
+      const bool vertex_only = gfx->shaders[MESA_SHADER_FRAGMENT] == dev->null_fs &&
+         !render->tilebuffer.sample_size_B && !render->depth_att.iview &&
+         !render->stencil_att.iview && !gfx->occlusion.mode;
+      const bool discard = hk_rast_discard(cmd) || vertex_only;
+      const VkViewport *vp = &dyn->vp.viewports[0];
+      const VkRect2D *scissor = &dyn->vp.scissors[0];
+      const struct agx_unlinked_uvs_layout *uvs = hw_vs ? &hw_vs->info.uvs : NULL;
+      const bool psiz = uvs && uvs->group_offs[UVS_LAYER_VIEWPORT] ==
+                                 uvs->group_offs[UVS_PSIZ] + 1;
+      const unsigned clip_words = uvs && uvs->size >= uvs->group_offs[UVS_CLIP_DIST]
+         ? uvs->size - uvs->group_offs[UVS_CLIP_DIST] : 0;
+      const unsigned components = uvs ? uvs->user_size : 0;
+      const bool layer_word = uvs && uvs->group_offs[UVS_CLIP_DIST] ==
+                                    uvs->group_offs[UVS_LAYER_VIEWPORT] + 1;
+
+      const bool vertex_layout = uvs && components <= 124 &&
+         uvs->size <= 128 && clip_words <= 8 &&
+         uvs->group_offs[UVS_POSITION] == 0 &&
+         uvs->group_offs[UVS_VARYINGS] == 4 &&
+         uvs->group_offs[UVS_PSIZ] == 4 + components &&
+         uvs->group_offs[UVS_LAYER_VIEWPORT] == 4 + components + psiz &&
+         uvs->group_offs[UVS_CLIP_DIST] == 4 + components + psiz + layer_word &&
+         uvs->size == 4 + components + psiz + layer_word + clip_words &&
+         uvs->osel.opaque[0] == ((psiz ? 0x40000 : 0) |
+                               (layer_word ? 0x180000 : 0) |
+                               ((UINT32_C(1) << clip_words) - 1));
+      const struct agx_varyings_fs *cf = linked ? &linked->b.cf : NULL;
+      const struct agx_cf_binding *color =
+         cf && cf->nr_bindings >= 1 && cf->nr_bindings <= 3
+            ? &cf->bindings[cf->nr_bindings - 1] : NULL;
+      bool color_varying = vertex_layout && components <= 8 && components && color &&
+         color->slot != VARYING_SLOT_POS && gfx->varyings &&
+         hk_g17p_smooth_varyings_supported(cmd, linked) &&
+         gfx->linked_varyings.counts_16.opaque[0] == 0 &&
+         gfx->linked_varyings.counts_32.opaque[0] ==
+            (components << (!color->smooth ? 8 : color->perspective ? 0 : 16));
+      unsigned coefficient_count = 0;
+      if (color_varying) {
+         for (unsigned i = 0; i < cf->nr_bindings; ++i) {
+            const struct agx_cf_binding *b = &cf->bindings[i];
+            coefficient_count += b->count;
+            if (b->slot == VARYING_SLOT_POS) {
+               color_varying &= b->count == 1 && b->smooth && !b->perspective &&
+                  ((b->offset == 3 && i == 0) ||
+                   (b->offset == 2 && cf->reads_z));
+            } else {
+               color_varying &= b->slot < ARRAY_SIZE(uvs->components) &&
+                  b->count && b->offset + b->count <= uvs->components[b->slot] &&
+                  b->smooth == color->smooth && b->perspective == color->perspective;
+            }
+         }
+         color_varying &= coefficient_count == cf->nr_cf &&
+            (!color->perspective ||
+             (cf->nr_bindings >= 2 &&
+              cf->bindings[0].slot == VARYING_SLOT_POS &&
+              cf->bindings[0].offset == 3));
+      }
+      /* Native HAL300 uses the same 4-byte VIT rows and interpolation-count
+       * groups as Neo. Preserve the checked shared linker rather than infer
+       * every binding's interpolation from the final color input. */
+      bool general_varyings = vertex_layout && cf && cf->nr_bindings &&
+         gfx->varyings && hk_g17p_smooth_varyings_supported(cmd, linked) &&
+         gfx->linked_varyings.counts_16.opaque[0] == 0;
+      if (general_varyings) {
+         for (unsigned i = 0; i < cf->nr_bindings; ++i) {
+            const struct agx_cf_binding *b = &cf->bindings[i];
+            /* Native HAL300 VIT source2 supplies PointCoord without a UVS
+             * word. Source3 supplies PrimitiveID through the shared VDM
+             * generator and the PPP cull2 enable below. */
+            const bool user =
+               (b->slot >= VARYING_SLOT_VAR0 && b->slot <= VARYING_SLOT_VAR31) ||
+               b->slot == VARYING_SLOT_LAYER || b->slot == VARYING_SLOT_VIEWPORT ||
+               b->slot == VARYING_SLOT_CLIP_DIST0 ||
+               b->slot == VARYING_SLOT_CLIP_DIST1 ||
+               b->slot == VARYING_SLOT_CULL_PRIMITIVE ||
+               b->slot == VARYING_SLOT_CULL_PRIMITIVE + 1;
+            general_varyings &= b->slot == VARYING_SLOT_POS || user ||
+               b->slot == VARYING_SLOT_PNTC ||
+               b->slot == VARYING_SLOT_PRIMITIVE_ID;
+         }
+      }
+      const bool no_varyings = vertex_layout && cf && !cf->nr_bindings &&
+         !cf->nr_cf && !cf->reads_z && !gfx->varyings;
+      const bool reads_rhw = cf && cf->nr_bindings &&
+         cf->bindings[0].slot == VARYING_SLOT_POS &&
+         cf->bindings[0].offset == 3;
+      bool system_varyings = vertex_layout && cf && cf->nr_bindings &&
+         gfx->varyings &&
+         gfx->linked_varyings.counts_32.opaque[0] == 0 &&
+         gfx->linked_varyings.counts_16.opaque[0] == 0;
+      if (system_varyings) {
+         /* The shared linker emits optional 1/W, then optional Z. These
+          * system coefficients consume no user varying word; retain the
+          * same native bindings when the fragment reads only FragCoord. */
+         unsigned position_mask = 0;
+         for (unsigned i = 0; i < cf->nr_bindings; ++i) {
+            const struct agx_cf_binding *b = &cf->bindings[i];
+            if (b->slot != VARYING_SLOT_POS || b->count != 1 ||
+                !b->smooth || b->perspective || b->cf_base != i ||
+                (b->offset != 2 && b->offset != 3) ||
+                (b->offset == 3 && i != 0)) {
+               system_varyings = false;
+               break;
+            }
+            const unsigned bit = 1u << b->offset;
+            if (position_mask & bit) {
+               system_varyings = false;
+               break;
+            }
+            position_mask |= bit;
+         }
+         system_varyings &= cf->nr_cf == cf->nr_bindings &&
+            ((position_mask & (1u << 2)) != 0) == cf->reads_z;
+      }
+      const VkFormat depth_plane_format =
+         hk_depth_stencil_plane_format(render->depth_att.vk_format, 0);
+      const VkFormat stencil_plane_format = hk_depth_stencil_plane_format(
+         render->stencil_att.vk_format,
+         hk_split_depth_stencil(render->stencil_att.vk_format));
+      const bool depth_format = !render->depth_att.iview ||
+         depth_plane_format == VK_FORMAT_D32_SFLOAT ||
+         depth_plane_format == VK_FORMAT_D16_UNORM;
+      const bool stencil_format = !render->stencil_att.iview ||
+         stencil_plane_format == VK_FORMAT_S8_UINT;
+      const bool depth_active = render->depth_att.iview && dyn->ds.depth.test_enable;
+      /* NEVER with late side effects is emulated by exported NaN depth. */
+      const VkCompareOp depth_compare =
+         gfx->descriptors.root.draw.force_never_in_shader
+            ? VK_COMPARE_OP_ALWAYS : dyn->ds.depth.compare_op;
+      const bool stencil_active = hk_stencil_test_enabled(cmd);
+      struct vk_stencil_test_face_state front_stencil_state = dyn->ds.stencil.front;
+      struct vk_stencil_test_face_state back_stencil_state = dyn->ds.stencil.back;
+      // S8 compares and writes only the low eight API mask bits.
+      front_stencil_state.compare_mask &= 0xffu;
+      front_stencil_state.write_mask &= 0xffu;
+      back_stencil_state.compare_mask &= 0xffu;
+      back_stencil_state.write_mask &= 0xffu;
+      const struct agx_fragment_stencil_packed front_stencil =
+         hk_pack_stencil_face(front_stencil_state, stencil_active);
+      const struct agx_fragment_stencil_packed back_stencil =
+         hk_pack_stencil_face(back_stencil_state, stencil_active);
+      const unsigned fragment_pass = linked ? linked->b.fragment_control.opaque[0] >> 29 : 0;
+      const bool has_zs_attachment =
+         render->depth_att.iview || render->stencil_att.iview;
+      /* v70 depth928 and depth/stencil945 use the ordinary four-sample
+       * 32x16 ZLS profile. The native ZLS struct/stride producer is sample
+       * independent; kext selects both four/eight-sample tile geometry.
+       * Native G17G DEPTHF writes select one covered sample per word,
+       * preserving SR39's API-mask byte. Depth/stencil resolves run through
+       * hk_meta_resolve_rendering after the multisample source is stored;
+       * their presence does not change this draw's ZLS profile. */
+      const bool plain_ms_zs =
+         (render->tilebuffer.nr_samples == 2 ||
+          render->tilebuffer.nr_samples == 4 ||
+          render->tilebuffer.nr_samples == 8) && fs &&
+         !gfx->descriptors.root.draw.force_never_in_shader;
+
+      const struct vk_properties *limits = &dev->vk.physical->properties;
+      const float max_viewport_height = limits->maxViewportDimensions[1];
+      /* HAL300 publishes full float32 viewport transforms. Validate against
+       * the device's API domain rather than the render target dimensions. */
+      bool bounded_viewport = dyn->vp.viewport_count > 0 &&
+         dyn->vp.viewport_count <= limits->maxViewports &&
+         dyn->vp.scissor_count == dyn->vp.viewport_count;
+      bool bounded_depth_range = true;
+      for (unsigned i = 0; bounded_viewport && i < dyn->vp.viewport_count; ++i) {
+         const VkViewport *v = &dyn->vp.viewports[i];
+         bounded_viewport &=
+            v->width > 0 && v->width <= limits->maxViewportDimensions[0] &&
+            /* Vulkan 1.1 permits zero height. Keep the degenerate transform:
+             * resize-time Zink draws still execute pre-raster shader work. */
+            v->height >= -max_viewport_height &&
+            v->height <= max_viewport_height &&
+            v->x >= limits->viewportBoundsRange[0] &&
+            v->x <= limits->viewportBoundsRange[1] &&
+            v->x + v->width >= limits->viewportBoundsRange[0] &&
+            v->x + v->width <= limits->viewportBoundsRange[1] &&
+            v->y >= limits->viewportBoundsRange[0] &&
+            v->y <= limits->viewportBoundsRange[1] &&
+            v->y + v->height >= limits->viewportBoundsRange[0] &&
+            v->y + v->height <= limits->viewportBoundsRange[1];
+         bounded_depth_range &= v->minDepth >= 0 && v->minDepth <= 1 &&
+                                v->maxDepth >= 0 && v->maxDepth <= 1;
+      }
+
+      if (!hw_vs || !fs || !linked ||
+          (!no_varyings && !system_varyings && !color_varying &&
+           !general_varyings) ||
+          (gfx->object_type != AGX_OBJECT_TYPE_TRIANGLE &&
+           gfx->object_type != AGX_OBJECT_TYPE_LINE &&
+           gfx->object_type != AGX_OBJECT_TYPE_POINT_SPRITE_UV01) ||
+          (render->color_att_count > 1 && !hk_m5_mrt_layout_supported(render)) ||
+          !depth_format || !stencil_format ||
+          (!discard && render->tilebuffer.nr_samples != 1 &&
+           !((render->tilebuffer.nr_samples == 2 ||
+              render->tilebuffer.nr_samples == 4 ||
+              render->tilebuffer.nr_samples == 8) &&
+             (render->tilebuffer.tile_size == 32 * 32 ||
+              render->tilebuffer.tile_size == 32 * 16 ||
+              render->tilebuffer.tile_size == 16 * 16) &&
+             render->tilebuffer.nr_samples * render->tilebuffer.tile_size <=
+                render->tilebuffer.max_samples_per_tile &&
+             (unsigned)render->tilebuffer.sample_size_B *
+                render->tilebuffer.nr_samples * render->tilebuffer.tile_size <=
+                render->tilebuffer.max_bytes_per_tile &&
+             (!has_zs_attachment || plain_ms_zs))) ||
+          fragment_pass > AGX_PASS_TYPE_TRANSLUCENT_PUNCH_THROUGH ||
+          (!discard && (dyn->vp.depth_clip_negative_one_to_one ||
+          dyn->ds.depth.bounds_test.enable ||
+          !bounded_viewport || !bounded_depth_range))) {
+         {
+            fprintf(stderr, "M5 PPP profile: uvs=%u osel=%#x provoking=%u object=%u "
+                    "depth=%u stencil=%u varyings=%#llx cf=%u/%u/%u "
+                    "viewport=%g,%g,%g,%g,%g,%g negative_one=%u "
+                    "scissor=%d,%d,%u,%u area=%d,%d,%u,%u\n",
+                    hw_vs ? hw_vs->info.uvs.size : 0,
+                    hw_vs ? hw_vs->info.uvs.osel.opaque[0] : 0,
+                    gfx->provoking, gfx->object_type,
+                    render->depth_att.iview != NULL, render->stencil_att.iview != NULL,
+                    (unsigned long long)gfx->varyings,
+                    linked ? linked->b.cf.nr_bindings : 0,
+                    linked ? linked->b.cf.nr_cf : 0,
+                    linked ? linked->b.cf.reads_z : 0,
+                    vp->x, vp->y, vp->width, vp->height, vp->minDepth, vp->maxDepth,
+                    dyn->vp.depth_clip_negative_one_to_one,
+                    scissor->offset.x, scissor->offset.y,
+                    scissor->extent.width, scissor->extent.height,
+                    render->area.offset.x, render->area.offset.y,
+                    render->area.extent.width, render->area.extent.height);
+            fprintf(stderr, "M5 PPP flags: vertex_layout=%u psiz=%u bounded_viewport=%u "
+                    "color=%u layers=%u viewmask=%u samples=%u pass=%u/%u "
+                    "writes=%u discard=%u polygon=%u line=%g bias=%u/%g/%g/%g clamp=%u "
+                    "clip=%u bounds=%u query=%u viewports=%u scissors=%u size=%u,%u\n",
+                    vertex_layout, psiz, bounded_viewport,
+                    render->color_att_count, render->layer_count, render->view_mask,
+                    render->tilebuffer.nr_samples,
+                    linked ? linked->b.fragment_control.opaque[0] >> 29 : 0,
+                    AGX_PASS_TYPE_OPAQUE, fs ? fs->info.fs.writes_memory : 0,
+                    hk_rast_discard(cmd), dyn->rs.polygon_mode, dyn->rs.line.width,
+                    dyn->rs.depth_bias.enable, dyn->rs.depth_bias.constant_factor,
+                    dyn->rs.depth_bias.slope_factor, dyn->rs.depth_bias.clamp,
+                    dyn->rs.depth_clamp_enable,
+                    vk_rasterization_state_depth_clip_enable(&dyn->rs),
+                    dyn->ds.depth.bounds_test.enable, gfx->occlusion.mode,
+                    dyn->vp.viewport_count, dyn->vp.scissor_count,
+                    render->cr.width, render->cr.height);
+            fprintf(stderr, "M5 PPP color_valid=%u depth_valid=%u depth_format=%u "
+                            "depth_test=%u write=%u compare=%u counts=%#x/%#x\n",
+                    color_varying, depth_format, render->depth_att.vk_format,
+                    dyn->ds.depth.test_enable, dyn->ds.depth.write_enable,
+                    dyn->ds.depth.compare_op, gfx->linked_varyings.counts_32.opaque[0],
+                    gfx->linked_varyings.counts_16.opaque[0]);
+         }
+         mesa_loge("M5 draw exceeds the native color/depth PPP profile");
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+         return;
+      }
+
+      if (fs->b.info.has_preamble && !fs->b.info.agx3_preamble_staging_known) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+         return;
+      }
+      /* Software tessellation determines the generated triangle winding.
+       * Match the shared CULL path when the TES feeds rasterization directly. */
+      bool front_face_ccw =
+         dyn->rs.front_face == VK_FRONT_FACE_COUNTER_CLOCKWISE;
+      if (gfx->shaders[MESA_SHADER_TESS_CTRL] &&
+          !gfx->shaders[MESA_SHADER_GEOMETRY]) {
+         front_face_ccw ^= gfx->tess.info.ccw;
+         front_face_ccw ^= dyn->ts.domain_origin ==
+                           VK_TESSELLATION_DOMAIN_ORIGIN_LOWER_LEFT;
+      }
+      struct agx_ppp_hal300_graphics_fields fields;
+      /* Zeroed padding lets m5_ppp_pack_memo compare it bytewise. */
+      memset(&fields, 0, sizeof(fields));
+      /* A vertex-stage draw selects its provoking vertex through the VDM
+       * vertex program template (hk_agx3_publish_vertex_state), but
+       * the PPP flat-shading vertex must match it too. With it left at
+       * vertex 0, the EDS tests' oversized last-vertex triangles shaded
+       * from the wrong vertex on hardware while in-viewport ones were
+       * right; presumably clipping takes the flat inputs of the vertices
+       * it generates from the PPP selector. A native mesh draw has no VDM
+       * template; the PPP selector alone selects it (hardware: EDS
+       * last-vertex mesh draws pass with it and fail with the VDM
+       * mesh-state word). */
+      fields.flat_shading_vertex = gfx->provoking;
+      fields.viewport_count = hk_effective_viewport_count(cmd);
+      fields.general_varyings = general_varyings;
+      fields.cf_register_count = cf->nr_cf;
+      fields.varying_counts_32 = gfx->linked_varyings.counts_32.opaque[0];
+      fields.varying_counts_16 = gfx->linked_varyings.counts_16.opaque[0];
+      fields.output_select = uvs->osel.opaque[0] | linked->b.osel.opaque[0];
+      fields.uvs_words = uvs->size;
+      fields.fragment_program = hk_upload_usc_words(cmd, fs, linked);
+      fields.varying_table = gfx->varyings;
+      fields.cf_binding_count = cf->nr_bindings;
+      fields.fragment_ti_spill_B = fs->b.info.has_preamble
+            ? fs->b.info.preamble_scratch_size : 0;
+      fields.fragment_ti_staging_B = fs->b.info.has_preamble
+            ? fs->b.info.agx3_preamble_staging_halfs * 2u : 0;
+      fields.no_color_attachment = !hk_m5_has_color(&render->tilebuffer);
+      /* An attached target can have no colour transaction when the
+       * shader omits it or its write mask is zero (stencil-only draws). */
+      fields.no_color_writes = linked->b.g17p_colour_classes == 0;
+      fields.scissor_enable = !discard;
+      fields.depth_bias_enable = !discard && dyn->rs.depth_bias.enable &&
+            gfx->object_type == AGX_OBJECT_TYPE_TRIANGLE &&
+            (dyn->rs.depth_bias.constant_factor != 0.0f ||
+             dyn->rs.depth_bias.slope_factor != 0.0f);
+      fields.depth_clip_disable = !vk_rasterization_state_depth_clip_enable(&dyn->rs);
+      fields.polygon_mode = translate_polygon_mode(dyn->rs.polygon_mode);
+      fields.primitive_msaa = gfx->object_type == AGX_OBJECT_TYPE_LINE &&
+            dyn->rs.line.mode == VK_LINE_RASTERIZATION_MODE_BRESENHAM_KHR;
+      fields.line_width = agx_pack_line_width(dyn->rs.line.width);
+      fields.rasterizer_discard = discard;
+      fields.cull_mode = dyn->rs.cull_mode;
+      fields.front_face_ccw = front_face_ccw;
+      fields.point_size_output = psiz;
+      fields.layer_output = layer_word;
+      fields.clip_distance_words = clip_words;
+      fields.needs_primitive_id = gfx->generate_primitive_id;
+      fields.object_type = gfx->object_type == AGX_OBJECT_TYPE_POINT_SPRITE_UV01
+                           ? AGX_PPP_HAL300_POINTS
+                           : gfx->object_type == AGX_OBJECT_TYPE_LINE
+                                ? AGX_PPP_HAL300_LINES
+                                : AGX_PPP_HAL300_TRIANGLES;
+      fields.vertex_varying_components = components;
+      fields.fragment_varying_components = color_varying ? components : 0;
+      fields.interpolation = !color_varying || color->perspective
+                             ? AGX_PPP_HAL300_SMOOTH
+                             : color->smooth ? AGX_PPP_HAL300_LINEAR
+                                             : AGX_PPP_HAL300_FLAT;
+      fields.fragment_pass = fragment_pass;
+      /* AGX2 merges the linked FRAGMENT_CONTROL into its secondary word;
+       * HAL300 needs the tag-write disable carried explicitly, or a
+       * fragment that stores nothing (every channel of its format masked)
+       * hides the tile beneath it. */
+      fields.tag_write_disable =
+            (linked->b.fragment_control.opaque[0] & BITFIELD_BIT(21)) != 0;
+      fields.disable_tri_merging =
+            (linked->b.fragment_control.opaque[0] & BITFIELD_BIT(26)) != 0;
+      fields.fragment_writes_depth = fs->info.fs.epilog_key.write_z;
+      fields.fragment_depth_mode = agx_ppp_fragment_depth_mode(
+            agx_translate_depth_layout(fs->b.info.depth_layout),
+            fs->b.info.early_fragment_tests, fs->info.fs.writes_memory);
+      fields.fragment_reads_depth = cf->reads_z;
+      fields.fragment_reads_reciprocal_w = reads_rhw;
+      fields.depth_active = depth_active;
+      fields.depth_compare = depth_active ? depth_compare : 0;
+      fields.depth_write = depth_active && dyn->ds.depth.write_enable;
+      fields.stencil_active = stencil_active;
+      fields.front_stencil_reference = stencil_active
+            ? dyn->ds.stencil.front.reference & 0xffu : 0;
+      fields.back_stencil_reference = stencil_active
+            ? dyn->ds.stencil.back.reference & 0xffu : 0;
+      fields.front_stencil = front_stencil.opaque[0];
+      fields.back_stencil = back_stencil.opaque[0];
+      fields.visibility_mode = gfx->occlusion.mode;
+      fields.visibility_index = gfx->occlusion.index;
+      /* The stream already latched this state (see hk_cs). */
+      if (cs->m5_ppp_emitted_valid &&
+          !memcmp(&cs->m5_ppp_emitted, &fields, sizeof(fields)))
+         return;
+
+      __typeof__(gfx->m5_ppp_pack_memo) *pack = &gfx->m5_ppp_pack_memo;
+      if (!pack->valid || memcmp(&pack->fields, &fields, sizeof(fields))) {
+         pack->valid = false;
+         if (!agx_ppp_hal300_graphics_core_pack(dev->dev.chip, &fields,
+                                                &pack->state)) {
+            vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+            return;
+         }
+         pack->fields = fields;
+         pack->valid = true;
+      }
+      const struct agx_ppp_hal300_graphics_core state = pack->state;
+
+      /* Each record is a self-contained PPP update and the PPP keeps the
+       * others, so only the records that differ from what the stream latched
+       * are emitted. Every record the PPP processes stalls tiling for about
+       * 50 ns, close to half a microsecond per draw for the whole core. */
+      unsigned changed = 0;
+      for (unsigned i = 0; i < AGX_PPP_HAL300_GRAPHICS_RECORDS; ++i) {
+         if (!cs->m5_ppp_emitted_valid ||
+             memcmp(cs->m5_ppp_emitted_state.bytes + state.offsets[i],
+                    state.bytes + state.offsets[i], state.lengths[i]))
+            changed |= BITFIELD_BIT(i);
+      }
+
+      if (changed) {
+         struct agx_ptr upload =
+            hk_pool_vdm_alloc(cmd, sizeof(state.bytes), 64);
+         if (!upload.cpu)
+            return;
+         memcpy(upload.cpu, state.bytes, sizeof(state.bytes));
+
+         uint8_t *start = *out;
+         u_foreach_bit(i, changed) {
+            if (!hk_ppp_emit_state(dev->dev.chip, out,
+                                   upload.gpu + state.offsets[i],
+                                   state.lengths[i])) {
+               vk_command_buffer_set_error(&cmd->vk,
+                                           VK_ERROR_FEATURE_NOT_PRESENT);
+               return;
+            }
+         }
+         if (unlikely(dev->frame_anatomy))
+            cs->anatomy_ppp += (*out - start) / AGX_PPP_STATE_LENGTH;
+      }
+      cs->m5_ppp_emitted = fields;
+      cs->m5_ppp_emitted_state = state;
+      cs->m5_ppp_emitted_valid = true;
+      return;
+   }
+
+
    bool hw_vs_dirty = IS_SHADER_DIRTY(VERTEX) || IS_SHADER_DIRTY(TESS_EVAL) ||
                       IS_SHADER_DIRTY(GEOMETRY);
    bool fs_dirty = IS_SHADER_DIRTY(FRAGMENT);
 
    struct hk_linked_shader *linked_fs = gfx->linked[MESA_SHADER_FRAGMENT];
    bool linked_fs_dirty = IS_LINKED_DIRTY(FRAGMENT);
+   /* The launch follows the linked program's own layout, if it has one. */
+   if (linked_fs && linked_fs->g17p_layout)
+      fs = linked_fs->g17p_layout;
 
    bool varyings_dirty = gfx->dirty & HK_DIRTY_VARYINGS;
 
@@ -7982,7 +9812,7 @@ hk_fill_fast_link_key_vs(struct hk_cmd_buffer *cmd, struct hk_shader *sw_vs,
 
       /* G17P materialises adjacency into an index buffer before the hardware
        * vertex program. Applying the remap again in the prolog is wrong. */
-      if (!agx_native_graphics(&dev->dev))
+      if (!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip))
          key->prolog.adjacency = prim;
    }
 
@@ -8031,7 +9861,9 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
    bool defer_api_sample_mask = agx_native_graphics(&dev->dev) &&
                                 has_sample_mask && !nontrivial_force_early;
 
-   if (agx_native_graphics(&dev->dev) && fs->info.fs.uses_sample_shading &&
+   const bool native_samples = agx_native_graphics(&dev->dev) ||
+                               agx_chip_g17_m5(dev->dev.chip);
+   if (native_samples && fs->info.fs.uses_sample_shading &&
        dyn->ms.rasterization_samples > 1) {
       if (report_errors)
          mesa_loge("G17P: per-sample fragment execution needs a fixed sample "
@@ -8040,7 +9872,7 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
       return false;
    }
 
-   if (agx_native_graphics(&dev->dev) &&
+   if (native_samples &&
        fs->info.fs.epilog_key.sample_shading &&
        fs->info.fs.g17p_samples != samples) {
       if (report_errors)
@@ -8050,7 +9882,7 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
       return false;
    }
 
-   if (agx_native_graphics(&dev->dev) && fs->b.info.writes_sample_mask &&
+   if (native_samples && fs->b.info.writes_sample_mask &&
        fs->info.fs.g17p_samples != samples) {
       if (report_errors)
          mesa_loge("G17P: fragment coverage updates compiled for %u samples "
@@ -8061,21 +9893,24 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
 
    unsigned samples_shaded = 0;
    if (fs->info.fs.epilog_key.sample_shading &&
-       !agx_native_graphics(&dev->dev))
+       !native_samples)
       samples_shaded = dyn->ms.rasterization_samples;
 
    *key = (struct hk_fast_link_key_fs){
       .prolog.statistics = in->statistics,
       .prolog.cull_distance_size = in->cull_distance_size,
       /* AGX2 must apply the API mask before an explicit-early main part
-       * counts samples, just as upstream does. Only the native path folds
-       * that mask into its forced-early coverage word. */
+       * counts samples, just as upstream does. Only the native and M5 paths
+       * fold that mask into their forced-early coverage word
+       * (agx_nir_apply_api_mask_to_force_early()). */
       .prolog.api_sample_mask =
          has_sample_mask && !defer_api_sample_mask &&
-               (!agx_native_graphics(&dev->dev) || !nontrivial_force_early)
+               ((!agx_native_graphics(&dev->dev) &&
+                 !agx_chip_g17_m5(dev->dev.chip)) ||
+                !nontrivial_force_early)
             ? api_sample_mask
             : 0xff,
-      .prolog.nr_samples = agx_native_graphics(&dev->dev) ? samples : 0,
+      .prolog.nr_samples = native_samples ? samples : 0,
       .nr_samples_shaded = samples_shaded,
    };
 
@@ -8108,8 +9943,10 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
    }
    agx_fs_epilog_apply_device_policy(dev->dev.chip, &key->epilog);
 
-   key->epilog.native_spills = agx_native_graphics(&dev->dev) &&
-                               agx_tilebuffer_spills(in->tib);
+   key->epilog.native_spills =
+      (agx_native_graphics(&dev->dev) ||
+       agx_chip_g17_m5(dev->dev.chip)) &&
+      agx_tilebuffer_spills(in->tib);
    if (key->epilog.native_spills) {
       uint16_t base;
       if (!hk_g17p_relocated_blend_constant_base(fs, &base) ||
@@ -8117,9 +9954,17 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
          return false;
       key->epilog.emrt_heap_base = ALIGN_POT(base, 4);
       key->epilog.blend_constant_base = key->epilog.emrt_heap_base + 4;
+      /* The M5 image-write encoder accepts the low 256 uniform halves. The
+       * resource heaps occupy 0..7 and the root starts at 12; the mandatory
+       * inline program can use the reserved 8..11 gap for its spill heap. */
+      if (fs->info.fs.m5_inline_sample_shading &&
+          !fs->info.agx3_gen1_fragment_rq &&
+          fs->info.agx3_root_uniform == AGX_ABI_FUNI_ROOT)
+         key->epilog.emrt_heap_base = 8;
    }
 
-   if (!key->epilog.native_spills && agx_native_graphics(&dev->dev) &&
+   if (!key->epilog.native_spills &&
+       (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
        hk_uses_blend_constant(&dyn->cb) &&
        (fs->info.agx3_combined_sampler_valid ||
         fs->info.agx3_indexed_sampler_valid) &&
@@ -8134,7 +9979,9 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
    }
 
    key->epilog.link.already_ran_zs |= nontrivial_force_early;
+   key->epilog.native_imageblock = in->tib->native_imageblock;
    key->epilog.max_sample_size_B = in->tib->max_sample_size_B;
+   key->epilog.resident_mask = in->tib->resident_mask;
    key->epilog.max_bytes_per_tile = in->tib->max_bytes_per_tile;
    key->epilog.max_samples_per_tile = in->tib->max_samples_per_tile;
 
@@ -8142,7 +9989,7 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
       const VkFormat format = *(const VkFormat *)(
          (const uint8_t *)in->color_formats + i * in->color_formats_stride);
       key->epilog.rt_formats[i] = hk_format_to_pipe_format(format);
-      if (agx_native_graphics(&dev->dev)) {
+      if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
          const enum pipe_format logical = key->epilog.rt_formats[i];
          key->epilog.rt_formats[i] = hk_padded_image_format(logical);
          const enum pipe_format integer =
@@ -8207,9 +10054,10 @@ hk_fs_link_key_build(const struct hk_device *dev, struct hk_shader *fs,
 /* Colour attachments a draw with this fragment link key writes: the
  * attachment is mapped to a location the shader writes (location 0 for
  * every attachment with gl_FragColor broadcast) and its write mask is not
- * empty. */
+ * empty, or, with preserved_written (M5), the epilog preserves it. */
 static uint8_t
-hk_fs_color_written(const struct hk_fast_link_key_fs *key)
+hk_fs_color_written(const struct hk_fast_link_key_fs *key,
+                    bool preserved_written)
 {
    const struct agx_fs_epilog_link_info *link = &key->epilog.link;
    uint8_t mask = 0;
@@ -8219,7 +10067,9 @@ hk_fs_color_written(const struct hk_fast_link_key_fs *key)
          continue;
       const unsigned src = link->broadcast_rt0 ? 0 : loc;
       if ((link->loc_written & BITFIELD_BIT(src)) &&
-          key->epilog.blend.rt[rt].colormask)
+          (key->epilog.blend.rt[rt].colormask ||
+           (preserved_written &&
+            (key->epilog.preserve_rt & BITFIELD_BIT(rt)))))
          mask |= BITFIELD_BIT(rt);
    }
    return mask;
@@ -8287,7 +10137,7 @@ hk_prelink_pipeline_fragment(struct hk_device *dev,
    struct agx_tilebuffer_layout tib = hk_tilebuffer_layout(
       dev, state->rp->color_attachment_formats, sizeof(VkFormat),
       state->rp->color_attachment_count, samples, view_mask != 0,
-      samples > 1 && view_mask != 0);
+      samples > 1 && view_mask != 0, 0);
    if (!hk_agx3_g17p_rt0_tile_layout_supported(&tib))
       return;
 
@@ -8483,6 +10333,12 @@ hk_agx3_publish_vertex_state(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    gfx->g17p_vdm_update_memo.valid = true;
 
 tail:
+   if (unlikely(dev->frame_anatomy)) {
+      cs->anatomy_vdm++;
+      if (pipeline_program != cs->anatomy_last_program)
+         cs->anatomy_vdm_new++;
+      cs->anatomy_last_program = pipeline_program;
+   }
    if (agx_native_graphics(&dev->dev)) {
       if (!hk_g17p_emit_native_bind_tail(cmd, cs, &out,
                                            tail_known_match)) {
@@ -8533,6 +10389,8 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
 
    if (!hk_gfx_republishes_render_state(cmd))
       return;
+   if (unlikely(dev->frame_anatomy))
+      cs->anatomy_republish++;
 
    struct hk_descriptor_state *desc = &cmd->state.gfx.descriptors;
 
@@ -8546,6 +10404,7 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    bool vgt_dirty = IS_SHADER_DIRTY(VERTEX) || IS_SHADER_DIRTY(MESH) || gt_dirty;
    bool fs_dirty = IS_SHADER_DIRTY(FRAGMENT);
    if (native_mesh && gfx->linked[MESA_SHADER_VERTEX] != hw_vs->only_linked) {
+      gfx->fast_link_vs.valid = false;
       gfx->linked[MESA_SHADER_VERTEX] = hw_vs->only_linked;
       gfx->linked_dirty |= BITFIELD_BIT(MESA_SHADER_VERTEX);
    }
@@ -8744,7 +10603,8 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
          cs->current = out;
          return;
       }
-      gfx->fast_link_fs.color_written = hk_fs_color_written(&key);
+      gfx->fast_link_fs.color_written =
+         hk_fs_color_written(&key, agx_chip_g17_m5(dev->dev.chip));
       gfx->fast_link_fs.reads_tib = fs->b.info.reads_tib;
    }
    gfx->render.color_written |= gfx->fast_link_fs.color_written;
@@ -8780,14 +10640,20 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
       gfx->g17p_draw_id_host_valid = false;
    }
 
-   if (IS_DIRTY(IA_PRIMITIVE_TOPOLOGY) || gt_dirty || vgt_dirty) {
+   bool topology_changed = false;
+   if (IS_DIRTY(IA_PRIMITIVE_TOPOLOGY) || gt_dirty || vgt_dirty || IS_SHADER_DIRTY(MESH)) {
       enum mesa_prim prim = native_mesh ? hw_vs->info.mesh.prim : hk_rast_prim(cmd);
+      const unsigned topology = translate_hw_primitive_topology(prim);
 
-      gfx->topology = translate_hw_primitive_topology(prim);
+      topology_changed = topology != gfx->topology && IS_SHADER_DIRTY(MESH);
+      gfx->topology = topology;
       gfx->object_type = translate_object_type(prim);
    }
 
-   if (IS_DIRTY(IA_PRIMITIVE_TOPOLOGY) || IS_DIRTY(RS_PROVOKING_VERTEX)) {
+   /* The selector depends on the topology, which binding or unbinding a
+    * mesh shader changes without dirtying the IA state. */
+   if (IS_DIRTY(IA_PRIMITIVE_TOPOLOGY) || IS_DIRTY(RS_PROVOKING_VERTEX) ||
+       topology_changed) {
       unsigned provoking = hk_provoking_selector(dyn, gfx->topology);
 
       if (provoking != gfx->provoking) {
@@ -8829,7 +10695,7 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    if (IS_DIRTY(MS_SAMPLE_LOCATIONS) || IS_DIRTY(MS_SAMPLE_LOCATIONS_ENABLE) ||
        IS_DIRTY(MS_RASTERIZATION_SAMPLES)) {
 
-      uint32_t ctrl;
+      uint64_t ctrl;
       if (dyn->ms.sample_locations_enable) {
          ctrl = hk_pack_ppp_multisamplectrl(dyn->ms.sample_locations);
       } else {
@@ -8849,8 +10715,10 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
          assert(dont_commit || cs->ppp_multisamplectl == ctrl);
       }
 
-      if (gfx->descriptors.root.draw.ppp_multisamplectl != ctrl) {
-         gfx->descriptors.root.draw.ppp_multisamplectl = ctrl;
+      if (gfx->descriptors.root.draw.ppp_multisamplectl != (uint32_t)ctrl ||
+          gfx->descriptors.root.ppp_multisamplectl_hi != (ctrl >> 32)) {
+         gfx->descriptors.root.draw.ppp_multisamplectl = (uint32_t)ctrl;
+         gfx->descriptors.root.ppp_multisamplectl_hi = ctrl >> 32;
          gfx->descriptors.root_dirty = true;
       }
    }
@@ -8881,7 +10749,8 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
          size_t linkage_size =
             AGX_CF_BINDING_HEADER_LENGTH + (bindings * AGX_CF_BINDING_LENGTH);
 
-         const bool native_vit = agx_native_graphics(&dev->dev);
+         const bool native_vit = agx_native_graphics(&dev->dev) ||
+                                 agx_chip_g17_m5(dev->dev.chip);
          struct agx_ptr t =
             hk_pool_usc_alloc(cmd, linkage_size, native_vit ? 64 : 16);
          if (!t.cpu)
@@ -8971,14 +10840,13 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
       struct hk_linked_shader *linked_hw_vs =
          (hw_vs == sw_vs) ? linked_vs : hw_vs->only_linked;
       const uint64_t pipeline_program = native_mesh
-         ? hk_upload_native_mesh_program(cmd, hw_vs, linked_hw_vs, false)
+         ? hk_upload_native_mesh_program(cmd, hw_vs, linked_hw_vs, false, false)
          : hk_upload_usc_words(cmd, hw_vs, linked_hw_vs);
       if (native_mesh) {
-         struct hk_shader *object =
-            &gfx->shaders[MESA_SHADER_MESH]->variants[gfx->mesh.object_params
-            ? HK_MESH_VARIANT_PREDICATED_OBJECT : HK_MESH_VARIANT_RAST];
+         bool task;
+         struct hk_shader *object = hk_native_mesh_object(gfx, &task);
          const uint64_t object_program = hk_upload_native_mesh_program(
-            cmd, object, object->only_linked, true);
+            cmd, object, object->only_linked, true, task);
          uint32_t mesh_dimensions[3] = {hw_vs->b.info.workgroup_size[0],
             hw_vs->b.info.workgroup_size[1],hw_vs->b.info.workgroup_size[2]};
          uint32_t mesh_threadgroup;
@@ -8996,10 +10864,8 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
             vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
             return;
          }
-         const char *native_raster =
-            getenv("ASAHI_MESH_NATIVE_RASTER_ENABLE_DIAGNOSTIC");
-         const bool native_raster_enabled = native_raster &&
-            !strcmp(native_raster, "1");
+         const bool native_raster_enabled =
+            debug_get_option_mesh_native_raster_enable();
          struct agx_vdm_mesh_state_info state = {
             
 
@@ -9035,6 +10901,20 @@ hk_flush_dynamic_state_slow(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
                dev->dev.chip == AGX_CHIP_G17P
                   ? DIV_ROUND_UP(hw_vs->b.info.local_size, 16) : 9},
          };
+         if (dev->dev.chip == AGX_CHIP_G17S) {
+            if (!agx_vdm_m5_mesh_set_output_limits(
+                   &state, hw_vs->info.mesh.max_vertices,
+                   hw_vs->info.mesh.max_primitives, hw_vs->info.uvs.size)) {
+               vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+               return;
+            }
+            /* Native object->mesh kicks (native726 mesh-object, 0x1ca48
+             * bound) set tail[0]; task kicks with the native record do too.
+             * Payload-free task kicks without it keep the mesh-only value. */
+            if ((task && hk_m5_task_record(dev->dev.params.features)) ||
+                debug_get_option_mesh_pp_tail0())
+               state.raw_tail[0] = 1;
+         }
          if (!agx_vdm_mesh_state_pack(dev->dev.chip, &state, out)) {
             vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
             return;
@@ -9232,7 +11112,7 @@ hk_needs_index_robustness(struct hk_cmd_buffer *cmd, struct agx_draw *draw,
     * page instead.
     */
    if (draw->index_buffer_range_B == 0 ||
-       (agx_native_graphics(&dev->dev) &&
+       ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
         draw->index_buffer == AGX_ZERO_PAGE_ADDRESS &&
         draw->index_buffer_range_B == 4)) {
       draw->index_buffer = AGX_ZERO_PAGE_ADDRESS;
@@ -9344,9 +11224,11 @@ hk_handle_passthrough_gs(struct hk_cmd_buffer *cmd, struct agx_draw draw)
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    perf_debug(cmd, "Binding passthrough GS for%s\n", xfb_outputs ? " XFB" : "");
 
-   gs = hk_meta_shader(dev, hk_nir_passthrough_gs_native, key, key_size);
+   VkResult result;
+   gs = hk_meta_shader(dev, hk_nir_passthrough_gs_native, key, key_size,
+                       &result);
    if (!gs) {
-      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      vk_command_buffer_set_error(&cmd->vk, result);
       return;
    }
    gs->is_passthrough = true;
@@ -9398,6 +11280,10 @@ hk_flush_gfx_state(struct hk_cmd_buffer *cmd, uint32_t draw_id,
       return NULL;
    }
 
+   if (unlikely(gfx->occlusion.pending_pool) &&
+       !hk_ensure_occlusion_query_for_draw(cmd))
+      return NULL;
+
    /* An ordered render continuation (hk_m5_pass_merge_begin()) ends before a
     * draw that could break its memory equivalence; the draw goes to a
     * continuation stream ordered after the whole merged pass. Meta draws
@@ -9535,7 +11421,8 @@ hk_flush_gfx_state(struct hk_cmd_buffer *cmd, uint32_t draw_id,
     * dereference the heap. Direct adjacency without restart uses no heap. */
    const bool needs_heap = geom || tess || (*needs_idx_robust && !adj) ||
       (adj && (draw->restart ||
-               (agx_native_graphics(&dev->dev) && agx_is_indirect(draw->b))));
+               ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+                agx_is_indirect(draw->b))));
    if (needs_heap && !hk_cmd_ensure_geometry_heap(cmd))
       return NULL;
 
@@ -9545,7 +11432,8 @@ hk_flush_gfx_state(struct hk_cmd_buffer *cmd, uint32_t draw_id,
     * publication path as well; the generated body carries the matching
     * context-relative selector when that publication is fresh. */
    const bool g17p_robust_index_path =
-      agx_native_graphics(&dev->dev) && *needs_idx_robust;
+      (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+      *needs_idx_robust;
    const bool ordinary_draw_path =
       !geom && !tess && !adj &&
       (!*needs_idx_robust || g17p_robust_index_path);
@@ -9560,7 +11448,8 @@ hk_flush_gfx_state(struct hk_cmd_buffer *cmd, uint32_t draw_id,
     * adjacency topology failed at vkEndCommandBuffer with
     * VK_ERROR_FEATURE_NOT_PRESENT. */
    const bool g17p_transformed_path =
-      agx_native_graphics(&dev->dev) && (geom || tess || adj);
+      (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+      (geom || tess || adj);
    if (desc->push_dirty)
       hk_cmd_buffer_flush_push_descriptors(cmd, desc);
 
@@ -9568,7 +11457,8 @@ hk_flush_gfx_state(struct hk_cmd_buffer *cmd, uint32_t draw_id,
     * record. Geometry/XFB lowers even a non-indexed API draw to that packet,
     * so emitting the legacy state here as well produces two consecutive
     * restart records before the synthetic indexed draw. */
-   if ((draw->restart || geom) && !agx_native_graphics(&dev->dev))
+   if ((draw->restart || geom) && !agx_native_graphics(&dev->dev) &&
+       !agx_chip_g17_m5(dev->dev.chip))
       hk_flush_index(cmd, cs);
 
    /* Everything that can dirty render state for this draw has now run, so
@@ -10039,6 +11929,11 @@ hk_emit_robust_index(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
          return;
       }
       empty_draw_jump = compact;
+   } else if (agx_chip_g17_m5(dev->dev.chip)) {
+      if (!agx_m5_vdm_compact(target + size_B, &empty_draw_jump)) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+         return;
+      }
    }
 
    const struct libagx_robust_index_draw job = {
@@ -10068,10 +11963,10 @@ hk_emit_robust_index(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    }
 }
 
-/* G17P: a draw whose pre-rasterization shaders may read images orders its
- * render's vertex half after the image barriers before it (HK_SYNC_GFX_IMG).
- * Software tessellation, geometry, task and mesh stages are assumed to
- * read.
+/* G17P and M5: a draw whose pre-rasterization shaders may read images
+ * orders its render's vertex half after the image barriers before it
+ * (HK_SYNC_GFX_IMG). Software tessellation, geometry, task and mesh stages
+ * are assumed to read.
  *
  * An image barrier's memory dependency covers only its image, but its
  * execution dependency covers all work in its stages, so vertex work that
@@ -10128,16 +12023,17 @@ hk_draw_impl(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw_,
     * older path does in hk_needs_index_robustness. The one-dword range bounds
     * hardware fetches to the zero page; it is not a count limit.
     */
-   if (agx_native_graphics(&dev->dev) && draw_.indexed &&
-       draw_.index_buffer_range_B == 0) {
+   if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+       draw_.indexed && draw_.index_buffer_range_B == 0) {
       draw_.index_buffer = AGX_ZERO_PAGE_ADDRESS;
       draw_.index_buffer_range_B = 4;
       draw_.start = 0;
    }
    const struct agx_draw api_draw = draw_;
    bool conditional_direct_indexed = false;
-   const uint64_t primitive_bound = agx_native_graphics(&dev->dev)
-      ? hk_draw_primitive_bound(cmd, &api_draw) : 0;
+   const uint64_t primitive_bound =
+      (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
+         ? hk_draw_primitive_bound(cmd, &api_draw) : 0;
 
    /* There is no native conditional-render packet on AGX. Lower the command
     * to the API's indirect record and let an ordered compute kernel either
@@ -10295,11 +12191,22 @@ hk_draw_impl(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw_,
                       stat_c_prims, stat_c_inv);
       }
 
+      /* Without geometry or tessellation the primitives generated are the
+       * assembled primitives, counted like the clipper statistic; the
+       * tessellator and the geometry pipeline count their own outputs. */
+      const uint64_t prims_generated = hk_prims_generated_addr(cmd, 0);
+      if (prims_generated && !geom && !tess) {
+         hk_ia_update(cmd, draw, AGX_SCRATCH_PAGE_ADDRESS,
+                      AGX_SCRATCH_PAGE_ADDRESS, AGX_SCRATCH_PAGE_ADDRESS,
+                      AGX_SCRATCH_PAGE_ADDRESS, prims_generated);
+      }
+
       if (tess) {
          cmd->state.gfx.tess.tes_direct_threads =
             hk_tes_direct_query_threads(cmd, draw);
          draw = hk_launch_tess(cmd, ccs, draw, geom ? 0 : stat_c_prims,
-                               geom ? 0 : stat_c_inv);
+                               geom ? 0 : stat_c_inv,
+                               geom ? 0 : prims_generated);
       }
 
       if (geom)
@@ -10323,7 +12230,7 @@ hk_draw_impl(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw_,
             uint64_t out_draw = hk_pool_alloc(cmd, size, 4).gpu;
             struct hk_device *dev = hk_cmd_buffer_device(cmd);
 
-            if (agx_native_graphics(&dev->dev)) {
+            if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
                /* As in the direct case above, a G17P hardware vertex shader
                 * never sees the prolog's remap, so the mapping has to reach
                 * the input assembler as real indices.  The output count is
@@ -10365,7 +12272,8 @@ hk_draw_impl(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw_,
              * draw fetched in[i] where it owed in[map(i)].  Materialise the
              * mapping as an index buffer instead and let the input assembler
              * apply it, which is the one path the vertex stage does see. */
-            if (count > 0 && agx_native_graphics(&dev->dev)) {
+            if (count > 0 &&
+                (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))) {
                unsigned ib_B = count * sizeof(uint32_t);
                uint64_t ib = hk_pool_alloc(cmd, ib_B, 4).gpu;
 
@@ -10421,6 +12329,11 @@ hk_draw_impl(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw_,
       enum agx_primitive topology = cmd->state.gfx.topology;
       if ((!agx_native_graphics(&dev->dev) || agx_is_indirect(draw.b)) &&
           !agx_vdm_draw_supported(dev->dev.chip, draw, topology)) {
+         if (debug_get_option_m5_launch_debug())
+            mesa_loge("draw admission: indexed=%u indirect=%u size=%u ib=%#llx range=%u arg=%#llx topology=%u",
+                      draw.indexed, agx_is_indirect(draw.b), draw.index_size,
+                      (unsigned long long)draw.index_buffer, draw.index_buffer_range_B,
+                      (unsigned long long)draw.b.ptr, topology);
          vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
          return;
       }
@@ -10445,14 +12358,37 @@ hk_draw_impl(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw_,
                                    g17p_native_bind_tail_emitted))
                return;
          } else {
-            cs->current = (void *)agx_vdm_draw(
-               (uint32_t *)out, dev->dev.chip, draw, topology,
-               hk_draw_restart_index(cmd, draw));
+            /* HAL300 direct Index Lists consume persistent restart state;
+             * the indirect packet carries its own state update. Geometry
+             * may have synthesized an index width unrelated to the API IB. */
+            if (agx_chip_g17_m5(dev->dev.chip) && draw.indexed &&
+                draw.restart && !agx_is_indirect(draw.b)) {
+               const uint32_t index =
+                  agx_primitive_restart_index(draw.index_size);
+               if (cs->restart_index != index) {
+                  agx_push(out, VDM_STATE, cfg) {
+                     cfg.restart_index_present = true;
+                  }
+                  agx_push(out, VDM_STATE_RESTART_INDEX, cfg) {
+                     cfg.value = index;
+                  }
+                  cs->restart_index = index;
+               }
+            }
+            cs->current = (void *)agx_vdm_draw((uint32_t *)out,
+                                               dev->dev.chip, draw, topology,
+                                               hk_draw_restart_index(cmd, draw));
          }
       }
 
       hk_g17p_note_layer_zero_draw(cmd, cs);
       hk_note_pre_raster_images(cmd, cs);
+      hk_m5_note_render_resources(cmd, cs);
+      if (unlikely(dev->frame_anatomy)) {
+         hk_frame_anatomy_note_fs(cs, hk_bound_fs(cmd));
+         hk_frame_anatomy_note_vs(cs, hk_bound_hw_vs(&cmd->state.gfx));
+         hk_frame_anatomy_note_m5_fragment(cs);
+      }
       cs->stats.cmds++;
    }
 }
@@ -10464,13 +12400,19 @@ hk_draw(struct hk_cmd_buffer *cmd, uint16_t draw_id, struct agx_draw draw)
 }
 
 static void
-hk_draw_mesh_native(struct hk_cmd_buffer *cmd, uint64_t grid, uint32_t draw_id,
-                     uint64_t count_address)
+hk_draw_mesh_native(struct hk_cmd_buffer *cmd, uint64_t grid,
+                    const uint32_t *cpu_grid, uint32_t draw_id,
+                    uint64_t count_address)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
    struct hk_graphics_state *gfx = &cmd->state.gfx;
-   if (dev->dev.chip != AGX_CHIP_G17P || !gfx->shaders[MESA_SHADER_MESH] ||
-       gfx->shaders[MESA_SHADER_TASK] || hk_mesh_mode() != HK_MESH_MODE_NATIVE_PROBE) {
+   struct hk_api_shader *task_api = gfx->shaders[MESA_SHADER_TASK];
+   if (!gfx->shaders[MESA_SHADER_MESH] ||
+       (task_api && (dev->dev.chip != AGX_CHIP_G17S || !hk_m5_task_enabled())) ||
+       (dev->dev.chip == AGX_CHIP_G17P
+          ? hk_mesh_mode() != HK_MESH_MODE_NATIVE_PROBE
+          : (dev->dev.chip != AGX_CHIP_G17S ||
+             !(dev->dev.params.features & DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2)))) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
       return;
    }
@@ -10480,22 +12422,58 @@ hk_draw_mesh_native(struct hk_cmd_buffer *cmd, uint64_t grid, uint32_t draw_id,
       hk_xfer_check_draw(cmd, count_address, count_address + sizeof(uint32_t));
    
 
+   const bool conditional = hk_conditional_render_is_active(
+      &cmd->state.conditional, cmd->in_meta, cmd->conditional_meta);
    const struct hk_mesh_info *mesh_info =
       &hk_main_mesh_variant(gfx->shaders[MESA_SHADER_MESH])->info.mesh;
+   const struct hk_shader *task = task_api ? &task_api->variants[0] : NULL;
+   /* A task draw whose grid, draw count or predicate lives in GPU memory is
+    * recorded with one object workgroup; libagx_m5_task_dispatch patches the
+    * object thread counts before the render and publishes whether the draw
+    * executes (hk_m5_task_predicate). */
+   const bool task_gpu_dispatch = task_api &&
+      (!cpu_grid || count_address || conditional);
+   uint64_t task_flag = 0;
+   if (task) {
+      if (!task->only_linked) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+         return;
+      }
+      if (task_gpu_dispatch) {
+         struct agx_ptr flag = hk_pool_alloc(cmd, 4, 4);
+         if (!flag.cpu) {
+            vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+            return;
+         }
+         task_flag = flag.gpu;
+         for (unsigned i = 0; i < 3; ++i)
+            gfx->mesh.task_grid[i] = 1;
+      } else {
+         memcpy(gfx->mesh.task_grid, cpu_grid, sizeof(gfx->mesh.task_grid));
+      }
+   }
+   gfx->mesh.grid = grid;
+   /* Only a stage that reads gl_DrawID or gl_NumWorkGroups needs the root
+    * republished per draw; others keep the bound root. With a task stage,
+    * the API grid belongs to the task shader. */
    struct hk_descriptor_state *desc = &gfx->descriptors;
-   if (mesh_info->reads_num_workgroups &&
+   if (task && desc->root.mesh_task_flag != task_flag) {
+      desc->root.mesh_task_flag = task_flag;
+      desc->root_dirty = true;
+   }
+   if ((mesh_info->reads_num_workgroups ||
+        (task && task->info.mesh.reads_num_workgroups)) &&
        desc->root.mesh_grid_addr != grid) {
       desc->root.mesh_grid_addr = grid;
       desc->root_dirty = true;
    }
-   if (mesh_info->reads_draw_id && desc->root.mesh_draw_id != draw_id) {
+   if ((mesh_info->reads_draw_id || (task && task->info.mesh.reads_draw_id)) &&
+       desc->root.mesh_draw_id != draw_id) {
       desc->root.mesh_draw_id = draw_id;
       desc->root_dirty = true;
    }
-   const bool conditional = hk_conditional_render_is_active(
-      &cmd->state.conditional, cmd->in_meta, cmd->conditional_meta);
    gfx->mesh.object_params = 0;
-   {
+   if (!task && (dev->dev.chip == AGX_CHIP_G17P || count_address || conditional)) {
       const struct {
          uint64_t grid, count, predicate;
          uint32_t draw_id, inverted;
@@ -10524,15 +12502,45 @@ hk_draw_mesh_native(struct hk_cmd_buffer *cmd, uint64_t grid, uint32_t draw_id,
       memcpy(cs->current, &draw_state_offset, sizeof(draw_state_offset));
       cs->current += sizeof(draw_state_offset);
    }
-   const struct agx_vdm_mesh_dispatch_info dispatch = {
+   /* The record is an Index List block; its Primitive field must name the
+    * mesh output topology. With the triangle value, point and line meshes
+    * of 22 or more primitives (more than 64 bytes at three index bytes per
+    * primitive) lost primitives on hardware. */
+   struct agx_vdm_mesh_dispatch_info dispatch = {
       .threads = {3, 1, 1}, .local_size = {3, 1, 1},
+      .primitive = mesh_info->prim == MESA_PRIM_POINTS ? 0
+                 : mesh_info->prim == MESA_PRIM_LINES  ? 1 : 6,
    };
+   if (task) {
+      for (unsigned i = 0; i < 3; ++i) {
+         const uint64_t threads =
+            (uint64_t)gfx->mesh.task_grid[i] * task->b.info.workgroup_size[i];
+         if (!threads || threads > UINT32_MAX) {
+            vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+            return;
+         }
+         dispatch.threads[i] = threads;
+         dispatch.local_size[i] = task->b.info.workgroup_size[i];
+      }
+   }
+   /* The record's object thread counts follow its header word. */
+   const uint64_t dispatch_threads = hk_cs_current_addr(cs) + 4;
    if (!agx_vdm_mesh_dispatch_pack(dev->dev.chip, &dispatch, cs->current)) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
       return;
    }
    cs->current += AGX_VDM_MESH_DISPATCH_LENGTH;
-
+   if (task_gpu_dispatch) {
+      libagx_m5_task_dispatch(cmd, agx_1d(1), AGX_BARRIER_ALL | AGX_PREGFX,
+                              dispatch_threads, task_flag, grid, count_address,
+                              conditional ? cmd->state.conditional.address : 0,
+                              draw_id,
+                              conditional && cmd->state.conditional.inverted,
+                              task->b.info.workgroup_size[0],
+                              task->b.info.workgroup_size[1],
+                              task->b.info.workgroup_size[2]);
+   }
+   hk_m5_note_render_resources(cmd, cs);
    cs->stats.calls++;
    hk_upload_stat(cmd, draws, 1);
    hk_note_pre_raster_images(cmd, cs);
@@ -10550,7 +12558,8 @@ hk_CmdDrawMeshTasksEXT(VkCommandBuffer commandBuffer, uint32_t groupCountX,
    if (!groupCountX || !groupCountY || !groupCountZ)
       return;
    const uint32_t grid[4] = {groupCountX, groupCountY, groupCountZ, 0};
-   hk_draw_mesh_native(cmd, hk_pool_upload(cmd, grid, sizeof(grid), 16), 0, 0);
+   hk_draw_mesh_native(cmd, hk_pool_upload(cmd, grid, sizeof(grid), 16), grid,
+                       0, 0);
 }
 
 VKAPI_ATTR void VKAPI_CALL
@@ -10561,7 +12570,8 @@ hk_CmdDrawMeshTasksIndirectEXT(VkCommandBuffer commandBuffer, VkBuffer buffer,
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    VK_FROM_HANDLE(hk_buffer, src, buffer);
    for (unsigned i = 0; i < drawCount; ++i)
-      hk_draw_mesh_native(cmd, hk_buffer_address_rw(src, offset + (uint64_t)i * stride), i, 0);
+      hk_draw_mesh_native(cmd, hk_buffer_address_rw(src, offset + (uint64_t)i * stride),
+                          NULL, i, 0);
 }
 
 VKAPI_ATTR void VKAPI_CALL
@@ -10576,7 +12586,7 @@ hk_CmdDrawMeshTasksIndirectCountEXT(VkCommandBuffer commandBuffer, VkBuffer buff
    const uint64_t count_address = hk_buffer_address_rw(count, countBufferOffset);
    for (unsigned i = 0; i < maxDrawCount; ++i)
       hk_draw_mesh_native(cmd, hk_buffer_address_rw(src, offset + (uint64_t)i * stride),
-                          i, count_address);
+                          NULL, i, count_address);
 }
 
 VKAPI_ATTR void VKAPI_CALL
@@ -10696,7 +12706,7 @@ hk_g17p_normalize_indirect_draw(const struct hk_cmd_buffer *cmd,
 }
 
 /*
- * Records of an indirect (multi-)draw on G17P whose draws would find nothing
+ * Records of an indirect (multi-)draw on G17P or M5 whose draws would find nothing
  * to do in hk_draw_impl() / hk_flush_gfx_state() /
  * hk_flush_dynamic_state_slow() except their own publication: no dirty
  * dynamic, shader, link, root, descriptor or vertex-buffer state, and none of
@@ -10712,6 +12722,10 @@ hk_g17p_normalize_indirect_draw(const struct hk_cmd_buffer *cmd,
  * later records' state publication followed by a reserved repair slot instead
  * (hk_emit_robust_index); no other compute work or stream split intervenes.
  *
+ * On M5 the general path also derives the HAL300 graphics core, which is then
+ * the one the stream already latched (hk_cs.m5_ppp_emitted), so it emits
+ * nothing there either.
+ *
  * The check reads no per-record value except the indirect address, and
  * emitting a record changes none of what it reads, so a multi-draw checks it
  * once (draw is already normalised).  It has no side effects.
@@ -10755,7 +12769,7 @@ hk_g17p_indirect_records_eligible(struct hk_cmd_buffer *cmd,
          VK_QUERY_PIPELINE_STATISTIC_VERTEX_SHADER_INVOCATIONS_BIT |
          VK_QUERY_PIPELINE_STATISTIC_CLIPPING_INVOCATIONS_BIT |
          VK_QUERY_PIPELINE_STATISTIC_CLIPPING_PRIMITIVES_BIT)) ||
-       dyn->rs.depth_bias.enable ||
+       hk_prims_generated_active(cmd) || dyn->rs.depth_bias.enable ||
        (dyn->ms.rasterization_samples && !gfx->render.tilebuffer.nr_samples))
       return false;
 
@@ -10837,7 +12851,8 @@ hk_g17p_emit_indirect_record(struct hk_cmd_buffer *cmd, uint16_t draw_id,
       /* A record after the first of a multi-draw republishes the previous
        * record's program with its own draw parameters. */
       const uint64_t pipeline_program =
-         continuing && agx_chip_native_agx3(dev->dev.chip)
+         continuing && (agx_chip_native_agx3(dev->dev.chip) ||
+                        agx_chip_g17_m5(dev->dev.chip))
             ? hk_m5_multi_draw_vertex_program(cmd, hw_vs, linked_hw_vs)
             : hk_upload_usc_words(cmd, hw_vs, linked_hw_vs);
 
@@ -10865,12 +12880,26 @@ hk_g17p_emit_indirect_record(struct hk_cmd_buffer *cmd, uint16_t draw_id,
    hk_upload_stat(cmd, draws, 1);
    if (batch) {
       hk_emit_robust_index(cmd, cs, draw, gfx->topology, tail_emitted, batch);
-   } else if (!hk_g17p_emit_draw(cmd, cs, cs->current, draw, gfx->topology,
-                                 tail_emitted)) {
-      return;
+   } else if (agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev)) {
+      if (!hk_g17p_emit_draw(cmd, cs, cs->current, draw, gfx->topology,
+                             tail_emitted))
+         return;
+   } else {
+      /* As hk_draw_impl() emits an indirect HAL300 draw: the packet carries
+       * its own restart state. */
+      cs->current = (void *)agx_vdm_draw((uint32_t *)cs->current,
+                                         hk_cmd_buffer_device(cmd)->dev.chip,
+                                         draw, gfx->topology,
+                                         hk_draw_restart_index(cmd, draw));
    }
    hk_g17p_note_layer_zero_draw(cmd, cs);
    hk_note_pre_raster_images(cmd, cs);
+   hk_m5_note_render_resources(cmd, cs);
+   if (unlikely(dev->frame_anatomy)) {
+      hk_frame_anatomy_note_fs(cs, hk_bound_fs(cmd));
+      hk_frame_anatomy_note_vs(cs, hk_bound_hw_vs(gfx));
+      hk_frame_anatomy_note_m5_fragment(cs);
+   }
    cs->stats.cmds++;
 }
 
@@ -10924,7 +12953,9 @@ hk_draw_indirect_inner(VkCommandBuffer commandBuffer, uint64_t base,
       assert(stride >= sizeof(VkDrawIndirectCommand));
    }
 
-   int eligible = agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) ? -1 : 0;
+   const struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   int eligible =
+      agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip) ? -1 : 0;
    for (unsigned draw_id = 0; draw_id < drawCount; ++draw_id) {
       uint64_t addr = base + stride * draw_id;
       hk_draw_indirect_record(cmd, draw_id, agx_draw_indirect(addr),
@@ -11036,7 +13067,9 @@ hk_draw_indexed_indirect_inner(VkCommandBuffer commandBuffer, uint64_t buffer,
    if (hk_g17p_draw_robust_multi(cmd, buffer, drawCount, stride))
       return;
 
-   int eligible = agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) ? -1 : 0;
+   const struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   int eligible =
+      agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip) ? -1 : 0;
    for (unsigned draw_id = 0; draw_id < drawCount; ++draw_id) {
       uint64_t addr = buffer + stride * draw_id;
       struct hk_addr_range buf = cmd->state.gfx.index.buffer;
diff --git a/src/asahi/vulkan/hk_cmd_meta.c b/src/asahi/vulkan/hk_cmd_meta.c
index b217455f63ff..7d63670b28a7 100644
--- a/src/asahi/vulkan/hk_cmd_meta.c
+++ b/src/asahi/vulkan/hk_cmd_meta.c
@@ -11,6 +11,7 @@
 #include "vulkan/vulkan_core.h"
 #include "agx_pack.h"
 #include "hk_buffer.h"
+#include "hk_buffer_view.h"
 #include "hk_cmd_buffer.h"
 #include "hk_device.h"
 #include "hk_entrypoints.h"
@@ -65,6 +66,7 @@ hk_device_init_meta(struct hk_device *dev)
    dev->meta.use_gs_for_layer = false;
    dev->meta.use_layer_output = true;
    dev->meta.use_view_relative_3d_layer = agx_native_graphics(&dev->dev);
+   dev->meta.specialize_single_layer_buffer_upload = agx_is_m4_pro(&dev->dev);
    /* G17P has ordinary fixed-function stencil testing/updates, but its
     * fragment stencil-export operand is not part of the grounded AGX3B ABI.
     * Let common meta lower a stencil blit to eight bit-selective discard
@@ -280,6 +282,7 @@ enum hk_meta_object_key_type {
    HK_META_OBJECT_KEY_SPARSE_BUFFER_COPY,
    HK_META_OBJECT_KEY_BLOCK_COPY,
    HK_META_OBJECT_KEY_G17P_MIP,
+   HK_META_OBJECT_KEY_G17P_BUFFER_FILL_PBE,
 };
 
 struct hk_meta_event_wait_info {
@@ -443,7 +446,8 @@ hk_g17p_sparse_buffer_copy(struct hk_cmd_buffer *cmd,
                            uint64_t dst_addr, VkDeviceSize size)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
-   assert(agx_native_graphics(&dev->dev) && src_buffer->sparse_map);
+   assert((agx_native_graphics(&dev->dev) ||
+           agx_chip_g17_m5(dev->dev.chip)) && src_buffer->sparse_map);
 
    /* Resident writes use the primary VA while nonresident reads must use the
     * zero-page shadow. Read the live map in the copy shader so a sparse bind
@@ -964,7 +968,8 @@ build_image_copy_shader(const struct vk_meta_image_copy_key *key,
 
    image->data.descriptor_set = 0;
    image->data.binding = BINDING_OUTPUT;
-   image->data.access = ACCESS_NON_READABLE;
+   /* Partial tile copies read the destination before preserving its pixels. */
+   image->data.access = key->block_based ? 0 : ACCESS_NON_READABLE;
 
    /* Multisampled stores use a linear PBE view. AGX2's address lowering
     * needs the texel size to convert the byte layer stride to samples.
@@ -1217,6 +1222,7 @@ get_image_copy_pipeline(struct vk_device *device, struct vk_meta_device *meta,
        * Meta pipelines are cached per device, so no extra key is needed. */
       .nir = build_image_copy_shader(key,
          !agx_native_graphics(&container_of(device, struct hk_device, vk)->dev) &&
+         !agx_chip_g17_m5(container_of(device, struct hk_device, vk)->dev.chip) &&
          container_of(device, struct hk_device, vk)->dev.chip != AGX_CHIP_G16G),
    };
    const VkPipelineShaderStageCreateInfo cs_info = {
@@ -1275,9 +1281,16 @@ hk_meta_copy_image_to_buffer2(struct vk_command_buffer *cmd,
 
          VkImageAspectFlags aspect = region->imageSubresource.aspectMask;
          VkFormat aspect_fmt = aspect_format(image->format, aspect);
+         /* M5 reads uncompressed colour through the same integer alias as
+          * G17P: a canonical R9G9B9E5_FLOAT (or UNORM) view decodes and
+          * re-encodes, which renormalizes shared-exponent texels and is not
+          * the bit-exact copy Vulkan specifies (queue_transfer e5b9g9r9). */
+         const enum agx_chip chip =
+            container_of(device, struct hk_device, vk)->dev.chip;
          const bool g17p_color =
-            agx_native_graphics(
-               &container_of(device, struct hk_device, vk)->dev) &&
+            (agx_native_graphics(
+                &container_of(device, struct hk_device, vk)->dev) ||
+             agx_chip_g17_m5(chip)) &&
             !vk_format_is_depth_or_stencil(image->format);
          const struct hk_image *hk_image =
             container_of(image, struct hk_image, vk);
@@ -1848,11 +1861,19 @@ hk_meta_bitwise_copy_format(VkFormat format)
 }
 
 static inline struct vk_meta_copy_image_properties
-hk_meta_copy_get_image_properties(struct hk_image *img, struct hk_device *dev)
+hk_meta_copy_get_image_properties(struct hk_image *img, struct hk_device *dev,
+                                  bool g17p, bool msaa_depth_stencil_aspect_views)
 {
-   bool g17p = agx_native_graphics(&dev->dev);
    const bool agx2 =
       agx_chip_isa_encoding(dev->dev.chip) == AGX_ISA_ENCODING_AGX2;
+   /* A split depth/stencil image has separate depth and stencil planes. The
+    * interleaved R32G32_UINT alias below is a colour view, which selects
+    * image plane 0 (the depth plane) for both aspects, so on M5 copy each
+    * plane through its own raw alias at every sample count. */
+   const bool aspect_views =
+      g17p && hk_split_depth_stencil(img->vk.format) &&
+      (img->vk.samples == VK_SAMPLE_COUNT_1_BIT ||
+       msaa_depth_stencil_aspect_views);
    /* Keep the existing multisample render-target formats: G17P's native
     * multisample tile path does not support these integer aliases yet. */
    g17p &= img->vk.samples == VK_SAMPLE_COUNT_1_BIT;
@@ -1862,7 +1883,7 @@ hk_meta_copy_get_image_properties(struct hk_image *img, struct hk_device *dev)
        * COLOR instead of the original aspect would view the depth plane as
        * interleaved RG32 data and lose the stencil plane altogether. */
       .depth_stencil_aspect_views =
-         (g17p || agx2) && hk_split_depth_stencil(img->vk.format),
+         aspect_views || (agx2 && hk_split_depth_stencil(img->vk.format)),
    };
    const struct vk_format_ycbcr_info *ycbcr =
       vk_format_get_ycbcr_info(img->vk.format);
@@ -2880,7 +2901,8 @@ hk_xfer_copybuffer2(VkCommandBuffer commandBuffer, const VkCopyBufferInfo2 *info
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) ||
+       agx_chip_g17_m5(dev->dev.chip)) {
       struct hk_meta_save save;
       hk_meta_begin(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
       /* Retain the aligned 16-byte bulk path even when the original
@@ -3010,7 +3032,8 @@ hk_g17p_copy_buffer_to_image_supported(
 static bool
 hk_g17p_block_copy_format(struct hk_device *dev, VkFormat format)
 {
-   if (!dev || !agx_native_graphics(&dev->dev))
+   if (!dev || (!agx_native_graphics(&dev->dev) &&
+                !agx_chip_g17_m5(dev->dev.chip)))
       return false;
 
    switch (hk_g17p_block_format_family(format)) {
@@ -3290,7 +3313,8 @@ hk_g17p_copy_padded_rgb_buffer_image(
    VkImageLayout image_layout, bool to_image)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
-   if (!agx_native_graphics(&dev->dev) || region_count == 0 ||
+   if ((!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) ||
+       region_count == 0 ||
        image->vk.samples != VK_SAMPLE_COUNT_1_BIT ||
        (image_layout != (to_image ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
                                   : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) &&
@@ -3466,7 +3490,8 @@ hk_g17p_copy_uncompressed_buffer_to_image_supported(
    struct hk_device *dev, const struct hk_buffer *buffer,
    const struct hk_image *image, const VkCopyBufferToImageInfo2 *info)
 {
-   if (!agx_native_graphics(&dev->dev) || info->regionCount == 0)
+   if ((!agx_native_graphics(&dev->dev) &&
+        !agx_chip_g17_m5(dev->dev.chip)) || info->regionCount == 0)
       return false;
 
    for (uint32_t r = 0; r < info->regionCount; ++r) {
@@ -3756,7 +3781,8 @@ hk_g17p_raw_image_copy_supported(struct hk_device *dev,
                                  const struct hk_image *dst,
                                  const VkCopyImageInfo2 *info)
 {
-   if (!agx_native_graphics(&dev->dev) || info->regionCount == 0 ||
+   if ((!agx_native_graphics(&dev->dev) && !agx_chip_g17_m5(dev->dev.chip)) ||
+       info->regionCount == 0 ||
        src->vk.samples != dst->vk.samples ||
        (src->vk.samples != VK_SAMPLE_COUNT_1_BIT &&
         src->vk.samples != VK_SAMPLE_COUNT_2_BIT &&
@@ -3984,8 +4010,21 @@ hk_xfer_copybuffertoimage2(VkCommandBuffer commandBuffer,
    VK_FROM_HANDLE(hk_image, dst_image, pCopyBufferToImageInfo->dstImage);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
 
+   /* Share the native raw tiled upload with M5. It preserves partial
+    * rectangles without an imageblock round trip or extra firmware job. */
+   if (agx_chip_g17_m5(dev->dev.chip) &&
+       hk_g17p_copy_uncompressed_buffer_to_image_supported(
+          dev, src_buffer, dst_image, pCopyBufferToImageInfo)) {
+      struct hk_meta_save save;
+      hk_meta_begin(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
+      hk_g17p_copy_uncompressed_buffer_to_image(
+         cmd, src_buffer, dst_image, pCopyBufferToImageInfo);
+      hk_meta_end(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
+      return;
+   }
+
    bool gfx = hk_copy_requires_gfx(dst_image);
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       uint8_t plane;
       VkFormat format;
       uint32_t api_block_B, physical_block_B;
@@ -4056,8 +4095,16 @@ hk_xfer_copybuffertoimage2(VkCommandBuffer commandBuffer,
    hk_meta_begin(cmd, &save, bind_point);
 
    if (gfx) {
+      /* M5 reaches this graphics upload only for a destination the raw
+       * upload above refuses, such as a compressed depth plane. A split
+       * depth/stencil destination needs the per-plane aspect views here as
+       * well: the interleaved R32G32_UINT alias is a view of plane 0 alone. */
+      const bool m5_split_zs = agx_chip_g17_m5(dev->dev.chip) &&
+                               hk_split_depth_stencil(dst_image->vk.format);
       struct vk_meta_copy_image_properties dst_props =
-         hk_meta_copy_get_image_properties(dst_image, dev);
+         hk_meta_copy_get_image_properties(
+            dst_image, dev, agx_native_graphics(&dev->dev) || m5_split_zs,
+            m5_split_zs);
 
       if (agx_native_graphics(&dev->dev)) {
          /* Native single-slice EOT views already select the mip and array
@@ -4252,7 +4299,7 @@ hk_xfer_copyimagetobuffer2(VkCommandBuffer commandBuffer,
    VkFormat format;
    uint32_t api_block_B, physical_block_B;
    const bool padded_rgb =
-      agx_native_graphics(&dev->dev) &&
+      (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
       pCopyImageToBufferInfo->regionCount > 0 &&
       hk_g17p_buffer_upload_plane(
          src_image,
@@ -4268,8 +4315,7 @@ hk_xfer_copyimagetobuffer2(VkCommandBuffer commandBuffer,
              pCopyImageToBufferInfo->pRegions,
              pCopyImageToBufferInfo->srcImageLayout, false))
          vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
-   } else if (agx_native_graphics(&dev->dev) &&
-              hk_g17p_block_copy_format(dev, src_image->vk.format)) {
+   } else if (hk_g17p_block_copy_format(dev, src_image->vk.format)) {
       /* A compressed Vulkan copy preserves the encoded block bytes.  The
        * common meta path samples and decodes the source, so use the symmetric
        * raw AIL-addressed path on G17P instead. */
@@ -4308,6 +4354,7 @@ hk_xfer_copyimage2(VkCommandBuffer commandBuffer,
     * The common meta path preserves partial rectangles, layers and format
     * reinterpretation without the AGX2 compute imageblock protocol. */
    bool gfx = agx_native_graphics(&dev->dev) ||
+              agx_chip_g17_m5(dev->dev.chip) ||
               hk_copy_requires_gfx(dst_image);
    VkPipelineBindPoint bind_point = hk_bind_point(gfx);
 
@@ -4315,10 +4362,13 @@ hk_xfer_copyimage2(VkCommandBuffer commandBuffer,
    hk_meta_begin(cmd, &save, bind_point);
 
    if (gfx) {
+      const bool native = agx_native_graphics(&dev->dev) ||
+                          agx_chip_g17_m5(dev->dev.chip);
+      const bool m5 = agx_chip_g17_m5(dev->dev.chip);
       struct vk_meta_copy_image_properties src_props =
-         hk_meta_copy_get_image_properties(src_image, dev);
+         hk_meta_copy_get_image_properties(src_image, dev, native, m5);
       struct vk_meta_copy_image_properties dst_props =
-         hk_meta_copy_get_image_properties(dst_image, dev);
+         hk_meta_copy_get_image_properties(dst_image, dev, native, m5);
 
       if (agx_native_graphics(&dev->dev)) {
          /* The qualified HAL200 BG/EOT contract is single-layer.  Preserve
@@ -4371,6 +4421,173 @@ hk_xfer_copyimage2(VkCommandBuffer commandBuffer,
    hk_meta_end(cmd, &save, bind_point);
 }
 
+static void
+hk_g17p_fill_buffer_pbe(struct hk_cmd_buffer *cmd, VkBuffer buffer,
+                        VkDeviceSize offset, VkDeviceSize size, uint32_t data,
+                        const struct hk_native_pbe_descriptor *expected)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const uint32_t key = HK_META_OBJECT_KEY_G17P_BUFFER_FILL_PBE;
+   const VkDescriptorSetLayoutBinding binding = {
+      .binding = 0,
+      .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
+      .descriptorCount = 1,
+      .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
+   };
+   const VkDescriptorSetLayoutCreateInfo desc_info = {
+      .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
+      .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
+      .bindingCount = 1,
+      .pBindings = &binding,
+   };
+   const struct {
+      uint32_t elements;
+      uint32_t data;
+   } push = {size / 16, data};
+   const VkPushConstantRange push_range = {
+      .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
+      .size = sizeof(push),
+   };
+   VkPipelineLayout layout;
+   VkResult result = vk_meta_get_pipeline_layout(
+      &dev->vk, &dev->meta, &desc_info, &push_range,
+      &key, sizeof(key), &layout);
+   if (result != VK_SUCCESS) {
+      vk_command_buffer_set_error(&cmd->vk, result);
+      return;
+   }
+
+   VkPipeline pipeline = vk_meta_lookup_pipeline(&dev->meta, &key, sizeof(key));
+   if (pipeline == VK_NULL_HANDLE) {
+      nir_builder b = nir_builder_init_simple_shader(
+         MESA_SHADER_COMPUTE, NULL, "hk-meta-fill-buffer-pbe");
+      b.shader->info.workgroup_size[0] = 64;
+      b.shader->info.workgroup_size[1] = 1;
+      b.shader->info.workgroup_size[2] = 1;
+      nir_variable *image = nir_variable_create(
+         b.shader, nir_var_image,
+         glsl_image_type(GLSL_SAMPLER_DIM_BUF, false, GLSL_TYPE_UINT), "dest");
+      image->data.descriptor_set = 0;
+      image->data.binding = 0;
+      image->data.image.format = PIPE_FORMAT_R32G32B32A32_UINT;
+      image->data.access = ACCESS_NON_READABLE;
+      nir_def *index = nir_channel(&b, nir_load_global_invocation_id(&b, 32), 0);
+      nir_def *elements = nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0));
+      nir_def *value = nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 4));
+      nir_push_if(&b, nir_ult(&b, index, elements));
+      nir_image_deref_store(
+         &b, &nir_build_deref_var(&b, image)->def, nir_pad_vec4(&b, index),
+         nir_imm_int(&b, 0), nir_replicate(&b, value, 4), nir_imm_int(&b, 0),
+         .image_dim = GLSL_SAMPLER_DIM_BUF, .image_array = false,
+         .format = PIPE_FORMAT_R32G32B32A32_UINT, .src_type = nir_type_uint32);
+      nir_pop_if(&b, NULL);
+
+      const VkPipelineShaderStageNirCreateInfoMESA nir_info = {
+         .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_NIR_CREATE_INFO_MESA,
+         .nir = b.shader,
+      };
+      const VkComputePipelineCreateInfo info = {
+         .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+         .stage = {
+            .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+            .pNext = &nir_info,
+            .stage = VK_SHADER_STAGE_COMPUTE_BIT,
+            .pName = "main",
+         },
+         .layout = layout,
+      };
+      result = vk_meta_create_compute_pipeline(
+         &dev->vk, &dev->meta, &info, &key, sizeof(key), &pipeline);
+      ralloc_free(b.shader);
+      if (result != VK_SUCCESS) {
+         vk_command_buffer_set_error(&cmd->vk, result);
+         return;
+      }
+   }
+
+   /* The view is owned by this recording until its normal reset/destruction.
+    * Its PBE addresses the original buffer; no image or alias is allocated. */
+   const VkBufferViewCreateInfo view_info = {
+      .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
+      .buffer = buffer,
+      .format = VK_FORMAT_R32G32B32A32_UINT,
+      .offset = offset,
+      .range = size,
+   };
+   VkBufferView view;
+   result = vk_meta_create_buffer_view(&cmd->vk, &dev->meta, &view_info, &view);
+   if (result != VK_SUCCESS) {
+      vk_command_buffer_set_error(&cmd->vk, result);
+      return;
+   }
+
+   /* CreateBufferView can represent a refused format with a null PBE.
+    * Require the exact descriptor accepted by our existing codec preflight. */
+   VK_FROM_HANDLE(hk_buffer_view, native_view, view);
+   if (memcmp(&native_view->pbe, expected, sizeof(*expected)) != 0) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FORMAT_NOT_SUPPORTED);
+      return;
+   }
+
+   const VkWriteDescriptorSet write = {
+      .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
+      .dstBinding = 0,
+      .descriptorCount = 1,
+      .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
+      .pTexelBufferView = &view,
+   };
+   const struct vk_device_dispatch_table *disp = &dev->vk.dispatch_table;
+   VkCommandBuffer cb = vk_command_buffer_to_handle(&cmd->vk);
+   disp->CmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
+   disp->CmdPushDescriptorSetKHR(cb, VK_PIPELINE_BIND_POINT_COMPUTE,
+                                layout, 0, 1, &write);
+   disp->CmdPushConstants(cb, layout, VK_SHADER_STAGE_COMPUTE_BIT,
+                          0, sizeof(push), &push);
+   disp->CmdDispatch(cb, DIV_ROUND_UP(push.elements, 64), 1, 1);
+}
+
+static bool
+hk_g17p_fill_buffer_pbe_supported(const struct hk_device *dev,
+                                  const struct hk_buffer *buffer,
+                                  uint64_t address, uint64_t offset,
+                                  uint64_t size,
+                                  struct hk_native_pbe_descriptor *pbe)
+{
+   const struct hk_physical_device *pdev =
+      (const struct hk_physical_device *)dev->vk.physical;
+   if (dev->dev.chip != AGX_CHIP_G17P || buffer->va ||
+       size < (64u << 20) || ((address | offset | size) & 15) ||
+       !address || size > UINT64_MAX - address ||
+       size / 16 > pdev->vk.properties.maxTexelBufferElements)
+      return false;
+
+   /* Match CreateBufferView's ordinary RGBA32_UINT linear PBE. Let the
+    * existing codec prove address, stride and dimension representability. */
+   struct AGX_PBE semantic = {
+      AGX_PBE_header,
+      .dimension = AGX_TEXTURE_DIMENSION_2D,
+      .layout = AGX_LAYOUT_LINEAR,
+      .channels = AGX_CHANNELS_R32G32B32A32,
+      .type = AGX_TEXTURE_TYPE_UINT,
+      .swizzle_r = 0, .swizzle_g = 1, .swizzle_b = 2, .swizzle_a = 3,
+      .buffer = address,
+      .width = AGX_TEXTURE_BUFFER_WIDTH,
+      .height = DIV_ROUND_UP(size / 16, AGX_TEXTURE_BUFFER_WIDTH),
+      .stride = AGX_TEXTURE_BUFFER_WIDTH * 16 - 4,
+      .layers = 1,
+      .levels = 1,
+   };
+   const unsigned generation = agx_chip_descriptor_generation(dev->dev.chip);
+   if (!hk_pack_native_pbe(generation, pbe, &semantic))
+      return false;
+
+   /* The last stored texel must stay in the codec's supported address domain
+    * too; size and address were already checked for alignment and overflow. */
+   semantic.buffer = address + size - 16;
+   struct hk_native_pbe_descriptor last;
+   return hk_pack_native_pbe(generation, &last, &semantic);
+}
+
 static void
 hk_xfer_fillbuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer,
                  VkDeviceSize dstOffset, VkDeviceSize dstRange, uint32_t data)
@@ -4387,7 +4604,12 @@ hk_xfer_fillbuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer,
          vk_meta_buffer_address(&dev->vk, dstBuffer, dstOffset, dstRange);
       struct hk_meta_save save;
       hk_meta_begin(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
-      vk_meta_fill_buffer_addr(&cmd->vk, &dev->meta, address, size, data);
+      struct hk_native_pbe_descriptor pbe;
+      if (hk_g17p_fill_buffer_pbe_supported(dev, buffer, address, dstOffset,
+                                           size, &pbe))
+         hk_g17p_fill_buffer_pbe(cmd, dstBuffer, dstOffset, size, data, &pbe);
+      else
+         vk_meta_fill_buffer_addr(&cmd->vk, &dev->meta, address, size, data);
       hk_meta_end(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
       if (size && cmd->current_cs.cs) {
          hk_drm_attachment_set_add(&cmd->current_cs.cs->compute_attachments,
diff --git a/src/asahi/vulkan/hk_cmd_pool.c b/src/asahi/vulkan/hk_cmd_pool.c
index 012626c16b84..1becf61a6493 100644
--- a/src/asahi/vulkan/hk_cmd_pool.c
+++ b/src/asahi/vulkan/hk_cmd_pool.c
@@ -500,7 +500,8 @@ hk_cmd_pool_free_bo_list(struct hk_cmd_pool *pool, enum hk_cmd_bo_type type,
 {
    const struct hk_device *dev = hk_cmd_pool_device(pool);
    const unsigned max_free =
-      agx_native_graphics(&dev->dev) && type == HK_CMD_BO_MAIN
+      (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+         type == HK_CMD_BO_MAIN
          ? HK_G17P_CMD_POOL_MAIN_BO_MAX
          : HK_CMD_POOL_BO_MAX;
 
diff --git a/src/asahi/vulkan/hk_cmd_pool.h b/src/asahi/vulkan/hk_cmd_pool.h
index f2731a2e1f4b..72a1007a2679 100644
--- a/src/asahi/vulkan/hk_cmd_pool.h
+++ b/src/asahi/vulkan/hk_cmd_pool.h
@@ -34,6 +34,9 @@ struct hk_cmd_bo {
    /* Full VA of the G17P VDM alias in the architectural user window. */
    uint64_t g17p_vdm_alias;
 
+   /* This recording allocated the BO only through uploader.streams. */
+   bool stream_only;
+
    /** Link in a matching command-pool or command-buffer BO list. */
    struct list_head link;
 };
diff --git a/src/asahi/vulkan/hk_cmd_trace_rays.c b/src/asahi/vulkan/hk_cmd_trace_rays.c
index 7d723b8059dd..8cd0984fb073 100644
--- a/src/asahi/vulkan/hk_cmd_trace_rays.c
+++ b/src/asahi/vulkan/hk_cmd_trace_rays.c
@@ -5,10 +5,21 @@
  * vkCmdTraceRaysKHR and its indirect forms.
  *
  * Every form publishes the same record, VkTraceRaysIndirectCommand2KHR, and
- * dispatches the pipeline's megakernel with the record's address in the compute
+ * dispatches the pipeline's kernels with the record's address in the compute
  * root descriptor table.  The direct form fills the record on the host; the
  * indirect forms let one GPU invocation fill in the launch size, because that
  * is the only part that is not known until the command executes.
+ *
+ * Elsewhere a pipeline is one megakernel; on M5 it has one kernel per
+ * ray-generation group.  Which group a trace runs
+ * is named by the raygen shader binding table record, and that record is
+ * application memory whose contents are only required to be valid when the
+ * command executes -- not when it is recorded.  So for a pipeline with several
+ * raygen groups the selection is made on the GPU: a setup kernel reads the
+ * record's group handle and publishes one VkDispatchIndirectCommand per kernel
+ * (the real threadgroup count for the matching kernel, zero for the others),
+ * and one indirect dispatch is recorded per kernel.  A single-raygen pipeline
+ * needs no selection and keeps the previous direct dispatch.
  */
 
 #include "hk_buffer.h"
@@ -19,6 +30,11 @@
 #include "hk_shader.h"
 
 #include "asahi/lib/agx_bo.h"
+#include "asahi/genxml/agx3_cmdstream.h"
+#include "asahi/lib/agx3_compute_program_core.h"
+#include "util/u_debug.h"
+#include <inttypes.h>
+#include <stdio.h>
 #include "libagx_shaders.h"
 
 static VkTraceRaysIndirectCommand2KHR
@@ -50,12 +66,13 @@ hk_trace_rays_record(const VkStridedDeviceAddressRegionKHR *raygen,
    };
 }
 
-static struct hk_api_shader *
-hk_bound_rt_shader(struct hk_cmd_buffer *cmd)
+static struct hk_ray_tracing_pipeline *
+hk_bound_rt_pipeline(struct hk_cmd_buffer *cmd)
 {
    struct hk_ray_tracing_pipeline *pipeline = cmd->state.rt.pipeline;
 
-   if (pipeline == NULL || pipeline->shader == NULL) {
+   if (pipeline == NULL ||
+       (pipeline->shader == NULL && pipeline->kernel_count == 0)) {
       /* A pipeline library is never executable and a failed create returns
        * VK_NULL_HANDLE.  Recording a trace against either is application
        * error; report it rather than dispatching nothing silently. */
@@ -63,7 +80,175 @@ hk_bound_rt_shader(struct hk_cmd_buffer *cmd)
       return NULL;
    }
 
-   return pipeline->shader;
+   return pipeline;
+}
+
+/* The one kernel a trace dispatches without a GPU-side selection: the
+ * megakernel, or the only per-raygen kernel of an M5 pipeline. */
+static struct hk_api_shader *
+hk_rt_single_kernel(const struct hk_ray_tracing_pipeline *pipeline)
+{
+   return pipeline->shader ? pipeline->shader : pipeline->kernels[0].shader;
+}
+
+/*
+ * Publish the per-kernel threadgroup counts from the GPU and record one
+ * indirect dispatch per kernel.  `record_addr` is the record the kernels will
+ * read; `dims_addr` is where the three launch dimensions live right now;
+ * `record_dims_addr` is where they must be copied to, or zero when the record
+ * already contains them.
+ */
+static bool
+hk_m5_rt_single_kick_enabled(void)
+{
+   static int enabled = -1;
+   if (enabled < 0)
+      enabled = debug_get_bool_option("ASAHI_M5_RT_SINGLE_KICK", true);
+   return enabled;
+}
+
+/*
+ * Single-kick form of the per-raygen selection on M5 (default; set
+ * ASAHI_M5_RT_SINGLE_KICK=0 to record K indirect dispatches instead), shaped like
+ * Metal's indirect command buffers (as the native M5 driver executes them):
+ * the host encodes each kernel's direct launch as a template,
+ * one select invocation writes the chosen launch into a slot, and the control
+ * stream reaches the slot through barrier 0x60000960 and a Stream Link with
+ * return, so the CDM sees exactly one launch instead of K indirect ones.
+ * Returns false (and records nothing) if a kernel could not be templated.
+ */
+static bool
+hk_trace_rays_single_kick(struct hk_cmd_buffer *cmd,
+                          struct hk_ray_tracing_pipeline *pipeline,
+                          uint64_t record_addr, uint64_t record_dims_addr,
+                          uint64_t dims_addr, uint64_t handles_addr)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const uint32_t count = pipeline->kernel_count;
+   const unsigned T = AGX3_G17P_NATIVE_COMPUTE_DISPATCH_SIZE;
+   if (T != 44)
+      return false;
+
+   struct agx_ptr templates = hk_pool_alloc(cmd, count * T, 64);
+   struct agx_ptr slot = hk_pool_alloc(cmd, 13 * 4, 64);
+   struct agx_ptr groups =
+      hk_pool_alloc(cmd, sizeof(VkDispatchIndirectCommand), 8);
+   if (!templates.cpu || !slot.cpu || !groups.cpu)
+      return false;
+   memset(templates.cpu, 0, count * T);
+   memset(groups.cpu, 0, sizeof(VkDispatchIndirectCommand));
+
+   /* Template every kernel first: this records no stream words, only each
+    * kernel's IOTO program and resource profile. */
+   cmd->m5_rt_state.share = true;
+   cmd->m5_rt_state.state = (struct agx_ptr){0};
+   cmd->m5_rt_state.uniforms = (struct agx_ptr){0};
+   bool ok = true;
+   for (uint32_t k = 0; k < count && ok; ++k) {
+      cmd->m5_rt_capture = (uint8_t *)templates.cpu + T * k;
+      cmd->m5_rt_captured = false;
+      hk_cmd_dispatch_rays(cmd, pipeline->kernels[k].shader, record_addr,
+                           groups.gpu, agx_3d(1, 1, 1));
+      ok = cmd->m5_rt_captured;
+   }
+   cmd->m5_rt_capture = NULL;
+   cmd->m5_rt_captured = false;
+   cmd->m5_rt_state.share = false;
+   cmd->m5_rt_state.state = (struct agx_ptr){0};
+   cmd->m5_rt_state.uniforms = (struct agx_ptr){0};
+   if (!ok)
+      return false;
+
+   uint32_t *words = slot.cpu;
+   memset(words, 0, 13 * 4);
+   words[0] = 0x60000000; /* the empty barrier slot Metal leads a record with */
+   words[12] = 0x80000000; /* Stream Return */
+
+   libagx_trace_rays_select_launch(cmd, agx_1d(1), AGX_BARRIER_ALL,
+                                   record_addr, record_dims_addr, groups.gpu,
+                                   dims_addr, handles_addr, templates.gpu,
+                                   slot.gpu, count,
+                                   HK_RT_WORKGROUP_SIZE_X_LOG2);
+
+   struct hk_cs *cs = hk_cmd_buffer_get_cs(cmd, true /* compute */);
+   if (!cs)
+      return true;
+   hk_ensure_cs_has_space(cmd, cs, 0x40);
+   const struct agx3_cdm_barrier_info barrier = {
+      .chip = dev->dev.chip, .flags = 2, /* 0x60000960, as Metal's */
+   };
+   const struct agx3_cdm_stream_link_info link = {
+      .chip = dev->dev.chip, .target = slot.gpu, .with_return = true,
+   };
+   uint8_t *out = (uint8_t *)cs->current;
+   if (!agx3_cdm_barrier_pack(&barrier, out) ||
+       !agx3_cdm_stream_link_pack(&link, out + AGX3_CDM_BARRIER_SIZE)) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return true;
+   }
+   cs->current = (uint32_t *)(out + AGX3_CDM_BARRIER_SIZE +
+                              AGX3_CDM_STREAM_LINK_SIZE);
+   cs->stats.cmds++;
+   if (debug_get_bool_option("ASAHI_M5_LAUNCH_DEBUG", false))
+      fprintf(stderr, "M5 single-kick: %u templates, slot 0x%" PRIx64 "\n",
+              count, slot.gpu);
+   return true;
+}
+
+static void
+hk_trace_rays_select_and_dispatch(struct hk_cmd_buffer *cmd,
+                                  struct hk_ray_tracing_pipeline *pipeline,
+                                  uint64_t record_addr,
+                                  uint64_t record_dims_addr,
+                                  uint64_t dims_addr)
+{
+   const uint32_t count = pipeline->kernel_count;
+   if (debug_get_bool_option("ASAHI_M5_LAUNCH_DEBUG", false))
+      fprintf(stderr, "M5 trace rays: %u raygen kernels, single kick %d\n",
+              count, hk_m5_rt_single_kick_enabled());
+
+   uint32_t handles[count];
+   for (uint32_t k = 0; k < count; ++k)
+      handles[k] = pipeline->kernels[k].raygen_handle_index;
+
+   const uint64_t handles_addr =
+      hk_pool_upload(cmd, handles, sizeof(handles), 4);
+   if (!handles_addr)
+      return;
+
+   if (agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip) &&
+       hk_m5_rt_single_kick_enabled() &&
+       hk_trace_rays_single_kick(cmd, pipeline, record_addr, record_dims_addr,
+                                 dims_addr, handles_addr))
+      return;
+
+   const uint64_t groups_addr =
+      hk_pool_alloc(cmd, count * sizeof(VkDispatchIndirectCommand), 8).gpu;
+   if (!groups_addr)
+      return;
+
+   libagx_trace_rays_select_setup(cmd, agx_1d(1), AGX_BARRIER_ALL, record_addr,
+                                  record_dims_addr, groups_addr, dims_addr,
+                                  handles_addr, count,
+                                  HK_RT_WORKGROUP_SIZE_X_LOG2);
+
+   /* Exactly one kernel gets threadgroups; the others are empty dispatches.
+    * Let them share one SCS state block (when the device block is disabled)
+    * and one set of traversal uniforms and AS table. */
+   cmd->m5_rt_state.share = true;
+   cmd->m5_rt_state.state = (struct agx_ptr){0};
+   cmd->m5_rt_state.uniforms = (struct agx_ptr){0};
+   for (uint32_t k = 0; k < count; ++k) {
+      const uint64_t group_count_addr =
+         groups_addr + k * sizeof(VkDispatchIndirectCommand);
+
+      hk_cmd_dispatch_rays(cmd, pipeline->kernels[k].shader, record_addr,
+                           group_count_addr,
+                           agx_grid_indirect(group_count_addr));
+   }
+   cmd->m5_rt_state.share = false;
+   cmd->m5_rt_state.state = (struct agx_ptr){0};
+   cmd->m5_rt_state.uniforms = (struct agx_ptr){0};
 }
 
 VKAPI_ATTR void VKAPI_CALL
@@ -76,8 +261,8 @@ hk_CmdTraceRaysKHR(VkCommandBuffer commandBuffer,
 {
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
 
-   struct hk_api_shader *shader = hk_bound_rt_shader(cmd);
-   if (!shader)
+   struct hk_ray_tracing_pipeline *pipeline = hk_bound_rt_pipeline(cmd);
+   if (!pipeline)
       return;
 
    if (width == 0 || height == 0 || depth == 0)
@@ -93,6 +278,14 @@ hk_CmdTraceRaysKHR(VkCommandBuffer commandBuffer,
    if (!record_addr)
       return;
 
+   if (pipeline->kernel_count > 1) {
+      /* The record already holds the dimensions; only the selection is
+       * execution-time work. */
+      hk_trace_rays_select_and_dispatch(cmd, pipeline, record_addr, 0,
+                                        record_addr + HK_RT_CMD_WIDTH);
+      return;
+   }
+
    uint32_t groups[3];
    hk_rt_launch_group_count(width, height, depth, groups);
 
@@ -108,21 +301,29 @@ hk_CmdTraceRaysKHR(VkCommandBuffer commandBuffer,
 
    /* hk_cmd_dispatch_rays() takes the same grid every API dispatch takes:
     * threadgroups, which the compute path multiplies by the workgroup size. */
-   hk_cmd_dispatch_rays(cmd, shader, record_addr, group_count_addr,
+   hk_cmd_dispatch_rays(cmd, hk_rt_single_kernel(pipeline), record_addr,
+                        group_count_addr,
                         agx_3d(groups[0], groups[1], groups[2]));
 }
 
 /*
  * The shared tail of both indirect forms.  `record_addr` is the record the
- * megakernel will read; `dims_addr` is where the three launch dimensions live
+ * kernels will read; `dims_addr` is where the three launch dimensions live
  * right now; `record_dims_addr` is where they must be copied to, or zero when
  * the record already contains them.
  */
 static void
-hk_trace_rays_indirect(struct hk_cmd_buffer *cmd, struct hk_api_shader *shader,
+hk_trace_rays_indirect(struct hk_cmd_buffer *cmd,
+                       struct hk_ray_tracing_pipeline *pipeline,
                        uint64_t record_addr, uint64_t record_dims_addr,
                        uint64_t dims_addr)
 {
+   if (pipeline->kernel_count > 1) {
+      hk_trace_rays_select_and_dispatch(cmd, pipeline, record_addr,
+                                        record_dims_addr, dims_addr);
+      return;
+   }
+
    const uint64_t group_count_addr =
       hk_pool_alloc(cmd, sizeof(VkDispatchIndirectCommand), 8).gpu;
    if (!group_count_addr)
@@ -132,8 +333,8 @@ hk_trace_rays_indirect(struct hk_cmd_buffer *cmd, struct hk_api_shader *shader,
                                     record_dims_addr, group_count_addr,
                                     dims_addr, HK_RT_WORKGROUP_SIZE_X_LOG2);
 
-   hk_cmd_dispatch_rays(cmd, shader, record_addr, group_count_addr,
-                        agx_grid_indirect(group_count_addr));
+   hk_cmd_dispatch_rays(cmd, hk_rt_single_kernel(pipeline), record_addr,
+                        group_count_addr, agx_grid_indirect(group_count_addr));
 }
 
 VKAPI_ATTR void VKAPI_CALL
@@ -147,8 +348,8 @@ hk_CmdTraceRaysIndirectKHR(
 {
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
 
-   struct hk_api_shader *shader = hk_bound_rt_shader(cmd);
-   if (!shader)
+   struct hk_ray_tracing_pipeline *pipeline = hk_bound_rt_pipeline(cmd);
+   if (!pipeline)
       return;
 
    /* The application's buffer holds only a VkTraceRaysIndirectCommandKHR, so
@@ -163,7 +364,7 @@ hk_CmdTraceRaysIndirectKHR(
    if (!record_addr)
       return;
 
-   hk_trace_rays_indirect(cmd, shader, record_addr,
+   hk_trace_rays_indirect(cmd, pipeline, record_addr,
                           record_addr + HK_RT_CMD_WIDTH, indirectDeviceAddress);
 }
 
@@ -173,13 +374,13 @@ hk_CmdTraceRaysIndirect2KHR(VkCommandBuffer commandBuffer,
 {
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
 
-   struct hk_api_shader *shader = hk_bound_rt_shader(cmd);
-   if (!shader)
+   struct hk_ray_tracing_pipeline *pipeline = hk_bound_rt_pipeline(cmd);
+   if (!pipeline)
       return;
 
-   /* The application supplied the whole record, so the megakernel can read it
+   /* The application supplied the whole record, so the kernels can read it
     * where it is; only the threadgroup counts still have to be derived. */
-   hk_trace_rays_indirect(cmd, shader, indirectDeviceAddress, 0,
+   hk_trace_rays_indirect(cmd, pipeline, indirectDeviceAddress, 0,
                           indirectDeviceAddress + HK_RT_CMD_WIDTH);
 }
 
@@ -187,7 +388,7 @@ VKAPI_ATTR void VKAPI_CALL
 hk_CmdSetRayTracingPipelineStackSizeKHR(VkCommandBuffer commandBuffer,
                                         uint32_t pipelineStackSize)
 {
-   /* The megakernel's stack is static scratch inside its compiled binary, sized
+   /* Each kernel's stack is static scratch inside its compiled binary, sized
     * at pipeline creation for maxPipelineRayRecursionDepth.  A dynamic size
     * cannot shrink it and must not grow it, so this is deliberately a no-op
     * rather than a partial implementation.  Documented in
diff --git a/src/asahi/vulkan/hk_cmd_upload.c b/src/asahi/vulkan/hk_cmd_upload.c
index 2bf0fd858311..efaf8dfbd0c4 100644
--- a/src/asahi/vulkan/hk_cmd_upload.c
+++ b/src/asahi/vulkan/hk_cmd_upload.c
@@ -4,6 +4,10 @@
 #include "hk_device.h"
 #include "hk_shader.h"
 #include "util/u_cpu_detect.h"
+#include "util/u_debug.h"
+
+DEBUG_GET_ONCE_BOOL_OPTION(m5_fragment_root_shadow,
+                           "ASAHI_M5_FRAGMENT_ROOT_SHADOW", true)
 
 static struct agx_ptr
 hk_pool_root_alloc(struct hk_cmd_buffer *cmd, uint32_t size,
@@ -38,6 +42,10 @@ hk_cmd_buffer_upload_root(struct hk_cmd_buffer *cmd,
       dev->samplers.table.bo && dev->samplers.table.bo->va
          ? dev->samplers.table.bo->va->addr
          : 0;
+   root->physical_sparse_map = dev->physical_sparse_map
+      ? dev->physical_sparse_map->va->addr : 0;
+   root->physical_sparse_base = dev->dev.main_heap_base;
+   root->physical_sparse_size = dev->dev.main_heap_size;
 
    /* Dynamic buffers past the highest one ever bound are read by no bound
     * layout. Never read the uninitialized hole in an earlier upload. */
@@ -60,6 +68,8 @@ hk_cmd_buffer_upload_root(struct hk_cmd_buffer *cmd,
              cmd->state.gfx.root &&
              desc->uploaded_root.gpu != cmd->state.gfx.root)
             cmd->state.gfx.g17p_bind_tail.valid = false;
+         if (bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS)
+            cmd->state.gfx.root_cpu = desc->uploaded_root.cpu;
          hk_upload_stat(cmd, root_reuses, 1);
          return desc->uploaded_root.gpu;
       }
@@ -69,12 +79,24 @@ hk_cmd_buffer_upload_root(struct hk_cmd_buffer *cmd,
    if (!root_ptr.gpu)
       return 0;
 
+   if (bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS)
+      cmd->state.gfx.root_cpu = root_ptr.cpu;
+
    /* A HAL200 tail built for the old root names it and must be rebuilt
     * before publication. Byte-identical roots keep that identity. */
    if (bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS &&
        cmd->state.gfx.root && root_ptr.gpu != cmd->state.gfx.root)
       cmd->state.gfx.g17p_bind_tail.valid = false;
    root->root_desc_addr = root_ptr.gpu;
+   /* Capture from the CPU source after every publication-time augmentation.
+    * Reading the GPU-mapped root back here would pay uncached load latency. */
+   if (bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS &&
+       agx_chip_g17_m5(dev->dev.chip) &&
+       debug_get_option_m5_fragment_root_shadow()) {
+      struct hk_graphics_state *gfx = &cmd->state.gfx;
+      hk_root_fragment_snapshot_publish(gfx, root_ptr.cpu, root);
+   }
+
    memcpy(root_ptr.cpu, root, head_B);
    if (tail_B < root_size)
       memcpy((uint8_t *)root_ptr.cpu + tail_B, (const uint8_t *)root + tail_B,
diff --git a/src/asahi/vulkan/hk_descriptor_set.c b/src/asahi/vulkan/hk_descriptor_set.c
index 670c45ce2d3d..1d8d9ab7979c 100644
--- a/src/asahi/vulkan/hk_descriptor_set.c
+++ b/src/asahi/vulkan/hk_descriptor_set.c
@@ -93,6 +93,12 @@ write_accel_struct_desc(struct hk_descriptor_set *set,
    VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, _accel_struct);
    uint64_t addr =
       accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0;
+   if (accel_struct) {
+      struct hk_device *dev = container_of(accel_struct->base.device,
+                                             struct hk_device, vk);
+      if (dev->m5_native_as)
+         addr = ALIGN_POT(addr, UINT64_C(1024));
+   }
 
    write_desc(set, binding, elem, &addr, sizeof(addr));
 }
@@ -426,13 +432,15 @@ write_storage_image_view_desc(struct hk_descriptor_set *set,
          view->planes[0].storage_pbe_valid;
       supported = hk_native_pbe_supported(generation) || native_basic;
       /* A readable native texture does not require a writable PBE. In
-       * particular, linear storage views have a valid texture descriptor
-       * even when the native PBE packer cannot represent their layout.
+       * particular, linear and multisampled storage views have a valid
+       * texture descriptor even when the native PBE packer cannot represent
+       * their layout.
        * Keep imageLoad/imageSize bound to that view instead of replacing
        * both halves with a null descriptor because stores are unsupported.
        */
       texture_supported = supported ||
-         (generation == 9 && view->plane_count == 1);
+         (hk_descriptor_generation_is_agx3(generation) &&
+          view->plane_count == 1);
       if (texture_supported) {
          /* Storage images are always single plane */
          assert(view->plane_count == 1);
@@ -558,9 +566,11 @@ write_buffer_view_desc(struct hk_descriptor_set *set,
       hk_storage_image_descriptor_layout(generation);
    uint32_t heap_index = 0;
 
-   /* G17P buffer views carry a Gen9 linear PBE; imageBuffer reads still use
-    * the native texture heap. */
-   if ((!storage || generation == 9 || hk_native_pbe_supported(generation)) &&
+   /* Native buffer-image reads use the same texture heap and software
+    * metadata as uniform texel buffers. A missing writable PBE must not
+    * replace their readable texture with the null descriptor. */
+   if ((!storage || hk_descriptor_generation_is_agx3(generation) ||
+        hk_native_pbe_supported(generation)) &&
        bufferView != VK_NULL_HANDLE) {
       VK_FROM_HANDLE(hk_buffer_view, view, bufferView);
 
diff --git a/src/asahi/vulkan/hk_descriptor_set_layout.c b/src/asahi/vulkan/hk_descriptor_set_layout.c
index d02515976008..4b674c08759d 100644
--- a/src/asahi/vulkan/hk_descriptor_set_layout.c
+++ b/src/asahi/vulkan/hk_descriptor_set_layout.c
@@ -55,7 +55,7 @@ hk_descriptor_stride_align_for_type(
    case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
       *stride = hk_storage_image_descriptor_layout(
                    agx_chip_descriptor_generation(pdev->dev.chip)).stride;
-      if (agx_native_graphics(&pdev->dev)) {
+      if (agx_native_graphics(&pdev->dev) || agx_chip_g17_m5(pdev->dev.chip)) {
          *stride += HK_G17P_STORAGE_SW_SIZE;
          /* imgwci addresses PBE records in a heap, and stores treat the
           * descriptor set as that heap. The PBE descriptor starts 32 bytes
@@ -78,7 +78,7 @@ hk_descriptor_stride_align_for_type(
        * heap indices and 24 bytes of buffer size/offset/address metadata.
        * The shader must never decode AGX2 software fields from Gen9 bytes.
        */
-      if (agx_native_graphics(&pdev->dev)) {
+      if (agx_native_graphics(&pdev->dev) || agx_chip_g17_m5(pdev->dev.chip)) {
          *stride = AGX_G17P_BUFFER_TEXTURE_DESCRIPTOR_SIZE;
          /* Texel-buffer stores address the PBE through the set's PBE heap,
           * like storage images. */
@@ -118,7 +118,8 @@ hk_descriptor_stride_align_for_type(
                hk_sampled_image_descriptor_layout(generation).stride;
          } else {
             *stride = HK_MAX_DESCRIPTOR_SIZE;
-            *alignment = agx_native_graphics(&pdev->dev)
+            *alignment = (agx_native_graphics(&pdev->dev) ||
+                          agx_chip_g17_m5(pdev->dev.chip))
                             ? HK_G17P_STORAGE_ALIGNMENT
                             : 8;
          }
@@ -202,7 +203,8 @@ hk_CreateDescriptorSetLayout(VkDevice device,
    layout->binding_count = num_bindings;
    layout->descriptor_generation =
       agx_chip_descriptor_generation(pdev->dev.chip);
-   layout->native_sampler_indices = agx_native_graphics(&pdev->dev);
+   layout->native_sampler_indices = agx_native_graphics(&pdev->dev) ||
+                                    agx_chip_g17_m5(pdev->dev.chip);
 
    for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
       const VkDescriptorSetLayoutBinding *binding = &pCreateInfo->pBindings[j];
diff --git a/src/asahi/vulkan/hk_descriptor_table.c b/src/asahi/vulkan/hk_descriptor_table.c
index a9467806deed..28a4a7c788ff 100644
--- a/src/asahi/vulkan/hk_descriptor_table.c
+++ b/src/asahi/vulkan/hk_descriptor_table.c
@@ -134,7 +134,7 @@ hk_descriptor_table_reserve(struct hk_device *dev,
    }
 
    if (agx_bo_bind(&dev->dev, dev->dev.zero_bo, table->va->addr, size_B, 0,
-                   DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_SINGLE_PAGE)) {
+                   DRM_ASAHI_BIND_READ | agx_fill_bind_flags(&dev->dev))) {
       agx_va_free(&dev->dev, table->va, false);
       table->va = NULL;
       return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
diff --git a/src/asahi/vulkan/hk_device.c b/src/asahi/vulkan/hk_device.c
index 6b23d70a7e44..670632153f93 100644
--- a/src/asahi/vulkan/hk_device.c
+++ b/src/asahi/vulkan/hk_device.c
@@ -5,6 +5,7 @@
  * SPDX-License-Identifier: MIT
  */
 #include "hk_device.h"
+#include "hk_frame_anatomy.h"
 
 #include "hk_acceleration_structure.h"
 
@@ -43,6 +44,9 @@
 #include "vk_pipeline_cache.h"
 #include "vk_shader.h"
 
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
 #include <fcntl.h>
 #include <xf86drm.h>
 
@@ -139,7 +143,7 @@ hk_upload_rodata(struct hk_device *dev)
       memcpy(map + offs + i * sampler_size, &txf, sampler_size);
    offs += sampler_count * sampler_size;
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       const uint64_t pointers[2] = {
          dev->sampled_images.base_va,
          dev->samplers.table.bo->va->addr,
@@ -258,14 +262,16 @@ hk_init_sampler_heap(struct hk_device *dev, struct hk_sampler_heap *h)
 
    VkResult result =
       hk_descriptor_table_init(dev, &h->table, sampler_size, 1024,
-                               agx_native_graphics(&dev->dev) ? 4096 : 1024);
+                               agx_native_graphics(&dev->dev) ? 4096
+                               : agx_chip_g17_m5(dev->dev.chip) ? 2048
+                                                                : 1024);
 
    if (result != VK_SUCCESS) {
       ralloc_free(h->ht);
       return result;
    }
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       /* Native indexed read() and sample() share the sampler table prefix.
        * Reserve texel-fetch sampler zero once, so mixed shaders need no
        * per-fetch pointer switch or descriptor copy. Preserve at least the
@@ -768,6 +774,18 @@ hk_init_g17p_grid_helper(struct hk_device *dev,
       mesa_logw("G17P sparse indirect compute unavailable: helper refused");
 }
 
+/* Direct stderr provenance is diagnostic-only: some initialization helpers
+ * return an error without passing through Vulkan's debug logger. */
+static void
+hk_device_init_diag(const void *pdev, const void *dev, const char *stage,
+                    VkResult result, int error)
+{
+   const int saved_errno = errno;
+   fprintf(stderr, "M5_DEVICE_INIT pid=%ld pdev=%p dev=%p stage=%s result=%d errno=%d\n",
+           (long)getpid(), pdev, dev, stage, (int)result, error);
+   errno = saved_errno;
+}
+
 VKAPI_ATTR VkResult VKAPI_CALL
 hk_CreateDevice(VkPhysicalDevice physicalDevice,
                 const VkDeviceCreateInfo *pCreateInfo,
@@ -776,12 +794,26 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    VK_FROM_HANDLE(hk_physical_device, pdev, physicalDevice);
    VkResult result = VK_ERROR_OUT_OF_HOST_MEMORY;
    struct hk_device *dev;
+   const bool init_diag =
+      debug_get_bool_option("ASAHI_M5_DEVICE_INIT_DIAG", false);
+   if (init_diag)
+      hk_device_init_diag(pdev, NULL, "enter", VK_SUCCESS, 0);
+
+#define HK_INIT_FAIL(label, stage, error) do { \
+   if (init_diag) \
+      hk_device_init_diag(pdev, dev, stage, result, error); \
+   goto label; \
+} while (0)
    struct hk_instance *instance = (struct hk_instance *)pdev->vk.instance;
 
    dev = vk_zalloc2(&instance->vk.alloc, pAllocator, sizeof(*dev), 8,
                     VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
-   if (!dev)
+   if (!dev) {
+      if (init_diag)
+         hk_device_init_diag(pdev, NULL, "allocate-device",
+                             VK_ERROR_OUT_OF_HOST_MEMORY, 0);
       return vk_error(pdev, VK_ERROR_OUT_OF_HOST_MEMORY);
+   }
 
    struct vk_device_dispatch_table dispatch_table;
 
@@ -811,7 +843,7 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    result = vk_device_init(&dev->vk, &pdev->vk, &dispatch_table, pCreateInfo,
                            pAllocator);
    if (result != VK_SUCCESS)
-      goto fail_alloc;
+      HK_INIT_FAIL(fail_alloc, "vk-device-init", 0);
 
    dev->vk.shader_ops = &hk_device_shader_ops;
    dev->vk.command_dispatch_table = &dev->cmd_dispatch;
@@ -821,16 +853,17 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    if (ret != 0) {
       result = vk_errorf(dev, VK_ERROR_INITIALIZATION_FAILED,
                          "Failed to get DRM device: %m");
-      goto fail_init;
+      HK_INIT_FAIL(fail_init, "drm-device-query", 0);
    }
 
    const char *path = drm_device->nodes[DRM_NODE_RENDER];
    dev->dev.fd = open(path, O_RDWR | O_CLOEXEC);
    if (dev->dev.fd < 0) {
+      const int open_error = errno;
       result = vk_errorf(dev, VK_ERROR_INITIALIZATION_FAILED,
                          "failed to open device %s", path);
       drmFreeDevice(&drm_device);
-      goto fail_init;
+      HK_INIT_FAIL(fail_init, "open-render-node", open_error);
    }
 
    dev->perftest = debug_get_flags_option("HK_PERFTEST", hk_perf_options, 0);
@@ -856,7 +889,7 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    if (!succ) {
       result = vk_errorf(dev, VK_ERROR_INITIALIZATION_FAILED,
                          "Failed to get DRM device: %m");
-      goto fail_fd;
+      HK_INIT_FAIL(fail_fd, "agx-open-device", 0);
    }
 
    vk_device_set_drm_fd(&dev->vk, dev->dev.fd);
@@ -874,18 +907,37 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
              dev->dev.user_timestamp_to_ns.den &&
           "user timestamps are in ns");
 
+   if (agx_chip_g17_m5(dev->dev.chip) &&
+       dev->vk.enabled_features.sparseResidencyBuffer) {
+      const uint64_t pages = dev->dev.main_heap_size / AIL_PAGESIZE;
+      const uint64_t bytes = align64(16 + DIV_ROUND_UP(pages, 8), AIL_PAGESIZE);
+      dev->physical_sparse_map = agx_bo_create(
+         &dev->dev, bytes, AIL_PAGESIZE,
+         AGX_BO_WRITEBACK | AGX_BO_ZERO_ON_REUSE,
+         "Physical sparse buffer page map");
+      if (!dev->physical_sparse_map || !agx_bo_map(dev->physical_sparse_map)) {
+         result = vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+         HK_INIT_FAIL(fail_dev, "sparse-map-allocation", 0);
+      }
+      memset(agx_bo_map(dev->physical_sparse_map), 0, bytes);
+      if (agx_bo_sync_for_device(dev->physical_sparse_map, 0, bytes)) {
+         result = vk_error(dev, VK_ERROR_DEVICE_LOST);
+         HK_INIT_FAIL(fail_dev, "sparse-map-sync", 0);
+      }
+   }
+
    result = hk_init_sampler_heap(dev, &dev->samplers);
    if (result != VK_SUCCESS)
-      goto fail_dev;
+      HK_INIT_FAIL(fail_dev, "sampler-heap", 0);
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       /* Capacity stays HK_MAX_DESCRIPTORS, but memory is added a chunk at a
        * time as image and buffer views are created. */
       result = hk_descriptor_table_init(dev, &dev->sampled_images, 32,
                                         HK_DESCRIPTOR_TABLE_CHUNK_B / 32,
                                         HK_MAX_DESCRIPTORS);
       if (result != VK_SUCCESS)
-         goto fail_samplers;
+         HK_INIT_FAIL(fail_samplers, "sampled-image-table", 0);
 
       struct hk_native_texture_descriptor null_texture;
       hk_set_null_native_texture(agx_chip_descriptor_generation(dev->dev.chip),
@@ -894,7 +946,7 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
       result = hk_descriptor_table_add(dev, &dev->sampled_images,
                                        &null_texture, 32, &null_index);
       if (result != VK_SUCCESS)
-         goto fail_sampled_images;
+         HK_INIT_FAIL(fail_sampled_images, "null-texture-descriptor", 0);
       assert(null_index == 0);
    }
 
@@ -902,17 +954,17 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
       dev, &dev->occlusion_queries, sizeof(uint64_t), AGX_MAX_OCCLUSION_QUERIES,
       AGX_MAX_OCCLUSION_QUERIES);
    if (result != VK_SUCCESS)
-      goto fail_sampled_images;
+      HK_INIT_FAIL(fail_sampled_images, "occlusion-query-table", 0);
 
    result = hk_upload_rodata(dev);
    if (result != VK_SUCCESS)
-      goto fail_queries;
+      HK_INIT_FAIL(fail_queries, "rodata", 0);
 
    /* XXX: error handling, and should this even go on the device? */
    agx_bg_eot_init(&dev->bg_eot, &dev->dev);
    if (!dev->bg_eot.ht) {
       result = VK_ERROR_OUT_OF_HOST_MEMORY;
-      goto fail_rodata;
+      HK_INIT_FAIL(fail_rodata, "bg-eot", 0);
    }
    hk_g17p_bg_eot_dev_cache_init(dev);
 
@@ -923,23 +975,23 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    if (agx_native_graphics(&dev->dev) &&
        !hk_g17p_prewarm_poly_kernels(dev)) {
       result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
-      goto fail_bg_eot;
+      HK_INIT_FAIL(fail_bg_eot, "poly-prewarm", 0);
    }
 
    result = hk_init_internal_shaders(&dev->prolog_epilog);
    if (result != VK_SUCCESS)
-      goto fail_bg_eot;
+      HK_INIT_FAIL(fail_bg_eot, "prolog-epilog-shaders", 0);
 
    result = hk_init_internal_shaders(&dev->kernels);
    if (result != VK_SUCCESS)
-      goto fail_internal_shaders;
+      HK_INIT_FAIL(fail_internal_shaders, "internal-kernels", 0);
 
    for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
       for (unsigned q = 0; q < pCreateInfo->pQueueCreateInfos[i].queueCount;
            q++) {
          result = hk_queue_init(dev, &pCreateInfo->pQueueCreateInfos[i], q);
          if (result != VK_SUCCESS)
-            goto fail_queues;
+            HK_INIT_FAIL(fail_queues, "queue-init", 0);
       }
    }
 
@@ -949,16 +1001,16 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    dev->vk.mem_cache = vk_pipeline_cache_create(&dev->vk, &cache_info, NULL);
    if (dev->vk.mem_cache == NULL) {
       result = VK_ERROR_OUT_OF_HOST_MEMORY;
-      goto fail_queues;
+      HK_INIT_FAIL(fail_queues, "pipeline-cache", 0);
    }
 
    result = hk_device_init_accel_struct_state(dev);
    if (result != VK_SUCCESS)
-      goto fail_mem_cache;
+      HK_INIT_FAIL(fail_mem_cache, "accel-struct-state", 0);
 
    result = hk_device_init_meta(dev);
    if (result != VK_SUCCESS)
-      goto fail_mem_cache;
+      HK_INIT_FAIL(fail_accel_struct, "meta", 0);
 
    /* Precompile an empty fragment shader that can be used to handle API-level
     * null fragment shaders. We do this at device-time to make binds cheap.
@@ -974,7 +1026,7 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    hk_compile_shader(dev, &info, NULL, NULL, pAllocator, &dev->null_fs);
    if (!dev->null_fs) {
       result = VK_ERROR_OUT_OF_HOST_MEMORY;
-      goto fail_meta;
+      HK_INIT_FAIL(fail_meta, "empty-fragment-shader", 0);
    }
 
    hk_init_g17p_grid_helper(dev, pAllocator);
@@ -993,11 +1045,19 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    dev->external_bos.counts = UTIL_DYNARRAY_INIT;
    dev->external_bos.list = UTIL_DYNARRAY_INIT;
 
+   /* Opt-in profiler; never fails device creation. */
+   hk_frame_anatomy_init(dev);
+
+   if (init_diag)
+      hk_device_init_diag(pdev, dev, "success", VK_SUCCESS, 0);
    return VK_SUCCESS;
 
 fail_meta:
-   hk_gen1_rq_finish(dev);
+   if (!agx_chip_g17_m5(dev->dev.chip))
+      hk_gen1_rq_finish(dev);
    hk_device_finish_meta(dev);
+fail_accel_struct:
+   hk_device_finish_accel_struct_state(dev);
 fail_mem_cache:
    vk_pipeline_cache_destroy(dev->vk.mem_cache, NULL);
 fail_queues:
@@ -1020,12 +1080,13 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
 fail_queries:
    hk_descriptor_table_finish(dev, &dev->occlusion_queries);
 fail_sampled_images:
-   if (agx_native_graphics(&dev->dev))
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
       hk_descriptor_table_finish(dev, &dev->sampled_images);
 fail_samplers:
    hk_destroy_sampler_heap(dev, &dev->samplers);
 fail_dev:
    hk_finish_vm_status_mirror(dev);
+   agx_bo_unreference(&dev->dev, dev->physical_sparse_map);
    agx_bo_unreference(&dev->dev, dev->code_heap.chunk);
    agx_close_device(&dev->dev);
    /* agx_close_device owns the DRM fd after agx_open_device succeeds. */
@@ -1036,6 +1097,7 @@ hk_CreateDevice(VkPhysicalDevice physicalDevice,
    vk_device_finish(&dev->vk);
 fail_alloc:
    vk_free(&dev->vk.alloc, dev);
+#undef HK_INIT_FAIL
    return result;
 }
 
@@ -1051,15 +1113,20 @@ hk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator)
    util_dynarray_fini(&dev->external_bos.list);
    u_rwlock_destroy(&dev->external_bos.lock);
 
-   hk_gen1_rq_finish(dev);
+   if (!agx_chip_g17_m5(dev->dev.chip))
+      hk_gen1_rq_finish(dev);
    hk_device_finish_meta(dev);
    hk_shader_binary_cache_finish(dev);
    hk_g17p_compute_program_cache_finish(dev);
+   hk_device_finish_accel_struct_state(dev);
    hk_destroy_internal_shaders(dev, &dev->kernels, false);
    hk_destroy_internal_shaders(dev, &dev->prolog_epilog, true);
 
    vk_pipeline_cache_destroy(dev->vk.mem_cache, NULL);
 
+   /* Retire pending profiler records while the queue syncobjs exist. */
+   hk_frame_anatomy_finish(dev);
+
    vk_foreach_queue_safe(iter, &dev->vk) {
       struct hk_queue *queue = container_of(iter, struct hk_queue, vk);
       hk_queue_finish(dev, queue);
@@ -1094,7 +1161,7 @@ hk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator)
    }
 
    hk_destroy_sampler_heap(dev, &dev->samplers);
-   if (agx_native_graphics(&dev->dev))
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
       hk_descriptor_table_finish(dev, &dev->sampled_images);
    hk_descriptor_table_finish(dev, &dev->occlusion_queries);
    agx_bo_unreference(&dev->dev, dev->rodata.heap_bo);
@@ -1104,6 +1171,7 @@ hk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator)
    hk_g17p_bg_eot_dev_cache_finish(dev);
    agx_bg_eot_cleanup(&dev->bg_eot);
    hk_finish_vm_status_mirror(dev);
+   agx_bo_unreference(&dev->dev, dev->physical_sparse_map);
    agx_bo_unreference(&dev->dev, dev->code_heap.chunk);
    simple_mtx_destroy(&dev->code_heap.lock);
    agx_close_device(&dev->dev);
diff --git a/src/asahi/vulkan/hk_device.h b/src/asahi/vulkan/hk_device.h
index c024699bf63f..fc4bfa1e5b45 100644
--- a/src/asahi/vulkan/hk_device.h
+++ b/src/asahi/vulkan/hk_device.h
@@ -119,11 +119,19 @@ struct hk_gen1_as_window {
    uint32_t used[8];
    uint32_t live;
 };
+struct hk_frame_anatomy;
 
 struct hk_device {
    struct vk_device vk;
    struct agx_device dev;
+   /* Scratch-filled sparse VA (M5: bounded by sparseAddressSpaceSize). */
+   uint64_t sparse_scratch_B;
+   /* One bit per main-heap page: only unbound sparse buffer pages are set. */
+   struct agx_bo *physical_sparse_map;
    struct agxdecode_ctx *decode_ctx;
+   /* Opt-in frame anatomy profiler (hk_frame_anatomy.h); NULL unless the
+    * HK_FRAME_ANATOMY environment variable is set. */
+   struct hk_frame_anatomy *frame_anatomy;
 
    struct hk_descriptor_table occlusion_queries;
    struct hk_sampler_heap samplers;
@@ -179,6 +187,33 @@ struct hk_device {
       struct hk_queue *queue;
       VkCommandPool pool;
    } host_as;
+   bool m5_native_as;
+   /* Explicit diagnostic runtime, loaded only from the caller's capture. */
+   bool m5_native_rq;
+   bool m5_rq_instances;
+   simple_mtx_t m5_as_lock;
+   struct util_dynarray m5_ases;
+   /* Stable native AS-ID -> address table. VA is reserved once; backing
+    * pages are added as IDs are allocated, so recorded/pending commands
+    * also see acceleration structures created after recording. */
+   struct agx_va *m5_as_table_va;
+   struct util_dynarray m5_as_table_pages;
+   /* Only AS objects created before their ordinary storage buffer is bound.
+    * Binding publishes their final addresses without a submit-time walk. */
+   struct util_dynarray m5_pending_as_ids;
+   uint32_t m5_pending_as_count;
+   struct agx_bo *m5_rq_code;
+   struct agx_va *m5_rq_code_alias;
+   struct agx_bo *m5_rq_ift;
+   struct agx_bo *m5_rq_diag;
+   /* Device-lifetime SCS state block (hk_cmd_dispatch.c). */
+   struct agx_bo *m5_rt_state_bo;
+   uint64_t m5_rt_state_packed, m5_rt_state_size;
+   char *m5_rq_diag_dump;
+   void *m5_rq_user_image;
+   uint32_t m5_rq_uniforms[80];
+   uint64_t m5_rq_default_program;
+   uint64_t m5_rq_default_control;
    struct agx_bg_eot_cache bg_eot;
    /* frg-codegen v4 BG/EOT device cache (hk_cmd_draw.c). */
    struct hk_g17p_bg_eot_dev_cache *g17p_bg_eot_dev_cache;
diff --git a/src/asahi/vulkan/hk_device_memory.c b/src/asahi/vulkan/hk_device_memory.c
index 4120a267e564..0de931119c28 100644
--- a/src/asahi/vulkan/hk_device_memory.c
+++ b/src/asahi/vulkan/hk_device_memory.c
@@ -238,8 +238,26 @@ hk_AllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo,
    if (pAllocateInfo->allocationSize > UINT64_MAX - (alignment - 1))
       return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
 
+   /* No size beyond the heap can be satisfied. Refuse it before creating a
+    * BO: a request whose page rounding below wraps to 0 would otherwise
+    * reach agx_bo_create() as a 0-byte BO and fail in the kernel. */
+   if (!(fd_info && fd_info->handleType) &&
+       pAllocateInfo->allocationSize > heap->size) {
+      return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
+                       "allocationSize %" PRIu64 " exceeds heap %u "
+                       "(%" PRIu64 " bytes)",
+                       (uint64_t)pAllocateInfo->allocationSize,
+                       type->heapIndex, (uint64_t)heap->size);
+   }
+
+   /* The spec requires allocationSize > 0 when nothing is imported
+    * (VUID-VkMemoryAllocateInfo-allocationSize-07897), but CTS
+    * cooperative_matrix.op_constant_null creates zero-sized buffers and
+    * allocates their zero-sized requirements. Back such an allocation with
+    * one page, so it can still be bound and mapped, rather than fail it with
+    * VK_ERROR_OUT_OF_DEVICE_MEMORY. */
    const uint64_t aligned_size =
-      align64(pAllocateInfo->allocationSize, alignment);
+      align64(MAX2(pAllocateInfo->allocationSize, 1), alignment);
    uint64_t used = p_atomic_read(heap->used);
    if (used > heap->size || aligned_size > heap->size - used)
       return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
diff --git a/src/asahi/vulkan/hk_drirc_gen.py b/src/asahi/vulkan/hk_drirc_gen.py
index 79be73a00a06..2d661377fc38 100644
--- a/src/asahi/vulkan/hk_drirc_gen.py
+++ b/src/asahi/vulkan/hk_drirc_gen.py
@@ -34,6 +34,11 @@ def declare_options():
         B("hk_enable_vertex_pipeline_stores_atomics", False,
           "Enable vertexPipelineStoresAndAtomics",
           c_name="enable_vertex_pipeline_stores_atomics"),
+        B("hk_mutable_format_srgb_pair", False,
+          "Assume a mutable-format colour image created without a format "
+          "list is only viewed as itself or its sRGB/UNORM twin, so that it "
+          "can be compressed",
+          c_name="mutable_format_srgb_pair"),
     ]
 
     debug_options = []
diff --git a/src/asahi/vulkan/hk_drm_prior_core.h b/src/asahi/vulkan/hk_drm_prior_core.h
index c64b57b372c5..6accdc159442 100644
--- a/src/asahi/vulkan/hk_drm_prior_core.h
+++ b/src/asahi/vulkan/hk_drm_prior_core.h
@@ -78,6 +78,26 @@ hk_drm_rebase_prior(struct drm_asahi_cmd_header *h, void *payload,
    }
    h->vdm_barrier = barriers[0];
    h->cdm_barrier = barriers[1];
+   if (exact && h->cmd_type == DRM_ASAHI_CMD_RENDER &&
+       (r->legacy.flags & DRM_ASAHI_RENDER_COMPUTE_STAGE_BARRIERS)) {
+      /* The tiling half's compute prefix is rebased like the header's. */
+      struct drm_asahi_cmd_render_with_stage_prior *s = payload;
+      if (h->size != sizeof(*s))
+         return false;
+      const uint16_t value = s->ta_cdm_barrier;
+      if (value != DRM_ASAHI_BARRIER_NONE && value != 0) {
+         if (value > before[1]) {
+            s->ta_cdm_barrier = value - before[1];
+         } else {
+            if (initial[1] > UINT64_MAX - value)
+               return false;
+            const uint64_t absolute = initial[1] + value;
+            if (absolute > s->ta_prior_cdm)
+               s->ta_prior_cdm = absolute;
+            s->ta_cdm_barrier = DRM_ASAHI_BARRIER_NONE;
+         }
+      }
+   }
    if (exact && h->cmd_type == DRM_ASAHI_CMD_RENDER) {
       r->prior_vdm = prior[0];
       r->prior_cdm = prior[1];
diff --git a/src/asahi/vulkan/hk_drm_render_core.c b/src/asahi/vulkan/hk_drm_render_core.c
index a633045259bf..eed340198929 100644
--- a/src/asahi/vulkan/hk_drm_render_core.c
+++ b/src/asahi/vulkan/hk_drm_render_core.c
@@ -45,6 +45,19 @@ hk_drm_g17p_zls_pixels(uint32_t width, uint32_t height,
    return (width - 1) | ((height - 1) << 15);
 }
 
+uint32_t
+hk_drm_m5_zls_pixels(uint32_t width, uint32_t height,
+                     uint64_t depth_base, uint64_t depth_meta,
+                     uint64_t stencil_base, uint64_t stencil_meta)
+{
+   if (!hk_drm_g17p_has_zls_storage(depth_base, depth_meta, stencil_base,
+                                    stencil_meta))
+      return 0;
+   assert(width >= 1 && width <= 0x10000);
+   assert(height >= 1 && height <= 0x10000);
+   return (width - 1) | ((height - 1) << 16);
+}
+
 uint32_t
 hk_drm_g17p_depth_clear(uint32_t depth_clear,
                         uint64_t depth_base, uint64_t depth_meta,
@@ -198,6 +211,21 @@ hk_drm_next_command_span(const void *cmdbuf, size_t cmdbuf_size,
              header->cdm_barrier != DRM_ASAHI_BARRIER_NONE ||
              header->size % sizeof(struct drm_asahi_attachment) != 0)
             return false;
+      } else if (header->cmd_type == DRM_ASAHI_SET_RENDER_RESOURCES) {
+         if (header->vdm_barrier != DRM_ASAHI_BARRIER_NONE ||
+             header->cdm_barrier != DRM_ASAHI_BARRIER_NONE ||
+             header->size % sizeof(struct drm_asahi_render_program) != 0)
+            return false;
+      } else if (header->cmd_type == DRM_ASAHI_SET_RENDER_RESOURCES_V2) {
+         if (header->vdm_barrier != DRM_ASAHI_BARRIER_NONE ||
+             header->cdm_barrier != DRM_ASAHI_BARRIER_NONE ||
+             header->size % sizeof(struct drm_asahi_render_program_v2) != 0)
+            return false;
+      } else if (header->cmd_type == DRM_ASAHI_SET_COMPUTE_RESOURCES) {
+         if (header->vdm_barrier != DRM_ASAHI_BARRIER_NONE ||
+             header->cdm_barrier != DRM_ASAHI_BARRIER_NONE ||
+             header->size % sizeof(struct drm_asahi_compute_program) != 0)
+            return false;
       } else {
          return false;
       }
diff --git a/src/asahi/vulkan/hk_drm_render_core.h b/src/asahi/vulkan/hk_drm_render_core.h
index f2ae82f50421..e2b89cba3fdf 100644
--- a/src/asahi/vulkan/hk_drm_render_core.h
+++ b/src/asahi/vulkan/hk_drm_render_core.h
@@ -38,6 +38,10 @@ uint32_t hk_drm_g17p_zls_pixels(uint32_t width, uint32_t height,
                                 uint64_t depth_base, uint64_t depth_meta,
                                 uint64_t stencil_base, uint64_t stencil_meta);
 
+uint32_t hk_drm_m5_zls_pixels(uint32_t width, uint32_t height,
+                              uint64_t depth_base, uint64_t depth_meta,
+                              uint64_t stencil_base, uint64_t stencil_meta);
+
 uint32_t hk_drm_g17p_depth_clear(uint32_t depth_clear,
                                  uint64_t depth_base, uint64_t depth_meta,
                                  uint64_t stencil_base, uint64_t stencil_meta);
diff --git a/src/asahi/vulkan/hk_format.c b/src/asahi/vulkan/hk_format.c
index 955a7645c74e..93c4971aaf64 100644
--- a/src/asahi/vulkan/hk_format.c
+++ b/src/asahi/vulkan/hk_format.c
@@ -31,7 +31,7 @@ uint64_t agx_best_modifiers[] = {
 
 static VkFormatFeatureFlags2
 hk_modifier_features(const struct agx_device *dev, uint64_t mod,
-                     VkFormat vk_format, const VkFormatProperties *props)
+                     VkFormat vk_format, const VkFormatProperties3 *props)
 {
    /* There's no corresponding fourcc, so don't advertise modifiers */
    if (vk_format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 ||
@@ -39,20 +39,24 @@ hk_modifier_features(const struct agx_device *dev, uint64_t mod,
       return 0;
    }
 
-   /* Don't advertise compression for the uncompressable */
+   /* Don't advertise compression for the uncompressable. A compressed image
+    * has no host-copy layout (images with host-transfer usage are never
+    * compressed). */
    if (mod == DRM_FORMAT_MOD_APPLE_GPU_TILED_COMPRESSED) {
       if (!hk_can_compress_format(dev, vk_format))
          return 0;
 
-      /* G17P shares compressed images without storage or host-copy access
-       * (hk_g17p_shared_compression_shape()). */
-      if (dev->chip == AGX_CHIP_G17P)
+      /* G17P and M5 share compressed images without storage or host-copy
+       * access (hk_g17p_shared_compression_shape()). */
+      if (dev->chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->chip))
          return props->optimalTilingFeatures &
                 ~(VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT |
                   VK_FORMAT_FEATURE_2_STORAGE_IMAGE_ATOMIC_BIT |
                   VK_FORMAT_FEATURE_2_STORAGE_READ_WITHOUT_FORMAT_BIT |
                   VK_FORMAT_FEATURE_2_STORAGE_WRITE_WITHOUT_FORMAT_BIT |
                   VK_FORMAT_FEATURE_2_HOST_IMAGE_TRANSFER_BIT_EXT);
+      return props->optimalTilingFeatures &
+             ~VK_FORMAT_FEATURE_2_HOST_IMAGE_TRANSFER_BIT_EXT;
    }
 
    /* Interchange: the 32-bit RGBA formats on G17P; never storage or host
@@ -77,7 +81,7 @@ hk_modifier_features(const struct agx_device *dev, uint64_t mod,
 static void
 get_drm_format_modifier_properties_list(
    const struct hk_physical_device *physical_device, VkFormat vk_format,
-   VkDrmFormatModifierPropertiesListEXT *list, const VkFormatProperties *props)
+   VkDrmFormatModifierPropertiesListEXT *list, const VkFormatProperties3 *props)
 {
    VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierPropertiesEXT, out,
                           list->pDrmFormatModifierProperties,
@@ -97,7 +101,8 @@ get_drm_format_modifier_properties_list(
          *out_props = (VkDrmFormatModifierPropertiesEXT){
             .drmFormatModifier = mod,
             .drmFormatModifierPlaneCount = vk_format_get_plane_count(vk_format),
-            .drmFormatModifierTilingFeatures = flags,
+            .drmFormatModifierTilingFeatures =
+               vk_format_features2_to_features(flags),
          };
       };
    }
@@ -106,7 +111,7 @@ get_drm_format_modifier_properties_list(
 static void
 get_drm_format_modifier_properties_list_2(
    const struct hk_physical_device *physical_device, VkFormat vk_format,
-   VkDrmFormatModifierPropertiesList2EXT *list, const VkFormatProperties *props)
+   VkDrmFormatModifierPropertiesList2EXT *list, const VkFormatProperties3 *props)
 {
    VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierProperties2EXT, out,
                           list->pDrmFormatModifierProperties,
@@ -152,6 +157,14 @@ hk_GetPhysicalDeviceFormatProperties2(VkPhysicalDevice physicalDevice,
       .bufferFeatures = vk_format_features2_to_features(buffer2),
    };
 
+   /* Modifier lists report the 64-bit feature sets (host image transfer,
+    * formatless storage, ...), which VkFormatProperties cannot carry. */
+   const VkFormatProperties3 props3 = {
+      .linearTilingFeatures = linear2,
+      .optimalTilingFeatures = optimal2,
+      .bufferFeatures = buffer2,
+   };
+
    vk_foreach_struct(sType, ext, pFormatProperties->pNext) {
       switch (sType) {
       case VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3: {
@@ -163,13 +176,13 @@ hk_GetPhysicalDeviceFormatProperties2(VkPhysicalDevice physicalDevice,
       }
 
       case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT:
-         get_drm_format_modifier_properties_list(
-            pdevice, format, ext, &pFormatProperties->formatProperties);
+         get_drm_format_modifier_properties_list(pdevice, format, ext,
+                                                 &props3);
          break;
 
       case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT:
-         get_drm_format_modifier_properties_list_2(
-            pdevice, format, ext, &pFormatProperties->formatProperties);
+         get_drm_format_modifier_properties_list_2(pdevice, format, ext,
+                                                   &props3);
          break;
 
       default:
diff --git a/src/asahi/vulkan/hk_frame_anatomy.c b/src/asahi/vulkan/hk_frame_anatomy.c
new file mode 100644
index 000000000000..9da74cf4e606
--- /dev/null
+++ b/src/asahi/vulkan/hk_frame_anatomy.c
@@ -0,0 +1,992 @@
+/*
+ * Copyright 2026 Aurora Silicon contributors
+ * SPDX-License-Identifier: MIT
+ *
+ * Opt-in frame anatomy profiler. See hk_frame_anatomy.h.
+ *
+ * Timestamps ride on the kernel's native per-command timestamp writes
+ * (drm_asahi_cmd_render::ts_vtx/ts_frag, drm_asahi_cmd_compute::ts), which
+ * the firmware fills from its pass records; the profiler owns a dedicated
+ * timestamp object and only claims slots no query pool already uses. Results
+ * are read back lazily, under a non-blocking syncobj query, at the next
+ * sampled submit, and with a bounded wait at device destruction.
+ *
+ * The uapi in include/drm-uapi/asahi_drm.h exposes no TVB overflow or
+ * partial-render counter; the kernel keeps the firmware's per-command
+ * tvb_overflow_count and its TVB growth messages to itself (TVBStats debug
+ * flag, dmesg). So no such indicator is logged here; "pass" shows only the
+ * userspace-visible split of an API render pass into several streams.
+ */
+#include "hk_frame_anatomy.h"
+#include "nir.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <xf86drm.h>
+
+#include "agx_bo.h"
+#include "agx_chip.h"
+#include "agx_device.h"
+#include "agx_pack.h"
+#include "asahi/libagx/query.h"
+#include "drm-uapi/asahi_drm.h"
+#include "hk_cmd_buffer.h"
+#include "hk_device.h"
+#include "hk_queue.h"
+#include "util/format/u_format.h"
+#include "util/hash_table.h"
+#include "util/log.h"
+#include "util/macros.h"
+#include "util/os_misc.h"
+#include "util/os_time.h"
+#include "util/simple_mtx.h"
+#include "util/u_debug.h"
+#include "util/u_dynarray.h"
+
+#define HK_FA_TEXT_MAX       1024
+#define HK_FA_STAMPS         4
+#define HK_FA_TS_BO_SIZE     (256 * 1024)
+#define HK_FA_RING_RECS      (HK_FA_TS_BO_SIZE / (HK_FA_STAMPS * sizeof(uint64_t)))
+#define HK_FA_FINISH_WAIT_NS (2 * 1000000000ll)
+
+enum hk_fa_ts_mode {
+   HK_FA_TS_ALL,
+   HK_FA_TS_EDGES,
+   HK_FA_TS_NONE,
+};
+
+struct hk_frame_anatomy_rec {
+   uint32_t syncobj;
+   uint64_t point;
+   uint32_t slot;        /* ring record index, UINT32_MAX when none was free */
+   uint32_t header_off;  /* payload offset of the drm_asahi_cmd_header */
+   uint8_t own;          /* bit i: stamp i belongs to the profiler */
+   uint8_t query;        /* bit i: stamp i belongs to a query pool */
+   uint8_t nr_stamps;    /* 4 render, 2 compute */
+   char kind;            /* 'R' render, 'C' compute */
+   bool barriers_known;
+   uint16_t vdm_barrier, cdm_barrier;
+   uint64_t prior_vdm, prior_cdm;
+   char text[HK_FA_TEXT_MAX];
+};
+
+struct hk_frame_anatomy_submit {
+   struct util_dynarray recs; /* struct hk_frame_anatomy_rec */
+   uint64_t seq;
+   int64_t cpu_ns;
+   unsigned command_count;
+   unsigned dropped;
+};
+
+struct hk_frame_anatomy {
+   FILE *file;
+   bool own_file;
+   simple_mtx_t lock;
+   uint64_t every;
+   enum hk_fa_ts_mode ts_mode;
+   uint64_t submit_seq;
+   uint64_t sampled;
+
+   struct agx_bo *ts_bo;
+   uint32_t ts_handle;
+   uint64_t *ts_map;
+
+   /* Records in flight, FIFO; their slots form a contiguous ring span. */
+   uint32_t ring_head, ring_count;
+   struct util_dynarray pending; /* struct hk_frame_anatomy_rec */
+
+   /* Free-form notes written so far and dropped at the limit
+    * (hk_frame_anatomy_note); the limit is HK_FRAME_ANATOMY_NOTES. */
+   uint32_t notes, notes_dropped, note_limit;
+
+   /* HK_FRAME_ANATOMY_DUMP=<dir>: each sampled fragment program's binary is
+    * written once to <dir>/fs-<hash>.bin. */
+   const char *dump_dir;
+   uint32_t dumped[1024];
+   unsigned nr_dumped;
+
+   bool merge_notes;
+};
+
+void
+hk_frame_anatomy_note(struct hk_device *dev, const char *fmt, ...)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   if (likely(!fa))
+      return;
+
+   /* Render pass continuation notes arrive for nearly every pass; they are
+    * written only with HK_FRAME_ANATOMY_MERGE_NOTES so other notes fit. */
+   if (!fa->merge_notes && (!strncmp(fmt, "pass-merge", 10) ||
+                            !strncmp(fmt, "MSAA pass merge", 15)))
+      return;
+
+   simple_mtx_lock(&fa->lock);
+   if (fa->notes < fa->note_limit) {
+      va_list ap;
+      va_start(ap, fmt);
+      fputs("N ", fa->file);
+      vfprintf(fa->file, fmt, ap);
+      va_end(ap);
+      if (++fa->notes == fa->note_limit)
+         fputs(" (note limit reached)", fa->file);
+      fputc('\n', fa->file);
+      fflush(fa->file);
+   } else {
+      fa->notes_dropped++;
+   }
+   simple_mtx_unlock(&fa->lock);
+}
+
+uint8_t
+hk_frame_anatomy_encode_op(bool present, unsigned hw_op,
+                           VkAttachmentLoadOp load, VkAttachmentStoreOp store)
+{
+   unsigned l = load == VK_ATTACHMENT_LOAD_OP_LOAD        ? 0
+                : load == VK_ATTACHMENT_LOAD_OP_CLEAR     ? 1
+                : load == VK_ATTACHMENT_LOAD_OP_DONT_CARE ? 2
+                                                          : 3;
+   unsigned s = store == VK_ATTACHMENT_STORE_OP_STORE       ? 0
+                : store == VK_ATTACHMENT_STORE_OP_DONT_CARE ? 1
+                                                            : 2;
+   return (present ? HK_FA_OP_PRESENT : 0) | (hw_op & 3) | (l << 2) | (s << 4);
+}
+
+static inline uint64_t
+hk_fa_ns(struct hk_device *dev, uint64_t stamp)
+{
+   return stamp ? agx_gpu_timestamp_to_ns(&dev->dev, stamp) : 0;
+}
+
+static void
+hk_fa_emit(struct hk_frame_anatomy *fa, struct hk_device *dev,
+           const struct hk_frame_anatomy_rec *rec, bool complete)
+{
+   uint64_t t[HK_FA_STAMPS] = {0};
+   if (rec->slot != UINT32_MAX) {
+      for (unsigned i = 0; i < HK_FA_STAMPS; ++i) {
+         if (rec->own & BITFIELD_BIT(i))
+            t[i] = hk_fa_ns(dev, fa->ts_map[rec->slot * HK_FA_STAMPS + i]);
+      }
+   }
+
+   fputs(rec->text, fa->file);
+
+   /* Barriers exactly as submitted: NONE prints as '-', else the relative
+    * command index inside the ioctl; exact-prior ordinals when nonzero. */
+   if (rec->barriers_known) {
+      fputs(" bar=", fa->file);
+      if (rec->vdm_barrier == DRM_ASAHI_BARRIER_NONE)
+         fputs("v-", fa->file);
+      else
+         fprintf(fa->file, "v%u", rec->vdm_barrier);
+      if (rec->cdm_barrier == DRM_ASAHI_BARRIER_NONE)
+         fputs(",c-", fa->file);
+      else
+         fprintf(fa->file, ",c%u", rec->cdm_barrier);
+      if (rec->prior_vdm || rec->prior_cdm)
+         fprintf(fa->file, " prior=v%" PRIu64 ",c%" PRIu64, rec->prior_vdm,
+                 rec->prior_cdm);
+   } else {
+      fputs(" bar=?", fa->file);
+   }
+
+   /* Which stamps the command carries: p profiler, q query pool, - none. */
+   fputs(" ts=", fa->file);
+   for (unsigned i = 0; i < rec->nr_stamps; ++i) {
+      fputc((rec->own & BITFIELD_BIT(i))     ? 'p'
+            : (rec->query & BITFIELD_BIT(i)) ? 'q'
+                                             : '-',
+            fa->file);
+   }
+
+   if (rec->kind == 'R') {
+      /* t0 vertex start, t1 vertex end, t2 fragment start, t3 fragment end */
+      fprintf(fa->file, " ts_ns=%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64,
+              t[0], t[1], t[2], t[3]);
+      if (t[0] && t[1])
+         fprintf(fa->file, " vtx_us=%.1f", (t[1] - t[0]) / 1000.0);
+      if (t[1] && t[2])
+         fprintf(fa->file, " gap_us=%.1f",
+                 ((int64_t)t[2] - (int64_t)t[1]) / 1000.0);
+      if (t[2] && t[3])
+         fprintf(fa->file, " frg_us=%.1f", (t[3] - t[2]) / 1000.0);
+      if (t[0] && t[3])
+         fprintf(fa->file, " total_us=%.1f", (t[3] - t[0]) / 1000.0);
+   } else {
+      fprintf(fa->file, " ts_ns=%" PRIu64 ",%" PRIu64, t[0], t[1]);
+      if (t[0] && t[1])
+         fprintf(fa->file, " dur_us=%.1f", (t[1] - t[0]) / 1000.0);
+   }
+
+   if (rec->slot == UINT32_MAX)
+      fputs(" (no timestamp slot)", fa->file);
+   else if (!rec->point)
+      fputs(" (submit failed)", fa->file);
+   else if (!complete)
+      fputs(" (not retired at device destruction)", fa->file);
+   fputc('\n', fa->file);
+}
+
+/* Retire completed records in FIFO order. With wait (device teardown only),
+ * block up to HK_FA_FINISH_WAIT_NS per queue point and then emit every
+ * record, flagging those whose point never completed. */
+static void
+hk_fa_flush(struct hk_frame_anatomy *fa, struct hk_device *dev, bool wait)
+{
+   const unsigned n =
+      util_dynarray_num_elements(&fa->pending, struct hk_frame_anatomy_rec);
+   struct hk_frame_anatomy_rec *recs = fa->pending.data;
+   uint32_t cached_syncobj = 0;
+   uint64_t cached_value = 0;
+   bool have_cached = false;
+   unsigned done = 0;
+
+   for (unsigned i = 0; i < n; ++i) {
+      const struct hk_frame_anatomy_rec *rec = &recs[i];
+      bool complete = true;
+      if (rec->point) {
+         if (!have_cached || cached_syncobj != rec->syncobj ||
+             cached_value < rec->point) {
+            uint32_t syncobj = rec->syncobj;
+            if (wait) {
+               uint64_t point = rec->point;
+               drmSyncobjTimelineWait(dev->dev.fd, &syncobj, &point, 1,
+                                      os_time_get_nano() + HK_FA_FINISH_WAIT_NS,
+                                      0, NULL);
+            }
+            uint64_t value = 0;
+            if (drmSyncobjQuery(dev->dev.fd, &syncobj, &value, 1))
+               value = 0;
+            cached_syncobj = rec->syncobj;
+            cached_value = value;
+            have_cached = true;
+         }
+         if (cached_value < rec->point) {
+            if (!wait)
+               break;
+            complete = false;
+         }
+      }
+      hk_fa_emit(fa, dev, rec, complete);
+      done++;
+   }
+
+   if (done) {
+      memmove(recs, recs + done, (n - done) * sizeof(*recs));
+      fa->pending.size -= done * sizeof(*recs);
+      /* Every record holds exactly one ring slot. */
+      fa->ring_head = (fa->ring_head + done) % HK_FA_RING_RECS;
+      fa->ring_count -= done;
+      fflush(fa->file);
+   }
+}
+
+/* Caller holds fa->lock. */
+static bool
+hk_fa_alloc_slot(struct hk_frame_anatomy *fa, uint32_t *slot)
+{
+   if (fa->ring_count >= HK_FA_RING_RECS)
+      return false;
+   *slot = (fa->ring_head + fa->ring_count) % HK_FA_RING_RECS;
+   fa->ring_count++;
+   memset(&fa->ts_map[*slot * HK_FA_STAMPS], 0, HK_FA_STAMPS * sizeof(uint64_t));
+   return true;
+}
+
+void
+hk_frame_anatomy_init(struct hk_device *dev)
+{
+   dev->frame_anatomy = NULL;
+
+   const char *path = os_get_option("HK_FRAME_ANATOMY");
+   if (!path || !*path)
+      return;
+
+   struct hk_frame_anatomy *fa = calloc(1, sizeof(*fa));
+   if (!fa)
+      return;
+
+   int64_t every = debug_get_num_option("HK_FRAME_ANATOMY_EVERY", 200);
+   fa->every = every > 0 ? (uint64_t)every : 1;
+   int64_t notes = debug_get_num_option("HK_FRAME_ANATOMY_NOTES",
+                                        HK_FA_NOTE_LIMIT);
+   fa->note_limit = notes > 0 ? (uint32_t)MIN2(notes, INT32_MAX) : 0;
+
+   fa->merge_notes = debug_get_bool_option("HK_FRAME_ANATOMY_MERGE_NOTES", false);
+   fa->dump_dir = os_get_option("HK_FRAME_ANATOMY_DUMP");
+   if (fa->dump_dir && !*fa->dump_dir)
+      fa->dump_dir = NULL;
+
+   const char *ts = os_get_option("HK_FRAME_ANATOMY_TS");
+   fa->ts_mode = HK_FA_TS_ALL;
+   if (ts && !strcmp(ts, "edges"))
+      fa->ts_mode = HK_FA_TS_EDGES;
+   else if (ts && !strcmp(ts, "none"))
+      fa->ts_mode = HK_FA_TS_NONE;
+   else if (ts && *ts && strcmp(ts, "all"))
+      mesa_logw("HK_FRAME_ANATOMY_TS=%s unknown; using all", ts);
+
+   if (!strcmp(path, "-") || !strcmp(path, "stderr")) {
+      fa->file = stderr;
+   } else {
+      fa->file = fopen(path, "a");
+      fa->own_file = true;
+   }
+   if (!fa->file) {
+      mesa_logw("HK_FRAME_ANATOMY: cannot open %s: %s", path, strerror(errno));
+      free(fa);
+      return;
+   }
+
+   /* The kernel requires timestamp buffers to be SHARED (hk_query_pool.c). */
+   fa->ts_bo = agx_bo_create(&dev->dev, HK_FA_TS_BO_SIZE, 0,
+                             AGX_BO_WRITEBACK | AGX_BO_SHARED,
+                             "Frame anatomy timestamps");
+   if (fa->ts_bo)
+      fa->ts_map = agx_bo_map(fa->ts_bo);
+   if (!fa->ts_map ||
+       agx_bind_timestamps(&dev->dev, fa->ts_bo, &fa->ts_handle) ||
+       !fa->ts_handle) {
+      mesa_logw("HK_FRAME_ANATOMY: timestamp buffer setup failed; disabled");
+      if (fa->ts_bo)
+         agx_bo_unreference(&dev->dev, fa->ts_bo);
+      if (fa->own_file)
+         fclose(fa->file);
+      free(fa);
+      return;
+   }
+   memset(fa->ts_map, 0, HK_FA_TS_BO_SIZE);
+
+   simple_mtx_init(&fa->lock, mtx_plain);
+   util_dynarray_init(&fa->pending, NULL);
+
+   static const char *const mode_names[] = {"all", "edges", "none"};
+   fprintf(fa->file,
+           "# hk frame anatomy pid=%d chip=%s(0x%x) ts_hz=%" PRIu64
+           " every=%" PRIu64 " ts=%s ring_recs=%u features=0x%" PRIx64 "\n",
+           (int)getpid(), agx_chip_name(dev->dev.chip), dev->dev.chip,
+           (uint64_t)dev->dev.params.command_timestamp_frequency_hz,
+           fa->every, mode_names[fa->ts_mode], (unsigned)HK_FA_RING_RECS,
+           (uint64_t)dev->dev.params.features);
+   fprintf(fa->file,
+           "# S: sampled submit. R: render stream, C: compute stream; i: "
+           "hardware command index in the submit.\n"
+           "# rtN=<format>:<bg><eot>:<api load><api store>:<c|u>[/spill]  bg: "
+           "-none C clear L load; eot: -none S store; api load: L C D N; api "
+           "store: S D N; c compressed u uncompressed; '?' unknown\n"
+           "# z/s=<fmt>:<L|-><S|->:<c|u>  (ZLS control as submitted)  "
+           "pass=<B|.><E|.> B: stream begins the API pass, E: ends it; '..' "
+           "is a continuation\n"
+           "# bar=v<n>,c<n>: header vdm/cdm barrier as submitted ('-' = NONE, "
+           "n = relative command count in the ioctl); prior=: exact-prior "
+           "ordinals\n"
+           "# ts=: per stamp p profiler / q query pool / - none; ts_ns: render "
+           "vtx start,vtx end,frag start,frag end; compute start,end; 0 = not "
+           "written\n"
+           "# no TVB/partial-render indicator exists in the asahi uapi\n");
+   fflush(fa->file);
+
+   dev->frame_anatomy = fa;
+}
+
+void
+hk_frame_anatomy_finish(struct hk_device *dev)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   if (!fa)
+      return;
+   dev->frame_anatomy = NULL;
+
+   simple_mtx_lock(&fa->lock);
+   hk_fa_flush(fa, dev, true);
+   const unsigned left =
+      util_dynarray_num_elements(&fa->pending, struct hk_frame_anatomy_rec);
+   fprintf(fa->file,
+           "# end submits=%" PRIu64 " sampled=%" PRIu64
+           " unretired=%u notes=%u dropped=%u\n",
+           fa->submit_seq, fa->sampled, left, fa->notes, fa->notes_dropped);
+   fflush(fa->file);
+   simple_mtx_unlock(&fa->lock);
+
+   if (dev->dev.ops.bo_unbind_object)
+      dev->dev.ops.bo_unbind_object(&dev->dev, fa->ts_handle);
+   agx_bo_unreference(&dev->dev, fa->ts_bo);
+   util_dynarray_fini(&fa->pending);
+   simple_mtx_destroy(&fa->lock);
+   if (fa->own_file)
+      fclose(fa->file);
+   free(fa);
+}
+
+struct hk_frame_anatomy_submit *
+hk_frame_anatomy_begin_submit(struct hk_device *dev, struct hk_queue *queue,
+                              unsigned command_count)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+
+   simple_mtx_lock(&fa->lock);
+   const uint64_t seq = ++fa->submit_seq;
+   const bool sample = ((seq - 1) % fa->every) == 0;
+   if (sample) {
+      fa->sampled++;
+      hk_fa_flush(fa, dev, false);
+   }
+   simple_mtx_unlock(&fa->lock);
+
+   if (!sample)
+      return NULL;
+
+   struct hk_frame_anatomy_submit *s = calloc(1, sizeof(*s));
+   if (!s)
+      return NULL;
+   util_dynarray_init(&s->recs, NULL);
+   s->seq = seq;
+   s->cpu_ns = os_time_get_nano();
+   s->command_count = command_count;
+   return s;
+}
+
+/* Every record takes a ring slot (so FIFO retirement stays one slot per
+ * record); only the stamps actually claimed are ever read. */
+static struct hk_frame_anatomy_rec *
+hk_fa_new_rec(struct hk_frame_anatomy_submit *s, struct hk_device *dev,
+              char kind, size_t header_offset)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   struct hk_frame_anatomy_rec *rec =
+      util_dynarray_grow(&s->recs, struct hk_frame_anatomy_rec, 1);
+   if (!rec)
+      return NULL;
+   memset(rec, 0, sizeof(*rec));
+   rec->kind = kind;
+   rec->nr_stamps = kind == 'R' ? 4 : 2;
+   rec->slot = UINT32_MAX;
+   rec->header_off = header_offset;
+
+   simple_mtx_lock(&fa->lock);
+   if (!hk_fa_alloc_slot(fa, &rec->slot)) {
+      rec->slot = UINT32_MAX;
+      s->dropped++;
+   }
+   simple_mtx_unlock(&fa->lock);
+   return rec;
+}
+
+/* Claim stamp i of the record for the kernel to write unless a query owns it. */
+static void
+hk_fa_claim(struct hk_frame_anatomy *fa, struct hk_frame_anatomy_rec *rec,
+            unsigned i, struct drm_asahi_timestamp *ts)
+{
+   if (ts->handle) {
+      if (ts->handle != fa->ts_handle)
+         rec->query |= BITFIELD_BIT(i);
+      return;
+   }
+   if (rec->slot == UINT32_MAX)
+      return;
+   ts->handle = fa->ts_handle;
+   ts->offset = (rec->slot * HK_FA_STAMPS + i) * sizeof(uint64_t);
+   rec->own |= BITFIELD_BIT(i);
+}
+
+static void
+hk_fa_note_query(struct hk_frame_anatomy_rec *rec, unsigned i,
+                 const struct drm_asahi_timestamp *ts)
+{
+   if (ts->handle)
+      rec->query |= BITFIELD_BIT(i);
+}
+
+static void
+hk_fa_claim_render(struct hk_frame_anatomy *fa,
+                   struct hk_frame_anatomy_rec *rec,
+                   struct drm_asahi_cmd_render *c)
+{
+   hk_fa_claim(fa, rec, 0, &c->ts_vtx.start);
+   hk_fa_claim(fa, rec, 1, &c->ts_vtx.end);
+   hk_fa_claim(fa, rec, 2, &c->ts_frag.start);
+   hk_fa_claim(fa, rec, 3, &c->ts_frag.end);
+}
+
+static void
+hk_fa_claim_compute(struct hk_frame_anatomy *fa,
+                    struct hk_frame_anatomy_rec *rec,
+                    struct drm_asahi_cmd_compute *c)
+{
+   hk_fa_claim(fa, rec, 0, &c->ts.start);
+   hk_fa_claim(fa, rec, 1, &c->ts.end);
+}
+
+#define HK_FA_APPEND(...)                                                     \
+   do {                                                                       \
+      if (len < sizeof(rec->text)) {                                          \
+         int _r = snprintf(rec->text + len, sizeof(rec->text) - len,          \
+                           __VA_ARGS__);                                      \
+         if (_r > 0)                                                          \
+            len += MIN2((size_t)_r, sizeof(rec->text) - 1 - len);             \
+      }                                                                       \
+   } while (0)
+
+static char
+hk_fa_hw_char(unsigned op)
+{
+   static const char c[4] = {'-', 'C', 'L', 'S'};
+   return c[op & 3];
+}
+
+void
+hk_frame_anatomy_dump_nir(struct hk_device *dev, const struct hk_shader *shader,
+                          struct nir_shader *nir, unsigned stage)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   if (!fa || !fa->dump_dir || !shader->code_size ||
+       (stage != MESA_SHADER_FRAGMENT && stage != MESA_SHADER_COMPUTE))
+      return;
+
+   char path[512];
+   snprintf(path, sizeof(path), "%s/%s-%08x.nir", fa->dump_dir,
+            stage == MESA_SHADER_FRAGMENT ? "fs" : "cs",
+            _mesa_hash_data(shader->code_ptr, shader->code_size));
+   if (access(path, F_OK) == 0)
+      return;
+   FILE *f = fopen(path, "w");
+   if (f) {
+      nir_print_shader(nir, f);
+      fclose(f);
+   }
+}
+
+static void
+hk_fa_dump_fs(struct hk_frame_anatomy *fa, uint32_t hash,
+              const struct hk_shader *fs, const char *prefix)
+{
+   if (!fa->dump_dir || !fs->code_size)
+      return;
+
+   simple_mtx_lock(&fa->lock);
+   bool dump = fa->nr_dumped < ARRAY_SIZE(fa->dumped);
+   for (unsigned i = 0; dump && i < fa->nr_dumped; ++i)
+      dump = fa->dumped[i] != hash;
+   if (dump)
+      fa->dumped[fa->nr_dumped++] = hash;
+   simple_mtx_unlock(&fa->lock);
+   if (!dump)
+      return;
+
+   char path[512];
+   snprintf(path, sizeof(path), "%s/%s-%08x.bin", fa->dump_dir, prefix, hash);
+   FILE *f = fopen(path, "wb");
+   if (f) {
+      fwrite(fs->code_ptr, 1, fs->code_size, f);
+      fclose(f);
+   }
+}
+
+void
+hk_frame_anatomy_render(struct hk_frame_anatomy_submit *s,
+                        struct hk_device *dev, const struct hk_cs *cs,
+                        struct drm_asahi_cmd_render *c, unsigned cmdbuf_index,
+                        unsigned command_index, size_t header_offset)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   struct hk_frame_anatomy_rec *rec = hk_fa_new_rec(s, dev, 'R', header_offset);
+   if (!rec)
+      return;
+
+   if (fa->ts_mode == HK_FA_TS_ALL) {
+      hk_fa_claim_render(fa, rec, c);
+   } else {
+      hk_fa_note_query(rec, 0, &c->ts_vtx.start);
+      hk_fa_note_query(rec, 1, &c->ts_vtx.end);
+      hk_fa_note_query(rec, 2, &c->ts_frag.start);
+      hk_fa_note_query(rec, 3, &c->ts_frag.end);
+   }
+
+   size_t len = 0;
+   HK_FA_APPEND("R s=%" PRIu64 " cb=%u i=%u fb=%ux%ux%u ms=%u tile=%ux%u "
+                "tib=%uB draws=%u api=%u pass=%c%c flags=0x%x",
+                s->seq, cmdbuf_index, command_index, c->width_px,
+                c->height_px, c->layers, c->samples, c->utile_width_px,
+                c->utile_height_px, c->sample_size_B, cs->stats.cmds,
+                cs->stats.calls, cs->g17p_pass_begins ? 'B' : '.',
+                cs->g17p_pass_ends ? 'E' : '.', c->flags);
+
+   /* Depth/stencil exactly as the ZLS unit is programmed for this command. */
+   agx_unpack(NULL, &cs->cr.zls_control, ZLS_CONTROL, zls);
+   if (cs->cr.depth.buffer) {
+      HK_FA_APPEND(" z=%s:%c%c:%c",
+                   zls.z_format == AGX_ZLS_FORMAT_32F ? "z32f" : "z16",
+                   zls.z_load ? 'L' : '-', zls.z_store ? 'S' : '-',
+                   cs->cr.depth.meta ? 'c' : 'u');
+   }
+   if (cs->cr.stencil.buffer) {
+      HK_FA_APPEND(" s=s8:%c%c:%c", zls.s_load ? 'L' : '-',
+                   zls.s_store ? 'S' : '-', cs->cr.stencil.meta ? 'c' : 'u');
+   }
+   if (cs->cr.depth.buffer || cs->cr.stencil.buffer)
+      HK_FA_APPEND(" zls=%ux%u", cs->cr.zls_width, cs->cr.zls_height);
+
+   for (unsigned rt = 0; rt < HK_MAX_RTS; ++rt) {
+      const enum pipe_format fmt = cs->tib.logical_format[rt];
+      const uint8_t bg = cs->cr.bg.main.anatomy_ops[rt];
+      const uint8_t eot = cs->cr.eot.main.anatomy_ops[rt];
+      if (fmt == PIPE_FORMAT_NONE && !(bg & HK_FA_OP_PRESENT))
+         continue;
+
+      static const char load_c[4] = {'L', 'C', 'D', 'N'};
+      static const char store_c[4] = {'S', 'D', 'N', '?'};
+      HK_FA_APPEND(" rt%u=%s:", rt,
+                   fmt == PIPE_FORMAT_NONE ? "none" : util_format_short_name(fmt));
+      if (bg & HK_FA_OP_PRESENT) {
+         HK_FA_APPEND("%c%c:%c%c", hk_fa_hw_char(HK_FA_OP_HW(bg)),
+                      hk_fa_hw_char(HK_FA_OP_HW(eot)),
+                      load_c[HK_FA_OP_LOAD(bg)], store_c[HK_FA_OP_STORE(bg)]);
+      } else {
+         HK_FA_APPEND("??:??");
+      }
+      HK_FA_APPEND(":%c%s",
+                   (cs->cr.color_compressed_mask & BITFIELD_BIT(rt)) ? 'c' : 'u',
+                   cs->tib.spilled[rt] ? "/spill" : "");
+   }
+
+   if (cs->anatomy_area.extent.width)
+      HK_FA_APPEND(" area=%ux%u+%d+%d", cs->anatomy_area.extent.width,
+                   cs->anatomy_area.extent.height, cs->anatomy_area.offset.x,
+                   cs->anatomy_area.offset.y);
+   HK_FA_APPEND(" pet=%u scratch=%c%c", cs->cr.process_empty_tiles ? 1 : 0,
+                (c->flags & DRM_ASAHI_RENDER_VERTEX_SCRATCH) ? 'v' : '-',
+                c->fragment_helper.data ? 'f' : '-');
+   if (cs->g17p_primitive_bound != UINT64_MAX)
+      HK_FA_APPEND(" prims<=%" PRIu64, cs->g17p_primitive_bound);
+
+   /* Fragment programs: hash of the main binary, draws, then the compiler's
+    * instruction count, registers, threads, spills+fills, scratch bytes and
+    * sample-rate shading. */
+   for (unsigned i = 0; i < HK_FA_FS_MAX && cs->anatomy_fs[i].fs; ++i) {
+      const struct hk_shader *fs = cs->anatomy_fs[i].fs;
+      const struct agx2_stats *st = &fs->b.info.stats;
+      const uint32_t hash =
+         fs->code_size ? _mesa_hash_data(fs->code_ptr, fs->code_size) : 0;
+      hk_fa_dump_fs(fa, hash, fs, "fs");
+      HK_FA_APPEND(" fs=%08x:d%u:i%u:g%u:t%u:sp%u:sc%u:ss%u", hash,
+                   cs->anatomy_fs[i].draws, st->instrs, st->gprs, st->threads,
+                   st->spills + st->fills, st->scratch,
+                   fs->info.fs.uses_sample_shading);
+   }
+   if (cs->anatomy_fs_other)
+      HK_FA_APPEND(" fs_other=%u", cs->anatomy_fs_other);
+   /* Vertex programs: hash, draws, instructions, registers, threads,
+    * spills+fills. */
+   for (unsigned i = 0; i < HK_FA_FS_MAX && cs->anatomy_vs[i].vs; ++i) {
+      const struct hk_shader *vs = cs->anatomy_vs[i].vs;
+      const struct agx2_stats *st = &vs->b.info.stats;
+      const uint32_t hash =
+         vs->code_size ? _mesa_hash_data(vs->code_ptr, vs->code_size) : 0;
+      HK_FA_APPEND(" vs=%08x:d%u:i%u:g%u:t%u:sp%u", hash,
+                   cs->anatomy_vs[i].draws, st->instrs, st->gprs, st->threads,
+                   st->spills + st->fills);
+   }
+   if (cs->anatomy_vs_other)
+      HK_FA_APPEND(" vs_other=%u", cs->anatomy_vs_other);
+   if (cs->anatomy_covered)
+      HK_FA_APPEND(" cov=%x", cs->anatomy_covered);
+   if (cs->anatomy_republish)
+      HK_FA_APPEND(" st=r%u:v%u:n%u:p%u", cs->anatomy_republish,
+                   cs->anatomy_vdm, cs->anatomy_vdm_new, cs->anatomy_ppp);
+   if (cs->anatomy_pass[0] | cs->anatomy_pass[1] | cs->anatomy_pass[2] |
+       cs->anatomy_pass[3])
+      HK_FA_APPEND(" pt=o%u:t%u:p%u:tp%u:nm%u:zw%u:lz%u", cs->anatomy_pass[0],
+                   cs->anatomy_pass[1], cs->anatomy_pass[2],
+                   cs->anatomy_pass[3], cs->anatomy_nomerge,
+                   cs->anatomy_zwrite, cs->anatomy_late_z);
+   if (cs->anatomy_api_barriers)
+      HK_FA_APPEND(" apibar=%u:%" PRIx64 ":%" PRIx64 ":%" PRIx64 ":%" PRIx64,
+                   cs->anatomy_api_barriers, cs->anatomy_api_src,
+                   cs->anatomy_api_dst, cs->anatomy_api_src_access,
+                   cs->anatomy_api_dst_access);
+   if (cs->anatomy_api_barriers)
+      HK_FA_APPEND(" apibar_n=m%u:b%u:i%u", cs->anatomy_api_mem,
+                   cs->anatomy_api_buf, cs->anatomy_api_img);
+}
+
+void
+hk_frame_anatomy_note_fs(struct hk_cs *cs, const struct hk_shader *fs)
+{
+   if (!fs)
+      return;
+
+   /* T6040's inline blend can replace the API fragment main with a
+    * monolithic program owning its constant-program layout. Counting the
+    * API main here reports only its export copies and hides the blend,
+    * registers and scratch actually launched. The linked shader owns this
+    * layout for the same lifetime as the API shader. Compute streams keep
+    * recording their compute shader, and other products keep their existing
+    * profiler output. This hook is reached only with frame anatomy enabled.
+    */
+   if (cs->type == HK_CS_VDM &&
+       agx_is_m4_pro(&hk_cmd_buffer_device(cs->cmd)->dev)) {
+      const struct hk_linked_shader *linked =
+         cs->cmd->state.gfx.linked[MESA_SHADER_FRAGMENT];
+      if (linked && linked->g17p_layout)
+         fs = linked->g17p_layout;
+   }
+
+   for (unsigned i = 0; i < HK_FA_FS_MAX; ++i) {
+      if (cs->anatomy_fs[i].fs == fs || !cs->anatomy_fs[i].fs) {
+         cs->anatomy_fs[i].fs = fs;
+         cs->anatomy_fs[i].draws++;
+         return;
+      }
+   }
+   cs->anatomy_fs_other++;
+}
+
+void
+hk_frame_anatomy_note_grid(struct hk_cs *cs, const struct hk_shader *shader,
+                           uint64_t threads, uint64_t groups, bool indirect)
+{
+   for (unsigned i = 0; i < HK_FA_FS_MAX; ++i) {
+      if (cs->anatomy_fs[i].fs == shader) {
+         if (indirect)
+            cs->anatomy_fs[i].indirect++;
+         cs->anatomy_fs[i].threads += threads;
+         cs->anatomy_fs[i].groups += groups;
+         return;
+      }
+   }
+}
+
+void
+hk_frame_anatomy_note_m5_fragment(struct hk_cs *cs)
+{
+   if (!cs->m5_ppp_emitted_valid)
+      return;
+   const struct agx_ppp_hal300_graphics_fields *f = &cs->m5_ppp_emitted;
+   cs->anatomy_pass[f->fragment_pass & 3]++;
+   cs->anatomy_nomerge += f->disable_tri_merging;
+   cs->anatomy_zwrite += f->depth_active && f->depth_write;
+   cs->anatomy_late_z += f->fragment_depth_mode != 3;
+}
+
+void
+hk_frame_anatomy_note_vs(struct hk_cs *cs, const struct hk_shader *vs)
+{
+   if (!vs)
+      return;
+   for (unsigned i = 0; i < HK_FA_FS_MAX; ++i) {
+      if (cs->anatomy_vs[i].vs == vs || !cs->anatomy_vs[i].vs) {
+         cs->anatomy_vs[i].vs = vs;
+         cs->anatomy_vs[i].draws++;
+         return;
+      }
+   }
+   cs->anatomy_vs_other++;
+}
+
+void
+hk_frame_anatomy_note_precomp(struct hk_cs *cs, unsigned program,
+                              struct agx_grid grid)
+{
+   const uint64_t threads =
+      agx_is_indirect(grid)
+         ? UINT64_MAX
+         : (uint64_t)grid.count[0] * grid.count[1] * grid.count[2];
+   for (unsigned i = 0; i < HK_FA_FS_MAX; ++i) {
+      if (!cs->anatomy_precomp[i].dispatches ||
+          cs->anatomy_precomp[i].program == program) {
+         cs->anatomy_precomp[i].program = program;
+         cs->anatomy_precomp[i].dispatches++;
+         if (threads == UINT64_MAX ||
+             cs->anatomy_precomp[i].threads == UINT64_MAX)
+            cs->anatomy_precomp[i].threads = UINT64_MAX;
+         else
+            cs->anatomy_precomp[i].threads += threads;
+         return;
+      }
+   }
+   cs->anatomy_precomp_other++;
+}
+
+void
+hk_frame_anatomy_compute(struct hk_frame_anatomy_submit *s,
+                         struct hk_device *dev, const struct hk_cs *cs,
+                         struct drm_asahi_cmd_compute *c, unsigned cmdbuf_index,
+                         unsigned command_index, size_t header_offset)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   struct hk_frame_anatomy_rec *rec = hk_fa_new_rec(s, dev, 'C', header_offset);
+   if (!rec)
+      return;
+
+   if (fa->ts_mode == HK_FA_TS_ALL) {
+      hk_fa_claim_compute(fa, rec, c);
+   } else {
+      hk_fa_note_query(rec, 0, &c->ts.start);
+      hk_fa_note_query(rec, 1, &c->ts.end);
+   }
+
+   size_t len = 0;
+   HK_FA_APPEND("C s=%" PRIu64 " cb=%u i=%u dispatches=%u api=%u flushes=%u "
+                "merged=%u imm=%u rq=%u matrix=%u scratch=%c",
+                s->seq, cmdbuf_index, command_index, cs->stats.cmds,
+                cs->stats.calls, cs->stats.flushes, cs->stats.merged,
+                (unsigned)util_dynarray_num_elements(&cs->imm_writes,
+                                                     struct libagx_imm_write),
+                cs->m5_native_rq || cs->gen1_native_rq ? 1 : 0,
+                cs->m5_matrix_ops ? 1 : 0, c->helper.data ? 'c' : '-');
+   /* Compute programs, as for fragment programs in render records. */
+   for (unsigned i = 0; i < HK_FA_FS_MAX && cs->anatomy_fs[i].fs; ++i) {
+      const struct hk_shader *prog = cs->anatomy_fs[i].fs;
+      const struct agx2_stats *st = &prog->b.info.stats;
+      const uint32_t hash = prog->code_size
+         ? _mesa_hash_data(prog->code_ptr, prog->code_size) : 0;
+      hk_fa_dump_fs(fa, hash, prog, "cs");
+      HK_FA_APPEND(" cs=%08x:d%u:i%u:g%u:t%u:sp%u:sc%u:n%" PRIu64 ":w%" PRIu64
+                   ":x%u", hash,
+                   cs->anatomy_fs[i].draws, st->instrs, st->gprs, st->threads,
+                   st->spills + st->fills, st->scratch,
+                   cs->anatomy_fs[i].threads, cs->anatomy_fs[i].groups,
+                   cs->anatomy_fs[i].indirect);
+   }
+   if (cs->anatomy_fs_other)
+      HK_FA_APPEND(" cs_other=%u", cs->anatomy_fs_other);
+   if (cs->anatomy_api_barriers)
+      HK_FA_APPEND(" apibar=%u:%" PRIx64 ":%" PRIx64 ":%" PRIx64 ":%" PRIx64,
+                   cs->anatomy_api_barriers, cs->anatomy_api_src,
+                   cs->anatomy_api_dst, cs->anatomy_api_src_access,
+                   cs->anatomy_api_dst_access);
+   if (cs->anatomy_api_barriers)
+      HK_FA_APPEND(" apibar_n=m%u:b%u:i%u", cs->anatomy_api_mem,
+                   cs->anatomy_api_buf, cs->anatomy_api_img);
+   for (unsigned i = 0; i < HK_FA_FS_MAX && cs->anatomy_precomp[i].dispatches;
+        ++i) {
+      if (cs->anatomy_precomp[i].threads == UINT64_MAX)
+         HK_FA_APPEND(" pre=%u:d%u:indirect", cs->anatomy_precomp[i].program,
+                      cs->anatomy_precomp[i].dispatches);
+      else
+         HK_FA_APPEND(" pre=%u:d%u:t%" PRIu64, cs->anatomy_precomp[i].program,
+                      cs->anatomy_precomp[i].dispatches,
+                      cs->anatomy_precomp[i].threads);
+   }
+   if (cs->anatomy_precomp_other)
+      HK_FA_APPEND(" pre_other=%u", cs->anatomy_precomp_other);
+}
+
+/* The command struct a record's header introduces, if the payload holds it. */
+static void *
+hk_fa_payload_cmd(const struct hk_frame_anatomy_rec *rec, void *payload,
+                  size_t payload_size, struct drm_asahi_cmd_header **hdr)
+{
+   if ((size_t)rec->header_off + sizeof(**hdr) > payload_size)
+      return NULL;
+   struct drm_asahi_cmd_header *h =
+      (struct drm_asahi_cmd_header *)((uint8_t *)payload + rec->header_off);
+   const uint16_t expect =
+      rec->kind == 'R' ? DRM_ASAHI_CMD_RENDER : DRM_ASAHI_CMD_COMPUTE;
+   if (h->cmd_type != expect ||
+       (size_t)rec->header_off + sizeof(*h) + h->size > payload_size)
+      return NULL;
+   const size_t need = rec->kind == 'R' ? sizeof(struct drm_asahi_cmd_render)
+                                        : sizeof(struct drm_asahi_cmd_compute);
+   if (h->size < need)
+      return NULL;
+   if (hdr)
+      *hdr = h;
+   return h + 1;
+}
+
+void
+hk_frame_anatomy_prepare(struct hk_frame_anatomy_submit *s,
+                         struct hk_device *dev, void *payload,
+                         size_t payload_size)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   if (fa->ts_mode != HK_FA_TS_EDGES)
+      return;
+
+   const unsigned n =
+      util_dynarray_num_elements(&s->recs, struct hk_frame_anatomy_rec);
+   if (!n)
+      return;
+
+   /* First and last recorded hardware command of the submit (one command
+    * when n == 1). Claiming happens in the payload copy the kernel reads. */
+   const unsigned edges[2] = {0, n - 1};
+   for (unsigned e = 0; e < (n == 1 ? 1u : 2u); ++e) {
+      struct hk_frame_anatomy_rec *rec =
+         util_dynarray_element(&s->recs, struct hk_frame_anatomy_rec, edges[e]);
+      void *cmd = hk_fa_payload_cmd(rec, payload, payload_size, NULL);
+      if (!cmd)
+         continue;
+      if (rec->kind == 'R')
+         hk_fa_claim_render(fa, rec, cmd);
+      else
+         hk_fa_claim_compute(fa, rec, cmd);
+   }
+}
+
+void
+hk_frame_anatomy_note_barriers(struct hk_frame_anatomy_submit *s,
+                               const void *payload, size_t begin, size_t end)
+{
+   util_dynarray_foreach(&s->recs, struct hk_frame_anatomy_rec, rec) {
+      if (rec->header_off < begin || rec->header_off >= end)
+         continue;
+      struct drm_asahi_cmd_header *h = NULL;
+      const void *cmd = hk_fa_payload_cmd(rec, (void *)payload, end, &h);
+      if (!cmd)
+         continue;
+      rec->vdm_barrier = h->vdm_barrier;
+      rec->cdm_barrier = h->cdm_barrier;
+      rec->barriers_known = true;
+      rec->prior_vdm = rec->prior_cdm = 0;
+      if (rec->kind == 'R' &&
+          h->size >= sizeof(struct drm_asahi_cmd_render_with_prior)) {
+         const struct drm_asahi_cmd_render_with_prior *r = cmd;
+         rec->prior_vdm = r->prior_vdm;
+         rec->prior_cdm = r->prior_cdm;
+      } else if (rec->kind == 'C' &&
+                 h->size >= offsetof(struct drm_asahi_cmd_compute_with_prior,
+                                     prior_cdm) +
+                               sizeof(uint64_t)) {
+         const struct drm_asahi_cmd_compute_with_prior *k = cmd;
+         rec->prior_vdm = k->prior_vdm;
+         rec->prior_cdm = k->prior_cdm;
+      }
+   }
+}
+
+void
+hk_frame_anatomy_end_submit(struct hk_frame_anatomy_submit *s,
+                            struct hk_device *dev, struct hk_queue *queue,
+                            VkResult result)
+{
+   struct hk_frame_anatomy *fa = dev->frame_anatomy;
+   const unsigned n =
+      util_dynarray_num_elements(&s->recs, struct hk_frame_anatomy_rec);
+
+   simple_mtx_lock(&fa->lock);
+   fprintf(fa->file,
+           "S s=%" PRIu64 " q=%u pt=%" PRIu64 " cmds=%u recs=%u dropped=%u "
+           "cpu_ns=%" PRId64 " result=%d\n",
+           s->seq, queue->drm.syncobj, queue->drm.timeline_value,
+           s->command_count, n, s->dropped, s->cpu_ns, (int)result);
+
+   util_dynarray_foreach(&s->recs, struct hk_frame_anatomy_rec, rec) {
+      rec->syncobj = queue->drm.syncobj;
+      rec->point = result == VK_SUCCESS ? queue->drm.timeline_value : 0;
+      util_dynarray_append(&fa->pending, *rec);
+   }
+   if (result != VK_SUCCESS)
+      hk_fa_flush(fa, dev, false);
+   simple_mtx_unlock(&fa->lock);
+
+   util_dynarray_fini(&s->recs);
+   free(s);
+}
diff --git a/src/asahi/vulkan/hk_frame_anatomy.h b/src/asahi/vulkan/hk_frame_anatomy.h
new file mode 100644
index 000000000000..2955e74388ea
--- /dev/null
+++ b/src/asahi/vulkan/hk_frame_anatomy.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2026 Aurora Silicon contributors
+ * SPDX-License-Identifier: MIT
+ *
+ * Opt-in "frame anatomy" profiler: one compact log line per submitted
+ * control stream (render or compute) with the hardware's framebuffer and
+ * attachment load/store configuration, the barriers exactly as submitted and
+ * firmware GPU timestamps.
+ *
+ * Enabled only by HK_FRAME_ANATOMY=<path> (or "-" for stderr). Every
+ * HK_FRAME_ANATOMY_EVERY-th queue submit is sampled (default 200).
+ * HK_FRAME_ANATOMY_TS selects which commands of a sampled submit get profiler
+ * timestamps: "all" (default), "edges" (first and last hardware command only;
+ * the M5 kernel turns every timestamped command into a host-prefix producer,
+ * so this is the low-perturbation whole-submit measurement) or "none".
+ * When HK_FRAME_ANATOMY is unset, hk_device::frame_anatomy stays NULL and
+ * every hook below reduces to one predictable NULL test; no control stream,
+ * shader or kernel command is changed.
+ */
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "util/macros.h"
+#include "vulkan/vulkan_core.h"
+
+struct hk_device;
+struct hk_queue;
+struct hk_cs;
+struct hk_shader;
+struct drm_asahi_cmd_render;
+struct drm_asahi_cmd_compute;
+struct hk_frame_anatomy_submit;
+
+/* Per-render-target load/store summary carried by struct hk_bg_eot, recorded
+ * only while the profiler is enabled. Bit 7 marks an attachment with an image
+ * view; bits 0-1 hold the op the program actually performs (0 none, 1 clear,
+ * 2 load, 3 store); bits 2-3 hold the API loadOp (0 LOAD, 1 CLEAR, 2
+ * DONT_CARE, 3 NONE); bits 4-5 hold the API storeOp (0 STORE, 1 DONT_CARE,
+ * 2 NONE). */
+#define HK_FA_OP_PRESENT 0x80u
+#define HK_FA_OP_HW(x) ((x) & 3u)
+#define HK_FA_OP_LOAD(x) (((x) >> 2) & 3u)
+#define HK_FA_OP_STORE(x) (((x) >> 4) & 3u)
+
+uint8_t hk_frame_anatomy_encode_op(bool present, unsigned hw_op,
+                                   VkAttachmentLoadOp load,
+                                   VkAttachmentStoreOp store);
+
+void hk_frame_anatomy_init(struct hk_device *dev);
+void hk_frame_anatomy_finish(struct hk_device *dev);
+
+/* One free-form "N <text>" line about a recording-time decision (not tied to
+ * a sampled submit). Nothing unless the profiler is enabled; at most
+ * HK_FRAME_ANATOMY_NOTES (default HK_FA_NOTE_LIMIT) lines per device, so a
+ * hot path cannot flood the log; the end line counts the dropped ones. */
+#define HK_FA_NOTE_LIMIT 4096
+void hk_frame_anatomy_note(struct hk_device *dev, const char *fmt, ...)
+   PRINTFLIKE(2, 3);
+
+/* Count one draw of a render stream with fragment program fs (NULL for no
+ * fragment shader). Called only while the profiler is enabled. */
+void hk_frame_anatomy_note_fs(struct hk_cs *cs, const struct hk_shader *fs);
+void hk_frame_anatomy_note_vs(struct hk_cs *cs, const struct hk_shader *vs);
+void hk_frame_anatomy_note_m5_fragment(struct hk_cs *cs);
+void hk_frame_anatomy_note_grid(struct hk_cs *cs, const struct hk_shader *cs_shader,
+                                 uint64_t threads, uint64_t groups, bool indirect);
+
+/* HK_FRAME_ANATOMY_DUMP only: write the final NIR of a compiled fragment or
+ * compute program to <dir>/{fs,cs}-<binary hash>.nir. */
+struct nir_shader;
+void hk_frame_anatomy_dump_nir(struct hk_device *dev,
+                               const struct hk_shader *shader,
+                               struct nir_shader *nir, unsigned stage);
+
+/* Count one dispatch of precompiled internal program `program`. */
+struct agx_grid;
+void hk_frame_anatomy_note_precomp(struct hk_cs *cs, unsigned program,
+                                   struct agx_grid grid);
+
+/* Returns NULL unless this submit is sampled. */
+struct hk_frame_anatomy_submit *
+hk_frame_anatomy_begin_submit(struct hk_device *dev, struct hk_queue *queue,
+                              unsigned command_count);
+
+/* Record one hardware command of a sampled submit. header_offset is the
+ * payload offset at which the command's drm_asahi_cmd_header is appended
+ * next. In "all" mode this claims the command's free timestamp slots (never
+ * one a query already owns). */
+void hk_frame_anatomy_render(struct hk_frame_anatomy_submit *s,
+                             struct hk_device *dev, const struct hk_cs *cs,
+                             struct drm_asahi_cmd_render *c,
+                             unsigned cmdbuf_index, unsigned command_index,
+                             size_t header_offset);
+void hk_frame_anatomy_compute(struct hk_frame_anatomy_submit *s,
+                              struct hk_device *dev, const struct hk_cs *cs,
+                              struct drm_asahi_cmd_compute *c,
+                              unsigned cmdbuf_index, unsigned command_index,
+                              size_t header_offset);
+
+/* Payload complete, before any ioctl: "edges" mode claims the timestamp
+ * slots of the first and last recorded command in place. */
+void hk_frame_anatomy_prepare(struct hk_frame_anatomy_submit *s,
+                              struct hk_device *dev, void *payload,
+                              size_t payload_size);
+
+/* Capture each command's header barriers (and exact-prior ordinals) from the
+ * payload bytes [begin, end) exactly as they are about to reach the kernel. */
+void hk_frame_anatomy_note_barriers(struct hk_frame_anatomy_submit *s,
+                                    const void *payload, size_t begin,
+                                    size_t end);
+
+/* Queue the records for readback once queue->drm.timeline_value completes;
+ * a failed result retires them immediately without timestamps. */
+void hk_frame_anatomy_end_submit(struct hk_frame_anatomy_submit *s,
+                                 struct hk_device *dev, struct hk_queue *queue,
+                                 VkResult result);
diff --git a/src/asahi/vulkan/hk_g17p_compression.h b/src/asahi/vulkan/hk_g17p_compression.h
index de4f12a2fe04..4b4004a3d706 100644
--- a/src/asahi/vulkan/hk_g17p_compression.h
+++ b/src/asahi/vulkan/hk_g17p_compression.h
@@ -5,6 +5,7 @@
 #pragma once
 #include "asahi/lib/agx_chip.h"
 #include "hk_g17p_meta_copy_core.h"
+#include "drm-uapi/asahi_drm.h"
 #include "vulkan/vulkan_core.h"
 #include <stdbool.h>
 #include <stddef.h>
@@ -128,17 +129,27 @@ hk_g17p_compression_depth_format(VkFormat format)
  * compatibility, ail_can_compress, the AGX-wide nocompress debug flag) still
  * apply. */
 static inline bool
-hk_g17p_compression_eligible(enum agx_chip chip, unsigned plane,
-                             const VkImageCreateInfo *info)
+hk_hal200_compression_eligible(enum agx_chip chip, unsigned plane,
+                               const VkImageCreateInfo *info, bool g16s)
 {
    if (chip != AGX_CHIP_G17P || !info ||
        (plane != 0 && !(plane == 1 &&
                         info->format == VK_FORMAT_D32_SFLOAT_S8_UINT)))
       return false;
 
-   /* Shared images keep their plain layout contract. Read each header by
-    * copy: the chain's structures have other types, and Mesa builds with
-    * strict aliasing (gcc -O2 dropped the stores of a stack chain). */
+   /* Keep depth/stencil on the existing single-level/layer/sample policy.
+    * The expanded G16S ZLS allocation shapes also reach colour-alias transfer
+    * and clear paths; those do not yet support every compressed D/S shape.
+    * Admitting them at allocation makes legal commands fail at EndCommandBuffer.
+    * Colour's layered/MSAA packet support is independent of this restriction.
+    */
+   g16s = g16s && hk_g17p_compression_color_format(info->format);
+
+   /* Shared images keep their plain layout contract. Read each header field
+    * by copy: the chain's structures have other types, and Mesa builds with
+    * strict aliasing (gcc -O2 dropped the stores of a stack chain). Copy pNext
+    * straight into next: through a copied VkBaseInStructure, GCC 16's
+    * points-to analysis loses the chain and drops its stores again. */
    for (const void *next = info->pNext; next;) {
       VkStructureType type;
       const void *following;
@@ -158,6 +169,11 @@ hk_g17p_compression_eligible(enum agx_chip chip, unsigned plane,
 
    VkImageCreateFlags allowed_flags =
       VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
+   if (g16s && info->imageType == VK_IMAGE_TYPE_2D)
+      allowed_flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
+   if (g16s && info->imageType == VK_IMAGE_TYPE_3D)
+      allowed_flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT |
+                       VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT;
    /* Zink marks every depth attachment sample-location compatible, including
     * single-sample images. This changes sample-location tracking, not the
     * depth plane's storage class. The one-sample restriction below remains.
@@ -182,14 +198,41 @@ hk_g17p_compression_eligible(enum agx_chip chip, unsigned plane,
    if (hk_g17p_compression_depth_format(info->format))
       allowed_usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
 
-   if (info->imageType != VK_IMAGE_TYPE_2D || info->extent.depth != 1 ||
-       info->arrayLayers != 1 || info->samples != VK_SAMPLE_COUNT_1_BIT ||
-       info->tiling != VK_IMAGE_TILING_OPTIMAL || info->mipLevels < 1 ||
-       (info->flags & ~allowed_flags) || (info->usage & ~allowed_usage) ||
-       info->extent.width < 16 || info->extent.height < 16)
+   const bool multisampled = info->samples != VK_SAMPLE_COUNT_1_BIT;
+   const bool three_d = info->imageType == VK_IMAGE_TYPE_3D;
+   if (!g16s) {
+      if (info->imageType != VK_IMAGE_TYPE_2D || info->extent.depth != 1 ||
+          info->arrayLayers != 1 || info->samples != VK_SAMPLE_COUNT_1_BIT ||
+          info->tiling != VK_IMAGE_TILING_OPTIMAL || info->mipLevels < 1 ||
+          (info->flags & ~allowed_flags) || (info->usage & ~allowed_usage) ||
+          info->extent.width < 16 || info->extent.height < 16)
+         return false;
+   } else if ((info->imageType != VK_IMAGE_TYPE_2D && !three_d) ||
+              !info->extent.depth || !info->arrayLayers ||
+              (three_d ? (info->arrayLayers != 1 || multisampled)
+                       : info->extent.depth != 1) ||
+              (multisampled &&
+               ((info->samples != VK_SAMPLE_COUNT_2_BIT &&
+                 info->samples != VK_SAMPLE_COUNT_4_BIT) || info->mipLevels != 1)) ||
+              ((info->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) &&
+               (multisampled || info->extent.width != info->extent.height ||
+                info->arrayLayers % 6)) ||
+              info->tiling != VK_IMAGE_TILING_OPTIMAL || info->mipLevels < 1 ||
+              (info->flags & ~allowed_flags) || (info->usage & ~allowed_usage) ||
+              info->extent.width < 16 || info->extent.height < 16) {
       return false;
+   }
 
    if (hk_g17p_compression_color_format(info->format)) {
+      if (g16s && (three_d || info->arrayLayers > 1) && info->mipLevels != 1)
+         return false;
+      /* #66: a legal MRT mix can spill any MSAA colour attachment. Keep
+       * those plain until transient backing has a measured performance case. */
+      if (g16s && multisampled &&
+          (info->usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
+                         VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT)))
+         return false;
+
       /* An RGBA8/BGRA8 UNORM mip chain the application never renders to is
        * written only by uploads and mip blits. A plain image takes each
        * upload as one raw compute copy and each mip level as a raw compute
@@ -214,16 +257,33 @@ hk_g17p_compression_eligible(enum agx_chip chip, unsigned plane,
       return true;
    }
 
+   if (g16s) {
+      return (hk_g17p_compression_depth_format(info->format) ||
+              (info->format == VK_FORMAT_S8_UINT && !multisampled)) &&
+             !three_d && info->extent.width <= 16384 &&
+             info->extent.height <= 16384 && info->mipLevels <= 16;
+   }
    return hk_g17p_compression_depth_format(info->format) &&
           info->mipLevels == 1;
 }
 
-/* G17P colour images shared through DRM_FORMAT_MOD_APPLE_GPU_TILED_COMPRESSED.
+/* Preserve the qualified Neo policy for every pre-existing caller. T6040's
+ * extended shapes are selected by physical identity at image allocation. */
+static inline bool
+hk_g17p_compression_eligible(enum agx_chip chip, unsigned plane,
+                             const VkImageCreateInfo *info)
+{
+   return hk_hal200_compression_eligible(chip, plane, info, false);
+}
+
+/* G17P and M5 colour images shared through
+ * DRM_FORMAT_MOD_APPLE_GPU_TILED_COMPRESSED.
  *
  * The modifier is the sharing contract. With one level, one layer and one
  * sample, the G17P (legacy AIL policy) compressed allocation is a function of
  * the format's block size and the extent alone: GPU tiling, the image, then
- * its metadata, in one memory plane. Exporter and importer therefore derive
+ * its metadata, in one memory plane. The M5 allocation (AIL_LAYOUT_POLICY_M5
+ * and its metadata, one memory plane) is too. Exporter and importer therefore derive
  * the same layout, and import already follows the modifier through
  * hk_map_compression(). The packets that read and write it are the ones the
  * private policy above uses for single-level colour images: the compressed
@@ -256,7 +316,8 @@ hk_g17p_shared_compression_shape(enum agx_chip chip, VkFormat format,
       VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
       VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
 
-   return chip == AGX_CHIP_G17P && type == VK_IMAGE_TYPE_2D &&
+   return (chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip)) &&
+          type == VK_IMAGE_TYPE_2D &&
           !(flags & ~allowed_flags) && !(usage & ~allowed_usage) &&
           hk_g17p_compression_color_format(format);
 }
@@ -274,3 +335,279 @@ hk_g17p_shared_compression_eligible(enum agx_chip chip, unsigned plane,
           info->arrayLayers == 1 && info->samples == VK_SAMPLE_COUNT_1_BIT &&
           info->extent.width >= 16 && info->extent.height >= 16;
 }
+
+/* M5 lossless compression of colour render targets.
+ *
+ * The Gen10 compressed PBE (write) packet and texture (read) descriptor are
+ * native-verified for GPU-tiled one-layer 2D images (byte-identical to
+ * the native M5 PBE and texture descriptor encoders), and the allocation is
+ * the measured M5 layout with its metadata (AIL_LAYOUT_POLICY_M5). Every
+ * write of such an image then goes through the PBE: end-of-tile stores,
+ * and the graphics meta paths hk_xfer_copyimage2() and the buffer uploads
+ * take for a compressed destination (the raw tiled copiers refuse one).
+ *
+ * Admitted: the G17P colour formats (1-8 byte texels, so a single-sample
+ * tilebuffer never spills a compressed target), one layer, any mip count
+ * (one level for one-byte texels, see hk_m5_compression_one_byte()),
+ * one sample, a colour attachment, and otherwise only sampled,
+ * input-attachment and transfer usages. Shared images (external memory,
+ * WSI) stay plain.
+ *
+ * Every level of a compressed mip chain is described by the level-0 surface
+ * and metadata addresses plus the level fields, in the PBE and
+ * texture packets alike, as native does; the hardware finds the
+ * level, so AIL_LAYOUT_POLICY_M5's per-level metadata placement must be the
+ * hardware's.
+ */
+/* One-byte texels. The M5 hardware places a compressed level after a 32x32
+ * one-byte level 2 KiB on, where AIL_LAYOUT_POLICY_M5 puts it 1 KiB on: the
+ * end-of-tile store and texture reads use the hardware's offset, AIL-addressed
+ * writes (fast clears, copies) the other. Two- to eight-byte texels agree with
+ * the hardware at every level tested (64x64 to 2976x1872, odd sizes). */
+static inline bool
+hk_m5_compression_one_byte(VkFormat format)
+{
+   return format == VK_FORMAT_R8_UNORM || format == VK_FORMAT_R8_SNORM ||
+          format == VK_FORMAT_R8_UINT || format == VK_FORMAT_R8_SINT;
+}
+
+static inline bool
+hk_m5_color_compression_eligible(enum agx_chip chip, unsigned plane,
+                                 const VkImageCreateInfo *info)
+{
+   if (!agx_chip_g17_m5(chip) || plane != 0 || !info)
+      return false;
+
+   /* Read the chain as hk_g17p_compression_eligible() does. */
+   for (const void *next = info->pNext; next;) {
+      VkStructureType type;
+      const void *following;
+      memcpy(&type, next, sizeof(type));
+      memcpy(&following, (const char *)next + offsetof(VkBaseInStructure, pNext),
+             sizeof(following));
+      if (type == VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO) {
+         VkExternalMemoryImageCreateInfo external;
+         memcpy(&external, next, sizeof(external));
+         if (external.handleTypes)
+            return false;
+      }
+      if (type == HK_G17P_WSI_IMAGE_CREATE_INFO_MESA)
+         return false;
+      next = following;
+   }
+
+   const VkImageCreateFlags allowed_flags =
+      VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
+   const VkImageUsageFlags allowed_usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
+
+   return info->imageType == VK_IMAGE_TYPE_2D && info->extent.depth == 1 &&
+          info->arrayLayers == 1 &&
+          (info->mipLevels == 1 || !hk_m5_compression_one_byte(info->format)) &&
+          info->samples == VK_SAMPLE_COUNT_1_BIT &&
+          info->tiling == VK_IMAGE_TILING_OPTIMAL &&
+          !(info->flags & ~allowed_flags) &&
+          (info->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
+          !(info->usage & ~allowed_usage) && info->extent.width >= 16 &&
+          info->extent.height >= 16 &&
+          hk_g17p_compression_color_format(info->format);
+}
+
+/* Multisample colour formats with a qualified compressed metadata shape:
+ * RGBA16F, and the 4-byte RGBA8/BGRA8 UNORM and sRGB formats, which share the
+ * R32_UINT texel size of the witnessed raw copy packet. */
+static inline bool
+hk_m5_msaa_color_compression_format(VkFormat format)
+{
+   switch (format) {
+   case VK_FORMAT_R16G16B16A16_SFLOAT:
+   case VK_FORMAT_R8G8B8A8_UNORM:
+   case VK_FORMAT_R8G8B8A8_SRGB:
+   case VK_FORMAT_B8G8R8A8_UNORM:
+   case VK_FORMAT_B8G8R8A8_SRGB:
+      return true;
+   default:
+      return false;
+   }
+}
+
+/* Qualified multisample metadata shapes. The sample mask retains an
+ * explicit diagnostic opt-out. Sharing and aliasing flags are not admitted;
+ * a mutable format is, and hk_can_compress() then requires a format list of
+ * compression-compatible views.
+ */
+static inline bool
+hk_m5_msaa_color_compression_eligible(enum agx_chip chip, unsigned plane,
+                                     const VkImageCreateInfo *info,
+                                     unsigned enabled_samples)
+{
+   const VkImageUsageFlags allowed_usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+
+   if (chip != AGX_CHIP_G17S || plane != 0 || !info)
+      return false;
+
+   for (const void *next = info->pNext; next;) {
+      VkStructureType type;
+      memcpy(&type, next, sizeof(type));
+      if (type != VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO)
+         return false;
+      memcpy(&next, (const char *)next + offsetof(VkBaseInStructure, pNext),
+             sizeof(next));
+   }
+
+   return !(info->flags & ~VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) &&
+          info->imageType == VK_IMAGE_TYPE_2D &&
+          info->extent.depth == 1 && info->extent.width >= 16 &&
+          info->extent.height >= 16 && info->arrayLayers == 1 &&
+          info->mipLevels == 1 &&
+          hk_m5_msaa_color_compression_format(info->format) &&
+          (info->samples == VK_SAMPLE_COUNT_2_BIT ||
+           info->samples == VK_SAMPLE_COUNT_4_BIT) &&
+          (enabled_samples & info->samples) &&
+          info->tiling == VK_IMAGE_TILING_OPTIMAL &&
+          (info->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
+          !(info->usage & ~allowed_usage);
+}
+
+/* M5 lossless compression of depth attachments.
+ *
+ * The native HAL300 framebuffer setup sets ZLS control bits 2 and
+ * 6 exactly when the attached depth level has compression metadata, bits 4
+ * and 8 for stencil metadata and bit 26 for Depth16Unorm, and programs the
+ * metadata address and its cache-line layer stride beside the surface: the
+ * encoding agx_pack_zls_control() and hk's render setup already produce.
+ * Depth is read through the same Gen10 texture descriptor as colour, and the
+ * copy paths write it through its R16/R32_UINT colour alias with the
+ * compressed PBE.
+ *
+ * Admitted: D16_UNORM, D32_SFLOAT and both planes of D32_SFLOAT_S8_UINT
+ * (stencil is read through its R8_UINT alias and copied through the same
+ * graphics path), one layer, one mip, one sample,
+ * a depth attachment, and otherwise only sampled, input-attachment,
+ * feedback-loop and transfer usages. Zink marks its depth attachments
+ * sample-location compatible and feedback-loop capable, which changes
+ * neither. Shared images (external memory, WSI) stay plain.
+ */
+static inline bool
+hk_m5_depth_compression_eligible(enum agx_chip chip, unsigned plane,
+                                 const VkImageCreateInfo *info)
+{
+   if (!agx_chip_g17_m5(chip) || !info ||
+       (plane != 0 && !(plane == 1 &&
+                        info->format == VK_FORMAT_D32_SFLOAT_S8_UINT)))
+      return false;
+
+   /* Read the chain as hk_g17p_compression_eligible() does. */
+   for (const void *next = info->pNext; next;) {
+      VkStructureType type;
+      const void *following;
+      memcpy(&type, next, sizeof(type));
+      memcpy(&following, (const char *)next + offsetof(VkBaseInStructure, pNext),
+             sizeof(following));
+      if (type == VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO) {
+         VkExternalMemoryImageCreateInfo external;
+         memcpy(&external, next, sizeof(external));
+         if (external.handleTypes)
+            return false;
+      }
+      if (type == HK_G17P_WSI_IMAGE_CREATE_INFO_MESA)
+         return false;
+      next = following;
+   }
+
+   const VkImageCreateFlags allowed_flags =
+      VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT |
+      VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT;
+   const VkImageUsageFlags allowed_usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+
+   return info->imageType == VK_IMAGE_TYPE_2D && info->extent.depth == 1 &&
+          info->arrayLayers == 1 && info->mipLevels == 1 &&
+          info->samples == VK_SAMPLE_COUNT_1_BIT &&
+          info->tiling == VK_IMAGE_TILING_OPTIMAL &&
+          !(info->flags & ~allowed_flags) &&
+          (info->usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT) &&
+          !(info->usage & ~allowed_usage) && info->extent.width >= 16 &&
+          info->extent.height >= 16 &&
+          hk_g17p_compression_depth_format(info->format);
+}
+
+/* Stage-one experiment: native ZLS plus texture reads, no PBE destination.
+ * The capability is advertised only by an explicitly enabled private kernel.
+ * Transfer/image-copy and additional image semantics require separate gates.
+ */
+static inline bool
+hk_m5_depth_msaa_compression_eligible(enum agx_chip chip, unsigned plane,
+                                     const VkImageCreateInfo *info,
+                                     uint64_t kernel_features, bool opted_in)
+{
+   const uint64_t capability = DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION;
+   const VkImageUsageFlags allowed_usage =
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
+   /* The stencil plane of D32_SFLOAT_S8_UINT compresses with depth where the
+    * kernel admits compressed multisample stencil. */
+   const bool plane_ok =
+      plane == 0 ||
+      (plane == 1 && info && info->format == VK_FORMAT_D32_SFLOAT_S8_UINT &&
+       (kernel_features & DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION));
+   return opted_in && (kernel_features & capability) &&
+          agx_chip_g17_m5(chip) && plane_ok && info && !info->pNext &&
+          info->imageType == VK_IMAGE_TYPE_2D && info->extent.depth == 1 &&
+          info->arrayLayers == 1 && info->mipLevels == 1 &&
+          (info->samples == VK_SAMPLE_COUNT_2_BIT ||
+           info->samples == VK_SAMPLE_COUNT_4_BIT) &&
+          info->tiling == VK_IMAGE_TILING_OPTIMAL && info->flags == 0 &&
+          (info->usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT) &&
+          !(info->usage & ~allowed_usage) && info->extent.width >= 16 &&
+          info->extent.height >= 16 &&
+          (info->format == VK_FORMAT_D32_SFLOAT ||
+           info->format == VK_FORMAT_D32_SFLOAT_S8_UINT);
+}
+
+/* Private transfer qualification: preserve the stage-one shape and capability
+ * checks while admitting only the complete DS/sampled/transfer usage set.
+ * Stencil stays plain; its sole permitted extension excludes sampled usage.
+ */
+static inline bool
+hk_m5_depth_msaa_transfer_eligible(enum agx_chip chip, unsigned plane,
+                                  const VkImageCreateInfo *info,
+                                  uint64_t kernel_features,
+                                  bool compression_opted_in,
+                                  bool transfer_opted_in)
+{
+   const VkImageUsageFlags usage =
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+   if (!compression_opted_in || !transfer_opted_in || !info ||
+       info->usage != usage)
+      return false;
+
+   if (info->pNext) {
+      VkBaseInStructure extension;
+      memcpy(&extension, info->pNext, sizeof(extension));
+      if (info->format != VK_FORMAT_D32_SFLOAT_S8_UINT ||
+          extension.sType != VK_STRUCTURE_TYPE_IMAGE_STENCIL_USAGE_CREATE_INFO ||
+          extension.pNext)
+         return false;
+      VkImageStencilUsageCreateInfo stencil;
+      memcpy(&stencil, info->pNext, sizeof(stencil));
+      if (stencil.stencilUsage != (usage & ~VK_IMAGE_USAGE_SAMPLED_BIT))
+         return false;
+   }
+
+   VkImageCreateInfo base = *info;
+   base.pNext = NULL;
+   base.usage &= ~(VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+                   VK_IMAGE_USAGE_TRANSFER_DST_BIT);
+   return hk_m5_depth_msaa_compression_eligible(
+      chip, plane, &base, kernel_features, compression_opted_in);
+}
diff --git a/src/asahi/vulkan/hk_g17p_meta_copy_core.c b/src/asahi/vulkan/hk_g17p_meta_copy_core.c
index 2c99c3f1fefa..31d139dc6225 100644
--- a/src/asahi/vulkan/hk_g17p_meta_copy_core.c
+++ b/src/asahi/vulkan/hk_g17p_meta_copy_core.c
@@ -234,7 +234,8 @@ hk_g17p_compressed_copy_plan(
    if (plan)
       memset(plan, 0, sizeof(*plan));
 
-   if (!copy || !plan || !agx_chip_native_agx3(copy->chip) ||
+   if (!copy || !plan ||
+       (!agx_chip_native_agx3(copy->chip) && !agx_chip_g17_m5(copy->chip)) ||
        hk_g17p_block_format_family(copy->format) ==
           HK_G17P_BLOCK_FORMAT_NONE ||
        !(image_is_source
@@ -508,7 +509,8 @@ hk_g17p_buffer_to_image_copy_supported(
    const bool one_dimensional =
       copy && copy->image_type == VK_IMAGE_TYPE_1D;
    const bool volume = copy && copy->image_type == VK_IMAGE_TYPE_3D;
-   if (!copy || !agx_chip_native_agx3(copy->chip) ||
+   if (!copy || (!agx_chip_native_agx3(copy->chip) &&
+                 !agx_chip_g17_m5(copy->chip)) ||
        !hk_g17p_transfer_dst_layout_supported(copy->layout) ||
        (!one_dimensional && copy->image_type != VK_IMAGE_TYPE_2D && !volume) ||
        copy->samples != VK_SAMPLE_COUNT_1_BIT ||
@@ -672,7 +674,8 @@ hk_g17p_image_copy_slice(enum agx_chip chip, VkImageType src_type,
    const bool two_d_three_d = (src_1d_or_2d && dst_3d) ||
                               (dst_1d_or_2d && src_3d);
    const bool valid_pair = same_type || one_d_two_d || two_d_three_d;
-   if (!agx_chip_native_agx3(chip) || !region || !out || !valid_types ||
+   if ((!agx_chip_native_agx3(chip) && !agx_chip_g17_m5(chip)) ||
+       !region || !out || !valid_types ||
        !valid_pair || src_layers == 0 ||
        src_layers != dst_layers || slice >= src_layers ||
        ((src_3d || dst_3d) && region->extent.depth != src_layers) ||
diff --git a/src/asahi/vulkan/hk_g17p_zls.h b/src/asahi/vulkan/hk_g17p_zls.h
new file mode 100644
index 000000000000..466711dbd86e
--- /dev/null
+++ b/src/asahi/vulkan/hk_g17p_zls.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include "asahi/layout/layout.h"
+
+struct hk_g17p_zls_plane {
+   uint64_t buffer, meta;
+   uint32_t stride, meta_stride;
+   uint32_t width, height;
+   enum ail_tiling tiling;
+};
+
+/* Build one aspect's attachment state. The caller supplies that aspect's
+ * allocation, not the other plane of a combined depth/stencil image. */
+static inline struct hk_g17p_zls_plane
+hk_g17p_zls_plane(const struct ail_layout *layout, uint64_t base,
+                  unsigned layer, unsigned level)
+{
+   assert(level < layout->levels && layer < layout->depth_px);
+   assert(layout->tiling == AIL_TILING_GPU ||
+          layout->tiling == AIL_TILING_TWIDDLED);
+
+   struct hk_g17p_zls_plane state = {
+      .buffer = base + ail_get_layer_level_B(layout, layer, level),
+      .width = u_minify(layout->width_px, level),
+      .height = u_minify(layout->height_px, level),
+      .tiling = layout->tiling,
+   };
+   assert(state.width <= 16384 && state.height <= 16384);
+   assert(util_is_aligned(state.buffer, AIL_CACHELINE));
+
+   assert(layout->depth_px == 1 ||
+          util_is_aligned(layout->layer_stride_B, AIL_PAGESIZE));
+   unsigned pages = layout->layer_stride_B / AIL_PAGESIZE;
+   /* A single layer does not use the stride. Keep sub-page strides zero,
+    * as required by the native kernel's ZLS validation. */
+   if (pages)
+      state.stride = ((pages - 1) << 14) | 1;
+
+   if (layout->compressed) {
+      /* The narrow tail can have reserved metadata without being compressed.
+       * It uses twiddled data and must not address those metadata slots. */
+      state.tiling = ail_is_level_twiddled_uncompressed(layout, level)
+                        ? AIL_TILING_TWIDDLED
+                        : layout->tiling;
+      if (base && ail_is_level_logically_compressed(layout, level) &&
+          layout->compression_layer_stride_B >= AIL_CACHELINE) {
+         assert(
+            util_is_aligned(layout->compression_layer_stride_B, AIL_CACHELINE));
+         state.meta = base + layout->metadata_offset_B +
+                      (uint64_t)layer * layout->compression_layer_stride_B +
+                      layout->level_offsets_compressed_B[level];
+         assert(util_is_aligned(state.meta, AIL_CACHELINE));
+         state.meta_stride =
+            (layout->compression_layer_stride_B / AIL_CACHELINE - 1) << 14;
+      }
+   }
+
+   return state;
+}
diff --git a/src/asahi/vulkan/hk_image.c b/src/asahi/vulkan/hk_image.c
index b3583d605a7d..dedc3bba71a2 100644
--- a/src/asahi/vulkan/hk_image.c
+++ b/src/asahi/vulkan/hk_image.c
@@ -5,6 +5,8 @@
  * SPDX-License-Identifier: MIT
  */
 #include "hk_image.h"
+#include "hk_frame_anatomy.h"
+#include "hk_instance.h"
 #include "hk_g17p_compression.h"
 #include "agx_bg_eot_key.h"
 #include "hk_native_descriptor.h"
@@ -36,6 +38,12 @@
  */
 #define HK_SPARSE_ALIGN_B 65536
 
+DEBUG_GET_ONCE_BOOL_OPTION(m5_depth_msaa_transfer,
+                          "ASAHI_M5_DEPTH_MSAA_TRANSFER", true)
+
+DEBUG_GET_ONCE_BOOL_OPTION(m5_depth_msaa_compression,
+                          "ASAHI_M5_DEPTH_MSAA_COMPRESSION", true)
+
 static VkFormatFeatureFlags2
 hk_get_image_plane_format_features(struct hk_physical_device *pdev,
                                    VkFormat vk_format, VkImageTiling tiling)
@@ -86,9 +94,10 @@ hk_get_image_plane_format_features(struct hk_physical_device *pdev,
       return 0;
 
    enum pipe_format p_format = hk_format_to_pipe_format(vk_format);
-   /* D16S8 uses the existing independent D16 and S8 planes. */
+   /* D16S8 uses the existing independent D16 and S8 planes, as D32S8 does
+    * on G17P and M5. */
    if (vk_format == VK_FORMAT_D16_UNORM_S8_UINT) {
-      if (pdev->dev.chip != AGX_CHIP_G17P)
+      if (pdev->dev.chip != AGX_CHIP_G17P && !agx_chip_g17_m5(pdev->dev.chip))
          return 0;
       p_format = PIPE_FORMAT_Z16_UNORM;
    }
@@ -96,7 +105,8 @@ hk_get_image_plane_format_features(struct hk_physical_device *pdev,
       return 0;
 
    enum pipe_format padded = hk_padded_image_format(p_format);
-   if (agx_native_graphics(&pdev->dev) && padded != p_format) {
+   if ((agx_native_graphics(&pdev->dev) || agx_chip_g17_m5(pdev->dev.chip)) &&
+       padded != p_format) {
       /* Linear and external modifier layouts must retain the API's packed
        * texel stride. This path is restricted to driver-owned optimal images. */
       if (tiling != VK_IMAGE_TILING_OPTIMAL)
@@ -138,6 +148,10 @@ hk_get_image_plane_format_features(struct hk_physical_device *pdev,
          if (!util_format_is_pure_integer(padded))
             rgb |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT;
       }
+      if ((rgb & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT) &&
+          agx_chip_sampler_generation(pdev->dev.chip) ==
+             AGX_SAMPLER_GENERATION_GEN5)
+         rgb |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT;
       return rgb;
    }
 
@@ -223,12 +237,17 @@ hk_get_image_plane_format_features(struct hk_physical_device *pdev,
    /* Our image atomic lowering doesn't bother to handle linear */
    if ((p_format == PIPE_FORMAT_R32_UINT || p_format == PIPE_FORMAT_R32_SINT ||
         (p_format == PIPE_FORMAT_R32_FLOAT &&
-         pdev->dev.chip == AGX_CHIP_G17P)) &&
+         (pdev->dev.chip == AGX_CHIP_G17P ||
+          agx_chip_g17_m5(pdev->dev.chip)))) &&
        tiling == VK_IMAGE_TILING_OPTIMAL) {
 
       features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_ATOMIC_BIT;
    }
 
+   if ((features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT) &&
+       agx_chip_sampler_generation(pdev->dev.chip) == AGX_SAMPLER_GENERATION_GEN5)
+      features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT;
+
    if (features != 0) {
       features |= VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT;
       features |= VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
@@ -374,11 +393,15 @@ vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)
    }
 }
 
+/* Enable the validated multisample color layouts on G17S. */
+DEBUG_GET_ONCE_NUM_OPTION(m5_msaa_color_compression,
+                         "ASAHI_M5_MSAA_COLOR_COMPRESSION", 6)
+
 static bool
 hk_can_compress(const struct agx_device *dev, VkFormat format, unsigned plane,
                 unsigned width, unsigned height, unsigned samples,
                 VkImageCreateFlagBits flags, VkImageUsageFlagBits usage,
-                const void *pNext, bool allow_agx3)
+                const void *pNext, bool allow_agx3, bool srgb_pair_views)
 {
    const struct vk_format_ycbcr_info *ycbcr_info =
       vk_format_get_ycbcr_info(format);
@@ -406,7 +429,7 @@ hk_can_compress(const struct agx_device *dev, VkFormat format, unsigned plane,
 
    /* AGX3 compresses only what its policies in hk_g17p_compression.h
     * admitted (allow_agx3). The private policy excludes images shared through
-    * external memory; G17P shares colour images compressed only through the
+    * external memory; G17P and M5 share colour images compressed only through the
     * GPU_TILED_COMPRESSED modifier (hk_g17p_shared_compression_eligible()),
     * which is all hk_can_compress_format() advertises on AGX3. */
    if (hk_descriptor_generation_is_agx3(
@@ -434,15 +457,16 @@ hk_can_compress(const struct agx_device *dev, VkFormat format, unsigned plane,
     * Feedback loops are problematic with compression. The GL driver bans them.
     * Interestingly, the relevant CTS tests pass on G13G and G14C, but not on
     * G13D. Keep that restriction except for depth images already admitted by
-    * the G17P policy. Rejecting those here contradicts the depth policy and
+    * the G17P or M5 policy. Rejecting those here contradicts the depth policy and
     * makes the host-copy performance query report a plain device layout;
     * Zink then retains HOST_TRANSFER and never compresses its depth images.
     */
    /* Decided on the API format: the stencil plane of D32_SFLOAT_S8_UINT is
     * S8_UINT here, and zink gives its depth/stencil images feedback-loop
     * usage. */
-   const bool depth_feedback = dev->chip == AGX_CHIP_G17P && allow_agx3 &&
-                               hk_g17p_compression_depth_format(api_format);
+   const bool depth_feedback =
+      (dev->chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->chip)) &&
+      allow_agx3 && hk_g17p_compression_depth_format(api_format);
    if ((usage & (VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT |
                  VK_IMAGE_USAGE_STORAGE_BIT)) ||
        ((usage & VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT) &&
@@ -465,10 +489,19 @@ hk_can_compress(const struct agx_device *dev, VkFormat format, unsigned plane,
       const struct VkImageFormatListCreateInfo *format_list =
          (void *)vk_find_struct_const(pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
 
-      if (!format_list || format_list->viewFormatCount == 0)
-         return false;
+      if (!format_list || format_list->viewFormatCount == 0) {
+         /* Without a list any compatible format may view the image, and
+          * compressed data only decodes through its own channel layout. The
+          * hk_mutable_format_srgb_pair application option asserts that only
+          * the format's sRGB/UNORM twin (same channels) will. */
+         const bool has_twin = util_format_srgb(p_format) != PIPE_FORMAT_NONE ||
+                               util_format_is_srgb(p_format);
+         if (!srgb_pair_views || !has_twin)
+            return false;
+      }
 
-      for (unsigned i = 0; i < format_list->viewFormatCount; ++i) {
+      for (unsigned i = 0; format_list && i < format_list->viewFormatCount;
+           ++i) {
          if (format_list->pViewFormats[i] == VK_FORMAT_UNDEFINED)
             continue;
 
@@ -495,9 +528,112 @@ hk_can_compress(const struct agx_device *dev, VkFormat format, unsigned plane,
    return true;
 }
 
+/* M5 lossless storage compression of formats without image atomics: the
+ * half-float formats, RG32F, RGBA32F and the 8- and 16-bit-channel integer
+ * and normalized formats have none, and R32F has them only with the float32
+ * image atomic features, so it is compressed only when the device did not
+ * enable them (and its views cannot reinterpret it). The single-layer/level
+ * native read/write packets use the allocator's recovered Gen10 metadata
+ * layout, the same layout colour attachments use, so a storage image may also
+ * be a colour or input attachment and a transfer destination: end-of-tile
+ * stores, shader image stores, clears and uploads all write it through
+ * compressed PBE descriptors, and an input attachment reads it as a sampled
+ * colour attachment does. Bits 0-10 select RGBA16F, RG16F, R16F, R32F, RG32F, RGBA32F,
+ * RGBA8 UNORM, the other RGBA16 formats, the other RG16 formats, the other
+ * RGBA8 formats and the other R16 formats, respectively; bit 12 admits
+ * mipmapped images, whose levels compress as those of colour attachments
+ * do. Zero retains the uncompressed diagnostic control. */
+DEBUG_GET_ONCE_NUM_OPTION(m5_storage_compression,
+                         "ASAHI_M5_STORAGE_COMPRESSION", 6143)
+
+static bool
+hk_m5_storage_compression_eligible(const struct agx_device *dev, unsigned plane,
+                                 const VkImageCreateInfo *info,
+                                 bool float_image_atomics)
+{
+   /* Typeless resources of D3D translation layers are mutable and list
+    * their view formats: the formats of one channel layout (UNORM, SNORM,
+    * UINT, SINT, FLOAT). None of these formats has image atomics, so such an
+    * image compresses when hk_can_compress() finds only formats of the
+    * image's channel layout in the list. */
+   const VkImageCreateFlags allowed_flags =
+      VK_IMAGE_CREATE_EXTENDED_USAGE_BIT | VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
+   const VkBaseInStructure *ext = info->pNext;
+   const bool format_list_only =
+      (info->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) && ext &&
+      ext->sType == VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO &&
+      !ext->pNext;
+
+   if (dev->chip != AGX_CHIP_G17S || plane ||
+       (info->pNext && !format_list_only) ||
+       (info->flags & ~allowed_flags) ||
+       info->imageType != VK_IMAGE_TYPE_2D || info->extent.depth != 1 ||
+       info->extent.width < 16 || info->extent.height < 16 ||
+       (info->mipLevels != 1 &&
+        !(debug_get_option_m5_storage_compression() & 4096)) ||
+       info->arrayLayers != 1 ||
+       info->samples != VK_SAMPLE_COUNT_1_BIT ||
+       info->tiling != VK_IMAGE_TILING_OPTIMAL ||
+       !(info->usage & VK_IMAGE_USAGE_STORAGE_BIT) ||
+       (info->usage & ~(VK_IMAGE_USAGE_STORAGE_BIT |
+                        VK_IMAGE_USAGE_SAMPLED_BIT |
+                        VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+                        VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+                        VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
+                        VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)))
+      return false;
+
+   unsigned bit;
+   switch (info->format) {
+   case VK_FORMAT_R16G16B16A16_SFLOAT: bit = 1; break;
+   case VK_FORMAT_R16G16_SFLOAT: bit = 2; break;
+   case VK_FORMAT_R16_SFLOAT: bit = 4; break;
+   case VK_FORMAT_R32_SFLOAT:
+      if (float_image_atomics)
+         return false;
+      bit = 8;
+      break;
+   case VK_FORMAT_R32G32_SFLOAT: bit = 16; break;
+   case VK_FORMAT_R32G32B32A32_SFLOAT: bit = 32; break;
+   case VK_FORMAT_R8G8B8A8_UNORM: bit = 64; break;
+   case VK_FORMAT_R16G16B16A16_UNORM:
+   case VK_FORMAT_R16G16B16A16_SNORM:
+   case VK_FORMAT_R16G16B16A16_UINT:
+   case VK_FORMAT_R16G16B16A16_SINT: bit = 128; break;
+   case VK_FORMAT_R16G16_UNORM:
+   case VK_FORMAT_R16G16_SNORM:
+   case VK_FORMAT_R16G16_UINT:
+   case VK_FORMAT_R16G16_SINT: bit = 256; break;
+   case VK_FORMAT_R8G8B8A8_SNORM:
+   case VK_FORMAT_R8G8B8A8_UINT:
+   case VK_FORMAT_R8G8B8A8_SINT: bit = 512; break;
+   case VK_FORMAT_R16_UNORM:
+   case VK_FORMAT_R16_SNORM:
+   case VK_FORMAT_R16_UINT:
+   case VK_FORMAT_R16_SINT: bit = 1024; break;
+   default: return false;
+   }
+   /* A mutable image's views may be FLOAT when the image is not: the list
+    * check in hk_can_compress() keeps them to the image's channel layout. */
+   if ((info->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) && !format_list_only)
+      return false;
+   return debug_get_option_m5_storage_compression() & bit;
+}
+
 static bool
 hk_can_compress_create_info(const struct agx_device *dev, unsigned plane,
-                            const VkImageCreateInfo *info);
+                            const VkImageCreateInfo *info,
+                            bool float_image_atomics, bool srgb_pair_views);
+
+/* Whether shaders of this device may use atomics on R32_SFLOAT images. */
+static bool
+hk_float_image_atomics(const struct hk_device *dev)
+{
+   const struct vk_features *f = &dev->vk.enabled_features;
+   return f->shaderImageFloat32Atomics || f->shaderImageFloat32AtomicAdd ||
+          f->shaderImageFloat32AtomicMinMax || f->sparseImageFloat32Atomics ||
+          f->sparseImageFloat32AtomicAdd || f->sparseImageFloat32AtomicMinMax;
+}
 
 /* Compressed data is only decodable through its own channel class. Every
  * G17P writer of a colour image is a PBE; the end-of-tile PBE may store a
@@ -561,12 +697,13 @@ hk_can_compress_format(const struct agx_device *dev, VkFormat format)
     *
     * Likewise, we do not set flags as flags only disable compression.
     *
-    * On AGX3 only G17P advertises the modifier, for the colour formats it
-    * shares compressed (hk_g17p_shared_compression_supported()).
+    * On AGX3 only G17P and M5 advertise the modifier, for the colour formats
+    * they share compressed (hk_g17p_shared_compression_supported()).
     */
    const bool g17p_shared = hk_g17p_shared_compression_supported(
       dev, format, VK_IMAGE_TYPE_2D, 0, 0, NULL);
-   return hk_can_compress(dev, format, 0, 64, 64, 1, 0, 0, NULL, g17p_shared);
+   return hk_can_compress(dev, format, 0, 64, 64, 1, 0, 0, NULL, g17p_shared,
+                          false);
 }
 
 bool
@@ -801,7 +938,7 @@ hk_GetPhysicalDeviceImageFormatProperties2(
       return VK_ERROR_FORMAT_NOT_SUPPORTED;
 
    const bool g17p_compressed =
-      pdev->dev.chip == AGX_CHIP_G17P &&
+      (pdev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(pdev->dev.chip)) &&
       pImageFormatInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
       drm_mod_info &&
       drm_mod_info->drmFormatModifier ==
@@ -854,7 +991,8 @@ hk_GetPhysicalDeviceImageFormatProperties2(
        !(pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)) {
 
       sampleCounts =
-         VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_2_BIT | VK_SAMPLE_COUNT_4_BIT;
+         VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_2_BIT | VK_SAMPLE_COUNT_4_BIT |
+         (agx_chip_g17_m5(pdev->dev.chip) ? VK_SAMPLE_COUNT_8_BIT : 0);
 
       VkSampleCountFlags attachmentSampleCounts = UINT32_MAX;
       if (features & VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT)
@@ -1044,7 +1182,10 @@ hk_GetPhysicalDeviceImageFormatProperties2(
             .format = pImageFormatInfo->format,
             .extent = {64, 64, 1},
             .mipLevels = 1,
-            .arrayLayers = 1,
+            .arrayLayers = agx_is_m4_pro(&pdev->dev) &&
+                                 (pImageFormatInfo->flags &
+                                  VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)
+                              ? 6 : 1,
             .samples = VK_SAMPLE_COUNT_1_BIT,
             .tiling = pImageFormatInfo->tiling,
             .usage = (pImageFormatInfo->usage &
@@ -1056,7 +1197,7 @@ hk_GetPhysicalDeviceImageFormatProperties2(
          if (pImageFormatInfo->tiling == VK_IMAGE_TILING_OPTIMAL) {
             for (unsigned plane = 0; plane < plane_count; ++plane) {
                can_compress |= hk_can_compress_create_info(
-                  &pdev->dev, plane, &device_copy_info);
+                  &pdev->dev, plane, &device_copy_info, false, false);
             }
          }
 
@@ -1157,13 +1298,29 @@ hk_GetPhysicalDeviceSparseImageFormatProperties2(
 
 static bool
 hk_can_compress_create_info(const struct agx_device *dev, unsigned plane,
-                            const VkImageCreateInfo *info)
+                            const VkImageCreateInfo *info,
+                            bool float_image_atomics, bool srgb_pair_views)
 {
    /* AGX3: G17P compresses every image hk_g17p_compression.h admits. Other
     * AGX3 chips stay plain until their packets are recovered. */
+   const bool m5_storage =
+      hk_m5_storage_compression_eligible(dev, plane, info, float_image_atomics);
    bool allow_agx3 =
-      hk_g17p_compression_eligible(dev->chip, plane, info) ||
-      hk_g17p_shared_compression_eligible(dev->chip, plane, info);
+      hk_m5_msaa_color_compression_eligible(
+         dev->chip, plane, info,
+         debug_get_option_m5_msaa_color_compression()) ||
+      hk_m5_depth_msaa_compression_eligible(
+         dev->chip, plane, info, dev->params.features,
+         debug_get_option_m5_depth_msaa_compression()) ||
+      hk_m5_depth_msaa_transfer_eligible(
+         dev->chip, plane, info, dev->params.features,
+         debug_get_option_m5_depth_msaa_compression(),
+         debug_get_option_m5_depth_msaa_transfer()) ||
+      m5_storage || hk_hal200_compression_eligible(dev->chip, plane, info,
+                                                  agx_is_m4_pro(dev)) ||
+      hk_g17p_shared_compression_eligible(dev->chip, plane, info) ||
+      hk_m5_color_compression_eligible(dev->chip, plane, info) ||
+      hk_m5_depth_compression_eligible(dev->chip, plane, info);
 
    if (allow_agx3 && !vk_format_is_depth_or_stencil(info->format) &&
        !hk_g17p_store_class_matches(dev, info->format))
@@ -1171,7 +1328,9 @@ hk_can_compress_create_info(const struct agx_device *dev, unsigned plane,
 
    return hk_can_compress(dev, info->format, plane, info->extent.width,
                           info->extent.height, info->samples, info->flags,
-                          info->usage, info->pNext, allow_agx3);
+                          m5_storage ? info->usage & ~VK_IMAGE_USAGE_STORAGE_BIT
+                                     : info->usage,
+                          info->pNext, allow_agx3, srgb_pair_views);
 }
 
 static enum ail_tiling
@@ -1202,7 +1361,10 @@ hk_map_compression(struct hk_device *dev, const VkImageCreateInfo *info,
       return false;
 
    case VK_IMAGE_TILING_OPTIMAL:
-      return hk_can_compress_create_info(&dev->dev, plane, info);
+      return hk_can_compress_create_info(
+         &dev->dev, plane, info, hk_float_image_atomics(dev),
+         hk_physical_device_instance(hk_device_physical(dev))
+            ->drirc.misc.mutable_format_srgb_pair);
 
    case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
       return ail_is_drm_modifier_compressed(modifier);
@@ -1252,7 +1414,8 @@ choose_drm_format_mod(struct hk_device *dev, uint8_t plane_count,
    bool can_compress = true;
 
    for (uint8_t plane = 0; plane < plane_count; plane++) {
-      if (!hk_can_compress_create_info(&dev->dev, plane, info))
+      if (!hk_can_compress_create_info(&dev->dev, plane, info,
+                                       hk_float_image_atomics(dev), false))
          can_compress = false;
    }
 
@@ -1372,16 +1535,16 @@ hk_image_init(struct hk_device *dev, struct hk_image *image,
                           mod_explicit_info->pPlaneLayouts[0].rowPitch, pitch_B);
    }
 
-   /* G17P shares compressed colour images only in the shape both sides
+   /* G17P and M5 share compressed colour images only in the shape both sides
     * allocate identically (hk_g17p_shared_compression_eligible()). */
    if (image->vk.drm_format_mod == DRM_FORMAT_MOD_APPLE_GPU_TILED_COMPRESSED &&
-       dev->dev.chip == AGX_CHIP_G17P &&
+       (dev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->dev.chip)) &&
        !(hk_g17p_shared_compression_supported(
             &dev->dev, pCreateInfo->format, pCreateInfo->imageType,
             pCreateInfo->flags, pCreateInfo->usage, pCreateInfo->pNext) &&
          hk_g17p_shared_compression_eligible(dev->dev.chip, 0, pCreateInfo)))
       return vk_errorf(dev, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT,
-                       "compressed modifier on an ineligible G17P image");
+                       "compressed modifier on an ineligible image");
 
    const struct vk_format_ycbcr_info *ycbcr_info =
       vk_format_get_ycbcr_info(pCreateInfo->format);
@@ -1400,7 +1563,7 @@ hk_image_init(struct hk_device *dev, struct hk_image *image,
 
       const enum pipe_format p_format = hk_format_to_pipe_format(format);
       const enum pipe_format layout_format =
-         agx_native_graphics(&dev->dev) &&
+         (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
                pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL
             ? hk_padded_image_format(p_format)
             : p_format;
@@ -1428,6 +1591,8 @@ hk_image_init(struct hk_device *dev, struct hk_image *image,
          .tiling = hk_map_tiling(dev, pCreateInfo, image->vk.drm_format_mod),
          .compressed = compressed,
          .mipmapped_z = pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
+         .sparse_residency =
+            pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT,
          .format = layout_format,
 
          .linear_stride_B = linear_stride_B,
@@ -1477,7 +1642,11 @@ hk_image_init(struct hk_device *dev, struct hk_image *image,
       } else if (image->disjoint) {
          image->planes[plane].offset_B = 0;
       } else if (!hk_image_implicit_plane_range(
-                    implicit_plane_cursor_B, image->planes[plane].layout.size_B,
+                    /* Sparse binds map each plane in whole pages. */
+                    (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
+                       ? align64(implicit_plane_cursor_B, HK_SPARSE_ALIGN_B)
+                       : implicit_plane_cursor_B,
+                    image->planes[plane].layout.size_B,
                     &image->planes[plane].offset_B, &plane_end_B)) {
          return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
                           "image plane layout exceeds the address space");
@@ -1515,13 +1684,23 @@ hk_image_plane_alloc_vma(struct hk_device *dev, struct hk_image_plane *plane,
        * In the future we could optimize this out using the PBE sparse support
        * but that needs more reverse-engineering.
        */
-      VkResult result = hk_bind_scratch(dev, plane->va, 0, size);
+      VkResult result = hk_reserve_sparse_scratch(dev, size);
       if (result != VK_SUCCESS) {
          agx_va_free(&dev->dev, plane->va, true);
          plane->va = NULL;
          plane->addr = 0;
          return result;
       }
+      plane->sparse_scratch_B = agx_chip_g17_m5(dev->dev.chip) ? size : 0;
+      result = hk_bind_scratch(dev, plane->va, 0, size);
+      if (result != VK_SUCCESS) {
+         hk_release_sparse_scratch(dev, plane->sparse_scratch_B);
+         plane->sparse_scratch_B = 0;
+         agx_va_free(&dev->dev, plane->va, true);
+         plane->va = NULL;
+         plane->addr = 0;
+         return result;
+      }
    }
 
    if (sparse_resident) {
@@ -1556,6 +1735,8 @@ hk_image_plane_finish(struct hk_device *dev, struct hk_image_plane *plane,
    if (plane->va) {
       agx_va_free(&dev->dev, plane->va, true);
    }
+   hk_release_sparse_scratch(dev, plane->sparse_scratch_B);
+   plane->sparse_scratch_B = 0;
 
    agx_bo_unreference(&dev->dev, plane->sparse_map);
 }
@@ -1597,6 +1778,34 @@ hk_CreateImage(VkDevice _device, const VkImageCreateInfo *pCreateInfo,
       return result;
    }
 
+   /* Frame anatomy profiler: the compression decision for every image large
+    * enough to matter for bandwidth. */
+   if (unlikely(dev->frame_anatomy) && pCreateInfo->extent.width >= 128 &&
+       pCreateInfo->extent.height >= 128) {
+      /* The view formats a mutable image lists, which the compression
+       * decision depends on. */
+      char views[256] = "";
+      const VkImageFormatListCreateInfo *list =
+         vk_find_struct_const(pCreateInfo->pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
+      for (unsigned i = 0; list && i < list->viewFormatCount; ++i) {
+         const size_t len = strlen(views);
+         snprintf(views + len, sizeof(views) - len, "%s%s", i ? "," : " views=",
+                  util_format_short_name(
+                     hk_format_to_pipe_format(list->pViewFormats[i])));
+      }
+      hk_frame_anatomy_note(
+         dev,
+         "image %ux%ux%u fmt=%s samples=%u mips=%u layers=%u usage=0x%x "
+         "flags=0x%x tiling=%u pnext=%u compressed=%u%s",
+         pCreateInfo->extent.width, pCreateInfo->extent.height,
+         pCreateInfo->extent.depth,
+         util_format_short_name(hk_format_to_pipe_format(pCreateInfo->format)),
+         pCreateInfo->samples, pCreateInfo->mipLevels,
+         pCreateInfo->arrayLayers, pCreateInfo->usage, pCreateInfo->flags,
+         pCreateInfo->tiling, pCreateInfo->pNext != NULL,
+         image->planes[0].layout.compressed, views);
+   }
+
    for (uint8_t plane = 0; plane < image->plane_count; plane++) {
       result = hk_image_plane_alloc_vma(dev, &image->planes[plane],
                                         image->vk.create_flags);
diff --git a/src/asahi/vulkan/hk_image.h b/src/asahi/vulkan/hk_image.h
index 4649cccb7539..1be3d0917719 100644
--- a/src/asahi/vulkan/hk_image.h
+++ b/src/asahi/vulkan/hk_image.h
@@ -57,6 +57,9 @@ struct hk_image_plane {
    uint64_t addr;
    struct agx_va *va;
 
+   /* Bytes of that VA counted in hk_device::sparse_scratch_B. */
+   uint64_t sparse_scratch_B;
+
    /* For host image copy */
    void *map;
    uint32_t rem;
diff --git a/src/asahi/vulkan/hk_image_view.c b/src/asahi/vulkan/hk_image_view.c
index bbf328df92d9..d7e3fc4d3747 100644
--- a/src/asahi/vulkan/hk_image_view.c
+++ b/src/asahi/vulkan/hk_image_view.c
@@ -16,6 +16,7 @@
 #include "agx_pack.h"
 #include "hk_device.h"
 #include "hk_entrypoints.h"
+#include "hk_frame_anatomy.h"
 #include "hk_image.h"
 #include "hk_physical_device.h"
 
@@ -240,7 +241,8 @@ pack_texture(struct hk_device *dev, struct hk_image_view *view,
    if (packed_ycbcr)
       p_format = packed_ycbcr == 8 ? PIPE_FORMAT_R8G8B8A8_UNORM :
                                     PIPE_FORMAT_R16G16B16A16_UNORM;
-   const bool padded_rgb = agx_native_graphics(&dev->dev) &&
+   const bool padded_rgb =
+      (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
                           hk_padded_image_format(p_format) != p_format;
    if (padded_rgb)
       p_format = hk_padded_image_format(p_format);
@@ -463,7 +465,7 @@ pack_pbe(struct hk_device *dev, struct hk_image_view *view, unsigned view_plane,
    unsigned layer = view->vk.base_array_layer;
 
    enum pipe_format p_format = format_for_plane(view, view_plane);
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       p_format = hk_padded_image_format(p_format);
       enum pipe_format integer = hk_scaled_texture_format(p_format);
       if (integer != PIPE_FORMAT_NONE)
@@ -479,7 +481,8 @@ pack_pbe(struct hk_device *dev, struct hk_image_view *view, unsigned view_plane,
    if (eot)
       p_format = util_format_linear(p_format);
 
-   if (eot && agx_native_graphics(&dev->dev))
+   if (eot && (agx_native_graphics(&dev->dev) ||
+                agx_chip_g17_m5(dev->dev.chip)))
       p_format = agx_g17p_rt0_pbe_format(p_format);
 
    const struct util_format_description *desc =
@@ -642,7 +645,7 @@ pack_pbe(struct hk_device *dev, struct hk_image_view *view, unsigned view_plane,
        * sample count [62:63].  The
        * shader-side Gen9 decoder consumes this word independently of the
        * native PBE. */
-      if (atomic_sw_out && generation == 9 && !cfg.extended &&
+      if (atomic_sw_out && (generation == 9 || generation == 10) && !cfg.extended &&
           (layout->writeable_image || usage == HK_DESC_USAGE_EMRT)) {
          /* A linear image addresses texels through the native PBE's row
           * stride and is never layered here, so its sideband carries no tile
@@ -738,7 +741,7 @@ hk_image_view_init(struct hk_device *dev, struct hk_image_view *view,
       if (!textures_valid)
          return VK_ERROR_FORMAT_NOT_SUPPORTED;
 
-      if (agx_native_graphics(&dev->dev)) {
+      if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
          VkResult result = hk_descriptor_table_add(
             dev, &dev->sampled_images, &view->planes[p].sampled, 32,
             &view->planes[p].sampled_heap_index);
@@ -789,7 +792,8 @@ hk_image_view_init(struct hk_device *dev, struct hk_image_view *view,
          view->planes[p].emrt_pbe_valid = pack_pbe(
             dev, view, p, HK_DESC_USAGE_EMRT, &view->planes[p].emrt_pbe,
             &view->planes[p].emrt_atomic_sw);
-         if (agx_native_graphics(&dev->dev) &&
+         if ((agx_native_graphics(&dev->dev) ||
+              agx_chip_g17_m5(dev->dev.chip)) &&
              (view->vk.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
              view->planes[p].emrt_pbe_valid) {
             const struct ail_layout *layout =
@@ -854,7 +858,7 @@ hk_DestroyImageView(VkDevice _device, VkImageView imageView,
    if (!view)
       return;
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       for (unsigned p = 0; p < view->plane_count; ++p) {
          if (view->planes[p].emrt_read_heap_index)
             hk_descriptor_table_remove(dev, &dev->sampled_images,
@@ -904,6 +908,37 @@ hk_CreateImageView(VkDevice _device, const VkImageViewCreateInfo *pCreateInfo,
       return result;
    }
 
+   /* Compressed data only decodes through its own channel layout. Only a
+    * mutable image created without a format list under the
+    * hk_mutable_format_srgb_pair application option can reach this. */
+   const struct hk_image *image = container_of(view->vk.image, struct hk_image, vk);
+   if (view->plane_count == 1 && view->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT &&
+       !ail_is_view_compatible(
+          &image->planes[view->planes[0].image_plane].layout,
+          hk_format_to_pipe_format(view->vk.view_format))) {
+      mesa_loge_once("compressed %s image viewed as %s: the "
+                     "hk_mutable_format_srgb_pair assumption does not hold",
+                     util_format_short_name(
+                        hk_format_to_pipe_format(image->vk.format)),
+                     util_format_short_name(
+                        hk_format_to_pipe_format(view->vk.view_format)));
+   }
+
+   /* Frame anatomy profiler: the formats that reinterpret a mutable image,
+    * which its compression decision could not see without a format list. */
+   const struct vk_image *vk_image = view->vk.image;
+   if (unlikely(dev->frame_anatomy) &&
+       (vk_image->create_flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) &&
+       view->vk.view_format != vk_image->format) {
+      hk_frame_anatomy_note(
+         dev, "view %ux%u fmt=%s samples=%u -> %s usage=0x%x",
+         vk_image->extent.width, vk_image->extent.height,
+         util_format_short_name(hk_format_to_pipe_format(vk_image->format)),
+         vk_image->samples,
+         util_format_short_name(hk_format_to_pipe_format(view->vk.view_format)),
+         (unsigned)view->vk.usage);
+   }
+
    *pView = hk_image_view_to_handle(view);
 
    return VK_SUCCESS;
diff --git a/src/asahi/vulkan/hk_instance.c b/src/asahi/vulkan/hk_instance.c
index bef98ef5c52e..8630624d7164 100644
--- a/src/asahi/vulkan/hk_instance.c
+++ b/src/asahi/vulkan/hk_instance.c
@@ -118,6 +118,11 @@ hk_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
    if (result != VK_SUCCESS)
       goto fail_alloc;
 
+   /* Keep existing error provenance visible for opt-in device-init diagnosis
+    * without requiring validation layers or changing initialization policy. */
+   if (debug_get_bool_option("ASAHI_M5_DEVICE_INIT_DIAG", false))
+      instance->vk.enable_debug_logging = true;
+
    hk_init_dri_options(instance);
 
    instance->vk.physical_devices.try_create_for_drm =
diff --git a/src/asahi/vulkan/hk_m5_acceleration_structure.c b/src/asahi/vulkan/hk_m5_acceleration_structure.c
new file mode 100644
index 000000000000..63d8c93fa226
--- /dev/null
+++ b/src/asahi/vulkan/hk_m5_acceleration_structure.c
@@ -0,0 +1,1973 @@
+/*
+ * Copyright © 2021 Google
+ * Copyright © 2023 Valve Corporation
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * VK_KHR_acceleration_structure for Honeykrisp.
+ *
+ * The heavy lifting -- Morton codes, the radix sort, PLOC/HPLOC or LBVH
+ * construction -- is Mesa's common builder, which produces a driver-neutral
+ * binary tree in scratch memory.  All this file adds is the conversion of
+ * that tree into the node format the software traversal reads (hk_bvh.h),
+ * which happens in one compute dispatch per structure, plus the entry points
+ * and the handful of vk_device hooks the common builder calls back into.
+ */
+
+#include "hk_acceleration_structure.h"
+#include "hk_m5_acceleration_structure.h"
+#include "hk_agx3_as_layout.h"
+#include "asahi/compiler/agx3_pack.h"
+#include "asahi/compiler/agx3_mem_core.h"
+#include "hk_m5_ift_diag_inventory.h"
+#include "util/u_debug.h"
+#include "util/cache_ops.h"
+#include "util/u_atomic.h"
+#include <errno.h>
+#include <unistd.h>
+
+#include "hk_buffer.h"
+#include "hk_cmd_buffer.h"
+#include "hk_device.h"
+#include "hk_entrypoints.h"
+#include "hk_physical_device.h"
+#include "libagx_dgc.h"
+#include "libagx_shaders.h"
+
+#include "vk_acceleration_structure.h"
+#include "vk_common_entrypoints.h"
+#include "vk_meta.h"
+
+#include "bvh/hk_build_interface.h"
+#include "hk_agx3_as_format.h"
+
+#include "util/u_math.h"
+
+static const uint32_t agx3_batch_build_leaf_spv[] = {
+#include "bvh/hk_agx3_batch_build_leaf.spv.h"
+};
+
+static const uint32_t encode_spv[] = {
+#include "bvh/hk_encode.spv.h"
+};
+
+static const uint32_t write_spv[] = {
+#include "bvh/hk_bvh_write.spv.h"
+};
+
+static const uint32_t agx3_node_spv[] = {
+#include "bvh/hk_agx3_m5_legacy_leaf.spv.h"
+};
+
+static const uint32_t agx3_envelope_spv[] = {
+#include "bvh/hk_agx3_m5_legacy_envelope.spv.h"
+};
+
+static const uint32_t agx3_plan_spv[] = {
+#include "bvh/hk_agx3_m5_legacy_plan.spv.h"
+};
+
+static const uint32_t agx3_batch_plan_spv[] = {
+#include "bvh/hk_agx3_batch_plan.spv.h"
+};
+static const uint32_t agx3_batch_node_spv[] = {
+#include "bvh/hk_agx3_batch_leaf.spv.h"
+};
+static const uint32_t agx3_batch_envelope_spv[] = {
+#include "bvh/hk_agx3_batch_envelope.spv.h"
+};
+
+static const uint32_t agx3_refit_spv[] = {
+#include "bvh/hk_agx3_m5_legacy_refit.spv.h"
+};
+
+const uint32_t *
+hk_m5_agx3_node_encoder_spirv(bool batch, size_t *size_B)
+{
+   *size_B = batch ? sizeof(agx3_batch_node_spv) : sizeof(agx3_node_spv);
+   return batch ? agx3_batch_node_spv : agx3_node_spv;
+}
+
+const uint32_t *
+hk_m5_agx3_refit_spirv(size_t *size_B)
+{
+   *size_B = sizeof(agx3_refit_spv);
+   return agx3_refit_spv;
+}
+
+static void hk_flush_buffer_write_cp(VkCommandBuffer);
+static void hk_agx3_refit_phase(VkCommandBuffer,
+   const struct vk_acceleration_structure_build_state *, unsigned);
+
+static void hk_as_phase_begin(VkCommandBuffer,
+   struct vk_acceleration_structure_build_marker *);
+static void hk_as_phase_end(VkCommandBuffer,
+   struct vk_acceleration_structure_build_marker *);
+
+/* Native tables have 16-byte entries and reserve ID zero. Keep the same
+ * representable ID range as the old command-pool snapshots, without allocating
+ * four GiB of physical memory. Each page is independently mapped once and lives
+ * until device destruction; neither command recording nor submission copies or
+ * patches the table. Updates of unrelated IDs can proceed while work is pending.
+ */
+#define HK_M5_AS_TABLE_PAGE_SIZE 0x4000u
+#define HK_M5_AS_TABLE_SIZE (UINT64_C(1) << 32)
+#define HK_M5_AS_TABLE_MAX_ID (UINT32_MAX / 16u - 1u)
+
+/* Called under m5_as_lock. BO mappings are uncached CPU mappings, like the
+ * default descriptor-pool path. Fence the published address before returning
+ * from creation, including updates to a descriptor in pending work.
+ */
+static uint64_t *
+hk_m5_as_table_entry(struct hk_device *dev, unsigned id)
+{
+   unsigned page = id / (HK_M5_AS_TABLE_PAGE_SIZE / 16);
+   while (util_dynarray_num_elements(&dev->m5_as_table_pages,
+                                     struct agx_bo *) <= page) {
+      unsigned next = util_dynarray_num_elements(&dev->m5_as_table_pages,
+                                                 struct agx_bo *);
+      struct agx_bo *bo = agx_bo_create(&dev->dev, HK_M5_AS_TABLE_PAGE_SIZE,
+                                      HK_M5_AS_TABLE_PAGE_SIZE, 0,
+                                      "Native AS address table");
+      if (!bo)
+         return NULL;
+      memset(agx_bo_map(bo), 0, HK_M5_AS_TABLE_PAGE_SIZE);
+      struct agx_bo **slot = util_dynarray_grow(&dev->m5_as_table_pages,
+                                               struct agx_bo *, 1);
+      if (!slot) {
+         agx_bo_unreference(&dev->dev, bo);
+         return NULL;
+      }
+      const uint64_t address = dev->m5_as_table_va->addr +
+                               (uint64_t)next * HK_M5_AS_TABLE_PAGE_SIZE;
+      if (agx_bo_bind(&dev->dev, bo, address, HK_M5_AS_TABLE_PAGE_SIZE, 0,
+                      DRM_ASAHI_BIND_READ)) {
+         /* The reserved slot is uninitialized; roll it back without reading it. */
+         dev->m5_as_table_pages.size -= sizeof(*slot);
+         agx_bo_unreference(&dev->dev, bo);
+         return NULL;
+      }
+      *slot = bo;
+   }
+   struct agx_bo *bo = util_dynarray_element(&dev->m5_as_table_pages,
+                                             struct agx_bo *, page)[0];
+   return (uint64_t *)agx_bo_map(bo) +
+          2 * (id % (HK_M5_AS_TABLE_PAGE_SIZE / 16));
+}
+
+/* Native instance records and the AS header name entries in the same table.
+ * Per-lane replacement of those entries aliases child IDs with root IDs. */
+static uint32_t
+hk_m5_as_id(struct hk_device *dev, struct vk_acceleration_structure *as)
+{
+   uint32_t result = 0, i = 1;
+   simple_mtx_lock(&dev->m5_as_lock);
+   util_dynarray_foreach(&dev->m5_ases, struct vk_acceleration_structure *, p) {
+      if (*p == as) { result = i; break; }
+      ++i;
+   }
+   simple_mtx_unlock(&dev->m5_as_lock);
+   return result;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CreateAccelerationStructureKHR(VkDevice device,
+   const VkAccelerationStructureCreateInfoKHR *info,
+   const VkAllocationCallbacks *alloc, VkAccelerationStructureKHR *out)
+{
+   VK_FROM_HANDLE(hk_device, dev, device);
+   VkResult result = vk_common_CreateAccelerationStructureKHR(device, info, alloc, out);
+   if (result != VK_SUCCESS || !dev->m5_native_as)
+      return result;
+   VK_FROM_HANDLE(vk_acceleration_structure, as, *out);
+   bool inserted = false;
+   simple_mtx_lock(&dev->m5_as_lock);
+   unsigned index = 0;
+   util_dynarray_foreach(&dev->m5_ases, struct vk_acceleration_structure *, p) {
+      if (!*p)
+         break;
+      ++index;
+   }
+   if (index < HK_M5_AS_TABLE_MAX_ID) {
+      uint64_t *entry = hk_m5_as_table_entry(dev, index + 1);
+      if (entry) {
+         /* Vulkan permits AS creation before the buffer's memory binding.
+          * Ordinary Honeykrisp buffers acquire their address at that binding,
+          * so publish it again then rather than freezing an offset-only VA. */
+         const bool pending = as->buffer->device_address == 0;
+         uint32_t *pending_id = pending
+            ? util_dynarray_grow(&dev->m5_pending_as_ids, uint32_t, 1) : NULL;
+         if (!pending || pending_id) {
+            struct vk_acceleration_structure **slot;
+            if (index == util_dynarray_num_elements(&dev->m5_ases,
+                                                    struct vk_acceleration_structure *))
+               slot = util_dynarray_grow(&dev->m5_ases,
+                                         struct vk_acceleration_structure *, 1);
+            else
+               slot = util_dynarray_element(&dev->m5_ases,
+                                             struct vk_acceleration_structure *, index);
+            if (slot) {
+               *slot = as;
+               if (pending) {
+                  *pending_id = index + 1;
+                  p_atomic_set(&dev->m5_pending_as_count,
+                     util_dynarray_num_elements(&dev->m5_pending_as_ids, uint32_t));
+               }
+               *entry = ALIGN_POT(vk_acceleration_structure_get_va(as),
+                                  UINT64_C(1024));
+               util_post_flush_fence();
+               inserted = true;
+            } else if (pending) {
+               /* No ID was stored in the reserved slot. */
+               dev->m5_pending_as_ids.size -= sizeof(*pending_id);
+            }
+         }
+      }
+   }
+   simple_mtx_unlock(&dev->m5_as_lock);
+   if (!inserted) {
+      vk_common_DestroyAccelerationStructureKHR(device, *out, alloc);
+      *out = VK_NULL_HANDLE;
+      return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+   }
+   return VK_SUCCESS;
+}
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_DestroyAccelerationStructureKHR(VkDevice device, VkAccelerationStructureKHR handle,
+                                   const VkAllocationCallbacks *alloc)
+{
+   VK_FROM_HANDLE(hk_device, dev, device);
+   VK_FROM_HANDLE(vk_acceleration_structure, as, handle);
+   if (as && dev->m5_native_as) {
+      simple_mtx_lock(&dev->m5_as_lock);
+      unsigned id = 1;
+      util_dynarray_foreach(&dev->m5_ases, struct vk_acceleration_structure *, p) {
+         if (*p == as) {
+            *p = NULL;
+            util_dynarray_delete_unordered(&dev->m5_pending_as_ids, uint32_t, id);
+            p_atomic_set(&dev->m5_pending_as_count,
+               util_dynarray_num_elements(&dev->m5_pending_as_ids, uint32_t));
+            break;
+         }
+         ++id;
+      }
+      simple_mtx_unlock(&dev->m5_as_lock);
+   }
+   vk_common_DestroyAccelerationStructureKHR(device, handle, alloc);
+}
+
+/* Called after a successful buffer binding. A normally pre-bound AS never
+ * enters this list; its creation and every subsequent queue submission keep
+ * the existing path. Only addresses which were unresolved at AS creation are
+ * visited, with no shader or command-buffer mutation. */
+void
+hk_m5_as_buffer_bound(struct hk_device *dev, struct vk_buffer *buffer)
+{
+   if (!dev->m5_native_as || !p_atomic_read(&dev->m5_pending_as_count))
+      return;
+
+   bool published = false;
+   simple_mtx_lock(&dev->m5_as_lock);
+   for (unsigned i = 0;
+        i < util_dynarray_num_elements(&dev->m5_pending_as_ids, uint32_t);) {
+      uint32_t *pending = util_dynarray_element(&dev->m5_pending_as_ids,
+                                               uint32_t, i);
+      const uint32_t id = *pending;
+      struct vk_acceleration_structure *as =
+         *util_dynarray_element(&dev->m5_ases,
+                                 struct vk_acceleration_structure *, id - 1);
+      assert(as);
+      if (as->buffer != buffer) {
+         ++i;
+         continue;
+      }
+      uint64_t *entry = hk_m5_as_table_entry(dev, id);
+      assert(entry); /* The AS was published only after its page was mapped. */
+      *entry = ALIGN_POT(vk_acceleration_structure_get_va(as), UINT64_C(1024));
+      *pending = util_dynarray_pop(&dev->m5_pending_as_ids, uint32_t);
+      published = true;
+   }
+   if (published)
+      util_post_flush_fence();
+   p_atomic_set(&dev->m5_pending_as_count,
+      util_dynarray_num_elements(&dev->m5_pending_as_ids, uint32_t));
+   simple_mtx_unlock(&dev->m5_as_lock);
+}
+
+static uint64_t
+hk_m5_upload_as_table_count(struct hk_cmd_buffer *cmd, unsigned *table_count)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   simple_mtx_lock(&dev->m5_as_lock);
+   unsigned count = util_dynarray_num_elements(&dev->m5_ases,
+                                              struct vk_acceleration_structure *);
+   if (table_count)
+      *table_count = count;
+   struct agx_ptr table = hk_pool_alloc(cmd, (count + 1) * 16, 16);
+   if (table.cpu) {
+      memset(table.cpu, 0, (count + 1) * 16);
+      uint64_t *words = table.cpu;
+      unsigned i = 1;
+      util_dynarray_foreach(&dev->m5_ases, struct vk_acceleration_structure *, p) {
+         if (*p)
+            words[2*i] = ALIGN_POT(vk_acceleration_structure_get_va(*p), UINT64_C(1024));
+         ++i;
+      }
+   }
+   simple_mtx_unlock(&dev->m5_as_lock);
+   return table.cpu ? table.gpu : 0;
+}
+
+uint64_t
+hk_m5_upload_as_table(struct hk_cmd_buffer *cmd)
+{
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   return dev->m5_as_table_va ? dev->m5_as_table_va->addr : 0;
+}
+
+static_assert(sizeof(struct hk_bvh_triangle_node) == 48, "node format");
+static_assert(sizeof(struct hk_bvh_aabb_node) == 32, "node format");
+static_assert(sizeof(struct hk_bvh_instance_node) == 120, "node format");
+static_assert(sizeof(struct hk_bvh_box_node) == 56, "node format");
+static_assert(sizeof(struct hk_agx3_leaf_job) == 64, "native node job ABI");
+/* hk_build_interface.h is shared with the GEN1 encoders, whose leaf and plan
+ * arguments end in fields (counts; pairs) that M5 leaves zero and its
+ * encoders never read. */
+static_assert(sizeof(struct hk_agx3_leaf_args) == 32, "native node push ABI");
+static_assert(sizeof(struct hk_agx3_envelope_job) == 48, "native envelope job ABI");
+static_assert(sizeof(struct hk_agx3_envelope_args) == 64, "native envelope push ABI");
+static_assert(sizeof(struct hk_agx3_plan_node) == 16, "native planner node ABI");
+static_assert(sizeof(struct hk_agx3_plan_state) == 32, "native planner state ABI");
+static_assert(sizeof(struct hk_agx3_plan_args) == 120, "native planner push ABI");
+/* The M5 layout: the plan state alone after its scratch offset, 48-byte
+ * bounding-box records within the 64-byte child slots, 1 KiB images, and one
+ * node per 8-lane encoding workgroup (bvh/agx3_leaf.comp). */
+static_assert(HK_AGX3_M5_PLAN_TAIL_B == sizeof(struct hk_agx3_plan_state),
+              "M5 scratch: no node-encoding dispatch arguments");
+static_assert(HK_AGX3_M5_AABB_RECORD_B == 48 &&
+              HK_AGX3_M5_AABB_RECORD_B < HK_AGX3_AS_SLOT_B,
+              "M5 bounding-box records pack below the slot size");
+static_assert(HK_AGX3_M5_IMAGE_ALIGN == 1024, "M5 image alignment");
+static_assert(HK_AGX3_M5_NODE_GROUP_LANES == 8, "M5 encoder workgroup");
+static_assert(sizeof(struct hk_bvh_box_node) == HK_BVH_NODE_PREFETCH_SIZE,
+              "the traversal prefetches a whole box node before it knows the "
+              "node type, so no node may be larger than one");
+
+static VkResult
+get_pipeline_spv(struct hk_device *dev, const char *name, const uint32_t *spv,
+                 uint32_t spv_size, unsigned push_constant_size,
+                 VkPipeline *pipeline, VkPipelineLayout *layout)
+{
+   size_t key_size = strlen(name);
+
+   const VkPushConstantRange pc_range = {
+      .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
+      .offset = 0,
+      .size = push_constant_size,
+   };
+
+   VkResult result = vk_meta_get_pipeline_layout(
+      &dev->vk, &dev->meta, NULL, &pc_range, name, key_size, layout);
+   if (result != VK_SUCCESS)
+      return result;
+
+   VkPipeline cached = vk_meta_lookup_pipeline(&dev->meta, name, key_size);
+   if (cached != VK_NULL_HANDLE) {
+      *pipeline = cached;
+      return VK_SUCCESS;
+   }
+
+   const VkShaderModuleCreateInfo module_info = {
+      .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
+      .codeSize = spv_size,
+      .pCode = spv,
+   };
+   const VkComputePipelineCreateInfo pipeline_info = {
+      .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+      .stage =
+         {
+            .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+            .pNext = &module_info,
+            .stage = VK_SHADER_STAGE_COMPUTE_BIT,
+            .pName = "main",
+         },
+      .layout = *layout,
+   };
+
+   return vk_meta_create_compute_pipeline(&dev->vk, &dev->meta, &pipeline_info,
+                                          name, key_size, pipeline);
+}
+
+VkResult
+hk_m5_backend_cmd_encode_agx3_nodes(VkCommandBuffer command_buffer,
+                        const struct hk_agx3_leaf_args *args)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!args->job_count)
+      return VK_SUCCESS;
+
+   /* The native phase uses eight invocations per independently writable
+    * node. Job planning bounds the count before it reaches this helper. */
+   assert(args->job_count <= UINT32_MAX / 8);
+   VkPipeline pipeline;
+   VkPipelineLayout layout;
+   VkResult result = get_pipeline_spv(
+      dev, "hk-agx3-node-encode", agx3_node_spv, sizeof(agx3_node_spv),
+      sizeof(*args), &pipeline, &layout);
+   if (result != VK_SUCCESS)
+      return result;
+
+   vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                            pipeline);
+   vk_common_CmdPushConstants(command_buffer, layout,
+                             VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(*args), args);
+   hk_cmd_dispatch_unaligned(command_buffer, args->job_count * 8, 1, 1);
+   return VK_SUCCESS;
+}
+
+VkResult
+hk_m5_backend_cmd_finalize_agx3_as(VkCommandBuffer command_buffer,
+                       const struct hk_agx3_envelope_args *args)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!args->job_count)
+      return VK_SUCCESS;
+
+   VkPipeline pipeline;
+   VkPipelineLayout layout;
+   VkResult result = get_pipeline_spv(
+      dev, "hk-agx3-as-envelope", agx3_envelope_spv, sizeof(agx3_envelope_spv),
+      sizeof(*args), &pipeline, &layout);
+   if (result != VK_SUCCESS)
+      return result;
+
+   vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                            pipeline);
+   vk_common_CmdPushConstants(command_buffer, layout,
+                             VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(*args), args);
+   hk_cmd_dispatch_unaligned(command_buffer, args->job_count, 1, 1);
+   return VK_SUCCESS;
+}
+
+static VkResult
+hk_cmd_build_agx3_from_ir_mode(VkCommandBuffer command_buffer,
+                        const struct hk_agx3_plan_args *args,
+                        VkDeviceAddress node_status,
+                        VkDeviceAddress envelope_status,
+                        bool single_leaf)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   const struct {
+      const char *name;
+      const uint32_t *code;
+      size_t size;
+      unsigned push_size;
+   } stages[] = {
+      {"hk-agx3-plan", agx3_plan_spv, sizeof(agx3_plan_spv), sizeof(*args)},
+      {"hk-agx3-node-encode", agx3_node_spv, sizeof(agx3_node_spv),
+       sizeof(struct hk_agx3_leaf_args)},
+      {"hk-agx3-as-envelope", agx3_envelope_spv, sizeof(agx3_envelope_spv),
+       sizeof(struct hk_agx3_envelope_args)},
+   };
+   VkPipeline pipeline[3];
+   VkPipelineLayout layout[3];
+   for (unsigned i = 0; i < ARRAY_SIZE(stages); ++i) {
+      VkResult result = get_pipeline_spv(
+         dev, stages[i].name, stages[i].code, stages[i].size,
+         stages[i].push_size, &pipeline[i], &layout[i]);
+      if (result != VK_SUCCESS)
+         return result;
+   }
+
+   vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                            pipeline[0]);
+   const VkMemoryBarrier barrier = {
+      .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+      .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
+      .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
+                       VK_ACCESS_INDIRECT_COMMAND_READ_BIT,
+   };
+   for (unsigned i = 0; i < (single_leaf ? 1 : 5); ++i) {
+      unsigned phase = single_leaf ? 5 : i;
+      if (debug_get_bool_option("ASAHI_M5_AS_PHASE_DIAG", false)) {
+         struct vk_acceleration_structure_build_marker marker = {
+            .step = VK_ACCELERATION_STRUCTURE_BUILD_STEP_TOP /* no ENCODE step upstream */,
+            .encode = {.key = 0x100 + phase, .leaf_node_count = args->leaf_bound},
+         };
+         hk_as_phase_begin(command_buffer, &marker);
+         vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                                  pipeline[0]);
+      }
+      struct hk_agx3_plan_args push = *args;
+      push.phase = phase;
+      vk_common_CmdPushConstants(command_buffer, layout[0],
+                                VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push),
+                                &push);
+      hk_cmd_dispatch_unaligned(command_buffer, args->node_bound, 1, 1);
+      vk_common_CmdPipelineBarrier(
+         command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+         VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
+         0, 1, &barrier, 0, NULL, 0, NULL);
+   }
+
+   const struct hk_agx3_leaf_args encode = {
+      .jobs = args->jobs, .status = node_status, .job_count = args->node_bound,
+      .slot_bias = args->m5_aabb_stride,
+   };
+   const struct hk_agx3_envelope_args finish = {
+      .jobs = args->envelope, .status = envelope_status, .job_count = 1,
+   };
+   const bool instances = args->geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR;
+   for (unsigned step = 0; step < 2; ++step) {
+      bool envelope = instances ? step == 0 : step == 1;
+      unsigned stage = envelope ? 2 : 1;
+      if (debug_get_bool_option("ASAHI_M5_AS_PHASE_DIAG", false)) {
+         struct vk_acceleration_structure_build_marker marker = {
+            .step = VK_ACCELERATION_STRUCTURE_BUILD_STEP_TOP /* no ENCODE step upstream */,
+            .encode = {.key = 0x200 + stage, .leaf_node_count = args->leaf_bound},
+         };
+         hk_as_phase_begin(command_buffer, &marker);
+      }
+      vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                               pipeline[stage]);
+      vk_common_CmdPushConstants(command_buffer, layout[stage],
+                                VK_SHADER_STAGE_COMPUTE_BIT, 0,
+                                envelope ? sizeof(finish) : sizeof(encode),
+                                envelope ? (const void *)&finish : (const void *)&encode);
+      if (envelope)
+         hk_cmd_dispatch_unaligned(command_buffer, 1, 1, 1);
+      else
+         hk_cmd_dispatch_indirect_addr(command_buffer, args->state);
+      if (step == 0)
+         vk_common_CmdPipelineBarrier(command_buffer,
+            VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+            VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
+            0, 1, &barrier, 0, NULL, 0, NULL);
+   }
+   return VK_SUCCESS;
+}
+
+/* External prebuilt-IR import callers keep the full topology planner. */
+VkResult
+hk_m5_backend_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
+                        const struct hk_agx3_plan_args *args,
+                        VkDeviceAddress node_status,
+                        VkDeviceAddress envelope_status)
+{
+   return hk_cmd_build_agx3_from_ir_mode(command_buffer, args, node_status,
+                                       envelope_status, false);
+}
+
+/* ---------------------------------------------------------------- */
+/* Layout                                                            */
+
+static void
+hk_leaf_node_sizes(VkGeometryTypeKHR geometry_type, uint32_t *ir_size,
+                   uint32_t *out_size)
+{
+   switch (geometry_type) {
+   case VK_GEOMETRY_TYPE_TRIANGLES_KHR:
+      *ir_size = sizeof(struct vk_ir_triangle_node);
+      *out_size = sizeof(struct hk_bvh_triangle_node);
+      return;
+   case VK_GEOMETRY_TYPE_AABBS_KHR:
+      *ir_size = sizeof(struct vk_ir_aabb_node);
+      *out_size = sizeof(struct hk_bvh_aabb_node);
+      return;
+   default:
+      *ir_size = sizeof(struct vk_ir_instance_node);
+      *out_size = sizeof(struct hk_bvh_instance_node);
+      return;
+   }
+}
+
+/* Common-builder leaf record consumed by the native planner/encoder. */
+static uint32_t
+hk_agx3_ir_leaf_size(VkGeometryTypeKHR geometry_type)
+{
+   switch (geometry_type) {
+   case VK_GEOMETRY_TYPE_INSTANCES_KHR:
+      return sizeof(struct vk_ir_instance_node);
+   case VK_GEOMETRY_TYPE_AABBS_KHR:
+      return sizeof(struct vk_ir_aabb_node);
+   default:
+      return sizeof(struct vk_ir_triangle_node);
+   }
+}
+
+static VkDeviceSize
+hk_as_size(VkGeometryTypeKHR geometry_type, uint32_t leaf_node_count)
+{
+   /* The common builder always produces MAX2(leaves, 2) - 1 internal nodes,
+    * so a one-primitive structure still has a root to point at. */
+   uint32_t internal_node_count = MAX2(leaf_node_count, 2) - 1;
+
+   uint32_t ir_size, out_size;
+   hk_leaf_node_sizes(geometry_type, &ir_size, &out_size);
+
+   uint32_t nodes_size = internal_node_count * sizeof(struct hk_bvh_box_node) +
+                         leaf_node_count * out_size;
+
+   /* Pad so the traversal's unconditional whole-node prefetch of the last
+    * node cannot read past the allocation. */
+   nodes_size = util_align_npot(nodes_size, HK_BVH_NODE_PREFETCH_SIZE);
+
+   return sizeof(struct hk_bvh_header) + nodes_size;
+}
+
+static VkDeviceSize
+hk_agx3_retained_offset(const struct vk_acceleration_structure_build_state *state)
+{
+   struct hk_agx3_as_layout layout;
+   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout))
+      return 0;
+   return ALIGN_POT(layout.as_bytes +
+      (vk_get_as_geometry_type(state->build_info) == VK_GEOMETRY_TYPE_INSTANCES_KHR
+          ? (uint64_t)state->leaf_node_count * 80 : 0), 8);
+}
+
+static VkDeviceSize
+hk_get_as_size(VkDevice _device,
+               const struct vk_acceleration_structure_build_state *state)
+{
+   VK_FROM_HANDLE(hk_device, dev, _device);
+   if (dev->m5_native_as) {
+      struct hk_agx3_as_layout layout;
+      if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout))
+         return 0;
+      if (state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR)
+         return hk_agx3_retained_offset(state) + sizeof(struct vk_ir_header) +
+                (uint64_t)state->internal_node_count * sizeof(struct vk_ir_box_node);
+      return layout.as_bytes +
+         (vk_get_as_geometry_type(state->build_info) == VK_GEOMETRY_TYPE_INSTANCES_KHR
+             ? (uint64_t)state->leaf_node_count * 80 : 0);
+   }
+   return hk_as_size(vk_get_as_geometry_type(state->build_info),
+                     state->leaf_node_count);
+}
+
+static VkDeviceSize
+hk_get_encode_scratch_size(VkDevice _device,
+                           const struct vk_acceleration_structure_build_state *state)
+{
+   VK_FROM_HANDLE(hk_device, dev, _device);
+   struct hk_agx3_as_layout layout;
+   return dev->m5_native_as &&
+          hk_agx3_as_layout_init(state->leaf_node_count, &layout)
+             ? layout.encode_bytes : 0;
+}
+
+static uint64_t
+hk_as_native_va(struct vk_acceleration_structure *as)
+{
+   return ALIGN_POT(vk_acceleration_structure_get_va(as), UINT64_C(1024));
+}
+
+VKAPI_ATTR VkDeviceAddress VKAPI_CALL
+hk_m5_backend_GetAccelerationStructureDeviceAddressKHR(
+   VkDevice _device, const VkAccelerationStructureDeviceAddressInfoKHR *info)
+{
+   VK_FROM_HANDLE(hk_device, dev, _device);
+   VK_FROM_HANDLE(vk_acceleration_structure, as, info->accelerationStructure);
+   return dev->m5_native_as ? hk_as_native_va(as)
+                            : vk_acceleration_structure_get_va(as);
+}
+
+/* ---------------------------------------------------------------- */
+/* Encode                                                            */
+
+static VkResult
+hk_encode_prepare(VkCommandBuffer commandBuffer,
+                  const struct vk_acceleration_structure_build_state *state)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+
+   if (dev->m5_native_as)
+      return VK_SUCCESS;
+
+   VkPipeline pipeline;
+   VkPipelineLayout layout;
+   VkResult result =
+      get_pipeline_spv(dev, "hk-bvh-encode", encode_spv, sizeof(encode_spv),
+                       sizeof(struct hk_encode_args), &pipeline, &layout);
+   if (result != VK_SUCCESS)
+      return result;
+
+   vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                             pipeline);
+   return VK_SUCCESS;
+}
+
+static void
+hk_encode_as(VkCommandBuffer commandBuffer,
+             const struct vk_acceleration_structure_build_state *state)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   VK_FROM_HANDLE(vk_acceleration_structure, dst,
+                  state->build_info->dstAccelerationStructure);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+
+   if (dev->m5_native_as) {
+      /* Keep only header/internal topology; updated vertices are read from
+       * input buffers, with no Morton sort or tree reconstruction. */
+      if ((state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR) &&
+          (state->build_info->mode == VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR ||
+           state->build_info->srcAccelerationStructure != state->build_info->dstAccelerationStructure)) {
+         hk_agx3_refit_phase(commandBuffer, state, 0);
+         hk_flush_buffer_write_cp(commandBuffer);
+      }
+      struct hk_agx3_as_layout layout;
+      if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout)) {
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+         return;
+      }
+      const uint64_t scratch = state->build_info->scratchData.deviceAddress;
+      const uint64_t encode = scratch + state->scratch.header_offset +
+                              sizeof(struct vk_ir_header);
+      const struct hk_agx3_plan_args args = {
+         .intermediate_bvh = scratch + state->scratch.ir_offset,
+         .header = scratch + state->scratch.header_offset,
+         .nodes = encode + layout.nodes,
+         .jobs = encode + layout.jobs,
+         .envelope = encode + layout.envelope,
+         .state = encode + layout.state,
+         .output_bvh = hk_as_native_va(dst),
+         .root_offset = state->leaf_node_count *
+            hk_agx3_ir_leaf_size(vk_get_as_geometry_type(state->build_info)),
+         .leaf_bound = state->leaf_node_count,
+         .node_bound = layout.node_bound,
+         .slot_bound = layout.slot_bound,
+         .resource_id = hk_m5_as_id(dev, dst),
+         .geometry_type = vk_get_as_geometry_type(state->build_info),
+         /* Native bounding-box records are 48 bytes (record class 4). The
+          * 64-byte class-5 slot form is a hardware A/B diagnostic only. */
+         .m5_aabb_stride = debug_get_num_option("ASAHI_M5_AABB_RECORD_STRIDE", 48) == 64
+                        ? 64 : 48,
+      };
+      VkResult result = hk_cmd_build_agx3_from_ir_mode(
+         commandBuffer, &args, encode + layout.node_status,
+         encode + layout.envelope_status,
+         false);
+      if (result != VK_SUCCESS)
+         vk_command_buffer_set_error(&cmd->vk, result);
+      return;
+   }
+
+   VkGeometryTypeKHR geometry_type =
+      vk_get_as_geometry_type(state->build_info);
+
+   uint32_t ir_size, out_size;
+   hk_leaf_node_sizes(geometry_type, &ir_size, &out_size);
+
+   VkDeviceAddress scratch = state->build_info->scratchData.deviceAddress;
+   uint32_t internal_bound = MAX2(state->leaf_node_count, 2) - 1;
+
+   VkPipeline pipeline;
+   VkPipelineLayout layout;
+   if (get_pipeline_spv(dev, "hk-bvh-encode", encode_spv, sizeof(encode_spv),
+                        sizeof(struct hk_encode_args), &pipeline,
+                        &layout) != VK_SUCCESS)
+      return;
+
+   const struct hk_encode_args args = {
+      .intermediate_bvh = scratch + state->scratch.ir_offset,
+      .output_bvh = vk_acceleration_structure_get_va(dst),
+      .header = scratch + state->scratch.header_offset,
+      .root_offset = state->leaf_node_count * ir_size,
+      .internal_node_bound = internal_bound,
+      .leaf_node_bound = state->leaf_node_count,
+      .geometry_type = geometry_type,
+      .as_size = hk_as_size(geometry_type, state->leaf_node_count),
+   };
+   vk_common_CmdPushConstants(commandBuffer, layout,
+                              VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(args),
+                              &args);
+
+   /* One invocation per output node.  The shader clamps against the counts
+    * the builder wrote into the header, which can be smaller than this bound
+    * when leaves turn out inactive. */
+   hk_cmd_dispatch_unaligned(commandBuffer,
+                             internal_bound + state->leaf_node_count, 1, 1);
+}
+
+static void
+hk_agx3_refit_phase(VkCommandBuffer commandBuffer,
+                    const struct vk_acceleration_structure_build_state *state,
+                    unsigned phase)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   struct hk_agx3_as_layout layout;
+   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout)) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+      return;
+   }
+   VkPipeline pipeline;
+   VkPipelineLayout pipeline_layout;
+   VkResult result = get_pipeline_spv(dev, "hk-agx3-refit", agx3_refit_spv,
+      sizeof(agx3_refit_spv), sizeof(struct hk_m5_refit_args), &pipeline, &pipeline_layout);
+   if (result != VK_SUCCESS) {
+      vk_command_buffer_set_error(&cmd->vk, result);
+      return;
+   }
+   VK_FROM_HANDLE(vk_acceleration_structure, retained_as,
+      phase == 1 ? state->build_info->srcAccelerationStructure : state->build_info->dstAccelerationStructure);
+   uint64_t scratch = state->build_info->scratchData.deviceAddress;
+   struct hk_m5_refit_args args = {
+      .retained = vk_acceleration_structure_get_va(retained_as) + hk_agx3_retained_offset(state),
+      .intermediate_bvh = scratch + state->scratch.ir_offset,
+      .header = scratch + state->scratch.header_offset,
+      .parents = scratch + state->scratch.size,
+      .root_offset = state->leaf_node_count *
+         hk_agx3_ir_leaf_size(vk_get_as_geometry_type(state->build_info)),
+      .leaf_bound = state->leaf_node_count,
+      .node_bound = layout.node_bound, .phase = phase,
+   };
+   unsigned count = phase <= 1
+      ? MAX2(layout.node_bound, (sizeof(struct vk_ir_header) +
+               state->internal_node_count * sizeof(struct vk_ir_box_node)) / 4)
+      : phase == 2 ? state->internal_node_count : state->leaf_node_count;
+   vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
+   vk_common_CmdPushConstants(commandBuffer, pipeline_layout,
+      VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(args), &args);
+   hk_cmd_dispatch_unaligned(commandBuffer, count, 1, 1);
+}
+
+static void
+hk_native_build_config(VkDevice device,
+                       struct vk_acceleration_structure_build_state *state)
+{
+   if (state->build_info->mode == VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR)
+      state->config.internal_type = VK_INTERNAL_BUILD_TYPE_UPDATE;
+   state->config.updateable =
+      (state->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR) != 0;
+   /* The upstream runtime always builds the internal topology; the former
+    * single-leaf shortcut needed a runtime hook this base does not have. */
+}
+
+static VkDeviceSize
+hk_update_scratch_size(VkDevice device,
+                       const struct vk_acceleration_structure_build_state *state)
+{
+   /* Leaf generation writes sort IDs, which alias native encode scratch.
+    * Keep live refit parent/counter state beyond both workspaces. */
+   struct hk_agx3_as_layout layout;
+   if (!hk_agx3_as_layout_init(state->leaf_node_count, &layout))
+      return 0;
+   return (VkDeviceSize)state->scratch.size + (uint64_t)layout.node_bound * 16;
+}
+
+static void
+hk_init_update_scratch(VkCommandBuffer commandBuffer,
+                       const struct vk_acceleration_structure_build_state *states,
+                       uint32_t count)
+{
+   for (unsigned i = 0; i < count; ++i)
+      if (states[i].config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE)
+         hk_agx3_refit_phase(commandBuffer, &states[i], 1);
+}
+
+static void
+hk_update_prepare(VkCommandBuffer commandBuffer,
+                  const struct vk_acceleration_structure_build_state *state,
+                  bool flushed_cp, bool flushed_compute)
+{
+   hk_flush_buffer_write_cp(commandBuffer);
+}
+
+static void
+hk_update_as(VkCommandBuffer commandBuffer,
+             const struct vk_acceleration_structure_build_state *state)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   hk_agx3_refit_phase(commandBuffer, state, 2);
+   VkResult result = vk_cmd_build_as_leaves(commandBuffer, &dev->vk, &dev->meta,
+      &dev->accel_struct_args, state);
+   if (result != VK_SUCCESS) {
+      vk_command_buffer_set_error(&cmd->vk, result);
+      return;
+   }
+   hk_flush_buffer_write_cp(commandBuffer);
+   hk_agx3_refit_phase(commandBuffer, state, 3);
+   hk_flush_buffer_write_cp(commandBuffer);
+   hk_encode_as(commandBuffer, state);
+}
+
+/* Same per-AS native records, planned and encoded concurrently. */
+static VkResult
+hk_cmd_encode_agx3_batch(VkCommandBuffer command_buffer,
+                         const struct hk_agx3_batch_job *entries,
+                         uint32_t count, uint32_t map_bound,
+                         uint32_t max_node_bound)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   static_assert(sizeof(struct hk_agx3_plan_args) == 120, "native plan ABI");
+   static_assert(sizeof(struct hk_agx3_batch_job) == 152, "batch array stride");
+   static_assert(sizeof(struct hk_agx3_batch_args) == 32, "batch push ABI");
+   static_assert(sizeof(struct hk_agx3_batch_map) == 8, "dense map stride");
+   static_assert(offsetof(struct hk_agx3_batch_job, first_job) == 140,
+                 "GPU-written dense offset ABI");
+
+   const struct {
+      const char *name;
+      const uint32_t *code;
+      size_t size;
+   } stages[] = {
+      {"hk-agx3-batch-plan", agx3_batch_plan_spv, sizeof(agx3_batch_plan_spv)},
+      {"hk-agx3-batch-node-encode", agx3_batch_node_spv, sizeof(agx3_batch_node_spv)},
+      {"hk-agx3-batch-as-envelope", agx3_batch_envelope_spv,
+       sizeof(agx3_batch_envelope_spv)},
+   };
+   VkPipeline pipeline[3];
+   VkPipelineLayout layout[3];
+   for (unsigned i = 0; i < ARRAY_SIZE(stages); ++i) {
+      VkResult result = get_pipeline_spv(dev, stages[i].name, stages[i].code,
+         stages[i].size, sizeof(struct hk_agx3_batch_args), &pipeline[i], &layout[i]);
+      if (result != VK_SUCCESS)
+         return result;
+   }
+   if (debug_get_bool_option("ASAHI_M5_AS_BATCH_DIAG", false))
+      fprintf(stderr, "M5_AS_BATCH builds=%u map_bound=%u max_node_bound=%u\n",
+              count, map_bound, max_node_bound);
+   const struct hk_agx3_batch_dispatch indirect = {.x = 0, .y = 1, .z = 1};
+   struct hk_agx3_batch_args push = {
+      .jobs = hk_pool_upload(cmd, entries, (size_t)count * sizeof(*entries), 8),
+      .map = hk_pool_alloc(cmd, (size_t)map_bound * sizeof(struct hk_agx3_batch_map), 8).gpu,
+      .dispatch = hk_pool_upload(cmd, &indirect, sizeof(indirect), 8),
+      .job_count = count,
+   };
+   if (!push.jobs || !push.map || !push.dispatch)
+      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+   const VkMemoryBarrier barrier = {
+      .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+      .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
+      .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
+                       VK_ACCESS_INDIRECT_COMMAND_READ_BIT,
+   };
+   bool all_single = true;
+   for (unsigned i = 0; i < count; ++i)
+      all_single &= entries[i].single_leaf != 0;
+
+   vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline[0]);
+   for (unsigned phase = 0; phase < 5; ++phase) {
+      if (all_single && (phase == 1 || phase == 2))
+         continue;
+      push.phase = phase;
+      vk_common_CmdPushConstants(command_buffer, layout[0], VK_SHADER_STAGE_COMPUTE_BIT,
+                                0, sizeof(push), &push);
+      hk_cmd_dispatch_unaligned(command_buffer, max_node_bound, count, 1);
+      vk_common_CmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+         VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
+         0, 1, &barrier, 0, NULL, 0, NULL);
+   }
+
+   /* Instance metadata must precede their records, while BLAS headers follow
+    * completed nodes, just as in the existing scalar encoder. */
+   const bool instances = entries[0].plan.geometry_type == VK_GEOMETRY_TYPE_INSTANCES_KHR;
+   for (unsigned step = 0; step < 2; ++step) {
+      const bool envelope = instances ? step == 0 : step == 1;
+      const unsigned stage = envelope ? 2 : 1;
+      vk_common_CmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline[stage]);
+      vk_common_CmdPushConstants(command_buffer, layout[stage], VK_SHADER_STAGE_COMPUTE_BIT,
+                                0, sizeof(push), &push);
+      if (envelope)
+         hk_cmd_dispatch_unaligned(command_buffer, count, 1, 1);
+      else
+         hk_cmd_dispatch_indirect_addr(command_buffer, push.dispatch);
+      if (step == 0)
+         vk_common_CmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+            VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
+            0, 1, &barrier, 0, NULL, 0, NULL);
+   }
+   return VK_SUCCESS;
+}
+
+static bool
+hk_native_encode_batch(VkCommandBuffer command_buffer,
+                        const struct vk_acceleration_structure_build_state *states,
+                        uint32_t count)
+{
+   if (!debug_get_bool_option("ASAHI_M5_AS_BATCH_ENCODE_PROBE", false))
+      return false;
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, command_buffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   unsigned builds = 0;
+   for (uint32_t i = 0; i < count; ++i)
+      builds += states[i].config.internal_type != VK_INTERNAL_BUILD_TYPE_UPDATE;
+   if (builds < 2)
+      return false;
+
+   /* Bounded chunks increase concurrency over the scalar path. The map's
+    * upper bound also bounds the eventual indirect X dimension without a
+    * CPU readback of GPU-produced active-node counts. */
+   const uint32_t max_builds = MIN2(1024u,
+      dev->vk.physical->properties.maxComputeWorkGroupCount[1]);
+   const uint32_t max_jobs = dev->vk.physical->properties.maxComputeWorkGroupCount[0];
+   struct hk_agx3_batch_job *entries = calloc(max_builds, sizeof(*entries));
+   if (!entries) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_HOST_MEMORY);
+      return true;
+   }
+   VkResult result = VK_SUCCESS;
+   for (unsigned kind = 0; kind < 2 && result == VK_SUCCESS; ++kind) {
+      uint32_t used = 0, map_bound = 0, max_bound = 0;
+      for (uint32_t i = 0; i <= count && result == VK_SUCCESS; ++i) {
+         const struct vk_acceleration_structure_build_state *s = i < count ? &states[i] : NULL;
+         VkGeometryTypeKHR type = s ? vk_get_as_geometry_type(s->build_info)
+                                   : VK_GEOMETRY_TYPE_TRIANGLES_KHR;
+         if (s && (s->config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE ||
+                   (type == VK_GEOMETRY_TYPE_INSTANCES_KHR) != (kind == 1)))
+            continue;
+         struct hk_agx3_as_layout native = {0};
+         if (s && !hk_agx3_as_layout_init(s->leaf_node_count, &native)) {
+            result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
+            break;
+         }
+         if (used && (!s || used == max_builds ||
+                      (uint64_t)map_bound + native.node_bound > max_jobs)) {
+            result = hk_cmd_encode_agx3_batch(command_buffer, entries, used,
+                                             map_bound, max_bound);
+            used = map_bound = max_bound = 0;
+            if (result != VK_SUCCESS)
+               break;
+         }
+         if (!s)
+            continue;
+         /* A single enormous tree keeps its already-qualified indirect path. */
+         if (native.node_bound > max_jobs) {
+            hk_encode_as(command_buffer, s);
+            continue;
+         }
+         if (s->build_info->flags & VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR) {
+            hk_agx3_refit_phase(command_buffer, s, 0);
+            hk_flush_buffer_write_cp(command_buffer);
+         }
+         VK_FROM_HANDLE(vk_acceleration_structure, dst,
+                        s->build_info->dstAccelerationStructure);
+         uint64_t scratch = s->build_info->scratchData.deviceAddress;
+         uint64_t encode = scratch + s->scratch.header_offset + sizeof(struct vk_ir_header);
+         struct hk_agx3_batch_job *entry = &entries[used++];
+         *entry = (struct hk_agx3_batch_job){
+            .plan = {
+               .intermediate_bvh = scratch + s->scratch.ir_offset,
+               .header = scratch + s->scratch.header_offset,
+               .nodes = encode + native.nodes, .jobs = encode + native.jobs,
+               .envelope = encode + native.envelope, .state = encode + native.state,
+               .output_bvh = hk_as_native_va(dst),
+               .root_offset = s->leaf_node_count * hk_agx3_ir_leaf_size(type),
+               .leaf_bound = s->leaf_node_count, .node_bound = native.node_bound,
+               .slot_bound = native.slot_bound, .resource_id = hk_m5_as_id(dev, dst),
+               .geometry_type = type,
+               .m5_aabb_stride = debug_get_num_option("ASAHI_M5_AABB_RECORD_STRIDE", 48) == 64 ? 64 : 48,
+            },
+            .node_status = encode + native.node_status,
+            .envelope_status = encode + native.envelope_status,
+            .single_leaf = false,
+         };
+         map_bound += native.node_bound;
+         max_bound = MAX2(max_bound, native.node_bound);
+      }
+   }
+   free(entries);
+   if (result != VK_SUCCESS)
+      vk_command_buffer_set_error(&cmd->vk, result);
+   return true;
+}
+
+/* The upstream runtime hands encoding every state of a batch at once, after
+ * the common builder finished the intermediate trees of the non-update
+ * states. Keep the M5 per-state prepare/encode/update sequence inside it. */
+static void
+hk_encode_dispatch(VkCommandBuffer commandBuffer, struct vk_device *device,
+                   struct vk_meta_device *meta,
+                   const struct vk_acceleration_structure_build_args *build_args,
+                   struct vk_acceleration_structure_build_state *states,
+                   uint32_t build_count, bool flushed_compute_after_init_update_scratch)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   bool prepared = false;
+
+   /* The common builder hands encoding the completed intermediate tree. */
+   vk_bvh_build_barrier_compute_to_compute(commandBuffer, false);
+   /* Opt-in batched native encode (ASAHI_M5_AS_BATCH_ENCODE_PROBE): when it
+    * handles every non-update state, only updates remain. */
+   const bool batched = hk_native_encode_batch(commandBuffer, states, build_count);
+   for (uint32_t i = 0; i < build_count; i++) {
+      if (batched && states[i].config.internal_type != VK_INTERNAL_BUILD_TYPE_UPDATE)
+         continue;
+      if (states[i].config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE) {
+         hk_update_prepare(commandBuffer, &states[i], true,
+                           flushed_compute_after_init_update_scratch);
+         hk_update_as(commandBuffer, &states[i]);
+         continue;
+      }
+      if (!prepared) {
+         VkResult result = hk_encode_prepare(commandBuffer, &states[i]);
+         if (result != VK_SUCCESS) {
+            vk_command_buffer_set_error(&cmd->vk, result);
+            return;
+         }
+         prepared = true;
+      }
+      hk_encode_as(commandBuffer, &states[i]);
+   }
+}
+
+struct hk_leaf_task {
+   struct leaf_args leaf;
+   uint32_t first;
+   uint32_t count;
+};
+
+static VkResult
+hk_build_leaves_batched(VkCommandBuffer commandBuffer, struct vk_device *device,
+                        struct vk_meta_device *meta,
+                        const struct vk_acceleration_structure_build_args *args,
+                        struct vk_acceleration_structure_build_state *states,
+                        uint32_t build_count, uint32_t build_flags)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   static_assert(sizeof(struct hk_leaf_task) == 88, "leaf task shader ABI");
+   static_assert(offsetof(struct hk_leaf_task, first) == 80, "leaf task first ABI");
+   assert(args->subgroup_size == 32);
+   uint64_t task_count = 0;
+   for (unsigned i = 0; i < build_count; ++i) {
+      if (states[i].config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE ||
+          (states[i].config.build_flags & VK_LEAF_BUILD_FLAGS) != build_flags)
+         continue;
+      for (unsigned j = 0; j < states[i].build_info->geometryCount; ++j) {
+         uint32_t count = states[i].build_range_infos[j].primitiveCount;
+         task_count += DIV_ROUND_UP((uint64_t)count, 32);
+      }
+   }
+   if (!task_count)
+      return VK_SUCCESS;
+   if (task_count > SIZE_MAX / sizeof(struct hk_leaf_task))
+      return VK_ERROR_OUT_OF_HOST_MEMORY;
+   if (task_count > UINT32_MAX / sizeof(struct hk_leaf_task))
+      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+   struct hk_leaf_task *tasks = malloc(task_count * sizeof(*tasks));
+   if (!tasks)
+      return VK_ERROR_OUT_OF_HOST_MEMORY;
+
+   VkPipeline pipeline;
+   VkPipelineLayout layout;
+   /* This callback replaces every leaf launch on this device, so the common
+    * leaf key remains unique, including all original build-flag variants. */
+   VkResult result = vk_get_bvh_build_pipeline_spv(device, meta,
+      VK_META_OBJECT_KEY_LEAF, agx3_batch_build_leaf_spv,
+      sizeof(agx3_batch_build_leaf_spv), sizeof(uint64_t), args, build_flags,
+      &pipeline, false);
+   if (result != VK_SUCCESS) {
+      free(tasks);
+      return result;
+   }
+   result = vk_get_bvh_build_pipeline_layout(device, meta, sizeof(uint64_t), &layout);
+   if (result != VK_SUCCESS) {
+      free(tasks);
+      return result;
+   }
+
+   uint64_t next = 0;
+   for (unsigned i = 0; i < build_count; ++i) {
+      if (states[i].config.internal_type == VK_INTERNAL_BUILD_TYPE_UPDATE ||
+          (states[i].config.build_flags & VK_LEAF_BUILD_FLAGS) != build_flags)
+         continue;
+      const VkAccelerationStructureBuildGeometryInfoKHR *info = states[i].build_info;
+      uint64_t scratch = info->scratchData.deviceAddress;
+      struct leaf_args leaf = {
+         .bvh = scratch + states[i].scratch.ir_offset,
+         .header = scratch + states[i].scratch.header_offset,
+         .ids = scratch + states[i].scratch.sort_buffer_offset[0],
+      };
+      for (unsigned j = 0; j < info->geometryCount; ++j) {
+         const VkAccelerationStructureBuildRangeInfoKHR *range = &states[i].build_range_infos[j];
+         if (!range->primitiveCount)
+            continue;
+         const VkAccelerationStructureGeometryKHR *geom = info->pGeometries
+            ? &info->pGeometries[j] : info->ppGeometries[j];
+         leaf.geom_data = vk_fill_geometry_data(info->type, states[i].leaf_node_count,
+                                                j, geom, range);
+         for (uint64_t first = 0; first < range->primitiveCount; first += 32)
+            tasks[next++] = (struct hk_leaf_task) {
+               .leaf = leaf, .first = first,
+               .count = MIN2(32, range->primitiveCount - first),
+            };
+         states[i].leaf_node_count += range->primitiveCount;
+      }
+   }
+   assert(next == task_count);
+   uint64_t gpu = hk_pool_upload(cmd, tasks, task_count * sizeof(*tasks), 8);
+   free(tasks);
+   if (!gpu)
+      return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+   struct vk_acceleration_structure_build_marker marker = {
+      .step = VK_ACCELERATION_STRUCTURE_BUILD_STEP_BUILD_LEAVES,
+   };
+   if (args->emit_markers)
+      device->as_build_ops->begin_debug_marker(commandBuffer, &marker);
+   device->dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
+   uint32_t limit = MIN2(device->physical->properties.maxComputeWorkGroupCount[0],
+                         UINT32_MAX / 32);
+   for (uint64_t first = 0; first < task_count;) {
+      uint32_t count = MIN2(task_count - first, limit);
+      uint64_t address = gpu + first * sizeof(struct hk_leaf_task);
+      device->dispatch_table.CmdPushConstants(commandBuffer, layout,
+         VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(address), &address);
+      device->cmd_dispatch_unaligned(commandBuffer, count * 32, 1, 1);
+      first += count;
+   }
+   if (args->emit_markers)
+      device->as_build_ops->end_debug_marker(commandBuffer, &marker);
+   return VK_SUCCESS;
+}
+
+static const struct vk_acceleration_structure_build_ops hk_native_batched_leaf_build_ops = {
+   .begin_debug_marker = hk_as_phase_begin,
+   .end_debug_marker = hk_as_phase_end,
+   .get_build_config = hk_native_build_config,
+   .get_as_size = hk_get_as_size,
+   .get_encode_scratch_size = hk_get_encode_scratch_size,
+   .get_update_scratch_size = hk_update_scratch_size,
+   .init_update_scratch = hk_init_update_scratch,
+   .encode = hk_encode_dispatch,
+   .build_leaves = hk_build_leaves_batched,
+};
+
+static const struct vk_acceleration_structure_build_ops hk_native_as_build_ops = {
+   .begin_debug_marker = hk_as_phase_begin,
+   .end_debug_marker = hk_as_phase_end,
+   .get_build_config = hk_native_build_config,
+   .get_as_size = hk_get_as_size,
+   .get_encode_scratch_size = hk_get_encode_scratch_size,
+   .get_update_scratch_size = hk_update_scratch_size,
+   .init_update_scratch = hk_init_update_scratch,
+   .encode = hk_encode_dispatch,
+};
+
+static const struct vk_acceleration_structure_build_ops hk_as_build_ops = {
+   .begin_debug_marker = hk_as_phase_begin,
+   .end_debug_marker = hk_as_phase_end,
+   .get_as_size = hk_get_as_size,
+   .get_encode_scratch_size = hk_get_encode_scratch_size,
+   .encode = hk_encode_dispatch,
+};
+
+
+/* ---------------------------------------------------------------- */
+/* vk_device hooks the common builder calls                          */
+
+static void
+hk_write_buffer_cp(VkCommandBuffer commandBuffer, VkDeviceAddress addr,
+                   void *data, uint32_t size)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+
+   struct {
+      uint64_t dst;
+      uint32_t count;
+      uint32_t data[28];
+   } args = {
+      .dst = addr,
+      .count = DIV_ROUND_UP(size, 4),
+   };
+
+   /* IR headers and the 112-byte driver empty-AS header fit this payload.
+    * Refuse larger writes rather than truncating initialized metadata. */
+   assert(size <= sizeof(args.data));
+   if (size > sizeof(args.data))
+      return;
+   memcpy(args.data, data, size);
+
+   VkPipeline pipeline;
+   VkPipelineLayout layout;
+   VkResult result = get_pipeline_spv(dev, "hk-bvh-write", write_spv,
+      sizeof(write_spv), sizeof(args), &pipeline, &layout);
+   if (result != VK_SUCCESS) {
+      vk_command_buffer_set_error(&cmd->vk, result);
+      return;
+   }
+
+   vk_common_CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE,
+                             pipeline);
+   vk_common_CmdPushConstants(commandBuffer, layout,
+                              VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(args),
+                              &args);
+   hk_cmd_dispatch_unaligned(commandBuffer, args.count, 1, 1);
+}
+
+static void
+hk_flush_buffer_write_cp(VkCommandBuffer commandBuffer)
+{
+   /* The common builder's initialization barrier has TOP_OF_PIPE as its
+    * source: that orders a CP write but cannot order our compute replacement.
+    * Publish the initialized header before build_leaves reads and atomically
+    * updates its bounds/counters. Keep the dispatches in the same CDM stream. */
+   const VkMemoryBarrier barrier = {
+      .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+      .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
+      .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT,
+   };
+   vk_common_CmdPipelineBarrier(commandBuffer,
+      VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+      0, 1, &barrier, 0, NULL, 0, NULL);
+}
+
+/* Explicitly opt-in build-phase witness, written to the test caller's mapped
+ * buffer. Ordered markers identify the last entered phase before a hang;
+ * they add work/cache flushes and must not be used by the production path. */
+static void
+hk_as_phase_write(VkCommandBuffer command_buffer,
+                  struct vk_acceleration_structure_build_marker *marker,
+                  bool end)
+{
+   const char *text = getenv("ASAHI_M5_AS_PHASE_ADDRESS");
+   if (!text)
+      return;
+   uint64_t address = strtoull(text, NULL, 0);
+   if (!address || (address & 3))
+      return;
+   hk_flush_buffer_write_cp(command_buffer);
+   struct vk_acceleration_structure_build_marker record = *marker;
+   if (end)
+      record.step |= 0x80000000u;
+   hk_write_buffer_cp(command_buffer, address, &record, sizeof(record));
+   hk_flush_buffer_write_cp(command_buffer);
+}
+
+static void
+hk_as_phase_begin(VkCommandBuffer command_buffer,
+                  struct vk_acceleration_structure_build_marker *marker)
+{
+   hk_as_phase_write(command_buffer, marker, false);
+}
+
+static void
+hk_as_phase_end(VkCommandBuffer command_buffer,
+                struct vk_acceleration_structure_build_marker *marker)
+{
+   hk_as_phase_write(command_buffer, marker, true);
+}
+
+static void
+hk_cmd_fill_buffer_addr(VkCommandBuffer commandBuffer, VkDeviceAddress addr,
+                        VkDeviceSize size, uint32_t data)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+
+   vk_meta_fill_buffer_addr(&cmd->vk, &dev->meta, addr, size, data);
+}
+
+/* ---------------------------------------------------------------- */
+/* Device state                                                      */
+
+/* Private integration profile. The external capture is an oracle while owned
+ * traversal bodies are developed; it is never embedded in the driver. */
+static bool
+hk_m5_rq_init_fail(struct hk_device *dev, const char *stage, int error)
+{
+   const int saved_errno = errno;
+   if (debug_get_bool_option("ASAHI_M5_DEVICE_INIT_DIAG", false))
+      fprintf(stderr, "M5_RQ_INIT pid=%ld dev=%p stage=%s errno=%d\n",
+              (long)getpid(), (void *)dev, stage, error);
+   errno = saved_errno;
+   return false;
+}
+
+/* Whether the query traversal files ASAHI_M5_RQ_CAPTURE_DIR names are
+ * readable in this process. A container that replaces /usr (Steam's runtime)
+ * may not see them; ray queries are then not offered rather than failing
+ * device creation. */
+bool
+hk_m5_rq_assets_available(void)
+{
+   static const char *const names[] = {
+      "driver-rebase-b3.bin", "user-image.bin", "uniforms.bin",
+   };
+   const char *dir = getenv("ASAHI_M5_RQ_CAPTURE_DIR");
+   if (!dir)
+      return false;
+   for (unsigned i = 0; i < ARRAY_SIZE(names); ++i) {
+      char path[4096];
+      if (snprintf(path, sizeof(path), "%s/%s", dir, names[i]) >= sizeof(path) ||
+          access(path, R_OK))
+         return false;
+   }
+   return true;
+}
+
+static bool
+hk_m5_rq_read(const char *dir, const char *name, void *data, size_t bytes)
+{
+   char path[4096];
+   if (snprintf(path, sizeof(path), "%s/%s", dir, name) >= sizeof(path))
+      return false;
+   FILE *f = fopen(path, "rb");
+   if (!f) {
+      const int error = errno;
+      if (debug_get_bool_option("ASAHI_M5_DEVICE_INIT_DIAG", false)) {
+         fprintf(stderr, "M5_RQ_READ pid=%ld file=%s stage=open errno=%d\n",
+                 (long)getpid(), name, error);
+      }
+      errno = error;
+      return false;
+   }
+   bool ok = fread(data, 1, bytes, f) == bytes && fgetc(f) == EOF;
+   const int read_errno = errno;
+   if (!ok && debug_get_bool_option("ASAHI_M5_DEVICE_INIT_DIAG", false)) {
+      const int error = ferror(f) ? read_errno : 0;
+      const int saved_errno = read_errno;
+      fprintf(stderr, "M5_RQ_READ pid=%ld file=%s stage=exact-size errno=%d ferror=%d feof=%d\n",
+              (long)getpid(), name, error, ferror(f), feof(f));
+      errno = saved_errno;
+   }
+   if (!ok)
+      errno = read_errno;
+   fclose(f);
+   return ok;
+}
+
+/* Explicit diagnostic only. A native IFT program store has not been qualified
+ * on hardware. The exact captured call setup overwrites halves0..3 before the
+ * native body executes; no additional caller registers or uniforms are used. */
+static bool
+hk_m5_ift_diag_setup(struct hk_device *dev)
+{
+   const char *dump = getenv("ASAHI_M5_RQ_IFT_DIAG_DUMP");
+   if (!dump || !dump[0])
+      return true;
+   dev->m5_rq_diag_dump = strdup(dump);
+   dev->m5_rq_diag = agx_bo_create(&dev->dev, 0x4000, 0x4000,
+                                 AGX_BO_WRITEBACK, "M5 IFT diagnostic bitmap");
+   if (!dev->m5_rq_diag_dump || !dev->m5_rq_diag)
+      return false;
+   memset(agx_bo_map(dev->m5_rq_diag), 0, 0x4000);
+   const uint64_t address = dev->m5_rq_diag->va->addr;
+   /* Each callback stores its slotAddressLow at its inventory slot. Repeated writers
+    * store the same word. This observes entry execution, not callback counts
+    * or individual lanes. Require a nonzero marker and a native48-bit VA. */
+   if (!(uint32_t)address || (address >> 48))
+      return false;
+   uint8_t *code = agx_bo_map(dev->m5_rq_code);
+   /* Verify every original before writing any entry. The inventory also
+    * qualifies the full GPR0..3 overwrite and entry-specific prefix padding. */
+   for (unsigned i = 0; i < ARRAY_SIZE(hk_m5_ift_diag_entries); ++i) {
+      const struct hk_m5_ift_diag_entry *e = &hk_m5_ift_diag_entries[i];
+      if (memcmp(code + e->offset, hk_m5_ift_diag_original + e->original,
+                 e->bytes))
+         return false;
+      for (unsigned j = 0; j < e->max_prefix; ++j) {
+         if (code[e->offset + e->bytes + j])
+            return false;
+      }
+   }
+   uint8_t prefixes[ARRAY_SIZE(hk_m5_ift_diag_entries)][32];
+   uint8_t lengths[ARRAY_SIZE(hk_m5_ift_diag_entries)];
+   for (unsigned i = 0; i < ARRAY_SIZE(hk_m5_ift_diag_entries); ++i) {
+      const struct hk_m5_ift_diag_entry *e = &hk_m5_ift_diag_entries[i];
+      uint8_t *prefix = prefixes[i], bytes;
+      const uint64_t marker_address = address + e->slot * 4;
+      size_t length = 0;
+      struct agx3_encoded_ldimm imm;
+      if (!agx3_pack_ldimm_u32(AGX_ISA_ENCODING_AGX3C, 0, marker_address, &imm))
+         return false;
+      memcpy(prefix + length, imm.bytes, imm.length);
+      length += imm.length;
+      if (!agx3_pack_ldimm_u32(AGX_ISA_ENCODING_AGX3C, 2, marker_address >> 32, &imm))
+         return false;
+      memcpy(prefix + length, imm.bytes, imm.length);
+      length += imm.length;
+      const struct agx3_mem_desc store = {
+         .data = 0, .width = AGX3_MEM_W32, .mask = 1, .base = 0,
+         .idx_kind = AGX3_MEM_IDX_IMM, .byte_off = 0,
+      };
+      if (!agx3_mem_encode_store(AGX_ISA_ENCODING_AGX3C, &store,
+                                prefix + length, &bytes))
+         return false;
+      length += bytes;
+      if (length > e->max_prefix)
+         return false;
+      lengths[i] = length;
+   }
+   for (unsigned i = 0; i < ARRAY_SIZE(hk_m5_ift_diag_entries); ++i) {
+      const struct hk_m5_ift_diag_entry *e = &hk_m5_ift_diag_entries[i];
+      memmove(code + e->offset + lengths[i], code + e->offset, e->bytes);
+      memcpy(code + e->offset, prefixes[i], lengths[i]);
+   }
+   return true;
+}
+
+static void
+hk_m5_ift_diag_finish(struct hk_device *dev)
+{
+   if (!dev->m5_rq_diag || !dev->m5_rq_diag_dump)
+      return;
+   /* The caller must have completed its work before destroying the device.
+    * The diagnostic fixture performs vkDeviceWaitIdle before VkDestroyDevice. */
+   FILE *f = fopen(dev->m5_rq_diag_dump, "wb");
+   if (f) {
+      fwrite(agx_bo_map(dev->m5_rq_diag), 1, 0x4000, f);
+      fclose(f);
+   }
+   char path[4096];
+   int n = snprintf(path, sizeof(path), "%s.patched-driver.bin", dev->m5_rq_diag_dump);
+   if (n >= 0 && (size_t)n < sizeof(path)) {
+      f = fopen(path, "wb");
+      if (f) {
+         fwrite(agx_bo_map(dev->m5_rq_code), 1, 0x800000, f);
+         fclose(f);
+      }
+   }
+   n = snprintf(path, sizeof(path), "%s.address.txt", dev->m5_rq_diag_dump);
+   if (n >= 0 && (size_t)n < sizeof(path)) {
+      f = fopen(path, "w");
+      if (f) {
+         fprintf(f, "0x%llx\n", (unsigned long long)dev->m5_rq_diag->va->addr);
+         fclose(f);
+      }
+   }
+}
+
+static bool
+hk_m5_rq_init(struct hk_device *dev, const char *dir)
+{
+   dev->m5_rq_instances = debug_get_bool_option("ASAHI_M5_RQ_INSTANCES", false);
+   const uint64_t alias = UINT64_C(0x10010000000);
+   dev->m5_rq_code_alias = agx_va_alloc(&dev->dev, 0x800000, 0x4000,
+                                        AGX_VA_USC | AGX_VA_FIXED, alias);
+   if (!dev->m5_rq_code_alias)
+      return hk_m5_rq_init_fail(dev, "fixed-code-alias", 0);
+   dev->m5_rq_code = agx_bo_create(&dev->dev, 0x800000, 0x4000,
+                                    AGX_BO_EXEC | AGX_BO_WRITEBACK, "M5 query oracle");
+   dev->m5_rq_ift = agx_bo_create(&dev->dev, 0x8000, 0x4000,
+                                   AGX_BO_WRITEBACK, "M5 intersection table");
+   dev->m5_rq_user_image = malloc(0x800000);
+   const int image_error = dev->m5_rq_user_image ? 0 : errno;
+   if (!dev->m5_rq_code)
+      return hk_m5_rq_init_fail(dev, "code-bo", 0);
+   if (!dev->m5_rq_ift)
+      return hk_m5_rq_init_fail(dev, "ift-bo", 0);
+   if (!dev->m5_rq_user_image)
+      return hk_m5_rq_init_fail(dev, "user-image-allocation", image_error);
+   if (!hk_m5_rq_read(dir, "driver-rebase-b3.bin", agx_bo_map(dev->m5_rq_code), 0x800000) ||
+       !hk_m5_rq_read(dir, "user-image.bin", dev->m5_rq_user_image, 0x800000) ||
+       !hk_m5_rq_read(dir, "uniforms.bin", dev->m5_rq_uniforms, 296))
+      return hk_m5_rq_init_fail(dev, "oracle-read", 0);
+   if (dev->m5_rq_instances && !hk_m5_rq_read(dir, "instance-uniforms.bin", dev->m5_rq_uniforms, sizeof(dev->m5_rq_uniforms)))
+      return hk_m5_rq_init_fail(dev, "instance-uniforms-read", 0);
+   if (!hk_m5_ift_diag_setup(dev))
+      return hk_m5_rq_init_fail(dev, "ift-diagnostic-setup", 0);
+   if (agx_bo_bind(&dev->dev, dev->m5_rq_code, alias, 0x800000, 0,
+                   DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_WRITE))
+      return hk_m5_rq_init_fail(dev, "code-alias-bind", 0);
+   uint8_t *table = agx_bo_map(dev->m5_rq_ift);
+   memset(table, 0, 0x8000);
+   memcpy(table + 128, (uint8_t *)dev->m5_rq_user_image + 0x558080, 112 * 128);
+   for (unsigned i = 0; i < 112 * 8; ++i) {
+      uint64_t lo; memcpy(&lo, table + 128 + 16*i, 8);
+      uint64_t address = lo & UINT64_C(0xffffffffffff);
+      if (!address) continue;
+      if (address < UINT64_C(0x1000000000) || address >= UINT64_C(0x1000800000))
+         return hk_m5_rq_init_fail(dev, "ift-address-range", 0);
+      lo = (lo & UINT64_C(0xffff000000000000)) | (alias + address - UINT64_C(0x1000000000));
+      memcpy(table + 128 + 16*i, &lo, 8);
+   }
+   dev->m5_rq_default_program = alias + 0x4d8800;
+   dev->m5_rq_default_control = UINT64_C(0x16000000000);
+   dev->m5_rq_instances = debug_get_bool_option("ASAHI_M5_RQ_INSTANCES", false);
+   dev->m5_native_rq = true;
+   return true;
+}
+
+VkResult
+hk_m5_backend_device_init_accel_struct_state(struct hk_device *dev)
+{
+   dev->m5_native_as = agx_chip_g17_m5(dev->dev.chip) &&
+      debug_get_bool_option("ASAHI_M5_NATIVE_AS", false);
+   dev->accel_struct_args = (struct vk_acceleration_structure_build_args){
+      .subgroup_size = 32,
+      .bvh_bounds_offset = dev->m5_native_as ? 0x50 :
+         offsetof(struct hk_bvh_header, bounds),
+      /* The common builder's spec_map gives BUILD_FLAGS_ID and
+       * ROOT_FLAGS_OFFSET_ID the same push-constant offset, so root_flags
+       * would receive the build flags.  It is only read when cull-flag
+       * propagation is on, so leave that off until that is fixed upstream. */
+      .root_flags_offset = 0,
+      .morton_sort_workgroup_size = 512,
+      .morton_sort_kvs_per_thread = 2,
+      .propagate_cull_flags = false,
+      /* Private progress probe adds dispatches and flushes: never default.
+       * Its performance cost is unmeasured and it is not a root-cause fix. */
+      .emit_markers = debug_get_bool_option("ASAHI_M5_AS_PHASE_DIAG", false),
+   };
+
+   dev->vk.as_build_ops = !dev->m5_native_as ? &hk_as_build_ops :
+      debug_get_bool_option("ASAHI_M5_BATCH_LEAVES", true)
+         ? &hk_native_batched_leaf_build_ops : &hk_native_as_build_ops;
+   dev->vk.write_buffer_cp = hk_write_buffer_cp;
+   dev->vk.flush_buffer_write_cp = hk_flush_buffer_write_cp;
+   dev->vk.cmd_dispatch_unaligned = hk_cmd_dispatch_unaligned;
+   dev->vk.cmd_fill_buffer_addr = hk_cmd_fill_buffer_addr;
+
+   simple_mtx_init(&dev->m5_as_lock, mtx_plain);
+   util_dynarray_init(&dev->m5_ases, NULL);
+   util_dynarray_init(&dev->m5_as_table_pages, NULL);
+   util_dynarray_init(&dev->m5_pending_as_ids, NULL);
+   if (dev->m5_native_as) {
+      dev->m5_as_table_va = agx_va_alloc(&dev->dev, HK_M5_AS_TABLE_SIZE,
+                                       HK_M5_AS_TABLE_PAGE_SIZE, 0, 0);
+      if (!dev->m5_as_table_va || !hk_m5_as_table_entry(dev, 0)) {
+         hk_device_finish_accel_struct_state(dev);
+         return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+      }
+   }
+   /* Only a device that enables ray queries or ray tracing pipelines runs
+    * the query traversal code, so only such a device loads it. */
+   const char *capture = getenv("ASAHI_M5_RQ_CAPTURE_DIR");
+   const bool queries = dev->vk.enabled_features.rayQuery ||
+                        dev->vk.enabled_features.rayTracingPipeline;
+   if (dev->m5_native_as && capture && queries &&
+       !hk_m5_rq_init(dev, capture)) {
+      hk_device_finish_accel_struct_state(dev);
+      return VK_ERROR_INITIALIZATION_FAILED;
+   }
+   return VK_SUCCESS;
+}
+
+void
+hk_device_finish_accel_struct_state(struct hk_device *dev)
+{
+   if (!agx_chip_g17_m5(dev->dev.chip))
+      return;
+   hk_m5_ift_diag_finish(dev);
+   if (dev->m5_rq_code_alias) {
+      agx_bo_bind(&dev->dev, NULL, dev->m5_rq_code_alias->addr, 0x800000, 0,
+                  DRM_ASAHI_BIND_UNBIND);
+      agx_va_free(&dev->dev, dev->m5_rq_code_alias, false);
+   }
+   agx_bo_unreference(&dev->dev, dev->m5_rq_code);
+   agx_bo_unreference(&dev->dev, dev->m5_rq_ift);
+   agx_bo_unreference(&dev->dev, dev->m5_rq_diag);
+   agx_bo_unreference(&dev->dev, dev->m5_rt_state_bo);
+   free(dev->m5_rq_diag_dump);
+   free(dev->m5_rq_user_image);
+   bool table_unmapped = true;
+   unsigned page = 0;
+   util_dynarray_foreach(&dev->m5_as_table_pages, struct agx_bo *, bo) {
+      if (agx_bo_bind(&dev->dev, NULL, dev->m5_as_table_va->addr +
+                      (uint64_t)page * HK_M5_AS_TABLE_PAGE_SIZE,
+                      HK_M5_AS_TABLE_PAGE_SIZE, 0, DRM_ASAHI_BIND_UNBIND))
+         table_unmapped = false;
+      agx_bo_unreference(&dev->dev, *bo);
+      ++page;
+   }
+   /* Never recycle the reservation if an alias failed to unbind. */
+   if (table_unmapped)
+      agx_va_free(&dev->dev, dev->m5_as_table_va, false);
+   util_dynarray_fini(&dev->m5_pending_as_ids);
+   util_dynarray_fini(&dev->m5_as_table_pages);
+   util_dynarray_fini(&dev->m5_ases);
+   simple_mtx_destroy(&dev->m5_as_lock);
+}
+
+/* ---------------------------------------------------------------- */
+/* Entry points                                                      */
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_GetAccelerationStructureBuildSizesKHR(
+   VkDevice _device, VkAccelerationStructureBuildTypeKHR buildType,
+   const VkAccelerationStructureBuildGeometryInfoKHR *pBuildInfo,
+   const uint32_t *pMaxPrimitiveCounts,
+   VkAccelerationStructureBuildSizesInfoKHR *pSizeInfo)
+{
+   VK_FROM_HANDLE(hk_device, dev, _device);
+
+   if (dev->m5_native_as) {
+      uint64_t leaves = 0;
+      for (uint32_t i = 0; i < pBuildInfo->geometryCount; ++i)
+         leaves += pMaxPrimitiveCounts[i];
+      if (!leaves) {
+         struct hk_agx3_as_layout layout;
+         hk_agx3_as_layout_init(0, &layout);
+         pSizeInfo->accelerationStructureSize = layout.as_bytes;
+         pSizeInfo->buildScratchSize = 0;
+         pSizeInfo->updateScratchSize = 0;
+         return;
+      }
+   }
+
+   vk_get_as_build_sizes(_device, buildType, pBuildInfo, pMaxPrimitiveCounts,
+                         pSizeInfo, &dev->accel_struct_args);
+}
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdBuildAccelerationStructuresKHR(
+   VkCommandBuffer commandBuffer, uint32_t infoCount,
+   const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
+   const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+
+   uint32_t empty_count = 0;
+   if (dev->m5_native_as) {
+      for (uint32_t i = 0; i < infoCount; ++i) {
+         const VkAccelerationStructureGeometryKHR *geometry =
+            pInfos[i].geometryCount
+               ? (pInfos[i].pGeometries ? pInfos[i].pGeometries :
+                                        pInfos[i].ppGeometries[0]) : NULL;
+         uint64_t primitives = 0;
+         bool compatible = geometry != NULL ||
+            (pInfos[i].type == VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR &&
+             pInfos[i].geometryCount == 0);
+         for (uint32_t g = 0; g < pInfos[i].geometryCount; ++g) {
+            const VkAccelerationStructureGeometryKHR *next =
+               pInfos[i].pGeometries ? &pInfos[i].pGeometries[g] :
+                                      pInfos[i].ppGeometries[g];
+            compatible &= next->geometryType == geometry->geometryType;
+            primitives += ppBuildRangeInfos[i][g].primitiveCount;
+         }
+         struct hk_agx3_as_layout layout;
+         /* Bottom-level structures hold triangles or procedural AABBs; the
+          * native encoder emits the captured record for either. */
+         bool triangles =
+            pInfos[i].type == VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR &&
+            (!geometry || geometry->geometryType == VK_GEOMETRY_TYPE_TRIANGLES_KHR ||
+             geometry->geometryType == VK_GEOMETRY_TYPE_AABBS_KHR);
+         bool instances = geometry && pInfos[i].geometryCount == 1 &&
+            pInfos[i].type == VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR &&
+            geometry->geometryType == VK_GEOMETRY_TYPE_INSTANCES_KHR;
+         if ((pInfos[i].mode != VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR &&
+              pInfos[i].mode != VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR) ||
+             !compatible || (!triangles && !instances) || primitives > UINT32_MAX ||
+             !hk_agx3_as_layout_init(primitives, &layout)) {
+            if (debug_get_bool_option("ASAHI_M5_LAUNCH_DEBUG", false))
+               fprintf(stderr, "M5 AS build refused: type=%u mode=%u geometries=%u geometry=%u flags=%x count=%u\n",
+                       pInfos[i].type, pInfos[i].mode, pInfos[i].geometryCount,
+                       geometry ? geometry->geometryType : ~0u,
+                       geometry ? geometry->flags : 0,
+                       ppBuildRangeInfos[i][0].primitiveCount);
+            vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+            return;
+         }
+         empty_count += primitives == 0;
+      }
+   }
+
+   const uint32_t nonempty_count = infoCount - empty_count;
+   VkAccelerationStructureBuildGeometryInfoKHR *nonempty_infos = NULL;
+   const VkAccelerationStructureBuildRangeInfoKHR **nonempty_ranges = NULL;
+   if (empty_count && nonempty_count) {
+      nonempty_infos = malloc(nonempty_count * sizeof(*nonempty_infos));
+      nonempty_ranges = malloc(nonempty_count * sizeof(*nonempty_ranges));
+      if (!nonempty_infos || !nonempty_ranges) {
+         free(nonempty_infos);
+         free(nonempty_ranges);
+         vk_command_buffer_set_error(&cmd->vk, VK_ERROR_OUT_OF_HOST_MEMORY);
+         return;
+      }
+   }
+   /* A build binds its own compute pipelines and push constants, so the
+    * application's compute state has to survive it. */
+   struct hk_meta_save save;
+   hk_meta_begin(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
+
+   if (empty_count) {
+      uint32_t n = 0;
+      for (uint32_t i = 0; i < infoCount; ++i) {
+         uint64_t primitives = 0;
+         for (uint32_t g = 0; g < pInfos[i].geometryCount; ++g)
+            primitives += ppBuildRangeInfos[i][g].primitiveCount;
+         if (!primitives) {
+            VK_FROM_HANDLE(vk_acceleration_structure, dst,
+                           pInfos[i].dstAccelerationStructure);
+            uint32_t header[28];
+            hk_agx3_as_empty_header(header);
+            hk_write_buffer_cp(commandBuffer, hk_as_native_va(dst) + 64,
+                               header, sizeof(header));
+         } else if (nonempty_infos) {
+            nonempty_infos[n] = pInfos[i];
+            nonempty_ranges[n++] = ppBuildRangeInfos[i];
+         }
+      }
+      hk_flush_buffer_write_cp(commandBuffer);
+   }
+   /* Keep all remaining structures in one common build batch. */
+   if (nonempty_count)
+      vk_cmd_build_acceleration_structures(commandBuffer, &dev->vk, &dev->meta,
+         nonempty_count, nonempty_infos ? nonempty_infos : pInfos,
+         nonempty_ranges ? nonempty_ranges : ppBuildRangeInfos,
+         &dev->accel_struct_args);
+
+   hk_meta_end(cmd, &save, VK_PIPELINE_BIND_POINT_COMPUTE);
+   free(nonempty_infos);
+   free(nonempty_ranges);
+}
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_GetDeviceAccelerationStructureCompatibilityKHR(
+   VkDevice _device, const VkAccelerationStructureVersionInfoKHR *pVersionInfo,
+   VkAccelerationStructureCompatibilityKHR *pCompatibility)
+{
+   VK_FROM_HANDLE(hk_device, dev, _device);
+   struct hk_physical_device *pdev = hk_device_physical(dev);
+
+   bool compat = memcmp(pVersionInfo->pVersionData,
+                        pdev->vk.properties.driverUUID, VK_UUID_SIZE) == 0;
+   *pCompatibility = compat
+                        ? VK_ACCELERATION_STRUCTURE_COMPATIBILITY_COMPATIBLE_KHR
+                        : VK_ACCELERATION_STRUCTURE_COMPATIBILITY_INCOMPATIBLE_KHR;
+}
+
+/* The native image is smaller than its Vulkan allocation: the latter also
+ * reserves the prefix needed to align the root to 1KiB. Launch for the
+ * allocation's upper bound and let the GPU header bound the actual copy.
+ * This keeps build -> copy -> trace entirely on the GPU, including builds
+ * recorded earlier in this same command buffer. */
+static uint32_t
+hk_as_copy_threads(struct hk_cmd_buffer *cmd,
+                   struct vk_acceleration_structure *as)
+{
+   const uint64_t padding = hk_as_native_va(as) -
+                            vk_acceleration_structure_get_va(as);
+   if (padding > as->size || as->size - padding < 192 ||
+       DIV_ROUND_UP(as->size - padding, 16) > UINT32_MAX) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return 0;
+   }
+   return DIV_ROUND_UP(as->size - padding, 16);
+}
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdCopyAccelerationStructureKHR(
+   VkCommandBuffer commandBuffer, const VkCopyAccelerationStructureInfoKHR *info)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   VK_FROM_HANDLE(vk_acceleration_structure, src, info->src);
+   VK_FROM_HANDLE(vk_acceleration_structure, dst, info->dst);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!dev->m5_native_as ||
+       (info->mode != VK_COPY_ACCELERATION_STRUCTURE_MODE_CLONE_KHR &&
+        info->mode != VK_COPY_ACCELERATION_STRUCTURE_MODE_COMPACT_KHR)) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   uint32_t threads = hk_as_copy_threads(cmd, src);
+   if (!threads)
+      return;
+   libagx_agx3_as_copy(cmd, agx_1d(threads), AGX_BARRIER_ALL,
+                       hk_as_native_va(dst), hk_as_native_va(src),
+                       hk_m5_as_id(dev, dst));
+}
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdCopyAccelerationStructureToMemoryKHR(
+   VkCommandBuffer commandBuffer,
+   const VkCopyAccelerationStructureToMemoryInfoKHR *info)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   VK_FROM_HANDLE(vk_acceleration_structure, src, info->src);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!dev->m5_native_as ||
+       info->mode != VK_COPY_ACCELERATION_STRUCTURE_MODE_SERIALIZE_KHR) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   uint32_t threads = hk_as_copy_threads(cmd, src);
+   if (!threads)
+      return;
+   uint32_t uuid[4];
+   memcpy(uuid, hk_device_physical(dev)->vk.properties.driverUUID, sizeof(uuid));
+   libagx_agx3_as_serialize(cmd, agx_1d(threads), AGX_BARRIER_ALL,
+                            info->dst.deviceAddress, hk_as_native_va(src),
+                            uuid[0], uuid[1], uuid[2], uuid[3]);
+}
+
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdCopyMemoryToAccelerationStructureKHR(
+   VkCommandBuffer commandBuffer,
+   const VkCopyMemoryToAccelerationStructureInfoKHR *info)
+{
+   VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
+   VK_FROM_HANDLE(vk_acceleration_structure, dst, info->dst);
+   struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (!dev->m5_native_as ||
+       info->mode != VK_COPY_ACCELERATION_STRUCTURE_MODE_DESERIALIZE_KHR) {
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   uint32_t threads = hk_as_copy_threads(cmd, dst);
+   if (!threads)
+      return;
+   unsigned count;
+   uint64_t table = hk_m5_upload_as_table_count(cmd, &count);
+   if (!table)
+      return;
+   libagx_agx3_as_deserialize(cmd, agx_1d(threads), AGX_BARRIER_ALL,
+                              hk_as_native_va(dst), info->src.deviceAddress,
+                              hk_m5_as_id(dev, dst));
+   /* IDs are assigned at creation. Vulkan permits the replacement BLAS to be
+    * unbuilt here, so resolve its address through the uploaded object table,
+    * rather than reading a header which a later build will initialize. */
+   libagx_agx3_as_relink(cmd, agx_1d(threads), AGX_BARRIER_ALL,
+                         hk_as_native_va(dst), info->src.deviceAddress,
+                         table, count);
+}
+
+/*
+ * Host-side builds and copies are unimplemented and unadvertised:
+ * accelerationStructureHostCommands
+ * stays false, so a conformant application never reaches them.  They fail
+ * loudly rather than silently returning success on an untouched structure.
+ */
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_BuildAccelerationStructuresKHR(
+   VkDevice _device, VkDeferredOperationKHR deferredOperation,
+   uint32_t infoCount,
+   const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
+   const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos)
+{
+   return VK_ERROR_FEATURE_NOT_PRESENT;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CopyAccelerationStructureKHR(VkDevice _device,
+                                VkDeferredOperationKHR deferredOperation,
+                                const VkCopyAccelerationStructureInfoKHR *pInfo)
+{
+   return VK_ERROR_FEATURE_NOT_PRESENT;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CopyMemoryToAccelerationStructureKHR(
+   VkDevice _device, VkDeferredOperationKHR deferredOperation,
+   const VkCopyMemoryToAccelerationStructureInfoKHR *pInfo)
+{
+   return VK_ERROR_FEATURE_NOT_PRESENT;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CopyAccelerationStructureToMemoryKHR(
+   VkDevice _device, VkDeferredOperationKHR deferredOperation,
+   const VkCopyAccelerationStructureToMemoryInfoKHR *pInfo)
+{
+   return VK_ERROR_FEATURE_NOT_PRESENT;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_WriteAccelerationStructuresPropertiesKHR(
+   VkDevice _device, uint32_t accelerationStructureCount,
+   const VkAccelerationStructureKHR *pAccelerationStructures,
+   VkQueryType queryType, size_t dataSize, void *pData, size_t stride)
+{
+   return VK_ERROR_FEATURE_NOT_PRESENT;
+}
diff --git a/src/asahi/vulkan/hk_m5_acceleration_structure.h b/src/asahi/vulkan/hk_m5_acceleration_structure.h
new file mode 100644
index 000000000000..5022091038fb
--- /dev/null
+++ b/src/asahi/vulkan/hk_m5_acceleration_structure.h
@@ -0,0 +1,77 @@
+#pragma once
+#include "hk_acceleration_structure.h"
+
+bool hk_m5_rq_assets_available(void);
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_BuildAccelerationStructuresKHR(
+   VkDevice _device, VkDeferredOperationKHR deferredOperation,
+   uint32_t infoCount,
+   const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
+   const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdBuildAccelerationStructuresKHR(
+   VkCommandBuffer commandBuffer, uint32_t infoCount,
+   const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
+   const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdCopyAccelerationStructureKHR(
+   VkCommandBuffer commandBuffer, const VkCopyAccelerationStructureInfoKHR *info);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdCopyAccelerationStructureToMemoryKHR(
+   VkCommandBuffer commandBuffer,
+   const VkCopyAccelerationStructureToMemoryInfoKHR *info);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_CmdCopyMemoryToAccelerationStructureKHR(
+   VkCommandBuffer commandBuffer,
+   const VkCopyMemoryToAccelerationStructureInfoKHR *info);
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CopyAccelerationStructureKHR(VkDevice _device,
+                                VkDeferredOperationKHR deferredOperation,
+                                const VkCopyAccelerationStructureInfoKHR *pInfo);
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CopyAccelerationStructureToMemoryKHR(
+   VkDevice _device, VkDeferredOperationKHR deferredOperation,
+   const VkCopyAccelerationStructureToMemoryInfoKHR *pInfo);
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CopyMemoryToAccelerationStructureKHR(
+   VkDevice _device, VkDeferredOperationKHR deferredOperation,
+   const VkCopyMemoryToAccelerationStructureInfoKHR *pInfo);
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_CreateAccelerationStructureKHR(VkDevice device,
+   const VkAccelerationStructureCreateInfoKHR *info,
+   const VkAllocationCallbacks *alloc, VkAccelerationStructureKHR *out);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_DestroyAccelerationStructureKHR(VkDevice device, VkAccelerationStructureKHR handle,
+                                   const VkAllocationCallbacks *alloc);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_GetAccelerationStructureBuildSizesKHR(
+   VkDevice _device, VkAccelerationStructureBuildTypeKHR buildType,
+   const VkAccelerationStructureBuildGeometryInfoKHR *pBuildInfo,
+   const uint32_t *pMaxPrimitiveCounts,
+   VkAccelerationStructureBuildSizesInfoKHR *pSizeInfo);
+VKAPI_ATTR VkDeviceAddress VKAPI_CALL
+hk_m5_backend_GetAccelerationStructureDeviceAddressKHR(
+   VkDevice _device, const VkAccelerationStructureDeviceAddressInfoKHR *info);
+VKAPI_ATTR void VKAPI_CALL
+hk_m5_backend_GetDeviceAccelerationStructureCompatibilityKHR(
+   VkDevice _device, const VkAccelerationStructureVersionInfoKHR *pVersionInfo,
+   VkAccelerationStructureCompatibilityKHR *pCompatibility);
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_m5_backend_WriteAccelerationStructuresPropertiesKHR(
+   VkDevice _device, uint32_t accelerationStructureCount,
+   const VkAccelerationStructureKHR *pAccelerationStructures,
+   VkQueryType queryType, size_t dataSize, void *pData, size_t stride);
+VkResult
+hk_m5_backend_cmd_build_agx3_from_ir(VkCommandBuffer command_buffer,
+                        const struct hk_agx3_plan_args *args,
+                        VkDeviceAddress node_status,
+                        VkDeviceAddress envelope_status);
+VkResult
+hk_m5_backend_cmd_encode_agx3_nodes(VkCommandBuffer command_buffer,
+                        const struct hk_agx3_leaf_args *args);
+VkResult
+hk_m5_backend_cmd_finalize_agx3_as(VkCommandBuffer command_buffer,
+                       const struct hk_agx3_envelope_args *args);
+VkResult
+hk_m5_backend_device_init_accel_struct_state(struct hk_device *dev);
diff --git a/src/asahi/vulkan/hk_m5_ift_diag_inventory.h b/src/asahi/vulkan/hk_m5_ift_diag_inventory.h
new file mode 100644
index 000000000000..b1716fedadef
--- /dev/null
+++ b/src/asahi/vulkan/hk_m5_ift_diag_inventory.h
@@ -0,0 +1,3215 @@
+/* Private captured-loader diagnostic inventory. Exact c2500589 oracle. */
+struct hk_m5_ift_diag_entry { uint32_t offset, bytes, original, slot, max_prefix; };
+static const struct hk_m5_ift_diag_entry hk_m5_ift_diag_entries[] = {
+   {0x4d8800, 180, 0, 832, 24},
+   {0x4d8900, 196, 180, 0, 24},
+   {0x4d8a00, 180, 376, 768, 24},
+   {0x4d8b00, 220, 556, 1, 24},
+   {0x4d8c00, 180, 776, 769, 24},
+   {0x4d8d00, 196, 956, 2, 24},
+   {0x4d8e00, 180, 1152, 770, 24},
+   {0x4d8f00, 220, 1332, 3, 24},
+   {0x4d9000, 180, 1552, 771, 24},
+   {0x4d9100, 228, 1732, 4, 24},
+   {0x4d9200, 180, 1960, 772, 24},
+   {0x4d9300, 220, 2140, 5, 24},
+   {0x4d9400, 180, 2360, 773, 24},
+   {0x4d9500, 196, 2540, 512, 24},
+   {0x4d9600, 204, 2736, 513, 24},
+   {0x4d9700, 324, 2940, 64, 24},
+   {0x4d9880, 324, 3264, 65, 24},
+   {0x4d9a00, 220, 3588, 8, 24},
+   {0x4d9b00, 180, 3808, 776, 24},
+   {0x4d9c00, 220, 3988, 9, 24},
+   {0x4d9d00, 180, 4208, 777, 24},
+   {0x4d9e00, 220, 4388, 10, 24},
+   {0x4d9f00, 180, 4608, 778, 24},
+   {0x4da000, 220, 4788, 11, 24},
+   {0x4da100, 180, 5008, 779, 24},
+   {0x4da200, 228, 5188, 12, 24},
+   {0x4da300, 180, 5416, 780, 24},
+   {0x4da400, 220, 5596, 13, 24},
+   {0x4da500, 180, 5816, 781, 24},
+   {0x4da600, 196, 5996, 520, 24},
+   {0x4da700, 228, 6192, 521, 24},
+   {0x4da800, 316, 6420, 72, 24},
+   {0x4da980, 316, 6736, 73, 24},
+   {0x4dab00, 220, 7052, 16, 24},
+   {0x4dac00, 180, 7272, 784, 24},
+   {0x4dad00, 220, 7452, 17, 24},
+   {0x4dae00, 180, 7672, 785, 24},
+   {0x4daf00, 220, 7852, 18, 24},
+   {0x4db000, 180, 8072, 786, 24},
+   {0x4db100, 220, 8252, 19, 24},
+   {0x4db200, 180, 8472, 787, 24},
+   {0x4db300, 228, 8652, 20, 24},
+   {0x4db400, 180, 8880, 788, 24},
+   {0x4db500, 220, 9060, 21, 24},
+   {0x4db600, 180, 9280, 789, 24},
+   {0x4db700, 220, 9460, 528, 24},
+   {0x4db800, 228, 9680, 529, 24},
+   {0x4db900, 316, 9908, 80, 24},
+   {0x4dba80, 316, 10224, 81, 24},
+   {0x4dbc00, 220, 10540, 24, 24},
+   {0x4dbd00, 180, 10760, 792, 24},
+   {0x4dbe00, 220, 10940, 25, 24},
+   {0x4dbf00, 180, 11160, 793, 24},
+   {0x4dc000, 220, 11340, 26, 24},
+   {0x4dc100, 180, 11560, 794, 24},
+   {0x4dc200, 220, 11740, 27, 24},
+   {0x4dc300, 180, 11960, 795, 24},
+   {0x4dc400, 228, 12140, 28, 24},
+   {0x4dc500, 180, 12368, 796, 24},
+   {0x4dc600, 220, 12548, 29, 24},
+   {0x4dc700, 180, 12768, 797, 24},
+   {0x4dc800, 196, 12948, 536, 24},
+   {0x4dc900, 228, 13144, 537, 24},
+   {0x4dca00, 316, 13372, 88, 24},
+   {0x4dcb80, 316, 13688, 89, 24},
+   {0x4dcd00, 244, 14004, 32, 24},
+   {0x4dce40, 180, 14248, 800, 24},
+   {0x4dcf40, 244, 14428, 33, 24},
+   {0x4dd080, 180, 14672, 801, 24},
+   {0x4dd180, 244, 14852, 34, 24},
+   {0x4dd2c0, 180, 15096, 802, 24},
+   {0x4dd3c0, 244, 15276, 35, 24},
+   {0x4dd500, 180, 15520, 803, 24},
+   {0x4dd600, 252, 15700, 36, 24},
+   {0x4dd740, 180, 15952, 804, 24},
+   {0x4dd840, 244, 16132, 37, 24},
+   {0x4dd980, 180, 16376, 805, 24},
+   {0x4dda80, 220, 16556, 544, 24},
+   {0x4ddb80, 228, 16776, 545, 24},
+   {0x4ddc80, 340, 17004, 96, 24},
+   {0x4dde00, 340, 17344, 97, 24},
+   {0x4ddf80, 268, 17684, 40, 24},
+   {0x4de0c0, 180, 17952, 808, 24},
+   {0x4de1c0, 268, 18132, 41, 24},
+   {0x4de300, 180, 18400, 809, 24},
+   {0x4de400, 268, 18580, 42, 24},
+   {0x4de540, 180, 18848, 810, 24},
+   {0x4de640, 268, 19028, 43, 24},
+   {0x4de780, 180, 19296, 811, 24},
+   {0x4de880, 276, 19476, 44, 24},
+   {0x4de9c0, 180, 19752, 812, 24},
+   {0x4deac0, 268, 19932, 45, 24},
+   {0x4dec00, 180, 20200, 813, 24},
+   {0x4ded00, 244, 20380, 552, 24},
+   {0x4dee40, 252, 20624, 553, 24},
+   {0x4def80, 348, 20876, 104, 24},
+   {0x4df100, 340, 21224, 105, 24},
+   {0x4df280, 268, 21564, 48, 24},
+   {0x4df3c0, 180, 21832, 816, 24},
+   {0x4df4c0, 268, 22012, 49, 24},
+   {0x4df600, 180, 22280, 817, 24},
+   {0x4df700, 292, 22460, 50, 24},
+   {0x4df840, 180, 22752, 818, 24},
+   {0x4df940, 292, 22932, 51, 24},
+   {0x4dfa80, 180, 23224, 819, 24},
+   {0x4dfb80, 276, 23404, 52, 24},
+   {0x4dfcc0, 180, 23680, 820, 24},
+   {0x4dfdc0, 268, 23860, 53, 24},
+   {0x4dff00, 180, 24128, 821, 24},
+   {0x4e0000, 244, 24308, 560, 24},
+   {0x4e0140, 252, 24552, 561, 24},
+   {0x4e0280, 340, 24804, 112, 24},
+   {0x4e0400, 340, 25144, 113, 24},
+   {0x4e0580, 268, 25484, 56, 24},
+   {0x4e06c0, 180, 25752, 824, 24},
+   {0x4e07c0, 268, 25932, 57, 24},
+   {0x4e0900, 180, 26200, 825, 24},
+   {0x4e0a00, 292, 26380, 58, 24},
+   {0x4e0b40, 180, 26672, 826, 24},
+   {0x4e0c40, 292, 26852, 59, 24},
+   {0x4e0d80, 180, 27144, 827, 24},
+   {0x4e0e80, 276, 27324, 60, 24},
+   {0x4e0fc0, 180, 27600, 828, 24},
+   {0x4e10c0, 268, 27780, 61, 24},
+   {0x4e1200, 180, 28048, 829, 24},
+   {0x4e1300, 244, 28228, 568, 24},
+   {0x4e1440, 252, 28472, 569, 24},
+   {0x4e1580, 340, 28724, 120, 24},
+   {0x4e1700, 340, 29064, 121, 24},
+   {0x4e1880, 284, 29404, 576, 24},
+   {0x4e19c0, 204, 29688, 577, 24},
+   {0x4e1ac0, 276, 29892, 640, 24},
+   {0x4e1c00, 204, 30168, 641, 24},
+   {0x4e1d00, 276, 30372, 642, 24},
+   {0x4e1e40, 204, 30648, 643, 24},
+   {0x4e1f40, 180, 30852, 704, 24},
+   {0x4e2040, 148, 31032, 705, 24},
+   {0x4e2100, 204, 31180, 706, 24},
+   {0x4e2200, 204, 31384, 707, 24},
+   {0x4e2300, 284, 31588, 584, 24},
+   {0x4e2440, 180, 31872, 585, 24},
+   {0x4e2540, 284, 32052, 648, 24},
+   {0x4e2680, 180, 32336, 649, 24},
+   {0x4e2780, 284, 32516, 650, 24},
+   {0x4e28c0, 180, 32800, 651, 24},
+   {0x4e29c0, 180, 32980, 712, 24},
+   {0x4e2ac0, 148, 33160, 713, 24},
+   {0x4e2b80, 204, 33308, 714, 24},
+   {0x4e2c80, 204, 33512, 715, 24},
+   {0x4e2d80, 308, 33716, 592, 24},
+   {0x4e2f00, 276, 34024, 593, 24},
+   {0x4e3040, 308, 34300, 656, 24},
+   {0x4e31c0, 276, 34608, 657, 24},
+   {0x4e3300, 308, 34884, 658, 24},
+   {0x4e3480, 276, 35192, 659, 24},
+   {0x4e35c0, 180, 35468, 720, 24},
+   {0x4e36c0, 148, 35648, 721, 24},
+   {0x4e3780, 204, 35796, 722, 24},
+   {0x4e3880, 204, 36000, 723, 24},
+   {0x4e3980, 276, 36204, 600, 24},
+   {0x4e3ac0, 180, 36480, 601, 24},
+   {0x4e3bc0, 308, 36660, 664, 24},
+   {0x4e3d40, 180, 36968, 665, 24},
+   {0x4e3e40, 308, 37148, 666, 24},
+   {0x4e3fc0, 180, 37456, 667, 24},
+   {0x4e40c0, 180, 37636, 728, 24},
+   {0x4e41c0, 148, 37816, 729, 24},
+   {0x4e4280, 204, 37964, 730, 24},
+   {0x4e4380, 204, 38168, 731, 24},
+   {0x4e4480, 300, 38372, 608, 24},
+   {0x4e4600, 276, 38672, 609, 24},
+   {0x4e4740, 308, 38948, 672, 24},
+   {0x4e48c0, 276, 39256, 673, 24},
+   {0x4e4a00, 308, 39532, 674, 24},
+   {0x4e4b80, 276, 39840, 675, 24},
+   {0x4e4cc0, 180, 40116, 736, 24},
+   {0x4e4dc0, 148, 40296, 737, 24},
+   {0x4e4e80, 204, 40444, 738, 24},
+   {0x4e4f80, 204, 40648, 739, 24},
+   {0x4e5080, 284, 40852, 616, 24},
+   {0x4e51c0, 204, 41136, 617, 24},
+   {0x4e52c0, 300, 41340, 680, 24},
+   {0x4e5440, 204, 41640, 681, 24},
+   {0x4e5540, 300, 41844, 682, 24},
+   {0x4e56c0, 204, 42144, 683, 24},
+   {0x4e57c0, 180, 42348, 744, 24},
+   {0x4e58c0, 148, 42528, 745, 24},
+   {0x4e5980, 204, 42676, 746, 24},
+   {0x4e5a80, 204, 42880, 747, 24},
+   {0x4e5b80, 284, 43084, 624, 24},
+   {0x4e5cc0, 180, 43368, 625, 24},
+   {0x4e5dc0, 300, 43548, 688, 24},
+   {0x4e5f40, 180, 43848, 689, 24},
+   {0x4e6040, 300, 44028, 690, 24},
+   {0x4e61c0, 180, 44328, 691, 24},
+   {0x4e62c0, 180, 44508, 752, 24},
+   {0x4e63c0, 148, 44688, 753, 24},
+   {0x4e6480, 204, 44836, 754, 24},
+   {0x4e6580, 204, 45040, 755, 24},
+   {0x4e6680, 308, 45244, 632, 24},
+   {0x4e6800, 252, 45552, 633, 24},
+   {0x4e6940, 300, 45804, 696, 24},
+   {0x4e6ac0, 252, 46104, 697, 24},
+   {0x4e6c00, 300, 46356, 698, 24},
+   {0x4e6d80, 252, 46656, 699, 24},
+   {0x4e6ec0, 180, 46908, 760, 24},
+   {0x4e6fc0, 148, 47088, 761, 24},
+   {0x4e7080, 204, 47236, 762, 24},
+   {0x4e7180, 204, 47440, 763, 24},
+   {0x4e7280, 156, 47644, 448, 24},
+   {0x4e7340, 156, 47800, 449, 24},
+};
+static const uint8_t hk_m5_ift_diag_original[] = {
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x16,0x00,0x00,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x04,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x08,0x08,0x00,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x06,0x58,0x0e,
+   0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x10,0x08,0x00,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x12,0x50,0x0e,0x00,0x00,0x00,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0c,0x00,0x01,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x14,0x58,0x0e,
+   0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x14,0x00,0x01,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x24,0x50,0x0e,0x00,0x00,0x00,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x12,0x08,0x01,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x26,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x1a,0x08,0x01,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x35,0x50,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x02,0x04,0x02,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x37,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0a,0x04,0x02,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x49,0x50,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x0e,0x0c,0x02,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x4b,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x16,0x0c,0x02,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x5c,0x50,0x0e,0x00,0x00,0x00,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x04,0x03,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x5e,0x58,0x0e,
+   0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x00,0x08,0x03,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x6f,0x50,0x0e,0x00,0x00,0x00,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x02,0x0c,0x03,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x00,0x58,0x0e,0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x0a,0x00,0x04,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x09,0x58,0x0e,0x00,0x00,0x60,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x0c,0x00,0x04,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x00,0x58,0x0e,
+   0x00,0x00,0x37,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x50,0x49,0x04,
+   0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,
+   0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0e,0x00,0x04,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4e,
+   0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0xd5,0x58,0x0e,0x00,0x00,0x37,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,
+   0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x0a,0x08,0x04,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0x84,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x12,0x08,0x04,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x93,0x50,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x16,0x00,0x05,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x95,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1e,0x00,0x05,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xa6,0x50,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x00,0x0c,0x05,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0xa8,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x0c,0x05,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0xb8,0x50,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x16,0x04,0x06,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0xba,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1e,0x04,0x06,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xce,0x50,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x0e,0x00,0x07,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0xd0,0x58,0x0e,0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x16,0x00,0x07,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xe4,0x50,0x0e,0x00,0x00,0x00,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x0c,0x07,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xe6,0x58,0x0e,
+   0x00,0x00,0x00,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x10,0x0c,0x07,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xfa,0x50,0x0e,0x00,0x00,0x00,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x12,0x0c,0x07,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x16,0x58,0x0e,0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x14,0x0c,0x07,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x20,0x58,0x0e,0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x16,0x0c,0x07,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0xc8,0x58,0x0e,0x00,0x00,0x38,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x0c,0x07,0x00,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4e,
+   0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0xa0,0x58,0x0e,0x00,0x00,0x39,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,
+   0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x1c,0x0c,0x07,0x00,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x82,0x58,0x0e,
+   0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1e,0x0c,0x07,0x00,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x92,0x50,0x0e,0x00,0x00,0x3a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x16,0x00,0xd5,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x93,0x58,0x0e,
+   0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x1e,0x00,0xd5,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xa6,0x50,0x0e,0x00,0x00,0x3a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x10,0x0c,0xd5,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xa8,0x58,0x0e,
+   0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x18,0x0c,0xd5,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xbd,0x50,0x0e,0x00,0x00,0x3a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x10,0x08,0xd6,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xbf,0x58,0x0e,
+   0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x18,0x08,0xd6,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xd5,0x50,0x0e,0x00,0x00,0x3a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x10,0x04,0xd7,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xd7,0x58,0x0e,
+   0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x04,0xd7,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0xed,0x50,0x0e,0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x1a,0x04,0xd7,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x80,0x58,0x0e,0x00,0x00,0x3b,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x02,0x08,0xd7,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xef,0x50,0x0e,0x00,0x00,0x3a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x14,0x0c,0xd7,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x2d,0x58,0x0e,0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x16,0x0c,0xd7,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xbd,0x58,0x0e,0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x18,0x0c,0xd7,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0xa3,0x58,0x0e,0x00,0x00,0x3b,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x1c,0x0c,0xd7,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4e,
+   0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x7f,0x58,0x0e,0x00,0x00,0x3c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,
+   0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0e,0x08,0xeb,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x65,0x58,0x0e,
+   0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x16,0x08,0xeb,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x74,0x50,0x0e,0x00,0x00,0x3d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1c,0x00,0xec,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x76,0x58,0x0e,
+   0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x04,0x04,0xec,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x88,0x50,0x0e,0x00,0x00,0x3d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x10,0x0c,0xec,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x8a,0x58,0x0e,
+   0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x18,0x0c,0xec,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x9d,0x50,0x0e,0x00,0x00,0x3d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0a,0x08,0xed,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x9f,0x58,0x0e,
+   0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x12,0x08,0xed,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xb3,0x50,0x0e,0x00,0x00,0x3d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x04,0x04,0xee,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xb5,0x58,0x0e,
+   0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0c,0x04,0xee,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0xca,0x50,0x0e,0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x00,0x00,0xef,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0xcc,0x58,0x0e,0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x00,0xef,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0xe0,0x50,0x0e,0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x14,0x04,0xef,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xcc,0x58,0x0e,
+   0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x06,0x0c,0xef,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xd8,0x58,0x0e,0x00,0x00,0x60,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x12,0x08,0xf6,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xfa,0x58,0x0e,
+   0x00,0x00,0x3d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x50,0x49,0x04,
+   0xfb,0x52,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,
+   0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x06,0x0c,0xfd,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x8f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,
+   0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xd5,0x58,0x0e,0x00,0x00,0x3e,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,
+   0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,
+   0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x14,0x04,0xfe,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xba,0x58,0x0e,
+   0x00,0x00,0x3f,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x1c,0x04,0xfe,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xce,0x50,0x0e,0x00,0x00,0x3f,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x12,0x00,0xff,0x01,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0xd0,0x58,0x0e,0x00,0x00,0x3f,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1a,0x00,0xff,0x01,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xe5,0x50,0x0e,0x00,0x00,0x3f,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x14,0x0c,0xff,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xe7,0x58,0x0e,
+   0x00,0x00,0x3f,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1e,0x0c,0xff,0x01,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xfe,0x50,0x0e,0x00,0x00,0x3f,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x1e,0x08,0x00,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x00,0x58,0x0e,0x00,0x00,0x40,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x06,0x0c,0x00,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x18,0x50,0x0e,0x00,0x00,0x40,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x08,0x08,0x01,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x1a,0x58,0x0e,
+   0x00,0x00,0x40,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x10,0x08,0x01,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x33,0x50,0x0e,0x00,0x00,0x40,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x12,0x04,0x02,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x35,0x58,0x0e,
+   0x00,0x00,0x40,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x1a,0x04,0x02,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x4d,0x50,0x0e,0x00,0x00,0x40,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x16,0x0c,0x02,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xe6,0x58,0x0e,
+   0x00,0x00,0x60,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x18,0x04,0x03,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x8f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xf6,0x58,0x0e,0x00,0x00,0x60,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x08,0x0a,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xcf,0x18,0x83,0x00,0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0x6e,0x58,0x0e,0x00,0x00,0x40,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x2b,0x58,0x89,0x04,0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,
+   0xfb,0x52,0x49,0x04,0x0b,0x54,0x89,0x04,0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1e,0x0c,0x11,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x83,0x00,0xa8,0x0c,0x00,0x50,
+   0x8f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,
+   0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x53,0x58,0x0e,0x00,0x00,0x41,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x58,0x89,0x04,0x3b,0x5a,0x89,0x04,
+   0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0x0b,0x54,0x89,0x04,0x1b,0x56,0x89,0x04,
+   0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,
+   0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x0c,0x08,0x12,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x41,0x58,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x14,0x08,0x12,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x54,0x50,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x0a,0x04,0x13,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x56,0x58,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x12,0x04,0x13,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x6b,0x50,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x10,0x00,0x14,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x6d,0x58,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x00,0x14,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x85,0x50,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x1e,0x0c,0x14,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x87,0x58,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x00,0x15,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0xa0,0x50,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x0a,0x0c,0x15,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0xa2,0x58,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x14,0x0c,0x15,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xbb,0x50,0x0e,0x00,0x00,0x42,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x08,0x16,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xbd,0x58,0x0e,
+   0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x02,0x0c,0x16,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xd7,0x50,0x0e,0x00,0x00,0x42,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x08,0x00,0x17,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x08,0x58,0x0e,0x00,0x00,0x61,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1a,0x04,0x17,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x13,0x58,0x0e,0x00,0x00,0x61,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x12,0x04,0x1e,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xcf,0x18,0x83,0x00,0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0xef,0x58,0x0e,0x00,0x00,0x42,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x2b,0x58,0x89,0x04,0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,
+   0xfb,0x52,0x49,0x04,0x0b,0x54,0x89,0x04,0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x14,0x08,0x25,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x83,0x00,0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4e,
+   0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0xcd,0x58,0x0e,0x00,0x00,0x43,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x58,0x89,0x04,0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,
+   0x0b,0x54,0x89,0x04,0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,
+   0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x04,0x26,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xb6,0x58,0x0e,
+   0x00,0x00,0x44,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x10,0x04,0x26,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xcb,0x50,0x0e,0x00,0x00,0x44,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0c,0x00,0x27,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xcd,0x58,0x0e,
+   0x00,0x00,0x44,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x16,0x00,0x27,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xe4,0x50,0x0e,0x00,0x00,0x44,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1a,0x0c,0x27,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0xe6,0x58,0x0e,0x00,0x00,0x44,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x04,0x00,0x28,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0xff,0x50,0x0e,0x00,0x00,0x44,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x10,0x0c,0x28,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x01,0x58,0x0e,
+   0x00,0x00,0x45,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1a,0x0c,0x28,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x1d,0x50,0x0e,0x00,0x00,0x45,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x04,0x0c,0x29,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x1f,0x58,0x0e,
+   0x00,0x00,0x45,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0c,0x0c,0x29,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x3a,0x50,0x0e,0x00,0x00,0x45,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x1a,0x08,0x2a,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x3c,0x58,0x0e,0x00,0x00,0x45,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x02,0x0c,0x2a,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x57,0x50,0x0e,0x00,0x00,0x45,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x0a,0x00,0x2b,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x21,0x58,0x0e,
+   0x00,0x00,0x61,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1c,0x04,0x2b,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x2c,0x58,0x0e,0x00,0x00,0x61,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x1e,0x04,0x32,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x70,0x58,0x0e,
+   0x00,0x00,0x45,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x58,0x89,0x04,
+   0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0x0b,0x54,0x89,0x04,
+   0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,
+   0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x0c,0x0c,0x39,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x83,0x00,0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4e,
+   0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x50,0x58,0x0e,0x00,0x00,0x46,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x58,0x89,0x04,0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,
+   0x0b,0x54,0x89,0x04,0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,
+   0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x06,0x08,0x3a,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x3c,0x58,0x0e,
+   0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x10,0x08,0x3a,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x52,0x50,0x0e,0x00,0x00,0x47,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x12,0x04,0x3b,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x54,0x58,0x0e,
+   0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1c,0x04,0x3b,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x6d,0x50,0x0e,0x00,0x00,0x47,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x04,0x3c,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0x6f,0x58,0x0e,0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x10,0x04,0x3c,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x8b,0x50,0x0e,0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x06,0x04,0x3d,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x8d,0x58,0x0e,
+   0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x10,0x04,0x3d,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xaa,0x50,0x0e,0x00,0x00,0x47,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x02,0x04,0x3e,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xac,0x58,0x0e,
+   0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0a,0x04,0x3e,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xc9,0x50,0x0e,0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x00,0x04,0x3f,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0xcb,0x58,0x0e,0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x0a,0x04,0x3f,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0xe9,0x50,0x0e,0x00,0x00,0x47,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x12,0x08,0x3f,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x3a,0x58,0x0e,
+   0x00,0x00,0x61,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x04,0x00,0x40,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x46,0x58,0x0e,0x00,0x00,0x61,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x18,0x00,0x47,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x83,0x00,
+   0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4e,0x4f,0x18,0x83,0x00,
+   0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x02,0x58,0x0e,
+   0x00,0x00,0x48,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x58,0x89,0x04,
+   0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,0x0b,0x54,0x89,0x04,
+   0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0xcb,0x4c,0x49,0x04,
+   0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x16,0x08,0x4e,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x83,0x00,0xa8,0x0c,0x00,0x50,0x8f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4e,
+   0x4f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4c,0x0f,0x18,0x83,0x00,0xe8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0xe7,0x58,0x0e,0x00,0x00,0x48,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x58,0x89,0x04,0x3b,0x5a,0x89,0x04,0xeb,0x50,0x49,0x04,0xfb,0x52,0x49,0x04,
+   0x0b,0x54,0x89,0x04,0x1b,0x56,0x89,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,
+   0xcb,0x4c,0x49,0x04,0xdb,0x4e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x0c,0x00,0x4f,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0xd6,0x58,0x0e,0x00,0x00,0x49,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x18,0x00,0x4f,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0xe4,0x58,0x0e,0x00,0x00,0x49,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x0c,0x0c,0x4f,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xe6,0x58,0x0e,
+   0x00,0x00,0x49,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x16,0x0c,0x4f,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0xfc,0x58,0x0e,0x00,0x00,0x49,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x10,0x08,0x50,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xfe,0x58,0x0e,
+   0x00,0x00,0x49,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x1a,0x08,0x50,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x15,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x08,0x0c,0x50,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x17,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x0e,0x0c,0x50,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x1a,0x48,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x08,0x00,0x51,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x1c,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x06,0x04,0x51,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x23,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,
+   0x1e,0x08,0x51,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x2a,0x58,0x0e,
+   0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x08,0x0c,0x51,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x38,0x50,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,
+   0x1e,0x04,0x52,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x3b,0x58,0x0e,
+   0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x08,0x08,0x52,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x50,0x50,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,
+   0x04,0x04,0x53,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x53,0x58,0x0e,
+   0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x0e,0x04,0x53,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x6a,0x50,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x1c,0x04,0x53,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x6c,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x02,0x08,0x53,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x70,0x48,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x1e,0x08,0x53,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x71,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x1c,0x0c,0x53,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x78,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x10,0x08,0x54,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x80,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x0a,0x0c,0x54,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x95,0x58,0x0e,0x00,0x00,0x4a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x1e,0x08,0x55,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x9b,0x58,0x0e,
+   0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x0c,0x55,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0xb8,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x14,0x0c,0x56,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0xbf,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x0e,0x00,0x57,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xde,0x58,0x0e,0x00,0x00,0x4a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1c,0x00,0x57,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xe4,0x58,0x0e,
+   0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x02,0x04,0x57,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xe8,0x48,0x0e,0x00,0x00,0x4a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x1e,0x04,0x57,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xe9,0x58,0x0e,
+   0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1c,0x08,0x57,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xf0,0x58,0x0e,0x00,0x00,0x4a,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x14,0x00,0x58,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0xf8,0x58,0x0e,0x00,0x00,0x4a,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1e,0x00,0x58,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x06,0x50,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x10,0x00,0x59,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x08,0x58,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1a,0x00,0x59,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x25,0x50,0x0e,0x00,0x00,0x4b,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x18,0x00,0x5a,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x27,0x58,0x0e,
+   0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x02,0x04,0x5a,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x47,0x50,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x10,0x04,0x5a,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x49,0x58,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x16,0x04,0x5a,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x4d,0x48,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x12,0x08,0x5a,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x4e,0x58,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x10,0x0c,0x5a,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x55,0x58,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,
+   0x06,0x08,0x5b,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0x5d,0x58,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x00,0x0c,0x5b,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x72,0x58,0x0e,0x00,0x00,0x4b,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x04,0x0c,0x5b,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x87,0x00,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x0e,0x00,0x61,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x7a,0x58,0x0e,0x00,0x00,0x4b,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x0c,0x04,0x62,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x24,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x06,0x08,0x62,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x83,0x00,0xa8,0x0c,0x00,0x4a,
+   0xcf,0x18,0x43,0x00,0xe8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x4c,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x1e,0x0c,0x5b,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0x52,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1a,0x08,0x62,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0xc7,0x55,0x48,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x14,0x0c,0x62,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x57,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,
+   0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x14,0x00,0x63,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x07,0x5e,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,
+   0x06,0x08,0x63,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x65,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x14,0x08,0x63,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x72,0x58,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x04,0x04,0x64,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x75,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x12,0x04,0x64,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x47,0x8a,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x08,0x00,0x65,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x8d,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x14,0x00,0x65,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xa3,0x58,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x02,0x04,0x65,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xa6,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x08,0x04,0x65,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xa9,0x48,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x02,0x08,0x65,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xaa,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x00,0x0c,0x65,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xb1,0x58,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,0x16,0x00,0x66,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x07,0xb9,0x58,0x0e,0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x00,0x04,0x66,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xc6,0x58,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0xc0,0x02,0x10,0x14,0x0c,0x66,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xc9,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x00,0x00,0x67,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0xde,0x58,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,0x1a,0x08,0x67,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0xe0,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x04,0x0c,0x67,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xf7,0x58,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x12,0x0c,0x67,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xfa,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x18,0x0c,0x67,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0xfd,0x48,0x0e,0x00,0x00,0x4c,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,
+   0x14,0x00,0x68,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0xff,0x58,0x0e,
+   0x00,0x00,0x4c,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x12,0x04,0x68,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x05,0x58,0x0e,0x00,0x00,0x4d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,0x04,0x00,0x69,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x0d,0x58,0x0e,
+   0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x1e,0x00,0x69,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x22,0x58,0x0e,0x00,0x00,0x4d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,0x12,0x00,0x6a,0x02,0xfc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x28,0x58,0x0e,
+   0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,
+   0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,
+   0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,
+   0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,
+   0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,
+   0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0xc0,0x02,0x10,0x0c,0x04,0x6a,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,
+   0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,
+   0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x45,0x58,0x0e,0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,
+   0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,
+   0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xec,0x80,0x02,0x10,
+   0x06,0x04,0x6b,0x02,0xfc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x12,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x30,0x00,0x08,0x00,0x01,0x84,0x00,0x00,0x2f,0x1c,0x83,0x00,
+   0xe8,0x0c,0x00,0x4a,0xef,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x48,0xaf,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x46,0x6f,0x1c,0x43,0x00,0xe8,0x0c,0x00,0x44,0x2f,0x1c,0x43,0x00,
+   0xe8,0x0c,0x00,0x42,0xef,0x1c,0x03,0x00,0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0x47,0x4c,0x58,0x0e,0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x14,0x00,0x14,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x2b,0x18,0x09,0x44,
+   0x3b,0x1a,0x09,0x04,0x0b,0x14,0x09,0x04,0x1b,0x16,0x09,0x04,0x8b,0x44,0x49,0x04,
+   0x9b,0x46,0x49,0x04,0xab,0x48,0x49,0x04,0xbb,0x4a,0x49,0x04,0x4b,0x3c,0x49,0x04,
+   0x5b,0x3e,0x49,0x04,0x6b,0x40,0x49,0x04,0x7b,0x42,0x49,0x04,0x0b,0x34,0x49,0x04,
+   0x1b,0x36,0x49,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,0xcb,0x2c,0x09,0x04,
+   0xdb,0x2e,0x09,0x04,0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x00,0x08,0x6b,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x43,0x00,0xa8,0x0c,0x00,0x48,
+   0x8f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x46,0x4f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x87,0x6a,0x58,0x0e,0x00,0x00,0x4d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x38,0x49,0x04,0x3b,0x3a,0x49,0x04,
+   0xeb,0x30,0x09,0x04,0xfb,0x32,0x09,0x04,0x0b,0x34,0x49,0x04,0x1b,0x36,0x49,0x04,
+   0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,0xcb,0x2c,0x09,0x04,0xdb,0x2e,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x10,0x08,0x6b,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0x0f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x71,0x58,0x0e,
+   0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x6b,0x20,0x09,0x04,
+   0x7b,0x22,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x16,0x08,0x6b,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0xc7,0x74,0x48,0x0e,0x00,0x00,0x4d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,
+   0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,
+   0x10,0x0c,0x6b,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,
+   0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,
+   0xa8,0x0c,0x00,0x44,0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,
+   0xe8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x47,0x76,0x58,0x0e,
+   0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,
+   0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,
+   0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,
+   0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,
+   0xbb,0x2a,0x09,0x04,0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,
+   0x9b,0x26,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,
+   0x5b,0x1e,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,
+   0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,0xcc,0x80,0x02,0x10,0x10,0x00,0x6c,0x02,
+   0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,
+   0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,0x4f,0x18,0x43,0x00,0xa8,0x0c,0x00,0x44,
+   0x0f,0x18,0x43,0x00,0xe8,0x0c,0x00,0x42,0xcf,0x18,0x03,0x00,0xe8,0x0c,0x00,0x40,
+   0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x07,0x7d,0x58,0x0e,0x00,0x00,0x4d,0x10,
+   0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,
+   0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,
+   0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,
+   0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,0xab,0x28,0x09,0x04,0xbb,0x2a,0x09,0x04,
+   0x6b,0x20,0x09,0x04,0x7b,0x22,0x09,0x04,0x8b,0x24,0x09,0x04,0x9b,0x26,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x4b,0x1c,0x09,0x04,0x5b,0x1e,0x09,0x04,
+   0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,
+   0x00,0x00,0x03,0x00,0xcc,0xc0,0x02,0x10,0x16,0x00,0x6c,0x02,0xdc,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,
+   0x01,0x84,0x00,0x00,0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,
+   0x00,0x00,0x00,0x00,0xc7,0x84,0x48,0x0e,0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,
+   0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,
+   0x34,0x81,0x01,0x00,0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x1a,0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,
+   0x04,0x00,0x00,0x00,0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,
+   0x1b,0x16,0x09,0x04,0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x23,0x00,0x07,0x02,
+   0x42,0x20,0xa0,0x02,0x0e,0x00,0x00,0x00,0x4c,0x80,0x02,0x00,0x00,0x00,0x03,0x00,
+   0xcc,0x80,0x02,0x10,0x1e,0x00,0x6c,0x02,0xdc,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0xa7,0x00,0x01,0x02,0x26,0x81,0xa8,0x0a,0x10,0x04,0x08,0x00,0x01,0x84,0x00,0x00,
+   0xcf,0x18,0x03,0x00,0xa8,0x0c,0x00,0x40,0x07,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,
+   0x87,0x86,0x48,0x0e,0x00,0x00,0x4d,0x10,0x00,0x01,0x07,0x00,0x00,0x0d,0x00,0x00,
+   0x00,0x00,0x00,0x00,0x1c,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x34,0x81,0x01,0x00,
+   0x20,0x00,0xa0,0x02,0x0c,0x80,0x02,0x00,0x10,0x00,0x00,0x00,0x27,0x04,0x04,0x1a,
+   0x21,0x00,0xa3,0x02,0x88,0x80,0x00,0x00,0x1c,0x80,0x02,0x00,0x04,0x00,0x00,0x00,
+   0x27,0x04,0x04,0x00,0x30,0x00,0xa4,0x12,0x0b,0x14,0x09,0x44,0x1b,0x16,0x09,0x04,
+   0x2b,0x18,0x09,0x04,0x3b,0x1a,0x09,0x04,0x23,0x00,0x07,0x02,0x42,0x20,0xa0,0x02,
+   0x0e,0x00,0x00,0x00,
+};
diff --git a/src/asahi/vulkan/hk_native_descriptor_pack.c b/src/asahi/vulkan/hk_native_descriptor_pack.c
index bb3ced99a0a6..b6b0dd240dac 100644
--- a/src/asahi/vulkan/hk_native_descriptor_pack.c
+++ b/src/asahi/vulkan/hk_native_descriptor_pack.c
@@ -75,6 +75,11 @@ hk_pack_native_texture(unsigned generation,
       return false;
 
    if (generation == 9) {
+      if (legacy->compressed &&
+          (!extended || !legacy->acceleration_buffer ||
+           (multisampled && legacy->samples == AGX_SAMPLE_COUNT_8)))
+         return false;
+
       if (!legacy->width || legacy->width > 16384 || !legacy->height ||
           legacy->height > 16384)
          return false;
@@ -179,7 +184,9 @@ hk_pack_native_texture(unsigned generation,
          .unk_mipmapped = legacy->unk_mipmapped,
          .compressed = legacy->compressed,
          .image_mode_enable = legacy->mode != AGX_IMAGE_MODE_NORMAL,
-         .image_mode = legacy->mode,
+         /* The HAL300 texture encoder writes 1 for 16 KiB, 2 for 64 KiB and 3
+          * for 256 KiB sparse pages. AIL uses 16 KiB pages. */
+         .image_mode = legacy->mode == AGX_IMAGE_MODE_SPARSE ? 1 : legacy->mode,
          .address = legacy->address,
          .srgb = legacy->srgb,
          .srgb_2_channel = legacy->srgb_2_channel,
@@ -231,33 +238,75 @@ hk_pack_native_pbe(unsigned generation,
       return false;
 
    /* The recovered subset excludes fields whose Gen9/10 semantic placement
-    * remains incomplete: sparse mode, Gen10 gamma and the software-addressing
-    * sideband. The host-only HAL200 StateFields oracle matches the complete
+    * remains incomplete: sparse mode and the software-addressing sideband. The host-only HAL200 PBE encoder oracle matches the complete
     * GPU-tiled 2D compressed packet, including the metadata GPU address and
     * mip extension. Other compressed shapes remain closed. */
-   const bool array = generation == 9 &&
+   const bool array = (generation == 9 || generation == 10) &&
                       legacy->dimension == AGX_TEXTURE_DIMENSION_2D_ARRAY;
    const bool array_multisample =
-      generation == 9 &&
+      (generation == 9 || generation == 10) &&
       legacy->dimension == AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED;
-   const bool three_d = generation == 9 &&
+   const bool three_d = (generation == 9 || generation == 10) &&
                         legacy->dimension == AGX_TEXTURE_DIMENSION_3D;
    const bool layered = array || array_multisample || three_d;
-   /* Multisample stores use the recovered Gen9 dimension-4 packet for one
-    * layer and dimension 8 for a layered render target. */
+   /* Both generations have native dimension-4 single-layer MS stores.
+    * Gen9 additionally has the qualified dimension-8 array packet. */
    const bool multisample =
-      generation == 9 &&
+      (generation == 9 || generation == 10) &&
       (legacy->dimension == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED ||
        array_multisample);
    const bool interchange = legacy->layout == AGX_LAYOUT_INTERCHANGE;
-   const bool compressed_2d = legacy->compressed && generation == 9 &&
-                              legacy->dimension == AGX_TEXTURE_DIMENSION_2D &&
-                              (legacy->layout == AGX_LAYOUT_GPU || interchange) &&
-                              !multisample;
+   const bool compressed_gen9 = legacy->compressed && generation == 9 &&
+      (legacy->layout == AGX_LAYOUT_GPU ||
+       (interchange && !layered && !multisample));
+   const bool compressed_2d = legacy->compressed &&
+      (generation == 9 || generation == 10) &&
+      (legacy->dimension == AGX_TEXTURE_DIMENSION_2D ||
+       (generation == 10 && array && legacy->layers == 1 &&
+        legacy->level == 0 && legacy->levels == 1)) &&
+      (legacy->layout == AGX_LAYOUT_GPU || (generation == 9 && interchange)) &&
+      !multisample;
+   /* Private destination capability for raw D32 copy views. The independently
+    * witnessed packet combines only Gen10 R32_UINT with compressed 2x/4x.
+    * Image allocation policy remains separate and unchanged. */
+   const bool compressed_msaa_r32_uint =
+      generation == 10 &&
+      legacy->dimension == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED &&
+      legacy->layout == AGX_LAYOUT_GPU &&
+      legacy->channels == AGX_CHANNELS_R32 &&
+      legacy->type == AGX_TEXTURE_TYPE_UINT && !legacy->srgb &&
+      legacy->layers == 1 && legacy->level == 0 && legacy->levels == 1 &&
+      !legacy->unk_mipmapped &&
+      (legacy->samples == AGX_SAMPLE_COUNT_2 ||
+       legacy->samples == AGX_SAMPLE_COUNT_4);
+   const bool compressed_msaa_rgba16 =
+      generation == 10 &&
+      legacy->dimension == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED &&
+      legacy->layout == AGX_LAYOUT_GPU &&
+      legacy->channels == AGX_CHANNELS_R16G16B16A16 &&
+      (legacy->type == AGX_TEXTURE_TYPE_FLOAT ||
+       legacy->type == AGX_TEXTURE_TYPE_UINT) && !legacy->srgb &&
+      legacy->layers == 1 && legacy->level == 0 && legacy->levels == 1 &&
+      !legacy->unk_mipmapped &&
+      (legacy->samples == AGX_SAMPLE_COUNT_2 ||
+       legacy->samples == AGX_SAMPLE_COUNT_4);
+   /* RGBA8/BGRA8 (UNORM and sRGB) share the R32_UINT texel size and so its
+    * compressed multisample metadata layout; see
+    * agx_pbe_agx3_2d_multisample_pack(). */
+   const bool compressed_msaa_rgba8 =
+      generation == 10 &&
+      legacy->dimension == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED &&
+      legacy->layout == AGX_LAYOUT_GPU &&
+      legacy->channels == AGX_CHANNELS_R8G8B8A8 &&
+      legacy->type == AGX_TEXTURE_TYPE_UNORM &&
+      legacy->layers == 1 && legacy->level == 0 && legacy->levels == 1 &&
+      !legacy->unk_mipmapped &&
+      (legacy->samples == AGX_SAMPLE_COUNT_2 ||
+       legacy->samples == AGX_SAMPLE_COUNT_4);
    if ((!layered && !multisample &&
         legacy->dimension != AGX_TEXTURE_DIMENSION_2D) ||
        (legacy->layout == AGX_LAYOUT_LINEAR &&
-        (generation != 9 || legacy->level != 0 || legacy->levels != 1 ||
+        (legacy->level != 0 || legacy->levels != 1 ||
          legacy->unk_mipmapped)) ||
        (legacy->layout != AGX_LAYOUT_LINEAR &&
         legacy->layout != AGX_LAYOUT_TWIDDLED &&
@@ -266,17 +315,21 @@ hk_pack_native_pbe(unsigned generation,
           legacy->level == 0 && legacy->levels == 1)) ||
        legacy->unk_52 || legacy->rotate_90 || legacy->flip_vertical ||
        (multisample ? (legacy->samples != AGX_SAMPLE_COUNT_2 &&
-                       legacy->samples != AGX_SAMPLE_COUNT_4)
+                       legacy->samples != AGX_SAMPLE_COUNT_4 &&
+                       !(generation == 10 && legacy->samples == AGX_SAMPLE_COUNT_8))
                     : legacy->samples != AGX_SAMPLE_COUNT_2) ||
-       (legacy->compressed && !compressed_2d) ||
+       (legacy->compressed && !compressed_gen9 && !compressed_2d &&
+                          !compressed_msaa_r32_uint &&
+                          !compressed_msaa_rgba16 && !compressed_msaa_rgba8) ||
        legacy->mode != AGX_IMAGE_MODE_NORMAL ||
        (!layered && legacy->layers != 1) ||
        (multisample && (legacy->layout == AGX_LAYOUT_LINEAR ||
                         legacy->level != 0 || legacy->levels != 1 ||
-                        legacy->unk_mipmapped || legacy->compressed)) ||
-       (generation != 9 && (legacy->level != 0 || legacy->levels != 1 ||
-                            legacy->unk_mipmapped)) ||
-       (generation != 9 && legacy->srgb) ||
+                        legacy->unk_mipmapped ||
+                        (legacy->compressed && !compressed_gen9 &&
+                          !compressed_msaa_r32_uint &&
+                          !compressed_msaa_rgba16 &&
+                          !compressed_msaa_rgba8))) ||
        (legacy->extended != legacy->compressed) ||
        (legacy->layout == AGX_LAYOUT_LINEAR || interchange
            ? legacy->stride < 12 || legacy->stride > UINT32_MAX - 4
@@ -313,9 +366,11 @@ hk_pack_native_pbe(unsigned generation,
       ? (array_multisample
             ? agx_pbe_agx3_2d_array_multisample_pack(
                  generation, &fields, legacy->layers,
+                 legacy->samples == AGX_SAMPLE_COUNT_8 ? 8u :
                  legacy->samples == AGX_SAMPLE_COUNT_4 ? 4u : 2u, &packed)
             : agx_pbe_agx3_2d_multisample_pack(
                  generation, &fields,
+                 legacy->samples == AGX_SAMPLE_COUNT_8 ? 8u :
                  legacy->samples == AGX_SAMPLE_COUNT_4 ? 4u : 2u, &packed))
       : layered
       ? (array ? agx_pbe_agx3_2d_array_mip_pack(
@@ -324,10 +379,8 @@ hk_pack_native_pbe(unsigned generation,
                : agx_pbe_agx3_3d_mip_pack(generation, &fields, legacy->layers,
                                           legacy->level, legacy->levels,
                                           &packed))
-      : generation == 9
-         ? agx_pbe_agx3_2d_mip_pack(generation, &fields, legacy->level,
-                                    legacy->levels, &packed)
-         : agx_pbe_agx3_2d_pack(generation, &fields, &packed);
+      : agx_pbe_agx3_2d_mip_pack(generation, &fields, legacy->level,
+                                 legacy->levels, &packed);
    if (!valid)
       return false;
 
diff --git a/src/asahi/vulkan/hk_native_write_class.h b/src/asahi/vulkan/hk_native_write_class.h
new file mode 100644
index 000000000000..dae11634c277
--- /dev/null
+++ b/src/asahi/vulkan/hk_native_write_class.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: MIT */
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "compiler/nir/nir.h"
+#include "drm-uapi/asahi_drm.h"
+
+/* The external-write predicate itself is hk_shader_writes_external_memory()
+ * in hk_shader.c. */
+static inline void
+hk_m5_accumulate_shader_writes(bool *command_writes, bool shader_writes)
+{
+   *command_writes |= shader_writes;
+}
+
+static inline uint32_t
+hk_m5_shader_write_class_flags(uint64_t features, bool command_writes)
+{
+   return command_writes &&
+          (features & DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES)
+             ? DRM_ASAHI_RENDER_SHADER_WRITES : 0;
+}
diff --git a/src/asahi/vulkan/hk_nir_lower_descriptors.c b/src/asahi/vulkan/hk_nir_lower_descriptors.c
index d1cada399969..e81703a1611d 100644
--- a/src/asahi/vulkan/hk_nir_lower_descriptors.c
+++ b/src/asahi/vulkan/hk_nir_lower_descriptors.c
@@ -43,6 +43,23 @@ struct lower_descriptors_ctx {
    mesa_shader_stage stage;
 };
 
+static bool
+storage_address_has_map(const struct lower_descriptors_ctx *ctx)
+{
+   return ctx->ssbo_addr_format == nir_address_format_64bit_global_32bit_offset_map ||
+          ctx->ssbo_addr_format == nir_address_format_64bit_bounded_global_map;
+}
+
+static nir_def *
+storage_address(nir_builder *b, nir_def *desc,
+                const struct lower_descriptors_ctx *ctx)
+{
+   nir_def *parts[] = {nir_channel(b, desc, 0), nir_channel(b, desc, 1),
+      nir_channel(b, desc, 2), nir_imm_int(b, 0),
+      ctx->sparse_buffer_residency ? nir_channel(b, desc, 3) : nir_imm_int(b, 0)};
+   return nir_vec(b, parts, 5);
+}
+
 static const struct hk_descriptor_set_binding_layout *
 get_binding_layout(uint32_t set, uint32_t binding,
                    const struct lower_descriptors_ctx *ctx)
@@ -145,6 +162,10 @@ load_descriptor(nir_builder *b, unsigned num_components, unsigned bit_size,
 {
    const struct hk_descriptor_set_binding_layout *binding_layout =
       get_binding_layout(set, binding, ctx);
+   const bool map_address = storage_address_has_map(ctx) &&
+      (binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ||
+       binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC);
+   const unsigned load_components = map_address ? 4 : num_components;
 
    if (ctx->clamp_desc_array_bounds)
       index =
@@ -164,11 +185,15 @@ load_descriptor(nir_builder *b, unsigned num_components, unsigned bit_size,
          b, nir_amul_imm(b, index, sizeof(struct hk_buffer_address)),
          hk_root_descriptor_offset(dynamic_buffers));
 
-      assert(num_components == 4 && bit_size == 32);
+      assert(num_components == (map_address ? 5 : 4) && bit_size == 32);
       nir_def *desc = load_root(b, 4, 32, root_desc_offset, 16);
 
+      if (map_address)
+         return storage_address(b, desc, ctx);
+
       if (binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC &&
-          ctx->sparse_buffer_residency && ctx->descriptor_generation == 9)
+          ctx->sparse_buffer_residency &&
+          (ctx->descriptor_generation == 9 || ctx->descriptor_generation == 10))
          return desc;
 
       /* We know a priori that the .w component (offset) is zero. */
@@ -183,7 +208,7 @@ load_descriptor(nir_builder *b, unsigned num_components, unsigned bit_size,
       const uint32_t binding_size = binding_layout->array_size;
 
       /* Convert it to nir_address_format_64bit_bounded_global */
-      assert(num_components == 4 && bit_size == 32);
+      assert(num_components == (map_address ? 5 : 4) && bit_size == 32);
       return nir_vec4(b, nir_unpack_64_2x32_split_x(b, base_addr),
                       nir_unpack_64_2x32_split_y(b, base_addr),
                       nir_imm_int(b, binding_size), nir_imm_int(b, 0));
@@ -203,16 +228,20 @@ load_descriptor(nir_builder *b, unsigned num_components, unsigned bit_size,
       nir_def *desc;
       nir_def *set_addr = nir_load_descriptor_set_agx(b, set);
       desc = nir_load_global_constant_offset(
-         b, num_components, bit_size, set_addr, desc_ubo_offset,
+         b, load_components, bit_size, set_addr, desc_ubo_offset,
          .align_mul = desc_align_mul, .align_offset = desc_align_offset,
          .access = ACCESS_CAN_SPECULATE);
 
       if (binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER ||
           binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) {
-         assert(num_components == 4 && bit_size == 32);
+         assert(num_components == (map_address ? 5 : 4) && bit_size == 32);
+         if (map_address)
+            return storage_address(b, desc, ctx);
+
          if (!(binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER &&
                ctx->sparse_buffer_residency &&
-               ctx->descriptor_generation == 9)) {
+               (ctx->descriptor_generation == 9 ||
+                ctx->descriptor_generation == 10))) {
             /* We know a priori that the .w component (offset) is zero. */
             desc = nir_vector_insert_imm(b, desc, nir_imm_int(b, 0), 3);
          }
@@ -258,7 +287,10 @@ load_descriptor_for_idx_intrin(nir_builder *b, nir_intrinsic_instr *intrin,
        nir_descriptor_type_acceleration_structure)
       return load_descriptor(b, 1, 64, set, binding, index, 0, ctx);
 
-   return load_descriptor(b, 4, 32, set, binding, index, 0, ctx);
+   const unsigned components =
+      nir_intrinsic_desc_type(intrin) == nir_descriptor_type_storage_buffer &&
+      storage_address_has_map(ctx) ? 5 : 4;
+   return load_descriptor(b, components, 32, set, binding, index, 0, ctx);
 }
 
 static bool
@@ -525,6 +557,9 @@ lower_uvs_index(nir_builder *b, nir_intrinsic_instr *intrin, void *data)
    case nir_intrinsic_load_sample_positions_agx:
       return lower_sysval_to_root_table(b, intrin, draw.ppp_multisamplectl);
 
+   case nir_intrinsic_load_sample_positions_hi_agx:
+      return lower_sysval_to_root_table(b, intrin, ppp_multisamplectl_hi);
+
    case nir_intrinsic_load_depth_never_agx:
       return lower_sysval_to_root_table(b, intrin, draw.force_never_in_shader);
 
@@ -580,9 +615,10 @@ lower_uvs_index(nir_builder *b, nir_intrinsic_instr *intrin, void *data)
    case nir_intrinsic_load_draw_id: {
       b->cursor = nir_instr_remove(&intrin->instr);
 
-      
-
-      if (args->sw_stage == MESA_SHADER_MESH &&
+      /* Mesh programs have no vertex uniform ABI; the draw root carries
+       * the draw index of each vkCmdDrawMeshTasks* draw. */
+      if ((args->sw_stage == MESA_SHADER_MESH ||
+           args->sw_stage == MESA_SHADER_TASK) &&
           intrin->intrinsic == nir_intrinsic_load_draw_id) {
          unsigned offset = hk_root_descriptor_offset(mesh_draw_id);
          nir_def *val = load_root(b, 1, 32, nir_imm_int(b, offset), 4);
@@ -613,10 +649,11 @@ lower_uvs_index(nir_builder *b, nir_intrinsic_instr *intrin, void *data)
 
       unsigned off1 = hk_root_descriptor_offset(draw.pipeline_stats);
       unsigned off2 = hk_root_descriptor_offset(draw.pipeline_stats_flags);
-      static_assert(hk_root_descriptor_offset(draw.pipeline_stats) == 432,
+      static_assert(hk_root_descriptor_offset(draw.pipeline_stats) ==
+                       304 + 8 * HK_MAX_SETS,
                     "pipeline-statistics compiler oracle ABI");
       static_assert(hk_root_descriptor_offset(draw.pipeline_stats_flags) ==
-                       440,
+                       312 + 8 * HK_MAX_SETS,
                     "pipeline-statistics compiler oracle ABI");
 
       nir_def *base = load_root(b, 1, 64, nir_imm_int(b, off1), 8);
@@ -741,9 +778,11 @@ try_lower_intrin(nir_builder *b, nir_intrinsic_instr *intrin,
    case nir_intrinsic_load_num_workgroups: {
       b->cursor = nir_instr_remove(&intrin->instr);
 
-      
-
-      if (b->shader->info.stage == MESA_SHADER_MESH) {
+      /* Mesh and task shaders run under the graphics root, whose cs arm
+       * overlays the draw arm: read the draw's own grid instead. With a task
+       * stage this is the API task grid, which only the task stage reads. */
+      if (b->shader->info.stage == MESA_SHADER_MESH ||
+          b->shader->info.stage == MESA_SHADER_TASK) {
          unsigned offset = hk_root_descriptor_offset(mesh_grid_addr);
          nir_def *ptr = load_root(b, 1, 64, nir_imm_int(b, offset), 8);
          nir_def_rewrite_uses(&intrin->def, load_speculatable(b, 3, 32, ptr, 4));
@@ -823,7 +862,8 @@ static bool
 lower_compute_lod_query(nir_builder *b, nir_tex_instr *tex,
                         const struct lower_descriptors_ctx *ctx)
 {
-   if (tex->op != nir_texop_lod || ctx->descriptor_generation != 9 ||
+   if (tex->op != nir_texop_lod ||
+       (ctx->descriptor_generation != 9 && ctx->descriptor_generation != 10) ||
        b->shader->info.stage != MESA_SHADER_COMPUTE ||
        b->shader->info.derivative_group == DERIVATIVE_GROUP_NONE ||
        agx_nir_compute_implicit_lod_diagnostic())
@@ -856,21 +896,35 @@ lower_compute_lod_query(nir_builder *b, nir_tex_instr *tex,
       hk_sampled_image_descriptor_layout(ctx->descriptor_generation);
    const uint64_t plane_offset_B = plane * native.stride;
 
-   nir_def *index = load_resource_deref_desc(
-      b, 1, 16, sampler, plane_offset_B + native.sampler_index, ctx);
-   nir_def *api_bias = load_resource_deref_desc(
-      b, 1, 32, sampler,
-      plane_offset_B + native.sampler +
-         offsetof(struct hk_sampled_image_descriptor_sw, g17p.lod_bias),
-      ctx);
-   nir_def *heap = load_root(
-      b, 1, 64, nir_imm_int(b, hk_root_descriptor_offset(sampler_heap_address)),
-      8);
-   nir_def *addr = nir_iadd(
-      b, heap,
-      nir_imul_imm(b, nir_u2u64(b, index),
-                   agx_chip_sampler_heap_element_size(AGX_CHIP_G17P)));
-   nir_def *word = load_speculatable(b, 1, 32, addr, 8);
+   nir_def *word, *api_bias;
+   if (ctx->descriptor_generation == 9) {
+      nir_def *index = load_resource_deref_desc(
+         b, 1, 16, sampler, plane_offset_B + native.sampler_index, ctx);
+      api_bias = load_resource_deref_desc(
+         b, 1, 32, sampler,
+         plane_offset_B + native.sampler +
+            offsetof(struct hk_sampled_image_descriptor_sw, g17p.lod_bias),
+         ctx);
+      nir_def *heap = load_root(
+         b, 1, 64,
+         nir_imm_int(b, hk_root_descriptor_offset(sampler_heap_address)), 8);
+      nir_def *addr = nir_iadd(
+         b, heap,
+         nir_imul_imm(b, nir_u2u64(b, index),
+                      agx_chip_sampler_heap_element_size(AGX_CHIP_G17P)));
+      word = load_speculatable(b, 1, 32, addr, 8);
+   } else {
+      /* A Gen10 descriptor carries its plane's Gen5 sampler word itself
+       * (hk_rc_sampler::key, the heap entry's packed state), with the API
+       * bias in the word's signed 5:6 LOD bias field at bits 43..53 --
+       * the bias Gen5 hardware applies when sampling with this sampler. */
+      nir_def *packed = load_resource_deref_desc(
+         b, 2, 32, sampler, plane_offset_B + native.sampler, ctx);
+      word = nir_channel(b, packed, 0);
+      nir_def *bias_fixed =
+         nir_ishr_imm(b, nir_ishl_imm(b, nir_channel(b, packed, 1), 10), 21);
+      api_bias = nir_fmul_imm(b, nir_i2f32(b, bias_fixed), 1.0 / 64.0);
+   }
    nir_def *min_lod =
       nir_fmul_imm(b, nir_u2f32(b, nir_iand_imm(b, word, 0x3ff)), 1.0 / 64.0);
    nir_def *max_lod = nir_fmul_imm(
@@ -1260,6 +1314,11 @@ lower_ssbo_resource_index(nir_builder *b, nir_intrinsic_instr *intrin,
                             nir_unpack_64_2x32_split_y(b, binding_addr),
                             nir_imm_int(b, binding_size), offset_in_binding);
 
+   if (storage_address_has_map(ctx)) {
+      nir_def *parts[] = {nir_channel(b, addr, 0), nir_channel(b, addr, 1),
+         nir_channel(b, addr, 2), nir_channel(b, addr, 3), nir_imm_int(b, 0)};
+      addr = nir_vec(b, parts, 5);
+   }
    nir_def_rewrite_uses(&intrin->def, addr);
 
    return true;
@@ -1303,7 +1362,8 @@ lower_load_ssbo_descriptor(nir_builder *b, nir_intrinsic_instr *intrin,
 
    nir_def *desc;
    switch (ctx->ssbo_addr_format) {
-   case nir_address_format_64bit_global_32bit_offset: {
+   case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map: {
       nir_def *base = nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2));
       nir_def *offset = nir_channel(b, addr, 3);
       /* Mask off the binding stride */
@@ -1314,7 +1374,8 @@ lower_load_ssbo_descriptor(nir_builder *b, nir_intrinsic_instr *intrin,
       break;
    }
 
-   case nir_address_format_64bit_bounded_global: {
+   case nir_address_format_64bit_bounded_global:
+   case nir_address_format_64bit_bounded_global_map: {
       nir_def *base = nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2));
       nir_def *size = nir_channel(b, addr, 2);
       nir_def *offset = nir_channel(b, addr, 3);
@@ -1330,6 +1391,8 @@ lower_load_ssbo_descriptor(nir_builder *b, nir_intrinsic_instr *intrin,
       UNREACHABLE("Unknown address mode");
    }
 
+   if (storage_address_has_map(ctx))
+      desc = storage_address(b, desc, ctx);
    nir_def_rewrite_uses(&intrin->def, desc);
 
    return true;
@@ -1392,7 +1455,7 @@ lower_sparse_buffer_deref(nir_builder *b, nir_intrinsic_instr *intr,
    nir_deref_path_init(&path, leaf, NULL);
    nir_deref_instr *root = path.path[0];
    if (root->deref_type != nir_deref_type_cast ||
-       root->parent.ssa->num_components != 4 ||
+       root->parent.ssa->num_components != (storage_address_has_map(ctx) ? 5 : 4) ||
        root->parent.ssa->bit_size != 32) {
       nir_deref_path_finish(&path);
       return false;
@@ -1409,8 +1472,13 @@ lower_sparse_buffer_deref(nir_builder *b, nir_intrinsic_instr *intr,
        is_buffer_deref_atomic(intr->intrinsic)) {
       static_assert(AIL_PAGESIZE == (1u << 14), "update shader lookup");
 
-      nir_def *map_page = nir_channel(b, addr, 3);
+      nir_def *map_page = nir_channel(b, addr, storage_address_has_map(ctx) ? 4 : 3);
       nir_def *has_map = nir_ine_imm(b, map_page, 0);
+
+      /* Ordinary buffers have no residency map. Keep their loads on the
+       * direct path instead of executing the sparse address translation and
+       * reading the zero page for every access. */
+      nir_if *map_if = nir_push_if(b, has_map);
       nir_def *map_base =
          nir_ishl_imm(b, nir_u2u64(b, map_page), 14);
       nir_def *safe_map_base = nir_bcsel(
@@ -1431,6 +1499,8 @@ lower_sparse_buffer_deref(nir_builder *b, nir_intrinsic_instr *intr,
          .access = ACCESS_CAN_REORDER | ACCESS_CAN_SPECULATE);
       nir_def *byte_offset =
          nir_build_deref_offset(b, leaf, glsl_get_natural_size_align_bytes);
+      if (storage_address_has_map(ctx))
+         byte_offset = nir_iadd(b, byte_offset, nir_channel(b, addr, 3));
       nir_def *rw_base = nir_iand_imm(b, base, ~ctx->sparse_ro_offset);
       nir_def *access_addr =
          nir_iadd(b, rw_base, nir_u2u64(b, byte_offset));
@@ -1465,12 +1535,28 @@ lower_sparse_buffer_deref(nir_builder *b, nir_intrinsic_instr *intr,
          nir_def *zero_base = nir_ior_imm(b, base, ctx->sparse_ro_offset);
          base = nir_bcsel(b, resident, base, zero_base);
       }
+
+      nir_def *mapped_base = base;
+      nir_def *mapped_resident = resident;
+      nir_push_else(b, map_if);
+      nir_def *direct_base =
+         nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2));
+      nir_def *direct_resident = nir_imm_true(b);
+      nir_pop_if(b, map_if);
+      base = nir_if_phi(b, mapped_base, direct_base);
+      resident = nir_if_phi(b, mapped_resident, direct_resident);
    }
 
    nir_def *lowered_addr =
       nir_vec4(b, nir_unpack_64_2x32_split_x(b, base),
                nir_unpack_64_2x32_split_y(b, base), nir_channel(b, addr, 2),
                nir_imm_int(b, 0));
+   if (storage_address_has_map(ctx)) {
+      nir_def *parts[] = {nir_channel(b, lowered_addr, 0),
+         nir_channel(b, lowered_addr, 1), nir_channel(b, lowered_addr, 2),
+         nir_channel(b, addr, 3), nir_imm_int(b, 0)};
+      lowered_addr = nir_vec(b, parts, 5);
+   }
    nir_deref_instr *lowered = nir_build_deref_cast_with_alignment(
       b, lowered_addr, root->modes, root->type, root->cast.ptr_stride,
       root->cast.align_mul, root->cast.align_offset);
@@ -1538,7 +1624,7 @@ hk_nir_lower_descriptors(nir_shader *nir,
          rs->images != VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DISABLED_EXT,
 
       .ssbo_addr_format = hk_buffer_addr_format(
-         rs->storage_buffers, rs->null_storage_buffer_descriptor),
+         rs->storage_buffers, rs->null_storage_buffer_descriptor, false),
       .ubo_addr_format = hk_ubo_addr_format(rs->uniform_buffers),
    };
 
@@ -1554,6 +1640,10 @@ hk_nir_lower_descriptors(nir_shader *nir,
       }
    }
 
+   ctx.ssbo_addr_format = hk_buffer_addr_format(
+      rs->storage_buffers, rs->null_storage_buffer_descriptor,
+      ctx.descriptor_generation == 10);
+
    /* First lower everything but complex SSBOs, then lower complex SSBOs.
     *
     * TODO: See if we can unify this, not sure if the fast path matters on
@@ -1570,9 +1660,10 @@ hk_nir_lower_descriptors(nir_shader *nir,
       nir, lower_ssbo_descriptor, nir_metadata_control_flow, &ctx);
 
    bool pass_sparse_buffers = false;
-   if (ctx.sparse_buffer_residency && ctx.descriptor_generation == 9) {
+   if (ctx.sparse_buffer_residency &&
+       (ctx.descriptor_generation == 9 || ctx.descriptor_generation == 10)) {
       pass_sparse_buffers = nir_shader_intrinsics_pass(
-         nir, lower_sparse_buffer_deref, nir_metadata_control_flow, &ctx);
+         nir, lower_sparse_buffer_deref, nir_metadata_none, &ctx);
    }
 
    return pass_lower_descriptors || pass_lower_ssbo || pass_sparse_buffers;
diff --git a/src/asahi/vulkan/hk_nir_lower_gs.c b/src/asahi/vulkan/hk_nir_lower_gs.c
index 96bd3fad2152..4e6503a343a1 100644
--- a/src/asahi/vulkan/hk_nir_lower_gs.c
+++ b/src/asahi/vulkan/hk_nir_lower_gs.c
@@ -8,11 +8,13 @@
 bool
 hk_nir_lower_gs(nir_shader *gs, nir_shader **count, nir_shader **rast,
                 nir_shader **pre_gs, struct poly_gs_info *info,
-                const struct agx_gen1_ray_query_layout *rq_layout)
+                const struct agx_gen1_ray_query_layout *rq_layout,
+                bool main_side_effects, poly_nir_remove_dead_fn remove_dead)
 {
    /* rq_layout is supplied only by the G17P native ray-query profile. */
    poly_nir_lower_gs_with_options(gs, count, rast, pre_gs, info,
-                                  rq_layout != NULL);
+                                  rq_layout != NULL, main_side_effects,
+                                  remove_dead);
 
    if (!rq_layout)
       return true;
diff --git a/src/asahi/vulkan/hk_nir_lower_gs.h b/src/asahi/vulkan/hk_nir_lower_gs.h
index ecb7624b2b74..bd6d64534ff3 100644
--- a/src/asahi/vulkan/hk_nir_lower_gs.h
+++ b/src/asahi/vulkan/hk_nir_lower_gs.h
@@ -5,15 +5,20 @@
 #define HK_NIR_LOWER_GS_H
 
 #include "nir.h"
+#include "poly/nir/poly_nir.h"
 
 struct agx_gen1_ray_query_layout;
 struct poly_gs_info;
 
 /* Split a GS, then lower native queries for each execution stage. Returns
  * success, not pass progress. The caller owns all output NIR even on failure.
+ * main_side_effects and remove_dead are those of
+ * poly_nir_lower_gs_with_options().
  */
 bool hk_nir_lower_gs(nir_shader *gs, nir_shader **count, nir_shader **rast,
                      nir_shader **pre_gs, struct poly_gs_info *info,
-                     const struct agx_gen1_ray_query_layout *rq_layout);
+                     const struct agx_gen1_ray_query_layout *rq_layout,
+                     bool main_side_effects,
+                     poly_nir_remove_dead_fn remove_dead);
 
 #endif
diff --git a/src/asahi/vulkan/hk_nir_lower_mesh.h b/src/asahi/vulkan/hk_nir_lower_mesh.h
index 6a4cd2441b11..013459b02825 100644
--- a/src/asahi/vulkan/hk_nir_lower_mesh.h
+++ b/src/asahi/vulkan/hk_nir_lower_mesh.h
@@ -6,6 +6,7 @@
 #include <string.h>
 #include "compiler/nir/nir.h"
 #include "compiler/shader_enums.h"
+#include "drm-uapi/asahi_drm.h"
 
 
 enum hk_mesh_mode { HK_MESH_MODE_OFF=0, HK_MESH_MODE_NATIVE_PROBE };
@@ -15,6 +16,23 @@ static inline enum hk_mesh_mode hk_mesh_mode(void) {
                                                : HK_MESH_MODE_OFF;
 }
 
+/* M5 task (object) shaders, in native-probe mesh mode on G17S. taskShader is
+ * a required feature of VK_EXT_mesh_shader. The native object->mesh record
+ * (child grid and payload at the SR62/SR63 payload address) needs the kernel
+ * to bind the TA mesh rendering buffer (0x1ca48) to task kicks, which it
+ * reports with DRM_ASAHI_FEATURE_TASK_MESH_STORAGE; taskShader is
+ * advertised only then. Without it only payload-free task shaders are
+ * accepted. */
+static inline bool hk_m5_task_enabled(void) {
+   return hk_mesh_mode()==HK_MESH_MODE_NATIVE_PROBE;
+}
+static inline bool hk_m5_task_record(uint64_t kernel_features) {
+   return hk_m5_task_enabled() &&
+          (kernel_features & DRM_ASAHI_FEATURE_TASK_MESH_STORAGE);
+}
+static inline unsigned hk_m5_task_payload_limit(uint64_t kernel_features) {
+   return hk_m5_task_record(kernel_features) ? 16384 : 0;
+}
 
 struct hk_mesh_info {
    uint32_t max_vertices;
diff --git a/src/asahi/vulkan/hk_nir_widen_16bit_varyings.c b/src/asahi/vulkan/hk_nir_widen_16bit_varyings.c
index 05c2f24dfa8e..bf29ae089a6a 100644
--- a/src/asahi/vulkan/hk_nir_widen_16bit_varyings.c
+++ b/src/asahi/vulkan/hk_nir_widen_16bit_varyings.c
@@ -29,7 +29,8 @@ widen_16bit_varying(nir_builder *b, nir_intrinsic_instr *intr,
 
    switch (intr->intrinsic) {
    case nir_intrinsic_store_output:
-   case nir_intrinsic_store_per_vertex_output: {
+   case nir_intrinsic_store_per_vertex_output:
+   case nir_intrinsic_store_per_primitive_output: {
       if (!(mode & HK_NIR_WIDEN_16BIT_OUTPUTS) ||
           stage == MESA_SHADER_FRAGMENT || intr->src[0].ssa->bit_size != 16)
          return false;
@@ -46,12 +47,15 @@ widen_16bit_varying(nir_builder *b, nir_intrinsic_instr *intr,
 
    case nir_intrinsic_load_input:
    case nir_intrinsic_load_per_vertex_input:
+   case nir_intrinsic_load_per_primitive_input:
    case nir_intrinsic_load_interpolated_input:
    case nir_intrinsic_load_output:
-   case nir_intrinsic_load_per_vertex_output: {
+   case nir_intrinsic_load_per_vertex_output:
+   case nir_intrinsic_load_per_primitive_output: {
       const bool output =
          intr->intrinsic == nir_intrinsic_load_output ||
-         intr->intrinsic == nir_intrinsic_load_per_vertex_output;
+         intr->intrinsic == nir_intrinsic_load_per_vertex_output ||
+         intr->intrinsic == nir_intrinsic_load_per_primitive_output;
       const unsigned required_mode = output ? HK_NIR_WIDEN_16BIT_OUTPUTS
                                             : HK_NIR_WIDEN_16BIT_INPUTS;
       if (!(mode & required_mode) ||
diff --git a/src/asahi/vulkan/hk_physical_device.c b/src/asahi/vulkan/hk_physical_device.c
index c28af1830188..ac48a0805b15 100644
--- a/src/asahi/vulkan/hk_physical_device.c
+++ b/src/asahi/vulkan/hk_physical_device.c
@@ -21,6 +21,7 @@
 #include "hk_entrypoints.h"
 #include "hk_image.h"
 #include "hk_instance.h"
+#include "hk_m5_acceleration_structure.h"
 #include "hk_memory_budget_core.h"
 #include "hk_agx3_vdm_decode.h"
 #include "hk_native_descriptor.h"
@@ -74,23 +75,37 @@ hk_hash_device_generation_contract(blake3_hasher *ctx,
       agx_chip_descriptor_generation(dev->chip),
       agx_chip_sampler_generation(dev->chip),
       agx_chip_sampler_heap_element_size(dev->chip),
-      hk_g17p_rt_enabled(dev),
+      hk_gen1_as_enabled(dev),
       hk_g17p_rt_enabled(dev),
    };
 
    _mesa_blake3_update(ctx, values, sizeof(values));
 }
 
+/* VK_KHR_shader_fma: G17P with its explicit-RTE tininess repair
+ * (agx_nir_lower_fma_rte()) and M5. G13-G15 do not pass the newly exposed
+ * cases. */
+static bool
+hk_shader_fma_supported(enum agx_chip chip)
+{
+   return (chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip)) &&
+          agx_chip_exact_fma32(chip);
+}
+
 static void
 hk_get_device_extensions(const struct agx_device *dev,
                          const struct hk_instance *instance,
                          struct vk_device_extension_table *ext)
 {
-   /* G15S restores the extended profile qualified on M3 Pro. Preserve the
-    * conservative AGX2 and other G15 profiles, and newer native exposure.
+   /* Keep the upstream AGX2 extension profile until the newly added common
+    * implementations have been qualified there. G15S restores the extended
+    * profile qualified on M3 Pro; the other G15 profiles stay conservative.
+    * Native AGX3 and M5 keep their existing exposure.
     */
-   const bool native_agx3 = agx_native_graphics(dev) &&
-      (dev->params.gpu_generation > 15 || dev->chip == AGX_CHIP_G15S);
+   const bool native_agx3 = (agx_native_graphics(dev) &&
+                             (dev->params.gpu_generation > 15 ||
+                              dev->chip == AGX_CHIP_G15S)) ||
+                            agx_chip_g17_m5(dev->chip);
 
    /*
     * VK_KHR_ray_query and everything it depends on.  Software traversal:
@@ -322,11 +337,14 @@ hk_get_device_features(
       .tessellationShader = true,
 
       
-      .taskShader = false,
+      .taskShader = dev->chip == AGX_CHIP_G17S &&
+                    supported_extensions->EXT_mesh_shader &&
+                    hk_m5_task_record(dev->params.features),
       .meshShader = supported_extensions->EXT_mesh_shader,
       .multiviewMeshShader = false,
       .primitiveFragmentShadingRateMeshShader = false,
-      .meshShaderQueries = supported_extensions->EXT_mesh_shader,
+      .meshShaderQueries =
+         !agx_chip_g17_m5(dev->chip) && supported_extensions->EXT_mesh_shader,
       .sampleRateShading = true,
       .dualSrcBlend = true,
       .logicOp = true,
@@ -346,7 +364,9 @@ hk_get_device_features(
       .textureCompressionASTC_LDR = true,
       .occlusionQueryPrecise = true,
       .pipelineStatisticsQuery = true,
-      .vertexPipelineStoresAndAtomics = instance->drirc.misc.enable_vertex_pipeline_stores_atomics,
+      .vertexPipelineStoresAndAtomics =
+         instance->drirc.misc.enable_vertex_pipeline_stores_atomics ||
+         agx_chip_g17_m5(dev->chip),
       .fragmentStoresAndAtomics = true,
       .shaderTessellationAndGeometryPointSize = true,
       .shaderImageGatherExtended = true,
@@ -389,7 +409,8 @@ hk_get_device_features(
       .storagePushConstant16 = true,
       /* HAL200 interpolation is 32-bit, but the compiler performs matching
        * type-correct widening at ST_VARY and narrowing after interpolation. */
-      .storageInputOutput16 = agx_chip_native_agx3(dev->chip),
+      .storageInputOutput16 = agx_chip_native_agx3(dev->chip) ||
+                              agx_chip_g17_m5(dev->chip),
       .multiview = true,
       /* ViewIndex is lowered into the layer output of the last
        * pre-rasterization stage and the draw is duplicated per view, so
@@ -410,6 +431,11 @@ hk_get_device_features(
       .shaderBufferInt64Atomics = false,
       .shaderSharedInt64Atomics = false,
       .shaderFloat16 = true,
+      .cooperativeMatrix = supported_extensions->KHR_cooperative_matrix,
+      .cooperativeMatrixRobustBufferAccess = false,
+      .shaderBFloat16Type = supported_extensions->KHR_shader_bfloat16,
+      .shaderBFloat16CooperativeMatrix = supported_extensions->KHR_shader_bfloat16,
+      .shaderBFloat16DotProduct = false,
       .shaderInt8 = true,
       .descriptorIndexing = true,
       .shaderInputAttachmentArrayDynamicIndexing = true,
@@ -432,7 +458,8 @@ hk_get_device_features(
       .descriptorBindingPartiallyBound = true,
       .descriptorBindingVariableDescriptorCount = true,
       .runtimeDescriptorArray = true,
-      .samplerFilterMinmax = instance->drirc.misc.fake_minmax,
+      .samplerFilterMinmax = supported_extensions->EXT_sampler_filter_minmax ||
+                             instance->drirc.misc.fake_minmax,
       .scalarBlockLayout = true,
       .imagelessFramebuffer = true,
       .uniformBufferStandardLayout = true,
@@ -448,11 +475,14 @@ hk_get_device_features(
        * and a conformant application therefore never reaches them. */
       .accelerationStructure = supported_extensions->KHR_acceleration_structure,
       .accelerationStructureCaptureReplay =
-         supported_extensions->KHR_acceleration_structure,
+         !agx_chip_g17_m5(dev->chip) &&
+            supported_extensions->KHR_acceleration_structure,
       .accelerationStructureIndirectBuild =
-         supported_extensions->KHR_acceleration_structure,
+         !agx_chip_g17_m5(dev->chip) &&
+            supported_extensions->KHR_acceleration_structure,
       .accelerationStructureHostCommands =
-         supported_extensions->KHR_acceleration_structure,
+         !agx_chip_g17_m5(dev->chip) &&
+            supported_extensions->KHR_acceleration_structure,
       .descriptorBindingAccelerationStructureUpdateAfterBind =
          supported_extensions->KHR_acceleration_structure,
       .rayQuery = supported_extensions->KHR_ray_query,
@@ -544,6 +574,7 @@ hk_get_device_features(
 
       /* VK_KHR_maintenance9 */
       .maintenance9 = true,
+      .maintenance10 = true,
 
       /* VK_KHR_maintenance10 */
       .maintenance10 = true,
@@ -575,10 +606,13 @@ hk_get_device_features(
 
       /* VK_KHR_shader_float_controls2 */
       .shaderFloatControls2 = true,
+      .shaderFmaFloat16 = true,
+      .shaderFmaFloat32 = true,
+      .shaderFmaFloat64 = false,
 
       /* VK_KHR_shader_fma */
-      .shaderFmaFloat16 = (dev->chip == AGX_CHIP_G17P && agx_chip_exact_fma32(dev->chip)),
-      .shaderFmaFloat32 = (dev->chip == AGX_CHIP_G17P && agx_chip_exact_fma32(dev->chip)),
+      .shaderFmaFloat16 = hk_shader_fma_supported(dev->chip),
+      .shaderFmaFloat32 = hk_shader_fma_supported(dev->chip),
       .shaderFmaFloat64 = false,
 
       /* VK_KHR_shader_maximal_reconvergence */
@@ -745,10 +779,15 @@ hk_get_device_features(
       .primitiveTopologyListRestart = true,
       .primitiveTopologyPatchListRestart = false,
 
-      /* VK_EXT_primitives_generated_query */
-      .primitivesGeneratedQuery = false,
-      .primitivesGeneratedQueryWithNonZeroStreams = false,
-      .primitivesGeneratedQueryWithRasterizerDiscard = false,
+      /* VK_EXT_primitives_generated_query: counted in the pre-graphics
+       * helpers (assembled, tessellated or geometry-shader primitives per
+       * stream), independent of rasterizer discard. */
+      .primitivesGeneratedQuery =
+         supported_extensions->EXT_primitives_generated_query,
+      .primitivesGeneratedQueryWithNonZeroStreams =
+         supported_extensions->EXT_primitives_generated_query,
+      .primitivesGeneratedQueryWithRasterizerDiscard =
+         supported_extensions->EXT_primitives_generated_query,
 
       /* VK_EXT_provoking_vertex */
       .provokingVertexLast = true,
@@ -934,6 +973,13 @@ hk_get_device_features(
       }
    }
 
+   /* Compute derivatives are NIR quad-swap lowering plus explicit-gradient
+    * samples, both available on M5; quad control stays off as on G17P. */
+   if (agx_chip_g17_m5(dev->chip)) {
+      features->computeDerivativeGroupQuads = true;
+      features->computeDerivativeGroupLinear = true;
+   }
+
    /* A multiview promise for a stage this device does not expose is
     * meaningless, so keep the pair consistent however the stage bits above
     * resolved. */
@@ -990,7 +1036,8 @@ hk_get_device_properties(const struct agx_device *dev,
 
    /* Note: update nir_shader_compiler_options.max_samples when changing this. */
    const VkSampleCountFlagBits sample_counts =
-      VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_2_BIT | VK_SAMPLE_COUNT_4_BIT;
+      VK_SAMPLE_COUNT_1_BIT | VK_SAMPLE_COUNT_2_BIT | VK_SAMPLE_COUNT_4_BIT |
+      (agx_chip_g17_m5(dev->chip) ? VK_SAMPLE_COUNT_8_BIT : 0);
 
    /* Integer colour attachments and sampled multisample images use the same
     * native tile store/reload and texel-fetch paths as other colour formats.
@@ -1023,10 +1070,13 @@ hk_get_device_properties(const struct agx_device *dev,
    unsigned max_vgt_output_components = dev->chip == AGX_CHIP_G17P ? 128 : 116;
 
    *properties = (struct vk_properties){
+      .cooperativeMatrixSupportedStages = features->cooperativeMatrix
+         ? VK_SHADER_STAGE_COMPUTE_BIT : 0,
       .apiVersion = hk_get_vk_version(),
       .driverVersion = vk_get_driver_version(),
       .vendorID = instance->drirc.debug.force_vk_vendor ?: VK_VENDOR_ID_MESA,
-      .deviceID = agx_chip_native_agx3(dev->chip) ? dev->params.chip_id : 0,
+      .deviceID = (agx_chip_native_agx3(dev->chip) || agx_chip_g17_m5(dev->chip))
+                     ? dev->params.chip_id : 0,
       .deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
 
       /* Vulkan 1.0 limits */
@@ -1036,7 +1086,8 @@ hk_get_device_properties(const struct agx_device *dev,
        * shared texture descriptor fields can encode larger arrays.
        */
       .maxImageDimension3D =
-         (dev->chip == AGX_CHIP_G17P || dev->chip == AGX_CHIP_G16G) ? 2048 : 16384,
+         (dev->chip == AGX_CHIP_G17P || dev->chip == AGX_CHIP_G16G ||
+          agx_chip_g17_m5(dev->chip)) ? 2048 : 16384,
       .maxImageDimensionCube = 16384,
       .maxImageArrayLayers = 2048,
       /* The G17P texture/PBE includes the software alignment tail in its
@@ -1060,7 +1111,9 @@ hk_get_device_properties(const struct agx_device *dev,
       .maxMemoryAllocationCount = 4096,
       .maxSamplerAllocationCount = 4000,
       .bufferImageGranularity = 0x400,
-      .sparseAddressSpaceSize = HK_SPARSE_ADDR_SPACE_SIZE,
+      .sparseAddressSpaceSize = agx_chip_g17_m5(dev->chip)
+                                   ? HK_M5_SPARSE_ADDR_SPACE_SIZE
+                                   : HK_SPARSE_ADDR_SPACE_SIZE,
       /* The classic AGX2 root upload has one USC uniform record, limited
        * to 64 halfwords. Keep upstream's eight-set limit: the native
        * sixteen-set root needs 68 halfwords and uses a separate loader.
@@ -1191,7 +1244,8 @@ hk_get_device_properties(const struct agx_device *dev,
       .maxComputeSharedMemorySize = HK_MAX_SHARED_SIZE,
       .maxComputeWorkGroupCount = {0x7fffffff, 65535, 65535},
       .maxComputeWorkGroupInvocations = 1024,
-      .maxComputeWorkGroupSize = {1024, 1024, 64},
+      .maxComputeWorkGroupSize = {
+         1024, 1024, agx_chip_g17_m5(dev->chip) ? 1024 : 64},
       .subPixelPrecisionBits = 8,
       .subTexelPrecisionBits = 8,
       .mipmapPrecisionBits = 8,
@@ -1290,10 +1344,12 @@ hk_get_device_properties(const struct agx_device *dev,
       .independentResolveNone = true,
       .independentResolve = true,
       .driverID = VK_DRIVER_ID_MESA_HONEYKRISP,
-      /* The G16G port has not completed a conformance submission. Keep the
+      /* The M4 ports have not completed a conformance submission. Keep the
        * implemented API/features available without claiming a passed CTS version.
+       * T6040 uses the G17P driver path, so check its physical identity too.
        */
-      .conformanceVersion = dev->chip == AGX_CHIP_G16G
+      .conformanceVersion = (dev->chip == AGX_CHIP_G16G ||
+                             dev->params.chip_id == 0x6040)
                                ? (VkConformanceVersion){0, 0, 0, 0}
                                : (VkConformanceVersion){1, 4, 6, 2},
       .denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL,
@@ -1344,8 +1400,11 @@ hk_get_device_properties(const struct agx_device *dev,
       .maxDescriptorSetUpdateAfterBindSampledImages = HK_MAX_DESCRIPTORS,
       .maxDescriptorSetUpdateAfterBindStorageImages = HK_MAX_DESCRIPTORS,
       .maxDescriptorSetUpdateAfterBindInputAttachments = HK_MAX_DESCRIPTORS,
-      .filterMinmaxSingleComponentFormats = instance->drirc.misc.fake_minmax,
-      .filterMinmaxImageComponentMapping = false,
+      .filterMinmaxSingleComponentFormats =
+         agx_chip_sampler_generation(dev->chip) == AGX_SAMPLER_GENERATION_GEN5 ||
+         instance->drirc.misc.fake_minmax,
+      .filterMinmaxImageComponentMapping =
+         agx_chip_sampler_generation(dev->chip) == AGX_SAMPLER_GENERATION_GEN5,
       .maxTimelineSemaphoreValueDifference = UINT64_MAX,
       .framebufferIntegerColorSampleCounts = sample_counts,
 
@@ -1395,7 +1454,8 @@ hk_get_device_properties(const struct agx_device *dev,
 
       /* VK_KHR_maintenance5 */
       .earlyFragmentMultisampleCoverageAfterSampleCounting = false,
-      .earlyFragmentSampleMaskTestBeforeSampleCounting = true,
+      .earlyFragmentSampleMaskTestBeforeSampleCounting =
+         !agx_chip_g17_m5(dev->chip),
       .depthStencilSwizzleOneSupport = true,
       .polygonModePointSize = false,
       .nonStrictSinglePixelWideLinesUseParallelogram = false,
@@ -1860,6 +1920,10 @@ hk_create_drm_physical_device(struct vk_instance *_instance,
 
    struct vk_device_extension_table supported_extensions;
    hk_get_device_extensions(&pdev->dev, instance, &supported_extensions);
+   if (hk_g16s_as_enabled(&pdev->dev)) {
+      supported_extensions.KHR_acceleration_structure = true;
+      supported_extensions.KHR_deferred_host_operations = true;
+   }
    if (hk_g17p_rt_enabled(&pdev->dev)) {
       supported_extensions.KHR_acceleration_structure = true;
       supported_extensions.KHR_deferred_host_operations = true;
@@ -1872,6 +1936,39 @@ hk_create_drm_physical_device(struct vk_instance *_instance,
       supported_extensions.KHR_ray_tracing_position_fetch = true;
    }
 
+   /* Explicit bring-up profile; query/pipeline features remain disabled until
+    * their complete runtime contract is available. */
+   if (agx_chip_g17_m5(pdev->dev.chip) &&
+       debug_get_bool_option("ASAHI_M5_NATIVE_AS", false)) {
+      supported_extensions.KHR_acceleration_structure = true;
+      supported_extensions.KHR_deferred_host_operations = true;
+      supported_extensions.KHR_ray_query = hk_m5_rq_assets_available();
+      /* Explicit pipeline qualification profile; the default remains gated. */
+      bool native_rt = supported_extensions.KHR_ray_query &&
+         debug_get_bool_option("ASAHI_M5_NATIVE_RT", false);
+      supported_extensions.KHR_ray_tracing_pipeline = native_rt;
+      supported_extensions.KHR_ray_tracing_maintenance1 = native_rt;
+      supported_extensions.KHR_pipeline_library = native_rt;
+   }
+   supported_extensions.KHR_cooperative_matrix =
+      agx_chip_g17_m5(pdev->dev.chip) &&
+      debug_get_bool_option("ASAHI_M5_COOPERATIVE_MATRIX", true);
+
+   supported_extensions.KHR_shader_bfloat16 =
+      supported_extensions.KHR_cooperative_matrix &&
+      debug_get_bool_option("ASAHI_M5_BFLOAT16", true);
+
+   /* Hardware-qualified on M5 only. */
+   supported_extensions.EXT_primitives_generated_query =
+      agx_chip_g17_m5(pdev->dev.chip);
+
+   supported_extensions.EXT_mesh_shader = pdev->dev.chip == AGX_CHIP_G17S &&
+      hk_mesh_mode() == HK_MESH_MODE_NATIVE_PROBE &&
+      (pdev->dev.params.features & DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2);
+
+   /* Gen5 adds native component-wise minimum/maximum sampler reduction. */
+   supported_extensions.EXT_sampler_filter_minmax =
+      agx_chip_sampler_generation(pdev->dev.chip) == AGX_SAMPLER_GENERATION_GEN5;
 
    if (agx_native_graphics(&pdev->dev)) {
       /* This extension controls divergent quad derivatives, not the standard
@@ -1880,8 +1977,7 @@ hk_create_drm_physical_device(struct vk_instance *_instance,
    }
 
    /* VK_KHR_shader_fma makes shaderFmaFloat32 mandatory. */
-   if (pdev->dev.chip != AGX_CHIP_G17P ||
-       !agx_chip_exact_fma32(pdev->dev.chip))
+   if (!hk_shader_fma_supported(pdev->dev.chip))
       supported_extensions.KHR_shader_fma = false;
 
    if (pdev->dev.chip == AGX_CHIP_G17P) {
@@ -1894,9 +1990,20 @@ hk_create_drm_physical_device(struct vk_instance *_instance,
       supported_extensions.EXT_texture_compression_astc_hdr = true;
    }
 
+   if (agx_chip_g17_m5(pdev->dev.chip))
+      supported_extensions.KHR_compute_shader_derivatives = true;
+
+   /* AGX3C has no witnessed native float atomic; agx_nir_lower_atomics turns
+    * fp32 add/min/max on buffers, shared memory and (lowered) images into
+    * compare-and-swap loops, so the same feature set is exact on M5. */
+   if (agx_chip_g17_m5(pdev->dev.chip)) {
+      supported_extensions.EXT_shader_atomic_float = true;
+      supported_extensions.EXT_shader_atomic_float2 = true;
+   }
+
    struct vk_features supported_features;
-   supported_extensions.EXT_mesh_shader = pdev->dev.chip == AGX_CHIP_G17P &&
-      hk_mesh_mode() == HK_MESH_MODE_NATIVE_PROBE;
+   if (pdev->dev.chip == AGX_CHIP_G17P)
+      supported_extensions.EXT_mesh_shader = hk_mesh_mode() == HK_MESH_MODE_NATIVE_PROBE;
    hk_get_device_features(&pdev->dev, instance, &supported_extensions,
                           &supported_features);
 
@@ -2004,7 +2111,11 @@ hk_create_drm_physical_device(struct vk_instance *_instance,
    }
 
    uint32_t queue_count = 1;
-   if (pdev->dev.params.features & DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES) {
+   /* Under virtio the simple-ioctl path forwards the request struct, not the
+    * buffer .pointer refers to, so the queue-limits query cannot work there.
+    */
+   if ((pdev->dev.params.features & DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES) &&
+       !pdev->dev.is_virtio) {
 #define HK_ADVERTISED_QUEUES 8u
       struct drm_asahi_queue_limits limits = {0};
       struct drm_asahi_get_params query = {
@@ -2040,6 +2151,26 @@ hk_create_drm_physical_device(struct vk_instance *_instance,
 
       .queue_count = queue_count,
    };
+
+   /* With independent kernel queues, also expose compute-only and
+    * transfer-only families. Kernel queues carry no capability class: every
+    * queue accepts render and compute commands, and hk records copies,
+    * clears and blits the same way on any family. These families only
+    * narrow what the application may record, so they reuse the universal
+    * queue implementation, and like it they create their kernel queue on
+    * first submission, in the device's one VM. The G17P and M3 kernels
+    * report independent queues too, but the families are qualified on M5. */
+   if (agx_chip_g17_m5(pdev->dev.chip) &&
+       (pdev->dev.params.features & DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES)) {
+      pdev->queue_families[pdev->queue_family_count++] = (struct hk_queue_family){
+         .queue_flags = VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT,
+         .queue_count = queue_count,
+      };
+      pdev->queue_families[pdev->queue_family_count++] = (struct hk_queue_family){
+         .queue_flags = VK_QUEUE_TRANSFER_BIT,
+         .queue_count = queue_count,
+      };
+   }
    assert(pdev->queue_family_count <= ARRAY_SIZE(pdev->queue_families));
 
    unsigned st_idx = 0;
@@ -2234,3 +2365,74 @@ hk_GetPhysicalDeviceMultisamplePropertiesEXT(
       pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){0, 0};
    }
 }
+
+VKAPI_ATTR VkResult VKAPI_CALL
+hk_GetPhysicalDeviceCooperativeMatrixPropertiesKHR(
+   VkPhysicalDevice physicalDevice, uint32_t *pPropertyCount,
+   VkCooperativeMatrixPropertiesKHR *pProperties)
+{
+   VK_FROM_HANDLE(hk_physical_device, pdev, physicalDevice);
+   VK_OUTARRAY_MAKE_TYPED(VkCooperativeMatrixPropertiesKHR, out, pProperties,
+                          pPropertyCount);
+   if (pdev->vk.supported_features.cooperativeMatrix) {
+      /* The order of this list is implementation-defined. List the 8x8x8
+       * FP32 x FP32 configuration first: CTS
+       * cooperative_matrix.op_constant_null steps through the device's
+       * configurations and, for the last one, needs it in its own table,
+       * which has no FP32 A/B operands; when the last configuration is
+       * missing there it builds a zero configuration and crashes. */
+      vk_outarray_append_typed(VkCooperativeMatrixPropertiesKHR, &out, p) {
+         *p = (VkCooperativeMatrixPropertiesKHR) {
+            .sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR,
+            .MSize = 8, .NSize = 8, .KSize = 8,
+            .AType = VK_COMPONENT_TYPE_FLOAT32_KHR,
+            .BType = VK_COMPONENT_TYPE_FLOAT32_KHR,
+            .CType = VK_COMPONENT_TYPE_FLOAT32_KHR,
+            .ResultType = VK_COMPONENT_TYPE_FLOAT32_KHR,
+            .saturatingAccumulation = VK_FALSE,
+            .scope = VK_SCOPE_SUBGROUP_KHR,
+         };
+      }
+      unsigned max_size = (pdev->dev.params.features & DRM_ASAHI_FEATURE_COMPUTE_MATRIX) ? 16 : 8;
+      for (unsigned size = max_size; size >= 8; size /= 2) {
+         const VkComponentTypeKHR types[] = {VK_COMPONENT_TYPE_FLOAT16_KHR,
+            VK_COMPONENT_TYPE_BFLOAT16_KHR};
+         for (unsigned i = 0; i < ARRAY_SIZE(types); ++i) {
+            if (types[i] == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
+                !pdev->vk.supported_features.shaderBFloat16CooperativeMatrix)
+               continue;
+            unsigned modes = types[i] == VK_COMPONENT_TYPE_BFLOAT16_KHR ? 2 : 1;
+            for (unsigned accum = 0; accum < modes; ++accum) {
+               vk_outarray_append_typed(VkCooperativeMatrixPropertiesKHR, &out, p) {
+                  *p = (VkCooperativeMatrixPropertiesKHR) {
+                     .sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR,
+                     .MSize = size, .NSize = size, .KSize = size,
+                     .AType = types[i], .BType = types[i],
+                     .CType = accum ? VK_COMPONENT_TYPE_BFLOAT16_KHR : VK_COMPONENT_TYPE_FLOAT32_KHR,
+                     .ResultType = accum ? VK_COMPONENT_TYPE_BFLOAT16_KHR : VK_COMPONENT_TYPE_FLOAT32_KHR,
+                     .saturatingAccumulation = VK_FALSE,
+                     .scope = VK_SCOPE_SUBGROUP_KHR,
+                  };
+               }
+            }
+         }
+         if (size == 16 && pdev->vk.supported_features.shaderBFloat16CooperativeMatrix) {
+            for (unsigned swap = 0; swap < 2; ++swap) {
+               vk_outarray_append_typed(VkCooperativeMatrixPropertiesKHR, &out, p) {
+                  *p = (VkCooperativeMatrixPropertiesKHR) {
+                     .sType = VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR,
+                     .MSize = 16, .NSize = 16, .KSize = 16,
+                     .AType = swap ? VK_COMPONENT_TYPE_FLOAT16_KHR : VK_COMPONENT_TYPE_BFLOAT16_KHR,
+                     .BType = swap ? VK_COMPONENT_TYPE_BFLOAT16_KHR : VK_COMPONENT_TYPE_FLOAT16_KHR,
+                     .CType = VK_COMPONENT_TYPE_FLOAT32_KHR,
+                     .ResultType = VK_COMPONENT_TYPE_FLOAT32_KHR,
+                     .saturatingAccumulation = VK_FALSE,
+                     .scope = VK_SCOPE_SUBGROUP_KHR,
+                  };
+               }
+            }
+         }
+      }
+   }
+   return vk_outarray_status(&out);
+}
diff --git a/src/asahi/vulkan/hk_ppp_state.h b/src/asahi/vulkan/hk_ppp_state.h
new file mode 100644
index 000000000000..9561a5dd85d5
--- /dev/null
+++ b/src/asahi/vulkan/hk_ppp_state.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: MIT */
+#pragma once
+
+#include "agx_m5_va.h"
+#include "agx_ppp.h"
+
+static inline bool
+hk_ppp_emit_state(enum agx_chip chip, uint8_t **out, uint64_t gpu_base,
+                   size_t size)
+{
+   if (agx_chip_g17_m5(chip)) {
+      if (!agx_m5_vdm_compact(gpu_base, &gpu_base))
+         return false;
+   }
+   return agx_ppp_emit_state(chip, out, gpu_base, size);
+}
diff --git a/src/asahi/vulkan/hk_private.h b/src/asahi/vulkan/hk_private.h
index db1475f4bc33..1bd05b9695eb 100644
--- a/src/asahi/vulkan/hk_private.h
+++ b/src/asahi/vulkan/hk_private.h
@@ -39,6 +39,15 @@
 #define HK_MAX_MULTIVIEW_VIEW_COUNT    32
 
 #define HK_SPARSE_ADDR_SPACE_SIZE (1ull << 39)
+/* M5 fills unbound sparse VA with scratch through the hypervisor's SPTM
+ * provider, which spends one page-table frame of its own heap per 32 MiB of
+ * filled VA and never returns it to the heap. Bound what one device fills.
+ * Interim capability limit: concurrent 4 x 64 GiB fills still reset the
+ * machine (at >= 7930 live table frames) while x1n1's real headroom is
+ * unknown; 16 GiB keeps one device near 1024 frames under the kernel's
+ * 2048-frame budget.
+ */
+#define HK_M5_SPARSE_ADDR_SPACE_SIZE (1ull << 34)
 #define HK_MAX_BUFFER_SIZE        (1ull << 37)
 #define HK_MAX_SHARED_SIZE        (32 * 1024)
 
@@ -77,6 +86,28 @@ hk_g17p_rt_enabled(const struct agx_device *dev)
           !debug_get_bool_option("ASAHI_G17P_RT_DISABLE", false);
 }
 
+/* T6040's 25G83 firmware does not yet admit intersection/SCS commands. AS
+ * construction, copies and refits are ordinary compute and can use the same
+ * GEN1 node layout without creating a traversal context. Require the bounded
+ * AS execution allowance as well as the exact-prior submission contract.
+ * Do not infer traversal support from this predicate.
+ */
+static inline bool
+hk_g16s_as_enabled(const struct agx_device *dev)
+{
+   const uint64_t required = DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS |
+                             DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD;
+   return agx_is_m4_pro(dev) &&
+          (dev->params.features & required) == required &&
+          !debug_get_bool_option("ASAHI_G17P_RT_DISABLE", false);
+}
+
+static inline bool
+hk_gen1_as_enabled(const struct agx_device *dev)
+{
+   return hk_g17p_rt_enabled(dev) || hk_g16s_as_enabled(dev);
+}
+
 /* Whether a logical device enables any ray-tracing feature. Only such a
  * device sets up the native G17P ray-tracing state: the ray query program
  * image and its fixed code binds, the acceleration-structure tables, and the
@@ -84,7 +115,7 @@ hk_g17p_rt_enabled(const struct agx_device *dev)
  * (hk_device_init_accel_struct_state()). Without one of these features no
  * acceleration structure, ray query or ray-tracing pipeline can exist on the
  * device. Every feature here is advertised only with the ray-tracing
- * extensions, so on any device without hk_g17p_rt_enabled() this is false.
+ * extensions. G16S also admits an AS-only device without traversal state.
  *
  * Shaders compiled for the two kinds of device differ, so this is part of
  * every shader cache key (HK_FEAT_RAY_TRACING). */
diff --git a/src/asahi/vulkan/hk_query_oq.c b/src/asahi/vulkan/hk_query_oq.c
new file mode 100644
index 000000000000..1d757a28f176
--- /dev/null
+++ b/src/asahi/vulkan/hk_query_oq.c
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * Copyright 2024 Alyssa Rosenzweig
+ * Copyright 2022-2023 Collabora Ltd. and Red Hat Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+/* Deferred occlusion availability (hk_query_oq.h): the decisions made at the
+ * end of a command buffer, kept apart from the rest of the query code so they
+ * can be tested without a device. */
+
+#include "hk_query_oq.h"
+
+#include "hk_cmd_buffer.h"
+#include "hk_device.h"
+#include "hk_sync.h"
+#include "asahi/libagx/query.h"
+#include "util/list.h"
+#include "util/u_dynarray.h"
+
+bool
+hk_defer_oq_available(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   const unsigned writes =
+      util_dynarray_num_elements(&cs->imm_writes, struct libagx_imm_write);
+   const unsigned stamps =
+      util_dynarray_num_elements(&cs->oq_stamps, struct hk_oq_stamp);
+
+   if (!stamps || stamps != writes || cs->stats.cmds ||
+       cs->stats.flushes || cs->timestamp.start.handle ||
+       cs->timestamp.end.handle || cs->query_reset_pending)
+      return false;
+
+   const struct hk_oq_stamp *s = cs->oq_stamps.data;
+   for (unsigned i = 1; i < stamps; ++i) {
+      if (s[i].render_seq != s[0].render_seq)
+         return false;
+   }
+
+   cs->oq_avail_deferred = true;
+   return true;
+}
+
+static bool
+hk_dep_reaches(const struct hk_sync_dep *dep, const struct hk_cs *cs)
+{
+   return dep->cut && dep->cut->sync.seq >= cs->sync.seq;
+}
+
+enum hk_oq_resolution {
+   /* Emit the store in place. */
+   HK_OQ_KEEP,
+   /* The render carries the availability; drop the stream. */
+   HK_OQ_STAMPED,
+   /* Nothing depends on the stream or (from compute) on its render, but the
+    * render has too few free slots: the store may move to a later deferred
+    * stream that waits on a later render. */
+   HK_OQ_MOVABLE,
+};
+
+/* Whether the render can carry the availability of the deferred stream post:
+ * nothing later in the command buffer waits on post, no compute waits on the
+ * render (or anything after it on the render subqueue), the command buffer's
+ * tail requirement does not name it, and enough timestamp slots are free.
+ */
+static enum hk_oq_resolution
+hk_stamp_oq_available(struct hk_cmd_buffer *cmd, struct hk_cs *post)
+{
+   const struct hk_oq_stamp *stamps = post->oq_stamps.data;
+   const unsigned count =
+      util_dynarray_num_elements(&post->oq_stamps, struct hk_oq_stamp);
+
+   /* While deferred, the stream may have become the tail an API timestamp
+    * rides on (hk_sync_write_timestamp()). It then stays, with its store. */
+   if (post->timestamp.start.handle || post->timestamp.end.handle ||
+       post->stats.cmds || post->stats.flushes)
+      return HK_OQ_KEEP;
+
+   struct hk_cs *render = NULL;
+   list_for_each_entry(struct hk_cs, it, &cmd->control_streams, node) {
+      if (it == post)
+         break;
+      if (it->type == HK_CS_VDM && it->sync.seq == stamps[0].render_seq)
+         render = it;
+   }
+   if (!render || post->sync.dep[HK_SYNC_VDM].cut != render)
+      return HK_OQ_KEEP;
+
+   list_for_each_entry(struct hk_cs, it, &cmd->control_streams, node) {
+      if (it == post || it == render)
+         continue;
+      if (hk_dep_reaches(&it->sync.dep[HK_SYNC_CDM], post) ||
+          hk_dep_reaches(&it->sync.dep_ta[HK_SYNC_CDM], post))
+         return HK_OQ_KEEP;
+      if (it->type == HK_CS_CDM &&
+          (hk_dep_reaches(&it->sync.dep[HK_SYNC_VDM], render) ||
+           hk_dep_reaches(&it->sync.dep_ta[HK_SYNC_VDM], render)))
+         return HK_OQ_KEEP;
+   }
+
+   for (unsigned k = 0; k < HK_SYNC_NUM_KINDS; ++k) {
+      if (hk_dep_reaches(&cmd->sync.req[k][HK_SYNC_VDM], render) ||
+          hk_dep_reaches(&cmd->sync.req[k][HK_SYNC_CDM], post))
+         return HK_OQ_KEEP;
+   }
+
+   /* Fragment end first, then the slots no API timestamp uses, then vertex
+    * start. A slot an API timestamp rides on stays with it. */
+   const unsigned order[] = {3, 1, 2, 0};
+   unsigned free_slots[4], nfree = 0;
+   for (unsigned i = 0; i < ARRAY_SIZE(order); ++i) {
+      const unsigned slot = order[i];
+      if ((slot == 0 && render->timestamp.start.handle) ||
+          (slot == 3 && render->timestamp.end.handle) ||
+          render->oq_stamp[slot].handle)
+         continue;
+      free_slots[nfree++] = slot;
+   }
+   if (nfree < count)
+      return HK_OQ_MOVABLE;
+
+   for (unsigned i = 0; i < count; ++i) {
+      render->oq_stamp[free_slots[i]] = (struct agx_timestamp_req){
+         .handle = stamps[i].handle,
+         .offset_B = stamps[i].offset_B,
+      };
+   }
+   return HK_OQ_STAMPED;
+}
+
+static void
+hk_drop_oq_stream(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   hk_sync_forget_stream(cmd, cs);
+   list_del(&cs->node);
+   hk_cs_destroy(cs);
+}
+
+/* Recording order: while an earlier stream is judged, later deferred streams
+ * still wait on later renders, so only the last can ride on its render. */
+static void
+hk_resolve_oq_forward(struct hk_cmd_buffer *cmd)
+{
+   struct hk_cs *movable = NULL;
+
+   list_for_each_entry_safe(struct hk_cs, cs, &cmd->control_streams, node) {
+      if (cs->type != HK_CS_CDM || !cs->oq_avail_deferred)
+         continue;
+
+      cs->oq_avail_deferred = false;
+      switch (hk_stamp_oq_available(cmd, cs)) {
+      case HK_OQ_STAMPED:
+         hk_drop_oq_stream(cmd, cs);
+         break;
+      case HK_OQ_MOVABLE:
+         /* The later stream's render wait covers every earlier render. */
+         if (movable && movable->sync.dep[HK_SYNC_VDM].cut->sync.seq >
+                           cs->sync.dep[HK_SYNC_VDM].cut->sync.seq) {
+            hk_dispatch_imm_writes(cmd, movable);
+            movable = NULL;
+         }
+         if (movable) {
+            util_dynarray_append_dynarray(&cs->imm_writes, &movable->imm_writes);
+            hk_drop_oq_stream(cmd, movable);
+         }
+         movable = cs;
+         break;
+      case HK_OQ_KEEP:
+         hk_dispatch_imm_writes(cmd, cs);
+         break;
+      }
+   }
+
+   if (movable)
+      hk_dispatch_imm_writes(cmd, movable);
+}
+
+/* Whether the stores of the deferred stream post can join into, a later
+ * movable stream: post carries nothing else, into waits on a render at or
+ * after the one post waits on (renders complete in order, as
+ * hk_resolve_oq_forward() also relies on), and nothing else in the command
+ * buffer waits on post, so dropping it moves its stores later without moving
+ * them past a consumer. Unlike in hk_stamp_oq_available(), compute waiting on
+ * the render does not prevent this: into is that compute, and it is the one
+ * command left for these stores. Stamping the render instead would make into
+ * wait for the host-side retirement of the render. */
+static bool
+hk_oq_joins(struct hk_cmd_buffer *cmd, struct hk_cs *post, struct hk_cs *into)
+{
+   if (post->timestamp.start.handle || post->timestamp.end.handle ||
+       post->stats.cmds || post->stats.flushes)
+      return false;
+
+   const struct hk_cs *render = post->sync.dep[HK_SYNC_VDM].cut;
+   const struct hk_cs *later = into->sync.dep[HK_SYNC_VDM].cut;
+   if (!render || !later || later->sync.seq < render->sync.seq)
+      return false;
+
+   list_for_each_entry(struct hk_cs, it, &cmd->control_streams, node) {
+      if (it == post || it == into)
+         continue;
+      if (hk_dep_reaches(&it->sync.dep[HK_SYNC_CDM], post) ||
+          hk_dep_reaches(&it->sync.dep_ta[HK_SYNC_CDM], post))
+         return false;
+   }
+
+   for (unsigned k = 0; k < HK_SYNC_NUM_KINDS; ++k) {
+      if (hk_dep_reaches(&cmd->sync.req[k][HK_SYNC_CDM], post))
+         return false;
+   }
+
+   return true;
+}
+
+/* From the end of the command buffer. A later stream whose render carries its
+ * stores is dropped before earlier streams are judged, so it no longer counts
+ * as compute waiting on the render subqueue, and an earlier render can carry
+ * its own stores too. A stream that stays (kept, or movable) still counts.
+ * Once a movable stream remains, every earlier stream that can join it does
+ * (hk_oq_joins()), whether or not its own render could have carried the
+ * stores: the movable stream is emitted anyway and waits on all of those
+ * renders, so one compute command serves the whole command buffer. */
+static void
+hk_resolve_oq_reverse(struct hk_cmd_buffer *cmd)
+{
+   struct hk_cs *movable = NULL;
+
+   list_for_each_entry_safe_rev(struct hk_cs, cs, &cmd->control_streams, node) {
+      if (cs->type != HK_CS_CDM || !cs->oq_avail_deferred)
+         continue;
+
+      cs->oq_avail_deferred = false;
+      if (movable && hk_oq_joins(cmd, cs, movable)) {
+         util_dynarray_append_dynarray(&movable->imm_writes, &cs->imm_writes);
+         hk_drop_oq_stream(cmd, cs);
+         continue;
+      }
+
+      switch (hk_stamp_oq_available(cmd, cs)) {
+      case HK_OQ_STAMPED:
+         hk_drop_oq_stream(cmd, cs);
+         break;
+      case HK_OQ_MOVABLE:
+         /* The later stream's render wait covers this stream's render. */
+         if (movable && movable->sync.dep[HK_SYNC_VDM].cut->sync.seq >=
+                           cs->sync.dep[HK_SYNC_VDM].cut->sync.seq) {
+            util_dynarray_append_dynarray(&movable->imm_writes, &cs->imm_writes);
+            hk_drop_oq_stream(cmd, cs);
+            break;
+         }
+         if (movable)
+            hk_dispatch_imm_writes(cmd, movable);
+         movable = cs;
+         break;
+      case HK_OQ_KEEP:
+         hk_dispatch_imm_writes(cmd, cs);
+         break;
+      }
+   }
+
+   if (movable)
+      hk_dispatch_imm_writes(cmd, movable);
+}
+
+bool
+hk_oq_render_stamps_supported(enum agx_chip chip)
+{
+   /* G17P and M5 (G17G/G17S) both run the shared G17 kernel runtime: a render
+    * completes only once its completion stamps cover it, after the
+    * end-of-render maintenance, and the host then writes its four stage
+    * timestamps into the requested slots before signalling the fence
+    * (g17/queue/render/retirement.rs, g17/completion.rs). */
+   return chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip);
+}
+
+void
+hk_resolve_deferred_oq_available(struct hk_cmd_buffer *cmd)
+{
+   /* The reverse order is qualified on G17P; the other native AGX3 chips keep
+    * recording order until it is measured there. */
+   if (hk_cmd_buffer_device(cmd)->dev.chip == AGX_CHIP_G17P)
+      hk_resolve_oq_reverse(cmd);
+   else
+      hk_resolve_oq_forward(cmd);
+}
diff --git a/src/asahi/vulkan/hk_query_oq.h b/src/asahi/vulkan/hk_query_oq.h
new file mode 100644
index 000000000000..77fffc7cd5dc
--- /dev/null
+++ b/src/asahi/vulkan/hk_query_oq.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * Copyright 2024 Alyssa Rosenzweig
+ * Copyright 2022-2023 Collabora Ltd. and Red Hat Inc.
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "asahi/lib/agx_chip.h"
+
+/*
+ * Native AGX3 occlusion availability.
+ *
+ * A render's retirement record carries four timestamp slots (vertex start
+ * and end, fragment start and end). The G17 kernel writes every requested
+ * slot when it retires the render, which is after the end-of-render cache
+ * maintenance that makes the render's occlusion counts visible, and before
+ * it signals the submission. A nonzero timestamp in a query's 64-bit
+ * availability slot is therefore exactly "available" there.
+ *
+ * That holds only for a kernel that publishes the slots at retirement (see
+ * hk_oq_render_stamps_supported()). The G15 firmware writes each slot from
+ * its stage's own start or end record: the vertex end lands when tiling
+ * finishes, before the fragment work writes the counts. Those chips keep the
+ * queued store and the submission wait on readback.
+ *
+ * The queued post-graphics store remains the general mechanism: it is
+ * recorded as usual, and the stream that would carry it is kept empty until
+ * the end of the command buffer. Only then is it known whether any later
+ * compute work in this command buffer depends on the render; such work would
+ * have to wait for the render's host-side retirement once the render writes
+ * timestamps, where it now waits in firmware. When nothing does and the
+ * render has free slots, the render carries the availability and the extra
+ * compute command is dropped. Otherwise the store is emitted as before.
+ */
+/* Whether renders on @chip may carry occlusion availability in their
+ * retirement timestamp slots: only chips whose kernel writes the slots at
+ * retirement, which is the G17 runtime (G17P and M5). */
+bool hk_oq_render_stamps_supported(enum agx_chip chip);
+
+struct hk_oq_stamp {
+   uint32_t handle;
+   uint32_t offset_B;
+   uint32_t render_seq;
+};
diff --git a/src/asahi/vulkan/hk_query_pool.c b/src/asahi/vulkan/hk_query_pool.c
index 049a9a1bbc14..bd3690330a76 100644
--- a/src/asahi/vulkan/hk_query_pool.c
+++ b/src/asahi/vulkan/hk_query_pool.c
@@ -5,6 +5,8 @@
  * SPDX-License-Identifier: MIT
  */
 #include "hk_query_pool.h"
+#include "hk_query_oq.h"
+#include "hk_frame_anatomy.h"
 
 #include <errno.h>
 #include <xf86drm.h>
@@ -17,6 +19,7 @@
 #include "hk_entrypoints.h"
 #include "hk_queue.h"
 #include "hk_pipeline_statistics_core.h"
+#include "hk_queue.h"
 #include "hk_shader.h"
 #include "hk_acceleration_structure.h"
 
@@ -114,25 +117,55 @@ hk_query_command_cs(struct hk_cmd_buffer *cmd)
    struct hk_sync_dep dep[HK_SYNC_NUM_SQ];
    hk_sync_src_dep(cmd, HK_SYNC_ALL, cmd->current_cs.cs, dep);
    struct hk_cs *cs = hk_sync_get_cs_after(cmd, dep);
-   if (cs && agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev))
+   if (cs && (agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) ||
+              agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip)))
       cs->g17p_timestamp_query = true;
    return cs;
 }
 
 static uint64_t
-hk_query_available_addr(struct hk_query_pool *pool, uint32_t query)
+hk_query_available_size(const struct hk_query_pool *pool)
 {
-   assert(hk_has_available(pool));
-   assert(query < pool->vk.query_count);
-   return pool->bo->va->addr + query * sizeof(uint32_t);
+   return sizeof(uint32_t) << pool->avail_shift;
 }
 
-static uint32_t *
-hk_query_available_map(struct hk_query_pool *pool, uint32_t query)
+static uint64_t
+hk_query_available_offset(struct hk_query_pool *pool, uint32_t query)
 {
    assert(hk_has_available(pool));
    assert(query < pool->vk.query_count);
-   return (uint32_t *)agx_bo_map(pool->bo) + query;
+   return (uint64_t)query * hk_query_available_size(pool);
+}
+
+static uint64_t
+hk_query_available_addr(struct hk_query_pool *pool, uint32_t query)
+{
+   return pool->bo->va->addr + hk_query_available_offset(pool, query);
+}
+
+/* The availability slot of a query, nonzero once available. A 64-bit slot
+ * holds 1 when a queued store made the query available and a retirement
+ * timestamp when the kernel did (hk_cmd_end_query_stamped()). */
+static uint64_t
+hk_query_available_value(struct hk_query_pool *pool, uint32_t query)
+{
+   void *slot = (char *)agx_bo_map(pool->bo) +
+                hk_query_available_offset(pool, query);
+   if (pool->avail_shift)
+      return p_atomic_read((uint64_t *)slot);
+   return p_atomic_read((uint32_t *)slot);
+}
+
+static void
+hk_query_set_available_host(struct hk_query_pool *pool, uint32_t query,
+                            bool available)
+{
+   void *slot = (char *)agx_bo_map(pool->bo) +
+                hk_query_available_offset(pool, query);
+   if (pool->avail_shift)
+      *(uint64_t *)slot = available;
+   else
+      *(uint32_t *)slot = available;
 }
 
 static uint64_t
@@ -163,6 +196,9 @@ hk_query_report_map(struct hk_device *dev, struct hk_query_pool *pool,
       uint64_t *queries = (uint64_t *)agx_bo_map(dev->occlusion_queries.bo);
       uint16_t *oq_index = hk_pool_oq_index_ptr(pool);
 
+      if (oq_index[query] == LIBAGX_QUERY_INDEX_UNUSED)
+         return NULL;
+
       return (struct hk_query_report *)&queries[oq_index[query]];
    } else {
       return (void *)((char *)agx_bo_map(pool->bo) +
@@ -179,12 +215,23 @@ hk_CmdWriteAccelerationStructuresPropertiesKHR(
    VK_FROM_HANDLE(hk_cmd_buffer, cmd, commandBuffer);
    VK_FROM_HANDLE(hk_query_pool, pool, queryPool);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
-
-   /* Live sidecar metadata includes retained update topology. Serialization
-    * requires the matching native serialization format and
-    * relocation path; do not invent a size for an unimplemented format. */
-   if (!dev->g17p_native_as || pool->vk.query_type != queryType ||
-       !hk_is_acceleration_structure_query(queryType) ||
+   uint32_t kind;
+   switch (queryType) {
+   case VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR:
+   case VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SIZE_KHR:
+      kind = 0;
+      break;
+   case VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE_KHR:
+      kind = 1;
+      break;
+   case VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_BOTTOM_LEVEL_POINTERS_KHR:
+      kind = 2;
+      break;
+   default:
+      vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
+      return;
+   }
+   if ((!dev->g17p_native_as && !dev->m5_native_as) || pool->vk.query_type != queryType ||
        firstQuery > pool->vk.query_count ||
        accelerationStructureCount > pool->vk.query_count - firstQuery) {
       vk_command_buffer_set_error(&cmd->vk, VK_ERROR_FEATURE_NOT_PRESENT);
@@ -192,24 +239,26 @@ hk_CmdWriteAccelerationStructuresPropertiesKHR(
    }
    if (!accelerationStructureCount || !hk_query_command_cs(cmd))
       return;
-
    for (uint32_t i = 0; i < accelerationStructureCount; ++i) {
       VK_FROM_HANDLE(vk_acceleration_structure, as, pAccelerationStructures[i]);
-      uint64_t metadata = hk_gen1_as_metadata_addr(cmd, as);
-      if (!metadata)
-         return;
-      libagx_gen1_as_property(cmd, agx_1d(1), AGX_BARRIER_ALL,
-         vk_acceleration_structure_get_va(as),
-         metadata, hk_query_report_addr(dev, pool, firstQuery + i),
-         queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR ? 0 :
-         queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SIZE_KHR ? 1 :
-         queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE_KHR ? 2 : 3);
+      if (dev->g17p_native_as) {
+         uint64_t metadata = hk_gen1_as_metadata_addr(cmd, as);
+         if (!metadata)
+            return;
+         libagx_gen1_as_property(cmd, agx_1d(1), AGX_BARRIER_ALL,
+            vk_acceleration_structure_get_va(as),
+            metadata, hk_query_report_addr(dev, pool, firstQuery + i),
+            queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR ? 0 :
+            queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SIZE_KHR ? 1 :
+            queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE_KHR ? 2 : 3);
+      } else {
+         libagx_agx3_as_property(cmd, agx_1d(1), AGX_BARRIER_ALL,
+            ALIGN_POT(vk_acceleration_structure_get_va(as), UINT64_C(1024)),
+            hk_query_report_addr(dev, pool, firstQuery + i), kind);
+      }
       if (vk_command_buffer_has_error(&cmd->vk))
          return;
-      /* The helper's strong barrier completes the value before the existing
-       * queue-write path makes availability system-wide coherent. */
-      hk_queue_write(cmd, hk_query_available_addr(pool, firstQuery + i), 1,
-                      false);
+      hk_queue_write(cmd, hk_query_available_addr(pool, firstQuery + i), 1, false);
    }
 }
 
@@ -302,6 +351,46 @@ hk_queue_write(struct hk_cmd_buffer *cmd, uint64_t address, uint32_t value,
    libagx_write_u32(cmd, agx_1d(1), AGX_BARRIER_ALL, address, value);
 }
 
+/* Publish @value after the pre-graphics helpers already recorded for the
+ * current render, in the same stream. */
+static void
+hk_queue_write_pre_gfx(struct hk_cmd_buffer *cmd, uint64_t address,
+                       uint32_t value)
+{
+   struct hk_cs *cs =
+      hk_cmd_buffer_get_cs_general(cmd, &cmd->current_cs.pre_gfx, true);
+   if (!cs)
+      return;
+
+   /* Make the producers' counters visible before the availability word, as
+    * hk_queue_write() does. */
+   hk_cdm_cache_flush(hk_cmd_buffer_device(cmd), cs);
+   libagx_write_u32(cmd, agx_1d(1), AGX_BARRIER_ALL | AGX_PREGFX, address,
+                    value);
+}
+
+static void
+hk_cmd_end_query_stamped(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
+                         uint32_t query)
+{
+   const uint32_t render_seq = cmd->current_cs.gfx->sync.seq;
+   hk_queue_write(cmd, hk_query_available_addr(pool, query), 1, true);
+
+   struct hk_cs *cs = cmd->current_cs.post_gfx;
+   if (!cs || vk_command_buffer_has_error(&cmd->vk))
+      return;
+
+   if (!cs->oq_stamps.data)
+      cs->oq_stamps = UTIL_DYNARRAY_INIT;
+
+   const struct hk_oq_stamp stamp = {
+      .handle = pool->handle,
+      .offset_B = hk_query_available_offset(pool, query),
+      .render_seq = render_seq,
+   };
+   util_dynarray_append(&cs->oq_stamps, stamp);
+}
+
 static void
 emit_zero_queries_g17p(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
                        uint32_t first_index, uint32_t num_queries,
@@ -350,6 +439,7 @@ emit_zero_queries_g17p(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
       .first_query = first_index,
       .reports_per_query = reports,
       .set_available = set_available && !publish_available,
+      .availability_shift = pool->avail_shift,
    };
    struct hk_cs *cs = cmd->current_cs.cs;
    assert(set_available || cs);
@@ -376,13 +466,13 @@ emit_zero_queries_g17p(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
    if (hk_has_available(pool)) {
       hk_drm_attachment_set_add(&cs->compute_attachments,
          hk_query_available_addr(pool, first_index),
-         (uint64_t)num_queries * sizeof(uint32_t));
+         (uint64_t)num_queries * hk_query_available_size(pool));
    }
    if (pool->oq_queries) {
-      for (uint32_t i = 0; i < num_queries; ++i) {
-         hk_drm_attachment_set_add(&cs->compute_attachments,
-            hk_query_report_addr(dev, pool, first_index + i), sizeof(uint64_t));
-      }
+      /* A later-recorded command may assign a counter before this reset runs. */
+      hk_drm_attachment_set_add(&cs->compute_attachments,
+         dev->occlusion_queries.bo->va->addr,
+         (uint64_t)AGX_MAX_OCCLUSION_QUERIES * sizeof(uint64_t));
    } else {
       hk_drm_attachment_set_add(&cs->compute_attachments,
          hk_query_report_addr(dev, pool, first_index),
@@ -392,7 +482,7 @@ emit_zero_queries_g17p(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
    if (publish_available) {
       hk_g17p_meta_fill_addr(cmd, &cmd->current_cs.cs,
          hk_query_available_addr(pool, first_index),
-         (uint64_t)num_queries * sizeof(uint32_t), 1);
+         (uint64_t)num_queries * hk_query_available_size(pool), 1);
    }
 }
 
@@ -419,6 +509,7 @@ emit_zero_queries(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
       .first_query = first_index,
       .reports_per_query = hk_reports_per_query(pool),
       .set_available = set_available,
+      .availability_shift = pool->avail_shift,
    };
 
    libagx_reset_query_struct(cmd, agx_1d(num_queries), AGX_BARRIER_ALL, info);
@@ -450,14 +541,14 @@ host_zero_queries(struct hk_device *dev, struct hk_query_pool *pool,
 
       uint64_t value = 0;
       if (hk_has_available(pool)) {
-         uint32_t *available = hk_query_available_map(pool, first_index + i);
-         *available = set_available;
+         hk_query_set_available_host(pool, first_index + i, set_available);
       } else {
          value = set_available ? 0 : LIBAGX_QUERY_UNAVAILABLE;
       }
 
-      for (unsigned j = 0; j < hk_reports_per_query(pool); ++j) {
-         reports[j].value = value;
+      if (reports) {
+         for (unsigned j = 0; j < hk_reports_per_query(pool); ++j)
+            reports[j].value = value;
       }
    }
 }
@@ -479,6 +570,13 @@ hk_CreateQueryPool(VkDevice device, const VkQueryPoolCreateInfo *pCreateInfo,
    if (!pool)
       return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
 
+   /* Native AGX3 occlusion availability lives in 64-bit slots of a bound
+    * timestamp buffer (hk_cmd_end_query_stamped()), on chips whose kernel
+    * writes them at render retirement (hk_oq_render_stamps_supported()). */
+   const bool stamped = occlusion && agx_native_graphics(&dev->dev) &&
+                        hk_oq_render_stamps_supported(dev->dev.chip);
+   pool->avail_shift = stamped ? 1 : 0;
+
    /* We place the availability first and then data. Pipeline-statistics pools
     * use the independently tested 64-bit layout path so large query counts
     * cannot wrap a 32-bit BO size. */
@@ -499,7 +597,8 @@ hk_CreateQueryPool(VkDevice device, const VkQueryPoolCreateInfo *pCreateInfo,
       pool->query_start = 0;
       if (hk_has_available(pool)) {
          pool->query_start =
-            align64((uint64_t)pool->vk.query_count * sizeof(uint32_t),
+            align64((uint64_t)pool->vk.query_count *
+                       hk_query_available_size(pool),
                     sizeof(struct hk_query_report));
       }
 
@@ -521,7 +620,7 @@ hk_CreateQueryPool(VkDevice device, const VkQueryPoolCreateInfo *pCreateInfo,
 
       /* The kernel requires that timestamp buffers are SHARED */
       enum agx_bo_flags flags = AGX_BO_WRITEBACK;
-      if (timestamp)
+      if (timestamp || stamped)
          flags |= AGX_BO_SHARED;
 
       pool->bo = agx_bo_create(&dev->dev, bo_size, 0, flags, "Query pool");
@@ -543,6 +642,10 @@ hk_CreateQueryPool(VkDevice device, const VkQueryPoolCreateInfo *pCreateInfo,
 
          assert(pool->handle && "handles are nonzero");
       }
+
+      /* Without a binding, every query end keeps the queued store. */
+      if (stamped && agx_bind_timestamps(&dev->dev, pool->bo, &pool->handle))
+         pool->handle = 0;
    }
 
    uint16_t *oq_index = hk_pool_oq_index_ptr(pool);
@@ -561,22 +664,12 @@ hk_CreateQueryPool(VkDevice device, const VkQueryPoolCreateInfo *pCreateInfo,
       }
    }
 
-   for (unsigned i = 0; i < occlusion_queries; ++i) {
-      uint64_t zero = 0;
-      unsigned index;
-
-      VkResult result = hk_descriptor_table_add(
-         dev, &dev->occlusion_queries, &zero, sizeof(uint64_t), &index);
-
-      if (result != VK_SUCCESS) {
-         hk_DestroyQueryPool(device, hk_query_pool_to_handle(pool), pAllocator);
-         return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
-      }
-
-      /* We increment as we go so we can clean up properly if we run out */
-      assert(pool->oq_queries < occlusion_queries);
-      oq_index[pool->oq_queries++] = index;
-   }
+   /* Applications may reserve far more queries than they use. Keep the pool's
+    * logical indices independent of the finite hardware counter heap, and
+    * assign a counter only when a command needs one. */
+   if (occlusion_queries)
+      memset(oq_index, 0xff, occlusion_queries * sizeof(*oq_index));
+   pool->oq_queries = occlusion_queries;
 
    if (pCreateInfo->flags & VK_QUERY_POOL_CREATE_RESET_BIT_KHR)
       host_zero_queries(dev, pool, 0, pool->vk.query_count, false);
@@ -599,7 +692,8 @@ hk_DestroyQueryPool(VkDevice device, VkQueryPool queryPool,
    uint16_t *oq_index = hk_pool_oq_index_ptr(pool);
 
    for (unsigned i = 0; oq_index && i < pool->oq_queries; ++i) {
-      hk_descriptor_table_remove(dev, &dev->occlusion_queries, oq_index[i]);
+      if (oq_index[i] != LIBAGX_QUERY_INDEX_UNUSED)
+         hk_descriptor_table_remove(dev, &dev->occlusion_queries, oq_index[i]);
    }
 
    if (pool->handle)
@@ -664,6 +758,12 @@ hk_CmdWriteTimestamp2(VkCommandBuffer commandBuffer,
    hk_flush_deferred_clears(cmd);
    VK_FROM_HANDLE(hk_query_pool, pool, queryPool);
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
+   if (unlikely(dev->frame_anatomy)) {
+      hk_frame_anatomy_note(dev, "timestamp stage=0x%llx query=%u in_pass=%u",
+                            (unsigned long long)stage, query,
+                            cmd->state.gfx.render.tilebuffer.nr_samples != 0 &&
+                               cmd->current_cs.gfx != NULL);
+   }
 
    const struct agx_timestamp_req req = {
       .addr = hk_query_report_addr(dev, pool, query),
@@ -701,17 +801,78 @@ hk_CmdWriteTimestamp2(VkCommandBuffer commandBuffer,
    }
 }
 
+static VkResult
+hk_ensure_occlusion_query(struct hk_device *dev, struct hk_query_pool *pool,
+                          uint32_t query)
+{
+   uint16_t *entry = &hk_pool_oq_index_ptr(pool)[query];
+   if (p_atomic_read(entry) != LIBAGX_QUERY_INDEX_UNUSED)
+      return VK_SUCCESS;
+
+   const uint64_t zero = 0;
+   unsigned index;
+   VkResult result = hk_descriptor_table_add(
+      dev, &dev->occlusion_queries, &zero, sizeof(zero), &index);
+   if (result != VK_SUCCESS)
+      return result;
+
+   /* Different command buffers may record the same query concurrently. Publish
+    * one initialized counter and return any redundant allocation to the heap. */
+   uint16_t previous =
+      p_atomic_cmpxchg(entry, LIBAGX_QUERY_INDEX_UNUSED, (uint16_t)index);
+   if (previous != LIBAGX_QUERY_INDEX_UNUSED)
+      hk_descriptor_table_remove(dev, &dev->occlusion_queries, index);
+   return VK_SUCCESS;
+}
+
+bool
+hk_ensure_occlusion_query_for_draw(struct hk_cmd_buffer *cmd)
+{
+   struct hk_graphics_state *gfx = &cmd->state.gfx;
+   struct hk_query_pool *pool = gfx->occlusion.pending_pool;
+   if (!pool || gfx->occlusion.mode == AGX_VISIBILITY_MODE_NONE)
+      return true;
+
+   uint32_t query = gfx->occlusion.pending_query;
+   VkResult result =
+      hk_ensure_occlusion_query(hk_cmd_buffer_device(cmd), pool, query);
+   if (result != VK_SUCCESS) {
+      vk_command_buffer_set_error(&cmd->vk, result);
+      return false;
+   }
+
+   gfx->occlusion.index = hk_pool_oq_index_ptr(pool)[query];
+   gfx->occlusion.pending_pool = NULL;
+   gfx->dirty |= HK_DIRTY_OCCLUSION;
+   return true;
+}
+
 static void
 hk_cmd_begin_end_query(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
                        uint32_t query, uint32_t index,
                        VkQueryControlFlags flags, bool end)
 {
    struct hk_device *dev = hk_cmd_buffer_device(cmd);
-   bool graphics = false;
+   bool graphics = false, pre_gfx = false;
 
    switch (pool->vk.query_type) {
    case VK_QUERY_TYPE_OCCLUSION: {
       assert(query < pool->oq_queries);
+      if (agx_chip_g17_m5(dev->dev.chip)) {
+         /* A reset precedes each legal query use and zeros any assigned
+          * counter at execution time. An empty Begin/End pair therefore
+          * only needs to publish availability, without reserving a counter. */
+         cmd->state.gfx.occlusion.pending_pool = end ? NULL : pool;
+         cmd->state.gfx.occlusion.pending_query = query;
+      } else {
+         VkResult result = hk_ensure_occlusion_query(dev, pool, query);
+         if (result != VK_SUCCESS) {
+            vk_command_buffer_set_error(&cmd->vk, result);
+            return;
+         }
+         uint16_t *oq_index = hk_pool_oq_index_ptr(pool);
+         cmd->state.gfx.occlusion.index = oq_index[query];
+      }
 
       if (end) {
          cmd->state.gfx.occlusion.mode = AGX_VISIBILITY_MODE_NONE;
@@ -719,10 +880,10 @@ hk_cmd_begin_end_query(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
          cmd->state.gfx.occlusion.mode = flags & VK_QUERY_CONTROL_PRECISE_BIT
                                             ? AGX_VISIBILITY_MODE_COUNTING
                                             : AGX_VISIBILITY_MODE_BOOLEAN;
+         cmd->state.gfx.occlusion.begin_render_seq =
+            cmd->current_cs.gfx ? cmd->current_cs.gfx->sync.seq : 0;
       }
 
-      uint16_t *oq_index = hk_pool_oq_index_ptr(pool);
-      cmd->state.gfx.occlusion.index = oq_index[query];
       cmd->state.gfx.dirty |= HK_DIRTY_OCCLUSION;
       if (end && pool->oq_submit) {
          const struct hk_oq_end oq_end = {.pool = pool, .query = query};
@@ -735,6 +896,31 @@ hk_cmd_begin_end_query(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
    case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT: {
       uint64_t addr = hk_query_report_addr(dev, pool, query);
       cmd->state.gfx.xfb_query[index] = end ? 0 : addr;
+
+      /* The counters are written by the geometry helpers in the
+       * pre-graphics stream of the render that draws. The general compute
+       * stream is not ordered after that stream: it stays open across
+       * render begins and need not wait on later pre-graphics work. On M5,
+       * publish availability after the helpers in the same pre-graphics
+       * stream, or after the graphics work when the current render has no
+       * such stream (or there is no current render). */
+      if (agx_chip_g17_m5(dev->dev.chip)) {
+         if (cmd->current_cs.gfx && cmd->current_cs.pre_gfx)
+            pre_gfx = true;
+         else
+            graphics = true;
+      }
+      break;
+   }
+
+   case VK_QUERY_TYPE_PRIMITIVES_GENERATED_EXT: {
+      /* Counted like the transform feedback query's primitives-needed
+       * report: in the pre-graphics helpers of each draw. */
+      uint64_t addr = hk_query_report_addr(dev, pool, query);
+      cmd->state.gfx.prims_generated_query[index] = end ? 0 : addr;
+      /* As for the pipeline statistics written by the same helpers, publish
+       * availability after the graphics work. */
+      graphics = true;
       break;
    }
 
@@ -773,7 +959,21 @@ hk_cmd_begin_end_query(struct hk_cmd_buffer *cmd, struct hk_query_pool *pool,
    /* We need to set available=1 after the graphics work finishes. */
    if (end) {
       perf_debug(cmd, "Query ending, type %u", pool->vk.query_type);
-      hk_queue_write(cmd, hk_query_available_addr(pool, query), 1, graphics);
+      /* Renders of one queue may finish out of order: a query that spans
+       * renders keeps the store, which waits on every render before it.
+       * Only native AGX3 occlusion pools have a bound availability buffer
+       * (pool->handle, see hk_CreateQueryPool()); M5 transform feedback
+       * queries publish after their pre-graphics helpers. */
+      if (pre_gfx)
+         hk_queue_write_pre_gfx(cmd, hk_query_available_addr(pool, query), 1);
+      else if (pool->vk.query_type == VK_QUERY_TYPE_OCCLUSION && pool->handle &&
+               cmd->current_cs.gfx &&
+               cmd->current_cs.gfx->sync.seq ==
+                  cmd->state.gfx.occlusion.begin_render_seq)
+         hk_cmd_end_query_stamped(cmd, pool, query);
+      else
+         hk_queue_write(cmd, hk_query_available_addr(pool, query), 1,
+                        graphics);
    }
 }
 
@@ -828,8 +1028,7 @@ hk_query_is_available(struct hk_device *dev, struct hk_query_pool *pool,
                       uint32_t query)
 {
    if (hk_has_available(pool)) {
-      uint32_t *available = hk_query_available_map(pool, query);
-      return p_atomic_read(available) != 0;
+      return hk_query_available_value(pool, query) != 0;
    } else {
       const struct hk_query_report *report =
          hk_query_report_map(dev, pool, query);
@@ -932,12 +1131,50 @@ hk_oq_submission_done(struct hk_device *dev, struct hk_query_pool *pool,
    }
 }
 
+/* Whether every queue has completed all the work submitted to it: nothing
+ * waits in a submit thread and each queue timeline reached its last point. */
+static bool
+hk_queues_idle(struct hk_device *dev)
+{
+   vk_foreach_queue(vk_queue, &dev->vk) {
+      struct hk_queue *queue = container_of(vk_queue, struct hk_queue, vk);
+
+      mtx_lock(&vk_queue->submit.mutex);
+      const bool queued = !list_is_empty(&vk_queue->submit.submits);
+      mtx_unlock(&vk_queue->submit.mutex);
+      if (queued)
+         return false;
+
+      const uint64_t submitted = p_atomic_read(&queue->drm.timeline_value);
+      uint64_t point = 0;
+      if (!queue->drm.syncobj ||
+          drmSyncobjQuery(dev->dev.fd, &queue->drm.syncobj, &point, 1) ||
+          point < submitted)
+         return false;
+   }
+   return true;
+}
+
+/* Wait until query becomes available. *never is set, with VK_SUCCESS, for a
+ * query that no submitted work will make available.
+ *
+ * On M5 the kernel fails a command caught in another command's GPU recovery
+ * as a victim: its fence completes with an error, the VM stays usable (its
+ * status word stays zero; hk_check_status() loses the device otherwise),
+ * and the command does not write the queries it would have ended or
+ * timestamped. Vulkan only says a query that never becomes available "may"
+ * lose the device (vkGetQueryPoolResults, VK_QUERY_RESULT_WAIT_BIT); losing
+ * it here would turn the survivable victim into a device loss. So once
+ * every queue is idle with the device intact, report the query unavailable
+ * instead of waiting for the timeout. */
 static VkResult
 hk_query_wait_for_available(struct hk_device *dev, struct hk_query_pool *pool,
-                            uint32_t query)
+                            uint32_t query, bool *never)
 {
    uint64_t abs_timeout_ns = os_time_get_absolute_timeout(HK_QUERY_TIMEOUT);
+   const bool victims = agx_chip_g17_m5(dev->dev.chip);
 
+   *never = false;
    while (os_time_get_nano() < abs_timeout_ns) {
       if (hk_query_is_available(dev, pool, query))
          return VK_SUCCESS;
@@ -945,6 +1182,21 @@ hk_query_wait_for_available(struct hk_device *dev, struct hk_query_pool *pool,
       VkResult status = vk_device_check_status(&dev->vk);
       if (status != VK_SUCCESS)
          return status;
+
+      if (victims && hk_queues_idle(dev)) {
+         /* The last producer may have retired after the check above. */
+         if (hk_query_is_available(dev, pool, query))
+            return VK_SUCCESS;
+         status = vk_device_check_status(&dev->vk);
+         if (status != VK_SUCCESS)
+            return status;
+
+         mesa_logw_once("query %u became unavailable for good: all submitted "
+                        "work completed without writing it (a command the "
+                        "kernel failed as a recovery victim)", query);
+         *never = true;
+         return VK_SUCCESS;
+      }
    }
 
    return vk_device_set_lost(&dev->vk, "query timeout");
@@ -968,15 +1220,19 @@ hk_GetQueryPoolResults(VkDevice device, VkQueryPool queryPool,
 
       bool available = hk_query_is_available(dev, pool, query);
 
+      bool never = false;
       if (!available && (flags & VK_QUERY_RESULT_WAIT_BIT)) {
-         status = hk_query_wait_for_available(dev, pool, query);
+         status = hk_query_wait_for_available(dev, pool, query, &never);
          if (status != VK_SUCCESS)
             return status;
 
-         available = true;
+         available = !never;
       }
 
-      if (available) {
+      /* A retirement timestamp (any value but the queued store's 1) was
+       * written after the end-of-render maintenance: no submission wait. */
+      if (available &&
+          !(pool->avail_shift && hk_query_available_value(pool, query) > 1)) {
          VkResult done = hk_oq_submission_done(
             dev, pool, query, flags & VK_QUERY_RESULT_WAIT_BIT, &available);
          if (done != VK_SUCCESS)
@@ -988,8 +1244,17 @@ hk_GetQueryPoolResults(VkDevice device, VkQueryPool queryPool,
       void *dst = (char *)pData + i * stride;
 
       uint32_t reports = hk_reports_per_query(pool);
-      if (!hk_query_write_cpu_result(dst, &src[0].value, reports, available,
-                                     flags))
+      if (never) {
+         /* Zero results, reported unavailable: nothing was written. */
+         uint64_t zero[32] = {0};
+         assert(reports <= ARRAY_SIZE(zero));
+         hk_query_write_cpu_result(dst, zero, reports, false,
+                                   flags | VK_QUERY_RESULT_PARTIAL_BIT);
+         continue;
+      }
+      const uint64_t zero = 0;
+      if (!hk_query_write_cpu_result(dst, src ? &src[0].value : &zero,
+                                     reports, available, flags))
          status = VK_NOT_READY;
    }
 
@@ -1055,6 +1320,16 @@ hk_cmd_copy_query_pool_results(struct hk_cmd_buffer *cmd,
          return;
       }
 
+      /* The copy reads each counter through the pool's index table. Assign
+       * the counters now even if their producers have not been recorded. */
+      for (uint32_t i = 0; pool->oq_queries && i < queryCount; i++) {
+         VkResult result = hk_ensure_occlusion_query(dev, pool, firstQuery + i);
+         if (result != VK_SUCCESS) {
+            vk_command_buffer_set_error(&cmd->vk, result);
+            return;
+         }
+      }
+
       struct libagx_copy_query_g17p_args info = {
          .availability = hk_has_available(pool) ? pool->bo->va->addr : 0,
          .results = pool->oq_queries ? dev->occlusion_queries.bo->va->addr
@@ -1065,6 +1340,7 @@ hk_cmd_copy_query_pool_results(struct hk_cmd_buffer *cmd,
          .dst_stride = stride,
          .reports_per_query = reports,
          .flags = flags,
+         .availability_shift = pool->avail_shift,
       };
 
       struct hk_cs *cs = cmd->current_cs.cs;
diff --git a/src/asahi/vulkan/hk_query_pool.h b/src/asahi/vulkan/hk_query_pool.h
index 12916e4c48ba..7a0797907950 100644
--- a/src/asahi/vulkan/hk_query_pool.h
+++ b/src/asahi/vulkan/hk_query_pool.h
@@ -21,11 +21,18 @@ struct hk_query_pool {
    struct agx_bo *bo;
    void *bo_map;
 
-   /* For timestamp queries, the kernel-assigned timestamp buffer handle. Unused
-    * for all other query types
+   /* For timestamp queries, the kernel-assigned timestamp buffer handle. On
+    * the native AGX3 path, occlusion pools bind their availability slots the
+    * same way, so the render that ends a query can mark it available through
+    * its own retirement timestamp (0 if binding failed). Unused for all other
+    * query types.
     */
    uint32_t handle;
 
+   /* log2 of the availability slot size in 32-bit words: 1 for native AGX3
+    * occlusion pools (64-bit timestamp slots), 0 otherwise. */
+   uint8_t avail_shift;
+
    unsigned oq_queries;
 
    /* Occlusion queries: the queue and timeline point of the latest
diff --git a/src/asahi/vulkan/hk_queue.c b/src/asahi/vulkan/hk_queue.c
index 4d872f2e9b56..4c7eb7eeee4f 100644
--- a/src/asahi/vulkan/hk_queue.c
+++ b/src/asahi/vulkan/hk_queue.c
@@ -10,6 +10,7 @@
  * SPDX-License-Identifier: MIT
  */
 #include "hk_queue.h"
+#include "hk_native_write_class.h"
 #include "hk_acceleration_structure.h"
 #include "hk_buffer.h"
 
@@ -21,6 +22,7 @@
 #include "agx_vdm_hal200_bind_tail_core.h"
 #include "decode.h"
 #include "hk_cmd_buffer.h"
+#include "hk_frame_anatomy.h"
 #include "hk_device.h"
 #include "hk_drm_render_core.h"
 #include "hk_drm_prior_core.h"
@@ -51,6 +53,8 @@
 #include "vk_drm_syncobj.h"
 #include "vk_sync.h"
 
+DEBUG_GET_ONCE_BOOL_OPTION(m5_chunk_timelines, "ASAHI_M5_CHUNK_TIMELINES", true)
+
 /* Signal scope includes all earlier commands on this Vulkan queue. A DRM
  * timeline point retains its predecessor fence chain; an individual submit's
  * aggregate fence covers only that submit. Transferring the queue point orders
@@ -120,6 +124,7 @@ asahi_fill_cdm_command(struct hk_device *dev, struct hk_cs *cs,
    size_t len = cs->stream_linked ? 65536 /* XXX */ : (cs->current - cs->start);
 
    *cmd = (struct drm_asahi_cmd_compute){
+      .flags = cs->m5_matrix_ops ? DRM_ASAHI_COMPUTE_USES_MATRIX : 0,
       .cdm_ctrl_stream_base = cs->addr,
       .cdm_ctrl_stream_end = cs->addr + len,
 
@@ -197,8 +202,12 @@ asahi_fill_vdm_command(struct hk_device *dev, struct hk_cs *cs,
    static_assert(sizeof(c->zls_ctrl) == sizeof(cs->cr.zls_control));
    memcpy(&c->zls_ctrl, &cs->cr.zls_control, sizeof(cs->cr.zls_control));
 
-   if (agx_native_graphics(&dev->dev)) {
-      c->isp_zls_pixels = hk_drm_g17p_zls_pixels(
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
+      c->isp_zls_pixels = agx_chip_g17_m5(dev->dev.chip)
+         ? hk_drm_m5_zls_pixels(
+              cs->cr.zls_width, cs->cr.zls_height, cs->cr.depth.buffer,
+              cs->cr.depth.meta, cs->cr.stencil.buffer, cs->cr.stencil.meta)
+         : hk_drm_g17p_zls_pixels(
          cs->cr.zls_width, cs->cr.zls_height, cs->cr.depth.buffer,
          cs->cr.depth.meta, cs->cr.stencil.buffer, cs->cr.stencil.meta);
       c->isp_bgobjdepth = hk_drm_g17p_depth_clear(
@@ -228,13 +237,17 @@ asahi_fill_vdm_command(struct hk_device *dev, struct hk_cs *cs,
     * render pass whose known primitive count is at most four, and never for
     * a continued pass or one with uncounted (indirect, mesh) work. This
     * stream's bound counts every draw of the whole pass, so it proves that. */
-   if (agx_native_graphics(&dev->dev) &&
+   if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
        (dev->dev.params.features & DRM_ASAHI_FEATURE_FEW_PRIMITIVES) &&
        cs->g17p_pass_begins && cs->g17p_pass_ends &&
        cs->g17p_primitive_bound <= 4) {
       c->flags |= DRM_ASAHI_RENDER_FEW_PRIMITIVES;
    }
 
+   if (agx_chip_g17_m5(dev->dev.chip))
+      c->flags |= hk_m5_shader_write_class_flags(dev->dev.params.features,
+                                                cs->m5_shader_writes_memory);
+
    if ((dev->dev.debug & AGX_DBG_NOCLUSTER) &&
        dev->dev.chip != AGX_CHIP_G17P) {
       c->flags |= DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING;
@@ -315,6 +328,18 @@ asahi_fill_vdm_command(struct hk_device *dev, struct hk_cs *cs,
       c->ts_frag.end.handle = cs->timestamp.end.handle;
       c->ts_frag.end.offset = cs->timestamp.end.offset_B;
    }
+
+   /* Occlusion availability (hk_stamp_oq_available()). */
+   struct drm_asahi_timestamp *slots[] = {
+      &c->ts_vtx.start, &c->ts_vtx.end, &c->ts_frag.start, &c->ts_frag.end,
+   };
+   for (unsigned i = 0; i < ARRAY_SIZE(slots); ++i) {
+      if (cs->oq_stamp[i].handle) {
+         assert(!slots[i]->handle);
+         slots[i]->handle = cs->oq_stamp[i].handle;
+         slots[i]->offset = cs->oq_stamp[i].offset_B;
+      }
+   }
 }
 
 static void
@@ -400,16 +425,17 @@ asahi_append_fragment_attachments(struct hk_device *dev,
    }
 }
 
-/* Submit all ordered control streams together up to the kernel's advertised
- * limit.  Splitting every stream into a separate ioctl serializes real games
- * at the queue boundary; keep single-stream submission only as an explicit
- * diagnostic control.
+/* Native AGX3 benefits from submitting ordered control streams together.
+ * G13X already serializes both engines, and separate ioctls let the kernel
+ * start a render while the following compute submission is prepared. This
+ * matters for post-graphics query availability and its CPU round trip.
  */
 static inline unsigned
 max_commands_per_submit(struct hk_device *dev)
 {
    return hk_drm_submit_command_limit(
-      !HK_PERF(dev, NOBATCH), dev->dev.params.max_commands_per_submission);
+      dev->dev.chip != AGX_CHIP_G13X && !HK_PERF(dev, NOBATCH),
+      dev->dev.params.max_commands_per_submission);
 }
 
 static VkResult
@@ -435,9 +461,37 @@ queue_submit_single(struct hk_device *dev, struct drm_asahi_submit *submit,
    if (dev->dev.is_virtio)
       u_rwlock_rdunlock(&dev->external_bos.lock);
 
-   if (ret)
+   if (ret) {
+      /* Opt-in diagnostic: preserve the exact rejected submission so the
+       * kernel's validation can be replayed offline. No effect unless set. */
+      const char *dump = os_get_option("ASAHI_SUBMIT_DUMP_ON_ERROR");
+      if (dump && *dump) {
+         const int err = errno;
+         static unsigned seq;
+         char path[512];
+         snprintf(path, sizeof(path), "%s/submit-%d-%u.bin", dump, (int)getpid(),
+                  p_atomic_inc_return(&seq));
+         FILE *f = fopen(path, "wb");
+         if (f) {
+            /* Header: errno, struct size, then the ioctl struct, the command
+             * stream, and the sync array (16 bytes per entry). */
+            const uint32_t hdr[4] = {(uint32_t)err, sizeof(*submit),
+                                     submit->cmdbuf_size,
+                                     submit->in_sync_count + submit->out_sync_count};
+            fwrite(hdr, sizeof(hdr), 1, f);
+            fwrite(submit, sizeof(*submit), 1, f);
+            fwrite((const void *)(uintptr_t)submit->cmdbuf, 1, submit->cmdbuf_size, f);
+            if (submit->syncs)
+               fwrite((const void *)(uintptr_t)submit->syncs,
+                      sizeof(struct drm_asahi_sync), hdr[3], f);
+            fclose(f);
+            mesa_loge("hk: rejected submission (errno %d) saved to %s", err, path);
+         }
+         errno = err;
+      }
       return vk_device_set_lost(&dev->vk,
                                 "DRM_IOCTL_ASAHI_SUBMIT failed: %m");
+   }
 
    return VK_SUCCESS;
 }
@@ -451,16 +505,26 @@ queue_submit_single(struct hk_device *dev, struct drm_asahi_submit *submit,
 static VkResult
 queue_submit_looped(struct hk_device *dev, struct hk_queue *queue,
                     struct drm_asahi_submit *submit, unsigned command_count,
-                    unsigned ring_idx, const uint64_t prior_base[2])
+                    unsigned ring_idx, const uint64_t prior_base[2],
+                    bool chunk_timelines, uint64_t first_chunk_point,
+                    struct hk_frame_anatomy_submit *anatomy)
 {
    uint8_t *cmdbuf = (uint8_t *)(uintptr_t)submit->cmdbuf;
    uint32_t offs = 0;
    unsigned submitted_vdm = 0, submitted_cdm = 0;
    unsigned commands_remaining = command_count;
+   unsigned chunk_index = 0;
+   uint64_t last_attempted_point = first_chunk_point - 1;
 
    uint64_t out_syncs =
       submit->syncs + sizeof(struct drm_asahi_sync) * submit->in_sync_count;
 
+   struct drm_asahi_sync *chunk_signal = (void *)(uintptr_t)out_syncs;
+   if (chunk_timelines)
+      assert(submit->out_sync_count == 1 &&
+             chunk_signal->sync_type == DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ &&
+             chunk_signal->handle == queue->drm.syncobj);
+
    while (commands_remaining) {
       bool first = commands_remaining == command_count;
       bool last = commands_remaining <= max_commands_per_submit(dev);
@@ -475,6 +539,8 @@ queue_submit_looped(struct hk_device *dev, struct hk_queue *queue,
       struct hk_drm_command_span span;
       if (!hk_drm_next_command_span(cmdbuf, submit->cmdbuf_size, base_offs,
                                     count, &span)) {
+         if (chunk_timelines)
+            queue->drm.timeline_value = last_attempted_point;
          return vk_device_set_lost(
             &dev->vk, "Invalid internal Asahi command stream framing");
       }
@@ -482,6 +548,8 @@ queue_submit_looped(struct hk_device *dev, struct hk_queue *queue,
       unsigned cdm_count = span.compute_count;
       unsigned vdm_count = span.render_count;
       if (last && offs != submit->cmdbuf_size) {
+         if (chunk_timelines)
+            queue->drm.timeline_value = last_attempted_point;
          return vk_device_set_lost(
             &dev->vk, "Trailing synthetic Asahi commands without hardware");
       }
@@ -497,17 +565,17 @@ queue_submit_looped(struct hk_device *dev, struct hk_queue *queue,
 
          if (!hk_drm_rebase_prior(cmd, cmd + 1, submitted_vdm,
                                   submitted_cdm, prior_base,
-                                  queue->sync.exact_prior))
+                                  queue->sync.exact_prior)) {
+            if (chunk_timelines)
+               queue->drm.timeline_value = last_attempted_point;
             return vk_device_set_lost(&dev->vk,
                                       "Invalid historical Asahi prefix");
+         }
       }
 
-      /* We can't signal the out-syncobjs until all prior work finishes. Since
-       * only the last ioctl will signal, make sure it waits on prior ioctls.
-       *
-       * TODO: there might be a more performant way to do this.
-      */
-      if (last && !first) {
+      /* Without per-chunk completion points, only the final ioctl contributes
+       * to the output fence. It must cover the preceding chunks explicitly. */
+      if (last && !first && !chunk_timelines) {
          assert(span.first_hardware_offset != SIZE_MAX);
          struct drm_asahi_cmd_header *cmd =
             (void *)(cmdbuf + span.first_hardware_offset);
@@ -524,7 +592,15 @@ queue_submit_looped(struct hk_device *dev, struct hk_queue *queue,
        * retain the first-command chain and receive the waits only once. */
       bool has_in_syncs = first ||
          (dev->dev.params.features & DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS);
-      bool has_out_syncs = last;
+      /* Distinct timeline points join completion without making this
+       * chunk execute after every command in its predecessor. */
+      bool has_out_syncs = last || chunk_timelines;
+      if (chunk_timelines)
+         chunk_signal->timeline_value = first_chunk_point + chunk_index;
+
+      /* Profiler: barriers of this chunk exactly as rebased above. */
+      if (unlikely(anatomy))
+         hk_frame_anatomy_note_barriers(anatomy, cmdbuf, base_offs, offs);
 
       struct drm_asahi_submit submit_ioctl = {
          .flags = submit->flags,
@@ -537,9 +613,15 @@ queue_submit_looped(struct hk_device *dev, struct hk_queue *queue,
          .out_sync_count = has_out_syncs ? submit->out_sync_count : 0,
       };
 
+      if (chunk_timelines)
+         last_attempted_point = chunk_signal->timeline_value;
       VkResult result = queue_submit_single(dev, &submit_ioctl, ring_idx);
-      if (result != VK_SUCCESS)
+      if (result != VK_SUCCESS) {
+         if (chunk_timelines)
+            queue->drm.timeline_value = last_attempted_point;
          return result;
+      }
+      chunk_index++;
 
       submitted_cdm += cdm_count;
       submitted_vdm += vdm_count;
@@ -603,7 +685,7 @@ hk_flush_bind(struct hk_bind_builder *b)
       op = (struct drm_asahi_gem_bind_op){
          .handle = b->dev->dev.scratch_bo->uapi_handle,
          .flags = DRM_ASAHI_BIND_READ | DRM_ASAHI_BIND_WRITE |
-                  DRM_ASAHI_BIND_SINGLE_PAGE,
+                  agx_fill_bind_flags(&b->dev->dev),
          .addr = b->va->addr + b->resourceOffset,
          .range = b->size,
       };
@@ -654,6 +736,11 @@ hk_sparse_buffer_map_commit(struct hk_bind_builder *b)
       const bool resident =
          op->handle != b->dev->dev.scratch_bo->uapi_handle;
 
+      VkResult result = hk_update_physical_sparse_map(
+         b->dev, op->addr, op->range, !resident);
+      if (result != VK_SUCCESS)
+         return result;
+
       for (uint64_t page = 0; page < pages; ++page) {
          if (first + page >= b->buffer->sparse_map_entries)
             break;
@@ -1058,7 +1145,8 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
       if (result != VK_SUCCESS)
          return result;
 
-      flush_sstt = dev->dev.chip == AGX_CHIP_G17P &&
+      flush_sstt = (dev->dev.chip == AGX_CHIP_G17P ||
+                    agx_chip_g17_m5(dev->dev.chip)) &&
                    hk_sparse_submission_updates_sstt(submit);
    }
 
@@ -1095,13 +1183,33 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
       queue->sync.submitted[HK_SYNC_VDM], queue->sync.submitted[HK_SYNC_CDM],
    };
 
-   /* G17P permits independent engines and physical render queues to finish
-    * out of order. Publish API signals from the cumulative queue point after
-    * the ioctl, preserving independent execution while covering earlier work.
+   /* Independent engines and physical render queues may finish
+    * out of order. Publish API signals from the cumulative queue point,
+    * preserving independent execution while covering earlier work: on G17P
+    * in the ioctl when the kernel accepts a cumulative output
+    * (DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS), otherwise by a transfer after
+    * the ioctl. M5 always transfers, also for its chunked timelines.
     */
-   const bool cumulative_signals = dev->dev.chip == AGX_CHIP_G17P;
+   const bool cumulative_signals =
+      dev->dev.chip == AGX_CHIP_G17P ||
+      (agx_chip_g17_m5(dev->dev.chip) &&
+       (dev->dev.params.features & DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES));
+   const bool kernel_cumulative = dev->dev.chip == AGX_CHIP_G17P &&
+      submit->signal_count > 0 &&
+      (dev->dev.params.features & DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS);
    const unsigned kernel_signal_count =
-      cumulative_signals ? 0 : submit->signal_count;
+      cumulative_signals && !kernel_cumulative ? 0 : submit->signal_count;
+   const bool chunk_timelines = cumulative_signals &&
+      (dev->dev.params.features & DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) &&
+      agx_chip_g17_m5(dev->dev.chip) &&
+      debug_get_option_m5_chunk_timelines() &&
+      command_count > max_commands_per_submit(dev);
+   const uint64_t point_count = chunk_timelines
+      ? 1 + (command_count - 1) / max_commands_per_submit(dev) : 1;
+   if (queue->drm.timeline_value > UINT64_MAX - point_count)
+      return vk_device_set_lost(&dev->vk, "Asahi queue timeline overflow");
+   const uint64_t first_chunk_point = queue->drm.timeline_value + 1;
+   queue->drm.timeline_value += point_count;
    unsigned wait_count = 0;
    struct drm_asahi_sync *syncs =
       alloca((submit->wait_count + kernel_signal_count + 1) *
@@ -1142,9 +1250,11 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
 
    /* Signal progress on the queue itself */
    syncs[wait_count + kernel_signal_count] = (struct drm_asahi_sync){
-      .sync_type = DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ,
+      .sync_type = kernel_cumulative
+         ? DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE
+         : DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ,
       .handle = queue->drm.syncobj,
-      .timeline_value = ++queue->drm.timeline_value,
+      .timeline_value = queue->drm.timeline_value,
    };
 
    for (unsigned i = 0; i < submit->command_buffer_count; ++i) {
@@ -1152,9 +1262,54 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                            queue, queue->drm.timeline_value);
    }
 
+   /* Opt-in frame anatomy profiler (HK_FRAME_ANATOMY): NULL unless enabled
+    * and this submit is sampled, so the hooks below cost one NULL test. */
+   struct hk_frame_anatomy_submit *anatomy =
+      unlikely(dev->frame_anatomy)
+         ? hk_frame_anatomy_begin_submit(dev, queue, command_count)
+         : NULL;
+
+   /* Anatomy: which waits are still unsignalled at submission; the kernel
+    * holds every command of the submission until they signal. */
+   if (unlikely(anatomy) && submit->wait_count) {
+      unsigned timeline = 0, pending = 0, pending_timeline = 0;
+      for (unsigned i = 0; i < submit->wait_count; ++i) {
+         struct vk_sync *sync = submit->waits[i].sync;
+         if (!vk_sync_type_is_drm_syncobj(sync->type))
+            continue;
+         uint32_t handle = vk_sync_as_drm_syncobj(sync)->syncobj;
+         bool done;
+         if (sync->flags & VK_SYNC_IS_TIMELINE) {
+            uint64_t point = 0;
+            timeline++;
+            done = !drmSyncobjQuery(dev->dev.fd, &handle, &point, 1) &&
+                   point >= submit->waits[i].wait_value;
+         } else {
+            done = !drmSyncobjWait(dev->dev.fd, &handle, 1, 0, 0, NULL);
+         }
+         if (!done) {
+            pending++;
+            pending_timeline += !!(sync->flags & VK_SYNC_IS_TIMELINE);
+         }
+      }
+      hk_frame_anatomy_note(dev, "submit waits=%u timeline=%u pending=%u "
+                            "pending_timeline=%u signals=%u commands=%u",
+                            submit->wait_count, timeline, pending,
+                            pending_timeline, submit->signal_count,
+                            command_count);
+   }
+
    /* Now setup the command structs */
    struct util_dynarray payload = UTIL_DYNARRAY_INIT;
 
+   /* A small G16S submit otherwise grows the heap allocation once for its
+    * header and again for the command. Keep those payloads on the stack;
+    * util_dynarray moves to the heap if attachments or more commands need
+    * additional space. The ioctl consumes the payload before we return.
+    */
+   if (agx_is_m4_pro(&dev->dev))
+      util_dynarray_init_from_stack(&payload, alloca(1024), 1024);
+
    unsigned nr_vdm = 0, nr_cdm = 0;
 
    if (flush_sstt) {
@@ -1203,6 +1358,8 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                             DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS),
                           completed)) {
       util_dynarray_fini(&payload);
+      if (unlikely(anatomy))
+         hk_frame_anatomy_end_submit(anatomy, dev, queue, VK_ERROR_DEVICE_LOST);
       return vk_device_set_lost(&dev->vk, "Asahi command ordinal overflow");
    }
 
@@ -1268,11 +1425,37 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                    cs->timestamp.end.handle || cs->timestamp.start.handle);
 
             asahi_append_compute_attachments(dev, &payload, cs);
+            if (agx_chip_g17_m5(dev->dev.chip) &&
+                (dev->dev.params.features & DRM_ASAHI_FEATURE_COMPUTE_RESOURCES)) {
+               /* The header size is 16 bits; never let it truncate. */
+               if (cs->compute_resources.size > UINT16_MAX) {
+                  util_dynarray_fini(&payload);
+                  if (unlikely(anatomy))
+                     hk_frame_anatomy_end_submit(anatomy, dev, queue,
+                                                 VK_ERROR_DEVICE_LOST);
+                  return vk_device_set_lost(
+                     &dev->vk, "M5 compute resource list too large (%u bytes)",
+                     cs->compute_resources.size);
+               }
+               const struct drm_asahi_cmd_header resources = {
+                  .cmd_type = DRM_ASAHI_SET_COMPUTE_RESOURCES,
+                  .size = cs->compute_resources.size,
+                  .vdm_barrier = DRM_ASAHI_BARRIER_NONE,
+                  .cdm_barrier = DRM_ASAHI_BARRIER_NONE,
+               };
+               util_dynarray_append(&payload, resources);
+               util_dynarray_append_array(&payload, uint8_t,
+                                         cs->compute_resources.data,
+                                         cs->compute_resources.size);
+            }
             struct drm_asahi_cmd_compute_with_prior cmd = {0};
             asahi_fill_cdm_command(dev, cs, &cmd.legacy);
             cmd.legacy.flags |= hk_as_build_flags(dev->dev.chip,
                                                   dev->dev.params.features,
                                                   cs->as_build);
+            if (unlikely(anatomy))
+               hk_frame_anatomy_compute(anatomy, dev, cs, &cmd.legacy, i,
+                                        nr_vdm + nr_cdm, payload.size);
             /* Native ray queries exist only on kernels that accept the
              * extended command (hk_g17p_rt_enabled). */
             assert(!cs->gen1_native_rq || dev->gen1_native_rq);
@@ -1284,6 +1467,20 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                cmd.intersection_first=1;cmd.intersection_count=112;
                cmd.intersection_default_program=UINT64_C(0x10010020000);
                cmd.intersection_default_control=UINT64_C(0x16000000000);
+            } else if (cs->m5_native_rq) {
+               header.size = sizeof(cmd);
+               cmd.scs_layout = dev->m5_rq_instances
+                  ? ((debug_get_bool_option("ASAHI_M5_RQ_CANDIDATE_PROBE", false) || debug_get_bool_option("ASAHI_M5_RQ_CANDIDATE_API", false))
+                     ? UINT64_C(0x01140000600100e8) /* 276-entry IFT50 tail */
+                     : UINT64_C(0x00480000600100e8))
+                  : UINT64_C(0x0028000060000070);
+               cmd.cdm_auxiliary_allocation = 14;
+               cmd.intersection_pool = dev->m5_rq_ift->va->addr;
+               cmd.intersection_pool_bytes = 0x8000;
+               cmd.intersection_first = 1;
+               cmd.intersection_count = 112;
+               cmd.intersection_default_program = dev->m5_rq_default_program;
+               cmd.intersection_default_control = dev->m5_rq_default_control;
             }
             if (queue->sync.exact_prior) {
                header.size = sizeof(cmd);
@@ -1307,6 +1504,41 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                    cs->timestamp.end.handle || cs->timestamp.start.handle);
 
             asahi_append_fragment_attachments(dev, &payload, cs);
+            if (dev->dev.params.features & DRM_ASAHI_FEATURE_RENDER_RESOURCES) {
+               /* V2 kernels take 104-byte drm_asahi_render_program_v2 records
+                * through command 7; the tile record is widened to match. */
+               const bool v2 = dev->dev.params.features &
+                               DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2;
+               const struct drm_asahi_render_program_v2 tile = {
+                  .legacy = cs->tile_resources};
+               const size_t tile_bytes =
+                  v2 ? sizeof(tile) : sizeof(cs->tile_resources);
+               /* The header size is 16 bits; never let it truncate. */
+               if (cs->render_resources.size + tile_bytes > UINT16_MAX) {
+                  util_dynarray_fini(&payload);
+                  if (unlikely(anatomy))
+                     hk_frame_anatomy_end_submit(anatomy, dev, queue,
+                                                 VK_ERROR_DEVICE_LOST);
+                  return vk_device_set_lost(
+                     &dev->vk, "M5 render resource list too large (%u bytes)",
+                     cs->render_resources.size);
+               }
+               const struct drm_asahi_cmd_header resources = {
+                  .cmd_type = v2 ? DRM_ASAHI_SET_RENDER_RESOURCES_V2
+                                 : DRM_ASAHI_SET_RENDER_RESOURCES,
+                  .size = cs->render_resources.size + tile_bytes,
+                  .vdm_barrier = DRM_ASAHI_BARRIER_NONE,
+                  .cdm_barrier = DRM_ASAHI_BARRIER_NONE,
+               };
+               util_dynarray_append(&payload, resources);
+               util_dynarray_append_array(
+                  &payload, uint8_t,
+                  v2 ? (const void *)&tile : (const void *)&cs->tile_resources,
+                  tile_bytes);
+               util_dynarray_append_array(&payload, uint8_t,
+                                         cs->render_resources.data,
+                                         cs->render_resources.size);
+            }
             struct drm_asahi_cmd_render_with_prior cmd = {0};
             asahi_fill_vdm_command(dev, cs, &cmd.legacy);
 
@@ -1325,13 +1557,35 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                 (cs->sync.barrier[HK_SYNC_VDM] != DRM_ASAHI_BARRIER_NONE ||
                  cs->sync.prior[HK_SYNC_VDM] != 0))
                cmd.legacy.flags |= DRM_ASAHI_RENDER_FRAGMENT_BARRIERS;
+
+            /* Likewise for compute: when the vertex half depends on less
+             * compute work than the fragment half (image-only producers
+             * ordered by image barriers, fragment-only destinations), the
+             * kernel orders the tiling half after that prefix alone. */
+            STATIC_ASSERT(sizeof(struct drm_asahi_cmd_render_with_stage_prior) ==
+                          sizeof(struct drm_asahi_cmd_render_with_prior) + 16);
+            const bool stage_cdm =
+               agx_chip_g17_m5(dev->dev.chip) && queue->sync.exact_prior &&
+               (dev->dev.params.features &
+                DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS) &&
+               cs->sync.ta_split &&
+               (cs->sync.barrier_ta[HK_SYNC_CDM] !=
+                   cs->sync.barrier[HK_SYNC_CDM] ||
+                cs->sync.prior_ta[HK_SYNC_CDM] != cs->sync.prior[HK_SYNC_CDM]);
+            if (stage_cdm)
+               cmd.legacy.flags |= DRM_ASAHI_RENDER_COMPUTE_STAGE_BARRIERS;
+            if (unlikely(anatomy))
+               hk_frame_anatomy_render(anatomy, dev, cs, &cmd.legacy, i,
+                                       nr_vdm + nr_cdm, payload.size);
             header.size = hk_drm_render_command_size(
-               agx_native_graphics(&dev->dev), queue->sync.exact_prior);
+               agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip),
+               queue->sync.exact_prior);
             if (queue->sync.exact_prior) {
                cmd.prior_vdm = command_prior[HK_SYNC_VDM];
                cmd.prior_cdm = command_prior[HK_SYNC_CDM];
             }
-            if (cs->gen1_native_fragment_rq || cs->gen1_native_vertex_rq) {
+            if (!agx_chip_g17_m5(dev->dev.chip) &&
+                (cs->gen1_native_fragment_rq || cs->gen1_native_vertex_rq)) {
                struct drm_asahi_cmd_render_with_ray_query extended = {.legacy = cmd};
                for (unsigned stage = 0; stage < 2; ++stage) {
                   if (!(stage ? cs->gen1_native_fragment_rq
@@ -1349,6 +1603,16 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
                header.size = sizeof(extended);
                util_dynarray_append(&payload, header);
                util_dynarray_append_array(&payload, uint8_t, &extended, header.size);
+            } else if (stage_cdm) {
+               const struct drm_asahi_cmd_render_with_stage_prior staged = {
+                  .legacy = cmd,
+                  .ta_cdm_barrier = cs->sync.barrier_ta[HK_SYNC_CDM],
+                  .ta_prior_cdm = cs->sync.prior_ta[HK_SYNC_CDM],
+               };
+               header.size = sizeof(staged);
+               util_dynarray_append(&payload, header);
+               util_dynarray_append_array(&payload, uint8_t, &staged,
+                                          header.size);
             } else {
                util_dynarray_append(&payload, header);
                util_dynarray_append_array(&payload, uint8_t, &cmd, header.size);
@@ -1375,15 +1639,22 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
       .cmdbuf = (uint64_t)(uintptr_t)(payload.data),
    };
 
+   /* Profiler: "edges" timestamp claims happen in the final payload. */
+   if (unlikely(anatomy))
+      hk_frame_anatomy_prepare(anatomy, dev, payload.data, payload.size);
+
    VkResult result;
    if (command_count <= max_commands_per_submit(dev)) {
+      if (unlikely(anatomy))
+         hk_frame_anatomy_note_barriers(anatomy, payload.data, 0, payload.size);
       result =
          queue_submit_single(dev, &submit_ioctl, queue->drm.virt_ring_idx);
    } else {
       result = queue_submit_looped(dev, queue, &submit_ioctl, command_count,
-                                   queue->drm.virt_ring_idx, prior_base);
+                                   queue->drm.virt_ring_idx, prior_base,
+                                   chunk_timelines, first_chunk_point, anatomy);
    }
-   if (cumulative_signals) {
+   if (cumulative_signals && !kernel_cumulative) {
       /* The kernel can publish an error fence even when submission fails.
        * Forward it when present, without waiting for a missing queue point
        * or replacing the original submit error with a transfer error. */
@@ -1394,6 +1665,9 @@ queue_submit(struct hk_device *dev, struct hk_queue *queue,
 
    util_dynarray_fini(&payload);
 
+   if (unlikely(anatomy))
+      hk_frame_anatomy_end_submit(anatomy, dev, queue, result);
+
    return result;
 }
 
@@ -1470,9 +1744,18 @@ hk_queue_init(struct hk_device *dev, const VkDeviceQueueCreateInfo *pCreateInfo,
       priority_info ? priority_info->globalPriority
                     : VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR;
 
-   /* High-priority scheduling still needs kernel qualification. */
+   /* On M5, quantize relative priorities to the two unprivileged firmware
+    * classes. An explicit global class takes precedence; the kernel checks
+    * permission for elevated classes. No scheduling work is needed during
+    * submission. Elsewhere high-priority scheduling still needs kernel
+    * qualification.
+    */
    enum drm_asahi_priority drm_priority = translate_priority(priority);
-   if (drm_priority >= DRM_ASAHI_PRIORITY_HIGH) {
+   if (agx_chip_g17_m5(dev->dev.chip)) {
+      if (!priority_info &&
+          pCreateInfo->pQueuePriorities[index_in_family] < 0.5f)
+         drm_priority = DRM_ASAHI_PRIORITY_LOW;
+   } else if (drm_priority >= DRM_ASAHI_PRIORITY_HIGH) {
       vk_free(&dev->vk.alloc, queue);
       return VK_ERROR_NOT_PERMITTED_EXT;
    }
@@ -1484,6 +1767,8 @@ hk_queue_init(struct hk_device *dev, const VkDeviceQueueCreateInfo *pCreateInfo,
    }
 
    queue->vk.driver_submit = hk_queue_submit;
+   queue->sync.precise_tile_evictions = agx_is_m4_pro(&dev->dev);
+   queue->sync.direct_single_footprints = agx_is_m4_pro(&dev->dev);
    queue->sync.serialize = dev->dev.params.gpu_generation >= 13 &&
                            dev->dev.params.gpu_generation <= 14;
    /* A serialized queue orders every command after both engines' complete
@@ -1523,7 +1808,8 @@ hk_queue_init(struct hk_device *dev, const VkDeviceQueueCreateInfo *pCreateInfo,
    queue->drm.timeline_value = initial_value;
    queue->drm.signalled_value = initial_value;
 
-   if (dev->dev.chip == AGX_CHIP_G17P &&
+   if ((dev->dev.chip == AGX_CHIP_G17P ||
+        agx_chip_g17_m5(dev->dev.chip)) &&
        dev->vk.enabled_features.sparseBinding) {
       queue->g17p_sstt_flush_bo =
          agx_bo_create(&dev->dev, AIL_PAGESIZE, 0, 0, "SSTT flush stream");
@@ -1534,9 +1820,9 @@ hk_queue_init(struct hk_device *dev, const VkDeviceQueueCreateInfo *pCreateInfo,
 
       uint8_t *stream = agx_bo_map(queue->g17p_sstt_flush_bo);
       if (!stream ||
-          !agx3_cdm_sstt_flush_invalidate_pack(AGX_CHIP_G17P, stream) ||
+          !agx3_cdm_sstt_flush_invalidate_pack(dev->dev.chip, stream) ||
           !agx3_cdm_stream_terminate_pack(
-             AGX_CHIP_G17P,
+             dev->dev.chip,
              stream + AGX3_CDM_SSTT_FLUSH_INVALIDATE_SIZE)) {
          hk_queue_finish(dev, queue);
          return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
diff --git a/src/asahi/vulkan/hk_ray_tracing_pipeline.c b/src/asahi/vulkan/hk_ray_tracing_pipeline.c
index bb897e719b00..c84998b662c3 100644
--- a/src/asahi/vulkan/hk_ray_tracing_pipeline.c
+++ b/src/asahi/vulkan/hk_ray_tracing_pipeline.c
@@ -13,6 +13,21 @@
  * (hk_nir_ray_tracing.c) is already a port of lavapipe's, so the two halves fit
  * without reinterpretation.
  *
+ * On M5, two departures from lavapipe's single megakernel, both for
+ * throughput:
+ *
+ *  - One kernel is compiled PER RAY-GENERATION GROUP rather than one for the
+ *    whole pipeline.  A trace runs exactly one raygen, so inlining every raygen
+ *    into one kernel only makes each dispatch pay for the union: its code
+ *    size, its register count (the occupancy of every trace was that of the
+ *    worst raygen) and its scratch (the stack was sized for the largest raygen
+ *    frame).  hk_cmd_trace_rays.c selects the kernel on the GPU from the SBT.
+ *
+ *  - A stage that several groups name (a hit group with and without an
+ *    any-hit shader commonly share one closest-hit) is inlined once per call
+ *    site, guarded by the disjunction of the groups' handle tests, instead of
+ *    once per group.
+ *
  * The one real divergence is how the trace-rays record reaches the shader.
  * lavapipe binds it as an internal SSBO; hk has no internal descriptor set, so
  * the record's device address is published in the compute root descriptor table
@@ -312,10 +327,18 @@ struct hk_rt_compiler {
    VkPipelineCreateFlags2KHR flags;
    bool position_fetch;
    bool native_agx;
-   nir_variable *native_query, *native_hit_kind;
+   bool native_m5;
+   /* M5 compiles one kernel per ray-generation group and shares a stage's
+    * call site between the groups naming it. Other chips keep the single
+    * megakernel with one call site per group. */
+   bool m5;
    bool callable_dispatch;
 
    struct hk_rt_shader_state state;
+   nir_variable *native_query;
+   /* Hit kind reported by the intersection shader of a committed generated
+    * (procedural) hit; triangle hits derive theirs from the facing bit. */
+   nir_variable *native_hit_kind;
 
    struct hash_table **functions;
    struct hash_table *function_depths;
@@ -516,6 +539,98 @@ hk_call_ray_tracing_stage(nir_builder *b, struct hk_rt_compiler *compiler,
    }
 }
 
+/*
+ * Which stage of a shader group a call site dispatches through.
+ * HK_RT_GROUP_RECURSIVE is the group's callable stage (general or closest-hit),
+ * filtered by `stage_filter`; the other two are reached from inside traversal.
+ */
+enum hk_rt_group_stage {
+   HK_RT_GROUP_RECURSIVE,
+   HK_RT_GROUP_AHIT,
+   HK_RT_GROUP_ISEC,
+};
+
+struct hk_rt_stage_call {
+   nir_shader *stage;
+   /* The any-hit shader paired with an intersection shader's group, else NULL:
+    * reportIntersectionEXT lowering binds it to the intersection body. */
+   nir_shader *ahit;
+   nir_def *cond;
+   /* Without merged call sites (cond == NULL), the group's handle test is
+    * emitted where the call is. */
+   nir_def *handle;
+   uint32_t handle_index;
+};
+
+/*
+ * Collect the distinct stage bodies the pipeline's groups select for `handle`
+ * through `which`, each guarded by the disjunction of its groups' handle
+ * tests.  Several groups commonly name the same stage; emitting one inlined
+ * copy per call site instead of one per group keeps a closest-hit shared by N
+ * hit groups from appearing N times at every traceRayEXT.  The groups are
+ * indistinguishable inside the stage: the shader record pointer was already
+ * taken from the SBT record itself, not from the group.  `calls` must hold
+ * group_count entries.
+ */
+static unsigned
+hk_rt_collect_stage_calls(nir_builder *b, struct hk_rt_compiler *compiler,
+                          nir_def *handle, enum hk_rt_group_stage which,
+                          mesa_shader_stage stage_filter,
+                          struct hk_rt_stage_call *calls)
+{
+   unsigned count = 0;
+
+   for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
+      const struct hk_ray_tracing_group *group = compiler->pipeline->groups + i;
+      const uint32_t index = which == HK_RT_GROUP_AHIT   ? group->ahit_index
+                             : which == HK_RT_GROUP_ISEC ? group->isec_index
+                                                         : group->recursive_index;
+      if (index == VK_SHADER_UNUSED_KHR)
+         continue;
+
+      nir_shader *stage = compiler->pipeline->stages[index]->nir;
+      if (which == HK_RT_GROUP_RECURSIVE && stage->info.stage != stage_filter)
+         continue;
+
+      nir_shader *ahit = NULL;
+      if (which == HK_RT_GROUP_ISEC && group->ahit_index != VK_SHADER_UNUSED_KHR)
+         ahit = compiler->pipeline->stages[group->ahit_index]->nir;
+
+      if (!compiler->m5) {
+         calls[count++] = (struct hk_rt_stage_call){
+            .stage = stage, .ahit = ahit, .handle = handle,
+            .handle_index = group->handle.index,
+         };
+         continue;
+      }
+
+      nir_def *test = nir_ieq_imm(b, handle, group->handle.index);
+
+      unsigned j;
+      for (j = 0; j < count; ++j) {
+         if (calls[j].stage == stage && calls[j].ahit == ahit)
+            break;
+      }
+
+      if (j == count) {
+         calls[count++] = (struct hk_rt_stage_call){
+            .stage = stage, .ahit = ahit, .cond = test,
+         };
+      } else {
+         calls[j].cond = nir_ior(b, calls[j].cond, test);
+      }
+   }
+
+   return count;
+}
+
+static nir_def *
+hk_rt_stage_call_cond(nir_builder *b, const struct hk_rt_stage_call *call)
+{
+   return call->cond ? call->cond
+                     : nir_ieq_imm(b, call->handle, call->handle_index);
+}
+
 /* Preserve invocation state across nested calls. SSA/DCE eliminates fields
  * that the caller never reads after the return; no fixed spill frame is added. */
 struct hk_rt_saved_state {
@@ -593,18 +708,13 @@ hk_execute_callable_inline(nir_builder *b, struct hk_rt_compiler *compiler,
    nir_store_var(b, state->shader_call_data_offset,
                  nir_iadd_imm(b, payload, -(int32_t)stack_size), 0x1);
 
-   for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
-      struct hk_ray_tracing_group *group = compiler->pipeline->groups + i;
-      if (group->recursive_index == VK_SHADER_UNUSED_KHR)
-         continue;
-
-      nir_shader *stage =
-         compiler->pipeline->stages[group->recursive_index]->nir;
-      if (stage->info.stage != MESA_SHADER_CALLABLE)
-         continue;
-
-      nir_push_if(b, nir_ieq_imm(b, callable_entry.value, group->handle.index));
-      hk_call_ray_tracing_stage(b, compiler, stage, false);
+   struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+   const unsigned count = hk_rt_collect_stage_calls(
+      b, compiler, callable_entry.value, HK_RT_GROUP_RECURSIVE,
+      MESA_SHADER_CALLABLE, calls);
+   for (unsigned i = 0; i < count; i++) {
+      nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+      hk_call_ray_tracing_stage(b, compiler, calls[i].stage, false);
       nir_pop_if(b, NULL);
    }
 
@@ -934,24 +1044,17 @@ hk_handle_aabb_intersection(nir_builder *b,
    nir_store_var(b, state->shader_record_ptr, isec_entry.shader_record_ptr,
                  0x1);
 
-   for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
-      struct hk_ray_tracing_group *group = compiler->pipeline->groups + i;
-      if (group->isec_index == VK_SHADER_UNUSED_KHR)
-         continue;
-
-      nir_shader *stage = compiler->pipeline->stages[group->isec_index]->nir;
-
-      nir_push_if(b, nir_ieq_imm(b, isec_entry.value, group->handle.index));
-      hk_call_ray_tracing_stage(b, compiler, stage, true);
+   struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+   const unsigned count = hk_rt_collect_stage_calls(
+      b, compiler, isec_entry.value, HK_RT_GROUP_ISEC, MESA_SHADER_NONE, calls);
+   for (unsigned i = 0; i < count; i++) {
+      nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+      hk_call_ray_tracing_stage(b, compiler, calls[i].stage, true);
       nir_pop_if(b, NULL);
 
-      nir_shader *ahit_stage = NULL;
-      if (group->ahit_index != VK_SHADER_UNUSED_KHR)
-         ahit_stage = compiler->pipeline->stages[group->ahit_index]->nir;
-
       struct hk_lower_isec_state isec_state = {
          .compiler = compiler,
-         .ahit = ahit_stage,
+         .ahit = calls[i].ahit,
       };
       nir_shader_intrinsics_pass(b->shader, hk_lower_isec_intrinsic,
                                  nir_metadata_none, &isec_state);
@@ -1029,15 +1132,13 @@ hk_handle_triangle_intersection(nir_builder *b,
       nir_store_var(b, state->shader_record_ptr,
                     ahit_entry.shader_record_ptr, 0x1);
 
-      for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
-         struct hk_ray_tracing_group *group = compiler->pipeline->groups + i;
-         if (group->ahit_index == VK_SHADER_UNUSED_KHR)
-            continue;
-
-         nir_shader *stage = compiler->pipeline->stages[group->ahit_index]->nir;
-
-         nir_push_if(b, nir_ieq_imm(b, ahit_entry.value, group->handle.index));
-         hk_call_ray_tracing_stage(b, compiler, stage, true);
+      struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+      const unsigned count = hk_rt_collect_stage_calls(
+         b, compiler, ahit_entry.value, HK_RT_GROUP_AHIT, MESA_SHADER_NONE,
+         calls);
+      for (unsigned i = 0; i < count; i++) {
+         nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+         hk_call_ray_tracing_stage(b, compiler, calls[i].stage, true);
          nir_pop_if(b, NULL);
       }
    }
@@ -1221,6 +1322,205 @@ static void hk_rt_dispatch_hit_or_miss(nir_builder *, struct hk_rt_compiler *,
                                        nir_def *, nir_def *);
 static nir_variable *hk_deref_base_var(nir_deref_instr *);
 
+static void
+hk_rt_m5_load_hit_state(nir_builder *b, struct hk_rt_compiler *compiler,
+                        nir_def *q, nir_def *as, bool committed,
+                        nir_def *generated)
+{
+   struct hk_rt_shader_state *state = &compiler->state;
+   nir_def *instance_id = nir_rq_load(b, 1, 32, q, .committed = committed,
+      .ray_query_value = nir_ray_query_value_intersection_instance_id);
+   /* Native pointer table entries contain record-relative-offset +4. */
+   nir_def *table = nir_load_global(b, 1, 64, nir_iadd_imm(b, as, 128),
+                                    .align_mul = 8);
+   nir_def *entry = nir_load_global(b, 1, 64,
+      nir_iadd(b, nir_iadd(b, as, table), nir_u2u64(b, nir_ishl_imm(b, instance_id, 4))),
+      .align_mul = 8);
+   nir_def *instance = nir_iadd_imm(b, nir_iadd(b, as, entry), -4);
+   nir_store_var(b, state->instance_addr, instance, 1);
+   nir_def *primitive = nir_rq_load(b, 1, 32, q, .committed = committed,
+      .ray_query_value = nir_ray_query_value_intersection_primitive_index);
+   nir_store_var(b, state->primitive_id, primitive, 1);
+   nir_def *geometry = nir_rq_load(b, 1, 32, q, .committed = committed,
+      .ray_query_value = nir_ray_query_value_intersection_geometry_index);
+   nir_store_var(b, state->geometry_id_and_flags, geometry, 1);
+   nir_def *distance = nir_rq_load(b, 1, 32, q, .committed = committed,
+      .ray_query_value = nir_ray_query_value_intersection_t);
+   nir_store_var(b, state->tmax, distance, 1);
+   nir_def *front = nir_rq_load(b, 1, 1, q, .committed = committed,
+      .ray_query_value = nir_ray_query_value_intersection_front_face);
+   nir_def *hit_kind =
+      nir_bcsel(b, front, nir_imm_int(b, 0xfe), nir_imm_int(b, 0xff));
+   if (generated)
+      hit_kind = nir_bcsel(b, generated,
+                           nir_load_var(b, compiler->native_hit_kind), hit_kind);
+   nir_store_var(b, state->hit_kind, hit_kind, 1);
+   /* A generated hit's attributes were written by the intersection shader
+    * into the same frame; do not overwrite them with barycentrics. */
+   if (generated)
+      nir_push_if(b, nir_inot(b, generated));
+   nir_def *bary = nir_rq_load(b, 2, 32, q, .committed = committed,
+      .ray_query_value = nir_ray_query_value_intersection_barycentrics);
+   nir_store_scratch(b, bary, nir_load_var(b, state->stack_ptr),
+                     .align_mul = 8, .write_mask = 3);
+   if (generated)
+      nir_pop_if(b, NULL);
+   nir_def *otw = nir_iadd(b, instance,
+      nir_load_global(b, 1, 64, nir_iadd_imm(b, instance, 104), .align_mul = 8));
+   nir_def *sbt = nir_load_global(b, 1, 32, nir_iadd_imm(b, otw, 48), .align_mul = 4);
+   nir_store_var(b, state->traversal.sbt_offset_and_flags, sbt, 1);
+   nir_store_var(b, state->sbt_index,
+                 hk_hit_sbt_index(b, state, geometry), 1);
+}
+
+/* Procedural candidate of the native query: run the hit group's intersection
+ * shader with the candidate state loaded. hk_lower_isec_intrinsic turns each
+ * reportIntersectionEXT into a [tmin, committed t] window check, the any-hit
+ * stage and an accept/rollback of tmax/hit kind, exactly as on the software
+ * traversal. An accepted report then commits the candidate at the final
+ * reported distance through rayQueryGenerateIntersection semantics. */
+static void
+hk_trace_ray_m5_aabb_candidate(nir_builder *b, struct hk_rt_compiler *compiler,
+                               nir_def *q, nir_def *flags)
+{
+   struct hk_rt_shader_state *state = &compiler->state;
+
+   bool has_isec = false;
+   for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
+      if (compiler->pipeline->groups[i].isec_index != VK_SHADER_UNUSED_KHR) {
+         has_isec = true;
+         break;
+      }
+   }
+   /* Without an intersection shader a procedural geometry has no defined
+    * intersection: the candidate is left unconfirmed. */
+   if (!has_isec)
+      return;
+
+   /* gl_RayTmaxEXT inside an intersection shader is the current committed
+    * distance, not the candidate record's distance. */
+   nir_store_var(b, state->tmax, nir_rq_load(b, 1, 32, q, .committed = true,
+      .ray_query_value = nir_ray_query_value_intersection_t), 1);
+   nir_store_var(b, state->opaque, nir_rq_load(b, 1, 1, q, .committed = false,
+      .ray_query_value = nir_ray_query_value_intersection_candidate_aabb_opaque), 1);
+   nir_store_var(b, state->accept, nir_imm_false(b), 0x1);
+   nir_store_var(b, state->terminate,
+      nir_test_mask(b, flags, SpvRayFlagsTerminateOnFirstHitKHRMask), 1);
+
+   struct hk_sbt_entry isec_entry = hk_load_sbt_entry(
+      b, nir_load_var(b, state->sbt_index), HK_RT_CMD_HIT_ADDRESS);
+   nir_store_var(b, state->shader_record_ptr, isec_entry.shader_record_ptr,
+                 0x1);
+
+   struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+   const unsigned count = hk_rt_collect_stage_calls(
+      b, compiler, isec_entry.value, HK_RT_GROUP_ISEC, MESA_SHADER_NONE, calls);
+   for (unsigned i = 0; i < count; i++) {
+      nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+      hk_call_ray_tracing_stage(b, compiler, calls[i].stage, true);
+      nir_pop_if(b, NULL);
+
+      struct hk_lower_isec_state isec_state = {
+         .compiler = compiler,
+         .ahit = calls[i].ahit,
+      };
+      nir_shader_intrinsics_pass(b->shader, hk_lower_isec_intrinsic,
+                                 nir_metadata_none, &isec_state);
+   }
+
+   nir_push_if(b, nir_load_var(b, state->accept));
+   {
+      nir_store_var(b, compiler->native_hit_kind,
+                    nir_load_var(b, state->hit_kind), 1);
+      nir_rq_generate_intersection(b, q, nir_load_var(b, state->tmax));
+      nir_push_if(b, nir_load_var(b, state->terminate));
+      nir_rq_terminate(b, q);
+      nir_pop_if(b, NULL);
+   }
+   nir_pop_if(b, NULL);
+}
+
+/* Native AS buffers use the captured Apple layout. Reuse the Vulkan query
+ * lowering for traversal; software hk_bvh traversal cannot interpret them. */
+static void
+hk_trace_ray_m5(nir_builder *b, struct hk_rt_compiler *compiler,
+                nir_def *as, nir_def *flags, nir_def *mask,
+                nir_def *origin, nir_def *tmin, nir_def *dir, nir_def *tmax)
+{
+   struct hk_rt_shader_state *state = &compiler->state;
+   /* Each generated traversal drains/terminates its native queue and copies
+    * the committed hit into state before invoking closest-hit or miss code.
+    * Those stages therefore reuse this context for their subsequent traces.
+    * Any application ray-query variables remain separate and retain the
+    * lowerer's multiple-live-query guard. */
+   if (!compiler->native_query) {
+      compiler->native_query = nir_variable_create(b->shader, nir_var_shader_temp,
+         glsl_uint64_t_type(), "rt.query");
+      compiler->native_query->data.ray_query = true;
+      compiler->native_hit_kind = nir_variable_create(b->shader,
+         nir_var_shader_temp, glsl_uint_type(), "rt.hit_kind");
+   }
+   nir_variable *query = compiler->native_query;
+   nir_def *q = &nir_build_deref_var(b, query)->def;
+   /* SkipClosestHitShader selects a pipeline stage after traversal; it
+    * does not change the native intersection query. */
+   nir_def *query_flags = nir_iand_imm(b, flags,
+      ~SpvRayFlagsSkipClosestHitShaderKHRMask);
+   nir_rq_initialize(b, q, as, query_flags, mask, origin, tmin, dir, tmax);
+   nir_push_loop(b);
+   nir_break_if(b, nir_inot(b, nir_rq_proceed(b, 1, q)));
+   if (debug_get_bool_option("ASAHI_M5_RQ_CANDIDATE_API", false)) {
+      /* The native query API internally commits opaque hits. Each returned
+       * triangle is nonopaque and must run the selected any-hit stage before
+       * confirmation. Rejected candidates leave traversal uncommitted. */
+      hk_rt_m5_load_hit_state(b, compiler, q, as, false, NULL);
+      /* A procedural (AABB) candidate runs the hit group's intersection
+       * shader instead of the any-hit stage; reportIntersectionEXT commits
+       * it at the reported distance after the optional any-hit stage. */
+      nir_def *candidate_type = nir_rq_load(b, 1, 32, q, .committed = false,
+         .ray_query_value = nir_ray_query_value_intersection_type);
+      nir_push_if(b, nir_ieq_imm(b, candidate_type, 1));
+      {
+         hk_trace_ray_m5_aabb_candidate(b, compiler, q, flags);
+      }
+      nir_push_else(b, NULL);
+      {
+         nir_store_var(b, state->accept, nir_imm_true(b), 1);
+         nir_store_var(b, state->terminate,
+            nir_test_mask(b, flags, SpvRayFlagsTerminateOnFirstHitKHRMask), 1);
+         struct hk_sbt_entry ahit_entry = hk_load_sbt_entry(
+            b, nir_load_var(b, state->sbt_index), HK_RT_CMD_HIT_ADDRESS);
+         nir_store_var(b, state->shader_record_ptr, ahit_entry.shader_record_ptr, 1);
+         struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+         const unsigned count = hk_rt_collect_stage_calls(
+            b, compiler, ahit_entry.value, HK_RT_GROUP_AHIT, MESA_SHADER_NONE,
+            calls);
+         for (unsigned i = 0; i < count; ++i) {
+            nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+            hk_call_ray_tracing_stage(b, compiler, calls[i].stage, true);
+            nir_pop_if(b, NULL);
+         }
+         nir_push_if(b, nir_load_var(b, state->accept));
+         nir_rq_confirm_intersection(b, q);
+         nir_push_if(b, nir_load_var(b, state->terminate));
+         nir_rq_terminate(b, q);
+         nir_pop_if(b, NULL);
+         nir_pop_if(b, NULL);
+      }
+      nir_pop_if(b, NULL);
+   }
+   nir_pop_loop(b, NULL);
+   /* A miss after rejected candidates keeps the original ray maximum. */
+   nir_store_var(b, state->tmax, tmax, 1);
+   nir_def *type = nir_rq_load(b, 1, 32, q, .committed = true,
+                               .ray_query_value = nir_ray_query_value_intersection_type);
+   nir_store_var(b, state->traversal.hit, nir_ine_imm(b, type, 0), 1);
+   nir_push_if(b, nir_ine_imm(b, type, 0));
+   /* Committed type 2 is a generated (procedural) hit. */
+   hk_rt_m5_load_hit_state(b, compiler, q, as, true, nir_ieq_imm(b, type, 2));
+   nir_pop_if(b, NULL);
+}
+
 static void
 hk_trace_ray(nir_builder *b, struct hk_rt_compiler *compiler,
              nir_intrinsic_instr *instr)
@@ -1312,6 +1612,19 @@ hk_trace_ray(nir_builder *b, struct hk_rt_compiler *compiler,
    if (compiler->native_agx)
       hk_trace_ray_native(b, compiler, accel_struct, flags, cull_mask,
                       origin, tmin, dir, tmax);
+   else if (compiler->native_m5) {
+      /* Pipeline skip flags cull like the matching ray flags, but are not
+       * part of the shader-visible IncomingRayFlags. */
+      uint32_t pipeline_flags = 0;
+      if (compiler->flags & VK_PIPELINE_CREATE_2_RAY_TRACING_SKIP_TRIANGLES_BIT_KHR)
+         pipeline_flags |= SpvRayFlagsSkipTrianglesKHRMask;
+      if (compiler->flags & VK_PIPELINE_CREATE_2_RAY_TRACING_SKIP_AABBS_BIT_KHR)
+         pipeline_flags |= SpvRayFlagsSkipAABBsKHRMask;
+      hk_trace_ray_m5(b, compiler, accel_struct,
+                      pipeline_flags ? nir_ior_imm(b, flags, pipeline_flags)
+                                     : flags,
+                      cull_mask, origin, tmin, dir, tmax);
+   }
    else
       hk_build_ray_traversal(b, &args);
    nir_pop_if(b, NULL);
@@ -1325,7 +1638,7 @@ static void
 hk_rt_load_wto(nir_builder *b, const struct hk_rt_compiler *compiler,
                 nir_def *instance, nir_def *rows[3])
 {
-   if (!compiler->native_agx) {
+   if (!compiler->native_agx && !compiler->native_m5) {
       hk_load_wto_matrix(b, instance, NULL, rows);
       return;
    }
@@ -1357,18 +1670,13 @@ hk_rt_dispatch_hit_or_miss(nir_builder *b, struct hk_rt_compiler *compiler,
       nir_store_var(b, state->shader_record_ptr,
                     chit_entry.shader_record_ptr, 0x1);
 
-      for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
-         struct hk_ray_tracing_group *group = compiler->pipeline->groups + i;
-         if (group->recursive_index == VK_SHADER_UNUSED_KHR)
-            continue;
-
-         nir_shader *stage =
-            compiler->pipeline->stages[group->recursive_index]->nir;
-         if (stage->info.stage != MESA_SHADER_CLOSEST_HIT)
-            continue;
-
-         nir_push_if(b, nir_ieq_imm(b, chit_entry.value, group->handle.index));
-         hk_call_ray_tracing_stage(b, compiler, stage, true);
+      struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+      const unsigned count = hk_rt_collect_stage_calls(
+         b, compiler, chit_entry.value, HK_RT_GROUP_RECURSIVE,
+         MESA_SHADER_CLOSEST_HIT, calls);
+      for (unsigned i = 0; i < count; i++) {
+         nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+         hk_call_ray_tracing_stage(b, compiler, calls[i].stage, true);
          nir_pop_if(b, NULL);
       }
 
@@ -1381,18 +1689,13 @@ hk_rt_dispatch_hit_or_miss(nir_builder *b, struct hk_rt_compiler *compiler,
       nir_store_var(b, state->shader_record_ptr,
                     miss_entry.shader_record_ptr, 0x1);
 
-      for (uint32_t i = 0; i < compiler->pipeline->group_count; i++) {
-         struct hk_ray_tracing_group *group = compiler->pipeline->groups + i;
-         if (group->recursive_index == VK_SHADER_UNUSED_KHR)
-            continue;
-
-         nir_shader *stage =
-            compiler->pipeline->stages[group->recursive_index]->nir;
-         if (stage->info.stage != MESA_SHADER_MISS)
-            continue;
-
-         nir_push_if(b, nir_ieq_imm(b, miss_entry.value, group->handle.index));
-         hk_call_ray_tracing_stage(b, compiler, stage, true);
+      struct hk_rt_stage_call calls[compiler->pipeline->group_count];
+      const unsigned count = hk_rt_collect_stage_calls(
+         b, compiler, miss_entry.value, HK_RT_GROUP_RECURSIVE,
+         MESA_SHADER_MISS, calls);
+      for (unsigned i = 0; i < count; i++) {
+         nir_push_if(b, hk_rt_stage_call_cond(b, &calls[i]));
+         hk_call_ray_tracing_stage(b, compiler, calls[i].stage, true);
          nir_pop_if(b, NULL);
       }
    }
@@ -2017,7 +2320,7 @@ hk_lower_ray_tracing_instr(nir_builder *b, nir_instr *instr, void *data)
       nir_def *custom_instance_and_mask = nir_load_global(
          b, 1, 32,
          nir_iadd_imm(b, instance_node_addr,
-                      compiler->native_agx ? 96 :
+                      (compiler->native_agx || compiler->native_m5) ? 96 :
                          offsetof(struct hk_bvh_instance_node,
                                   custom_instance_and_mask)));
       def = nir_iand_imm(b, custom_instance_and_mask, 0xFFFFFF);
@@ -2035,7 +2338,7 @@ hk_lower_ray_tracing_instr(nir_builder *b, nir_instr *instr, void *data)
       def = nir_load_global(
          b, 1, 32,
          nir_iadd_imm(b, instance_node_addr,
-                      compiler->native_agx ? 100 :
+                      (compiler->native_agx || compiler->native_m5) ? 100 :
                          offsetof(struct hk_bvh_instance_node, instance_id)));
       break;
    }
@@ -2061,7 +2364,7 @@ hk_lower_ray_tracing_instr(nir_builder *b, nir_instr *instr, void *data)
    case nir_intrinsic_load_ray_object_to_world: {
       const unsigned c = nir_intrinsic_column(intr);
       nir_def *instance_node_addr = nir_load_var(b, state->instance_addr);
-      if (compiler->native_agx) {
+      if (compiler->native_agx || compiler->native_m5) {
          nir_def *relative = nir_load_global(b, 1, 64,
             nir_iadd_imm(b, instance_node_addr, 104), .align_mul = 8);
          def = nir_load_global(b, 3, 32,
@@ -2170,6 +2473,15 @@ hk_ray_tracing_pipeline_destroy(struct vk_device *vk_dev,
    if (pipeline->shader)
       hk_api_shader_destroy(vk_dev, &pipeline->shader->vk, pAllocator);
 
+   if (pipeline->kernels) {
+      for (uint32_t i = 0; i < pipeline->kernel_count; i++) {
+         if (pipeline->kernels[i].shader)
+            hk_api_shader_destroy(vk_dev, &pipeline->kernels[i].shader->vk,
+                                  pAllocator);
+      }
+      free(pipeline->kernels);
+   }
+
    if (pipeline->stages) {
       for (uint32_t i = 0; i < pipeline->stage_count; i++)
          hk_rt_stage_nir_unref(pipeline->stages[i]);
@@ -2205,17 +2517,42 @@ hk_ray_tracing_pipeline_get_shader(struct vk_pipeline *vk_pipeline,
    return NULL;
 }
 
-/* All RT stages are inlined into a single compute megakernel. Report that
- * executable once, with the API stages it implements rather than COMPUTE. A
- * library only retains stage NIR and has no executable to enumerate.
+/* All RT stages are inlined into a single compute megakernel, except on M5,
+ * which compiles one kernel per ray-generation group. Report each executable
+ * with the API stages it implements rather than COMPUTE. A library only
+ * retains stage NIR and has no executable to enumerate.
  */
 static VkResult
 hk_rt_pipeline_get_executable_properties(
-   UNUSED struct vk_device *device, struct vk_pipeline *vk_pipeline,
+   struct vk_device *device, struct vk_pipeline *vk_pipeline,
    uint32_t *executable_count, VkPipelineExecutablePropertiesKHR *properties)
 {
    struct hk_ray_tracing_pipeline *pipeline =
       hk_pipeline_to_ray_tracing(vk_pipeline);
+
+   if (pipeline->kernel_count) {
+      const uint32_t capacity = properties ? *executable_count : 0;
+      *executable_count = properties ? MIN2(capacity, pipeline->kernel_count)
+                                     : pipeline->kernel_count;
+      for (uint32_t i = 0; properties && i < *executable_count; i++) {
+         struct vk_shader *shader = &pipeline->kernels[i].shader->vk;
+         uint32_t count = 1;
+         VkResult result = shader->ops->get_executable_properties(
+            device, shader, &count, &properties[i]);
+         if (result != VK_SUCCESS)
+            return result;
+         assert(count == 1);
+         properties[i].stages = pipeline->base.stages;
+         snprintf(properties[i].name, sizeof(properties[i].name),
+                  "Ray generation %u", pipeline->kernels[i].raygen_handle_index);
+         snprintf(properties[i].description, sizeof(properties[i].description),
+                  "Ray-tracing megakernel for ray-generation group %u",
+                  pipeline->kernels[i].raygen_handle_index);
+      }
+      return properties && capacity < pipeline->kernel_count ? VK_INCOMPLETE
+                                                             : VK_SUCCESS;
+   }
+
    VK_OUTARRAY_MAKE_TYPED(VkPipelineExecutablePropertiesKHR, out, properties,
                          executable_count);
 
@@ -2232,16 +2569,27 @@ hk_rt_pipeline_get_executable_properties(
    return vk_outarray_status(&out);
 }
 
+static const struct vk_shader *
+hk_rt_pipeline_executable(const struct hk_ray_tracing_pipeline *pipeline,
+                          uint32_t executable_index)
+{
+   if (pipeline->kernel_count) {
+      assert(executable_index < pipeline->kernel_count);
+      return &pipeline->kernels[executable_index].shader->vk;
+   }
+
+   assert(pipeline->shader && executable_index == 0);
+   return &pipeline->shader->vk;
+}
+
 static VkResult
 hk_rt_pipeline_get_executable_statistics(
    struct vk_device *device, struct vk_pipeline *vk_pipeline,
    uint32_t executable_index, uint32_t *statistic_count,
    VkPipelineExecutableStatisticKHR *statistics)
 {
-   struct hk_ray_tracing_pipeline *pipeline =
-      hk_pipeline_to_ray_tracing(vk_pipeline);
-   assert(pipeline->shader && executable_index == 0);
-   const struct vk_shader *shader = &pipeline->shader->vk;
+   const struct vk_shader *shader = hk_rt_pipeline_executable(
+      hk_pipeline_to_ray_tracing(vk_pipeline), executable_index);
 
    return shader->ops->get_executable_statistics(
       device, shader, 0, statistic_count, statistics);
@@ -2253,10 +2601,8 @@ hk_rt_pipeline_get_internal_representations(
    uint32_t executable_index, uint32_t *representation_count,
    VkPipelineExecutableInternalRepresentationKHR *representations)
 {
-   struct hk_ray_tracing_pipeline *pipeline =
-      hk_pipeline_to_ray_tracing(vk_pipeline);
-   assert(pipeline->shader && executable_index == 0);
-   const struct vk_shader *shader = &pipeline->shader->vk;
+   const struct vk_shader *shader = hk_rt_pipeline_executable(
+      hk_pipeline_to_ray_tracing(vk_pipeline), executable_index);
 
    return shader->ops->get_executable_internal_representations(
       device, shader, 0, representation_count, representations);
@@ -2277,6 +2623,10 @@ hk_rt_stage_to_nir(struct hk_device *dev,
                    const VkPipelineShaderStageCreateInfo *stage_info,
                    nir_shader **nir_out)
 {
+   if (pipeline_flags &
+       VK_PIPELINE_CREATE_2_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_KHR)
+      return VK_PIPELINE_COMPILE_REQUIRED;
+
    struct hk_physical_device *pdev = hk_device_physical(dev);
    struct vk_pipeline_robustness_state rstate;
 
@@ -2448,6 +2798,17 @@ hk_compile_ray_tracing_stages(
       NIR_PASS(_, nir, nir_split_struct_vars, nir_var_function_temp);
       NIR_PASS(_, nir, nir_lower_vars_to_ssa);
 
+      /* nir_lower_vars_to_ssa promotes most locals to SSA but leaves their
+       * declarations behind, and nir_lower_vars_to_explicit_types below
+       * assigns a frame slot to every surviving function_temp declaration,
+       * promoted or not.  Every call site of a GLSL function was inlined
+       * with its own copy of the callee's locals, so the dead declarations
+       * dominate the frame: RTGL1's IndirectInit raygen declared a 28,416-byte
+       * frame of which the kernel touched 24 bytes (the payload window), and
+       * the per-dispatch stack reservation multiplies that by the recursion
+       * depth.  Drop the dead declarations so the frame holds only what is
+       * still addressed (payload, hit attributes, dynamically indexed arrays,
+       * ray queries). */
       NIR_PASS(_, nir, nir_opt_dce);
       NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_function_temp, NULL);
 
@@ -2521,6 +2882,10 @@ hk_compile_ray_tracing_stages(
    return VK_SUCCESS;
 }
 
+/* Lowest address a capture/replay handle pointer may hold: the smallest page
+ * size, below which no process memory can be mapped. */
+#define HK_RT_MIN_REPLAY_HANDLE_ADDRESS 4096
+
 static VkResult
 hk_init_ray_tracing_groups(
    struct hk_device *dev, struct hk_ray_tracing_pipeline *pipeline,
@@ -2538,31 +2903,21 @@ hk_init_ray_tracing_groups(
              group_info->anyHitShader, group_info->intersectionShader))
          return VK_ERROR_FEATURE_NOT_PRESENT;
 
-      /* A group index past the end of this pipeline's own stage array is
-       * application error, and the validation layer catches it -- but the
-       * megakernel builder dereferences it, so without a check the driver
-       * faults in a release build instead of refusing.  This is the same
-       * process-boundary argument agx_emit_imadshl_agx() makes: refuse
-       * loudly rather than take the process down. */
+      /* Library stages are appended to the local stage array before group
+       * initialization. Newly declared groups may reference any stage in
+       * that combined array, including a shader-only library's stages. */
       const struct hk_ray_tracing_group *g = pipeline->groups + i;
-      uint32_t own_stage_count = create_info->stageCount;
-      if (create_info->pLibraryInfo) {
-         for (uint32_t l = 0; l < create_info->pLibraryInfo->libraryCount; l++) {
-            VK_FROM_HANDLE(vk_pipeline, vk_lib,
-                           create_info->pLibraryInfo->pLibraries[l]);
-            own_stage_count += hk_pipeline_to_ray_tracing(vk_lib)->stage_count;
-         }
-      }
+      const uint32_t total_stage_count = pipeline->stage_count;
       if ((g->recursive_index != VK_SHADER_UNUSED_KHR &&
-           g->recursive_index >= own_stage_count) ||
+           g->recursive_index >= total_stage_count) ||
           (g->ahit_index != VK_SHADER_UNUSED_KHR &&
-           g->ahit_index >= own_stage_count) ||
+           g->ahit_index >= total_stage_count) ||
           (g->isec_index != VK_SHADER_UNUSED_KHR &&
-           g->isec_index >= own_stage_count)) {
+           g->isec_index >= total_stage_count)) {
          return vk_errorf(dev, VK_ERROR_INITIALIZATION_FAILED,
                           "ray tracing group %u names a shader index past "
-                          "stageCount %u",
-                          i, own_stage_count);
+                          "combined stageCount %u",
+                          i, total_stage_count);
       }
 
       /* Handles are unique per device, not per pipeline, so a handle an
@@ -2571,9 +2926,29 @@ hk_init_ray_tracing_groups(
       if ((vk_rt_pipeline_create_flags(create_info) &
            VK_PIPELINE_CREATE_2_RAY_TRACING_SHADER_GROUP_HANDLE_CAPTURE_REPLAY_BIT_KHR) &&
           group_info->pShaderGroupCaptureReplayHandle) {
+         /* A replay handle points at the 4-byte value that
+          * vkGetRayTracingCaptureReplayShaderGroupHandlesKHR returned. No
+          * application memory lives in the zero page; a pointer there is a
+          * tool's offset from a NULL base (gfxreconstruct replays a 0-byte
+          * captured handle set as NULL + 4 * group). Report that the handle
+          * cannot be reused instead of faulting on it. */
+         if ((uintptr_t)group_info->pShaderGroupCaptureReplayHandle <
+             HK_RT_MIN_REPLAY_HANDLE_ADDRESS) {
+            return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS,
+                             "ray tracing group %u: capture/replay handle "
+                             "pointer %p is in the zero page",
+                             i, group_info->pShaderGroupCaptureReplayHandle);
+         }
          uint32_t replay;
          memcpy(&replay, group_info->pShaderGroupCaptureReplayHandle,
                 sizeof(replay));
+         /* Handles start at 1 (p_atomic_inc_return below), so 0 was never
+          * returned by vkGetRayTracingCaptureReplayShaderGroupHandlesKHR. */
+         if (!replay) {
+            return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS,
+                             "ray tracing group %u: capture/replay handle 0 "
+                             "was never issued", i);
+         }
          pipeline->groups[i].handle.index = replay;
          uint32_t current = p_atomic_read(&dev->rt_group_handle_alloc);
          while (current < replay) {
@@ -2619,11 +2994,22 @@ hk_init_ray_tracing_groups(
    return VK_SUCCESS;
 }
 
+/*
+ * Compile the kernel for one ray-generation group: that raygen, plus every
+ * stage reachable from it through the shader binding table (hit, miss,
+ * intersection, callable), inlined.  Without a group, compile the single
+ * megakernel that selects between every raygen group.  The kernel still checks the raygen
+ * record's handle against its own group, so a kernel dispatched for another
+ * group's record retires without tracing; the GPU-side selection in
+ * hk_cmd_trace_rays.c dispatches the others with zero threadgroups.
+ */
 static VkResult
-hk_compile_ray_tracing_pipeline(
+hk_compile_rt_kernel(
    struct hk_device *dev, struct hk_ray_tracing_pipeline *pipeline,
    const VkRayTracingPipelineCreateInfoKHR *create_info,
-   const VkAllocationCallbacks *pAllocator)
+   const VkAllocationCallbacks *pAllocator,
+   const struct hk_ray_tracing_group *raygen_group,
+   struct hk_rt_kernel *kernel)
 {
    struct hk_physical_device *pdev = hk_device_physical(dev);
    struct vk_pipeline_robustness_state rstate;
@@ -2642,6 +3028,8 @@ hk_compile_ray_tracing_pipeline(
       MESA_SHADER_COMPUTE, nir_options, "ray tracing pipeline");
    nir_builder *b = &_b;
 
+   kernel->raygen_handle_index = raygen_group ? raygen_group->handle.index : 0;
+
    /* nir_builder_init_simple_shader() marks the shader internal, and
     * hk_compile_nir() turns that into `has_scratch = false`
     * (hk_shader.c:2633), which makes agx_ra() assert "internal shaders are
@@ -2660,6 +3048,8 @@ hk_compile_ray_tracing_pipeline(
       .flags = vk_rt_pipeline_create_flags(create_info),
       .position_fetch = dev->vk.enabled_features.rayTracingPositionFetch,
       .native_agx = dev->gen1_native_rq && dev->gen1_rq_instances,
+      .native_m5 = dev->m5_native_rq && dev->m5_rq_instances,
+      .m5 = agx_chip_g17_m5(dev->dev.chip),
       .callable_dispatch = hk_callable_dispatch_needed(pipeline),
       .max_depth = create_info->maxPipelineRayRecursionDepth,
    };
@@ -2704,18 +3094,29 @@ hk_compile_ray_tracing_pipeline(
    nir_store_var(b, compiler.state.shader_record_ptr,
                  raygen_entry.shader_record_ptr, 0x1);
 
-   for (uint32_t i = 0; i < pipeline->group_count; i++) {
-      struct hk_ray_tracing_group *group = pipeline->groups + i;
-      if (group->recursive_index == VK_SHADER_UNUSED_KHR)
-         continue;
-
-      nir_shader *stage = pipeline->stages[group->recursive_index]->nir;
-      if (stage->info.stage != MESA_SHADER_RAYGEN)
-         continue;
+   if (raygen_group) {
+      nir_shader *stage = pipeline->stages[raygen_group->recursive_index]->nir;
+      assert(stage->info.stage == MESA_SHADER_RAYGEN);
 
-      nir_push_if(b, nir_ieq_imm(b, raygen_entry.value, group->handle.index));
+      nir_push_if(b, nir_ieq_imm(b, raygen_entry.value,
+                                 raygen_group->handle.index));
       hk_call_ray_tracing_stage(b, &compiler, stage, false);
       nir_pop_if(b, NULL);
+   } else {
+      /* Single megakernel: every raygen group, selected by its handle. */
+      for (uint32_t i = 0; i < pipeline->group_count; i++) {
+         struct hk_ray_tracing_group *group = pipeline->groups + i;
+         if (group->recursive_index == VK_SHADER_UNUSED_KHR)
+            continue;
+
+         nir_shader *stage = pipeline->stages[group->recursive_index]->nir;
+         if (stage->info.stage != MESA_SHADER_RAYGEN)
+            continue;
+
+         nir_push_if(b, nir_ieq_imm(b, raygen_entry.value, group->handle.index));
+         hk_call_ray_tracing_stage(b, &compiler, stage, false);
+         nir_pop_if(b, NULL);
+      }
    }
 
    nir_shader_instructions_pass(b->shader, hk_lower_ray_tracing_instr,
@@ -2769,14 +3170,30 @@ hk_compile_ray_tracing_pipeline(
             nir_intrinsic_load_ray_tracing_stack_base_lvp,
             b->shader->scratch_size);
 
-   pipeline->stack_sizes = compiler.sizes;
-   pipeline->stack_size = hk_rt_pipeline_stack_size(
+   /* Each kernel reserves exactly what its own raygen needs; the pipeline
+    * reports the maximum over its kernels. */
+   const uint32_t stack_size = hk_rt_pipeline_stack_size(
       &compiler.sizes, create_info->maxPipelineRayRecursionDepth);
-   b->shader->scratch_size += pipeline->stack_size;
+   b->shader->scratch_size += stack_size;
+
+   pipeline->stack_sizes.raygen =
+      MAX2(pipeline->stack_sizes.raygen, compiler.sizes.raygen);
+   pipeline->stack_sizes.any_hit =
+      MAX2(pipeline->stack_sizes.any_hit, compiler.sizes.any_hit);
+   pipeline->stack_sizes.closest_hit =
+      MAX2(pipeline->stack_sizes.closest_hit, compiler.sizes.closest_hit);
+   pipeline->stack_sizes.miss =
+      MAX2(pipeline->stack_sizes.miss, compiler.sizes.miss);
+   pipeline->stack_sizes.intersection =
+      MAX2(pipeline->stack_sizes.intersection, compiler.sizes.intersection);
+   pipeline->stack_sizes.callable =
+      MAX2(pipeline->stack_sizes.callable, compiler.sizes.callable);
+   pipeline->stack_size = MAX2(pipeline->stack_size, stack_size);
 
    hk_rt_optimize_nir(b->shader);
 
-   if (dev->dev.chip == AGX_CHIP_G17P) {
+   if (agx_chip_isa_encoding(dev->dev.chip) == AGX_ISA_ENCODING_AGX3C ||
+       dev->dev.chip == AGX_CHIP_G17P) {
       bool indirect = false;
       nir_foreach_function_impl(impl, b->shader) {
          nir_foreach_block(block, impl) {
@@ -2847,14 +3264,88 @@ hk_compile_ray_tracing_pipeline(
    };
 
    return hk_compile_shader(dev, &info, NULL, &dev->vk.enabled_features,
-                            pAllocator, &pipeline->shader);
+                            pAllocator, &kernel->shader);
+}
+
+/*
+ * On M5, one kernel per ray-generation group.  A pipeline whose groups name no
+ * raygen (only a library can be created that way) has nothing to compile;
+ * hk_cmd_trace_rays.c refuses to trace with it.
+ */
+static VkResult
+hk_compile_ray_tracing_pipeline(
+   struct hk_device *dev, struct hk_ray_tracing_pipeline *pipeline,
+   const VkRayTracingPipelineCreateInfoKHR *create_info,
+   const VkAllocationCallbacks *pAllocator)
+{
+   /* Only M5 compiles one kernel per ray-generation group; elsewhere the
+    * pipeline is one megakernel. */
+   if (!agx_chip_g17_m5(dev->dev.chip)) {
+      struct hk_rt_kernel kernel = {0};
+      VkResult result = hk_compile_rt_kernel(dev, pipeline, create_info,
+                                             pAllocator, NULL, &kernel);
+      pipeline->shader = kernel.shader;
+      return result;
+   }
+   uint32_t raygen_count = 0;
+   for (uint32_t i = 0; i < pipeline->group_count; i++) {
+      const struct hk_ray_tracing_group *group = pipeline->groups + i;
+      if (group->recursive_index != VK_SHADER_UNUSED_KHR &&
+          pipeline->stages[group->recursive_index]->nir->info.stage ==
+             MESA_SHADER_RAYGEN)
+         raygen_count++;
+   }
+
+   if (raygen_count == 0)
+      return VK_SUCCESS;
+
+   if (vk_rt_pipeline_create_flags(create_info) &
+       VK_PIPELINE_CREATE_2_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_KHR)
+      return VK_PIPELINE_COMPILE_REQUIRED;
+
+   pipeline->kernels = calloc(raygen_count, sizeof(*pipeline->kernels));
+   if (!pipeline->kernels)
+      return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+   for (uint32_t i = 0; i < pipeline->group_count; i++) {
+      const struct hk_ray_tracing_group *group = pipeline->groups + i;
+      if (group->recursive_index == VK_SHADER_UNUSED_KHR ||
+          pipeline->stages[group->recursive_index]->nir->info.stage !=
+             MESA_SHADER_RAYGEN)
+         continue;
+
+      VkResult result = hk_compile_rt_kernel(
+         dev, pipeline, create_info, pAllocator, group,
+         &pipeline->kernels[pipeline->kernel_count]);
+      if (result != VK_SUCCESS)
+         return result;
+
+      pipeline->kernel_count++;
+   }
+
+   assert(pipeline->kernel_count == raygen_count);
+   return VK_SUCCESS;
 }
 
 /* The common binary entrypoints own vk_rt_pipeline objects.  Honeykrisp's
- * megakernel and retained library NIR have a different layout and lifetime. */
+ * megakernel (or, on M5, per-raygen kernels) and retained library NIR have a
+ * different layout and lifetime. */
 #define HK_RT_BINARY_MAGIC   0x42524b48u
 #define HK_RT_BINARY_VERSION 1u
 
+/* What an RT binary carries after the stage NIR. */
+enum hk_rt_binary_executable {
+   HK_RT_BINARY_LIBRARY = 0,
+   HK_RT_BINARY_MEGAKERNEL = 1,
+   HK_RT_BINARY_RAYGEN_KERNELS = 2, /* M5 */
+};
+
+static bool
+hk_rt_binaries_native(const struct hk_device *dev)
+{
+   return dev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->dev.chip);
+}
+
 static void
 hk_rt_pipeline_key(struct hk_device *dev,
                    const VkRayTracingPipelineCreateInfoKHR *info,
@@ -2934,7 +3425,9 @@ hk_rt_binary_serialize(struct hk_device *dev,
    blob_write_uint32(blob, pipeline->stage_count);
    blob_write_uint32(blob, pipeline->group_count);
    blob_write_uint32(blob, pipeline->base.stages);
-   blob_write_uint32(blob, pipeline->shader != NULL);
+   blob_write_uint32(blob, pipeline->kernel_count ? HK_RT_BINARY_RAYGEN_KERNELS
+                           : pipeline->shader ? HK_RT_BINARY_MEGAKERNEL
+                                              : HK_RT_BINARY_LIBRARY);
    blob_write_bytes(blob, &pipeline->stack_sizes,
                     sizeof(pipeline->stack_sizes));
    blob_write_uint32(blob, pipeline->stack_size);
@@ -2954,6 +3447,15 @@ hk_rt_binary_serialize(struct hk_device *dev,
    if (pipeline->shader && !pipeline->shader->vk.ops->serialize(
                               &dev->vk, &pipeline->shader->vk, blob))
       return false;
+   if (pipeline->kernel_count) {
+      blob_write_uint32(blob, pipeline->kernel_count);
+      for (uint32_t i = 0; i < pipeline->kernel_count; ++i) {
+         const struct vk_shader *shader = &pipeline->kernels[i].shader->vk;
+         blob_write_uint32(blob, pipeline->kernels[i].raygen_handle_index);
+         if (!shader->ops->serialize(&dev->vk, shader, blob))
+            return false;
+      }
+   }
    return !blob->out_of_memory;
 }
 
@@ -2964,7 +3466,7 @@ hk_CreatePipelineBinariesKHR(VkDevice device,
                              VkPipelineBinaryHandlesInfoKHR *out)
 {
    VK_FROM_HANDLE(hk_device, dev, device);
-   if (dev->dev.chip != AGX_CHIP_G17P)
+   if (!hk_rt_binaries_native(dev))
       return vk_common_CreatePipelineBinariesKHR(device, info, alloc, out);
 
    if (info->pPipelineCreateInfo &&
@@ -3016,7 +3518,7 @@ hk_GetPipelineKeyKHR(VkDevice device, const VkPipelineCreateInfoKHR *info,
        !vk_find_struct_const(info->pNext, RAY_TRACING_PIPELINE_CREATE_INFO_KHR))
       return vk_common_GetPipelineKeyKHR(device, info, key);
    VK_FROM_HANDLE(hk_device, dev, device);
-   if (dev->dev.chip != AGX_CHIP_G17P)
+   if (!hk_rt_binaries_native(dev))
       return vk_common_GetPipelineKeyKHR(device, info, key);
 
    const VkRayTracingPipelineCreateInfoKHR *rt =
@@ -3069,9 +3571,13 @@ hk_rt_binary_deserialize(struct hk_device *dev,
    uint32_t stages = blob_read_uint32(&blob);
    uint32_t groups = blob_read_uint32(&blob);
    pipeline->base.stages = blob_read_uint32(&blob);
-   bool executable = blob_read_uint32(&blob);
+   const uint32_t executable = blob_read_uint32(&blob);
+   const uint32_t kernels_kind = agx_chip_g17_m5(dev->dev.chip)
+                                    ? HK_RT_BINARY_RAYGEN_KERNELS
+                                    : HK_RT_BINARY_MEGAKERNEL;
    if (blob.overrun ||
-       executable ==
+       (executable != HK_RT_BINARY_LIBRARY && executable != kernels_kind) ||
+       (executable != HK_RT_BINARY_LIBRARY) ==
           !!(pipeline->base.flags & VK_PIPELINE_CREATE_2_LIBRARY_BIT_KHR) ||
        stages > size / sizeof(uint32_t) ||
        groups > size / sizeof(pipeline->groups[0]))
@@ -3110,7 +3616,7 @@ hk_rt_binary_deserialize(struct hk_device *dev,
          goto done;
       }
    }
-   if (executable) {
+   if (executable == HK_RT_BINARY_MEGAKERNEL) {
       struct vk_shader *shader = NULL;
       result = dev->vk.shader_ops->deserialize(
          &dev->vk, &blob, dev->vk.physical->properties.shaderBinaryVersion,
@@ -3118,6 +3624,32 @@ hk_rt_binary_deserialize(struct hk_device *dev,
       if (result != VK_SUCCESS)
          goto done;
       pipeline->shader = container_of(shader, struct hk_api_shader, vk);
+   } else if (executable == HK_RT_BINARY_RAYGEN_KERNELS) {
+      const uint32_t kernel_count = blob_read_uint32(&blob);
+      if (blob.overrun || kernel_count == 0 || kernel_count > groups)
+         goto done;
+      pipeline->kernels = calloc(kernel_count, sizeof(*pipeline->kernels));
+      if (!pipeline->kernels) {
+         result = VK_ERROR_OUT_OF_HOST_MEMORY;
+         goto done;
+      }
+      for (uint32_t i = 0; i < kernel_count; ++i) {
+         const uint32_t handle = blob_read_uint32(&blob);
+         struct vk_shader *shader = NULL;
+         if (blob.overrun) {
+            result = VK_ERROR_UNKNOWN;
+            goto done;
+         }
+         result = dev->vk.shader_ops->deserialize(
+            &dev->vk, &blob, dev->vk.physical->properties.shaderBinaryVersion,
+            alloc, &shader);
+         if (result != VK_SUCCESS)
+            goto done;
+         pipeline->kernels[i].shader =
+            container_of(shader, struct hk_api_shader, vk);
+         pipeline->kernels[i].raygen_handle_index = handle;
+         pipeline->kernel_count++;
+      }
    }
    result =
       blob.overrun || blob.current != blob.end ? VK_ERROR_UNKNOWN : VK_SUCCESS;
@@ -3163,8 +3695,7 @@ hk_create_ray_tracing_pipeline(
 
    const VkPipelineBinaryInfoKHR *binaries =
       vk_find_struct_const(create_info->pNext, PIPELINE_BINARY_INFO_KHR);
-   if (dev->dev.chip == AGX_CHIP_G17P &&
-       binaries && binaries->binaryCount) {
+   if (hk_rt_binaries_native(dev) && binaries && binaries->binaryCount) {
       if (binaries->binaryCount != 1) {
          result = VK_ERROR_UNKNOWN;
          goto fail;
@@ -3176,7 +3707,7 @@ hk_create_ray_tracing_pipeline(
       *out_pipeline = vk_pipeline_to_handle(&pipeline->base);
       return VK_SUCCESS;
    }
-   if (dev->dev.chip == AGX_CHIP_G17P)
+   if (hk_rt_binaries_native(dev))
       hk_rt_pipeline_key(dev, create_info, pipeline->binary_key);
 
    pipeline->stage_count = create_info->stageCount;
diff --git a/src/asahi/vulkan/hk_ray_tracing_pipeline.h b/src/asahi/vulkan/hk_ray_tracing_pipeline.h
index 79a89ad116a5..c82673df404e 100644
--- a/src/asahi/vulkan/hk_ray_tracing_pipeline.h
+++ b/src/asahi/vulkan/hk_ray_tracing_pipeline.h
@@ -57,14 +57,29 @@ struct hk_ray_tracing_pipeline {
    struct hk_rt_stage_nir **stages;
    struct hk_ray_tracing_group *groups;
 
-   /* The compiled megakernel.  NULL for a pipeline library, which is never
-    * executable. */
+   /* The compiled megakernel.  NULL on M5, which compiles kernels instead,
+    * and for a pipeline library, which is never executable. */
    struct hk_api_shader *shader;
 
-   /* What the megakernel reserves above its own frame, reported through
-    * vkGetRayTracingShaderGroupStackSizeKHR and used by nothing else: the
-    * stack is static scratch inside the compiled binary, so the dynamic
-    * stack-size state cannot change it. */
+   /*
+    * On M5, one compute kernel per ray-generation group.  Each kernel inlines only
+    * its own raygen plus every hit/miss/intersection/callable stage reachable
+    * through the shader binding table; vkCmdTraceRays selects the kernel on the
+    * GPU from the raygen record's group handle (hk_cmd_trace_rays.c), because
+    * the record's contents are only required to be valid at execution time.
+    * Empty elsewhere and for a pipeline library, which is never executable.
+    */
+   uint32_t kernel_count;
+   struct hk_rt_kernel {
+      struct hk_api_shader *shader;
+      /* The hk_ray_tracing_group_handle::index this kernel's raygen answers to. */
+      uint32_t raygen_handle_index;
+   } *kernels;
+
+   /* What the kernels reserve above their own frame (maximum over kernels),
+    * reported through vkGetRayTracingShaderGroupStackSizeKHR and used by
+    * nothing else: the stack is static scratch inside each compiled binary,
+    * so the dynamic stack-size state cannot change it. */
    struct hk_rt_stack_sizes stack_sizes;
    uint32_t stack_size;
 };
diff --git a/src/asahi/vulkan/hk_shader.c b/src/asahi/vulkan/hk_shader.c
index 4ddaaf778ca2..77d3dbdf13bc 100644
--- a/src/asahi/vulkan/hk_shader.c
+++ b/src/asahi/vulkan/hk_shader.c
@@ -7,9 +7,14 @@
  * SPDX-License-Identifier: MIT
  */
 #include "hk_shader.h"
+#include "hk_frame_anatomy.h"
 #include "hk_nir_lower_gs.h"
 #include "hk_nir_root_size.h"
 #include "hk_nir_lower_tes_rq.h"
+#include "hk_native_write_class.h"
+#include "asahi/compiler/agx_nir_m5_ray_query.h"
+#include "asahi/compiler/agx_nir_m5_mesh.h"
+#include "asahi/compiler/agx3_mesh_core.h"
 #include "hk_nir_sampler_bias.h"
 #include "hk_gen1_rt_indirect_contract.h"
 #include "asahi/compiler/agx_nir_m5_mesh.h"
@@ -41,6 +46,7 @@
 #include "nir_serialize.h"
 #include "util/blob.h"
 #include "nir_builder.h"
+#include "nir_deref.h"
 
 #include "agx_bo.h"
 #include "hk_cmd_buffer.h"
@@ -106,6 +112,9 @@ struct hk_fs_key {
     */
    uint8_t g17p_samples;
 
+   /** M5 sample-position width selection; 0 means dynamic or unknown. */
+   uint8_t m5_position_samples;
+
    /** AGX3: alpha-to-coverage is statically disabled, so no fragment epilog of
     * this shader discards unless the shader writes depth or stencil
     * (agx_fs_epilog_may_discard()). The main part then always runs the
@@ -119,6 +128,8 @@ struct hk_fs_key {
     * colour stores are pruned before varying linking.
     */
    bool discard_color_outputs : 1;
+
+   uint8_t pad[3];
 };
 
 struct hk_shader_binary_cache_entry {
@@ -132,7 +143,7 @@ struct hk_shader_binary_cache_entry {
    uint32_t refcount;
 };
 
-static_assert(sizeof(struct hk_fs_key) == 4, "packed");
+static_assert(sizeof(struct hk_fs_key) == 8, "packed");
 
 struct hk_vs_key {
    struct agx_velem_key attribs[32];
@@ -193,7 +204,8 @@ hk_get_spirv_options(struct vk_physical_device *vk_pdev,
 {
    return (struct spirv_to_nir_options){
       .ssbo_addr_format = hk_buffer_addr_format(
-         rs->storage_buffers, rs->null_storage_buffer_descriptor),
+         rs->storage_buffers, rs->null_storage_buffer_descriptor,
+         agx_chip_g17_m5(((struct hk_physical_device *)vk_pdev)->dev.chip)),
       .phys_ssbo_addr_format = nir_address_format_64bit_global,
       .ubo_addr_format = hk_ubo_addr_format(rs->uniform_buffers),
       .shared_addr_format = nir_address_format_32bit_offset,
@@ -303,6 +315,68 @@ hk_cf_list_has_ray_query(struct exec_list *cf_list)
    return false;
 }
 
+/* A ray query addressed through a non-constant array index. Unrolling the
+ * loop that produces the index lets query arrays split into fixed native
+ * contexts, so such loops are left to the unroller. */
+static bool
+hk_cf_list_has_indexed_ray_query(struct exec_list *cf_list)
+{
+   foreach_list_typed(nir_cf_node, node, node, cf_list) {
+      switch (node->type) {
+      case nir_cf_node_block:
+         nir_foreach_instr(instr, nir_cf_node_as_block(node)) {
+            if (instr->type != nir_instr_type_intrinsic ||
+                !nir_intrinsic_is_ray_query(nir_instr_as_intrinsic(instr)->intrinsic))
+               continue;
+            nir_deref_instr *deref =
+               nir_src_as_deref(nir_instr_as_intrinsic(instr)->src[0]);
+            for (; deref && deref->deref_type != nir_deref_type_var;
+                 deref = nir_deref_instr_parent(deref)) {
+               if (deref->deref_type == nir_deref_type_array &&
+                   !nir_src_is_const(deref->arr.index))
+                  return true;
+            }
+         }
+         break;
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(node);
+         if (hk_cf_list_has_indexed_ray_query(&nif->then_list) ||
+             hk_cf_list_has_indexed_ray_query(&nif->else_list))
+            return true;
+         break;
+      }
+      case nir_cf_node_loop:
+         if (hk_cf_list_has_indexed_ray_query(&nir_cf_node_as_loop(node)->body))
+            return true;
+         break;
+      default:
+         break;
+      }
+   }
+   return false;
+}
+
+/* Keep loops that run whole native queries. Unrolled, each iteration copies
+ * the complete traversal (a 64-iteration TES loop compiled to 317 KB).
+ * Opt-in (ASAHI_M5_RQ_KEEP_LOOPS) until its CTS qualification is complete. */
+static void
+hk_keep_m5_ray_query_loops(struct exec_list *cf_list)
+{
+   foreach_list_typed(nir_cf_node, node, node, cf_list) {
+      if (node->type == nir_cf_node_if) {
+         nir_if *nif = nir_cf_node_as_if(node);
+         hk_keep_m5_ray_query_loops(&nif->then_list);
+         hk_keep_m5_ray_query_loops(&nif->else_list);
+      } else if (node->type == nir_cf_node_loop) {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         if (hk_cf_list_has_ray_query(&loop->body) &&
+             !hk_cf_list_has_indexed_ray_query(&loop->body))
+            loop->control = nir_loop_control_dont_unroll;
+         hk_keep_m5_ray_query_loops(&loop->body);
+      }
+   }
+}
+
 static void
 hk_keep_ray_query_loops(struct exec_list *cf_list)
 {
@@ -338,9 +412,36 @@ hk_nir_has_ray_query(nir_shader *nir)
    return false;
 }
 
+/* M5 lowers graphics-stage ray queries to intersect_ray_agx before the vertex
+ * ABI is assigned. Those shaders own uniform halves 0..167 for query state. */
+static bool
+hk_nir_has_m5_native_query(const struct hk_device *dev, nir_shader *nir)
+{
+   if (!dev->m5_native_rq)
+      return false;
+   if (hk_nir_has_ray_query(nir))
+      return true;
+   nir_foreach_function_impl(impl, nir) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type == nir_instr_type_intrinsic &&
+                nir_instr_as_intrinsic(instr)->intrinsic ==
+                   nir_intrinsic_intersect_ray_agx)
+               return true;
+         }
+      }
+   }
+   return false;
+}
+
 void
 hk_preprocess_nir_internal(struct vk_physical_device *vk_pdev, nir_shader *nir)
 {
+   if (agx_chip_g17_m5(((struct hk_physical_device *)vk_pdev)->dev.chip) &&
+       debug_get_bool_option("ASAHI_M5_RQ_KEEP_LOOPS", false)) {
+      nir_foreach_function_impl(impl, nir)
+         hk_keep_m5_ray_query_loops(&impl->body);
+   }
    if ((nir->info.stage == MESA_SHADER_VERTEX ||
         nir->info.stage == MESA_SHADER_TESS_EVAL) &&
        debug_get_bool_option("ASAHI_G17P_GRAPHICS_RQ_KEEP_LOOPS_DIAGNOSTIC", false)) {
@@ -414,7 +515,8 @@ hk_preprocess_nir_internal(struct vk_physical_device *vk_pdev, nir_shader *nir)
     * move, and nir_opt_large_constants only ever looks at function_temp
     * (nir_opt_large_constants.c:545-561).
     */
-   if (agx_native_graphics(&((struct hk_physical_device *)vk_pdev)->dev)) {
+   if (agx_native_graphics(&((struct hk_physical_device *)vk_pdev)->dev) ||
+       agx_chip_g17_m5(((struct hk_physical_device *)vk_pdev)->dev.chip)) {
       if (nir->info.stage == MESA_SHADER_COMPUTE) {
          NIR_PASS(_, nir, nir_opt_large_constants, NULL, 32);
       } else if ((nir->info.stage == MESA_SHADER_VERTEX ||
@@ -433,6 +535,8 @@ hk_preprocess_nir_internal(struct vk_physical_device *vk_pdev, nir_shader *nir)
    if (agx_native_graphics(&((struct hk_physical_device *)vk_pdev)->dev) &&
        nir->info.stage != MESA_SHADER_MESH)
       agx_preprocess_nir_g17p(nir);
+   else if (agx_chip_g17_m5(((struct hk_physical_device *)vk_pdev)->dev.chip))
+      agx_preprocess_nir_m5(nir);
    else
       agx_preprocess_nir(nir);
 
@@ -446,14 +550,15 @@ hk_preprocess_nir(struct vk_physical_device *vk_pdev, nir_shader *nir,
                   UNUSED const struct vk_pipeline_robustness_state *rs)
 {
    hk_preprocess_nir_internal(vk_pdev, nir);
+   const enum agx_chip chip = ((struct hk_physical_device *)vk_pdev)->dev.chip;
    const bool g17p_compute_derivatives =
-      ((struct hk_physical_device *)vk_pdev)->dev.chip == AGX_CHIP_G17P &&
+      (chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip)) &&
       nir->info.stage == MESA_SHADER_COMPUTE &&
       nir->info.derivative_group != DERIVATIVE_GROUP_NONE;
    nir_lower_compute_system_values_options csv_options = {
-      
-
-
+      /* Only vkCmdDispatchBase has a base workgroup. Task/mesh draws have
+       * none, and their root is the graphics arm: root.cs.base_group
+       * (byte 88) overlays root.draw.attrib_clamps[2..4] there. */
       .has_base_workgroup_id = nir->info.stage == MESA_SHADER_COMPUTE,
    };
    if (g17p_compute_derivatives &&
@@ -540,11 +645,17 @@ hk_populate_fs_key(struct hk_fs_key *key,
        !BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_MS_RASTERIZATION_SAMPLES))
       key->g17p_samples = state->ms->rasterization_samples;
 
-   if (agx_chip_native_agx3(chip) && state->ms != NULL &&
+   if (agx_chip_g17_m5(chip) && state->ms != NULL &&
+       !BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_MS_RASTERIZATION_SAMPLES))
+      key->m5_position_samples = state->ms->rasterization_samples;
+
+   /* M5 has no read-only early reject to rule out. */
+   if ((agx_chip_native_agx3(chip) || agx_chip_g17_m5(chip)) &&
+       state->ms != NULL &&
        !BITSET_TEST(state->dynamic,
                     MESA_VK_DYNAMIC_MS_ALPHA_TO_COVERAGE_ENABLE) &&
        !state->ms->alpha_to_coverage_enable &&
-       hk_g17p_static_no_early_reject(state))
+       (agx_chip_g17_m5(chip) || hk_g17p_static_no_early_reject(state)))
       key->static_tests = true;
 
    if (state->pipeline_flags &
@@ -575,7 +686,10 @@ hk_populate_fs_key(struct hk_fs_key *key,
    /* A fragment library without output-interface state may acquire color
     * attachments later. Alpha-to-coverage also consumes color even without
     * an attachment, so require it to be statically disabled. */
-   if (chip == AGX_CHIP_G17P && state->rp &&
+   if ((chip == AGX_CHIP_G17P ||
+        (agx_chip_g17_m5(chip) &&
+         debug_get_bool_option("ASAHI_M5_PRUNE_DEPTH_COLOR", true))) &&
+       state->rp &&
        (state->cal || BITSET_TEST(state->dynamic,
                                    MESA_VK_DYNAMIC_COLOR_ATTACHMENT_MAP)) &&
        vk_render_pass_state_has_attachment_info(state->rp) &&
@@ -784,6 +898,13 @@ hk_hash_graphics_state(struct vk_physical_device *device,
                           sizeof(no_fragment_shader));
    }
 
+   /* M5 triangle mesh point-size removal is a pipeline-only lowering. */
+   if (agx_chip_g17_m5(((struct hk_physical_device *)device)->dev.chip) &&
+       (stages & VK_SHADER_STAGE_MESH_BIT_EXT)) {
+      const bool pipeline_mesh = state != NULL;
+      _mesa_blake3_update(&blake3_ctx, &pipeline_mesh, sizeof(pipeline_mesh));
+   }
+
    enum hk_feature_key feature_key = hk_make_feature_key(features);
    _mesa_blake3_update(&blake3_ctx, &feature_key, sizeof(feature_key));
 
@@ -817,6 +938,45 @@ is_op(nir_scalar s, nir_op op)
    return nir_scalar_is_alu(s) && nir_scalar_alu_op(s) == op;
 }
 
+/* Prove low offset bits from SSA, independently of the final-address
+ * alignment metadata: an aligned address does not imply an aligned offset
+ * when its base is unaligned. Fail closed on phis, calls and unknown values. */
+static bool
+m5_offset_aligned(nir_scalar s, uint32_t mask, unsigned depth)
+{
+   if (!mask)
+      return true;
+   if (nir_scalar_is_const(s))
+      return (nir_scalar_as_uint(s) & mask) == 0;
+   if (!depth || s.def->bit_size != 32 || !nir_scalar_is_alu(s))
+      return false;
+   const nir_op op = nir_scalar_alu_op(s);
+   if (op == nir_op_mov)
+      return m5_offset_aligned(nir_scalar_chase_alu_src(s, 0), mask, depth - 1);
+   nir_scalar x = nir_scalar_chase_alu_src(s, 0);
+   nir_scalar y;
+   switch (op) {
+   case nir_op_iadd: case nir_op_isub: case nir_op_ior: case nir_op_ixor:
+      y = nir_scalar_chase_alu_src(s, 1);
+      return m5_offset_aligned(x, mask, depth - 1) &&
+             m5_offset_aligned(y, mask, depth - 1);
+   case nir_op_imul: case nir_op_amul: case nir_op_iand:
+      y = nir_scalar_chase_alu_src(s, 1);
+      return m5_offset_aligned(x, mask, depth - 1) ||
+             m5_offset_aligned(y, mask, depth - 1);
+   case nir_op_ishl:
+      y = nir_scalar_chase_alu_src(s, 1);
+      return nir_scalar_is_const(y) &&
+             m5_offset_aligned(x, mask >> (nir_scalar_as_uint(y) & 31),
+                               depth - 1);
+   case nir_op_bcsel:
+      return m5_offset_aligned(nir_scalar_chase_alu_src(s, 1), mask, depth - 1) &&
+             m5_offset_aligned(nir_scalar_chase_alu_src(s, 2), mask, depth - 1);
+   default:
+      return false;
+   }
+}
+
 static nir_def *
 check_in_bounds(nir_builder *b, nir_intrinsic_instr *intr)
 {
@@ -842,13 +1002,481 @@ check_in_bounds(nir_builder *b, nir_intrinsic_instr *intr)
       }
    }
 
-   /* TODO: handle also the iadd(amul) pattern, this is important */
+   /* For word-aligned 32-bit accesses, count words instead of bytes. The
+    * shifted offset is at most 2^30-1, so adding at most15 never wraps. This
+    * keeps partial trailing bytes and top-u32 ranges exact without saturation. */
+   if (debug_get_bool_option("ASAHI_M5_BOUNDED_VEC_PROBE", false) &&
+       bit_size == 32 && offset->bit_size == 32 && intr->num_components <= 16 &&
+       m5_offset_aligned(offset_s, 3, 12)) {
+      return nir_ult(b, nir_iadd_imm(b, nir_ushr_imm(b, offset, 2),
+                                   intr->num_components - 1),
+                        nir_ushr_imm(b, bound, 2));
+   }
+
+   /* For offset=q*size, offset+size<=bound is exactly q<floor(bound/size).
+    * This also rejects the last aligned u32 offset when its end would be
+    * 2^32. There is no rounded-bound or aligned-base assumption here. */
+   if (debug_get_bool_option("ASAHI_M5_BOUNDS_FACTOR_PROBE", false) &&
+       offset->bit_size == 32 && util_is_power_of_two_nonzero(load_size) &&
+       m5_offset_aligned(offset_s, load_size - 1, 12)) {
+      unsigned shift = util_logbase2(load_size);
+      return nir_ult(b, nir_ushr_imm(b, offset, shift),
+                        nir_ushr_imm(b, bound, shift));
+   }
 
    /* Otherwise bounds check in bytes */
    nir_def *sat_offset = nir_umin_imm(b, offset, UINT32_MAX - (load_size - 1));
    return nir_ult(b, nir_iadd_imm(b, sat_offset, load_size - 1), bound);
 }
 
+struct m5_bounded_offset {
+   nir_scalar term;
+   uint32_t constant;
+   uint32_t scale;
+   bool valid;
+};
+
+static struct m5_bounded_offset
+m5_bounded_offset(nir_scalar s, unsigned depth)
+{
+   if (nir_scalar_is_const(s))
+      return (struct m5_bounded_offset){.constant = nir_scalar_as_uint(s), .valid = true};
+   if (depth && is_op(s, nir_op_iadd)) {
+      struct m5_bounded_offset x = m5_bounded_offset(nir_scalar_chase_alu_src(s, 0), depth - 1);
+      struct m5_bounded_offset y = m5_bounded_offset(nir_scalar_chase_alu_src(s, 1), depth - 1);
+      if (!x.valid || !y.valid || (x.term.def && y.term.def))
+         return (struct m5_bounded_offset){0};
+      return (struct m5_bounded_offset){.term = x.term.def ? x.term : y.term,
+         .constant = x.constant + y.constant,
+         .scale = x.term.def ? x.scale : y.scale, .valid = true};
+   }
+   if (depth && is_op(s, nir_op_ior)) {
+      nir_scalar a = nir_scalar_chase_alu_src(s, 0);
+      nir_scalar b = nir_scalar_chase_alu_src(s, 1);
+      if (nir_scalar_is_const(a) || nir_scalar_is_const(b)) {
+         uint32_t k = nir_scalar_as_uint(nir_scalar_is_const(a) ? a : b);
+         nir_scalar term = nir_scalar_is_const(a) ? b : a;
+         uint32_t mask = k ? UINT32_MAX >> (32 - util_last_bit(k)) : 0;
+         if (m5_offset_aligned(term, mask, depth)) {
+            struct m5_bounded_offset value = m5_bounded_offset(term, depth - 1);
+            value.constant += k;
+            return value;
+         }
+      }
+   }
+   if (depth && is_op(s, nir_op_ishl)) {
+      nir_scalar shift = nir_scalar_chase_alu_src(s, 1);
+      if (nir_scalar_is_const(shift)) {
+         unsigned bits = nir_scalar_as_uint(shift) & 31;
+         struct m5_bounded_offset value = m5_bounded_offset(
+            nir_scalar_chase_alu_src(s, 0), depth - 1);
+         value.constant <<= bits;
+         value.scale <<= bits;
+         return value;
+      }
+   }
+   if (depth && (is_op(s, nir_op_amul) || is_op(s, nir_op_imul))) {
+      nir_scalar a = nir_scalar_chase_alu_src(s, 0);
+      nir_scalar b = nir_scalar_chase_alu_src(s, 1);
+      if (nir_scalar_is_const(a) || nir_scalar_is_const(b)) {
+         uint32_t k = nir_scalar_as_uint(nir_scalar_is_const(a) ? a : b);
+         struct m5_bounded_offset value = m5_bounded_offset(
+            nir_scalar_is_const(a) ? b : a, depth - 1);
+         value.constant *= k;
+         value.scale *= k;
+         return value;
+      }
+   }
+   return (struct m5_bounded_offset){.term = s, .scale = 1, .valid = true};
+}
+
+static bool
+m5_bounded_candidate(nir_shader *nir, struct hash_table *ranges,
+                     nir_intrinsic_instr *in)
+{
+   if ((in->intrinsic != nir_intrinsic_load_global_bounded &&
+        in->intrinsic != nir_intrinsic_load_global_constant_bounded) ||
+       in->num_components != 1 || in->def.bit_size != 32 ||
+       !nir_intrinsic_can_reorder(in) || nir_intrinsic_align(in) < 4)
+      return false;
+   /* Legacy scalar amul(x,4) checks x, rather than the wrapped byte
+    * offset. Only admit it when unsigned range analysis proves no wrap,
+    * making the reconstructed byte predicate exactly equivalent. */
+   nir_scalar off = nir_scalar_resolved(in->src[1].ssa, 0);
+   if (is_op(off, nir_op_amul)) {
+      for (unsigned s = 0; s < 2; s++) {
+         nir_scalar src = nir_scalar_chase_alu_src(off, s);
+         if (nir_scalar_is_const(src) && nir_scalar_as_uint(src) == 4) {
+            nir_scalar index = nir_scalar_chase_alu_src(off, 1 - s);
+            if (nir_unsigned_upper_bound(nir, ranges, index) > UINT32_MAX / 4)
+               return false;
+         }
+      }
+   }
+   return true;
+}
+
+/* Keep memory/control barriers intact even for readonly candidates. */
+static bool
+m5_bounded_can_move(nir_instr *first, nir_instr *last)
+{
+   for (nir_instr *instr = nir_instr_next(first); instr != last;
+        instr = nir_instr_next(instr)) {
+      if (instr->type == nir_instr_type_intrinsic &&
+          !nir_intrinsic_can_reorder(nir_instr_as_intrinsic(instr)))
+         return false;
+      if (instr->type == nir_instr_type_call || instr->type == nir_instr_type_jump)
+         return false;
+   }
+   return true;
+}
+
+struct m5_bounded_group {
+   nir_intrinsic_instr *load[4];
+   unsigned count;
+};
+
+/* A full-range vector load is the common path; a partial or wrapped range
+ * executes the original scalar intrinsics, preserving every component's
+ * zeroing rule. Both paths still use bounded intrinsics, so later speculative
+ * flattening cannot issue an unsafe vector address. No access qualifiers or
+ * application robustness modes are weakened. */
+static bool
+m5_vectorize_bounded_loads(nir_shader *nir)
+{
+   struct hash_table *ranges = _mesa_hash_table_create(
+      nir, _mesa_hash_pointer, _mesa_key_pointer_equal);
+   struct util_dynarray groups;
+   util_dynarray_init(&groups, nir);
+   nir_foreach_function_impl(impl, nir) {
+      nir_index_instrs(impl);
+      nir_foreach_block(block, impl) {
+         struct util_dynarray candidates;
+         util_dynarray_init(&candidates, nir);
+         nir_foreach_instr(instr, block) {
+            if (instr->type == nir_instr_type_intrinsic &&
+                m5_bounded_candidate(nir, ranges, nir_instr_as_intrinsic(instr))) {
+               instr->pass_flags = 0;
+               nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr);
+               util_dynarray_append(&candidates, in);
+            }
+         }
+         unsigned n = util_dynarray_num_elements(&candidates, nir_intrinsic_instr *);
+         nir_intrinsic_instr **loads = candidates.data;
+         for (unsigned i = 0; i < n; i++) {
+            if (loads[i]->instr.pass_flags) continue;
+            struct m5_bounded_group g = {.load = {loads[i]}, .count = 1};
+            struct m5_bounded_offset low = m5_bounded_offset(nir_scalar_resolved(loads[i]->src[1].ssa, 0), 12);
+            if (!low.valid) continue;
+            for (unsigned j = i + 1; j < n && g.count < 4; j++) {
+               nir_intrinsic_instr *in = loads[j];
+               if (in->instr.pass_flags || in->intrinsic != loads[i]->intrinsic ||
+                   in->src[0].ssa != loads[i]->src[0].ssa ||
+                   in->src[2].ssa != loads[i]->src[2].ssa ||
+                   nir_intrinsic_access(in) != nir_intrinsic_access(loads[i]) ||
+                   !m5_bounded_can_move(&loads[i]->instr, &in->instr)) continue;
+               struct m5_bounded_offset off = m5_bounded_offset(nir_scalar_resolved(in->src[1].ssa, 0), 12);
+               if (off.valid && off.term.def == low.term.def &&
+                   off.term.comp == low.term.comp && off.scale == low.scale &&
+                   off.constant == low.constant + 4 * g.count)
+                  g.load[g.count++] = in;
+            }
+            if (g.count < 2) continue;
+            for (unsigned k = 0; k < g.count; k++) g.load[k]->instr.pass_flags = 1;
+            util_dynarray_append(&groups, g);
+         }
+         util_dynarray_fini(&candidates);
+      }
+   }
+   unsigned count = util_dynarray_num_elements(&groups, struct m5_bounded_group);
+   /* Process backwards so earlier groups keep their original dominance. */
+   for (unsigned index = count; index-- > 0;) {
+      struct m5_bounded_group *g = util_dynarray_element(&groups, struct m5_bounded_group, index);
+      nir_intrinsic_instr *low = g->load[0];
+      nir_builder b = nir_builder_create(nir_cf_node_get_function(&low->instr.block->cf_node));
+      b.cursor = nir_before_instr(&low->instr);
+      nir_intrinsic_instr *guard = nir_instr_as_intrinsic(nir_instr_clone(nir, &low->instr));
+      guard->num_components = g->count;
+      guard->def.num_components = g->count;
+      nir_def *valid = check_in_bounds(&b, guard);
+      nir_instr_free(&guard->instr);
+      nir_def *offset[4];
+      for (unsigned j = 0; j < g->count; j++)
+         offset[j] = nir_iadd_imm(&b, low->src[1].ssa, 4 * j);
+      nir_push_if(&b, valid);
+      nir_def *fast;
+      if (low->intrinsic == nir_intrinsic_load_global_bounded)
+         fast = nir_load_global_bounded(&b, g->count, 32, low->src[0].ssa,
+            low->src[1].ssa, low->src[2].ssa, .access = nir_intrinsic_access(low),
+            .align_mul = nir_intrinsic_align_mul(low), .align_offset = nir_intrinsic_align_offset(low));
+      else
+         fast = nir_load_global_constant_bounded(&b, g->count, 32, low->src[0].ssa,
+            low->src[1].ssa, low->src[2].ssa, .access = nir_intrinsic_access(low),
+            .align_mul = nir_intrinsic_align_mul(low), .align_offset = nir_intrinsic_align_offset(low));
+      nir_push_else(&b, NULL);
+      nir_def *values[4];
+      for (unsigned j = 0; j < g->count; j++) {
+         nir_intrinsic_instr *slow = nir_instr_as_intrinsic(nir_instr_clone(nir, &g->load[j]->instr));
+         /* The normalized affine offset is exactly equal modulo u32. Any
+          * original scalar element predicate was proven non-wrapping before
+          * admitting it, so replacing its spelling preserves the bound test. */
+         nir_builder_instr_insert(&b, &slow->instr);
+         nir_src_rewrite(&slow->src[1], offset[j]);
+         values[j] = &slow->def;
+      }
+      nir_def *slow = nir_vec(&b, values, g->count);
+      nir_pop_if(&b, NULL);
+      nir_def *result = nir_if_phi(&b, fast, slow);
+      for (unsigned j = 0; j < g->count; j++)
+         nir_def_replace(&g->load[j]->def, nir_channel(&b, result, j));
+      nir_metadata_invalidate(nir);
+   }
+   unsigned words = 0;
+   for (unsigned i = 0; i < count; i++)
+      words += util_dynarray_element(&groups, struct m5_bounded_group, i)->count;
+   if (count) fprintf(stderr, "M5_BOUNDED_VEC groups=%u words=%u\n", count, words);
+   util_dynarray_fini(&groups);
+   _mesa_hash_table_destroy(ranges, NULL);
+   return count != 0;
+}
+
+struct m5_word_read {
+   nir_intrinsic_instr *load;
+   /* The 64-bit base and, for a robust read, the descriptor's byte size. */
+   nir_def *base, *bound;
+   /* The read's 32-bit byte offset from base. */
+   nir_def *offset_def;
+   struct m5_bounded_offset offset;
+};
+
+static int
+m5_word_read_cmp(const void *pa, const void *pb)
+{
+   const struct m5_word_read *a = pa, *b = pb;
+   const nir_intrinsic_instr *x = a->load, *y = b->load;
+#define M5_CMP(u, v) if ((u) != (v)) return (u) < (v) ? -1 : 1
+   M5_CMP(x->intrinsic, y->intrinsic);
+   M5_CMP(a->base->index, b->base->index);
+   M5_CMP(a->bound ? a->bound->index + 1 : 0, b->bound ? b->bound->index + 1 : 0);
+   M5_CMP(nir_intrinsic_access(x), nir_intrinsic_access(y));
+   M5_CMP(a->offset.term.def ? a->offset.term.def->index + 1 : 0,
+          b->offset.term.def ? b->offset.term.def->index + 1 : 0);
+   M5_CMP(a->offset.term.comp, b->offset.term.comp);
+   M5_CMP(a->offset.scale, b->offset.scale);
+   M5_CMP(a->offset.constant, b->offset.constant);
+   M5_CMP(x->instr.index, y->instr.index);
+#undef M5_CMP
+   return 0;
+}
+
+
+static bool
+m5_word_reads_same_record(const struct m5_word_read *a, const struct m5_word_read *b)
+{
+   return a->load->intrinsic == b->load->intrinsic && a->base == b->base &&
+          a->bound == b->bound &&
+          nir_intrinsic_access(a->load) == nir_intrinsic_access(b->load) &&
+          a->offset.term.def == b->offset.term.def &&
+          a->offset.term.comp == b->offset.term.comp &&
+          a->offset.scale == b->offset.scale;
+}
+
+static bool
+m5_word_reads_adjacent(const struct m5_word_read *a, const struct m5_word_read *b)
+{
+   return m5_word_reads_same_record(a, b) &&
+          b->offset.constant == a->offset.constant + 4;
+}
+
+/* Match a 64-bit address base + u2u64(offset) with a 32-bit offset. */
+static bool
+m5_split_address(nir_def *addr, nir_def **base, nir_def **offset)
+{
+   nir_scalar s = nir_scalar_resolved(addr, 0);
+   if (addr->bit_size != 64 || !is_op(s, nir_op_iadd))
+      return false;
+   for (unsigned i = 0; i < 2; i++) {
+      nir_scalar u = nir_scalar_chase_alu_src(s, i);
+      nir_scalar other = nir_scalar_chase_alu_src(s, 1 - i);
+      if (is_op(u, nir_op_u2u64) && other.def->num_components == 1) {
+         nir_scalar off = nir_scalar_chase_alu_src(u, 0);
+         if (off.def->bit_size == 32 && off.def->num_components == 1 &&
+             !is_op(other, nir_op_u2u64)) {
+            *base = other.def;
+            *offset = off.def;
+            return true;
+         }
+      }
+   }
+   return false;
+}
+
+/* M5: a structure read field by field (a light record in a loop) issues one
+ * load per 32-bit word, each with its own address. Merge the reads of up to
+ * four consecutive words at one base and offset term into one vector load.
+ *
+ * Buffer reads address base + u2u64(X + c) with a 32-bit offset, so the words
+ * of one vector load differ from their separate reads only where X + c wraps
+ * 2^32. Such an offset lies past the end of any buffer: undefined without
+ * robustness, and zeroed by the robust forms here. A robust read (soft fault:
+ * load, then zero the word when its end lies past the descriptor's byte size)
+ * keeps each word's own check:
+ *
+ *    word j of a load at X: bounds_agx(v.j, uadd_sat(X, 4 * (j + 1)), size)
+ *
+ * which is exact, except that a word whose offset wraps is now zeroed instead
+ * of read at its wrapped offset. Only such a word's address differs from its
+ * separate read, and soft fault covers it. */
+static bool
+m5_merge_buffer_words(nir_shader *nir)
+{
+   bool any = false;
+   struct util_dynarray reads;
+   util_dynarray_init(&reads, NULL);
+   nir_foreach_function_impl(impl, nir) {
+      bool progress = false;
+      nir_index_ssa_defs(impl);
+      nir_index_instrs(impl);
+      nir_foreach_block(block, impl) {
+         util_dynarray_clear(&reads);
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr);
+            if (in->num_components != 1 || in->def.bit_size != 32 ||
+                !nir_intrinsic_can_reorder(in) || nir_intrinsic_align(in) < 4)
+               continue;
+            struct m5_word_read r = {.load = in};
+            switch (in->intrinsic) {
+            case nir_intrinsic_load_global_bounded:
+            case nir_intrinsic_load_global_constant_bounded:
+               r.bound = in->src[2].ssa;
+               FALLTHROUGH;
+            case nir_intrinsic_load_global_constant_offset:
+               r.base = in->src[0].ssa;
+               r.offset_def = in->src[1].ssa;
+               break;
+            case nir_intrinsic_load_global:
+            case nir_intrinsic_load_global_constant:
+               if (!m5_split_address(in->src[0].ssa, &r.base, &r.offset_def))
+                  continue;
+               break;
+            default:
+               continue;
+            }
+            if (r.offset_def->bit_size != 32)
+               continue;
+            r.offset = m5_bounded_offset(nir_scalar_resolved(r.offset_def, 0), 12);
+            if (r.offset.valid)
+               util_dynarray_append(&reads, r);
+         }
+
+         unsigned n = util_dynarray_num_elements(&reads, struct m5_word_read);
+         if (n < 2)
+            continue;
+         struct m5_word_read *r = reads.data;
+         qsort(r, n, sizeof(*r), m5_word_read_cmp);
+
+         for (unsigned g = 0; g < n;) {
+            /* The reads of one record: one base, size and offset term. */
+            unsigned g_end = g + 1, earliest = g;
+            while (g_end < n && m5_word_reads_same_record(&r[g], &r[g_end])) {
+               if (r[g_end].load->instr.index < r[earliest].load->instr.index)
+                  earliest = g_end;
+               g_end++;
+            }
+
+            /* Split the record into loads of up to four consecutive words. */
+            unsigned units[64][2], nr_units = 0;
+            for (unsigned i = g; i < g_end && nr_units < ARRAY_SIZE(units);) {
+               unsigned count = 1;
+               while (i + count < g_end && count < 4 &&
+                      m5_word_reads_adjacent(&r[i + count - 1], &r[i + count]))
+                  count++;
+               while (count > 1) {
+                  unsigned first = i, last = i;
+                  for (unsigned j = i + 1; j < i + count; j++) {
+                     if (r[j].load->instr.index < r[first].load->instr.index)
+                        first = j;
+                     if (r[j].load->instr.index > r[last].load->instr.index)
+                        last = j;
+                  }
+                  if (m5_bounded_can_move(&r[first].load->instr, &r[last].load->instr))
+                     break;
+                  count = 1;
+               }
+               units[nr_units][0] = i;
+               units[nr_units][1] = count;
+               nr_units++;
+               i += count;
+            }
+            const unsigned covered = units[nr_units - 1][0] +
+                                     units[nr_units - 1][1] - g;
+
+            bool merged = false;
+            for (unsigned u = 0; u < nr_units; u++)
+               merged |= units[u][1] > 1;
+            if (covered != g_end - g || !merged) {
+               g = g_end;
+               continue;
+            }
+
+            nir_builder b = nir_builder_at(nir_before_instr(&r[earliest].load->instr));
+            nir_def *x_min = nir_iadd_imm(&b, r[earliest].offset_def,
+               (uint32_t)(r[g].offset.constant - r[earliest].offset.constant));
+
+            for (unsigned u = 0; u < nr_units; u++) {
+               const unsigned i = units[u][0], count = units[u][1];
+               if (count < 2)
+                  continue;
+
+               unsigned first = i;
+               for (unsigned j = i + 1; j < i + count; j++) {
+                  if (r[j].load->instr.index < r[first].load->instr.index)
+                     first = j;
+               }
+               const uint32_t delta = r[i].offset.constant - r[g].offset.constant;
+               b.cursor = nir_before_instr(&r[first].load->instr);
+
+               nir_intrinsic_instr *low = r[i].load;
+               const unsigned align_mul = nir_intrinsic_align_mul(low);
+               const unsigned align_offset = nir_intrinsic_align_offset(low);
+               enum gl_access_qualifier access = nir_intrinsic_access(low);
+               const bool constant =
+                  low->intrinsic == nir_intrinsic_load_global_constant ||
+                  low->intrinsic == nir_intrinsic_load_global_constant_offset ||
+                  low->intrinsic == nir_intrinsic_load_global_constant_bounded;
+               if (r[i].bound && constant)
+                  access |= ACCESS_NON_WRITEABLE | ACCESS_CAN_REORDER;
+
+               nir_def *x = nir_iadd_imm(&b, x_min, delta);
+               nir_def *addr = nir_iadd(&b, r[i].base, nir_u2u64(&b, x));
+               nir_def *v = constant
+                  ? nir_load_global_constant(&b, count, 32, addr,
+                       .align_mul = align_mul, .align_offset = align_offset,
+                       .access = access)
+                  : nir_load_global(&b, count, 32, addr,
+                       .align_mul = align_mul, .align_offset = align_offset,
+                       .access = access);
+               for (unsigned j = 0; j < count; j++) {
+                  nir_def *word = nir_channel(&b, v, j);
+                  if (r[i].bound) {
+                     nir_def *end = nir_uadd_sat(&b, x, nir_imm_int(&b, 4 * (j + 1)));
+                     word = nir_bounds_agx(&b, word, end, r[i].bound);
+                  }
+                  nir_def_replace(&r[i + j].load->def, word);
+               }
+            }
+            progress = true;
+            g = g_end;
+         }
+      }
+      any |= nir_progress(progress, impl, nir_metadata_control_flow);
+   }
+   util_dynarray_fini(&reads);
+   return any;
+}
+
 static nir_def *
 bound_offset(nir_builder *b, nir_def *valid, nir_scalar offset)
 {
@@ -918,16 +1546,133 @@ lower_load_global_bounded(nir_builder *b, nir_intrinsic_instr *intr)
 
 struct global_load_lowering {
    bool soft_fault;
-   /* G17P: storage-buffer loads use the soft-fault form as well: load, then
-    * zero the value when the access is out of bounds. */
+   /* G17P and M5: storage-buffer loads use the soft-fault form as well:
+    * load, then zero the value when the access is out of bounds. */
    bool storage_result_select;
+   /* M5 diagnostic: single-word bounded loads become predicated loads. */
+   bool predicated;
 };
 
+static bool
+lower_physical_sparse_load(nir_builder *b, nir_intrinsic_instr *intr,
+                           UNUSED void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_load_deref)
+      return false;
+
+   nir_deref_instr *leaf = nir_src_as_deref(intr->src[0]);
+   if (!leaf || leaf->modes != nir_var_mem_global)
+      return false;
+
+   nir_deref_path path;
+   nir_deref_path_init(&path, leaf, NULL);
+   nir_deref_instr *root = path.path[0];
+   if (root->deref_type != nir_deref_type_cast ||
+       root->parent.ssa->num_components != 1 ||
+       root->parent.ssa->bit_size != 64) {
+      nir_deref_path_finish(&path);
+      return false;
+   }
+
+   /* Generated read-only aliases are not coherent with the writable VA
+    * within an M5 dispatch. Only unbound sparse pages need substituted
+    * storage; resident reads must keep the application's original VA. */
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_def *table = nir_load_root_agx(b);
+#define MAP_FIELD(field) nir_load_global_constant( \
+   b, 1, 64, nir_iadd_imm(b, table, hk_root_descriptor_offset(field)), \
+   .align_mul = 8)
+   nir_def *map = MAP_FIELD(physical_sparse_map);
+   nir_def *heap_base = MAP_FIELD(physical_sparse_base);
+   nir_def *heap_size = MAP_FIELD(physical_sparse_size);
+#undef MAP_FIELD
+   nir_def *address = path.path[0]->parent.ssa;
+   for (unsigned i = 1; path.path[i]; i++)
+      address = nir_explicit_io_address_from_deref(
+         b, path.path[i], address, nir_address_format_64bit_global);
+   const unsigned bits = intr->def.bit_size;
+   const unsigned memory_bits = bits == 1 ? 32 : bits;
+   const unsigned units = memory_bits == 64 ? 2 : 1;
+   const unsigned unit_bits = MIN2(memory_bits, 32);
+   const unsigned stride = glsl_get_explicit_stride(leaf->type);
+   uint32_t align_mul, align_offset;
+   if (!nir_get_explicit_deref_align(leaf, true, &align_mul, &align_offset)) {
+      align_mul = memory_bits / 8;
+      align_offset = 0;
+   }
+   nir_def *components[NIR_MAX_VEC_COMPONENTS];
+   nir_def *direct[NIR_MAX_VEC_COMPONENTS];
+   nir_def *count = nir_load_global_constant(b, 1, 32, map, .align_mul = 4);
+   nir_if *sparse_if = nir_push_if(b, nir_ine_imm(b, count, 0));
+   for (unsigned c = 0; c < intr->def.num_components; c++) {
+      nir_def *parts[2];
+      for (unsigned u = 0; u < units; u++) {
+         const unsigned byte = c * (stride ? stride : memory_bits / 8) +
+                               u * unit_bits / 8;
+         nir_def *ptr = nir_iadd_imm(b, address, byte);
+         nir_def *relative = nir_isub(b, ptr, heap_base);
+         nir_def *valid = nir_ult(b, relative, heap_size);
+         nir_def *page = nir_ushr_imm(b, relative, 14);
+         nir_def *word = nir_bcsel(b, valid, nir_ushr_imm(b, page, 5),
+                                   nir_imm_int64(b, 0));
+         nir_def *entry = nir_load_global_constant(
+            b, 1, 32, nir_iadd_imm(b, nir_iadd(b, map, nir_ishl_imm(b, word, 2)), 16),
+            .align_mul = 4);
+         nir_def *mask = nir_ishl(b, nir_imm_int(b, 1),
+            nir_iand_imm(b, nir_u2u32(b, page), 31));
+         nir_def *nonresident = nir_iand(b, valid,
+            nir_ine_imm(b, nir_iand(b, entry, mask), 0));
+         nir_def *read_address = nir_bcsel(b, nonresident,
+            nir_imm_int64(b, AGX_ZERO_PAGE_ADDRESS), ptr);
+         parts[u] = nir_load_global(b, 1, unit_bits, read_address,
+            .align_mul = MIN2(align_mul, unit_bits / 8),
+            .align_offset = (align_offset + byte) % MIN2(align_mul, unit_bits / 8),
+            .access = nir_intrinsic_access(intr));
+      }
+      components[c] = units == 2
+         ? nir_pack_64_2x32(b, nir_vec2(b, parts[0], parts[1]))
+         : bits == 1 ? nir_ine_imm(b, parts[0], 0) : parts[0];
+   }
+   nir_push_else(b, sparse_if);
+   for (unsigned c = 0; c < intr->def.num_components; c++) {
+      const unsigned byte = c * (stride ? stride : memory_bits / 8);
+      direct[c] = nir_load_global(b, 1, memory_bits,
+         nir_iadd_imm(b, address, byte),
+         .align_mul = align_mul,
+         .align_offset = (align_offset + byte) % align_mul,
+         .access = nir_intrinsic_access(intr));
+      if (bits == 1)
+         direct[c] = nir_ine_imm(b, direct[c], 0);
+   }
+   nir_pop_if(b, sparse_if);
+   for (unsigned c = 0; c < intr->def.num_components; c++)
+      components[c] = nir_if_phi(b, components[c], direct[c]);
+   nir_def_rewrite_uses(&intr->def,
+                        nir_vec(b, components, intr->def.num_components));
+   nir_instr_remove(&intr->instr);
+   nir_deref_path_finish(&path);
+   return true;
+}
+
+
 static bool
 lower_load_global_constant_offset_instr(nir_builder *b,
                                         nir_intrinsic_instr *intrin, void *data)
 {
    const struct global_load_lowering *state = data;
+   if (state->predicated &&
+       intrin->intrinsic == nir_intrinsic_load_global_bounded &&
+       intrin->def.bit_size == 32 && intrin->num_components == 1) {
+      b->cursor = nir_before_instr(&intrin->instr);
+      nir_def *valid = check_in_bounds(b, intrin);
+      nir_def *value = nir_load_global_predicated_agx(
+         b, 1, 32, intrin->src[0].ssa, intrin->src[1].ssa, valid,
+         .access = nir_intrinsic_access(intrin) & ~ACCESS_CAN_SPECULATE,
+         .align_mul = nir_intrinsic_align_mul(intrin),
+         .align_offset = nir_intrinsic_align_offset(intrin));
+      nir_def_replace(&intrin->def, value);
+      return true;
+   }
 
    if ((intrin->intrinsic == nir_intrinsic_load_global_bounded &&
         !(state->soft_fault && state->storage_result_select)) ||
@@ -2249,6 +2994,9 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
    NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_push_const,
             nir_address_format_32bit_offset);
 
+   if (agx_chip_g17_m5(dev->dev.chip))
+      NIR_PASS(_, nir, agx_nir_lower_cmat);
+
    /* Turn cache flushes into image coherency bits while we still have derefs */
    NIR_PASS(_, nir, nir_lower_memory_model);
 
@@ -2260,6 +3008,9 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
    const bool lower_g17p_robust_images =
       dev->dev.chip == AGX_CHIP_G17P &&
       rs->images != VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DISABLED_EXT;
+   const bool lower_agx3_robust_fetches =
+      (dev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->dev.chip)) &&
+      rs->images != VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DISABLED_EXT;
    const bool native_robust_images =
       agx_chip_explicit_robust_image_bounds(dev->dev.chip) &&
       rs->images != VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DISABLED_EXT;
@@ -2267,7 +3018,9 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
    /* A G17P image load does not see a store made earlier in the same
     * invocation; forward the stored value. Run it before the robust lowering
     * so the loads and stores it tracks are still unconditional. */
-   if (agx_chip_image_load_misses_own_store(dev->dev.chip))
+   if (agx_chip_image_load_misses_own_store(dev->dev.chip) ||
+       (agx_chip_g17_m5(dev->dev.chip) &&
+        debug_get_bool_option("ASAHI_M5_IMAGE_STORE_FORWARD", false)))
       NIR_PASS(_, nir, hk_nir_forward_image_stores,
                lower_g17p_robust_images || native_robust_images ||
                   (features & HK_FEAT_NULL_DESCRIPTOR));
@@ -2309,9 +3062,12 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
     * texture_load with a 32-bit immediate source, which trips
     * agx_validate_sources() in a debug build and, in a release build, reaches
     * an encoder with no offset field at all. */
-   if (agx_native_graphics(&dev->dev))
+   /* M5 keeps a depth comparison's bias in the sampler descriptor, as the
+    * M5 path always has; only a Gen4 sampler moves it into the shader. */
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
       NIR_PASS(_, nir, agx_nir_lower_g17p_texture_early, support_lod_bias,
-               agx_chip_sampler_has_lod_bias(dev->dev.chip));
+               agx_chip_sampler_has_lod_bias(dev->dev.chip) ||
+                  agx_chip_g17_m5(dev->dev.chip));
    else
       NIR_PASS(_, nir, agx_nir_lower_texture_early, support_lod_bias);
 
@@ -2332,6 +3088,12 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
       NIR_PASS(_, nir, agx_nir_lower_custom_border);
    }
 
+   if (dev->vk.enabled_features.sparseResidencyBuffer &&
+       agx_chip_g17_m5(dev->dev.chip)) {
+      NIR_PASS(_, nir, nir_shader_intrinsics_pass,
+               lower_physical_sparse_load, nir_metadata_none, NULL);
+   }
+
    NIR_PASS(_, nir, hk_nir_lower_descriptors, rs, set_layout_count,
             set_layouts, false, dev->dev.sparse_ro_offset,
             hk_device_physical(dev)->scaled_sampling,
@@ -2340,7 +3102,8 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
             nir_address_format_64bit_global);
    NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_ssbo,
             hk_buffer_addr_format(rs->storage_buffers,
-                                  rs->null_storage_buffer_descriptor));
+                                  rs->null_storage_buffer_descriptor,
+                                  agx_chip_g17_m5(dev->dev.chip)));
    NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_ubo,
             hk_ubo_addr_format(rs->uniform_buffers));
 
@@ -2370,6 +3133,43 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
             mesa_loge("Gen1 native ray query: unsupported shape or operation");
             return;
          }
+      } else if (dev->m5_native_rq) {
+         /* A query used in closest-hit/miss/callable code is only live after
+          * the TraceRay query of the same invocation has finished, and
+          * separate shader-level queries are often sequential too. Merge
+          * queries whose live ranges do not overlap, so only queries that are
+          * genuinely live at the same time need distinct native contexts. */
+         /* Constant-indexed query arrays become separate queries first. */
+         NIR_PASS(_, nir, agx_nir_m5_split_ray_query_arrays);
+         NIR_PASS(_, nir, nir_split_array_vars, nir_var_function_temp);
+         NIR_PASS(_, nir, nir_opt_ray_queries);
+         NIR_PASS(_, nir, nir_opt_ray_query_ranges);
+
+         const struct agx_m5_ray_query_layout layout = {
+            .payload_uniform = dev->m5_rq_instances ? 134 : 122,
+            .bvh_uniform = dev->m5_rq_instances ? 100 : 96, .status_uniform = 134,
+            .launch_header = 0x10000000,
+            .context_uniform = dev->m5_rq_instances ? 160 : 148,
+            .as_table_uniform = dev->m5_rq_instances ? 164 : 152,
+            .instance_traversal = dev->m5_rq_instances,
+            .compute_kernel = true,
+         };
+         /* Geometry queries are lowered here and then cloned into the poly
+          * kernels and the raster vertex program. Earlier recoveries in GS
+          * query draws predate the dead-query removal from the poly kernels
+          * (agx_nir_m5_remove_dead_ray_queries); with it, all 578 geometry
+          * ray-query CTS cases ran on one worker without a fault.
+          * ASAHI_M5_GS_RAY_QUERY_DISABLE=1 refuses them for diagnosis. */
+         const bool refused_stage = nir->info.stage == MESA_SHADER_GEOMETRY &&
+            debug_get_bool_option("ASAHI_M5_GS_RAY_QUERY_DISABLE", false);
+         if (refused_stage || !agx_nir_lower_m5_ray_query(nir, &layout)) {
+            mesa_loge(refused_stage
+               ? "M5 native ray query: geometry stage disabled"
+               : "M5 native ray query: unsupported query shape or operation");
+            if (debug_get_bool_option("ASAHI_M5_LAUNCH_DEBUG", false))
+               nir_print_shader(nir, stderr);
+            return;
+         }
       } else {
          NIR_PASS(_, nir, hk_nir_lower_ray_queries);
       }
@@ -2406,14 +3206,32 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
    NIR_PASS(progress, nir, nir_opt_sink, opts);
    NIR_PASS(progress, nir, nir_opt_move, opts);
 
+   if (debug_get_bool_option("ASAHI_M5_BOUNDED_VEC_PROBE", false) &&
+       !dev->vk.enabled_features.sparseResidencyBuffer) {
+      NIR_PASS(_, nir, nir_opt_cse);
+      NIR_PASS(_, nir, m5_vectorize_bounded_loads);
+   }
    struct global_load_lowering global_loads = {
       .soft_fault = agx_has_soft_fault(&dev->dev),
-      .storage_result_select = dev->dev.chip == AGX_CHIP_G17P,
+      .storage_result_select = dev->dev.chip == AGX_CHIP_G17P ||
+                               agx_chip_g17_m5(dev->dev.chip),
+      .predicated = agx_chip_isa_encoding(dev->dev.chip) ==
+         AGX_ISA_ENCODING_AGX3C &&
+         nir->info.stage == MESA_SHADER_COMPUTE && !nir->info.internal &&
+         !dev->vk.enabled_features.sparseResidencyBuffer &&
+         debug_get_bool_option("ASAHI_M5_PREDICATED_LOAD_PROBE", false),
    };
+   if (agx_chip_g17_m5(dev->dev.chip) && global_loads.soft_fault &&
+       !global_loads.predicated) {
+      /* Reads of one descriptor share its base and size only after CSE. */
+      NIR_PASS(_, nir, nir_opt_cse);
+      NIR_PASS(_, nir, m5_merge_buffer_words);
+   }
    NIR_PASS(_, nir, nir_shader_intrinsics_pass,
             lower_load_global_constant_offset_instr, nir_metadata_none,
             &global_loads);
 
+
    if (nir->info.stage == MESA_SHADER_TASK || nir->info.stage == MESA_SHADER_MESH) {
       NIR_PASS(_, nir, nir_lower_vars_to_explicit_types, nir_var_mem_task_payload,
                shared_var_info);
@@ -2450,7 +3268,7 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
    if (nir->info.stage == MESA_SHADER_FRAGMENT) {
       lower_indirect_modes |= nir_var_shader_out;
 
-      if (agx_native_graphics(&dev->dev))
+      if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
          lower_indirect_modes |= nir_var_shader_in;
    } else if (nir->info.stage == MESA_SHADER_VERTEX) {
       lower_indirect_modes |= nir_var_shader_in | nir_var_shader_out;
@@ -2461,12 +3279,12 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
 
    /* SPIR-V represents a fragment gl_Layer read as load_layer_id, and the
     * generic AGX options deliberately keep that system-value form for AGX2.
-    * G17P has no fragment layer system register: Apple's shaders consume the
+    * AGX3 reads the API fragment layer as a flat input: Apple's shaders consume the
     * raster layer through a flat coefficient. Convert the sysval back to a
     * shader input here, after the generic system-value lowering but before IO
     * lowering, so the regular varying linker builds the coefficient binding. */
    if (nir->info.stage == MESA_SHADER_FRAGMENT &&
-       agx_native_graphics(&dev->dev)) {
+       (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))) {
       const nir_lower_sysvals_to_varyings_options sysvals_to_varyings = {
          .layer_id = true,
       };
@@ -2474,6 +3292,27 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
                &sysvals_to_varyings);
    }
 
+   /* Tess levels are compact float arrays: nir_lower_io turns an indirect
+    * element index into an IO offset counted in components, while poly's
+    * TCS/TES buffer addressing counts IO offsets in slots. A control shader
+    * writing gl_TessLevelOuter[gl_InvocationID] therefore stored levels 1/2
+    * into other slots, left them zero and culled every patch. Make the
+    * levels vectors so indexed accesses become per-component accesses. */
+   if (agx_chip_g17_m5(dev->dev.chip) &&
+       (nir->info.stage == MESA_SHADER_TESS_CTRL ||
+        nir->info.stage == MESA_SHADER_TESS_EVAL))
+      NIR_PASS(_, nir, nir_lower_tess_level_array_vars_to_vec);
+
+   /* Mesh outputs are per-vertex/per-primitive arrays, so a component
+    * write such as gl_MeshVerticesEXT[i].gl_Position.y reaches here as an
+    * array deref of a vector, which nir_lower_io would store to component
+    * 0. Turn those into masked vector accesses first. */
+   if (nir->info.stage == MESA_SHADER_MESH)
+      NIR_PASS(_, nir, nir_lower_array_deref_of_vec, nir_var_shader_out, NULL,
+               nir_lower_direct_array_deref_of_vec_load |
+               nir_lower_indirect_array_deref_of_vec_load |
+               nir_lower_direct_array_deref_of_vec_store |
+               nir_lower_indirect_array_deref_of_vec_store);
    NIR_PASS(_, nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out,
             glsl_type_size,
             nir_lower_io_lower_64bit_to_32 |
@@ -2493,12 +3332,12 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
                hk_device_physical(dev));
    }
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       /* hk_nir_lower_robust_image_access() leaves formatless loads to this
        * pass. */
       NIR_PASS(_, nir, agx_nir_lower_texture_with_buffer_metadata,
                agx_chip_descriptor_generation(dev->dev.chip),
-               lower_g17p_robust_images, dev->dev.sparse_ro_offset,
+               lower_agx3_robust_fetches, dev->dev.sparse_ro_offset,
                dev->vk.enabled_features.sparseResidencyBuffer);
    } else {
       NIR_PASS(_, nir, agx_nir_lower_texture_for_generation,
@@ -2506,9 +3345,18 @@ hk_lower_nir(struct hk_device *dev, nir_shader *nir,
    }
    NIR_PASS(_, nir, agx_nir_lower_multisampled_image_store);
 
-   if (nir->info.stage != MESA_SHADER_MESH) {
+   /* Mesh adds its native UVB shared arena after this frontend lowering.
+    * Keep application shared accesses byte addressed until that arena exists;
+    * the mesh branch then preprocesses every shared access exactly once.
+    * Reprocessing here and there divides preexisting element indices again.
+    * The M5 task (object) branch likewise preprocesses after adding its
+    * child-grid words. */
+   if (nir->info.stage != MESA_SHADER_MESH &&
+       !(nir->info.stage == MESA_SHADER_TASK && agx_chip_g17_m5(dev->dev.chip))) {
       if (agx_native_graphics(&dev->dev))
          agx_preprocess_nir_g17p(nir);
+      else if (agx_chip_g17_m5(dev->dev.chip))
+         agx_preprocess_nir_m5(nir);
       else
          agx_preprocess_nir(nir);
    }
@@ -2757,26 +3605,6 @@ hk_g17p_lower_is_first_fan(nir_builder *b, nir_intrinsic_instr *intr,
    return true;
 }
 
-static bool
-hk_g17p_apply_api_mask_to_force_early(nir_builder *b,
-                                      nir_intrinsic_instr *intr, void *data)
-{
-   bool *done = data;
-   if (*done || intr->intrinsic != nir_intrinsic_sample_mask_agx)
-      return false;
-
-   /* Fold the API mask into the one forced-early coverage update.  A separate
-    * prolog MOVMSK followed by the main shader's forced-early MOVMSK loses the
-    * surviving samples on G17P; Apple's native program performs one combined
-    * update before reading the post-test coverage for fragment side effects. */
-   b->cursor = nir_before_instr(&intr->instr);
-   nir_def *api = nir_load_api_sample_mask_agx(b);
-   nir_src_rewrite(&intr->src[1],
-                   nir_iand(b, intr->src[1].ssa,
-                            nir_u2uN(b, api, nir_src_bit_size(intr->src[1]))));
-   *done = true;
-   return true;
-}
 
 static bool
 kill_psiz_write(nir_builder *b, nir_intrinsic_instr *intr, void *data)
@@ -2857,6 +3685,120 @@ hk_nir_uses_root(const nir_shader *nir)
    return false;
 }
 
+/* The value of an integer offset built from immediates: the descriptor
+ * lowering adds, scales and widens immediates without folding them. */
+static bool
+hk_nir_constant_offset(nir_scalar s, unsigned depth, uint64_t *value)
+{
+   s = nir_scalar_chase_movs(s);
+   if (nir_scalar_is_const(s)) {
+      *value = nir_scalar_as_uint(s);
+      return true;
+   }
+   if (!depth || !nir_scalar_is_alu(s))
+      return false;
+
+   const nir_op op = nir_scalar_alu_op(s);
+   uint64_t a, b = 0;
+   if (!hk_nir_constant_offset(nir_scalar_chase_alu_src(s, 0), depth - 1, &a))
+      return false;
+   if (nir_op_infos[op].num_inputs == 2 &&
+       !hk_nir_constant_offset(nir_scalar_chase_alu_src(s, 1), depth - 1, &b))
+      return false;
+
+   switch (op) {
+   case nir_op_u2u32:
+   case nir_op_u2u64:
+      *value = a;
+      break;
+   case nir_op_iadd:
+      *value = a + b;
+      break;
+   case nir_op_imul:
+   case nir_op_amul:
+      *value = a * b;
+      break;
+   case nir_op_ishl:
+      *value = a << (b & (s.def->bit_size - 1));
+      break;
+   default:
+      return false;
+   }
+   *value &= BITFIELD64_MASK(s.def->bit_size);
+   return true;
+}
+
+/* The HK_ROOT_READS_* classes of the root bytes one use of load_root_agx can
+ * read: an immediate offset loaded as is falls in the classes of its bytes,
+ * anything else could read any byte. */
+static uint8_t
+hk_nir_root_use_reads(nir_src *use)
+{
+   const uint8_t any = HK_ROOT_READS_ADDRESS | HK_ROOT_READS_VERTEX_INPUT |
+                       HK_ROOT_READS_DYNAMIC_BUFFERS;
+   if (nir_src_is_if(use) ||
+       nir_src_use_instr(use)->type != nir_instr_type_alu)
+      return any;
+
+   nir_alu_instr *add = nir_instr_as_alu(nir_src_use_instr(use));
+   if (add->op != nir_op_iadd)
+      return any;
+
+   const unsigned other = use == &add->src[0].src ? 1 : 0;
+   uint64_t offset;
+   if (!hk_nir_constant_offset(nir_get_scalar(add->src[other].src.ssa,
+                                              add->src[other].swizzle[0]),
+                               8, &offset))
+      return any;
+
+   uint8_t reads = 0;
+   nir_foreach_use_including_if(load_use, &add->def) {
+      if (nir_src_is_if(load_use) ||
+          nir_src_use_instr(load_use)->type != nir_instr_type_intrinsic)
+         return any;
+
+      nir_intrinsic_instr *load =
+         nir_instr_as_intrinsic(nir_src_use_instr(load_use));
+      if ((load->intrinsic != nir_intrinsic_load_global_constant &&
+           load->intrinsic != nir_intrinsic_load_global) ||
+          load_use != &load->src[0])
+         return any;
+
+      reads |= hk_root_reads_of(
+         offset, offset + load->def.num_components * load->def.bit_size / 8);
+   }
+   return reads;
+}
+
+/* hk_shader_info::agx3_root_reads, gathered where hk_nir_uses_root() is. */
+static uint8_t
+hk_nir_root_reads(const nir_shader *nir)
+{
+   uint8_t reads = 0;
+   nir_foreach_function_impl(impl, nir) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic ||
+                nir_instr_as_intrinsic(instr)->intrinsic !=
+                   nir_intrinsic_load_root_agx)
+               continue;
+
+            nir_foreach_use_including_if(use,
+                                         &nir_instr_as_intrinsic(instr)->def)
+               reads |= hk_nir_root_use_reads(use);
+         }
+      }
+   }
+
+   return reads;
+}
+
+struct hk_g17p_vertex_preamble_shift {
+   unsigned base;
+   /* Preamble loads below this half are fixed ABI state, not VBO uniforms. */
+   unsigned first;
+};
+
 static bool
 hk_g17p_shift_vertex_preamble(UNUSED nir_builder *b,
                               nir_intrinsic_instr *intr, void *data)
@@ -2864,9 +3806,10 @@ hk_g17p_shift_vertex_preamble(UNUSED nir_builder *b,
    if (intr->intrinsic != nir_intrinsic_load_preamble)
       return false;
 
-   const unsigned base = *(const unsigned *)data;
+   const struct hk_g17p_vertex_preamble_shift *shift = data;
+   const unsigned base = shift->base;
    const unsigned old = nir_intrinsic_base(intr);
-   if (old > UINT16_MAX - base)
+   if (old < shift->first || old > UINT16_MAX - base)
       return false;
 
    nir_intrinsic_set_base(intr, old + base);
@@ -2946,11 +3889,25 @@ hk_lower_hw_vs(nir_shader *nir, struct hk_shader *shader, bool kill_psiz,
  * lets it run per sample at all. hk_compile_nir() lays out the same thing
  * from the link info once the split has recorded it exactly.
  */
+static unsigned
+hk_native_fs_samples(enum agx_chip chip, const struct hk_fs_key *key)
+{
+   return agx_chip_g17_m5(chip) ? key->m5_position_samples : key->g17p_samples;
+}
+
+static unsigned
+hk_native_fs_registers(enum agx_chip chip)
+{
+   return agx_chip_g17_m5(chip) ? AGX_NUM_REGS : AGX3B_NUM_REGS;
+}
+
 static bool
-hk_g17p_unrolls_sample_shading(const nir_shader *nir,
-                               const struct hk_fs_key *key)
+hk_native_sample_shading_supported(const nir_shader *nir,
+                               const struct hk_fs_key *key, enum agx_chip chip)
 {
-   if (key->g17p_samples != 2 && key->g17p_samples != 4)
+   const unsigned samples = hk_native_fs_samples(chip, key);
+   if (samples != 2 && samples != 4 &&
+       !(agx_chip_g17_m5(chip) && samples == 8))
       return false;
 
    uint8_t locations =
@@ -2972,13 +3929,14 @@ hk_g17p_unrolls_sample_shading(const nir_shader *nir,
    struct agx_g17p_colour_layout layout;
 
    agx_g17p_colour_layout_init(&layout, locations, size_32, comps, false);
-   if (agx_g17p_sample_exports_end(&layout, key->g17p_samples) <=
-       AGX3B_NUM_REGS)
+   if (agx_g17p_sample_exports_end(&layout, samples) <=
+       hk_native_fs_registers(chip))
       return true;
 
    agx_g17p_colour_layout_init(&layout, locations, size_32, comps, true);
-   return agx_g17p_sample_exports_end(&layout, key->g17p_samples) <=
-          AGX3B_NUM_REGS;
+   return agx_g17p_sample_exports_end(&layout, samples) <=
+          hk_native_fs_registers(chip) ||
+          (agx_chip_g17_m5(chip) && samples == 8);
 }
 
 static void
@@ -3086,8 +4044,8 @@ hk_g17p_compile_vs_cprog(struct hk_device *dev, nir_shader **nir,
    } else {
       if (agx_get_compiler_debug() & AGX_DBG_SHADERDB)
          fprintf(stderr, "HK G17P VS CProg: declined:%s\n", declined);
-      mesa_loge("G17P vertex constant program failed to compile; "
-                "compiling without it");
+      mesa_loge("G17P vertex constant program failed to compile (%s); "
+                "compiling without it", agx_last_compile_refusal());
    }
 
    memset(&shader->b, 0, sizeof(shader->b));
@@ -3138,8 +4096,8 @@ hk_g17p_compile_fs_cprog(nir_shader **nir, struct agx_shader_key *key,
                 info->nr_preamble_gprs, info->agx3_preamble_staging_known);
       free(shader->b.binary);
    } else {
-      mesa_logw("G17P fragment constant program failed to compile; "
-                "compiling without it");
+      mesa_logw("G17P fragment constant program failed to compile (%s); "
+                "compiling without it", agx_last_compile_refusal());
    }
 
    memset(&shader->b, 0, sizeof(shader->b));
@@ -3433,6 +4391,28 @@ hk_vs_covers_clip_square(const nir_shader *vs)
    return true;
 }
 
+/* Query lowering clears info.ray_queries before backend compilation. The
+ * native traversal instructions retain the requirement for its fixed ABI. */
+static bool
+hk_m5_needs_query_abi(nir_shader *nir)
+{
+   if (nir->info.ray_queries)
+      return true;
+   nir_foreach_function_impl(impl, nir) {
+      nir_foreach_block(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_intrinsic)
+               continue;
+            nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;
+            if (op == nir_intrinsic_intersect_ray_agx ||
+                op == nir_intrinsic_continue_ray_agx)
+               return true;
+         }
+      }
+   }
+   return false;
+}
+
 static VkResult
 hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
                nir_shader *nir, VkShaderCreateFlagsEXT shader_flags,
@@ -3445,20 +4425,28 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
    const bool writes_external_memory = hk_shader_writes_external_memory(nir);
    const bool reads_images = hk_shader_reads_images(nir);
    /* Facts hk_g17p_note_first_draw() needs, before lowering obscures them. */
-   if (dev->dev.chip == AGX_CHIP_G17P && sw_stage == MESA_SHADER_FRAGMENT)
+   if ((dev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->dev.chip)) &&
+       sw_stage == MESA_SHADER_FRAGMENT)
       shader->info.fs.may_kill = hk_fs_may_kill(nir);
    unsigned nr_vbos = 0;
    bool kill_psiz = false;
    bool default_layer = true;
    if ((sw_stage == MESA_SHADER_FRAGMENT && dev->gen1_native_fragment_rq) ||
        (sw_stage != MESA_SHADER_FRAGMENT && sw_stage != MESA_SHADER_COMPUTE &&
-        dev->gen1_native_graphics_rq)) {
+        dev->gen1_native_graphics_rq) ||
+       /* The hardware evaluation variant becomes a vertex shader in
+        * poly_nir_lower_tes() below, after this scan. */
+       (dev->m5_native_rq && (nir->info.stage == MESA_SHADER_VERTEX ||
+                               nir->info.stage == MESA_SHADER_FRAGMENT ||
+                               (sw_stage == MESA_SHADER_TESS_EVAL && hw)))) {
       nir_foreach_function_impl(impl, nir) {
          nir_foreach_block(block, impl) {
             nir_foreach_instr(instr, block) {
-               if (instr->type == nir_instr_type_intrinsic &&
+               if ((instr->type == nir_instr_type_intrinsic &&
                    nir_instr_as_intrinsic(instr)->intrinsic ==
-                      nir_intrinsic_intersect_ray_gen1_agx)
+                      nir_intrinsic_intersect_ray_gen1_agx) ||
+                   (instr->type == nir_instr_type_intrinsic && dev->m5_native_rq &&
+                    nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_intersect_ray_agx))
                   shader->info.agx3_gen1_fragment_rq = true;
             }
          }
@@ -3497,10 +4485,12 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
           * 0..7. The monolithic VBO lowering may already have materialized
           * vertex ABI preamble loads, so move those as one unit before root
           * and descriptor-set lowering allocates the following ranges. */
-         const unsigned base = AGX_ABI_G17P_TEXTURE_POINTER_HALFS;
+         const struct hk_g17p_vertex_preamble_shift shift = {
+            .base = AGX_ABI_G17P_TEXTURE_POINTER_HALFS,
+         };
          NIR_PASS(_, nir, nir_shader_intrinsics_pass,
                   hk_g17p_shift_vertex_preamble,
-                  nir_metadata_control_flow, (void *)&base);
+                  nir_metadata_control_flow, (void *)&shift);
       }
    } else if (agx_native_graphics(&dev->dev) &&
               (nir->info.stage == MESA_SHADER_GEOMETRY ||
@@ -3633,6 +4623,24 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
       }
    }
 
+   if (agx_chip_g17_m5(dev->dev.chip) &&
+       agx_nir_lower_g17g_indexed_reads(nir)) {
+      shader->info.agx3_indexed_sampler_valid = true;
+      if (nir->info.stage == MESA_SHADER_VERTEX) {
+         /* Both native AGX3 shader ABIs reserve halves 0..7 for the texture
+          * and sampler heaps. Move already-lowered VBO uniforms together.
+          * A native query owns halves 0..167 and its heaps start at 168, so
+          * only the vertex ABI above the query state moves. */
+         const struct hk_g17p_vertex_preamble_shift shift = {
+            .base = AGX_ABI_G17P_TEXTURE_POINTER_HALFS,
+            .first = shader->info.agx3_gen1_fragment_rq ? 168 : 0,
+         };
+         NIR_PASS(_, nir, nir_shader_intrinsics_pass,
+                  hk_g17p_shift_vertex_preamble,
+                  nir_metadata_control_flow, (void *)&shift);
+      }
+   }
+
    /* For now, only shader objects are supported */
    if (sw_stage == MESA_SHADER_VERTEX) {
       nr_vbos = DIV_ROUND_UP(
@@ -3664,7 +4672,9 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
 
       NIR_PASS(_, nir, agx_nir_lower_fs_output_to_epilog_for_device,
                dev->dev.chip, &shader->info.fs.epilog_key);
-      if (agx_native_graphics(&dev->dev))
+      if (agx_chip_g17_m5(dev->dev.chip))
+         NIR_PASS(_, nir, agx_nir_lower_sample_mask_deferred_m5);
+      else if (agx_native_graphics(&dev->dev))
          NIR_PASS(_, nir, agx_nir_lower_sample_mask_deferred);
       else
          NIR_PASS(_, nir, agx_nir_lower_sample_mask);
@@ -3677,7 +4687,9 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
        * ones for every draw that can use it. Fold it, leaving the single
        * unconditional testing coverage update of the native form instead of
        * a uniform branch between a testing and a killing one. */
-      if (agx_chip_native_agx3(dev->dev.chip) && key && key->fs.static_tests &&
+      if ((agx_chip_native_agx3(dev->dev.chip) ||
+           agx_chip_g17_m5(dev->dev.chip)) &&
+          key && key->fs.static_tests &&
           !shader->info.fs.epilog_key.write_z &&
           !shader->info.fs.epilog_key.write_s) {
          bool folded = false;
@@ -3686,28 +4698,31 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
          shader->info.fs.static_tests = folded;
       }
 
-      if (agx_native_graphics(&dev->dev) &&
-          nir->info.fs.early_fragment_tests) {
-         bool done = false;
-         NIR_PASS(_, nir, nir_shader_intrinsics_pass,
-                  hk_g17p_apply_api_mask_to_force_early,
-                  nir_metadata_control_flow, &done);
-      }
+      NIR_PASS(_, nir, agx_nir_apply_api_mask_to_force_early, dev->dev.chip);
 
       if (nir->info.fs.uses_sample_shading &&
-          agx_native_graphics(&dev->dev)) {
-         /* hk_compile_shader() kept sample shading only for a fixed count.
-          * The linked epilog unrolls the same samples. */
-         assert(key && (key->fs.g17p_samples == 2 ||
-                        key->fs.g17p_samples == 4));
-
-         if (!agx_nir_lower_g17p_sample_shading(
-                nir, &shader->info.fs.epilog_key, key->fs.g17p_samples)) {
-            assert(!"the gate admitted a fragment whose exports do not fit");
-            ralloc_free(nir);
-            return vk_errorf(dev, VK_ERROR_UNKNOWN,
-                             "G17P: %u-sample colour exports exceed the register file",
-                             key->fs.g17p_samples);
+          (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))) {
+         /* A split shader exports each sample through fixed registers. M5
+          * fragments that exceed that file retain one sample's NIR until
+          * the epilog can be inlined, then unroll the complete body. */
+         const unsigned raster_samples = hk_native_fs_samples(dev->dev.chip, &key->fs);
+         const unsigned registers = hk_native_fs_registers(dev->dev.chip);
+         assert(raster_samples == 2 || raster_samples == 4 ||
+                (agx_chip_g17_m5(dev->dev.chip) && raster_samples == 8));
+
+         /* M5 unrolls up to eight samples in its larger register file, the
+          * same count and file hk_native_sample_shading_supported() admitted. */
+         if (!agx_nir_lower_g17p_sample_shading_file(
+                nir, &shader->info.fs.epilog_key, raster_samples, registers,
+                agx_chip_g17_m5(dev->dev.chip))) {
+            if (agx_chip_g17_m5(dev->dev.chip) && raster_samples == 8) {
+               shader->info.fs.m5_inline_sample_shading = true;
+            } else {
+               ralloc_free(nir);
+               return vk_errorf(dev, VK_ERROR_UNKNOWN,
+                                "G17P: %u-sample colour exports exceed the %u-half register file",
+                                raster_samples, registers);
+            }
          }
       } else if (nir->info.fs.uses_sample_shading) {
          /* Ensure the sample mask is preserved in register */
@@ -3724,17 +4739,22 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
 
       /* Explicit interpolateAtSample() is not sample shading. Fold its
        * single-sample clamp too, without cloning the per-pixel shader body
-       * or guessing a G17P samples-log2 special-register encoding. */
-      if (agx_native_graphics(&dev->dev) && key->fs.g17p_samples)
-         NIR_PASS(_, nir, agx_nir_specialize_sample_count,
-                  key->fs.g17p_samples);
-
-      NIR_PASS(_, nir, agx_nir_lower_fs_active_samples_to_register);
+       * or guessing a samples-log2 special-register encoding: neither G17P
+       * nor M5 has a grounded AGX3 encoding for that SR, and the packer
+       * refuses it. M5 keys the count as m5_position_samples. */
+      const unsigned native_samples =
+         hk_native_fs_samples(dev->dev.chip, &key->fs);
+      if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+          native_samples)
+         NIR_PASS(_, nir, agx_nir_specialize_sample_count, native_samples);
+
+      if (!shader->info.fs.m5_inline_sample_shading)
+         NIR_PASS(_, nir, agx_nir_lower_fs_active_samples_to_register);
       /* G17P exposes the raster layer through the flat coefficient table.
        * SR28 is the compute grid-Z register and does not hold a fragment
        * layer on this generation. */
       NIR_PASS(_, nir, agx_nir_lower_interpolation,
-               agx_native_graphics(&dev->dev));
+               agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip));
       if (agx_native_graphics(&dev->dev)) {
          NIR_PASS(_, nir, nir_shader_intrinsics_pass,
                   hk_g17p_lower_is_first_fan,
@@ -3795,8 +4815,10 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
                      hk_g17p_saturate_layer_viewport,
                      nir_metadata_control_flow, NULL);
          }
+         if (sw_stage == MESA_SHADER_VERTEX && agx_chip_g17_m5(dev->dev.chip))
+            NIR_PASS(_, nir, agx_nir_preserve_position_order);
          hk_lower_hw_vs(nir, shader, kill_psiz,
-                        agx_native_graphics(&dev->dev),
+                        agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip),
                         default_layer, sw_stage == MESA_SHADER_VERTEX);
          outputs = nir->info.outputs_written;
       } else {
@@ -3822,6 +4844,16 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
          : 0) +
       (sw_stage == MESA_SHADER_VERTEX && shader->info.agx3_gen1_fragment_rq
          ? 168 : 0);
+   /* TCS kernels retain their API stage through lowering. Reserve the native
+    * query prefix for them as well as compute, and only for live queries. */
+   const bool m5_native_query_abi =
+      (nir->info.stage == MESA_SHADER_COMPUTE ||
+       sw_stage == MESA_SHADER_TESS_CTRL ||
+       sw_stage == MESA_SHADER_GEOMETRY) && dev->m5_native_rq &&
+      (!debug_get_bool_option("ASAHI_M5_SHADER_LOCAL_RQ_ABI", true) ||
+       hk_m5_needs_query_abi(nir));
+   if (m5_native_query_abi)
+      root = dev->m5_rq_instances ? 176 : 164;
    if (sw_stage == MESA_SHADER_FRAGMENT)
       root = AGX_ABI_FUNI_ROOT;
    else if (sw_stage == MESA_SHADER_VERTEX)
@@ -3831,7 +4863,8 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
       ((sw_stage == MESA_SHADER_COMPUTE && dev->gen1_native_rq &&
         hk_nir_has_ray_query(nir)) ||
        shader->info.agx3_gen1_fragment_rq) ? 168 : 0;
-   if (sw_stage != MESA_SHADER_VERTEX || !shader->info.agx3_gen1_fragment_rq)
+   if ((sw_stage != MESA_SHADER_VERTEX || !shader->info.agx3_gen1_fragment_rq) &&
+       !(m5_native_query_abi && shader->info.agx3_gen1_fragment_rq))
       root += texture_uniform_base;
    shader->info.set_count = set_count;
    shader->info.agx3_root_uniform = root;
@@ -3843,8 +4876,10 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
     * no_epilog_discard mask). hk_lower_uvs_index turns those into root
     * accesses, so gather the resource requirement after that pass and before
     * lower_uniforms removes the load_root_agx markers. */
-   if (agx_native_graphics(&dev->dev))
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       shader->info.agx3_uses_root = hk_nir_uses_root(nir);
+      shader->info.agx3_root_reads = hk_nir_root_reads(nir);
+   }
 
    /* Measure while root markers are still explicit. Uniform lowering loads
     * the root pointer and set bases separately, so retain that prefix too.
@@ -3863,7 +4898,7 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
    NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_uniforms,
             nir_metadata_control_flow, &root);
 
-   if (agx_native_graphics(&dev->dev)) {
+   if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
       NIR_PASS(_, nir, nir_shader_intrinsics_pass,
                hk_g17p_lower_storage_records, nir_metadata_control_flow,
                NULL);
@@ -3878,9 +4913,34 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
       robust2_modes |= nir_var_mem_ssbo;
 #endif
 
+   /* Native M5 compute launches publish invariant inputs through TPS and
+    * execute a separate constant program. Apply that same compile/launch
+    * contract to every compute pipeline, including ordinary compute and
+    * meta compute, while excluding graphics variants dispatched on CDM. */
+   const bool m5_compute_cprog = agx_chip_g17_m5(dev->dev.chip) &&
+      sw_stage == MESA_SHADER_COMPUTE && nir->info.stage == MESA_SHADER_COMPUTE;
+
+   /* A fragment shader that reads descriptor-backed memory forms its native
+    * constant program, as on G17P. The native HAL300 fragment program state
+    * uses the same constant-calculation program setup of the state-load
+    * encoder as compute and publishes the TI
+    * spill/IPR quads in its PPP records (agx_ppp_hal300_graphics_core_pack).
+    * An M5 fragment unrolled inline over 8 samples keeps no preamble: its
+    * launch needs the monolithic program, whose preamble need not match.
+    * Ray-query fragments keep their qualified launch unchanged, as their
+    * vertex programs do. */
+   const bool fs_reads_descriptors =
+      nir->info.stage == MESA_SHADER_FRAGMENT &&
+      (nir->info.num_ubos || nir->info.num_ssbos || nir->info.num_textures ||
+       nir->info.num_images || hk_nir_has_tex(nir));
+   const bool m5_fs_cprog = agx_chip_g17_m5(dev->dev.chip) &&
+      sw_stage == MESA_SHADER_FRAGMENT && fs_reads_descriptors &&
+      !shader->info.fs.m5_inline_sample_shading &&
+      !shader->info.agx3_gen1_fragment_rq;
+
    struct agx_shader_key backend_key = {
       .reserved_preamble = root + (4 * (1 + set_count)),
-      .g17p_texture_uniform_base = texture_uniform_base,
+      .g17p_texture_uniform_base = m5_native_query_abi ? (dev->m5_rq_instances ? 168 : 156) : texture_uniform_base,
       .dev = agx_gather_device_key(&dev->dev),
       .no_stop = nir->info.stage == MESA_SHADER_FRAGMENT,
       .has_scratch = !nir->info.internal,
@@ -3897,20 +4957,21 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
        * the loop's uniform compare/early compare were lost). Descriptor
        * lowering above uses immediate native state slots, never the AGX2
        * texture/sampler-state-store protocol. Only G17P's constant program
-       * is qualified; other native chips never form a preamble. */
-      .no_preamble = agx_native_graphics(&dev->dev) &&
-         (!agx_chip_native_preamble(dev->dev.chip) ||
-          !(nir->info.stage == MESA_SHADER_FRAGMENT &&
-            (nir->info.num_ubos || nir->info.num_ssbos ||
-             nir->info.num_textures || nir->info.num_images ||
-             hk_nir_has_tex(nir)))),
+       * is qualified among the AGX3B chips; G15 never forms a preamble. */
+      .agx3_native_cprog = m5_compute_cprog,
+      .no_preamble = (agx_chip_g17_m5(dev->dev.chip) && !m5_compute_cprog &&
+                      !m5_fs_cprog) ||
+         (agx_native_graphics(&dev->dev) &&
+          (!agx_chip_native_preamble(dev->dev.chip) || !fs_reads_descriptors)),
    };
 
-   /* The native constant-program lowering applies exactly when G17P lets
-    * this compile form a preamble (fragment and vertex constant programs);
-    * carried in the key, never in the global debug word. */
+   /* The native constant-program lowering applies exactly when G17P or an
+    * M5 graphics stage lets this compile form a preamble (fragment and
+    * vertex constant programs); carried in the key, never in the global
+    * debug word. */
    backend_key.g17p_native_uniforms =
-      agx_native_graphics(&dev->dev) && !backend_key.no_preamble;
+      (agx_native_graphics(&dev->dev) || m5_fs_cprog) &&
+      !backend_key.no_preamble;
 
    /* Uniform DMA: the root table, every descriptor set pointer, the dynamic
     * buffers and the buffers of static sets are driver-resolvable
@@ -3919,9 +4980,22 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
     * fragment program copies only when it forms none. */
    const bool vs_dma = sw_stage == MESA_SHADER_VERTEX && hw &&
                        nir->info.stage == MESA_SHADER_VERTEX;
-   if (agx_native_graphics(&dev->dev) &&
-       ((backend_key.no_preamble && sw_stage == MESA_SHADER_FRAGMENT &&
-         nir->info.stage == MESA_SHADER_FRAGMENT) || vs_dma)) {
+   /* An ordinary compute pipeline forms no preamble, so without copies
+    * every thread loads its push data, dispatch base and descriptors from
+    * memory, one dependent load after another. On native AGX3 its launch
+    * copies them like a graphics stage's (hk_g17p_compute_uniform_ranges()),
+    * including meta compute pipelines; on M5 one with a native constant
+    * program does (hk_m5_compute_uniform_dma_ranges()). Compute-stage
+    * programs of graphics keep their own launch ABI; precompiled library
+    * helpers do not use this shader path. */
+   const bool cs_dma = sw_stage == MESA_SHADER_COMPUTE &&
+                       nir->info.stage == MESA_SHADER_COMPUTE;
+   const bool fs_dma = backend_key.no_preamble &&
+                       sw_stage == MESA_SHADER_FRAGMENT &&
+                       nir->info.stage == MESA_SHADER_FRAGMENT;
+   if ((agx_native_graphics(&dev->dev) && (fs_dma || vs_dma || cs_dma)) ||
+       (agx_chip_g17_m5(dev->dev.chip) &&
+        (fs_dma || vs_dma || (cs_dma && m5_compute_cprog)))) {
       backend_key.g17p_uniform_dma = true;
       backend_key.g17p_dma_root = root;
       backend_key.g17p_dma_set_mask =
@@ -3939,7 +5013,9 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
       backend_key.fs.inside_sample_loop = true;
 
    if (nir->info.stage == MESA_SHADER_FRAGMENT && key)
-      backend_key.fs.nr_samples = key->fs.g17p_samples;
+      backend_key.fs.nr_samples = agx_chip_g17_m5(dev->dev.chip)
+                                    ? key->fs.m5_position_samples
+                                    : key->fs.g17p_samples;
 
    /* Serialize compiles only while they print, so debug output
     * stays readable. Behaviour flags (native_uniforms, nopreamble, ...)
@@ -3957,14 +5033,15 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
 
    assert(nir->info.io_lowered);
 
-   /* Inline blend: keep what fast-link needs to rebuild this
-    * fragment program with its colour epilog inlined. Per-sample shading and
-    * internal shaders keep the split parts. */
-   if (agx_native_graphics(&dev->dev) &&
-       (agx_get_fs_structure_flags() & AGX_FS_STRUCTURE_INLINE_BLEND) &&
-       sw_stage == MESA_SHADER_FRAGMENT &&
-       nir->info.stage == MESA_SHADER_FRAGMENT && !nir->info.internal &&
-       !nir->info.fs.uses_sample_shading) {
+   /* Preserve NIR for optional G17P inline blending and mandatory M5
+    * sample shading whose split exports exceed the register file. */
+   if ((sw_stage == MESA_SHADER_FRAGMENT &&
+        shader->info.fs.m5_inline_sample_shading) ||
+       (agx_native_graphics(&dev->dev) &&
+        (agx_get_fs_structure_flags() & AGX_FS_STRUCTURE_INLINE_BLEND) &&
+        sw_stage == MESA_SHADER_FRAGMENT &&
+        nir->info.stage == MESA_SHADER_FRAGMENT && !nir->info.internal &&
+        !nir->info.fs.uses_sample_shading)) {
       struct blob blob;
       blob_init(&blob);
       nir_serialize(&blob, nir, false);
@@ -3977,18 +5054,59 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
          }
       }
       blob_finish(&blob);
+      if (shader->info.fs.m5_inline_sample_shading) {
+         if (!shader->g17p_mono_nir) {
+            ralloc_free(nir);
+            return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
+         }
+         /* This compile supplies resource metadata only. Command recording
+          * requires the complete inline program before any draw can run. */
+         NIR_PASS(_, nir, agx_nir_lower_g17p_1x_sample_shading);
+         backend_key.fs.inside_sample_loop = false;
+      }
+   }
+
+   /* Native query lowering and descriptor optimization can turn the last
+    * dynamic RT stack offset into a constant after the pipeline inliner has
+    * already tried promotion. Retry here, after preprocessing, so constant
+    * frame transport does not survive as per-lane memory traffic. */
+   if (agx_chip_g17_m5(dev->dev.chip) && nir->info.stage == MESA_SHADER_COMPUTE &&
+       nir->scratch_size) {
+      bool indirect = false, traversal = false;
+      nir_foreach_function_impl(impl, nir) {
+         nir_foreach_block(block, impl) {
+            nir_foreach_instr(instr, block) {
+               if (instr->type != nir_instr_type_intrinsic)
+                  continue;
+               nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+               traversal |= intr->intrinsic == nir_intrinsic_intersect_ray_agx;
+               if (intr->intrinsic == nir_intrinsic_load_scratch)
+                  indirect |= !nir_src_is_const(intr->src[0]);
+               else if (intr->intrinsic == nir_intrinsic_store_scratch)
+                  indirect |= !nir_src_is_const(intr->src[1]);
+            }
+         }
+      }
+      if (traversal && !indirect)
+         NIR_PASS(_, nir, nir_lower_scratch_to_var);
    }
 
    /* G17P hardware vertex programs (API VS, GS raster and native TES
     * variants) run per-draw uniform work once in a native vertex constant
     * program, as Apple's compiler does: e.g. ProjMat * ModelViewMat. */
-   const bool vs_cprog = agx_chip_native_preamble(dev->dev.chip) && hw &&
-                         nir->info.stage == MESA_SHADER_VERTEX &&
-                         (!shader->info.agx3_gen1_fragment_rq ||
-                          debug_get_bool_option("ASAHI_G17P_GRAPHICS_RQ_CPROG_DIAGNOSTIC", false));
+   /* HAL300 sets the same vertex constant program and packs its TI IPR/spill
+    * quads into every VDM template exactly as HAL200 does. */
+   const bool vs_cprog =
+      (agx_chip_native_preamble(dev->dev.chip) ||
+       agx_chip_g17_m5(dev->dev.chip)) &&
+      hw && nir->info.stage == MESA_SHADER_VERTEX &&
+      (!shader->info.agx3_gen1_fragment_rq ||
+       (agx_chip_native_preamble(dev->dev.chip) &&
+        debug_get_bool_option("ASAHI_G17P_GRAPHICS_RQ_CPROG_DIAGNOSTIC", false)));
    bool compiled = vs_cprog &&
       hk_g17p_compile_vs_cprog(dev, &nir, &backend_key, shader);
-   if (!compiled && agx_native_graphics(&dev->dev) &&
+   if (!compiled &&
+       (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
        nir->info.stage == MESA_SHADER_FRAGMENT && !backend_key.no_preamble)
       compiled = hk_g17p_compile_fs_cprog(&nir, &backend_key, shader);
    if (!compiled)
@@ -4035,6 +5153,9 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
    shader->code_ptr = shader->b.binary;
    shader->code_size = shader->b.info.binary_size;
 
+   if (unlikely(dev->frame_anatomy))
+      hk_frame_anatomy_dump_nir(dev, shader, nir, sw_stage);
+
    if (sw_stage == MESA_SHADER_FRAGMENT)
       shader->info.fs.g17p_samples = MAX2(backend_key.fs.nr_samples, 1);
 
@@ -4049,8 +5170,8 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
     * geometry only changes how its input vertices were produced.  Build the
     * same byte-grounded VertexProgramVariant payload from the final UVS
     * layout, while leaving compute-only GS variants untouched. */
-   if (agx_native_graphics(&dev->dev) && hw &&
-       shader->b.info.stage == MESA_SHADER_VERTEX) {
+   if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
+       hw && shader->b.info.stage == MESA_SHADER_VERTEX) {
       struct hk_agx3_g17p_vertex_compile_info compiler_info;
       hk_g17p_vertex_compile_info_init(shader, vs_cprog, &compiler_info);
       /* On failure the program info is cleared, so the draw paths see no
@@ -4093,12 +5214,57 @@ hk_compile_nir(struct hk_device *dev, const VkAllocationCallbacks *pAllocator,
 
 static const struct vk_shader_ops hk_shader_ops;
 
+struct hk_shader *
+hk_g17p_layout_shader_create(struct hk_device *dev, const struct hk_shader *api,
+                             struct agx_shader_part *part)
+{
+   struct hk_shader *s = malloc(sizeof(*s));
+   if (!s)
+      return NULL;
+
+   /* Everything but the program and its uniform layout is the API shader's:
+    * the draw still builds link keys, varyings and constant data from it.
+    * The copy shares the API shader's constant data, owns its code and
+    * links nothing itself. */
+   *s = *api;
+   s->b = *part;
+   part->binary = NULL;
+   s->code_ptr = s->b.binary;
+   s->code_size = s->b.info.binary_size;
+   s->bo = NULL;
+   s->bo_offset = 0;
+   s->preamble_addr = 0;
+   memset(&s->linked, 0, sizeof(s->linked));
+   s->only_linked = NULL;
+   s->g17p_mono_nir = NULL;
+   s->g17p_mono_nir_size = 0;
+
+   if (s->b.info.has_preamble || s->b.info.rodata.size_16) {
+      const size_t size = s->b.info.binary_size;
+      s->bo = size ? hk_code_alloc(dev, size, &s->bo_offset) : NULL;
+      if (!s->bo) {
+         free(s->b.binary);
+         free(s);
+         return NULL;
+      }
+      memcpy((uint8_t *)agx_bo_map(s->bo) + s->bo_offset, s->b.binary, size);
+      agx_bo_assert_range(s->bo, s->bo_offset, size);
+      s->preamble_addr = hk_shader_code_va(s) + s->b.info.preamble_offset;
+   }
+   return s;
+}
+
 static void
 hk_destroy_linked_shader(struct hk_device *dev, struct hk_linked_shader *linked)
 {
    hk_g17p_linked_compute_cache_finish(linked);
    hk_g17p_linked_graphics_cache_finish(dev, linked);
    agx_bo_unreference(&dev->dev, linked->b.bo);
+   if (linked->g17p_layout) {
+      free(linked->g17p_layout->b.binary);
+      agx_bo_unreference(&dev->dev, linked->g17p_layout->bo);
+      free(linked->g17p_layout);
+   }
    ralloc_free(linked);
 }
 
@@ -4278,10 +5444,21 @@ hk_compile_fs_sample_variant(struct hk_device *dev,
    const unsigned i = util_logbase2(samples);
    union hk_key key = src->key;
    key.fs.g17p_samples = samples;
+   if (agx_chip_g17_m5(dev->dev.chip))
+      key.fs.m5_position_samples = samples;
+
+   if (samples > 1 && nir->info.fs.uses_sample_shading &&
+       !hk_native_sample_shading_supported(nir, &key.fs, dev->dev.chip)) {
+      ralloc_free(nir);
+      return vk_error(dev, VK_ERROR_UNKNOWN);
+   }
 
    if (samples == 1)
       NIR_PASS(_, nir, agx_nir_lower_g17p_1x_sample_shading);
-   NIR_PASS(_, nir, agx_nir_lower_sample_intrinsics, false);
+   if (agx_chip_g17_m5(dev->dev.chip) && samples > 4)
+      NIR_PASS(_, nir, agx_nir_lower_sample_intrinsics_wide, false);
+   else
+      NIR_PASS(_, nir, agx_nir_lower_sample_intrinsics, false);
 
    VkResult result = hk_compile_nir(dev, pAllocator, nir, src->flags,
                                     &src->robustness, &key, src->features,
@@ -4322,14 +5499,10 @@ hk_fs_build_sample_variant(struct hk_device *dev, struct hk_api_shader *obj,
    return result;
 }
 
-/* twin_of: compile the custom-border-emulating twin of the shader that
- * saved these inputs; info then names its NIR and saved create inputs. */
-
-
-
-
+/* Parameter block addressed by the predicated object's uniform0 pointer.
+ * The ordinary mesh-only object keeps its smaller native passthrough ABI. */
 static nir_shader *
-hk_build_predicated_mesh_object(void)
+hk_build_predicated_mesh_object(enum agx_chip chip)
 {
    nir_builder b = nir_builder_init_simple_shader(
       MESA_SHADER_TASK, &agx_nir_options, "native predicated mesh object");
@@ -4368,11 +5541,20 @@ hk_build_predicated_mesh_object(void)
       nir_iadd(&b, grid, nir_u2u64(&b, nir_imul_imm(&b, lane, 4))), .align_mul = 4);
    nir_store_object_childcount_agx(&b, nir_bcsel(&b, nir_load_var(&b, enabled),
                                                value, nir_imm_int(&b, 0)));
-
-   agx_preprocess_nir_g17p(b.shader);
+   if (agx_chip_g17_m5(chip))
+      agx_preprocess_nir_m5(b.shader);
+   else
+      agx_preprocess_nir_g17p(b.shader);
    return b.shader;
 }
 
+/* twin_of: compile the custom-border-emulating twin of the shader that
+ * saved these inputs; info then names its NIR and saved create inputs. */
+
+
+
+
+
 static nir_def *
 hk_mesh_query_root(nir_builder *b, unsigned bit_size, unsigned offset)
 {
@@ -4430,6 +5612,64 @@ hk_lower_mesh_queries(nir_shader *nir)
    return nir_progress(true, impl, nir_metadata_none);
 }
 
+/* A mesh shader used with a task stage reads its child grid
+ * (gl_NumWorkGroups) from the native object->mesh record, not the API grid in
+ * the draw root. Run before hk_lower_nir, which would lower it to the root.
+ * Without the record (no kernel mesh rendering buffer) there is no source for
+ * it: returns false. */
+/* M5 task draws whose grid, draw count or predicate live in GPU memory
+ * (hk_draw_mesh_native) record one object workgroup and let
+ * libagx_m5_task_dispatch publish whether the draw executes. A task shader
+ * checks that word on entry: a disabled draw launches no mesh children. The
+ * root word is zero for every other draw, so only those loads the flag. */
+static void
+hk_m5_task_predicate(nir_shader *nir)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(nir);
+   nir_builder b = nir_builder_at(nir_before_impl(impl));
+   nir_def *flag = nir_load_global_constant(&b, 1, 64,
+      nir_iadd_imm(&b, nir_load_root_agx(&b),
+                   hk_root_descriptor_offset(mesh_task_flag)),
+      .align_mul = 8, .access = ACCESS_CAN_SPECULATE);
+   nir_push_if(&b, nir_ine_imm(&b, flag, 0));
+   {
+      nir_def *execute = nir_load_global(&b, 1, 32, flag, .align_mul = 4);
+      nir_push_if(&b, nir_ieq_imm(&b, execute, 0));
+      {
+         nir_launch_mesh_workgroups(&b, nir_imm_ivec3(&b, 0, 0, 0));
+         nir_jump(&b, nir_jump_halt);
+      }
+      nir_pop_if(&b, NULL);
+   }
+   nir_pop_if(&b, NULL);
+   nir_progress(true, impl, nir_metadata_none);
+}
+
+static bool
+hk_m5_mesh_task_grid(struct hk_device *dev, nir_shader *nir,
+                     VkShaderCreateFlagsEXT flags)
+{
+   if (nir->info.stage != MESA_SHADER_MESH || !hk_m5_task_enabled() ||
+       (flags & VK_SHADER_CREATE_NO_TASK_SHADER_BIT_EXT) ||
+       !hk_nir_reads_intrinsic(nir, nir_intrinsic_load_num_workgroups))
+      return true;
+   if (!hk_m5_task_record(dev->dev.params.features))
+      return false;
+   NIR_PASS(_, nir, agx_nir_lower_m5_mesh_task_record);
+   return true;
+}
+
+/* The native three-thread passthrough object program of an M5 mesh-only
+ * draw is composed directly (agx3_mesh_passthrough_object), so the compiler
+ * reports no staging size for it; its program declares compiler word d = 1.
+ * Applied when the program is built and when it is deserialized. */
+static void
+hk_m5_mesh_passthrough_linked(struct hk_shader *object)
+{
+   object->only_linked->b.g17p_compiler_d_valid = true;
+   object->only_linked->b.g17p_compiler_d = 1;
+}
+
 static VkResult
 hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *info,
                   const struct vk_graphics_pipeline_state *state,
@@ -4446,7 +5686,8 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
     * never qualifies then. */
    const bool vs_covers_clip_square =
       vs_covers >= 0 ? vs_covers > 0
-                     : dev->dev.chip == AGX_CHIP_G17P &&
+                     : (dev->dev.chip == AGX_CHIP_G17P ||
+                        agx_chip_g17_m5(dev->dev.chip)) &&
                           info->stage == MESA_SHADER_VERTEX &&
                           !info->nir->info.io_lowered &&
                           hk_vs_covers_clip_square(info->nir);
@@ -4464,29 +5705,41 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
    /* We consume the NIR, regardless of success or failure */
    nir_shader *nir = info->nir;
 
-   
-
-
-
+   /* hk_lower_nir turns these into draw-root loads. Scan for them first
+    * (after compute system-value lowering, which can introduce
+    * num_workgroups); NIR that arrives already lowered keeps the
+    * conservative answer. */
    const bool mesh_reads_draw_id =
-      nir->info.stage == MESA_SHADER_MESH &&
+      (nir->info.stage == MESA_SHADER_MESH ||
+       nir->info.stage == MESA_SHADER_TASK) &&
       (nir->info.io_lowered ||
        hk_nir_reads_intrinsic(nir, nir_intrinsic_load_draw_id));
    const bool mesh_reads_num_workgroups =
-      nir->info.stage == MESA_SHADER_MESH &&
+      (nir->info.stage == MESA_SHADER_MESH ||
+       nir->info.stage == MESA_SHADER_TASK) &&
       (nir->info.io_lowered ||
        hk_nir_reads_intrinsic(nir, nir_intrinsic_load_num_workgroups));
 
+   if (!nir->info.io_lowered &&
+       !hk_m5_mesh_task_grid(dev, nir, info->flags)) {
+      ralloc_free(nir);
+      return vk_errorf(dev, VK_ERROR_FEATURE_NOT_PRESENT,
+                       "mesh gl_NumWorkGroups under a task stage needs the task record");
+   }
    if (!nir->info.io_lowered) {
        hk_lower_nir(dev, nir, info->robustness, false, info->next_stage_mask,
                     info->set_layout_count, info->set_layouts, features);
    }
    if (dev->dev.chip == AGX_CHIP_G17P &&
        nir->info.stage == MESA_SHADER_FRAGMENT &&
-       !agx_nir_lower_m5_mesh_fragment(nir)) {
+       !agx_nir_lower_gen1_mesh_fragment(nir)) {
       ralloc_free(nir);
       return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT);
    }
+   /* M5 mesh shaders export per-primitive words on private vertices. */
+   if (agx_chip_g17_m5(dev->dev.chip) &&
+       nir->info.stage == MESA_SHADER_FRAGMENT)
+      NIR_PASS(_, nir, agx_nir_lower_m5_mesh_fragment);
 
 
    /* G17P compiles the custom-border emulation in only once the physical
@@ -4550,13 +5803,16 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
           (nir->info.outputs_written &
            (BITFIELD64_BIT(FRAG_RESULT_DEPTH) |
             BITFIELD64_BIT(FRAG_RESULT_STENCIL))));
-      sample_variants = agx_native_graphics(&dev->dev) &&
-         key_tmp.fs.g17p_samples == 0 &&
+      sample_variants = (agx_native_graphics(&dev->dev) ||
+                         agx_chip_g17_m5(dev->dev.chip)) &&
+         hk_native_fs_samples(dev->dev.chip, &key_tmp.fs) == 0 &&
          (nir->info.fs.uses_sample_shading || key_tmp.fs.force_sample_shading ||
           coverage_updates || agx_nir_needs_sample_count(nir));
    }
 
-   size_t size = hk_api_shader_size(info->stage, sample_variants);
+   const unsigned sample_variant_count = sample_variants
+      ? (agx_chip_g17_m5(dev->dev.chip) ? 4 : 3) : 0;
+   size_t size = hk_api_shader_size(info->stage, sample_variant_count);
    struct hk_api_shader *obj =
       vk_shader_zalloc(&dev->vk, &hk_shader_ops, info->stage, pAllocator, size);
 
@@ -4566,7 +5822,7 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
       return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
    }
 
-   obj->g17p_sample_variants = sample_variants;
+   obj->g17p_sample_variants = sample_variant_count;
    obj->border_emulating = border_emulating;
 
    if (border_nir) {
@@ -4593,6 +5849,8 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
    }
 
    mesa_shader_stage sw_stage = nir->info.stage;
+   const bool api_writes_external_memory =
+      hk_shader_writes_external_memory(nir);
 
    if (sw_stage == MESA_SHADER_FRAGMENT) {
       key = &key_tmp;
@@ -4604,10 +5862,10 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
        * Canonicalization is exact only at one sample; keep the draw-time
        * refusal for unsupported fixed-count export footprints.
        */
-      if (agx_native_graphics(&dev->dev) &&
+      if ((agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) &&
           nir->info.fs.uses_sample_shading &&
           !sample_variants &&
-          !hk_g17p_unrolls_sample_shading(nir, &key->fs)) {
+          !hk_native_sample_shading_supported(nir, &key->fs, dev->dev.chip)) {
          g17p_collapsed_sample_shading = true;
          NIR_PASS(_, nir, agx_nir_lower_g17p_1x_sample_shading);
       }
@@ -4625,8 +5883,14 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
          hk_fs_force_shader_depth_write(nir);
       }
 
-      if (!sample_variants)
-         NIR_PASS(_, nir, agx_nir_lower_sample_intrinsics, false);
+      if (!sample_variants) {
+         if (agx_chip_g17_m5(dev->dev.chip) &&
+             (key_tmp.fs.m5_position_samples == 0 ||
+              key_tmp.fs.m5_position_samples > 4))
+            NIR_PASS(_, nir, agx_nir_lower_sample_intrinsics_wide, false);
+         else
+            NIR_PASS(_, nir, agx_nir_lower_sample_intrinsics, false);
+      }
    } else if (sw_stage == MESA_SHADER_VERTEX) {
       key = &key_tmp;
    } else if (sw_stage == MESA_SHADER_TESS_CTRL) {
@@ -4648,17 +5912,22 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
 
    if (sw_stage == MESA_SHADER_MESH) {
       struct hk_shader *shader = hk_main_mesh_variant(obj);
-      if (dev->dev.chip != AGX_CHIP_G17P ||
+      if ((dev->dev.chip != AGX_CHIP_G17P && dev->dev.chip != AGX_CHIP_G17S) ||
           hk_mesh_mode() != HK_MESH_MODE_NATIVE_PROBE) {
          ralloc_free(nir);
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return vk_error(dev, VK_ERROR_UNKNOWN);
       }
-      if (nir->info.mesh.primitive_type == MESA_PRIM_POINTS &&
+      if (agx_chip_g17_m5(dev->dev.chip))
+         NIR_PASS(_, nir, agx_nir_m5_canonicalize_empty_mesh);
+      if ((dev->dev.chip == AGX_CHIP_G17P || agx_chip_g17_m5(dev->dev.chip)) &&
+          nir->info.mesh.primitive_type == MESA_PRIM_POINTS &&
           !(nir->info.outputs_written & VARYING_BIT_PSIZ))
          NIR_PASS(_, nir, nir_shader_intrinsics_pass,
                   hk_mesh_default_point_size, nir_metadata_control_flow, NULL);
-      if (nir->info.mesh.primitive_type == MESA_PRIM_TRIANGLES &&
+      if ((dev->dev.chip == AGX_CHIP_G17P ||
+           (state && agx_chip_g17_m5(dev->dev.chip))) &&
+          nir->info.mesh.primitive_type == MESA_PRIM_TRIANGLES &&
           !(nir->info.outputs_read & VARYING_BIT_PSIZ))
          NIR_PASS(_, nir, nir_shader_intrinsics_pass,
                   kill_mesh_triangle_psiz_write,
@@ -4677,32 +5946,45 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
       shader->info.mesh.vertex_outputs =
          nir->info.outputs_written & ~nir->info.per_primitive_outputs;
       shader->info.mesh.primitive_outputs =
-         nir->info.per_primitive_outputs & ~VARYING_BIT_PRIMITIVE_INDICES;
+         nir->info.per_primitive_outputs &
+         (agx_chip_g17_m5(dev->dev.chip) ? nir->info.outputs_written : ~UINT64_C(0)) &
+         ~VARYING_BIT_PRIMITIVE_INDICES;
       
 
 
 
 
       NIR_PASS(_, nir, nir_opt_constant_folding);
-      NIR_PASS(_, nir, hk_lower_mesh_queries);
+      if (dev->dev.chip == AGX_CHIP_G17P)
+         NIR_PASS(_, nir, hk_lower_mesh_queries);
       struct agx_m5_mesh_varyings varyings;
-      bool layout_ok = agx_m5_mesh_gather_varyings(nir, &varyings);
+      if (agx_chip_g17_m5(dev->dev.chip)) {
+         NIR_PASS(_, nir, nir_lower_io_to_scalar, nir_var_shader_out, NULL, NULL);
+         NIR_PASS(_, nir, agx_nir_lower_cull_distance_vs);
+      }
+      const bool m5_mesh = agx_chip_g17_m5(dev->dev.chip);
+      bool layout_ok = m5_mesh ? agx_m5_mesh_gather_varyings(nir, &varyings)
+                              : agx_gen1_mesh_gather_varyings(nir, &varyings);
 
       nir_builder layout = nir_builder_init_simple_shader(
          MESA_SHADER_VERTEX, &agx_nir_options, "native mesh raster interface");
       layout.shader->info.io_lowered = true;
+      if (m5_mesh) {
+         layout.shader->info.clip_distance_array_size = nir->info.clip_distance_array_size;
+         layout.shader->info.cull_distance_array_size = nir->info.cull_distance_array_size;
+      }
       layout.shader->info.outputs_written = VARYING_BIT_POS;
       nir_store_output(&layout, nir_undef(&layout, 4, 32),
          nir_imm_int(&layout, 0), .write_mask = 15,
          .io_semantics = {.location = VARYING_SLOT_POS, .num_slots = 1});
-      if (layout_ok && varyings.lv) {
+      if (layout_ok && !m5_mesh && varyings.lv) {
          layout.shader->info.outputs_written |= VARYING_BIT_LAYER;
          nir_store_output(&layout, nir_undef(&layout, 1, 32),
             nir_imm_int(&layout, 0), .write_mask = 1,
             .src_type = nir_type_uint32,
             .io_semantics = {.location = VARYING_SLOT_LAYER, .num_slots = 1});
       }
-      if (layout_ok && varyings.clip_count) {
+      if (layout_ok && !m5_mesh && varyings.clip_count) {
          layout.shader->info.clip_distance_array_size = varyings.clip_count;
          for (unsigned slot = VARYING_SLOT_CLIP_DIST0;
               slot <= VARYING_SLOT_CLIP_DIST1; ++slot) {
@@ -4716,14 +5998,14 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
                .io_semantics = {.location = slot, .num_slots = 1});
          }
       }
-      if (layout_ok && varyings.components[VARYING_SLOT_PSIZ]) {
+      if (layout_ok && !m5_mesh && varyings.components[VARYING_SLOT_PSIZ]) {
          layout.shader->info.outputs_written |= VARYING_BIT_PSIZ;
          nir_store_output(&layout, nir_undef(&layout, 1, 32),
             nir_imm_int(&layout, 0), .write_mask = 1,
             .src_type = nir_type_float32,
             .io_semantics = {.location = VARYING_SLOT_PSIZ, .num_slots = 1});
       }
-      for (unsigned slot = VARYING_SLOT_VAR0; layout_ok && slot < 64; ++slot) {
+      for (unsigned slot = m5_mesh ? 0 : VARYING_SLOT_VAR0; layout_ok && slot < 64; ++slot) {
          const unsigned n = varyings.components[slot];
          if (!n)
             continue;
@@ -4740,7 +6022,10 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
             .write_mask = BITFIELD_MASK(page_words), .src_type = nir_type_uint32,
             .io_semantics = {.location = AGX_MESH_PRIMITIVE_PAGE_SLOT, .num_slots = 1});
       }
-      agx_nir_lower_uvs(layout.shader, &shader->info.uvs, false);
+      /* An M5 mesh layer/viewport is also a user word (see
+       * agx_m5_mesh_gather_varyings), as for the M5 vertex stage. */
+      agx_nir_lower_uvs(layout.shader, &shader->info.uvs,
+                        m5_mesh && layout_ok && varyings.lv);
       shader->info.uvs.mesh_primitive_page = varyings.primitive_stride_B != 0;
       shader->info.mesh.primitive_stride_B = varyings.primitive_stride_B;
       ralloc_free(layout.shader);
@@ -4755,22 +6040,40 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
                           nir->info.perspective_varyings),
                         nir->info.linear_varyings);
          for (unsigned slot = 0; slot < 64; ++slot) {
-            if (varyings.components[slot])
+            if (varyings.components[slot] && slot != VARYING_SLOT_PSIZ)
                varyings.first[slot] = linked.slots[slot];
          }
+         if (varyings.components[VARYING_SLOT_PSIZ])
+            varyings.first[VARYING_SLOT_PSIZ] = shader->info.uvs.group_offs[UVS_PSIZ];
          if (varyings.lv)
             varyings.lv_first = shader->info.uvs.group_offs[UVS_LAYER_VIEWPORT];
          varyings.dynamic = false;
       }
       layout_ok &= varyings.size == shader->info.uvs.size;
-      if (!layout_ok || !agx_nir_lower_m5_task_payload_probe(nir) ||
-          !agx_nir_lower_m5_mesh(nir, &varyings)) {
+      layout_ok &= !varyings.clip_distance_count ||
+         varyings.clip_distance_first == shader->info.uvs.group_offs[UVS_CLIP_DIST];
+      layout_ok &= !m5_mesh || !varyings.lv ||
+         varyings.lv_first == shader->info.uvs.group_offs[UVS_LAYER_VIEWPORT];
+      if (!layout_ok ||
+          !(m5_mesh ? (hk_m5_task_enabled()
+                          ? agx_nir_lower_m5_task_payload(nir,
+                               hk_m5_task_payload_limit(dev->dev.params.features),
+                               hk_m5_task_record(dev->dev.params.features) ? 16 : 0)
+                          : agx_nir_lower_m5_task_payload_probe(nir))
+                    : agx_nir_lower_gen1_task_payload_probe(nir)) ||
+          !(m5_mesh ? agx_nir_lower_m5_mesh(nir, &varyings)
+                    : agx_nir_lower_gen1_mesh(nir, &varyings))) {
          ralloc_free(nir);
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return vk_errorf(dev, VK_ERROR_UNKNOWN,
                           "native mesh layout outside captured triangle contract");
       }
-      agx_preprocess_nir_g17p(nir);
+      /* Per-primitive outputs can raise the exported vertex count. */
+      shader->info.mesh.max_vertices = nir->info.mesh.max_vertices_out;
+      if (agx_chip_g17_m5(dev->dev.chip))
+         agx_preprocess_nir_m5(nir);
+      else
+         agx_preprocess_nir_g17p(nir);
       result = hk_compile_nir(dev, pAllocator, nir, info->flags,
                              info->robustness, NULL, features, shader,
                              sw_stage, true, NULL, info->set_layout_count,
@@ -4779,11 +6082,15 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return result;
       }
+      /* Native mesh lowering introduces private transport stores after the
+       * API write class was gathered. They do not change this declaration. */
+      shader->info.writes_external_memory = api_writes_external_memory;
+      struct hk_shader *object = &obj->variants[HK_MESH_VARIANT_RAST];
+      if (dev->dev.chip == AGX_CHIP_G17P) {
       
 
 
-      struct hk_shader *object = &obj->variants[HK_MESH_VARIANT_RAST];
-      nir_shader *object_nir = hk_build_predicated_mesh_object();
+      nir_shader *object_nir = hk_build_predicated_mesh_object(dev->dev.chip);
       if (!object_nir) {
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return vk_error(dev, VK_ERROR_UNKNOWN);
@@ -4795,9 +6102,44 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return result;
       }
+      object->info.writes_external_memory = false;
+      } else {
+      /* Mesh-only Vulkan pipelines use Apple's native three-thread
+       * passthrough object stage, whose operand-bearing code is composed,
+       * not copied from a fixed shader blob. */
+      uint8_t body[64], size;
+      if (!agx3_mesh_passthrough_object(0, body, &size)) {
+         hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
+         return vk_error(dev, VK_ERROR_UNKNOWN);
+      }
+      object->b.binary = malloc(size);
+      if (!object->b.binary) {
+         hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
+         return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
+      }
+      memcpy(object->b.binary, body, size);
+      object->code_ptr = object->b.binary;
+      object->code_size = size;
+      object->info.stage = MESA_SHADER_TASK;
+      object->b.info.stage = MESA_SHADER_TASK;
+      object->b.info.binary_size = size;
+      object->b.info.main_size = size;
+      object->b.info.nr_gprs = 8;
+      object->b.info.workgroup_size[0] = 3;
+      object->b.info.workgroup_size[1] = object->b.info.workgroup_size[2] = 1;
+      result = hk_init_link_ht(object, MESA_SHADER_TASK);
+      if (result == VK_SUCCESS)
+         result = hk_upload_shader(dev, object);
+      if (result != VK_SUCCESS) {
+         hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
+         return vk_error(dev, result);
+      }
+      hk_m5_mesh_passthrough_linked(object);
+      object->compiled = true;
+      }
       struct hk_shader *predicated =
          &obj->variants[HK_MESH_VARIANT_PREDICATED_OBJECT];
-      object_nir = hk_build_predicated_mesh_object();
+      nir_shader *object_nir = hk_build_predicated_mesh_object(dev->dev.chip);
       if (!object_nir) {
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return vk_error(dev, VK_ERROR_UNKNOWN);
@@ -4809,22 +6151,43 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return result;
       }
+      /* This compiler-generated object only transports the mesh launch. */
+      predicated->info.writes_external_memory = false;
    } else if (sw_stage == MESA_SHADER_TASK) {
-      
-
-
-
-      if (true) {
+      /* Compile the actual API object shader through captured native payload
+       * addressing and child-grid UVSW, never as a CDM mesh producer.
+       * taskShader is advertised only with the kernel's task mesh storage
+       * (hk_m5_task_record). */
+      if (hk_m5_task_enabled())
+         hk_m5_task_predicate(nir);
+      if (dev->dev.chip != AGX_CHIP_G17S ||
+          hk_mesh_mode() != HK_MESH_MODE_NATIVE_PROBE ||
+          !(hk_m5_task_enabled()
+               ? agx_nir_lower_m5_task(nir,
+                    hk_m5_task_payload_limit(dev->dev.params.features),
+                    hk_m5_task_record(dev->dev.params.features))
+               : agx_nir_lower_m5_task_probe(nir))) {
          ralloc_free(nir);
          hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
          return vk_errorf(dev, VK_ERROR_UNKNOWN,
                           "native task layout outside captured object contract");
       }
-      agx_preprocess_nir_g17p(nir);
+      if (agx_chip_g17_m5(dev->dev.chip))
+         agx_preprocess_nir_m5(nir);
+      else
+         agx_preprocess_nir_g17p(nir);
       result = hk_compile_nir(dev, pAllocator, nir, info->flags,
                              info->robustness, NULL, features, &obj->variants[0],
                              sw_stage, true, NULL, info->set_layout_count,
                              static_set_mask);
+      if (result == VK_SUCCESS) {
+         obj->variants[0].info.writes_external_memory = api_writes_external_memory;
+         /* hk_draw_mesh_native publishes the API grid and draw index to the
+          * object stage's draw root only when it reads them. */
+         obj->variants[0].info.mesh.reads_draw_id = mesh_reads_draw_id;
+         obj->variants[0].info.mesh.reads_num_workgroups =
+            mesh_reads_num_workgroups;
+      }
    } else if (sw_stage == MESA_SHADER_GEOMETRY) {
       struct hk_shader *main_variant = hk_main_gs_variant(obj);
       struct hk_shader *count_variant = hk_count_gs_variant(obj);
@@ -4835,9 +6198,16 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
          hk_gen1_ray_query_layout(dev, nir, 0);
       /* The raster replay is always a hardware vertex shader. */
       rq_layout.graphics_stage = dev->gen1_native_graphics_rq;
+      /* On M5 the main kernel performs the invocation's API stores and
+       * atomics; the count and raster variants drop them, together with any
+       * native ray query that only fed them. */
+      const bool m5 = agx_chip_g17_m5(dev->dev.chip);
       if (!hk_nir_lower_gs(nir, &count, &rast, &pre_gs,
                            &count_variant->info.gs,
-                           hk_native_rq_profile(dev) ? &rq_layout : NULL)) {
+                           hk_native_rq_profile(dev) ? &rq_layout : NULL, m5,
+                           m5 && dev->m5_native_rq
+                              ? agx_nir_m5_remove_dead_ray_queries
+                              : NULL)) {
          ralloc_free(nir);
          ralloc_free(count);
          ralloc_free(rast);
@@ -4847,10 +6217,22 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
                           "Gen1 native GS ray query lowering failed");
       }
 
+      /* A main-kernel query whose results feed nothing is dead too. */
+      if (dev->m5_native_rq)
+         NIR_PASS(_, nir, agx_nir_m5_remove_dead_ray_queries);
+
       agx_preprocess_nir(count);
       agx_preprocess_nir(rast);
       agx_preprocess_nir(pre_gs);
 
+      /* The raster variant is a hardware vertex shader. On M5, match the
+       * ordinary VS/TES default when the geometry producer has no point-size
+       * output, including a producer that emits no vertices. Other chips keep
+       * agx3b's raster variant, which writes no default point size.
+       */
+      if (agx_chip_g17_m5(dev->dev.chip))
+         NIR_PASS(_, rast, nir_lower_default_point_size);
+
       struct hk_shader *shader = &obj->variants[HK_GS_VARIANT_RAST];
       if (agx_native_graphics(&dev->dev)) {
          NIR_PASS(_, rast, nir_shader_intrinsics_pass,
@@ -4858,7 +6240,8 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
                   NULL);
       }
       hk_lower_hw_vs(rast, shader, false,
-                     agx_native_graphics(&dev->dev), true, false);
+                     agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip),
+                     true, false);
       shader->info.gs = count_variant->info.gs;
       main_variant->info.gs = count_variant->info.gs;
 
@@ -5026,7 +6409,8 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
                         : AGX_ROBUSTNESS_DISABLED,
                };
 
-               if (agx_native_graphics(&dev->dev))
+               if (agx_native_graphics(&dev->dev) ||
+                   agx_chip_g17_m5(dev->dev.chip))
                   agx_nir_lower_vbo_raw(clone, attribs, agx_rs,
                                         !key->vs.static_strides);
                else
@@ -5035,8 +6419,14 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
 
                unsigned nr = DIV_ROUND_UP(
                   BITSET_LAST_BIT(shader->info.vs.attrib_components_read), 4);
+               /* Native query state owns uniform halves 0..167; the vertex
+                * ABI then starts at 168, as the prolog key and the bound
+                * uniform ranges already assume. */
+               const bool query_prefix =
+                  (dev->gen1_native_graphics_rq && hk_nir_has_ray_query(clone)) ||
+                  hk_nir_has_m5_native_query(dev, clone);
                agx_nir_lower_non_monolithic_uniforms_at(clone, nr,
-                  dev->gen1_native_graphics_rq && hk_nir_has_ray_query(clone) ? 168 : 0);
+                                                        query_prefix ? 168 : 0);
             }
          }
 
@@ -5063,9 +6453,12 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
        * library that never meets a multisampled draw does not pay for them.
        * Refuse a shape no sample count can export now, as before. */
       if (nir->info.fs.uses_sample_shading) {
-         for (unsigned samples = 2; samples <= 4; samples *= 2) {
+         const unsigned max_samples = agx_chip_g17_m5(dev->dev.chip) ? 8 : 4;
+         for (unsigned samples = 2; samples <= max_samples; samples *= 2) {
             key->fs.g17p_samples = samples;
-            if (!hk_g17p_unrolls_sample_shading(nir, &key->fs)) {
+            if (agx_chip_g17_m5(dev->dev.chip))
+               key->fs.m5_position_samples = samples;
+            if (!hk_native_sample_shading_supported(nir, &key->fs, dev->dev.chip)) {
                ralloc_free(nir);
                hk_api_shader_destroy(&dev->vk, &obj->vk, pAllocator);
                /* Same contract as above: not a legal code for any caller of
@@ -5075,6 +6468,8 @@ hk_compile_shader_impl(struct hk_device *dev, struct vk_shader_compile_info *inf
          }
       }
       key->fs.g17p_samples = 0;
+      if (agx_chip_g17_m5(dev->dev.chip))
+         key->fs.m5_position_samples = 0;
 
       result = hk_fs_sample_source_init(obj, nir, key, info, features,
                                         static_set_mask);
@@ -5320,9 +6715,17 @@ hk_compile_shaders(struct vk_device *vk_dev, uint32_t shader_count,
          }
       }
 
-      vs_covers[i] = dev->dev.chip == AGX_CHIP_G17P &&
+      vs_covers[i] = (dev->dev.chip == AGX_CHIP_G17P ||
+                      agx_chip_g17_m5(dev->dev.chip)) &&
                      nir->info.stage == MESA_SHADER_VERTEX &&
                      hk_vs_covers_clip_square(nir);
+      if (!hk_m5_mesh_task_grid(dev, nir, info->flags)) {
+         for (uint32_t j = 0; j < shader_count; j++)
+            ralloc_free(infos[j].nir);
+         memset(shaders_out, 0, shader_count * sizeof(*shaders_out));
+         return vk_errorf(dev, VK_ERROR_FEATURE_NOT_PRESENT,
+                          "mesh gl_NumWorkGroups under a task stage needs the task record");
+      }
       hk_lower_nir(dev, nir, info->robustness, is_multiview,
                    info->next_stage_mask, info->set_layout_count,
                     info->set_layouts, hk_features);
@@ -5331,14 +6734,14 @@ hk_compile_shaders(struct vk_device *vk_dev, uint32_t shader_count,
                   hk_fs_mesh_builtin_per_primitive, nir_metadata_control_flow,
                   NULL);
 
-      if (agx_native_graphics(&dev->dev)) {
+      if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip)) {
          enum hk_nir_widen_16bit_varyings_mode mode = 0;
 
          if (nir->info.stage == MESA_SHADER_FRAGMENT) {
             mode = HK_NIR_WIDEN_16BIT_INPUTS;
          } else if (nir->info.stage == MESA_SHADER_MESH) {
-            
-
+            /* Native mesh output reaches the same 32-bit UVS words and
+             * fragment interpolation as a vertex stage. */
             mode = HK_NIR_WIDEN_16BIT_OUTPUTS;
          } else if (nir->info.stage == MESA_SHADER_VERTEX ||
                     nir->info.stage == MESA_SHADER_TESS_EVAL ||
@@ -5366,7 +6769,7 @@ hk_compile_shaders(struct vk_device *vk_dev, uint32_t shader_count,
          nir_io_add_intrinsic_xfb_info(nir);
       }
 
-      if (agx_native_graphics(&dev->dev))
+      if (agx_native_graphics(&dev->dev) || agx_chip_g17_m5(dev->dev.chip))
          NIR_PASS(_, nir, nir_shader_intrinsics_pass, hk_widen_integer_io,
                   nir_metadata_control_flow, NULL);
 
@@ -5413,8 +6816,8 @@ hk_compile_shaders(struct vk_device *vk_dev, uint32_t shader_count,
       }
       struct agx_m5_mesh_varyings mesh_varyings;
       if (mesh_nir && fs_nir &&
-          agx_m5_mesh_gather_varyings(mesh_nir, &mesh_varyings))
-         agx_nir_lower_m5_mesh_fragment_layout(fs_nir, &mesh_varyings);
+          agx_gen1_mesh_gather_varyings(mesh_nir, &mesh_varyings))
+         agx_nir_lower_gen1_mesh_fragment_layout(fs_nir, &mesh_varyings);
    }
 
    /* Only a complete pipeline known to have no fragment shader can fix an
@@ -5673,14 +7076,22 @@ hk_deserialize_api_shader_impl(struct vk_device *vk_dev,
    mesa_shader_stage stage = blob_read_uint8(blob);
    const unsigned variant_flags = blob_read_uint8(blob);
    const bool sample_variants = variant_flags & 1;
+   const bool m5_sample_variants = variant_flags & 32;
    const bool has_source = variant_flags & 2;
    const bool has_mono = variant_flags & 4;
    const bool has_border_source = variant_flags & 8;
    const bool border_emulating = variant_flags & 16;
-   if (blob->overrun || stage >= MESA_SHADER_STAGES || variant_flags > 31 ||
+   /* Graphics, compute, task and mesh stages. */
+   if (blob->overrun || stage >= MESA_SHADER_MESH_STAGES || variant_flags > 63 ||
        (has_source && !sample_variants) ||
-       ((variant_flags & 7) && (stage != MESA_SHADER_FRAGMENT ||
-                                !agx_native_graphics(&dev->dev))))
+       (m5_sample_variants &&
+        (!sample_variants || !agx_chip_g17_m5(dev->dev.chip))) ||
+       (sample_variants &&
+        (stage != MESA_SHADER_FRAGMENT ||
+         (agx_chip_g17_m5(dev->dev.chip) ? !m5_sample_variants
+                                        : !agx_native_graphics(&dev->dev)))) ||
+       (has_mono && (stage != MESA_SHADER_FRAGMENT ||
+                    !agx_native_graphics(&dev->dev))))
       return vk_error(dev, VK_ERROR_INCOMPATIBLE_SHADER_BINARY_EXT);
 
    /* Compiled with a custom-border emulation that this GPU, with no sampler
@@ -5690,7 +7101,9 @@ hk_deserialize_api_shader_impl(struct vk_device *vk_dev,
        !p_atomic_read(&hk_device_physical(dev)->border_emulation))
       return VK_ERROR_INCOMPATIBLE_SHADER_BINARY_EXT;
 
-   size_t size = hk_api_shader_size(stage, sample_variants);
+   const unsigned sample_variant_count = sample_variants
+      ? (m5_sample_variants ? 4 : 3) : 0;
+   size_t size = hk_api_shader_size(stage, sample_variant_count);
 
    struct hk_api_shader *obj =
       vk_shader_zalloc(&dev->vk, &hk_shader_ops, stage, pAllocator, size);
@@ -5698,7 +7111,7 @@ hk_deserialize_api_shader_impl(struct vk_device *vk_dev,
    if (obj == NULL)
       return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
 
-   obj->g17p_sample_variants = sample_variants;
+   obj->g17p_sample_variants = sample_variant_count;
    obj->border_emulating = border_emulating;
 
    hk_foreach_variant(obj, shader) {
@@ -5719,6 +7132,13 @@ hk_deserialize_api_shader_impl(struct vk_device *vk_dev,
       }
    }
 
+   /* Restore what the compile set on the composed passthrough object. */
+   if (stage == MESA_SHADER_MESH && agx_chip_g17_m5(dev->dev.chip)) {
+      struct hk_shader *object = &obj->variants[HK_MESH_VARIANT_RAST];
+      if (object->compiled && object->only_linked)
+         hk_m5_mesh_passthrough_linked(object);
+   }
+
    if (has_source) {
       VkResult result = hk_fs_sample_source_deserialize(obj, blob);
       if (result != VK_SUCCESS) {
@@ -6056,9 +7476,10 @@ hk_api_shader_serialize(struct vk_device *vk_dev,
       mono |= shader->g17p_mono_nir != NULL;
    const struct hk_border_source *border = obj->border_source;
    blob_write_uint8(blob, vk_shader->stage);
-   blob_write_uint8(blob, obj->g17p_sample_variants | (src ? 2 : 0) |
+   blob_write_uint8(blob, (obj->g17p_sample_variants ? 1 : 0) | (src ? 2 : 0) |
                              (mono ? 4 : 0) | (border ? 8 : 0) |
-                             (obj->border_emulating ? 16 : 0));
+                             (obj->border_emulating ? 16 : 0) |
+                             (obj->g17p_sample_variants == 4 ? 32 : 0));
 
    hk_foreach_variant(obj, shader) {
       const unsigned i = shader - obj->variants;
diff --git a/src/asahi/vulkan/hk_shader.h b/src/asahi/vulkan/hk_shader.h
index 6eb4652b90eb..9e8dd7ffbeb2 100644
--- a/src/asahi/vulkan/hk_shader.h
+++ b/src/asahi/vulkan/hk_shader.h
@@ -105,7 +105,8 @@ struct hk_shader_info {
          /* G17P: the shader may leave pixels unwritten: it discards, demotes
           * or writes the sample mask. */
          bool may_kill;
-         uint8_t _pad[3];
+         bool m5_inline_sample_shading;
+         uint8_t _pad[2];
       } fs;
 
       struct {
@@ -159,6 +160,8 @@ struct hk_shader_info {
    bool agx3_combined_sampler_valid;
    bool agx3_uses_root;
    bool agx3_gen1_fragment_rq;
+   /* HK_ROOT_READS_* classes of root bytes the shader may read. */
+   uint8_t agx3_root_reads;
    /* Actual uniform address selected after stage and resource lowering. */
    unsigned agx3_root_uniform;
    bool agx3_txf_texture_valid;
@@ -306,13 +309,13 @@ static const char *hk_gs_variant_name[] = {
 /* clang-format on */
 
 enum hk_mesh_variant {
-   
+   /* Main compute kernel writing vertex/primitive/index records */
    HK_MESH_VARIANT_MAIN,
 
-   
+   /* Hardware vertex shader rasterizing those records */
    HK_MESH_VARIANT_RAST,
 
-   
+   /* Native object predicate/count kernel; selected only when required. */
    HK_MESH_VARIANT_PREDICATED_OBJECT,
 
    HK_MESH_VARIANTS,
@@ -399,8 +402,8 @@ struct hk_api_shader {
    /* Is this an internal passthrough geometry shader? */
    bool is_passthrough;
 
-   /* Dynamic G17P sample-rate/coverage shaders have native 1x/2x/4x binaries. */
-   bool g17p_sample_variants;
+   /* Native sample-count variants: three slots on G17P, four on M5. */
+   uint8_t g17p_sample_variants;
 
    /* The one-sample binary compiles with the shader; the multisample ones
     * compile from this source on first use (hk_fs_build_sample_variant). */
@@ -436,13 +439,14 @@ bool hk_shader_ensure_link_ht_locked(struct hk_shader *shader);
 static inline unsigned
 hk_api_num_variants(const struct hk_api_shader *obj)
 {
-   return obj->g17p_sample_variants ? 3 : hk_num_variants(obj->vk.stage);
+   return obj->g17p_sample_variants ? obj->g17p_sample_variants
+                                    : hk_num_variants(obj->vk.stage);
 }
 
 static inline size_t
-hk_api_shader_size(mesa_shader_stage stage, bool samples)
+hk_api_shader_size(mesa_shader_stage stage, unsigned samples)
 {
-   unsigned count = samples ? 3 : hk_num_variants(stage);
+   unsigned count = samples ? samples : hk_num_variants(stage);
    return sizeof(struct hk_api_shader) + count * sizeof(struct hk_shader);
 }
 
@@ -458,7 +462,7 @@ hk_variant_name(struct hk_api_shader *obj, struct hk_shader *variant)
    assert(i < hk_api_num_variants(obj));
 
    if (obj->g17p_sample_variants) {
-      static const char *names[] = {"1 sample", "2 samples", "4 samples"};
+      static const char *names[] = {"1 sample", "2 samples", "4 samples", "8 samples"};
       return names[i];
    } else if (hk_api_num_variants(obj) == 1) {
       return NULL;
@@ -508,7 +512,8 @@ hk_fs_variant(struct hk_api_shader *obj, unsigned samples)
 
    assert(obj->vk.stage == MESA_SHADER_FRAGMENT);
    if (obj->g17p_sample_variants)
-      assert(samples == 0 || samples == 1 || samples == 2 || samples == 4);
+      assert(samples == 0 || samples == 1 || samples == 2 || samples == 4 ||
+             (samples == 8 && obj->g17p_sample_variants == 4));
    if (!obj->g17p_sample_variants)
       return hk_only_variant(obj);
 
@@ -560,6 +565,9 @@ struct hk_linked_shader {
     * parallel and are released with this linked executable. */
    void *g17p_compute_templates;
 
+   /* M5 compute loaders retain their exact native load schedule. */
+   void *m5_compute_templates;
+
    /* G17P stage programs of this executable, one per uniform-range layout,
     * patched with each draw's range addresses (struct hk_g17p_gfx_loader). */
    void *g17p_gfx_loaders;
@@ -585,6 +593,13 @@ struct hk_linked_shader {
     * the descriptor root even when the application fragment shader does not. */
    bool g17p_prolog_uses_root;
 
+   /* An inline-blend fragment program that forms its own constant program
+    * (hk_g17p_inline_epilog_part()) has its own uniform layout: this shader
+    * describes it, and the draw takes the program's launch (uniform ranges,
+    * constant program, scratch) from it instead of the API shader. Owned by
+    * this linked shader; NULL when the API shader's layout applies. */
+   struct hk_shader *g17p_layout;
+
    /* Distinct from hk_shader::counts due to addition of cf_binding_count, which
     * is delayed since it depends on cull distance.
     */
@@ -606,6 +621,13 @@ struct hk_linked_shader *hk_fast_link_part(struct hk_device *dev, bool fragment,
                                            struct agx_shader_part *prolog,
                                            struct agx_shader_part *epilog,
                                            unsigned nr_samples_shaded);
+/* A copy of `api` whose code, uniform layout and constant program are
+ * `part`'s (an inline-blend program with its own constant program); takes
+ * ownership of part->binary. */
+struct hk_shader *hk_g17p_layout_shader_create(struct hk_device *dev,
+                                               const struct hk_shader *api,
+                                               struct agx_shader_part *part);
+
 struct hk_linked_shader *hk_fast_link(struct hk_device *dev, bool fragment,
                                       struct hk_shader *main,
                                       struct agx_shader_part *prolog,
@@ -647,21 +669,24 @@ hk_ubo_addr_format(VkPipelineRobustnessBufferBehaviorEXT robustness)
 
 static inline nir_address_format
 hk_buffer_addr_format(VkPipelineRobustnessBufferBehaviorEXT robustness,
-                      bool null_descriptor)
+                      bool null_descriptor, bool map_metadata)
 {
    /* A null buffer descriptor has a zero range.  Preserve that range through
     * explicit-I/O lowering even when ordinary robustness is disabled so null
     * loads return zero and null stores are discarded before touching the
     * read-only zero page. */
    if (null_descriptor)
-      return nir_address_format_64bit_bounded_global;
+      return map_metadata ? nir_address_format_64bit_bounded_global_map
+                          : nir_address_format_64bit_bounded_global;
 
    switch (robustness) {
    case VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DISABLED_EXT:
-      return nir_address_format_64bit_global_32bit_offset;
+      return map_metadata ? nir_address_format_64bit_global_32bit_offset_map
+                          : nir_address_format_64bit_global_32bit_offset;
    case VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_ROBUST_BUFFER_ACCESS_EXT:
    case VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_ROBUST_BUFFER_ACCESS_2_EXT:
-      return nir_address_format_64bit_bounded_global;
+      return map_metadata ? nir_address_format_64bit_bounded_global_map
+                          : nir_address_format_64bit_bounded_global;
    default:
       UNREACHABLE("Invalid robust buffer access behavior");
    }
@@ -699,7 +724,7 @@ hk_get_nir_options(struct vk_physical_device *vk_pdev, mesa_shader_stage stage,
 
 struct hk_api_shader *hk_meta_shader(struct hk_device *dev,
                                      hk_internal_builder_t builder, void *data,
-                                     size_t data_size);
+                                     size_t data_size, VkResult *result);
 struct spirv_to_nir_options
 hk_get_spirv_options(struct vk_physical_device *vk_pdev,
                      UNUSED mesa_shader_stage stage,
diff --git a/src/asahi/vulkan/hk_sync.c b/src/asahi/vulkan/hk_sync.c
index 51743a742fbd..80dd841ba96c 100644
--- a/src/asahi/vulkan/hk_sync.c
+++ b/src/asahi/vulkan/hk_sync.c
@@ -98,6 +98,26 @@ hk_sync_reset(struct hk_cmd_buffer *cmd)
    util_dynarray_clear(&cmd->sync_pass_records);
 }
 
+DEBUG_GET_ONCE_BOOL_OPTION(sync_tile_index, "ASAHI_SYNC_TILE_INDEX", true)
+
+#define TILE_BUCKETS 256
+#define TILE_INDEX_NONE UINT32_MAX
+
+/* Buckets only narrow candidates. The full predicate still checks image,
+ * level, layers, preservation and tile overlap after hash collisions. */
+static unsigned
+tile_bucket(const struct hk_sync_footprint *fp)
+{
+   const uintptr_t image = (uintptr_t)fp->image;
+   return ((image >> 4) ^ (image >> 12) ^ (fp->level * 0x9e3779b9u)) &
+          (TILE_BUCKETS - 1);
+}
+
+struct recent_tile_index {
+   uint16_t head[TILE_BUCKETS];
+   uint16_t next[HK_SYNC_RECENT_FOOTPRINTS];
+};
+
 /* Tile hazards: see hk_sync.h. */
 static bool
 footprints_share_tile(const struct hk_sync_footprint *a,
@@ -479,6 +499,38 @@ hk_sync_order_xfer(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    xfer_forget_upto(st, cs->sync.seq);
 }
 
+static void
+barrier_source_dependencies(struct hk_cmd_buffer *cmd, uint8_t src_kinds,
+                            struct hk_sync_dep dep[HK_SYNC_NUM_SQ])
+{
+   hk_sync_src_dep(cmd, src_kinds, NULL, dep);
+
+   struct hk_sync_dep pending[HK_SYNC_NUM_SQ];
+   req_for_kinds(cmd, src_kinds, pending);
+   for (unsigned sq = 0; sq < HK_SYNC_NUM_SQ; ++sq)
+      dep_max(&dep[sq], &pending[sq]);
+}
+
+static void note_last(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
+
+/* The latest HK_SYNC_GFX_MEM stream at or before a cut: a prefix wait on it
+ * covers every such stream the cut covers. */
+static struct hk_cs *
+memory_cut(struct hk_cs *cut)
+{
+   return !cut || (cut->sync.kind & HK_SYNC_GFX_MEM) ? cut
+                                                     : cut->sync.prev_mem;
+}
+
+void
+hk_sync_note_memory_write(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   if (cs->sync.kind & HK_SYNC_GFX_MEM)
+      return;
+   cs->sync.kind |= HK_SYNC_GFX_MEM;
+   note_last(cmd, cs);
+}
+
 void
 hk_sync_barrier(struct hk_cmd_buffer *cmd, uint8_t src_kinds,
                 uint8_t dst_kinds)
@@ -495,7 +547,27 @@ hk_sync_barrier(struct hk_cmd_buffer *cmd, uint8_t src_kinds,
       xfer_forget_upto(&cmd->sync, UINT32_MAX);
 
    struct hk_sync_dep dep[HK_SYNC_NUM_SQ];
-   hk_sync_src_dep(cmd, src_kinds, NULL, dep);
+   barrier_source_dependencies(cmd, src_kinds, dep);
+
+   /* M5: of the renders this dependency orders pre-rasterization work after,
+    * only those that wrote memory other than their attachments
+    * (HK_SYNC_GFX_MEM) can hand that work anything it reads without an
+    * image. Order the vertex half (HK_SYNC_GFX) after the latest of them;
+    * the fragment half and image-reading vertex work keep the whole
+    * dependency. The compute subqueue's requirement is unchanged: the kernel
+    * orders both halves after compute work either way. */
+   if ((src_kinds & HK_SYNC_GFX) && (dst_kinds & HK_SYNC_GFX) &&
+       agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip)) {
+      struct hk_sync_dep vtx = dep[HK_SYNC_VDM];
+      vtx.cut = memory_cut(vtx.cut);
+      if (vtx.prior & HK_SYNC_GFX)
+         vtx.prior = (vtx.prior & ~HK_SYNC_GFX) | HK_SYNC_GFX_MEM;
+      const unsigned gfx = util_logbase2(HK_SYNC_GFX);
+      dep_max(&cmd->sync.req[gfx][HK_SYNC_VDM], &vtx);
+      dep_max(&cmd->sync.req[gfx][HK_SYNC_CDM], &dep[HK_SYNC_CDM]);
+      dst_kinds = (dst_kinds & ~HK_SYNC_GFX) | HK_SYNC_GFX_FRAG |
+                  HK_SYNC_GFX_IMG;
+   }
 
    u_foreach_bit(d, dst_kinds) {
       for (unsigned sq = 0; sq < HK_SYNC_NUM_SQ; ++sq)
@@ -514,7 +586,7 @@ hk_sync_pass_barrier(struct hk_cmd_buffer *cmd, uint8_t src_kinds,
    st->epoch++;
 
    struct hk_sync_dep dep[HK_SYNC_NUM_SQ];
-   hk_sync_src_dep(cmd, src_kinds, NULL, dep);
+   barrier_source_dependencies(cmd, src_kinds, dep);
    for (unsigned sq = 0; sq < HK_SYNC_NUM_SQ; ++sq) {
       /* Neither scope includes work before the pass instance. */
       dep[sq].prior = 0;
@@ -653,6 +725,17 @@ hk_sync_stream_created(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    struct hk_cs *gfx = cmd->current_cs.gfx;
    const bool pre_gfx = compute && slot == &cmd->current_cs.pre_gfx && gfx &&
                         gfx != cs;
+   if (unlikely(hk_cmd_buffer_device(cmd)->frame_anatomy)) {
+      cs->anatomy_api_barriers = cmd->anatomy_bar.count;
+      cs->anatomy_api_mem = cmd->anatomy_bar.mem;
+      cs->anatomy_api_buf = cmd->anatomy_bar.buf;
+      cs->anatomy_api_img = cmd->anatomy_bar.img;
+      cs->anatomy_api_src = cmd->anatomy_bar.src;
+      cs->anatomy_api_dst = cmd->anatomy_bar.dst;
+      cs->anatomy_api_src_access = cmd->anatomy_bar.src_access;
+      cs->anatomy_api_dst_access = cmd->anatomy_bar.dst_access;
+      memset(&cmd->anatomy_bar, 0, sizeof(cmd->anatomy_bar));
+   }
    const bool post_gfx = compute && slot == &cmd->current_cs.post_gfx;
    st->epoch++;
 
@@ -668,6 +751,11 @@ hk_sync_stream_created(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
    }
 
    cs->sync.kind = stream_kind(cmd, slot, compute);
+   if (cs->sync.kind != HK_SYNC_GFX || slot == &cmd->current_cs.pre_gfx ||
+       slot == &cmd->current_cs.post_gfx)
+      cs->sync.kind |= HK_SYNC_GFX_MEM;
+   cs->sync.prev_mem =
+      st->last[util_logbase2(HK_SYNC_GFX_MEM)][hk_sync_sq(cs)];
 
    /* The recorded requirement for this kind of work. A render's vertex half
     * needs only the part whose destination is not fragment-only. */
@@ -806,6 +894,53 @@ cs_reusable(struct hk_cmd_buffer *cmd, struct hk_cs *cs, uint8_t kinds)
    return true;
 }
 
+/* Whether the compute stream cs already waits for the render r: directly,
+ * or through the compute prefix it waits on, each stream of which completes
+ * only after the renders it waits on. */
+static bool
+cdm_waits_for_render(struct hk_cmd_buffer *cmd, const struct hk_cs *cs,
+                     const struct hk_cs *r)
+{
+   const struct hk_cs *v = cs->sync.dep[HK_SYNC_VDM].cut;
+   if (v && v->sync.seq >= r->sync.seq)
+      return true;
+
+   const struct hk_cs *prefix = cs->sync.dep[HK_SYNC_CDM].cut;
+   if (!prefix || prefix == cs)
+      return false;
+
+   list_for_each_entry(struct hk_cs, it, &cmd->control_streams, node) {
+      if (it->type != HK_CS_CDM || it == cs ||
+          it->sync.seq > prefix->sync.seq)
+         continue;
+      v = it->sync.dep[HK_SYNC_VDM].cut;
+      if (v && v->sync.seq >= r->sync.seq)
+         return true;
+   }
+   return false;
+}
+
+/* M5: whether a requirement on the render subqueue would make the work a
+ * compute stream already holds wait for a render it does not wait for yet.
+ * A requirement recorded after that work (a barrier between two dispatches)
+ * has none of it in its second scope, so the work keeps the requirements it
+ * was recorded under and only later work takes the new one. */
+static bool
+cdm_raise_delays_work(struct hk_cmd_buffer *cmd, const struct hk_cs *cs,
+                      const struct hk_cs *r)
+{
+   return r && cs->stats.cmds &&
+          agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip) &&
+          !cdm_waits_for_render(cmd, cs, r);
+}
+
+bool
+hk_sync_merge_delays(struct hk_cmd_buffer *cmd, const struct hk_cs *into,
+                     const struct hk_cs *from)
+{
+   return cdm_raise_delays_work(cmd, into, from->sync.dep[HK_SYNC_VDM].cut);
+}
+
 void
 hk_sync_check_cs(struct hk_cmd_buffer *cmd)
 {
@@ -820,12 +955,30 @@ hk_sync_check_cs(struct hk_cmd_buffer *cmd)
       return;
    }
 
-   /* The stream absorbs the requirement of the work it now carries. */
    struct hk_sync_dep dep[HK_SYNC_NUM_SQ];
    req_for_kinds(cmd, kinds, dep);
+
+   /* The new work goes into a stream of its own, created under the whole
+    * requirement; a first scope that names compute work orders it after
+    * this stream through last[]. */
+   if (cdm_raise_delays_work(cmd, cs, dep[HK_SYNC_VDM].cut)) {
+      hk_cmd_buffer_end_compute(cmd);
+      return;
+   }
+
+   /* The stream absorbs the requirement of the work it now carries. */
    hk_sync_stream_wait(cs, dep);
    cs->sync.kind |= kinds;
+   if (cs->sync.kind != HK_SYNC_GFX)
+      cs->sync.kind |= HK_SYNC_GFX_MEM;
    note_last(cmd, cs);
+
+   if (agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip) &&
+       cmd->sync.pending_start_ts.handle && !cs->timestamp.start.handle &&
+       !cs->timestamp.end.handle) {
+      cs->timestamp.start = cmd->sync.pending_start_ts;
+      cmd->sync.pending_start_ts = (struct agx_timestamp_req){0};
+   }
 }
 
 struct hk_cs *
@@ -884,7 +1037,8 @@ hk_sync_end_cs_if_timestamped(struct hk_cmd_buffer *cmd)
       return;
 
    const bool g17p_start =
-      agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) &&
+      (agx_native_graphics(&hk_cmd_buffer_device(cmd)->dev) ||
+       agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip)) &&
       cs->timestamp.start.handle;
    if (cs->timestamp.end.handle || g17p_start) {
       stamp_carrier_launch(cmd);
@@ -962,6 +1116,7 @@ hk_sync_forget_stream(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    list_for_each_entry(struct hk_cs, it, &cmd->control_streams, node) {
       replace_pointer(&it->sync.dep[sq].cut, cs, prev);
       replace_pointer(&it->sync.dep_ta[sq].cut, cs, prev);
+      replace_pointer(&it->sync.prev_mem, cs, cs->sync.prev_mem);
    }
 
    /* A pre-graphics or continuation stream that was waiting on nothing but
@@ -1032,6 +1187,7 @@ hk_sync_merge(struct hk_cmd_buffer *cmd, struct hk_cs *into,
       if (it != into && it != from) {
          replace_pointer(&it->sync.dep[sq].cut, from, into);
          replace_pointer(&it->sync.dep_ta[sq].cut, from, into);
+         replace_pointer(&it->sync.prev_mem, from, into);
       }
    }
 }
@@ -1053,12 +1209,32 @@ hk_sync_merge(struct hk_cmd_buffer *cmd, struct hk_cs *into,
  * earlier work orders that command's completion: for a render, its fragment
  * half alone. */
 static void
-stamp_wait(struct hk_cs *cs, const struct hk_sync_dep dep[HK_SYNC_NUM_SQ])
-{
-   if (cs->type == HK_CS_CDM)
-      hk_sync_stream_wait(cs, dep);
-   else
+stamp_wait(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
+           const struct hk_sync_dep dep[HK_SYNC_NUM_SQ])
+{
+   if (cs->type == HK_CS_CDM) {
+      const struct hk_device *dev = hk_cmd_buffer_device(cmd);
+      if (agx_native_graphics(&dev->dev) && dev->dev.chip == AGX_CHIP_G17P &&
+          (dev->dev.params.features & DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS)) {
+         /* The logical compute queue already orders these GPU commands by
+          * their mandatory kick parents. Adding the timestamp's same-CDM
+          * prefix also waits for earlier CPU timestamp publication, which
+          * serializes otherwise independent timed dispatches. Keep genuine
+          * query-memory requirements already present on the stream, and the
+          * timestamp's cross-engine scope, unchanged. Exact priors also carry
+          * standalone barrier requirements across submissions; legacy kernels
+          * retain the original conservative timestamp prefix.
+          */
+         struct hk_sync_dep timestamp_dep[HK_SYNC_NUM_SQ];
+         memcpy(timestamp_dep, dep, sizeof(timestamp_dep));
+         timestamp_dep[HK_SYNC_CDM] = (struct hk_sync_dep){0};
+         hk_sync_stream_wait(cs, timestamp_dep);
+      } else {
+         hk_sync_stream_wait(cs, dep);
+      }
+   } else {
       stream_wait_frag(cs, dep);
+   }
 }
 
 static struct hk_cs *
@@ -1141,7 +1317,7 @@ hk_sync_write_timestamp(struct hk_cmd_buffer *cmd, VkPipelineStageFlags2 stage,
              dep[sq].cut->sync.seq > gfx->sync.seq)
             dep[sq].cut = latest_before(cmd, sq, gfx);
       }
-      stamp_wait(gfx, dep);
+      stamp_wait(cmd, gfx, dep);
 
       if (gfx->timestamp.end.handle) {
          libagx_copy_timestamp(cmd, agx_1d(1), AGX_BARRIER_ALL | AGX_POSTGFX,
@@ -1155,6 +1331,8 @@ hk_sync_write_timestamp(struct hk_cmd_buffer *cmd, VkPipelineStageFlags2 stage,
    if (!src_kinds) {
       hk_sync_flush_pending_timestamp(cmd);
       st->pending_start_ts = req;
+      if (agx_chip_g17_m5(hk_cmd_buffer_device(cmd)->dev.chip))
+         st->cs_check_pending = true;
       return;
    }
 
@@ -1166,7 +1344,7 @@ hk_sync_write_timestamp(struct hk_cmd_buffer *cmd, VkPipelineStageFlags2 stage,
 
    if (tail && !tail->timestamp.end.handle) {
       tail->timestamp.end = req;
-      stamp_wait(tail, dep);
+      stamp_wait(cmd, tail, dep);
 
       /* Later compute would move the stamp past itself; keep the stream
        * boundary so the stamp stays at the recorded position. */
@@ -1394,6 +1572,23 @@ recent_may_run(const struct hk_sync_queue *q, uint64_t completed)
    return q->recent_count && q->recent_max > completed;
 }
 
+/* Complete writes only need tile-hazard ordering against a pass preserving
+ * pixels/layers. Keep the full footprints: a future preserving pass must
+ * still wait for complete writes that have not retired.
+ */
+static bool
+submit_preserves_tiles(struct hk_cmd_buffer *const *cmdbufs, unsigned count)
+{
+   for (unsigned i = 0; i < count; ++i) {
+      util_dynarray_foreach(&cmdbufs[i]->sync_pass_records,
+                            struct hk_sync_pass_record, rec) {
+         if (rec->fp.preserve)
+            return true;
+      }
+   }
+   return false;
+}
+
 bool
 hk_sync_submit_needs_completed(const struct hk_queue *queue,
                                struct hk_cmd_buffer *const *cmdbufs,
@@ -1404,6 +1599,9 @@ hk_sync_submit_needs_completed(const struct hk_queue *queue,
     * submission can share a tile with this one while it runs. */
    if (q->serialize || !q->recent_count)
       return false;
+   if (q->precise_tile_evictions && !q->last_preserve &&
+       !submit_preserves_tiles(cmdbufs, count))
+      return false;
    for (unsigned i = 0; i < count; ++i) {
       if (util_dynarray_num_elements(&cmdbufs[i]->sync_pass_records,
                                      struct hk_sync_pass_record))
@@ -1414,14 +1612,31 @@ hk_sync_submit_needs_completed(const struct hk_queue *queue,
 
 static uint8_t
 recent_conflict(const struct hk_sync_queue *q, const struct hk_sync_footprint *fp,
-                uint64_t completed)
+                uint64_t completed, const struct recent_tile_index *index)
 {
    if (!recent_may_run(q, completed))
       return false;
-   /* A dropped entry may still be running: assume it shares the tile. */
-   if (q->evicted > completed)
+   if (q->precise_tile_evictions && !fp->preserve && !q->last_preserve)
+      return false;
+   /* A dropped entry may still be running: assume it shares the tile.
+    * When this pass is a complete write, only an evicted preserving pass
+    * can conflict. A preserving pass still considers every evicted write.
+    */
+   const uint64_t evicted = q->precise_tile_evictions && !fp->preserve
+      ? q->evicted_preserve : q->evicted;
+   if (evicted > completed)
       return BITFIELD_MASK(HK_SYNC_NUM_SQ);
    uint8_t engines = 0;
+   if (index) {
+      for (uint16_t i = index->head[tile_bucket(fp)]; i != UINT16_MAX;
+           i = index->next[i]) {
+         const uint8_t engine = BITFIELD_BIT(q->recent[i].sq);
+         if (!(engines & engine) &&
+             footprints_share_tile(&q->recent[i].fp, fp))
+            engines |= engine;
+      }
+      return engines;
+   }
    const uint32_t n = MIN2(q->recent_count, HK_SYNC_RECENT_FOOTPRINTS);
    for (uint32_t i = 0; i < n; ++i) {
       if (q->recent[i].point > completed &&
@@ -1436,8 +1651,13 @@ recent_add(struct hk_sync_queue *q, const struct hk_sync_footprint *fp,
            uint64_t point, enum hk_sync_sq sq)
 {
    struct hk_sync_recent *slot = &q->recent[q->recent_next];
-   if (q->recent_count >= HK_SYNC_RECENT_FOOTPRINTS)
+   if (q->recent_count >= HK_SYNC_RECENT_FOOTPRINTS) {
       q->evicted = MAX2(q->evicted, slot->point);
+      if (q->precise_tile_evictions && slot->fp.preserve)
+         q->evicted_preserve = MAX2(q->evicted_preserve, slot->point);
+   }
+   if (q->precise_tile_evictions && fp->preserve)
+      q->last_preserve = MAX2(q->last_preserve, point);
    *slot = (struct hk_sync_recent){.fp = *fp, .point = point, .sq = sq};
    q->recent_next = (q->recent_next + 1) % HK_SYNC_RECENT_FOOTPRINTS;
    q->recent_count = MIN2(q->recent_count + 1, HK_SYNC_RECENT_FOOTPRINTS);
@@ -1458,6 +1678,7 @@ struct submit_footprint {
    struct hk_sync_footprint fp;
    hk_sync_value index;
    enum hk_sync_sq sq;
+   uint32_t next_by_image;
 };
 
 /* The barrier and prior fields of one requirement on subqueue sq, with the
@@ -1511,9 +1732,43 @@ hk_sync_resolve_submit(struct hk_queue *queue,
       }
    }
 
+   bool indexed = debug_get_option_sync_tile_index() &&
+      (!queue->sync.precise_tile_evictions || queue->sync.last_preserve ||
+       submit_preserves_tiles(cmdbufs, count));
+   if (indexed) {
+      indexed = false;
+      for (unsigned i = 0; i < count && !indexed; ++i) {
+         indexed = util_dynarray_num_elements(
+            &cmdbufs[i]->sync_pass_records, struct hk_sync_pass_record) != 0;
+      }
+   }
+   struct recent_tile_index recent_index;
+   uint32_t submitted_head[TILE_BUCKETS];
+   if (indexed) {
+      memset(&recent_index, 0xff, sizeof(recent_index));
+      memset(submitted_head, 0xff, sizeof(submitted_head));
+      const struct hk_sync_queue *q = &queue->sync;
+      const uint32_t n = MIN2(q->recent_count, HK_SYNC_RECENT_FOOTPRINTS);
+      for (uint32_t i = 0; i < n; ++i) {
+         if (q->recent[i].point <= completed)
+            continue;
+         const unsigned bucket = tile_bucket(&q->recent[i].fp);
+         recent_index.next[i] = recent_index.head[bucket];
+         recent_index.head[bucket] = i;
+      }
+   }
+
    struct submit_state ss = {.queue = queue};
    struct util_dynarray submitted;
    util_dynarray_init(&submitted, NULL);
+   /* The temporary list orders different command buffers of this submit.
+    * With only one, recorded dependencies already order all its passes.
+    * Publish its original records to queue history after resolving hazards,
+    * avoiding an allocation and a second copy without exposing this submit
+    * as earlier history to itself. Keep other devices on their existing path.
+    */
+   const bool direct_footprints =
+      queue->sync.direct_single_footprints && count == 1;
 
    for (unsigned k = 0; k < HK_SYNC_NUM_KINDS; ++k) {
       for (unsigned sq = 0; sq < HK_SYNC_NUM_SQ; ++sq) {
@@ -1580,11 +1835,24 @@ hk_sync_resolve_submit(struct hk_queue *queue,
          if (!rec->first || !rec->last)
             continue;
          hk_sync_value v[HK_SYNC_NUM_SQ] = {HK_SYNC_NONE, HK_SYNC_NONE};
-         util_dynarray_foreach(&submitted, struct submit_footprint, e) {
-            if (e->index > v[e->sq] && footprints_share_tile(&e->fp, &rec->fp))
-               v[e->sq] = e->index;
+         if (indexed) {
+            for (uint32_t j = submitted_head[tile_bucket(&rec->fp)];
+                 j != TILE_INDEX_NONE;) {
+               const struct submit_footprint *e = util_dynarray_element(
+                  &submitted, struct submit_footprint, j);
+               if (e->index > v[e->sq] &&
+                   footprints_share_tile(&e->fp, &rec->fp))
+                  v[e->sq] = e->index;
+               j = e->next_by_image;
+            }
+         } else {
+            util_dynarray_foreach(&submitted, struct submit_footprint, e) {
+               if (e->index > v[e->sq] && footprints_share_tile(&e->fp, &rec->fp))
+                  v[e->sq] = e->index;
+            }
          }
-         const uint8_t recent = recent_conflict(&queue->sync, &rec->fp, completed);
+         const uint8_t recent = recent_conflict(
+            &queue->sync, &rec->fp, completed, indexed ? &recent_index : NULL);
          for (unsigned sq = 0; sq < HK_SYNC_NUM_SQ; ++sq) {
             if (v[sq] == HK_SYNC_NONE && (recent & BITFIELD_BIT(sq)))
                v[sq] = 0;
@@ -1594,13 +1862,21 @@ hk_sync_resolve_submit(struct hk_queue *queue,
       }
       util_dynarray_foreach(&cmd->sync_pass_records,
                             struct hk_sync_pass_record, rec) {
-         if (!rec->last)
+         if (!rec->last || direct_footprints)
             continue;
+         if (indexed && util_dynarray_num_elements(
+               &submitted, struct submit_footprint) >= TILE_INDEX_NONE)
+            indexed = false;
          const struct submit_footprint e = {
             .fp = rec->fp,
             .index = (hk_sync_value)rec->last->sync.index,
             .sq = hk_sync_sq(rec->last),
+            .next_by_image = indexed ? submitted_head[tile_bucket(&rec->fp)]
+                                     : TILE_INDEX_NONE,
          };
+         if (indexed)
+            submitted_head[tile_bucket(&rec->fp)] = util_dynarray_num_elements(
+               &submitted, struct submit_footprint);
          util_dynarray_append(&submitted, e);
       }
 
@@ -1710,8 +1986,17 @@ hk_sync_resolve_submit(struct hk_queue *queue,
     * timeline point it signals (queue_submit has already advanced it).
     * Serialized queues order the next submission after this one whole. */
    if (!queue->sync.serialize) {
-      util_dynarray_foreach(&submitted, struct submit_footprint, e)
-         recent_add(&queue->sync, &e->fp, queue->drm.timeline_value, e->sq);
+      if (direct_footprints) {
+         util_dynarray_foreach(&cmdbufs[0]->sync_pass_records,
+                               struct hk_sync_pass_record, rec) {
+            if (rec->last)
+               recent_add(&queue->sync, &rec->fp, queue->drm.timeline_value,
+                          hk_sync_sq(rec->last));
+         }
+      } else {
+         util_dynarray_foreach(&submitted, struct submit_footprint, e)
+            recent_add(&queue->sync, &e->fp, queue->drm.timeline_value, e->sq);
+      }
    }
    util_dynarray_fini(&submitted);
 
diff --git a/src/asahi/vulkan/hk_sync.h b/src/asahi/vulkan/hk_sync.h
index b5d658f87ffd..a1d38e6bec03 100644
--- a/src/asahi/vulkan/hk_sync.h
+++ b/src/asahi/vulkan/hk_sync.h
@@ -73,15 +73,28 @@ enum hk_sync_kind {
    /* Destination scopes only: pre-rasterization shaders that read images.
     * An image barrier can only order the image's own accesses, and a
     * pre-rasterization stage accesses an image only through a shader that
-    * reads one. On G17P, image barriers therefore name this class (and the
-    * fragment class) where other dependencies name HK_SYNC_GFX: a render's
-    * fragment half always waits on it, its vertex half only once it draws
-    * with a pre-rasterization shader that reads images
+    * reads one. On G17P and M5, image barriers therefore name this class
+    * (and the fragment class) where other dependencies name HK_SYNC_GFX: a
+    * render's fragment half always waits on it, its vertex half only once it
+    * draws with a pre-rasterization shader that reads images
     * (hk_sync_vertex_images). Other chips never record it. */
    HK_SYNC_GFX_IMG = 1 << 4,
+
+   /* Stream kinds only, never a destination scope: work that can write
+    * memory a later pre-rasterization stage reads other than through an
+    * image. Graphics work writes such memory only from shaders (storage,
+    * atomics), through transform feedback, through attachments that can
+    * alias other resources, and in its pre- and post-graphics streams; a
+    * render without any of those writes only its attachments, which a later
+    * pre-rasterization stage can read only through an image (HK_SYNC_GFX_IMG).
+    * Every stream of another kind is counted too. On M5 a dependency from
+    * graphics work to pre-rasterization stages orders a render's vertex half
+    * only after the latest such stream on the render subqueue
+    * (hk_sync_barrier()). */
+   HK_SYNC_GFX_MEM = 1 << 5,
 };
 
-#define HK_SYNC_NUM_KINDS 5
+#define HK_SYNC_NUM_KINDS 6
 
 /* Subqueue index: 0 render (VDM), 1 compute (CDM). */
 enum hk_sync_sq {
@@ -212,6 +225,10 @@ struct hk_sync_stream {
     * its render so it sorts before it, where the list places it. */
    uint32_t seq;
 
+   /* The latest HK_SYNC_GFX_MEM stream on this stream's subqueue recorded
+    * before it. */
+   struct hk_cs *prev_mem;
+
    /* Set at submission: 1-based position among the commands of this
     * subqueue in the ioctl being built. */
    uint32_t index;
@@ -343,6 +360,14 @@ struct hk_sync_queue {
    struct hk_sync_recent recent[HK_SYNC_RECENT_FOOTPRINTS];
    uint32_t recent_next, recent_count;
    uint64_t recent_max, evicted;
+   /* G16S: complete writes need no preservation query against other
+    * complete writes, including entries evicted from the bounded ring.
+    * Retain all writes for a future partial-tile or sparse-layer pass.
+    */
+   bool precise_tile_evictions;
+   /* A single command buffer already owns the complete footprint list. */
+   bool direct_single_footprints;
+   uint64_t last_preserve, evicted_preserve;
 };
 
 void hk_sync_reset(struct hk_cmd_buffer *cmd);
@@ -374,6 +399,7 @@ void hk_sync_src_dep(struct hk_cmd_buffer *cmd, uint8_t src_kinds,
 /* The render stream draws with a pre-rasterization shader that reads images:
  * its vertex half joins the image-barrier requirements (HK_SYNC_GFX_IMG). */
 void hk_sync_vertex_images(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
+void hk_sync_note_memory_write(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
 
 /* Every later stream of the given kinds waits on cs. */
 void hk_sync_after_stream(struct hk_cmd_buffer *cmd, struct hk_cs *cs,
@@ -394,6 +420,11 @@ void hk_sync_forget_stream(struct hk_cmd_buffer *cmd, struct hk_cs *cs);
 void hk_sync_merge(struct hk_cmd_buffer *cmd, struct hk_cs *into,
                    struct hk_cs *from);
 
+/* M5: merging compute stream from into the adjacent stream into would make
+ * the work into already holds wait for a render it does not wait for. */
+bool hk_sync_merge_delays(struct hk_cmd_buffer *cmd, const struct hk_cs *into,
+                          const struct hk_cs *from);
+
 /* Obtain the general compute stream, ended and recreated if it is not
  * positioned after dep. The returned stream carries dep. */
 struct hk_cs *hk_sync_get_cs_after(struct hk_cmd_buffer *cmd,
diff --git a/src/asahi/vulkan/meson.build b/src/asahi/vulkan/meson.build
index 7e5dadd7c250..7519b6e2716f 100644
--- a/src/asahi/vulkan/meson.build
+++ b/src/asahi/vulkan/meson.build
@@ -4,6 +4,7 @@ hk_files = files(
   'hk_barrier.c',
   'hk_barrier.h',
   'hk_acceleration_structure.c',
+  'hk_m5_acceleration_structure.c',
   'hk_host_as.c',
   'hk_acceleration_structure.h',
   'hk_buffer.c',
@@ -90,6 +91,7 @@ hk_files = files(
   'hk_pipeline_statistics_core.c',
   'hk_pipeline_statistics_core.h',
   'hk_private.h',
+  'hk_query_oq.c',
   'hk_query_pool.c',
   'hk_query_reset.c',
   'hk_query_pool.h',
@@ -98,6 +100,8 @@ hk_files = files(
   'hk_execution_allowance.h',
   'hk_queue.c',
   'hk_queue.h',
+  'hk_frame_anatomy.c',
+  'hk_frame_anatomy.h',
   'hk_sampler.c',
   'hk_sampler.h',
   'hk_shader.c',
@@ -222,8 +226,62 @@ if with_tests
       suite : ['asahi'],
     )
 
+    # The deferred occlusion-availability resolver against mock control
+    # streams, with the real hk_sync stream bookkeeping it drops streams from.
+    test(
+      'hk_query_oq',
+      executable(
+        'hk_query_oq_test',
+        files('tests/test-hk-query-oq.c', 'hk_query_oq.c', 'hk_sync.c'),
+        include_directories : [
+          include_directories('.'),
+          inc_gallium, inc_gallium_aux, inc_include, inc_src, inc_asahi,
+          inc_virtio_gpu, inc_virtio_vdrm,
+        ],
+        dependencies : [
+          idep_nir_headers, idep_mesautil, idep_libagx,
+          idep_libagx_shaders_h, idep_agx_pack,
+          idep_vulkan_runtime_headers, idep_vulkan_util_headers,
+        ],
+        link_args : [ld_args_gc_sections],
+        gnu_symbol_visibility : 'hidden',
+      ),
+      suite : ['asahi'],
+    )
+
   endif
 
+  test(
+    'hk_root_fragment',
+    executable(
+      'hk_root_fragment_test',
+      files('tests/test-hk-root-fragment.c'),
+      include_directories : [
+        include_directories('.'),
+        inc_gallium, inc_gallium_aux, inc_include, inc_src, inc_asahi,
+        inc_virtio_gpu, inc_virtio_vdrm,
+      ],
+      dependencies : [
+        idep_nir, idep_mesautil, idep_libagx,
+        idep_libagx_shaders_h, idep_agx_pack,
+        idep_vulkan_runtime_headers, idep_vulkan_util_headers,
+      ],
+      gnu_symbol_visibility : 'hidden',
+    ),
+    suite : ['asahi'],
+  )
+
+  test(
+    'hk_render_resource_span',
+    executable(
+      'hk_render_resource_span_test',
+      files('tests/test-hk-render-resource-span.c', 'hk_drm_render_core.c'),
+      include_directories : [include_directories('.'), inc_include],
+      c_args : [c_msvc_compat_args],
+    ),
+    suite : ['asahi'],
+  )
+
   test(
     'hk_query_program',
     executable(
@@ -231,7 +289,7 @@ if with_tests
       files('tests/test-hk-query-program.c'),
       include_directories : [inc_include, inc_src],
       dependencies : [
-        idep_nir_headers, idep_libagx, idep_libagx_shaders_h,
+        idep_nir_headers, idep_mesautil, idep_libagx, idep_libagx_shaders_h,
       ],
       link_with : [libasahi_lib],
     ),
@@ -258,6 +316,31 @@ if with_tests
     suite : ['asahi'],
   )
 
+  test(
+    'hk_g16s_compression',
+    executable(
+      'hk_g16s_compression_test',
+      files('tests/test-hk-g16s-compression.c'),
+      include_directories : [inc_include, inc_src],
+    ),
+    suite : ['asahi'],
+  )
+
+  test(
+    'hk_g17p_zls',
+    executable(
+      'hk_g17p_zls_test',
+      files('tests/test-hk-g17p-zls.c', 'hk_drm_render_core.c'),
+      include_directories : [include_directories('.'), inc_include, inc_src,
+                             inc_asahi],
+      dependencies : [idep_mesautil, idep_nir, idep_libagx,
+                      idep_libagx_shaders_h, idep_agx_pack],
+      link_with : [libasahi_layout],
+      gnu_symbol_visibility : 'hidden',
+    ),
+    suite : ['asahi'],
+  )
+
   test(
     'hk_g17p_metadata_grid',
     executable(
@@ -503,6 +586,13 @@ if with_tests and ld_args_gc_sections.length() > 0
       dependencies : [hk_deps],
       link_args : [ld_args_gc_sections, hk_wrap_test_link_args,
         '-Wl,--wrap=util_flush_range']), suite : ['asahi'])
+  test('hk_agx3_as_format', executable('hk_agx3_as_format_test',
+    files('tests/test-hk-agx3-as-format.c'),
+    include_directories : [include_directories('.'), inc_gallium, inc_gallium_aux,
+      inc_include, inc_src, inc_asahi, inc_virtio_gpu, inc_virtio_vdrm],
+    dependencies : [hk_deps], link_with : [libhk],
+    link_args : [ld_args_gc_sections, hk_wrap_test_link_args]),
+    suite : ['asahi'])
   test('hk_as_registry', executable('hk_as_registry_test',
     files('tests/test-hk-as-registry.c'),
     include_directories : [include_directories('.'), inc_gallium, inc_gallium_aux,
@@ -624,6 +714,15 @@ if with_tests
     gnu_symbol_visibility : 'hidden',
   )
 
+  g16s_vulkan_buffer_upload = executable(
+    'g16s-vulkan-buffer-upload',
+    files('tests/g16s_vulkan_buffer_upload.c'),
+    include_directories : [inc_include],
+    dependencies : [cc.find_library('vulkan')],
+    c_args : [c_msvc_compat_args],
+    gnu_symbol_visibility : 'hidden',
+  )
+
   g17p_vulkan_compressed_layer_copy = executable(
     'g17p-vulkan-compressed-layer-copy',
     files('tests/g17p_vulkan_compressed_layer_copy.c'),
diff --git a/src/asahi/vulkan/tests/g16s_vulkan_buffer_upload.c b/src/asahi/vulkan/tests/g16s_vulkan_buffer_upload.c
new file mode 100644
index 000000000000..239c2e626d12
--- /dev/null
+++ b/src/asahi/vulkan/tests/g16s_vulkan_buffer_upload.c
@@ -0,0 +1,222 @@
+/* SPDX-License-Identifier: MIT
+ * Buffer uploads to single-slice graphics attachment views. Nonzero array/Z
+ * origins, mip levels, row/image padding and unaligned rectangle edges must
+ * preserve the source bytes and every untouched destination texel.
+ * --record-only uses drm-shim and never submits; it cannot check GPU results.
+ */
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <vulkan/vulkan.h>
+
+#define CHECK(call) do { VkResult r = (call); if (r != VK_SUCCESS) { \
+   fprintf(stderr, "%s: %d\n", #call, r); exit(2); } } while (0)
+
+static VkDevice dev;
+static VkPhysicalDevice physical;
+static VkQueue queue;
+static VkCommandPool pool;
+static bool record_only;
+
+static VkDeviceMemory
+memory(VkMemoryRequirements req, VkMemoryPropertyFlags flags)
+{
+   VkPhysicalDeviceMemoryProperties props;
+   vkGetPhysicalDeviceMemoryProperties(physical, &props);
+   unsigned i;
+   for (i = 0; i < props.memoryTypeCount; ++i)
+      if ((req.memoryTypeBits & (1u << i)) &&
+          (props.memoryTypes[i].propertyFlags & flags) == flags) break;
+   if (i == props.memoryTypeCount) exit(3);
+   VkMemoryAllocateInfo ai = { .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+      .allocationSize = req.size, .memoryTypeIndex = i };
+   VkDeviceMemory mem;
+   CHECK(vkAllocateMemory(dev, &ai, NULL, &mem));
+   return mem;
+}
+
+static VkBuffer
+buffer(VkDeviceSize size, VkDeviceMemory *mem, void **map)
+{
+   VkBufferCreateInfo ci = { .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+      .size = size, .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT };
+   VkBuffer buf;
+   CHECK(vkCreateBuffer(dev, &ci, NULL, &buf));
+   VkMemoryRequirements req;
+   vkGetBufferMemoryRequirements(dev, buf, &req);
+   *mem = memory(req, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
+   CHECK(vkBindBufferMemory(dev, buf, *mem, 0));
+   CHECK(vkMapMemory(dev, *mem, 0, VK_WHOLE_SIZE, 0, map));
+   return buf;
+}
+
+static unsigned
+check_case(VkFormat format, unsigned bytes, unsigned shape, unsigned level)
+{
+   const bool volume = shape == 3, one_d = shape == 4;
+   unsigned width = 129, height = one_d ? 1 : 127;
+   unsigned depth = volume ? 4 : 1;
+   unsigned layers = shape == 1 || one_d ? 3 : shape == 2 ? 6 : 1;
+   if (shape == 2) height = width;
+   VkImageCreateInfo ci = { .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+      .flags = shape == 2 ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0,
+      .imageType = volume ? VK_IMAGE_TYPE_3D : one_d ? VK_IMAGE_TYPE_1D : VK_IMAGE_TYPE_2D,
+      .format = format, .extent = {width, height, depth}, .mipLevels = 2,
+      .arrayLayers = layers, .samples = VK_SAMPLE_COUNT_1_BIT,
+      .tiling = VK_IMAGE_TILING_OPTIMAL,
+      .usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT };
+   VkImageFormatProperties props;
+   VkResult supported = vkGetPhysicalDeviceImageFormatProperties(physical, format,
+      ci.imageType, ci.tiling, ci.usage, ci.flags, &props);
+   if (supported == VK_ERROR_FORMAT_NOT_SUPPORTED) {
+      printf("SKIP format=%u shape=%u level=%u unsupported\n", format, shape, level);
+      return 0;
+   }
+   CHECK(supported);
+   VkImage image;
+   CHECK(vkCreateImage(dev, &ci, NULL, &image));
+   VkMemoryRequirements req;
+   vkGetImageMemoryRequirements(dev, image, &req);
+   VkDeviceMemory imem = memory(req, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+   CHECK(vkBindImageMemory(dev, image, imem, 0));
+   width >>= level; height = one_d ? 1 : height >> level;
+   depth = volume ? depth >> level : layers;
+   size_t full_size = (size_t)width * height * depth * bytes;
+   const unsigned px = 1, py = one_d ? 0 : 3;
+   const unsigned pw = width - 3, ph = one_d ? 1 : height - 5;
+   const unsigned first = depth > 1 ? 1 : 0, count = depth - first;
+   const unsigned row = pw + 2, rows = ph + 3;
+   const size_t patch_offset = (full_size + 63) & ~(size_t)63;
+   const size_t patch_size = (size_t)row * rows * count * bytes;
+   VkDeviceMemory smem, rmem;
+   uint8_t *src, *readback;
+   VkBuffer source = buffer(patch_offset + patch_size, &smem, (void **)&src);
+   VkBuffer result = buffer(full_size + 64, &rmem, (void **)&readback);
+   uint8_t *expected = malloc(full_size);
+   if (!expected) exit(3);
+   for (size_t i = 0; i < patch_offset + patch_size; ++i)
+      src[i] = (uint8_t)((i * 37 + (i >> 8) * 19 + shape * 23 + level * 11) ^ 0xa5);
+   memcpy(expected, src, full_size);
+   memset(readback, 0xcd, full_size + 64);
+   for (unsigned z = 0; z < count; ++z)
+      for (unsigned y = 0; y < ph; ++y)
+         memcpy(expected + (((size_t)(z + first) * height + y + py) * width + px) * bytes,
+                src + patch_offset + ((size_t)z * rows + y) * row * bytes, pw * bytes);
+
+   VkCommandBufferAllocateInfo ai = { .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
+      .commandPool = pool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1 };
+   VkCommandBuffer cmd;
+   CHECK(vkAllocateCommandBuffers(dev, &ai, &cmd));
+   VkCommandBufferBeginInfo begin = { .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO };
+   CHECK(vkBeginCommandBuffer(cmd, &begin));
+   VkImageMemoryBarrier ib = { .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+      .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
+      .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+      .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+      .image = image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, level, 1, 0, layers} };
+   vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+      0, 0, NULL, 0, NULL, 1, &ib);
+   VkBufferImageCopy all = { .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, volume ? 1 : layers},
+      .imageExtent = {width, height, volume ? depth : 1} };
+   vkCmdCopyBufferToImage(cmd, source, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &all);
+   ib.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+   ib.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+   vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+      0, 0, NULL, 0, NULL, 1, &ib);
+   VkBufferImageCopy patch = { .bufferOffset = patch_offset,
+      .bufferRowLength = row, .bufferImageHeight = one_d ? 0 : rows,
+      .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, volume ? 0 : first, volume ? 1 : count},
+      .imageOffset = {px, py, volume ? first : 0}, .imageExtent = {pw, ph, volume ? count : 1} };
+   /* For 1D arrays Vulkan's source layer stride is bufferRowLength, since
+    * there is only one row per image. */
+   if (one_d) {
+      patch.bufferImageHeight = 0;
+      for (unsigned z = 0; z < count; ++z)
+         memcpy(expected + ((size_t)(z + first) * width + px) * bytes,
+                src + patch_offset + (size_t)z * row * bytes, pw * bytes);
+   }
+   vkCmdCopyBufferToImage(cmd, source, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &patch);
+   ib.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+   ib.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+   vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+      0, 0, NULL, 0, NULL, 1, &ib);
+   vkCmdCopyImageToBuffer(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, result, 1, &all);
+   VkBufferMemoryBarrier host = { .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
+      .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_HOST_READ_BIT,
+      .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+      .buffer = result, .size = VK_WHOLE_SIZE };
+   vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT,
+      0, 0, NULL, 1, &host, 0, NULL);
+   CHECK(vkEndCommandBuffer(cmd));
+   unsigned failure = 0;
+   if (!record_only) {
+      VkFenceCreateInfo fci = { .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO };
+      VkFence fence;
+      CHECK(vkCreateFence(dev, &fci, NULL, &fence));
+      VkSubmitInfo submit = { .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
+         .commandBufferCount = 1, .pCommandBuffers = &cmd };
+      CHECK(vkQueueSubmit(queue, 1, &submit, fence));
+      CHECK(vkWaitForFences(dev, 1, &fence, VK_TRUE, 30ull * 1000000000));
+      vkDestroyFence(dev, fence, NULL);
+      for (size_t i = 0; i < full_size + 64; ++i) {
+         uint8_t want = i < full_size ? expected[i] : 0xcd;
+         if (readback[i] != want) {
+            fprintf(stderr, "MISMATCH format=%u shape=%u level=%u byte=%zu got=%02x expected=%02x\n",
+               format, shape, level, i, readback[i], want);
+            failure = 1;
+            break;
+         }
+      }
+   }
+   printf("%s format=%u bytes=%u shape=%u level=%u\n",
+      record_only ? "RECORDED" : failure ? "FAIL" : "PASS", format, bytes, shape, level);
+   fflush(stdout);
+   free(expected);
+   vkFreeCommandBuffers(dev, pool, 1, &cmd);
+   vkDestroyImage(dev, image, NULL); vkFreeMemory(dev, imem, NULL);
+   vkUnmapMemory(dev, smem); vkUnmapMemory(dev, rmem);
+   vkDestroyBuffer(dev, source, NULL); vkDestroyBuffer(dev, result, NULL);
+   vkFreeMemory(dev, smem, NULL); vkFreeMemory(dev, rmem, NULL);
+   return failure;
+}
+
+int main(int argc, char **argv)
+{
+   if (argc > 2 || (argc == 2 && strcmp(argv[1], "--record-only"))) return 2;
+   record_only = argc == 2;
+   VkApplicationInfo app = { .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .apiVersion = VK_API_VERSION_1_3 };
+   VkInstanceCreateInfo ici = { .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, .pApplicationInfo = &app };
+   VkInstance instance;
+   CHECK(vkCreateInstance(&ici, NULL, &instance));
+   uint32_t count = 1;
+   CHECK(vkEnumeratePhysicalDevices(instance, &count, &physical));
+   VkPhysicalDeviceProperties props;
+   vkGetPhysicalDeviceProperties(physical, &props);
+   fprintf(stderr, "Device: %s; mode: %s\n", props.deviceName, record_only ? "record only, no GPU submit" : "GPU readback");
+   float priority = 1;
+   VkDeviceQueueCreateInfo qci = { .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
+      .queueFamilyIndex = 0, .queueCount = 1, .pQueuePriorities = &priority };
+   VkDeviceCreateInfo dci = { .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
+      .queueCreateInfoCount = 1, .pQueueCreateInfos = &qci };
+   CHECK(vkCreateDevice(physical, &dci, NULL, &dev));
+   vkGetDeviceQueue(dev, 0, 0, &queue);
+   VkCommandPoolCreateInfo pci = { .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .queueFamilyIndex = 0 };
+   CHECK(vkCreateCommandPool(dev, &pci, NULL, &pool));
+   const struct { VkFormat format; unsigned bytes; } formats[] = {
+      {VK_FORMAT_R8G8B8A8_UNORM, 4}, {VK_FORMAT_R8G8B8A8_SRGB, 4},
+      {VK_FORMAT_B8G8R8A8_UNORM, 4}, {VK_FORMAT_R8G8B8A8_UINT, 4},
+      {VK_FORMAT_R32_UINT, 4}, {VK_FORMAT_R32_SFLOAT, 4},
+      {VK_FORMAT_R16G16_SFLOAT, 4}, {VK_FORMAT_R8_UINT, 1},
+      {VK_FORMAT_R16_UINT, 2}, {VK_FORMAT_R32G32_UINT, 8},
+      {VK_FORMAT_R32G32B32A32_UINT, 16},
+   };
+   unsigned failures = 0;
+   for (unsigned f = 0; f < sizeof(formats)/sizeof(formats[0]); ++f)
+      for (unsigned shape = 0; shape < 5; ++shape)
+         for (unsigned level = 0; level < 2; ++level)
+            failures += check_case(formats[f].format, formats[f].bytes, shape, level);
+   vkDestroyCommandPool(dev, pool, NULL); vkDestroyDevice(dev, NULL); vkDestroyInstance(instance, NULL);
+   return failures ? 1 : 0;
+}
diff --git a/src/asahi/vulkan/tests/g17p_vulkan_compressed_layer_copy.c b/src/asahi/vulkan/tests/g17p_vulkan_compressed_layer_copy.c
index 04ab81c29a90..9c6c5ce316be 100644
--- a/src/asahi/vulkan/tests/g17p_vulkan_compressed_layer_copy.c
+++ b/src/asahi/vulkan/tests/g17p_vulkan_compressed_layer_copy.c
@@ -47,6 +47,36 @@ static VkDevice dev;
 static VkQueue queue;
 static VkCommandPool pool;
 
+static FILE *driver_log;
+static int saved_stderr = -1;
+
+static void
+restore_stderr(void)
+{
+   if (saved_stderr < 0)
+      return;
+   fflush(stderr);
+   dup2(saved_stderr, STDERR_FILENO);
+   close(saved_stderr);
+   saved_stderr = -1;
+}
+
+/* CHECK() exits during setup too. Keep the loader error and the failing
+ * Vulkan call visible even when the normal log checks are never reached. */
+static void
+replay_driver_log(void)
+{
+   restore_stderr();
+   if (!driver_log)
+      return;
+   rewind(driver_log);
+   char line[1024];
+   while (fgets(line, sizeof(line), driver_log))
+      fputs(line, stderr);
+   fclose(driver_log);
+   driver_log = NULL;
+}
+
 static uint32_t
 memory_type(uint32_t bits, VkMemoryPropertyFlags flags)
 {
@@ -358,10 +388,15 @@ main(int argc, char **argv)
 
    /* Capture the driver's log to check it below. */
    fflush(stderr);
-   const int saved_stderr = dup(2);
-   FILE *log = tmpfile();
-   if (saved_stderr < 0 || !log || dup2(fileno(log), 2) < 0)
+   saved_stderr = dup(STDERR_FILENO);
+   driver_log = tmpfile();
+   if (saved_stderr < 0 || !driver_log ||
+       atexit(replay_driver_log) != 0 ||
+       dup2(fileno(driver_log), STDERR_FILENO) < 0) {
+      replay_driver_log();
+      fprintf(stderr, "could not capture driver log\n");
       return 2;
+   }
 
    const VkApplicationInfo app = {
       .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
@@ -435,18 +470,20 @@ main(int argc, char **argv)
    vkDestroyDevice(dev, NULL);
    vkDestroyInstance(instance, NULL);
 
-   fflush(stderr);
-   dup2(saved_stderr, 2);
-   rewind(log);
+   restore_stderr();
+   rewind(driver_log);
    bool partial_view = false, dst_compressed = false;
    char line[1024];
-   while (fgets(line, sizeof(line), log)) {
+   while (fgets(line, sizeof(line), driver_log)) {
       fputs(line, stderr);
       partial_view |= strstr(line, "addresses only the first") != NULL;
       dst_compressed |=
          strstr(line, "compression enabled: r32g32_float 129x127") != NULL;
    }
 
+   fclose(driver_log);
+   driver_log = NULL;
+
    if (partial_view) {
       fprintf(stderr, "FAIL: a copy viewed several layers through an "
                       "uncompressed view of the block-compressed source\n");
diff --git a/src/asahi/vulkan/tests/test-hk-agx3-as-format.c b/src/asahi/vulkan/tests/test-hk-agx3-as-format.c
new file mode 100644
index 000000000000..c384c51f5dbe
--- /dev/null
+++ b/src/asahi/vulkan/tests/test-hk-agx3-as-format.c
@@ -0,0 +1,134 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ *
+ * Every chip gets one native AS format end to end: the format the AS entry
+ * points dispatch on (hk_agx3_as_format_for_chip()), the encoding scratch
+ * and image layout its builds size and address, and the node encoder and
+ * refit programs its pipelines are created from. GEN1 (G17P and the other
+ * non-M5 chips) and M5 (G17G/G17S) must not share any of them.
+ */
+#undef NDEBUG
+#include "hk_acceleration_structure.h"
+#include "hk_agx3_as_layout.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+/* x of the LocalSize execution mode of a SPIR-V compute module. */
+static unsigned
+spirv_local_size_x(const uint32_t *code, size_t size_B)
+{
+   assert(size_B >= 20 && size_B % 4 == 0 && code[0] == 0x07230203);
+   unsigned x = 0, modes = 0;
+   for (size_t i = 5; i < size_B / 4;) {
+      const unsigned words = code[i] >> 16, op = code[i] & 0xffff;
+      assert(words && i + words <= size_B / 4);
+      if (op == 16 /* OpExecutionMode */ && words >= 6 &&
+          code[i + 2] == 17 /* LocalSize */) {
+         x = code[i + 3];
+         assert(code[i + 4] == 1 && code[i + 5] == 1);
+         modes++;
+      }
+      i += words;
+   }
+   assert(modes == 1);
+   return x;
+}
+
+static void
+check_layout(enum hk_agx3_as_format format, uint32_t leaves,
+             const struct hk_agx3_as_layout *l)
+{
+   const uint32_t align = format == HK_AGX3_AS_FORMAT_M5 ? 1024 : 16384;
+   const uint32_t tail = format == HK_AGX3_AS_FORMAT_M5 ? 32 : 48;
+   assert(hk_agx3_as_image_align(format) == align);
+   assert(hk_agx3_as_plan_tail(format) == tail);
+   if (!leaves) {
+      assert(l->slot_bound == 3 && l->encode_bytes == 0);
+      assert(l->as_bytes == 192 + (align == 16384 ? 0 : align - 1));
+      return;
+   }
+   const uint64_t nodes = (uint64_t)leaves + (leaves > 1 ? leaves : 2) - 1;
+   const uint64_t slots = (uint64_t)leaves * 4 + 1;
+   assert(l->node_bound == nodes && l->slot_bound == slots);
+   assert(l->jobs == nodes * 16 && l->envelope == l->jobs + nodes * 64);
+   assert(l->state == l->envelope + 48);
+   /* GEN1's plan writes the indirect encode dispatch after its state. */
+   assert(l->node_status == l->state + tail);
+   assert(l->envelope_status == l->node_status + nodes * 4);
+   const uint64_t refit_levels = (l->envelope_status + 4 + 7) & ~7u;
+   assert(l->refit_levels == refit_levels);
+   assert(l->encode_bytes == refit_levels +
+          (format == HK_AGX3_AS_FORMAT_GEN1 ? 16 * 12 : 0));
+   assert(l->as_bytes == slots * HK_AGX3_AS_SLOT_B + 184 +
+                            (align == 16384 ? 0 : align - 1));
+}
+
+int
+main(void)
+{
+   unsigned m5 = 0, gen1 = 0;
+   for (enum agx_chip chip = 0; chip < AGX_CHIP_UNKNOWN; ++chip) {
+      const enum hk_agx3_as_format format = hk_agx3_as_format_for_chip(chip);
+      const bool is_m5 = chip == AGX_CHIP_G17G || chip == AGX_CHIP_G17S;
+      assert(format == (is_m5 ? HK_AGX3_AS_FORMAT_M5 : HK_AGX3_AS_FORMAT_GEN1));
+      m5 += is_m5;
+      gen1 += !is_m5;
+
+      /* The scratch and image layout the chip's builds use. */
+      static const uint32_t leaves[] = {0, 1, 2, 3, 7, 64, 1000, 1u << 20};
+      for (unsigned i = 0; i < sizeof(leaves) / sizeof(leaves[0]); ++i) {
+         struct hk_agx3_as_layout l, other;
+         assert(hk_agx3_as_layout_init_format(format, leaves[i], &l));
+         assert(is_m5 ? hk_agx3_as_layout_init(leaves[i], &other)
+                      : hk_gen1_as_layout_init(leaves[i], &other));
+         assert(!memcmp(&l, &other, sizeof(l)));
+         check_layout(format, leaves[i], &l);
+      }
+
+      /* The node encoder: one node per 8 lanes on M5, four 8-lane jobs per
+       * workgroup on GEN1. */
+      size_t size_B;
+      const uint32_t *encoder = hk_agx3_node_encoder_spirv(chip, &size_B);
+      assert(encoder && spirv_local_size_x(encoder, size_B) ==
+                           (is_m5 ? HK_AGX3_M5_NODE_GROUP_LANES
+                                  : HK_AGX3_GEN1_NODE_GROUP_LANES));
+      size_t own_B;
+      const uint32_t *own = is_m5 ? hk_m5_agx3_node_encoder_spirv(false, &own_B)
+                                  : hk_gen1_agx3_node_encoder_spirv(false, &own_B);
+      assert(own == encoder && own_B == size_B);
+
+      /* The refit of the same records. */
+      const uint32_t *refit = hk_agx3_refit_spirv(chip, &size_B);
+      own = is_m5 ? hk_m5_agx3_refit_spirv(&own_B) : hk_gen1_agx3_refit_spirv(&own_B);
+      assert(refit && own == refit && own_B == size_B);
+   }
+   assert(m5 == 2 && gen1 == AGX_CHIP_UNKNOWN - 2);
+
+   /* The two formats share no program, and each format's variants keep its
+    * workgroup shape: GEN1 triangle pairs, M5 batched builds. */
+   size_t gen1_B, m5_B, pairs_B, batch_B, gen1_refit_B, m5_refit_B;
+   const uint32_t *gen1_enc = hk_gen1_agx3_node_encoder_spirv(false, &gen1_B);
+   const uint32_t *m5_enc = hk_m5_agx3_node_encoder_spirv(false, &m5_B);
+   const uint32_t *pairs = hk_gen1_agx3_node_encoder_spirv(true, &pairs_B);
+   const uint32_t *batch = hk_m5_agx3_node_encoder_spirv(true, &batch_B);
+   assert(gen1_enc != m5_enc && pairs != gen1_enc && batch != m5_enc);
+   assert(gen1_B != m5_B || memcmp(gen1_enc, m5_enc, gen1_B));
+   assert(spirv_local_size_x(pairs, pairs_B) == HK_AGX3_GEN1_NODE_GROUP_LANES);
+   assert(spirv_local_size_x(batch, batch_B) == HK_AGX3_M5_NODE_GROUP_LANES);
+   const uint32_t *gen1_refit = hk_gen1_agx3_refit_spirv(&gen1_refit_B);
+   const uint32_t *m5_refit = hk_m5_agx3_refit_spirv(&m5_refit_B);
+   assert(gen1_refit != m5_refit &&
+          (gen1_refit_B != m5_refit_B ||
+           memcmp(gen1_refit, m5_refit, gen1_refit_B)));
+
+   /* The layouts differ exactly where the formats do. */
+   struct hk_agx3_as_layout g, m;
+   assert(hk_gen1_as_layout_init(64, &g) && hk_agx3_as_layout_init(64, &m));
+   assert(g.state == m.state && g.node_status == m.node_status + 16);
+   assert(m.as_bytes == g.as_bytes + 1023);
+
+   printf("hk_agx3_as_format: %u M5 and %u GEN1 chips, each with its own "
+          "layout, encoder and refit\n", m5, gen1);
+   return 0;
+}
diff --git a/src/asahi/vulkan/tests/test-hk-execution-allowance.c b/src/asahi/vulkan/tests/test-hk-execution-allowance.c
index 35661b32acee..b784ba021465 100644
--- a/src/asahi/vulkan/tests/test-hk-execution-allowance.c
+++ b/src/asahi/vulkan/tests/test-hk-execution-allowance.c
@@ -40,11 +40,49 @@ check_merge(bool first, bool second, bool linked)
    assert(links == before + linked);
 }
 
+static void
+check_as_contracts(void)
+{
+   struct agx_device dev = {
+      .chip = AGX_CHIP_G17P,
+      .params = {.chip_id = 0x6040, .gpu_generation = 16, .gpu_variant = 'S'},
+   };
+   /* Exercise every combination: AS construction cannot imply traversal,
+    * and bounded execution alone cannot select the extended-prior ABI. */
+   for (unsigned bits = 0; bits < 8; ++bits) {
+      dev.params.features =
+         ((bits & 1) ? DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS : 0) |
+         ((bits & 2) ? DRM_ASAHI_FEATURE_RENDER_RAY_QUERY : 0) |
+         ((bits & 4) ? DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD : 0);
+      assert(hk_g16s_as_enabled(&dev) == ((bits & 5) == 5));
+      assert(hk_g17p_rt_enabled(&dev) == ((bits & 3) == 3));
+      assert(hk_gen1_as_enabled(&dev) ==
+             (((bits & 5) == 5) || ((bits & 3) == 3)));
+   }
+   dev.params.features = UINT64_MAX;
+   dev.params.chip_id = 0x8140;
+   assert(!hk_g16s_as_enabled(&dev) && hk_g17p_rt_enabled(&dev));
+   dev.params.chip_id = 0x6040;
+   dev.params.gpu_variant = 'G';
+   assert(!hk_g16s_as_enabled(&dev));
+   dev.params.gpu_variant = 'S';
+   dev.params.gpu_generation = 17;
+   assert(!hk_g16s_as_enabled(&dev));
+   dev.params.gpu_generation = 16;
+   for (enum agx_chip chip = AGX_CHIP_G13G; chip < AGX_CHIP_UNKNOWN; ++chip) {
+      dev.chip = chip;
+      if (chip != AGX_CHIP_G17P)
+         assert(!hk_g16s_as_enabled(&dev) && !hk_g17p_rt_enabled(&dev));
+   }
+}
+
 int main(void)
 {
+   check_as_contracts();
    for (enum agx_chip chip = AGX_CHIP_G13G; chip < AGX_CHIP_UNKNOWN; ++chip)
       assert(hk_shader_binary_version(chip) ==
-             (chip == AGX_CHIP_G17P ? 35 :
+             (chip == AGX_CHIP_G17P || chip == AGX_CHIP_G17G ||
+              chip == AGX_CHIP_G17S ? 36 :
               chip <= AGX_CHIP_G15P ? 36 : 34));
    assert(hk_as_build_flags(AGX_CHIP_G17P, 0, true) == 0);
    assert(hk_as_build_flags(AGX_CHIP_G17P, DRM_ASAHI_FEATURE_RENDER_RAY_QUERY, true) == 0);
diff --git a/src/asahi/vulkan/tests/test-hk-g16s-compression.c b/src/asahi/vulkan/tests/test-hk-g16s-compression.c
new file mode 100644
index 000000000000..89ba6afdaf48
--- /dev/null
+++ b/src/asahi/vulkan/tests/test-hk-g16s-compression.c
@@ -0,0 +1,620 @@
+/* Copyright 2026 Aurora Silicon; SPDX-License-Identifier: MIT
+ *
+ * Pure checks of the G17P lossless-compression admission policy
+ * (hk_g17p_compression.h). The generic hk_can_compress() checks are applied
+ * separately by the driver.
+ */
+#include "asahi/vulkan/hk_g17p_compression.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+/* The physical T6040 selector in hk_image.c supplies this capability.
+ * The pre-existing test-hk-g17p-compression separately pins false/Neo. */
+static bool
+g16s_eligible(enum agx_chip chip, unsigned plane, const VkImageCreateInfo *info)
+{
+   return hk_hal200_compression_eligible(chip, plane, info, true);
+}
+
+static unsigned checks;
+#define CHECK(x)                                                              \
+   do {                                                                       \
+      checks++;                                                               \
+      if (!(x)) {                                                             \
+         fprintf(stderr, "line %u: %s\n", __LINE__, #x);                      \
+         exit(1);                                                             \
+      }                                                                       \
+   } while (0)
+
+/* Exercise mutable extension chains through a runtime call, as the driver
+ * does, rather than letting the compiler fold the whole stack fixture. */
+static bool (*volatile runtime_eligible)(enum agx_chip, unsigned,
+                                         const VkImageCreateInfo *) =
+   g16s_eligible;
+#define ELIGIBLE(i) runtime_eligible(AGX_CHIP_G17P, 0, &(i))
+
+static VkImageCreateInfo
+image(VkFormat format, uint32_t w, uint32_t h, uint32_t mips,
+      VkImageUsageFlags usage)
+{
+   return (VkImageCreateInfo){
+      .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+      .imageType = VK_IMAGE_TYPE_2D,
+      .format = format,
+      .extent = {w, h, 1},
+      .mipLevels = mips,
+      .arrayLayers = 1,
+      .samples = VK_SAMPLE_COUNT_1_BIT,
+      .tiling = VK_IMAGE_TILING_OPTIMAL,
+      .usage = usage,
+      .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+      .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+   };
+}
+
+static void
+check_formats(void)
+{
+   const VkImageUsageFlags color_usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+
+   /* Every core format below VK_FORMAT_ASTC_12x12_SRGB_BLOCK. */
+   unsigned color = 0, depth = 0;
+   for (unsigned f = 1; f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK; ++f) {
+      const bool c = hk_g17p_compression_color_format((VkFormat)f);
+      const bool d = hk_g17p_compression_depth_format((VkFormat)f) ||
+                     f == VK_FORMAT_S8_UINT;
+      CHECK(!(c && d));
+      color += c;
+      depth += d;
+
+      VkImageCreateInfo i = image((VkFormat)f, 64, 64, 1, color_usage);
+      CHECK(ELIGIBLE(i) == (c || hk_g17p_compression_depth_format((VkFormat)f)));
+   }
+   CHECK(color == 46);
+   CHECK(depth == 4);
+
+   /* No 16-byte texels (tilebuffer spill), no packed/padded RGB, no packed
+    * depth/stencil, no block-compressed or YCbCr formats. */
+   static const VkFormat rejected[] = {
+      VK_FORMAT_R32G32B32A32_SFLOAT,
+      VK_FORMAT_R32G32B32A32_UINT,
+      VK_FORMAT_R32G32B32A32_SINT,
+      VK_FORMAT_R32G32B32_SFLOAT,
+      VK_FORMAT_R8G8B8_UNORM,
+      VK_FORMAT_R16G16B16_SFLOAT,
+      VK_FORMAT_B10G11R11_UFLOAT_PACK32,
+      VK_FORMAT_E5B9G9R9_UFLOAT_PACK32,
+      VK_FORMAT_R4G4B4A4_UNORM_PACK16,
+      VK_FORMAT_A1R5G5B5_UNORM_PACK16,
+      VK_FORMAT_D24_UNORM_S8_UINT,
+      VK_FORMAT_D16_UNORM_S8_UINT,
+      VK_FORMAT_X8_D24_UNORM_PACK32,
+      VK_FORMAT_BC1_RGBA_UNORM_BLOCK,
+      VK_FORMAT_ETC2_R8G8B8A8_UNORM_BLOCK,
+      VK_FORMAT_G8_B8R8_2PLANE_420_UNORM,
+      VK_FORMAT_G8B8G8R8_422_UNORM,
+      VK_FORMAT_R64_UINT,
+   };
+   for (unsigned r = 0; r < sizeof(rejected) / sizeof(rejected[0]); ++r) {
+      VkImageCreateInfo i = image(rejected[r], 64, 64, 1, color_usage);
+      CHECK(!ELIGIBLE(i));
+   }
+}
+
+static void
+check_shape(VkFormat format, bool is_zs)
+{
+   const bool is_depth = hk_g17p_compression_depth_format(format);
+   const VkImageUsageFlags usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT |
+      (is_zs ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT
+             : VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
+   const VkImageCreateInfo base = image(format, 2408, 1508, 1, usage);
+   CHECK(ELIGIBLE(base));
+
+   /* Chip and plane. */
+   for (unsigned chip = 0; chip <= AGX_CHIP_UNKNOWN; ++chip)
+      CHECK(g16s_eligible((enum agx_chip)chip, 0, &base) ==
+            (chip == AGX_CHIP_G17P));
+   /* Plane 1 is the stencil plane of D32_SFLOAT_S8_UINT alone. */
+   CHECK(g16s_eligible(AGX_CHIP_G17P, 1, &base) ==
+         (format == VK_FORMAT_D32_SFLOAT_S8_UINT));
+   CHECK(!g16s_eligible(AGX_CHIP_G17P, 2, &base));
+   CHECK(!g16s_eligible(AGX_CHIP_G17P, 0, NULL));
+   CHECK(!g16s_eligible(AGX_CHIP_G17P, 1, NULL));
+
+   /* Usage: only texture/PBE/ZLS usages are admitted. */
+   VkImageUsageFlags allowed =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
+   if (is_depth)
+      allowed |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+   for (unsigned bit = 0; bit < 32; ++bit) {
+      VkImageCreateInfo i = base;
+      i.usage |= 1u << bit;
+      CHECK(ELIGIBLE(i) == !!((1u << bit) & allowed));
+   }
+   for (unsigned usage_bits = 0; usage_bits < 256; ++usage_bits) {
+      VkImageCreateInfo i = base;
+      i.usage = usage_bits & allowed;
+      CHECK(ELIGIBLE(i));
+   }
+   static const VkImageUsageFlags refused[] = {
+      VK_IMAGE_USAGE_STORAGE_BIT, VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT,
+      VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT,
+      VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR,
+      VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR,
+      VK_IMAGE_USAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR,
+      VK_IMAGE_USAGE_FRAGMENT_DENSITY_MAP_BIT_EXT,
+   };
+   for (unsigned r = 0; r < sizeof(refused) / sizeof(refused[0]); ++r) {
+      VkImageCreateInfo i = base;
+      i.usage |= refused[r];
+      CHECK(ELIGIBLE(i) == (is_depth && refused[r] ==
+                           VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT));
+   }
+
+   /* Sample-location tracking is also admitted for single-sample depth. */
+   for (unsigned bit = 0; bit < 32; ++bit) {
+      VkImageCreateInfo i = base;
+      i.flags = 1u << bit;
+      CHECK(ELIGIBLE(i) ==
+            ((1u << bit) == VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT ||
+             (1u << bit) == VK_IMAGE_CREATE_EXTENDED_USAGE_BIT ||
+             (is_depth && (1u << bit) ==
+                          VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT)));
+   }
+
+   /* Geometry. */
+   VkImageCreateInfo i = base;
+   i.imageType = VK_IMAGE_TYPE_1D;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.imageType = VK_IMAGE_TYPE_3D;
+   CHECK(ELIGIBLE(i) == !is_zs);
+   i = base; i.extent.depth = 2;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.arrayLayers = 2;
+   CHECK(ELIGIBLE(i));
+   i = base; i.arrayLayers = 6;
+   CHECK(ELIGIBLE(i));
+   for (unsigned s = 2; s <= 64; s <<= 1) {
+      i = base; i.samples = (VkSampleCountFlagBits)s;
+      CHECK(ELIGIBLE(i) == (is_depth && s <= 4));
+   }
+   i = base; i.tiling = VK_IMAGE_TILING_LINEAR;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.mipLevels = 0;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.mipLevels = 12;
+   CHECK(ELIGIBLE(i));
+   i = base; i.extent.width = 15;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.extent.height = 15;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.extent.width = 16; i.extent.height = 16;
+   CHECK(ELIGIBLE(i));
+   i = base; i.extent.width = 17; i.extent.height = 31;
+   CHECK(ELIGIBLE(i));
+   i = base; i.extent.width = 16384; i.extent.height = 16384;
+   CHECK(ELIGIBLE(i));
+   i = base; i.sharingMode = VK_SHARING_MODE_CONCURRENT;
+   CHECK(ELIGIBLE(i));
+
+   /* Shared images stay plain; an empty external handle set does not. */
+   VkExternalMemoryImageCreateInfo external = {
+      .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
+      .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
+   };
+   VkImageFormatListCreateInfo list = {
+      .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO,
+      .pNext = &external,
+   };
+   i = base; i.pNext = &external;
+   CHECK(!ELIGIBLE(i));
+   i = base; i.pNext = &list;
+   CHECK(!ELIGIBLE(i));
+   external.handleTypes = 0;
+   CHECK(ELIGIBLE(i));
+   list.pNext = NULL;
+   CHECK(ELIGIBLE(i));
+   VkBaseInStructure wsi = {.sType = HK_G17P_WSI_IMAGE_CREATE_INFO_MESA};
+   i = base; i.pNext = &wsi;
+   CHECK(!ELIGIBLE(i));
+}
+
+/* Compression is chosen for the image's whole lifetime. Its first render
+ * pass, attachment slot and neighbouring formats are unknown at allocation.
+ * Cover every admitted colour format, including the smallest ones that can
+ * spill next to wide targets, and input/transient attachment combinations.
+ */
+static void
+check_colour_msaa_spill_policy(void)
+{
+   for (unsigned f = 1; f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK; ++f) {
+      if (!hk_g17p_compression_color_format((VkFormat)f))
+         continue;
+
+      for (unsigned samples = 1; samples <= 4; samples *= 2) {
+         for (unsigned layers = 1; layers <= 4; layers *= 4) {
+            VkImageCreateInfo i = image((VkFormat)f, 64, 64, 1,
+               VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+               VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT);
+            i.samples = (VkSampleCountFlagBits)samples;
+            i.arrayLayers = layers;
+            CHECK(ELIGIBLE(i));
+            for (unsigned mask = 0; mask < 4; ++mask) {
+               VkImageCreateInfo attachment = i;
+               attachment.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+               if (mask & 1)
+                  attachment.usage |= VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
+               if (mask & 2)
+                  attachment.usage |= VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT;
+               CHECK(ELIGIBLE(attachment) == (samples == 1));
+               attachment.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT |
+                                  VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
+               CHECK(ELIGIBLE(attachment) == (samples == 1));
+            }
+         }
+      }
+   }
+}
+
+static void
+check_layered_and_msaa(void)
+{
+   VkImageCreateInfo i =
+      image(VK_FORMAT_R8G8B8A8_SRGB, 256, 256, 1,
+            VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+               VK_IMAGE_USAGE_SAMPLED_BIT);
+   i.arrayLayers = 12;
+   i.flags = VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
+   CHECK(ELIGIBLE(i));
+   i.mipLevels = 5;
+   CHECK(!ELIGIBLE(i));
+   i.mipLevels = 1;
+   i.flags = 0;
+   for (unsigned samples = 1; samples <= 64; samples <<= 1) {
+      i.samples = (VkSampleCountFlagBits)samples;
+      CHECK(ELIGIBLE(i) == (samples <= 4));
+      i.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+      CHECK(ELIGIBLE(i) == (samples == 1));
+      i.usage &= ~VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+      i.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
+      CHECK(!ELIGIBLE(i));
+      i.usage &= ~VK_IMAGE_USAGE_STORAGE_BIT;
+   }
+   i.samples = VK_SAMPLE_COUNT_1_BIT;
+   i.arrayLayers = 1;
+   i.imageType = VK_IMAGE_TYPE_3D;
+   i.extent.depth = 9;
+   i.flags = VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT |
+             VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT;
+   CHECK(ELIGIBLE(i));
+   i.mipLevels = 5;
+   CHECK(!ELIGIBLE(i));
+   i.mipLevels = 1;
+   i.samples = VK_SAMPLE_COUNT_2_BIT;
+   CHECK(!ELIGIBLE(i));
+   i.samples = VK_SAMPLE_COUNT_1_BIT;
+   i.arrayLayers = 2;
+   CHECK(!ELIGIBLE(i));
+   i.arrayLayers = 1;
+   i.format = VK_FORMAT_D32_SFLOAT;
+   CHECK(!ELIGIBLE(i));
+
+
+}
+
+/* Preserve the pre-port depth/stencil policy across the shapes observed in
+ * depth sampling, feedback loops, host copies and transfer clears. Allocation
+ * tests alone did not catch the unsupported colour-alias command paths. */
+static void
+check_legacy_depth_policy(void)
+{
+   const VkFormat formats[] = { VK_FORMAT_D16_UNORM, VK_FORMAT_D32_SFLOAT,
+      VK_FORMAT_D32_SFLOAT_S8_UINT, VK_FORMAT_S8_UINT };
+   const VkImageUsageFlags usages[] = {
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
+         VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT,
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT,
+   };
+   for (unsigned f = 0; f < 4; ++f) {
+      for (unsigned u = 0; u < 4; ++u) {
+         for (unsigned levels = 1; levels <= 8; levels *= 2) {
+            for (unsigned layers = 1; layers <= 6; ++layers) {
+               for (unsigned samples = 1; samples <= 4; samples *= 2) {
+                  VkImageCreateInfo i = image(formats[f], 128, 128, levels,
+                                               usages[u]);
+                  i.arrayLayers = layers;
+                  i.samples = (VkSampleCountFlagBits)samples;
+                  const bool expected = f < 3 && u < 3 && levels == 1 &&
+                                        layers == 1 && samples == 1;
+                  CHECK(ELIGIBLE(i) == expected);
+                  CHECK(g16s_eligible(AGX_CHIP_G17P, 1, &i) ==
+                        (expected && f == 2));
+               }
+            }
+         }
+      }
+   }
+}
+
+static void
+check_minecraft(void)
+{
+   /* The live Minecraft 26.3 images: two 2408x1508 RGBA8 colour targets
+    * (usage 0x17), the 2048^2 five-level blocks atlas (0x17) and the
+    * 2408x1508 D32 depth target (0x27). All take the compressed path. */
+   const VkImageCreateInfo color =
+      image(VK_FORMAT_R8G8B8A8_UNORM, 2408, 1508, 1, 0x17);
+   const VkImageCreateInfo atlas =
+      image(VK_FORMAT_R8G8B8A8_UNORM, 2048, 2048, 5, 0x17);
+   const VkImageCreateInfo depth = image(VK_FORMAT_D32_SFLOAT, 2408, 1508, 1,
+                                         0x27);
+   CHECK(ELIGIBLE(color));
+   CHECK(ELIGIBLE(atlas));
+   CHECK(ELIGIBLE(depth));
+   /* Small blit-generated RGBA8/BGRA8 UNORM mip chains the application
+    * never renders to stay on the raw compute path; from
+    * HK_G17P_MIP_PBE_MIN_TEXELS texels, and everything else, compresses. */
+   const VkImageUsageFlags texture_usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+      VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
+   const VkFormat compute_blit[] = {VK_FORMAT_R8G8B8A8_UNORM,
+                                    VK_FORMAT_B8G8R8A8_UNORM};
+   for (unsigned f = 0; f < 2; ++f) {
+      VkImageCreateInfo t = image(compute_blit[f], 256, 256, 9, texture_usage);
+      CHECK(!ELIGIBLE(t));
+      t.mipLevels = 1;
+      CHECK(ELIGIBLE(t));
+      t.mipLevels = 9;
+      t.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+      CHECK(ELIGIBLE(t));
+
+      /* 362x362 is 131,044 texels, just below 2^17; 512x256 is 2^17. */
+      VkImageCreateInfo big = image(compute_blit[f], 362, 362, 9, texture_usage);
+      CHECK(!ELIGIBLE(big));
+      big = image(compute_blit[f], 512, 256, 10, texture_usage);
+      CHECK(ELIGIBLE(big));
+      big = image(compute_blit[f], 4096, 4096, 13, texture_usage);
+      CHECK(ELIGIBLE(big));
+   }
+   VkImageCreateInfo srgb = image(VK_FORMAT_R8G8B8A8_SRGB, 256, 256, 9,
+                                  texture_usage);
+   CHECK(ELIGIBLE(srgb));
+   VkImageCreateInfo half = image(VK_FORMAT_R16G16B16A16_SFLOAT, 256, 256, 9,
+                                  texture_usage);
+   CHECK(ELIGIBLE(half));
+
+   /* The swapchain is external/WSI memory and stays plain. */
+   VkExternalMemoryImageCreateInfo external = {
+      .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
+      .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
+   };
+   VkImageCreateInfo i = image(VK_FORMAT_B8G8R8A8_UNORM, 2408, 1508, 1, 0x13);
+   CHECK(ELIGIBLE(i));
+   i.pNext = &external;
+   CHECK(!ELIGIBLE(i));
+}
+
+static void
+check_zink(void)
+{
+   /* Portal 2 under zink (t-1079): GL render targets declare
+    * feedback-loop usage and keep the uncompressed layout. The GL Z24S8
+    * depth buffer is D32S8 on Honeykrisp; both its depth and its stencil
+    * plane compress. */
+   const VkImageUsageFlags rt = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+      VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+      VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+   const VkImageUsageFlags ds = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+      VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+   VkImageCreateInfo color = image(VK_FORMAT_R8G8B8A8_SRGB, 2408, 1506, 1, rt);
+   color.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
+   const VkFormat views[] = {VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8G8B8A8_SRGB};
+   VkImageFormatListCreateInfo list = {
+      .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO,
+      .viewFormatCount = 2,
+      .pViewFormats = views,
+   };
+   color.pNext = &list;
+   CHECK(!ELIGIBLE(color));
+   VkImageCreateInfo depth =
+      image(VK_FORMAT_D32_SFLOAT_S8_UINT, 2408, 1506, 1, ds);
+   depth.flags = VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT;
+   CHECK(ELIGIBLE(depth));
+   CHECK(g16s_eligible(AGX_CHIP_G17P, 1, &depth));
+   /* This is Portal's actual shape. A device-copy performance query must
+    * find its compressed layout when host transfer is removed; otherwise
+    * zink retains HOST_TRANSFER and never uses depth compression. */
+   depth.usage |= VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
+   CHECK(!ELIGIBLE(depth));
+   depth.usage &= ~VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
+   CHECK(ELIGIBLE(depth));
+   depth.samples = VK_SAMPLE_COUNT_4_BIT;
+   CHECK(!ELIGIBLE(depth));
+   const VkImageCreateInfo d32 = image(VK_FORMAT_D32_SFLOAT, 2408, 1506, 1, ds);
+   CHECK(ELIGIBLE(d32));
+   /* A host-transfer image keeps the plain layout host copies require. */
+   VkImageCreateInfo hic = color;
+   hic.usage |= VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
+   CHECK(!ELIGIBLE(hic));
+}
+
+#define SHARED(i) hk_g17p_shared_compression_eligible(AGX_CHIP_G17P, 0, &(i))
+
+static VkImageCreateInfo
+swapchain_image(VkFormat format, uint32_t w, uint32_t h,
+                VkImageUsageFlags usage)
+{
+   VkImageCreateInfo i = image(format, w, h, 1, usage);
+   i.flags = VK_IMAGE_CREATE_ALIAS_BIT;
+   i.tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
+   return i;
+}
+
+static void
+check_shared(void)
+{
+   /* vkmark's swapchain (headless and Wayland): one colour attachment. The
+    * WSI image carries ALIAS, the Mesa WSI marker and external memory, all
+    * of which the shared contract admits. */
+   VkExternalMemoryImageCreateInfo external = {
+      .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
+      .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
+   };
+   VkBaseInStructure wsi = {
+      .sType = HK_G17P_WSI_IMAGE_CREATE_INFO_MESA,
+      .pNext = (const VkBaseInStructure *)&external,
+   };
+   VkImageCreateInfo vkmark = swapchain_image(
+      VK_FORMAT_B8G8R8A8_SRGB, 1920, 1080, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
+   vkmark.pNext = &wsi;
+   CHECK(SHARED(vkmark));
+   /* The private policy keeps refusing it: sharing goes through the
+    * modifier contract only. */
+   CHECK(!ELIGIBLE(vkmark));
+   vkmark.tiling = VK_IMAGE_TILING_OPTIMAL;
+   CHECK(!SHARED(vkmark));
+   vkmark.tiling = VK_IMAGE_TILING_LINEAR;
+   CHECK(!SHARED(vkmark));
+
+   /* Every private colour format, and nothing else (depth, stencil,
+    * 16-byte texels, packed RGB, YCbCr, block compression). */
+   unsigned shared = 0;
+   for (unsigned f = 1; f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK; ++f) {
+      VkImageCreateInfo i = swapchain_image((VkFormat)f, 64, 64,
+                                            VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
+      CHECK(SHARED(i) == hk_g17p_compression_color_format((VkFormat)f));
+      shared += SHARED(i);
+   }
+   CHECK(shared == 46);
+   static const VkFormat refused_formats[] = {
+      VK_FORMAT_D32_SFLOAT, VK_FORMAT_D16_UNORM, VK_FORMAT_D32_SFLOAT_S8_UINT,
+      VK_FORMAT_S8_UINT, VK_FORMAT_R32G32B32A32_SFLOAT,
+      VK_FORMAT_B10G11R11_UFLOAT_PACK32, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32,
+      VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, VK_FORMAT_BC1_RGBA_UNORM_BLOCK,
+   };
+   for (unsigned r = 0;
+        r < sizeof(refused_formats) / sizeof(refused_formats[0]); ++r) {
+      VkImageCreateInfo i = swapchain_image(refused_formats[r], 64, 64,
+                                            VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
+      CHECK(!SHARED(i));
+   }
+
+   const VkImageCreateInfo base = swapchain_image(
+      VK_FORMAT_R8G8B8A8_UNORM, 2408, 1506,
+      VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT);
+   CHECK(SHARED(base));
+
+   /* G17P only, plane 0 only. */
+   for (unsigned chip = 0; chip <= AGX_CHIP_UNKNOWN; ++chip)
+      CHECK(hk_g17p_shared_compression_eligible((enum agx_chip)chip, 0,
+                                                &base) ==
+            (chip == AGX_CHIP_G17P || agx_chip_g17_m5((enum agx_chip)chip)));
+   CHECK(!hk_g17p_shared_compression_eligible(AGX_CHIP_G17P, 1, &base));
+   CHECK(!hk_g17p_shared_compression_eligible(AGX_CHIP_G17P, 0, NULL));
+
+   /* Usage: texture/PBE usages plus import-side feedback loops. Storage and
+    * host transfer access the image raw. */
+   const VkImageUsageFlags allowed_usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+   for (unsigned bit = 0; bit < 32; ++bit) {
+      VkImageCreateInfo i = base;
+      i.usage |= 1u << bit;
+      CHECK(SHARED(i) == !!((1u << bit) & allowed_usage));
+   }
+   /* No colour-attachment requirement: a blit-only swapchain is a transfer
+    * destination, which hk_image_init() makes renderable. */
+   VkImageCreateInfo blit_only = base;
+   blit_only.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+   CHECK(SHARED(blit_only));
+
+   /* Flags: mutable/extended views and WSI's ALIAS only. */
+   for (unsigned bit = 0; bit < 32; ++bit) {
+      VkImageCreateInfo i = base;
+      i.flags = 1u << bit;
+      CHECK(SHARED(i) == ((1u << bit) == VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT ||
+                          (1u << bit) == VK_IMAGE_CREATE_EXTENDED_USAGE_BIT ||
+                          (1u << bit) == VK_IMAGE_CREATE_ALIAS_BIT));
+   }
+
+   /* One level, one layer, one sample, 2D, at least 16x16. */
+   VkImageCreateInfo i = base;
+   i.mipLevels = 2;
+   CHECK(!SHARED(i));
+   i = base; i.arrayLayers = 2;
+   CHECK(!SHARED(i));
+   i = base; i.extent.depth = 2;
+   CHECK(!SHARED(i));
+   i = base; i.imageType = VK_IMAGE_TYPE_3D;
+   CHECK(!SHARED(i));
+   i = base; i.imageType = VK_IMAGE_TYPE_1D;
+   CHECK(!SHARED(i));
+   for (unsigned s = 2; s <= 64; s <<= 1) {
+      i = base; i.samples = (VkSampleCountFlagBits)s;
+      CHECK(!SHARED(i));
+   }
+   i = base; i.extent.width = 15;
+   CHECK(!SHARED(i));
+   i = base; i.extent.height = 15;
+   CHECK(!SHARED(i));
+   i = base; i.extent.width = 16; i.extent.height = 16;
+   CHECK(SHARED(i));
+   i = base; i.extent.width = 16384; i.extent.height = 16384;
+   CHECK(SHARED(i));
+
+   /* The format-query shape (no extent) agrees with the create shape. */
+   CHECK(hk_g17p_shared_compression_shape(AGX_CHIP_G17P, base.format,
+                                          base.imageType, base.flags,
+                                          base.usage));
+   CHECK(!hk_g17p_shared_compression_shape(AGX_CHIP_G17P, base.format,
+                                           base.imageType, base.flags,
+                                           VK_IMAGE_USAGE_STORAGE_BIT));
+   CHECK(!hk_g17p_shared_compression_shape(AGX_CHIP_G14G, base.format,
+                                           base.imageType, base.flags,
+                                           base.usage));
+
+   /* zink's kopper swapchain and imports carry feedback-loop usage. The
+    * shape admits it so an import of a compressed buffer succeeds;
+    * hk_can_compress() still refuses it when allocating. */
+   VkImageCreateInfo kopper = base;
+   kopper.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
+                   VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
+                   VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+   CHECK(SHARED(kopper));
+}
+
+int
+main(void)
+{
+   check_formats();
+   check_shape(VK_FORMAT_R8G8B8A8_UNORM, false);
+   check_shape(VK_FORMAT_B8G8R8A8_SRGB, false);
+   check_shape(VK_FORMAT_R16G16B16A16_SFLOAT, false);
+   check_shape(VK_FORMAT_R8_UNORM, false);
+   check_legacy_depth_policy();
+   check_layered_and_msaa();
+   check_colour_msaa_spill_policy();
+   check_minecraft();
+   check_zink();
+   check_shared();
+   printf("G17P compression policy: %u checks passed\n", checks);
+   return 0;
+}
diff --git a/src/asahi/vulkan/tests/test-hk-g17p-compression.c b/src/asahi/vulkan/tests/test-hk-g17p-compression.c
index 2e14b90e3276..3ccec5257a19 100644
--- a/src/asahi/vulkan/tests/test-hk-g17p-compression.c
+++ b/src/asahi/vulkan/tests/test-hk-g17p-compression.c
@@ -378,11 +378,12 @@ check_shared(void)
       VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT);
    CHECK(SHARED(base));
 
-   /* G17P only, plane 0 only. */
+   /* G17P and M5 only, plane 0 only. */
    for (unsigned chip = 0; chip <= AGX_CHIP_UNKNOWN; ++chip)
       CHECK(hk_g17p_shared_compression_eligible((enum agx_chip)chip, 0,
                                                 &base) ==
-            (chip == AGX_CHIP_G17P));
+            (chip == AGX_CHIP_G17P || chip == AGX_CHIP_G17G ||
+             chip == AGX_CHIP_G17S));
    CHECK(!hk_g17p_shared_compression_eligible(AGX_CHIP_G17P, 1, &base));
    CHECK(!hk_g17p_shared_compression_eligible(AGX_CHIP_G17P, 0, NULL));
 
@@ -460,6 +461,269 @@ check_shared(void)
    CHECK(SHARED(kopper));
 }
 
+static void
+check_m5_depth(void)
+{
+   /* Minecraft under zink: a D32 depth buffer with zink's usages and the
+    * sample-location flag zink sets on depth attachments. */
+   const VkImageUsageFlags ds = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+      VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
+   static const VkFormat formats[] = {
+      VK_FORMAT_D16_UNORM, VK_FORMAT_D32_SFLOAT, VK_FORMAT_D32_SFLOAT_S8_UINT,
+   };
+   for (unsigned f = 0; f < sizeof(formats) / sizeof(formats[0]); ++f) {
+      VkImageCreateInfo i = image(formats[f], 2976, 1872, 1, ds);
+      i.flags = VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT;
+      for (unsigned chip = 0; chip <= AGX_CHIP_UNKNOWN; ++chip)
+         CHECK(hk_m5_depth_compression_eligible((enum agx_chip)chip, 0, &i) ==
+               agx_chip_g17_m5((enum agx_chip)chip));
+      /* Of the second planes, only the single-sample stencil plane of
+       * D32_SFLOAT_S8_UINT is admitted. */
+      CHECK(hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 1, &i) ==
+            (formats[f] == VK_FORMAT_D32_SFLOAT_S8_UINT));
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, NULL));
+      /* Neither policy admits a depth image as colour, nor colour as depth. */
+      CHECK(!hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &i));
+
+      VkImageCreateInfo c = i;
+      c.usage |= VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.usage &= ~VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.samples = VK_SAMPLE_COUNT_4_BIT;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.mipLevels = 2;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.arrayLayers = 2;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      c.extent.width = 15;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c = i;
+      VkExternalMemoryImageCreateInfo external = {
+         .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
+         .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
+      };
+      c.pNext = &external;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+
+      /* Later links are read too: external memory or the WSI marker behind
+       * another structure still refuses compression. */
+      VkImageFormatListCreateInfo list = {
+         .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO,
+      };
+      VkBaseInStructure wsi = {.sType = HK_G17P_WSI_IMAGE_CREATE_INFO_MESA};
+      c = i;
+      c.pNext = &list;
+      CHECK(hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      list.pNext = &external;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      list.pNext = &wsi;
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &c));
+   }
+
+   /* No stencil-only or other depth/stencil formats. */
+   static const VkFormat rejected[] = {
+      VK_FORMAT_S8_UINT, VK_FORMAT_D24_UNORM_S8_UINT,
+      VK_FORMAT_D16_UNORM_S8_UINT, VK_FORMAT_X8_D24_UNORM_PACK32,
+      VK_FORMAT_R32_SFLOAT,
+   };
+   for (unsigned r = 0; r < sizeof(rejected) / sizeof(rejected[0]); ++r) {
+      VkImageCreateInfo i = image(rejected[r], 64, 64, 1, ds);
+      CHECK(!hk_m5_depth_compression_eligible(AGX_CHIP_G17S, 0, &i));
+   }
+}
+
+static void
+check_m5_color(void)
+{
+   const VkImageUsageFlags usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+      VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+      VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+   /* Minecraft's blocks atlas: RGBA8, 2048x2048, five mips; zink gives its
+    * sampled textures colour-attachment usage. */
+   VkImageCreateInfo i = image(VK_FORMAT_R8G8B8A8_UNORM, 2048, 2048, 5, usage);
+   for (unsigned chip = 0; chip <= AGX_CHIP_UNKNOWN; ++chip)
+      CHECK(hk_m5_color_compression_eligible((enum agx_chip)chip, 0, &i) ==
+            agx_chip_g17_m5((enum agx_chip)chip));
+   i.mipLevels = 1;
+   CHECK(hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &i));
+   i.mipLevels = 12;
+   CHECK(hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &i));
+   VkImageCreateInfo c = i;
+   c.arrayLayers = 2;
+   CHECK(!hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+   c = i;
+   c.usage &= ~VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+   CHECK(!hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+   c = i;
+   c.format = VK_FORMAT_R32G32B32A32_SFLOAT;
+   CHECK(!hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+   c = i;
+   c.samples = VK_SAMPLE_COUNT_4_BIT;
+   c.mipLevels = 1;
+   CHECK(!hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+   /* One-byte texels compress only as a single level. */
+   static const VkFormat one_byte[] = {
+      VK_FORMAT_R8_UNORM, VK_FORMAT_R8_SNORM, VK_FORMAT_R8_UINT, VK_FORMAT_R8_SINT,
+   };
+   for (unsigned f = 0; f < sizeof(one_byte) / sizeof(one_byte[0]); ++f) {
+      c = i;
+      c.format = one_byte[f];
+      CHECK(!hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+      c.mipLevels = 1;
+      CHECK(hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+   }
+   c = i;
+   c.format = VK_FORMAT_R8G8_UNORM;
+   CHECK(hk_m5_color_compression_eligible(AGX_CHIP_G17S, 0, &c));
+}
+
+static void
+check_m5_depth_msaa(void)
+{
+   _Static_assert(DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION ==
+                     (UINT64_C(1) << 23), "depth capability allocation");
+   const uint64_t capability = DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION;
+   const uint64_t feature_sets[] = {0, UINT64_C(1) << 15,
+                                  UINT64_C(1) << 16, capability, UINT64_MAX};
+   const VkFormat formats[] = {VK_FORMAT_D32_SFLOAT,
+                               VK_FORMAT_D32_SFLOAT_S8_UINT};
+   const VkImageUsageFlags uses[] = {
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT,
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT,
+      VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+         VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+   };
+   for (unsigned f = 0; f < 2; ++f) {
+      for (unsigned n = 2; n <= 4; n *= 2) {
+         for (unsigned u = 0; u < 3; ++u) {
+            VkImageCreateInfo i = image(formats[f], 3024, 1964, 1, uses[u]);
+            i.samples = n;
+            for (unsigned c = 0; c < 5; ++c) {
+               for (unsigned options = 0; options < 4; ++options) {
+                  const bool primary = options & 1, transfer = options & 2;
+                  const bool capable = (feature_sets[c] & capability) != 0;
+                  CHECK(hk_m5_depth_msaa_compression_eligible(
+                           AGX_CHIP_G17S, 0, &i, feature_sets[c], primary) ==
+                        (capable && primary && u < 2));
+                  CHECK(hk_m5_depth_msaa_transfer_eligible(
+                           AGX_CHIP_G17S, 0, &i, feature_sets[c], primary,
+                           transfer) == (capable && primary && transfer && u == 2));
+               }
+            }
+         }
+      }
+   }
+
+   VkImageCreateInfo base = image(VK_FORMAT_D32_SFLOAT_S8_UINT, 17, 19, 1,
+                                 uses[2]);
+   base.samples = VK_SAMPLE_COUNT_4_BIT;
+   for (unsigned bad = 0; bad < 15; ++bad) {
+      VkImageCreateInfo i = base;
+      unsigned plane = 0;
+      switch (bad) {
+      case 0: i.samples = VK_SAMPLE_COUNT_1_BIT; break;
+      case 1: i.samples = VK_SAMPLE_COUNT_8_BIT; break;
+      case 2: i.format = VK_FORMAT_D16_UNORM; break;
+      case 3: i.extent.width = 15; break;
+      case 4: i.extent.height = 15; break;
+      case 5: i.extent.depth = 2; break;
+      case 6: i.arrayLayers = 2; break;
+      case 7: i.mipLevels = 2; break;
+      case 8: i.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; break;
+      case 9: i.flags = VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; break;
+      case 10: i.usage |= VK_IMAGE_USAGE_STORAGE_BIT; break;
+      case 11: i.usage |= VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT; break;
+      case 12: i.tiling = VK_IMAGE_TILING_LINEAR; break;
+      case 13: i.imageType = VK_IMAGE_TYPE_3D; break;
+      case 14: plane = 1; break;
+      }
+      CHECK(!hk_m5_depth_msaa_compression_eligible(AGX_CHIP_G17S, plane,
+                                                 &i, capability, true));
+      CHECK(!hk_m5_depth_msaa_transfer_eligible(AGX_CHIP_G17S, plane, &i,
+                                              capability, true, true));
+   }
+   VkImageStencilUsageCreateInfo stencil = {
+      .sType = VK_STRUCTURE_TYPE_IMAGE_STENCIL_USAGE_CREATE_INFO,
+      .stencilUsage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
+                      VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+                      VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+   };
+   base.pNext = &stencil;
+   CHECK(hk_m5_depth_msaa_transfer_eligible(AGX_CHIP_G17S, 0, &base,
+                                           capability, true, true));
+   stencil.stencilUsage |= VK_IMAGE_USAGE_SAMPLED_BIT;
+   CHECK(!hk_m5_depth_msaa_transfer_eligible(AGX_CHIP_G17S, 0, &base,
+                                            capability, true, true));
+   CHECK(!hk_m5_depth_msaa_transfer_eligible(AGX_CHIP_G17S, 0, NULL,
+                                            capability, true, true));
+}
+
+static void
+check_m5_msaa_diagnostic(void)
+{
+   const VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
+      VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
+      VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+   VkImageCreateInfo i = image(VK_FORMAT_R16G16B16A16_SFLOAT, 3024, 1964, 1,
+                               usage);
+   for (unsigned samples = 1; samples <= 8; samples *= 2) {
+      i.samples = (VkSampleCountFlagBits)samples;
+      for (unsigned enabled = 0; enabled < 8; enabled++) {
+         const bool expected = (samples == 2 || samples == 4) &&
+                               (enabled & samples);
+         CHECK(hk_m5_msaa_color_compression_eligible(
+                  AGX_CHIP_G17S, 0, &i, enabled) == expected);
+      }
+   }
+   i.samples = VK_SAMPLE_COUNT_4_BIT;
+   for (unsigned chip = 0; chip <= AGX_CHIP_UNKNOWN; chip++)
+      CHECK(hk_m5_msaa_color_compression_eligible(
+               (enum agx_chip)chip, 0, &i, 4) == (chip == AGX_CHIP_G17S));
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 1, &i, 4));
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, NULL, 4));
+   VkImageCreateInfo c = i;
+   c.pNext = &i;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   /* A mutable format is admitted here; hk_can_compress() then requires a
+    * format list of compatible views. Aliasing is not. */
+   c = i; c.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
+   CHECK(hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.flags = VK_IMAGE_CREATE_ALIAS_BIT;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.arrayLayers = 2;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.mipLevels = 2;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.format = VK_FORMAT_R8G8B8A8_UNORM;
+   CHECK(hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.format = VK_FORMAT_B8G8R8A8_SRGB;
+   CHECK(hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.format = VK_FORMAT_R32G32B32A32_SFLOAT;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.usage &= ~VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.extent.width = 15;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+   c = i; c.imageType = VK_IMAGE_TYPE_3D;
+   CHECK(!hk_m5_msaa_color_compression_eligible(AGX_CHIP_G17S, 0, &c, 4));
+}
+
 int
 main(void)
 {
@@ -474,6 +738,10 @@ main(void)
    check_minecraft();
    check_zink();
    check_shared();
+   check_m5_depth();
+   check_m5_color();
+   check_m5_msaa_diagnostic();
+   check_m5_depth_msaa();
    printf("G17P compression policy: %u checks passed\n", checks);
    return 0;
 }
diff --git a/src/asahi/vulkan/tests/test-hk-g17p-metadata-grid.c b/src/asahi/vulkan/tests/test-hk-g17p-metadata-grid.c
index 4b9e14dba67b..c60a7b89f036 100644
--- a/src/asahi/vulkan/tests/test-hk-g17p-metadata-grid.c
+++ b/src/asahi/vulkan/tests/test-hk-g17p-metadata-grid.c
@@ -137,9 +137,87 @@ check_mip_split(void)
    EXPECT(!hk_g17p_mip_level_through_pbe(1, 1), "1x1 should take compute");
 }
 
+/* A layer owns a complete metadata miptree, including 3D Z slots whose
+ * higher levels are unused. Samples expand X/Y, never the layer count. */
+static void
+check_layered_layout(uint32_t width, uint32_t height, unsigned layers,
+                     unsigned samples, unsigned levels, bool three_d)
+{
+   struct ail_layout layout = {
+      .width_px = width,
+      .height_px = height,
+      .depth_px = layers,
+      .sample_count_sa = samples,
+      .levels = levels,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R8G8B8A8_UNORM,
+      .compressed = true,
+      .renderable = true,
+      .mipmapped_z = three_d,
+   };
+   ail_make_miptree(&layout);
+
+   unsigned a = (width * (samples == 4 ? 2 : 1) + 15) & ~15u;
+   unsigned b = (height * (samples >= 2 ? 2 : 1) + 15) & ~15u;
+   uint64_t stride = 0;
+   unsigned allocated = 0;
+   for (unsigned l = 0; l < levels && MAX2(a, b) >= 16; ++l) {
+      const unsigned tw = MAX2(util_next_power_of_two(a), 16) / 16;
+      const unsigned th = MAX2(util_next_power_of_two(b), 16) / 16;
+      const uint64_t size = ((uint64_t)tw * th * 8 + 127) & ~UINT64_C(127);
+      EXPECT(layout.level_offsets_compressed_B[l] == stride,
+             "%ux%u, %ux, level %u: offset mismatch", width, height, samples,
+             l);
+      stride += size;
+      a = (a + 1) / 2;
+      b = (b + 1) / 2;
+      allocated++;
+   }
+   EXPECT(layout.compression_layer_stride_B == stride,
+          "%ux%u, %ux: metadata stride mismatch", width, height, samples);
+   EXPECT(layout.size_B == layout.metadata_offset_B + layers * stride,
+          "%ux%u, %ux, %u layers: metadata reserve mismatch", width, height,
+          samples, layers);
+   EXPECT(layers == 1 ||
+             (layout.page_aligned_layers && layout.layer_stride_B % 16384 == 0),
+          "renderable data layers must be page aligned");
+   for (unsigned l = 0; l < allocated; ++l) {
+      const unsigned valid_layers = three_d ? MAX2(layers >> l, 1) : layers;
+      const uint64_t level_offset = layout.level_offsets_compressed_B[l];
+      for (unsigned k = 0; k < valid_layers; ++k) {
+         const uint64_t address =
+            layout.metadata_offset_B + k * stride + level_offset;
+         EXPECT(address % 128 == 0 && address < layout.size_B,
+                "metadata layer/level outside allocation");
+         EXPECT(k == 0 || address >= layout.metadata_offset_B + k * stride,
+                "metadata layers overlap");
+      }
+   }
+}
+
+static void
+check_layered_layouts(void)
+{
+   static const unsigned sizes[][2] = {
+      {16, 16}, {17, 31}, {129, 127}, {300, 17}, {17, 4096}, {1920, 1080},
+   };
+   for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); ++i) {
+      const unsigned w = sizes[i][0], h = sizes[i][1];
+      for (unsigned samples = 1; samples <= 4; samples *= 2) {
+         check_layered_layout(w, h, 1, samples, 1, false);
+         check_layered_layout(w, h, 6, samples, 1, false);
+         check_layered_layout(w, h, 12, samples, 1, false);
+      }
+      const unsigned levels = util_logbase2(MAX2(w, h)) + 1;
+      check_layered_layout(w, h, 12, 1, levels, false);
+      check_layered_layout(w, h, 9, 1, levels, true);
+   }
+}
+
 int
 main(void)
 {
+   check_layered_layouts();
    check_mip_split();
    for (uint32_t h = 1; h <= 70; ++h) {
       for (uint32_t w = 1; w <= 70; ++w)
diff --git a/src/asahi/vulkan/tests/test-hk-g17p-zls.c b/src/asahi/vulkan/tests/test-hk-g17p-zls.c
new file mode 100644
index 000000000000..e51fcd6478df
--- /dev/null
+++ b/src/asahi/vulkan/tests/test-hk-g17p-zls.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#undef NDEBUG
+#include "asahi/lib/agx_helpers.h"
+#include "hk_drm_render_core.h"
+#include "hk_g17p_zls.h"
+
+#include <stdio.h>
+
+static unsigned checks;
+#define CHECK(x)                                                               \
+   do {                                                                        \
+      checks++;                                                                \
+      if (!(x)) {                                                              \
+         fprintf(stderr, "line %u: %s\n", __LINE__, #x);                       \
+         abort();                                                              \
+      }                                                                        \
+   } while (0)
+
+static struct ail_layout
+make_layout(enum pipe_format format, unsigned w, unsigned h, unsigned layers,
+            bool compressed, unsigned samples, bool mipmapped)
+{
+   struct ail_layout layout = {
+      .width_px = w,
+      .height_px = h,
+      .depth_px = layers,
+      .sample_count_sa = samples,
+      .levels = mipmapped ? util_logbase2(MAX2(w, h)) + 1 : 1,
+      .tiling = AIL_TILING_GPU,
+      .format = format,
+      .compressed = compressed,
+      .renderable = true,
+      .writeable_image = !compressed,
+   };
+   ail_make_miptree(&layout);
+   return layout;
+}
+
+/* Check the entire metadata reserve, including unused narrow-tail slots. */
+static void
+check_metadata(const struct ail_layout *layout)
+{
+   unsigned w =
+      ALIGN_POT(layout->width_px * (layout->sample_count_sa == 4 ? 2 : 1), 16);
+   unsigned h =
+      ALIGN_POT(layout->height_px * (layout->sample_count_sa >= 2 ? 2 : 1), 16);
+   uint64_t offset = 0;
+   for (unsigned l = 0; l < layout->levels; l++) {
+      const unsigned major = MAX2(
+         ALIGN_POT(layout->width_px * (layout->sample_count_sa == 4 ? 2 : 1),
+                   16),
+         ALIGN_POT(layout->height_px * (layout->sample_count_sa >= 2 ? 2 : 1),
+                   16));
+      if ((major >> l) < 16)
+         break;
+      CHECK(layout->level_offsets_compressed_B[l] == offset);
+      const unsigned tw = DIV_ROUND_UP(util_next_power_of_two(w), 16);
+      const unsigned th = DIV_ROUND_UP(util_next_power_of_two(h), 16);
+      offset += ALIGN_POT((uint64_t)tw * th * 8, 128);
+      w = DIV_ROUND_UP(w, 2);
+      h = DIV_ROUND_UP(h, 2);
+   }
+   CHECK(layout->compression_layer_stride_B == offset);
+   CHECK(layout->size_B ==
+         layout->metadata_offset_B + offset * layout->depth_px);
+}
+
+static struct ail_layout
+selected_layout(const struct ail_layout *layout,
+                const struct hk_g17p_zls_plane *state)
+{
+   struct ail_layout selected = *layout;
+   selected.compressed = state->meta != 0;
+   selected.tiling = state->tiling;
+   return selected;
+}
+
+/* All load/store combinations cover the first, partial and final stream
+ * words. Compression and tiling must not depend on those actions. */
+static void
+check_control(const struct ail_layout *z, const struct ail_layout *s)
+{
+   for (unsigned actions = 0; actions < 16; actions++) {
+      struct agx_zls args = {
+         .z_load = actions & 1,
+         .z_store = actions & 2,
+         .s_load = actions & 4,
+         .s_store = actions & 8,
+      };
+      struct agx_zls_control_packed packed;
+      agx_pack_zls_control(&packed, z, s, &args);
+      struct AGX_ZLS_CONTROL cfg;
+      CHECK(AGX_ZLS_CONTROL_unpack(NULL, (const uint8_t *)&packed, &cfg));
+      CHECK(cfg.z_load == (z && args.z_load));
+      CHECK(cfg.z_store == (z && args.z_store));
+      CHECK(cfg.s_load == (s && args.s_load));
+      CHECK(cfg.s_store == (s && args.s_store));
+      CHECK(cfg.z_load_compress == (z && z->compressed));
+      CHECK(cfg.z_store_compress == (z && z->compressed));
+      CHECK(cfg.s_load_compress == (s && s->compressed));
+      CHECK(cfg.s_store_compress == (s && s->compressed));
+      CHECK(cfg.z_load_tiling == (z && z->tiling == AIL_TILING_TWIDDLED));
+      CHECK(cfg.z_store_tiling == (z && z->tiling == AIL_TILING_TWIDDLED));
+      CHECK(cfg.s_load_tiling == (s && s->tiling == AIL_TILING_TWIDDLED));
+      CHECK(cfg.s_store_tiling == (s && s->tiling == AIL_TILING_TWIDDLED));
+      CHECK(cfg.z_format == (z && z->format == PIPE_FORMAT_Z16_UNORM
+                                ? AGX_ZLS_FORMAT_16
+                                : AGX_ZLS_FORMAT_32F));
+   }
+}
+
+static void
+check_plane(const struct ail_layout *layout, uint64_t base)
+{
+   if (layout->compressed)
+      check_metadata(layout);
+
+   for (unsigned l = 0; l < layout->levels; l++) {
+      const unsigned w = MAX2(layout->width_px >> l, 1);
+      const unsigned h = MAX2(layout->height_px >> l, 1);
+      const bool compressed =
+         layout->compressed &&
+         w * (layout->sample_count_sa == 4 ? 2 : 1) >= 16 &&
+         h * (layout->sample_count_sa >= 2 ? 2 : 1) >= 16;
+      for (unsigned k = 0; k < layout->depth_px; k++) {
+         struct hk_g17p_zls_plane state = hk_g17p_zls_plane(layout, base, k, l);
+         CHECK(state.width == w && state.height == h);
+         const uint32_t pixels = hk_drm_g17p_zls_pixels(
+            state.width, state.height, state.buffer, state.meta, 0, 0);
+         CHECK(pixels == (w - 1) + ((h - 1) << 15));
+         CHECK(!(pixels & ((1u << 14) | (1u << 29))));
+         CHECK(state.buffer ==
+               base + k * layout->layer_stride_B + layout->level_offsets_B[l]);
+         CHECK(state.buffer % 128 == 0);
+         CHECK(state.buffer < base + layout->metadata_offset_B ||
+               !layout->compressed);
+         const uint64_t stride = layout->layer_stride_B;
+         CHECK(state.stride ==
+               (stride >= 16384 ? (((stride / 16384) - 1) << 14) | 1 : 0));
+         CHECK(state.tiling == (layout->compressed && !compressed
+                                   ? AIL_TILING_TWIDDLED
+                                   : layout->tiling));
+         if (compressed) {
+            CHECK(state.meta == base + layout->metadata_offset_B +
+                                   k * layout->compression_layer_stride_B +
+                                   layout->level_offsets_compressed_B[l]);
+            CHECK(state.meta % 128 == 0);
+            CHECK(state.meta < base + layout->size_B);
+            CHECK(state.meta_stride ==
+                  (layout->compression_layer_stride_B / 128 - 1) << 14);
+            CHECK(state.meta + layout->compression_layer_stride_B ==
+                  base + layout->metadata_offset_B +
+                     (k + 1) * layout->compression_layer_stride_B +
+                     layout->level_offsets_compressed_B[l]);
+         } else {
+            CHECK(state.meta == 0 && state.meta_stride == 0);
+         }
+         struct ail_layout selected = selected_layout(layout, &state);
+         if (layout->format == PIPE_FORMAT_S8_UINT)
+            check_control(NULL, &selected);
+         else
+            check_control(&selected, NULL);
+      }
+   }
+}
+
+static void
+check_combined(unsigned w, unsigned h, unsigned layers)
+{
+   struct ail_layout z =
+      make_layout(PIPE_FORMAT_Z32_FLOAT, w, h, layers, true, 1, true);
+   struct ail_layout s =
+      make_layout(PIPE_FORMAT_S8_UINT, w, h, layers, true, 1, true);
+   const uint64_t zbase = UINT64_C(0x100000000);
+   const uint64_t sbase = zbase + ALIGN_POT(z.size_B, 16384);
+   CHECK(z.layer_stride_B >= s.layer_stride_B);
+   const struct hk_g17p_zls_plane absent = hk_g17p_zls_plane(&z, 0, 0, 0);
+   CHECK(absent.meta == 0 && absent.meta_stride == 0);
+   for (unsigned l = 0; l < z.levels; l++) {
+      for (unsigned k = 0; k < layers; k++) {
+         const struct hk_g17p_zls_plane zs = hk_g17p_zls_plane(&z, zbase, k, l);
+         const struct hk_g17p_zls_plane ss = hk_g17p_zls_plane(&s, sbase, k, l);
+         struct ail_layout zl = selected_layout(&z, &zs);
+         struct ail_layout sl = selected_layout(&s, &ss);
+         check_control(&zl, &sl);
+         CHECK(zs.buffer < sbase && ss.buffer >= sbase);
+         if (zs.meta)
+            CHECK(zs.meta < sbase && ss.meta >= sbase);
+
+         /* A missing metadata allocation on either plane clears only that
+          * aspect, regardless of the other aspect's admission. */
+         struct ail_layout missing = z;
+         missing.compression_layer_stride_B = 0;
+         struct hk_g17p_zls_plane ms = hk_g17p_zls_plane(&missing, zbase, k, l);
+         CHECK(ms.meta == 0 && ms.meta_stride == 0);
+         struct ail_layout ml = selected_layout(&missing, &ms);
+         check_control(&ml, &sl);
+         missing = s;
+         missing.compressed = false;
+         ms = hk_g17p_zls_plane(&missing, sbase, k, l);
+         CHECK(ms.meta == 0 && ms.meta_stride == 0);
+         ml = selected_layout(&missing, &ms);
+         check_control(&zl, &ml);
+      }
+   }
+}
+
+int
+main(void)
+{
+   static const unsigned sizes[][2] = {
+      {16, 16},  {17, 31},   {32, 32},     {33, 65},       {129, 127},
+      {300, 17}, {17, 4096}, {1920, 1080}, {16384, 16384},
+   };
+   static const unsigned layers[] = {1, 2, 6, 12};
+   static const enum pipe_format formats[] = {
+      PIPE_FORMAT_Z16_UNORM,
+      PIPE_FORMAT_Z32_FLOAT,
+      PIPE_FORMAT_S8_UINT,
+   };
+   for (unsigned i = 0; i < ARRAY_SIZE(sizes); i++) {
+      for (unsigned j = 0; j < ARRAY_SIZE(layers); j++) {
+         /* Keep the maximum-extent fixtures within maxResourceSize. */
+         if (sizes[i][0] == 16384 && layers[j] > 2)
+            continue;
+         for (unsigned f = 0; f < ARRAY_SIZE(formats); f++) {
+            struct ail_layout layout = make_layout(
+               formats[f], sizes[i][0], sizes[i][1], layers[j], true, 1, true);
+            check_plane(&layout, UINT64_C(0x100000000));
+            for (unsigned samples = 1; samples <= (sizes[i][0] <= 4096 ? 4 : 1);
+                 samples *= 2) {
+               layout = make_layout(formats[f], sizes[i][0], sizes[i][1],
+                                    layers[j], true, samples, false);
+               check_plane(&layout, UINT64_C(0x100000000));
+            }
+            layout = make_layout(formats[f], sizes[i][0], sizes[i][1],
+                                 layers[j], false, 1, true);
+            check_plane(&layout, UINT64_C(0x100000000));
+         }
+         check_combined(sizes[i][0], sizes[i][1], layers[j]);
+      }
+   }
+   printf("hk_g17p_zls: %u checks passed\n", checks);
+   return 0;
+}
diff --git a/src/asahi/vulkan/tests/test-hk-gs-ray-query.c b/src/asahi/vulkan/tests/test-hk-gs-ray-query.c
index 15641575177c..49f3da05275d 100644
--- a/src/asahi/vulkan/tests/test-hk-gs-ray-query.c
+++ b/src/asahi/vulkan/tests/test-hk-gs-ray-query.c
@@ -107,7 +107,8 @@ check_split(bool dynamic_count, bool instances)
    };
    nir_shader *count = NULL, *rast = NULL, *pre_gs = NULL;
    struct poly_gs_info info;
-   CHECK(hk_nir_lower_gs(b.shader, &count, &rast, &pre_gs, &info, &layout));
+   CHECK(hk_nir_lower_gs(b.shader, &count, &rast, &pre_gs, &info, &layout, false,
+                         NULL));
    CHECK(rast && rast->info.stage == MESA_SHADER_VERTEX);
    CHECK(!!count == dynamic_count);
    CHECK(check_variant(rast, true, true, instances));
@@ -189,7 +190,7 @@ check_atomic_guard(bool native, bool instances)
       .graphics_stage = true,
    };
    CHECK(hk_nir_lower_gs(b.shader, &count, &rast, &pre, &info,
-                        native ? &layout : NULL));
+                        native ? &layout : NULL, false, NULL));
    CHECK(!count);
    unsigned atomics = 0, queries = 0;
    nir_foreach_block(block, nir_shader_get_entrypoint(b.shader)) {
diff --git a/src/asahi/vulkan/tests/test-hk-native-descriptor.c b/src/asahi/vulkan/tests/test-hk-native-descriptor.c
index 7e29844a3bc4..b1c137de469e 100644
--- a/src/asahi/vulkan/tests/test-hk-native-descriptor.c
+++ b/src/asahi/vulkan/tests/test-hk-native-descriptor.c
@@ -8,6 +8,7 @@
 
 #include "hk_native_descriptor_pack.h"
 #include "asahi/lib/agx_abi.h"
+#include "asahi/genxml/agx3_cmdstream.h"
 #include "asahi/lib/agx_pbe_agx3_core.h"
 
 #include <assert.h>
@@ -469,6 +470,75 @@ check_layered_multisample_pbe(void)
    semantic.layers = 0;
    assert(!hk_pack_native_pbe(9, &packed, &semantic));
    semantic.layers = 12;
+   assert(hk_pack_native_pbe(10, &packed, &semantic));
+   memcpy(&word0, (const uint8_t *)packed.opaque, sizeof(word0));
+   memcpy(&word1, (const uint8_t *)packed.opaque + 8, sizeof(word1));
+   assert((word0 & 0xf) == 8);
+   assert(((word0 >> 58) & 3) == 0);
+   assert(((word1 >> 44) & 0xffff) == 11);
+}
+
+static void
+check_compressed_msaa_pbe(void)
+{
+   /* Independent offline constructor outputs, including the metadata
+    * extension, for a 3024x1964 RGBA16F surface. */
+   const uint64_t expected[2][4] = {
+      {UINT64_C(0x2003d58bcfe48ca4), UINT64_C(0x8000000001234500),
+       UINT64_C(0x4567800), 0},
+      {UINT64_C(0x2403d58bcfe48ca4), UINT64_C(0x8000000001234500),
+       UINT64_C(0x4567800), 0},
+   };
+   const uint64_t expected_read[2][4] = {
+      {UINT64_C(0x203d58bcf6888ca4), UINT64_C(0x8000000001234500),
+       UINT64_C(0x4567800), 0},
+      {UINT64_C(0x243d58bcf6888ca4), UINT64_C(0x8000000001234500),
+       UINT64_C(0x4567800), 0},
+   };
+   struct AGX_PBE semantic = {
+      AGX_PBE_header,
+      .dimension = AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED,
+      .layout = AGX_LAYOUT_GPU,
+      .channels = AGX_CHANNELS_R16G16B16A16,
+      .type = AGX_TEXTURE_TYPE_FLOAT,
+      .swizzle_r = AGX_CHANNEL_R, .swizzle_g = AGX_CHANNEL_G,
+      .swizzle_b = AGX_CHANNEL_B, .swizzle_a = AGX_CHANNEL_A,
+      .width = 3024, .height = 1964,
+      .buffer = UINT64_C(0x12345000),
+      .compressed = true, .extended = true,
+      .acceleration_buffer = UINT64_C(0x45678000),
+   };
+   struct hk_native_pbe_descriptor packed;
+   for (unsigned i = 0; i < 2; i++) {
+      semantic.samples = i ? AGX_SAMPLE_COUNT_4 : AGX_SAMPLE_COUNT_2;
+      assert(hk_pack_native_pbe(10, &packed, &semantic));
+      assert(!memcmp(&packed, expected[i], sizeof(expected[i])));
+      assert(hk_pack_native_pbe(9, &packed, &semantic));
+      const struct AGX_TEXTURE read = {
+         AGX_TEXTURE_header,
+         .dimension = AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED,
+         .layout = AGX_LAYOUT_GPU,
+         .channels = AGX_CHANNELS_R16G16B16A16,
+         .type = AGX_TEXTURE_TYPE_FLOAT,
+         .swizzle_r = AGX_CHANNEL_R, .swizzle_g = AGX_CHANNEL_G,
+         .swizzle_b = AGX_CHANNEL_B, .swizzle_a = AGX_CHANNEL_A,
+         .width = 3024, .height = 1964, .depth = 1,
+         .address = UINT64_C(0x12345000),
+         .compressed = true, .extended = true,
+         .acceleration_buffer = UINT64_C(0x45678000),
+         .samples = i ? AGX_SAMPLE_COUNT_4 : AGX_SAMPLE_COUNT_2,
+      };
+      struct hk_native_texture_descriptor read_packed;
+      assert(hk_pack_native_texture(10, &read_packed, &read));
+      assert(!memcmp(&read_packed, expected_read[i], sizeof(expected_read[i])));
+   }
+   semantic.samples = AGX_SAMPLE_COUNT_8;
+   assert(!hk_pack_native_pbe(10, &packed, &semantic));
+   semantic.samples = AGX_SAMPLE_COUNT_4;
+   semantic.acceleration_buffer = 0;
+   assert(!hk_pack_native_pbe(10, &packed, &semantic));
+   semantic.acceleration_buffer = UINT64_C(0x45678000);
+   semantic.dimension = AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED;
    assert(!hk_pack_native_pbe(10, &packed, &semantic));
 }
 
@@ -494,7 +564,28 @@ check_captured_rgba8_lossless_pbe(void)
    struct hk_native_pbe_descriptor actual;
    assert(hk_pack_native_pbe(9, &actual, &p));
    assert(memcmp(&actual, expected, sizeof(expected)) == 0);
-   assert(!hk_pack_native_pbe(10, &actual, &p));
+   const uint64_t expected10[4] = {
+      UINT64_C(0x2001ff83ffe40a22), UINT64_C(0x8000001000008000),
+      UINT64_C(0x0000001000048000), 0,
+   };
+   assert(hk_pack_native_pbe(10, &actual, &p));
+   assert(memcmp(&actual, expected10, sizeof(expected10)) == 0);
+   /* Pinned native Gen10 PBE encoder (offline execution, compression-oracle
+    * native_pbe.py): RGBA8 UNORM 64x64, level 2 of 5, compressed. */
+   {
+      struct AGX_PBE m = p;
+      m.width = m.height = 64;
+      m.buffer = UINT64_C(0x12345000);
+      m.acceleration_buffer = UINT64_C(0x45678000);
+      m.level = 2;
+      m.levels = 5;
+      const uint64_t native_mip10[4] = {
+         UINT64_C(0x30001f803fe40a22), UINT64_C(0x8000000001234500),
+         UINT64_C(0x0004200004567800), 0,
+      };
+      assert(hk_pack_native_pbe(10, &actual, &m));
+      assert(memcmp(&actual, native_mip10, sizeof(native_mip10)) == 0);
+   }
    const struct AGX_PBE original = p;
 #define REJECT_COMPRESSED(change) do { p = original; change; \
    assert(!hk_pack_native_pbe(9, &actual, &p)); } while (0)
@@ -510,9 +601,211 @@ check_captured_rgba8_lossless_pbe(void)
 #undef REJECT_COMPRESSED
 }
 
+static void
+check_sparse_translation(void)
+{
+   const struct AGX_TEXTURE texture = {
+      AGX_TEXTURE_header,
+      .dimension = AGX_TEXTURE_DIMENSION_2D,
+      .layout = AGX_LAYOUT_GPU,
+      .channels = AGX_CHANNELS_R32,
+      .type = AGX_TEXTURE_TYPE_UINT,
+      .width = 128,
+      .height = 64,
+      .address = 0x80000,
+      .mode = AGX_IMAGE_MODE_SPARSE,
+   };
+   struct hk_native_texture_descriptor packed;
+   assert(hk_pack_native_texture(10, &packed, &texture));
+   struct AGX_TEXTURE_GEN10 gen10;
+   assert(AGX_TEXTURE_GEN10_unpack(NULL, (const uint8_t *)packed.opaque,
+                                   &gen10));
+   /* The native HAL300 texture descriptor encoder: enabled with
+    * page size 0x4000 maps to selector1, not the legacy sparse mode3. */
+   assert(gen10.image_mode_enable);
+   assert(gen10.image_mode == 1);
+   assert((gen10.address & ~UINT64_C(0x30)) == texture.address);
+
+   assert(hk_pack_native_texture(9, &packed, &texture));
+   struct AGX_TEXTURE_GEN9 gen9;
+   assert(AGX_TEXTURE_GEN9_unpack(NULL, (const uint8_t *)packed.opaque,
+                                  &gen9));
+   assert((gen9.state_control & 3) == AGX_IMAGE_MODE_SPARSE);
+
+   /* Both native HALs emit this exact sparse-table maintenance command. */
+   const uint8_t expected[] = {0x02, 0x41, 0x02, 0x60};
+   const enum agx_chip chips[] = {
+      AGX_CHIP_G17P, AGX_CHIP_G17G, AGX_CHIP_G17S,
+   };
+   for (unsigned i = 0; i < ARRAY_SIZE(chips); ++i) {
+      uint8_t stream[AGX3_CDM_SSTT_FLUSH_INVALIDATE_SIZE];
+      assert(agx3_cdm_sstt_flush_invalidate_pack(chips[i], stream));
+      assert(memcmp(stream, expected, sizeof(expected)) == 0);
+      assert(agx3_cdm_sstt_flush_invalidate_validate(chips[i], stream,
+                                                   sizeof(stream)));
+      assert(!agx3_cdm_sstt_flush_invalidate_validate(chips[i], stream, 3));
+   }
+}
+
+/* Check each field independently of the packer's dimension dispatch. The
+ * colour codec and footprint remain zero for every lossless shape. */
+static void
+check_lossless_layered_packets(void)
+{
+   static const struct {
+      enum agx_texture_dimension dimension;
+      unsigned encoded_dimension;
+      unsigned layers;
+      unsigned samples;
+   } cases[] = {
+      {AGX_TEXTURE_DIMENSION_2D, 2, 1, 1},
+      {AGX_TEXTURE_DIMENSION_2D_ARRAY, 3, 1, 1},
+      {AGX_TEXTURE_DIMENSION_2D_ARRAY, 3, 12, 1},
+      {AGX_TEXTURE_DIMENSION_2D_ARRAY, 3, 65536, 1},
+      {AGX_TEXTURE_DIMENSION_3D, 5, 9, 1},
+      {AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED, 4, 1, 2},
+      {AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED, 4, 1, 4},
+      {AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED, 8, 12, 2},
+      {AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED, 8, 12, 4},
+   };
+   for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) {
+      const bool layered = cases[i].encoded_dimension == 3 ||
+                           cases[i].encoded_dimension == 5 ||
+                           cases[i].encoded_dimension == 8;
+      struct AGX_PBE p = {
+         AGX_PBE_header,
+         .dimension = cases[i].dimension,
+         .layout = AGX_LAYOUT_GPU,
+         .channels = AGX_CHANNELS_R8G8B8A8,
+         .type = AGX_TEXTURE_TYPE_UNORM,
+         .swizzle_r = AGX_CHANNEL_R,
+         .swizzle_g = AGX_CHANNEL_G,
+         .swizzle_b = AGX_CHANNEL_B,
+         .swizzle_a = AGX_CHANNEL_A,
+         .width = 16384,
+         .height = 127,
+         .buffer = UINT64_C(0x123456789ab0),
+         .acceleration_buffer = UINT64_C(0x23456789abc0),
+         .layers = cases[i].layers,
+         .samples =
+            cases[i].samples == 4 ? AGX_SAMPLE_COUNT_4 : AGX_SAMPLE_COUNT_2,
+         .compressed = true,
+         .extended = true,
+         .page_aligned_layers = true,
+      };
+      struct hk_native_pbe_descriptor packed;
+      assert(hk_pack_native_pbe(9, &packed, &p));
+      uint64_t w[4];
+      memcpy(w, packed.opaque, sizeof(w));
+      assert((w[0] & 15) == cases[i].encoded_dimension);
+      assert(((w[0] >> 4) & 3) == 2);
+      assert(((w[0] >> 24) & 0x3fff) == p.width - 1);
+      assert(((w[0] >> 38) & 0x3fff) == p.height - 1);
+      assert(((w[0] >> 56) & 3) == (cases[i].samples == 4));
+      assert((w[0] >> 59) == 1);
+      assert((w[1] & ((UINT64_C(1) << 44) - 1)) == p.buffer >> 4);
+      assert(((w[1] >> 44) & 0xffff) == p.layers - 1);
+      assert(((w[1] >> 60) & 1) == layered);
+      assert((w[1] >> 61) == 4);
+      /* GPU-tiled compression has no explicit metadata layer stride and
+       * does not combine the secondary stride and mip-range unions. */
+      assert(w[2] == p.acceleration_buffer >> 4);
+      assert(w[3] == 0);
+      assert(!hk_pack_native_pbe(10, &packed, &p));
+
+      const struct AGX_PBE original = p;
+#define REFUSE(change)                                                         \
+   do {                                                                        \
+      p = original;                                                            \
+      change;                                                                  \
+      memset(&packed, 0xff, sizeof(packed));                                   \
+      assert(!hk_pack_native_pbe(9, &packed, &p));                             \
+      assert(memcmp(&packed, &(struct hk_native_pbe_descriptor){0},            \
+                    sizeof(packed)) == 0);                                     \
+   } while (0)
+      REFUSE(p.acceleration_buffer = 0);
+      REFUSE(p.acceleration_buffer++);
+      REFUSE(p.acceleration_buffer = UINT64_C(1) << 48);
+      REFUSE(p.buffer++);
+      REFUSE(p.buffer = UINT64_C(1) << 48);
+      REFUSE(p.extended = false);
+      REFUSE(p.samples = AGX_SAMPLE_COUNT_8);
+      REFUSE(p.layout = AGX_LAYOUT_TWIDDLED);
+      if (layered) {
+         REFUSE(p.layers = 0);
+         REFUSE(p.layers = 65537);
+         REFUSE(p.levels = 2);
+      }
+#undef REFUSE
+   }
+
+   static const enum agx_texture_dimension dimensions[] = {
+      AGX_TEXTURE_DIMENSION_2D_ARRAY,
+      AGX_TEXTURE_DIMENSION_CUBE,
+      AGX_TEXTURE_DIMENSION_CUBE_ARRAY,
+      AGX_TEXTURE_DIMENSION_3D,
+      AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED,
+      AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED,
+   };
+   for (unsigned i = 0; i < sizeof(dimensions) / sizeof(dimensions[0]); ++i) {
+      const bool ms =
+         dimensions[i] == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED ||
+         dimensions[i] == AGX_TEXTURE_DIMENSION_2D_ARRAY_MULTISAMPLED;
+      struct AGX_TEXTURE t = {
+         AGX_TEXTURE_header,
+         .dimension = dimensions[i],
+         .layout = AGX_LAYOUT_GPU,
+         .channels = AGX_CHANNELS_R8G8B8A8,
+         .type = AGX_TEXTURE_TYPE_UINT,
+         .width = 256,
+         .height = 128,
+         /* Cubes count cubes here; the allocation counts physical faces. */
+         .depth =
+            dimensions[i] == AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED ? 1 : 2,
+         .address = UINT64_C(0x123456789ab0),
+         .acceleration_buffer = UINT64_C(0x23456789abc0),
+         .compressed = true,
+         .extended = true,
+         .page_aligned_layers = true,
+         .samples = ms ? AGX_SAMPLE_COUNT_4 : AGX_SAMPLE_COUNT_2,
+      };
+      struct hk_native_texture_descriptor packed;
+      assert(hk_pack_native_texture(9, &packed, &t));
+      struct AGX_TEXTURE_GEN9 read;
+      assert(
+         AGX_TEXTURE_GEN9_unpack(NULL, (const uint8_t *)packed.opaque, &read));
+      assert(read.dimension == t.dimension);
+      assert(read.width == t.width && read.height == t.height);
+      assert(read.depth == t.depth);
+      assert(read.samples == t.samples);
+      assert(read.compressed && read.extended);
+      assert(read.address == t.address);
+      assert(read.acceleration_buffer == t.acceleration_buffer);
+      assert(read.page_aligned_layers);
+      uint64_t w[4];
+      memcpy(w, packed.opaque, sizeof(w));
+      assert(((w[2] >> 60) & 3) == 0 && w[3] == 0);
+      if (ms) {
+         t.samples = AGX_SAMPLE_COUNT_8;
+         assert(!hk_pack_native_texture(9, &packed, &t));
+      } else {
+         t.first_level = 1;
+         t.last_level = 3;
+         t.unk_mipmapped = true;
+         assert(hk_pack_native_texture(9, &packed, &t));
+         memcpy(w, packed.opaque, sizeof(w));
+         assert(((w[2] >> 44) & 0xffff) == 0x31);
+         assert((w[2] & ((UINT64_C(1) << 44) - 1)) ==
+                t.acceleration_buffer >> 4);
+      }
+   }
+}
+
 int
 main(void)
 {
+   check_sparse_translation();
+   check_lossless_layered_packets();
    check_buffer_metadata();
    check_layered_multisample_pbe();
    uint64_t atomic_sw = 0;
@@ -624,6 +917,8 @@ main(void)
 
    excluded = plain;
    excluded.compressed = true;
+   excluded.extended = true;
+   excluded.acceleration_buffer = UINT64_C(0x100000a0000);
    check_gen9_layer_mode_preserved(excluded);
 
    excluded = plain;
@@ -754,6 +1049,28 @@ main(void)
    assert(hk_pack_native_texture(10, &native_packed, &semantic));
    assert(memcmp(&native_packed, m5_texture, sizeof(m5_texture)) == 0);
 
+   /* Pinned native texture encoder (offline execution, compression-oracle
+    * native_read.py): a compressed 64x64 RGBA8 mip chain viewed from level 2
+    * with 5 levels takes the level-0 data and metadata addresses (the
+    * metadata address query passes level 0), word0 bit60 and
+    * first/last level 2/6 in word2. */
+   {
+      struct AGX_TEXTURE mip = semantic;
+      mip.width = mip.height = 64;
+      mip.address = UINT64_C(0x12345000);
+      mip.acceleration_buffer = UINT64_C(0x45678000);
+      mip.unk_mipmapped = true;
+      mip.first_level = 2;
+      mip.last_level = 6;
+      const uint64_t native_mip_texture[4] = {
+         UINT64_C(0x3001f803f6880a22), UINT64_C(0x8000000001234500),
+         UINT64_C(0x0006200004567800), 0,
+      };
+      assert(hk_pack_native_texture(10, &native_packed, &mip));
+      assert(memcmp(&native_packed, native_mip_texture,
+                    sizeof(native_mip_texture)) == 0);
+   }
+
    enum agx_sample_count samples;
    assert(hk_translate_native_texture_sample_count(9, 2, &samples));
    assert(samples == AGX_SAMPLE_COUNT_2);
@@ -766,6 +1083,8 @@ main(void)
    assert(!hk_translate_native_texture_sample_count(10, 1, &samples));
 
    semantic.dimension = AGX_TEXTURE_DIMENSION_2D_MULTISAMPLED;
+   semantic.compressed = semantic.extended = false;
+   semantic.acceleration_buffer = 0;
    static const struct {
       enum agx_sample_count samples;
       unsigned selector;
@@ -929,8 +1248,19 @@ main(void)
       assert(memcmp(&native_pbe, compressed_color_targets[i].words,
                     sizeof(native_pbe)) == 0);
    }
+   /* The same atlas on Gen10, from the pinned native PBE encoder (offline
+    * execution, compression-oracle native_pbe.py, levels = 5 at stack +0x48,
+    * level at +0x44): the compressed single-level words plus the chain. */
+   for (unsigned level = 0; level < 5; ++level) {
+      const uint64_t gen10_atlas[4] = {
+         0x3003ff87ffe40a22ull, 0x8000001000000000ull,
+         0x000400100000a000ull | ((uint64_t)level << 44), 0,
+      };
+      compressed_atlas.level = level;
+      assert(hk_pack_native_pbe(10, &native_pbe, &compressed_atlas));
+      assert(memcmp(&native_pbe, gen10_atlas, sizeof(gen10_atlas)) == 0);
+   }
    compressed_atlas.level = 0;
-   assert(!hk_pack_native_pbe(10, &native_pbe, &compressed_atlas));
    compressed_atlas.acceleration_buffer = 0;
    assert(!hk_pack_native_pbe(9, &native_pbe, &compressed_atlas));
    compressed_atlas.acceleration_buffer = 0x100000a0001ull;
@@ -1472,7 +1802,7 @@ main(void)
    assert(hk_pack_native_pbe(9, &native_pbe, &pbe));
    assert(memcmp(&native_pbe, gen9_array_pbe,
                  sizeof(gen9_array_pbe)) == 0);
-   assert(!hk_pack_native_pbe(10, &native_pbe, &pbe));
+   assert(hk_pack_native_pbe(10, &native_pbe, &pbe));
    /* A one-layer array keeps dimension 3 with a zero layer field. */
    pbe.layers = 1;
    uint8_t expected_layered[32];
@@ -1489,7 +1819,7 @@ main(void)
    assert(hk_pack_native_pbe(9, &native_pbe, &pbe));
    assert(memcmp(&native_pbe, expected_layered,
                  sizeof(expected_layered)) == 0);
-   assert(!hk_pack_native_pbe(10, &native_pbe, &pbe));
+   assert(hk_pack_native_pbe(10, &native_pbe, &pbe));
    /* A level of a mipmapped 3D or array texture keeps the whole chain's
     * geometry and adds the native mip extension: word0 bit58, word1 bit63,
     * level at word2[47:44] and levels-1 at word2[51:48]. */
@@ -1529,12 +1859,13 @@ main(void)
    pbe.level = 0;
    CHECK_PBE_REJECT(pbe.levels = 17);
    pbe.levels = 1;
-   /* Gen9 converts stores to sRGB with word1 bit 61; Gen10's field is not
-    * recovered. */
+   /* Gen9 and Gen10 convert stores to sRGB with word1 bit 61 (Gen10:
+    * hardware sweep, port/srgbsweep). */
    pbe.srgb = true;
    assert(hk_pack_native_pbe(9, &native_pbe, &pbe));
    assert(((const uint8_t *)&native_pbe)[15] & 0x20);
-   assert(!hk_pack_native_pbe(10, &native_pbe, &pbe));
+   assert(hk_pack_native_pbe(10, &native_pbe, &pbe));
+   assert(((const uint8_t *)&native_pbe)[15] & 0x20);
    pbe.srgb = false;
    CHECK_PBE_REJECT(pbe.mode = AGX_IMAGE_MODE_SPARSE);
    pbe.mode = AGX_IMAGE_MODE_NORMAL;
@@ -1562,6 +1893,75 @@ main(void)
    assert(memcmp(&null_pbe10, &(struct hk_native_pbe_descriptor){0},
                  sizeof(null_pbe10)) != 0);
 
+   {
+      const struct AGX_PBE linear = {
+         AGX_PBE_header,
+         .dimension = AGX_TEXTURE_DIMENSION_2D,
+         .layout = AGX_LAYOUT_LINEAR,
+         .channels = AGX_CHANNELS_R8G8B8A8,
+         .type = AGX_TEXTURE_TYPE_UNORM,
+         .swizzle_r = AGX_CHANNEL_R, .swizzle_g = AGX_CHANNEL_G,
+         .swizzle_b = AGX_CHANNEL_B, .swizzle_a = AGX_CHANNEL_A,
+         .width = 1920, .height = 1080,
+         .buffer = UINT64_C(0x5ffdf6c000), .stride = 7680 - 4,
+      };
+      const uint8_t expected[32] = {
+         0x02,0x0a,0xe4,0x7f,0x87,0x1b,0x02,0x00,
+         0x00,0x6c,0xdf,0xff,0x05,0xf0,0x1d,0x00,
+      };
+      struct hk_native_pbe_descriptor packed;
+      assert(hk_pack_native_pbe(10, &packed, &linear));
+      assert(!memcmp(&packed, expected, sizeof(expected)));
+      struct AGX_PBE oversized = linear;
+      oversized.stride = (1u << 20) + 16 - 4;
+      assert(!hk_pack_native_pbe(10, &packed, &oversized));
+   }
+   {
+      /* Pinned native Gen10 PBE and texture encoder offline oracle, 64x64
+       * RGBA8, one sample/layer/mip, data=0x12345000, aux=0x45678000.
+       * Gen10 moves the PBE compression flag from Gen9 bit59 to bit61. */
+      const uint64_t expected_pbe[4] = {
+         UINT64_C(0x20001f803fe40a22), UINT64_C(0x8000000001234500),
+         UINT64_C(0x4567800), 0,
+      };
+      struct AGX_PBE semantic = {
+         AGX_PBE_header,
+         .dimension = AGX_TEXTURE_DIMENSION_2D,
+         .layout = AGX_LAYOUT_GPU,
+         .channels = AGX_CHANNELS_R8G8B8A8,
+         .type = AGX_TEXTURE_TYPE_UNORM,
+         .swizzle_r = AGX_CHANNEL_R, .swizzle_g = AGX_CHANNEL_G,
+         .swizzle_b = AGX_CHANNEL_B, .swizzle_a = AGX_CHANNEL_A,
+         .width = 64, .height = 64, .buffer = UINT64_C(0x12345000),
+         .compressed = true, .extended = true,
+         .acceleration_buffer = UINT64_C(0x45678000),
+      };
+      struct hk_native_pbe_descriptor packed;
+      assert(hk_pack_native_pbe(10, &packed, &semantic));
+      assert(!memcmp(&packed, expected_pbe, sizeof(expected_pbe)));
+      struct agx_pbe_agx3_2d_fields decoded;
+      assert(agx_pbe_agx3_2d_unpack(10, &packed, sizeof(packed), &decoded));
+      assert(decoded.compressed &&
+             decoded.acceleration_buffer == semantic.acceleration_buffer);
+      semantic.buffer += 16;
+      assert(!hk_pack_native_pbe(10, &packed, &semantic));
+      semantic.buffer -= 16;
+      semantic.page_aligned_layers = true;
+      assert(!hk_pack_native_pbe(10, &packed, &semantic));
+      semantic.page_aligned_layers = false;
+      semantic.dimension = AGX_TEXTURE_DIMENSION_2D_ARRAY;
+      assert(hk_pack_native_pbe(10, &packed, &semantic));
+      uint64_t array_expected[4];
+      memcpy(array_expected, expected_pbe, sizeof(array_expected));
+      array_expected[0] |= 1;
+      assert(!memcmp(&packed, array_expected, sizeof(array_expected)));
+      semantic.layers = 2;
+      assert(!hk_pack_native_pbe(10, &packed, &semantic));
+      semantic.layers = 1;
+      semantic.levels = 2;
+      assert(!hk_pack_native_pbe(10, &packed, &semantic));
+   }
+   check_compressed_msaa_pbe();
    check_captured_rgba8_lossless_pbe();
    puts("honeykrisp native descriptor geometry/policy: passed");
    return 0;
diff --git a/src/asahi/vulkan/tests/test-hk-native-sampler.c b/src/asahi/vulkan/tests/test-hk-native-sampler.c
index c4ff704a3389..a00e59abfee6 100644
--- a/src/asahi/vulkan/tests/test-hk-native-sampler.c
+++ b/src/asahi/vulkan/tests/test-hk-native-sampler.c
@@ -55,9 +55,65 @@ check_chip(enum agx_chip chip, enum agx_sampler_generation generation,
    }
 }
 
+/* Full sampler words from the native HAL300 Gen5 constructor at
+ * macOS 26A428, with normalized
+ * coordinates, linear min/mag filters, no mip filter and clamp-to-edge.
+ * Exercise reduction alongside both LOD bias signs and anisotropy. */
+static void
+check_native_minmax(void)
+{
+   static const struct {
+      unsigned reduction, anisotropy;
+      float bias;
+      uint64_t word;
+   } cases[] = {
+      {1, 1, 0.00f, UINT64_C(0x00000080028e0000)},
+      {1, 4, 0.00f, UINT64_C(0x0000008002ae0000)},
+      {1, 16, 0.00f, UINT64_C(0x0000008002ce0000)},
+      {1, 1, 1.50f, UINT64_C(0x00030080028e0000)},
+      {1, 4, 1.50f, UINT64_C(0x0003008002ae0000)},
+      {1, 16, 1.50f, UINT64_C(0x0003008002ce0000)},
+      {1, 1, -2.25f, UINT64_C(0x003b8080028e0000)},
+      {1, 4, -2.25f, UINT64_C(0x003b808002ae0000)},
+      {1, 16, -2.25f, UINT64_C(0x003b808002ce0000)},
+      {2, 1, 0.00f, UINT64_C(0x00000100028e0000)},
+      {2, 4, 0.00f, UINT64_C(0x0000010002ae0000)},
+      {2, 16, 0.00f, UINT64_C(0x0000010002ce0000)},
+      {2, 1, 1.50f, UINT64_C(0x00030100028e0000)},
+      {2, 4, 1.50f, UINT64_C(0x0003010002ae0000)},
+      {2, 16, 1.50f, UINT64_C(0x0003010002ce0000)},
+      {2, 1, -2.25f, UINT64_C(0x003b8100028e0000)},
+      {2, 4, -2.25f, UINT64_C(0x003b810002ae0000)},
+      {2, 16, -2.25f, UINT64_C(0x003b810002ce0000)},
+   };
+   for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) {
+      struct hk_native_sampler_state state = {
+         .maximum_lod = 14.0f,
+         .maximum_anisotropy = cases[i].anisotropy,
+         .lod_bias = cases[i].bias,
+         .magnify = AGX_FILTER_LINEAR,
+         .minify = AGX_FILTER_LINEAR,
+         .reduction_mode = (enum hk_sampler_reduction_mode)cases[i].reduction,
+      };
+      struct hk_native_sampler_descriptor out;
+      assert(hk_pack_native_sampler(AGX_CHIP_G17G, &out, &state));
+      uint64_t word;
+      memcpy(&word, out.opaque, sizeof(word));
+      assert(word == cases[i].word);
+
+      /* hk_nir_lower_descriptors' Gen10 compute LOD query reads the
+       * descriptor's copy of this word: bias is signed 5:6 at bits 43..53
+       * of the high half (bits 11..21), and max LOD is 4:6 at bits 10..19. */
+      const uint32_t high = (uint32_t)(word >> 32);
+      assert((int32_t)(high << 10) >> 21 == (int32_t)(cases[i].bias * 64));
+      assert(((uint32_t)word >> 10 & 0x3ff) == 14 * 64);
+   }
+}
+
 int
 main(void)
 {
+   check_native_minmax();
    check_chip(AGX_CHIP_G13G, AGX_SAMPLER_GENERATION_GEN4, 8);
    check_chip(AGX_CHIP_G13X, AGX_SAMPLER_GENERATION_GEN4, 8);
    check_chip(AGX_CHIP_G14G, AGX_SAMPLER_GENERATION_GEN4, 8);
@@ -98,6 +154,14 @@ main(void)
    for (unsigned i = sizeof(expected_hal200); i < sizeof(g17p); ++i)
       assert(((const uint8_t *)&g17p)[i] == 0);
    assert(memcmp(&gen5, expected_gen5, sizeof(expected_gen5)) == 0);
+   {
+      /* The fields the Gen10 compute LOD query decodes from this word. */
+      const uint32_t lo = gen5.opaque[0], hi = gen5.opaque[1];
+      assert((lo & 0x3ff) == 64);                             /* min LOD 1 */
+      assert((lo >> 10 & 0x3ff) == 15 * 64);       /* max LOD 100 -> 15 */
+      assert((lo >> 27 & 3) == AGX_MIP_FILTER_LINEAR);
+      assert((int32_t)(hi << 10) >> 21 == -65);     /* bias -1.015625 */
+   }
 
    /* A disabled compare must retain the exact mature Gen4 bytes: compare
     * function bits were historically left at their header default. */
diff --git a/src/asahi/vulkan/tests/test-hk-query-copy.c b/src/asahi/vulkan/tests/test-hk-query-copy.c
index e99a58f89699..0c3c62a9d7d6 100644
--- a/src/asahi/vulkan/tests/test-hk-query-copy.c
+++ b/src/asahi/vulkan/tests/test-hk-query-copy.c
@@ -51,7 +51,7 @@ test_flags(unsigned reports, bool timestamp, bool indexed)
          unsigned q = i + 1;
          libagx_copy_query_g17p_result(
             timestamp ? NULL : availability, values, indexed ? remap : NULL,
-            (uintptr_t)output + 16, stride, 1, flags, reports, i);
+            (uintptr_t)output + 16, stride, 1, flags, reports, 0, i);
 
          uint8_t *dst = (uint8_t *)expected + 16 + i * stride;
          if (availability[q] || (flags & VK_QUERY_RESULT_PARTIAL_BIT)) {
@@ -110,7 +110,7 @@ test_wait(bool timestamp)
       VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT |
          VK_QUERY_RESULT_WAIT_BIT |
          (timestamp ? 0 : VK_QUERY_RESULT_PARTIAL_BIT),
-      timestamp ? 1 : 2, 0);
+      timestamp ? 1 : 2, 0, 0);
    alarm(0);
    assert(p->output[0] == UINT64_C(0x1122334455667788));
    if (!timestamp)
@@ -122,6 +122,34 @@ test_wait(bool timestamp)
    assert(munmap(p, sizeof(*p)) == 0);
 }
 
+static void
+test_stamp_slots(void)
+{
+   /* 64-bit availability slots: a retirement timestamp may have either half
+    * zero, the queued store writes 1 to the low half, a reset writes both. */
+   uint64_t availability[] = {0, 1, UINT64_C(0x100000000),
+                              UINT64_C(0x1234567800000000), 0,
+                              UINT64_C(0x00000000deadbeef)};
+   uint64_t values[6];
+   for (unsigned q = 0; q < 6; ++q)
+      values[q] = 1000 + q;
+
+   for (unsigned q = 0; q < 6; ++q) {
+      uint64_t output[2];
+      memset(output, 0xa5, sizeof(output));
+      libagx_copy_query_g17p_result(
+         (uint32_t *)availability, values, NULL, (uintptr_t)output, 0, q,
+         VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT, 1, 1,
+         0);
+      const bool available = availability[q] != 0;
+      assert(output[1] == available);
+      if (available)
+         assert(output[0] == values[q]);
+      else
+         assert(output[0] == UINT64_C(0xa5a5a5a5a5a5a5a5));
+   }
+}
+
 static void
 test_large_stride(void)
 {
@@ -138,7 +166,8 @@ test_large_stride(void)
    for (unsigned i = 0; i < 3; ++i) {
       libagx_copy_query_g17p_result(
          NULL, values, NULL, (uintptr_t)dst, stride, 0,
-         VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT, 1, i);
+         VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT, 1, 0,
+         i);
       assert(*(uint64_t *)(dst + i * stride) == values[i]);
       assert(*(uint64_t *)(dst + i * stride + 8) == 1);
    }
@@ -154,8 +183,10 @@ main(void)
    test_flags(1, true, false);   /* Timestamp sentinel availability. */
    test_wait(false);
    test_wait(true);
+   test_stamp_slots();
    test_large_stride();
    puts(
-      "query range copies: flags, remaps, tuples, WAIT and 64-bit strides passed");
+      "query range copies: flags, remaps, tuples, WAIT, stamp slots and 64-bit "
+      "strides passed");
    return 0;
 }
diff --git a/src/asahi/vulkan/tests/test-hk-query-oq.c b/src/asahi/vulkan/tests/test-hk-query-oq.c
new file mode 100644
index 000000000000..b6d914c9b5d1
--- /dev/null
+++ b/src/asahi/vulkan/tests/test-hk-query-oq.c
@@ -0,0 +1,554 @@
+/* SPDX-License-Identifier: MIT
+ * Calls the real deferred occlusion-availability resolver (hk_query_oq.c) and
+ * hk_sync on mock control streams; no GPU or DRM calls.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "util/ralloc.h"
+#include "hk_cmd_buffer.h"
+#include "hk_device.h"
+#include "hk_query_oq.h"
+#include "asahi/libagx/query.h"
+
+static unsigned tests, failures;
+static void *test_context;
+
+#define T(name, cond)                                                          \
+   do {                                                                        \
+      tests++;                                                                 \
+      if (!(cond)) {                                                           \
+         failures++;                                                           \
+         printf("FAIL %s line%d\n", name, __LINE__);                           \
+      } else                                                                   \
+         printf("PASS %s\n", name);                                            \
+   } while (0)
+
+/* hk_dispatch_imm_writes() emits the queued stores of a kept stream. The mock
+ * records which stream it was called for and how many stores it carried. */
+#define MAX_DISPATCH 8
+static struct {
+   struct hk_cs *cs;
+   unsigned writes;
+} dispatched[MAX_DISPATCH];
+static unsigned ndispatched;
+
+void
+hk_dispatch_imm_writes(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   (void)cmd;
+   if (ndispatched == MAX_DISPATCH)
+      abort();
+   dispatched[ndispatched].cs = cs;
+   dispatched[ndispatched].writes =
+      util_dynarray_num_elements(&cs->imm_writes, struct libagx_imm_write);
+   ndispatched++;
+}
+
+/* hk_sync.c references these for recording paths these fixtures never take. */
+void
+hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   (void)cmd;
+   (void)cs;
+   abort();
+}
+
+void
+hk_g17p_meta_fill_addr(struct hk_cmd_buffer *cmd, struct hk_cs **target,
+                       uint64_t address, uint64_t size, uint32_t value)
+{
+   (void)cmd; (void)target; (void)address; (void)size; (void)value;
+   abort();
+}
+
+struct agx_ptr
+hk_pool_alloc_from_slow(struct hk_cmd_buffer *cmd, struct hk_uploader *uploader,
+                         uint32_t size, uint32_t alignment,
+                         enum hk_cmd_bo_type type)
+{
+   (void)cmd; (void)uploader; (void)size; (void)alignment; (void)type;
+   abort();
+}
+
+struct agx_ptr
+hk_pool_cs_alloc(struct hk_cmd_buffer *cmd, uint32_t size, uint32_t alignment)
+{
+   (void)cmd; (void)size; (void)alignment;
+   abort();
+}
+
+void
+hk_drm_attachment_set_init(struct hk_drm_attachment_set *set)
+{
+   (void)set;
+   abort();
+}
+
+void
+hk_flush_deferred_clears_why(struct hk_cmd_buffer *cmd, const char *why)
+{
+   (void)cmd;
+   (void)why;
+   abort();
+}
+
+void
+hk_dispatch_precomp(struct hk_cmd_buffer *cmd, struct agx_grid grid,
+                     enum agx_barrier barrier, enum libagx_program idx,
+                     void *data, size_t data_size)
+{
+   (void)cmd; (void)grid; (void)barrier; (void)idx; (void)data; (void)data_size;
+   abort();
+}
+
+/* The resolver reads only the chip from the device. */
+static enum agx_chip test_chip = AGX_CHIP_G17P;
+
+static struct hk_cmd_buffer *
+new_cmd(void)
+{
+   struct hk_cmd_buffer *c = rzalloc_size(test_context, sizeof(*c));
+   struct hk_device *dev = rzalloc_size(test_context, sizeof(*dev));
+   if (!c || !dev)
+      abort();
+   dev->dev.chip = test_chip;
+   c->vk.base.device = &dev->vk;
+   list_inithead(&c->control_streams);
+   util_dynarray_init(&c->sync_pass_records, NULL);
+   ndispatched = 0;
+   return c;
+}
+
+/* Streams are heap-allocated like the driver's: a dropped stream is freed by
+ * hk_cs_destroy(). */
+static struct hk_cs *
+add(struct hk_cmd_buffer *c, bool compute)
+{
+   struct hk_cs *s = calloc(1, sizeof(*s));
+   if (!s)
+      abort();
+   s->type = compute ? HK_CS_CDM : HK_CS_VDM;
+   list_addtail(&s->node, &c->control_streams);
+   hk_sync_stream_created(c, s, &s, compute);
+   return s;
+}
+
+static unsigned
+nstreams(struct hk_cmd_buffer *c)
+{
+   return list_length(&c->control_streams);
+}
+
+static bool
+listed(struct hk_cmd_buffer *c, const struct hk_cs *cs)
+{
+   list_for_each_entry(struct hk_cs, it, &c->control_streams, node) {
+      if (it == cs)
+         return true;
+   }
+   return false;
+}
+
+/* A post-graphics stream holding `count` availability stores for queries
+ * ended by `render`, deferred as hk_cmd_buffer_end_compute_internal() would. */
+static struct hk_cs *
+add_post(struct hk_cmd_buffer *c, struct hk_cs *render, unsigned count,
+         uint32_t first_handle)
+{
+   struct hk_cs *post = add(c, true);
+   post->sync.dep[HK_SYNC_VDM].cut = render;
+   post->imm_writes = UTIL_DYNARRAY_INIT;
+   post->oq_stamps = UTIL_DYNARRAY_INIT;
+   for (unsigned i = 0; i < count; ++i) {
+      const struct libagx_imm_write write = {0};
+      const struct hk_oq_stamp stamp = {
+         .handle = first_handle + i,
+         .offset_B = 8 * (first_handle + i),
+         .render_seq = render->sync.seq,
+      };
+      util_dynarray_append(&post->imm_writes, write);
+      util_dynarray_append(&post->oq_stamps, stamp);
+   }
+   return post;
+}
+
+static bool
+deferred(struct hk_cmd_buffer *c, struct hk_cs *post)
+{
+   return hk_defer_oq_available(c, post) && post->oq_avail_deferred;
+}
+
+static bool
+slot_is(const struct hk_cs *render, unsigned slot, uint32_t handle)
+{
+   return render->oq_stamp[slot].handle == handle &&
+          render->oq_stamp[slot].offset_B == 8 * handle;
+}
+
+static void
+test_defer(void)
+{
+   struct hk_cmd_buffer *c = new_cmd();
+   struct hk_cs *render = add(c, false);
+
+   struct hk_cs *post = add_post(c, render, 2, 1);
+   T("defer accepts stores that all ride on one render", deferred(c, post));
+
+   post = add_post(c, render, 0, 1);
+   T("defer refuses a stream without availability stores",
+     !hk_defer_oq_available(c, post));
+
+   post = add_post(c, render, 1, 1);
+   const struct libagx_imm_write extra = {0};
+   util_dynarray_append(&post->imm_writes, extra);
+   T("defer refuses other immediate writes", !hk_defer_oq_available(c, post));
+
+   post = add_post(c, render, 1, 1);
+   post->stats.cmds = 1;
+   T("defer refuses a stream with commands", !hk_defer_oq_available(c, post));
+
+   post = add_post(c, render, 1, 1);
+   post->timestamp.end.handle = 9;
+   T("defer refuses a stream an API timestamp rides on",
+     !hk_defer_oq_available(c, post));
+
+   post = add_post(c, render, 1, 1);
+   post->query_reset_pending = true;
+   T("defer refuses a stream with a pending query reset",
+     !hk_defer_oq_available(c, post));
+
+   struct hk_cs *other = add(c, false);
+   post = add_post(c, render, 2, 1);
+   ((struct hk_oq_stamp *)post->oq_stamps.data)[1].render_seq = other->sync.seq;
+   T("defer refuses stores ended by different renders",
+     !hk_defer_oq_available(c, post));
+}
+
+static void
+test_stamped(void)
+{
+   struct hk_cmd_buffer *c = new_cmd();
+   struct hk_cs *render = add(c, false);
+   struct hk_cs *post = add_post(c, render, 1, 5);
+   T("stamped fixture defers", deferred(c, post));
+   const unsigned before = nstreams(c);
+   hk_resolve_deferred_oq_available(c);
+   T("stamped: the render carries the store in the fragment-end slot",
+     slot_is(render, 3, 5));
+   T("stamped: the stream is dropped", nstreams(c) == before - 1);
+   T("stamped: nothing is dispatched", ndispatched == 0);
+
+   /* API timestamps ride on vertex start (0) and fragment end (3): the stores
+    * take the free slots in order 1, 2. */
+   c = new_cmd();
+   render = add(c, false);
+   render->timestamp.start.handle = 40;
+   render->timestamp.end.handle = 41;
+   post = add_post(c, render, 2, 7);
+   T("stamped with API timestamps fixture defers", deferred(c, post));
+   hk_resolve_deferred_oq_available(c);
+   T("stamped: API timestamp slots are left alone",
+     render->oq_stamp[0].handle == 0 && render->oq_stamp[3].handle == 0);
+   T("stamped: stores take slots 1 then 2",
+     slot_is(render, 1, 7) && slot_is(render, 2, 8));
+   T("stamped: no dispatch with API timestamps", ndispatched == 0);
+
+   /* Another occlusion store already rides on the fragment-end slot. */
+   c = new_cmd();
+   render = add(c, false);
+   render->oq_stamp[3] = (struct agx_timestamp_req){.handle = 11, .offset_B = 88};
+   post = add_post(c, render, 1, 12);
+   T("stamped with an occupied slot fixture defers", deferred(c, post));
+   hk_resolve_deferred_oq_available(c);
+   T("stamped: the store skips the occupied slot",
+     render->oq_stamp[3].handle == 11 && slot_is(render, 1, 12));
+   T("stamped: dropped with an occupied slot", !listed(c, post) && ndispatched == 0);
+}
+
+static void
+test_keep(void)
+{
+   /* The stream became the tail an API timestamp rides on while deferred. */
+   struct hk_cmd_buffer *c = new_cmd();
+   struct hk_cs *render = add(c, false);
+   struct hk_cs *post = add_post(c, render, 1, 1);
+   T("keep (timestamp) fixture defers", deferred(c, post));
+   post->timestamp.end.handle = 9;
+   hk_resolve_deferred_oq_available(c);
+   T("keep: a later API timestamp keeps the store",
+     ndispatched == 1 && dispatched[0].cs == post && listed(c, post) &&
+        render->oq_stamp[3].handle == 0);
+
+   /* A later query reset (or any compute) waits on the deferred stream. */
+   c = new_cmd();
+   render = add(c, false);
+   post = add_post(c, render, 1, 1);
+   T("keep (reset) fixture defers", deferred(c, post));
+   struct hk_cs *reset = add(c, true);
+   reset->sync.dep[HK_SYNC_CDM].cut = post;
+   hk_resolve_deferred_oq_available(c);
+   T("keep: a later query reset waiting on the stream keeps the store",
+     ndispatched == 1 && dispatched[0].cs == post && render->oq_stamp[3].handle == 0);
+
+   /* Later compute waits on the render. */
+   c = new_cmd();
+   render = add(c, false);
+   post = add_post(c, render, 1, 1);
+   T("keep (render wait) fixture defers", deferred(c, post));
+   struct hk_cs *later = add(c, true);
+   later->sync.dep[HK_SYNC_VDM].cut = render;
+   hk_resolve_deferred_oq_available(c);
+   T("keep: compute waiting on the render keeps the store",
+     ndispatched == 1 && dispatched[0].cs == post);
+
+   /* The command buffer's tail requirement names the render. */
+   c = new_cmd();
+   render = add(c, false);
+   post = add_post(c, render, 1, 1);
+   T("keep (tail) fixture defers", deferred(c, post));
+   c->sync.req[0][HK_SYNC_VDM].cut = render;
+   hk_resolve_deferred_oq_available(c);
+   T("keep: a tail requirement on the render keeps the store",
+     ndispatched == 1 && dispatched[0].cs == post);
+
+   /* The stream does not wait on the render that ended the queries. */
+   c = new_cmd();
+   render = add(c, false);
+   struct hk_cs *next = add(c, false);
+   post = add_post(c, render, 1, 1);
+   post->sync.dep[HK_SYNC_VDM].cut = next;
+   T("keep (wrong render) fixture defers", deferred(c, post));
+   hk_resolve_deferred_oq_available(c);
+   T("keep: a stream not waiting on its render keeps the store",
+     ndispatched == 1 && dispatched[0].cs == post && render->oq_stamp[3].handle == 0);
+}
+
+/* Fill every slot of a render: API timestamps on 0 and 3, other stores on 1/2. */
+static void
+fill_slots(struct hk_cs *render, uint32_t base)
+{
+   render->timestamp.start.handle = base;
+   render->timestamp.end.handle = base + 1;
+   render->oq_stamp[1] = (struct agx_timestamp_req){.handle = base + 2};
+   render->oq_stamp[2] = (struct agx_timestamp_req){.handle = base + 3};
+}
+
+static void
+test_movable(void)
+{
+   /* More stores than free slots on the render, and nothing later: the stream
+    * is MOVABLE and, with no later movable stream, dispatched at the end. */
+   struct hk_cmd_buffer *c = new_cmd();
+   struct hk_cs *r1 = add(c, false);
+   r1->timestamp.end.handle = 300;
+   r1->oq_stamp[1] = (struct agx_timestamp_req){.handle = 301};
+   struct hk_cs *p1 = add_post(c, r1, 3, 1);
+   T("short-of-slots fixture defers", deferred(c, p1));
+   hk_resolve_deferred_oq_available(c);
+   T("movable: too few free slots dispatches the stores, slots unchanged",
+     ndispatched == 1 && dispatched[0].cs == p1 && dispatched[0].writes == 3 &&
+        r1->oq_stamp[2].handle == 0 && r1->oq_stamp[0].handle == 0);
+
+   /* Two full renders in order. The later stream is MOVABLE and stays, a
+    * compute stream waiting on the later render; the earlier stream joins it,
+    * so one dispatch carries both stores. */
+   c = new_cmd();
+   r1 = add(c, false);
+   fill_slots(r1, 100);
+   p1 = add_post(c, r1, 1, 1);
+   struct hk_cs *r2 = add(c, false);
+   fill_slots(r2, 200);
+   struct hk_cs *p2 = add_post(c, r2, 1, 2);
+   T("in-order movable fixtures defer", deferred(c, p1) && deferred(c, p2));
+   hk_resolve_deferred_oq_available(c);
+   T("movable in order: the earlier stream joins the movable one",
+     ndispatched == 1 && dispatched[0].cs == p2 && dispatched[0].writes == 2 &&
+        !listed(c, p1) && listed(c, p2));
+
+   /* The earlier stream waits on the later render: nothing after it waits on
+    * that render, so it is MOVABLE; the later stream (waiting on the earlier
+    * render) is then kept because the movable one still waits past it. */
+   c = new_cmd();
+   r1 = add(c, false);
+   fill_slots(r1, 100);
+   r2 = add(c, false);
+   fill_slots(r2, 200);
+   p1 = add_post(c, r2, 1, 1);
+   p2 = add_post(c, r1, 1, 2);
+   T("out-of-order movable fixtures defer", deferred(c, p1) && deferred(c, p2));
+   hk_resolve_deferred_oq_available(c);
+   T("movable out of order: the kept stream goes first, the movable one last",
+     ndispatched == 2 && dispatched[0].cs == p2 && dispatched[1].cs == p1 &&
+        dispatched[0].writes == 1 && dispatched[1].writes == 1);
+
+   /* A free render whose store was deferred after a movable stream that waits
+    * on a later render: the movable stream counts as compute waiting past the
+    * free render, so its store is kept too (in place), the movable one last. */
+   c = new_cmd();
+   r1 = add(c, false);
+   r2 = add(c, false);
+   fill_slots(r2, 200);
+   p2 = add_post(c, r2, 1, 2);
+   p1 = add_post(c, r1, 1, 1);
+   T("movable-then-free fixtures defer", deferred(c, p2) && deferred(c, p1));
+   hk_resolve_deferred_oq_available(c);
+   T("a free render's store is kept behind a movable stream's later wait",
+     r1->oq_stamp[3].handle == 0 && ndispatched == 2 && dispatched[0].cs == p1 &&
+        dispatched[1].cs == p2);
+}
+
+/* Resolution runs from the end of the command buffer: a later stream whose
+ * render carries its stores is gone before earlier streams are judged. */
+static void
+test_order(void)
+{
+   struct hk_cmd_buffer *c = new_cmd();
+   struct hk_cs *r[3], *p[3];
+   for (unsigned i = 0; i < 3; ++i) {
+      r[i] = add(c, false);
+      p[i] = add_post(c, r[i], 1, 1 + i);
+   }
+   T("three passes defer", deferred(c, p[0]) && deferred(c, p[1]) && deferred(c, p[2]));
+   const unsigned before = nstreams(c);
+   hk_resolve_deferred_oq_available(c);
+   T("order: every render carries its own store",
+     slot_is(r[0], 3, 1) && slot_is(r[1], 3, 2) && slot_is(r[2], 3, 3));
+   T("order: all three streams dropped, nothing dispatched",
+     nstreams(c) == before - 3 && ndispatched == 0);
+
+   /* The last render is full: its stream stays as compute waiting on that
+    * render, so the earlier render may not carry its store; the earlier
+    * store joins the remaining stream instead of a compute command of its
+    * own. */
+   c = new_cmd();
+   r[0] = add(c, false);
+   p[0] = add_post(c, r[0], 1, 1);
+   r[1] = add(c, false);
+   fill_slots(r[1], 200);
+   p[1] = add_post(c, r[1], 1, 2);
+   T("free-then-full fixtures defer", deferred(c, p[0]) && deferred(c, p[1]));
+   hk_resolve_deferred_oq_available(c);
+   T("order: the earlier store joins the remaining later stream",
+     r[0]->oq_stamp[3].handle == 0 && ndispatched == 1 &&
+        dispatched[0].cs == p[1] && dispatched[0].writes == 2 &&
+        !listed(c, p[0]));
+
+   /* As above, but a later reset of the first query waits on its stream: the
+    * store may not move past that reset, so it stays in place. */
+   c = new_cmd();
+   r[0] = add(c, false);
+   p[0] = add_post(c, r[0], 1, 1);
+   r[1] = add(c, false);
+   fill_slots(r[1], 200);
+   p[1] = add_post(c, r[1], 1, 2);
+   T("free-then-full-reset fixtures defer", deferred(c, p[0]) && deferred(c, p[1]));
+   struct hk_cs *waiter = add(c, true);
+   waiter->sync.dep[HK_SYNC_CDM].cut = p[0];
+   hk_resolve_deferred_oq_available(c);
+   T("order: a store something waits on does not join a later stream",
+     r[0]->oq_stamp[3].handle == 0 && ndispatched == 2 &&
+        dispatched[0].cs == p[0] && dispatched[1].cs == p[1] &&
+        listed(c, p[0]));
+
+   /* Many queries in each of three passes, more than any render has slots
+    * for: one compute command carries every store. */
+   c = new_cmd();
+   for (unsigned i = 0; i < 3; ++i) {
+      r[i] = add(c, false);
+      p[i] = add_post(c, r[i], 8, 1 + 8 * i);
+   }
+   T("full-passes fixtures defer",
+     deferred(c, p[0]) && deferred(c, p[1]) && deferred(c, p[2]));
+   hk_resolve_deferred_oq_available(c);
+   T("order: stores of passes without enough slots share one dispatch",
+     ndispatched == 1 && dispatched[0].cs == p[2] && dispatched[0].writes == 24 &&
+        !listed(c, p[0]) && !listed(c, p[1]) && r[0]->oq_stamp[3].handle == 0 &&
+        r[1]->oq_stamp[3].handle == 0 && r[2]->oq_stamp[3].handle == 0);
+
+   /* A later query reset waits on the last stream: both stores stay. */
+   c = new_cmd();
+   r[0] = add(c, false);
+   p[0] = add_post(c, r[0], 1, 1);
+   r[1] = add(c, false);
+   p[1] = add_post(c, r[1], 1, 2);
+   T("reset-after-two fixtures defer", deferred(c, p[0]) && deferred(c, p[1]));
+   struct hk_cs *reset = add(c, true);
+   reset->sync.dep[HK_SYNC_CDM].cut = p[1];
+   hk_resolve_deferred_oq_available(c);
+   T("order: a later reset keeps both stores in place",
+     ndispatched == 2 && dispatched[0].cs == p[1] && dispatched[1].cs == p[0] &&
+        r[0]->oq_stamp[3].handle == 0 && r[1]->oq_stamp[3].handle == 0);
+
+   /* A later reset of the first query only: the later render still carries its
+    * store, the first stays. */
+   c = new_cmd();
+   r[0] = add(c, false);
+   p[0] = add_post(c, r[0], 1, 1);
+   r[1] = add(c, false);
+   p[1] = add_post(c, r[1], 1, 2);
+   T("reset-first fixtures defer", deferred(c, p[0]) && deferred(c, p[1]));
+   reset = add(c, true);
+   reset->sync.dep[HK_SYNC_CDM].cut = p[0];
+   hk_resolve_deferred_oq_available(c);
+   T("order: a reset of the first query keeps only the first store",
+     slot_is(r[1], 3, 2) && !listed(c, p[1]) && ndispatched == 1 &&
+        dispatched[0].cs == p[0]);
+}
+
+/* The other native AGX3 chips keep recording order: while an earlier stream is
+ * judged, the later deferred stream still waits on its later render. */
+static void
+test_order_forward(void)
+{
+   test_chip = AGX_CHIP_G16G;
+   struct hk_cmd_buffer *c = new_cmd();
+   struct hk_cs *r[3], *p[3];
+   for (unsigned i = 0; i < 3; ++i) {
+      r[i] = add(c, false);
+      p[i] = add_post(c, r[i], 1, 1 + i);
+   }
+   T("forward: three passes defer", deferred(c, p[0]) && deferred(c, p[1]) && deferred(c, p[2]));
+   hk_resolve_deferred_oq_available(c);
+   T("forward: only the last render carries its store",
+     r[0]->oq_stamp[3].handle == 0 && r[1]->oq_stamp[3].handle == 0 &&
+        slot_is(r[2], 3, 3) && !listed(c, p[2]));
+   T("forward: the earlier stores are dispatched in order",
+     ndispatched == 2 && dispatched[0].cs == p[0] && dispatched[1].cs == p[1]);
+   test_chip = AGX_CHIP_G17P;
+}
+
+/* Only the G17 runtime (G17P and M5) publishes render timestamps at
+ * retirement. G15 firmware writes the vertex-end slot when tiling finishes,
+ * before the fragment work writes the counts: such a stamp is not
+ * availability. */
+static void
+test_stamp_chips(void)
+{
+   T("chips: G17P and M5 renders carry availability",
+     hk_oq_render_stamps_supported(AGX_CHIP_G17P) &&
+        hk_oq_render_stamps_supported(AGX_CHIP_G17G) &&
+        hk_oq_render_stamps_supported(AGX_CHIP_G17S));
+   T("chips: G15 renders keep the queued store",
+     !hk_oq_render_stamps_supported(AGX_CHIP_G15G) &&
+        !hk_oq_render_stamps_supported(AGX_CHIP_G15S) &&
+        !hk_oq_render_stamps_supported(AGX_CHIP_G15C));
+   T("chips: other native chips keep the queued store",
+     !hk_oq_render_stamps_supported(AGX_CHIP_G16G));
+}
+
+int
+main(void)
+{
+   test_context = ralloc_context(NULL);
+   test_defer();
+   test_stamped();
+   test_keep();
+   test_movable();
+   test_order();
+   test_order_forward();
+   test_stamp_chips();
+   ralloc_free(test_context);
+   printf("%u tests, %u failures\n", tests, failures);
+   return failures ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/src/asahi/vulkan/tests/test-hk-query-reset.c b/src/asahi/vulkan/tests/test-hk-query-reset.c
index 9e83b608d6e7..cb74b155fa75 100644
--- a/src/asahi/vulkan/tests/test-hk-query-reset.c
+++ b/src/asahi/vulkan/tests/test-hk-query-reset.c
@@ -36,9 +36,10 @@ hk_cdm_cache_flush(struct hk_device *dev, struct hk_cs *cs)
  * paths must not allocate, change dependencies, discard a stream or emit
  * unrelated work while draining query resets. */
 void
-hk_flush_deferred_clears(struct hk_cmd_buffer *cmd)
+hk_flush_deferred_clears_why(struct hk_cmd_buffer *cmd, const char *why)
 {
    (void)cmd;
+   (void)why;
    abort();
 }
 
@@ -92,6 +93,14 @@ hk_dispatch_imm_writes(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    abort();
 }
 
+bool
+hk_defer_oq_available(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   (void)cmd;
+   (void)cs;
+   abort();
+}
+
 void
 hk_sync_forget_stream(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
 {
diff --git a/src/asahi/vulkan/tests/test-hk-render-resource-span.c b/src/asahi/vulkan/tests/test-hk-render-resource-span.c
new file mode 100644
index 000000000000..c10f61cb0a05
--- /dev/null
+++ b/src/asahi/vulkan/tests/test-hk-render-resource-span.c
@@ -0,0 +1,47 @@
+/* Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+#include "hk_drm_render_core.h"
+#include <assert.h>
+#include <string.h>
+
+int main(void)
+{
+   struct {
+      struct drm_asahi_cmd_header resources;
+      struct drm_asahi_render_program programs[2];
+      struct drm_asahi_cmd_header draw;
+      struct drm_asahi_cmd_header reset;
+      struct drm_asahi_cmd_header compute;
+   } stream = {
+      .resources = {
+         .cmd_type = DRM_ASAHI_SET_RENDER_RESOURCES,
+         .size = 2 * sizeof(struct drm_asahi_render_program),
+         .vdm_barrier = DRM_ASAHI_BARRIER_NONE,
+         .cdm_barrier = DRM_ASAHI_BARRIER_NONE,
+      },
+      .programs = {{.stage = 0}, {.stage = 1, .primary_bytes = 272}},
+      .draw = {.cmd_type = DRM_ASAHI_CMD_RENDER},
+      .reset = {
+         .cmd_type = DRM_ASAHI_SET_RENDER_RESOURCES,
+         .vdm_barrier = DRM_ASAHI_BARRIER_NONE,
+         .cdm_barrier = DRM_ASAHI_BARRIER_NONE,
+      },
+      .compute = {.cmd_type = DRM_ASAHI_CMD_COMPUTE},
+   };
+   struct hk_drm_command_span span;
+   assert(hk_drm_next_command_span(&stream, sizeof(stream), 0, 2, &span));
+   assert(span.hardware_count == 2 && span.render_count == 1 && span.compute_count == 1);
+   assert(span.first_hardware_offset == offsetof(__typeof__(stream), draw));
+   assert(span.end_offset == sizeof(stream));
+   assert(hk_drm_next_command_span(&stream, sizeof(stream), 0, 1, &span));
+   assert(span.end_offset == offsetof(__typeof__(stream), reset));
+   assert(hk_drm_next_command_span(&stream, sizeof(stream), span.end_offset, 1, &span));
+   assert(span.compute_count == 1 && span.render_count == 0);
+   stream.resources.vdm_barrier = 0;
+   assert(!hk_drm_next_command_span(&stream, sizeof(stream), 0, 2, &span));
+   stream.resources.vdm_barrier = DRM_ASAHI_BARRIER_NONE;
+   stream.resources.size--;
+   assert(!hk_drm_next_command_span(&stream, sizeof(stream), 0, 2, &span));
+   return 0;
+}
diff --git a/src/asahi/vulkan/tests/test-hk-root-fragment.c b/src/asahi/vulkan/tests/test-hk-root-fragment.c
new file mode 100644
index 000000000000..c42e2015d1cc
--- /dev/null
+++ b/src/asahi/vulkan/tests/test-hk-root-fragment.c
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: MIT */
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "hk_cmd_buffer.h"
+
+static uint64_t checks;
+#define CHECK(condition)                                                       \
+   do {                                                                        \
+      ++checks;                                                                \
+      if (!(condition)) {                                                      \
+         fprintf(stderr, "line %d: %s\n", __LINE__, #condition);               \
+         exit(1);                                                              \
+      }                                                                        \
+   } while (0)
+
+/* Independent reference: select by comparing the actual published roots. */
+static uint64_t
+fresh_select(struct hk_graphics_state *gfx, uint8_t reads)
+{
+   const uint8_t hidden = HK_ROOT_READS_ADDRESS | HK_ROOT_READS_VERTEX_INPUT |
+                          HK_ROOT_READS_DYNAMIC_BUFFERS;
+   if (gfx->fs_root && gfx->fs_root != gfx->root && gfx->root_cpu &&
+       !(reads & hidden) &&
+       hk_root_matches_for_fragment(gfx->fs_root_cpu, gfx->root_cpu))
+      return gfx->fs_root;
+   gfx->fs_root = gfx->root;
+   gfx->fs_root_cpu = gfx->root_cpu;
+   return gfx->root;
+}
+
+static void
+select_and_check(struct hk_graphics_state *fresh,
+                 struct hk_graphics_state *cached, uint8_t reads, bool memo,
+                 bool shadow)
+{
+   CHECK(fresh_select(fresh, reads) ==
+         hk_m5_fragment_root_select(cached, reads, memo, shadow));
+   CHECK(fresh->fs_root == cached->fs_root);
+   CHECK(fresh->fs_root_cpu == cached->fs_root_cpu);
+}
+
+/* Publication uses a mutable cached CPU source and an immutable GPU copy.
+ * Poison skipped dynamic bytes so accidental observation cannot pass. */
+static void
+publish(struct hk_graphics_state *fresh, struct hk_graphics_state *cached,
+        struct hk_root_descriptor_table *gpu,
+        const struct hk_root_descriptor_table *source, unsigned dynamic_end,
+        uint64_t va, bool shadow)
+{
+   const size_t head = hk_root_descriptor_offset(dynamic_buffers) +
+                       dynamic_end * sizeof(source->dynamic_buffers[0]);
+   const size_t tail = hk_root_descriptor_offset(set_dynamic_buffer_start);
+   memset(gpu, 0xa7, sizeof(*gpu));
+   if (shadow)
+      hk_root_fragment_snapshot_publish(cached, gpu, source);
+   memcpy(gpu, source, head);
+   memcpy((uint8_t *)gpu + tail, (const uint8_t *)source + tail,
+          sizeof(*gpu) - tail);
+   fresh->root = cached->root = va;
+   fresh->root_cpu = cached->root_cpu = gpu;
+   if (shadow) {
+      size_t out = 0;
+      for (size_t b = 0; b < sizeof(*gpu); ++b) {
+         if (!hk_root_reads_of(b, b + 1)) {
+            CHECK(cached->root_fragment_snapshot.bytes[out] ==
+                  ((const uint8_t *)gpu)[b]);
+            ++out;
+         }
+      }
+      CHECK(out == HK_ROOT_FRAGMENT_VISIBLE_SIZE);
+   }
+}
+
+int
+main(void)
+{
+   struct hk_graphics_state *fresh = calloc(1, sizeof(*fresh));
+   struct hk_graphics_state *cached = calloc(1, sizeof(*cached));
+   struct hk_root_descriptor_table source, gpu[35];
+   CHECK(fresh && cached);
+
+   for (unsigned mode = 0; mode < 4; ++mode) {
+      const bool memo = mode & 1, shadow = mode & 2;
+      for (unsigned end = 0; end <= HK_MAX_DYNAMIC_BUFFERS; ++end) {
+         for (unsigned byte = 0; byte < sizeof(source); ++byte) {
+            memset(fresh, 0, sizeof(*fresh));
+            memset(cached, 0, sizeof(*cached));
+            memset(&source, 0, sizeof(source));
+            publish(fresh, cached, &gpu[0], &source, end, 0x1000, shadow);
+            select_and_check(fresh, cached, 0, memo, shadow);
+            const struct hk_root_fragment_snapshot prior =
+               cached->fs_root_fragment_snapshot;
+            const struct hk_root_descriptor_table *prior_cpu =
+               cached->fs_root_fragment_snapshot_cpu;
+            ((uint8_t *)&source)[byte] ^= 0x5a;
+            publish(fresh, cached, &gpu[1], &source, end, 0x2000, shadow);
+            for (unsigned reads = 0; reads < 8; ++reads) {
+               /* Each mask must decide between the old and current roots.
+                * A previous hidden-read selection must not pre-adopt it. */
+               fresh->fs_root = cached->fs_root = 0x1000;
+               fresh->fs_root_cpu = cached->fs_root_cpu = &gpu[0];
+               cached->fs_root_checked_root = 0;
+               cached->fs_root_fragment_snapshot = prior;
+               cached->fs_root_fragment_snapshot_cpu = prior_cpu;
+               select_and_check(fresh, cached, reads, memo, shadow);
+               select_and_check(fresh, cached, reads, memo, shadow);
+            }
+         }
+      }
+
+      /* Every publication-time global and the complete tail are visible. */
+      memset(fresh, 0, sizeof(*fresh));
+      memset(cached, 0, sizeof(*cached));
+      memset(&source, 0, sizeof(source));
+      for (unsigned globals = 0; globals < 32; ++globals) {
+         source.root_desc_addr = 0x100000000ull + globals;
+         source.printf_buffer_address = globals & 1 ? 0x12340000 : 0;
+         source.sampler_heap_address = globals & 2 ? 0x23450000 : 0;
+         source.physical_sparse_map = globals & 4 ? 0x34560000 : 0;
+         source.physical_sparse_base = globals & 8 ? 0x45670000 : 0;
+         source.physical_sparse_size = globals & 16 ? 0x100000 : 0;
+         publish(fresh, cached, &gpu[globals], &source, 0,
+                 0x3000 + globals * 0x1000, shadow);
+         select_and_check(fresh, cached, 0, memo, shadow);
+      }
+
+      /* Mutable descriptor restoration cannot change a published snapshot. */
+      memset(fresh, 0, sizeof(*fresh));
+      memset(cached, 0, sizeof(*cached));
+      memset(&source, 0, sizeof(source));
+      publish(fresh, cached, &gpu[0], &source, 0, 0x1000, shadow);
+      select_and_check(fresh, cached, 0, memo, shadow);
+      publish(fresh, cached, &gpu[1], &source, 0, 0x2000, shadow);
+      memset(&source, 0x6d, sizeof(source));
+      select_and_check(fresh, cached, 0, memo, shadow);
+      select_and_check(fresh, cached, 7, memo, shadow);
+
+      /* Failed publication retains the old CPU copy while its GPU VA is 0. */
+      source.push[0] ^= 1;
+      publish(fresh, cached, &gpu[2], &source, 64, 0x3000, shadow);
+      fresh->root = cached->root = 0;
+      select_and_check(fresh, cached, 0, memo, shadow);
+      select_and_check(fresh, cached, 7, memo, shadow);
+      cached->root_fragment_snapshot_cpu = NULL;
+      select_and_check(fresh, cached, 0, memo, shadow);
+
+      /* Missing current/prior metadata must use the fresh comparison. */
+      memset(fresh, 0, sizeof(*fresh));
+      memset(cached, 0, sizeof(*cached));
+      memset(&source, 0, sizeof(source));
+      publish(fresh, cached, &gpu[0], &source, 0, 0x1000, shadow);
+      select_and_check(fresh, cached, 0, memo, shadow);
+      source.push[0] = 1;
+      publish(fresh, cached, &gpu[1], &source, 0, 0x2000, shadow);
+      cached->root_fragment_snapshot_cpu = NULL;
+      select_and_check(fresh, cached, 0, memo, shadow);
+      publish(fresh, cached, &gpu[2], &source, 0, 0x3000, shadow);
+      cached->fs_root_fragment_snapshot_cpu = NULL;
+      select_and_check(fresh, cached, 0, memo, shadow);
+
+      /* Command-buffer reset permits both pointer and VA reuse. */
+      memset(fresh, 0, sizeof(*fresh));
+      memset(cached, 0, sizeof(*cached));
+      publish(fresh, cached, &gpu[0], &source, 0, 0x1000, shadow);
+      select_and_check(fresh, cached, 0, memo, shadow);
+      fresh->root_cpu = cached->root_cpu = NULL;
+      fresh->root = cached->root = 0;
+      select_and_check(fresh, cached, 0, memo, shadow);
+   }
+   printf("root fragment: %" PRIu64 " checks passed\n", checks);
+   free(fresh);
+   free(cached);
+   return 0;
+}
diff --git a/src/asahi/vulkan/tests/test-hk-rt-pipeline-binary.c b/src/asahi/vulkan/tests/test-hk-rt-pipeline-binary.c
index 287d126b1f61..91364fe02db9 100644
--- a/src/asahi/vulkan/tests/test-hk-rt-pipeline-binary.c
+++ b/src/asahi/vulkan/tests/test-hk-rt-pipeline-binary.c
@@ -313,7 +313,7 @@ __wrap_vk_common_CreatePipelineBinariesKHR(
    VkDevice d, const VkPipelineBinaryCreateInfoKHR *ci,
    const VkAllocationCallbacks *a, VkPipelineBinaryHandlesInfoKHR *out)
 {
-   if (dev.dev.chip != AGX_CHIP_G17P) {
+   if (dev.dev.chip != AGX_CHIP_G17P && !agx_chip_g17_m5(dev.dev.chip)) {
       classic_delegations++;
       return VK_NOT_READY;
    }
@@ -328,7 +328,7 @@ VkResult
 __wrap_vk_common_GetPipelineKeyKHR(
    VkDevice d, const VkPipelineCreateInfoKHR *ci, VkPipelineBinaryKeyKHR *key)
 {
-   if (dev.dev.chip != AGX_CHIP_G17P) {
+   if (dev.dev.chip != AGX_CHIP_G17P && !agx_chip_g17_m5(dev.dev.chip)) {
       classic_delegations++;
       return VK_NOT_READY;
    }
@@ -362,7 +362,7 @@ main(void)
    pdev.vk.properties.maxSubgroupSize = 32;
    device = hk_device_to_handle(&dev);
    for (enum agx_chip chip = AGX_CHIP_G13G; chip < AGX_CHIP_UNKNOWN; ++chip) {
-      if (chip == AGX_CHIP_G17P)
+      if (chip == AGX_CHIP_G17P || agx_chip_g17_m5(chip))
          continue;
       dev.dev.chip = chip;
       VkRayTracingPipelineCreateInfoKHR rt = {
diff --git a/src/asahi/vulkan/tests/test-hk-sampler-bias.c b/src/asahi/vulkan/tests/test-hk-sampler-bias.c
index d0d3aac780fd..34875a1b3585 100644
--- a/src/asahi/vulkan/tests/test-hk-sampler-bias.c
+++ b/src/asahi/vulkan/tests/test-hk-sampler-bias.c
@@ -92,7 +92,7 @@ int main(void)
             assert(!nir_shader_tex_pass(b.shader, hk_lower_g17p_sampler_bias,
                                        nir_metadata_control_flow, NULL));
          }
-         nir_validate_shader(b.shader, "sampler bias test");
+         /* nir_validate_shader is compiled out of this NDEBUG build. */
          ralloc_free(b.shader);
          checks++;
       }
diff --git a/src/asahi/vulkan/tests/test-hk-sync-prior.c b/src/asahi/vulkan/tests/test-hk-sync-prior.c
index 791d40ec17e8..fdc5e2c6e499 100644
--- a/src/asahi/vulkan/tests/test-hk-sync-prior.c
+++ b/src/asahi/vulkan/tests/test-hk-sync-prior.c
@@ -7,6 +7,7 @@
 #include <string.h>
 #include "util/ralloc.h"
 #include "hk_cmd_buffer.h"
+#include "hk_device.h"
 #include "hk_drm_prior_core.h"
 #include "hk_queue.h"
 _Static_assert(sizeof(struct drm_asahi_cmd_render) == 320, "legacy render");
@@ -75,6 +76,9 @@ static struct hk_cmd_buffer *
 new_cmd(void)
 {
    struct hk_cmd_buffer *c = test_alloc(sizeof(*c));
+   /* Stream creation reads device-wide options (the frame anatomy flag):
+    * give each fixture a zeroed device. */
+   c->vk.base.device = test_alloc(sizeof(struct hk_device));
    list_inithead(&c->control_streams);
    util_dynarray_init(&c->sync_pass_records, NULL);
    return c;
@@ -94,6 +98,131 @@ submit(struct hk_queue *q, struct hk_cmd_buffer *c)
    struct hk_cmd_buffer *cs[] = {c};
    return hk_sync_resolve_submit(q, cs, 1, false, false, 0);
 }
+
+static struct hk_cmd_buffer *
+timestamp_cmd(enum agx_chip chip, bool exact)
+{
+   struct hk_cmd_buffer *cmd = new_cmd();
+   struct hk_device *dev = test_alloc(sizeof(*dev));
+   dev->dev.chip = chip;
+   dev->dev.params.features = exact ? DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS : 0;
+   cmd->vk.base.device = &dev->vk;
+   return cmd;
+}
+
+static void
+end_timestamp(struct hk_cmd_buffer *cmd, uint32_t query)
+{
+   const struct agx_timestamp_req req = {
+      .handle = 1, .addr = 0x10000 + 8 * query, .offset_B = 8 * query};
+   hk_sync_write_timestamp(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, req);
+}
+
+/* Exercise actual timestamp placement, barriers and the submission resolver.
+ * Existing query-command scopes are not removed by the timestamp scope's
+ * redundant same-engine execution edge. No command payload or GPU is needed.
+ */
+static void
+test_timestamp_execution_scopes(void)
+{
+   for (unsigned exact = 0; exact < 2; ++exact) {
+      struct hk_queue *queue = new_queue(exact);
+      struct hk_cmd_buffer *first = timestamp_cmd(AGX_CHIP_G17P, exact);
+      add(first, true);
+      end_timestamp(first, 0);
+      T("first timed compute resolves", submit(queue, first));
+      struct hk_cmd_buffer *second = timestamp_cmd(AGX_CHIP_G17P, exact);
+      struct hk_cs *timed = add(second, true);
+      end_timestamp(second, 1);
+      T("next timed compute resolves", submit(queue, second));
+      T("timestamp execution scope follows the exact-prior capability",
+        exact ? timed->sync.barrier[HK_SYNC_CDM] == DRM_ASAHI_BARRIER_NONE &&
+                   timed->sync.prior[HK_SYNC_CDM] == 0
+              : timed->sync.barrier[HK_SYNC_CDM] == 0);
+
+      /* Query copy/reset commands use an ALL earlier-work scope. Record
+       * that genuine scope through the production barrier entry point before
+       * the consumer, then add a timestamp to it and resolve both submissions.
+       */
+      struct hk_cmd_buffer *query = timestamp_cmd(AGX_CHIP_G17P, exact);
+      hk_sync_barrier(query, HK_SYNC_ALL, HK_SYNC_ALL);
+      struct hk_cs *copy = add(query, true);
+      copy->g17p_timestamp_query = true;
+      end_timestamp(query, 2);
+      T("query consumer plus timestamp resolves", submit(queue, query));
+      T("query memory consumer retains the completed producer prefix",
+        exact ? copy->sync.prior[HK_SYNC_CDM] == 2
+              : copy->sync.barrier[HK_SYNC_CDM] == 0);
+
+      struct hk_cmd_buffer *barrier = timestamp_cmd(AGX_CHIP_G17P, exact);
+      hk_sync_barrier(barrier, HK_SYNC_ALL, HK_SYNC_ALL);
+      T("standalone query ordering resolves", submit(queue, barrier));
+      struct hk_cmd_buffer *after = timestamp_cmd(AGX_CHIP_G17P, exact);
+      struct hk_cs *inherited = add(after, true);
+      end_timestamp(after, 3);
+      T("later timestamp with carried requirements resolves", submit(queue, after));
+      T("timestamp preserves a carried query-memory requirement",
+        exact ? inherited->sync.prior[HK_SYNC_CDM] == 3
+              : inherited->sync.barrier[HK_SYNC_CDM] == 0);
+
+      queue = new_queue(exact);
+      first = timestamp_cmd(AGX_CHIP_G17P, exact);
+      add(first, false);
+      T("earlier render resolves", submit(queue, first));
+      second = timestamp_cmd(AGX_CHIP_G17P, exact);
+      timed = add(second, true);
+      end_timestamp(second, 4);
+      T("timestamp after render resolves", submit(queue, second));
+      T("timestamp cross-engine scope is preserved",
+        exact ? timed->sync.prior[HK_SYNC_VDM] == 1
+              : timed->sync.barrier[HK_SYNC_VDM] == 0);
+
+      queue = new_queue(exact);
+      first = timestamp_cmd(AGX_CHIP_G17P, exact);
+      add(first, true);
+      end_timestamp(first, 5);
+      T("reset producer timestamp resolves", submit(queue, first));
+      struct hk_cmd_buffer *reset_cmd = timestamp_cmd(AGX_CHIP_G17P, exact);
+      hk_sync_barrier(reset_cmd, HK_SYNC_ALL, HK_SYNC_ALL);
+      struct hk_cs *reset = add(reset_cmd, true);
+      reset->g17p_timestamp_query = true;
+      hk_sync_after_stream(reset_cmd, reset, HK_SYNC_ALL);
+      struct hk_cs *later = add(reset_cmd, true);
+      end_timestamp(reset_cmd, 6);
+      T("reset then later stamped stream resolves", submit(queue, reset_cmd));
+      T("reset keeps the earlier timestamp publication requirement",
+        exact ? reset->sync.prior[HK_SYNC_CDM] == 1
+              : reset->sync.barrier[HK_SYNC_CDM] == 0);
+      T("later stamp keeps reset stream ordering",
+        later->sync.barrier[HK_SYNC_CDM] == 1);
+
+      queue = new_queue(exact);
+      first = timestamp_cmd(AGX_CHIP_G17P, exact);
+      add(first, true);
+      T("render timestamp compute source resolves", submit(queue, first));
+      second = timestamp_cmd(AGX_CHIP_G17P, exact);
+      struct hk_cs *render = add(second, false);
+      end_timestamp(second, 7);
+      T("render end timestamp resolves", submit(queue, second));
+      T("render timestamp keeps the original compute scope",
+        exact ? render->sync.prior[HK_SYNC_CDM] == 1
+              : render->sync.barrier[HK_SYNC_CDM] == 0);
+   }
+
+   for (unsigned chip = 0; chip < 2; ++chip) {
+      const enum agx_chip target = chip ? AGX_CHIP_G16G : AGX_CHIP_G13G;
+      struct hk_queue *queue = new_queue(true);
+      struct hk_cmd_buffer *first = timestamp_cmd(target, true);
+      add(first, true);
+      T("other target source resolves", submit(queue, first));
+      struct hk_cmd_buffer *second = timestamp_cmd(target, true);
+      struct hk_cs *timed = add(second, true);
+      end_timestamp(second, 0);
+      T("other target timestamp resolves", submit(queue, second));
+      T("unqualified target retains the original timestamp prefix",
+        timed->sync.prior[HK_SYNC_CDM] == 1);
+   }
+}
 /* The fixture has no open compute stream. A request to close one is a bug. */
 void
 hk_dispatch_imm_writes(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
@@ -103,6 +232,14 @@ hk_dispatch_imm_writes(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
    abort();
 }
 
+bool
+hk_defer_oq_available(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
+{
+   (void)cmd;
+   (void)cs;
+   abort();
+}
+
 /* This fixture never records query resets. */
 void
 hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
@@ -112,6 +249,58 @@ hk_flush_query_resets(struct hk_cmd_buffer *cmd, struct hk_cs *cs)
       abort();
 }
 
+/* Timestamp recording references payload/allocation entry points for other
+ * branches. These tests attach end stamps to already-created closed streams;
+ * reaching a payload emission branch would invalidate that fixture contract.
+ */
+void
+hk_g17p_meta_fill_addr(struct hk_cmd_buffer *cmd, struct hk_cs **target,
+                       uint64_t address, uint64_t size, uint32_t value)
+{
+   (void)cmd; (void)target; (void)address; (void)size; (void)value;
+   abort();
+}
+
+struct agx_ptr
+hk_pool_alloc_from_slow(struct hk_cmd_buffer *cmd, struct hk_uploader *uploader,
+                         uint32_t size, uint32_t alignment,
+                         enum hk_cmd_bo_type type)
+{
+   (void)cmd; (void)uploader; (void)size; (void)alignment; (void)type;
+   abort();
+}
+
+struct agx_ptr
+hk_pool_cs_alloc(struct hk_cmd_buffer *cmd, uint32_t size, uint32_t alignment)
+{
+   (void)cmd; (void)size; (void)alignment;
+   abort();
+}
+
+void
+hk_drm_attachment_set_init(struct hk_drm_attachment_set *set)
+{
+   (void)set;
+   abort();
+}
+
+void
+hk_flush_deferred_clears_why(struct hk_cmd_buffer *cmd, const char *why)
+{
+   (void)cmd;
+   (void)why;
+   abort();
+}
+
+void
+hk_dispatch_precomp(struct hk_cmd_buffer *cmd, struct agx_grid grid,
+                     enum agx_barrier barrier, enum libagx_program idx,
+                     void *data, size_t data_size)
+{
+   (void)cmd; (void)grid; (void)barrier; (void)idx; (void)data; (void)data_size;
+   abort();
+}
+
 /* Record a real full-frame pass with a sparse-mask preservation footprint.
  * Layer ranges describe BG/EOT's physical access, not just enabled views. */
 static struct hk_cs *
@@ -131,6 +320,83 @@ render_pass(struct hk_cmd_buffer *cmd, const void *image, uint32_t layers,
 }
 
 
+static void
+test_precise_tile_evictions(void)
+{
+   const void *image = (void *)0x9876;
+   struct hk_cmd_buffer *full = new_cmd(), *partial = new_cmd();
+   struct hk_cs *write = render_pass(full, image, 1, false);
+   struct hk_cs *preserve = render_pass(partial, image, 1, true);
+   struct hk_cmd_buffer *one[] = {full};
+   struct hk_queue *q = new_queue(true);
+   q->sync.precise_tile_evictions = true;
+   bool no_query = true, no_barrier = true, resolved = true;
+   for (unsigned i = 1; i <= HK_SYNC_RECENT_FOOTPRINTS + 2; ++i) {
+      q->drm.timeline_value = i;
+      no_query &= !hk_sync_submit_needs_completed(q, one, 1);
+      resolved &= submit(q, full);
+      no_barrier &= write->sync.barrier[HK_SYNC_VDM] == DRM_ASAHI_BARRIER_NONE;
+   }
+   T("complete writes skip retirement query through ring wrap", no_query);
+   T("complete writes keep overlap through ring wrap", resolved && no_barrier);
+   T("complete history retained for later partial pass",
+     q->sync.recent_count == HK_SYNC_RECENT_FOOTPRINTS && q->sync.evicted > 0 &&
+     q->sync.last_preserve == 0 && q->sync.evicted_preserve == 0);
+   struct hk_cmd_buffer *p[] = {partial};
+   T("later preserving pass queries complete history",
+     hk_sync_submit_needs_completed(q, p, 1));
+   const uint64_t point = ++q->drm.timeline_value;
+   T("later preserving pass resolves", submit(q, partial));
+   T("later preserving pass waits on evicted complete writes",
+     preserve->sync.barrier[HK_SYNC_VDM] == 0 && q->sync.last_preserve == point);
+   T("complete pass queries preserving history",
+     hk_sync_submit_needs_completed(q, one, 1));
+   ++q->drm.timeline_value;
+   T("complete pass resolves against preserving history", submit(q, full));
+   T("complete pass waits on live preserving pass",
+     write->sync.barrier[HK_SYNC_VDM] == 0);
+
+   /* Evict the preserving entry, then retire only through that entry. Later
+    * complete writes still run. They must remain independent of a full write,
+    * while a new partial write must wait for those same complete writes. */
+   for (unsigned i = 0; i < HK_SYNC_RECENT_FOOTPRINTS; ++i) {
+      ++q->drm.timeline_value;
+      resolved &= submit(q, full);
+   }
+   T("preserving eviction tracked separately from complete writes",
+     resolved && q->sync.evicted_preserve == point && q->sync.evicted > point);
+   ++q->drm.timeline_value;
+   T("retired preserving pass allows complete write",
+     hk_sync_resolve_submit(q, one, 1, false, false, point) &&
+     write->sync.barrier[HK_SYNC_VDM] == DRM_ASAHI_BARRIER_NONE);
+   ++q->drm.timeline_value;
+   T("partial pass still waits on live evicted complete writes",
+     hk_sync_resolve_submit(q, p, 1, false, false, point) &&
+     preserve->sync.barrier[HK_SYNC_VDM] == 0);
+
+   q = new_queue(true);
+   q->sync.precise_tile_evictions = true;
+   q->drm.timeline_value = 1;
+   struct hk_cmd_buffer *pair[] = {partial, full};
+   T("partial then full in one submit retains dependency",
+     hk_sync_resolve_submit(q, pair, 2, false, false, 0) &&
+     write->sync.barrier[HK_SYNC_VDM] == preserve->sync.index);
+   q = new_queue(true);
+   q->sync.precise_tile_evictions = true;
+   q->drm.timeline_value = 1;
+   struct hk_cmd_buffer *reverse[] = {full, partial};
+   T("full then partial in one submit retains dependency",
+     hk_sync_resolve_submit(q, reverse, 2, false, false, 0) &&
+     preserve->sync.barrier[HK_SYNC_VDM] == write->sync.index);
+
+   q = new_queue(true);
+   q->drm.timeline_value = 1;
+   T("other-device complete history resolves", submit(q, full));
+   T("other devices retain retirement query policy",
+     hk_sync_submit_needs_completed(q, one, 1));
+}
+
+
 /* Transfer one range [lo, hi) into a new compute stream, as the G17P
  * transfer entry points and hk_cmd_buffer_get_cs_general() do. */
 static struct hk_cs *
@@ -880,6 +1146,122 @@ test_classic_submission_order(void)
       DRM_ASAHI_BARRIER_NONE && compute->sync.prior[0] == 0);
 }
 
+static void
+test_barrier_chains_without_work(void)
+{
+   struct hk_cmd_buffer *cmd = new_cmd();
+   cmd->sync.kind_override = HK_SYNC_XFER;
+   struct hk_cs *copy = add(cmd, true);
+   cmd->sync.kind_override = 0;
+   hk_sync_barrier(cmd, HK_SYNC_XFER, HK_SYNC_COMPUTE);
+   hk_sync_barrier(cmd, HK_SYNC_COMPUTE, HK_SYNC_GFX);
+   struct hk_cs *draw = add(cmd, false);
+   T("transfer-compute-graphics chain retains transfer source",
+     draw->sync.dep[HK_SYNC_CDM].cut == copy);
+   T("transfer-compute-graphics chain reaches vertex work",
+     draw->sync.dep_ta[HK_SYNC_CDM].cut == copy);
+
+   cmd = new_cmd();
+   cmd->sync.kind_override = HK_SYNC_XFER;
+   copy = add(cmd, true);
+   cmd->sync.kind_override = 0;
+   hk_sync_barrier(cmd, HK_SYNC_XFER, HK_SYNC_COMPUTE);
+   hk_sync_barrier(cmd, HK_SYNC_GFX, HK_SYNC_GFX);
+   draw = add(cmd, false);
+   T("disjoint barrier scopes do not serialize unrelated graphics",
+     draw->sync.dep[HK_SYNC_CDM].cut == NULL);
+
+   cmd = new_cmd();
+   cmd->sync.kind_override = HK_SYNC_XFER;
+   copy = add(cmd, true);
+   cmd->sync.kind_override = 0;
+   hk_sync_barrier(cmd, HK_SYNC_XFER, HK_SYNC_GFX_FRAG);
+   hk_sync_barrier(cmd, HK_SYNC_GFX, HK_SYNC_COMPUTE);
+   struct hk_cs *compute = add(cmd, true);
+   T("fragment destination chains into a later graphics source",
+     compute->sync.dep[HK_SYNC_CDM].cut == copy);
+}
+
+/* Differential check of production resolution and history, including ring
+ * wrap, retired points, mixed engines, empty buffers and multi-buffer fallback.
+ * The two queues consume the same immutable recordings in the same order.
+ */
+static void
+test_direct_single_footprints(void)
+{
+   const unsigned sizes[] = {0, 1, 2, HK_SYNC_RECENT_FOOTPRINTS + 1};
+   for (unsigned exact = 0; exact < 2; ++exact) {
+      for (unsigned serialized = 0; serialized < 2; ++serialized) {
+         for (unsigned count = 0; count <= 2; ++count) {
+            for (unsigned s = 0; s < ARRAY_SIZE(sizes); ++s) {
+               struct hk_queue *a = new_queue(exact), *b = new_queue(exact);
+               a->sync.serialize = b->sync.serialize = serialized;
+               a->sync.precise_tile_evictions = b->sync.precise_tile_evictions = true;
+               b->sync.direct_single_footprints = true;
+               struct hk_cmd_buffer *cmds[] = {new_cmd(), new_cmd()};
+               struct hk_cs **streams = test_alloc(
+                  sizeof(*streams) * MAX2(1, 2 * sizes[s]));
+               struct hk_sync_stream *saved = test_alloc(
+                  sizeof(*saved) * MAX2(1, 2 * sizes[s]));
+               unsigned n = 0;
+               for (unsigned c = 0; c < count; ++c) {
+                  for (unsigned p = 0; p < sizes[s]; ++p) {
+                     const void *image = (void *)(uintptr_t)(0x1000 + (p % 3) * 16);
+                     streams[n++] = (p & 1)
+                        ? compute_clear_pass(cmds[c], image, 0, true)
+                        : render_pass(cmds[c], image, 1, p % 3 == 0);
+                  }
+               }
+               bool same = true;
+               const unsigned rounds = sizes[s]
+                  ? HK_SYNC_RECENT_FOOTPRINTS / sizes[s] + 3 : 3;
+               for (unsigned r = 0; r < rounds; ++r) {
+                  a->drm.timeline_value = b->drm.timeline_value = r + 1;
+                  const uint64_t completed = r % 4 ? 0 : r;
+                  const bool flush = r % 7 == 0, chain = r % 3 == 0;
+                  same &= hk_sync_resolve_submit(a, cmds, count, flush, chain, completed);
+                  for (unsigned j = 0; j < n; ++j)
+                     saved[j] = streams[j]->sync;
+                  same &= hk_sync_resolve_submit(b, cmds, count, flush, chain, completed);
+                  for (unsigned j = 0; j < n; ++j) {
+                     same &= saved[j].index == streams[j]->sync.index &&
+                             saved[j].ta_split == streams[j]->sync.ta_split;
+                     for (unsigned sq = 0; sq < HK_SYNC_NUM_SQ; ++sq) {
+                        same &= saved[j].barrier[sq] == streams[j]->sync.barrier[sq] &&
+                                saved[j].prior[sq] == streams[j]->sync.prior[sq] &&
+                                saved[j].barrier_ta[sq] == streams[j]->sync.barrier_ta[sq] &&
+                                saved[j].prior_ta[sq] == streams[j]->sync.prior_ta[sq];
+                     }
+                  }
+                  same &= a->sync.recent_count == b->sync.recent_count &&
+                          a->sync.recent_next == b->sync.recent_next &&
+                          a->sync.recent_max == b->sync.recent_max &&
+                          a->sync.evicted == b->sync.evicted &&
+                          a->sync.last_preserve == b->sync.last_preserve &&
+                          a->sync.evicted_preserve == b->sync.evicted_preserve;
+                  same &= memcmp(a->sync.ever, b->sync.ever, sizeof(a->sync.ever)) == 0 &&
+                          memcmp(a->sync.tail, b->sync.tail, sizeof(a->sync.tail)) == 0 &&
+                          memcmp(a->sync.last_absolute, b->sync.last_absolute,
+                                 sizeof(a->sync.last_absolute)) == 0 &&
+                          memcmp(a->sync.submitted, b->sync.submitted,
+                                 sizeof(a->sync.submitted)) == 0;
+                  for (unsigned j = 0; j < a->sync.recent_count; ++j) {
+                     const struct hk_sync_recent *x = &a->sync.recent[j], *y = &b->sync.recent[j];
+                     same &= x->point == y->point && x->sq == y->sq &&
+                             x->fp.image == y->fp.image && x->fp.level == y->fp.level &&
+                             x->fp.preserve == y->fp.preserve &&
+                             x->fp.layer0 == y->fp.layer0 && x->fp.layer1 == y->fp.layer1 &&
+                             x->fp.cx0 == y->fp.cx0 && x->fp.cx1 == y->fp.cx1 &&
+                             x->fp.cy0 == y->fp.cy0 && x->fp.cy1 == y->fp.cy1;
+                  }
+               }
+               T("direct single-buffer footprint resolution/history matches", same);
+            }
+         }
+      }
+   }
+}
+
 int
 main(void)
 {
@@ -887,7 +1269,10 @@ main(void)
    test_classic_submission_order();
    if (!test_context)
       return EXIT_FAILURE;
+   test_precise_tile_evictions();
+   test_direct_single_footprints();
    test_xfer_write_order();
+   test_barrier_chains_without_work();
    test_multiview_layer_preservation();
    test_compute_clear_layer_preservation();
    test_loaded_color_preservation();
@@ -895,6 +1280,7 @@ main(void)
    test_image_barrier_vertex_class();
    test_chain_first_skips_empty_buffers();
    test_rt_kernel_gate();
+   test_timestamp_execution_scopes();
    /* Each matching destination retains the exact old source prefix, including
     * reuse of a recording and independent physical destinations on the queue. */
    struct hk_queue *q = new_queue(true);
diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build
index 70c6415d25f4..8a12fc5fe63e 100644
--- a/src/compiler/nir/meson.build
+++ b/src/compiler/nir/meson.build
@@ -458,6 +458,7 @@ if with_tests and not with_nir_headers_only
           'tests/opt_licm_tests.cpp',
           'tests/opt_loop_tests.cpp',
           'tests/opt_peephole_select.cpp',
+          'tests/opt_reassociate_tests.cpp',
           'tests/opt_shrink_vectors_tests.cpp',
           'tests/opt_vectorize_tests.cpp',
           'tests/opt_varyings_tests_bicm_binary_alu.cpp',
diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h
index b2f76709d462..96421abd6215 100644
--- a/src/compiler/nir/nir.h
+++ b/src/compiler/nir/nir.h
@@ -7085,6 +7085,7 @@ typedef enum {
 
 bool nir_opt_if(nir_shader *shader, nir_opt_if_options options);
 bool nir_opt_if_empty_then(nir_shader *shader);
+bool nir_opt_if_merge(nir_shader *shader);
 
 bool nir_opt_intrinsics(nir_shader *shader);
 
@@ -7210,6 +7211,14 @@ typedef enum {
     * ALU, since non-divergence is a decent proxy for uniformity.
     */
    nir_reassociate_scalar_math = (1 << 1),
+
+   /* Within each of the classes above, rank sources that are invariant in
+    * the innermost loop around a chain below the sources varying with it.
+    * The invariant part of the chain is then computed first and can leave
+    * the loop through nir_opt_preamble or loop-invariant code motion, for
+    * example the per-iteration constants of an accumulation.
+    */
+   nir_reassociate_loop_invariant = (1 << 2),
 } nir_reassociate_options;
 
 bool nir_opt_reassociate(nir_shader *shader, nir_reassociate_options opts);
@@ -7362,6 +7371,10 @@ typedef struct nir_opt_preamble_options {
    /* True if load_workgroup_size is supported in the preamble. */
    bool load_workgroup_size_allowed;
 
+   /* Keep texture instructions in the main program, including when they are
+    * dependencies of another candidate. Default false. */
+   bool no_textures;
+
    /* size/align/class for load/store_preamble.
     *
     * Defs with class "general" will always be allocated as general. Other
diff --git a/src/compiler/nir/nir_convert_address_format.c b/src/compiler/nir/nir_convert_address_format.c
index 3dd1293a2aef..d400a19f8cd3 100644
--- a/src/compiler/nir/nir_convert_address_format.c
+++ b/src/compiler/nir/nir_convert_address_format.c
@@ -30,6 +30,12 @@ nir_build_convert_address_format(nir_builder *b, nir_def *addr,
    switch (from) {
    case nir_address_format_64bit_global:
       switch (to) {
+      case nir_address_format_64bit_global_32bit_offset_map: {
+         nir_def *parts[] = {nir_unpack_64_2x32_split_x(b, addr),
+            nir_unpack_64_2x32_split_y(b, addr), nir_imm_int(b, 0),
+            nir_imm_int(b, 0), nir_imm_int(b, 0)};
+         return nir_vec(b, parts, 5);
+      }
       case nir_address_format_64bit_global_32bit_offset:
          return nir_vec4(b, nir_unpack_64_2x32_split_x(b, addr),
                          nir_unpack_64_2x32_split_y(b, addr), nir_imm_int(b, 0),
@@ -40,6 +46,7 @@ nir_build_convert_address_format(nir_builder *b, nir_def *addr,
       }
 
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
       switch (to) {
       case nir_address_format_64bit_global:
          return nir_iadd(b, nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2)),
diff --git a/src/compiler/nir/nir_defines.h b/src/compiler/nir/nir_defines.h
index 43b3ef3b492b..69962a0df02e 100644
--- a/src/compiler/nir/nir_defines.h
+++ b/src/compiler/nir/nir_defines.h
@@ -247,6 +247,8 @@ typedef enum {
     * you can always chase the chain.
     */
    nir_address_format_logical,
+   nir_address_format_64bit_global_32bit_offset_map,
+   nir_address_format_64bit_bounded_global_map,
 } nir_address_format;
 
 typedef union {
diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c
index 2e366d038574..e05bc59ff977 100644
--- a/src/compiler/nir/nir_divergence_analysis.c
+++ b/src/compiler/nir/nir_divergence_analysis.c
@@ -987,6 +987,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state)
    case nir_intrinsic_load_scs_indexed_agx:
    case nir_intrinsic_load_scs_uniform_agx:
    case nir_intrinsic_load_scs_base_agx:
+   case nir_intrinsic_load_scs_register_agx:
    case nir_intrinsic_load_scratch:
    case nir_intrinsic_load_scratch_nv:
    case nir_intrinsic_load_scratch_intel:
@@ -1124,6 +1125,7 @@ visit_intrinsic(nir_intrinsic_instr *instr, struct divergence_state *state)
    case nir_intrinsic_load_helper_arg_hi_agx:
    case nir_intrinsic_stack_map_agx:
    case nir_intrinsic_stack_unmap_agx:
+   case nir_intrinsic_enter_query_agx:
    case nir_intrinsic_load_exported_agx:
    case nir_intrinsic_load_local_pixel_agx:
    case nir_intrinsic_load_coefficients_agx:
diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py
index f8026b78116b..11617f3143fa 100644
--- a/src/compiler/nir/nir_intrinsics.py
+++ b/src/compiler/nir/nir_intrinsics.py
@@ -253,6 +253,7 @@ def intrinsic(name, src_comp=[], dest_comp=-1, indices=[],
 
 # The nir_alu_type of input data to a store or conversion
 index("nir_alu_type", "src_type")
+index("nir_alu_type", "src_type2")
 
 # The nir_alu_type of the data output from a load or conversion
 index("nir_alu_type", "dest_type")
@@ -1570,6 +1571,8 @@ def store(name, srcs, indices=[], flags=[]):
 intrinsic("cmat_copy", src_comp=[-1, -1])
 intrinsic("cmat_transpose", src_comp=[-1, -1], indices=[SATURATE, CMAT_SIGNED_MASK, FP_MATH_CTRL])
 intrinsic("cmat_get_coordinate", src_comp=[1], dest_comp=2, indices=[CMAT_DESC], bit_sizes=[32])
+intrinsic("cmat_muladd_agx", src_comp=[0, 0, 0], dest_comp=0, bit_sizes=[32], indices=[SRC_TYPE, SRC_TYPE2],
+          flags=[CAN_ELIMINATE, SUBGROUP])
 
 # src[] = { deref }.
 load("buffer_ptr_deref", [-1], [ACCESS, RESOURCE_TYPE],
@@ -2606,6 +2609,10 @@ def store(name, srcs, indices=[], flags=[]):
 # src[] = { sample mask, depth, stencil }
 intrinsic("store_zs_agx", [1, 1, 1], indices=[BASE], flags=[])
 
+# M5: open (FLAGS = 0) or close (FLAGS = 1) the depth/coverage pixel
+# transaction around a loop of store_zs_agx carrying AGX_ZS_COVERED_SAMPLE.
+intrinsic("zs_transaction_agx", indices=[FLAGS])
+
 # Store a block from local memory into a bound image. Used to write out render
 # targets within the end-of-tile shader, although it is valid in general compute
 # kernels.
@@ -2697,6 +2704,9 @@ def store(name, srcs, indices=[], flags=[]):
 # Loads the sample position array as fixed point packed into a 32-bit word
 system_value("sample_positions_agx", 1, bit_sizes=[32])
 
+# Upper four fixed-point sample positions on an eight-sample framebuffer.
+system_value("sample_positions_hi_agx", 1, bit_sizes=[32])
+
 # In a non-monolithic fragment shader part, returns whether this shader part is
 # responsible for Z/S testing after its final discard. ~0/0 boolean.
 system_value("shader_part_tests_zs_agx", 1, bit_sizes=[16])
@@ -2749,6 +2759,16 @@ def store(name, srcs, indices=[], flags=[]):
 intrinsic("load_scs_base_agx", [1], dest_comp=0, bit_sizes=[16, 32],
           indices=[BASE, FORMAT, COMPONENT, FLAGS])
 intrinsic("store_scs_base_agx", [0, 1], indices=[BASE, FORMAT, COMPONENT])
+intrinsic("load_scs_register_agx", [1, 1], dest_comp=0, bit_sizes=[16, 32],
+          indices=[BASE, FORMAT, COMPONENT, FLAGS])
+
+# Two 64-bit native operands followed by the three 32-bit launch words
+# published to staging halves 0/2/4. Completion is observed through SCS.
+intrinsic("intersect_ray_agx", [1, 1, 1, 1, 1])
+intrinsic("continue_ray_agx", [1])
+# Native query stack entry. The hardware writes a 32-bit query pointer to the
+# f2 register; BASE is the native allocation index (f24, witnessed 4/5).
+intrinsic("enter_query_agx", [], dest_comp=1, bit_sizes=[32], indices=[BASE])
 
 
 # Helper shader intrinsics
@@ -3402,19 +3422,31 @@ def store(name, srcs, indices=[], flags=[]):
 # Loads the viewport Z range for a given viewport index.
 load("viewport_z_range_kk", [1], [], [CAN_ELIMINATE, CAN_REORDER])
 
-# Native G17P/G17X SR59 identifies the resident SCS thread.
 system_value("scs_thread_index_agx", 1, bit_sizes=[32])
 
+# Deferred guarded global read. Source0 is the64-bit base, source1 the32-bit
+# byte offset, and source2 the exact frontend bounds predicate. False returns
+# zero without a memory access. AGX lowers this after all speculation passes.
+load("global_predicated_agx", [1, 1, 1],
+     indices=[ACCESS, ALIGN_MUL, ALIGN_OFFSET], flags=[CAN_ELIMINATE])
+# Native mesh allocation consumes two scalar 16-bit operands. UVST reads
+# a contiguous vec2-u32 descriptor; FLAGS selects the witnessed primitive leaf.
 intrinsic("allocate_mesh_uvb_agx", [1, 1])
 intrinsic("store_mesh_uvb_agx", [2], indices=[FLAGS])
+# Object invocation0/1/2 publishes one child-grid dimension through UVSW.
 intrinsic("store_object_childcount_agx", [1])
 
+# Native object/mesh payload pointer from witnessed SR62 low32/SR63 high16.
 intrinsic("load_mesh_payload_address_agx", dest_comp=2, bit_sizes=[32],
           flags=[CAN_ELIMINATE, CAN_REORDER])
 
+# Native per-workgroup mesh per-primitive record base: SR64 low32 / SR65 high16
+# (primitive_color and primitive_id witnesses, main and write_uvb helper).
 intrinsic("load_mesh_primitive_storage_agx", dest_comp=2, bit_sizes=[32],
           flags=[CAN_ELIMINATE, CAN_REORDER])
 
+# Exact captured native mesh payload entry barrier; anonymous selector fields.
 intrinsic("mesh_payload_entry_barrier_agx")
 
+# Native object.write_childcount payload publication barrier.
 intrinsic("object_payload_exit_barrier_agx")
diff --git a/src/compiler/nir/nir_lower_explicit_io.c b/src/compiler/nir/nir_lower_explicit_io.c
index 93671509e049..aa2ccc05e514 100644
--- a/src/compiler/nir/nir_lower_explicit_io.c
+++ b/src/compiler/nir/nir_lower_explicit_io.c
@@ -134,7 +134,8 @@ build_addr_ushr_imm(nir_builder *b, nir_def *addr,
          b, addr, nir_ushr_imm(b, nir_channel(b, addr, 1), shift), 1);
 
    case nir_address_format_64bit_global_32bit_offset:
-      assert(addr->num_components == 4);
+   case nir_address_format_64bit_global_32bit_offset_map:
+      assert(addr->num_components == nir_address_format_num_components(addr_format));
       return nir_vector_insert_imm(
          b, addr, nir_ushr_imm(b, nir_channel(b, addr, 3), shift), 3);
 
@@ -154,7 +155,8 @@ build_addr_for_var(nir_builder *b, nir_variable *var,
                             nir_var_mem_push_const | nir_var_mem_constant));
 
    const nir_address_format base_addr_format =
-      addr_format == nir_address_format_64bit_global_32bit_offset
+      (addr_format == nir_address_format_64bit_global_32bit_offset ||
+       addr_format == nir_address_format_64bit_global_32bit_offset_map)
          ? nir_address_format_64bit_global
          : addr_format;
    const unsigned num_comps = nir_address_format_num_components(base_addr_format);
@@ -164,7 +166,8 @@ build_addr_for_var(nir_builder *b, nir_variable *var,
    case nir_address_format_2x32bit_global:
    case nir_address_format_32bit_global:
    case nir_address_format_64bit_global:
-   case nir_address_format_64bit_global_32bit_offset: {
+   case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map: {
       nir_def *base_addr;
       switch (var->data.mode) {
       case nir_var_shader_temp:
@@ -276,8 +279,10 @@ nir_address_format_bit_size(nir_address_format addr_format)
    case nir_address_format_64bit_global:
       return 64;
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
       return 32;
    case nir_address_format_64bit_bounded_global:
+   case nir_address_format_64bit_bounded_global_map:
       return 32;
    case nir_address_format_32bit_index_offset:
       return 32;
@@ -301,6 +306,9 @@ unsigned
 nir_address_format_num_components(nir_address_format addr_format)
 {
    switch (addr_format) {
+   case nir_address_format_64bit_global_32bit_offset_map:
+   case nir_address_format_64bit_bounded_global_map:
+      return 5;
    case nir_address_format_32bit_global:
       return 1;
    case nir_address_format_2x32bit_global:
@@ -381,8 +389,10 @@ addr_format_is_global(nir_address_format addr_format,
    return addr_format == nir_address_format_32bit_global ||
           addr_format == nir_address_format_2x32bit_global ||
           addr_format == nir_address_format_64bit_global ||
-          addr_format == nir_address_format_64bit_global_32bit_offset ||
-          addr_format == nir_address_format_64bit_bounded_global;
+          (addr_format == nir_address_format_64bit_global_32bit_offset ||
+       addr_format == nir_address_format_64bit_global_32bit_offset_map) ||
+          (addr_format == nir_address_format_64bit_bounded_global ||
+       addr_format == nir_address_format_64bit_bounded_global_map);
 }
 
 static bool
@@ -412,8 +422,10 @@ addr_to_global(nir_builder *b, nir_def *addr,
       return addr;
 
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
    case nir_address_format_64bit_bounded_global:
-      assert(addr->num_components == 4);
+   case nir_address_format_64bit_bounded_global_map:
+      assert(addr->num_components == nir_address_format_num_components(addr_format));
       return nir_iadd(b, nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2)),
                       nir_u2u64(b, nir_channel(b, addr, 3)));
 
@@ -432,15 +444,17 @@ addr_to_global(nir_builder *b, nir_def *addr,
 static bool
 addr_format_needs_bounds_check(nir_address_format addr_format)
 {
-   return addr_format == nir_address_format_64bit_bounded_global;
+   return (addr_format == nir_address_format_64bit_bounded_global ||
+       addr_format == nir_address_format_64bit_bounded_global_map);
 }
 
 static nir_def *
 addr_is_in_bounds(nir_builder *b, nir_def *addr,
                   nir_address_format addr_format, unsigned size)
 {
-   assert(addr_format == nir_address_format_64bit_bounded_global);
-   assert(addr->num_components == 4);
+   assert((addr_format == nir_address_format_64bit_bounded_global ||
+       addr_format == nir_address_format_64bit_bounded_global_map));
+   assert(addr->num_components == nir_address_format_num_components(addr_format));
    assert(size > 0);
    return nir_ult(b, nir_iadd_imm(b, nir_channel(b, addr, 3), size - 1),
                   nir_channel(b, addr, 2));
@@ -665,9 +679,11 @@ build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin,
    case nir_intrinsic_load_deref:
       switch (mode) {
       case nir_var_mem_ubo:
-         if (addr_format == nir_address_format_64bit_global_32bit_offset)
+         if ((addr_format == nir_address_format_64bit_global_32bit_offset ||
+       addr_format == nir_address_format_64bit_global_32bit_offset_map))
             op = nir_intrinsic_load_global_constant_offset;
-         else if (addr_format == nir_address_format_64bit_bounded_global)
+         else if ((addr_format == nir_address_format_64bit_bounded_global ||
+       addr_format == nir_address_format_64bit_bounded_global_map))
             op = nir_intrinsic_load_global_constant_bounded;
          else if (addr_format_is_global(addr_format, mode))
             op = nir_intrinsic_load_global_constant;
@@ -675,7 +691,8 @@ build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin,
             op = nir_intrinsic_load_ubo;
          break;
       case nir_var_mem_ssbo:
-         if (addr_format == nir_address_format_64bit_bounded_global &&
+         if ((addr_format == nir_address_format_64bit_bounded_global ||
+       addr_format == nir_address_format_64bit_bounded_global_map) &&
              b->shader->options->has_load_global_bounded)
             op = nir_intrinsic_load_global_bounded;
          else if (addr_format_is_global(addr_format, mode))
@@ -770,13 +787,15 @@ build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin,
 
    if (op == nir_intrinsic_load_global_constant_offset ||
        op == nir_intrinsic_load_global_offset) {
-      assert(addr_format == nir_address_format_64bit_global_32bit_offset);
+      assert((addr_format == nir_address_format_64bit_global_32bit_offset ||
+       addr_format == nir_address_format_64bit_global_32bit_offset_map));
       load->src[0] = nir_src_for_ssa(
          nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2)));
       load->src[1] = nir_src_for_ssa(nir_channel(b, addr, 3));
    } else if (op == nir_intrinsic_load_global_bounded ||
               op == nir_intrinsic_load_global_constant_bounded) {
-      assert(addr_format == nir_address_format_64bit_bounded_global);
+      assert((addr_format == nir_address_format_64bit_bounded_global ||
+       addr_format == nir_address_format_64bit_bounded_global_map));
       load->src[0] = nir_src_for_ssa(
          nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2)));
       load->src[1] = nir_src_for_ssa(nir_channel(b, addr, 3));
@@ -1015,7 +1034,8 @@ build_explicit_io_store(nir_builder *b, nir_intrinsic_instr *intrin,
 
    store->src[0] = nir_src_for_ssa(value);
    if (op == nir_intrinsic_store_global_offset) {
-      assert(addr_format == nir_address_format_64bit_global_32bit_offset);
+      assert((addr_format == nir_address_format_64bit_global_32bit_offset ||
+       addr_format == nir_address_format_64bit_global_32bit_offset_map));
       store->src[1] = nir_src_for_ssa(
          nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2)));
       store->src[2] = nir_src_for_ssa(nir_channel(b, addr, 3));
@@ -1750,7 +1770,9 @@ lower_explicit_io_array_length(nir_builder *b, nir_intrinsic_instr *intrin,
    nir_def *offset, *size;
    switch (addr_format) {
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
    case nir_address_format_64bit_bounded_global:
+   case nir_address_format_64bit_bounded_global_map:
       offset = nir_channel(b, addr, 3);
       size = nir_channel(b, addr, 2);
       break;
@@ -2307,7 +2329,9 @@ nir_address_format_null_value(nir_address_format addr_format)
       [nir_address_format_2x32bit_global] = { { 0 } },
       [nir_address_format_64bit_global] = { { 0 } },
       [nir_address_format_64bit_global_32bit_offset] = { { 0 } },
+      [nir_address_format_64bit_global_32bit_offset_map] = { { 0 } },
       [nir_address_format_64bit_bounded_global] = { { 0 } },
+      [nir_address_format_64bit_bounded_global_map] = { { 0 } },
       [nir_address_format_32bit_index_offset] = { { .u32 = ~0 }, { .u32 = ~0 } },
       [nir_address_format_32bit_index_offset_pack64] = { { .u64 = ~0ull } },
       [nir_address_format_vec2_index_32bit_offset] = { { .u32 = ~0 }, { .u32 = ~0 }, { .u32 = ~0 } },
@@ -2326,6 +2350,9 @@ nir_build_addr_ieq(nir_builder *b, nir_def *addr0, nir_def *addr1,
                    nir_address_format addr_format)
 {
    switch (addr_format) {
+   case nir_address_format_64bit_bounded_global_map:
+      return nir_ball_iequal(b, nir_trim_vector(b, addr0, 4),
+                               nir_trim_vector(b, addr1, 4));
    case nir_address_format_32bit_global:
    case nir_address_format_2x32bit_global:
    case nir_address_format_64bit_global:
@@ -2337,6 +2364,7 @@ nir_build_addr_ieq(nir_builder *b, nir_def *addr0, nir_def *addr1,
       return nir_ball_iequal(b, addr0, addr1);
 
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
       return nir_ball_iequal(b, nir_channels(b, addr0, 0xb),
                              nir_channels(b, addr1, 0xb));
 
@@ -2379,7 +2407,9 @@ nir_build_addr_isub(nir_builder *b, nir_def *addr0, nir_def *addr1,
       return nir_u2u64(b, nir_isub(b, nir_u2u32(b, addr0), nir_u2u32(b, addr1)));
 
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
    case nir_address_format_64bit_bounded_global:
+   case nir_address_format_64bit_bounded_global_map:
       return nir_isub(b, addr_to_global(b, addr0, addr_format),
                       addr_to_global(b, addr1, addr_format));
 
@@ -2434,8 +2464,10 @@ nir_build_addr_iadd(nir_builder *b, nir_def *addr,
       return nir_u2u64(b, nir_iadd(b, nir_u2u32(b, addr), offset));
 
    case nir_address_format_64bit_global_32bit_offset:
+   case nir_address_format_64bit_global_32bit_offset_map:
    case nir_address_format_64bit_bounded_global:
-      assert(addr->num_components == 4);
+   case nir_address_format_64bit_bounded_global_map:
+      assert(addr->num_components == nir_address_format_num_components(addr_format));
       assert(addr->bit_size == offset->bit_size);
       return nir_vector_insert_imm(b, addr, nir_iadd(b, nir_channel(b, addr, 3), offset), 3);
 
diff --git a/src/compiler/nir/nir_opt_if.c b/src/compiler/nir/nir_opt_if.c
index 3a144fbdc07d..c1c069fc0bc4 100644
--- a/src/compiler/nir/nir_opt_if.c
+++ b/src/compiler/nir/nir_opt_if.c
@@ -1629,6 +1629,53 @@ nir_opt_if_empty_then(nir_shader *shader)
    return progress;
 }
 
+/* Only the merge of adjacent if-statements with the same condition, for
+ * callers whose late code motion empties the block between two such ifs and
+ * which must not have the rest of nir_opt_if reshape their control flow. */
+static bool
+opt_if_merge_cf_list(struct exec_list *cf_list)
+{
+   bool progress = false;
+   foreach_list_typed(nir_cf_node, cf_node, node, cf_list) {
+      switch (cf_node->type) {
+      case nir_cf_node_block:
+         break;
+
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(cf_node);
+         progress |= opt_if_merge_cf_list(&nif->then_list);
+         progress |= opt_if_merge_cf_list(&nif->else_list);
+         /* Absorb every following if with the same condition. */
+         while (opt_if_merge(nif))
+            progress = true;
+         break;
+      }
+
+      case nir_cf_node_loop: {
+         nir_loop *loop = nir_cf_node_as_loop(cf_node);
+         assert(!nir_loop_has_continue_construct(loop));
+         progress |= opt_if_merge_cf_list(&loop->body);
+         break;
+      }
+
+      case nir_cf_node_function:
+         UNREACHABLE("Invalid cf type");
+      }
+   }
+   return progress;
+}
+
+bool
+nir_opt_if_merge(nir_shader *shader)
+{
+   bool progress = false;
+   nir_foreach_function_impl(impl, shader) {
+      bool impl_progress = opt_if_merge_cf_list(&impl->body);
+      progress |= nir_progress(impl_progress, impl, nir_metadata_none);
+   }
+   return progress;
+}
+
 bool
 nir_opt_if(nir_shader *shader, nir_opt_if_options options)
 {
diff --git a/src/compiler/nir/nir_opt_preamble.c b/src/compiler/nir/nir_opt_preamble.c
index 54a37dee8e49..280ed4361fa3 100644
--- a/src/compiler/nir/nir_opt_preamble.c
+++ b/src/compiler/nir/nir_opt_preamble.c
@@ -267,6 +267,9 @@ can_move_intrinsic(nir_intrinsic_instr *instr, opt_preamble_ctx *ctx)
    case nir_intrinsic_image_samples_identical:
    case nir_intrinsic_bindless_image_load:
    case nir_intrinsic_image_heap_load:
+      if (ctx->options->no_textures)
+         return false;
+      FALLTHROUGH;
    case nir_intrinsic_load_global_bounded:
    case nir_intrinsic_load_ssbo:
    case nir_intrinsic_load_ssbo_intel:
@@ -293,6 +296,8 @@ can_move_instr(nir_instr *instr, opt_preamble_ctx *ctx)
 
    switch (instr->type) {
    case nir_instr_type_tex: {
+      if (ctx->options->no_textures)
+         return false;
       nir_tex_instr *tex = nir_instr_as_tex(instr);
       /* See note below about derivatives. We have special code to convert tex
        * to txd, though, because it's a common case.
diff --git a/src/compiler/nir/nir_opt_reassociate.c b/src/compiler/nir/nir_opt_reassociate.c
index b3ce295b2941..2ce0317b260a 100644
--- a/src/compiler/nir/nir_opt_reassociate.c
+++ b/src/compiler/nir/nir_opt_reassociate.c
@@ -154,25 +154,135 @@ lookup_pair_freq(struct hash_table *ht, struct pair_key key)
    return (uintptr_t)(_mesa_hash_table_search(ht, &key)->data);
 }
 
+/* Bound on the expression depth walked to prove a source loop-invariant. */
+#define MAX_INVARIANT_DEPTH 32
+
+struct reassociate_state {
+   nir_reassociate_options opts;
+
+   /* Loop invariance memo: nir_def -> (loop << 1) | invariant. */
+   struct hash_table *invariant;
+};
+
+static nir_loop *
+innermost_loop(nir_block *block)
+{
+   for (nir_cf_node *node = block->cf_node.parent; node; node = node->parent) {
+      if (node->type == nir_cf_node_loop)
+         return nir_cf_node_as_loop(node);
+   }
+
+   return NULL;
+}
+
+static bool
+loop_contains(nir_loop *loop, nir_block *block)
+{
+   for (nir_cf_node *node = block->cf_node.parent; node; node = node->parent) {
+      if (node == &loop->cf_node)
+         return true;
+   }
+
+   return false;
+}
+
+struct invariant_walk {
+   struct reassociate_state *state;
+   nir_loop *loop;
+   unsigned depth;
+
+   /* The depth bound cut the walk short somewhere below the current def. */
+   bool limited;
+};
+
+static bool def_loop_invariant(struct invariant_walk *w, nir_def *def);
+
+static bool
+src_loop_invariant(nir_src *src, void *data)
+{
+   return def_loop_invariant(data, src->ssa);
+}
+
+/*
+ * Whether def has the same value in every iteration of the loop: defined
+ * outside it, or computed inside it without side effects from such values.
+ * This only steers the order of a chain, so being conservative only costs
+ * optimization.
+ */
+static bool
+def_loop_invariant(struct invariant_walk *w, nir_def *def)
+{
+   nir_instr *instr = nir_def_instr(def);
+   if (!loop_contains(w->loop, instr->block))
+      return true;
+
+   struct hash_entry *ent = _mesa_hash_table_search(w->state->invariant, def);
+   if (ent && ((uintptr_t)ent->data & ~(uintptr_t)1) == (uintptr_t)w->loop)
+      return (uintptr_t)ent->data & 1;
+
+   const bool outer_limited = w->limited;
+   w->limited = w->depth >= MAX_INVARIANT_DEPTH;
+
+   bool invariant = false;
+   if (!w->limited) {
+      switch (instr->type) {
+      case nir_instr_type_load_const:
+      case nir_instr_type_undef:
+         invariant = true;
+         break;
+      case nir_instr_type_intrinsic:
+         if (!nir_intrinsic_can_reorder(nir_instr_as_intrinsic(instr)))
+            break;
+         FALLTHROUGH;
+      case nir_instr_type_alu:
+      case nir_instr_type_tex:
+         w->depth++;
+         invariant = nir_foreach_src(instr, src_loop_invariant, w);
+         w->depth--;
+         break;
+      default:
+         break;
+      }
+   }
+
+   /* A result cut short by the depth bound is not final. */
+   if (invariant || !w->limited) {
+      _mesa_hash_table_insert(w->state->invariant, def,
+                              (void *)((uintptr_t)w->loop | invariant));
+   }
+
+   w->limited |= outer_limited;
+   return invariant;
+}
+
 static int
-rank(nir_scalar s)
+rank(struct reassociate_state *state, nir_loop *loop, nir_scalar s)
 {
    /* Constants are rank 0. This promotes constant folding. */
    if (nir_scalar_is_const(s))
       return 0;
 
-   /* Convergent expressions are rank 1, promoting preambles and scalar ALU */
-   if (!s.def->divergent)
-      return 1;
+   /* Convergent expressions rank below divergent ones, promoting preambles
+    * and scalar ALU.
+    */
+   int r = s.def->divergent ? 2 : 1;
+
+   /* Optionally split each class by loop invariance, so the invariant part
+    * of a chain inside a loop is computed first and can leave the loop.
+    */
+   if (state->opts & nir_reassociate_loop_invariant) {
+      struct invariant_walk w = { .state = state, .loop = loop };
+      r = 2 * r - (!loop || def_loop_invariant(&w, s.def) ? 1 : 0);
+   }
 
-   /* Everything else is rank 2. TODO: Promote loop-invariant code motion. */
-   return 2;
+   return r;
 }
 
 struct chain {
    nir_alu_instr *root;
    unsigned length;
    nir_scalar srcs[MAX_CHAIN_LENGTH];
+   int ranks[MAX_CHAIN_LENGTH];
    bool do_global_cse;
    unsigned fp_math_ctrl;
 };
@@ -284,8 +394,8 @@ record_pairs(struct chain *c, struct hash_table *pair_freq)
  * ensures we do not duplicate chains and keeps `find_chains` O(instructions).
  */
 static void
-find_chains(nir_function_impl *impl, struct hash_table *pair_freq,
-            struct util_dynarray *chains)
+find_chains(nir_function_impl *impl, struct reassociate_state *state,
+            struct hash_table *pair_freq, struct util_dynarray *chains)
 {
    nir_foreach_block_reverse(block, impl) {
       nir_foreach_instr_reverse(instr, block) {
@@ -311,20 +421,24 @@ find_chains(nir_function_impl *impl, struct hash_table *pair_freq,
          if (c.length < 3)
             continue;
 
+         nir_loop *loop = innermost_loop(block);
+         for (unsigned i = 0; i < c.length; ++i)
+            c.ranks[i] = rank(state, loop, c.srcs[i]);
+
          /* Analyze the chain to feed our heuristic */
          unsigned lowest_rank = UINT32_MAX, nr_lowest = 0;
          unsigned highest_rank = 0, nr_highest = 0;
          bool local = true;
 
          for (unsigned i = 0; i < c.length; ++i) {
-            lowest_rank = MIN2(rank(c.srcs[i]), lowest_rank);
-            highest_rank = MAX2(rank(c.srcs[i]), highest_rank);
+            lowest_rank = MIN2(c.ranks[i], lowest_rank);
+            highest_rank = MAX2(c.ranks[i], highest_rank);
             local &= nir_def_block(c.srcs[i].def) == block;
          }
 
          for (unsigned i = 0; i < c.length; ++i) {
-            nr_lowest += (rank(c.srcs[i]) == lowest_rank);
-            nr_highest += (rank(c.srcs[i]) == highest_rank);
+            nr_lowest += (c.ranks[i] == lowest_rank);
+            nr_highest += (c.ranks[i] == highest_rank);
          }
 
          /* If we don't have the pair_freq table, the caller doesn't want to use
@@ -419,7 +533,7 @@ find_best_pair_in_chain(struct chain *c, void *pair_freq)
    foreach_pair(c, i, j) {
       struct pair_key key = get_pair_key(c->root->op, c->srcs[i], c->srcs[j]);
       unsigned freq = lookup_pair_freq(pair_freq, key);
-      unsigned max_rank = MAX2(rank(c->srcs[i]), rank(c->srcs[j]));
+      unsigned max_rank = MAX2(c->ranks[i], c->ranks[j]);
 
       if (freq > best_freq || (freq == best_freq && max_rank < best_max_rank)) {
          best = (struct pair){i, j};
@@ -431,17 +545,46 @@ find_best_pair_in_chain(struct chain *c, void *pair_freq)
    return best_freq > 1 ? best : (struct pair){0};
 }
 
+struct ranked_src {
+   nir_scalar src;
+   int rank;
+};
+
 /* Compare ranks. Tie break to ensure the sort-by-rank sort is stable */
 static int
 cmp_rank(const void *a_, const void *b_)
 {
-   const nir_scalar *a = a_, *b = b_;
-   int ra = rank(*a), rb = rank(*b);
+   const struct ranked_src *a = a_, *b = b_;
 
-   if (ra != rb)
-      return ra - rb;
+   if (a->rank != b->rank)
+      return a->rank - b->rank;
    else
-      return a->def->index - b->def->index;
+      return a->src.def->index - b->src.def->index;
+}
+
+static void
+swap_srcs(struct chain *c, unsigned a, unsigned b)
+{
+   SWAP(c->srcs[a], c->srcs[b]);
+   SWAP(c->ranks[a], c->ranks[b]);
+}
+
+/* Sort the sources from `first` by rank. */
+static void
+sort_by_rank(struct chain *c, unsigned first)
+{
+   struct ranked_src srcs[MAX_CHAIN_LENGTH];
+   const unsigned n = c->length - first;
+
+   for (unsigned i = 0; i < n; ++i)
+      srcs[i] = (struct ranked_src){c->srcs[first + i], c->ranks[first + i]};
+
+   qsort(srcs, n, sizeof(srcs[0]), cmp_rank);
+
+   for (unsigned i = 0; i < n; ++i) {
+      c->srcs[first + i] = srcs[i].src;
+      c->ranks[first + i] = srcs[i].rank;
+   }
 }
 
 static bool
@@ -458,13 +601,13 @@ reassociate_chain(struct chain *c, void *pair_freq)
 
       if (best_pair.i != best_pair.j) {
          /* Pin the best pair at the front. The rest is sorted by rank. */
-         SWAP(c->srcs[0], c->srcs[best_pair.i]);
-         SWAP(c->srcs[1], c->srcs[best_pair.j]);
+         swap_srcs(c, 0, best_pair.i);
+         swap_srcs(c, 1, best_pair.j);
          pinned = 2;
       }
    }
 
-   qsort(c->srcs + pinned, c->length - pinned, sizeof(c->srcs[0]), cmp_rank);
+   sort_by_rank(c, pinned);
 
    /* Reassociate according to the new order */
    nir_def *new_root = nir_mov_scalar(&b, c->srcs[0]);
@@ -509,6 +652,10 @@ nir_opt_reassociate(nir_shader *nir, nir_reassociate_options opts)
    bool cse_heuristic = opts & nir_reassociate_cse_heuristic;
    struct hash_table *pair_freq =
       cse_heuristic ? pair_key_table_create(NULL) : NULL;
+   struct reassociate_state state = {
+      .opts = opts,
+      .invariant = _mesa_pointer_hash_table_create(NULL),
+   };
    struct util_dynarray chains;
    bool progress = false;
 
@@ -529,10 +676,11 @@ nir_opt_reassociate(nir_shader *nir, nir_reassociate_options opts)
 
       bool impl_progress = false;
       _mesa_hash_table_clear(pair_freq, NULL);
+      _mesa_hash_table_clear(state.invariant, NULL);
       util_dynarray_clear(&chains);
 
       /* Step 1: find all chains in the function */
-      find_chains(impl, pair_freq, &chains);
+      find_chains(impl, &state, pair_freq, &chains);
 
       /* Step 2: reassociate all chains */
       util_dynarray_foreach(&chains, struct chain, chain) {
@@ -544,6 +692,7 @@ nir_opt_reassociate(nir_shader *nir, nir_reassociate_options opts)
    }
 
    ralloc_free(pair_freq);
+   ralloc_free(state.invariant);
    util_dynarray_fini(&chains);
    return progress;
 }
@@ -564,8 +713,10 @@ nir_opt_reassociate_loop(nir_shader *nir, nir_reassociate_options in_opts)
    for (unsigned i = 0; i < 2; ++i) {
       nir_reassociate_options opts = in_opts;
       if (i >= 1) {
-         if (in_opts == (nir_reassociate_cse_heuristic | nir_reassociate_scalar_math))
-            opts = nir_reassociate_scalar_math;
+         const nir_reassociate_options both =
+            nir_reassociate_cse_heuristic | nir_reassociate_scalar_math;
+         if ((in_opts & both) == both)
+            opts = in_opts & ~nir_reassociate_cse_heuristic;
          else
             break;
       }
diff --git a/src/compiler/nir/tests/opt_if_tests.cpp b/src/compiler/nir/tests/opt_if_tests.cpp
index 3bf4c58de11e..83e21f10d232 100644
--- a/src/compiler/nir/tests/opt_if_tests.cpp
+++ b/src/compiler/nir/tests/opt_if_tests.cpp
@@ -341,3 +341,46 @@ TEST_P(nir_opt_if_merge_test, opt_if_merge)
    ASSERT_EQ(exec_list_length((&nir_if_first_then_block(next_if)->instr_list)), instr_in_2nd_then);
    ASSERT_EQ(exec_list_length((&nir_if_first_else_block(next_if)->instr_list)), instr_in_2nd_else);
 }
+
+TEST_F(nir_opt_if_test, opt_if_merge_run)
+{
+   /* nir_opt_if_merge folds a run of same-condition ifs, separated by nothing
+    * but phis, into the first one in a single call:
+    *
+    * if (c) { a = in + 1 } else {}  p = phi(a, 0)
+    * if (c) { b = p + 2 } else {}   q = phi(b, 0)
+    * if (c) { d = q + 3 } else {}   r = phi(d, 0)
+    */
+   nir_def *zero = nir_imm_int(b, 0);
+   nir_def *cond = nir_ine(b, in_def, zero);
+   nir_def *value = in_def;
+   nir_if *first = NULL;
+
+   for (int i = 1; i <= 3; ++i) {
+      nir_if *nif = nir_push_if(b, cond);
+      first = first ? first : nif;
+      nir_def *sum = nir_iadd_imm(b, value, i);
+      nir_pop_if(b, NULL);
+      value = nir_if_phi(b, sum, zero);
+   }
+
+   nir_store_var(b, out_var, value, 1);
+
+   ASSERT_TRUE(nir_opt_if_merge(b->shader));
+   nir_validate_shader(b->shader, NULL);
+
+   unsigned ifs = 0;
+   foreach_list_typed(nir_cf_node, node, node, &b->impl->body)
+      ifs += node->type == nir_cf_node_if;
+   ASSERT_EQ(ifs, 1u);
+
+   /* The three additions now chain inside the merged then-branch. */
+   unsigned adds = 0;
+   nir_foreach_instr(instr, nir_if_first_then_block(first)) {
+      adds += instr->type == nir_instr_type_alu &&
+              nir_instr_as_alu(instr)->op == nir_op_iadd;
+   }
+   ASSERT_EQ(adds, 3u);
+
+   ASSERT_FALSE(nir_opt_if_merge(b->shader));
+}
diff --git a/src/compiler/nir/tests/opt_reassociate_tests.cpp b/src/compiler/nir/tests/opt_reassociate_tests.cpp
new file mode 100644
index 000000000000..82f455f2164c
--- /dev/null
+++ b/src/compiler/nir/tests/opt_reassociate_tests.cpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026 Aurora Silicon
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nir_test.h"
+
+class nir_opt_reassociate_test : public nir_test {
+protected:
+   nir_opt_reassociate_test()
+      : nir_test::nir_test("nir_opt_reassociate_test", MESA_SHADER_COMPUTE)
+   {
+   }
+
+   nir_def *x, *inv, *root;
+
+   /* Inside a loop, r = ((v1 + |x|) + v2) + x with v1 and v2 loaded in the
+    * loop body and x loaded before it. */
+   void build()
+   {
+      nir_def *addr = nir_undef(b, 1, 64);
+      x = nir_load_global(b, 1, 32, addr);
+
+      nir_loop *loop = nir_push_loop(b);
+      nir_def *v1 = nir_load_global(b, 1, 32, addr);
+      inv = nir_iabs(b, x);
+      nir_def *v2 = nir_load_global(b, 1, 32, addr);
+      root = nir_iadd(b, nir_iadd(b, nir_iadd(b, v1, inv), v2), x);
+      nir_store_global(b, root, addr);
+      nir_break_if(b, nir_undef(b, 1, 1));
+      nir_pop_loop(b, loop);
+   }
+
+   /* Whether some iadd adds exactly the two loop-invariant terms. */
+   bool invariant_pair()
+   {
+      nir_foreach_block(block, b->impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_alu)
+               continue;
+            nir_alu_instr *alu = nir_instr_as_alu(instr);
+            if (alu->op != nir_op_iadd || list_is_empty(&alu->def.uses))
+               continue;
+            nir_def *a = alu->src[0].src.ssa, *c = alu->src[1].src.ssa;
+            while (nir_def_is_alu(a) && nir_def_as_alu(a)->op == nir_op_mov)
+               a = nir_def_as_alu(a)->src[0].src.ssa;
+            while (nir_def_is_alu(c) && nir_def_as_alu(c)->op == nir_op_mov)
+               c = nir_def_as_alu(c)->src[0].src.ssa;
+            if ((a == x && c == inv) || (a == inv && c == x))
+               return true;
+         }
+      }
+      return false;
+   }
+};
+
+TEST_F(nir_opt_reassociate_test, loop_invariant_terms_first)
+{
+   build();
+   ASSERT_TRUE(nir_opt_reassociate(b->shader, nir_reassociate_loop_invariant));
+   nir_validate_shader(b->shader, NULL);
+   nir_opt_dce(b->shader);
+   ASSERT_TRUE(invariant_pair());
+}
+
+TEST_F(nir_opt_reassociate_test, loop_invariance_off)
+{
+   /* Without the option every source has the same rank: nothing to do. */
+   build();
+   ASSERT_FALSE(nir_opt_reassociate(b->shader, (nir_reassociate_options)0));
+   ASSERT_FALSE(invariant_pair());
+}
diff --git a/src/gallium/drivers/asahi/agx_pipe.c b/src/gallium/drivers/asahi/agx_pipe.c
index bb4b20e3969f..c769f062345d 100644
--- a/src/gallium/drivers/asahi/agx_pipe.c
+++ b/src/gallium/drivers/asahi/agx_pipe.c
@@ -1949,12 +1949,18 @@ agx_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
       priority = DRM_ASAHI_PRIORITY_LOW;
    else if (flags & PIPE_CONTEXT_PRIORITY_MEDIUM)
       priority = DRM_ASAHI_PRIORITY_MEDIUM;
-
-   /* TODO: High/realtime need us to handle errors since we might not have
-    * permission. Sort this out later.
-    */
+   else if (flags & (PIPE_CONTEXT_PRIORITY_HIGH | PIPE_CONTEXT_PRIORITY_REALTIME))
+      priority = DRM_ASAHI_PRIORITY_HIGH;
 
    ctx->queue_id = agx_create_command_queue(agx_device(screen), priority);
+
+   /* Kernels grant HIGH only on G17, to CAP_SYS_NICE or DRM-master callers;
+    * otherwise fall back to MEDIUM.
+    */
+   if (ctx->queue_id == UINT32_MAX && priority == DRM_ASAHI_PRIORITY_HIGH) {
+      priority = DRM_ASAHI_PRIORITY_MEDIUM;
+      ctx->queue_id = agx_create_command_queue(agx_device(screen), priority);
+   }
    if (ctx->queue_id == UINT32_MAX) {
       u_upload_destroy(pctx->stream_uploader);
       ralloc_free(ctx);
diff --git a/src/gallium/drivers/asahi/agx_state.c b/src/gallium/drivers/asahi/agx_state.c
index 1deda9b1c748..6785dd575bc5 100644
--- a/src/gallium/drivers/asahi/agx_state.c
+++ b/src/gallium/drivers/asahi/agx_state.c
@@ -7163,6 +7163,16 @@ agx_batch_geometry_params(struct agx_batch *batch, uint64_t input_index_buffer,
          agx_get_query_address(batch, batch->ctx->prims_generated[i]);
    }
 
+   /* The pre-GS kernel also adds to query_prims_generated_counter, which
+    * backs Vulkan's VK_EXT_primitives_generated_query. GL's primitives
+    * generated query already counts through prims_generated_counter, so
+    * point this one at the scratch page rather than leaving it zero.
+    */
+   for (unsigned i = 0; i < ARRAY_SIZE(params.query_prims_generated_counter);
+        ++i) {
+      params.query_prims_generated_counter[i] = AGX_SCRATCH_PAGE_ADDRESS;
+   }
+
    for (unsigned i = 0; i < ARRAY_SIZE(batch->ctx->tf_prims_generated); ++i) {
       params.xfb_prims_generated_counter[i] =
          agx_get_query_address(batch, batch->ctx->tf_prims_generated[i]);
@@ -7930,7 +7940,7 @@ agx_draw_patches(struct agx_context *ctx, const struct pipe_draw_info *info,
    libagx_tessellate(batch, tess_grid, AGX_BARRIER_ALL, mode,
                      POLY_TESS_MODE_COUNT, state);
    libagx_prefix_sum_tess(batch, agx_1d(1024), AGX_BARRIER_ALL, state, c_prims,
-                          c_invs, c_prims || c_invs);
+                          c_invs, 0, c_prims || c_invs);
    libagx_tessellate(batch, tess_grid, AGX_BARRIER_ALL, mode,
                      POLY_TESS_MODE_WITH_COUNTS, state);
 
diff --git a/src/gallium/drivers/zink/meson.build b/src/gallium/drivers/zink/meson.build
index 35ec6f1f1e48..7251b2a26b10 100644
--- a/src/gallium/drivers/zink/meson.build
+++ b/src/gallium/drivers/zink/meson.build
@@ -128,6 +128,7 @@ if with_tests
         ['zink_query_test.c', zink_device_info, zink_instance, vk_dispatch_table, spirv_info_h],
         dependencies : [idep_nir_headers, idep_mesautil, idep_vulkan_util_headers,
                         idep_vulkan_util, idep_vulkan_wsi_defines, dep_libdrm, vulkan_wsi_deps],
+        link_with : libgallium,
         include_directories : [inc_include, inc_src, inc_gallium, inc_gallium_aux, inc_vulkan_util],
         c_args : ['-UNDEBUG', '-ffunction-sections', '-fdata-sections'] + cc.get_supported_arguments('-fno-tree-slp-vectorize'),
         link_args : ['-Wl,--gc-sections'],
diff --git a/src/gallium/drivers/zink/tests/ssbo-block-limit.c b/src/gallium/drivers/zink/tests/ssbo-block-limit.c
index c7f61e329dd4..a00468cfe2a6 100644
--- a/src/gallium/drivers/zink/tests/ssbo-block-limit.c
+++ b/src/gallium/drivers/zink/tests/ssbo-block-limit.c
@@ -61,25 +61,40 @@ int main(int argc, char **argv)
    GLint64 limit = 0;
    glGetInteger64v(GL_MAX_SHADER_STORAGE_BLOCK_SIZE, &limit);
    printf("renderer=%s\nssbo_limit=%lld\n", glGetString(GL_RENDERER), (long long)limit);
-   const uint64_t max_size = limit;
    const struct { const char *name; unsigned size, alignment; } types[] = {
       {"float", 4, 4}, {"int", 4, 4}, {"uint", 4, 4}, {"double", 8, 8},
       {"ivec2", 8, 8}, {"uvec2", 8, 8}, {"vec2", 8, 8},
       {"ivec3", 12, 16}, {"uvec3", 12, 16}, {"vec3", 12, 16}, {"dvec3", 24, 32}
    };
-   const struct { const char *name; GLenum type; const char *preamble; } stages[] = {
-      {"compute", GL_COMPUTE_SHADER, "layout(local_size_x=1) in;\n"},
-      {"vertex", GL_VERTEX_SHADER, ""},
-      {"fragment", GL_FRAGMENT_SHADER, ""},
-      {"geometry", GL_GEOMETRY_SHADER, "layout(points) in;layout(points,max_vertices=1) out;\n"},
-      {"tess-control", GL_TESS_CONTROL_SHADER, "layout(vertices=1) out;\n"},
-      {"tess-eval", GL_TESS_EVALUATION_SHADER, "layout(triangles) in;\n"},
+   /* std140 rounds the block extent to at least 16 bytes; dvec3
+    * raises the alignment to 32. Test a usable boundary within the reported
+    * limit, including when that limit is INT_MAX rather than a multiple. */
+   unsigned block_alignment = 16;
+   for (unsigned i = 0; i < sizeof(types) / sizeof(types[0]); i++)
+      if (types[i].alignment > block_alignment)
+         block_alignment = types[i].alignment;
+   const uint64_t max_size = (uint64_t)limit & ~(uint64_t)(block_alignment - 1);
+   const struct { const char *name; GLenum type, block_limit; const char *preamble; } stages[] = {
+      {"compute", GL_COMPUTE_SHADER, GL_MAX_COMPUTE_SHADER_STORAGE_BLOCKS, "layout(local_size_x=1) in;\n"},
+      {"vertex", GL_VERTEX_SHADER, GL_MAX_VERTEX_SHADER_STORAGE_BLOCKS, ""},
+      {"fragment", GL_FRAGMENT_SHADER, GL_MAX_FRAGMENT_SHADER_STORAGE_BLOCKS, ""},
+      {"geometry", GL_GEOMETRY_SHADER, GL_MAX_GEOMETRY_SHADER_STORAGE_BLOCKS, "layout(points) in;layout(points,max_vertices=1) out;\n"},
+      {"tess-control", GL_TESS_CONTROL_SHADER, GL_MAX_TESS_CONTROL_SHADER_STORAGE_BLOCKS, "layout(vertices=1) out;\n"},
+      {"tess-eval", GL_TESS_EVALUATION_SHADER, GL_MAX_TESS_EVALUATION_SHADER_STORAGE_BLOCKS, "layout(triangles) in;\n"},
    };
    unsigned checked = 0, failures = 0;
    for (unsigned stage = 0; stage < sizeof(stages) / sizeof(stages[0]); stage++) {
+   GLint64 stage_blocks = 0;
+   glGetInteger64v(stages[stage].block_limit, &stage_blocks);
    for (unsigned i = 0; i < sizeof(types) / sizeof(types[0]); i++) {
-      /* Same upper-bound window as EnhancedLayouts' invalid-offset CTS. */
-      for (uint64_t off = max_size - 2 * types[i].size; off <= max_size - types[i].size; off++) {
+      /* Keep the original valid/unaligned boundary window and add one
+       * aligned declaration beyond the actual reported limit. */
+      for (unsigned range = 0; range < 2; range++) {
+      const uint64_t oversized = ((uint64_t)limit + types[i].alignment - 1) &
+                                 ~(uint64_t)(types[i].alignment - 1);
+      const uint64_t first = range ? oversized : max_size - 2 * types[i].size;
+      const uint64_t last = range ? oversized : max_size - types[i].size;
+      for (uint64_t off = first; off <= last; off++) {
          char src[1024];
          snprintf(src, sizeof(src), "#version 460\n%s"
                   "layout(std140,binding=0) buffer Block {layout(offset=%llu) %s member;};\n"
@@ -96,7 +111,8 @@ int main(int argc, char **argv)
          glLinkProgram(prog);
          GLint linked = 0;
          glGetProgramiv(prog, GL_LINK_STATUS, &linked);
-         const int expected = (off % types[i].alignment) == 0;
+         const int expected = !range && stage_blocks >= 2 &&
+                              (off % types[i].alignment) == 0;
          checked++;
          if (!!linked != expected) {
             char log[1024] = {0};
@@ -108,6 +124,7 @@ int main(int argc, char **argv)
          glDeleteProgram(prog);
          glDeleteShader(sh);
       }
+      }
    }
    }
    eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
diff --git a/src/gallium/drivers/zink/zink_query.c b/src/gallium/drivers/zink/zink_query.c
index 8beed46638a6..493eeda807ae 100644
--- a/src/gallium/drivers/zink/zink_query.c
+++ b/src/gallium/drivers/zink/zink_query.c
@@ -1161,6 +1161,16 @@ zink_end_query(struct pipe_context *pctx,
    return !query->failed;
 }
 
+static bool
+can_direct_poll_query_result(struct zink_query *query)
+{
+   /* One QBO can contain several Vulkan starts after a flush. The direct
+    * readback sees only the last start, while the full path sums them all.
+    */
+   return query->buffer_count == 1 && get_num_starts(query) == 1 &&
+          get_num_results(query) == 1 && query->type != PIPE_QUERY_TIME_ELAPSED;
+}
+
 static bool
 zink_get_query_result(struct pipe_context *pctx,
                       struct pipe_query *q,
@@ -1210,8 +1220,7 @@ zink_get_query_result(struct pipe_context *pctx,
    if (!wait && !zink_screen_usage_check_completion_fast(screen, query->batch_uses)) {
       if (query->result_check_counter++ < NOWAIT_CHECK_THRESHOLD)
          return false;
-      /* simple queries can use the "fast" path which (probably) avoids directly accessing a syncobj */
-      if (query->buffer_count == 1 && get_num_results(query) == 1 && query->type != PIPE_QUERY_TIME_ELAPSED) {
+      if (can_direct_poll_query_result(query)) {
          struct zink_query_start *start = util_dynarray_top_ptr(&query->starts, struct zink_query_start);
          unsigned query_id = start->vkq[0]->query_id;
          VkResult ret = VKCTX(GetQueryPoolResults)(screen->dev, start->vkq[0]->pool->query_pool, query_id, 1,
diff --git a/src/gallium/drivers/zink/zink_query_test.c b/src/gallium/drivers/zink/zink_query_test.c
index c0be1ddc8eff..c9c2eef62d59 100644
--- a/src/gallium/drivers/zink/zink_query_test.c
+++ b/src/gallium/drivers/zink/zink_query_test.c
@@ -131,6 +131,21 @@ main(void)
    assert(get_num_starts(&q) == 1 && first->vkq[0] == previous);
    teardown();
 
-   puts("zink query allocation failures: PASS");
+   /* One QBO still needs aggregation when a flush splits an active query. */
+   setup(PIPE_QUERY_OCCLUSION_COUNTER, VK_QUERY_TYPE_OCCLUSION);
+   q.buffer_count = 1;
+   struct zink_query_start split = {0};
+   util_dynarray_append(&q.starts, split);
+   assert(can_direct_poll_query_result(&q));
+   util_dynarray_append(&q.starts, split);
+   assert(!can_direct_poll_query_result(&q));
+   uint64_t values[] = {2048, 2048};
+   union pipe_query_result result;
+   util_query_clear_result(&result, q.type);
+   check_query_results(&q, &result, 2, values, NULL);
+   assert(result.u64 == 4096);
+   teardown();
+
+   puts("zink query allocation and aggregation: PASS");
    return 0;
 }
diff --git a/src/poly/cl/geometry.cl b/src/poly/cl/geometry.cl
index c7aa8a10bb4b..8c83b08fc887 100644
--- a/src/poly/cl/geometry.cl
+++ b/src/poly/cl/geometry.cl
@@ -286,6 +286,7 @@ poly_pre_gs(global struct poly_geometry_params *p, uint streams,
                                                  prefix_sum, unrolled_in_prims);
 
       *(p->prims_generated_counter[i]) += in_prims[i];
+      *(p->query_prims_generated_counter[i]) += in_prims[i];
    }
 
    uint4 prims = in_prims;
diff --git a/src/poly/geometry.h b/src/poly/geometry.h
index 9b1a78892fbe..d2e94d2da179 100644
--- a/src/poly/geometry.h
+++ b/src/poly/geometry.h
@@ -283,6 +283,9 @@ struct poly_geometry_params {
    /* Address of the primitives generated counters */
    DEVICE(uint) prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
    DEVICE(uint) xfb_prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
+   /* VK_EXT_primitives_generated_query, alongside a transform feedback
+    * query on the same stream. */
+   DEVICE(uint) query_prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
    DEVICE(uint) xfb_overflow[POLY_MAX_VERTEX_STREAMS];
    DEVICE(uint) xfb_any_overflow;
 
@@ -343,8 +346,8 @@ struct poly_geometry_params {
     */
    uint32_t input_topology;
 } PACKED;
-static_assert(sizeof(struct poly_geometry_params) == 79 * 4,
-              "struct poly_geometry_params must be 79 words");
+static_assert(sizeof(struct poly_geometry_params) == 87 * 4,
+              "struct poly_geometry_params must be 87 words");
 
 static inline void
 poly_geometry_params_init(struct poly_geometry_params *p,
diff --git a/src/poly/nir/meson.build b/src/poly/nir/meson.build
index 098b8a013b47..6a91d70dbc1a 100644
--- a/src/poly/nir/meson.build
+++ b/src/poly/nir/meson.build
@@ -20,6 +20,17 @@ libpoly_nir = static_library(
 )
 
 if with_tests
+  test(
+    'poly-gs-output-types',
+    executable(
+      'poly-gs-output-types',
+      '../tests/poly-gs-output-types.c',
+      include_directories : [inc_poly],
+      link_with : [libpoly_nir],
+      dependencies : [idep_nir, idep_mesautil, idep_libpoly],
+    ),
+    suite : 'poly',
+  )
   test(
     'poly-gs-outputs',
     executable(
diff --git a/src/poly/nir/poly_nir.h b/src/poly/nir/poly_nir.h
index f29cac90c71b..3d55053f3c21 100644
--- a/src/poly/nir/poly_nir.h
+++ b/src/poly/nir/poly_nir.h
@@ -48,17 +48,29 @@ struct poly_gs_info {
    uint8_t topology[64];
 };
 
+/* Optional driver pass that removes dead work NIR DCE keeps, such as a
+ * lowered ray query, so the API side effects it guarded can be stripped. */
+typedef bool (*poly_nir_remove_dead_fn)(struct nir_shader *shader);
+
 bool poly_nir_lower_gs(struct nir_shader *gs, struct nir_shader **gs_count,
                        struct nir_shader **gs_copy, struct nir_shader **pre_gs,
                        struct poly_gs_info *info);
 
-/* G17P native GS replay may discard unobserved invocation-local queries. */
+/* G17P native GS replay may discard unobserved invocation-local queries.
+ *
+ * With main_side_effects, the main kernel performs the API stores and atomics
+ * once per invocation and the count and raster variants drop them, together
+ * with what only fed them; remove_dead is then an optional driver pass for
+ * work NIR DCE keeps (M5). Otherwise the raster variant performs them, as
+ * poly_nir_lower_gs() does. */
 bool poly_nir_lower_gs_with_options(struct nir_shader *gs,
                                     struct nir_shader **gs_count,
                                     struct nir_shader **gs_copy,
                                     struct nir_shader **pre_gs,
                                     struct poly_gs_info *info,
-                                    bool eliminate_local_ray_queries);
+                                    bool eliminate_local_ray_queries,
+                                    bool main_side_effects,
+                                    poly_nir_remove_dead_fn remove_dead);
 
 bool poly_nir_lower_tcs(struct nir_shader *tcs,
                         bool can_ignore_shader_out_barriers);
diff --git a/src/poly/nir/poly_nir_lower_gs.c b/src/poly/nir/poly_nir_lower_gs.c
index f33c82f633d2..d4a236b65d5f 100644
--- a/src/poly/nir/poly_nir_lower_gs.c
+++ b/src/poly/nir/poly_nir_lower_gs.c
@@ -85,7 +85,7 @@ rewrite_intrinsics(nir_builder *b, nir_intrinsic_instr *intr, void *state_)
 }
 
 static bool
-lower_gs_intrinsics(nir_shader *shader)
+lower_gs_intrinsics(nir_shader *shader, bool main_side_effects)
 {
    struct state state;
    nir_function_impl *impl = nir_shader_get_entrypoint(shader);
@@ -124,10 +124,12 @@ lower_gs_intrinsics(nir_shader *shader)
       }
    }
 
-   /* If we have side effects, make sure we run the geometry shader at least
-    * once by outputting a dummy primitive if we wouldn't output anything.
+   /* If the raster variant performs the side effects, make sure we run the
+    * geometry shader at least once by outputting a dummy primitive if we
+    * wouldn't output anything. When the main kernel performs them, no dummy
+    * primitive is needed: it runs for every input primitive.
     */
-   if (shader->info.writes_memory) {
+   if (!main_side_effects && shader->info.writes_memory) {
       unsigned n = nir_verts_in_output_prim(shader);
       shader->info.gs.vertices_out = MAX2(shader->info.gs.vertices_out, n);
 
@@ -241,8 +243,10 @@ collect_output_types(nir_builder *b, nir_intrinsic_instr *intr, void *data)
    unsigned component = nir_intrinsic_component(intr);
    assert(component < ARRAY_SIZE(output->component_types));
    nir_alu_type *type = &output->component_types[component];
-   assert(*type == nir_type_invalid || *type == nir_intrinsic_src_type(intr));
-   *type = nir_intrinsic_src_type(intr);
+   if (*type == nir_type_invalid)
+      *type = nir_intrinsic_src_type(intr);
+   else if (*type != nir_intrinsic_src_type(intr))
+      *type = nir_type_uint | nir_alu_type_get_type_size(*type);
    output->nr_components = MAX2(output->nr_components, component + 1);
 
    return false;
@@ -499,11 +503,20 @@ lower_id(nir_builder *b, nir_intrinsic_instr *intr, void *data)
  * summed prior to running the real geometry shader. This is skipped if the
  * counts are statically known.
  */
+static void strip_api_side_effects(nir_shader *s,
+                                   poly_nir_remove_dead_fn remove_dead,
+                                   bool eliminate_local_ray_queries);
+
 static nir_shader *
-create_geometry_count_shader(nir_shader *gs, struct lower_gs_state *state)
+create_geometry_count_shader(nir_shader *gs, struct lower_gs_state *state,
+                             bool main_side_effects,
+                             poly_nir_remove_dead_fn remove_dead,
+                             bool eliminate_local_ray_queries)
 {
    /* Don't muck up the original shader */
    nir_shader *shader = nir_shader_clone(NULL, gs);
+   if (main_side_effects)
+      strip_api_side_effects(shader, remove_dead, eliminate_local_ray_queries);
 
    if (shader->info.name) {
       shader->info.name =
@@ -623,9 +636,10 @@ lower_to_gs_rast(nir_builder *b, nir_intrinsic_instr *intr, void *data)
    }
 }
 
+/* Remove an atomic whose result is unused and, with data, an API store or
+ * image store. */
 static bool
-strip_side_effect_from_main(nir_builder *b, nir_intrinsic_instr *intr,
-                            void *data)
+strip_api_side_effect(nir_builder *b, nir_intrinsic_instr *intr, void *data)
 {
    switch (intr->intrinsic) {
    case nir_intrinsic_global_atomic:
@@ -657,6 +671,34 @@ strip_side_effect_from_main(nir_builder *b, nir_intrinsic_instr *intr,
    }
 }
 
+/*
+ * A geometry shader invocation's stores and atomics must happen once. The
+ * main kernel runs once per invocation and performs them; the count and
+ * raster variants re-run the shader (the raster one once per output vertex)
+ * and must not repeat them. Strip them there, with everything that only fed
+ * them. remove_dead lets the driver remove work NIR DCE cannot, such as a
+ * lowered ray query, so the side effects it guarded can go too. An atomic
+ * whose result reaches an output stays, because the output depends on it.
+ * eliminate_local_ray_queries is the dead-CF option of
+ * poly_nir_lower_gs_with_options().
+ */
+static void
+strip_api_side_effects(nir_shader *s, poly_nir_remove_dead_fn remove_dead,
+                       bool eliminate_local_ray_queries)
+{
+   bool progress;
+   do {
+      progress = false;
+      NIR_PASS(progress, s, nir_shader_intrinsics_pass, strip_api_side_effect,
+               nir_metadata_control_flow, (void *)true);
+      NIR_PASS(progress, s, nir_opt_dce);
+      NIR_PASS(progress, s, nir_opt_dead_cf_with_options,
+               eliminate_local_ray_queries);
+      if (remove_dead)
+         NIR_PASS(progress, s, remove_dead);
+   } while (progress);
+}
+
 /*
  * The stream # is encoded into the lower bits of an index. The stream
  * multiplier is the factor to multiply vertex IDs before adding the stream #.
@@ -673,10 +715,15 @@ stream_multiplier(const nir_shader *gs)
  * shades each rasterized output vertex in parallel.
  */
 static nir_shader *
-create_gs_rast_shader(const nir_shader *gs, const struct lower_gs_state *state)
+create_gs_rast_shader(const nir_shader *gs, const struct lower_gs_state *state,
+                      bool main_side_effects,
+                      poly_nir_remove_dead_fn remove_dead,
+                      bool eliminate_local_ray_queries)
 {
    /* Don't muck up the original shader */
    nir_shader *shader = nir_shader_clone(NULL, gs);
+   if (main_side_effects)
+      strip_api_side_effects(shader, remove_dead, eliminate_local_ray_queries);
 
    /* Turn into a vertex shader run only for rasterization. Transform feedback
     * was handled in the prepass.
@@ -1254,7 +1301,9 @@ bool
 poly_nir_lower_gs_with_options(nir_shader *gs, nir_shader **gs_count,
                                nir_shader **gs_copy, nir_shader **pre_gs,
                                struct poly_gs_info *info,
-                               bool eliminate_local_ray_queries)
+                               bool eliminate_local_ray_queries,
+                               bool main_side_effects,
+                               poly_nir_remove_dead_fn remove_dead)
 {
    /* Lower I/O as assumed by the rest of GS lowering */
    if (gs->xfb_info != NULL) {
@@ -1291,7 +1340,7 @@ poly_nir_lower_gs_with_options(nir_shader *gs, nir_shader **gs_count,
    /* Lower geometry shader writes to contain all of the required counts, so we
     * know where in the various buffers we should write vertices.
     */
-   NIR_PASS(_, gs, lower_gs_intrinsics);
+   NIR_PASS(_, gs, lower_gs_intrinsics, main_side_effects);
 
    /* Clean up after all that lowering we did */
    bool progress = false;
@@ -1361,7 +1410,8 @@ poly_nir_lower_gs_with_options(nir_shader *gs, nir_shader **gs_count,
    /* Ensure that outputs_written is still accurate after DCE. */
    nir_shader_gather_info(gs, nir_shader_get_entrypoint(gs));
 
-   *gs_copy = create_gs_rast_shader(gs, &gs_state);
+   *gs_copy = create_gs_rast_shader(gs, &gs_state, main_side_effects,
+                                    remove_dead, eliminate_local_ray_queries);
 
    NIR_PASS(_, gs, nir_shader_intrinsics_pass, lower_id,
             nir_metadata_control_flow, NULL);
@@ -1374,21 +1424,27 @@ poly_nir_lower_gs_with_options(nir_shader *gs, nir_shader **gs_count,
 
    /* If there is any unknown count, we need a geometry count shader */
    if (info->count_words > 0)
-      *gs_count = create_geometry_count_shader(gs, &gs_state);
+      *gs_count = create_geometry_count_shader(gs, &gs_state,
+                                               main_side_effects, remove_dead,
+                                               eliminate_local_ray_queries);
    else
       *gs_count = NULL;
 
-   /* Strip stores and atomics */
-   do {
-      progress = false;
-      NIR_PASS(progress, gs, nir_shader_intrinsics_pass,
-               strip_side_effect_from_main, nir_metadata_control_flow,
-               (void *)true);
-
-      NIR_PASS(progress, gs, nir_opt_dce);
-      NIR_PASS(progress, gs, nir_opt_dead_cf_with_options,
-               eliminate_local_ray_queries);
-   } while (progress);
+   /* With main_side_effects the main kernel keeps the API stores and
+    * atomics: it is the one variant that runs once per invocation (see
+    * strip_api_side_effects). Otherwise strip them here. */
+   if (!main_side_effects) {
+      do {
+         progress = false;
+         NIR_PASS(progress, gs, nir_shader_intrinsics_pass,
+                  strip_api_side_effect, nir_metadata_control_flow,
+                  (void *)true);
+
+         NIR_PASS(progress, gs, nir_opt_dce);
+         NIR_PASS(progress, gs, nir_opt_dead_cf_with_options,
+                  eliminate_local_ray_queries);
+      } while (progress);
+   }
 
    NIR_PASS(_, gs, nir_shader_intrinsics_pass, lower_gs_instr,
             nir_metadata_none, &gs_state);
@@ -1413,16 +1469,18 @@ poly_nir_lower_gs_with_options(nir_shader *gs, nir_shader **gs_count,
    } while (progress);
 
    /* Strip remaining atomics, but not stores - since those are from us */
-   do {
-      progress = false;
-      NIR_PASS(progress, gs, nir_shader_intrinsics_pass,
-               strip_side_effect_from_main, nir_metadata_control_flow,
-               (void *)false);
-
-      NIR_PASS(progress, gs, nir_opt_dce);
-      NIR_PASS(progress, gs, nir_opt_dead_cf_with_options,
-               eliminate_local_ray_queries);
-   } while (progress);
+   if (!main_side_effects) {
+      do {
+         progress = false;
+         NIR_PASS(progress, gs, nir_shader_intrinsics_pass,
+                  strip_api_side_effect, nir_metadata_control_flow,
+                  (void *)false);
+
+         NIR_PASS(progress, gs, nir_opt_dce);
+         NIR_PASS(progress, gs, nir_opt_dead_cf_with_options,
+                  eliminate_local_ray_queries);
+      } while (progress);
+   }
 
    /* All those variables we created should've gone away by now */
    NIR_PASS(_, gs, nir_remove_dead_variables, nir_var_function_temp, NULL);
@@ -1542,5 +1600,5 @@ poly_nir_lower_gs(nir_shader *gs, nir_shader **gs_count, nir_shader **gs_copy,
                   nir_shader **pre_gs, struct poly_gs_info *info)
 {
    return poly_nir_lower_gs_with_options(gs, gs_count, gs_copy, pre_gs, info,
-                                        false);
+                                        false, false, NULL);
 }
diff --git a/src/poly/tests/poly-gs-output-types.c b/src/poly/tests/poly-gs-output-types.c
new file mode 100644
index 000000000000..a0fa85a7bf06
--- /dev/null
+++ b/src/poly/tests/poly-gs-output-types.c
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: MIT */
+#include "nir_builder.h"
+#include "poly_nir.h"
+
+#include <stdio.h>
+
+static bool
+check_types(nir_alu_type first, nir_alu_type second, bool overwrite)
+{
+   static const nir_shader_compiler_options options = {0};
+   nir_builder b = nir_builder_init_simple_shader(
+      MESA_SHADER_GEOMETRY, &options, "mixed GS outputs");
+   b.shader->info.gs.invocations = 1;
+   b.shader->info.gs.vertices_in = 1;
+   b.shader->info.gs.vertices_out = 1;
+   b.shader->info.gs.input_primitive = MESA_PRIM_POINTS;
+   b.shader->info.gs.output_primitive = MESA_PRIM_POINTS;
+   b.shader->info.gs.active_stream_mask = 1;
+   b.shader->info.io_lowered = true;
+
+   nir_store_output(&b, nir_imm_int(&b, 0xdeadbeef), nir_imm_int(&b, 0),
+                    .component = 0, .src_type = first,
+                    .io_semantics.location = VARYING_SLOT_VAR0);
+   nir_store_output(&b, nir_imm_int(&b, 0x80000001), nir_imm_int(&b, 0),
+                    .component = 1, .src_type = second,
+                    .io_semantics.location = VARYING_SLOT_VAR0);
+   /* One packed component can also be assigned different source types. */
+   if (overwrite) {
+      nir_store_output(&b, nir_imm_int(&b, 0x3f800000), nir_imm_int(&b, 0),
+                       .component = 0, .src_type = second,
+                       .io_semantics.location = VARYING_SLOT_VAR0);
+   }
+   nir_emit_vertex(&b, 0);
+   nir_end_primitive(&b, 0);
+
+   nir_shader *count = NULL, *copy = NULL, *pre = NULL;
+   struct poly_gs_info info;
+   poly_nir_lower_gs(b.shader, &count, &copy, &pre, &info);
+   nir_validate_shader(copy, "GS output types");
+
+   nir_alu_type found[2] = {nir_type_invalid, nir_type_invalid};
+   nir_foreach_block(block, nir_shader_get_entrypoint(copy)) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+         if (intr->intrinsic != nir_intrinsic_store_output ||
+             nir_intrinsic_io_semantics(intr).location != VARYING_SLOT_VAR0)
+            continue;
+         unsigned mask = nir_intrinsic_write_mask(intr);
+         for (unsigned c = 0; c < 2; ++c) {
+            if (mask & BITFIELD_BIT(c))
+               found[c] = nir_intrinsic_src_type(intr);
+         }
+      }
+   }
+   nir_alu_type expected = overwrite && first != second ? nir_type_uint32 : first;
+   bool valid = found[0] == expected && found[1] == second;
+   if (!valid)
+      fprintf(stderr, "GS types %u/%u became %u/%u\n",
+              first, second, found[0], found[1]);
+   ralloc_free(b.shader);
+   ralloc_free(count);
+   ralloc_free(copy);
+   ralloc_free(pre);
+   return valid;
+}
+
+int
+main(void)
+{
+   glsl_type_singleton_init_or_ref();
+   bool valid = true;
+   const nir_alu_type types[] = {nir_type_float32, nir_type_uint32,
+                                 nir_type_int32};
+   for (unsigned a = 0; a < ARRAY_SIZE(types); ++a) {
+      for (unsigned b = 0; b < ARRAY_SIZE(types); ++b) {
+         valid &= check_types(types[a], types[b], false);
+         valid &= check_types(types[a], types[b], true);
+      }
+   }
+   glsl_type_singleton_decref();
+   return valid ? 0 : 1;
+}
diff --git a/src/vulkan/runtime/bvh/hploc_internal.comp b/src/vulkan/runtime/bvh/hploc_internal.comp
index 1bc38b625364..62f75cae1b27 100644
--- a/src/vulkan/runtime/bvh/hploc_internal.comp
+++ b/src/vulkan/runtime/bvh/hploc_internal.comp
@@ -126,6 +126,12 @@ main(void)
             if (prev_range == 0xffffffff) {
                is_active = false;
             } else {
+               /* The sibling subtree's invocations stored its cluster ids and
+                * box nodes before their exchange (and released them with the
+                * barrier at the end of their iteration). Acquire them before
+                * the merge phase loads them. */
+               memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer,
+                             gl_SemanticsAcquire | gl_SemanticsMakeVisible);
                if (use_right_parent)
                   range_end = prev_range;
                else
diff --git a/src/vulkan/runtime/bvh/leaf.h b/src/vulkan/runtime/bvh/leaf.h
index 5dfeb50e8af8..31d8d0a3b8ed 100644
--- a/src/vulkan/runtime/bvh/leaf.h
+++ b/src/vulkan/runtime/bvh/leaf.h
@@ -31,9 +31,13 @@ spirv_execution_mode(extensions = ["SPV_KHR_float_controls"],
 
 layout(local_size_x_id = SUBGROUP_SIZE_ID, local_size_y = 1, local_size_z = 1) in;
 
+#ifdef VK_BATCH_LEAVES
+leaf_args args;
+#else
 layout(push_constant) uniform CONSTS {
    leaf_args args;
 };
+#endif
 
 bool
 build_triangle(inout vk_aabb bounds, VOID_REF dst_ptr, vk_bvh_geometry_data geom_data, uint32_t global_id)
@@ -158,10 +162,11 @@ build_instance(inout vk_aabb bounds, VOID_REF src_ptr, VOID_REF dst_ptr, uint32_
 
    AccelerationStructureInstance instance = DEREF(REF(AccelerationStructureInstance)(src_ptr));
 
-   /* An inactive instance is one whose acceleration structure handle is VK_NULL_HANDLE. Since the active terminology is
-    * only relevant for BVH updates, which we do not implement, we can also skip instances with mask == 0.
+   /* A null reference is inactive even when retaining updateable topology.
+    * A masked instance remains active and can be unmasked by an update.
     */
-   if (instance.accelerationStructureReference == 0 || instance.custom_instance_and_mask < (1u << 24u))
+   if (instance.accelerationStructureReference == 0 ||
+       (!VK_TEST_BUILD_FLAG_ALWAYS_ACTIVE && instance.custom_instance_and_mask < (1u << 24u)))
       return false;
 
    DEREF(node).base_ptr = instance.accelerationStructureReference;
@@ -190,8 +195,16 @@ build_instance(inout vk_aabb bounds, VOID_REF src_ptr, VOID_REF dst_ptr, uint32_
       bounds = CALCULATE_FINE_INSTANCE_NODE_BOUNDS(instance.accelerationStructureReference, mat3x4(transform));
 #endif
 
-   if (any(isnan(bounds.min)) || any(isnan(bounds.max)))
-      return false;
+   if (any(isnan(bounds.min)) || any(isnan(bounds.max))) {
+      if (!VK_TEST_BUILD_FLAG_ALWAYS_ACTIVE)
+         return false;
+
+      /* An instance of an empty BLAS is degenerate, not inactive. Retain
+       * its origin so an update can replace it with a non-empty BLAS.
+       */
+      bounds.min = vec4(0.0, 0.0, 0.0, 1.0) * mat3x4(transform);
+      bounds.max = bounds.min;
+   }
 
    DEREF(node).base.aabb = bounds;
    DEREF(node).custom_instance_and_mask = instance.custom_instance_and_mask;
@@ -270,7 +283,8 @@ main(void)
       is_active = build_instance(bounds, src_ptr, dst_ptr, global_id);
    }
 
-   if (VK_TEST_BUILD_FLAG_ALWAYS_ACTIVE && in_range)
+   if (VK_TEST_BUILD_FLAG_ALWAYS_ACTIVE && in_range &&
+       geom_data.geometry_type != VK_GEOMETRY_TYPE_INSTANCES_KHR)
       is_active = true;
 
    uint32_t id = is_active ? pack_ir_node_id(dst_offset, node_type) : VK_BVH_INVALID_NODE;
@@ -280,7 +294,9 @@ main(void)
    if (subgroupElect())
       atomicAdd(DEREF(args.header).active_leaf_count, subgroupBallotBitCount(ballot));
 
-   if (is_active) {
+   /* Uniform-address atomics are issued per lane where the compiler does not
+    * combine them, so a refit, which never reads these bounds, skips them. */
+   if (is_active && !VK_TEST_BUILD_FLAG_NO_HEADER_BOUNDS) {
       atomicMin(DEREF(args.header).min_bounds[0], to_emulated_float(bounds.min.x));
       atomicMin(DEREF(args.header).min_bounds[1], to_emulated_float(bounds.min.y));
       atomicMin(DEREF(args.header).min_bounds[2], to_emulated_float(bounds.min.z));
diff --git a/src/vulkan/runtime/bvh/vk_bvh_defines.h b/src/vulkan/runtime/bvh/vk_bvh_defines.h
index b20637efe3a7..7e949af06be1 100644
--- a/src/vulkan/runtime/bvh/vk_bvh_defines.h
+++ b/src/vulkan/runtime/bvh/vk_bvh_defines.h
@@ -446,10 +446,13 @@ TYPE(vk_prefix_scan_partition, 4);
 #define VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS (1 << 1)
 #define VK_BUILD_FLAG_64BIT_KEYS (1 << 2)
 #define VK_BUILD_FLAG_HAS_QUADS (1 << 3)
-#define VK_BUILD_FLAG_COUNT 4
+/* Leaf generation for a refit of retained topology: no Morton codes follow,
+ * so the header scene bounds are not accumulated. */
+#define VK_BUILD_FLAG_NO_HEADER_BOUNDS (1 << 4)
+#define VK_BUILD_FLAG_COUNT 5
 
 #define VK_LEAF_BUILD_FLAGS (VK_BUILD_FLAG_ALWAYS_ACTIVE | VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS | \
-                             VK_BUILD_FLAG_HAS_QUADS)
+                             VK_BUILD_FLAG_HAS_QUADS | VK_BUILD_FLAG_NO_HEADER_BOUNDS)
 
 struct leaf_args {
    VOID_REF bvh;
diff --git a/src/vulkan/runtime/bvh/vk_bvh_helpers.h b/src/vulkan/runtime/bvh/vk_bvh_helpers.h
index c51815a83151..ad3d6ec00081 100644
--- a/src/vulkan/runtime/bvh/vk_bvh_helpers.h
+++ b/src/vulkan/runtime/bvh/vk_bvh_helpers.h
@@ -457,6 +457,12 @@ vk_prefix_scan(uvec4 ballot, REF(vk_prefix_scan_partition) partitions, uint32_t
 #define VK_TEST_BUILD_FLAG_64BIT_KEYS ((BUILD_FLAGS & VK_BUILD_FLAG_64BIT_KEYS) != 0)
 #endif
 
+#if ((VK_USED_BUILD_FLAGS & VK_BUILD_FLAG_NO_HEADER_BOUNDS) != 0)
+#define VK_TEST_BUILD_FLAG_NO_HEADER_BOUNDS ((BUILD_FLAGS & VK_BUILD_FLAG_NO_HEADER_BOUNDS) != 0)
+#else
+#define VK_TEST_BUILD_FLAG_NO_HEADER_BOUNDS false
+#endif
+
 REF(vk_ir_triangle_node_quad)
 vk_ir_triangle_node_get_quad_ref(REF(vk_ir_triangle_node) node)
 {
diff --git a/src/vulkan/runtime/meson.build b/src/vulkan/runtime/meson.build
index 6c9f537d90a2..1f8548cf1127 100644
--- a/src/vulkan/runtime/meson.build
+++ b/src/vulkan/runtime/meson.build
@@ -339,6 +339,21 @@ idep_vulkan_runtime = declare_dependency(
   ]
 )
 
+if with_tests
+  test(
+    'vk_queue_dropped_signal',
+    executable(
+      'vk_queue_dropped_signal_test',
+      'tests/vk_queue_dropped_signal_test.c',
+      include_directories : [inc_include, inc_src],
+      dependencies : [vulkan_lite_runtime_deps, idep_vulkan_lite_runtime_headers],
+      link_with : [libvulkan_lite_runtime, libvulkan_lite_instance],
+    ),
+    suite : ['vulkan'],
+    timeout : 10,
+  )
+endif
+
 if with_tests and dep_libdrm.found()
   test(
     'vk_drm_syncobj_pending_wait',
diff --git a/src/vulkan/runtime/tests/vk_queue_dropped_signal_test.c b/src/vulkan/runtime/tests/vk_queue_dropped_signal_test.c
new file mode 100644
index 000000000000..775b8a7a71ef
--- /dev/null
+++ b/src/vulkan/runtime/tests/vk_queue_dropped_signal_test.c
@@ -0,0 +1,278 @@
+/* SPDX-License-Identifier: MIT */
+
+#include "vk_alloc.h"
+#include "vk_common_entrypoints.h"
+#include "vk_queue.h"
+#include "vk_semaphore.h"
+#include "vk_sync_timeline.h"
+#include "util/os_time.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+struct test_sync {
+   struct vk_sync base;
+   bool signalled;
+};
+
+static unsigned signal_calls;
+static unsigned fail_call;
+static VkResult unmarked_failure;
+static int wait_entered;
+static VkResult (*real_timeline_wait)(struct vk_device *, struct vk_sync *,
+                                     uint64_t, enum vk_sync_wait_flags, uint64_t);
+
+static void
+require(bool condition, const char *message)
+{
+   if (!condition) {
+      fprintf(stderr, "%s\n", message);
+      exit(EXIT_FAILURE);
+   }
+}
+
+static VkResult
+sync_init(struct vk_device *device, struct vk_sync *sync, uint64_t value)
+{
+   ((struct test_sync *)sync)->signalled = value != 0;
+   return VK_SUCCESS;
+}
+
+static void
+sync_finish(struct vk_device *device, struct vk_sync *sync)
+{
+}
+
+static VkResult
+sync_reset(struct vk_device *device, struct vk_sync *sync)
+{
+   ((struct test_sync *)sync)->signalled = false;
+   return VK_SUCCESS;
+}
+
+static VkResult
+sync_signal(struct vk_device *device, struct vk_sync *sync, uint64_t value)
+{
+   if (++signal_calls == fail_call)
+      return VK_ERROR_OUT_OF_HOST_MEMORY;
+   ((struct test_sync *)sync)->signalled = true;
+   return VK_SUCCESS;
+}
+
+static VkResult
+sync_wait(struct vk_device *device, struct vk_sync *sync, uint64_t value,
+          enum vk_sync_wait_flags flags, uint64_t timeout)
+{
+   /* Model a backend that releases its waits on loss. The public wait must
+    * still return DEVICE_LOST, regardless of these untrusted payloads.
+    */
+   return vk_device_is_lost_no_report(device) ||
+          ((struct test_sync *)sync)->signalled ? VK_SUCCESS : VK_TIMEOUT;
+}
+
+static VkResult
+timeline_wait_witness(struct vk_device *device, struct vk_sync *sync,
+                       uint64_t value, enum vk_sync_wait_flags flags,
+                       uint64_t timeout)
+{
+   /* This is after WaitSemaphores' initial device-loss check. A wait entered
+    * here before the submission fails must not remain blocked forever.
+    */
+   p_atomic_inc(&wait_entered);
+   return real_timeline_wait(device, sync, value, flags, timeout);
+}
+
+struct wait_context {
+   struct vk_device *device;
+   VkSemaphore semaphores[2];
+   VkResult result;
+   int done;
+};
+
+static int
+wait_thread(void *data)
+{
+   struct wait_context *ctx = data;
+   const uint64_t values[] = {5, 5};
+   const VkSemaphoreWaitInfo info = {
+      .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
+      .semaphoreCount = 2,
+      .pSemaphores = ctx->semaphores,
+      .pValues = values,
+   };
+   ctx->result = vk_common_WaitSemaphores(vk_device_to_handle(ctx->device),
+                                          &info, UINT64_MAX);
+   p_atomic_set(&ctx->done, 1);
+   return 0;
+}
+
+static void
+wait_for(int *flag, const char *message)
+{
+   const int64_t deadline = os_time_get_nano() + 2 * ONE_SECOND_IN_NS;
+   while (!p_atomic_read(flag)) {
+      require(os_time_get_nano() < deadline, message);
+      const struct timespec delay = {.tv_nsec = 1000000};
+      thrd_sleep(&delay, NULL);
+   }
+}
+
+static const struct vk_sync_type test_sync_type = {
+   .size = sizeof(struct test_sync),
+   .features = VK_SYNC_FEATURE_BINARY | VK_SYNC_FEATURE_GPU_WAIT |
+               VK_SYNC_FEATURE_GPU_MULTI_WAIT | VK_SYNC_FEATURE_CPU_WAIT |
+               VK_SYNC_FEATURE_CPU_RESET | VK_SYNC_FEATURE_CPU_SIGNAL,
+   .init = sync_init,
+   .finish = sync_finish,
+   .reset = sync_reset,
+   .signal = sync_signal,
+   .wait = sync_wait,
+};
+
+static VkResult
+fail_submit(struct vk_queue *queue, struct vk_queue_submit *submit)
+{
+   /* The submit thread normally records loss after submit_final returns.
+    * Exercise that interval when the best-effort signal itself fails: loss
+    * must be recorded before publishing the point and waking its waiters.
+    * unmarked_failure returns an error without recording loss at all.
+    */
+   if (fail_call)
+      return VK_ERROR_DEVICE_LOST;
+   if (unmarked_failure != VK_SUCCESS)
+      return unmarked_failure;
+   return vk_queue_set_lost(queue, "injected submission failure");
+}
+
+static void
+run(bool timeline, unsigned fail, VkResult unmarked)
+{
+   struct vk_device device = {
+      .alloc = *vk_default_allocator(),
+      .submit_mode = VK_QUEUE_SUBMIT_MODE_IMMEDIATE,
+      .timeline_mode = timeline ? VK_DEVICE_TIMELINE_MODE_EMULATED :
+                                 VK_DEVICE_TIMELINE_MODE_NONE,
+   };
+   vk_object_base_init(NULL, &device.base, VK_OBJECT_TYPE_DEVICE);
+   device._lost.reported = true;
+   list_inithead(&device.queues);
+   struct vk_queue queue;
+   const VkDeviceQueueCreateInfo create = {
+      .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
+      .queueCount = 1,
+   };
+   require(vk_queue_init(&queue, &device, &create, 0) == VK_SUCCESS,
+           "queue initialization failed");
+   queue.driver_submit = fail_submit;
+
+   struct vk_sync_timeline_type timeline_type =
+      vk_sync_timeline_get_type(&test_sync_type);
+   real_timeline_wait = timeline_type.sync.wait;
+   timeline_type.sync.wait = timeline_wait_witness;
+   const struct vk_sync_type *type = timeline ? &timeline_type.sync :
+                                               &test_sync_type;
+   struct vk_semaphore *semaphores[2];
+   VkSemaphoreSubmitInfo signals[2];
+   for (unsigned i = 0; i < 2; ++i) {
+      semaphores[i] = calloc(1, sizeof(struct vk_semaphore) + type->size);
+      require(semaphores[i] != NULL, "semaphore allocation failed");
+      vk_object_base_init(&device, &semaphores[i]->base,
+                          VK_OBJECT_TYPE_SEMAPHORE);
+      semaphores[i]->type = timeline ? VK_SEMAPHORE_TYPE_TIMELINE :
+                                      VK_SEMAPHORE_TYPE_BINARY;
+      require(vk_sync_init(&device, &semaphores[i]->permanent, type,
+                           timeline ? VK_SYNC_IS_TIMELINE : 0, 0) == VK_SUCCESS,
+              "sync initialization failed");
+      signals[i] = (VkSemaphoreSubmitInfo) {
+         .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO,
+         .semaphore = vk_semaphore_to_handle(semaphores[i]),
+         .value = timeline ? 5 : 0,
+         .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
+      };
+   }
+   const VkSubmitInfo2 submit = {
+      .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
+      .signalSemaphoreInfoCount = 2,
+      .pSignalSemaphoreInfos = signals,
+   };
+   signal_calls = 0;
+   fail_call = fail;
+   unmarked_failure = unmarked;
+   /* An out-of-memory failure leaves the device usable and must not touch
+    * the semaphores, so nothing would ever wake a waiter: don't start one.
+    */
+   const bool oom = unmarked == VK_ERROR_OUT_OF_HOST_MEMORY;
+   struct wait_context wait = {
+      .device = &device,
+      .semaphores = {signals[0].semaphore, signals[1].semaphore},
+   };
+   thrd_t thread;
+   if (timeline && !oom) {
+      p_atomic_set(&wait_entered, 0);
+      require(thrd_create(&thread, wait_thread, &wait) == thrd_success,
+              "wait thread creation failed");
+      wait_for(&wait_entered, "wait did not enter the timeline implementation");
+   }
+   const VkResult submitted =
+      vk_common_QueueSubmit2(vk_queue_to_handle(&queue), 1, &submit,
+                             VK_NULL_HANDLE);
+   if (oom) {
+      require(submitted == VK_ERROR_OUT_OF_HOST_MEMORY,
+              "out-of-memory failure was not propagated");
+      require(signal_calls == 0, "out-of-memory failure signalled semaphores");
+      require(!vk_device_is_lost_no_report(&device),
+              "out-of-memory failure lost the device");
+   } else {
+      require(submitted == VK_ERROR_DEVICE_LOST,
+              "submission failure was not propagated");
+      require(signal_calls == 2, "a failed signal prevented later signals");
+      require(vk_device_is_lost_no_report(&device),
+              "submission failure was not recorded before waking waiters");
+   }
+   if (timeline && !oom) {
+      wait_for(&wait.done, "device loss stranded an existing timeline waiter");
+      require(thrd_join(thread, NULL) == thrd_success, "wait thread join failed");
+      require(wait.result == VK_ERROR_DEVICE_LOST,
+              "waiter trusted a payload after device loss");
+   }
+   for (unsigned i = 0; i < 2; ++i) {
+      const bool expected = !oom && fail != i + 1;
+      struct vk_sync *sync = &semaphores[i]->permanent;
+      if (timeline) {
+         struct vk_sync_timeline *tl = vk_sync_as_timeline(sync);
+         const uint64_t value = oom ? 0 : 5;
+         require(tl->state->highest_pending == value &&
+                 tl->state->highest_past == value,
+                 oom ? "out-of-memory failure installed a time point" :
+                       "timeline did not release its pending wait");
+         require(tl->state->refcount == 1,
+                 "dropped submission leaked its time point reference");
+      } else {
+         require(((struct test_sync *)sync)->signalled == expected,
+                 "incorrect binary signal state");
+      }
+      vk_sync_finish(&device, sync);
+      vk_object_base_finish(&semaphores[i]->base);
+      free(semaphores[i]);
+   }
+   vk_queue_finish(&queue);
+   vk_object_base_finish(&device.base);
+   printf("%s, failed signal %u, unmarked %d: PASS\n",
+          timeline ? "timeline" : "binary", fail, unmarked);
+}
+
+int
+main(void)
+{
+   for (unsigned fail = 0; fail <= 2; ++fail) {
+      run(false, fail, VK_SUCCESS);
+      run(true, fail, VK_SUCCESS);
+   }
+   /* The driver fails without recording loss and every signal succeeds:
+    * waiters woken by the dropped signals must still see the loss. */
+   run(false, 0, VK_ERROR_DEVICE_LOST);
+   run(true, 0, VK_ERROR_DEVICE_LOST);
+   run(false, 0, VK_ERROR_OUT_OF_HOST_MEMORY);
+   run(true, 0, VK_ERROR_OUT_OF_HOST_MEMORY);
+   return EXIT_SUCCESS;
+}
diff --git a/src/vulkan/runtime/vk_acceleration_structure.c b/src/vulkan/runtime/vk_acceleration_structure.c
index 0508284c32b6..7c8edd5159f7 100644
--- a/src/vulkan/runtime/vk_acceleration_structure.c
+++ b/src/vulkan/runtime/vk_acceleration_structure.c
@@ -563,10 +563,10 @@ vk_accel_struct_cmd_begin_debug_marker_formatted(VkCommandBuffer commandBuffer,
    va_list ap;
    va_start(ap, format);
    char *name = NULL;
-   vasprintf(&name, format, ap);
+   int ret = vasprintf(&name, format, ap);
    va_end(ap);
 
-   if (!name)
+   if (ret < 0)
       return;
 
    VkDebugMarkerMarkerInfoEXT marker_info = {
@@ -631,6 +631,10 @@ build_leaves(VkCommandBuffer commandBuffer, struct vk_device *device,
              struct vk_acceleration_structure_build_state *states,
              uint32_t build_count, uint32_t build_flags)
 {
+   if (device->as_build_ops->build_leaves)
+      return device->as_build_ops->build_leaves(commandBuffer, device, meta, args,
+                                                states, build_count, build_flags);
+
    VkPipeline pipeline;
    VkPipelineLayout layout;
 
@@ -724,11 +728,18 @@ vk_cmd_build_as_leaves(VkCommandBuffer commandBuffer, struct vk_device *device,
                       const struct vk_acceleration_structure_build_args *args,
                       const struct vk_acceleration_structure_build_state *state)
 {
+   /* Refresh the leaf IR of an update in place: run the ordinary leaf pass on
+    * a copy that is not typed as an update, so the builder emits leaves without
+    * touching the retained internal topology. */
    struct vk_acceleration_structure_build_state leaves = *state;
    leaves.config.internal_type = VK_INTERNAL_BUILD_TYPE_LBVH;
+   /* Only the update path calls this: the retained topology needs leaf nodes
+    * and the active count, not the scene bounds that feed Morton codes. */
+   leaves.config.build_flags |= VK_BUILD_FLAG_NO_HEADER_BOUNDS;
    leaves.leaf_node_count = 0;
+   leaves.processed = false;
    return build_leaves(commandBuffer, device, meta, args, &leaves, 1,
-                       state->config.build_flags & VK_LEAF_BUILD_FLAGS);
+                       leaves.config.build_flags & VK_LEAF_BUILD_FLAGS);
 }
 
 static VkResult
diff --git a/src/vulkan/runtime/vk_acceleration_structure.h b/src/vulkan/runtime/vk_acceleration_structure.h
index b1c8b9da86b2..504665e02bd3 100644
--- a/src/vulkan/runtime/vk_acceleration_structure.h
+++ b/src/vulkan/runtime/vk_acceleration_structure.h
@@ -177,6 +177,12 @@ struct vk_acceleration_structure_build_ops {
                   const struct vk_acceleration_structure_build_args *args, struct vk_acceleration_structure_build_state *states,
                   uint32_t build_count, bool flushed_compute_after_init_update_scratch);
 
+   VkResult (*build_leaves)(VkCommandBuffer commandBuffer, struct vk_device *device,
+                            struct vk_meta_device *meta,
+                            const struct vk_acceleration_structure_build_args *args,
+                            struct vk_acceleration_structure_build_state *states,
+                            uint32_t build_count, uint32_t build_flags);
+
    const uint32_t *leaf_spirv_override;
    size_t leaf_spirv_override_size;
 
@@ -241,6 +247,12 @@ void vk_get_as_build_sizes(VkDevice _device, VkAccelerationStructureBuildTypeKHR
 
 bool vk_acceleration_struct_vtx_format_supported(VkFormat format);
 
+/* Refresh leaf IR while retaining an existing internal topology. */
+VkResult vk_cmd_build_as_leaves(VkCommandBuffer commandBuffer,
+   struct vk_device *device, struct vk_meta_device *meta,
+   const struct vk_acceleration_structure_build_args *args,
+   const struct vk_acceleration_structure_build_state *state);
+
 static inline VkGeometryTypeKHR
 vk_get_as_geometry_type(const VkAccelerationStructureBuildGeometryInfoKHR *build_info)
 {
diff --git a/src/vulkan/runtime/vk_meta.h b/src/vulkan/runtime/vk_meta.h
index 621aa975766d..156dc671edb2 100644
--- a/src/vulkan/runtime/vk_meta.h
+++ b/src/vulkan/runtime/vk_meta.h
@@ -138,6 +138,12 @@ struct vk_meta_device {
     * destination Z when addressing the source buffer. */
    bool use_view_relative_3d_layer;
 
+   /* Specialize graphics buffer uploads whose attachment view has exactly
+    * one layer. Requires non-layered rendering: fragment Layer is then zero
+    * even when the attachment selects an array layer or a 3D slice.
+    */
+   bool specialize_single_layer_buffer_upload;
+
    bool use_rect_list_pipeline;
 
    struct {
diff --git a/src/vulkan/runtime/vk_meta_copy_fill_update.c b/src/vulkan/runtime/vk_meta_copy_fill_update.c
index 4ca0455309e7..30e18eb9a975 100644
--- a/src/vulkan/runtime/vk_meta_copy_fill_update.c
+++ b/src/vulkan/runtime/vk_meta_copy_fill_update.c
@@ -1058,7 +1058,11 @@ build_buffer_to_image_fs(const struct vk_meta_device *meta,
 
    enum pipe_format buf_pfmt = vk_format_to_pipe_format(buf_fmt);
    nir_def *out_coord_xy = nir_f2u32(b, nir_load_frag_coord(b));
-   nir_def *out_layer = nir_load_layer_id(b);
+   const bool single_layer =
+      meta->specialize_single_layer_buffer_upload &&
+      key->img.view.type == VK_IMAGE_VIEW_TYPE_2D;
+   nir_def *out_layer = single_layer ? nir_imm_int(b, 0)
+                                    : nir_load_layer_id(b);
 
    nir_def *img_offs = nir_vec3(b,
       load_info(b, struct vk_meta_copy_buffer_image_info, img.offset.x),
@@ -1090,7 +1094,9 @@ build_buffer_to_image_fs(const struct vk_meta_device *meta,
    nir_def *extent = nir_vec3(b,
       load_info(b, struct vk_meta_copy_buffer_image_info, copy_id_range.end.x),
       load_info(b, struct vk_meta_copy_buffer_image_info, copy_id_range.end.y),
-      load_info(b, struct vk_meta_copy_buffer_image_info, copy_id_range.end.z));
+      single_layer ? nir_imm_int(b, 1)
+                   : load_info(b, struct vk_meta_copy_buffer_image_info,
+                               copy_id_range.end.z));
    nir_def *in_bounds = nir_ball(b, nir_ult(b, coords, extent));
    coords = nir_bcsel(b, in_bounds, coords, nir_imm_ivec3(b, 0, 0, 0));
 
@@ -1620,12 +1626,25 @@ copy_buffer_to_image_region_gfx(
    /* We only special-case 1D_ARRAY to move the layer ID to the second
     * component instead of the third. For all other view types, let's pick an
     * invalid VkImageViewType value so we don't end up creating the same
-    * pipeline multiple times. */
+    * pipeline multiple times. The opted-in single-layer specialization below
+    * gets a separate key. */
    VkImageViewType view_type =
       img->image_type == VK_IMAGE_TYPE_1D && img->array_layers > 1
          ? VK_IMAGE_VIEW_TYPE_1D_ARRAY
          : (VkImageViewType)-1;
 
+   /* A single-layer attachment view has Layer == 0. Keep the source bounds
+    * protection for helpers, but avoid loading and checking a varying layer
+    * coordinate and multiplying it by the source image stride. Drivers that
+    * split array/3D uploads have already rebased the source address per slice.
+    */
+   if (meta->specialize_single_layer_buffer_upload &&
+       !meta->use_layered_rendering &&
+       img->image_type != VK_IMAGE_TYPE_1D &&
+       region->imageExtent.depth == 1 &&
+       vk_image_subresource_layer_count(img, &region->imageSubresource) == 1)
+      view_type = VK_IMAGE_VIEW_TYPE_2D;
+
    struct vk_meta_copy_buffer_image_key key = {
       .key_type = VK_META_OBJECT_KEY_COPY_BUFFER_TO_IMAGE_GFX,
       .img = {
diff --git a/src/vulkan/runtime/vk_queue.c b/src/vulkan/runtime/vk_queue.c
index 737a508c28ba..63e42efeaea7 100644
--- a/src/vulkan/runtime/vk_queue.c
+++ b/src/vulkan/runtime/vk_queue.c
@@ -231,6 +231,48 @@ vk_queue_submit_cleanup(struct vk_queue *queue,
    }
 }
 
+/* A submission that will never execute on a lost queue still owns its
+ * signal operations. Signal them on the CPU so that host and device waiters
+ * (including WAIT_FOR_SUBMIT waits on timeline points) wake up and observe the
+ * lost device through check_status, instead of waiting forever for a payload
+ * that is never submitted. The device is already lost, so no result produced
+ * after this point is trusted.
+ */
+static void
+vk_queue_submit_signal_dropped(struct vk_queue *queue,
+                               struct vk_queue_submit *submit)
+{
+   if (submit->_signals_dropped)
+      return;
+   submit->_signals_dropped = true;
+
+   for (uint32_t i = 0; i < submit->signal_count; i++) {
+      struct vk_sync *sync = submit->signals[i].sync;
+      if (sync == NULL || !(sync->type->features & VK_SYNC_FEATURE_CPU_SIGNAL))
+         continue;
+
+      /* After vk_sync_signal_unwrap, an emulated timeline signal targets the
+       * binary sync of a new time point that must also be installed.
+       */
+      VkResult result = vk_sync_signal(queue->base.device, sync,
+                                      submit->signals[i].signal_value);
+      if (unlikely(result != VK_SUCCESS))
+         vk_queue_set_lost(queue, "Failed to signal a dropped submission");
+
+      /* Publish the point even if the best-effort CPU signal failed. A
+       * waiter may already be blocked on the timeline's condition variable;
+       * it must reach the underlying sync/device-loss checks. The payload
+       * is untrusted on the lost device, as are all subsequent results.
+       */
+      if (submit->_signal_points != NULL &&
+          submit->_signal_points[i] != NULL) {
+         vk_sync_timeline_point_install(queue->base.device,
+                                        submit->_signal_points[i]);
+         submit->_signal_points[i] = NULL;
+      }
+   }
+}
+
 static void
 vk_queue_submit_free(struct vk_queue *queue,
                      struct vk_queue_submit *submit)
@@ -554,11 +596,39 @@ vk_queue_push_submit(struct vk_queue *queue,
                      struct vk_queue_submit *submit)
 {
    mtx_lock(&queue->submit.mutex);
+   if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED &&
+       queue->_lost.lost) {
+      /* The submit thread stopped with the queue lost and drained everything
+       * queued before; this submission would never be processed.
+       */
+      vk_queue_submit_signal_dropped(queue, submit);
+      vk_queue_submit_destroy(queue, submit);
+      mtx_unlock(&queue->submit.mutex);
+      return;
+   }
    list_addtail(&submit->link, &queue->submit.submits);
    cnd_signal(&queue->submit.push);
    mtx_unlock(&queue->submit.mutex);
 }
 
+/* The queue is lost: every queued submission will never be processed. Drop
+ * them, signalling their signal operations. Caller holds submit.mutex.
+ */
+static void
+vk_queue_drop_pending_locked(struct vk_queue *queue)
+{
+   assert(queue->_lost.lost);
+   while (!list_is_empty(&queue->submit.submits)) {
+      struct vk_queue_submit *submit =
+         list_first_entry(&queue->submit.submits,
+                          struct vk_queue_submit, link);
+      list_del(&submit->link);
+      vk_queue_submit_signal_dropped(queue, submit);
+      vk_queue_submit_destroy(queue, submit);
+   }
+   cnd_broadcast(&queue->submit.pop);
+}
+
 static VkResult
 vk_queue_drain(struct vk_queue *queue)
 {
@@ -655,8 +725,21 @@ vk_queue_submit_final(struct vk_queue *queue,
       vk_queue_unlock(queue);
    }
 
-   if (unlikely(result != VK_SUCCESS))
+   if (unlikely(result != VK_SUCCESS)) {
+      /* Nothing of this submission will run. Its signal operations are
+       * published below so that waiters wake up, and those waiters must then
+       * see the loss through check_status, so record it first. An immediate
+       * submission that failed without losing the device (out of memory)
+       * returns its error with the semaphores untouched, as vkQueueSubmit
+       * requires; the deferred and threaded callers lose the queue anyway.
+       */
+      if (result == VK_ERROR_DEVICE_LOST || vk_queue_is_lost(queue) ||
+          queue->submit.mode != VK_QUEUE_SUBMIT_MODE_IMMEDIATE) {
+         result = vk_queue_set_lost(queue, "queue::driver_submit failed");
+         vk_queue_submit_signal_dropped(queue, submit);
+      }
       return result;
+   }
 
    if (submit->_signal_points) {
       for (uint32_t i = 0; i < submit->signal_count; i++) {
@@ -706,6 +789,7 @@ vk_queue_flush(struct vk_queue *queue, uint32_t *submit_count_out)
             goto done;
          } else if (result != VK_SUCCESS) {
             result = vk_queue_set_lost(queue, "Wait for time points failed");
+            vk_queue_drop_pending_locked(queue);
             goto done;
          }
       }
@@ -713,6 +797,7 @@ vk_queue_flush(struct vk_queue *queue, uint32_t *submit_count_out)
       result = vk_queue_submit_final(queue, submit);
       if (unlikely(result != VK_SUCCESS)) {
          result = vk_queue_set_lost(queue, "queue::driver_submit failed");
+         vk_queue_drop_pending_locked(queue);
          goto done;
       }
 
@@ -766,12 +851,18 @@ vk_queue_submit_thread_func(void *_data)
                                  VK_SYNC_WAIT_PENDING, UINT64_MAX);
       if (unlikely(result != VK_SUCCESS)) {
          vk_queue_set_lost(queue, "Wait for time points failed");
+         mtx_lock(&queue->submit.mutex);
+         vk_queue_drop_pending_locked(queue);
+         mtx_unlock(&queue->submit.mutex);
          return 1;
       }
 
       result = vk_queue_submit_final(queue, submit);
       if (unlikely(result != VK_SUCCESS)) {
          vk_queue_set_lost(queue, "queue::driver_submit failed");
+         mtx_lock(&queue->submit.mutex);
+         vk_queue_drop_pending_locked(queue);
+         mtx_unlock(&queue->submit.mutex);
          return 1;
       }
 
diff --git a/src/vulkan/runtime/vk_queue.h b/src/vulkan/runtime/vk_queue.h
index 82b8d1401f0e..ca3a88818569 100644
--- a/src/vulkan/runtime/vk_queue.h
+++ b/src/vulkan/runtime/vk_queue.h
@@ -296,6 +296,9 @@ struct vk_queue_submit {
    VkSparseImageMemoryBind *_image_bind_entries;
 
    bool _has_binary_permanent_semaphore_wait;
+   /* Set once the signals of a submission that never reached the driver were
+    * signalled on the CPU (lost queue); see vk_queue_submit_signal_dropped. */
+   bool _signals_dropped;
    struct vk_sync **_wait_temps;
    struct vk_sync_timeline_point **_wait_points;
    struct vk_sync_timeline_point **_signal_points;