From 85adc6e3cad99c73b25c254eb88c07e5b8f8d217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 5 Nov 2026 10:15:28 +0100 Subject: [PATCH 01/33] apple: add G5P render support for J71t with the original device firmware Add the A9 (S8003) G5P render driver used by the merged PowerVR/G5P Mesa winsys as kernel patches, enable it on J71t in the device tree, set CONFIG_DRM_APPLE_G5P=y for apple-16k, and add an install-time recipe that extracts the original META LDR (apple/agx-g5p-c0.ldr) from the device's own iOS kernelcache into /lib/firmware. No firmware is committed. The driver provides TYPED_V3 rendering, GEM buffers, VM/arena mappings, PRIME import/export, syncobj, bounded retirement and recovery. It does not expose the old compute or fixed-fixture ABI. Hardware: the exact kernel this derives from (brkosebay/linux a9/gpu-pr1 407b7bf2cbbe) booted on a J71t with the merged Mesa, rendered labwc on the GPU with KMS scanout at 60 Hz, and passed the G5P device rungs with zero GPU events. This Orchard adaptation itself has not yet been built by orchard or run on hardware. Assisted-by: LLM --- devices/j71t/firmware.json | 10 + docs/firmware-recipes.md | 12 + kernel/configs/apple-16k/config | 1 + ...dts-apple-enable-the-G5P-GPU-on-J71t.patch | 50 + ...ine-the-G5P-merged-render-winsys-ABI.patch | 565 + ...e-A9-firmware-and-transport-core-for.patch | 14551 ++++++++++++++++ ...PED_V3-jobs-for-the-merged-G5P-Mesa-.patch | 8888 ++++++++++ ...gpu-describe-the-S8003-G5P-resources.patch | 46 + src/orchard/apple/agx_g5p.py | 19 + src/orchard/recipe.py | 7 +- tests/test_agx_g5p.py | 29 + 11 files changed, 24177 insertions(+), 1 deletion(-) create mode 100644 kernel/patches/devices/s8000/0007-arm64-dts-apple-enable-the-G5P-GPU-on-J71t.patch create mode 100644 kernel/patches/drivers/drm/0008-drm-apple-define-the-G5P-merged-render-winsys-ABI.patch create mode 100644 kernel/patches/drivers/drm/0009-drm-apple-add-the-A9-firmware-and-transport-core-for.patch create mode 100644 kernel/patches/drivers/drm/0010-drm-apple-run-TYPED_V3-jobs-for-the-merged-G5P-Mesa-.patch create mode 100644 kernel/patches/drivers/drm/0011-dt-bindings-gpu-describe-the-S8003-G5P-resources.patch create mode 100644 src/orchard/apple/agx_g5p.py create mode 100644 tests/test_agx_g5p.py diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index 3321e246..442cabbd 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -279,6 +279,16 @@ "calibration_size": 1024 } ] + }, + { + "destination": "lib/firmware/apple/agx-g5p-c0.ldr", + "sha256": "33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca", + "steps": [ + { + "step": "agx-g5p-ldr-from-kernelcache@1", + "input": "kernelcache" + } + ] } ] } diff --git a/docs/firmware-recipes.md b/docs/firmware-recipes.md index ec20c4bc..1983e8e2 100644 --- a/docs/firmware-recipes.md +++ b/docs/firmware-recipes.md @@ -143,3 +143,15 @@ The decompressed N61 kernel's SHA-256 is These fields are explicit recipe parameters, including zero-valued overrides. Physical qualification of firmware loading and input events remains separate from converting this shared firmware file. + +## S8003 G5P META firmware + +`agx-g5p-ldr-from-kernelcache@1` reads the unique `__TEXT,__const` section +of `com.apple.AGXFirmwareKextG5P` in the user's own legacy prelinked +kernelcache. It rejects any size other than 119112 bytes or any digest other +than `33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca`. +The existing J71t kernelcache input names the active device kernelcache and +the pinned restore-image fallback. The normal recipe installer writes the +verified output to `/lib/firmware/apple/agx-g5p-c0.ldr`. No payload patch, +calibration, firmware bytes or captured job is stored in the source tree. +Other C0 revisions and other boards require separate digest qualification. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index 82fc1e24..6e64a4ad 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -3935,6 +3935,7 @@ CONFIG_DRM_SIMPLEDRM=y # CONFIG_DRM_VKMS is not set # CONFIG_DRM_VMWGFX is not set CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DRM_APPLE_G5P=y # # Frame buffer Devices diff --git a/kernel/patches/devices/s8000/0007-arm64-dts-apple-enable-the-G5P-GPU-on-J71t.patch b/kernel/patches/devices/s8000/0007-arm64-dts-apple-enable-the-G5P-GPU-on-J71t.patch new file mode 100644 index 00000000..e17f719a --- /dev/null +++ b/kernel/patches/devices/s8000/0007-arm64-dts-apple-enable-the-G5P-GPU-on-J71t.patch @@ -0,0 +1,50 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 5 Nov 2026 09:01:41 +0000 +Subject: [PATCH] arm64: dts: apple: enable the G5P GPU on J71t + +Keep other S8003 boards disabled pending qualification. Resources match the device-tested fork node. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + arch/arm64/boot/dts/apple/s8003.dtsi | 17 +++++++++++++++++ + 2 files changed, 21 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc660..7e4d67d7f8 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -24,3 +24,7 @@ &touchscreen { + &wifi { + brcm,board-type = "apple,feara"; + }; ++ ++&gpu_diag { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/apple/s8003.dtsi b/arch/arm64/boot/dts/apple/s8003.dtsi +index b515889416..67cdd227a9 100644 +--- a/arch/arm64/boot/dts/apple/s8003.dtsi ++++ b/arch/arm64/boot/dts/apple/s8003.dtsi +@@ -336,3 +336,20 @@ map0 { + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/* G5P resources; enable only on a board qualified with this driver. */ ++&{/soc} { ++ gpu_diag: gpu@201000000 { ++ status = "disabled"; ++ compatible = "gpu,s8003"; ++ reg = <0x2 0x01000000 0x0 0x100000>, ++ <0x2 0x01e00000 0x0 0x100000>; ++ reg-names = "rgx", "fender"; ++ power-domains = <&ps_gfx>; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ }; ++}; diff --git a/kernel/patches/drivers/drm/0008-drm-apple-define-the-G5P-merged-render-winsys-ABI.patch b/kernel/patches/drivers/drm/0008-drm-apple-define-the-G5P-merged-render-winsys-ABI.patch new file mode 100644 index 00000000..f597024c --- /dev/null +++ b/kernel/patches/drivers/drm/0008-drm-apple-define-the-G5P-merged-render-winsys-ABI.patch @@ -0,0 +1,565 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 4 Nov 2026 21:55:55 +0000 +Subject: [PATCH] drm/apple: define the G5P merged render winsys ABI + +Expose BOs, VM mappings, render contexts, TYPED_V3 passes, targets and +synchronization. Keep the reserved legacy command numbers unused. V3 +embeds the preceding pass layouts without offering their submit +profiles. + +[Orchard: from brkosebay/linux 0ada05ee7e1c5b953ba613c7048d9ef753111bac; source unchanged.] + +Assisted-by: LLM + +Co-authored-by: Claude Opus 5.5 + +Co-authored-by: GPT-6.1 Sol +--- + include/uapi/drm/apple_g5p_drm.h | 537 +++++++++++++++++++++++++++++++ + 1 file changed, 537 insertions(+) + create mode 100644 include/uapi/drm/apple_g5p_drm.h + +diff --git a/include/uapi/drm/apple_g5p_drm.h b/include/uapi/drm/apple_g5p_drm.h +new file mode 100644 +index 0000000000..136ccca21b +--- /dev/null ++++ b/include/uapi/drm/apple_g5p_drm.h +@@ -0,0 +1,537 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++#ifndef _UAPI_APPLE_G5P_DRM_H_ ++#define _UAPI_APPLE_G5P_DRM_H_ ++ ++#include "drm.h" ++ ++/* G5P merged winsys ABI. Retired ioctl numbers remain reserved. */ ++#define DRM_G5P_UAPI_VERSION 1 ++#define DRM_G5P_CAP_BO_CPU (1ULL << 0) ++#define DRM_G5P_CAP_VM_PREPARE (1ULL << 1) ++#define DRM_G5P_CAP_SYNCOBJ (1ULL << 2) ++#define DRM_G5P_CAP_RENDER (1ULL << 5) ++#define DRM_G5P_CAP_PRIME (1ULL << 6) ++#define DRM_G5P_CAP_CODE_BASE_VERIFIED (1ULL << 7) ++#define DRM_G5P_CAP_VM_LIVE (1ULL << 8) ++#define DRM_G5P_CAP_CONTEXT_LIVE (1ULL << 9) ++#define DRM_G5P_CAP_JOB_STATUS (1ULL << 10) ++#define DRM_G5P_CAP_CONTEXT_STATUS (1ULL << 11) ++#define DRM_G5P_CAP_FAULT_INFO (1ULL << 13) ++ ++/* LIVE means the owned user catalogue has completed firmware attachment. */ ++#define DRM_G5P_CONTEXT_LIVE 3 ++#define DRM_G5P_CONTEXT_HOST_ONLY 0 ++#define DRM_G5P_CONTEXT_LOST 1 ++ ++struct drm_g5p_get_context_status { ++ __u32 context; ++ __u32 flags; ++ __u32 state; ++ __u32 map_count; ++ __u64 mapped_bytes; ++ __u64 reserved; ++}; ++ ++/* First terminal firmware event, retained independently of last_event. */ ++#define DRM_G5P_FAULT_NONE 0 ++#define DRM_G5P_FAULT_RECOVERY 1 ++#define DRM_G5P_FAULT_PAGE_REQUEST 2 ++#define DRM_G5P_FAULT_CHANNEL 3 ++#define DRM_G5P_FAULT_UNKNOWN 4 ++#define DRM_G5P_FAULT_VALID_CHANNEL (1U << 0) ++struct drm_g5p_get_fault_info { ++ __u32 flags; ++ __u32 kind; ++ __u64 event_serial; /* One-based arrival order; zero if no fault. */ ++ __u32 valid; ++ __u32 channel_class; /* Event8: 0 memory failure, 1 killed, others opaque. */ ++ __u32 channel_token; /* Firmware token, not a DRM context handle. */ ++ __u32 channel_stamp; ++ __u32 raw[11]; /* Includes unknown fields; no decoded fault VA is claimed. */ ++ __u32 reserved; ++}; ++ ++struct drm_g5p_get_info { ++ __u32 size; ++ __u32 flags; ++ __u32 version; ++ __u32 gpu_page_shift; ++ __u64 capabilities; ++ __u64 va_start; ++ __u64 va_end; ++ __u64 reserved_start; ++ __u64 reserved_end; ++ __u64 code_base; /* Zero until CODE_BASE_VERIFIED. */ ++ __u64 max_bo_size; ++ __u32 bvnc[4]; /* Static target, not a new MMIO measurement. */ ++ __u64 reserved[2]; ++}; ++ ++/* GPU access roles; CPU initialization/mmap is permitted for every role. */ ++#define DRM_G5P_BO_READ (1U << 0) ++#define DRM_G5P_BO_WRITE (1U << 1) ++#define DRM_G5P_BO_EXEC (1U << 2) ++struct drm_g5p_create_bo { ++ __u64 size; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_bo_mmap_offset { ++ __u32 handle; ++ __u32 flags; ++ __u64 offset; ++}; ++ ++#define DRM_G5P_BO_SYNC_TO_DEVICE 0 ++#define DRM_G5P_BO_SYNC_TO_CPU 1 ++struct drm_g5p_bo_sync { ++ __u32 handle; ++ __u32 op; ++ __u64 offset; ++ __u64 size; ++}; ++ ++struct drm_g5p_create_vm { ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_vm { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++#define DRM_G5P_VM_MAP 0 ++#define DRM_G5P_VM_UNMAP 1 ++struct drm_g5p_vm_bind { ++ __u32 vm; ++ __u32 bo; ++ __u64 va; ++ __u64 size; ++ __u64 offset; ++ __u32 op; ++ __u32 flags; ++}; ++ ++struct drm_g5p_create_context { ++ __u32 vm; ++ __u32 type; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_context { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* A point of zero uses a binary syncobj; nonzero uses a timeline point. */ ++#define DRM_G5P_SYNC_SIGNAL (1U << 0) ++struct drm_g5p_sync { ++ __u32 handle; ++ __u32 flags; ++ __u64 point; ++}; ++ ++/* Privileged one-shot kernel fixture; all payload/BO fields must be zero. ++ * The empty user context authorizes ownership only; its VM is not attached. ++ * Does not advertise general COMPUTE or attach user-provided GPU addresses. ++ */ ++ ++/* Geometry is all zero/NONE while CODE_BASE_VERIFIED is clear. */ ++#define DRM_G5P_BASE_NONE 0 ++#define DRM_G5P_BASE_DEVICE 1 ++#define DRM_G5P_BASE_VM 2 ++#define DRM_G5P_BASE_CONTEXT 3 ++struct drm_g5p_get_exec_info { ++ __u32 size; ++ __u32 flags; ++ __u32 base_scope; ++ __u32 pad; ++ __u64 usc_base; ++ __u64 pds_base; ++ __u64 window_size; ++ __u64 allocation_alignment; ++ __u32 usc_offset_bits; ++ __u32 pds_offset_bits; ++ __u64 reserved[3]; ++}; ++ ++#define DRM_G5P_JOB_PENDING 0 ++#define DRM_G5P_JOB_COMPLETE 1 ++struct drm_g5p_get_job_status { ++ __u64 job_id; ++ __u32 flags; ++ __u32 state; ++ __s32 error; /* -EINPROGRESS while pending; immutable after completion. */ ++ __u32 pad; ++ __u64 reset_generation; /* Zero: no reset/recovery support. */ ++}; ++ ++#define DRM_G5P_GET_INFO 0x00 ++#define DRM_G5P_CREATE_BO 0x01 ++#define DRM_G5P_BO_MMAP_OFFSET 0x02 ++#define DRM_G5P_BO_SYNC 0x03 ++#define DRM_G5P_CREATE_VM 0x04 ++#define DRM_G5P_DESTROY_VM 0x05 ++#define DRM_G5P_VM_BIND 0x06 ++#define DRM_G5P_CREATE_CONTEXT 0x07 ++#define DRM_G5P_DESTROY_CONTEXT 0x08 ++#define DRM_G5P_GET_EXEC_INFO 0x0b ++#define DRM_G5P_GET_JOB_STATUS 0x0c ++#define DRM_G5P_GET_CONTEXT_STATUS 0x0d ++#define DRM_G5P_GET_FAULT_INFO 0x0e ++ ++#define DRM_IOCTL_G5P_GET_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_INFO, struct drm_g5p_get_info) ++#define DRM_IOCTL_G5P_CREATE_BO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_BO, struct drm_g5p_create_bo) ++#define DRM_IOCTL_G5P_BO_MMAP_OFFSET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_BO_MMAP_OFFSET, \ ++ struct drm_g5p_bo_mmap_offset) ++#define DRM_IOCTL_G5P_BO_SYNC \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_BO_SYNC, struct drm_g5p_bo_sync) ++#define DRM_IOCTL_G5P_CREATE_VM \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_VM, struct drm_g5p_create_vm) ++#define DRM_IOCTL_G5P_DESTROY_VM \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_VM, \ ++ struct drm_g5p_destroy_vm) ++#define DRM_IOCTL_G5P_VM_BIND \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_VM_BIND, struct drm_g5p_vm_bind) ++#define DRM_IOCTL_G5P_CREATE_CONTEXT \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_CONTEXT, \ ++ struct drm_g5p_create_context) ++#define DRM_IOCTL_G5P_DESTROY_CONTEXT \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_CONTEXT, \ ++ struct drm_g5p_destroy_context) ++ ++#define DRM_IOCTL_G5P_GET_EXEC_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_EXEC_INFO, \ ++ struct drm_g5p_get_exec_info) ++#define DRM_IOCTL_G5P_GET_JOB_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_JOB_STATUS, \ ++ struct drm_g5p_get_job_status) ++ ++#define DRM_IOCTL_G5P_GET_CONTEXT_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_CONTEXT_STATUS, \ ++ struct drm_g5p_get_context_status) ++ ++#define DRM_IOCTL_G5P_GET_FAULT_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_FAULT_INFO, \ ++ struct drm_g5p_get_fault_info) ++ ++#define DRM_G5P_CAP_IDLE_VM_BIND (1ULL << 18) ++ ++#define DRM_G5P_CAP_BOUNDED_RENDER (1ULL << 19) ++#define DRM_G5P_CAP_TYPED_RENDER (1ULL << 20) ++#define DRM_G5P_CAP_RENDER_USER_GEOMETRY (1ULL << 21) ++#define DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY (1ULL << 22) ++#define DRM_G5P_CAP_TYPED_RENDER_V2 (1ULL << 23) ++/* Bit 24 is reserved for COMPUTE_BLIT_V1 and bit 25 is FIXED_DATA_B1 on ++ * a9/release, so the next bit free in every branch is 26. ++ */ ++#define DRM_G5P_CAP_TYPED_RENDER_V3 (1ULL << 26) ++ ++/** ++ * DRM_G5P_CAP_RENDER_MULTI_CONTEXT - several render contexts share the GPU ++ * ++ * Reported together with TYPED_RENDER_V3. Any number of render contexts, in any ++ * number of files, may submit TYPED_V3 jobs. Each context has its own VM and its ++ * own firmware slot pair, and the jobs of all contexts run one at a time in ++ * submission order. Without this bit, the first context to submit owns the GPU ++ * until reboot and every other context is refused. The other render profiles ++ * stay single-context. ++ */ ++#define DRM_G5P_CAP_RENDER_MULTI_CONTEXT (1ULL << 27) ++ ++/** ++ * DRM_G5P_CAP_RENDER_UNPRIVILEGED - TYPED_V3 render needs no CAP_SYS_RAWIO ++ * ++ * Reported together with RENDER_MULTI_CONTEXT. Any opener of the render node may ++ * create render contexts and submit TYPED_V3 jobs. A render VM then maps none of ++ * the firmware's memory beyond the kernel-built window the GPU's own requesters ++ * need, and the contexts close cleanly: a closed context's firmware binding is ++ * removed and its memory freed. Pinned memory is limited device-wide. The other ++ * render profiles, compute and the diagnostic ioctls still need CAP_SYS_RAWIO. ++ */ ++#define DRM_G5P_CAP_RENDER_UNPRIVILEGED (1ULL << 28) ++ ++/** ++ * DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH - TYPED_V3 carries pixel_output_width ++ * ++ * Reported together with TYPED_RENDER_V3. struct drm_g5p_render_pass_v3 has ++ * pixel_output_width, so a TYPED_V3 pass in the V1 dialect (all-zero ext) can ++ * write several colour targets. Without this bit the field is reserved and ++ * must be zero. ++ */ ++#define DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH (1ULL << 29) ++ ++/* drm_g5p_create_context.type; flags must be zero. */ ++#define DRM_G5P_CONTEXT_RENDER 1 ++ ++#define DRM_G5P_RENDER_PROFILE_TYPED_V3 4 ++ ++/* LINEAR64_V1: kind is also the resource's position in the canonical array. */ ++ ++#define DRM_G5P_RENDER_ACCESS_READ (1U << 0) ++#define DRM_G5P_RENDER_ACCESS_WRITE (1U << 1) ++#define DRM_G5P_RENDER_ACCESS_IMPLICIT (1U << 2) ++#define DRM_G5P_RENDER_ACCESS_CACHE (1U << 3) ++ ++#define DRM_G5P_RENDER_MAX_RESOURCES 256 ++#define DRM_G5P_RENDER_MAX_SYNCS 17 ++#define DRM_G5P_RENDER_MAX_VDM_BYTES 0x10000 ++ ++/* A GPU VA range [va, va + size) inside one existing mapping of handle's BO ++ * in the context VM, at BO byte offset `offset`. Never a CPU pointer, PA or ++ * META address. LINEAR64_V1 requires offset 0 and the exact whole BO. ++ */ ++struct drm_g5p_render_resource { ++ __u32 handle; ++ __u32 kind; ++ __u64 offset; ++ __u64 size; ++ __u64 va; ++}; ++ ++/** ++ * DRM_G5P_CAP_RENDER_MSAA - 4x multisampled render targets and passes ++ * ++ * Reported with TYPED_RENDER_V3 and RENDER_KERNEL_GEOMETRY. ++ * CREATE_RENDER_TARGET accepts samples = 4 (kernel-derived geometry; with ++ * DRM_G5P_RT_USER_GEOMETRY userspace may only restate it), and a TYPED_V3 ++ * pass carries samples, msaa_flags and sample_positions in struct ++ * drm_g5p_render_pass. Without this bit those fields are reserved. ++ */ ++#define DRM_G5P_CAP_RENDER_MSAA (1ULL << 31) ++ ++/* drm_g5p_render_pass.msaa_flags: the end of tile averages each pixel's ++ * samples into the (one-sample) surface the EOT's PBE words name. ++ */ ++#define DRM_G5P_RENDER_MSAA_RESOLVE (1U << 0) ++ ++struct drm_g5p_render_pass { ++ __u32 size; /* sizeof(struct drm_g5p_render_pass) */ ++ __u32 flags; /* zero */ ++ __u64 vdm_va; /* VDM control stream, READ, 4-byte aligned */ ++ __u32 vdm_size; /* bytes, multiple of 4, <= MAX_VDM_BYTES */ ++ __u32 vdm_control; /* V1/V2: TA W+0x88 value, multiple of 16; V3: zero */ ++ __u64 scratch_va; /* TA scratch block, WRITE, 16-byte aligned */ ++ __u64 scratch_size; ++ __u64 scratch_ptr[6]; /* TA scratch/marker pointers, in a WRITE range */ ++ __u64 pool7_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool7_size; ++ __u64 pool8_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool8_size; ++ __u32 context_pds[2][6]; /* TA context store/load PDS descriptors */ ++ __u32 resume_pds[2][4]; /* TA resume PDS descriptors */ ++ __u32 ta_pds[4]; /* TA state PDS descriptor; word 3 zero */ ++ __u32 background[8]; /* 3D background (load/clear) object */ ++ __u32 eot[4]; /* 3D end-of-tile object; word 3 zero */ ++ __u32 eot_resume[4]; /* 3D partial-render EOT; word 3 zero */ ++ __u32 restore[4]; /* 3D partial-render restore words */ ++ /* DRM_G5P_CAP_RENDER_MSAA, TYPED_V3 only; zero in every other profile. ++ * samples: 0 or 1 with a one-sample target, 4 with a four-sample one. ++ * sample_positions: one byte per sample, x in [3:0] and y in [7:4] in ++ * 1/16 px (PPP_MULTISAMPLECTL); 0 takes the target's standard pattern. ++ * The kernel writes the AA mode, ISP_CTL and output-partition words ++ * for the sample count. The draws' ISPCTL.upass (0x7 at four samples, ++ * 0xf at one), the PBE/EOT words and the fragment programs are ++ * userspace's, as for one sample; see docs/gpu/experiments/msaa.md. ++ */ ++ __u32 samples; ++ __u32 msaa_flags; /* DRM_G5P_RENDER_MSAA_* */ ++ __u64 sample_positions; ++ __u64 reserved[6]; ++}; ++ ++/* ++ * TYPED_V2 additions. Each field is a 3D register image that the firmware ++ * uploads from the normal-3D work object. The CR offsets are the firmware ++ * store targets (its 3D upload puts W3c0 on ROGUE_CR_ISP_CTL 0xf38); the ++ * Rogue names are inferred, graded in docs/gpu/render-uapi.md. ++ */ ++#define DRM_G5P_RENDER_MAX_TABLE_ENTRIES 65536 /* ISPDBSC 16-bit indices */ ++#define DRM_G5P_RENDER_ZLS_REGS 6 ++ ++struct drm_g5p_render_pass_ext { ++ /* ISP_SCISSOR_BASE (CR fb0): IPF_SCISSOR_WORD_0/1 pairs, 8 bytes each, ++ * indexed by TA_STATE_ISPDBSC.scindex. READ, 4-byte aligned. ++ */ ++ __u64 scissor_va; ++ __u32 scissor_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ __u32 dbias_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ /* ISP_DBIAS_BASE (CR fb8): {constant, slope, clamp} f32 triples, ++ * 12 bytes each, indexed by ISPDBSC.dbindex. READ, 4-byte aligned. ++ */ ++ __u64 dbias_va; ++ /* PDS_BGRND1_BASE (CR 6a8): background object texture/varying data ++ * (B-relative, like background[]); LOAD_OP_LOAD's texture state. ++ */ ++ __u64 background_texture; ++ __u32 bgobj_depth; /* ISP_BGOBJDEPTH (CR f80): depth clear, ZLS format */ ++ __u32 bgobj_vals; /* ISP_BGOBJVALS (CR f88): stencil [7:0], mask [8], bgtag [9] */ ++ /* Pixel output width (2, 4 or 8): the 6ca7ae0 inputs to ISP_CTL's ++ * partition fields and the six per-core output words. 2 for one ++ * 32-bpp colour attachment (the captured value). ++ */ ++ __u32 pixel_output_width; ++ /* Bit i: zls[i] is an address ([39:4]; [63:40] zero; [3:0] flags) ++ * inside a listed READ|WRITE range. The kernel does not decode ZLS. ++ */ ++ __u32 zls_address_mask; ++ /* Opaque ZLS register images, CR f48, f50, f58, f60, f68 and fd0: ++ * Rogue ISP_ZLSCTL, ZLOAD/ZSTORE and STENCIL_LOAD/STORE bases plus ++ * one more; positions proven, order unqualified. Zero: no ZLS. ++ */ ++ __u64 zls[DRM_G5P_RENDER_ZLS_REGS]; ++ __u64 reserved[4]; ++}; ++ ++struct drm_g5p_render_pass_v2 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++}; ++ ++#define DRM_G5P_RENDER_MAX_PROGRAM_BYTES 0x1000 ++#define DRM_G5P_RENDER_STREAM_TERMINATE 0xc0000000U ++ ++struct drm_g5p_render_pass_v3 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++ __u64 ta_program_va; /* -> W+0x88 / R0+0x470 as an offset from pds_base */ ++ __u32 ta_program_size; /* bytes, 16..MAX_PROGRAM_BYTES, multiple of 16 */ ++ __u32 vdm_end; /* bytes from vdm_va past STREAM_TERMINATE, <= vdm_size */ ++ __u32 pixel_output_width; /* V1 dialect: 0 (= 2), 2, 4 or 8 */ ++ __u32 pad; /* zero */ ++ __u64 reserved[3]; ++}; ++ ++struct drm_g5p_submit_render_typed { ++ __u32 context; ++ __u32 flags; /* zero */ ++ __u32 profile; ++ __u32 render_target; /* handle from CREATE_RENDER_TARGET */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[] */ ++ __u64 pass; /* struct drm_g5p_render_pass */ ++ __u32 pass_size; ++ __u32 pad; ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; ++}; ++ ++/* ++ * Render-target memory geometry: the AGXRenderTargetMemoryG5 arithmetic ++ * outputs (firmware round16 section 3, R-offsets in comments). The kernel ++ * computes them for its verified table (64x64x1), or with ++ * RENDER_KERNEL_GEOMETRY for any size; with RENDER_USER_GEOMETRY userspace ++ * supplies them and the kernel bounds the memory they imply (and, with ++ * RENDER_KERNEL_GEOMETRY, requires its own result). ++ */ ++struct drm_g5p_rt_geometry { ++ __u32 header_bytes; /* R10 */ ++ __u32 region_bytes; /* R20 */ ++ __u64 packed_state; /* Rb0 */ ++ __u32 state_word; /* Rb8 */ ++ __u32 padded_dims; /* Rbc */ ++ __u64 scale[3]; /* Rc0, Rc8, Rd0 packed float pairs */ ++ __u64 ta_state[2]; /* Rf0, Rf8 */ ++ __u64 ta_tiling[2]; /* R100, R108 */ ++ __u32 ta_tiling_word; /* R110 */ ++ __u32 tile_count; /* R118 */ ++ __u64 secondary_bytes; /* R138 */ ++ __u64 secondary_align; /* R178 */ ++ __u64 reserved[2]; ++}; ++ ++#define DRM_G5P_RT_USER_GEOMETRY (1U << 0) ++struct drm_g5p_create_render_target { ++ __u32 context; /* render context; RT memory lives in its VM */ ++ __u32 flags; ++ __u32 width; ++ __u32 height; ++ __u32 samples; /* 1, or 4 with DRM_G5P_CAP_RENDER_MSAA */ ++ __u32 layers; /* 1 */ ++ __u32 handle; /* output */ ++ __u32 pad; ++ __u64 main_va; /* output: kernel-private GPU VA */ ++ __u64 main_bytes; /* output */ ++ __u64 secondary_va; /* output */ ++ __u64 secondary_bytes; /* output: mapped bytes */ ++ struct drm_g5p_rt_geometry geometry; /* input if USER, else output */ ++ __u32 pm_blocks; /* output with RENDER_KERNEL_GEOMETRY: context PM pool */ ++ __u32 pad2; ++ __u64 reserved; ++}; ++ ++struct drm_g5p_destroy_render_target { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* GET_RENDER_INFO.flags, output only (input must be zero). ++ * VARIANT_ACTIVE: a render_user_* hypothesis knob (stage, cache/mmu/timestamp ++ * variant, 3d_after_ta, pm_policy, cache_repeat, wait_ms, pm/memory/rt/slot ++ * variant, profile1_vdm_control) differs from its default, so a context ++ * created now latches a non-default job behaviour. GET_RENDER_INFO has no ++ * context: it reports what CREATE_CONTEXT would latch at this moment. ++ */ ++#define DRM_G5P_RENDER_INFO_VARIANT_ACTIVE (1U << 0) ++ ++struct drm_g5p_get_render_info { ++ __u32 size; /* sizeof, exact */ ++ __u32 flags; /* in: zero; out: RENDER_INFO_* */ ++ __u32 profiles; /* bit N: profile N accepted */ ++ __u32 max_render_targets; ++ __u64 private_va_start; /* kernel window in render VMs */ ++ __u64 private_va_end; ++ __u64 user_va_min; /* lowest mappable VA in a render VM */ ++ __u64 usc_base; /* common USC/PDS base, 32-bit offsets */ ++ __u64 pds_base; ++ __u64 code_window; /* bytes above usc_base usable for code */ ++ __u32 pm_blocks; /* parameter buffer blocks per context */ ++ __u32 pm_block_bytes; ++ __u32 max_queued_jobs; ++ __u32 max_vdm_bytes; ++ __u32 job_timeout_ms; ++ __u32 retained_results; ++ /* RENDER_KERNEL_GEOMETRY only, else zero (64x64 only). */ ++ __u32 max_render_width; ++ __u32 max_render_height; ++ __u32 max_pm_blocks; /* PM pool growth bound per context */ ++ __u32 pad; ++ union { ++ __u64 reserved[2]; /* in: zero */ ++ struct { ++ /* TYPED_V3: every GPU VA the TA/3D read ends at or below ++ * this; 0 makes no statement. Out only. ++ */ ++ __u64 hw_va_limit; ++ __u64 reserved1; ++ }; ++ }; ++}; ++ ++#define DRM_G5P_CREATE_RENDER_TARGET 0x11 ++#define DRM_G5P_DESTROY_RENDER_TARGET 0x12 ++#define DRM_G5P_GET_RENDER_INFO 0x13 ++#define DRM_G5P_SUBMIT_RENDER_TYPED 0x14 ++ ++#define DRM_IOCTL_G5P_SUBMIT_RENDER_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_RENDER_TYPED, \ ++ struct drm_g5p_submit_render_typed) ++#define DRM_IOCTL_G5P_CREATE_RENDER_TARGET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_RENDER_TARGET, \ ++ struct drm_g5p_create_render_target) ++#define DRM_IOCTL_G5P_DESTROY_RENDER_TARGET \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_RENDER_TARGET, \ ++ struct drm_g5p_destroy_render_target) ++#define DRM_IOCTL_G5P_GET_RENDER_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_RENDER_INFO, \ ++ struct drm_g5p_get_render_info) ++ ++#endif diff --git a/kernel/patches/drivers/drm/0009-drm-apple-add-the-A9-firmware-and-transport-core-for.patch b/kernel/patches/drivers/drm/0009-drm-apple-add-the-A9-firmware-and-transport-core-for.patch new file mode 100644 index 00000000..9283f99f --- /dev/null +++ b/kernel/patches/drivers/drm/0009-drm-apple-add-the-A9-firmware-and-transport-core-for.patch @@ -0,0 +1,14551 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 4 Nov 2026 21:55:57 +0000 +Subject: [PATCH] drm/apple: add the A9 firmware and transport core for merged + rendering + +Load the device firmware directly and verify the original and packed +images. Retain the release power, IRQ transport, RAM publication and +terminal-access rules. Exclude firmware patches, captured programs, +compute execution and fixed-fixture experiments. + +[Orchard: from brkosebay/linux 67feb03244eaf36f3539f4c5eaddcbd8e7d9f21c; source unchanged.] + +Assisted-by: LLM + +Co-authored-by: Claude Opus 5.5 + +Co-authored-by: GPT-6.1 Sol +--- + drivers/gpu/apple-g5p/bits-compat.h | 20 + + drivers/gpu/apple-g5p/boot-layout.h | 306 ++++ + drivers/gpu/apple-g5p/boot-r73b.inc | 1180 +++++++++++++ + drivers/gpu/apple-g5p/boot-r74f.inc | 532 ++++++ + drivers/gpu/apple-g5p/boot-r74g.inc | 288 +++ + drivers/gpu/apple-g5p/boot.inc | 1662 ++++++++++++++++++ + drivers/gpu/apple-g5p/core.inc | 567 ++++++ + drivers/gpu/apple-g5p/diagnostics.inc | 181 ++ + drivers/gpu/apple-g5p/dirtystart108.h | 454 +++++ + drivers/gpu/apple-g5p/dirtystart108.inc | 273 +++ + drivers/gpu/apple-g5p/drm-validate.h | 40 + + drivers/gpu/apple-g5p/fender.h | 87 + + drivers/gpu/apple-g5p/gpu-oc.h | 149 ++ + drivers/gpu/apple-g5p/heartbeat116.h | 175 ++ + drivers/gpu/apple-g5p/kick.inc | 27 + + drivers/gpu/apple-g5p/latency123.h | 63 + + drivers/gpu/apple-g5p/ldr.h | 154 ++ + drivers/gpu/apple-g5p/lifecycle.inc | 1032 +++++++++++ + drivers/gpu/apple-g5p/meta71c114.h | 65 + + drivers/gpu/apple-g5p/mmio-licence.h | 148 ++ + drivers/gpu/apple-g5p/mmio.inc | 295 ++++ + drivers/gpu/apple-g5p/mmu-attrs.inc | 54 + + drivers/gpu/apple-g5p/mmu-tables.inc | 57 + + drivers/gpu/apple-g5p/pack.h | 89 + + drivers/gpu/apple-g5p/perf-envelope.h | 107 ++ + drivers/gpu/apple-g5p/power-r33.inc | 70 + + drivers/gpu/apple-g5p/power-r41.inc | 135 ++ + drivers/gpu/apple-g5p/r74e112.h | 137 ++ + drivers/gpu/apple-g5p/r87b-attach.h | 34 + + drivers/gpu/apple-g5p/readback.h | 40 + + drivers/gpu/apple-g5p/recover.inc | 101 ++ + drivers/gpu/apple-g5p/render-layout-r22.h | 54 + + drivers/gpu/apple-g5p/render-layout.h | 59 + + drivers/gpu/apple-g5p/render-user-bind.h | 285 +++ + drivers/gpu/apple-g5p/render-user-close.h | 91 + + drivers/gpu/apple-g5p/render-user-evwait.h | 52 + + drivers/gpu/apple-g5p/render-user-fault.h | 60 + + drivers/gpu/apple-g5p/render-user-faultreg.h | 124 ++ + drivers/gpu/apple-g5p/render-user-geometry.h | 144 ++ + drivers/gpu/apple-g5p/render-user-layout.h | 644 +++++++ + drivers/gpu/apple-g5p/render-user-loss.h | 120 ++ + drivers/gpu/apple-g5p/render-user-mmu.h | 60 + + drivers/gpu/apple-g5p/render-user-msaa.h | 88 + + drivers/gpu/apple-g5p/render-user-pmset.h | 77 + + drivers/gpu/apple-g5p/render-user-quiesce.h | 145 ++ + drivers/gpu/apple-g5p/render-user-recover.h | 107 ++ + drivers/gpu/apple-g5p/render-user-retire.h | 248 +++ + drivers/gpu/apple-g5p/render-user-sched.h | 155 ++ + drivers/gpu/apple-g5p/render-user-serial.h | 122 ++ + drivers/gpu/apple-g5p/render-user-sync.h | 50 + + drivers/gpu/apple-g5p/render-user-template.h | 24 + + drivers/gpu/apple-g5p/render-user-terminal.h | 103 ++ + drivers/gpu/apple-g5p/render-user-touch.h | 102 ++ + drivers/gpu/apple-g5p/render-user-wait.h | 266 +++ + drivers/gpu/apple-g5p/render-user-wake.h | 273 +++ + drivers/gpu/apple-g5p/render-user-window.h | 74 + + drivers/gpu/apple-g5p/render-variants-r25.h | 48 + + drivers/gpu/apple-g5p/retire108.h | 434 +++++ + drivers/gpu/apple-g5p/teardown.h | 740 ++++++++ + drivers/gpu/apple-g5p/teardown.inc | 692 ++++++++ + drivers/gpu/apple-g5p/transport-budget.inc | 6 + + drivers/gpu/apple-g5p/vm-mutation.h | 36 + + drivers/gpu/apple-g5p/work-layout.h | 21 + + 63 files changed, 14026 insertions(+) + create mode 100644 drivers/gpu/apple-g5p/bits-compat.h + create mode 100644 drivers/gpu/apple-g5p/boot-layout.h + create mode 100644 drivers/gpu/apple-g5p/boot-r73b.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74f.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74g.inc + create mode 100644 drivers/gpu/apple-g5p/boot.inc + create mode 100644 drivers/gpu/apple-g5p/core.inc + create mode 100644 drivers/gpu/apple-g5p/diagnostics.inc + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.h + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.inc + create mode 100644 drivers/gpu/apple-g5p/drm-validate.h + create mode 100644 drivers/gpu/apple-g5p/fender.h + create mode 100644 drivers/gpu/apple-g5p/gpu-oc.h + create mode 100644 drivers/gpu/apple-g5p/heartbeat116.h + create mode 100644 drivers/gpu/apple-g5p/kick.inc + create mode 100644 drivers/gpu/apple-g5p/latency123.h + create mode 100644 drivers/gpu/apple-g5p/ldr.h + create mode 100644 drivers/gpu/apple-g5p/lifecycle.inc + create mode 100644 drivers/gpu/apple-g5p/meta71c114.h + create mode 100644 drivers/gpu/apple-g5p/mmio-licence.h + create mode 100644 drivers/gpu/apple-g5p/mmio.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-attrs.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-tables.inc + create mode 100644 drivers/gpu/apple-g5p/pack.h + create mode 100644 drivers/gpu/apple-g5p/perf-envelope.h + create mode 100644 drivers/gpu/apple-g5p/power-r33.inc + create mode 100644 drivers/gpu/apple-g5p/power-r41.inc + create mode 100644 drivers/gpu/apple-g5p/r74e112.h + create mode 100644 drivers/gpu/apple-g5p/r87b-attach.h + create mode 100644 drivers/gpu/apple-g5p/readback.h + create mode 100644 drivers/gpu/apple-g5p/recover.inc + create mode 100644 drivers/gpu/apple-g5p/render-layout-r22.h + create mode 100644 drivers/gpu/apple-g5p/render-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-bind.h + create mode 100644 drivers/gpu/apple-g5p/render-user-close.h + create mode 100644 drivers/gpu/apple-g5p/render-user-evwait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-fault.h + create mode 100644 drivers/gpu/apple-g5p/render-user-faultreg.h + create mode 100644 drivers/gpu/apple-g5p/render-user-geometry.h + create mode 100644 drivers/gpu/apple-g5p/render-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-loss.h + create mode 100644 drivers/gpu/apple-g5p/render-user-mmu.h + create mode 100644 drivers/gpu/apple-g5p/render-user-msaa.h + create mode 100644 drivers/gpu/apple-g5p/render-user-pmset.h + create mode 100644 drivers/gpu/apple-g5p/render-user-quiesce.h + create mode 100644 drivers/gpu/apple-g5p/render-user-recover.h + create mode 100644 drivers/gpu/apple-g5p/render-user-retire.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sched.h + create mode 100644 drivers/gpu/apple-g5p/render-user-serial.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sync.h + create mode 100644 drivers/gpu/apple-g5p/render-user-template.h + create mode 100644 drivers/gpu/apple-g5p/render-user-terminal.h + create mode 100644 drivers/gpu/apple-g5p/render-user-touch.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wake.h + create mode 100644 drivers/gpu/apple-g5p/render-user-window.h + create mode 100644 drivers/gpu/apple-g5p/render-variants-r25.h + create mode 100644 drivers/gpu/apple-g5p/retire108.h + create mode 100644 drivers/gpu/apple-g5p/teardown.h + create mode 100644 drivers/gpu/apple-g5p/teardown.inc + create mode 100644 drivers/gpu/apple-g5p/transport-budget.inc + create mode 100644 drivers/gpu/apple-g5p/vm-mutation.h + create mode 100644 drivers/gpu/apple-g5p/work-layout.h + +diff --git a/drivers/gpu/apple-g5p/bits-compat.h b/drivers/gpu/apple-g5p/bits-compat.h +new file mode 100644 +index 0000000000..608e9b8eca +--- /dev/null ++++ b/drivers/gpu/apple-g5p/bits-compat.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_BITS_COMPAT_H ++#define G5P_BITS_COMPAT_H ++ ++#ifdef __KERNEL__ ++#include ++#include ++#else ++#ifndef BIT ++#define BIT(nr) (1UL << (nr)) ++#endif ++#ifndef BIT_ULL ++#define BIT_ULL(nr) (1ULL << (nr)) ++#endif ++#ifndef ARRAY_SIZE ++#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) ++#endif ++#endif ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-layout.h b/drivers/gpu/apple-g5p/boot-layout.h +new file mode 100644 +index 0000000000..752e6873f5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-layout.h +@@ -0,0 +1,306 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_BOOT_LAYOUT_H ++#define APPLE_G5P_BOOT_LAYOUT_H ++ ++#include "bits-compat.h" ++#include "pack.h" ++ ++#define G5P_GPU_PAGE 0x4000U ++#define G5P_ROOT_SIZE 0x8000U ++#define G5P_SHARED_SIZE 0x60000U ++#define G5P_IMAGE_VA 0xe1c0000000ULL ++#define G5P_SHARED_VA 0xe1c0200000ULL ++#define G5P_ROOT_VA 0xe1c3200000ULL ++#define G5P_PIO_VA 0xe1c4200000ULL ++#define G5P_PIO_SIZE 0x10000U ++ ++/* c95da0 descriptors, sizes at 5940b90; c97b6c packs on 64-byte boundaries. */ ++struct g5p_boot_layout { ++ u32 offset[0x90]; ++ u8 group[0x90]; ++ u32 used[2]; ++}; ++ ++static inline void g5p_put32(u8 *p, u32 value) ++{ ++ p[0] = value; ++ p[1] = value >> 8; ++ p[2] = value >> 16; ++ p[3] = value >> 24; ++} ++ ++static inline void g5p_put64(u8 *p, u64 value) ++{ ++ g5p_put32(p, value); ++ g5p_put32(p + 4, value >> 32); ++} ++ ++static inline u32 g5p_meta_address(u64 va) ++{ ++ /* ca0744: root range 8 bypasses META caches; other shared RAM is global. */ ++ u32 prefix = va >= G5P_ROOT_VA && va < G5P_ROOT_VA + 0x1000000 ? ++ 0x40000000 : ++ 0xc0000000; ++ ++ return ((u32)va + 0x40000000) | prefix; ++} ++ ++static inline int g5p_plan_layout(struct g5p_boot_layout *p) ++{ ++ static const struct { ++ u8 cpu, gpu, group; ++ u32 size; ++ } objects[] = { ++ { 0x44, 0x50, 0, 0x1490 }, ++ { 0x47, 0x51, 0, 0x990 }, ++ { 0x48, 0x52, 0, 0x8e0 }, ++ { 0x49, 0x53, 0, 0x8c0 }, ++ { 0x4a, 0x54, 0, 0x938 }, ++ { 0x4b, 0x55, 0, 0x28 }, ++ { 0x4c, 0x56, 0, 8 }, ++ { 0x4d, 0x57, 0, 0x78c }, ++ { 0x4e, 0x58, 0, 0x84 }, ++ { 0x5e, 0x5f, 1, 0x2058 }, ++ { 0x70, 0x78, 1, 0x20 }, ++ { 0x71, 0x79, 1, 0x80 }, ++ { 0x72, 0x7a, 1, 0x20 }, ++ { 0x73, 0x7b, 1, 0x20 }, ++ { 0x74, 0x7c, 1, 0x2c00 }, ++ { 0x75, 0x7d, 1, 0xb000 }, ++ { 0x4f, 0x59, 1, 0x36000 }, ++ { 0x76, 0x7e, 1, 0x2c00 }, ++ { 0x77, 0x7f, 1, 0x2400 }, ++ { 0x6c, 0x6e, 1, 0x20 }, ++ { 0x6d, 0x6f, 1, 0x1400 }, ++ { 0x60, 0x62, 1, 0x20 }, ++ { 0x64, 0x66, 1, 0x20 }, ++ { 0x68, 0x6a, 1, 0x20 }, ++ { 0x61, 0x63, 1, 0x1400 }, ++ { 0x65, 0x67, 1, 0x1400 }, ++ { 0x69, 0x6b, 1, 0x1400 }, ++ /* Separate c95da0 allocations, placed after the recovered group. */ ++ { 0x5c, 0x5d, 1, 8 }, ++ { 0x85, 0x88, 1, 0x400 }, ++ { 0x84, 0x87, 1, 0x400 }, ++ { 0x86, 0x89, 1, 0x800 }, ++ }; ++ unsigned int i; ++ ++ memset(p, 0, sizeof(*p)); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) { ++ u32 group = objects[i].group; ++ u32 off = (p->used[group] + 63) & ~63U; ++ ++ p->offset[objects[i].cpu] = off; ++ p->offset[objects[i].gpu] = off; ++ p->group[objects[i].cpu] = group; ++ p->group[objects[i].gpu] = group; ++ p->used[group] = off + objects[i].size; ++ } ++ return p->used[0] <= G5P_ROOT_SIZE && p->used[1] <= G5P_SHARED_SIZE ? ++ 0 : ++ -EINVAL; ++} ++ ++static inline u64 g5p_object_va(const struct g5p_boot_layout *p, u32 slot) ++{ ++ return (p->group[slot] ? G5P_SHARED_VA : G5P_ROOT_VA) + p->offset[slot]; ++} ++ ++static inline u8 *g5p_object(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 slot) ++{ ++ return (p->group[slot] ? shared : root) + p->offset[slot]; ++} ++ ++/* ca9b34 + cab44c + cac314: 40-bit PA, 16 KiB GPU pages. */ ++static inline u32 g5p_catalogue_entry(u64 pa) ++{ ++ return (u32)(pa >> 8) | 1; ++} ++ ++static inline u64 g5p_directory_entry(u64 pa) ++{ ++ /* Geometry bytes 3c..3f are 08 05 05 15: size, entry/PA/VA shifts. */ ++ return pa | 3; ++} ++ ++static inline u64 g5p_leaf_entry(u64 pa, u32 attributes) ++{ ++ /* caa6e4 range attributes -> ca9a60: META protect, CC / SLC bypass. */ ++ return (pa & 0xfffffff000ULL) | (attributes & 0xc) | ++ ((u64)(attributes & 0x10) << 58) | 1; ++} ++ ++static inline void g5p_make_boot_config(struct g5p_ldr_image *im, ++ const struct g5p_boot_layout *p) ++{ ++ static const u32 bases[] = { 0x10000000, 0x20000000, 0x40000000, ++ 0x60001000 }; ++ static const u32 sizes[] = { 0x18000, 0x4000, 0x4000, 0x4000 }; ++ static const u32 offsets[] = { 0x4000, 0x1c000, 0, 0x20000 }; ++ u8 *out = im->data + 0x200; ++ unsigned int seg, i; ++ ++ for (seg = 0; seg < 4; seg++) { ++ u64 va = G5P_IMAGE_VA + offsets[seg]; ++ u32 reg = 0x04850000 + seg * 16; ++ u32 values[] = { bases[seg] | 0xf02, sizes[seg] - 1, (u32)va, ++ (u32)(va >> 32) | 0x300000 }; ++ ++ if (seg == 2) { ++ g5p_put32(out, reg + 12); ++ g5p_put32(out + 4, (u32)(va >> 32) | 0x100); ++ out += 8; ++ continue; ++ } ++ for (i = 0; i < 4; i++, out += 8) { ++ g5p_put32(out, reg + i * 4); ++ g5p_put32(out + 4, values[i]); ++ } ++ } ++ for (i = 0; i < im->config_count; i++, out += 8) { ++ g5p_put32(out, im->config[i][0]); ++ g5p_put32(out + 4, im->config[i][1]); ++ } ++ g5p_put64(out, 0); ++ g5p_put32(out + 8, g5p_meta_address(G5P_IMAGE_VA + 0x3ff8)); ++ g5p_put32(im->data + 0x3ff8, g5p_meta_address(g5p_object_va(p, 0x50))); ++ g5p_put32(im->data + 0x3ffc, g5p_meta_address(g5p_object_va(p, 0x5f))); ++} ++ ++/* c96be8/c9edcc/ca3ccc. Unknown platform policy fields remain zero. */ ++static inline void g5p_make_init(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, const struct g5p_ldr_image *im, ++ u64 catalogue, u64 timestamp) ++{ ++ static const u8 roots[] = { ++ 0x62, 0x63, 0x66, 0x67, 0x6a, 0x6b, 0x6e, 0x6f, ++ 0x78, 0x7c, 0x79, 0x7d, 0x7a, 0x7e, 0x7b, 0x7f, ++ }; ++ static const u8 objects[] = { 0x51, 0x52, 0x53, 0x54, 0x55 }; ++ static const u8 tail[] = { 0x58, 0x56, 0x57, 0x88, 0x59 }; ++ static const u64 pio_pa[] = { 0x201e00000ULL, 0x20e100000ULL, ++ 0x20e104000ULL, 0x201000000ULL }; ++ static const u32 pio_size[] = { 0x5000, 0x4000, 0x4000, 0x20000 }; ++ static const u32 pio_offset[] = { 0, 0x8000, 0xc000, 0 }; ++ u8 *init = g5p_object(p, root, shared, 0x44); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *ctrl = g5p_object(p, root, shared, 0x5e); ++ unsigned int i; ++ ++ memset(root, 0, G5P_ROOT_SIZE); ++ memset(shared, 0, G5P_SHARED_SIZE); ++ for (i = 0; i < ARRAY_SIZE(roots); i++) ++ g5p_put32(init + i * 4, ++ g5p_meta_address(g5p_object_va(p, roots[i]))); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) ++ g5p_put32(init + 0x64 + i * 4, ++ g5p_meta_address(g5p_object_va(p, objects[i]))); ++ for (i = 0; i < ARRAY_SIZE(tail); i++) ++ g5p_put32(init + 0x44c + i * 4, ++ g5p_meta_address(g5p_object_va(p, tail[i]))); ++ for (i = 0; i < 3; i++) { ++ u8 *channel = g5p_object(p, root, shared, 0x47 + i); ++ u32 ptr = g5p_meta_address(g5p_object_va(p, 0x51 + i)); ++ ++ g5p_put32(channel + 4, ptr); ++ g5p_put32(channel + 12, ptr + 8); ++ } ++ g5p_put32(g5p_object(p, root, shared, 0x48) + 0x778, 0xffffffff); ++ g5p_put32(init + 0x480, ++ ((u32)(G5P_IMAGE_VA + 0x24000) + 0x40000000) | 0x40000000); ++ g5p_put64(init + 0x484, G5P_IMAGE_VA + 0x24000); ++ g5p_put32(init + 0x48c, 0x55dc); ++ g5p_put64(meta, 0x600000000ULL); ++ g5p_put64(meta + 8, 0xffc00000); ++ g5p_put64(meta + 0x10, 0x300000000ULL); ++ g5p_put64(meta + 0x18, 0x300000000ULL); ++ for (i = 0; i < ARRAY_SIZE(pio_pa); i++) { ++ u8 *entry = meta + 0x630 + i * 16; ++ ++ g5p_put64(entry, pio_pa[i]); ++ g5p_put32(entry + 8, i == 3 ? 0 : ++ g5p_meta_address(G5P_PIO_VA + ++ pio_offset[i])); ++ g5p_put32(entry + 12, pio_size[i]); ++ } ++ g5p_put32(meta + 0x6c0, 1); ++ g5p_put32(meta + 0x6c8, 1); ++ g5p_put64(meta + 0x6d8, 0x100000000ULL); ++ g5p_put32(meta + 0x6e0, 1); ++ g5p_put32(meta + 0x6e4, 1); ++ /* readChipInfo version 15 -> tables 5942650/5942674 -> 1,5; N=6. */ ++ g5p_put32(meta + 0x704, 1); ++ g5p_put32(meta + 0x708, 5); ++ g5p_put32(meta + 0x70c, 6); ++ g5p_put32(meta + 0x758, 0x14); ++ g5p_put32(meta + 0x75c, 0x4000); ++ g5p_put32(meta + 0x760, 0x1020); ++ g5p_put32(meta + 0x764, im->t1_pc); ++ g5p_put32(meta + 0x768, im->t1_sp); ++ g5p_put64(meta + 0x774, G5P_IMAGE_VA); ++ g5p_put64(meta + 0x77c, 0x300000); ++ g5p_put64(meta + 0x784, timestamp); ++ /* Fixed portion of c974f0; dynamic power-policy fields remain zero. */ ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 8, 0x400000000ULL); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x10, 0x3f800000); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x28, 0x100000001ULL); ++ g5p_put32(g5p_object(p, root, shared, 0x4a) + 0x38, 100); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x8f8, timestamp); ++ g5p_put32(ctrl + 0x24, 0x78); ++ g5p_put32(ctrl + 0x30, 1); ++ g5p_put32(ctrl + 0x1044, g5p_meta_address(g5p_object_va(p, 0x87))); ++ g5p_put32(ctrl + 0x1048, g5p_meta_address(g5p_object_va(p, 0x89))); ++ g5p_put32(ctrl + 0x104c, 1); ++ g5p_put64(g5p_object(p, root, shared, 0x86), catalogue); ++} ++ ++/* Explicit boot-time hypotheses; never modify a live root table. */ ++static inline void g5p_dc_init_variant(const struct g5p_boot_layout *p, ++ u8 *root, u8 *shared, u32 variant) ++{ ++ /* ca0744 uses 40 aliases for cache bypass. Applying that alias to DC ++ * is an unverified hypothesis, not recovered normal host behavior. ++ */ ++ if (variant == 1) ++ g5p_put32(root + 0x18, g5p_le32(root + 0x18) & ~0x80000000U); ++ if (variant == 1 || variant == 6) ++ g5p_put32(root + 0x1c, g5p_le32(root + 0x1c) & ~0x80000000U); ++ /* c96be8 chooses this policy from host+1af8 bits; both values exist. ++ * Whether zero is appropriate for our incomplete policy is unknown. ++ */ ++ if (variant == 5) ++ g5p_put32(shared + p->offset[0x5e] + 0x30, 0); ++ /* MINIM 10001ac8 enters policy1000c0d8 only when meta6c0/6c8 ++ * are both nonzero. Experimental pre-MTS bypass; no live length writes. ++ */ ++ if (variant == 7) ++ g5p_put32(g5p_object(p, root, shared, 0x4d) + 0x6c8, 0); ++ ++ /* Round13: recovered iOS inputs; variants 8/9 retain meta6c8=1. ++ * Neither is a complete DVFS policy (table+3c remains absent). ++ */ ++ if (variant == 8 || variant == 9) { ++ u8 *policy = g5p_object(p, root, shared, 0x4a); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *control = g5p_object(p, root, shared, 0x5e); ++ static const u32 mhz[] = { 0, 340, 474, 550, 723 }; ++ ++ g5p_put32(policy + 4, 16); ++ if (variant == 9) { ++ g5p_put32(meta + 0x710, 4); ++ g5p_put32(meta + 0x714, 5); ++ for (u32 i = 0; i < 16; i++) ++ g5p_put32(meta + 0x718 + 4 * i, ++ i < ARRAY_SIZE(mhz) ? mhz[i] : 0); ++ g5p_put32(control + 0x1028, 3348); ++ g5p_put32(control + 0x102c, 400); ++ g5p_put32(control + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++ } ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +new file mode 100644 +index 0000000000..2407ec1427 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -0,0 +1,1180 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R73b crash-first IRQ174 diagnostics. Included by boot.inc after struct ++ * g5p_boot. Case zero allocates nothing and keeps the r73 IRQ order. ++ * Rows live in CPU RAM only; they are formatted after the epoch, never in the ++ * IRQ service loop. The first 128 rows are retained, the last 128 roll. ++ */ ++#include ++#include ++#include ++#include ++#include "r87b-attach.h" ++#include "r74e112.h" ++#include "heartbeat116.h" ++ ++#define G5P_R73B_ROWS 256 ++#define G5P_R73B_HEAD 128 ++#define G5P_R73B_IRQ_CAP 8192 ++#define G5P_R73B_PANIC_ROWS 32 ++#define G5P_R73B_TRACE_ROWS 128 ++#define G5P_R73B_TRACE_WORDS 9 ++#define G5P_R74_IRQ_CAP 262144 ++#define G5P_R74_IRQ_CAP_MAX 1048576 ++#define G5P_R74_EVENT_FLOOR (256 + 4 * 65536) ++#define G5P_R74_STAGES 8 ++#define G5P_R74_MILESTONES 64 ++#define G5P_R74_MAX_TICKS 16 ++#define G5P_R74_RGX 9 ++#define G5P_R74_WORD1 8 ++#define G5P_R74F_KINDS 3 ++#define G5P_R74F_OFF_US 1000 ++#define G5P_R74F_ON_US 1800 ++#define G5P_R74F_BELL_MS 250 ++#define G5P_R74F_MMU_SLOTS 2 ++#define G5P_R74F_MMU_US 1000 ++#define G5P_R74G_GRACE_ALIVES 4 ++#define G5P_R74G_GRACE_US 20000 ++#define G5P_R74G_COMPANION 0x3980 ++#define G5P_R74G_RECOVERIES 0x3994 ++#define G5P_R74G_CLASS 0x3998 ++#define G5P_R74G_BAD_RING BIT(0) ++#define G5P_R74G_BAD_RECORD BIT(1) ++#define G5P_R74G_BAD_PA BIT(2) ++#define G5P_R74G_BAD_HOST BIT(3) ++#define G5P_R74G_BAD_CACHED BIT(4) ++#define G5P_R74G_BAD_FAILED BIT(5) ++#define G5P_R74G_BAD_COMPANION BIT(6) ++#define G5P_R74G_BAD_MUTATION BIT(7) ++#define G5P_R74H_TAIL 128 ++#define G5P_R74H_PHYS_MAX 1024 ++#define G5P_R74H_PTES 10 ++ ++/* First-ALIVE submit: wait (reads only) for a GPC request in flight to finish. */ ++static unsigned int boot_alive_settle_us = 2000; ++module_param(boot_alive_settle_us, uint, 0600); ++MODULE_PARM_DESC( ++ boot_alive_settle_us, ++ "default0 (single read, as before); 1..5000: when the first-ALIVE submit reads " ++ "R1+0x1000 with bit 4 set (a GPC performance-state request in flight), re-read " ++ "it every 10 us for up to this long before the unchanged state==1 check, as " ++ "Apple's setGPCPerformanceState waits for bit 4; R1 reads only; latched at map"); ++ ++enum g5p_r73b_phase { ++ G5P_R73B_RELEASE_ENTRY = 1, ++ G5P_R73B_RELEASE_FIFO, ++ G5P_R73B_RELEASE_GPC, ++ G5P_R73B_RELEASE_EXIT, ++ G5P_R73B_IRQ, ++ G5P_R73B_DRAIN_BEFORE, ++ G5P_R73B_DRAIN_AFTER, ++ G5P_R73B_DRAIN_BUSY, ++ G5P_R73B_CAP, ++ G5P_R73B_EVENTS_EXIT, ++ G5P_R73B_DC_PUBLISHED, ++ G5P_R73B_DC_KICKED, ++ G5P_R73B_WAIT_EXIT, ++ G5P_R73B_EXPORT, ++ G5P_R73B_PHASES, ++}; ++ ++static const char *const g5p_r73b_phase_names[G5P_R73B_PHASES] = { ++ "unknown", "release-entry", "release-fifo", "release-gpc", ++ "release-exit", "irq", "drain-before", "drain-after", ++ "drain-busy", "cap", "events-exit", "dc-published", ++ "dc-kicked", "wait-exit", "export", ++}; ++ ++enum g5p_r73b_owner { ++ G5P_R73B_NONE, ++ G5P_R73B_OWNER_IRQ, ++ G5P_R73B_OWNER_EVENTS, ++ G5P_R73B_OWNER_PREPARE, ++ G5P_R73B_OWNER_PUBLISH, ++ G5P_R73B_OWNER_KICK, ++ G5P_R73B_OWNER_WAIT, ++}; ++ ++/* Row columns: event, log lane 0, statistics, trace, device control. */ ++static const u8 g5p_r73b_row_slots[5] = { 0x70, 0x71, 0x72, 0x73, 0x6c }; ++ ++struct g5p_r73b_row { ++ u32 seq; /* sequence + 1, published last; zero means empty/torn */ ++ u32 phase; ++ u64 t_ns; ++ u32 irq, ack, released, owner, cap_seen; ++ s32 result; ++ u32 cp[5][2]; ++}; ++ ++enum g5p_r74_step { ++ G5P_R74_IRQ_TEST_REFUSED, ++ G5P_R74_PREPARE_BEGIN, ++ G5P_R74_PREPARE_END, ++ G5P_R74_ATTACH_BEGIN, ++ G5P_R74_ATTACH_PUBLISHED, ++ G5P_R74_ATTACH_DOORBELL_BEGIN, ++ G5P_R74_ATTACH_DOORBELL_END, ++ G5P_R74_ATTACH_CONSUMED, ++ G5P_R74_ATTACH_END, ++ G5P_R74_TA_BEGIN, ++ G5P_R74_TA_PUBLISH, ++ G5P_R74_TA_DOORBELL_RETURNED, ++ G5P_R74_OBSERVE, ++ G5P_R74_WAIT_TICK, ++ G5P_R74_WAIT_EXIT, ++ G5P_R74_REPORT_END, ++ G5P_R74_CMD8_CONSUMED, ++ G5P_R74_TA_CONSUMED, ++ G5P_R74_TA_RAM_COMPLETE, ++ G5P_R74_TA_RAM_TIMEOUT, ++ G5P_R74F_ATTACH_ARMED, ++ G5P_R74F_ATTACH_REACTIVATED, ++ G5P_R74F_ATTACH_BELL, ++ G5P_R74F_MMU_CLOSED, ++ G5P_R74F_TA_ARMED, ++ G5P_R74F_TA_REACTIVATED, ++ G5P_R74F_TA_BELL, ++ G5P_R74F_STOP, ++ G5P_R74G_FW_MMU, ++ G5P_R74G_ACK_SKIPPED, ++ G5P_R74G_TA_GRACE, ++ G5P_R74G_TA_FALLBACK, ++ G5P_R74_STEPS, ++}; ++ ++static const char *const g5p_r74_steps[G5P_R74_STEPS] = { ++ "irq-test-refused", ++ "prepare-begin", ++ "prepare-end", ++ "attach-begin", ++ "attach-dc3-published", ++ "attach-doorbell-begin", ++ "attach-doorbell-end", ++ "attach-consumed", ++ "attach-end", ++ "ta-begin", ++ "ta-publish", ++ "ta-doorbell-returned", ++ "observe", ++ "wait-tick", ++ "wait-exit", ++ "report-end", ++ "cmd8-consumed", ++ "ta-consumed", ++ "ta-ram-complete", ++ "ta-ram-timeout", ++ "attach-bell-armed", ++ "attach-reactivated", ++ "attach-bell", ++ "first-mmu-window-closed", ++ "ta-bell-armed", ++ "ta-reactivated", ++ "ta-bell", ++ "r74f-stop", ++ "firmware-mmu-complete", ++ "r31-ack-skipped", ++ "ta-grace-end", ++ "ta-fallback", ++}; ++ ++/* R74 read-only witnesses; every address is already read elsewhere in-tree. */ ++ ++struct g5p_r74_milestone { ++ u64 t_ns; ++ u32 step, irqs, events; ++ s32 ret; ++ u32 dc[2], ta[2]; ++}; ++ ++enum g5p_r74f_kind { ++ G5P_R74F_DC3, ++ G5P_R74F_TA, ++ G5P_R74F_3D, /* r96: ring 0x64, doorbell 0x11, owned by the ALIVE decoder like the TA's */ ++}; ++ ++/* idle -> published (gate closed) -> open (reactivated) -> firing -> ++ * rang; published/open -> cancelled. FIRING lives under event_lock. ++ */ ++enum g5p_r74f_state { ++ G5P_R74F_IDLE, ++ G5P_R74F_PUBLISHED, ++ G5P_R74F_OPEN, ++ G5P_R74F_FIRING, ++ G5P_R74F_RANG, ++ G5P_R74F_CANCELLED, ++ G5P_R74F_STATES, ++}; ++ ++enum g5p_r74f_cancel { ++ G5P_R74F_CANCEL_NONE, ++ G5P_R74F_CANCEL_REACTIVATION, ++ G5P_R74F_CANCEL_NO_ALIVE, ++ G5P_R74F_CANCEL_CONSUMED, ++ G5P_R74F_CANCEL_PROTOCOL, ++ G5P_R74F_CANCEL_ALIVE_ADVANCED, ++ G5P_R74F_CANCEL_SELF_DRAIN, ++ G5P_R74F_CANCELS, ++}; ++ ++enum g5p_r74f_stop { ++ G5P_R74F_STOP_NONE, ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ G5P_R74F_STOP_MMU_CLOSED, ++ G5P_R74F_STOP_TA_REACTIVATION, ++ G5P_R74F_STOP_TA_NOT_KICKED, ++ G5P_R74G_STOP_DC3_TAGS, ++ G5P_R74G_STOP_DC3_RAM, ++ G5P_R74G_STOP_COMPANION, ++ G5P_R74G_STOP_TA_NOT_CONSUMED, ++ G5P_R74G_STOP_TA_WINDOW_MISSED, ++ G5P_R74F_STOPS, ++}; ++ ++enum g5p_r74f_path { ++ G5P_R74F_PATH_NONE, ++ G5P_R74F_PATH_IMMEDIATE, ++ G5P_R74F_PATH_DEFERRED, ++ G5P_R74F_PATHS, ++}; ++ ++/* One generation. Its owner writes the receipt, then releases state. */ ++struct g5p_r74f_bell { ++ atomic_t state; ++ u32 kind, generation, ring, start, target, path, cancel; ++ u32 arm_alive, open_alive, bell_alive; ++ u32 first_fence, first_state, fence, bell_state; ++ s32 first_ret, bell_ret, power_ret; ++ u32 reactivated, core_fence, gpc_fence, off_raw, on_raw; ++ u32 attempted, kicked, complete; ++ u32 consumer_before, consumer_after, producer_after; ++ u32 alive_done; ++ u64 arm_ns, power_start_ns, off_ns, on_ns, open_ns; ++ u64 bell_start_ns, bell_ns, done_ns; ++}; ++ ++enum g5p_r74g_grace { ++ G5P_R74G_GRACE_NONE, ++ G5P_R74G_GRACE_CONSUMED, ++ G5P_R74G_GRACE_ALIVES_END, ++ G5P_R74G_GRACE_WALL, ++ G5P_R74G_GRACE_FAULT, ++ G5P_R74G_GRACES, ++}; ++ ++/* The RAM-visible results of the firmware DC3 handler: phase 0 at the DC3 ++ * consume, phase 1 again just before the TA producer moves. ++ */ ++struct g5p_r74g_dc3 { ++ u64 pa[2], expected_pa, t_ns; ++ u32 slot[2], host[2], cached[2], rec[2][5]; ++ u32 start, target, consumer, producer, tag; ++ u32 failed, event_failed, companion, recoveries, classification, bad; ++}; ++ ++struct g5p_r73b { ++ struct device *dev; ++ struct g5p_boot *boot; ++ u32 case_id, watchdog_ms, dc_variant, meta6c8, r73e_case; ++ bool ack_first, aux, fast_enabled, fast_window; ++ bool early_release_ready, early_power_abort; ++ atomic_t early_once, early_busy; ++ u64 early_irq_ns, early_power_start_ns; ++ u64 early_off_ns, early_on_ns; ++ u32 early_state, early_after_state, early_power_raw; ++ s32 early_ret, early_power_ret; ++ /* wake112 (r74e112.h): the R74E proof deferred to the first-ALIVE owner. */ ++ atomic_t early_proof; ++ u32 early_proof_gen, early_proof_gen_now, early_proof_why; ++ s32 early_defer_ret; ++ /* boot_alive_settle_us: limit, first in-flight value seen, re-reads, time spent. */ ++ u32 settle_us, settle_first, settle_samples; ++ u64 settle_ns; ++ u64 fast_release_ns, fast_alive_ns, fast_submit_ns; ++ u32 fast_state; ++ s32 fast_ret; ++ /* nopatch: Apple's unmodified image; policy record +0/+4/+8/+c/+34 taken before the first bell. */ ++ bool nopatch; ++ u32 np_pre[5]; ++ atomic_t seq, acks, cap_seen, busy, drains, drain_errors, exported, ++ trace_saved; ++ atomic_t watchdog_state; /* 0 off, 1 armed, 2 disarmed, 3 fired, 4 terminal */ ++ int watchdog_cpu; ++ u32 owner; ++ struct mutex drain_lock; /* single auxiliary-ring owner */ ++ struct hrtimer watchdog; ++ struct g5p_r73b_row rows[G5P_R73B_ROWS]; ++ u32 trace_rows[G5P_R73B_TRACE_ROWS][G5P_R73B_TRACE_WORDS]; ++ u32 irq_cap, r74_case, r74_mask, r74_ticks, r74_word1_count; ++ u32 r74_word1[G5P_R74_WORD1]; ++ atomic_t r74_seq, r74_irq_refused; ++ struct g5p_r74_milestone r74_milestones[G5P_R74_MILESTONES]; ++ ++ /* r74f_alive and FIRING are written under event_lock only. */ ++ u32 r74f_case, r74f_alive, r74f_active, r74f_generation, r74f_stop; ++ ++ u32 r87b_attach; ++ s32 r74f_stop_value; ++ atomic_t r74f_bells; ++ struct g5p_r74f_bell r74f_bell[G5P_R74F_KINDS]; ++ ++ /* R74g: no host MMU pass, publish-only TA grace, one fallback bell. */ ++ u32 r74g_case, r74g_ack_skipped, r74g_grace_end, r74g_opened; ++ u32 r74g_grace_alive; ++ u64 r74g_grace_ns; ++ struct g5p_r74g_dc3 r74g_dc3[2]; ++ /* R74h: RAM-only trace tail and checkpoints; NULL when off. */ ++ u32 r74h_case; ++ ++ /* R80: firmware trace observer; NULL when off. */ ++ ++ /* Heartbeat116 (heartbeat116.h): the R74 window detector; IRQ thread only. */ ++ bool hb_on; ++ u32 hb_seed, hb_empty, hb_cross; ++ atomic_t hb_late; ++ struct g5p_hb_cfg hb_cfg; ++ struct g5p_hb hb; ++}; ++ ++/* The ctrl+0 word installed before release: the r80 stage's, else the r78 control's. */ ++static u32 g5p_boot_ctrl0(struct g5p_boot *b) ++{ ++ return b->r73b && false ? 0 : b->r78_ctrl0; ++} ++ ++/* boot-r80.inc, included by boot.inc after boot-r74h.inc. */ ++ ++/* render-r96.inc: the receipts at export (RAM only) */ ++ ++/* render-r94.inc: the RAM state of the record and the code BO at export, read only */ ++ ++/* boot-r80.inc: the two IRQ-owned plateau banks, read-only */ ++ ++/* render-r84b.inc: the late raw dump and the CPU visibility step, RAM only */ ++ ++static int g5p_boot_drain(struct g5p_diag *g); ++static u32 g5p_transport_event_limit(void); ++static int g5p_r73c_fast_control(struct g5p_diag *g); ++static int g5p_r73e_prepare(struct g5p_diag *g); ++static int g5p_r73e_irq_submit(struct g5p_diag *g); ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner); ++ ++static const char *g5p_r73b_name(u32 id) ++{ ++ return id == 1 ? "meta1-bounded-all-rings" : ++ id == 2 ? "meta0-bounded-all-rings-control" : ++ id == 3 ? "meta1-event-only-reference" : ++ "off"; ++} ++ ++static bool g5p_r73b_sampled(u32 n) ++{ ++ return n <= 16 || is_power_of_2(n); ++} ++ ++static u32 *g5p_r73b_ptr(struct g5p_boot *b, u32 off) ++{ ++ return (u32 *)(b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r73b_word(struct g5p_boot *b, u32 off) ++{ ++ return READ_ONCE(*g5p_r73b_ptr(b, off)); ++} ++ ++static u32 g5p_r73b_slot(u32 seq) ++{ ++ if (seq < G5P_R73B_HEAD) ++ return seq; ++ return G5P_R73B_HEAD + ++ (seq - G5P_R73B_HEAD) % (G5P_R73B_ROWS - G5P_R73B_HEAD); ++} ++ ++/* Lockless: a writer owns its slot by sequence; seq is published last. */ ++static void g5p_r73b_crumb(struct g5p_boot *b, u32 phase, u32 owner, int result) ++{ ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r73b_row *row; ++ u32 seq; ++ ++ if (!r) ++ return; ++ seq = atomic_inc_return(&r->seq) - 1; ++ row = &r->rows[g5p_r73b_slot(seq)]; ++ WRITE_ONCE(row->seq, 0); ++ /* Invalidate before rewriting so a reader never pairs old seq/new data. */ ++ smp_wmb(); ++ row->phase = phase; ++ row->t_ns = ktime_get_mono_fast_ns(); ++ row->irq = atomic_read(&b->irqs); ++ row->ack = atomic_read(&r->acks); ++ row->released = READ_ONCE(b->released); ++ row->owner = owner; ++ row->result = result; ++ row->cap_seen = atomic_read(&r->cap_seen); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r73b_row_slots); i++) { ++ u32 ctl = b->layout.offset[g5p_r73b_row_slots[i]]; ++ ++ row->cp[i][0] = g5p_r73b_word(b, ctl); ++ row->cp[i][1] = g5p_r73b_word(b, ctl + 0x10); ++ } ++ /* Publish the row contents before its sequence. */ ++ smp_wmb(); ++ WRITE_ONCE(row->seq, seq + 1); ++} ++ ++static const char *g5p_r74_name(u32 id) ++{ ++ return id == 1 ? "forced-ready-meta1-natural-ta" : ++ id == 2 ? "meta0-unpatched-natural-ta-control" : ++ "off"; ++} ++ ++static const char *g5p_r74_step_name(u32 step) ++{ ++ return step < G5P_R74_STEPS ? g5p_r74_steps[step] : "unknown"; ++} ++ ++static u32 g5p_r74_case(struct g5p_boot *b) ++{ ++ return b && b->r73b ? 1 : 0; ++} ++ ++/* R74b case 1: after cmd8 retirement, optional host MMIO must not race the ++ * firmware policy loop power-gating RGX. Functional setup/kick sites opt in ++ * explicitly; this predicate gates observations only. ++ */ ++static bool g5p_r74b_ram_only(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ ++ return g5p_r74_case(b) == 1 && b && READ_ONCE(b->released) && ++ READ_ONCE(b->r72_meta6c8); ++} ++ ++/* Process context only, never per IRQ: one console line per r74 step, so a ++ * whole-SoC freeze is localized by the last line netconsole delivered. ++ */ ++static void g5p_r74_mark(struct g5p_diag *g, u32 step, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_r74_milestone m; ++ u32 seq; ++ ++ if (!r || !1) ++ return; ++ m.t_ns = ktime_get_ns(); ++ m.step = step; ++ m.ret = ret; ++ m.irqs = atomic_read(&b->irqs); ++ m.events = READ_ONCE(b->events); ++ m.dc[0] = g5p_r73b_word(b, b->layout.offset[0x6c]); ++ m.dc[1] = g5p_r73b_word(b, b->layout.offset[0x6c] + 0x10); ++ m.ta[0] = g5p_r73b_word(b, b->layout.offset[0x60]); ++ m.ta[1] = g5p_r73b_word(b, b->layout.offset[0x60] + 0x10); ++ seq = atomic_inc_return(&r->r74_seq) - 1; ++ if (seq < G5P_R74_MILESTONES) ++ r->r74_milestones[seq] = m; ++ dev_emerg( ++ g->dev, ++ "R74 milestone seq=%u step=%s ret=%d irqs=%u events=%u dc=%u/%u ta=%u/%u " ++ "t_ns=%llu end=r74\n", ++ seq, g5p_r74_step_name(step), ret, m.irqs, m.events, m.dc[0], ++ m.dc[1], m.ta[0], m.ta[1], m.t_ns); ++} ++ ++/* R74 keeps repeated ALIVEs quiet whatever their word1: each new value is ++ * printed once (eight at most), so a moved loop cannot flood the console. ++ */ ++static bool g5p_r74_word1(struct g5p_boot *b, u32 word1) ++{ ++ struct g5p_r73b *r = b->r73b; ++ ++ ; ++ ; ++ for (u32 i = 0; i < r->r74_word1_count; i++) ++ if (r->r74_word1[i] == word1) ++ return false; ++ if (r->r74_word1_count == G5P_R74_WORD1) ++ return false; ++ r->r74_word1[r->r74_word1_count++] = word1; ++ return true; ++} ++ ++static bool g5p_r73b_copy_row(struct g5p_r73b *r, u32 seq, ++ struct g5p_r73b_row *out) ++{ ++ struct g5p_r73b_row *row = &r->rows[g5p_r73b_slot(seq)]; ++ ++ if (READ_ONCE(row->seq) != seq + 1) ++ return false; ++ /* Pairs with the writer's publication barrier. */ ++ smp_rmb(); ++ *out = *row; ++ /* Re-check that the slot was not recycled while copying. */ ++ smp_rmb(); ++ return READ_ONCE(row->seq) == seq + 1; ++} ++ ++static int g5p_r73b_format(char *buf, size_t len, ++ const struct g5p_r73b_row *row) ++{ ++ u32 phase = row->phase < G5P_R73B_PHASES ? row->phase : 0; ++ ++ return scnprintf( ++ buf, len, ++ "R73B crumb seq=%u t_ns=%llu phase=%s irq=%u ack=%u released=%u event=%u/%u " ++ "log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d cap=%u end=r73b\n", ++ row->seq - 1, row->t_ns, g5p_r73b_phase_names[phase], row->irq, ++ row->ack, row->released, row->cp[0][0], row->cp[0][1], ++ row->cp[1][0], row->cp[1][1], row->cp[2][0], row->cp[2][1], ++ row->cp[3][0], row->cp[3][1], row->cp[4][0], row->cp[4][1], ++ row->owner, row->result, row->cap_seen); ++} ++ ++/* Snapshot the producer once and consume at most that modular distance ++ * (<= 255). One consumer store returns the whole batch; a producer that ++ * advances meanwhile is left for the next pass, never chased. ++ */ ++static int g5p_r73b_drain_ring(struct g5p_boot *b, u32 ctl_slot, u32 lane, ++ u32 data_slot, u32 stride, u32 ring, ++ u32 *consumed) ++{ ++ u32 ctl = b->layout.offset[ctl_slot] + lane * 0x20; ++ u32 data = b->layout.offset[data_slot] + ++ (ctl_slot == 0x71 ? lane * 0x2c00 : 0); ++ u32 consumer = g5p_r73b_word(b, ctl); ++ u32 producer = g5p_r73b_word(b, ctl + 0x10); ++ u32 count, hash = b->r73_aux_hash[ring] ?: 2166136261U; ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ count = (producer - consumer) & 255; ++ if (!count) ++ return 0; ++ /* Producer index before the slot payloads it covers. */ ++ dma_rmb(); ++ for (u32 n = 0; n < count; n++) { ++ u32 words[G5P_R73B_TRACE_WORDS] = { 0 }; ++ u32 off = data + consumer * stride; ++ ++ for (u32 i = 0; i < stride / 4; i++) { ++ u32 value = g5p_r73b_word(b, off + i * 4); ++ ++ if (i < ARRAY_SIZE(words)) ++ words[i] = value; ++ hash = (hash ^ value) * 16777619U; ++ } ++ if (ring == 5 && ++ atomic_read(&b->r73b->trace_saved) < G5P_R73B_TRACE_ROWS) { ++ u32 saved = atomic_read(&b->r73b->trace_saved); ++ ++ memcpy(b->r73b->trace_rows[saved], words, ++ sizeof(words)); ++ /* Publish this complete row before increasing the visible count. */ ++ smp_wmb(); ++ atomic_inc(&b->r73b->trace_saved); ++ }; ++ ; ++ if (!b->r73_aux_entries[ring]) ++ memcpy(b->r73_aux_first[ring], words, ++ sizeof(b->r73_aux_first[ring])); ++ memcpy(b->r73_aux_last[ring], words, ++ sizeof(b->r73_aux_last[ring])); ++ b->r73_aux_entries[ring]++; ++ consumer = (consumer + 1) & 255; ++ } ++ b->r73_aux_hash[ring] = hash; ++ /* dma_wmb() ordering plus the slot loads above: Apple returns the ++ * consumer only after its reads, with a full barrier. ++ */ ++ dma_mb(); ++ WRITE_ONCE(*g5p_r73b_ptr(b, ctl), consumer); ++ *consumed += count; ++ return 0; ++} ++ ++/* Apple order after the ACK: four log lanes, events, statistics, trace. */ ++static int g5p_r73b_aux_pass(struct g5p_diag *g, u32 *consumed) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret = 0; ++ ++ for (u32 lane = 0; lane < 4 && !ret; lane++) ++ ret = g5p_r73b_drain_ring(b, 0x71, lane, 0x75, 44, lane, ++ consumed); ++ if (!ret) ++ ret = g5p_boot_drain(g); ++ /* R80: ring 4 has one consumer, the drain thread (retains every record). */ ++ if (!ret && !false) ++ ret = g5p_r73b_drain_ring(b, 0x72, 0, 0x76, 44, 4, consumed); ++ if (!ret) ++ ret = g5p_r73b_drain_ring(b, 0x73, 0, 0x77, 36, 5, consumed); ++ return ret; ++} ++ ++static bool g5p_r73b_pending(struct g5p_boot *b, bool aux) ++{ ++ static const u8 slots[] = { 0x70, 0x71, 0x72, 0x73 }; ++ ++ for (u32 i = 0; i < (aux ? ARRAY_SIZE(slots) : 1); i++) { ++ for (u32 lane = 0; lane < (slots[i] == 0x71 ? 4 : 1); lane++) { ++ u32 ctl = b->layout.offset[slots[i]] + lane * 0x20; ++ ++ if (g5p_r73b_word(b, ctl) != ++ g5p_r73b_word(b, ctl + 0x10)) ++ return true; ++ } ++ } ++ return false; ++} ++ ++/* The IRQ thread only trylocks the auxiliary rings; a miss still runs the ++ * spinlock-protected event decoder. Process-context paths lock normally. ++ * Drains with work, errors or sampled entries record before/after rows. ++ */ ++static int g5p_r73b_service(struct g5p_diag *g, u32 owner, bool sampled) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 consumed = 0, before; ++ bool rows, aux; ++ int ret; ++ ++ if (!r) ++ return g5p_boot_drain(g); ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* The atomic r73c window services only the event ring. Auxiliary ++ * traffic is retained until cmd8 is either published or refused. */ ++ aux = 1 && !READ_ONCE(r->fast_window); ++ if (aux && owner == G5P_R73B_OWNER_IRQ && ++ !mutex_trylock(&r->drain_lock)) { ++ atomic_inc(&r->busy); ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BUSY, READ_ONCE(r->owner), ++ -EBUSY); ++ ret = g5p_boot_drain(g); ++ goto out; ++ } ++ if (aux && owner != G5P_R73B_OWNER_IRQ) ++ mutex_lock(&r->drain_lock); ++ if (aux) ++ WRITE_ONCE(r->owner, owner); ++ atomic_inc(&r->drains); ++ rows = sampled || g5p_r73b_pending(b, aux); ++ if (rows) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BEFORE, owner, 0); ++ before = READ_ONCE(b->events); ++ ret = aux ? g5p_r73b_aux_pass(g, &consumed) : g5p_boot_drain(g); ++ consumed += READ_ONCE(b->events) - before; ++ if (rows || ret) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_AFTER, owner, ret ?: consumed); ++ if (aux) { ++ WRITE_ONCE(r->owner, G5P_R73B_NONE); ++ mutex_unlock(&r->drain_lock); ++ } ++out: ++ if (ret) { ++ atomic_inc(&r->drain_errors); ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return ret; ++} ++ ++/* Heartbeat116: an R74 epoch bounds IRQ174 and its ring entries per window. */ ++static bool g5p_hb116_on(struct g5p_boot *b) ++{ ++ return b->r73b && b->r73b->hb_on; ++} ++ ++/* Auxiliary-ring entries consumed so far (RAM, monotonic, may wrap). */ ++static u32 g5p_hb116_aux(struct g5p_boot *b) ++{ ++ u32 sum = 0; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(b->r73_aux_entries); i++) ++ sum += READ_ONCE(b->r73_aux_entries[i]); ++ return sum; ++} ++ ++static u32 g5p_hb116_progress(struct g5p_boot *b) ++{ ++ return READ_ONCE(b->events) + g5p_hb116_aux(b); ++} ++ ++/* The lifetime cap's terminal stop, once: final row, cap_seen, failed, one disable. */ ++static void g5p_hb116_trip(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb *h = &r->hb; ++ u32 prior = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (atomic_cmpxchg(&r->cap_seen, 0, 1)) ++ return; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "HB116 trip why=%s prior_failed=%u windows=%u stalled=%u ms=%llu irqs=%u " ++ "events=%u acks=%u empty=%u aux=%u total_irqs=%u total_events=%u t_ns=%llu " ++ "stop=masked late=ack-only end=hb116\n", ++ g5p_hb_why_name[h->why < G5P_HBW_N ? h->why : 0], prior, ++ h->windows, h->stalled, h->trip.ns / NSEC_PER_MSEC, ++ h->trip.irqs, h->trip.events, h->trip.acks, h->trip.empty, ++ h->trip.aux, n, READ_ONCE(b->events), h->trip_ns); ++} ++ ++/* Receipts only: the first window past each Wake112 lifetime bound; nothing trips here. */ ++static void g5p_hb116_cross(struct g5p_diag *g, u32 irqs, u32 events) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ u32 limit = g5p_transport_event_limit(); ++ ++ if (!(r->hb_cross & 1) && irqs > r->irq_cap) { ++ r->hb_cross |= 1; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=irq-cap value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ r->irq_cap, irqs, events, r->hb.windows); ++ } ++ if (!(r->hb_cross & 2) && events >= limit) { ++ r->hb_cross |= 2; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=event-limit value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ limit, irqs, events, r->hb.windows); ++ } ++} ++ ++/* After the entry's ACK and service: account it, close a window, or trip. */ ++static void g5p_hb116_irq(struct g5p_diag *g, int irq, u32 n, u32 before) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb_win *w = &r->hb.last; ++ u32 seed = r->hb_seed, events = READ_ONCE(b->events); ++ struct g5p_hb_in in; ++ ++ if (g5p_hb116_progress(b) == before) ++ r->hb_empty++; ++ in.now_ns = ktime_get_mono_fast_ns(); ++ in.irqs = n + seed; ++ in.events = events + seed; ++ in.acks = atomic_read(&r->acks) + seed; ++ in.empty = r->hb_empty + seed; ++ in.aux = g5p_hb116_aux(b) + seed; ++ switch (g5p_hb_step(&r->hb, &r->hb_cfg, &in)) { ++ case G5P_HBR_OPEN: ++ case G5P_HBR_CLOSED: ++ break; ++ case G5P_HBR_TRIP: ++ g5p_hb116_trip(g, irq, n); ++ return; ++ } ++ g5p_hb116_cross(g, n, events); ++ if (g5p_hb_report(&r->hb)) ++ dev_info( ++ g->dev, ++ "HB116 window n=%u t_ns=%llu ms=%llu irqs=%u events=%u acks=%u empty=%u aux=%u " ++ "stalled=%u peak_irqs=%u peak_events=%u total_irqs=%u total_events=%u " ++ "total_acks=%d end=hb116\n", ++ r->hb.windows, in.now_ns, w->ns / NSEC_PER_MSEC, ++ w->irqs, w->events, w->acks, w->empty, w->aux, ++ r->hb.stalled, r->hb.peak_irqs, r->hb.peak_events, n, ++ events, atomic_read(&r->acks)); ++} ++ ++/* Every entry ACKs. Over the cap: ACK, final row, cap_seen, one disable; ++ * the kernel stays alive so the runner can export the record. ++ * Heartbeat116: an R74 epoch has no lifetime cap. The window detector runs ++ * after the ACK and the service; after its trip an entry only ACKs. ++ */ ++static irqreturn_t g5p_r73b_irq(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ bool sampled = g5p_r73b_sampled(n); ++ u32 before = 0; ++ ++ if (r->hb_on && atomic_read(&r->cap_seen)) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ atomic_inc(&r->hb_late); ++ return IRQ_HANDLED; ++ } ++ if (!r->hb_on && n > r->irq_cap) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ if (!atomic_read(&r->cap_seen)) { ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (!atomic_cmpxchg(&r->cap_seen, 0, 1)) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ } ++ } ++ return IRQ_HANDLED; ++ } ++ { ++ /* KPF level6G: Apple ACKs the AIC meta interrupt before ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ } ++ if (sampled) ++ g5p_r73b_crumb(b, G5P_R73B_IRQ, G5P_R73B_OWNER_IRQ, n); ++ if (r->hb_on) ++ before = g5p_hb116_progress(b); ++ if (READ_ONCE(b->released)) ++ g5p_r73b_service(g, G5P_R73B_OWNER_IRQ, sampled); ++ if (1 && READ_ONCE(b->alive)) ++ g5p_r73e_alive_submit(g, G5P_R73B_OWNER_IRQ); ++ ; ++ ; ++ ; ++ ; ++ ; ++ if (r->hb_on) ++ g5p_hb116_irq(g, irq, n, before); ++ return IRQ_HANDLED; ++} ++ ++static void g5p_r73b_disarm(struct g5p_r73b *r) ++{ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 2) == 1) ++ hrtimer_cancel(&r->watchdog); ++} ++ ++static void g5p_r73b_stop(void *data) ++{ ++ g5p_r73b_disarm(data); ++} ++ ++/* Cached CPU RAM only: no MMIO, no locks, then a deliberate panic so a ++ * silent hang becomes a pstore/ramoops record. ++ */ ++static enum hrtimer_restart g5p_r73b_watchdog_fire(struct hrtimer *timer) ++{ ++ struct g5p_r73b *r = container_of(timer, struct g5p_r73b, watchdog); ++ struct g5p_boot *b = r->boot; ++ struct g5p_r73b_row row; ++ u32 total = atomic_read(&r->seq), first, last; ++ char line[320]; ++ bool found = false; ++ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 3) != 1) ++ return HRTIMER_NORESTART; ++ /* A contained cap or a latched failure is terminal: the IRQ only ACKs, ++ * service returns -EIO and the runner sees failed=1. Keep that host ++ * alive for the export instead of turning it into a deliberate panic. ++ */ ++ if (atomic_read(&r->cap_seen) || READ_ONCE(b->failed)) { ++ atomic_set(&r->watchdog_state, 4); ++ dev_emerg( ++ r->dev, ++ "R73B watchdog terminal case=%u irqs=%d acks=%d cap_seen=%d failed=%u rows=%u " ++ "no-panic end=r73b\n", ++ r->case_id, atomic_read(&b->irqs), ++ atomic_read(&r->acks), atomic_read(&r->cap_seen), ++ READ_ONCE(b->failed), total); ++ return HRTIMER_NORESTART; ++ } ++ dev_emerg( ++ r->dev, ++ "R73B receipt case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u release=%s end=r73b\n", ++ r->case_id, g5p_r73b_name(r->case_id), r->ack_first, r->aux, ++ r->irq_cap, r->watchdog_ms, r->dc_variant, r->meta6c8, ++ init_utsname()->release); ++ first = total > G5P_R73B_PANIC_ROWS ? total - G5P_R73B_PANIC_ROWS : 0; ++ for (u32 seq = first; seq < total; seq++) { ++ if (!g5p_r73b_copy_row(r, seq, &row)) ++ continue; ++ g5p_r73b_format(line, sizeof(line), &row); ++ dev_emerg(r->dev, "%s", line); ++ } ++ for (last = total; last > first && !found; last--) ++ found = g5p_r73b_copy_row(r, last - 1, &row); ++ if (!found) ++ memset(&row, 0, sizeof(row)); ++ panic("G5P R73B watchdog: release epoch incomplete after %ums case=%u irqs=%d acks=%d " ++ "cap_seen=%d busy=%d rows=%u last seq=%u phase=%s irq=%u ack=%u released=%u " ++ "event=%u/%u log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d " ++ "cap=%u end=r73b\n", ++ 0, 1, atomic_read(&b->irqs), atomic_read(&r->acks), ++ atomic_read(&r->cap_seen), atomic_read(&r->busy), total, ++ row.seq ? row.seq - 1 : 0, ++ g5p_r73b_phase_names[row.phase < G5P_R73B_PHASES ? row.phase : 0], ++ row.irq, row.ack, row.released, row.cp[0][0], row.cp[0][1], ++ row.cp[1][0], row.cp[1][1], row.cp[2][0], row.cp[2][1], ++ row.cp[3][0], row.cp[3][1], row.cp[4][0], row.cp[4][1], row.owner, ++ row.result, row.cap_seen); ++ return HRTIMER_NORESTART; ++} ++ ++static void g5p_r73b_release_entry(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ ++ if (!r) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_ENTRY, G5P_R73B_NONE, 0); ++ { ++ return; ++ } ++} ++ ++static void g5p_r73b_release_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_EXIT, G5P_R73B_NONE, ret); ++ if (ret) ++ g5p_r73b_disarm(b->r73b); ++} ++ ++/* The release epoch ends when the host has returned from the ALIVE wait. */ ++static void g5p_r73b_events_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_EVENTS_EXIT, G5P_R73B_OWNER_EVENTS, ret); ++ g5p_r73b_disarm(b->r73b); ++} ++ ++static int g5p_r73b_latch(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const u32 id = (1), ms = (0); ++ const u32 fast = (1); ++ const u32 early = (1); ++ const u32 r74 = (1), cap = (1048576); ++ const u32 r74f = (1); ++ const u32 r74g = (1); ++ const u32 r74h = (0); ++ ++ struct g5p_r73b *r; ++ int ret; ++ ++ ; ++ ; ++ ; ++ ; ++ ; ++ ; ++ /* nopatch replaces r73f case 1 under the whole r74/r74f/r74g path, and nothing else. */ ++ /* nopatch's receipts and its r74 gate are the TYPED_V3 path's; any other path (r74 0) runs ++ * Apple's image anyway (only r73f patches it) and gets neither. ++ */ ++ if (1 != g5p_fw_unpatched_want((1), 0, 0, 0) || ++ (1 && r74 == 1 && (0 || r74f != 1 || r74g != 1))) { ++ dev_err(g->dev, ++ "NOPATCH reject phase=map unpatched=%u/%u r73f_case=%u path=%u/%u/%u end=nopatch\n", ++ 1, (1), 0, r74, r74f, r74g); ++ return -EINVAL; ++ } ++ /* R74 rides only on an exact r73f case 1 (or nopatch) or 4 with the raised limits. */ ++ if (r74 && (!id || r74 > 2 || 0 != (r74 == 1 ? (1 ? 0 : 1) : 4) || ++ 1 != (r74 == 1) || !(1) || cap < G5P_R74_IRQ_CAP || ++ cap > G5P_R74_IRQ_CAP_MAX || ++ g5p_transport_event_limit() < G5P_R74_EVENT_FLOOR)) { ++ dev_err(g->dev, ++ "R74 reject phase=map case=%u r73b_case=%u r73f_case=%u meta6c8=%u " ++ "render_engine=%u irq_cap=%u event_limit=%u end=r74\n", ++ r74, id, 0, b->r72_meta6c8, (1), cap, ++ g5p_transport_event_limit()); ++ return -EINVAL; ++ } ++ ++ /* R74f rides only on the exact r74 case 1 forced-ready epoch. */ ++ ; ++ ++ /* R74g rides only on the exact r74f case 1 epoch. */ ++ ; ++ ++ /* R74h observes only the exact r74g case 1 epoch. */ ++ ; ++ ++ /* The bounded serialized drain supersedes r73's IRQ-context drain. */ ++ ; ++ ; ++ ; ++ ; ++ if (id > 3 || fast > 1 || (fast && id == 3) || early > 2 || ++ (early && (!fast || id != early)) || b->r73_aux_drain || 7 != 7 || ++ 1 != (id == 2 ? 0 : 1) || (ms && (ms < 1000 || ms > 30000))) ++ return -EINVAL; ++ r = devm_kzalloc(g->dev, sizeof(*r), GFP_KERNEL); ++ if (!r) ++ return -ENOMEM; ++ r->dev = g->dev; ++ r->boot = b; ++ r->case_id = id; ++ r->watchdog_ms = ms; ++ r->dc_variant = 7; ++ r->meta6c8 = 1; ++ r->aux = id != 3; ++ r->ack_first = true; ++ r->fast_enabled = fast; ++ r->r73e_case = early; ++ r->r74_case = r74; ++ r->r74f_case = r74f; ++ r->r74g_case = r74g; ++ r->r74h_case = r74h; ++ r->settle_us = min_t(u32, READ_ONCE(boot_alive_settle_us), 5000); ++ r->nopatch = 1 && r74 == 1; ++ r->irq_cap = r74 ? cap : G5P_R73B_IRQ_CAP; ++ /* Heartbeat116: in an R74 epoch the window detector replaces both lifetime bounds. */ ++ r->hb_on = r74 != 0; ++ r->hb_seed = r74 ? (0) : 0; ++ g5p_hb_cfg_default(&r->hb_cfg); ++ atomic_set(&r->early_once, 0); ++ atomic_set(&r->early_busy, 0); ++ atomic_set(&r->early_proof, G5P_PX_NONE); ++ r->watchdog_cpu = -1; ++ mutex_init(&r->drain_lock); ++ hrtimer_setup(&r->watchdog, g5p_r73b_watchdog_fire, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL_PINNED_HARD); ++ ret = devm_add_action_or_reset(g->dev, g5p_r73b_stop, r); ++ if (ret) ++ return ret; ++ b->r73_aux_drain = 1; ++ b->r73b = r; ++ dev_info( ++ g->dev, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), r->ack_first, r->aux, r->irq_cap, ms, ++ r->dc_variant, r->meta6c8, G5P_R73B_ROWS, ++ init_utsname()->release); ++ g5p_blob_printf( ++ g, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), 1, 1, r->irq_cap, ms, 7, 1, ++ G5P_R73B_ROWS, init_utsname()->release); ++ g5p_blob_printf(g, "R73C latched fast=%u case=%u meta6c8=%u end=r73c\n", ++ 1, id, 1); ++ dev_info(g->dev, "R73E latched case=%u meta6c8=%u fast=%u end=r73e\n", ++ early, r->meta6c8, r->fast_enabled); ++ ; ++ ; ++ if (r->nopatch) ++ dev_info( ++ g->dev, ++ "NOPATCH latched fw=apple-original r73f_case=0 meta6c8=%u r73e_case=%u r74=%u " ++ "r74f=%u r74g=%u force_policy_ready=0 end=nopatch\n", ++ r->meta6c8, early, r74, r74f, r74g); ++ { ++ dev_info( ++ g->dev, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, r->meta6c8, ++ r->irq_cap, g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, 1, r->irq_cap, ++ g5p_transport_event_limit()); ++ dev_info( ++ g->dev, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ g5p_blob_printf( ++ g, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ g5p_blob_printf( ++ g, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ }; ++ /* TYPED_V3 production path: the self-drain DC3 attach policy of r87b rides on r74f + r74g ++ * alone, with none of the r74h / r80 instrumentation. r74g has the firmware own the first ++ * MMU pass, which is what makes self-drain safe (r87b-attach.h); r74h only observes. Every ++ * other r8x / r9x knob is still refused below without boot_r80_case. ++ */ ++ { ++ r->r87b_attach = G5P_R87B_ATTACH_SELF_DRAIN; ++ dev_info( ++ g->dev, ++ "R87B latched policy=1 name=self-drain r74f=%u r74g=%u r74h=%u r80=0 " ++ "production_path=1 end=r87b\n", ++ r74f, r74g, r74h); ++ } ++ { ++ { ++ { ++ { ++ { ++ if ((1) && !r->r87b_attach) { ++ dev_err(g->dev, ++ "R87B reject phase=map r87b knob without boot_r80_case policy=%u end=r87b\n", ++ (1)); ++ return -EBADMSG; ++ } else { ++ { ++ { ++ ; ++ ; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ } ++ return 0; ++} ++ ++/* Process context, cached CPU RAM only; never waits for the command lock. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +new file mode 100644 +index 0000000000..71cf79dbed +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -0,0 +1,532 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74f deferred doorbell. Included by boot.inc before the event decoder. ++ * Diagnostic bypass on the r74 forced-ready epoch only; not power policy. ++ * ++ * A DC3 pair or a natural TA is published once, append-only, before this ++ * code runs; its producer is never rewound, republished or torn down. If ++ * the immediate functional gate finds GPC inactive, that generation stays ++ * pending (never b->failed), the host repeats the r74e release transition ++ * once, and the first type-0 ALIVE decoded afterwards owns one doorbell ++ * attempt. The owner runs under event_lock: producer confirm, R1+14 fence, ++ * active check, dma_wmb(), doorbell 0x10. Receipts stay in RAM; console ++ * lines are printed afterwards from process context. ++ */ ++ ++static const char *const g5p_r74f_kinds[G5P_R74F_KINDS] = { "dc3", "ta", "3d" }; ++ ++static const char *const g5p_r74f_states[G5P_R74F_STATES] = { ++ "idle", "published", "open", "firing", "rang", "cancelled", ++}; ++ ++static const char *const g5p_r74f_cancels[G5P_R74F_CANCELS] = { ++ "none", "reactivation-failed", ++ "no-alive", "consumed-before-bell", ++ "protocol", "alive-during-reactivation", ++ "self-drain", ++}; ++ ++static const char *const g5p_r74f_stops[G5P_R74F_STOPS] = { ++ "none", ++ "attach-reactivation-failed", ++ "attach-published-not-kicked", ++ "attach-kicked-not-consumed", ++ "first-mmu-window-closed", ++ "ta-reactivation-failed", ++ "ta-published-not-kicked", ++ "firmware-dc3-tag-mismatch", ++ "dc3-ram-validation-refused", ++ "firmware-companion-recovery", ++ "ta-published-not-consumed", ++ "ta-window-missed", ++}; ++ ++static const char *const g5p_r74f_paths[G5P_R74F_PATHS] = { ++ "none", ++ "immediate", ++ "deferred", ++}; ++ ++static bool g5p_r74f_on(struct g5p_boot *b) ++{ ++ return b && b->r73b && 1; ++} ++ ++static u32 *g5p_r74f_ctl(struct g5p_boot *b, u32 kind) ++{ ++ return (u32 *)g5p_boot_shared(b, kind == G5P_R74F_TA ? 0x60 : ++ kind == G5P_R74F_3D ? 0x64 : ++ 0x6c); ++} ++ ++/* Owner only, event_lock held with interrupts off. Between the active check ++ * and the doorbell there is only dma_wmb(): no console, no other MMIO. ++ */ ++static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ u32 *fence, u32 *state) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 consumer = READ_ONCE(ctl[0]); ++ u32 producer = smp_load_acquire(&ctl[4]); ++ u32 queued = (f->target - f->start) & 255; ++ u32 progressed = (consumer - f->start) & 255; ++ u32 expected = f->kind == G5P_R74F_DC3 ? 2 : 1; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* Do not turn corrupted modulo-256 ring state into a doorbell. */ ++ if (consumer > 255 || producer > 255 || queued != expected || ++ progressed > queued) ++ return -EPROTO; ++ /* Acquire: pairs with the producer store published before the state. */ ++ if (producer != f->target) ++ return -EPROTO; ++ if (consumer == f->target) ++ return -EALREADY; ++ writel(7, g->window[1] + 0x14); ++ *fence = readl(g->window[1] + 0x14); ++ if (*fence & ~7U) ++ return -EPROTO; ++ *state = readl(g->window[1] + 0x1000); ++ /* perf60 (gpc_any_pstate): a GPC request in flight or a not-yet-awake GPC after the wake ++ * above; re-read for up to 2 ms, as Apple's setGPCPerformanceState waits for bit 4. Reads only. ++ */ ++ { ++ u64 t0 = ktime_get_mono_fast_ns(); ++ ++ while (!g5p_gpc_ok(*state) && ktime_get_mono_fast_ns() - t0 < ++ 2000ULL * NSEC_PER_USEC) { ++ udelay(10); ++ *state = readl(g->window[1] + 0x1000); ++ } ++ } ++ if (!g5p_gpc_ok(*state)) ++ return -EHOSTDOWN; ++ dma_wmb(); ++ writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); ++ return 0; ++} ++ ++/* Owner only: record the attempt, then publish it with the state release. */ ++static void g5p_r74f_rang(struct g5p_r73b *r, struct g5p_r74f_bell *f, int ret, ++ u32 fence, u32 state, u32 path) ++{ ++ f->bell_ns = ktime_get_mono_fast_ns(); ++ f->path = path; ++ f->fence = fence; ++ f->bell_state = state; ++ f->bell_ret = ret; ++ if (!ret) { ++ f->kicked = 1; ++ atomic_inc(&r->r74f_bells); ++ } else if (ret == -EALREADY) { ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ } else if (ret != -EHOSTDOWN) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ } ++ atomic_set_release(&f->state, G5P_R74F_RANG); ++} ++ ++/* g5p_boot_drain(), event_lock held: one type-0 ALIVE was just decoded. */ ++static void g5p_r74f_alive(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ struct g5p_r74f_bell *f; ++ u32 kind, fence = U32_MAX, state = U32_MAX; ++ int ret; ++ ++ WRITE_ONCE(r->r74f_alive, r->r74f_alive + 1); ++ kind = READ_ONCE(r->r74f_active); ++ if (!kind) ++ return; ++ f = &r->r74f_bell[kind - 1]; ++ if (atomic_read_acquire(&f->state) != G5P_R74F_OPEN || ++ r->r74f_alive <= f->open_alive) ++ return; ++ /* Sole owner: exactly one decoder moves OPEN to FIRING. */ ++ if (atomic_cmpxchg(&f->state, G5P_R74F_OPEN, G5P_R74F_FIRING) != ++ G5P_R74F_OPEN) ++ return; ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_DEFERRED); ++} ++ ++static int g5p_r74f_poll(struct g5p_diag *g, u32 alive, u32 expected, ++ u32 timeout_us, u64 *elapsed_ns, u32 *raw) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u64 start = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ *raw = U32_MAX; ++ for (;;) { ++ if (READ_ONCE(g->boot->failed)) { ++ ret = -EIO; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ /* Serialize the ordinal check and exactly one R1 read with the ++ * decoder. Once ALIVE advances, no further R1 MMIO is licensed. ++ */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ break; ++ } ++ *raw = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if ((*raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ return ret; ++} ++ ++/* Absolute freshness budget from the DC3 doorbell. The first MMU pass may ++ * touch R0 only while the same active window is still demonstrably fresh. ++ */ ++ ++/* The r74e release transition, once per generation, on r74f receipts and ++ * without console output: core request clear, GPC 0 within 1.0 ms, GPC 1 ++ * within 1.8 ms. Every R1 access is serialized against ALIVE decoding; ++ * ordinal advance cancels the generation before any further R1 MMIO. ++ */ ++static int g5p_r74f_reactivate(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u32 alive = f->arm_alive; ++ int ret; ++ ++ f->reactivated = 1; ++ f->power_start_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ f->power_ret = -EHOSTDOWN; ++ return -EHOSTDOWN; ++ } ++ writel(0, g->window[1] + 0x20000); ++ f->core_fence = readl(g->window[1] + 0x20000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, 0, G5P_R74F_OFF_US, &f->off_ns, ++ &f->off_raw); ++ if (!ret) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ goto out; ++ } ++ /* boot_gpu_perf_seed: the firmware believes pstate q; never drop the GPC under it */ ++ writel(0x10 | (b->perf_poweron ?: ++ b->perf_seed ?: ++ 1), ++ g->window[1] + 0x1000); ++ f->gpc_fence = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, ++ b->perf_poweron ?: ++ b->perf_seed ?: ++ 1, ++ G5P_R74F_ON_US, &f->on_ns, &f->on_raw); ++ } ++out: ++ f->power_ret = ret; ++ return ret; ++} ++ ++/* Process context, before any publication: no overlap with a pending ring. */ ++static int g5p_r74f_admit(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *dc = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ ++ if (!g5p_r74b_ram_only(g) || READ_ONCE(b->failed) || ++ READ_ONCE(r->r74f_stop) || ++ atomic_read(&r->r74f_bell[kind].state) != G5P_R74F_IDLE) ++ return -EPERM; ++ if (kind == G5P_R74F_DC3) ++ return READ_ONCE(r->r74f_active) ? -EBUSY : 0; ++ /* M7: the TA wakeup never shares a doorbell with a pending DC3 pair. */ ++ dma_rmb(); ++ if (READ_ONCE(r->r74f_active) != G5P_R74F_DC3 + 1 || ++ !r->r74f_bell[G5P_R74F_DC3].complete || ++ READ_ONCE(dc[0]) != READ_ONCE(dc[4])) ++ return -EBUSY; ++ return 0; ++} ++ ++static void g5p_r74f_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 state = atomic_read_acquire(&f->state); ++ char line[3][512]; ++ ++ dma_rmb(); ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74F bell kind=%s generation=%u ring=%02x start=%u target=%u state=%s path=%s " ++ "first_ret=%d first_fence=%08x first_state=%08x attempted=%u fence=%08x " ++ "bell_state=%08x bell_ret=%d kicked=%u bells=%d cancel=%s complete=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->ring, f->start, ++ f->target, g5p_r74f_states[state < G5P_R74F_STATES ? state : 0], ++ g5p_r74f_paths[f->path], f->first_ret, f->first_fence, ++ f->first_state, f->attempted, f->fence, f->bell_state, ++ f->bell_ret, f->kicked, atomic_read(&b->r73b->r74f_bells), ++ g5p_r74f_cancels[f->cancel], f->complete); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74F ring kind=%s generation=%u arm_alive=%u open_alive=%u bell_alive=%u " ++ "alive_now=%u consumer_before=%u consumer_after=%u producer_after=%u " ++ "consumer_now=%u producer_now=%u failed=%u event_failed=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->arm_alive, ++ f->open_alive, f->bell_alive, READ_ONCE(b->r73b->r74f_alive), ++ f->consumer_before, f->consumer_after, f->producer_after, ++ READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), READ_ONCE(b->failed), ++ READ_ONCE(b->event_failed)); ++ scnprintf( ++ line[2], sizeof(line[2]), ++ "R74F power kind=%s generation=%u reactivated=%u core_fence=%08x gpc_fence=%08x " ++ "off_ns=%llu on_ns=%llu off_raw=%08x on_raw=%08x power_ret=%d arm_ns=%llu " ++ "power_start_ns=%llu open_ns=%llu bell_start_ns=%llu bell_ns=%llu done_ns=%llu " ++ "end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->reactivated, ++ f->core_fence, f->gpc_fence, f->off_ns, f->on_ns, f->off_raw, ++ f->on_raw, f->power_ret, f->arm_ns, f->power_start_ns, ++ f->open_ns, f->bell_start_ns, f->bell_ns, f->done_ns); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++/* r87b receipt: the DC3 attach policy and what the host did after the publish. ++ * RAM reads only; one line per DC3 generation (console at the consume/timeout ++ * point, blob at export). ++ */ ++static void g5p_r87b_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 policy = READ_ONCE(b->r73b->r87b_attach); ++ char line[640]; ++ ++ dma_rmb(); ++ scnprintf( ++ line, sizeof(line), ++ "R87B attach policy=%u name=%s generation=%u gate_first_ret=%d " ++ "gate_first_state=%08x gate_closed=%u path=%s cancel=%s bell_rung=%u bells=%d " ++ "reactivated=%u cons_before=%u prod_before=%u prod_pub=%u cons_after=%u " ++ "prod_after=%u consumed=%u arm_alive=%u alive_done=%u alive_delta=%u " ++ "pub_to_done_ns=%llu irqs=%d events=%u end=r87b\n", ++ policy, policy ? "self-drain" : "original", f->generation, ++ f->first_ret, f->first_state, f->first_ret == -EHOSTDOWN, ++ g5p_r74f_paths[f->path], g5p_r74f_cancels[f->cancel], f->kicked, ++ atomic_read(&b->r73b->r74f_bells), f->reactivated, ++ f->consumer_before, f->start, f->target, f->consumer_after, ++ f->producer_after, f->complete, f->arm_alive, f->alive_done, ++ f->alive_done - f->arm_alive, ++ f->done_ns >= f->arm_ns ? f->done_ns - f->arm_ns : 0, ++ atomic_read(&b->irqs), READ_ONCE(b->events)); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* A retained boundary: the publication stays, nothing is failed. */ ++static int g5p_r74f_stop(struct g5p_diag *g, u32 stop, int value) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ ++ r->r74f_stop_value = value; ++ WRITE_ONCE(r->r74f_stop, stop); ++ g5p_r74_mark(g, G5P_R74F_STOP, stop); ++ dev_emerg(g->dev, "R74F stop reason=%s value=%d failed=%u end=r74f\n", ++ g5p_r74f_stops[stop], value, READ_ONCE(g->boot->failed)); ++ return -EINPROGRESS; ++} ++ ++/* After the reactivation: open the generation, wait (bounded) for the ++ * ALIVE owner while the event drain stays alive, then cancel if unrung. ++ */ ++static int g5p_r74f_open(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags, deadline; ++ int ret; ++ ++ ret = g5p_r74f_reactivate(g, f); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (ret) { ++ f->cancel = ret == -EHOSTDOWN ? G5P_R74F_CANCEL_ALIVE_ADVANCED : ++ G5P_R74F_CANCEL_REACTIVATION; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ f->open_alive = r->r74f_alive; ++ f->open_ns = ktime_get_mono_fast_ns(); ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (ret) ++ return 0; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ /* The IRQ path is the sole event consumer and bell owner. */ ++ if (READ_ONCE(b->failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* Process context, command mutex held, producer already published. Returns ++ * 0 after one doorbell, 1 if consumed without one, -EINPROGRESS when the ++ * publication is retained unrung, else a fatal protocol error. ++ */ ++static int g5p_r74f_bell(struct g5p_diag *g, u32 kind, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(b, kind); ++ u32 fence = U32_MAX, state = U32_MAX, now; ++ unsigned long flags; ++ int ret, drained = 0; ++ bool ta = kind == G5P_R74F_TA; ++ bool self_drain; ++ ++ if (kind == G5P_R74F_3D) ++ return -EINVAL; /* the 3D bell is armed by g5p_r96_3d_arm(): never the DC3 / self-drain path */ ++ f->kind = kind; ++ f->ring = ta ? 0x60 : 0x6c; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ /* The immediate functional gate r74e used, without console output. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, kind + 1); ++ atomic_set(&f->state, G5P_R74F_FIRING); ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ f->first_ret = ret; ++ f->first_fence = fence; ++ f->first_state = state; ++ /* r87b self-drain: a closed DC3 gate is only a phase sample of the ++ * firmware's GPC window. The host never repairs it: no R1+20000 clear, no ++ * GPC off/on, no bell. The caller waits RAM-only for the firmware to ++ * consume the pair. Only the DC3 kind; an active gate rang above. ++ */ ++ self_drain = ret == -EHOSTDOWN && kind == G5P_R74F_DC3 && ++ READ_ONCE(r->r87b_attach) == G5P_R87B_ATTACH_SELF_DRAIN; ++ if (self_drain) { ++ f->cancel = G5P_R74F_CANCEL_SELF_DRAIN; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (ret == -EHOSTDOWN) { ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ } else { ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_IMMEDIATE); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (self_drain) { ++ drained = 1; ++ g5p_r74_mark(g, G5P_R74F_ATTACH_ARMED, state); ++ } else if (ret == -EHOSTDOWN) { ++ drained = g5p_r74f_open(g, f); ++ /* Functional reactivation precedes all console/milestone output. */ ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_ARMED : G5P_R74F_ATTACH_ARMED, ++ state); ++ g5p_r74_mark(g, ++ ta ? G5P_R74F_TA_REACTIVATED : ++ G5P_R74F_ATTACH_REACTIVATED, ++ f->power_ret); ++ } ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_BELL : G5P_R74F_ATTACH_BELL, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ if (drained) ++ return drained; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_REACTIVATION || ++ f->cancel == G5P_R74F_CANCEL_ALIVE_ADVANCED) ++ return g5p_r74f_stop( ++ g, ++ ta ? G5P_R74F_STOP_TA_REACTIVATION : ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ f->power_ret); ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* RAM only: the consumer this generation reached, at consume or timeout. */ ++static void g5p_r74f_consumed(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_r74f_bell *f = &g->boot->r73b->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(g->boot, kind); ++ ++ dma_rmb(); ++ f->consumer_after = READ_ONCE(ctl[0]); ++ f->producer_after = READ_ONCE(ctl[4]); ++ f->complete = f->consumer_after == f->target; ++ f->done_ns = ktime_get_mono_fast_ns(); ++ f->alive_done = READ_ONCE(g->boot->r73b->r74f_alive); ++ if (kind == G5P_R74F_DC3) ++ g5p_r87b_line(g, f, false); ++} ++ ++/* g5p_r22_mmu(): attach's first pass only; records the gate it already read. */ ++ ++/* g5p_r22_command(): an r74f boundary keeps the epoch, drain and backing. */ ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74g.inc b/drivers/gpu/apple-g5p/boot-r74g.inc +new file mode 100644 +index 0000000000..6ff2693173 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74g.inc +@@ -0,0 +1,288 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74g. Included by boot.inc after boot-r74f.inc; r74f case 1 only. It is a ++ * diagnostic bypass on the forced-ready epoch, not power policy. ++ * ++ * The firmware DC3 handler (0x1000218c..0x10002218) writes e140/e148 for ++ * slot N, copies host_generation[N] to cached_generation[N], then issues ++ * the e138 (N << 3) | 4 catalogue invalidation and waits for it. The DC ++ * dispatcher advances the consumer only after the handler returns, so a ++ * consumed DC3 pair is the completion receipt. r74g checks the RAM-visible ++ * results and issues no host MMU MMIO. The natural TA is then published ++ * without a doorbell. A RAM-only grace (four type-0 ALIVEs, 20 ms cap) ++ * lets the firmware drain it on its own. If it stays pending, the r74f ++ * ALIVE owner gets one fenced fallback doorbell. Receipts print afterwards. ++ */ ++ ++/* perf60: with the policy held in state 0 the firmware is quiet (no ALIVE stream, no self-drain), so ++ * job 1's TA bell, owned by "the first ALIVE after the grace", would never ring. ++ */ ++/* release8 default */ ++ ++static const char *const g5p_r74g_graces[G5P_R74G_GRACES] = { ++ "none", "consumed", "alives", "wall", "fault", ++}; ++ ++static bool g5p_r74g_on(struct g5p_boot *b) ++{ ++ return g5p_r74f_on(b) && 1; ++} ++ ++static u32 g5p_r74g_root(struct g5p_boot *b, u32 off) ++{ ++ return g5p_le32(b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r74g_shared(struct g5p_boot *b, u32 at) ++{ ++ return g5p_le32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE); ++} ++ ++/* RAM only: the DC3 ring, both records in attach order (3D, then TA), the ++ * PA[N] entries, host and cached tags, both failure latches and the ++ * root+3980 companion word, whose value 2 is firmware recovery/escape. ++ */ ++static u32 g5p_r74g_check(struct g5p_diag *g, u32 phase, const u32 *slot, ++ u64 pa, u32 tag, u32 start) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r74g_dc3 *d = &b->r73b->r74g_dc3[phase]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ u32 bad = 0; ++ ++ dma_rmb(); ++ d->start = start; ++ d->target = (start + 2) & 255; ++ d->tag = tag; ++ d->expected_pa = pa; ++ d->consumer = READ_ONCE(ctl[0]); ++ d->producer = READ_ONCE(ctl[4]); ++ if (d->consumer != d->target || d->producer != d->target) ++ bad |= G5P_R74G_BAD_RING; ++ for (u32 i = 0; i < 2; i++) { ++ u32 at = b->layout.offset[0x6d] + ((start + i) & 255) * 20; ++ ++ for (u32 j = 0; j < 5; j++) { ++ d->rec[i][j] = g5p_r74g_shared(b, at + j * 4); ++ if (d->rec[i][j] != (j == 0 ? 3 : ++ j == 1 ? slot[1 - i] : ++ 0)) ++ bad |= G5P_R74G_BAD_RECORD; ++ } ++ } ++ for (u32 e = 0; e < 2; e++) { ++ u8 *p = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ ++ d->slot[e] = slot[e]; ++ d->pa[e] = (u64)g5p_le32(p + 4) << 32 | g5p_le32(p); ++ d->host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + slot[e] * 4); ++ d->cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + slot[e] * 4); ++ if (d->pa[e] != pa) ++ bad |= G5P_R74G_BAD_PA; ++ if (d->host[e] != tag) ++ bad |= G5P_R74G_BAD_HOST; ++ if (d->cached[e] != tag) ++ bad |= G5P_R74G_BAD_CACHED; ++ } ++ d->failed = READ_ONCE(b->failed); ++ d->event_failed = READ_ONCE(b->event_failed); ++ if (d->failed || d->event_failed) ++ bad |= G5P_R74G_BAD_FAILED; ++ d->companion = g5p_r74g_root(b, G5P_R74G_COMPANION); ++ d->recoveries = g5p_r74g_root(b, G5P_R74G_RECOVERIES); ++ d->classification = g5p_r74g_root(b, G5P_R74G_CLASS); ++ if (d->companion == 2) ++ bad |= G5P_R74G_BAD_COMPANION; ++ d->bad = bad; ++ d->t_ns = ktime_get_mono_fast_ns(); ++ return bad; ++} ++ ++/* A refused check is retained, never b->failed: companion 2 is firmware ++ * recovery, a cached-tag-only mismatch is the firmware tag copy, and ++ * anything else (a host tag included) is a refused RAM validation. ++ */ ++static int g5p_r74g_refuse(struct g5p_diag *g, u32 bad) ++{ ++ if (bad & G5P_R74G_BAD_COMPANION) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_COMPANION, bad); ++ if (bad == G5P_R74G_BAD_CACHED) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_TAGS, bad); ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_RAM, bad); ++} ++ ++/* Two bounded lines: every field at full width still ends in end=r74g. */ ++static void g5p_r74g_dc3_line(struct g5p_diag *g, u32 phase, bool blob) ++{ ++ const struct g5p_r74g_dc3 *d = &g->boot->r73b->r74g_dc3[phase]; ++ const char *name = phase ? "ta-preflight" : "dc3-consume"; ++ char line[2][512]; ++ ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74G dc3 phase=%s bad=%02x start=%u target=%u consumer=%u producer=%u tag=%08x " ++ "slot=%u/%u host=%08x/%08x cached=%08x/%08x failed=%u event_failed=%u " ++ "companion=%u recoveries=%u class=%u host_mmu=skipped t_ns=%llu end=r74g\n", ++ name, d->bad, d->start, d->target, d->consumer, d->producer, ++ d->tag, d->slot[0], d->slot[1], d->host[0], d->host[1], ++ d->cached[0], d->cached[1], d->failed, d->event_failed, ++ d->companion, d->recoveries, d->classification, d->t_ns); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74G dc3-ram phase=%s pa=%llx/%llx expected_pa=%llx rec0=%x,%x,%x,%x,%x " ++ "rec1=%x,%x,%x,%x,%x end=r74g\n", ++ name, d->pa[0], d->pa[1], d->expected_pa, d->rec[0][0], ++ d->rec[0][1], d->rec[0][2], d->rec[0][3], d->rec[0][4], ++ d->rec[1][0], d->rec[1][1], d->rec[1][2], d->rec[1][3], ++ d->rec[1][4]); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++static void g5p_r74g_ta_line(struct g5p_diag *g, bool blob) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ const struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 end = r->r74g_grace_end; ++ char line[256]; ++ ++ scnprintf( ++ line, sizeof(line), ++ "R74G ta grace_end=%s arm_alive=%u grace_alive=%u grace_ns=%llu opened=%u " ++ "open_alive=%u ack_skipped=%u end=r74g\n", ++ g5p_r74g_graces[end < G5P_R74G_GRACES ? end : 0], f->arm_alive, ++ r->r74g_grace_alive, r->r74g_grace_ns - f->arm_ns, ++ r->r74g_opened, f->open_alive, r->r74g_ack_skipped); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* Process context, command mutex held, TA producer already published once. ++ * Grace: RAM reads and sleeps only, no MMIO and no doorbell, while the IRQ ++ * thread keeps decoding. It ends at the consume, four type-0 ALIVEs after ++ * the arm, or the 20 ms wall cap. If the TA is still pending, the ++ * generation opens, and the first ALIVE decoded after that owns the single ++ * r74f fallback attempt. Returns 0 after one doorbell, 1 if consumed without ++ * one, -EINPROGRESS for a retained boundary, else a fatal error. ++ */ ++static int g5p_r74g_ta(struct g5p_diag *g, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_TA); ++ unsigned long flags, deadline; ++ u32 end = G5P_R74G_GRACE_NONE, now; ++ int ret = 0; ++ ++ f->kind = G5P_R74F_TA; ++ f->ring = 0x60; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, G5P_R74F_TA + 1); ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ while (end == G5P_R74G_GRACE_NONE) { ++ ; ++ dma_rmb(); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) ++ end = G5P_R74G_GRACE_FAULT; ++ else if (READ_ONCE(ctl[0]) == target) ++ end = G5P_R74G_GRACE_CONSUMED; ++ else if (READ_ONCE(r->r74f_alive) - f->arm_alive >= ++ G5P_R74G_GRACE_ALIVES) ++ end = G5P_R74G_GRACE_ALIVES_END; ++ else if (ktime_get_mono_fast_ns() - f->arm_ns >= ++ (u64)G5P_R74G_GRACE_US * 1000) ++ end = G5P_R74G_GRACE_WALL; ++ else ++ usleep_range(100, 200); ++ } ++ spin_lock_irqsave(&b->event_lock, flags); ++ r->r74g_grace_end = end; ++ r->r74g_grace_alive = r->r74f_alive; ++ r->r74g_grace_ns = ktime_get_mono_fast_ns(); ++ dma_rmb(); ++ if (end == G5P_R74G_GRACE_FAULT) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (READ_ONCE(ctl[0]) == target) { ++ /* Drained by the firmware policy loop: never ring. */ ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ r->r74g_opened = 1; ++ f->open_alive = r->r74f_alive; ++ f->open_ns = r->r74g_grace_ns; ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ /* ta1_host_bell: the host owns it instead, in the same lock section a decoder would. */ ++ if ((1) && atomic_cmpxchg(&f->state, G5P_R74F_OPEN, ++ G5P_R74F_FIRING) == G5P_R74F_OPEN) { ++ u32 fence = U32_MAX, state = U32_MAX; ++ int rung; ++ ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ rung = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, rung, fence, state, ++ G5P_R74F_PATH_IMMEDIATE); ++ } ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (end == G5P_R74G_GRACE_FAULT) ++ ret = -EIO; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, G5P_R74G_TA_GRACE, end); ++ if (r->r74g_opened) ++ g5p_r74_mark(g, G5P_R74G_TA_FALLBACK, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ g5p_r74g_ta_line(g, false); ++ if (ret) ++ return ret; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_CONSUMED) ++ return 1; ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_NOT_CONSUMED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_WINDOW_MISSED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +new file mode 100644 +index 0000000000..2f02127893 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -0,0 +1,1662 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Experimental G2 stages. Included after the bounded META slave primitives. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "boot-layout.h" ++#include "render-user-pmset.h" ++#include "render-user-recover.h" ++#include "fender.h" ++#include "readback.h" ++#include "meta71c114.h" ++#include "latency123.h" ++#include "perf-envelope.h" ++ ++/* Boot deltas are selected before map; no live render-time reconfiguration. */ ++ ++/* R46 (diag-fable51 T7): the first SOFT_RESET write of the host reset FIFO ++ * also asserts bits 56-59, VBS/VCE/TE3/TILING_CORE in the PBE2_IN_XE ++ * layout. Apple's mask 00e7fffffffffc1d is exactly the other layout's ++ * MASKFULL, in which those bits do not exist and TE/GPP/UVS/TLA/PM/VDM are ++ * already asserted; expected to be a no-op. The unchanged second write ++ * releases them. The FIFO forbids a readback here. ++ */ ++#define G5P_BOOT_RESET1 0x00e7fffffffffc1dULL ++#define G5P_BOOT_R46_RESET_BITS (0xfULL << 56) ++ ++/* R72b: isolate Apple meta+0x6c8 while retaining dc7 transport. */ ++ ++/* The event ring's stamp events (type 1) dump once per epoch, then a count at each power of two. */ ++ ++/* R73: mirror Apple's auxiliary firmware-ring returns on IRQ174. */ ++ ++/* Latency114: the pstate-1 frequency word meta+0x71c (meta71c114.h). */ ++ ++/* Latency123 (latency123.h): arm A boot words, and the live ctl+0x04 tick-service mask. */ ++ ++/* perf60 clock (perf-envelope.h): the firmware's own pstate request, pinned before release. */ ++/* release8 default */ ++/* GPU overclock (gpu-oc.h, docs/power/ds-gpu-dvfs.md §9 R5): PSINFO record 5 = stock record 4 ++ * with only the PLL multiplier moved (no voltage change), written before release while the GPC is at ++ * state 1, the firmware's table extended to state 5, the release powering on at 5. Needs ++ * boot_gpu_perf 500 and boot_gpu_perf_poweron 1. A thermal stop (drm-render-v3.inc) reboots the ++ * firmware at state 4 when a SoC zone reaches boot_gpu_oc_temp_mc. Record 5 is volatile PMGR ++ * state (LLB rewrites the table only at SoC reset; iBoot reads it once to build the ADT before us) ++ * and is cleared on every boot that does not overclock. ++ */ ++ ++static bool ++ g5p_oc_tripped; /* the thermal stop fired: every later epoch of this boot runs state 4 */ ++ ++static const char *const g5p_oc_zone_names[] = { "apple-a9-soc0", ++ "apple-a9-soc1", ++ "apple-a9-tcc2" }; ++ ++/* The hottest of the three SoC zones (millidegrees); the number of zones read. */ ++static u32 g5p_oc_temp(int *max_mc, const char **zone) ++{ ++ u32 n = 0; ++ ++ *max_mc = INT_MIN; ++ *zone = "none"; ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_zone_names); i++) { ++ struct thermal_zone_device *tz = ++ thermal_zone_get_zone_by_name(g5p_oc_zone_names[i]); ++ int t; ++ ++ if (IS_ERR_OR_NULL(tz) || thermal_zone_get_temp(tz, &t)) ++ continue; ++ n++; ++ if (t > *max_mc) { ++ *max_mc = t; ++ *zone = g5p_oc_zone_names[i]; ++ } ++ } ++ return n; ++} ++ ++static int g5p_oc_read(struct g5p_diag *g, u32 n, u32 w[4]) ++{ ++ for (u32 i = 0; i < 4; i++) ++ if (regmap_read(g->pmgr, G5P_PSINFO_REC(n) + 4 * i, &w[i])) ++ return -EIO; ++ return 0; ++} ++ ++/* Write record 5 from the live record 4 and read it back; the GPC is at state 1 (map time). ++ * Slot 5 is read first: nonzero and not a record this driver writes -> refused, left untouched. ++ */ ++ ++/* A boot without the overclock clears a record 5 this driver wrote; anything else stays. */ ++static void g5p_oc_clear(struct g5p_diag *g) ++{ ++ u32 r4[4], r5[4]; ++ ++ if (!g->pmgr || g5p_oc_read(g, 4, r4) || ++ g5p_oc_read(g, G5P_OC_STATE, r5) || ++ !(r5[0] || r5[1] || r5[2] || r5[3])) ++ return; ++ if (!g5p_oc_is_ours(r4, r5)) { ++ dev_info( ++ g->dev, ++ "GPUPERF oc slot 5 not ours, left: %08x %08x %08x %08x end=gpuperf\n", ++ r5[0], r5[1], r5[2], r5[3]); ++ return; ++ } ++ for (u32 i = 0; i < 4; i++) ++ regmap_write(g->pmgr, G5P_PSINFO_REC(G5P_OC_STATE) + 4 * i, 0); ++ g5p_oc_read(g, G5P_OC_STATE, r5); ++ dev_info( ++ g->dev, ++ "GPUPERF oc record 5 cleared (no overclock this epoch) word0=%08x end=gpuperf\n", ++ r5[0]); ++} ++ ++/* perf60 pin >100, the host's own route: power the GPC on at q (R1+0x1000 = 0x10 | q) in the release, ++ * as the GPC1 request the kext and we already make, with q in place of 1; the policy record carries ++ * the seed-1 words (+0x28 = +0x2c = q) and stays "never sampled" (+0 = 0), so the boot sampler only ++ * initialises; it derives +0x34 and its clock entry priv+0x2c4 from Fender (0x1001c838). ++ */ ++/* release8 default */ ++/* perf60 pin >100 (perf-envelope.h g5p_perf_seed): the boot sampler applies the pin before ALIVE. */ ++ ++/* perf60 stage 4 (render-user-pmset.h): firmware-GiB pages for per-context PM records. */ ++/* release8 default */ ++/* perf60 A/B (docs/gpu/experiments/perf60-clock.md): the firmware's per-tick full flush alone. */ ++ ++/* Latched by map, before firmware sees any pointers. Zero preserves round 9. */ ++ ++/* Explicit decoded VA input only: zero is a valid address when valid=true. ++ * Never infer an address or context from revision-dependent e150 bits. ++ */ ++static unsigned long long mmu_fault_va; ++ ++static const bool boot_experiment = 1; ++ ++struct g5p_dma_page { ++ u8 *cpu; ++ dma_addr_t dma; ++}; ++ ++struct g5p_boot { ++ u32 r27_variant, r27_mode_writes, r46_reset, r50_mode, r50_writes; ++ ++ bool table_clean, preinstall_context, compute_boot_vm; ++ u32 compute_slot; ++ ++ bool compute_numerical, user_compute_owned; ++ ++ struct g5p_boot_layout layout; ++ struct g5p_dma_page image[G5P_IMAGE_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page root[G5P_ROOT_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page shared[G5P_SHARED_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page render_meta[2]; ++ /* perf60 stage 4 (render-user-pmset.h): boot_pm_sets per-context PM sets, 3 pages each */ ++ u32 pm_sets; ++ struct g5p_dma_page pm_set[G5P_PMSET_MAX * G5P_PMSET_PAGES]; ++ struct g5p_dma_page catalogue, directory, zero, leaf[4]; ++ struct g5p_dma_page sink_directory, sink_leaf, sink_page; ++ u32 setup_variant; ++ bool sink_ready, sink_dumped; ++ u32 leaf_index[4]; ++ u32 leaves, pages, mapped, fifo, events, alive_word; ++ u32 fender_raw[ARRAY_SIZE(g5p_fender_regs)]; ++ u32 rgx_raw[ARRAY_SIZE(g5p_rgx_regs)]; ++ const char *snapshot_stage; ++ bool snapshot_valid, rgx_valid; ++ u32 rgx_count; ++ int fifo_error; ++ int irq; ++ atomic_t irqs; ++ spinlock_t event_lock; ++ void __iomem *swirq; ++ bool maps_ready, armed, copied, released, alive, failed, irq_ready; ++ bool release_ack, release_ack_attempted; ++ bool fault_va_valid; ++ u64 fault_va; ++ bool work_ready, work_submitted, work_complete; ++ u32 work_target, control_target; ++ u32 dc_variant, work_variant; ++ u32 r72_meta6c8; ++ u32 meta71c; ++ u32 fw_power_words; ++ u32 gpu_perf; ++ u32 perf_seed; /* boot_gpu_perf_seed: the seeded pstate q, 0 = not seeded */ ++ u32 perf_seed_mode; /* boot_gpu_perf_seed as latched (1..5) */ ++ u32 perf_poweron; /* boot_gpu_perf_poweron: the pstate q the release requests, 0 = 1 as before */ ++ u32 gpu_oc_mhz; /* boot_gpu_oc_mhz as applied (state 5), 0 = no overclock */ ++ /* record 5's voltage codes */ ++ u64 seed_stamp, ++ seed_gpc1; /* seed 3/4: the stamp and the GPC1 write (arch timer) */ ++ u32 meta6d4; ++ u32 r78_ctrl0; ++ u32 r73_aux_drain; ++ u32 r73_aux_entries[6]; ++ u32 r73_aux_hash[6]; ++ u32 r73_aux_first[6][3]; ++ u32 r73_aux_last[6][3]; ++ struct g5p_r73b *r73b; ++ bool irq_capped; ++ bool dc_observe; ++ ++ bool control_ready, control_submitted, control_complete; ++ bool irq_tested, stopped, parked; ++ bool control_timed_out, work_timed_out, event_failed; ++ bool context_submitted, context_complete, context_timed_out; ++ u32 context_target; ++ /* The epoch's last recovery packet (render-user-recover.h), under event_lock. */ ++ struct g5p_fw_recovery fw_recovery; ++ u32 fw_recoveries; ++ u32 event_n[10]; /* events drained this epoch by type (9: 9 and above) */ ++}; ++ ++#include "boot-r73b.inc" ++ ++/* The fault-status subset is read-only, including while META reset is held. ++ * Never access R0 with GPC off/busy; no slave-port transaction is involved. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage); ++ ++static int g5p_boot_alloc_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ p->cpu = dmam_alloc_coherent(g->dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ /* There is no GPU bus translation in the recovered GART path. */ ++ if ((p->dma & (G5P_GPU_PAGE - 1)) || p->dma >= BIT_ULL(40) || ++ dma_to_phys(g->dev, p->dma) != p->dma) { ++ dev_err(g->dev, ++ "unsupported DMA translation/alignment at %pad\n", ++ &p->dma); ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ g->boot->pages++; ++ return 0; ++} ++ ++/* Deliberately map one16KiB page only, not the iOS all-address fallback. ++ * Own distinct directory/leaf/backing; do not alias firmware tables or e160. ++ * No live table surgery: this helper runs only during first map construction. ++ */ ++static int g5p_boot_sink_init(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ; ++ ; ++ if (!b->catalogue.cpu || b->maps_ready || b->armed || b->released || ++ b->sink_ready || b->sink_directory.cpu || b->sink_leaf.cpu || ++ b->sink_page.cpu) ++ return -EPERM; ++ if (g5p_le32(b->catalogue.cpu)) ++ return -EEXIST; ++ ret = g5p_boot_alloc_page(g, &b->sink_directory); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_leaf); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_page); ++ if (ret) ++ return ret; ++ /* Round12 iOS optional dummy-tree leaf flags0: valid writable PA|1. ++ * Only cat[0]/dir[0]/leaf[0] are installed; every other hole stays absent. ++ */ ++ g5p_put64(b->sink_leaf.cpu, g5p_leaf_entry(b->sink_page.dma, 0)); ++ g5p_put64(b->sink_directory.cpu, g5p_directory_entry(b->sink_leaf.dma)); ++ dma_wmb(); ++ g5p_put32(b->catalogue.cpu, g5p_catalogue_entry(b->sink_directory.dma)); ++ dma_wmb(); ++ b->mapped++; ++ b->sink_ready = true; ++ g5p_mmu_record( ++ g, ++ "MMU sink-installed VA=0 bytes=4000 context=0 leaf_flags=0; pre-release " ++ "containment experiment\n"); ++ g5p_blob_printf( ++ g, ++ "MMU sink PC=%pad cat0=%08x dir=%pad dir0=%016llx leaf=%pad pte0=%016llx " ++ "backing=%pad; initially zero\n", ++ &b->catalogue.dma, g5p_catalogue_entry(b->sink_directory.dma), ++ &b->sink_directory.dma, g5p_directory_entry(b->sink_leaf.dma), ++ &b->sink_leaf.dma, g5p_leaf_entry(b->sink_page.dma, 0), ++ &b->sink_page.dma); ++ return 0; ++} ++ ++static int g5p_boot_map_page(struct g5p_diag *g, u64 va, u64 pa, u32 attributes) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ u32 ti = (va >> 14) & 127, i; ++ int ret; ++ ++ if ((va | pa) & (G5P_GPU_PAGE - 1)) ++ return -EINVAL; ++ /* This experiment only owns the recovered firmware heap's catalogue slot. */ ++ if (ci != ((G5P_IMAGE_VA >> 30) & 1023)) ++ return -ERANGE; ++ for (i = 0; i < b->leaves; i++) ++ if (b->leaf_index[i] == di) ++ break; ++ if (i == b->leaves) { ++ if (i == ARRAY_SIZE(b->leaf)) ++ return -ENOSPC; ++ ret = g5p_boot_alloc_page(g, &b->leaf[i]); ++ if (ret) ++ return ret; ++ b->leaf_index[i] = di; ++ g5p_put64(b->directory.cpu + di * 8, ++ g5p_directory_entry(b->leaf[i].dma)); ++ b->leaves++; ++ } ++ if (g5p_le32(b->leaf[i].cpu + ti * 8)) ++ return -EEXIST; ++ g5p_put64(b->leaf[i].cpu + ti * 8, g5p_leaf_entry(pa, attributes)); ++ g5p_put32(b->catalogue.cpu + ci * 4, ++ g5p_catalogue_entry(b->directory.dma)); ++ b->mapped++; ++ return 0; ++} ++ ++#include "mmu-attrs.inc" ++#include "mmu-tables.inc" ++ ++static int g5p_boot_region(struct g5p_diag *g, struct g5p_dma_page *p, u32 size, ++ u64 va) ++{ ++ u32 offset; ++ u32 attributes = va == G5P_IMAGE_VA ? 0x10 : 0x14; ++ int ret; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) { ++ ret = g5p_boot_alloc_page(g, p); ++ if (ret) ++ return ret; ++ ret = g5p_boot_map_page(g, va + offset, p->dma, attributes); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void g5p_boot_copy(struct g5p_dma_page *p, const u8 *data, u32 size) ++{ ++ u32 offset; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) ++ memcpy(p->cpu, data + offset, G5P_GPU_PAGE); ++} ++ ++static u8 *g5p_boot_shared(struct g5p_boot *b, u32 slot) ++{ ++ u32 off = b->layout.offset[slot]; ++ ++ return b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static int g5p_boot_maps(struct g5p_diag *g) ++{ ++ struct g5p_boot *b; ++ u8 *root = NULL, *shared = NULL; ++ bool m71c_wrote; ++ u32 i, m71c, seed; ++ int ret; ++ ++ if (!g->loaded || !g->id_checked || PAGE_SIZE != G5P_GPU_PAGE) ++ return -EPERM; ++ if (g->core_id != 0x011b0602 || g->core_revision != 0x00090f00) ++ return -ENODEV; ++ if (g->boot) ++ return g->boot->maps_ready ? 0 : -EIO; ++ b = devm_kzalloc(g->dev, sizeof(*b), GFP_KERNEL); ++ if (!b) ++ return -ENOMEM; ++ g->boot = b; ++ b->r50_mode = (0); ++ ; ++ b->r27_variant = (0); ++ ; ++ b->r46_reset = (0); ++ ; ++ b->r72_meta6c8 = (1); ++ b->r78_ctrl0 = (0); ++ if (b->r78_ctrl0 && b->r78_ctrl0 != 0x5f) { ++ b->failed = true; ++ return -EINVAL; ++ }; ++ b->r73_aux_drain = (0); ++ if (b->r73_aux_drain > 1 || (b->r73_aux_drain && !1)) { ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->release_ack = (0); ++ b->table_clean = (1); ++ b->compute_boot_vm = (0); ++ b->compute_slot = 0 ? 0 : (7); ++ b->preinstall_context = (0) || 0; ++ ; ++ b->setup_variant = (1); ++ ; ++ b->fault_va = READ_ONCE(mmu_fault_va); ++ b->fault_va_valid = (0); ++ ; ++ b->dc_variant = (7); ++ b->work_variant = (0); ++ b->dc_observe = (0); ++ ; ++ b->meta71c = (0); ++ m71c = g5p_m71c_check(b->meta71c, 7); ++ if (m71c != G5P_M71C_OK) { ++ dev_err(g->dev, ++ "META71C refused requested=%u variant=%u why=%s before_release=1 end=meta71c\n", ++ b->meta71c, b->dc_variant, g5p_m71c_whys[m71c]); ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->fw_power_words = (0); ++ b->gpu_perf = (400); ++ b->meta6d4 = (0); ++ seed = (0); ++ b->gpu_oc_mhz = 0; ++ ; ++ b->perf_poweron = ++ (1) ? (0 ? G5P_OC_STATE : g5p_perf_seed_q(b->gpu_perf)) : 0; ++ b->perf_seed = seed && !b->perf_poweron ? g5p_perf_seed_q(b->gpu_perf) : ++ 0; ++ b->perf_seed_mode = b->perf_seed ? seed : 0; ++ if (!(g5p_perf_valid(b->gpu_perf) || ++ (0 && b->gpu_perf == G5P_OC_PIN)) || ++ (b->gpu_perf && b->fw_power_words) || seed > 5 || (1) > 1 || ++ (b->gpu_perf > 100 && !(1)) || b->meta6d4 > 1 || ++ (b->meta6d4 && !b->gpu_perf)) { ++ dev_err(g->dev, ++ "GPUPERF refused pin=%u power_words=%u gpc_any_pstate=%u meta6d4=%u seed=%u " ++ "before_release=1 end=gpuperf\n", ++ b->gpu_perf, b->fw_power_words, (1), b->meta6d4, seed); ++ b->failed = true; ++ return -EINVAL; ++ } ++ if (b->fw_power_words > G5P_L123_FULL || (0) > G5P_L123_TICK_MASK) { ++ dev_err(g->dev, ++ "L123 refused power_words=%u tick_mask=%u before_release=1 end=l123\n", ++ b->fw_power_words, (0)); ++ b->failed = true; ++ return -EINVAL; ++ } ++ ret = g5p_r73b_latch(g); ++ if (ret) { ++ b->failed = true; ++ return ret; ++ }; ++ dev_info( ++ g->dev, ++ "DC matrix latched variant=%u observe=%u (changes require reboot)\n", ++ b->dc_variant, b->dc_observe); ++ dev_info( ++ g->dev, ++ "COMPUTE VM latched slot=%u boot_vm=%u preinstall=%u (changes require reboot)\n", ++ b->compute_slot, b->compute_boot_vm, b->preinstall_context); ++ spin_lock_init(&b->event_lock); ++ atomic_set(&b->irqs, 0); ++ ret = g5p_plan_layout(&b->layout); ++ if (ret) ++ goto out; ++ root = kvzalloc(G5P_ROOT_SIZE, GFP_KERNEL); ++ shared = kvzalloc(G5P_SHARED_SIZE, GFP_KERNEL); ++ if (!root || !shared) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_boot_alloc_page(g, &b->catalogue); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->directory); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->zero); ++ if (ret) ++ goto out; ++ ret = g5p_boot_sink_init(g); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->image, G5P_IMAGE_SIZE, G5P_IMAGE_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->root, G5P_ROOT_SIZE, G5P_ROOT_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->shared, G5P_SHARED_SIZE, G5P_SHARED_VA); ++ if (ret) ++ goto out; ++ /* R22 metadata mappings are installed before release, only behind its ++ * default-off gate. Compute/boot defaults allocate no additional pages. ++ */ ++ { ++ ret = g5p_boot_region(g, b->render_meta, 2 * G5P_GPU_PAGE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE); ++ if (ret) ++ goto out; ++ b->pm_sets = min((8), G5P_PMSET_MAX); ++ if (b->pm_sets) { ++ ret = g5p_boot_region(g, b->pm_set, ++ b->pm_sets * G5P_PMSET_PAGES * ++ G5P_GPU_PAGE, ++ G5P_PMSET_VA); ++ if (ret) ++ goto out; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-sets boot sets=%u va=%#llx..%#llx before_release=1 end=v3\n", ++ b->pm_sets, G5P_PMSET_VA, ++ g5p_pmset_va(b->pm_sets, 0)); ++ } ++ } ++ ret = 0; ++ if (ret) ++ goto out; ++ /* c80758: Fender (2 pages), AIC timer, AIC software interrupt. */ ++ for (i = 0; i < 4; i++) { ++ u64 pa = i < 2 ? 0x201e00000ULL + i * G5P_GPU_PAGE : ++ i == 2 ? 0x20e100000ULL : ++ 0x20e104000ULL; ++ ++ ret = g5p_boot_map_page(g, G5P_PIO_VA + i * G5P_GPU_PAGE, pa, ++ 0x18); ++ if (ret) ++ goto out; ++ } ++ g5p_make_boot_config(&g->image, &b->layout); ++ g5p_make_init(&b->layout, root, shared, &g->image, b->catalogue.dma, ++ arch_timer_read_counter()); ++ g5p_dc_init_variant(&b->layout, root, shared, 7); ++ /* meta+0x784 is the abstime OFFSET the firmware adds to its timebase (0x100221e8 = PIO-1 ++ * counter + priv+688, loaded from meta+0x784 at 0x1001ff58); iOS writes ++ * ml_get_abstime_offset(). Linux's timebase is the counter itself: 0. (make_init's absolute ++ * stamp put the firmware's clock ~uptime ahead of every host stamp in policy+0x8f8.) ++ */ ++ g5p_put64(g5p_object(&b->layout, root, shared, 0x4d) + 0x784, 0); ++ if (g5p_boot_ctrl0(b)) ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x5e), ++ g5p_boot_ctrl0(b)); ++ dev_info( ++ g->dev, ++ "R78 ctrl0 requested=%08x installed=%08x before_release=1 end=r78\n", ++ g5p_boot_ctrl0(b), ++ g5p_le32(g5p_object(&b->layout, root, shared, 0x5e))); ++ ; ++ ; ++ { ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x4d) + 0x6c8, ++ 1); ++ }; ++ m71c = g5p_m71c_install(&b->layout, root, shared, b->meta71c, ++ &m71c_wrote); ++ dev_info( ++ g->dev, ++ "META71C requested=%u installed=%08x variant=%u wrote=%u before_release=1 " ++ "end=meta71c\n", ++ b->meta71c, m71c, b->dc_variant, m71c_wrote); ++ if (m71c != b->meta71c) { ++ ret = -EIO; ++ goto out; ++ } ++ if (b->fw_power_words) ++ g5p_l123_arm_a(&b->layout, root, shared, b->fw_power_words); ++ if (b->gpu_perf) { ++ u8 *pol = g5p_object(&b->layout, root, shared, G5P_PERF_POLICY); ++ u8 *mt = g5p_object(&b->layout, root, shared, G5P_PERF_META); ++ u8 *ct = g5p_object(&b->layout, root, shared, G5P_PERF_CTL); ++ ++ g5p_perf_install(&b->layout, root, shared, b->gpu_perf); ++ { ++ g5p_oc_clear(g); ++ } ++ if (b->meta6d4) ++ g5p_put32(mt + 0x6d4, 1); ++ if (b->perf_poweron) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_poweron, ++ false); ++ dev_info( ++ g->dev, ++ "GPUPERF poweron q=%u policy28=%u/%u policy0=%u policy4=%u seed=%u ignored " ++ "before_release=1 end=gpuperf\n", ++ b->perf_poweron, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), seed); ++ } else { ++ dev_info( ++ g->dev, ++ "GPUPERF poweron ignored pin=%u (pins 200..400 only) before_release=1 end=gpuperf\n", ++ b->gpu_perf); ++ } ++ if (seed && !b->perf_seed && !b->perf_poweron) ++ dev_info( ++ g->dev, ++ "GPUPERF seed ignored pin=%u (seeds pins 200..400 only) before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf); ++ if (b->perf_seed) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_seed, ++ seed >= 2); ++ /* seed 5: meta+0x784 is the abstime OFFSET the firmware adds to its timebase ++ * (0x100221e8 = PIO-1 counter + priv+688, loaded from meta+0x784 at 0x1001ff58); ++ * iOS writes ml_get_abstime_offset(), make_init the absolute counter, so every ++ * policy+0x8f8 stamp lay ~counter in the firmware's past. Linux has no offset. ++ */ ++ if (seed == 5) ++ g5p_put64(mt + 0x784, 0); ++ dev_info( ++ g->dev, ++ "GPUPERF seed mode=%u q=%u policy28=%u/%u policy30=%u policy8=%u policy0=%u " ++ "policy4=%u meta784=%llx before_release=1 end=gpuperf\n", ++ seed, b->perf_seed, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0x30), ++ g5p_le32(pol + 8), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), ++ (u64)g5p_le32(mt + 0x784) | ++ (u64)g5p_le32(mt + 0x788) << 32); ++ } ++ dev_info( ++ g->dev, ++ "GPUPERF boot pin=%u table=%u/%u/%u,%u,%u,%u,%u limits=%u/%u/%u ovr=%u/%u " ++ "policy688=%u/%u policy6f0=%u/%u policy4=%u policy8=%u/%u q=%u mhz=%u " ++ "gpc_any_pstate=%u seed=%u/%u policy0=%u policy28=%u/%u before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf, g5p_le32(mt + 0x710), g5p_le32(mt + 0x714), ++ g5p_le32(mt + 0x718), g5p_le32(mt + 0x71c), ++ g5p_le32(mt + 0x720), g5p_le32(mt + 0x724), ++ g5p_le32(mt + 0x728), g5p_le32(ct + 0x1028), ++ g5p_le32(ct + 0x102c), g5p_le32(ct + 0x1030), ++ g5p_le32(ct + 0x1038), g5p_le32(ct + 0x1040), ++ g5p_le32(pol + 0x688), g5p_le32(pol + 0x68c), ++ g5p_le32(pol + 0x6f0), g5p_le32(pol + 0x6f4), ++ g5p_le32(pol + 4), g5p_le32(pol + 8), ++ g5p_le32(pol + 0xc), b->gpu_perf / 100, ++ b->gpu_perf / 100 < ARRAY_SIZE(g5p_perf_mhz) ? ++ g5p_perf_mhz[b->gpu_perf / 100] : ++ b->gpu_oc_mhz, ++ (1), seed, b->perf_seed, g5p_le32(pol + 0), ++ g5p_le32(pol + 0x28), g5p_le32(pol + 0x2c)); ++ } ++ g5p_put32(g5p_object(&b->layout, root, shared, G5P_L123_CTL) + 4, (0)); ++ { ++ u8 *meta = g5p_object(&b->layout, root, shared, G5P_L123_META); ++ ++ dev_info( ++ g->dev, ++ "L123 boot power_words=%u meta6d4=%u table=%u/%u/%u,%u,%u,%u,%u policy4=%u " ++ "ctl04=%u limits=%u/%u/%u before_release=1 end=l123\n", ++ b->fw_power_words, g5p_le32(meta + 0x6d4), ++ g5p_le32(meta + 0x710), g5p_le32(meta + 0x714), ++ g5p_le32(meta + 0x718), g5p_le32(meta + 0x71c), ++ g5p_le32(meta + 0x720), g5p_le32(meta + 0x724), ++ g5p_le32(meta + 0x728), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_POLICY) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1028), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x102c), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1030)); ++ /* the accessor's licence bound (mmio-access.md §4.2 step 4): meta+0x710 as written */ ++ WRITE_ONCE(g->mmio_max_pstate, ++ g5p_mmio_max_pstate(g5p_le32(meta + 0x710))); ++ }; ++ ; ++ g5p_boot_copy(b->image, g->image.data, G5P_IMAGE_SIZE); ++ g5p_boot_copy(b->root, root, G5P_ROOT_SIZE); ++ g5p_boot_copy(b->shared, shared, G5P_SHARED_SIZE); ++ dma_wmb(); ++ ret = 0; ++ if (ret) ++ goto out; ++ g5p_tables_clean(g, "boot-before-release", NULL, 0); ++ b->maps_ready = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 maps: %u x 16KiB allocations; %u PTEs, %u leaves, PC=%pad\n", ++ b->pages, b->mapped, b->leaves, &b->catalogue.dma); ++ dev_info( ++ g->dev, ++ "G2 init: root=%llx control=%llx event-ctl=%llx event-data=%llx\n", ++ g5p_object_va(&b->layout, 0x50), ++ g5p_object_va(&b->layout, 0x5f), ++ g5p_object_va(&b->layout, 0x78), ++ g5p_object_va(&b->layout, 0x7c)); ++ dev_info( ++ g->dev, ++ "G2 experimental zero policy fields; fault observation only in map\n"); ++out: ++ kvfree(root); ++ kvfree(shared); ++ if (ret) ++ b->failed = true; ++ g5p_ds_hook_maps(g, ret); ++ return ret; ++} ++ ++#include "boot-r74f.inc" ++#include "boot-r74g.inc" ++ ++static u32 g5p_transport_event_limit(void); ++ ++static int g5p_boot_drain(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ dev_err(g->dev, "G2 invalid event indices %u/%u\n", consumer, ++ producer); ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ for (n = 0; n < 16 && consumer != producer; n++) { ++ u32 words[11], off = b->layout.offset[0x74] + consumer * 44, t, ++ n; ++ u8 *src; ++ unsigned int i; ++ bool want; ++ ++ /* Heartbeat116: an R74 epoch bounds ring entries per window ++ * in the IRQ thread instead (heartbeat116.h EVENTS). ++ */ ++ if (!g5p_hb116_on(b) && ++ b->events >= g5p_transport_event_limit()) { ++ ret = -EOVERFLOW; ++ goto out; ++ } ++ /* Entries may straddle DMA pages. */ ++ for (i = 0; i < ARRAY_SIZE(words); i++) { ++ u32 at = off + i * 4; ++ ++ src = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ words[i] = g5p_le32(src); ++ } ++ /* A healthy meta6c8 setup loop can emit thousands of identical ++ * ALIVEs. Keep its first four packets and the exact aggregate, ++ * without making printk throughput part of the experiment. ++ * R74 also quiets a moved loop: each new word1 prints once. ++ */ ++ want = !b->r73b || words[0] || b->events < 4 || ++ (words[1] != 0x8000a120 && g5p_r74_word1(b, words[1])); ++ /* Stamp events (type 1, two per job) dump once per epoch, then only count: ++ * at a few jobs per frame they were most of the log and evicted the faults ++ * (release8 bench, 306 k lines). Every other type dumps as before. ++ */ ++ t = min_t(u32, words[0], ARRAY_SIZE(b->event_n) - 1); ++ n = ++b->event_n[t]; ++ if (words[0] == 1 && n > 1 && !(0)) { ++ if (is_power_of_2(n)) ++ dev_info( ++ g->dev, ++ "G2 events type=1 n=%u: stamp events counted, dumped once (boot_event_dump 1: each)\n", ++ n); ++ want = false; ++ } ++ /* Alive (0) and stamp (1) events are routine, one or more per job: job_log. Every other ++ * type (recovery, page request, channel error, fault) prints as before. ++ */ ++ if (want && words[0] > 1) { ++ dev_info(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ print_hex_dump(KERN_INFO, ++ "G2 event raw: ", DUMP_PREFIX_NONE, 16, ++ 4, words, sizeof(words), false); ++ } else if (want) { ++ g5p_job_printk(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ g5p_job_hex("G2 event raw: ", words, sizeof(words)); ++ }; ++ if (g->event_notify) ++ g->event_notify(g->event_cookie, words); ++ /* c99788 routes these to recovery/page-request/channel-error paths. */ ++ if (words[0] != 0 && words[0] != 1) { ++ WRITE_ONCE(b->failed, true); ++ switch (words[0]) { ++ case G5P_FW_EVENT_RECOVERY: { ++ /* The kext's fields (render-user-recover.h). */ ++ struct g5p_fw_recovery rec = ++ g5p_fw_recovery_decode(words); ++ ++ b->fw_recovery = rec; ++ b->fw_recoveries++; ++ dev_err(g->dev, ++ "G2 recovery event slot=%u flag=%u code=%08x w18=%08x w20=%08x w24=%08x n=%u: " ++ "the firmware stopped; iOS restarts the GPU\n", ++ rec.slot, rec.flag, rec.code, rec.w18, ++ rec.w20, rec.w24, b->fw_recoveries); ++ break; ++ } ++ case 7: ++ dev_err(g->dev, ++ "G2 page-request event payload=%08x/%08x/%08x; address encoding unresolved\n", ++ words[1], words[2], words[3]); ++ break; ++ case 8: ++ dev_err(g->dev, ++ "G2 channel-error event kind=%u engine=%u token=%08x stamp=%08x; no " ++ "acknowledgment command guessed\n", ++ words[1], words[2], words[3], words[4]); ++ break; ++ } ++ dev_err(g->dev, ++ "G2 unsupported/fault event=%u; epoch failed, entry retained (the render device " ++ "recovers it if it may)\n", ++ words[0]); ++ ret = -EIO; ++ break; ++ } ++ if (words[0] == 0 && READ_ONCE(b->released) && ++ !READ_ONCE(b->stopped)) { ++ b->alive_word = words[1]; ++ WRITE_ONCE(b->alive, true); ++ } ++ b->events++; ++ consumer = (consumer + 1) & 255; ++ /* c99788 uses a full barrier: finish slot reads before returning it. */ ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ /* Return the ALIVE slot before ringing work. No logging or ++ * other MMIO may be inserted between these two operations. ++ */ ++ if (!words[0] && g5p_r74f_on(b) && ++ READ_ONCE(b->r73b->owner) == G5P_R73B_OWNER_IRQ && ++ READ_ONCE(b->released) && !READ_ONCE(b->stopped)) ++ g5p_r74f_alive(g); ++ } ++out: ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++static irqreturn_t g5p_boot_irq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ struct g5p_boot *b = g->boot; ++ u32 n = atomic_inc_return(&b->irqs); ++ int ret = 0; ++ ++ if (b->r73b) ++ return g5p_r73b_irq(g, irq, n); ++ if (n > g5p_transport_event_limit()) { ++ /* Never leave the software source asserted, even when capped. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (!b->irq_capped) { ++ b->irq_capped = true; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "G2 IRQ174 storm capped; reboot required\n"); ++ } ++ return IRQ_HANDLED; ++ } ++ /* KPF level6G: Apple uses this AIC ACK before all ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (READ_ONCE(b->released)) ++ ret = g5p_boot_drain(g); ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return IRQ_HANDLED; ++} ++ ++static int g5p_boot_irq_setup(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b->irq_ready) ++ return 0; ++ b->swirq = devm_ioremap(g->dev, 0x20e104000ULL, 0x100); ++ if (!b->swirq) ++ return -ENOMEM; ++ b->irq = platform_get_irq(to_platform_device(g->dev), 4); ++ if (b->irq < 0) ++ return b->irq; ++ ret = devm_request_threaded_irq(g->dev, b->irq, NULL, g5p_boot_irq, ++ IRQF_ONESHOT, "g5p-meta", g); ++ if (ret) ++ return ret; ++ b->irq_ready = true; ++ dev_info(g->dev, "G2 IRQ174 registered as Linux IRQ %d\n", b->irq); ++ return 0; ++} ++ ++/* Cache first, then report: sysfs status never issues GPU MMIO. */ ++static void g5p_boot_snapshot(struct g5p_diag *g, const char *stage, ++ enum g5p_readback_state mode) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 i, state; ++ ++ b->snapshot_stage = stage; ++ b->snapshot_valid = false; ++ b->rgx_valid = false; ++ b->rgx_count = 0; ++ g5p_power_snapshot(g, stage); ++ dev_info(g->dev, "G2 snapshot %s: R1 first\n", stage); ++ for (i = 0; i < ARRAY_SIZE(g5p_fender_regs); i++) { ++ b->fender_raw[i] = readl(g->window[1] + g5p_fender_regs[i]); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x\n", g5p_fender_regs[i], ++ b->fender_raw[i]); ++ } ++ b->snapshot_valid = true; ++ state = b->fender_raw[4]; ++ /* GPC=1 did not make META a18 safe while bit32 was held. */ ++ b->rgx_count = g5p_readback_rgx_count(state, mode); ++ if (!b->rgx_count) { ++ dev_info(g->dev, "G2 R0 snapshot skipped: R1+1000=%08x\n", ++ state); ++ return; ++ } ++ for (i = 0; i < b->rgx_count; i++) { ++ dev_info(g->dev, "G2 R R0+%05x\n", g5p_rgx_regs[i]); ++ b->rgx_raw[i] = readl(g->rgx + g5p_rgx_regs[i]); ++ dev_info(g->dev, "G2 raw R0+%05x=%08x\n", g5p_rgx_regs[i], ++ b->rgx_raw[i]); ++ } ++ b->rgx_valid = true; ++} ++ ++static int g5p_fender_write(void *ctx, u32 reg, u32 value) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ ++ /* recovery108: never an R1 access after a FINAL teardown retired the device. */ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 W R1+%05x=%08x\n", reg, value); ++ writel(value, g->window[1] + reg); ++ raw = readl(g->window[1] + reg); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x (write fence)\n", reg, raw); ++ return 0; ++} ++ ++static int g5p_fender_poll(void *ctx, u32 reg, u32 mask, u32 expected) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 poll R1+%05x mask=%08x expected=%08x (100ms)\n", ++ reg, mask, expected); ++ ret = readl_poll_timeout(g->window[1] + reg, raw, ++ (raw & mask) == expected, 10, 100000); ++ dev_info(g->dev, "G2 poll raw R1+%05x=%08x ret=%d\n", reg, raw, ret); ++ return ret; ++} ++ ++static int g5p_r74e_power_poll(struct g5p_diag *g, u32 expected, u32 timeout_us, ++ u64 *elapsed_ns, u32 *samples) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u64 start = ktime_get_mono_fast_ns(); ++ u32 raw = U32_MAX, n = 0; ++ int ret; ++ ++ for (;;) { ++ if (READ_ONCE(b->alive)) { ++ ret = -EHOSTDOWN; ++ break; ++ } ++ raw = readl(g->window[1] + 0x1000); ++ n++; ++ if ((raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ /* wake112: the reads taken; 0 means raw is still the U32_MAX initializer, not a read. */ ++ *samples = n; ++ r->early_power_raw = raw; ++ return ret; ++} ++ ++/* wake112 (r74e112.h): a failed deferred proof is the cold power abort, absorbing. */ ++static void g5p_r74e_fail(struct g5p_r73b *r, u32 why) ++{ ++ r->early_proof_why = why; ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ atomic_set_release(&r->early_proof, G5P_PX_FAILED); ++} ++ ++/* The once-owner takes the deferred proof right before its one R1+0x1000 read. ++ * false: that read must not happen, and the proof has failed. ++ */ ++static bool g5p_r74e_take(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_px_pre p; ++ u32 why; ++ ++ if (atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, G5P_PX_TAKEN) != ++ G5P_PX_DEFERRED) ++ return false; ++ p = (struct g5p_px_pre){ ++ .gen = r->early_proof_gen, ++ .gen_now = g5p_ds_gen(), ++ .alive = READ_ONCE(b->alive), ++ .released = READ_ONCE(b->released), ++ .stopped = READ_ONCE(b->stopped), ++ .failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ .r1_allowed = g5p_r8_r1_allowed(), ++ }; ++ r->early_proof_gen_now = p.gen_now; ++ why = g5p_px_check(&p); ++ if (why) ++ g5p_r74e_fail(r, why); ++ return !why; ++} ++ ++/* The owner's read decides: only the state field 1 licenses the release. */ ++static void g5p_r74e_settle(struct g5p_r73b *r, u32 raw) ++{ ++ if (!g5p_px_on(raw)) { ++ g5p_r74e_fail(r, G5P_PXW_STATE); ++ return; ++ } ++ WRITE_ONCE(r->early_release_ready, true); ++ atomic_set_release(&r->early_proof, G5P_PX_PROVEN); ++} ++ ++/* No ALIVE wait ends with the proof still owed: close it, no register access. ++ * true: it was still owed. ++ */ ++static bool g5p_r74e_close(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ ++ if (!r || atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, ++ G5P_PX_TAKEN) != G5P_PX_DEFERRED) ++ return false; ++ g5p_r74e_fail(r, G5P_PXW_UNTAKEN); ++ return true; ++} ++ ++/* boot_gpu_perf_seed 3/4: the sampler's last-sample time policy+0x8f8 = now. Seed 3 stamps at the ++ * start of the release (flown 2026-11-03: 38 ms before the stream's end, the window was missed); ++ * seed 4 stamps as the last host write before the one that lets META run, the GPC1 request ++ * R1+0x1000 = 0x11 (writel orders the RAM store before it). ++ */ ++static void g5p_perf_seed_stamp(struct g5p_boot *b, u32 mode) ++{ ++ u32 o; ++ ++ if (mode == 4 ? b->perf_seed_mode < 4 : b->perf_seed_mode != mode) ++ return; /* the GPC1 stamp serves seeds 4 and 5 */ ++ o = b->layout.offset[G5P_PERF_POLICY] + 0x8f8; ++ b->seed_stamp = arch_timer_read_counter(); ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)b->seed_stamp)); ++ o += 4; ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)(b->seed_stamp >> 32))); ++ wmb(); ++} ++ ++static int g5p_r74e_fender_start(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxp latch; ++ bool on_ran = false; ++ u32 samples = ++ 0; /* wake112: the reads of the poll that ran last (the printed raw) */ ++ int ret; ++ ++ r->early_power_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_r74e_power_poll(g, 0, 1000, &r->early_off_ns, ++ &samples); ++ if (!ret) { ++ g5p_perf_seed_stamp(b, 4); ++ ret = g5p_fender_write(g, 0x1000, ++ 0x10 | (b->perf_poweron ?: 1)); ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ if (!ret) { ++ on_ran = true; ++ /* poweron: the first request above pstate 1, a voltage step; a longer bound */ ++ ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, ++ b->perf_poweron ? 10000 : 1800, ++ &r->early_on_ns, &samples); ++ if (b->perf_poweron) ++ dev_emerg( ++ g->dev, ++ "GPUPERF poweron q=%u r1_1000=%08x on_ns=%llu ret=%d end=gpuperf\n", ++ b->perf_poweron, r->early_power_raw, ++ r->early_on_ns, ret); ++ } ++ /* wake112: only ALIVE preempting the on-poll before its first read defers; the ++ * first-ALIVE owner owes the proof. A sampled on-poll (0x11, ...) aborts as before. ++ */ ++ latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples); ++ if (latch == G5P_PXP_DEFER) { ++ r->early_defer_ret = ret; ++ r->early_proof_gen = g5p_ds_gen(); ++ ret = 0; ++ atomic_set_release(&r->early_proof, G5P_PX_DEFERRED); ++ } ++ r->early_power_ret = ret; ++ if (latch == G5P_PXP_ABORT) { ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ } else if (latch == G5P_PXP_READY) { ++ WRITE_ONCE(r->early_release_ready, true); ++ } ++ dev_emerg( ++ g->dev, ++ "R74E power off_ns=%llu on_ns=%llu raw=%08x ret=%d release_ready=%u alive=%u " ++ "sampled=%u end=r74e\n", ++ r->early_off_ns, r->early_on_ns, r->early_power_raw, ret, ++ r->early_release_ready, READ_ONCE(b->alive), samples); ++ if (latch == G5P_PXP_DEFER) ++ dev_emerg( ++ g->dev, ++ "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u end=r74e112\n", ++ r->early_proof_gen, r->early_defer_ret, ++ r->early_power_raw, READ_ONCE(b->alive)); ++ return ret; ++} ++ ++static void g5p_fifo_word(struct g5p_diag *g, u32 word) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 raw; ++ ++ if (b->fifo_error) ++ return; ++ if (b->fifo < 0x10000 || b->fifo >= 0x10200) { ++ b->fifo_error = -EOVERFLOW; ++ return; ++ } ++ writel(word, g->window[1] + b->fifo); ++ raw = readl(g->window[1] + b->fifo); ++ if (raw != word) { ++ dev_err(g->dev, ++ "G2 FIFO verify R1+%05x actual=%08x expected=%08x\n", ++ b->fifo, raw, word); ++ b->fifo_error = -EIO; ++ } ++ b->fifo += 4; ++} ++ ++static void g5p_fifo32(struct g5p_diag *g, u32 reg, u32 value, u32 flags) ++{ ++ dev_info(g->dev, "G2 FIFO R0+%05x=%08x flags=%02x\n", reg, value, ++ flags); ++ g5p_fifo_word(g, (reg & 0xff00ffff) | (flags & 0xff) << 16); ++ g5p_fifo_word(g, value); ++} ++ ++static void g5p_fifo64(struct g5p_diag *g, u32 reg, u64 value) ++{ ++ g5p_fifo32(g, reg + 4, value >> 32, 0); ++ g5p_fifo32(g, reg, value, 0xff); ++} ++ ++static int g5p_boot_fifo_dump(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off, raw, end = min(b->fifo, 0x10200U); ++ ++ if (end <= 0x10000) ++ return -ENODATA; ++ dev_info(g->dev, "G2 FIFO dump R1+10000..%05x (<=128 words)\n", ++ end - 4); ++ for (off = 0x10000; off < end; off += 4) { ++ raw = readl(g->window[1] + off); ++ dev_info(g->dev, "G2 FIFO raw R1+%05x=%08x\n", off, raw); ++ } ++ return 0; ++} ++ ++static u64 g5p_snapshot64(struct g5p_boot *b, u32 reg) ++{ ++ u32 i; ++ ++ for (i = 0; i + 1 < b->rgx_count; i++) ++ if (g5p_rgx_regs[i] == reg && g5p_rgx_regs[i + 1] == reg + 4) ++ return (u64)b->rgx_raw[i + 1] << 32 | b->rgx_raw[i]; ++ return 0; ++} ++ ++static void g5p_release_observe(void *ctx, const char *stage) ++{ ++ ; ++ ; ++} ++ ++/* A diagnostic notification ACK only, never engine repair or a retry loop. ++ * Snapshot first, require the active wrapper, ACK only200 once, fence and ++ * re-sample. A cleared event alone is not evidence the underlying fault healed. ++ */ ++static int g5p_release_ack(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 state, event, after; ++ ++ if (!b->release_ack) ++ return 0; ++ if (!b->released || b->stopped || b->release_ack_attempted) ++ return -EPERM; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_fault_snapshot(g, "release-ack-before"); ++ g5p_mmu_progress(g, "release-ack-event", 0x130); ++ event = readl(g->rgx + 0x130); ++ b->release_ack_attempted = true; ++ if (event & 0x200) { ++ g5p_mmu_progress(g, "release-ack-write", 0x138); ++ writel(0x200, g->rgx + 0x138); ++ /* Ordered same-device read fences the single write. */ ++ g5p_mmu_progress(g, "release-ack-fence", 0x130); ++ after = readl(g->rgx + 0x130); ++ g5p_mmu_record( ++ g, ++ "MMU release-ack mask=00000200 before=%08x after=%08x; notification only\n", ++ event, after); ++ } else { ++ g5p_mmu_record(g, ++ "MMU release-ack skipped=no-event before=%08x\n", ++ event); ++ } ++ g5p_fault_snapshot(g, "release-ack-after"); ++ return 0; ++} ++ ++static int g5p_boot_program_stream(struct g5p_diag *g, bool release) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ u64 reset; ++ int ret; ++ ++ if (b->armed && !release) ++ return -EALREADY; ++ ret = g5p_boot_irq_setup(g); ++ if (ret) ++ return ret; ++ g5p_boot_snapshot(g, release ? "before-release" : "before-arm", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ ; ++ ret = 0; ++ if (ret) ++ goto out; ++ /* Slave-port access remains forbidden for the rest of this boot. */ ++ g->reset_transition = true; ++ /* c9fa10: core request false, GPC 0. Establish an actual state edge; ++ * wake left GPC at 1, so writing 1 again did not prove FIFO execution. ++ * No R0 accesses between this point and the successful active-state poll. ++ */ ++ dev_info(g->dev, "G2 Fender park: core=0, GPC=0\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ goto out; ++ /* c9fc10: core request true before prepareBoot c9feb4. */ ++ dev_info(g->dev, "G2 Fender prepare: core=20, require GPC=0\n"); ++ ret = g5p_fender_hold(&io); ++ if (ret) ++ goto out; ++ dev_info(g->dev, ++ "G2 init stream: master boot, release=%u; FIFO reset\n", ++ release); ++ ret = g5p_fender_write(g, 0x10, 0); ++ if (ret) ++ goto out; ++ /* setMode(3), after Fender prepare cleared +10, before firmware runs. ++ * Both arm and release execute prepare; never reapply after ALIVE. ++ */ ++ ; ++ ; ++ ; ++ g5p_fender_write(g, 0, 0x40000000); ++ ; ++ ; ++ b->fifo = 0x10000; ++ b->fifo_error = 0; ++ ; ++ ; ++ g5p_fifo64(g, 0x100, ++ G5P_BOOT_RESET1 | (0 ? G5P_BOOT_R46_RESET_BITS : 0)); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x00e7fe071ffffc1dULL); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x100000000ULL); ++ ; ++ ; ++ g5p_fifo32(g, 0xbf8, 1, 0xff); ++ ; ++ ; ++ g5p_fifo32(g, 0xe140, 0, 0xff); ++ ; ++ ; ++ g5p_fifo64(g, 0xe148, b->catalogue.dma >> 12); ++ ; ++ ; ++ g5p_fifo64(g, 0xe200, 3); ++ ; ++ ; ++ g5p_fifo64(g, 0xe208, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fifo64(g, 0xe210, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fender_write(g, 0x2010, 0xbf); ++ ; ++ ; ++ g5p_fender_write(g, 0x2014, 0x233); ++ ; ++ ; ++ g5p_fifo64(g, 0xe160, b->zero.dma); ++ ; ++ ; ++ g5p_fifo64(g, 0xbb8, 0); ++ ; ++ ; ++ if (release) { ++ g5p_fifo64(g, 0x100, 0); ++ ; ++ } ++ g5p_fifo_word(g, 0xffff0000); ++ ; ++ ; ++ ret = b->fifo_error; ++ if (ret) ++ goto out; ++ dma_wmb(); ++ dev_info( ++ g->dev, ++ "G2 FIFO stored and verified: %u words (not execution proof)\n", ++ (b->fifo - 0x10000) / 4); ++ if (release) { ++ WRITE_ONCE(b->released, true); ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_FIFO, G5P_R73B_NONE, 0); ++ ret = 0; ++ if (ret) ++ goto out; ++ } ++ /* c9f954: core request false, then GPC 1. */ ++ dev_info(g->dev, "G2 Fender execute: core=0, GPC=1 (from 0)\n"); ++ if (release && !(b->r73b && 1)) { ++ g5p_perf_seed_stamp( ++ b, ++ 4); /* the non-r74e start: as close as this path gets */ ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ ret = release && b->r73b && 1 ? g5p_r74e_fender_start(g) : ++ release ? g5p_fender_start_observed_q(&io, g5p_release_observe, ++ b->perf_poweron ?: 1) : ++ g5p_fender_start(&io); ++ if (release) ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_GPC, G5P_R73B_NONE, ret); ++ if (ret) ++ goto out; ++ /* r74e: after GPC1 is active, wait for ALIVE without optional MMIO. */ ++ if (release && b->r73b && 1) ++ return 0; ++ usleep_range(10, 20); ++ g5p_boot_snapshot(g, release ? "after-release" : "after-arm", ++ release ? G5P_READBACK_WRAPPER_ONLY : ++ G5P_READBACK_RESET_HELD); ++ ; ++ /* Only the fault-status subset above is sampled after release. */ ++ if (release) ++ return g5p_release_ack(g); ++ if (!b->rgx_valid) ++ return -EIO; ++ reset = g5p_snapshot64(b, 0x100); ++ dev_info(g->dev, "G2 verify reset actual=%016llx expected=%016llx\n", ++ reset, 0x100000000ULL); ++ dev_info( ++ g->dev, ++ "G2 expected catalogue=%llx scratch=%llx SLC=3 (not read: state safety)\n", ++ (u64)b->catalogue.dma >> 12, (u64)b->zero.dma); ++ if (reset != 0x100000000ULL) ++ return -EIO; ++ dev_info( ++ g->dev, ++ "G2 reset-held gate verified; remaining FIFO effects await firmware event\n"); ++ b->armed = true; ++ return 0; ++out: ++ /* A slow r74e epoch is scoreable without another unsafe MMIO read. */ ++ if (release && b->r73b && 1 && READ_ONCE(b->r73b->early_power_abort)) ++ return ret; ++ /* Wrapper-only snapshot if GPC remains off/busy. */ ++ g5p_boot_snapshot(g, "transition-or-FIFO-failed", ++ G5P_READBACK_WRAPPER_ONLY); ++ return ret; ++} ++ ++static int g5p_boot_arm(struct g5p_diag *g) ++{ ++ return g5p_boot_program_stream(g, false); ++} ++ ++static int g5p_boot_load(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off; ++ ++ /* c9f1bc/ca43b0: segment and LDR configuration pairs are supplied in ++ * mapped RAM. Polling MSLVCTRL1 in the reset-held state locks this SoC. ++ */ ++ for (off = 0; off < G5P_IMAGE_SIZE; off += G5P_GPU_PAGE) ++ if (memcmp(b->image[off / G5P_GPU_PAGE].cpu, ++ g->image.data + off, G5P_GPU_PAGE)) ++ return -EBADMSG; ++ dma_wmb(); ++ b->copied = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 RAM load verified bytes=%x segment_pairs=13 LDR_pairs=27; no slave-port MMIO\n", ++ G5P_IMAGE_SIZE); ++ return 0; ++} ++ ++static int g5p_boot_release(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u64 seed_ts = 0; ++ int ret; ++ ++ if (b->released) ++ return -EALREADY; ++ /* recovery108: fail closed before the release touches anything. */ ++ ret = g5p_ds_hook_release_gate(g); ++ if (ret) ++ return ret; ++ g5p_r73b_release_entry(g); ++ if (b->r73b && 1) { ++ ret = g5p_r73e_prepare(g); ++ if (ret) ++ return ret; ++ } ++ dev_info(g->dev, ++ "G2 release: META master boot; no GPU job submitted\n"); ++ /* Reproduce the complete original stream, including its final reset release. ++ * Do not assume the experimental arm stage preserves a FIFO read cursor. ++ */ ++ /* boot_gpu_perf_seed 3: the sampler's last-sample time = now, as close to the release as the ++ * host gets (perf-envelope.h); the firmware divides (now - policy+0x8f8) by 24000 * policy+4. ++ */ ++ if (b->perf_seed_mode >= 3) { ++ seed_ts = arch_timer_read_counter(); ++ g5p_perf_seed_stamp(b, 3); ++ } ++ ret = g5p_boot_program_stream(g, true); ++ if (seed_ts) { ++ u64 end = arch_timer_read_counter(), ++ rate = arch_timer_get_rate(); ++ ++ /* always printed on a seed 3/4 boot, before ALIVE is awaited */ ++ dev_emerg( ++ g->dev, ++ "GPUPERF seed mode=%u stamp=%llx release_to_gpc1_us=%llu stamp_to_gpc1_us=%llu " ++ "gpc1_to_stream_end_us=%llu stamp_to_stream_end_us=%llu period_us=16000 " ++ "before_alive=1 end=gpuperf\n", ++ b->perf_seed_mode, b->seed_stamp, ++ b->seed_gpc1 ? ++ div_u64((b->seed_gpc1 - seed_ts) * 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? div_u64((b->seed_gpc1 - b->seed_stamp) * ++ 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? ++ div_u64((end - b->seed_gpc1) * 1000000ULL, ++ rate) : ++ 0, ++ div_u64((end - b->seed_stamp) * 1000000ULL, rate)); ++ } ++ g5p_r73b_release_exit(g, ret); ++ g5p_ds_hook_release(g, ret); ++ if (ret) ++ return ret; ++ if (b->r73b && 1) ++ return g5p_r73c_fast_control(g); ++ return 0; ++} ++ ++static int g5p_boot_events_once(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 3 * HZ; ++ u64 submit_wait_start = 0; ++ u32 pass = 0, submit_wait_passes = 0; ++ int ret; ++ ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_EVENTS, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->alive)) { ++ if (b->r73b && 1) { ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ if (ret == -EAGAIN) ++ submit_wait_start = ++ ktime_get_mono_fast_ns(); ++ while (ret == -EAGAIN && ++ ktime_get_mono_fast_ns() - ++ submit_wait_start < ++ 100ULL * NSEC_PER_MSEC) { ++ usleep_range(50, 100); ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ submit_wait_passes++; ++ } ++ if (submit_wait_passes) ++ dev_info( ++ g->dev, ++ "R73C alive-submit synchronize wait_ns=%llu passes=%u ret=%d submitted=%u end=r73c\n", ++ ktime_get_mono_fast_ns() - ++ submit_wait_start, ++ submit_wait_passes, ret, ++ READ_ONCE( ++ b->control_submitted)); ++ if (ret) ++ return ret; ++ } ++ dev_info( ++ g->dev, ++ "G2 ALIVE type=0 word1=%08x events=%u IRQ174=%d\n", ++ b->alive_word, b->events, ++ atomic_read(&b->irqs)); ++ g5p_r41_bridge_firmware(g); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, ++ "G2 no alive event: events=%u IRQ174=%d; preserve baseline\n", ++ b->events, atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++/* recovery108: every ALIVE wait (release fast path and the events command) is an ALIVE event. */ ++static int g5p_boot_events(struct g5p_diag *g) ++{ ++ int ret = g5p_boot_events_once(g); ++ ++ /* wake112: no ALIVE wait ends with an R74E proof still owed (r74e112.h). */ ++ if (g5p_r74e_close(g)) { ++ dev_emerg( ++ g->dev, ++ "R74E112 close gen=%u why=untaken ret=%d end=r74e112\n", ++ g->boot->r73b->early_proof_gen, ret); ++ if (!ret) ++ ret = -EHOSTDOWN; ++ } ++ g5p_ds_hook_alive(g, ret); ++ return ret; ++} ++ ++static int g5p_boot_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b && b->stopped) ++ return -ESHUTDOWN; ++ if (sysfs_streq(command, "map")) ++ return g5p_boot_maps(g); ++ /* Read-only diagnostics remain available after a latched boot failure. */ ++ if (b && g->powered && g->fender_ready && g->id_checked) { ++ if (sysfs_streq(command, "snapshot")) { ++ g5p_boot_snapshot(g, "explicit", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ return 0; ++ } ++ if (sysfs_streq(command, "fifo")) ++ return g5p_boot_fifo_dump(g); ++ } ++ if (!boot_experiment || !b || !b->maps_ready || b->failed) ++ return -EPERM; ++ if (sysfs_streq(command, "arm")) { ++ ret = g5p_boot_arm(g); ++ } else if (sysfs_streq(command, "load") && b->armed && !b->released && ++ !b->copied) { ++ ret = g5p_boot_load(g); ++ g5p_ds_hook_load(g, ret); ++ } else if (sysfs_streq(command, "release") && b->copied) { ++ ret = g5p_boot_release(g); ++ } else if (sysfs_streq(command, "events") && b->released) { ++ ret = g5p_boot_events(g); ++ g5p_r73b_events_exit(g, ret); ++ } else { ++ return -EPERM; ++ } ++ if (ret && ret != -EALREADY) ++ b->failed = true; ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +new file mode 100644 +index 0000000000..8cd7085730 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -0,0 +1,567 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared private bring-up implementation for mutually exclusive frontends. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++/* perf60 clock: which GPC states license an R0 access (a bell, a TLB pass, a register sample). */ ++/* release8 default */ ++ ++/* The GPC is powered and not mid-transition: state 1 only, or (gpc_any_pstate) any nonzero ++ * pstate nibble with no request in flight. A parked GPC (nibble 0) never qualifies. ++ */ ++static inline bool g5p_gpc_ok(u32 raw) ++{ ++ ; ++ ; ++ return !(raw & 0x10) && (raw & 0xf); ++} ++ ++/* The pure headers (readback.h, r74e112.h, render-user-wake.h, dirtystart108.h) judge the GPC state ++ * through G5P_GPC_ON(); their default is the old == 1, for the host tests. The kernel takes the rule. ++ */ ++#define G5P_GPC_ON(raw) g5p_gpc_ok(raw) ++ ++#include "pack.h" ++#include "work-layout.h" ++#include "mmio-licence.h" ++#include "gpu-oc.h" ++#include ++#include "readback.h" ++ ++/* nopatch: the TYPED_V3 boot path (r74/r74f/r74g, meta6c8=1) on Apple's original image. The release8 ++ * default since its device pass (2026-11-03, release8-nopatch a10a8556b9): an explicit firmware patch ++ * request (boot_r73f_case 1..4, or the r75/r79 cases that ride on it) takes the patched path instead, ++ * so the patched fallback is boot_r73f_case 1 alone (g5p_fw_unpatched_want()). ++ */ ++/* release8 default */ ++ ++/* The effective nopatch latch: the parameter, unless a patched image is asked for. */ ++static u32 g5p_fw_unpatched_want(u32 param, u32 r73f, u32 r75, u32 r79) ++{ ++ return param == 1 && !r73f && !r75 && !r79; ++} ++ ++#define G5P_MSLVDATAX 0x0a00 ++#define G5P_MSLVDATAT 0x0a08 ++#define G5P_MSLVCTRL0 0x0a10 ++#define G5P_MSLVCTRL1 0x0a18 ++#define G5P_MSLVIRQSTATUS 0x0ac8 ++#define G5P_SP_READY (BIT(24) | BIT(26)) ++#define G5P_FW_NAME "apple/agx-g5p-c0.ldr" ++ ++struct g5p_boot; ++ ++struct g5p_diag { ++ struct g5p_diagnostics *diagnostics; ++ struct g5p_boot *boot; ++ ++ spinlock_t slot_lock; ++ DECLARE_BITMAP(user_slots, 256); ++ struct device *dev; ++ void __iomem *window[2]; ++ void __iomem *rgx; ++ struct regmap *pmgr; ++ /* Serializes commands and all hardware access. */ ++ struct mutex lock; ++ bool powered; ++ bool loaded; ++ ++ struct g5p_ldr_image image; ++ bool fender_ready; ++ bool id_checked; ++ int last_result; ++ u32 words_written; ++ u32 core_id, core_revision; ++ u32 last_sp_addr; ++ bool reset_transition; ++ void (*event_notify)(void *cookie, const u32 words[11]); ++ void *event_cookie; ++ /* The GPU register licence (mmio.inc, docs/gpu/experiments/mmio-access.md §4). mmio_lock is ++ * a leaf: taken inside event_lock where both are held, never the reverse, nothing inside. ++ * mmio_off forbids R1/R0 through the accessor from before any PMGR gfx gate until Fender is ++ * up again (g5p_mmio_gate(), cleared by g5p_fender_wake_once()). ++ */ ++ spinlock_t mmio_lock; ++ bool mmio_off; ++ u32 mmio_max_pstate; ++ struct g5p_mmio_self *mmio_self; ++}; ++ ++/* Before every PMGR gfx gate (boot R33 off, dirty-start, kexec quiesce and in-boot recovery ++ * cycles): no accessor MMIO from here until g5p_fender_wake_once() brings Fender up again. ++ * Apple clears fender+0xc inside setPlatformPower(0), before its gate (G5). ++ */ ++static void g5p_mmio_gate(struct g5p_diag *g) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = true; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++} ++ ++/* gpu-recovery108: retire predicates (teardown.inc) and dirty-start hooks (dirtystart108.inc). */ ++static bool g5p_r8_r1_allowed(void); ++static bool g5p_r8_is_retired(void); ++static void g5p_ds_hook_probe(struct g5p_diag *g); ++static void g5p_ds_hook_power(struct g5p_diag *g); ++static int g5p_ds_hook_reset(struct g5p_diag *g); ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret); ++static int g5p_ds_hook_release_gate(struct g5p_diag *g); ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret); ++/* wake112: the reset generation of a deferred R74E proof (dirtystart108.inc). */ ++static u32 g5p_ds_gen(void); ++ ++#include "diagnostics.inc" ++ ++/* The GPU ADT clock/power gate124 resolves to ps-regs[4]+index8*8. ++ * Read via its existing syscon, sharing the provider mapping without writes. ++ */ ++static void g5p_power_observer_init(struct g5p_diag *g) ++{ ++ struct device_node *domain, *parent; ++ u32 offset; ++ ++ domain = of_parse_phandle(g->dev->of_node, "power-domains", 0); ++ if (!domain) ++ return; ++ parent = of_get_parent(domain); ++ if (of_device_is_compatible(domain, "apple,s8000-pmgr-pwrstate") && ++ !of_property_read_u32(domain, "reg", &offset) && ++ offset == 0x80340 && parent && ++ of_device_is_compatible(parent, "apple,s8000-pmgr")) { ++ g->pmgr = syscon_node_to_regmap(parent); ++ if (IS_ERR(g->pmgr)) ++ g->pmgr = NULL; ++ } ++ of_node_put(parent); ++ of_node_put(domain); ++} ++ ++/* pmgr-pwrstate.c masks; history flags are sticky, not current clock state. */ ++static void g5p_power_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) { ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx unavailable: provider not resolved\n", ++ stage); ++ return; ++ } ++ dev_info(g->dev, "POWER %s reading PMGR gfx20e080340 gate124\n", stage); ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (ret) { ++ dev_info(g->dev, "POWER %s PMGR gfx read error=%d\n", stage, ++ ret); ++ return; ++ } ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx20e080340=%08x target=%x actual=%x auto=%u auto_state=%x " ++ "min=%x reset=%u parent_off=%u disabled=%u; current outer-domain fields only\n", ++ stage, raw, raw & 0xf, (raw >> 4) & 0xf, (raw >> 28) & 1, ++ (raw >> 24) & 0xf, (raw >> 16) & 0xf, raw >> 31, ++ (raw >> 11) & 1, (raw >> 10) & 1); ++ dev_info( ++ g->dev, ++ "POWER %s PMGR history was_clkgated=%u was_pwrgated=%u; sticky history, not " ++ "current state\n", ++ stage, (raw >> 9) & 1, (raw >> 8) & 1); ++} ++ ++static int g5p_power(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->powered) ++ return 0; ++ dev_info(g->dev, "request gfx power domain\n"); ++ ret = pm_runtime_resume_and_get(g->dev); ++ if (ret < 0) ++ return ret; ++ g->powered = true; ++ dev_info(g->dev, "gfx runtime PM active\n"); ++ g5p_ds_hook_power(g); ++ return 0; ++} ++ ++#include "power-r33.inc" ++#include "power-r41.inc" ++ ++static bool boot_gfx_off_done; ++ ++static int g5p_fender_wake_once(struct g5p_diag *g) ++{ ++ void __iomem *fender = g->window[1]; ++ u32 value; ++ int ret; ++ ++ ret = g5p_power(g); ++ if (ret) ++ return ret; ++ /* recovery108 mode 3: the one bounded incoming cycle, before any GPU MMIO. */ ++ ret = g5p_ds_hook_reset(g); ++ if (ret) ++ return ret; ++ /* R33: once per boot, before the first Fender write (no GPU MMIO yet). */ ++ ; ++ ret = g5p_r41_bridge_prepare(g); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "W R1+10000=ffff0000 (Apple ...6c93fe8)\n"); ++ writel(0xffff0000, fender + 0x10000); ++ dev_info(g->dev, "W R1+00000=40000000 (Apple ...6c93ff0)\n"); ++ writel(0x40000000, fender); ++ dev_info(g->dev, "W R1+01000=00000011 (Apple ...6c93ff8)\n"); ++ writel(0x11, fender + 0x1000); ++ dev_info(g->dev, "poll R1+01000 bit4 clear\n"); ++ ret = readl_poll_timeout(fender + 0x1000, value, !(value & BIT(4)), 10, ++ 100000); ++ dev_info(g->dev, "Fender wake status=%08x result=%d\n", value, ret); ++ if (ret) ++ return ret; ++ usleep_range(10, 20); ++ g->fender_ready = true; ++ { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = false; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++ } ++ return 0; ++} ++ ++static int g5p_fender_wake(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->fender_ready) ++ return 0; ++ ret = g5p_fender_wake_once(g); ++ g5p_ds_hook_wake(g, ret); ++ return ret; ++} ++ ++static int g5p_sp_wait(struct g5p_diag *g) ++{ ++ u32 value; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = readl_poll_timeout(g->rgx + G5P_MSLVCTRL1, value, ++ (value & G5P_SP_READY) == G5P_SP_READY, 10, ++ 100000); ++ if (ret) ++ dev_err(g->dev, "slave port timeout CTRL1=%08x words=%u\n", ++ value, g->words_written); ++ if (ret) ++ dev_err(g->dev, "slave target=%08x\n", g->last_sp_addr); ++ return ret; ++} ++ ++static int g5p_meta_read(struct g5p_diag *g, u32 addr, u32 *out) ++{ ++ int ret; ++ ++ g->last_sp_addr = addr; ++ ret = g5p_sp_wait(g); ++ ++ if (ret) ++ return ret; ++ writel(addr | 1, g->rgx + G5P_MSLVCTRL0); ++ readl(g->rgx + G5P_MSLVCTRL0); ++ ret = g5p_sp_wait(g); ++ if (ret) ++ return ret; ++ *out = readl(g->rgx + G5P_MSLVDATAX); ++ return 0; ++} ++ ++static int g5p_firmware(struct g5p_diag *g, bool prepare) ++{ ++ static const u8 expected[SHA256_DIGEST_SIZE] = { ++ 0x33, 0xae, 0x20, 0xcb, 0xbe, 0x24, 0x98, 0xe4, ++ 0xf1, 0x19, 0xba, 0x4e, 0x9a, 0x55, 0x46, 0x7e, ++ 0x4e, 0x01, 0x92, 0xd4, 0xc8, 0x6d, 0x5b, 0xf1, ++ 0x90, 0x3a, 0xdc, 0xe6, 0x3f, 0xfa, 0x5d, 0xca, ++ }; ++ /* SHA-256 of the packed image of that LDR, unmodified (nopatch's receipt) */ ++ static const u8 g5p_fw_packed_sha[SHA256_DIGEST_SIZE] = { ++ 0x48, 0xea, 0x03, 0x14, 0x08, 0x23, 0x96, 0x23, ++ 0xd8, 0x6f, 0x79, 0xb8, 0xa5, 0xda, 0x55, 0x7e, ++ 0x25, 0x3d, 0xb1, 0x6f, 0xb6, 0x23, 0x47, 0x95, ++ 0xe8, 0xed, 0xba, 0xa5, 0xcd, 0x9b, 0xb9, 0x1a, ++ }; ++ const struct firmware *fw; ++ u8 digest[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ ret = request_firmware_direct(&fw, G5P_FW_NAME, g->dev); ++ if (ret) ++ return ret; ++ sha256(fw->data, fw->size, digest); ++ if (fw->size != 119112 || memcmp(digest, expected, sizeof(expected))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, NULL, NULL); ++ if (ret) ++ goto out; ++ if (prepare && !g->loaded) { ++ g->image.data = kvzalloc(G5P_IMAGE_SIZE, GFP_KERNEL); ++ if (!g->image.data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, g5p_pack_record, ++ &g->image); ++ if (!ret && g->image.config_count != G5P_CONFIG_WRITES) ++ ret = -EINVAL; ++ if (!ret) { ++ sha256(g->image.data, G5P_IMAGE_SIZE, digest); ++ if (memcmp(digest, g5p_fw_packed_sha, sizeof(digest))) ++ ret = -EBADMSG; ++ } ++ if (ret) { ++ kvfree(g->image.data); ++ memset(&g->image, 0, sizeof(g->image)); ++ goto out; ++ } ++ g->loaded = true; ++ dev_info( ++ g->dev, ++ "NOPATCH packed apple_original=1 packed_sha256=%32phN end=nopatch\n", ++ digest); ++ } ++out: ++ release_firmware(fw); ++ return ret; ++} ++ ++static int g5p_sample(struct g5p_diag *g) ++{ ++ u32 value; ++ unsigned int thread, reg; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ if (!g->rgx) ++ return -EINVAL; ++ dev_info(g->dev, "RGX CTRL1=%08x IRQSTATUS=%08x\n", ++ readl(g->rgx + G5P_MSLVCTRL1), ++ readl(g->rgx + G5P_MSLVIRQSTATUS)); ++ for (thread = 0; thread < 2; thread++) { ++ for (reg = 0; reg <= 0x10; reg += 0x10) { ++ ret = g5p_meta_read( ++ g, 0x04800000 + thread * 0x1000 + reg, &value); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "META T%u %s=%08x\n", thread, ++ reg ? "STATUS" : "ENABLE", value); ++ } ++ } ++ return 0; ++} ++ ++static int g5p_read_id(struct g5p_diag *g) ++{ ++ u32 words[2], version; ++ static const u32 offsets[] = { 0x20, 0x18 }; ++ unsigned int i; ++ ++ if (!g->fender_ready || READ_ONCE(g->reset_transition)) ++ return -EPERM; ++ g->id_checked = false; ++ for (i = 0; i < ARRAY_SIZE(offsets); i++) { ++ dev_info(g->dev, "R R0+%05x\n", offsets[i]); ++ words[i] = readl(g->rgx + offsets[i]); ++ dev_info(g->dev, "R0+%05x=%08x\n", offsets[i], words[i]); ++ } ++ g->core_revision = words[0]; ++ g->core_id = words[1]; ++ if ((!words[0] && !words[1]) || words[0] == ~0U || words[1] == ~0U) ++ return -ENODEV; ++ /* Match Apple readChipInfo at ...6ca644c before deeper META accesses. */ ++ version = (words[0] >> 8) & 0xff; ++ if ((words[0] & 0xff0000) != 0x90000 || ++ (version != 7 && version != 10 && version != 13 && version != 15)) ++ return -ENODEV; ++ g->id_checked = true; ++ return 0; ++} ++ ++#include "boot.inc" ++#include "mmio.inc" ++#include "kick.inc" ++#include "lifecycle.inc" ++#include "teardown.inc" ++#include "dirtystart108.inc" ++#include "recover.inc" ++#include "transport-budget.inc" ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++ ++static ssize_t result_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ return sysfs_emit( ++ buf, ++ "last=%d loaded=%u maps=%u armed=%u copied=%u released=%u alive=%u " ++ "control_complete=%u failed=%u\n", ++ READ_ONCE(g->last_result), READ_ONCE(g->loaded), ++ b && b->maps_ready, b && b->armed, b && b->copied, ++ b && b->released, b && b->alive, b && b->control_complete, ++ b && b->failed); ++} ++static DEVICE_ATTR_RO(result); ++ ++/* Cached raw words from the last explicit/automatic snapshot: no GPU MMIO. */ ++ ++static ssize_t command_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ int ret; ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (sysfs_streq(buf, "td-kexec")) ++ return g5p_td_command(g, buf) ?: count; ++ if (g5p_r8_is_retired()) ++ return -ESHUTDOWN; ++ guard(mutex) ++ (&g->lock); ++ if (g->boot && (g->boot->stopped || g->boot->user_compute_owned)) ++ return -EBUSY; ++ if (sysfs_streq(buf, "validate") || sysfs_streq(buf, "prepare")) ++ ret = g5p_firmware(g, sysfs_streq(buf, "prepare")); ++ else if (sysfs_streq(buf, "wake")) ++ ret = g5p_fender_wake(g); ++ else if (sysfs_streq(buf, "id")) ++ ret = g5p_read_id(g); ++ else if (sysfs_streq(buf, "sample")) ++ ret = g->id_checked ? g5p_sample(g) : -EPERM; ++ else if (sysfs_streq(buf, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(buf, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(buf, "control-wait")) ++ ret = g5p_control_wait(g); ++ else if (sysfs_streq(buf, "map") || sysfs_streq(buf, "arm") || ++ sysfs_streq(buf, "load") || sysfs_streq(buf, "release") || ++ sysfs_streq(buf, "events")) ++ ret = g5p_boot_command(g, buf); ++ else ++ return -EOPNOTSUPP; ++ g->last_result = ret; ++ dev_info(dev, "command %.*s result=%d\n", (int)count, buf, ret); ++ return ret ? ret : count; ++} ++static DEVICE_ATTR_WO(command); ++ ++static struct attribute *g5p_attrs[] = { ++ &dev_attr_command.attr, ++ &dev_attr_result.attr, ++ &dev_attr_recovery_status.attr, ++ NULL, ++}; ++static const struct attribute_group g5p_group = { .attrs = g5p_attrs }; ++__ATTRIBUTE_GROUPS(g5p); ++ ++static int g5p_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct g5p_diag *g; ++ unsigned int i; ++ ++ g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL); ++ if (!g) ++ return -ENOMEM; ++ g->dev = dev; ++ if (g5p_diagnostics_init(g)) ++ return -ENOMEM; ++ mutex_init(&g->lock); ++ spin_lock_init(&g->slot_lock); ++ spin_lock_init(&g->mmio_lock); ++ g->mmio_max_pstate = 1; ++ bitmap_set(g->user_slots, 0, 7); ++ for (i = 0; i < 2; i++) { ++ g->window[i] = devm_platform_ioremap_resource(pdev, i); ++ if (IS_ERR(g->window[i])) ++ return PTR_ERR(g->window[i]); ++ } ++ g->rgx = g->window[0]; ++ g5p_power_observer_init(g); ++ platform_set_drvdata(pdev, g); ++ if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40))) ++ return -ENODEV; ++ pm_runtime_enable(dev); ++ g5p_td_attach(g); ++ g5p_ds_hook_probe(g); ++ ; ++ dev_info(dev, "mapped both ADT windows; awaiting explicit command\n"); ++ return 0; ++} ++ ++static void g5p_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ /* With teardown on, keep the PM reference: an outgoing PMGR off is unvalidated. */ ++ bool keep_power; ++ ++ ; ++ keep_power = g5p_td_remove(g); ++ if (g->powered && !keep_power) { ++ g5p_mmio_gate(g); ++ pm_runtime_put_sync(g->dev); ++ } ++ pm_runtime_disable(g->dev); ++ kvfree(g->image.data); ++} ++ ++/* Registration itself must not claim a device owned by another frontend. */ ++static int g5p_frontend_available(void) ++{ ++ struct device *dev; ++ int ret; ++ ++ if (!of_machine_is_compatible("apple,j71t")) ++ return -ENODEV; ++ dev = bus_find_device_by_name(&platform_bus_type, NULL, ++ "201000000.gpu"); ++ if (!dev) ++ return -ENODEV; ++ device_lock(dev); ++ ret = dev->driver ? -EBUSY : 0; ++ device_unlock(dev); ++ put_device(dev); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/diagnostics.inc b/drivers/gpu/apple-g5p/diagnostics.inc +new file mode 100644 +index 0000000000..184ce954b2 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/diagnostics.inc +@@ -0,0 +1,181 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Cached append-only text. Readers never take the command lock or touch MMIO. ++ * Preserve earliest records on overflow, expose dropped bytes explicitly. ++ */ ++#define G5P_BLOB_SIZE SZ_4M ++#define G5P_MMU_HISTORY_SIZE SZ_32K ++ ++struct g5p_diagnostics { ++ spinlock_t lock; ++ u32 used, history_used; ++ u64 dropped, history_dropped; ++ char stage[40]; ++ u32 next_register; ++ bool reading; ++ char history[G5P_MMU_HISTORY_SIZE]; ++ char blob[G5P_BLOB_SIZE]; ++}; ++ ++static void g5p_diagnostics_free(void *data) ++{ ++ kvfree(data); ++} ++ ++static int g5p_diagnostics_init(struct g5p_diag *g) ++{ ++ g->diagnostics = kvzalloc(sizeof(*g->diagnostics), GFP_KERNEL); ++ if (!g->diagnostics) ++ return -ENOMEM; ++ spin_lock_init(&g->diagnostics->lock); ++ return devm_add_action_or_reset(g->dev, g5p_diagnostics_free, ++ g->diagnostics); ++} ++ ++static void g5p_diagnostic_append(struct g5p_diag *g, bool history, ++ const char *text, size_t len) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ u32 *used = history ? &d->history_used : &d->used; ++ u64 *dropped = history ? &d->history_dropped : &d->dropped; ++ u32 capacity = history ? G5P_MMU_HISTORY_SIZE : G5P_BLOB_SIZE; ++ char *dst = history ? d->history : d->blob; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ /* Keep complete lines; once full, never replace earlier evidence. */ ++ if (*dropped || len > capacity - *used) { ++ *dropped += len; ++ } else { ++ memcpy(dst + *used, text, len); ++ *used += len; ++ } ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_blob_printf(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ char line[512]; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, false, line, len); ++} ++ ++static void g5p_blob_hex(struct g5p_diag *g, const char *name, const u8 *data, ++ u32 len) ++{ ++ for (u32 off = 0; off < len; off += 16) { ++ char text[80]; ++ ++ hex_dump_to_buffer(data + off, min_t(u32, len - off, 16), 16, 4, ++ text, sizeof(text), false); ++ g5p_blob_printf(g, "%s: %04x: %s\n", name, off, text); ++ } ++} ++ ++/* R1+1000 uses synthetic address11000; other offsets denote R0. */ ++static void g5p_mmu_progress(struct g5p_diag *g, const char *stage, u32 next) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ strscpy(d->stage, stage, sizeof(d->stage)); ++ d->next_register = next; ++ d->reading = true; ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_mmu_record(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ dev_info(g->dev, "%s", line); ++} ++ ++/* The routine per-job and per-event lines (a job's publish, wake and success receipt, the firmware's ++ * alive and stamp events). At desktop job rates they are hundreds of lines a second, more than netconsole ++ * can send: its queue grew to 700 MB of unreclaimable slab (release8, 55 min). Faults, losses, ++ * recoveries, terminal latches and every failed job keep their own levels and always print. ++ * job_log 0 dev_dbg (dynamic debug: off unless enabled in dynamic_debug/control) ++ * 1 KERN_DEBUG: in dmesg for the parsers (RENDER-USER job=, RENDER publish), below the ++ * default console loglevel, so not on a console or netconsole ++ * 2 KERN_INFO as before (short netconsole traces only: floods at desktop job rates) ++ */ ++static unsigned int job_log = 0; ++module_param(job_log, uint, 0600); ++MODULE_PARM_DESC( ++ job_log, ++ "default0 routine per-job/per-event lines are dev_dbg; 1 KERN_DEBUG (dmesg, no " ++ "console); 2 KERN_INFO as before (floods netconsole)"); ++ ++#define g5p_job_printk(dev, fmt, ...) \ ++ do { \ ++ switch (READ_ONCE(job_log)) { \ ++ case 0: \ ++ dev_dbg(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ case 1: \ ++ dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ default: \ ++ dev_info(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ } \ ++ } while (0) ++ ++/* g5p_mmu_record for a routine per-job line: the blob keeps it, the console sees it only at job_log 2. */ ++static __printf(2, 3) __maybe_unused ++ void g5p_mmu_record_job(struct g5p_diag *g, const char *fmt, ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ g5p_job_printk(g->dev, "%s", line); ++} ++ ++/* A routine event's raw words at the job_log level. */ ++static __maybe_unused void g5p_job_hex(const char *prefix, const void *buf, ++ size_t len) ++{ ++ switch (READ_ONCE(job_log)) { ++ case 0: ++ print_hex_dump_debug(prefix, DUMP_PREFIX_NONE, 16, 4, buf, len, ++ false); ++ break; ++ case 1: ++ print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ default: ++ print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ } ++} +diff --git a/drivers/gpu/apple-g5p/dirtystart108.h b/drivers/gpu/apple-g5p/dirtystart108.h +new file mode 100644 +index 0000000000..fcf7432fa8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.h +@@ -0,0 +1,454 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DIRTYSTART108_H ++#define G5P_DIRTYSTART108_H ++ ++#include "bits-compat.h" ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * G5P incoming dirty-start state machine (series gpu-recovery108, patch 0007). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 4. ++ * ++ * The bootstrap is stepped by userspace (map, load, release, events, ++ * control), so this is an event-driven machine, not one run: the glue feeds ++ * one event per driver step with the step's result and RAM-only inputs, and ++ * asks the machine at two gates (before release, at the DRM claim). It ++ * assumes nothing pristine: the origin of the power state is classified from ++ * the kexec flag, the requested incoming PMGR cycle and the PMGR word, and ++ * every step's own result is evidence. ++ * ++ * PROBE origin inputs: booted from kexec, kexec_gpu_cycle requested ++ * POWER the PMGR gfx word, read once after the PM reference (PMGR read) ++ * RESET the bounded incoming power cycle (g5p_r33_gfx_off(), one attempt ++ * per boot, only in mode 3, only before the first wrapper wake) ++ * WAKE the Fender wake result ++ * MAPS fresh shared state: every ring control word in range (after the wake: ++ * map needs the core id, read through the woken wrapper) ++ * LOAD the firmware image memcmp of the RAM copy (b->copied) ++ * RELEASE the reset-stream/release result, the R74E power result ++ * ALIVE the first ALIVE wait result and, when read, the first-ALIVE ++ * GPC word (r73e); 0x10/0x11 is the C3 closed gate ++ * CLAIM the DRM claim asks; answers from the ALIVE proof ++ * ++ * Modes (parameter dirty_start): ++ * 0 observe: receipts only, both gates open (combined107 behaviour) ++ * 1 gate (default): fail closed on evidence the driver itself produced (a ++ * failed or out-of-order step, a power abort, a GPC word other than ++ * 0x01 at the first ALIVE, a ring out of range); a claim needs ALIVE ++ * proven in the current reset generation. Equal to combined107 on every ++ * bootstrap that succeeds. ++ * 2 strict: also refuse release and claim for an origin that is not cold ++ * and not a witnessed cycle (warm-unproven, warm-uncycled, unknown), ++ * unless a reset succeeded in this generation ++ * 3 strict + reset: as 2, and the glue may spend the one RESET attempt ++ * on such an origin before the wrapper wake (destructive: PMGR write) ++ * ++ * Generations: gen starts at 1 at PROBE and steps once per RESET attempt. A ++ * reset voids every proof of the previous generation; ALIVE proven in an ++ * earlier generation never exposes. gen never decreases. ++ * ++ * Pure policy: no register access, no kernel API except scnprintf(). Each ++ * event emits exactly one DS108 line through the emit callback. ++ */ ++ ++enum g5p_ds_ev { ++ G5P_DSE_PROBE, ++ G5P_DSE_POWER, ++ G5P_DSE_RESET, ++ G5P_DSE_WAKE, ++ G5P_DSE_MAPS, ++ G5P_DSE_LOAD, ++ G5P_DSE_RELEASE, ++ G5P_DSE_ALIVE, ++ G5P_DSE_CLAIM, ++ G5P_DSE_RECOVER, /* an in-boot recovery cycled the gate after a proven epoch */ ++ G5P_DSE_N, ++}; ++ ++static const char *const g5p_ds_ev_name[G5P_DSE_N] = { ++ "probe", "power", "reset", "wake", "maps", ++ "load", "release", "alive", "claim", "recover", ++}; ++ ++enum g5p_ds_origin { ++ G5P_DSO_UNKNOWN, /* not probed, or the PMGR word could not be read */ ++ G5P_DSO_COLD, /* not booted from kexec */ ++ G5P_DSO_WARM_CYCLED, /* kexec, incoming cycle requested and witnessed (was_pwrgated) */ ++ G5P_DSO_WARM_UNPROVEN, /* kexec, cycle requested, no witness in the PMGR word */ ++ G5P_DSO_WARM_UNCYCLED, /* kexec, no incoming cycle requested */ ++ G5P_DSO_N, ++}; ++ ++static const char *const g5p_ds_origin_name[G5P_DSO_N] = { ++ "unknown", "cold", "warm-cycled", "warm-unproven", "warm-uncycled", ++}; ++ ++enum g5p_ds_why { ++ G5P_DSW_NONE, ++ G5P_DSW_PROTOCOL, /* a step out of order (its predecessor not ok in this gen) */ ++ G5P_DSW_WAKE, /* the wrapper wake failed */ ++ G5P_DSW_MAPS, /* map failed or a ring control word out of range */ ++ G5P_DSW_LOAD, /* the RAM image is not the firmware (memcmp) */ ++ G5P_DSW_RELEASE, /* release / reset stream failed */ ++ G5P_DSW_POWER_ABORT, /* the R74E power poll or the first-ALIVE gate aborted */ ++ G5P_DSW_GPC, /* first-ALIVE GPC word not 0x01 (0x10/0x11: C3) */ ++ G5P_DSW_ALIVE, /* no ALIVE, or the ALIVE wait failed */ ++ G5P_DSW_RESET, /* the incoming power cycle failed: power state unknown */ ++ G5P_DSW_N, ++}; ++ ++static const char *const g5p_ds_why_name[G5P_DSW_N] = { ++ "none", "protocol", "wake", "maps", "load", ++ "release", "power-abort", "gpc", "alive", "reset", ++}; ++ ++/* Gate answers (not latched): why a gate is closed right now. */ ++enum g5p_ds_gate { ++ G5P_DSG_OPEN, ++ G5P_DSG_FAILED, /* the fail-closed latch */ ++ G5P_DSG_NOT_PROVEN, /* no ALIVE proof in the current generation */ ++ G5P_DSG_ORIGIN, /* strict: origin neither cold nor a witnessed cycle */ ++ G5P_DSG_STATE, /* the claim snapshot contradicts the proof */ ++ G5P_DSG_N, ++}; ++ ++static const char *const g5p_ds_gate_name[G5P_DSG_N] = { ++ "open", "failed", "not-proven", "origin", "state", ++}; ++ ++#define G5P_DS_MODE_MAX 3 ++#define G5P_DS_GPC_ON 0x01u ++#define G5P_DS_PMGR_PWRGATED \ ++ BIT(8) /* sticky was_pwrgated history bit of 0x80340 */ ++ ++#define G5P_DSF_RINGS_BUSY \ ++ BIT(0) /* a ring not empty at map (data, not a failure) */ ++#define G5P_DSF_ALIVE_LATE \ ++ BIT(1) /* ALIVE first seen at the claim (no ALIVE event) */ ++#define G5P_DSF_RESET_RAN BIT(2) /* a RESET attempt ran this boot */ ++#define G5P_DSF_RESET_OK \ ++ BIT(3) /* ... and succeeded (gen of the success = gen) */ ++/* a second attempt or one after the wake was refused */ ++#define G5P_DSF_RESET_REFUSED BIT(4) ++#define G5P_DSF_DUP BIT(5) /* an event repeated (recorded, not a failure) */ ++#define G5P_DSF_PMGR_UNREAD BIT(6) /* the PMGR word could not be read */ ++ ++/* One flat input record; each event reads only its own fields (RAM or step results). */ ++struct g5p_ds_in { ++ bool kexec; /* PROBE: chosen linux,booted-from-kexec */ ++ bool cycle_requested; /* PROBE: apple_pmgr.kexec_gpu_cycle=1 on the cmdline */ ++ bool pmgr_ok; /* POWER: the PMGR read succeeded */ ++ u32 pmgr_raw; /* POWER: the gfx word */ ++ int ret; /* RESET/WAKE/MAPS/LOAD/RELEASE/ALIVE: the step's return */ ++ u32 rings_bad; /* MAPS: control words > 255 */ ++ u32 rings_busy; /* MAPS: rings not empty */ ++ int power_ret; /* RELEASE: R74E power result */ ++ u32 power_raw; /* RELEASE: its last R1+0x1000 sample */ ++ bool power_abort; /* RELEASE/ALIVE/CLAIM: r73b early_power_abort */ ++ bool gpc_valid; /* ALIVE: the r73e first-ALIVE word was read */ ++ u32 gpc; /* ALIVE: that word */ ++ bool alive, released, failed, ++ control_complete; /* ALIVE/CLAIM snapshot */ ++}; ++ ++struct g5p_ds { ++ u32 gen; /* reset generation, monotonic, 0 before PROBE */ ++ u32 seen, ok; /* event bitmasks, current generation */ ++ u32 flags; ++ enum g5p_ds_origin origin; ++ bool kexec, cycle; /* PROBE inputs, kept for POWER */ ++ u32 pmgr_raw; ++ u32 resets; /* RESET attempts this boot (at most one runs) */ ++ u32 reset_ok_gen; /* gen in which a RESET succeeded, 0 never */ ++ u32 alive_gen; /* gen in which ALIVE was proven, 0 never */ ++ bool failed; /* fail-closed latch (absorbing) */ ++ enum g5p_ds_why why; ++ u32 events, lines, claims, refused; ++ enum g5p_ds_ev last; ++ enum g5p_ds_gate last_gate; ++}; ++ ++static inline bool g5p_ds_ok(const struct g5p_ds *d, enum g5p_ds_ev e) ++{ ++ return d->ok & (1u << e); ++} ++ ++static inline void g5p_ds_fail(struct g5p_ds *d, enum g5p_ds_why why) ++{ ++ if (!d->failed) ++ d->why = why; ++ d->failed = true; ++} ++ ++static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool cycle, ++ bool pmgr_ok, u32 raw) ++{ ++ if (!pmgr_ok) ++ return G5P_DSO_UNKNOWN; ++ if (!kexec) ++ return G5P_DSO_COLD; ++ if (!cycle) ++ return G5P_DSO_WARM_UNCYCLED; ++ return (raw & G5P_DS_PMGR_PWRGATED) ? G5P_DSO_WARM_CYCLED : ++ G5P_DSO_WARM_UNPROVEN; ++} ++ ++/* The origin is trusted (cold, witnessed cycle) or a reset succeeded in this generation. */ ++static inline bool g5p_ds_origin_trusted(const struct g5p_ds *d) ++{ ++ return d->origin == G5P_DSO_COLD || d->origin == G5P_DSO_WARM_CYCLED || ++ (d->reset_ok_gen && d->reset_ok_gen == d->gen); ++} ++ ++/* Should the glue spend the one RESET attempt now (mode 3, before the wake)? */ ++static inline bool g5p_ds_reset_wanted(const struct g5p_ds *d, u32 mode) ++{ ++ return mode >= 3 && !d->failed && !d->resets && ++ !(d->seen & (1u << G5P_DSE_WAKE)) && ++ (d->seen & (1u << G5P_DSE_POWER)) && !g5p_ds_origin_trusted(d); ++} ++ ++/* The release gate (asked by the glue right before the reset stream runs). */ ++static inline enum g5p_ds_gate g5p_ds_release_gate(const struct g5p_ds *d, ++ u32 mode) ++{ ++ if (!mode) ++ return G5P_DSG_OPEN; ++ if (d->failed) ++ return G5P_DSG_FAILED; ++ if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ return G5P_DSG_ORIGIN; ++ return G5P_DSG_OPEN; ++} ++ ++/* The ALIVE proof the claim needs: proven in this generation, nothing failed since. */ ++static inline bool g5p_ds_proven(const struct g5p_ds *d) ++{ ++ return !d->failed && d->alive_gen && d->alive_gen == d->gen && ++ g5p_ds_ok(d, G5P_DSE_RELEASE) && g5p_ds_ok(d, G5P_DSE_LOAD); ++} ++ ++static inline int g5p_ds_line(struct g5p_ds *d, char *buf, size_t n, ++ enum g5p_ds_ev e, int ret, u32 a, u32 b) ++{ ++ d->lines++; ++ return scnprintf(buf, n, ++ "DS108 gen=%u ev=%s ret=%d origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x pmgr=%08x a=%x b=%x gate=%s", ++ d->gen, g5p_ds_ev_name[e], ret, g5p_ds_origin_name[d->origin], ++ d->failed, g5p_ds_why_name[d->why], d->seen, d->ok, d->flags, ++ d->pmgr_raw, a, b, g5p_ds_gate_name[d->last_gate]); ++} ++ ++/* ++ * Feed one event. Returns, for CLAIM, 0 (expose) or -EAGAIN/-EIO (refuse); ++ * for every other event 0. mode is the current dirty_start parameter; at mode 0 ++ * the claim answers 0 whatever the state (the legacy driver decides alone). ++ * line/n receive the event's receipt (always written). ++ */ ++static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in, u32 mode, char *line, ++ size_t n) ++{ ++ u32 bit = 1u << e, a = 0, b = 0; ++ int ret = 0; ++ ++ if (mode > G5P_DS_MODE_MAX) ++ mode = G5P_DS_MODE_MAX; ++ d->events++; ++ d->last = e; ++ if ((d->seen & bit) && e != G5P_DSE_CLAIM) ++ d->flags |= G5P_DSF_DUP; ++ switch (e) { ++ case G5P_DSE_PROBE: ++ /* The only event that may start a generation: probe runs once per boot. */ ++ if (!d->gen) ++ d->gen = 1; ++ d->origin = G5P_DSO_UNKNOWN; ++ a = in->kexec; ++ b = in->cycle_requested; ++ d->kexec = in->kexec; ++ d->cycle = in->cycle_requested; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_POWER: ++ if (!(d->seen & (1u << G5P_DSE_PROBE))) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->origin = g5p_ds_origin_of(d->kexec, d->cycle, in->pmgr_ok, ++ in->pmgr_raw); ++ if (!in->pmgr_ok) ++ d->flags |= G5P_DSF_PMGR_UNREAD; ++ d->pmgr_raw = in->pmgr_ok ? in->pmgr_raw : 0; ++ a = in->pmgr_ok; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RESET: ++ a = in->ret; ++ if (d->resets || (d->seen & (1u << G5P_DSE_WAKE))) { ++ /* The glue must not have run it; record a refused attempt and fail ++ * closed. ++ */ ++ d->flags |= G5P_DSF_RESET_REFUSED; ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->resets++; ++ d->flags |= G5P_DSF_RESET_RAN; ++ /* A reset voids the previous generation's proofs (power epoch boundary). */ ++ d->gen++; ++ /* Unreachable today (a reset runs before the wake, so before every later ++ * step); kept so that a reordered glue cannot carry a proof across it. ++ */ ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->flags |= G5P_DSF_RESET_OK; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_WAKE: ++ d->seen |= bit; ++ if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_WAKE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_MAPS: ++ d->seen |= bit; ++ a = in->rings_bad; ++ b = in->rings_busy; ++ if (in->rings_busy) ++ d->flags |= G5P_DSF_RINGS_BUSY; ++ /* map needs the core id, read through the woken wrapper */ ++ if (!g5p_ds_ok(d, G5P_DSE_WAKE)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret || in->rings_bad) ++ g5p_ds_fail(d, G5P_DSW_MAPS); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_LOAD: ++ d->seen |= bit; ++ if (!g5p_ds_ok(d, G5P_DSE_MAPS)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_LOAD); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RELEASE: ++ d->seen |= bit; ++ a = (u32)in->power_ret; ++ b = in->power_raw; ++ if (!g5p_ds_ok(d, G5P_DSE_LOAD)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->power_abort || in->power_ret) ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_RELEASE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_ALIVE: ++ d->seen |= bit; ++ a = in->gpc_valid; ++ b = in->gpc; ++ if (!g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ } else if (in->power_abort) { ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ } else if (in->gpc_valid && !G5P_GPC_ON(in->gpc)) { ++ g5p_ds_fail(d, G5P_DSW_GPC); ++ } else if (in->ret || !in->alive || in->failed) { ++ g5p_ds_fail(d, G5P_DSW_ALIVE); ++ } else { ++ d->ok |= bit; ++ d->alive_gen = d->gen; ++ } ++ break; ++ case G5P_DSE_RECOVER: ++ /* The render frontend's in-boot recovery (recover.inc): the gate was cycled ++ * after this generation's release, with the firmware frozen. Like a RESET it ++ * starts a generation that voids every proof of the last one, and it is a ++ * witnessed cycle for the origin gate; the bootstrap's steps then run again ++ * and must each succeed in the new generation. Not after a failure, and not ++ * before a release: there is no epoch to recover then. ++ */ ++ a = in->ret; ++ if (d->failed || !g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->gen++; ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ /* The old ALIVE proof is void; the claim's late path may prove the new one. */ ++ d->alive_gen = 0; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_CLAIM: ++ d->claims++; ++ d->seen |= bit; ++ a = in->alive | in->released << 1 | in->failed << 2 | ++ in->control_complete << 3 | in->power_abort << 4; ++ /* ALIVE reached through a path without the ALIVE event: accept the RAM ++ * latch as the proof only after a successful release of this generation. ++ */ ++ if (!d->failed && !d->alive_gen && in->alive && !in->failed && ++ !in->power_abort && g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ d->alive_gen = d->gen; ++ d->ok |= 1u << G5P_DSE_ALIVE; ++ d->flags |= G5P_DSF_ALIVE_LATE; ++ } ++ if (!mode) ++ d->last_gate = G5P_DSG_OPEN; ++ else if (d->failed) ++ d->last_gate = G5P_DSG_FAILED; ++ else if (!g5p_ds_proven(d)) ++ d->last_gate = G5P_DSG_NOT_PROVEN; ++ else if (!in->alive || !in->released || in->failed || ++ in->power_abort || !in->control_complete) ++ d->last_gate = G5P_DSG_STATE; ++ else if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ d->last_gate = G5P_DSG_ORIGIN; ++ else ++ d->last_gate = G5P_DSG_OPEN; ++ if (d->last_gate == G5P_DSG_OPEN) { ++ d->ok |= bit; ++ } else { ++ d->refused++; ++ ret = d->last_gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++ } ++ break; ++ default: ++ break; ++ } ++ g5p_ds_line(d, line, n, e, e == G5P_DSE_CLAIM ? ret : in->ret, a, b); ++ return ret; ++} ++ ++ ++ ++static inline void g5p_ds_init(struct g5p_ds *d) ++{ ++ *d = (struct g5p_ds){ .last = G5P_DSE_N }; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +new file mode 100644 +index 0000000000..037e5c56ed +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -0,0 +1,273 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the incoming dirty-start machine (dirtystart108.h), series ++ * gpu-recovery108 (0007). Design: docs/gpu/experiments/recovery108-driver.md. ++ * ++ * Included from core.inc after teardown.inc, so it may use the teardown ring ++ * read (RAM), g5p_r33_gfx_off() and the retire layer's state. Patch 0008 calls ++ * the hooks from the driver steps; until then nothing here runs. ++ * ++ * Register access added by this file: one PMGR read of the gfx word after the ++ * PM reference (POWER, the same read g5p_power_snapshot() makes), and in mode 3 ++ * only one bounded PMGR cycle through g5p_r33_gfx_off() before the first ++ * wrapper wake. Every other input is RAM or a step's own return value. ++ */ ++#include ++#include "dirtystart108.h" ++ ++static struct g5p_ds g5p_ds_state = { .last = G5P_DSE_N }; ++static DEFINE_SPINLOCK(g5p_ds_lock); ++ ++static u32 g5p_ds_mode(void) ++{ ++ return min_t(u32, (1), G5P_DS_MODE_MAX); ++} ++ ++/* One event, one DS108 line (dev_emerg: netconsole at any loglevel). */ ++static int g5p_ds_feed(struct g5p_diag *g, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in) ++{ ++ unsigned long flags; ++ char line[224]; ++ int ret; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ ret = g5p_ds_event(&g5p_ds_state, e, in, g5p_ds_mode(), line, ++ sizeof(line)); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "%s\n", line); ++ return ret; ++} ++ ++/* wake112: the current reset generation, recorded with a deferred R74E proof (RAM, leaf lock). */ ++static u32 g5p_ds_gen(void) ++{ ++ unsigned long flags; ++ u32 gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return gen; ++} ++ ++/* The last apple_pmgr.kexec_gpu_cycle= token of the chosen bootargs (the one the kernel obeys). */ ++static bool g5p_ds_cycle_requested(struct device_node *chosen) ++{ ++ static const char key[] = "apple_pmgr.kexec_gpu_cycle="; ++ const char *args, *p, *last = NULL; ++ char tok[8]; ++ bool v = false; ++ size_t n; ++ ++ if (!chosen || of_property_read_string(chosen, "bootargs", &args)) ++ return false; ++ for (p = strstr(args, key); p; p = strstr(p + 1, key)) ++ if (p == args || p[-1] == ' ') ++ last = p; ++ if (!last) ++ return false; ++ last += sizeof(key) - 1; ++ n = strcspn(last, " "); ++ if (!n || n >= sizeof(tok)) ++ return false; ++ memcpy(tok, last, n); ++ tok[n] = 0; ++ return !kstrtobool(tok, &v) && v; ++} ++ ++static void g5p_ds_hook_probe(struct g5p_diag *g) ++{ ++ struct device_node *chosen = of_find_node_by_path("/chosen"); ++ struct g5p_ds_in in = {}; ++ ++ in.kexec = chosen && ++ of_property_read_bool(chosen, "linux,booted-from-kexec"); ++ in.cycle_requested = g5p_ds_cycle_requested(chosen); ++ of_node_put(chosen); ++ g5p_ds_feed(g, G5P_DSE_PROBE, &in); ++} ++ ++/* After the PM reference is taken: one PMGR read (the gfx word), no write. */ ++static void g5p_ds_hook_power(struct g5p_diag *g) ++{ ++ struct g5p_ds_in in = {}; ++ u32 raw = 0; ++ ++ in.pmgr_ok = g->pmgr && !regmap_read(g->pmgr, 0x80340, &raw); ++ in.pmgr_raw = raw; ++ g5p_ds_feed(g, G5P_DSE_POWER, &in); ++} ++ ++/* Before the first wrapper wake. Mode 3 only, one attempt per boot, through the ++ * r33 helper (its own refusal rules: min/reset/auto or not on; every exit leaves ++ * the gate requested on). Returns the cycle's error; the wake then fails closed. ++ */ ++static int g5p_ds_hook_reset(struct g5p_diag *g) ++{ ++ u32 ms = clamp_t(u32, (10), 1, G5P_R33_GFX_OFF_MAX_MS); ++ struct g5p_ds_in in = {}; ++ unsigned long flags; ++ bool want; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ want = g5p_ds_reset_wanted(&g5p_ds_state, g5p_ds_mode()); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ if (!want) ++ return 0; ++ g5p_mmio_gate(g); ++ in.ret = g5p_r33_gfx_off(g->dev, g->pmgr, ms); ++ g5p_ds_feed(g, G5P_DSE_RESET, &in); ++ boot_gfx_off_done = true; /* one PMGR cycle per boot, whoever runs it */ ++ return in.ret; ++} ++ ++/* The legacy boot_gfx_off_ms cycle (R33) is the same mechanism: record it as the reset. */ ++ ++/* An in-boot recovery's gate cycle (recover.inc): a new generation, or a failed machine. */ ++static int g5p_ds_hook_recover(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ unsigned long flags; ++ bool failed; ++ ++ g5p_ds_feed(g, G5P_DSE_RECOVER, &in); ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ failed = g5p_ds_state.failed; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return failed ? -EIO : 0; ++} ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_WAKE, &in); ++} ++ ++/* Fresh shared state: every ring control word in range (RAM, the teardown ring read). */ ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ u32 i, cons, prod; ++ ++ for (i = 0; !ret && i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ in.rings_bad++; ++ continue; ++ } ++ if (cons > 255 || prod > 255) ++ in.rings_bad++; ++ else if (cons != prod) ++ in.rings_busy++; ++ } ++ g5p_ds_feed(g, G5P_DSE_MAPS, &in); ++} ++ ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_LOAD, &in); ++} ++ ++/* Asked before the release touches anything. -EIO: the fail-closed latch; -EAGAIN: strict origin. */ ++static int g5p_ds_hook_release_gate(struct g5p_diag *g) ++{ ++ enum g5p_ds_gate gate; ++ unsigned long flags; ++ u32 mode = g5p_ds_mode(), gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gate = g5p_ds_release_gate(&g5p_ds_state, mode); ++ g5p_ds_state.last_gate = gate; ++ if (gate != G5P_DSG_OPEN) ++ g5p_ds_state.refused++; ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "DS108 gen=%u gate=release mode=%u answer=%s\n", gen, ++ mode, g5p_ds_gate_name[gate]); ++ if (gate == G5P_DSG_OPEN) ++ return 0; ++ return gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++} ++ ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (r) { ++ in.power_ret = r->early_power_ret; ++ in.power_raw = r->early_power_raw; ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ } ++ g5p_ds_feed(g, G5P_DSE_RELEASE, &in); ++} ++ ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed); ++ } ++ if (r) { ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ /* r73e reads R1+0x1000 at the first ALIVE; 0 means not read. */ ++ in.gpc = READ_ONCE(r->early_state); ++ in.gpc_valid = 1 && in.gpc; ++ } ++ g5p_ds_feed(g, G5P_DSE_ALIVE, &in); ++} ++ ++/* The DRM claim: 0 expose, -EAGAIN not proven / strict origin, -EIO the fail-closed latch. */ ++static int __maybe_unused g5p_ds_hook_claim(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_ds_in in = {}; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ in.control_complete = READ_ONCE(b->control_complete); ++ in.power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ return g5p_ds_feed(g, G5P_DSE_CLAIM, &in); ++} ++ ++/* RAM only: the retire layer and the dirty-start machine, for capture without a log. */ ++static ssize_t recovery_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct g5p_r8 *s = &g5p_r8_state; ++ unsigned long flags; ++ struct g5p_ds d; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ d = g5p_ds_state; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return sysfs_emit( ++ buf, ++ "r8 gen=%u retired=%d mode=%s class=%s verdict=%s flags=%02x admit_closed=%u " ++ "inside=%u detach_queued=%u lines=%u\n" ++ "ds gen=%u mode=%u origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x " ++ "pmgr=%08x alive_gen=%u resets=%u reset_ok_gen=%u events=%u claims=%u refused=%u " ++ "gate=%s\n", ++ READ_ONCE(s->gen), READ_ONCE(s->retired), ++ g5p_td_mode_name[s->mode], g5p_r8_class_name[s->cls], ++ g5p_td_verdict_name[s->verdict], s->flags, ++ g5p_r8_admit_closed(&s->admit), g5p_r8_admit_inside(&s->admit), ++ READ_ONCE(s->detach_queued), s->lines, d.gen, g5p_ds_mode(), ++ g5p_ds_origin_name[d.origin], d.failed, g5p_ds_why_name[d.why], ++ d.seen, d.ok, d.flags, d.pmgr_raw, d.alive_gen, d.resets, ++ d.reset_ok_gen, d.events, d.claims, d.refused, ++ g5p_ds_gate_name[d.last_gate]); ++} ++static DEVICE_ATTR_RO(recovery_status); +diff --git a/drivers/gpu/apple-g5p/drm-validate.h b/drivers/gpu/apple-g5p/drm-validate.h +new file mode 100644 +index 0000000000..671e13ab8c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/drm-validate.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DRM_VALIDATE_H ++#define G5P_DRM_VALIDATE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_USER_VA_START 0x10000ULL ++#define G5P_USER_VA_END 0x10000000000ULL ++#define G5P_USER_RESERVED_START G5P_IMAGE_VA ++#define G5P_USER_RESERVED_END (G5P_PIO_VA + G5P_PIO_SIZE) ++#define G5P_MAX_BO_SIZE (64ULL << 20) ++/* VA bookkeeping: the sum of mapping sizes in one VM. A GEM bound twice ++ * (GBM alloc, then the EGL import of its own dma-buf) counts twice here. ++ */ ++#define G5P_MAX_VM_BYTES BIT_ULL(30) ++/* Memory behind one VM's mappings, counted once per GEM object. */ ++#define G5P_MAX_PINNED_BYTES (768ULL << 20) ++ ++static inline bool g5p_user_pa_valid(u64 pa) ++{ ++ return pa < 0x10000000000ULL && !(pa & (G5P_GPU_PAGE - 1)); ++} ++ ++static inline bool g5p_user_range_valid(u64 va, u64 size) ++{ ++ if (!size || ((va | size) & (G5P_GPU_PAGE - 1)) || ++ va < G5P_USER_VA_START || va >= G5P_USER_VA_END || ++ size > G5P_USER_VA_END - va) ++ return false; ++ return va >= G5P_USER_RESERVED_END || ++ va + size <= G5P_USER_RESERVED_START; ++} ++ ++static inline bool g5p_bo_range_valid(u64 offset, u64 size, u64 bo_size) ++{ ++ return size && !((offset | size) & (G5P_GPU_PAGE - 1)) && ++ offset < bo_size && size <= bo_size - offset; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/fender.h b/drivers/gpu/apple-g5p/fender.h +new file mode 100644 +index 0000000000..cbdf203c5b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/fender.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_FENDER_H ++#define G5P_FENDER_H ++ ++/* Every poll callback must enforce a deadline and retain its final raw word. ++ * Apple 20H350: stop c9fa10, prepare c9fc10, boot c9f954; ++ * Fender helpers ca4ccc (GPC state) and ca6544 (core-power request). ++ * Hardware interpretation of the 0 -> 1 transition remains experimental. ++ */ ++struct g5p_fender_io { ++ void *ctx; ++ int (*write)(void *ctx, u32 reg, u32 value); ++ int (*poll)(void *ctx, u32 reg, u32 mask, u32 expected); ++}; ++ ++static inline int g5p_fender_idle(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x1000, 0x10, 0); ++ if (ret) ++ return ret; ++ ret = io->write(io->ctx, 0x1000, 0x10); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++static inline int g5p_fender_hold(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0x20); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x20000, 0x20, 0x20); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++/* q: the pstate the GPC is requested at (R1+0x1000 = 0x10 | q, poll for q); 1 as before, ++ * or boot_gpu_perf_poweron's pin (boot.inc). ++ */ ++static inline int ++g5p_fender_start_observed_q(const struct g5p_fender_io *io, ++ void (*observe)(void *, const char *), u32 q) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-core-clear"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc0-verified"); ++ ret = io->write(io->ctx, 0x1000, 0x10 | q); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc1-requested"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, q); ++ if (!ret && observe) ++ observe(io->ctx, "release-gpc1-active"); ++ return ret; ++} ++ ++static inline int g5p_fender_start_observed(const struct g5p_fender_io *io, ++ void (*observe)(void *, ++ const char *)) ++{ ++ return g5p_fender_start_observed_q(io, observe, 1); ++} ++ ++static inline int g5p_fender_start(const struct g5p_fender_io *io) ++{ ++ return g5p_fender_start_observed(io, NULL); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/gpu-oc.h b/drivers/gpu/apple-g5p/gpu-oc.h +new file mode 100644 +index 0000000000..733aba9d4e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/gpu-oc.h +@@ -0,0 +1,149 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * GPU overclock record (boot_gpu_oc_mhz): docs/power/ds-gpu-dvfs.md §3-4, §9 R5; ++ * docs/gpu/experiments/clock-mechanisms.md §2. Host test: scripts/gpu/perf60-host/test_oc.c. ++ * ++ * The hardware applies PSINFO record n (PMGR+0x70000 + 0x10*n, written by LLB 180395fd8 for ++ * slots 0-4 once per SoC reset, volatile) when Fender is asked for pstate n. Word 0 = core-rail ++ * DAC code [31:24] | PLL valid [21] | multiplier [20:12] | predivider [8:4] | postdivider-1 [3:0]; ++ * word 1 [7:0] = buck8 code; word 2 = timing selectors. ++ * ++ * PLL: VCO = 24 MHz * mult / prediv, f = VCO / postdiv. Apple's GPU-domain (0x17) descriptors ++ * (LLB 0x1803bc718, all three chip sections) run VCOs from 800 to 2160 MHz (2160 = id 0x20 of ++ * section 0x2f0001ff, 90/1/3); the live states use 948..1650. The overclock keeps postdiv 2 (every ++ * live state at or above 474 MHz uses 2 or 3) and the VCO inside Apple's range: f <= 1080 MHz. ++ * Apple's own descriptors at 804 (67/1/2), 850 (425/6/2), 900 (75/1/2); elsewhere record 4's ++ * dividers (prediv 4, postdiv 2: f = 3 MHz * mult) on a 3 MHz grid. ++ * ++ * Voltage: record 5 keeps record 4's timing word; its core (rail 1, PMIC 0x11, 450 mV + 3.125 mV * ++ * code, GPU only) and buck8 (rail 8, PMIC 0x18, 600 mV + 3.125 mV * code) codes default to record ++ * 4's (138 = 881.25 mV, 124 = 987.5 mV), never lower, raised only on request in whole 3.125 mV ++ * codes, ++ * and never above the ceilings: the boot parameters (default state 4 + 50 mV, the CPU overclock's ++ * rail-offset cap) bounded by the compiled maxima below (state 4 + 150 mV; raising those needs a ++ * rebuild). The encoding itself reaches 1296 mV on both rails; that is not an operating point. ++ */ ++#ifndef G5P_GPU_OC_H ++#define G5P_GPU_OC_H ++ ++#include "bits-compat.h" ++ ++#define G5P_PSINFO_BASE 0x70000U /* PMGR syscon offset */ ++#define G5P_PSINFO_REC(n) (G5P_PSINFO_BASE + 0x10U * (n)) ++#define G5P_OC_STATE 5U ++#define G5P_OC_PIN 500U /* boot_gpu_perf for state 5 */ ++#define G5P_OC_STOCK4 \ ++ 0x8a2f1041U /* record 4 word 0 as LLB writes it on this chip */ ++#define G5P_OC_STOCK4_BUCK8 124U /* record 4 word 1 [7:0] */ ++#define G5P_OC_MIN_MHZ 726U /* the first grid step above 723 */ ++#define G5P_OC_VCO_MAX 2160U /* Apple's highest GPU-domain VCO */ ++#define G5P_OC_MAX_MHZ (G5P_OC_VCO_MAX / 2) /* postdiv 2: 1080 */ ++#define G5P_OC_VCO_STOCK_MAX \ ++ 1650U /* the highest VCO of a live state (550 MHz) */ ++#define G5P_OC_VALID BIT(21) ++#define G5P_OC_PLL_MASK 0x001ff1ffU /* mult, prediv, postdiv-1 */ ++#define G5P_OC_UV_STEP 3125U ++#define G5P_OC_CORE_BASE_UV 450000U ++#define G5P_OC_BUCK8_BASE_UV 600000U ++/* 881250 */ ++#define G5P_OC_CORE_STOCK_UV \ ++ (G5P_OC_CORE_BASE_UV + G5P_OC_UV_STEP * (G5P_OC_STOCK4 >> 24)) ++/* 987500 */ ++#define G5P_OC_BUCK8_STOCK_UV \ ++ (G5P_OC_BUCK8_BASE_UV + G5P_OC_UV_STEP * G5P_OC_STOCK4_BUCK8) ++/* the boot parameters' default ceiling: P4 + 50 mV */ ++#define G5P_OC_DEFAULT_HEADROOM_UV 50000U ++#define G5P_OC_ABS_HEADROOM_UV \ ++ 150000U /* compiled: P4 + 150 mV, no parameter goes above */ ++ ++struct g5p_oc_pll { ++ u32 mhz, mult, prediv, postdiv; ++ bool llb; /* Apple's own descriptor */ ++}; ++ ++static const struct g5p_oc_pll g5p_oc_llb[] = { ++ { 804, 67, 1, 2, true }, /* LLB id 0x22, all three GPU sections */ ++ { 850, 425, 6, 2, true }, /* LLB id 0x23, the live section */ ++ { 900, 75, 1, 2, true }, /* LLB id 0x24, the live section */ ++}; ++ ++/* ++ * The PLL for a requested MHz: Apple's descriptor at 804/850/900 (the request or its 3 MHz ++ * rounding), ++ * else the 3 MHz grid (rounded down) with record 4's dividers; false outside 726..1080. ++ */ ++static inline bool g5p_oc_pll(u32 want, struct g5p_oc_pll *p) ++{ ++ u32 mhz = want / 3 * 3; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_llb); i++) ++ if (want == g5p_oc_llb[i].mhz || mhz == g5p_oc_llb[i].mhz) { ++ *p = g5p_oc_llb[i]; ++ return true; ++ } ++ if (mhz < G5P_OC_MIN_MHZ || mhz > G5P_OC_MAX_MHZ) ++ return false; ++ *p = (struct g5p_oc_pll){ mhz, mhz / 3, 4, 2, false }; ++ return true; ++} ++ ++/* A rail voltage as a code: exact multiples of 3.125 mV above the base only. */ ++static inline bool g5p_oc_code(u32 uv, u32 base, u32 *code) ++{ ++ if (uv < base || (uv - base) % G5P_OC_UV_STEP || ++ (uv - base) / G5P_OC_UV_STEP > 0xff) ++ return false; ++ *code = (uv - base) / G5P_OC_UV_STEP; ++ return true; ++} ++ ++/* The voltage codes for a request (0 = state 4's), bounded below by state 4 and above by the ++ * ceilings (each already clamped to the compiled maximum by g5p_oc_ceiling()). ++ */ ++ ++/* Record 5's words 0 and 1 from the live record 4 (stock only), a PLL and the voltage codes. */ ++static inline bool g5p_oc_words(const u32 r4[4], u32 want, u32 core, u32 buck8, ++ u32 w[4], struct g5p_oc_pll *p) ++{ ++ if (r4[0] != G5P_OC_STOCK4 || (r4[1] & 0xff) != G5P_OC_STOCK4_BUCK8 || ++ !g5p_oc_pll(want, p) || core < (G5P_OC_STOCK4 >> 24) || ++ core > 0xff || buck8 < G5P_OC_STOCK4_BUCK8 || buck8 > 0xff) ++ return false; ++ w[0] = core << 24 | G5P_OC_VALID | p->mult << 12 | p->prediv << 4 | ++ (p->postdiv - 1); ++ w[1] = (r4[1] & ~0xffU) | buck8; ++ w[2] = r4[2]; ++ w[3] = r4[3]; ++ return true; ++} ++ ++/* Hz of a record word 0, as iBoot 87001c3ac decodes it (0 if the predivider is 0). */ ++static inline u64 g5p_oc_hz(u32 w0) ++{ ++ u32 mult = (w0 >> 12) & 0x1ff, prediv = (w0 >> 4) & 0x1f, ++ postdiv = (w0 & 0xf) + 1; ++ ++ return prediv ? 24000000ULL * mult / prediv / postdiv : 0; ++} ++ ++/* A nonzero record 5 that this driver writes: record 4's timing and word-3 words, valid, a PLL ++ * g5p_oc_pll() produces, voltage codes between state 4's and the compiled maxima. Anything else in ++ * slot 5 is not ours: never overwritten or cleared. ++ */ ++static inline bool g5p_oc_is_ours(const u32 r4[4], const u32 r5[4]) ++{ ++ u32 core = r5[0] >> 24, buck8 = r5[1] & 0xff, w[4], cmax, bmax; ++ struct g5p_oc_pll p; ++ u64 hz = g5p_oc_hz(r5[0]); ++ ++ if (!g5p_oc_code(G5P_OC_CORE_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_CORE_BASE_UV, &cmax) || ++ !g5p_oc_code(G5P_OC_BUCK8_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_BUCK8_BASE_UV, &bmax) || ++ core > cmax || buck8 > bmax || !hz || hz % 1000000ULL || ++ !g5p_oc_words(r4, (u32)(hz / 1000000ULL), core, buck8, w, &p)) ++ return false; ++ return w[0] == r5[0] && w[1] == r5[1] && w[2] == r5[2] && w[3] == r5[3]; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/heartbeat116.h b/drivers/gpu/apple-g5p/heartbeat116.h +new file mode 100644 +index 0000000000..428146764b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/heartbeat116.h +@@ -0,0 +1,175 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_HEARTBEAT116_H ++#define G5P_HEARTBEAT116_H ++ ++/* ++ * Heartbeat116: the R74 IRQ174 storm bound as a window rate, not a lifetime count. ++ * ++ * A healthy R74 epoch takes IRQ174 at about 575/s and consumes about 290 event-ring ++ * entries/s, idle or not: one type-0 ALIVE per firmware scheduler iteration, and two IRQ ++ * entries per ring entry (irqs = 2 * events - 18 over the Warm113 C4-prep R74 milestones; ++ * r81 measured 577/289 per s). Wake112 bounded both counts over the epoch's lifetime: ++ * IRQ entries against boot_r74_irq_cap (1048576) and ring entries against the compute event ++ * limit (524544). Either bound latched failed/event_failed, so an idle, healthy GPU faulted ++ * after about 30 minutes. ++ * ++ * Here both bounds are rates. A window opens at the first IRQ entry and closes at the first ++ * entry at least window_ns later, then re-bases on the totals of that entry. Totals are u32 ++ * and every in-window count is a u32 difference, so a wrap of a total is harmless. ++ * ++ * RATE more than irq_ceiling IRQ entries in the open window (checked on every entry) ++ * EVENTS more than event_ceiling ring entries in the open window (checked on every entry) ++ * STALL stall_windows consecutive closed windows in which ++ * irqs > ratio * events + slack. Healthy traffic is 2:1. A latch that re-fires ++ * without ring progress, or a heartbeat after a fault stopped the drain, is not. ++ * ++ * A trip is absorbing. Nothing here clears a failure; the glue adds the same terminal stop ++ * as the lifetime cap had (cap_seen, failed, event_failed, one disable_irq_nosync). ++ * ++ * Pure policy: no register access, no lock, no kernel API. The caller serializes (the R73B ++ * IRQ thread is the only caller). ++ */ ++ ++#define G5P_HB_WINDOW_NS 1000000000ULL ++#define G5P_HB_IRQ_CEILING 16384U ++#define G5P_HB_EVENT_CEILING 8192U ++#define G5P_HB_RATIO 8U ++#define G5P_HB_SLACK 512U ++#define G5P_HB_STALL_WINDOWS 3U ++ ++enum g5p_hb_why { ++ G5P_HBW_NONE, ++ G5P_HBW_RATE, ++ G5P_HBW_EVENTS, ++ G5P_HBW_STALL, ++ G5P_HBW_N, ++}; ++ ++static const char *const g5p_hb_why_name[G5P_HBW_N] = { ++ "none", ++ "rate", ++ "events", ++ "stall", ++}; ++ ++enum g5p_hb_res { ++ G5P_HBR_OPEN, /* counted in the open window */ ++ G5P_HBR_CLOSED, /* this entry closed a window (h->last) */ ++ G5P_HBR_TRIP, /* tripped now or earlier */ ++}; ++ ++struct g5p_hb_cfg { ++ u64 window_ns; ++ u32 irq_ceiling, event_ceiling, ratio, slack, stall_windows; ++}; ++ ++/* The totals the IRQ thread samples after servicing one entry. */ ++struct g5p_hb_in { ++ u64 now_ns; ++ u32 irqs; /* IRQ174 entries (b->irqs) */ ++ u32 events; /* event-ring entries consumed (b->events) */ ++ u32 acks; /* AIC ACKs written by the R73B handler */ ++ u32 empty; /* entries whose service consumed no ring entry */ ++ u32 aux; /* auxiliary-ring entries consumed */ ++}; ++ ++/* One window: its start and length and its in-window counts. */ ++struct g5p_hb_win { ++ u64 start_ns, ns; ++ u32 irqs, events, acks, empty, aux; ++}; ++ ++struct g5p_hb { ++ u32 open; /* zero until the first entry (kzalloc) */ ++ u32 why; /* enum g5p_hb_why, written once */ ++ u32 windows; /* windows closed */ ++ u32 stalled; /* consecutive unproductive closed windows */ ++ u32 peak_irqs, peak_events; ++ u64 start_ns; ++ struct g5p_hb_in base; /* totals at the window's first entry */ ++ struct g5p_hb_win last; /* the last closed window */ ++ struct g5p_hb_win trip; /* the open window at the trip */ ++ u64 trip_ns; ++}; ++ ++static inline void g5p_hb_cfg_default(struct g5p_hb_cfg *c) ++{ ++ c->window_ns = G5P_HB_WINDOW_NS; ++ c->irq_ceiling = G5P_HB_IRQ_CEILING; ++ c->event_ceiling = G5P_HB_EVENT_CEILING; ++ c->ratio = G5P_HB_RATIO; ++ c->slack = G5P_HB_SLACK; ++ c->stall_windows = G5P_HB_STALL_WINDOWS; ++} ++ ++static inline void g5p_hb_count(const struct g5p_hb *h, ++ const struct g5p_hb_in *in, ++ struct g5p_hb_win *w) ++{ ++ w->start_ns = h->start_ns; ++ w->ns = in->now_ns >= h->start_ns ? in->now_ns - h->start_ns : 0; ++ w->irqs = in->irqs - h->base.irqs; ++ w->events = in->events - h->base.events; ++ w->acks = in->acks - h->base.acks; ++ w->empty = in->empty - h->base.empty; ++ w->aux = in->aux - h->base.aux; ++} ++ ++static inline bool g5p_hb_unproductive(const struct g5p_hb_cfg *c, ++ const struct g5p_hb_win *w) ++{ ++ return (u64)w->irqs > (u64)c->ratio * w->events + c->slack; ++} ++ ++static inline enum g5p_hb_res ++g5p_hb_trip(struct g5p_hb *h, u32 why, const struct g5p_hb_win *w, u64 now_ns) ++{ ++ h->why = why; ++ h->trip = *w; ++ h->trip_ns = now_ns; ++ return G5P_HBR_TRIP; ++} ++ ++/* One serviced IRQ entry. */ ++static inline enum g5p_hb_res g5p_hb_step(struct g5p_hb *h, ++ const struct g5p_hb_cfg *c, ++ const struct g5p_hb_in *in) ++{ ++ struct g5p_hb_win w; ++ ++ if (h->why) ++ return G5P_HBR_TRIP; ++ if (!h->open) { ++ h->open = 1; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ return G5P_HBR_OPEN; ++ } ++ g5p_hb_count(h, in, &w); ++ if (w.irqs > c->irq_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_RATE, &w, in->now_ns); ++ if (w.events > c->event_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_EVENTS, &w, in->now_ns); ++ if (w.ns < c->window_ns) ++ return G5P_HBR_OPEN; ++ h->last = w; ++ h->windows++; ++ if (w.irqs > h->peak_irqs) ++ h->peak_irqs = w.irqs; ++ if (w.events > h->peak_events) ++ h->peak_events = w.events; ++ h->stalled = g5p_hb_unproductive(c, &w) ? h->stalled + 1 : 0; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ if (h->stalled >= c->stall_windows) ++ return g5p_hb_trip(h, G5P_HBW_STALL, &w, in->now_ns); ++ return G5P_HBR_CLOSED; ++} ++ ++/* Receipts: the first four closed windows, then every 64th. */ ++static inline bool g5p_hb_report(const struct g5p_hb *h) ++{ ++ return h->windows <= 4 || !(h->windows % 64); ++} ++ ++#endif /* G5P_HEARTBEAT116_H */ +diff --git a/drivers/gpu/apple-g5p/kick.inc b/drivers/gpu/apple-g5p/kick.inc +new file mode 100644 +index 0000000000..6ef7395c64 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/kick.inc +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared pre-kick step recovered from ca0358/ca05d4 and ca6644. */ ++static int g5p_prepare_kick(struct g5p_diag *g, const char *name) ++{ ++ u32 raw, state; ++ ++ if (gfx_bridge_replay && !strcmp(name, "RENDER-TA")) ++ g5p_r41_bridge_ta(g); ++ /* ca05d4 always invokes Fender +80 before inspecting GPC state. ++ * ca6644: W R1+14=7, then read fence. Hardware bit names unknown. ++ * This is distinct from acknowledging the host AIC interrupt. ++ */ ++ dev_info(g->dev, "%s Fender W R1+00014=00000007 before MTS kick\n", ++ name); ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ dev_info(g->dev, "%s Fender raw R1+00014=%08x (write fence)\n", name, ++ raw); ++ if (raw & ~7U) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ if (READ_ONCE(g->boot->failed)) ++ return -EIO; ++ return 0; ++} +diff --git a/drivers/gpu/apple-g5p/latency123.h b/drivers/gpu/apple-g5p/latency123.h +new file mode 100644 +index 0000000000..164dc8e684 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/latency123.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: firmware words a powered iOS GPU has and our bootstrap leaves zero ++ * (docs/gpu/firmware-re/wake-latency-re.md sections 3.4 and 4.2). ++ * ++ * Arm A. The firmware copies meta+0x6c0..0x758 into its private block once, at init ++ * (MINIM 0x100009ac -> 0x10003ca4), so these words only act from the boot image: ++ * meta+0x6d4 = 1 per-tick cache maintenance 0x1001b638 (688/1720/e230/e138) ++ * meta+0x710/0x714 pstate table max index / count ++ * meta+0x718.. pstate MHz; the tick period is table[Fender+0x1000 & 0xf] ++ * policy+0x4 = 16 governor sample period, ms (the governor reads it live) ++ * 1 = capped: table { 0, 340 }, max index 1. The firmware's own pstate setter ++ * (0x1001c88c, Fender+0x1000 = index | 0x10) can then only ask for pstate 1, the state ++ * the host already requests, so the clock does not move. ++ * 2 = full: table { 0, 340, 474, 550, 723 }, max index 4, plus the limits README rank 2 ++ * pairs with it (control+0x1028/0x102c/0x1030 = 3348/400/400, policy+0x7cc/0x6fc = ++ * 400). The firmware may then raise the GPU clock itself; nothing on Linux programs ++ * a GPU voltage table, so this arm runs after the capped one. ++ * ++ * Arm B. ctl+0x04 is the per-engine tick-service mask (iOS 7). Every tick re-copies ++ * ctl+0x00..0x20 (0x10001328), so the word is live; the boot image gets the value the ++ * parameter holds at map and the render path re-applies it before each job. ++ * ++ * RAM only. Temporary A/B code: the winning words become unconditional init writes. ++ */ ++#ifndef APPLE_G5P_LATENCY123_H ++#define APPLE_G5P_LATENCY123_H ++ ++#include "bits-compat.h" ++#define G5P_L123_POLICY 0x4aU ++#define G5P_L123_META 0x4dU ++#define G5P_L123_CTL 0x5eU ++#define G5P_L123_TICK_MASK 7U ++#define G5P_L123_CAPPED 1U ++#define G5P_L123_FULL 2U ++ ++static const u32 g5p_l123_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline void g5p_l123_arm_a(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 arm) ++{ ++ u8 *meta = g5p_object(p, root, shared, G5P_L123_META); ++ u8 *policy = g5p_object(p, root, shared, G5P_L123_POLICY); ++ u8 *ctl = g5p_object(p, root, shared, G5P_L123_CTL); ++ u32 count = arm == G5P_L123_FULL ? ARRAY_SIZE(g5p_l123_mhz) : 2; ++ u32 i; ++ ++ g5p_put32(meta + 0x6d4, 1); ++ g5p_put32(meta + 0x710, count - 1); ++ g5p_put32(meta + 0x714, count); ++ for (i = 0; i < count; i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_l123_mhz[i]); ++ g5p_put32(policy + 4, 16); ++ if (arm != G5P_L123_FULL) ++ return; ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, 400); ++ g5p_put32(ctl + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/ldr.h b/drivers/gpu/apple-g5p/ldr.h +new file mode 100644 +index 0000000000..165aacf85b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/ldr.h +@@ -0,0 +1,154 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Deliberately small Apple G5P META LDR reader. No allocation or MMIO here. */ ++#ifndef APPLE_G5P_LDR_H ++#define APPLE_G5P_LDR_H ++ ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_LDR_MAX_RECORDS 128 ++#define G5P_LDR_MAX_BYTES (256 * 1024) ++ ++enum g5p_ldr_op { G5P_LOAD, G5P_ZERO, G5P_CONFIG, G5P_START }; ++ ++typedef int (*g5p_ldr_emit)(void *ctx, enum g5p_ldr_op op, u32 address, ++ u32 size, const u8 *data); ++ ++static inline u32 g5p_le32(const u8 *p) ++{ ++ return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; ++} ++ ++static inline u16 g5p_le16(const u8 *p) ++{ ++ return (u16)p[0] | (u16)p[1] << 8; ++} ++ ++static inline bool g5p_ldr_bounds(size_t len, u32 off, u32 size) ++{ ++ return off <= len && size <= len - off; ++} ++ ++static inline int g5p_ldr_l2(const u8 *fw, size_t len, u32 off, u16 tag, ++ const u8 **data, u32 *size) ++{ ++ u16 bytes; ++ ++ if (!g5p_ldr_bounds(len, off, 4)) ++ return -EINVAL; ++ bytes = g5p_le16(fw + off + 2); ++ if (bytes < 6 || !g5p_ldr_bounds(len, off, bytes) || ++ g5p_le16(fw + off) != tag) ++ return -EINVAL; ++ *data = fw + off + 4; ++ *size = bytes - 6; ++ return 0; ++} ++ ++/* Validate in a first pass (emit=NULL); only then use a hardware callback. */ ++static inline int g5p_ldr_walk(const u8 *fw, size_t len, g5p_ldr_emit emit, ++ void *ctx) ++{ ++ u32 pos, seen[G5P_LDR_MAX_RECORDS], total = 0; ++ unsigned int nr = 0, starts = 0; ++ ++ if (len != 119112 || g5p_le32(fw) != 0x01aa5500 || ++ !g5p_ldr_bounds(len, g5p_le32(fw + 4), 1)) ++ return -EINVAL; ++ pos = g5p_le32(fw + 8); ++ while (pos != 0xffffffff) { ++ const u8 *p, *data; ++ u32 command, bytes, addr = 0, size = 0, l2, next; ++ enum g5p_ldr_op op; ++ unsigned int i; ++ int ret; ++ ++ if (nr == G5P_LDR_MAX_RECORDS || !g5p_ldr_bounds(len, pos, 12)) ++ return -EINVAL; ++ for (i = 0; i < nr; i++) ++ if (seen[i] == pos) ++ return -EINVAL; ++ seen[nr++] = pos; ++ p = fw + pos; ++ command = g5p_le16(p); ++ bytes = g5p_le16(p + 2); ++ next = g5p_le32(p + 4); ++ if (bytes < 12 || (bytes & 3) || ++ !g5p_ldr_bounds(len, pos, bytes)) ++ return -EINVAL; ++ if (command & 0x10) { ++ if (command != 0x10 || bytes != 16) ++ return -EINVAL; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), 0x8010, ++ &data, &size); ++ if (ret) ++ return ret; ++ goto next_record; ++ } ++ switch (command) { ++ case 0: ++ if (bytes != 20) ++ return -EINVAL; ++ op = G5P_LOAD; ++ addr = g5p_le32(p + 8); ++ l2 = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, l2, 0x8000, &data, &size); ++ if (ret) ++ return ret; ++ break; ++ case 4: ++ if (bytes != 24) ++ return -EINVAL; ++ op = G5P_ZERO; ++ addr = g5p_le32(p + 8); ++ size = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 16), 0x8004, ++ &data, &l2); ++ if (ret || l2) ++ return -EINVAL; ++ break; ++ case 5: ++ case 3: ++ if (bytes != 16) ++ return -EINVAL; ++ op = command == 5 ? G5P_CONFIG : G5P_START; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), ++ 0x8000 | command, &data, &size); ++ if (ret || !size || size % (command == 5 ? 12 : 16)) ++ return -EINVAL; ++ if (command == 5) { ++ for (i = 0; i < size; i += 12) ++ if (g5p_le32(data + i) != 2 || ++ (g5p_le32(data + i + 4) & 3)) ++ return -EINVAL; ++ } else { ++ if (++starts != 1 || size != 32 || ++ next != 0xffffffff || g5p_le32(data) != 1 || ++ g5p_le32(data + 16) != 0) ++ return -EINVAL; ++ } ++ break; ++ default: ++ return -EINVAL; ++ } ++ if (op == G5P_LOAD || op == G5P_ZERO) { ++ if ((addr & 3) || (size & 3) || !size || ++ size > G5P_LDR_MAX_BYTES || ++ addr > (u32)~0U - size || ++ total > G5P_LDR_MAX_BYTES - size) ++ return -EINVAL; ++ total += size; ++ } ++ if (emit) { ++ ret = emit(ctx, op, addr, size, data); ++ if (ret) ++ return ret; ++ } ++next_record: ++ pos = next; ++ } ++ return nr == 94 && starts == 1 ? 0 : -EINVAL; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +new file mode 100644 +index 0000000000..9a896569ca +--- /dev/null ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -0,0 +1,1032 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Post-alive transport and park experiments; never implicit recovery. */ ++static const bool control_experiment = 1; ++static const bool irq_experiment = 0; ++static const bool teardown_experiment = 1; ++ ++/* A complete sparse image: first sample omits only zero words, later samples ++ * emit every changed word (including transitions to zero). All allocated ++ * root/shared memory is covered, including unknown status/trace buffers. ++ * These are CPU coherent RAM observations, never proof of META cache contents. ++ * IRQs can update memory concurrently; this is deliberately not an atomic dump. ++ */ ++static u32 g5p_dc_root_word(struct g5p_boot *b, u32 offset) ++{ ++ return READ_ONCE(*(u32 *)(b->root[offset / G5P_GPU_PAGE].cpu + ++ offset % G5P_GPU_PAGE)); ++} ++ ++/* nopatch: the policy record (slot 0x4a) words 0x1000d044 decides readiness from: +8 state, ++ * +0xc substate, +0x34 pstate mirror; +0 and +4 (sample period) for context. iOS keeps +8 at 0. ++ */ ++static const u32 g5p_np_words[5] = { 0, 4, 8, 0xc, 0x34 }; ++ ++static void g5p_np_read(struct g5p_boot *b, u32 *w) ++{ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_np_words); i++) ++ w[i] = g5p_dc_root_word(b, b->layout.offset[0x4a] + ++ g5p_np_words[i]); ++} ++ ++static void g5p_np_print(struct g5p_diag *g, const char *stage, u32 seq, u32 e, ++ const u32 *w) ++{ ++ dev_info( ++ g->dev, ++ "NOPATCH policy stage=%s seq=%u engine=%u +0=%08x +4=%08x +8=%08x +c=%08x " ++ "+34=%08x events=%u end=nopatch\n", ++ stage, seq, e, w[0], w[1], w[2], w[3], w[4], ++ READ_ONCE(g->boot->events)); ++} ++ ++/* RAM only; nothing unless the unpatched image is latched. */ ++static void g5p_np_policy(struct g5p_diag *g, const char *stage, u32 seq, u32 e) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 w[ARRAY_SIZE(g5p_np_words)]; ++ ++ if (!b || !b->r73b || !b->r73b->nopatch) ++ return; ++ g5p_np_read(b, w); ++ g5p_np_print(g, stage, seq, e, w); ++} ++ ++/* Read fixed allocated policy objects, never follow firmware-owned pointers. ++ * 1000d044/1000d504 test these states;1001c900 stores requested pstate. ++ * These are sampled boundary predicates, NOT entry/return breadcrumbs or PC. ++ */ ++ ++/* Firmware MMU recovery uses event mask200 and reads e150/e158. ++ * Read raw qword halves without ACK/clear; concurrent updates can tear them. ++ * An absent sampled event does not exclude an earlier firmware-handled fault. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 event, lo, hi, lo2, hi2; ++ ++ g5p_mmu_progress(g, stage, 0x130); ++ event = readl(g->rgx + 0x130); ++ g5p_mmu_progress(g, stage, 0xe150); ++ lo = readl(g->rgx + 0xe150); ++ g5p_mmu_progress(g, stage, 0xe154); ++ hi = readl(g->rgx + 0xe154); ++ g5p_mmu_progress(g, stage, 0xe158); ++ lo2 = readl(g->rgx + 0xe158); ++ g5p_mmu_progress(g, stage, 0xe15c); ++ hi2 = readl(g->rgx + 0xe15c); ++ g5p_mmu_record( ++ g, ++ "MMU %s event130=%08x pending=%u e150=%08x:%08x e158=%08x:%08x raw-split read-only\n", ++ stage, event, !!(event & 0x200), hi, lo, hi2, lo2); ++} ++ ++/* Same active-wrapper gate as CDM observations. Never write/ACK a register. ++ * 1001c9b8 samples R0+164/160/164; bound host sampling even on rollover. ++ * A split sample with different high words is explicitly unstable. ++ */ ++ ++static void g5p_dc_snapshot(struct g5p_diag *g, const char *stage, bool first) ++{ ++ { ++ return; ++ } ++} ++ ++static int g5p_control_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 *work = (u32 *)g5p_boot_shared(b, 0x68); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ u32 ctl_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6e)); ++ u32 data_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6f)); ++ u8 *init = b->root[0].cpu; ++ u8 *handoff = b->image[0].cpu + 0x3ff8; ++ u32 *events = (u32 *)g5p_boot_shared(b, 0x70); ++ ++ /* c9f868/c9f884: validate the loaded boot handoff, not just its builder. */ ++ if (g5p_le32(handoff) != g5p_meta_address(G5P_ROOT_VA) || ++ g5p_le32(handoff + 4) != ++ g5p_meta_address(g5p_object_va(&b->layout, 0x5f))) ++ return -EPROTO; ++ /* Drain under event_lock before checking that ALIVE has been returned. */ ++ if (g5p_r73b_service(g, G5P_R73B_OWNER_PREPARE, true)) ++ return -EIO; ++ if (READ_ONCE(events[0]) > 255 || READ_ONCE(events[4]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(events[0]) != READ_ONCE(events[4])) ++ return -EBUSY; ++ dev_info(g->dev, ++ "DC boot handoff verified; event consumer=%u producer=%u\n", ++ READ_ONCE(events[0]), READ_ONCE(events[4])); ++ ++ ; ++ ; ++ ; ++ ; ++ /* c96be8: init+18/+1c are META addresses, never CPU/DMA pointers. */ ++ if (g5p_le32(init + 0x18) != ctl_meta || ++ g5p_le32(init + 0x1c) != data_meta) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC mapping ctl GPU=%llx META=%08x data GPU=%llx META=%08x; init verified\n", ++ g5p_object_va(&b->layout, 0x6e), ctl_meta, ++ g5p_object_va(&b->layout, 0x6f), data_meta); ++ ++ if (b->work_ready) ++ return -EBUSY; ++ if (b->control_ready) ++ return b->r73b && 1 ? 0 : -EALREADY; ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (consumer != producer || READ_ONCE(work[0]) != READ_ONCE(work[4]) || ++ (b->work_submitted && !b->work_complete)) ++ return -EBUSY; ++ /* c993f8, size=0: no payload copy, ctrl+1020=0, command={8,0,0,0,0}. ++ * No claim that this is an opcode-independent firmware NOP. ++ */ ++ g5p_put32(g5p_boot_shared(b, 0x5e) + 0x1020, 0); ++ b->control_target = (producer + 1) & 255; ++ b->control_ready = true; ++ dev_info( ++ g->dev, ++ "DC prepare cmd=8 payload_words=0 consumer=%u producer=%u target=%u; RAM only\n", ++ consumer, producer, b->control_target); ++ return 0; ++} ++ ++static int g5p_r73e_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ret = g5p_control_prepare(g); ++ if (!ret) ++ WRITE_ONCE(b->r73b->fast_window, true); ++ dev_info( ++ g->dev, ++ "R73E cmd8 prepared-before-release ret=%d target=%u end=r73e\n", ++ ret, b->control_target); ++ return ret; ++} ++ ++/* Apple's GPC precondition (AGXMetaFender_setGPCPerformanceState, ...6ca4ccc): a request in ++ * flight shows R1+0x1000 bit 4; wait for it to clear before judging the state. R1 reads only, ++ * bounded by boot_alive_settle_us; with 0 the first read stands, as before. ++ */ ++static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw) ++{ ++ u64 start; ++ ++ if (!r->settle_us || !(raw & 0x10)) ++ return raw; ++ if (!r->settle_first) ++ r->settle_first = raw; ++ start = ktime_get_mono_fast_ns(); ++ while ((raw & 0x10) && ktime_get_mono_fast_ns() - start < ++ (u64)r->settle_us * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(g->window[1] + 0x1000); ++ r->settle_samples++; ++ } ++ r->settle_ns += ktime_get_mono_fast_ns() - start; ++ return raw; ++} ++ ++static int g5p_r73e_irq_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 off, raw; ++ enum g5p_pxg gate; ++ ++ if (!b->control_ready || b->control_submitted) ++ return -EPERM; ++ /* wake112: release_ready, or a deferred R74E proof that the read below settles. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate != G5P_PXG_READY && gate != G5P_PXG_PROVE) ++ return -EHOSTDOWN; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ if (gate == G5P_PXG_PROVE && !g5p_r74e_take(g)) ++ return -EHOSTDOWN; ++ r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (gate == G5P_PXG_PROVE) ++ g5p_r74e_settle(r, r->early_state); ++ if (!g5p_gpc_ok(r->early_state)) ++ return -EHOSTDOWN; ++ /* nopatch: five RAM loads, the state the readiness gate will see before this bell. */ ++ if (r->nopatch) ++ g5p_np_read(b, r->np_pre); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_IRQ, 0); ++ /* Functional pre-kick sequence only; no optional diagnostics here. */ ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ if (raw & ~7U) ++ return -EPROTO; ++ r->early_after_state = ++ g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (!g5p_gpc_ok(r->early_after_state)) ++ return -EHOSTDOWN; ++ dev_emerg( ++ g->dev, ++ "R73E doorbell begin first-ALIVE W0+00b00 value=00000010 end=r73e\n"); ++ writel(0x10, g->rgx + 0xb00); ++ dev_emerg(g->dev, "R73E doorbell end first-ALIVE W0+00b00 end=r73e\n"); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_IRQ, 0); ++ return 0; ++} ++ ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxg gate; ++ u32 px; ++ int ret; ++ ++ /* wake112: a deferred R74E proof admits the once-owner as release_ready does. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate == G5P_PXG_REFUSE) ++ return -EHOSTDOWN; ++ if (gate == G5P_PXG_WAIT) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_busy, 0, 1)) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_once, 0, 1)) { ++ ret = READ_ONCE(r->early_ret); ++ goto out; ++ } ++ r->early_irq_ns = ktime_get_mono_fast_ns(); ++ r->early_ret = g5p_r73e_irq_submit(g); ++ ret = r->early_ret; ++ WRITE_ONCE(r->fast_window, false); ++ if (ret == -EHOSTDOWN) ++ WRITE_ONCE(r->early_power_abort, true); ++ dev_emerg( ++ g->dev, ++ "R74E ALIVE submit case=%u owner=%u state=%08x after=%08x ret=%d " ++ "release_ready=%u power_abort=%u submitted=%u end=r74e\n", ++ r->r73e_case, owner, r->early_state, r->early_after_state, ret, ++ r->early_release_ready, r->early_power_abort, ++ READ_ONCE(b->control_submitted)); ++ if (r->settle_first) ++ dev_emerg( ++ g->dev, ++ "R74E settle first=%08x state=%08x after=%08x samples=%u ns=%llu limit_us=%u " ++ "ret=%d end=r74e\n", ++ r->settle_first, r->early_state, r->early_after_state, ++ r->settle_samples, r->settle_ns, r->settle_us, ret); ++ if (r->nopatch && READ_ONCE(b->control_submitted)) ++ g5p_np_print(g, "pre-bell", 0, 0, r->np_pre); ++ px = atomic_read(&r->early_proof); ++ /* raw: the proof read, or U32_MAX when it did not happen (a refusal or no take). */ ++ if (px != G5P_PX_NONE) ++ dev_emerg( ++ g->dev, ++ "R74E112 proof owner=%u state=%s why=%s gen=%u/%u raw=%08x release_ready=%u " ++ "power_abort=%u end=r74e112\n", ++ owner, g5p_px_name[min_t(u32, px, G5P_PX_N - 1)], ++ g5p_px_why_name[min_t(u32, r->early_proof_why, ++ G5P_PXW_N - 1)], ++ r->early_proof_gen, r->early_proof_gen_now, ++ px == G5P_PX_PROVEN || ++ r->early_proof_why == G5P_PXW_STATE ? ++ r->early_state : ++ U32_MAX, ++ r->early_release_ready, r->early_power_abort); ++out: ++ atomic_set(&r->early_busy, 0); ++ return ret; ++} ++ ++static int g5p_control_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 state, off; ++ int ret; ++ ++ if (b->control_submitted) ++ return b->r73b && 1 ? 0 : -EPERM; ++ if (!b->control_ready) ++ return -EPERM; ++ /* R73 synchronous pre-publish pass closes the race where trace records ++ * do not raise a final IRQ. A negative result is scoreable only when the ++ * factor reaches cmd8 with every outbound auxiliary consumer caught up. ++ */ ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_PUBLISH, true); ++ if (ret) ++ return ret; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ dev_info(g->dev, "DC raw R1+01000=%08x (require active before kick)\n", ++ state); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_dc_snapshot(g, "before-publish", true); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ /* ca05d4: full dmb ish after producer, before Fender and MTS. */ ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_PUBLISH, 0); ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *src = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ if (g5p_le32(src) != (i ? 0 : 8)) ++ return -EBADMSG; ++ } ++ if (READ_ONCE(ctl[4]) != b->control_target) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC slot verified words=00000008/0/0/0/0 producer=%u; full barrier\n", ++ b->control_target); ++ ret = g5p_prepare_kick(g, "DC"); ++ if (ret) ++ return ret; ++ ; ++ if (b->r73_aux_drain) { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_KICK, true); ++ if (ret) ++ return ret; ++ } ++ g5p_dc_snapshot(g, "before-kick", false); ++ dev_info( ++ g->dev, ++ "DC publish cmd=8 slot=%u target=%u; W R0+00b00=%08x variant=%u\n", ++ producer, b->control_target, b->dc_variant == 4 ? 0x12 : 0x10, ++ b->dc_variant); ++ /* 0x12 is verified for CL, NOT for DC: variant 4 tests dispatch routing. */ ++ writel(7 == 4 ? 0x12 : 0x10, g->rgx + 0xb00); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_KICK, 0); ++ g5p_dc_snapshot(g, "after-kick", false); ++ return 0; ++} ++ ++/* perf60 pin >100 (seed 1-5, poweron): the policy record as the firmware left it, from RAM, plus the ++ * real Fender state through an R1-only ticket of the register licence (any pstate, bit 29 allowed). ++ * The sampler 0x1000aa80 stores its own "now" at policy+0x918 on every call (0x1000ab44) and the ++ * current state at +0x34 (0x1000ab18), so fw_now against the host's arch timer names the ++ * firmware's clock, and +0x34 / +0 / +0x8f8 name the branch it took. ++ */ ++static void g5p_perf_readback(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 pol = b->layout.offset[0x4a], meta = b->layout.offset[0x4d]; ++ u64 host = arch_timer_read_counter(), rate = arch_timer_get_rate(); ++ u64 p8f8, p918, p920, m784; ++ struct g5p_mmio_ticket t; ++ ++ if (!b->gpu_perf || (!b->perf_seed && !b->perf_poweron)) ++ return; ++ dma_rmb(); ++ p8f8 = g5p_dc_root_word(b, pol + 0x8f8) | ++ (u64)g5p_dc_root_word(b, pol + 0x8fc) << 32; ++ p918 = g5p_dc_root_word(b, pol + 0x918) | ++ (u64)g5p_dc_root_word(b, pol + 0x91c) << 32; ++ p920 = g5p_dc_root_word(b, pol + 0x920) | ++ (u64)g5p_dc_root_word(b, pol + 0x924) << 32; ++ m784 = g5p_dc_root_word(b, meta + 0x784) | ++ (u64)g5p_dc_root_word(b, meta + 0x788) << 32; ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ dev_emerg( ++ g->dev, ++ "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " ++ "pol28=%u pol2c=%u pol30=%u pol34=%u p8f8=%llx fw_now918=%llx p920=%llx " ++ "meta784=%llx host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " ++ "host_minus_fw_counter_us=%lld r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", ++ why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, ++ g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30), ++ g5p_dc_root_word(b, pol + 0x34), p8f8, p918, p920, m784, host, ++ b->seed_stamp, ++ b->seed_stamp ? ++ div_s64((s64)(p918 - m784 - b->seed_stamp) * 1000000LL, ++ rate) : ++ 0, ++ div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate), ++ g5p_mmio_did(&t), t.gpc, t.r1_0); ++} ++ ++static int g5p_r73c_fast_control(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ int ret; ++ ++ r->fast_release_ns = ktime_get_mono_fast_ns(); ++ WRITE_ONCE(r->fast_window, true); ++ ret = g5p_boot_events(g); ++ r->fast_alive_ns = ktime_get_mono_fast_ns(); ++ if (!ret) ++ ret = g5p_control_prepare(g); ++ if (!ret) { ++ r->fast_state = readl(g->window[1] + 0x1000); ++ ret = g5p_control_submit(g); ++ } ++ r->fast_submit_ns = ktime_get_mono_fast_ns(); ++ r->fast_ret = ret; ++ WRITE_ONCE(r->fast_window, false); ++ g5p_r73b_events_exit(g, ret); ++ dev_info( ++ g->dev, ++ "R73C fast release_to_alive_ns=%llu release_to_submit_ns=%llu state=%08x ret=%d " ++ "control_submitted=%u end=r73c\n", ++ r->fast_alive_ns - r->fast_release_ns, ++ r->fast_submit_ns - r->fast_release_ns, r->fast_state, ret, ++ READ_ONCE(b->control_submitted)); ++ g5p_perf_readback(g, ret ? "alive-timeout" : "alive"); ++ return ret; ++} ++ ++/* perf60 pin 200 (docs/gpu/experiments/perf60-clock.md "Pin 200: the guarded drain read" and ++ * "Clock-change drain RE" §4): at a boot control timeout with boot_gpu_perf set (before any job, ++ * the firmware alive), every read goes through the GPU register licence (mmio.inc) and nothing ++ * wakes the GPU (no R1+0x14 write, mmio-access.md G6): ++ * r1: R1 +0, +8, +0x14, +0x1000, +0x1004, +0x20000 (one R1-only ticket, any pstate) ++ * drain: 11f00 688 1720 1728 e230 e238 e138, then the clock words R0+0x0/0x4/0x8/0xc ++ * timer: R0+0x160/0x164, twice, 1 ms apart ++ * tx: T0/T1 TXENABLE/TXSTATUS (0x04800000/10, 0x04801000/10) through the META slave port ++ * pc: (bit 1 only) T0 PC: RQ 0x0480fff8 = 0x10005, poll bit 31, read 0x0480fff0; x8, 100 us ++ * apart. This is the driver's first slave-port write (CTRL0 = addr, DATAT = value). ++ * One ticket per group or slave-port op (<= G5P_MMIO_MAX_OPS), a refusal retried <= 2 ms. ++ */ ++ ++#define G5P_GPP_BUDGET_US 2000U ++ ++/* One META slave-port read in one ticket: ready, CTRL0 = addr | 1, read back, ready (<= 4 ++ * polls), DATAX: 8 ops. -EBUSY when the port is not ready within the ticket. ++ */ ++static int g5p_mmio_sp_read(struct g5p_diag *g, u32 addr, u32 *out, u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1 = 0; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr | 1)) { ++ g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL0); ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ *out = g5p_mmio_r0_read(g, &t, G5P_MSLVDATAX); ++ ret = 0; ++ break; ++ } ++ } ++ } ++ if (!g5p_mmio_release(g, &t) && !ret) ++ ret = -ESTALE; ++ return ret; ++} ++ ++/* One META slave-port write in one ticket: ready, CTRL0 = addr, DATAT = value, ready (<= 4). */ ++static int g5p_mmio_sp_write(struct g5p_diag *g, u32 addr, u32 value, ++ u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr) && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVDATAT, value)) { ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ ret = 0; ++ break; ++ } ++ } ++ } ++ g5p_mmio_release(g, &t); ++ return ret; ++} ++ ++/* A group of plain R0 reads (<= G5P_MMIO_MAX_OPS) under one ticket; returns the ticket's did. */ ++static const char *g5p_gpp_r0(struct g5p_diag *g, const u32 *off, u32 *v, u32 n, ++ u32 *tries, u32 *gpc) ++{ ++ struct g5p_mmio_ticket t; ++ ++ if (g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries)) { ++ *gpc = t.gpc; ++ return g5p_mmio_did(&t); ++ } ++ for (u32 i = 0; i < n; i++) ++ v[i] = g5p_mmio_r0_read(g, &t, off[i]); ++ g5p_mmio_release(g, &t); ++ *gpc = t.gpc; ++ g5p_mmio_sample_line(g, G5P_MMS_PROBE, &t, v, n, true); ++ return g5p_mmio_did(&t); ++} ++ ++/* What a slave-port op's return means, for the dc-probe lines. */ ++static const char *g5p_sp_why(int ret) ++{ ++ switch (ret) { ++ case 0: ++ return "ok"; ++ case -ENODATA: ++ return "not-run"; ++ case -EPERM: ++ return "forbidden-after-release"; /* g->reset_transition: boot.inc's post-release ban */ ++ case -ENODEV: ++ return "licence-off"; ++ case -EAGAIN: ++ return "licence-refused"; ++ case -EBUSY: ++ return "port-not-ready"; ++ case -ESTALE: ++ return "stale"; ++ case -ETIMEDOUT: ++ return "rq-no-ready"; ++ default: ++ return "error"; ++ } ++} ++ ++static void g5p_gpu_perf_probe(struct g5p_diag *g) ++{ ++ static const u32 drain[] = { 0x11f00, 0x688, 0x1720, 0x1728, ++ 0xe230, 0xe238, 0xe138 }; ++ static const u32 clk[] = { 0x0, 0x4, 0x8, 0xc }; ++ static const u32 timer[] = { 0x160, 0x164 }; ++ static const u32 r1off[] = { 0x0, 0x8, 0x14, 0x1000, 0x1004, 0x20000 }; ++ static const u32 tx[] = { 0x04800000, 0x04800010, 0x04801000, ++ 0x04801010 }; ++ u32 mode = (0); ++ struct g5p_boot *b = g->boot; ++ u32 vd[ARRAY_SIZE(drain)], vc[ARRAY_SIZE(clk)], vt[2][2], ++ r1[ARRAY_SIZE(r1off)]; ++ u32 vx[ARRAY_SIZE(tx)], pc[8], pcrq[8], gate = U32_MAX, tries = 0, ++ gpc = U32_MAX, gpc_x; ++ int txr[ARRAY_SIZE(tx)], pcr[8]; ++ const char *d_r1 = "skip-guard", *d_drain = "skip-guard", ++ *d_clk = "skip-guard"; ++ const char *d_t0 = "skip-guard", *d_t1 = "skip-guard"; ++ struct g5p_mmio_ticket t; ++ ++ if (!mode || !b->gpu_perf) ++ return; ++ memset(vd, 0xff, sizeof(vd)); ++ memset(vc, 0xff, sizeof(vc)); ++ memset(vt, 0xff, sizeof(vt)); ++ memset(r1, 0xff, sizeof(r1)); ++ memset(vx, 0xff, sizeof(vx)); ++ memset(pc, 0xff, sizeof(pc)); ++ memset(pcrq, 0xff, sizeof(pcrq)); ++ for (u32 i = 0; i < ARRAY_SIZE(txr); i++) ++ txr[i] = -ENODATA; ++ for (u32 i = 0; i < ARRAY_SIZE(pcr); i++) ++ pcr[i] = -ENODATA; ++ if (g->pmgr && regmap_read(g->pmgr, 0x80340, &gate)) ++ gate = U32_MAX; ++ if (READ_ONCE(b->failed)) ++ goto print; ++ /* r1: before anything else, R1-only (valid at any pstate and with bit 29 set) */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) { ++ for (u32 i = 0; i < ARRAY_SIZE(r1off); i++) ++ r1[i] = g5p_mmio_r1_read(g, &t, r1off[i]); ++ g5p_mmio_release(g, &t); ++ } ++ d_r1 = g5p_mmio_did(&t); ++ /* gpc: the drain group's licence read; every group's ticket also has its own sample line */ ++ d_drain = g5p_gpp_r0(g, drain, vd, ARRAY_SIZE(drain), &tries, &gpc); ++ d_clk = g5p_gpp_r0(g, clk, vc, ARRAY_SIZE(clk), &tries, &gpc_x); ++ d_t0 = g5p_gpp_r0(g, timer, vt[0], 2, &tries, &gpc_x); ++ usleep_range(1000, 1100); ++ d_t1 = g5p_gpp_r0(g, timer, vt[1], 2, &tries, &gpc_x); ++ for (u32 i = 0; i < ARRAY_SIZE(tx); i++) ++ txr[i] = g5p_mmio_sp_read(g, tx[i], &vx[i], &tries); ++ if (mode & 2) { ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) { ++ pcr[i] = g5p_mmio_sp_write(g, 0x0480fff8, 0x10005, ++ &tries); ++ for (u32 k = 0; !pcr[i] && k < 4; k++) { ++ pcr[i] = g5p_mmio_sp_read(g, 0x0480fff8, ++ &pcrq[i], &tries); ++ if (pcr[i] || (pcrq[i] & BIT(31))) ++ break; ++ udelay(10); ++ } ++ if (!pcr[i]) ++ pcr[i] = pcrq[i] & BIT(31) ? ++ g5p_mmio_sp_read(g, 0x0480fff0, ++ &pc[i], ++ &tries) : ++ -ETIMEDOUT; ++ udelay(100); ++ } ++ } ++print: ++ dev_err(g->dev, ++ "GPUPERF dc-probe did=%s gate=%08x r1=%s r1_0=%08x r1_8=%08x r1_14=%08x " ++ "r1_1000=%08x r1_1004=%08x r1_20000=%08x gpc=%08x tries=%u end=gpuperf\n", ++ d_drain, gate, d_r1, r1[0], r1[1], r1[2], r1[3], r1[4], r1[5], ++ gpc, tries); ++ dev_err(g->dev, ++ "GPUPERF dc-probe drain=%s 11f00=%08x 688=%08x 1720=%08x 1728=%08x e230=%08x " ++ "e238=%08x e138=%08x clk=%s clk=%08x:%08x clkstat=%08x:%08x end=gpuperf\n", ++ d_drain, vd[0], vd[1], vd[2], vd[3], vd[4], vd[5], vd[6], d_clk, ++ vc[1], vc[0], vc[3], vc[2]); ++ dev_err(g->dev, ++ "GPUPERF dc-probe timer=%s/%s 160=%08x/%08x 164=%08x/%08x gap_us=1000 " ++ "txen0=%08x/%s txst0=%08x/%s txen1=%08x/%s txst1=%08x/%s end=gpuperf\n", ++ d_t0, d_t1, vt[0][0], vt[1][0], vt[0][1], vt[1][1], vx[0], ++ g5p_sp_why(txr[0]), vx[1], g5p_sp_why(txr[1]), vx[2], ++ g5p_sp_why(txr[2]), vx[3], g5p_sp_why(txr[3])); ++ if (mode & 2) ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) ++ dev_err(g->dev, ++ "GPUPERF dc-probe pc i=%u ret=%s rq=%08x t0pc=%08x end=gpuperf\n", ++ i, g5p_sp_why(pcr[i]), pcrq[i], pc[i]); ++} ++ ++static int g5p_control_wait(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long start = jiffies, deadline = start + 3 * HZ; ++ bool sampled = false, rekicked = false; ++ u32 consumer, producer, pass = 0; ++ int ret; ++ ++ if (!b->control_submitted || b->control_complete) ++ return -EPERM; ++ do { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_WAIT, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ consumer = READ_ONCE(ctl[0]); ++ producer = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ if (consumer > 255 || producer != b->control_target || ++ (consumer != b->control_target && ++ consumer != ((b->control_target - 1) & 255))) ++ return -EPROTO; ++ if (consumer == b->control_target) { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (g5p_r74_case(b) == 1) { ++ WRITE_ONCE(b->control_complete, true); ++ g5p_r74_mark(g, G5P_R74_CMD8_CONSUMED, ++ consumer); ++ dev_emerg( ++ g->dev, ++ "R74B cmd8-consumed consumer=%u producer=%u IRQ174=%d ram_only=1 end=r74b\n", ++ consumer, producer, ++ atomic_read(&b->irqs)); ++ g5p_np_policy(g, "cmd8-consumed", consumer, 0); ++ return 0; ++ } ++ g5p_dc_snapshot(g, "consumed", false); ++ b->control_complete = true; ++ ; ++ dev_info( ++ g->dev, ++ "DC consumed cmd=8 consumer=%u producer=%u IRQ174=%d; transport only\n", ++ consumer, producer, atomic_read(&b->irqs)); ++ return 0; ++ } ++ if (!sampled && !time_before(jiffies, start + HZ / 10)) { ++ sampled = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "100ms", false); ++ g5p_np_policy(g, "100ms", consumer, 0); ++ } ++ if (7 == 3 && !rekicked && !time_before(jiffies, start + HZ)) { ++ /* Re-notify the SAME slot once; never publish a second command. */ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_prepare_kick(g, "DC re-kick"); ++ if (ret) ++ return ret; ++ mb(); ++ writel(0x10, g->rgx + 0xb00); ++ rekicked = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "re-kick", false); ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "timeout", false); ++ g5p_np_policy(g, "timeout", consumer, 0); ++ dev_err(g->dev, ++ "DC timeout consumer=%u producer=%u target=%u; retain RAM, no retry\n", ++ consumer, producer, b->control_target); ++ /* perf60 (boot_gpu_perf): the policy-state words at the stall, RAM only. pol34 = the pstate the ++ * firmware last requested (its setter stores it before the Fender write): q means it reached the ++ * setter, 1 that it stopped before (the clock-change drain 0x1001cec8 -> 0x1001b638). ++ */ ++ if (b->gpu_perf) { ++ u32 pol = b->layout.offset[0x4a]; ++ ++ dma_rmb(); ++ dev_err(g->dev, ++ "GPUPERF dc-timeout pin=%u pol8=%u/%u q=%u x=%u pol34=%u req=%u/%u ceil=%u/%u " ++ "seed=%u pol0=%u pol4=%u pol28=%u pol2c=%u pol30=%u end=gpuperf\n", ++ b->gpu_perf, g5p_dc_root_word(b, pol + 8), ++ g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x38), ++ g5p_dc_root_word(b, pol + 0x34), ++ g5p_dc_root_word(b, pol + 0x6f0), ++ g5p_dc_root_word(b, pol + 0x6f4), ++ g5p_dc_root_word(b, pol + 0x688), ++ g5p_dc_root_word(b, pol + 0x68c), b->perf_seed, ++ g5p_dc_root_word(b, pol + 0), ++ g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30)); ++ g5p_gpu_perf_probe(g); ++ g5p_perf_readback(g, "dc-timeout"); ++ } ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ b->control_timed_out = true; ++ return -ETIMEDOUT; ++} ++ ++static int g5p_irq_test(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + HZ / 10; ++ int before = atomic_read(&b->irqs), ret; ++ ++ if (b->irq_tested || !b->irq_ready || g5p_r74_case(b)) ++ return -EPERM; ++ /* ADT set/ack addresses, not a GPU slave-port transaction. Firmware may ++ * also interrupt concurrently: this tests delivery, not event provenance. ++ */ ++ dev_info(g->dev, "IRQ test baseline=%d W AIC+014=00004000\n", before); ++ writel(0x4000, b->swirq + 0x14); ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (atomic_read(&b->irqs) > before) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ b->irq_tested = true; ++ ; ++ dev_info( ++ g->dev, ++ "IRQ delivery observed before=%d after=%d; no shader completion implied\n", ++ before, atomic_read(&b->irqs)); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, "IRQ delivery timeout before=%d after=%d\n", before, ++ atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++static int g5p_boot_park(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ static const u32 slots[] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ bool failed_control = READ_ONCE(b->failed) && b->control_timed_out; ++ bool failed_work = READ_ONCE(b->failed) && b->work_timed_out; ++ bool failed_context = READ_ONCE(b->failed) && b->context_timed_out; ++ bool failed_timeout = failed_control || failed_work || failed_context; ++ int ret; ++ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ if (READ_ONCE(b->event_failed) || ++ (b->control_submitted && !b->control_complete && !failed_control) || ++ (b->work_submitted && !b->work_complete && !failed_work) || ++ (b->context_submitted && !b->context_complete && !failed_context)) ++ return -EBUSY; ++ /* A failed drain must never be bypassed. On the sole permitted failure ++ * path require the event ring empty below without consuming anything. ++ */ ++ if (!failed_timeout) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(slots); i++) { ++ u32 *ctl = (u32 *)g5p_boot_shared(b, slots[i]); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (((failed_control || failed_context) && slots[i] == 0x6c) || ++ (failed_work && slots[i] == 0x68)) { ++ u32 target = failed_context && slots[i] == 0x6c ? ++ b->context_target : ++ slots[i] == 0x6c ? b->control_target : ++ b->work_target; ++ ++ if (producer != target || ++ (consumer != producer && ++ consumer != ((producer - 1) & 255))) ++ return -EPROTO; ++ } else if (consumer != producer) { ++ return -EBUSY; ++ } ++ } ++ if (READ_ONCE(b->event_failed)) ++ return -EIO; ++ if (failed_timeout) ++ dev_info( ++ g->dev, ++ "PARK failed %s timeout; abandon command, preserve failure and RAM\n", ++ failed_context ? "CONTEXT" : ++ failed_work ? "CL" : ++ "DC"); ++ /* Freeze commands before the first transition; no retry even if it fails. ++ * c9fa10 enabled-Fender branch: core=0, GPC=0, no a18 access. ++ */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ dev_info( ++ g->dev, ++ "PARK begin core=0 GPC=0; wrapper only, DMA memory retained\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ return ret; ++ /* Keep the IRQ/ack handler and every allocation alive until reboot. ++ * GPC=0 alone is not proof that outstanding bus traffic has drained. ++ */ ++ b->parked = true; ++ dev_info( ++ g->dev, ++ "PARK GPC=0 verified; quarantine until reboot, no PMGR off or DMA free\n"); ++ return 0; ++} ++ ++static int g5p_lifecycle_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (!boot_experiment || !b || !b->alive || b->stopped) ++ return -EPERM; ++ if (b->failed && ++ !(sysfs_streq(command, "park") && teardown_experiment && ++ (b->control_timed_out || b->work_timed_out || ++ b->context_timed_out))) ++ return -EPERM; ++ if (!strncmp(command, "control-", 8) && control_experiment) { ++ if (sysfs_streq(command, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(command, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(command, "control-wait")) ++ ret = g5p_control_wait(g); ++ else ++ return -EINVAL; ++ } else if (sysfs_streq(command, "irq-test") && ++ (g5p_r74_case(b) || (1))) { ++ /* R74 never runs the AIC self-test r73f case 01 stopped in. */ ++ if (b->r73b) ++ atomic_inc(&b->r73b->r74_irq_refused); ++ g5p_r74_mark(g, G5P_R74_IRQ_TEST_REFUSED, -EPERM); ++ dev_err(g->dev, ++ "R74 refuse irq-test case=%u irq_tested=%u end=r74\n", ++ g5p_r74_case(b), b->irq_tested); ++ return -EPERM; ++ } else if (sysfs_streq(command, "irq-test") && irq_experiment) { ++ ret = g5p_irq_test(g); ++ } else if (sysfs_streq(command, "park") && teardown_experiment) { ++ ret = g5p_boot_park(g); ++ } else { ++ return -EPERM; ++ } ++ if (sysfs_streq(command, "control-wait")) ++ g5p_r73b_crumb(b, G5P_R73B_WAIT_EXIT, G5P_R73B_OWNER_WAIT, ret); ++ if (ret && ret != -EALREADY && ret != -EPERM) ++ WRITE_ONCE(b->failed, true); ++ return ret; ++} ++ ++/* Outgoing-side kexec/reboot quiesce; see docs/gpu/experiments/gpu-reset-recovery-production.md ++ * section 6.10. NOT a reset: Fender core=0/GPC=0 only, no PMGR off, no DMA free and ++ * no R0 access. It acts only when the command lock is free (no job or sysfs command is ++ * inside a hardware wait), the firmware was released and the pstate policy is ++ * host-owned (init meta+0x6c8 == 0); every other state is left alone. Unlike ++ * g5p_boot_park() it does not require drained rings: round 10 flew park after an ++ * unconsumed DC command, and at shutdown there is nothing left to drain for. ++ * GPC=0 is not a proof that bus traffic drained (see g5p_boot_park). ++ */ ++ ++static void g5p_td_shutdown(struct g5p_diag *g); ++ ++static void g5p_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ struct g5p_boot *b; ++ int ret; ++ ++ if (!g) ++ return; ++ /* Idempotent second chance after the reboot notifier; a no-op unless teardown != 0. */ ++ g5p_td_shutdown(g); ++ { ++ return; ++ }; ++ /* A worker or sysfs writer stuck in MMIO holds g->lock: never queue behind it. */ ++ if (!mutex_trylock(&g->lock)) { ++ dev_warn(g->dev, "SHUTDOWN park skipped: command lock busy\n"); ++ return; ++ } ++ b = g->boot; ++ /* R1 is touched only with ps_gfx held on (the retained runtime-PM reference), ++ * the wrapper woken and the firmware released. ++ */ ++ if (!b || !g->powered || !g->fender_ready || !b->maps_ready || ++ !b->released || b->stopped || b->parked) { ++ dev_info(g->dev, "SHUTDOWN park not needed\n"); ++ goto out; ++ } ++ /* With firmware-owned policy (init meta+0x6c8 != 0) the firmware also writes ++ * R1+0x1000, so a host write would race it (dsx-r74b). RAM-only read of the ++ * host-written init word, the same one g5p_dc_snapshot() reports. ++ */ ++ if (g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8)) { ++ dev_info( ++ g->dev, ++ "SHUTDOWN park refused: firmware-owned policy (meta+0x6c8 != 0)\n"); ++ goto out; ++ } ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ /* Freeze commands before the first transition; no retry even if it fails. */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ ret = g5p_fender_idle(&io); ++ if (!ret) ++ WRITE_ONCE(b->parked, true); ++ dev_info( ++ g->dev, ++ "SHUTDOWN park core=0 GPC=0 result=%d; DMA memory retained, no PMGR off\n", ++ ret); ++out: ++ mutex_unlock(&g->lock); ++} +diff --git a/drivers/gpu/apple-g5p/meta71c114.h b/drivers/gpu/apple-g5p/meta71c114.h +new file mode 100644 +index 0000000000..cc84dbd821 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/meta71c114.h +@@ -0,0 +1,65 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency114: one RAM dword, meta+0x71c, in the bootstrap image. ++ * ++ * meta (object 0x4d) +0x718 is the firmware's pstate frequency table in MHz. ++ * Its init copies the table to private+0x26c at boot (MINIM 0x1000122c..127c), ++ * and the scheduler loop reloads its timer from entry 1 on both branches ++ * (0x8000a1a4 via private+0x2c4 at pstate 1, 0x8000a338 via private+0x270). ++ * Linux leaves the table zero; Apple has { 0, 340, 474, 550, 723 }. The A/B ++ * installs entry 1 only: 0 is the control (nothing is stored, variant 7 leaves ++ * the word zero), 340 the factor. Everything else is refused at map, before ++ * any store. The store goes into the staging buffer, before g5p_boot_copy() ++ * publishes the image and long before the firmware is released. There is no ++ * register access here. ++ */ ++#ifndef APPLE_G5P_META71C114_H ++#define APPLE_G5P_META71C114_H ++ ++#define G5P_M71C_SLOT 0x4dU ++#define G5P_M71C_OFFSET 0x71cU ++#define G5P_M71C_CONTROL 0U ++#define G5P_M71C_FACTOR 340U ++#define G5P_M71C_VARIANT 7U ++ ++enum { ++ G5P_M71C_OK, ++ G5P_M71C_BAD_VALUE, ++ G5P_M71C_BAD_VARIANT, ++ G5P_M71C_WHYS, ++}; ++ ++static const char *const g5p_m71c_whys[G5P_M71C_WHYS] = { ++ "none", ++ "value", ++ "variant", ++}; ++ ++/* Latched at map: only the two pre-registered arms, the factor on dc7 only. */ ++static inline u32 g5p_m71c_check(u32 requested, u32 variant) ++{ ++ if (requested != G5P_M71C_CONTROL && requested != G5P_M71C_FACTOR) ++ return G5P_M71C_BAD_VALUE; ++ if (requested == G5P_M71C_FACTOR && variant != G5P_M71C_VARIANT) ++ return G5P_M71C_BAD_VARIANT; ++ return G5P_M71C_OK; ++} ++ ++/* ++ * One store into the staging image (the factor only) and its read-back. The ++ * caller fails the map unless the read-back equals the request. ++ */ ++static inline u32 g5p_m71c_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 requested, bool *wrote) ++{ ++ u8 *word = g5p_object(p, root, shared, G5P_M71C_SLOT) + G5P_M71C_OFFSET; ++ ++ *wrote = false; ++ if (requested == G5P_M71C_FACTOR) { ++ g5p_put32(word, requested); ++ *wrote = true; ++ } ++ return g5p_le32(word); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio-licence.h b/drivers/gpu/apple-g5p/mmio-licence.h +new file mode 100644 +index 0000000000..d9d23ca2ab +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio-licence.h +@@ -0,0 +1,148 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU register licence (docs/gpu/experiments/mmio-access.md §4), pure so the host test ++ * (scripts/gpu/perf60-host/test_mmio.c) drives the same code. iOS's ++ * AGXMetaFirmware_isGPURegisterAccessible (fffffff006ca10ec) licenses an R0 access when PMGR gfx ++ * is on and the wrapper is up (fender+0xc), R1+0 bit 29 is clear (the firmware's power-down / ++ * clock-change routine 0x1001cec8 raises it before the GPC goes to 0) and the GPC pstate nibble is ++ * 1..meta+0x710. We are stricter: R1+0x1000 bit 4 (a transition in flight) must be clear and ++ * R1+0 == 0 (a dead wrapper, Apple's ca05d4 vt+0x70 branch) refuses too. ++ * ++ * Order, and nothing else touches the bus: the RAM terms (no MMIO when any fails), then R1+0, then ++ * R1+0x1000, then at most G5P_MMIO_MAX_OPS R0 accesses, then R1+0x1000 and R1+0 again at release ++ * to account a sample whose licence changed under it (stale: data loss, not exclusion). ++ * A refusal never waits and never wakes (no R1+0x14 write): the caller decides whether to retry. ++ */ ++#ifndef G5P_MMIO_LICENCE_H ++#define G5P_MMIO_LICENCE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_MMIO_MAX_OPS 8U /* R0 accesses per ticket */ ++#define G5P_MMIO_R1_CTRL \ ++ 0x0U /* R1+0: bit 29 = firmware power-down in progress */ ++#define G5P_MMIO_R1_GPC \ ++ 0x1000U /* R1+0x1000: bit 4 transition, nibble = pstate */ ++#define G5P_MMIO_BIT29 BIT(29) ++ ++enum g5p_mmio_why { ++ G5P_MMIO_OK, ++ G5P_MMIO_OFF, /* RAM: power epoch not readable (no MMIO done) -> -ENODEV */ ++ G5P_MMIO_BUSY, /* the lock was held (try mode, IRQ census) -> -EAGAIN, no MMIO */ ++ G5P_MMIO_DEAD, /* R1+0 == 0 */ ++ G5P_MMIO_FWOFF, /* R1+0 bit 29 */ ++ G5P_MMIO_TRANS, /* R1+0x1000 bit 4 */ ++ G5P_MMIO_PARKED, /* nibble 0 */ ++ G5P_MMIO_OVERMAX, /* nibble > max pstate */ ++ G5P_MMIO_PARKEPOCH, /* R0 in a park epoch (g5p_mmio_r0_regime_ok) */ ++ G5P_MMIO_NR, ++}; ++ ++static const char *const g5p_mmio_why_names[G5P_MMIO_NR] = { ++ "read", "skip-off", "skip-busy", ++ "skip-dead", "skip-bit29", "skip-transition", ++ "skip-parked", "skip-overmax", "skip-park-epoch", ++}; ++ ++/* The RAM preconditions (§4.2 step 2). b->failed is deliberately not one: the fault read runs ++ * after it latches, as Apple's does. ++ */ ++struct g5p_mmio_ram { ++ bool powered, fender_ready, mmio_off, have_boot, released, stopped; ++}; ++ ++static inline bool g5p_mmio_ram_ok(const struct g5p_mmio_ram *r) ++{ ++ return r->powered && r->fender_ready && !r->mmio_off && r->have_boot && ++ r->released && !r->stopped; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_r1_0(u32 r1_0) ++{ ++ if (!r1_0) ++ return G5P_MMIO_DEAD; ++ if (r1_0 & G5P_MMIO_BIT29) ++ return G5P_MMIO_FWOFF; ++ return G5P_MMIO_OK; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_gpc(u32 gpc, u32 max) ++{ ++ if (gpc & 0x10) ++ return G5P_MMIO_TRANS; ++ if (!(gpc & 0xf)) ++ return G5P_MMIO_PARKED; ++ if ((gpc & 0xf) > max) ++ return G5P_MMIO_OVERMAX; ++ return G5P_MMIO_OK; ++} ++ ++/* ++ * R0 only where the firmware cannot park under it. The governor's park does not raise R1+0 ++ * bit 29 (mmio-access.md §2.3: only the power-down routine 0x1001cec8 does, a few calls before it ++ * drops the GPC), so against a park the licence's R1 checks are a race: a check that passes can be ++ * followed by the GPC going to 0 before the R0 load. The never-park envelope (boot_gpu_perf >= ++ * 100, latched at map: the policy writer's x >= 100, q >= 1, the governor never selects pstate 0) ++ * has no such window; at pin 400/500 the 3D-completion and clock samples saw bit 29 0 times, while ++ * the park tuple's rung 5 (2026-11-03) saw it on 749 of 983 3D completions and then froze the SoC. ++ * iOS reads R0 only at a fault event, never at idle. `override` = mmio_r0_park (DFU-class rungs ++ * only). ++ */ ++static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool override) ++{ ++ return override || gpu_perf >= 100; ++} ++ ++/* ++ * meta+0x710 as written into the boot image; 0 (no table) means the firmware stays at pstate 1. ++ */ ++static inline u32 g5p_mmio_max_pstate(u32 meta710) ++{ ++ return meta710 < 1 ? 1 : meta710 > 15 ? 15 : meta710; ++} ++ ++typedef u32 (*g5p_mmio_rd_t)(void *ctx, u32 off); ++ ++/* §4.2 steps 2-4 under the caller's lock: no bus access unless the RAM terms pass, R1+0 before ++ * R1+0x1000, nothing after a refusal. *r1_0 / *gpc stay U32_MAX when not read. ++ */ ++static inline enum g5p_mmio_why g5p_mmio_licence(const struct g5p_mmio_ram *ram, ++ u32 max, g5p_mmio_rd_t rd, ++ void *ctx, u32 *r1_0, u32 *gpc) ++{ ++ enum g5p_mmio_why why; ++ ++ *r1_0 = 0xffffffffU; ++ *gpc = 0xffffffffU; ++ if (!g5p_mmio_ram_ok(ram)) ++ return G5P_MMIO_OFF; ++ *r1_0 = rd(ctx, G5P_MMIO_R1_CTRL); ++ why = g5p_mmio_check_r1_0(*r1_0); ++ if (why) ++ return why; ++ *gpc = rd(ctx, G5P_MMIO_R1_GPC); ++ return g5p_mmio_check_gpc(*gpc, max); ++} ++ ++/* An R0 op is allowed while ops < G5P_MMIO_MAX_OPS; the counter counts refusals too. */ ++static inline bool g5p_mmio_op_take(u32 *ops) ++{ ++ return (*ops)++ < G5P_MMIO_MAX_OPS; ++} ++ ++/* §4.3 release: the licence fields (nibble, bit 4, bit 29) unchanged and still licensing. */ ++static inline bool g5p_mmio_stale(u32 r1_0, u32 gpc, u32 r1_0_end, u32 gpc_end, ++ u32 max) ++{ ++ if ((gpc & 0x1f) != (gpc_end & 0x1f) || ++ (r1_0 & G5P_MMIO_BIT29) != (r1_0_end & G5P_MMIO_BIT29)) ++ return true; ++ return g5p_mmio_check_r1_0(r1_0_end) || ++ g5p_mmio_check_gpc(gpc_end, max); ++} ++ ++/* mmio_selftest rung 3/5 expectations: CORE_REVISION and CORE_ID as read at bring-up. */ ++#define G5P_MMIO_EXPECT_20 0x00090f00U ++#define G5P_MMIO_EXPECT_18 0x011b0602U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio.inc b/drivers/gpu/apple-g5p/mmio.inc +new file mode 100644 +index 0000000000..b8c25bd8d7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio.inc +@@ -0,0 +1,295 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The one GPU register accessor (docs/gpu/experiments/mmio-access.md §4), Apple's licence from ++ * AGXMetaFirmware_isGPURegisterAccessible (mmio-licence.h) under g->mmio_lock: ++ * ++ * if (!g5p_mmio_hold(g, &t, 0)) { ++ * v = g5p_mmio_r0_read(g, &t, off); // <= G5P_MMIO_MAX_OPS, nothing else in between ++ * g5p_mmio_release(g, &t); // false: the licence changed under the sample ++ * } ++ * ++ * hold takes mmio_lock with IRQs off and returns with it held on success. It refuses instead of ++ * waiting and never wakes the GPU (no R1+0x14 write): -ENODEV when the power epoch is not ++ * readable (RAM only, no MMIO done), -EAGAIN when parked, bit 29, in transition, above the ++ * pstate table, or (G5P_MMIO_TRY) when the lock is taken; -EPERM for an R0 ticket in an epoch ++ * the firmware may park (boot_gpu_perf < 100, unless mmio_r0_park). Callable from hard IRQ, ++ * threaded IRQ and process context, inside event_lock and g->lock. Between hold and release: no ++ * printk, no other MMIO, no sleep, no allocation. ++ * ++ * mmio_selftest (0600, default 0, live) drives the proof rungs of §6: ++ * bit 0 rung 1/1b: 1 kHz hrtimer census, R1+0 and R1+0x1000 only (no R0) ++ * bit 1 rung 3/4: every 100 ms hold -> R0+0x20, R0+0x18 -> release (DFU-risk with parks: rung 4) ++ * bit 2 rung 5: the same at every 3D completion decision (drm side; DFU-risk, park tuple) ++ * bit 3 idle check: after 5, 30 and 120 s with no job, the PMGR gfx word, then (gfx on only) ++ * R1+0x1000 and R1+0; never R0 ++ * Lines: GPU-MMIO sample (per sample, at most one per source per second unless the outcome ++ * changes or the sample is stale/bad), GPU-MMIO idle, GPU-MMIO stats (every 10 s while on and ++ * once when it goes back to 0). ++ */ ++#define G5P_MMIO_TRY \ ++ BIT(0) /* trylock: refuse busy instead of spinning (the IRQ census) */ ++/* An R1-only ticket, counted in the census class: held whenever the RAM terms pass (R1 is ++ * readable while PMGR gfx is on, mmio-access.md §1.3), R1+0 and R1+0x1000 both read even when the ++ * licence refuses; R0 only if the licence passed (t->r0_ok). ++ */ ++#define G5P_MMIO_CENSUS BIT(1) ++ ++struct g5p_mmio_ticket { ++ u32 r1_0, gpc, r1_0_end, gpc_end, flags, ops; ++ u64 t_ns, t_end_ns; ++ unsigned long irq; ++ enum g5p_mmio_why why; ++ bool held, stale, r0_ok; ++}; ++ ++enum { ++ G5P_MMS_CENSUS, ++ G5P_MMS_IDLE, ++ G5P_MMS_3D, ++ G5P_MMS_FAULT, ++ G5P_MMS_PROBE, ++ G5P_MMS_NR ++}; ++static const char *const g5p_mms_names[G5P_MMS_NR] = { ++ "census", "idle-100ms", "3d-done", "fault-read", "dc-probe", ++}; ++ ++#define G5P_MMIO_RING 32 ++struct g5p_mmio_rec { ++ u64 t_ns; ++ u32 r1_0, gpc, us; ++ u8 why, stale; ++}; ++ ++struct g5p_mmio_self { ++ struct g5p_diag *g; ++ struct hrtimer timer; ++ struct delayed_work work; ++ bool census_on, was_on; ++ u64 stats_ns; ++ /* counters: [0] R1-only census tickets, [1] R0 tickets */ ++ atomic_t n[2][G5P_MMIO_NR]; ++ atomic_t stale[2], r0_ops, bad; ++ /* census transitions, written by the hrtimer, drained by the work */ ++ spinlock_t ring_lock; ++ struct g5p_mmio_rec ring[G5P_MMIO_RING]; ++ u32 ring_head, ring_tail, ring_drop; ++ struct g5p_mmio_rec census_last; ++ u8 census_prev_why, census_prev_stale; ++ /* per-source print limiter */ ++ u64 printed_ns[G5P_MMS_NR]; ++ u8 printed_why[G5P_MMS_NR]; ++ /* idle check */ ++ u64 job_ns; ++ u32 idle_done; ++ /* bit 4, the GPU clock: per Fender pstate nibble, the highest rate seen and the sample counts */ ++ u32 clk_max_khz[16], clk_cr_khz[16], clk_n[16], clk_busy[16], clk_reset; ++ u64 clk_printed_ns; ++}; ++ ++/* mmio_selftest bit 4: the gap between the two tickets of one clock sample */ ++ ++/* R0 tickets in a park epoch (boot_gpu_perf < 100): refused unless 1 (g5p_mmio_r0_regime_ok). */ ++ ++static u32 g5p_mmio_rd_r1(void *ctx, u32 off) ++{ ++ return readl((void __iomem *)ctx + off); ++} ++ ++static void g5p_mmio_count(struct g5p_diag *g, const struct g5p_mmio_ticket *t) ++{ ++ struct g5p_mmio_self *s = READ_ONCE(g->mmio_self); ++ u32 cls = t->flags & G5P_MMIO_CENSUS ? 0 : 1; ++ ++ if (!s) ++ return; ++ atomic_inc(&s->n[cls][t->why]); ++ if (t->stale) ++ atomic_inc(&s->stale[cls]); ++ if (t->ops && cls) ++ atomic_add(min_t(u32, t->ops, G5P_MMIO_MAX_OPS), &s->r0_ops); ++} ++ ++static int g5p_mmio_hold(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags) ++{ ++ struct g5p_mmio_ram ram = { 0 }; ++ struct g5p_boot *b; ++ unsigned long irq; ++ ++ memset(t, 0, sizeof(*t)); ++ t->flags = flags; ++ t->r1_0 = t->gpc = t->r1_0_end = t->gpc_end = U32_MAX; ++ if (flags & G5P_MMIO_TRY) { ++ if (!spin_trylock_irqsave(&g->mmio_lock, irq)) { ++ t->why = G5P_MMIO_BUSY; ++ goto refused; ++ } ++ } else { ++ spin_lock_irqsave(&g->mmio_lock, irq); ++ } ++ t->t_ns = ktime_get_mono_fast_ns(); ++ ram.mmio_off = g->mmio_off; ++ ram.powered = READ_ONCE(g->powered); ++ ram.fender_ready = READ_ONCE(g->fender_ready); ++ /* mmio_off first: a boot struct behind a gate is never dereferenced */ ++ b = ram.mmio_off ? NULL : READ_ONCE(g->boot); ++ if (b) { ++ ram.have_boot = true; ++ ram.released = READ_ONCE(b->released); ++ ram.stopped = READ_ONCE(b->stopped); ++ } ++ /* before any bus access: an R0 ticket in a park epoch is refused outright */ ++ if (!(flags & G5P_MMIO_CENSUS) && b && ++ !g5p_mmio_r0_regime_ok(b->gpu_perf, (0))) { ++ t->why = G5P_MMIO_PARKEPOCH; ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ t->why = g5p_mmio_licence(&ram, READ_ONCE(g->mmio_max_pstate), ++ g5p_mmio_rd_r1, g->window[1], &t->r1_0, ++ &t->gpc); ++ if (t->why && (!(flags & G5P_MMIO_CENSUS) || t->why == G5P_MMIO_OFF)) { ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ if (t->gpc == U32_MAX) ++ t->gpc = readl( ++ g->window[1] + ++ G5P_MMIO_R1_GPC); /* census: after an R1+0 refusal */ ++ t->r0_ok = !t->why; ++ t->irq = irq; ++ t->held = true; ++ return 0; ++refused: ++ g5p_mmio_count(g, t); ++ /* a park epoch does not change by waiting: -EPERM, never retried (g5p_mmio_hold_wait) */ ++ return t->why == G5P_MMIO_OFF ? -ENODEV : ++ t->why == G5P_MMIO_PARKEPOCH ? -EPERM : ++ -EAGAIN; ++} ++ ++/* Only between a successful hold and its release; past G5P_MMIO_MAX_OPS no bus access. */ ++static u32 g5p_mmio_r0_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->rgx + off); ++} ++ ++/* R0 writes count as ops too (the slave-port request words of the pin-200 probe). */ ++static bool g5p_mmio_r0_write(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off, u32 v) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return false; ++ writel(v, g->rgx + off); ++ return true; ++} ++ ++/* Any R1 word under a held ticket (census or licensed); counts as an op. */ ++static u32 g5p_mmio_r1_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->window[1] + off); ++} ++ ++/* Process context: refusals (-EAGAIN) retried outside the lock every 20-40 us, <= budget_us. */ ++static int g5p_mmio_hold_wait(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags, u32 budget_us, u32 *tries) ++{ ++ u64 t0 = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ might_sleep(); ++ for (;;) { ++ (*tries)++; ++ ret = g5p_mmio_hold(g, t, flags); ++ if (ret != -EAGAIN || ktime_get_mono_fast_ns() - t0 >= ++ (u64)budget_us * NSEC_PER_USEC) ++ return ret; ++ usleep_range(20, 40); ++ } ++} ++ ++static bool g5p_mmio_release(struct g5p_diag *g, struct g5p_mmio_ticket *t) ++{ ++ if (!t->held) ++ return false; ++ t->gpc_end = readl(g->window[1] + G5P_MMIO_R1_GPC); ++ t->r1_0_end = readl(g->window[1] + G5P_MMIO_R1_CTRL); ++ t->stale = t->r0_ok && ++ g5p_mmio_stale(t->r1_0, t->gpc, t->r1_0_end, t->gpc_end, ++ READ_ONCE(g->mmio_max_pstate)); ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ t->held = false; ++ spin_unlock_irqrestore(&g->mmio_lock, t->irq); ++ g5p_mmio_count(g, t); ++ return !t->stale; ++} ++ ++static const char *g5p_mmio_did(const struct g5p_mmio_ticket *t) ++{ ++ return t->stale ? "stale" : g5p_mmio_why_names[t->why]; ++} ++ ++static u32 g5p_mmio_us(const struct g5p_mmio_ticket *t) ++{ ++ return t->t_end_ns > t->t_ns ? ++ (u32)div_u64(t->t_end_ns - t->t_ns, NSEC_PER_USEC) : ++ 0; ++} ++ ++/* The per-sample line, after release. force: print regardless of the per-source limiter. */ ++static void g5p_mmio_sample_line(struct g5p_diag *g, u32 src, ++ const struct g5p_mmio_ticket *t, u32 *v, ++ u32 nv, bool force) ++{ ++ struct g5p_mmio_self *s = g->mmio_self; ++ u64 now = ktime_get_mono_fast_ns(); ++ char vs[9 * G5P_MMIO_MAX_OPS + 1] = "-"; ++ u32 len = 0; ++ ++ if (s && !force && !t->stale && s->printed_why[src] == t->why && ++ now - s->printed_ns[src] < NSEC_PER_SEC) ++ return; ++ if (s) { ++ s->printed_ns[src] = now; ++ s->printed_why[src] = t->why; ++ } ++ for (u32 i = 0; i < nv && i < G5P_MMIO_MAX_OPS; i++) ++ len += scnprintf(vs + len, sizeof(vs) - len, "%s%08x", ++ i ? "," : "", v[i]); ++ dev_info( ++ g->dev, ++ "GPU-MMIO sample why=%s did=%s r1_0=%08x gpc=%08x r1_0_end=%08x gpc_end=%08x " ++ "max=%u v=%s us=%u end=mmio\n", ++ g5p_mms_names[src], g5p_mmio_did(t), t->r1_0, t->gpc, ++ t->r1_0_end, t->gpc_end, READ_ONCE(g->mmio_max_pstate), vs, ++ g5p_mmio_us(t)); ++} ++ ++/* Rung 3/5: one licensed ticket, CORE_REVISION and CORE_ID. */ ++ ++/* drm side: rung 5 at the 3D completion decision, and the idle clock for bit 3. */ ++ ++/* Bit 4: the GPU core clock. CR_TIMER (R0+0x160) ticks once per 256 core clocks whether or not a ++ * phase is open (r3clk: 2823 kHz at pstate 4 = 722.7 MHz, 3703 kHz at 948 MHz), so CR_TIMER over ++ * host time x 256 is the clock at any moment. The Rogue PERF_TA_CYCLE / PERF_3D_CYCLE / ++ * PERF_TA_OR_3D_CYCLE counters (R0+0x6020/0x6028/0x6038, 32 bit) advance at the core clock while a ++ * TA or 3D phase is open (340.0 MHz through open TA phases, docs/gpu/experiments/46-device-tests.md): ++ * over host time, the highest rate per pstate (a window busy all through) is the clock again. One ++ * sample = two licensed tickets, mmio_clock_gap_us apart, each reading the three counters and ++ * CR_TIMER's low word back to back. A cycle rate above 2 GHz is a firmware counter clear ++ * (R0+0x6000 = 1, 0x1001ab00) inside the window and is dropped. ++ */ ++ ++/* Bit 3: does the GPC power down at full idle in state 0? The PMGR word is always readable; ++ * R1 only while the gate reads on (actual == target == 0xf), through a census ticket so the RAM ++ * terms and the lock still apply. A refused licence still returns the R1 words it read. ++ */ +diff --git a/drivers/gpu/apple-g5p/mmu-attrs.inc b/drivers/gpu/apple-g5p/mmu-attrs.inc +new file mode 100644 +index 0000000000..694552016a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-attrs.inc +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include ++#include ++ ++/* Read the actual stable kernel mapping of a retained DMA allocation. ++ * DMA memory may be a vmalloc PTE or a cached linear-map block. ++ */ ++static u64 g5p_table_cpu_descriptor(unsigned long va) ++{ ++ pgd_t *pgd = pgd_offset_k(va); ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ ++ if (pgd_none(READ_ONCE(*pgd)) || pgd_bad(READ_ONCE(*pgd))) ++ return 0; ++ p4d = p4d_offset(pgd, va); ++ if (p4d_none(READ_ONCE(*p4d)) || p4d_bad(READ_ONCE(*p4d))) ++ return 0; ++ pud = pud_offset(p4d, va); ++ if (pud_leaf(READ_ONCE(*pud))) ++ return pud_val(READ_ONCE(*pud)); ++ if (pud_none(READ_ONCE(*pud)) || pud_bad(READ_ONCE(*pud))) ++ return 0; ++ pmd = pmd_offset(pud, va); ++ if (pmd_leaf(READ_ONCE(*pmd))) ++ return pmd_val(READ_ONCE(*pmd)); ++ if (pmd_none(READ_ONCE(*pmd)) || pmd_bad(READ_ONCE(*pmd))) ++ return 0; ++ return pte_val(READ_ONCE(*pte_offset_kernel(pmd, va))); ++} ++ ++static void g5p_table_attrs(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *p) ++{ ++ u64 desc, mair = read_sysreg(mair_el1); ++ u32 index, attr; ++ ++ if (!p->cpu) ++ return; ++ desc = g5p_table_cpu_descriptor((unsigned long)p->cpu); ++ index = (desc & PTE_ATTRINDX_MASK) >> 2; ++ attr = (mair >> (index * 8)) & 0xff; ++ g5p_mmu_record( ++ g, ++ "MMU allocation %s DMA=%pad coherent=%u cpu_desc=%016llx AttrIndx=%u " ++ "MAIR=%016llx attr=%02x table_memtype=%s\n", ++ stage, &p->dma, dev_is_dma_coherent(g->dev), desc, index, mair, ++ attr, ++ !desc ? "unknown" : ++ attr == 0xff ? "Normal-WB" : ++ attr == 0x44 ? "Normal-NC" : ++ "other"); ++} +diff --git a/drivers/gpu/apple-g5p/mmu-tables.inc b/drivers/gpu/apple-g5p/mmu-tables.inc +new file mode 100644 +index 0000000000..94ae6ed8fc +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-tables.inc +@@ -0,0 +1,57 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Fixed, pre-release aliases only. No firmware patch or live table surgery. */ ++#define G5P_TABLE_ALIAS_VA (G5P_ROOT_VA + 0x100000ULL) ++#define G5P_TABLE_COOKIE 0x3ffcU ++ ++static void g5p_table_clean_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ unsigned long start; ++ ++ if (!p->cpu) ++ return; ++ start = (unsigned long)p->cpu; ++ /* Explicit architecture PoC operation, even if DT calls this coherent. ++ * dma_sync_single_for_device is not an API for coherent allocations. ++ */ ++ dcache_clean_poc(start, start + G5P_GPU_PAGE); ++} ++ ++static void g5p_tables_clean(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *user, u32 count) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ g5p_mmu_record( ++ g, ++ "MMU tables %s clean_poc=%u dma_coherent=%u dma_pgprot=%016llx; coherent DMA " ++ "API, extra clean is experimental\n", ++ stage, 1, dev_is_dma_coherent(g->dev), ++ (u64)pgprot_val(dma_pgprot(g->dev, PAGE_KERNEL, 0))); ++ for (u32 i = 0; i < count; i++) ++ g5p_table_attrs(g, stage, &user[i]); ++ g5p_table_attrs(g, stage, &b->catalogue); ++ g5p_table_attrs(g, stage, &b->directory); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_attrs(g, stage, &b->leaf[i]); ++ ; ++ ; ++ for (u32 i = 0; i < count; i++) ++ g5p_table_clean_page(g, &user[i]); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_clean_page(g, &b->leaf[i]); ++ g5p_table_clean_page(g, &b->sink_leaf); ++ g5p_table_clean_page(g, &b->sink_directory); ++ g5p_table_clean_page(g, &b->directory); ++ g5p_table_clean_page(g, &b->catalogue); ++ dsb(sy); ++ dma_wmb(); ++ g5p_mmu_record( ++ g, ++ "MMU tables %s PoC complete user_pages=%u boot_leaves=%u; dsb sy before publication\n", ++ stage, count, b->leaves); ++} ++ ++/* Deliberately separate from g5p_meta_read(): the generic slave guard stays ++ * latched after reset. This opt-in reader can address only the preinstalled ++ * table aliases and one immutable root control word, never private META RAM. ++ */ +diff --git a/drivers/gpu/apple-g5p/pack.h b/drivers/gpu/apple-g5p/pack.h +new file mode 100644 +index 0000000000..942b44ba33 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/pack.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_PACK_H ++#define APPLE_G5P_PACK_H ++ ++#include "bits-compat.h" ++#include "ldr.h" ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_IMAGE_SIZE 0x30000 ++#define G5P_CONFIG_WRITES 27 ++ ++struct g5p_ldr_image { ++ u8 *data; ++ u32 config[G5P_CONFIG_WRITES][2]; ++ u32 config_count; ++ u32 t1_pc; ++ u32 t1_sp; ++}; ++ ++/* Exact C0 ranges derived from AGXMetaFirmware_layoutAndLoadLDR ...6c9f1bc. ++ * GPU virtual mappings and the boot config at +0x200 are intentionally separate. ++ */ ++static inline int g5p_image_offset(u32 addr, u32 size, u32 *offset) ++{ ++ static const struct { ++ u32 base, size, packed; ++ } segments[] = { ++ { 0x10000000, 0x18000, 0x04000 }, ++ { 0x20000000, 0x04000, 0x1c000 }, ++ { 0xc0000000, 0x04000, 0x00000 }, ++ { 0xe0001000, 0x04000, 0x20000 }, ++ { 0x80000080, 0x055dc, 0x24000 }, ++ }; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(segments); i++) { ++ if (addr >= segments[i].base && size <= segments[i].size && ++ addr - segments[i].base <= segments[i].size - size) { ++ *offset = segments[i].packed + addr - segments[i].base; ++ return 0; ++ } ++ } ++ return -EINVAL; ++} ++ ++static inline int g5p_pack_record(void *ctx, enum g5p_ldr_op op, u32 addr, ++ u32 size, const u8 *data) ++{ ++ struct g5p_ldr_image *image = ctx; ++ u32 i, offset; ++ int ret; ++ ++ switch (op) { ++ case G5P_LOAD: ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memcpy(image->data + offset, data, size); ++ break; ++ case G5P_ZERO: ++ /* Apple does not apply the local/core-memory ZeroMem records here. */ ++ if (addr >= 0x80000000 && addr < 0x88000000) ++ break; ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memset(image->data + offset, 0, size); ++ break; ++ case G5P_CONFIG: ++ for (i = 0; i < size; i += 12) { ++ if (image->config_count == G5P_CONFIG_WRITES) ++ return -EINVAL; ++ image->config[image->config_count][0] = ++ g5p_le32(data + i + 4); ++ image->config[image->config_count++][1] = ++ g5p_le32(data + i + 8); ++ } ++ break; ++ case G5P_START: ++ image->t1_sp = g5p_le32(data + 4); ++ image->t1_pc = g5p_le32(data + 8); ++ break; ++ } ++ return 0; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/perf-envelope.h b/drivers/gpu/apple-g5p/perf-envelope.h +new file mode 100644 +index 0000000000..57387bcbae +--- /dev/null ++++ b/drivers/gpu/apple-g5p/perf-envelope.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 clock: the inputs of the firmware's policy-state writer (0x10008218) and of its governor, ++ * so the firmware itself moves the GPC from pstate 1 to q = pin / 100. Both compute ++ * x = min(policy+0x6f0, policy+0x688, ctrl+0x102c) ++ * (META CS is unsigned lower; iOS's live policy+0x38 equals this min in every sample), store ++ * policy+8 = 0 when x % 100 == 0 (2 when x < 100), and request pstate q = x / 100 (policy+0x28). ++ * +0x6f0 and +0x688 are recomputed by the firmware from its coefficient block unless their ++ * overrides +0x6f4 / +0x68c are nonzero, and the firmware itself reloads those overrides from ++ * the control object on every pass, whatever the record held: +0x6f4 = ctrl+0x1040 (0x1000a1e4, ++ * 0x1000b388, 0x1000c568) and +0x68c = ctrl+0x1038 (0x1000a74c, 0x1000b10c, 0x1000c2fc). iOS ++ * leaves both control words 0 and arms the sampler (policy+4 = 16); this boot pins them ++ * (ctrl+0x1040 = pin, ctrl+0x1038 = 400) and leaves the sampler off (policy+4 = 0), so there is ++ * no idle parking at pstate 0. The pstate table and limits are iOS's (live j71t capture, ++ * 2026-11-01): meta+0x710/0x714 = 4/5, MHz {0, 340, 474, 550, 723}, ctrl+0x1028/0x1030 = ++ * 3348/400; the floor ctrl+0x102c is the pin. RAM only, before release. ++ */ ++#ifndef APPLE_G5P_PERF_ENVELOPE_H ++#define APPLE_G5P_PERF_ENVELOPE_H ++ ++#include "bits-compat.h" ++#define G5P_PERF_POLICY 0x4aU ++#define G5P_PERF_META 0x4dU ++#define G5P_PERF_CTL 0x5eU ++#define G5P_PERF_TOP 400U ++ ++static const u32 g5p_perf_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline bool g5p_perf_valid(u32 pin) ++{ ++ return pin <= G5P_PERF_TOP && !(pin % 100); ++} ++ ++static inline void g5p_perf_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 pin) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ u8 *meta = g5p_object(p, root, shared, G5P_PERF_META); ++ u8 *ctl = g5p_object(p, root, shared, G5P_PERF_CTL); ++ u32 i; ++ ++ g5p_put32(meta + 0x710, ARRAY_SIZE(g5p_perf_mhz) - 1); ++ g5p_put32(meta + 0x714, ARRAY_SIZE(g5p_perf_mhz)); ++ for (i = 0; i < ARRAY_SIZE(g5p_perf_mhz); i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_perf_mhz[i]); ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, pin); ++ g5p_put32(ctl + 0x1030, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1038, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1040, pin); ++ g5p_put32(policy + 0x688, G5P_PERF_TOP); ++ g5p_put32(policy + 0x68c, G5P_PERF_TOP); ++ g5p_put32(policy + 0x6f0, pin); ++ g5p_put32(policy + 0x6f4, pin); ++} ++ ++/* boot_gpu_oc_mhz (gpu-oc.h): state 5 = the overclock record; the firmware's table, ceilings and ++ * policy limits extended to index 5 so the pin 500 and the power-on request of state 5 are in range ++ * (the firmware derives its clock entry priv+0x2c4 = meta+0x718[state] from Fender, 0x1001c838). ++ */ ++ ++/* ++ * boot_gpu_perf_seed (perf60-clock.md "Clock-change drain RE" §3): in policy state 0 the ++ * firmware ++ * never changes pstate while powered; only its boot sampler (0x1000aa80, before ALIVE) moves the ++ * GPC, when policy+0x28 != Fender & 0xf. Seeding the policy record with the firmware's own ++ * steady-state output for the pin, +0x28 = +0x2c = q, +0x30 = 0, +8 = 0, makes that sampler ++ * request q through its setter (0x1001c9b8 -> 0x1001c88c: pol34 = q, R1+0x1000 = q | 0x10), ++ * which is ++ * iOS's park -> reboot -> sampler path. q = pin / 100 for pins 200..400, else 0 (no seed). ++ */ ++static inline u32 g5p_perf_seed_q(u32 pin) ++{ ++ return pin >= 200 && pin <= G5P_PERF_TOP && !(pin % 100) ? pin / 100 : ++ 0; ++} ++ ++/* Seed 2 adds what the sampler needs to get past its first-call init: 0x1000aa80 at 0x1000abb0 ++ * takes policy+0 == 0 as "never sampled" and only initialises (0x1000ae88: +0 = 1, its time words) ++ * without reaching the target compare at 0x1000ada4 or the setter; its sampling path divides by ++ * policy+4 (0x10022f88 at 0x1000ac28). The boot call at 0x10000b54 is its only caller, so a fresh ++ * record never moves Fender. iOS's record persists across firmware reboots: +0 = 1 and +4 = 16 in ++ * every sample of the 2026-11-01 capture (idle and load). Seed 2 writes those two as iOS has them. ++ * Seed 2 on hardware (2026-11-03): no ALIVE, Fender 0. With +0 = 1 the sampler takes its sampling ++ * path when (now - policy+0x8f8) / 24000 / policy+4 >= 1 (0x1000ac10..ac5c): +0x8f8 is the host's ++ * map-time stamp, ~3 s old at release, so it runs the load governor (0x1000afa4 -> 0x1000b050, busy ++ * accumulators policy+0x62c..0x68c, all 0 on a fresh record) and lands on pstate 0. Seed 3 ++ * (boot.inc ++ * g5p_boot_release) restamps +0x8f8 right before the release stream: < 1 period elapsed, the load ++ * update is skipped (0x1000ac64 -> 0x1000acb0 -> 0x1000ada4) and the compare against +0x28 = q ++ * reaches the setter (0x1000ae68 -> 0x1001c9b8). ++ */ ++static inline void g5p_perf_seed(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 q, bool ios_record) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ ++ g5p_put64(policy + 0x28, (u64)q << 32 | q); ++ g5p_put32(policy + 0x30, 0); ++ g5p_put32(policy + 8, 0); ++ if (ios_record) { ++ g5p_put32(policy + 0, 1); ++ g5p_put32(policy + 4, 16); ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/power-r33.inc b/drivers/gpu/apple-g5p/power-r33.inc +new file mode 100644 +index 0000000000..ce8a18f3c1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r33.inc +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R33 (ds-r33-b knob A): the kexec GFX cycle holds ps_gfx off for ~10 us, and ++ * R0+0x418/0x41c (no reset) keeps the previous boot's value through it. At the ++ * first Fender wake, before any GPU MMIO and before any GPU IRQ is requested, ++ * hold the same PMGR gate (0x80340, gate124) off for boot_gfx_off_ms, then ++ * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset, ++ * auto). Every exit leaves the gate requested on. ++ */ ++#define G5P_R33_GFX_PS 0x80340U ++#define G5P_R33_GFX_OFF_MAX_MS 2000U ++#define G5P_R33_GFX_POLL_US 10000U ++ ++static inline bool g5p_r33_gfx_on(u32 raw) ++{ ++ return (raw & 0xf) == 0xf && ((raw >> 4) & 0xf) == 0xf; ++} ++ ++/* Only a plain, fully-on, software-controlled gate may be cycled. */ ++static inline bool g5p_r33_gfx_cyclable(u32 raw) ++{ ++ return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31) && ++ !((raw >> 28) & 1); ++} ++ ++static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) ++{ ++ u32 before, held, v = 0, after = 0; ++ u64 t0; ++ int ret, on; ++ ++ if (!ms) ++ return 0; ++ if (ms > G5P_R33_GFX_OFF_MAX_MS || !pmgr) ++ return -EINVAL; ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &before); ++ if (ret) ++ return ret; ++ if (!g5p_r33_gfx_cyclable(before)) { ++ dev_info( ++ dev, ++ "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset/auto or not on\n", ++ ms, before); ++ return -EBUSY; ++ } ++ ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0); ++ if (!ret) ++ ret = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, v, ++ !((v >> 4) & 0xf), 10, ++ G5P_R33_GFX_POLL_US); ++ t0 = ktime_get_ns(); ++ if (!ret) ++ msleep(ms); ++ held = (u32)((ktime_get_ns() - t0) / 1000); ++ if (!ret) ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &v); ++ /* Never leave the GPU domain off, whatever happened above. */ ++ on = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0xf); ++ if (!on) ++ on = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, after, ++ g5p_r33_gfx_on(after), 10, ++ G5P_R33_GFX_POLL_US); ++ dev_info( ++ dev, ++ "POWER gfx-off ms=%u held_us=%u before=%08x end=%08x after=%08x off_ret=%d " ++ "on_ret=%d\n", ++ ms, held, before, v, after, ret, on); ++ if (!ret && (v >> 4) & 0xf) ++ ret = -EIO; ++ return ret ?: on; ++} +diff --git a/drivers/gpu/apple-g5p/power-r41.inc b/drivers/gpu/apple-g5p/power-r41.inc +new file mode 100644 +index 0000000000..9e5c1e6683 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r41.inc +@@ -0,0 +1,135 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* J71t live ADT + ApplePMGR_restoreDeviceBridgeSettings (5a39918). ++ * Volatile fabric MMIO only. GFX selects bridge 8, then bridge 9; the two ++ * version-0 counter controls follow bridge 8's table, before bridge 9. ++ * Keep devm_ioremap/readl/writel identical to boot.inc's swirq mapping. ++ */ ++static const unsigned int gfx_bridge_replay = 0; ++ ++struct g5p_r41_write { ++ u32 bridge, offset, value; ++}; ++ ++static const struct g5p_r41_write g5p_r41_writes[] = { ++ { 8, 0x0000, 0x00000120 }, { 8, 0x0018, 0x80010005 }, ++ { 8, 0x001c, 0x00010001 }, { 8, 0x410c, 0x00000003 }, ++ { 8, 0x4100, 0x00000003 }, { 9, 0x0000, 0x00697101 }, ++ { 9, 0x0004, 0x00697101 }, { 9, 0x0008, 0x00697101 }, ++ { 9, 0x000c, 0x00697101 }, { 9, 0x0010, 0x00697101 }, ++ { 9, 0x0014, 0x00697101 }, { 9, 0x0018, 0x00697101 }, ++ { 9, 0x001c, 0x00697101 }, { 9, 0x0020, 0x00697101 }, ++ { 9, 0x0024, 0x00697101 }, { 9, 0x0028, 0x00697101 }, ++ { 9, 0x002c, 0x00697101 }, { 9, 0x0030, 0x00697101 }, ++ { 9, 0x0034, 0x00697101 }, { 9, 0x0038, 0x00697101 }, ++ { 9, 0x003c, 0x00697101 }, { 9, 0x0040, 0x00697101 }, ++ { 9, 0x0044, 0x00697101 }, { 9, 0x0048, 0x00697101 }, ++ { 9, 0x004c, 0x00697101 }, { 9, 0x0050, 0x00697101 }, ++ { 9, 0x0054, 0x00697101 }, { 9, 0x0058, 0x00697101 }, ++ { 9, 0x005c, 0x00697101 }, { 9, 0x0060, 0x00697101 }, ++ { 9, 0x0064, 0x00697101 }, { 9, 0x0068, 0x00697101 }, ++ { 9, 0x006c, 0x00697101 }, { 9, 0x0070, 0x00697101 }, ++ { 9, 0x0074, 0x00697101 }, { 9, 0x0078, 0x00697101 }, ++ { 9, 0x007c, 0x00697101 }, ++}; ++ ++static struct { ++ void __iomem *window[2]; ++ unsigned int mode; ++ int result; ++ bool attempted, ready, firmware_logged, ta_logged; ++} g5p_r41_bridge; ++ ++/* Reject anything outside the exact write allowlist, including unaligned ++ * offsets and other registers inside the two mapped ADT ranges. ++ */ ++static bool g5p_r41_valid(const struct g5p_r41_write *w) ++{ ++ if (w->offset & 3) ++ return false; ++ if (w->bridge == 8) ++ return w->offset < 0x5000 && ++ (w->offset == 0 || w->offset == 0x18 || ++ w->offset == 0x1c || w->offset == 0x410c || ++ w->offset == 0x4100); ++ return w->bridge == 9 && w->offset <= 0x7c; ++} ++ ++static void g5p_r41_bridge_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ u32 value = ++ readl(g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ ++ dev_info( ++ g->dev, ++ "BRIDGE r41 stage=%s mode=%u index=%u bridge=%u off=%04x value=%08x want=%08x\n", ++ stage, g5p_r41_bridge.mode, i, w->bridge, w->offset, ++ value, w->value); ++ } ++} ++ ++/* Called under g->lock after ACTIVE (and any r33 off/on), before Fender. ++ * Latch errors too: never retry a partial experiment in the same boot. ++ */ ++static int g5p_r41_bridge_prepare(struct g5p_diag *g) ++{ ++ unsigned int i, mode = gfx_bridge_replay; ++ ++ if (!mode) ++ return 0; ++ if (g5p_r41_bridge.attempted) ++ return g5p_r41_bridge.result; ++ g5p_r41_bridge.attempted = true; ++ g5p_r41_bridge.mode = mode; ++ g5p_r41_bridge.result = -EINVAL; ++ if (mode > 4) ++ return -EINVAL; ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) ++ if (!g5p_r41_valid(&g5p_r41_writes[i])) ++ return -EINVAL; ++ g5p_r41_bridge.result = -ENOMEM; ++ g5p_r41_bridge.window[0] = devm_ioremap(g->dev, 0x201f00000ULL, 0x5000); ++ if (!g5p_r41_bridge.window[0]) ++ return -ENOMEM; ++ g5p_r41_bridge.window[1] = devm_ioremap(g->dev, 0x201f80000ULL, 0x1000); ++ if (!g5p_r41_bridge.window[1]) ++ return -ENOMEM; ++ /* Read all 37 even in split modes; never read unmapped/partial state. */ ++ g5p_r41_bridge_snapshot(g, "before"); ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ ++ if (mode == 1 || (mode == 3 && w->bridge != 8) || ++ (mode == 4 && w->bridge != 9)) ++ continue; ++ writel(w->value, ++ g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ dev_info( ++ g->dev, ++ "BRIDGE r41 write mode=%u index=%u bridge=%u off=%04x value=%08x\n", ++ mode, i, w->bridge, w->offset, w->value); ++ } ++ g5p_r41_bridge_snapshot(g, "after"); ++ g5p_r41_bridge.ready = true; ++ g5p_r41_bridge.result = 0; ++ return 0; ++} ++ ++static void g5p_r41_bridge_firmware(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.firmware_logged) { ++ g5p_r41_bridge.firmware_logged = true; ++ g5p_r41_bridge_snapshot(g, "firmware-alive"); ++ } ++} ++ ++static void g5p_r41_bridge_ta(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.ta_logged) { ++ g5p_r41_bridge.ta_logged = true; ++ g5p_r41_bridge_snapshot(g, "first-TA-kick"); ++ } ++} +diff --git a/drivers/gpu/apple-g5p/r74e112.h b/drivers/gpu/apple-g5p/r74e112.h +new file mode 100644 +index 0000000000..cacac73ac0 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r74e112.h +@@ -0,0 +1,137 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R74E112_H ++#define G5P_R74E112_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * Wake112: the R74E GPC1 power proof when ALIVE preempts the on-poll. ++ * ++ * R74E (boot.inc) requests GPC=1 and polls R1+0x1000 until the state field reads 1. ++ * The poll stops as soon as ALIVE is decoded: after ALIVE only the first-ALIVE owner ++ * may read R1. On a warm, kexec-cycled boot the firmware can reach ALIVE before the ++ * poll's first read (Wake111 20261031T201022Z: off_ns=1500 on_ns=375 raw=ffffffff, ++ * the U32_MAX "not read" sentinel). Wake111 then latched a power abort although the ++ * state was never sampled. ++ * ++ * Wake112 defers that one proof to the owner instead, but only when the on-poll took ++ * no sample: the poll counts its reads, and the U32_MAX raw is never taken as "not read". ++ * The owner's existing first-ALIVE read of R1+0x1000 (g5p_r73e_irq_submit, before any ++ * register write) is the proof. State 1 sets release_ready and the unchanged cold ++ * sequence continues. Any other value (0, 0x10, 0x11, ffffffff, ...) is an absorbing ++ * power abort. An on-poll that reads state 1 never defers. An on-poll that sampled ++ * before ALIVE (0x11 in rounds 74e..79b, or any other word) and every other R74E failure ++ * (off-poll, timeout, GPC1 write refused) abort exactly as before. ++ * ++ * States (r73b->early_proof): ++ * NONE no deferral: the poll decided, or the release has not run ++ * DEFERRED ALIVE preempted the on-poll; one owner read is owed (generation recorded) ++ * TAKEN the once-owner claimed it; it settles with no exit in between ++ * PROVEN that read had state 1: release_ready ++ * FAILED any other value, another generation, a lost precondition, or an ALIVE ++ * wait that ended with the proof still owed: power abort, absorbing ++ * ++ * Pure policy: no register access, no lock, no kernel API. ++ */ ++ ++#define G5P_PX_STATE_MASK 0x1fU ++#define G5P_PX_STATE_ON 0x01U ++ ++enum g5p_px { ++ G5P_PX_NONE, ++ G5P_PX_DEFERRED, ++ G5P_PX_TAKEN, ++ G5P_PX_PROVEN, ++ G5P_PX_FAILED, ++ G5P_PX_N, ++}; ++ ++static const char *const g5p_px_name[G5P_PX_N] = { ++ "none", "deferred", "taken", "proven", "failed", ++}; ++ ++/* Why a deferred proof failed (NONE: it did not). */ ++enum g5p_px_why { ++ G5P_PXW_NONE, ++ G5P_PXW_STATE, /* the read's state field is not 1 */ ++ G5P_PXW_STALE, /* deferred in another reset generation */ ++ G5P_PXW_LOST, /* ALIVE or the release lost, stopped or failed since */ ++ G5P_PXW_RETIRED, /* a FINAL teardown forbids R1 */ ++ G5P_PXW_UNTAKEN, /* the ALIVE wait ended with the proof still owed */ ++ G5P_PXW_N, ++}; ++ ++static const char *const g5p_px_why_name[G5P_PXW_N] = { ++ "none", "state", "stale", "lost", "retired", "untaken", ++}; ++ ++/* What the R74E release path latches. */ ++enum g5p_pxp { ++ G5P_PXP_READY, /* the on-poll read state 1: release_ready (cold) */ ++ G5P_PXP_DEFER, /* ALIVE preempted the on-poll before its first read */ ++ G5P_PXP_ABORT, /* anything else: power abort (cold) */ ++}; ++ ++/* ++ * on_ran: the core request, the off-poll and the GPC1 write succeeded and the on-poll ++ * ran. ret: the last step's result. preempted: the on-poll stopped on ALIVE. ++ * samples: the on-poll's R1+0x1000 reads. Only a preemption with no read defers (no ++ * abort, no release_ready); a sampled one saw the state and aborts. ++ */ ++static inline enum g5p_pxp g5p_px_poll(bool on_ran, int ret, bool preempted, ++ u32 samples) ++{ ++ if (!on_ran) ++ return G5P_PXP_ABORT; ++ if (!ret) ++ return G5P_PXP_READY; ++ return preempted && !samples ? G5P_PXP_DEFER : G5P_PXP_ABORT; ++} ++ ++/* The first-ALIVE owner's admission. */ ++enum g5p_pxg { ++ G5P_PXG_READY, /* release_ready: the cold owner path */ ++ G5P_PXG_PROVE, /* a deferred proof is owed: the once-owner's read decides */ ++ G5P_PXG_WAIT, /* nothing proven yet, or another owner holds the proof */ ++ G5P_PXG_REFUSE, /* power abort or a failed proof: no register access */ ++}; ++ ++static inline enum g5p_pxg g5p_px_gate(bool power_abort, bool release_ready, ++ u32 px) ++{ ++ if (power_abort || px == G5P_PX_FAILED) ++ return G5P_PXG_REFUSE; ++ if (release_ready) ++ return G5P_PXG_READY; ++ if (px == G5P_PX_DEFERRED) ++ return G5P_PXG_PROVE; ++ return G5P_PXG_WAIT; ++} ++ ++/* RAM facts the once-owner samples after taking the proof, before its one read. */ ++struct g5p_px_pre { ++ u32 gen, gen_now; /* DS108 generation at the deferral and now */ ++ bool alive, released, stopped, failed; ++ bool r1_allowed; /* g5p_r8_r1_allowed() */ ++}; ++ ++static inline enum g5p_px_why g5p_px_check(const struct g5p_px_pre *p) ++{ ++ if (!p->gen || p->gen != p->gen_now) ++ return G5P_PXW_STALE; ++ if (!p->alive || !p->released || p->stopped || p->failed) ++ return G5P_PXW_LOST; ++ if (!p->r1_allowed) ++ return G5P_PXW_RETIRED; ++ return G5P_PXW_NONE; ++} ++ ++/* The proof: only the state field 1 is power-on. */ ++static inline bool g5p_px_on(u32 raw) ++{ ++ return G5P_GPC_ON(raw); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/r87b-attach.h b/drivers/gpu/apple-g5p/r87b-attach.h +new file mode 100644 +index 0000000000..81600ac06a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r87b-attach.h +@@ -0,0 +1,34 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R87B_ATTACH_H ++#define G5P_R87B_ATTACH_H ++/* R87b DIAGNOSTIC ONLY, never production-safe: the DC3 attach policy selector. ++ * (docs/gpu/experiments/87b-build-report.md) ++ * ++ * Two campaigns (r85 stage 2, r87 stage 1) never reached GPU work: the DC3 attach pair was ++ * published, the immediate gate (R1+0x1000 & 0x1f == 1) read 0x10/0x11, the host GPC off/on ++ * repair timed out (-110) and attach stopped with -115 before the RAM-only consume wait ran, ++ * after which the machine froze. Every retained instance of that failure shows the firmware ++ * consuming the pair on its own (dc=3/3 at export, no bell rung). boot_r87b_attach selects: ++ * 0 the original attach policy, exactly r87: a closed gate is repaired by the host (R1+0x20000 ++ * clear, GPC off/on, ALIVE-owned doorbell). The matched control. ++ * 1 self-drain (the r82b design, docs/gpu/experiments/82-attach-reactivation-triage.md): a ++ * closed DC3 gate is only a phase sample of the firmware's GPC window. The host does no ++ * repair and rings no bell; it keeps the RAM-only 3 s consume wait of render-r74f.inc and ++ * continues through the firmware-MMU attach path (r74g) when the firmware consumed the pair. ++ * An active gate still rings the bell at once (the unchanged 80 % path). The TA kind is ++ * never affected. ++ * The selector is latched at map and read only through the latched copy. A non-zero value needs ++ * the r80 epoch (which implies r74h, r74g and r74f case 1): self-drain is only safe because r74g ++ * has the firmware own the first MMU pass, so no host R0 access is licensed by a rung bell. ++ * The TYPED_V3 production path latches policy 1 with boot_r74f_case = boot_r74g_case = 1 and NO ++ * r80 / r74h (boot-r73b.inc, "production path"); g5p_r87b_attach_valid() describes the r80 ++ * epoch only. ++ */ ++#include ++#include ++ ++#define G5P_R87B_ATTACH_ORIGINAL 0U ++#define G5P_R87B_ATTACH_SELF_DRAIN 1U ++#define G5P_R87B_ATTACH_MAX 1U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/readback.h b/drivers/gpu/apple-g5p/readback.h +new file mode 100644 +index 0000000000..fdd71ab300 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/readback.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_READBACK_H ++#define G5P_READBACK_H ++ ++#include "bits-compat.h" ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++enum g5p_readback_state { ++ G5P_READBACK_WRAPPER_ONLY, ++ G5P_READBACK_BASELINE, ++ G5P_READBACK_RESET_HELD, ++}; ++ ++/* Returning wrapper registers in exp03; never a META slave-port register. */ ++static const u32 g5p_fender_regs[] = { ++ 0, 8, 0x10, 0x14, 0x1000, 0x1004, 0x2010, 0x2014, 0x20000, ++}; ++ ++/* Reset halves first: the only post-arm R0 readback needed by the gate. */ ++static const u32 g5p_rgx_regs[] = { ++ 0x100, 0x104, 0, 4, 8, 0xc, 0x18, 0x20, 0x80, 0x84, 0x88, 0x8c, ++}; ++ ++static inline u32 g5p_readback_rgx_count(u32 gpc, enum g5p_readback_state state) ++{ ++ if (!G5P_GPC_ON(gpc)) ++ return 0; ++ if (state == G5P_READBACK_RESET_HELD) ++ return 2; ++ return state == G5P_READBACK_BASELINE ? ARRAY_SIZE(g5p_rgx_regs) : 0; ++} ++ ++static inline bool g5p_slave_access_allowed(bool reset_transition) ++{ ++ return !reset_transition; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/recover.inc b/drivers/gpu/apple-g5p/recover.inc +new file mode 100644 +index 0000000000..f550d10102 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/recover.inc +@@ -0,0 +1,101 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * In-boot GPU recovery (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault isolation). ++ * ++ * A render job that stopped the firmware (its event stream went silent, or a fault) blocks the ++ * one TA/3D channel for every client, and the firmware has no abort. iOS ends that state with a ++ * firmware stop, a platform power cycle and a new firmware boot (kext 6c9e070, firmware-re README ++ * "Fault recovery"). This is the same, built from two proven pieces: ++ * - the reboot quiesce's freeze, IRQ mask and PMGR gfx gate cycle (teardown.inc g5p_kq_*), which ++ * on hardware cycled a wedged, failed device safely (deskfold reboot notifier: plan=-+cycle, ++ * then the next kernel came up warm-cycled); ++ * - the bootstrap's own stage functions (wake, map, arm, load, release, events, control), the ++ * same code the sysfs commands run after the next kernel's cycle on every warm hop. ++ * The old struct g5p_boot is frozen and left allocated (devm; a few hundred KiB per recovery), ++ * so a stale pointer anywhere still reads valid memory; a new one is built by the stages. id and ++ * sample are not rerun (they refuse once the boot saw a reset transition; map needs only the ++ * cached core id), nor is irq-test (refused in an r74 epoch). The firmware image is reused. ++ * ++ * Core lock held, no job in flight (the render scheduler's worker). Returns 0 with a live new ++ * epoch, or an error with the device left frozen and failed. ++ */ ++ ++/* The bootstrap's stages after the wake, as the sysfs runner orders them. */ ++static const char *const g5p_recover_stages[] = { ++ "map", ++ "arm", ++ "load", ++ "release", ++ "events", ++ "control-prepare", ++ "control-submit", ++ "control-wait", ++}; ++ ++static int __maybe_unused g5p_recover(struct g5p_diag *g, const char *why, ++ u32 *stage_out) ++{ ++ struct g5p_boot *old = g->boot; ++ struct g5p_kq_ctx kq = { .g = g }; ++ u64 t0 = ktime_get_ns(); ++ u32 stage = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ *stage_out = 0; ++ if (!old || !READ_ONCE(old->released) || g5p_r8_is_retired() || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || !g->pmgr || ++ !g->id_checked) ++ return -EPERM; ++ /* The quiesce's freeze and IRQ mask, then the line is given back for the new epoch. */ ++ g5p_kq_freeze(&kq); ++ g5p_kq_irq_off(&kq); ++ if (old->irq_ready && old->irq > 0) ++ devm_free_irq(g->dev, old->irq, g); ++ /* mmio-access.md G5: the GPU is marked unusable before its power is cut (Apple clears ++ * fender+0xc inside setPlatformPower(0), before the gate); g5p_kq_cycle() also sets mmio_off. ++ */ ++ WRITE_ONCE(g->fender_ready, false); ++ ret = g5p_kq_cycle(&kq); ++ ret = g5p_ds_hook_recover(g, ret) ?: ret; ++ if (ret) ++ goto out; ++ /* A new epoch: the stages build a new struct g5p_boot from here. */ ++ WRITE_ONCE(g->boot, NULL); ++ stage = 1; ++ ret = g5p_fender_wake(g); ++ for (u32 i = 0; !ret && i < ARRAY_SIZE(g5p_recover_stages); i++) { ++ const char *s = g5p_recover_stages[i]; ++ ++ stage = i + 2; ++ ret = strncmp(s, "control-", 8) ? g5p_boot_command(g, s) : ++ g5p_lifecycle_command(g, s); ++ if (ret == -EALREADY) ++ ret = 0; ++ } ++ if (!ret && (!g->boot || !READ_ONCE(g->boot->alive) || ++ !READ_ONCE(g->boot->control_complete))) ++ ret = -EIO; ++ /* The claim gate again: the held jobs need ALIVE proven in the new generation. */ ++ if (!ret) ++ ret = g5p_ds_hook_claim(g); ++ if (!ret) { ++ /* The render claim is kept: no diagnostic command may reset this epoch. */ ++ WRITE_ONCE(g->boot->user_compute_owned, true); ++ stage = 0; ++ } ++out: ++ if (ret && g->boot) ++ WRITE_ONCE(g->boot->failed, true); ++ g->last_result = ret; ++ *stage_out = stage; ++ dev_emerg( ++ g->dev, ++ "GPU-RECOVER why=%s ret=%d stage=%s ms=%llu gen=%u end=recover\n", ++ why, ret, ++ !stage ? (ret ? "cycle" : "done") : ++ stage == 1 ? "wake" : ++ g5p_recover_stages[stage - 2], ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC, g5p_ds_gen()); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/render-layout-r22.h b/drivers/gpu/apple-g5p/render-layout-r22.h +new file mode 100644 +index 0000000000..8391a8320f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout-r22.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_R22_H ++#define G5P_RENDER_LAYOUT_R22_H ++ ++#include "render-layout.h" ++ ++#define G5P_R22_TA 0x54000U ++#define G5P_R22_3D 0x58000U ++#define G5P_R22_PM 0x5c000U ++#define G5P_R22_W 0x400U ++#define G5P_R22_TA_SKU 0xa00U ++#define G5P_R22_3D_SKU 0xd00U ++#define G5P_R22_TQ 0x1800U ++#define G5P_R22_STAMP 0x1840U ++#define G5P_R22_DEP 0x1900U ++#define G5P_R22_BLOCKS 32U ++#define G5P_R22_PHYS_PAGES (G5P_R22_BLOCKS * 8) ++#define G5P_R22_AUX 0x101000000ULL ++#define G5P_R22_RT 0x101020000ULL ++#define G5P_R22_RT_AUX 0x101030000ULL ++ ++/* 6cb124c/13f4/14dc: PA PPNs, GPU F and META links are distinct. ++ * Caller provides complete 16KiB control/list/F allocations. Check the whole ++ * physical input before writing anything. No synthesized PM catalogue encoder. ++ */ ++ ++/* 6ca6d5c arithmetic oracle + 6cba480/6c7b64c. Native legacy mapping ++ * dialect has bit36 clear: L=H, Laux=Haux. No PA substitution or float math. ++ */ ++ ++static const u32 g5p_r22_wait_words[] = { ++ 0x201, 0x138204, 0x10, 0x8388204, 0, 0x104000a, 0x1f, ++ 0x1f, 0x8388204, 1, 0x104000a, 0x1f, 0x1f, 0xf48008, ++ 0x10000015, 4, 0, 0x14, 0x109f, 0x13000a, 0x4000, ++ 0x4000, 0x138204, 0x4000, 0x8388204, 0, 0x104000a, 8, ++ 8, 0x8388204, 1, 0x104000a, 8, 8, ++}; ++ ++/* Full normal timestamped recipes, independent stamp identities TA0/3D1. ++ * k is dependency+4 (zero on the proven fresh initializer), not MMU tag. ++ */ ++ ++/* PM bank0 save writes runtime qwords before the a0/a8/b0 backing anchors. */ ++static inline bool g5p_r22_pm_saved(const u8 *state) ++{ ++ for (u32 off = 8; off < 0xa0; off += 4) ++ if (g5p_le32(state + off)) ++ return true; ++ return false; ++} ++ ++/* Never trust a firmware pointer until it equals our retained stamp word. */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-layout.h b/drivers/gpu/apple-g5p/render-layout.h +new file mode 100644 +index 0000000000..f2a0373dad +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_H ++#define G5P_RENDER_LAYOUT_H ++ ++#include "boot-layout.h" ++ ++/* Offline construction primitives only. No TA/3D publication path exists. ++ * 6caf9f8/6cb2378: caller must recover N/S1/S2/S3, not infer them from USC ++ * count or target dimensions. Sizes are bounded to u32 allocation extents. ++ */ ++struct g5p_pm_sizes { ++ u32 control, page_lists, backing; ++}; ++ ++static inline int g5p_pm_sizes(u32 n, u32 blocks, u64 s1, u64 s2, u64 s3, ++ struct g5p_pm_sizes *out) ++{ ++ u64 limit = 0xffffffffULL; ++ u64 a, b; ++ ++ if (!n || !blocks || !s1 || !s2 || !s3) ++ return -EINVAL; ++ if (n > (limit - 0x80) / 0xc0 || blocks > (limit - 0x44) / 0x24 || ++ s1 > limit / n || s2 > limit / n || s3 > limit) ++ return -EOVERFLOW; ++ a = s1 * n; ++ b = s2 * n; ++ if (a > limit - b || a + b > limit - s3) ++ return -EOVERFLOW; ++ out->control = 0x80 + n * 0xc0; ++ out->page_lists = 0x44 + blocks * 0x24; ++ out->backing = a + b + s3; ++ return 0; ++} ++ ++/* Only four recovered backing-pointer writes, not a complete PM record. ++ * Caller owns a fully initialized0xc0 record and validates GPU mappings. ++ */ ++static inline int g5p_pm_record_backing(u8 *record, u32 index, u32 n, u64 base, ++ u64 s1, u64 s2, u64 s3, u32 meta) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (index >= n) ++ return -EINVAL; ++ ret = g5p_pm_sizes(n, 1, s1, s2, s3, &sizes); ++ if (ret) ++ return ret; ++ if (base > ~0ULL - sizes.backing) ++ return -EOVERFLOW; ++ g5p_put64(record + 0xa8, base + index * s1); ++ g5p_put64(record + 0xa0, base + n * s1 + index * s2); ++ g5p_put64(record + 0xb0, base + n * s1 + n * s2); ++ g5p_put32(record + 0xb8, meta); ++ return 0; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-bind.h b/drivers/gpu/apple-g5p/render-user-bind.h +new file mode 100644 +index 0000000000..3bbbb1e0fc +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-bind.h +@@ -0,0 +1,285 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Firmware slot pairs of render contexts (docs/gpu/multi-client-design.md, stage 1). ++ * ++ * Each render context owns a TA and a 3D firmware slot in 7..255, both pointing at its own ++ * page catalogue. The pair is bound once by two DC3 records {3, slot, 0, 0, 0}, 3D first, ++ * then TA, with the catalogue PA in 0x86[slot] and the host tag in 0x84[slot]. The firmware's ++ * DC3 handler installs CBASE and copies the host tag to 0x85[slot]. ++ * ++ * These helpers only read and write those RAM words, so they cannot reach a register and are ++ * safe after a job has retired. Pure functions, host-tested ++ * (scripts/gpu/multiclient1-host/test_render_bind.c). The caller provides g5p_le32(), ++ * g5p_put32() and g5p_put64(). ++ */ ++#ifndef G5P_RENDER_USER_BIND_H ++#define G5P_RENDER_USER_BIND_H ++ ++#define G5P_RU_SLOT_MIN 7U ++#define G5P_RU_SLOT_MAX 255U ++/* RAM-only wait for the firmware to consume a published DC3 batch. */ ++#define G5P_RU_BIND_MS 3000U ++/* Host tag of a pre-bound spare pair (render_user_spare_pairs): never zero. */ ++#define G5P_RU_SPARE_TAG 0x00400000U ++/* Host tag of a parked pair (render_user_close_park): never zero, never a spare's. */ ++#define G5P_RU_PARK_TAG 0x00500000U ++#define G5P_RU_MAX_SPARES 32U ++ ++/* The words one pair's binding touches. Pointers, because the DC records may straddle the ++ * shared pages. Index [e]: 0 TA slot, 1 3D slot. ++ */ ++struct g5p_ru_bind_mem { ++ u8 *rec[2][5]; /* the records at start and start + 1 */ ++ u8 *pa[2]; /* 0x86[slot], 64 bit */ ++ u8 *host[2]; /* 0x84[slot] */ ++ u8 *cached[2]; /* 0x85[slot] */ ++}; ++ ++/* One RAM sample of those words and of the DC ring control. */ ++struct g5p_ru_bind_sample { ++ u32 cons, prod; ++ u32 rec[2][5]; ++ u64 pa[2]; ++ u32 host[2], cached[2]; ++}; ++ ++/* g5p_ru_bind_busy() */ ++#define G5P_RU_BIND_BUSY_RING 0x1U /* the DC ring is not idle */ ++#define G5P_RU_BIND_BUSY_SLOT 0x2U /* a slot has a PA or a host tag */ ++ ++/* g5p_ru_bind_check(), the r74g criteria */ ++#define G5P_RU_BIND_BAD_RING 0x1U ++#define G5P_RU_BIND_BAD_RECORD 0x2U ++#define G5P_RU_BIND_BAD_PA 0x4U ++#define G5P_RU_BIND_BAD_HOST 0x8U ++#define G5P_RU_BIND_BAD_CACHED 0x10U ++ ++enum g5p_ru_bind_wait { ++ G5P_RU_BIND_PENDING, ++ G5P_RU_BIND_CONSUMED, ++ G5P_RU_BIND_PROTO, ++ G5P_RU_BIND_TIMEOUT, ++}; ++ ++static inline bool g5p_ru_slot_used(const unsigned long *map, u32 slot) ++{ ++ return (map[slot / BITS_PER_LONG] >> (slot % BITS_PER_LONG)) & 1; ++} ++ ++/* The slot pair of a new render context: the two lowest free slots in 7..255. The lower one ++ * is the 3D slot, or the TA slot with ta_first (render_user_slot_variant 1). False when fewer ++ * than two are free. slot[0] is TA, slot[1] is 3D. ++ */ ++static inline bool g5p_ru_slot_pair(const unsigned long *map, bool ta_first, ++ u32 *slot) ++{ ++ u32 pick[2], n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < 2; s++) ++ if (!g5p_ru_slot_used(map, s)) ++ pick[n++] = s; ++ if (n < 2) ++ return false; ++ slot[0] = pick[ta_first ? 0 : 1]; ++ slot[1] = pick[ta_first ? 1 : 0]; ++ return true; ++} ++ ++/* Word j of record i of a pair: {3, slot, 0, 0, 0}, the 3D slot's record first. */ ++static inline u32 g5p_ru_bind_record(const u32 *slot, u32 i, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot[1 - i] : 0; ++} ++ ++static inline void g5p_ru_bind_read(const struct g5p_ru_bind_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_bind_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ s->rec[i][j] = g5p_le32(m->rec[i][j]); ++ for (u32 e = 0; e < 2; e++) { ++ s->pa[e] = (u64)g5p_le32(m->pa[e] + 4) << 32 | ++ g5p_le32(m->pa[e]); ++ s->host[e] = g5p_le32(m->host[e]); ++ s->cached[e] = g5p_le32(m->cached[e]); ++ } ++} ++ ++/* Before a publish: the ring must be idle and both slots unused (zero PA and host tag). */ ++static inline u32 g5p_ru_bind_busy(const struct g5p_ru_bind_sample *s) ++{ ++ u32 busy = 0; ++ ++ if (s->prod > 255 || s->cons != s->prod) ++ busy |= G5P_RU_BIND_BUSY_RING; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] || s->host[e]) ++ busy |= G5P_RU_BIND_BUSY_SLOT; ++ return busy; ++} ++ ++/* ++ * A parked pair (render_user_close_park, release8 quiet2): a closed context's slots, bound to the ++ * kernel's park catalogue (it maps only the firmware window) with the park tag, so that the PM's ++ * late write-back of that context's free list resolves instead of faulting on an empty PC level. ++ * Both slots on `park`, both host tags the park tag. park 0: nothing is parked. ++ */ ++static inline bool g5p_ru_bind_parked(const struct g5p_ru_bind_sample *s, ++ u64 park) ++{ ++ if (!park) ++ return false; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] != park || s->host[e] != G5P_RU_PARK_TAG) ++ return false; ++ return true; ++} ++ ++/* PA and tag of both slots, then the two records. The caller cleans the catalogue first and ++ * stores the producer afterwards, each behind a barrier. ++ */ ++static inline void g5p_ru_bind_publish(const struct g5p_ru_bind_mem *m, ++ const u32 *slot, u64 root, u32 tag) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ u32 e = 1 - i; ++ ++ g5p_put64(m->pa[e], root); ++ g5p_put32(m->host[e], tag); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[i][j], g5p_ru_bind_record(slot, i, j)); ++ } ++} ++ ++/* After the ring reached `target`: the records intact, both slots on `root`, and the firmware ++ * tag equal to the host tag (the DC3 handler ran for both). ++ */ ++static inline u32 g5p_ru_bind_check(const struct g5p_ru_bind_sample *s, ++ const u32 *slot, u64 root, u32 tag, ++ u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[i][j] != g5p_ru_bind_record(slot, i, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ for (u32 e = 0; e < 2; e++) { ++ if (s->pa[e] != root) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host[e] != tag) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached[e] != tag) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ } ++ return bad; ++} ++ ++/* One sample of a published batch. The caller takes `expired` before reading the ring, so ++ * the sample taken at the deadline still counts. ++ */ ++static inline int g5p_ru_bind_step(u32 cons, u32 prod, u32 target, bool expired) ++{ ++ if (prod != target || cons > 255) ++ return G5P_RU_BIND_PROTO; ++ if (cons == target) ++ return G5P_RU_BIND_CONSUMED; ++ return expired ? G5P_RU_BIND_TIMEOUT : G5P_RU_BIND_PENDING; ++} ++ ++/* ++ * Unbind (iOS AGXGart_detachSlotsAndDestroyAddressSpace, firmware-re README round 15 §1): PA ++ * and host tag of a slot go to zero and one DC3 record {3, slot, 0, 0, 0} follows. The DC3 ++ * handler then installs CBASE 0, copies the zero tag to 0x85[slot] and invalidates the slot's ++ * PC level. The slots of one batch take consecutive ring entries, one record each, behind a ++ * single producer store. The caller must have retired every use of the slots first. ++ */ ++#define G5P_RU_USER_SLOTS (G5P_RU_SLOT_MAX - G5P_RU_SLOT_MIN + 1) ++ ++struct g5p_ru_slot_mem { ++ u8 *rec[5]; /* the slot's record in the batch */ ++ u8 *pa, *host, ++ *cached; /* 0x86[slot] (64 bit), 0x84[slot], 0x85[slot] */ ++}; ++ ++struct g5p_ru_slot_sample { ++ u32 cons, prod; ++ u32 rec[5]; ++ u64 pa; ++ u32 host, cached; ++}; ++ ++/* The user slots the firmware may hold: those with a PA or a host tag. At most `max`. */ ++static inline u32 g5p_ru_held_slots(const u8 *pa, const u8 *host, u32 *slots, ++ u32 max) ++{ ++ u32 n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < max; s++) ++ if (g5p_le32(pa + s * 8) || g5p_le32(pa + s * 8 + 4) || ++ g5p_le32(host + s * 4)) ++ slots[n++] = s; ++ return n; ++} ++ ++/* A batch of n records fits the 256-entry ring behind an idle consumer. */ ++static inline bool g5p_ru_unbind_fits(u32 n) ++{ ++ return n && n <= G5P_RU_USER_SLOTS; ++} ++ ++static inline u32 g5p_ru_unbind_record(u32 slot, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot : 0; ++} ++ ++static inline void g5p_ru_slot_read(const struct g5p_ru_slot_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_slot_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 j = 0; j < 5; j++) ++ s->rec[j] = g5p_le32(m->rec[j]); ++ s->pa = (u64)g5p_le32(m->pa + 4) << 32 | g5p_le32(m->pa); ++ s->host = g5p_le32(m->host); ++ s->cached = g5p_le32(m->cached); ++} ++ ++/* PA and host tag to zero, then the record. The caller stores the producer behind a barrier. */ ++static inline void g5p_ru_unbind_publish(const struct g5p_ru_slot_mem *m, ++ u32 slot) ++{ ++ g5p_put64(m->pa, 0); ++ g5p_put32(m->host, 0); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[j], g5p_ru_unbind_record(slot, j)); ++} ++ ++/* After the ring reached `target`: the record intact, PA and host tag still zero, and the ++ * firmware's tag zero too, which only its DC3 handler writes (the slot was bound with a ++ * non-zero tag). The G5P_RU_BIND_BAD_* bits. ++ */ ++static inline u32 g5p_ru_unbind_check(const struct g5p_ru_slot_sample *s, ++ u32 slot, u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[j] != g5p_ru_unbind_record(slot, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ if (s->pa) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ return bad; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-close.h b/drivers/gpu/apple-g5p/render-user-close.h +new file mode 100644 +index 0000000000..2aa461bce0 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-close.h +@@ -0,0 +1,91 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Release of a closed render context (docs/gpu/multi-client-design.md, stage 2). ++ * ++ * A TYPED_V3 context whose handle is gone (DESTROY_CONTEXT or file close) gets one close item. ++ * It runs after the context's last job, in the job order, and decides from plain values what ++ * the firmware may still hold: ++ * ++ * RELEASE the firmware never held the pair: drop the references, free everything; ++ * UNBIND the pair is bound and every use retired: RAM-only DC3 unbind ++ * (g5p_rv3_unbind_slots()), then free on success, retain on failure; ++ * RETAIN the firmware may still use the pair or its memory: keep the context, its ++ * slots and its memory until reboot (the kexec quiesce detaches them then). ++ * ++ * The rule follows iOS AGXGart_detachSlotsAndDestroyAddressSpace (firmware-re README round 15 ++ * §1, "Linux sequence" step 8): stop new jobs, wait for every engine use, publish PA and tag ++ * zero, consume the DC3, and only then recycle the slots and free the translations. A slot ++ * returns to the allocator only after its unbind was consumed and verified, so an old unbind ++ * can never clear a new owner's PA, and the lowest free pair may be reused at once (iOS pops ++ * a freed slot off its free stack first). ++ * ++ * Pure function, host-tested (scripts/gpu/multiclient2-host/test_render_close.c). ++ */ ++#ifndef G5P_RENDER_USER_CLOSE_H ++#define G5P_RENDER_USER_CLOSE_H ++ ++enum g5p_ru_close_plan { ++ G5P_RU_CLOSE_RELEASE, ++ G5P_RU_CLOSE_UNBIND, ++ G5P_RU_CLOSE_RETAIN, ++}; ++ ++/* Why a bound or published pair is retained (bits). */ ++#define G5P_RU_CLOSE_UNVERIFIED \ ++ 0x01U /* a bind was published but never checked out */ ++#define G5P_RU_CLOSE_BAD \ ++ 0x02U /* a job of this context failed on the hardware */ ++#define G5P_RU_CLOSE_TERMINAL 0x04U /* the epoch noted a timeout or fault */ ++#define G5P_RU_CLOSE_FAILED 0x08U /* the device failed */ ++#define G5P_RU_CLOSE_STOPPED 0x10U /* the device stopped (teardown, quiesce) */ ++#define G5P_RU_CLOSE_SHUTDOWN \ ++ 0x20U /* admission closed for good (reboot quiesce) */ ++#define G5P_RU_CLOSE_QUEUED 0x40U /* a job of this context is still queued */ ++#define G5P_RU_CLOSE_PENDING 0x80U /* an earlier DC batch was never consumed */ ++#define G5P_RU_CLOSE_LOCK 0x100U /* the command lock could not be taken */ ++ ++struct g5p_ru_close_in { ++ bool retained; /* the firmware may hold the pair: a DC3 bind was published */ ++ bool bound; /* that bind was consumed and checked */ ++ bool bad, term_failed, dev_failed, stopped, shutdown, dc_pending; ++ u32 queued; ++}; ++ ++static inline const char *g5p_ru_close_name(int plan) ++{ ++ return plan == G5P_RU_CLOSE_RELEASE ? "release" : ++ plan == G5P_RU_CLOSE_UNBIND ? "unbind" : ++ "retain"; ++} ++ ++/* The plan, and in *why the G5P_RU_CLOSE_* bits that made it RETAIN (0 otherwise). */ ++static inline int g5p_ru_close_plan(const struct g5p_ru_close_in *in, u32 *why) ++{ ++ u32 w = 0; ++ ++ *why = 0; ++ if (!in->retained) ++ return in->queued ? (*why = G5P_RU_CLOSE_QUEUED, ++ G5P_RU_CLOSE_RETAIN) : ++ G5P_RU_CLOSE_RELEASE; ++ if (!in->bound) ++ w |= G5P_RU_CLOSE_UNVERIFIED; ++ if (in->bad) ++ w |= G5P_RU_CLOSE_BAD; ++ if (in->term_failed) ++ w |= G5P_RU_CLOSE_TERMINAL; ++ if (in->dev_failed) ++ w |= G5P_RU_CLOSE_FAILED; ++ if (in->stopped) ++ w |= G5P_RU_CLOSE_STOPPED; ++ if (in->shutdown) ++ w |= G5P_RU_CLOSE_SHUTDOWN; ++ if (in->queued) ++ w |= G5P_RU_CLOSE_QUEUED; ++ if (in->dc_pending) ++ w |= G5P_RU_CLOSE_PENDING; ++ *why = w; ++ return w ? G5P_RU_CLOSE_RETAIN : G5P_RU_CLOSE_UNBIND; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +new file mode 100644 +index 0000000000..ccb7d2c8f0 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The phase wait's pause between two RAM samples (perf60-fillrate.md §11.3 cut 1): woken by the ++ * firmware event, not a fixed 1-2 ms sleep. Host test: scripts/gpu/perf60-host/test_evwait.c. ++ * ++ * Protocol (g5p_rv3_wait): drain -> seen = event_serial -> sample RAM -> decide -> pause. ++ * The IRQ thread's drain decodes each firmware event, bumps event_serial under status_lock and ++ * wakes the queue. The pause sleeps until event_serial != seen or the safety timeout, so an ++ * event decoded at any point after `seen` was read (before, during or after the sample) ends the ++ * pause at once: no lost wakeup. The timeout is the safety net for a RAM state that changes ++ * without an event; the predicate stays level-triggered on every sample, as before. ++ * Once the phase's own completion event is latched (s->event) but the RAM rules still wait ++ * (idle words, retire stamp, PM receipt land a little after the event), the pause is a short ++ * G5P_EW_POST_US poll: there is no further event to wait for. ++ */ ++#ifndef G5P_RENDER_USER_EVWAIT_H ++#define G5P_RENDER_USER_EVWAIT_H ++ ++#define G5P_EW_POST_US \ ++ 100U /* after the phase's own event: completion is imminent */ ++#define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ ++ ++enum g5p_ew_act { ++ G5P_EW_LEGACY, /* render_user_event_wake 0: usleep_range(1000, 2000), release8 */ ++ G5P_EW_SHORT, /* the phase's event seen: sleep G5P_EW_POST_US */ ++ G5P_EW_EVENT, /* sleep until any event since `seen`, at most *wait_us */ ++}; ++ ++static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, ++ u32 *wait_us) ++{ ++ if (!knob_us) { ++ *wait_us = 0; ++ return G5P_EW_LEGACY; ++ } ++ if (phase_event) { ++ *wait_us = G5P_EW_POST_US; ++ return G5P_EW_SHORT; ++ } ++ *wait_us = knob_us < G5P_EW_POST_US ? G5P_EW_POST_US : ++ knob_us > G5P_EW_MAX_US ? G5P_EW_MAX_US : ++ knob_us; ++ return G5P_EW_EVENT; ++} ++ ++/* The wait condition: an event was decoded since `seen` was read. */ ++static inline bool g5p_ew_woken(u64 seen, u64 serial) ++{ ++ return serial != seen; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-fault.h b/drivers/gpu/apple-g5p/render-user-fault.h +new file mode 100644 +index 0000000000..2af9f7513c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-fault.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * A client job that faults (docs/gpu/experiments/faultiso.md). Pure decisions, host-tested ++ * (scripts/gpu/faultiso-host/test_fault_iso.c). ++ * ++ * On this firmware a GPU page fault is not survivable in place. The firmware acknowledges the ++ * MMU fault itself (R0+0x138), reports it as event 4 and parks (firmware-re README "Fault ++ * ownership"); iOS answers event 4 only with its restart tail (kext 6c81dac -> 6c834a0: firmware ++ * stop, context and dependency reset, shared state rebuilt, firmware boot), and has no ++ * per-context ACK, channel reset or slot unbind for it. The no-reset path iOS does have, the ++ * firmware's own job kill (event 8, then device-control command 0xa), is not a page fault. So a ++ * job that stops the firmware, by a fault event or by a silent event stream, loses its context ++ * and the GPU is recovered in the boot (render_user_recover), or the epoch ends. ++ */ ++#ifndef G5P_RENDER_USER_FAULT_H ++#define G5P_RENDER_USER_FAULT_H ++ ++/* The errno a lost job reports: a timeout's own, else the fault's (-EIO for a fault event or a ++ * silent stream, -EPROTO, or the drain's error). `result` is the phase's G5P_RU_WAIT_*. ++ */ ++static inline int g5p_ru_lost_errno(u32 result, int err) ++{ ++ return result == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++} ++ ++/* ++ * Does a job that failed after touching the hardware fail the device too? The legacy profiles ++ * always do. A TYPED_V3 job does not when it timed out (nothing is wrong with the protocol and ++ * the firmware may still finish), when its bind failed (the DC ring is not on the path of the ++ * contexts already bound), or when it was lost (the executor ruled: the channel drains, or a ++ * recovery replaces the epoch). Its context is quarantined in every case. ++ */ ++static inline bool g5p_ru_job_fails_device(bool v3, int ret, bool bind_failed, ++ bool lost) ++{ ++ return !v3 || (ret != -ETIMEDOUT && !bind_failed && !lost); ++} ++ ++/* ++ * A device failure noted while a job or a drain runs (a fault event decoded mid-phase), or while ++ * a recovery is pending, is not yet ruled on: the executor either loses that job's context and ++ * recovers, or ends the epoch. Until then the other contexts wait (submit accepted and held, ++ * VM_BIND busy, context status not lost) instead of failing. The executor sets `recovering` ++ * before it clears `executing`, so a reader that loads `executing` with acquire and then ++ * `recovering` always sees one of them through a recovery. ++ */ ++static inline bool g5p_ru_ruling(bool executing, bool recovering) ++{ ++ return executing || recovering; ++} ++ ++/* render_user_test_fault_job: the context's job about to start (it has run `jobs` before) gets ++ * the unmapped control stream. Never the epoch's first job: its attach and bell are special. ++ */ ++static inline bool g5p_ru_fault_due(u32 fault_job, u32 jobs, bool first) ++{ ++ return fault_job && jobs + 1 == fault_job && !first; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-faultreg.h b/drivers/gpu/apple-g5p/render-user-faultreg.h +new file mode 100644 +index 0000000000..83cac51f7b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-faultreg.h +@@ -0,0 +1,124 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 fault address (render_user_fault_read, drm-render-v3.inc): decode of a GPU MMU fault ++ * status bank, R0+0xe150/e154 (bank 0) or R0+0xe158/e15c (bank 1), as high32:low32. ++ * ++ * The firmware's fault path (source 4 -> 1001a494) reads both banks with a 64-bit load, keeps ++ * only a class in its companion record and posts event 4 without the status, so the address ++ * exists only in these registers. iOS reads them itself in its fault handler (kext 6ca5c14, ++ * accelerator virtual+740) with exactly these fields (docs/gpu/firmware-re/README.md "Complete ++ * e150 decode"): bit 0 valid, 2:1 type (0 page invalid, 2 read-only, 3 PM/META protected), ++ * 3 read (0 write), 5:4 level (3 = catalogue), 11:6 requester (iOS table 6ff5ed8), 19:12 ++ * sideband, 27:20 MMU context (the slot), VA = (status >> 24) & 0xfffffffff0 (16-byte precision). ++ * Pure, host-tested (scripts/gpu/perf60-host/test_fault_decode.c). ++ */ ++#ifndef G5P_RENDER_USER_FAULTREG_H ++#define G5P_RENDER_USER_FAULTREG_H ++ ++#include "bits-compat.h" ++struct g5p_ru_mmu_fault { ++ bool valid, read; ++ u8 type, level, requester, sideband, context; ++ u64 va; ++}; ++ ++static inline struct g5p_ru_mmu_fault g5p_ru_mmu_fault_decode(u32 lo, u32 hi) ++{ ++ u64 s = (u64)hi << 32 | lo; ++ struct g5p_ru_mmu_fault f = { ++ .valid = s & 1, ++ .type = (s >> 1) & 3, ++ .read = (s >> 3) & 1, ++ .level = (s >> 4) & 3, ++ .requester = (s >> 6) & 0x3f, ++ .sideband = (s >> 12) & 0xff, ++ .context = (s >> 20) & 0xff, ++ .va = (s >> 24) & 0xfffffffff0ULL, ++ }; ++ ++ return f; ++} ++ ++static const char *const g5p_ru_mmu_fault_types[4] = { ++ "page-invalid", ++ "type1", ++ "read-only", ++ "pm-meta-protected", ++}; ++ ++/* Requester names, iOS kext table 6ff5ed8 (16-byte {name, class} entries; read from the j71t ++ * 16.7.10 kernelcache, 2026-11-03). 56 entries; a larger index prints its number only. ++ */ ++static const char *const g5p_ru_mmu_requesters[] = { ++ "META", ++ "TLA", ++ "HOST", ++ "VDMM", ++ "CDM", ++ "IPP", ++ "PM", ++ "TILING", ++ "MCU_TEXAS0", ++ "PDS_TEXAS0", ++ "PBE0_TEXAS0", ++ "PBE1_TEXAS0", ++ "VDMS_BLACKPEARL0", ++ "IPF_BLACKPEARL0", ++ "ISP_BLACKPEARL0", ++ "TPF_BLACKPEARL0", ++ "USCS_BLACKPEARL0", ++ "PPP_BLACKPEARL0", ++ "VCE_BLACKPEARL0", ++ "FBCDC_BLACKPEARL0", ++ "MCU_TEXAS1", ++ "PDS_TEXAS1", ++ "PBE0_TEXAS1", ++ "PBE1_TEXAS1", ++ "MCU_TEXAS2", ++ "PDS_TEXAS2", ++ "PBE0_TEXAS2", ++ "PBE1_TEXAS2", ++ "VDMS_BLACKPEARL1", ++ "IPF_BLACKPEARL1", ++ "ISP_BLACKPEARL1", ++ "TPF_BLACKPEARL1", ++ "USCS_BLACKPEARL1", ++ "PPP_BLACKPEARL1", ++ "VCE_BLACKPEARL1", ++ "FBCDC_BLACKPEARL1", ++ "MCU_TEXAS3", ++ "PDS_TEXAS3", ++ "PBE0_TEXAS3", ++ "PBE1_TEXAS3", ++ "MCU_TEXAS4", ++ "PDS_TEXAS4", ++ "PBE0_TEXAS4", ++ "PBE1_TEXAS4", ++ "VDMS_BLACKPEARL2", ++ "IPF_BLACKPEARL2", ++ "ISP_BLACKPEARL2", ++ "TPF_BLACKPEARL2", ++ "USCS_BLACKPEARL2", ++ "PPP_BLACKPEARL2", ++ "VCE_BLACKPEARL2", ++ "FBCDC_BLACKPEARL2", ++ "MCU_TEXAS5", ++ "PDS_TEXAS5", ++ "PBE0_TEXAS5", ++ "PBE1_TEXAS5", ++}; ++ ++static inline const char *g5p_ru_mmu_requester(u32 requester) ++{ ++ if (requester >= ARRAY_SIZE(g5p_ru_mmu_requesters)) ++ return "?"; ++ return g5p_ru_mmu_requesters[requester]; ++} ++ ++/* The range test of the VM edit log: does [va, va + size) hold the fault address? */ ++static inline bool g5p_ru_mmu_fault_in(u64 fault_va, u64 va, u64 size) ++{ ++ return fault_va >= va && fault_va - va < size; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-geometry.h b/drivers/gpu/apple-g5p/render-user-geometry.h +new file mode 100644 +index 0000000000..2117d567bc +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-geometry.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_GEOMETRY_H ++#define G5P_RENDER_USER_GEOMETRY_H ++ ++/* Kernel-derived render-target geometry: AGXRenderTargetMemoryG5's arithmetic ++ * leaf 6ca6d5c (firmware round16 section 3) for one sample at the pixel ++ * centre (8,8) or four samples (docs/gpu/experiments/msaa.md), in integer ++ * arithmetic only. It is Rogue's non-simple ++ * parameter format: 32x32 tiles, 4x4 macrotiles of ++ * ALIGN(DIV_ROUND_UP(tiles, 4), 4) tiles per axis, five-byte region headers ++ * per padded tile, a tail pointer cache of next_pow2(max(padded tiles))^2 ++ * eight-byte entries. The scale pairs are the leaf's IEEE singles ++ * 0x3e89309c / n, 0x3fddb3d9 / n (fdiv, round to nearest even) and n * 16. ++ * tools/gpu/test_rt_oracle.c compares every output with the unmodified leaf ++ * for every width and height in 1..G5P_RU_MAX_DIM. ++ * ++ * G5P_RU_MAX_DIM is rogue_get_render_size_max() for this tiling class; it ++ * is a memory bound (16 MiB private window, 8 MiB main ring), not a field ++ * limit: the 9-bit TE macrotile and 10-bit padded-size fields would hold ++ * twice as much. ++ */ ++#define G5P_RU_MAX_DIM 8192U ++#define G5P_RU_SCALE_A 0x3e89309cU ++#define G5P_RU_SCALE_B 0x3fddb3d9U ++ ++static inline u32 g5p_ru_fls(u32 v) ++{ ++ return v ? 32 - __builtin_clz(v) : 0; ++} ++ ++/* ucvtf of 1 <= n < 2^24: exact. */ ++static inline u32 g5p_ru_f32(u32 n) ++{ ++ u32 bits = g5p_ru_fls(n); ++ ++ return ((126 + bits) << 23) | ((n << (24 - bits)) & 0x7fffff); ++} ++ ++/* fdiv of a normal positive single by 1 <= n < 2^24, round to nearest even. ++ * The quotient q of the 24-bit significand shifted by fls(n) + 3 has 27 or ++ * 28 bits; s of them are rounding bits, the remainder is the sticky bit. ++ * Results stay normal for the two leaf constants and n <= G5P_RU_MAX_DIM. ++ */ ++static inline u32 g5p_ru_fdiv(u32 a, u32 n) ++{ ++ u32 shift = g5p_ru_fls(n) + 3; ++ u64 num = (u64)((a & 0x7fffff) | 0x800000) << shift; ++ u32 q = num / n, sticky = num % n != 0; ++ u32 s = g5p_ru_fls(q) - 24, m = q >> s, low = q & ((1U << s) - 1); ++ ++ if (low > 1U << (s - 1) || ++ (low == 1U << (s - 1) && (sticky || (m & 1)))) ++ m++; ++ if (m >> 24) { ++ m >>= 1; ++ s++; ++ } ++ return ((((a >> 23) & 0xff) - shift + s) << 23) | (m & 0x7fffff); ++} ++ ++/* Sample positions, one byte per sample: x in [3:0], y in [7:4], 1/16 px. ++ * One sample sits at the pixel centre; four use the standard Vulkan/D3D ++ * pattern, which is also Metal's (record+0x69c..0x6b8 of the iOS ++ * msaa4-resolve/-store captures: (6,2) (14,6) (2,10) (10,14)). ++ */ ++#define G5P_RU_POSITIONS_1X 0x88ULL ++#define G5P_RU_POSITIONS_4X 0xeaa26e26ULL ++ ++/* The leaf for one or four samples (6ca6d5c's 1, 2, 4, 8 table, of which ++ * Linux uses 1 and 4). Four samples are 2x2 per pixel: the region headers ++ * and the tail pointer cache are sized for twice the tiles in x and y, the ++ * ISP macrotile size (Rbc) doubles in y, Rb8 is log2(samples) (5940f40) ++ * and the TE word Rf0 is 5. Rb0 and Rf8 are the positions word, packed by ++ * the leaf as x0 | y0 << 4 | x1 << 8 | ... | y3 << 28. The scales, the TE ++ * tiling words and the tile count do not depend on the samples. ++ */ ++static inline int g5p_ru_rt_derive_samples(u32 width, u32 height, u32 samples, ++ u64 positions, struct g5p_ru_rt *rt) ++{ ++ u32 tx = (width + 31) / 32, ty = (height + 31) / 32; ++ u32 mx = (((tx + 3) >> 2) + 3) & ~3U, my = (((ty + 3) >> 2) + 3) & ~3U; ++ u32 s = samples == 4 ? 2 : 1; ++ u32 px = mx * s * 4, py = my * s * 4, tpc; ++ ++ if (!width || !height || width > G5P_RU_MAX_DIM || ++ height > G5P_RU_MAX_DIM || (samples != 1 && samples != 4) || ++ positions >> (samples * 8)) ++ return -EINVAL; ++ tpc = 1U << g5p_ru_fls((px > py ? px : py) - 1); ++ *rt = (struct g5p_ru_rt) { ++ .header = px * py * 5, ++ .region = mx * my * s * s * 5, ++ .packed = positions, ++ .state = s - 1 ? 2 : 0, ++ .padded = (mx & 0x3ff) << 16 | ((my * s) & 0x3ff), ++ .scale = { ++ g5p_ru_fdiv(G5P_RU_SCALE_A, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_A, height) << 32, ++ g5p_ru_fdiv(G5P_RU_SCALE_B, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_B, height) << 32, ++ (g5p_ru_f32(width) + (4U << 23)) | ++ (u64)(g5p_ru_f32(height) + (4U << 23)) << 32, ++ }, ++ .ta_state = { s - 1 ? 5 : 0, positions }, ++ .ta_tiling = { ++ (u64)((ty - 1) << 12 | (tx - 1)) << 32 | ++ ((height - 1) << 16 | (width - 1)), ++ (u64)(my << 18 | my << 10 | 3 * my) << 32 | ++ (mx << 18 | mx << 10 | 3 * mx), ++ }, ++ .ta_tiling_word = mx * my, ++ .tiles = tx * ty, ++ .secondary = (u64)tpc * tpc * 8, ++ .secondary_align = 0x40, ++ }; ++ return 0; ++} ++ ++/* g5p_ru_rt64 is this function's result for 64x64. */ ++ ++/* PM pool for a kernel-derived target. Linux policy, not hardware: Rogue's ++ * minimum free list (50 4KiB pages, non-XE cores) plus one 4KiB PM page per ++ * tile, in 128KiB blocks, never below the latched pool and at most one F ++ * page. 64x64 (4 tiles) needs 2, so the latched 32 stays. There is no PM ++ * growth during a job: a frame that exhausts the pool fails like any OOM. ++ */ ++static inline u32 g5p_ru_pm_blocks_for(u32 tiles, u32 latched) ++{ ++ u64 need = ((50ULL + tiles) * 0x1000 + G5P_RU_BLOCK_BYTES - 1) / ++ G5P_RU_BLOCK_BYTES; ++ ++ if (need < latched) ++ need = latched; ++ return need > G5P_RU_MAX_BLOCKS ? G5P_RU_MAX_BLOCKS : need; ++} ++ ++ ++ ++static inline int g5p_ru_rt_derive(u32 width, u32 height, struct g5p_ru_rt *rt) ++{ ++ return g5p_ru_rt_derive_samples(width, height, 1, G5P_RU_POSITIONS_1X, rt); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +new file mode 100644 +index 0000000000..25950b4581 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -0,0 +1,644 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_LAYOUT_H ++#define G5P_RENDER_USER_LAYOUT_H ++ ++#include "bits-compat.h" ++/* Parameterized forms of the r22 fixed constructors for userspace render. ++ * Pure byte builders: no allocation, MMIO, locking or firmware publication. ++ * With r22's inputs every builder reproduces render-layout-r22.h exactly; ++ * tools/gpu/test_render_user.c checks that equivalence byte for byte. ++ */ ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++#include "render-user-template.h" ++ ++/* Kernel-private window in every render VM. r22's aux/RT addresses are the ++ * first allocations, so a 64x64 target keeps the fixed path's W anchors. ++ * Root entry 0 carries the boot VA0 sink, so user maps start at 1 GiB. ++ */ ++#define G5P_RU_PRIVATE_VA G5P_R22_AUX ++#define G5P_RU_PRIVATE_SIZE 0x1000000ULL ++#define G5P_RU_USER_VA_MIN 0x40000000ULL ++/* Catalogue entry of the firmware GiB, borrowed from the boot tables. */ ++#define G5P_RU_FIRMWARE_CI ((G5P_IMAGE_VA >> 30) & 1023) ++#define G5P_RU_AUX_BYTES 0x10680U ++#define G5P_RU_AUX_PAGES 5U ++#define G5P_RU_RT_ALIGN 0x10000ULL ++#define G5P_RU_MAX_BLOCKS 128U ++#define G5P_RU_BLOCK_BYTES 0x20000U ++#define G5P_RU_MAX_HEADER 0x80000U ++#define G5P_RU_MAX_SECONDARY 0x100000ULL ++#define G5P_RU_TA_W_BYTES 0x560U ++#define G5P_RU_3D_W_BYTES 0x8c0U ++#define G5P_RU_CMD_BYTES 0x40U ++ ++/* Kernel mirror of drm_g5p_rt_geometry (firmware round16 R-offsets). */ ++struct g5p_ru_rt { ++ u32 header, region; /* R10, R20 */ ++ u64 packed; /* Rb0 */ ++ u32 state, padded; /* Rb8, Rbc */ ++ u64 scale[3]; /* Rc0, Rc8, Rd0 */ ++ u64 ta_state[2]; /* Rf0, Rf8 */ ++ u64 ta_tiling[2]; /* R100, R108 */ ++ u32 ta_tiling_word, tiles; /* R110, R118 */ ++ u64 secondary, secondary_align; /* R138, R178 */ ++}; ++ ++/* 6ca6d5c executed offline for 64,64,1,8,8 (round16 section 3). */ ++static const struct g5p_ru_rt g5p_ru_rt64 = { ++ .header = 0x500, ++ .region = 0x50, ++ .packed = 0x88, ++ .state = 0, ++ .padded = 0x40004, ++ .scale = { 0x3b89309c3b89309cULL, 0x3cddb3d93cddb3d9ULL, ++ 0x4480000044800000ULL }, ++ .ta_state = { 0, 0x88 }, ++ .ta_tiling = { 0x00001001003f003fULL, 0x0010100c0010100cULL }, ++ .ta_tiling_word = 0x10, ++ .tiles = 4, ++ .secondary = 0x800, ++ .secondary_align = 0x40, ++}; ++ ++#include "render-user-geometry.h" ++ ++struct g5p_ru_rt_sizes { ++ u64 slice, main, secondary_map; ++}; ++ ++/* 6cb3d68: slice=R10 rounded to 4KiB; main ring is the power-of-two capacity ++ * for sixteen slices. 6cb40e0 places the secondary extent at offset 0 of ++ * its own mapping. Bounds are Linux allocation policy, not hardware limits. ++ */ ++static inline int g5p_ru_rt_sizes(const struct g5p_ru_rt *rt, ++ struct g5p_ru_rt_sizes *out) ++{ ++ u64 main = 0x1000; ++ ++ if (!rt->header || rt->header > G5P_RU_MAX_HEADER || !rt->secondary || ++ rt->secondary > G5P_RU_MAX_SECONDARY || !rt->secondary_align || ++ (rt->secondary_align & (rt->secondary_align - 1)) || ++ rt->secondary_align > G5P_GPU_PAGE || !rt->tiles) ++ return -EINVAL; ++ out->slice = ((u64)rt->header + 0xfff) & ~0xfffULL; ++ while (main < out->slice * 16) ++ main <<= 1; ++ out->main = main < G5P_GPU_PAGE ? G5P_GPU_PAGE : main; ++ out->secondary_map = (rt->secondary + G5P_GPU_PAGE - 1) & ++ ~(u64)(G5P_GPU_PAGE - 1); ++ return 0; ++} ++ ++/* 6cba480/6c7b64c anchors. Native mapping dialect: L=H, Laux=Haux. ++ * Equal to g5p_r22_rt() for g5p_ru_rt64 at the r22 addresses. 3D W58 and ++ * Wa8 are the importer's literal Rb0 and R20>>2 for the verified geometry. ++ */ ++static inline void g5p_ru_rt_fill(u8 *ta, u8 *td, const struct g5p_ru_rt *rt, ++ u64 main_va, u64 secondary_va) ++{ ++ g5p_put64(ta + 0x28, main_va); ++ g5p_put64(ta + 0x30, secondary_va); ++ g5p_put64(ta + 0x38, rt->ta_state[0]); ++ g5p_put64(ta + 0x40, rt->ta_state[1]); ++ g5p_put64(ta + 0x444, secondary_va); ++ g5p_put32(ta + 0x44c, rt->secondary); ++ g5p_put64(ta + 0x3ac, rt->ta_tiling[0]); ++ g5p_put64(ta + 0x3b4, rt->ta_tiling[1]); ++ g5p_put32(ta + 0x3bc, rt->ta_tiling_word); ++ g5p_put64(td + 0x20, main_va); ++ g5p_put64(td + 0x28, rt->packed); ++ g5p_put32(td + 0x30, rt->state); ++ g5p_put32(td + 0x34, rt->padded); ++ g5p_put32(td + 0x38, rt->region); ++ g5p_put64(td + 0x3c, rt->scale[0]); ++ g5p_put64(td + 0x44, rt->scale[1]); ++ g5p_put64(td + 0x4c, rt->scale[2]); ++ g5p_put64(td + 0x58, rt->packed); ++ g5p_put32(td + 0xa8, rt->region >> 2); ++ g5p_put32(td + 0x3a0, rt->padded); ++ g5p_put64(td + 0x3d0, rt->scale[0]); ++ g5p_put64(td + 0x3d8, rt->scale[1]); ++ g5p_put32(td + 0x3e0, rt->scale[2]); ++ g5p_put32(td + 0x3e4, rt->scale[0]); ++} ++ ++/* Validated typed render inputs, already range-checked by the caller. */ ++struct g5p_ru_pass { ++ u64 vdm_va; ++ u32 vdm_size, vdm_control; ++ u64 scratch_va, scratch_size, scratch_ptr[6]; ++ u64 pool7_va, pool7_size, pool8_va, pool8_size; ++ u32 context_pds[2][6], resume_pds[2][4], ta_pds[4]; ++ u32 background[8], eot[4], eot_resume[4], restore[4]; ++}; ++ ++/* Importer chains 6cba8a0/6c7b100/6c7b34c -> 6ca9710/6ca7ae0 (offline ++ * differential map in make-render-user-template.py). Every user VA field ++ * and code-relative descriptor lands here; the rest of W is the frozen ++ * dialect-1 scalar table or a kernel RT/channel field. ++ */ ++static inline void g5p_ru_fill_ta(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_TA_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_ta_fixed); i++) ++ g5p_put32(w + g5p_ru_ta_fixed[i][0], g5p_ru_ta_fixed[i][1]); ++ g5p_put64(w + 0x48, p->scratch_ptr[4]); ++ g5p_put64(w + 0x50, p->scratch_ptr[5]); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put32(w + 0x58 + i * 4, p->context_pds[0][i]); ++ g5p_put32(w + 0x70 + i * 4, p->context_pds[1][i]); ++ } ++ g5p_put64(w + 0x88, p->vdm_control & ~0xfULL); ++ g5p_put64(w + 0x90, p->scratch_va & ~7ULL); ++ g5p_put64(w + 0x98, p->vdm_va & ~3ULL); ++ for (u32 i = 0; i < 4; i++) { ++ g5p_put32(w + 0xa0 + i * 4, p->resume_pds[0][i]); ++ g5p_put32(w + 0xb0 + i * 4, p->resume_pds[1][i]); ++ } ++ for (u32 i = 0; i < 3; i++) ++ g5p_put32(w + 0xc0 + i * 4, p->ta_pds[i]); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x468 + i * 8, p->scratch_ptr[i]); ++ g5p_put64(w + 0x488, p->scratch_ptr[4]); ++} ++ ++static inline void g5p_ru_fill_3d(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_3D_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_3d_fixed); i++) ++ g5p_put32(w + g5p_ru_3d_fixed[i][0], g5p_ru_3d_fixed[i][1]); ++ g5p_put64(w + 0x60, p->pool8_va & ~3ULL); ++ g5p_put64(w + 0x68, p->pool7_va & ~3ULL); ++ g5p_put64(w + 0x490, p->pool7_va); ++ g5p_put64(w + 0x4a0, p->pool8_va); ++ for (u32 i = 0; i < 8; i++) ++ g5p_put32(w + 0x5b8 + i * 4, p->background[i]); ++ g5p_put32(w + 0xb0, p->background[0]); ++ g5p_put32(w + 0xb4, p->background[1]); ++ for (u32 i = 4; i < 8; i++) ++ g5p_put32(w + 0xc0 + (i - 4) * 4, p->background[i]); ++ /* 6ca7ae0 swaps the first EOT qword's halves in its B upload. */ ++ g5p_put32(w + 0x3b4, p->eot[1]); ++ g5p_put32(w + 0x3b8, p->eot[0]); ++ g5p_put32(w + 0x3bc, p->eot[2]); ++ for (u32 i = 0; i < 3; i++) { ++ g5p_put32(w + 0x680 + i * 4, p->eot[i]); ++ g5p_put32(w + 0x694 + i * 4, p->eot_resume[i]); ++ } ++ g5p_put32(w + 0x600, p->restore[0]); ++ g5p_put32(w + 0x604, p->restore[1]); ++ g5p_put32(w + 0x610, p->restore[2]); ++ g5p_put32(w + 0x614, p->restore[3]); ++} ++ ++/* TYPED_V2 additions, already validated by g5p_ru_pass2_check(). */ ++struct g5p_ru_pass2 { ++ u64 scissor_va, dbias_va, background_texture, zls[6]; ++ u32 scissor_count, dbias_count, bgobj_depth, bgobj_vals; ++ u32 pixel_width, zls_mask; ++}; ++ ++/* Dialect-2 words over g5p_ru_fill_3d() (TYPED_V2 has zero pools). Each ++ * register image goes to its B-upload word and to the normal builder's ++ * copy (6c7a06c); make-render-user-dialect2.py proves every position by ++ * perturbing record 2d8/2e0 (pools), 2f0/310/338/360/388/308 (ZLS), 508 ++ * (background texture) and 4d0 (clears) through the r22 importer. ++ * Pixel output: 6ca7ae0 for D2f8 = width, one sample (D6e0 = 0), D630 = ++ * D6b2 = 0, six cores. Widths 2/4/8 keep two or more partitions, so W3c0 ++ * and W3c4 keep their dialect-1 values; width 2 is dialect 1 exactly. ++ */ ++static inline void g5p_ru_fill_3d_width(u8 *w, u32 width) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by D2f8 - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u32 count = 0x4000 / (width << 10); ++ u32 code = width_code[width - 2]; ++ ++ g5p_put32(w + 0x3c8, count & 0x1f); ++ g5p_put32(w + 0x3cc, code >> 4 | 0xf8); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(w + 0x108 + i * 8, code | (count - 1) << 7 | 0x43); ++ g5p_put32(w + 0x6b0, width); ++} ++ ++static inline void g5p_ru_fill_3d_v2(u8 *w, const struct g5p_ru_pass2 *e) ++{ ++ static const u16 zls_w[6][2] = { ++ { 0x80, 0x618 }, { 0x88, 0x628 }, { 0x90, 0x630 }, ++ { 0x98, 0x648 }, { 0xa0, 0x650 }, { 0x138, 0x6d0 }, ++ }; ++ ++ g5p_put64(w + 0x60, e->scissor_va & ~3ULL); ++ g5p_put64(w + 0x4a0, e->scissor_va); ++ g5p_put64(w + 0x68, e->dbias_va & ~3ULL); ++ g5p_put64(w + 0x490, e->dbias_va); ++ g5p_put64(w + 0xb8, e->background_texture); ++ g5p_put64(w + 0x5f0, e->background_texture); ++ g5p_put32(w + 0x3a4, e->bgobj_depth); ++ g5p_put32(w + 0x6a8, e->bgobj_depth); ++ g5p_put32(w + 0x3a8, e->bgobj_vals); ++ g5p_put32(w + 0x6ac, e->bgobj_vals); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put64(w + zls_w[i][0], e->zls[i]); ++ g5p_put64(w + zls_w[i][1], e->zls[i]); ++ } ++ g5p_ru_fill_3d_width(w, e->pixel_width); ++} ++ ++/* TYPED_V3 additions, already validated by g5p_ru_pass3_check(). */ ++struct g5p_ru_pass3 { ++ u64 ta_program_va; ++ u32 ta_program_size, vdm_end; ++ u32 pixel_width, pad; /* V1 dialect output width, 0 = 2 */ ++}; ++ ++/* Dialect-3 word over g5p_ru_fill_ta(): W+0x88 (-> R0+0x470) is the exact ++ * byte offset of the TA PDS program from pds_base, a plain 64-bit store with ++ * no mask. g5p_ru_fill_ta() leaves it zero for a V3 pass (vdm_control is ++ * zero), so V1/V2 bytes are untouched; the stream length never reaches W. ++ */ ++static inline void g5p_ru_fill_ta_v3(u8 *w, u64 program_off) ++{ ++ g5p_put64(w + 0x88, program_off); ++} ++ ++/* 6cb2378 selected state S at control+80: explicit zeros plus the four ++ * recovered backing pointers. N=1, PM ID0/record0. aux_va covers 10680. ++ */ ++static inline int g5p_ru_pm_state(u8 *control, u64 aux_va, u64 list_va) ++{ ++ memset(control + 0x80, 0, 0xc0); ++ return g5p_pm_record_backing(control + 0x80, 0, 1, aux_va, 0x80, ++ 0x10000, 0x600, ++ g5p_meta_address(list_va + 0x40)); ++} ++ ++/* 6cb124c/13f4/14dc pool for `blocks` resident = maximum blocks. Rewrites ++ * control+0..7f, the whole list and F; S at control+80 is not touched. ++ * physical[] holds blocks*8 16KiB PAs. Validate everything before writing. ++ */ ++static inline int g5p_ru_pm_check(u64 list_va, u64 free_va, const u64 *physical, ++ u32 blocks) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (!blocks || blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ ret = g5p_pm_sizes(1, blocks, 0x80, 0x10000, 0x600, &sizes); ++ if (ret) ++ return ret; ++ if ((list_va | free_va) & (G5P_GPU_PAGE - 1) || ++ list_va >= (1ULL << 40) || free_va >= (1ULL << 40) || ++ 0x80 + blocks * 0x24 > G5P_GPU_PAGE || ++ blocks * 32 * 4 > G5P_GPU_PAGE) ++ return -EINVAL; ++ for (u32 i = 0; i < blocks * 8; i++) ++ if ((physical[i] & (G5P_GPU_PAGE - 1)) || ++ physical[i] >= (1ULL << 40)) ++ return -ERANGE; ++ return 0; ++} ++ ++static inline int g5p_ru_pm_pool(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks) ++{ ++ u32 previous = 0; ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ memset(control, 0, 0x80); ++ memset(list, 0, G5P_GPU_PAGE); ++ memset(free, 0, G5P_GPU_PAGE); ++ for (u32 j = 0; j < blocks; j++) { ++ u32 at = 0x80 + j * 0x24; ++ ++ g5p_put32(list + at, previous); ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[j * 8 + k] >> 12; ++ ++ g5p_put32(list + at + 4 + k * 4, ppn); ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * 4, ++ ppn + p); ++ } ++ previous = g5p_meta_address(list_va + at); ++ } ++ g5p_put32(list + 0x60, 1); ++ g5p_put32(control + 0x10, previous); ++ g5p_put64(control + 0x14, free_va); ++ g5p_put32(control + 0x1c, blocks * 32 * 4); ++ g5p_put32(control + 0x20, blocks * 32); ++ g5p_put32(control + 0x24, blocks * 32 - 1); ++ g5p_put32(control + 0x34, G5P_GPU_PAGE); ++ g5p_put32(control + 0x40, g5p_meta_address(list_va)); ++ g5p_put32(control + 0x54, blocks * 32); ++ g5p_put32(control + 0x58, blocks * 32); ++ return 0; ++} ++ ++/* Complete fresh PM object: g5p_r22_pm() when blocks=32, aux=G5P_R22_AUX. */ ++static inline int g5p_ru_pm(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks, ++ u64 aux_va) ++{ ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ if ((aux_va & (G5P_GPU_PAGE - 1)) || ++ aux_va > (1ULL << 40) - G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ return -EINVAL; ++ memset(control, 0, G5P_GPU_PAGE); ++ ret = g5p_ru_pm_pool(control, list, free, list_va, free_va, physical, ++ blocks); ++ return ret ?: g5p_ru_pm_state(control, aux_va, list_va); ++} ++ ++/* The r25 latched selections that can still yield a frame. Context/host/SKU ++ * probes and the heap-relative RT dialect are diagnostic-only in the fixed ++ * path (pair rejected) and are not offered here; blocks is pm_blocks. ++ */ ++static inline bool g5p_ru_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return g5p_r25_valid(v) && !g5p_r25_diagnostic(v) && !v->blocks; ++} ++ ++/* r25 render_pm_variant 2/3 on a pool from g5p_ru_pm()/g5p_ru_pm_pool(): ++ * reverse only F's block order, or report F's whole mapped page as its ++ * extent. Equal to g5p_r25_pool() for 32, 64 and 128 blocks. ++ */ ++static inline void g5p_ru_pm_variant(u8 *control, u8 *free, const u64 *physical, ++ u32 blocks, u32 variant) ++{ ++ if (variant == 2) ++ for (u32 j = 0; j < blocks; j++) ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[(blocks - 1 - j) * 8 + k] >> ++ 12; ++ ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * ++ 4, ++ ppn + p); ++ } ++ if (variant == 3) ++ g5p_put32(control + 0x1c, G5P_GPU_PAGE); ++} ++ ++/* r25 render_pm_variant 1, after g5p_ru_job() built the TA page: tag6+10 ++ * names an owned holder with the linked head (outside CH/ring/W/SKU/TQ/ ++ * dependency); F/count/head start clear and last=ffffffff, so 10005368 ++ * rebuilds them. Equal to g5p_r25_inputs() for the same pool. ++ */ ++ ++/* Persistent per-channel objects (6c89420 layout): written once at attach. ++ * Control indices, TQ and the dependency object stay firmware-owned after. ++ */ ++static inline void g5p_ru_channel_init(u8 *p, u32 meta) ++{ ++ memset(p, 0, G5P_GPU_PAGE); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_R22_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_R22_TQ + 4, meta + G5P_R22_TQ); ++ g5p_put64(p + G5P_R22_DEP + 0x30, 1); ++ g5p_put64(p + G5P_R22_DEP + 0x48, ~0ULL); ++ g5p_put32(p + G5P_R22_DEP + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(p + G5P_R22_DEP + off + 8, 0xf); ++} ++ ++/* Pointer-ring positions of job `seq` (1-based): two entries per job. */ ++static inline u32 g5p_ru_ring_index(u32 seq) ++{ ++ return (2 * (seq - 1)) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_ru_endpoint(u32 seq) ++{ ++ return (2 * seq) % G5P_WORK_SLOTS; ++} ++ ++/* perf60 job slots (render_user_job_slots, drm-render-v3.inc). A job's command, W record and ++ * microsequence ("SKU") live in slot 0 (the r22 offsets) or slot 1, in the channel page's free ++ * tail; the firmware reaches both through the job's two pointer-ring entries. The channel header, ++ * control, pointer ring, TQ, stamps and dependency object stay shared. Slot 1 clears TQ/STAMP/DEP ++ * (0x1800..0x1b70) and the r25 holder (G5P_R25_HEAD, 0x3000). ++ */ ++#define G5P_RU_SLOT1 0x1c00U ++#define G5P_RU_SLOT1_END \ ++ (G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES + 0x180U) ++ ++static inline u32 g5p_ru_cmd_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 : G5P_WORK_CMD; ++} ++ ++static inline u32 g5p_ru_w_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 + G5P_RU_CMD_BYTES : G5P_R22_W; ++} ++ ++static inline u32 g5p_ru_sku_at(u32 js, bool td) ++{ ++ if (js) ++ return G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES; ++ return td ? G5P_R22_3D_SKU : G5P_R22_TA_SKU; ++} ++ ++/* Per-job objects for sequence `seq`: W image (fixed+typed+RT), channel W ++ * fields, the full timestamped microsequence, the tag6 PM bootstrap (TA) or ++ * tag4 TA-finish dependency (3D) and both pointer-ring entries. With seq=1 ++ * this equals g5p_r22_channel(). Stamp target and sequence are both seq. ++ * ta_finish is M(TA stamp word addressed by TA.W530), used by 3D only. ++ */ ++static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, ++ bool td, u32 seq, const u8 *image, ++ u32 ta_finish, bool zero_timestamps, u32 js, ++ u32 pmid) ++{ ++ u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; ++ u32 finish = meta + G5P_R22_STAMP + 4; ++ u32 sku = g5p_ru_sku_at(js, td), cmd = g5p_ru_cmd_at(js); ++ u32 ring = g5p_ru_ring_index(seq); ++ u8 *work = p + g5p_ru_w_at(js); ++ u32 cursor = sku, starts, finishes; ++ ++ memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); ++ memset(p + sku, 0, td ? 0x180 : 0xc0); ++ g5p_put32(work, td ? 1 : 0); ++ g5p_put32(work + 4, slot); ++ g5p_put32(work + (td ? 0x14 : 0xc), meta + G5P_R22_DEP); ++ g5p_put32(work + (td ? 0x18 : 0x14), pm); ++ if (!td && pmid) ++ g5p_put32(work + 0x10, ++ pmid); /* TA W+10: the PM identity (iOS Dcb0) */ ++ g5p_put32(work + (td ? 0x1c : 0x18), state); ++ g5p_put32(work + (td ? 0xc : 0x454), meta + sku); ++ g5p_put32(work + (td ? 0x10 : 0x458), td ? 0x180 : 0xc0); ++ g5p_put32(work + (td ? 0x888 : 0x52c), finish - 4); ++ g5p_put32(work + (td ? 0x88c : 0x530), finish); ++ g5p_put32(work + (td ? 0x890 : 0x534), seq); ++ g5p_put32(work + (td ? 0x894 : 0x538), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x898 : 0x53c), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x8a0 : 0x544), seq); ++ { ++ u32 ta_start[] = { ++ 0x23, ++ w + 0x3a4, ++ w + 0x20, ++ state, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ pmid, ++ 0, ++ 0, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ 0, ++ seq, ++ }; ++ u32 td_start[] = { ++ 0x25, ++ w + 0x3a0, ++ w + 0x58, ++ state, ++ global + 0x724, ++ w + 0x7f0, ++ w + 0x7f4, ++ w + 0x878, ++ meta, ++ w, ++ slot, ++ 0, ++ pmid, ++ g5p_le32(work + 0x800), ++ 0, ++ 1, ++ w + 0x6d8, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ 0, ++ seq, ++ }; ++ u32 ta_finish[] = { ++ 0x24, ++ state, ++ pm, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ seq, ++ finish, ++ seq, ++ (u32)-0x70, ++ }; ++ u32 td_finish[] = { ++ 0x26, ++ seq, ++ finish, ++ seq, ++ state, ++ pm, ++ 1, ++ global + 0x724, ++ w + 0x878, ++ w + 0x7f0, ++ meta, ++ w, ++ slot, ++ w + 0x6d8, ++ 0, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ (u32)-0x104, ++ }; ++ u32 result = w + (td ? 0x8ac : 0x550); ++ u32 timestamp = w + (td ? 0x8b4 : 0x558); ++ u32 ts0[] = { 0x80000018, result, timestamp, timestamp, seq }; ++ u32 ts1[] = { 0x18, result, timestamp, timestamp + 4, seq }; ++ ++ starts = td ? ARRAY_SIZE(td_start) : ARRAY_SIZE(ta_start); ++ finishes = td ? ARRAY_SIZE(td_finish) : ARRAY_SIZE(ta_finish); ++ for (u32 i = 0; i < starts; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_start[i] : ta_start[i]); ++ for (u32 i = 0; i < ARRAY_SIZE(ts0); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts0[i]); ++ if (td) { ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r22_wait_words); ++ i++, cursor += 4) ++ g5p_put32(p + cursor, g5p_r22_wait_words[i]); ++ } else { ++ g5p_put32(p + cursor, 0x801); ++ cursor += 4; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(ts1); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts1[i]); ++ for (u32 i = 0; i < finishes; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_finish[i] : ta_finish[i]); ++ g5p_put32(p + cursor, 0x40000017); ++ } ++ memset(p + cmd, 0, G5P_RU_CMD_BYTES); ++ g5p_put32(p + cmd, td ? 4 : 6); ++ g5p_put32(p + cmd + 4, td ? ta_finish : slot); ++ g5p_put32(p + cmd + 8, ++ td ? seq : pmid); /* tag6 {6, STA, PMid, M(control)} */ ++ g5p_put32(p + cmd + 12, td ? seq & 0x7fffffff : pm); ++ g5p_put32(p + G5P_WORK_RING + ring * 4, meta + cmd); ++ g5p_put32(p + G5P_WORK_RING + ((ring + 1) % G5P_WORK_SLOTS) * 4, w); ++} ++ ++/* Retirement of job `seq`: both W pointers still ours, finish word and ++ * retirement mirror at seq, all five channel watermarks at its endpoint. ++ * g5p_r22_retired() is the seq=1 case. ++ */ ++static inline bool g5p_ru_retired_at(const u8 *p, u32 meta, bool td, u32 seq, ++ u32 js) ++{ ++ const u8 *w = p + g5p_ru_w_at(js); ++ u32 endpoint = g5p_ru_endpoint(seq); ++ ++ if (g5p_le32(w + (td ? 0x888 : 0x52c)) != meta + G5P_R22_STAMP || ++ g5p_le32(w + (td ? 0x88c : 0x530)) != meta + G5P_R22_STAMP + 4 || ++ g5p_le32(w + (td ? 0x890 : 0x534)) != seq || ++ g5p_le32(p + G5P_R22_STAMP) != seq || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* A stamp beyond the published target, or a watermark not on the even ++ * two-entry grid, is a protocol error rather than "not yet retired". ++ */ ++static inline bool g5p_ru_stamp_sane(const u8 *p, u32 seq) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ s32 ahead = (s32)(g5p_le32(p + G5P_R22_STAMP + i * 4) - seq); ++ ++ if (ahead > 0) ++ return false; ++ } ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) >= G5P_WORK_SLOTS) ++ return false; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-loss.h b/drivers/gpu/apple-g5p/render-user-loss.h +new file mode 100644 +index 0000000000..2aad49d884 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-loss.h +@@ -0,0 +1,120 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context fault isolation (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault ++ * isolation). Pure decisions, host-tested (scripts/gpu/multiclient2-host/test_render_loss.c). ++ * ++ * The hardware gives no way to abandon a job: the firmware has no abort (iOS ends a ++ * non-progressing engine with a firmware stop, a platform power cycle and a new firmware boot, ++ * kext 6c9e070, firmware-re README "Fault recovery"), the TA and 3D channels are FIFO, and the ++ * one TA and one 3D engine serve every context. So one context's job can be *lost* (its client ++ * gets an error, its context is dead) without ending the device only while the firmware is ++ * alive, and only if the stuck phase still finishes: until then no other job may be published ++ * behind it (it would wait behind the same engine and be lost in turn). ++ * ++ * loss at the end of a phase that did not complete: ++ * NONE it completed; ++ * CONTEXT a phase timeout on a job other than the epoch's first, with the firmware ++ * alive (its event stream never went silent, no fault event, no protocol ++ * error) and isolation on: the job and its context are lost, the device is ++ * not; the channel is blocked behind the stuck phase; ++ * DEVICE anything else: the epoch is terminal, as before. ++ * drain while blocked, the scheduler runs no job and waits, RAM only, for the stuck phase: ++ * a late TA gets its 3D published (the channel and PM state then stay consistent), ++ * a late 3D ends it; the channel is then clear and every held job runs. A silent ++ * event stream, a fault, a refused 3D or the drain bound make the epoch terminal. ++ */ ++#ifndef G5P_RENDER_USER_LOSS_H ++#define G5P_RENDER_USER_LOSS_H ++ ++enum g5p_ru_loss { ++ G5P_RU_LOSS_NONE, ++ G5P_RU_LOSS_CONTEXT, ++ G5P_RU_LOSS_DEVICE, ++}; ++ ++struct g5p_ru_loss_in { ++ u32 result; /* G5P_RU_WAIT_* of the phase */ ++ bool first; /* the epoch's first job (its attach and bell) */ ++ bool isolate; /* render_user_drain_ms != 0 */ ++ bool silent; /* the event stream stopped: the firmware is dead */ ++ bool fault; /* a fault event, a failed boot, a protocol or drain error */ ++}; ++ ++static inline int g5p_ru_loss(const struct g5p_ru_loss_in *in) ++{ ++ if (in->result == G5P_RU_WAIT_COMPLETE) ++ return G5P_RU_LOSS_NONE; ++ if (in->result != G5P_RU_WAIT_TIMEOUT || in->first || !in->isolate || ++ in->silent || in->fault) ++ return G5P_RU_LOSS_DEVICE; ++ return G5P_RU_LOSS_CONTEXT; ++} ++ ++/* The drain of a blocked channel. */ ++enum g5p_ru_drain_state { ++ G5P_RU_DRAIN_TA, /* waiting for the stuck TA */ ++ G5P_RU_DRAIN_TD, /* waiting for the stuck job's 3D */ ++ G5P_RU_DRAIN_CLEAR, /* done: the channel is clear */ ++ G5P_RU_DRAIN_DEAD, /* the epoch is terminal */ ++}; ++ ++/* What the drain does next. */ ++enum g5p_ru_drain_action { ++ G5P_RU_DRAIN_WAIT, /* sample again */ ++ G5P_RU_DRAIN_PUBLISH_TD, /* publish the stuck job's 3D, wait for it */ ++ G5P_RU_DRAIN_UNBLOCK, /* release the held jobs */ ++ G5P_RU_DRAIN_TERMINAL, /* note the epoch terminal, release the held jobs (they fail) */ ++}; ++ ++/* One step from the result of the phase wait (G5P_RU_WAIT_*) in state *st. A refused 3D ++ * publish is reported by the caller as G5P_RU_WAIT_FAULT on the TD state. ++ */ ++static inline int g5p_ru_drain_step(u32 *st, u32 result) ++{ ++ if (*st >= G5P_RU_DRAIN_CLEAR) ++ return *st == G5P_RU_DRAIN_CLEAR ? G5P_RU_DRAIN_UNBLOCK : ++ G5P_RU_DRAIN_TERMINAL; ++ if (result == G5P_RU_WAIT_PENDING) ++ return G5P_RU_DRAIN_WAIT; ++ if (result != G5P_RU_WAIT_COMPLETE) { ++ *st = G5P_RU_DRAIN_DEAD; ++ return G5P_RU_DRAIN_TERMINAL; ++ } ++ if (*st == G5P_RU_DRAIN_TA) { ++ *st = G5P_RU_DRAIN_TD; ++ return G5P_RU_DRAIN_PUBLISH_TD; ++ } ++ *st = G5P_RU_DRAIN_CLEAR; ++ return G5P_RU_DRAIN_UNBLOCK; ++} ++ ++/* ++ * The drain bound must leave the held clients' own wait intact. A client waits for its job up to ++ * the job_timeout_ms GET_RENDER_INFO reports plus a margin (the ICD: + 3 s, so 2 * hard + obs + 3 s ++ * = 125 s with latency101 on, 33 s with it off), and a held job has already waited out the stuck ++ * phase (up to the hard bound, or render_user_wait_ms) before the drain starts, then runs after ++ * it. Valid: 0 (isolation off), or from 1000 up to hard + obs - 10000 with latency101 on (hard ++ * 60000, obs 2000: 52000), and up to 8000 with it off. An invalid value at the time of a loss ++ * turns isolation off for it. ++ */ ++static inline bool g5p_ru_drain_valid(u32 drain_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!drain_ms) ++ return true; ++ if (drain_ms < 1000) ++ return false; ++ if (!hard_ms) ++ return drain_ms <= 8000; ++ return hard_ms + obs_ms >= 11000 && ++ drain_ms <= hard_ms + obs_ms - 10000; ++} ++ ++ ++ ++static inline const char *g5p_ru_loss_name(int loss) ++{ ++ return loss == G5P_RU_LOSS_NONE ? "none" : ++ loss == G5P_RU_LOSS_CONTEXT ? "context" : "device"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-mmu.h b/drivers/gpu/apple-g5p/render-user-mmu.h +new file mode 100644 +index 0000000000..5a05dccbf7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-mmu.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 MMU invalidation arm (render_user_mmu_inval, drm-render-v3.inc), pure helpers. ++ * ++ * The only invalidation of a user MMU context besides DC3 is the firmware's per-kick tag compare ++ * (10017ac4): diff = host_tag[S] ^ cached_tag[S]; bits 31:22 set -> e138 = (S << 3) | 4 (PC), ++ * else bits 21:12 -> | 2 (PD), else bits 11:0 -> | 1 (PT); then cached = host. One invalidate per ++ * kick, the highest changed field (docs/gpu/firmware-re/README.md "generation producer"). A PC ++ * field change therefore asks for the firmware's PC-level invalidate of that context, the level ++ * Apple's own producer (6cabe70) and DC3 (1000218c) use as the strongest one. ++ * ++ * g5p_ru_mmu_pc_tag(): a host tag whose PC field differs from both slots' cached PC fields, so ++ * the next kick of each slot takes the PC branch. The PD and PT fields are the given tag's. ++ * Never 0 (0 is the unbound tag) and never equal to either cached tag. Host-tested ++ * (scripts/gpu/perf60-host/test_mmu_inval.c). ++ */ ++#ifndef G5P_RENDER_USER_MMU_H ++#define G5P_RENDER_USER_MMU_H ++ ++#define G5P_RU_MMU_PC_MASK 0xffc00000U ++#define G5P_RU_MMU_PC_ONE 0x00400000U ++ ++/* render_user_mmu_inval bits */ ++#define G5P_RU_MMU_EDIT \ ++ 0x1U /* every VM edit publication (and spare adoption) asks for PC */ ++#define G5P_RU_MMU_SWITCH \ ++ 0x2U /* a bound context's job after another context's job: PC retag */ ++#define G5P_RU_MMU_TOUCH \ ++ 0x4U /* NC read of every table line before the tag store */ ++#define G5P_RU_MMU_EVERY \ ++ 0x8U /* every job of a bound context: PC retag (superset of SWITCH) */ ++ ++static inline u32 g5p_ru_mmu_pc_tag(u32 tag, u32 cached0, u32 cached1) ++{ ++ u32 pc = (tag & G5P_RU_MMU_PC_MASK) + G5P_RU_MMU_PC_ONE; ++ ++ /* At most two values are excluded, so three candidates always contain a free one. */ ++ for (u32 i = 0; i < 3; i++, pc += G5P_RU_MMU_PC_ONE) { ++ u32 t = (pc & G5P_RU_MMU_PC_MASK) | (tag & ~G5P_RU_MMU_PC_MASK); ++ ++ if (((t ^ cached0) & G5P_RU_MMU_PC_MASK) && ++ ((t ^ cached1) & G5P_RU_MMU_PC_MASK) && t) ++ return t; ++ } ++ return 0; /* unreachable: kept for the host test */ ++} ++ ++/* ++ * Retag on a job: the published host tag, or the cached one if none, moved to a fresh PC field. ++ */ ++static inline bool g5p_ru_mmu_retag_due(u32 mode, bool bound_before, ++ bool switched) ++{ ++ if (!bound_before) ++ return false; /* this job's DC3 or adoption publishes its own tag */ ++ return (mode & G5P_RU_MMU_EVERY) || ++ ((mode & G5P_RU_MMU_SWITCH) && switched); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-msaa.h b/drivers/gpu/apple-g5p/render-user-msaa.h +new file mode 100644 +index 0000000000..f27f197a6c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-msaa.h +@@ -0,0 +1,88 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_MSAA_H ++#define G5P_RENDER_USER_MSAA_H ++ ++#include "bits-compat.h" ++/* TYPED_V3 4x multisampling: the per-job words a pass with a 4-sample render ++ * target needs on top of the V1/V2 3D words and the target's geometry ++ * (g5p_ru_rt_derive_samples()). Pure byte builders over the kernel's own ++ * work records: no allocation, MMIO, locking or firmware publication. ++ * ++ * Evidence (docs/gpu/experiments/msaa.md): the iOS gles3/gles3b captures ++ * 20261101T175626Z, msaa4-resolve, msaa4-store, g3b-a2c and g3b-samplemask, ++ * against g3-base, and the importer's 3D/TA fillers 6ca7ae0/6ca9710 executed ++ * with the 4-sample descriptor (build/gpu/msaa/filler-msaa.c): ++ * record+0x4d8 0xf -> 0x7 D6a8 -> 3D W3c0[3:0] (ISP_CTL; the draws' ++ * ISPCTL.upass drops 0xf -> 0x7 with it) ++ * record+0x640 0 -> 1 D6ad -> 3D W3c0 bit 17 and W86c (resolve only; ++ * bit 17 is ISP_CTL.PROCESS_EMPTY_TILES, which ++ * Linux's template sets for every pass since ++ * 0e05f993cf, so only W86c is new) ++ * RT Rb8 0 -> 2 D6e0 -> 3D Wd0 (AA mode, CR f30), and the ++ * pixel-output shift 10 -> 11 in W3c0 [21], ++ * W3c4, W3c8 and the six W108 words ++ * RT Rb0 / Rf8 3D W58 (CR fa8) / TA W40 (PPP_MULTISAMPLECTL) ++ * The rest of the target's geometry reaches W through g5p_ru_rt_fill(). ++ */ ++#define G5P_RU_MSAA_RESOLVE 1U /* DRM_G5P_RENDER_MSAA_RESOLVE */ ++ ++struct g5p_ru_msaa { ++ u32 samples, flags; /* 0 = 1 */ ++ u64 positions; /* 0 = the target's */ ++}; ++ ++/* samples 0 or 1 is the one-sample pass and must use a one-sample target; ++ * 4 must use a four-sample target. Positions are one byte per sample, so a ++ * word with bits above the sample count is malformed; zero takes the ++ * target's (the standard pattern). Resolve exists only with four samples. ++ * Nothing here is a hardware hazard: positions are a 4-bit grid and the ++ * resolve bit changes what the PBE writes, not where. ++ */ ++static inline int g5p_ru_msaa_check(const struct g5p_ru_msaa *m, u32 rt_samples) ++{ ++ u32 s = m->samples ? m->samples : 1; ++ ++ if ((s != 1 && s != 4) || s != rt_samples || ++ m->flags & ~G5P_RU_MSAA_RESOLVE || (m->flags && s == 1) || ++ m->positions >> (s * 8)) ++ return -EINVAL; ++ return 0; ++} ++ ++/* After g5p_ru_fill_3d(), the dialect's width words and g5p_ru_rt_fill(). ++ * A one-sample pass is left as it is, unless it names its position. ++ */ ++static inline void g5p_ru_fill_msaa(u8 *ta, u8 *td, const struct g5p_ru_msaa *m, ++ const struct g5p_ru_rt *rt) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by width - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u64 positions = m->positions ? m->positions : rt->packed; ++ bool resolve = m->flags & G5P_RU_MSAA_RESOLVE; ++ u32 width, count, mode, w3c0; ++ ++ if (m->samples != 4 && !m->positions) ++ return; ++ g5p_put64(ta + 0x40, positions); ++ g5p_put64(td + 0x58, positions); ++ if (m->samples != 4) ++ return; ++ /* The width the dialect wrote (2, 4 or 8); 6ca7ae0 shifts by 11 with an ++ * AA mode, so each width has half the output partitions. ++ */ ++ width = g5p_le32(td + 0x6b0); ++ count = 0x4000 / (width << 11); ++ mode = count > 1 ? 2 : count; ++ w3c0 = g5p_le32(td + 0x3c0) & ~(0xfU | BIT(21)); ++ g5p_put32(td + 0x3c0, ++ w3c0 | 0x7 | (resolve ? BIT(17) : 0) | (mode - 1) << 21); ++ g5p_put32(td + 0x3c4, mode * 0x204); ++ g5p_put32(td + 0x3c8, count & 0x1f); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(td + 0x108 + i * 8, ++ width_code[width - 2] | (count - 1) << 7 | 0x43); ++ g5p_put64(td + 0xd0, rt->state); ++ g5p_put32(td + 0x86c, resolve); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-pmset.h b/drivers/gpu/apple-g5p/render-user-pmset.h +new file mode 100644 +index 0000000000..170738f1f8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-pmset.h +@@ -0,0 +1,77 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context PM records (perf60 stage 4, docs/gpu/multi-client-design.md risk R3). ++ * ++ * iOS acquires a ParamBuffer ID per prepare from a 127-entry pool (6cb9afc) and builds a PM ++ * control/state/aux object and its block list per PM (6cb2378); one PM is never shared by two ++ * work queues. Linux built one device PM (control+S at shared+G5P_R22_PM, list and F in the ++ * two render_meta pages, aux at G5P_RU_PRIVATE_VA) for every render context; with the GPU ++ * never parked, one context's job then started from the PM state another context's job left. ++ * ++ * boot_pm_sets reserves N sets in the firmware GiB at boot, right after render_meta: set s is ++ * three pages {control + S, list, F} at G5P_PMSET_VA + s * 3 pages, mapped like the shared ++ * pages (they are firmware objects the firmware and the PM requesters address by meta VA). ++ * A render context takes one set, its own pool pages and its own aux pages. PM ID: 0 stays ++ * the device PM's; set s may carry s + 1. ++ * ++ * Pure helpers, host-tested (scripts/gpu/perf60-host/test_pmset.c). ++ */ ++#ifndef G5P_RENDER_USER_PMSET_H ++#define G5P_RENDER_USER_PMSET_H ++ ++#define G5P_PMSET_MAX 8U ++#define G5P_PMSET_PAGES 3U /* control + S, list, F */ ++#define G5P_PMSET_VA (G5P_SHARED_VA + G5P_SHARED_SIZE + 2 * G5P_GPU_PAGE) ++#define G5P_PMSET_END \ ++ (G5P_PMSET_VA + G5P_PMSET_MAX * G5P_PMSET_PAGES * G5P_GPU_PAGE) ++ ++enum { G5P_PMSET_CONTROL, G5P_PMSET_LIST, G5P_PMSET_FREE }; ++ ++static inline u64 g5p_pmset_va(u32 set, u32 part) ++{ ++ return G5P_PMSET_VA + ++ ((u64)set * G5P_PMSET_PAGES + part) * G5P_GPU_PAGE; ++} ++ ++static inline u32 g5p_pmset_id(u32 set) ++{ ++ return set + 1; ++} ++ ++/* ++ * The PM's write-back target (release8 bench-gaps3, 2026-11-03). After the GPU switches from ++ * context A's jobs to context B's, the firmware's PM keeps writing A's free list F back, through ++ * A's TA slot (its MMU context), on B's later jobs, until a job of a third context runs: two ++ * faults, one on the 2nd and one on the 3rd job of B after A's last, both the first job after A ++ * was unbound (page-invalid level 2 in A's MMU context, A's set F page; with A still bound the ++ * same writes landed). So a context's slots and set stay bound while it is the context of the ++ * last job or the previous *distinct* one: g5p_pmset_track() at every job end, g5p_pmset_holds() ++ * at close. That bounds when the slots may go, not when the PM stops: two later faults (release8 ++ * quiet2, seq 858 and 893) wrote the free list of a context two switches back, after this rule had ++ * released it. So a released pair is parked rather than left empty (render_user_close_park, ++ * drm-render-v3.inc), and a released set is reused last (g5p_pmset_pick()). ++ */ ++static inline bool g5p_pmset_switched(const void *last, const void *job) ++{ ++ return last != job; ++} ++ ++static inline bool g5p_pmset_holds(const void *last, const void *prev, ++ const void *ctx) ++{ ++ return ctx && (ctx == last || ctx == prev); ++} ++ ++/* A free set, round-robin from `next` (a released set is reused last). -1: none free. */ ++static inline int g5p_pmset_pick(const bool *used, u32 n, u32 next) ++{ ++ for (u32 i = 0; i < n; i++) { ++ u32 s = (next + i) % n; ++ ++ if (!used[s]) ++ return s; ++ } ++ return -1; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-quiesce.h b/drivers/gpu/apple-g5p/render-user-quiesce.h +new file mode 100644 +index 0000000000..3671e415e2 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-quiesce.h +@@ -0,0 +1,145 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU quiesce before a kexec, reboot or power-off (docs/gpu/experiments/mcfold2-kexec.md). ++ * ++ * Without it the firmware keeps running from the outgoing kernel's RAM across the relocation: ++ * it still writes its event rings, and the MMU still holds a CBASE for every bound render slot ++ * pointing at page tables the next kernel reuses, until the next kernel's PMGR GFX cycle about ++ * 0.4 s into its boot. The quiesce ends that window on the outgoing side, in this order: ++ * ++ * 1 close job admission and wait (bounded) for submitters inside; ++ * 2 take the command lock (bounded trylock), so no job or command is in flight; ++ * 3 DETACH: every user slot the firmware may hold is unbound by a RAM-only DC3 batch ++ * (render-user-bind.h), only when the lock is ours, the firmware is healthy, no timeout ++ * or fault was noted and the DC ring is idle; ++ * 4 freeze the boot state and mask the GPU IRQ (no host register access after this); ++ * 5 CYCLE: one bounded PMGR GFX off/on, the same gate cycle the next kernel performs with ++ * apple_pmgr.kexec_gpu_cycle=1, only when no host thread can be inside a GPU register ++ * access: the lock is ours, or its holder is the TYPED_V3 executor in its RAM-only regime. ++ * ++ * Pure functions over the ops below, host-tested (scripts/gpu/mcfold2-host/test_kexec_quiesce.c). ++ */ ++#ifndef G5P_RENDER_USER_QUIESCE_H ++#define G5P_RENDER_USER_QUIESCE_H ++ ++/* kexec_quiesce module parameter */ ++#define G5P_RU_KQ_OFF 0U ++#define G5P_RU_KQ_RAM 1U /* detach only, no PMGR access */ ++#define G5P_RU_KQ_FULL 2U /* detach and cycle (default) */ ++ ++/* Plan */ ++#define G5P_RU_KQ_DETACH 0x1U ++#define G5P_RU_KQ_CYCLE 0x2U ++ ++/* Why a step was skipped */ ++#define G5P_RU_KQ_WHY_OFF 0x001U /* kexec_quiesce=0 */ ++#define G5P_RU_KQ_WHY_DOWN 0x002U /* the firmware never ran on this boot */ ++#define G5P_RU_KQ_WHY_BUSY 0x004U /* the command lock is not ours: no detach */ ++#define G5P_RU_KQ_WHY_FAILED \ ++ 0x008U /* fault or protocol failure: firmware state unknown */ ++#define G5P_RU_KQ_WHY_WEDGED \ ++ 0x010U /* a TYPED_V3 timeout or fault: a job may hold its slots */ ++#define G5P_RU_KQ_WHY_RING 0x020U /* the DC ring is not idle */ ++#define G5P_RU_KQ_WHY_NONE 0x040U /* no user slot is held */ ++#define G5P_RU_KQ_WHY_HOLDER \ ++ 0x080U /* unknown lock holder, maybe inside MMIO: no cycle */ ++#define G5P_RU_KQ_WHY_NO_PMGR 0x100U ++#define G5P_RU_KQ_WHY_RAM 0x200U /* kexec_quiesce=1: no cycle */ ++#define G5P_RU_KQ_WHY_NO_DRM 0x400U /* no render frontend: nobody can detach */ ++ ++struct g5p_ru_kq_in { ++ u32 mode; /* kexec_quiesce */ ++ bool booted; /* the firmware was released on this boot */ ++ bool locked; /* the command lock is ours */ ++ bool failed; /* b->failed or b->event_failed */ ++ bool frontend; /* a render frontend that can detach is registered */ ++ bool term_failed; /* the TYPED_V3 latch noted a timeout or fault */ ++ bool ram_only; /* the TYPED_V3 latch is quiet: its executor touches no register */ ++ bool executing; /* the TYPED_V3 executor is inside a job */ ++ bool ring_idle; /* DC consumer == producer <= 255 */ ++ bool pmgr; /* the PMGR regmap is present */ ++ u32 held; /* user slots with a PA or a host tag */ ++ u32 cons, prod; /* the DC ring when sampled */ ++}; ++ ++static inline u32 g5p_ru_kq_plan(const struct g5p_ru_kq_in *in, u32 *why) ++{ ++ u32 plan = 0, w = 0; ++ ++ if (in->mode == G5P_RU_KQ_OFF || !in->booted) { ++ *why = in->mode == G5P_RU_KQ_OFF ? G5P_RU_KQ_WHY_OFF : ++ G5P_RU_KQ_WHY_DOWN; ++ return 0; ++ } ++ if (!in->locked) ++ w |= G5P_RU_KQ_WHY_BUSY; ++ if (in->failed) ++ w |= G5P_RU_KQ_WHY_FAILED; ++ if (in->term_failed) ++ w |= G5P_RU_KQ_WHY_WEDGED; ++ if (!in->ring_idle) ++ w |= G5P_RU_KQ_WHY_RING; ++ if (!in->held) ++ w |= G5P_RU_KQ_WHY_NONE; ++ if (!in->frontend) ++ w |= G5P_RU_KQ_WHY_NO_DRM; ++ if (!w) ++ plan |= G5P_RU_KQ_DETACH; ++ if (in->mode < G5P_RU_KQ_FULL) ++ w |= G5P_RU_KQ_WHY_RAM; ++ else if (!in->pmgr) ++ w |= G5P_RU_KQ_WHY_NO_PMGR; ++ else if (!in->locked && !(in->executing && in->ram_only)) ++ w |= G5P_RU_KQ_WHY_HOLDER; ++ else ++ plan |= G5P_RU_KQ_CYCLE; ++ *why = w; ++ return plan; ++} ++ ++struct g5p_ru_kq_ops { ++ void *ctx; ++ u32 (*admit_close)(void *ctx); /* close admission; submitters still inside */ ++ bool (*lock)(void *ctx); /* bounded trylock of the command lock */ ++ void (*unlock)(void *ctx); ++ void (*inputs)(void *ctx, struct g5p_ru_kq_in *in); /* RAM only */ ++ int (*detach)(void *ctx); /* RAM only */ ++ void (*freeze)(void *ctx); /* RAM only */ ++ void (*irq_off)(void *ctx); /* AIC ack, mask, wait for the handler */ ++ int (*cycle)(void *ctx); /* PMGR only */ ++}; ++ ++struct g5p_ru_kq_out { ++ struct g5p_ru_kq_in in; ++ u32 inside, plan, why; ++ int detach_ret, cycle_ret; ++ bool froze; ++}; ++ ++/* One quiesce. detach_ret/cycle_ret stay 0 for a step that did not run (see plan). */ ++static inline void g5p_ru_kq_run(const struct g5p_ru_kq_ops *o, u32 mode, ++ struct g5p_ru_kq_out *out) ++{ ++ *out = (struct g5p_ru_kq_out){ .in.mode = mode }; ++ if (mode == G5P_RU_KQ_OFF) { ++ out->why = G5P_RU_KQ_WHY_OFF; ++ return; ++ } ++ out->inside = o->admit_close(o->ctx); ++ out->in.locked = o->lock(o->ctx); ++ o->inputs(o->ctx, &out->in); ++ out->plan = g5p_ru_kq_plan(&out->in, &out->why); ++ if (out->plan & G5P_RU_KQ_DETACH) ++ out->detach_ret = o->detach(o->ctx); ++ if (out->in.booted) { ++ o->freeze(o->ctx); ++ o->irq_off(o->ctx); ++ out->froze = true; ++ } ++ if (out->plan & G5P_RU_KQ_CYCLE) ++ out->cycle_ret = o->cycle(o->ctx); ++ if (out->in.locked) ++ o->unlock(o->ctx); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-recover.h b/drivers/gpu/apple-g5p/render-user-recover.h +new file mode 100644 +index 0000000000..51982c5c5f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-recover.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * In-boot GPU recovery: the firmware's recovery packet and the recovery decision. Pure, ++ * host-tested (scripts/gpu/perf60-host/test_recover.c). ++ * ++ * The recovery packet is firmware event type 4 (44 bytes). The AGX kext's event drain ++ * (AGXFirmware_drainFirmwareEventRing 6c99788, case 4) reads +0x0c (the faulting slot, 0x80 for ++ * -1), +0x10 (a flag, accel+0x1af8 bit 3), +0x18, +0x20 and +0x24 (accel+0x1a1c/0x1af4/0x1af0), ++ * marks a recovery pending (accel+0x1a24 = 2) and calls IOGPUScheduler_signalHardwareError(sched, ++ * 1) (6c67b4c): "GPURestartSignaled", the scheduler enqueues one GPU restart (a second signal ++ * before it runs is "Redundant hardware error"). The restart is ++ * AGXMetaFirmware_stopForRecoveryOptionalPowerCycle (6ca1e40) and ++ * AGXMetaFirmware_rebootAfterRecovery (6ca1efc), which panics only if the firmware does not boot ++ * again. There is no per-boot limit: every recovery packet restarts the GPU. The ++ * kext does not read +0x04. ++ * ++ * On A9/Linux the slot matches the phase the host waited in: 0 on every TA fault, 1 on the 3D fault ++ * of cf7 (2026-11-02); +0x20 = 8 and +0x24 = 0x81818105 (TA) / 0x81810581 (3D) so far. ++ */ ++#ifndef G5P_RENDER_USER_RECOVER_H ++#define G5P_RENDER_USER_RECOVER_H ++ ++#define G5P_FW_EVENT_RECOVERY 4 ++#define G5P_FW_RECOVERY_NOSLOT 0x80 ++ ++struct g5p_fw_recovery { ++ u32 code; /* +0x04, not read by the kext */ ++ u32 slot; /* +0x0c, G5P_FW_RECOVERY_NOSLOT for -1 */ ++ u32 flag; /* +0x10 != 0 */ ++ u32 w18, w20, w24; ++}; ++ ++/* `w` is the 11-word event entry, type 4. */ ++static inline struct g5p_fw_recovery g5p_fw_recovery_decode(const u32 *w) ++{ ++ struct g5p_fw_recovery r = { ++ .code = w[1], ++ .slot = w[3] == 0xffffffffU ? G5P_FW_RECOVERY_NOSLOT : w[3], ++ .flag = w[4] != 0, ++ .w18 = w[6], ++ .w20 = w[8], ++ .w24 = w[9], ++ }; ++ ++ return r; ++} ++ ++/* ++ * May a job that stopped the firmware (a recovery packet, a silent event stream, a hard timeout) ++ * be recovered from in this boot, the job's context lost and every other kept? ++ * ++ * budget == 0 never (render_user_recover 0: the epoch ends, as before); ++ * retired/closed never (the boot is shutting the GPU down); ++ * streak_max == 0 the old rule: `budget` recoveries per boot, never on the epoch's first job; ++ * streak_max != 0 Apple's rule, every time, with two guards: at most streak_max recoveries in a ++ * row with no job completing between them (a GPU that faults again before it ++ * finishes anything is not recovering), and at most G5P_RU_RECOVER_BOOT_MAX per ++ * boot (recover.inc leaves each old epoch allocated, ~0.8 MiB: 50 x 16 KiB pages ++ * and its struct, so a stale pointer still reads valid memory; this bounds that ++ * to ~26 MiB). The boot's first epoch's first job stays terminal (a failed ++ * bring-up, not a client); the first job of a later epoch (the attach after a ++ * recovery) is recoverable like any other. ++ */ ++#define G5P_RU_RECOVER_BOOT_MAX 32 ++ ++struct g5p_ru_recover_in { ++ bool first; /* the job runs its epoch's attach */ ++ bool retired; /* the GPU was retired (kexec / reboot quiesce) */ ++ bool closed; /* admission closed */ ++ u32 recoveries; /* recoveries this boot */ ++ u32 streak; /* recoveries since the last job completed */ ++ u32 budget; /* render_user_recover */ ++ u32 streak_max; /* render_user_recover_streak */ ++}; ++ ++static inline bool g5p_ru_recoverable(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget || in->retired || in->closed) ++ return false; ++ if (!in->streak_max) ++ return !in->first && in->recoveries < in->budget; ++ if (in->first && !in->recoveries) ++ return false; ++ return in->streak < in->streak_max && ++ in->recoveries < G5P_RU_RECOVER_BOOT_MAX; ++} ++ ++/* Why g5p_ru_recoverable() said no (for the terminal line), or "-" when it said yes. */ ++static inline const char * ++g5p_ru_recover_refusal(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget) ++ return "recover-off"; ++ if (in->retired || in->closed) ++ return "shutdown"; ++ if (!in->streak_max) ++ return in->first ? "first-job" : ++ in->recoveries >= in->budget ? "per-boot-budget" : ++ "-"; ++ if (in->first && !in->recoveries) ++ return "first-job"; ++ if (in->streak >= in->streak_max) ++ return "streak"; ++ return in->recoveries >= G5P_RU_RECOVER_BOOT_MAX ? "boot-max" : "-"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-retire.h b/drivers/gpu/apple-g5p/render-user-retire.h +new file mode 100644 +index 0000000000..6e57290033 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-retire.h +@@ -0,0 +1,248 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 3D completion: target coherence and engine idle, decided separately. Pure ++ * functions, no kernel services beyond the integer types, so the host suite compiles this ++ * file as it is. Include render-user-wait.h first. ++ * ++ * combined107 P4R seq 4 (build/gpu/combined107/raw/20261031T145710Z): the 3D target state ++ * was coherent (ring consumer = producer = target 4, both stamps 4, W+0x888/0x88c ours, ++ * W+0x890 = 4, all five channel watermarks at endpoint 8, and the firmware's type-1 3D ++ * completion event with the endpoint pair 7/8 decoded at publish + 19999 ms), yet the ++ * wait timed out at 60002 ms. The wait is a 1-2 ms RAM poll (no waitqueue, no wake), so ++ * this was not a lost edge: the done predicate also required the 3D global's idle words ++ * (G+0x724/0x728/0x72c/0x734/0x738) to be zero, and they never cleared. ++ * ++ * The 3D phase is complete when either ++ * (a) the target is coherent and the idle words are zero (the flown r96 predicate), or ++ * (b) the target is coherent and the matching 3D completion event was decoded after this ++ * job's 3D publish (idle_stuck). The event is the firmware's completion report, the ++ * signal a fence follows on Asahi too; the idle words lag it by an arbitrary time. ++ * settle_ms (default 0) only adds a minimum length of the coherent run, or ++ * (c) the target has been coherent, unbroken, for G5P_RU_RT_HOLD_MS (at least settle_ms): ++ * tahang1, as the TA's rule (c), so a lost or late event does not hold a job whose ++ * stamps landed. The PM state needs no clause here: the TA phase, which the 3D ++ * publish waits for, required it saved. ++ * Target coherent = ring consumer == target && g5p_ru_retired(): both stamps == seq, the W ++ * stamp pointers ours, the W sequence word == seq, every channel watermark == endpoint. ++ * The stamps carry the identity (31-bit sequence, one per job); the watermarks and the ++ * event's endpoint pair repeat every eight jobs and only corroborate. The TA phase has its ++ * own rules (tahang1, below). ++ */ ++#ifndef G5P_RENDER_USER_RETIRE_H ++#define G5P_RENDER_USER_RETIRE_H ++ ++#define G5P_RU_RT_DEF_SETTLE_MS 0U /* default of render_user_idle_settle_ms */ ++#define G5P_RU_RT_MAX_SETTLE_MS \ ++ 5000U /* far below the smallest per-phase bound (20 s) */ ++#define G5P_RU_RT_GRID 16U /* the channel watermark grid, G5P_WORK_SLOTS */ ++#define G5P_RU_RT_HOLD_MS \ ++ 500U /* rule (c), raised to settle_ms when that is longer */ ++ ++/* Clauses of the 3D predicate, as the receipt's `unsat` bits (set = not satisfied). */ ++#define G5P_RU_RT_CONSUMED 0x1U /* ring consumer != target */ ++#define G5P_RU_RT_RETIRED 0x2U /* g5p_ru_retired() false */ ++#define G5P_RU_RT_IDLE 0x4U /* an idle word is non-zero */ ++#define G5P_RU_RT_EVENT 0x8U /* no matching completion event latched */ ++#define G5P_RU_RT_SETTLE \ ++ 0x10U /* coherent, run shorter than the settle or hold window */ ++#define G5P_RU_RT_PM 0x20U /* TA receipt only: the PM state is not saved */ ++ ++enum g5p_ru_rt_how { ++ G5P_RU_RT_PENDING, ++ G5P_RU_RT_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_RT_IDLE_STUCK, /* rule (b) */ ++ G5P_RU_RT_HOLD, /* rule (c) */ ++}; ++ ++struct g5p_ru_retire { ++ u64 settle_ns; /* minimum coherent run before rule (b); 0 = none */ ++ u64 hold_ns; /* rule (c): max(settle_ns, G5P_RU_RT_HOLD_MS) */ ++ u64 run_ns; /* first sample of the current coherent run; 0 = none */ ++ u64 coherent_ns; /* first coherent sample of the phase */ ++ u64 event_ns; /* first sample that saw the matching event latched */ ++ u64 busy_ns; /* first sample with an idle word set */ ++ u64 clear_ns; /* first all-clear sample after busy_ns */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample */ ++ u32 how; /* enum g5p_ru_rt_how of the last sample */ ++ u32 decided; /* enum g5p_ru_rt_how of the sample that decided COMPLETE */ ++}; ++ ++static inline bool g5p_ru_rt_settle_valid(u32 settle_ms) ++{ ++ return settle_ms <= G5P_RU_RT_MAX_SETTLE_MS; ++} ++ ++static inline void g5p_ru_rt_start(struct g5p_ru_retire *t, u32 settle_ms) ++{ ++ u32 hold_ms = settle_ms > G5P_RU_RT_HOLD_MS ? settle_ms : ++ G5P_RU_RT_HOLD_MS; ++ ++ *t = (struct g5p_ru_retire){ ++ .settle_ns = (u64)settle_ms * 1000000ULL, ++ .hold_ns = (u64)hold_ms * 1000000ULL, ++ }; ++} ++ ++/* The firmware's 3D completion is a type-1 event with notification bit 1 (engine 1) whose ++ * words 8/9 are the channel endpoint pair (2 seq - 1, 2 seq): 1/2, 3/4, 5/6, 7/8 for ++ * combined107 seq 1..4. The grid wraps at G5P_RU_RT_GRID, so the pair is compared modulo ++ * the grid whether the firmware reports it wrapped or as a running count: endpoint 0 ++ * (seq 8) matches 15/0 and 15/16 alike. ++ */ ++static inline bool g5p_ru_rt_event_match(const u32 words[11], u32 endpoint) ++{ ++ return words[0] == 1 && (words[1] & 2) && ++ words[9] % G5P_RU_RT_GRID == endpoint % G5P_RU_RT_GRID && ++ (words[9] - words[8]) % G5P_RU_RT_GRID == 1; ++} ++ ++/* One RAM sample of the 3D phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_rt_how); the caller passes `!= G5P_RU_RT_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_rt_step(struct g5p_ru_retire *t, u64 now_ns, ++ bool consumed, bool retired, bool idle, ++ bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ if (!idle && !t->busy_ns) ++ t->busy_ns = now_ns; ++ if (idle && t->busy_ns && !t->clear_ns) ++ t->clear_ns = now_ns; ++ t->how = G5P_RU_RT_PENDING; ++ if (!consumed || !retired) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ t->run_ns = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_RT_IDLE_CLEAR; ++ else if (event && now_ns - t->run_ns >= t->settle_ns) ++ t->how = G5P_RU_RT_IDLE_STUCK; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_RT_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_rt_note(struct g5p_ru_retire *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++/* ms from `from_ns` to `to_ns` for the receipt; -1 when either is missing or out of order. */ ++static inline s64 g5p_ru_rt_ms(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000000ULL); ++} ++ ++/* ++ * tahang1: the TA phase. mcfold1 desktop seq 251 (after T1) and seq 339 (one client): ring ++ * consumed at 2 ms, the firmware's type-1 TA completion event decoded at 7.7-8.4 ms, both ++ * stamps, the W words and the watermarks at the job and the PM state saved, yet a TA idle word ++ * (G+0x724/0x728/0x72c) set from 7-8 ms on never cleared, and the r74b predicate (which needs ++ * the idle words zero) timed out at 60 s (`unsat=0x4`). On 49 of 241 good TAs of the same ++ * boot that word was set at 7-12 ms and cleared 2 ms later. The 3D engine shows the same lag ++ * (rule (b) above). ++ * ++ * The TA phase is complete when the target is coherent (ring consumer == target and ++ * g5p_ru_retired()) with the PM state saved, and ++ * (a) the idle words are zero (the flown r74b predicate), or ++ * (b) a type-1 TA completion event was decoded after this job's TA publish, or ++ * (c) the target has stayed coherent, unbroken, for G5P_RU_TA_HOLD_MS. ++ * The stamps carry the job's identity; the TA event does not (its words are the same every ++ * job), it only reports that the firmware finished a TA after the publish. Rule (c) keeps a ++ * lost or late event from holding a job whose stamps already landed. ++ */ ++#define G5P_RU_TA_HOLD_MS 500U ++ ++enum g5p_ru_ta_how { ++ G5P_RU_TA_PENDING, ++ G5P_RU_TA_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_TA_EVENT, /* rule (b) */ ++ G5P_RU_TA_HOLD, /* rule (c) */ ++ G5P_RU_TA_HOWS, ++}; ++ ++struct g5p_ru_ta { ++ u64 hold_ns; /* rule (c) */ ++ u64 run_ns; /* first sample of the current coherent run */ ++ u64 coherent_ns; /* first coherent sample of the phase; 0 = none */ ++ u64 event_ns; /* first sample that saw the event latched; 0 = none */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample (SETTLE: run below the hold) */ ++ u32 how; /* enum g5p_ru_ta_how of the last sample */ ++ u32 decided; /* enum g5p_ru_ta_how of the sample that decided COMPLETE */ ++}; ++ ++static inline void g5p_ru_ta_start(struct g5p_ru_ta *t) ++{ ++ *t = (struct g5p_ru_ta){ .hold_ns = ++ (u64)G5P_RU_TA_HOLD_MS * 1000000ULL }; ++} ++ ++/* The firmware's TA completion: a type-1 event with notification bit 0 (engine 0). */ ++static inline bool g5p_ru_ta_event_match(const u32 words[11]) ++{ ++ return words[0] == 1 && (words[1] & 1); ++} ++ ++/* One RAM sample of the TA phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_ta_how); the caller passes `!= G5P_RU_TA_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_ta_step(struct g5p_ru_ta *t, u64 now_ns, bool consumed, ++ bool retired, bool pm, bool idle, bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT) | ++ (pm ? 0 : G5P_RU_RT_PM); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ t->how = G5P_RU_TA_PENDING; ++ if (!consumed || !retired || !pm) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_TA_IDLE_CLEAR; ++ else if (event) ++ t->how = G5P_RU_TA_EVENT; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_TA_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_ta_note(struct g5p_ru_ta *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sched.h b/drivers/gpu/apple-g5p/render-user-sched.h +new file mode 100644 +index 0000000000..174a29e230 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sched.h +@@ -0,0 +1,155 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Render job scheduling across contexts (docs/gpu/multi-client-design.md, stage 2), shaped ++ * like drm_sched so that stage 4 can swap drm_sched in without a UAPI or structural change: ++ * ++ * entity = one render context (drm_sched_entity): a FIFO of its accepted jobs, and at ++ * most one close item, which runs after its last job (drm_sched_entity_destroy()); ++ * run list = the entities with work, served round-robin: each pick takes the first entity ++ * whose head job is ready and moves it to the tail (drm_sched's RR policy); ++ * ready = every dependency of the head job has signalled, it failed before the hardware, ++ * its input deadline passed, or admission closed for good (the job then fails ++ * without the hardware). Readiness is the caller's predicate; the kernel's adds ++ * dma-fence callbacks (drm_sched_job_add_dependency()) so that an unsignalled ++ * input of one client never blocks another client's jobs. ++ * ++ * One credit: the single shared TA/3D channel and PM record run one job at a time, so the ++ * caller runs each picked item to completion before the next pick. Only the head of an ++ * entity is looked at, so each context's fences signal in submission order. ++ * ++ * The lists are plain list_heads; the caller holds its scheduler lock around every call. ++ * Host-tested (scripts/gpu/multiclient2-host/test_render_sched.c). ++ */ ++#ifndef G5P_RENDER_USER_SCHED_H ++#define G5P_RENDER_USER_SCHED_H ++ ++struct g5p_rs_job { ++ struct list_head link; ++}; ++ ++struct g5p_rs_entity { ++ struct list_head link; /* in g5p_rs.run while it has work */ ++ struct list_head jobs; /* FIFO of g5p_rs_job */ ++ bool in_run, close_pending; ++}; ++ ++struct g5p_rs { ++ struct list_head run; ++}; ++ ++/* g5p_rs_pick() result */ ++enum g5p_rs_pick { ++ G5P_RS_IDLE, /* nothing ready; *wake holds the earliest deadline if any job waits */ ++ G5P_RS_JOB, /* *job was popped from *entity */ ++ G5P_RS_CLOSE, /* *entity has no job left and its close item is due */ ++}; ++ ++/* Head-job predicate: true when it may run now. *deadline is the time at which it becomes ++ * ready on its own (its input deadline); the predicate leaves it alone if there is none. ++ */ ++typedef bool (*g5p_rs_ready_fn)(struct g5p_rs_job *job, void *cookie, ++ unsigned long *deadline); ++ ++static inline void g5p_rs_init(struct g5p_rs *s) ++{ ++ INIT_LIST_HEAD(&s->run); ++} ++ ++static inline void g5p_rs_entity_init(struct g5p_rs_entity *e) ++{ ++ INIT_LIST_HEAD(&e->link); ++ INIT_LIST_HEAD(&e->jobs); ++ e->in_run = false; ++ e->close_pending = false; ++} ++ ++static inline void g5p_rs_enlist(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (!e->in_run) { ++ list_add_tail(&e->link, &s->run); ++ e->in_run = true; ++ } ++} ++ ++/* An accepted job joins the end of its entity's FIFO. */ ++static inline void g5p_rs_push(struct g5p_rs *s, struct g5p_rs_entity *e, ++ struct g5p_rs_job *job) ++{ ++ list_add_tail(&job->link, &e->jobs); ++ g5p_rs_enlist(s, e); ++} ++ ++/* The entity's close item: due once its FIFO is empty. False if one is already queued. */ ++static inline bool g5p_rs_close(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (e->close_pending) ++ return false; ++ e->close_pending = true; ++ g5p_rs_enlist(s, e); ++ return true; ++} ++ ++/* After the close item ran: the entity leaves the scheduler for good. */ ++static inline void g5p_rs_closed(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ (void)s; ++ e->close_pending = false; ++ if (e->in_run) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } ++} ++ ++/* ++ * The next item to run, round-robin over the entities with work. A picked entity moves to ++ * the tail; one left without a job and without a close item leaves the run list. Entities ++ * whose head is not ready keep their place. `has_wake` reports whether *wake is set. ++ */ ++static inline int g5p_rs_pick(struct g5p_rs *s, g5p_rs_ready_fn ready, ++ void *cookie, struct g5p_rs_entity **entity, ++ struct g5p_rs_job **job, unsigned long *wake, ++ bool *has_wake) ++{ ++ struct g5p_rs_entity *e; ++ ++ *has_wake = false; ++ list_for_each_entry(e, &s->run, link) { ++ struct g5p_rs_job *head; ++ unsigned long deadline; ++ bool timed = false; ++ ++ if (list_empty(&e->jobs)) { ++ /* Without a close item it left the run list at its last pick. */ ++ if (!e->close_pending) ++ continue; ++ *entity = e; ++ *job = NULL; ++ list_move_tail(&e->link, &s->run); ++ return G5P_RS_CLOSE; ++ } ++ head = list_first_entry(&e->jobs, struct g5p_rs_job, link); ++ deadline = 0; ++ if (!ready(head, cookie, &deadline)) { ++ timed = deadline != 0; ++ if (timed && ++ (!*has_wake || (long)(deadline - *wake) < 0)) { ++ *wake = deadline; ++ *has_wake = true; ++ } ++ continue; ++ } ++ list_del_init(&head->link); ++ *entity = e; ++ *job = head; ++ if (list_empty(&e->jobs) && !e->close_pending) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } else { ++ list_move_tail(&e->link, &s->run); ++ } ++ return G5P_RS_JOB; ++ } ++ return G5P_RS_IDLE; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-serial.h b/drivers/gpu/apple-g5p/render-user-serial.h +new file mode 100644 +index 0000000000..e5cffa4e32 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-serial.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial jobs of every render context (a compositor and its clients: one TA+3D job ++ * per frame each), on the device's one TA/3D channel pair. ++ * Pure functions, host-tested (scripts/gpu/multiclient1-host/test_render_admit.c). ++ * ++ * Semantics: ++ * - jobs of all contexts run strictly one after another: the queue is FIFO, each job is ++ * TA publish -> TA complete -> 3D publish -> 3D complete before the next job starts, and ++ * there is never a TA(N+1) in flight with a 3D(N); ++ * - the epoch's first job brings the device up: PM, channels and its context's DC3 attach ++ * with the r74f/r74g bells (one-shot). Every later context binds its own slot pair at its ++ * first job with a DC3 pair published in RAM only (render-user-bind.h), no doorbell; ++ * - every job after the first only rebuilds the PM pool, builds its pages for the device's ++ * sequence number, publishes with NO doorbell and waits RAM-only; ++ * - the first job of a context that does not complete (timeout, fault, protocol error) ++ * marks that context bad: every job still queued behind it fails with -EIO without ++ * touching the hardware, and every later submit is refused (-EIO); ++ * - TYPED_V3 never mixes with the other profiles, on a context or on the device; ++ * - after any timeout or fault (render-user-terminal.h `failed`) no job starts at all, and ++ * once the epoch is quiet the device cannot be brought up (that rings a doorbell); binding ++ * another context is RAM only and stays allowed; ++ * - the sequence number is the device's, 31 bit; the channel watermark grid wraps ++ * (g5p_ru_endpoint). ++ */ ++#ifndef G5P_RENDER_USER_SERIAL_H ++#define G5P_RENDER_USER_SERIAL_H ++ ++enum g5p_ru_v3_refusal { ++ G5P_RU_V3_OK, ++ G5P_RU_V3_NO_TUPLE, /* boot tuple (r74g + r87b self-drain, no r74h/r80) absent */ ++ G5P_RU_V3_MIXED, /* another profile ran on this context or brought the device up */ ++ G5P_RU_V3_OVERFLOW, /* sequence space exhausted */ ++ G5P_RU_V3_SLOTS, /* not two distinct slots in 7..255 */ ++ G5P_RU_V3_VARIANT, /* r22/r25 variant, 3d_after_ta, timestamps or pm_policy */ ++ G5P_RU_V3_BOUND, /* per-phase bound outside 20000..30000 ms */ ++ G5P_RU_V3_STATE, /* executor only: bind state and job counts disagree */ ++ G5P_RU_V3_TERMINAL, /* terminal epoch: timeout/fault, or quiet and the device down */ ++ G5P_RU_V3_REFUSALS, ++}; ++ ++static const char *const g5p_ru_v3_why[G5P_RU_V3_REFUSALS] = { ++ "", ++ "transport is not ready for TYPED_V3 rendering", ++ "another profile ran on this context or brought the device up", ++ "sequence space exhausted", ++ "slots are not two distinct slots in 7..255", ++ "an r22/r25 variant, 3d_after_ta, timestamps or pm_policy is latched", ++ "per-phase bound outside 20000..30000 ms", ++ "bind state and job counts disagree", ++ "terminal epoch (timeout or fault, or quiet and the device not up): no hardware access", ++}; ++ ++/* Everything the admission needs, as plain values. */ ++struct g5p_ru_v3_state { ++ bool tuple; /* boot tuple present */ ++ bool v3_accepted; /* a V3 job was accepted on this context */ ++ bool bound; /* this context's slot pair was published */ ++ u32 jobs; /* jobs that started on this context */ ++ bool device_up; /* the epoch's first job built PM and channels */ ++ bool device_v3; /* ... through the TYPED_V3 attach */ ++ u32 sequence; /* the device's last job that started (0 = none) */ ++ u32 slot_ta, slot_3d; ++ bool variant; ++ u32 wait_ms; ++ bool quiet; /* a terminal note was made: mmio_ok is false */ ++ bool terminal_failed; /* ... and it was, or a later one is, a timeout or fault */ ++}; ++ ++/* Any two distinct user slots: unbinds leave holes, so a pair need not be adjacent. */ ++static inline bool g5p_ru_v3_slots_valid(u32 slot_ta, u32 slot_3d) ++{ ++ return slot_ta >= 7 && slot_3d >= 7 && slot_ta <= 255 && ++ slot_3d <= 255 && slot_ta != slot_3d; ++} ++ ++#define G5P_RU_V3_SEQ_MAX 0x7fffffffU ++ ++/* At submit time and again in the executor (`executing`): a refusal here touched nothing. */ ++static inline int g5p_ru_v3_admit(const struct g5p_ru_v3_state *s, ++ bool executing) ++{ ++ if (!s->tuple) ++ return G5P_RU_V3_NO_TUPLE; ++ /* After any timeout or fault nothing new starts (closes the window between a failing job's ++ * unlock and the context being marked bad). Once quiet, the device cannot be brought up, ++ * because that rings a doorbell; a job on an up device publishes with no doorbell and ++ * touches no register, and so does the bind of a new context. ++ */ ++ if (s->terminal_failed || (s->quiet && !s->device_up)) ++ return G5P_RU_V3_TERMINAL; ++ /* A job of another profile ran on this context, or another profile brought the device ++ * up: only V3 may follow V3. ++ */ ++ if ((s->jobs && !s->v3_accepted) || (s->device_up && !s->device_v3)) ++ return G5P_RU_V3_MIXED; ++ if (s->sequence >= G5P_RU_V3_SEQ_MAX) ++ return G5P_RU_V3_OVERFLOW; ++ if (!g5p_ru_v3_slots_valid(s->slot_ta, s->slot_3d)) ++ return G5P_RU_V3_SLOTS; ++ if (s->variant) ++ return G5P_RU_V3_VARIANT; ++ if (!g5p_ru_v3_bound_valid(s->wait_ms)) ++ return G5P_RU_V3_BOUND; ++ /* The executor: the device is up exactly when a job has started on it, and a context is ++ * bound exactly when it has run a job, always on an up device. ++ */ ++ if (executing && (s->device_up != !!s->sequence || ++ s->bound != !!s->jobs || (s->bound && !s->device_up))) ++ return G5P_RU_V3_STATE; ++ return G5P_RU_V3_OK; ++} ++ ++/* Verbose receipts for each context's first jobs and every failure; a compositor prints no ++ * line per frame. ++ */ ++static inline bool g5p_ru_v3_verbose(u32 jobs, int ret) ++{ ++ return jobs <= 8 || ret; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sync.h b/drivers/gpu/apple-g5p/render-user-sync.h +new file mode 100644 +index 0000000000..d212d31995 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sync.h +@@ -0,0 +1,50 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CPU cache maintenance by range (perf60-fillrate.md §11.3 cut 5, kernel half). Host test: ++ * scripts/gpu/perf60-host/test_sync_range.c. ++ * ++ * render_user_sync_range bit 0: BO_SYNC honours offset/size (already validated page aligned by ++ * g5p_bo_range_valid) instead of syncing the whole BO. Bit 1: a job's ACCESS_CACHE maintenance ++ * (clean before, invalidate after retirement) covers the resource's BO range [offset, offset + ++ * size), the bytes the job can reach through that resource, instead of the whole BO. The window ++ * is widened to whole GPU pages and clamped to the BO. Off: the whole BO, as release8. ++ * The window is walked over the sg_table's DMA segments in order (their lengths concatenate to ++ * the BO's bytes); each segment gets the intersection, relative to the segment. ++ */ ++#ifndef G5P_RENDER_USER_SYNC_H ++#define G5P_RENDER_USER_SYNC_H ++ ++#define G5P_SYNC_ALIGN 0x4000ULL ++ ++/* ++ * [lo, hi) = [off, off + size) widened to pages, clamped to the BO; false if empty or outside. ++ */ ++static inline bool g5p_sync_window(u64 off, u64 size, u64 bo_size, u64 *lo, ++ u64 *hi) ++{ ++ u64 end; ++ ++ if (!size || off >= bo_size) ++ return false; ++ end = size > bo_size - off ? bo_size : off + size; ++ *lo = off & ~(G5P_SYNC_ALIGN - 1); ++ *hi = (end + G5P_SYNC_ALIGN - 1) & ~(G5P_SYNC_ALIGN - 1); ++ if (*hi > bo_size) ++ *hi = bo_size; ++ return *hi > *lo; ++} ++ ++/* The part of segment [pos, pos + len) inside [lo, hi): *start relative to the segment. */ ++static inline bool g5p_sync_clip(u64 pos, u64 len, u64 lo, u64 hi, u64 *start, ++ u64 *n) ++{ ++ u64 a = pos > lo ? pos : lo, b = pos + len < hi ? pos + len : hi; ++ ++ if (b <= a) ++ return false; ++ *start = a - pos; ++ *n = b - a; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-template.h b/drivers/gpu/apple-g5p/render-user-template.h +new file mode 100644 +index 0000000000..92bd1604ae +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-template.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Generated by make-render-user-template.py from the hash-pinned r22 importer. */ ++#ifndef G5P_RENDER_USER_TEMPLATE_H ++#define G5P_RENDER_USER_TEMPLATE_H ++ ++/* Frozen dialect-1 W scalars {offset, value}: color-only, one sample, ++ * normal 3D, no depth/resource arrays. Typed and kernel fields excluded. ++ */ ++static const u32 g5p_ru_ta_fixed[][2] = { ++ { 0x020, 0x1e3ce508 }, { 0x024, 0x1e3ce508 }, { 0x3a4, 0x00000088 }, ++ { 0x3a8, 0x00000002 }, { 0x3c0, 0x00020000 }, { 0x514, 0x08002358 }, ++ { 0x528, 0x00000001 }, ++}; ++ ++static const u32 g5p_ru_3d_fixed[][2] = { ++ { 0x108, 0x000003c3 }, { 0x110, 0x000003c3 }, { 0x118, 0x000003c3 }, ++ { 0x120, 0x000003c3 }, { 0x128, 0x000003c3 }, { 0x130, 0x000003c3 }, ++ { 0x3a4, 0x3f800000 }, { 0x3a8, 0x00000300 }, { 0x3b0, 0x0000001f }, ++ { 0x3c0, 0x0023000f }, { 0x3c4, 0x00000408 }, { 0x3c8, 0x00000008 }, ++ { 0x3cc, 0x000000f8 }, { 0x6a8, 0x3f800000 }, { 0x6ac, 0x00000300 }, ++ { 0x6b0, 0x00000002 }, { 0x868, 0x08002358 }, ++}; ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-terminal.h b/drivers/gpu/apple-g5p/render-user-terminal.h +new file mode 100644 +index 0000000000..4ced26eafe +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-terminal.h +@@ -0,0 +1,103 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 terminal state of the epoch. Pure functions over one small struct; the ++ * kernel supplies READ_ONCE/WRITE_ONCE/cmpxchg and the host suite shims them, so the ++ * host compiles this file as it is. ++ * ++ * One latch, first writer wins. The first terminal event of the epoch (the natural TA ++ * complete, a timeout, a fault or a protocol error) fixes `kind`; from then on `quiet` ++ * is set and never cleared, and g5p_ru_mmio_ok() is false: the host touches no register ++ * (R0, R1, AIC, doorbell) and no diagnostic bank. RAM stays accessible. A later ++ * failure (the 3D after a completed TA, job N+1 after job N) cannot change `kind` but ++ * sets the sticky `failed`, which is what the admission uses: after any timeout or fault ++ * no new job starts, from any queue, even one that passed the pre-lock context checks ++ * before the failing job marked the context bad. ++ * ++ * The recovery series (patches/gpu-recovery) is independent of this file. Its ++ * g5p_td_note_terminal() takes {1 complete, 2 timeout}; g5p_ru_td_kind() maps ours. ++ */ ++#ifndef G5P_RENDER_USER_TERMINAL_H ++#define G5P_RENDER_USER_TERMINAL_H ++ ++enum g5p_ru_term_kind { ++ G5P_RU_TERM_NONE, ++ G5P_RU_TERM_COMPLETE, /* the natural TA completed (RAM) */ ++ G5P_RU_TERM_TIMEOUT, /* a phase ran out its deadline */ ++ G5P_RU_TERM_FAULT, /* protocol error, drain error, attach or publish failure */ ++ G5P_RU_TERM_KINDS, ++}; ++ ++static const char *const g5p_ru_term_names[G5P_RU_TERM_KINDS] = { ++ "none", ++ "complete", ++ "timeout", ++ "fault", ++}; ++ ++struct g5p_ru_terminal { ++ int kind; /* first writer wins */ ++ int quiet; /* set with the first note, never cleared */ ++ int failed; /* sticky: any timeout or fault was noted */ ++ int captured; /* the CAPTURE entry (recovery hook) was taken, once */ ++}; ++ ++/* A new firmware epoch (the in-boot recovery, recover.inc) starts a new latch; nothing else ++ * clears it. ++ */ ++static inline void g5p_ru_term_reset(struct g5p_ru_terminal *t) ++{ ++ WRITE_ONCE(t->kind, 0); ++ WRITE_ONCE(t->quiet, 0); ++ WRITE_ONCE(t->failed, 0); ++ WRITE_ONCE(t->captured, 0); ++} ++ ++/* The kind a job's return value stands for (only called for ret < 0, ret != -ECANCELED). */ ++static inline int g5p_ru_term_from_ret(int ret) ++{ ++ return ret == -ETIMEDOUT ? G5P_RU_TERM_TIMEOUT : G5P_RU_TERM_FAULT; ++} ++ ++/* Returns true for the first note of the epoch. */ ++static inline bool g5p_ru_term_note(struct g5p_ru_terminal *t, int kind) ++{ ++ bool first; ++ ++ if (kind <= G5P_RU_TERM_NONE || kind >= G5P_RU_TERM_KINDS) ++ kind = G5P_RU_TERM_FAULT; ++ first = cmpxchg(&t->kind, G5P_RU_TERM_NONE, kind) == G5P_RU_TERM_NONE; ++ if (kind >= G5P_RU_TERM_TIMEOUT) ++ WRITE_ONCE(t->failed, 1); ++ WRITE_ONCE(t->quiet, 1); ++ return first; ++} ++ ++static inline int g5p_ru_term_kind(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->kind); ++} ++ ++static inline bool g5p_ru_term_failed(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->failed); ++} ++ ++/* The single predicate for register access: false from the first terminal note on. */ ++static inline bool g5p_ru_mmio_ok(const struct g5p_ru_terminal *t) ++{ ++ return !READ_ONCE(t->quiet); ++} ++ ++/* True once per epoch for a timeout or fault: the caller may take the recovery CAPTURE entry. */ ++static inline bool g5p_ru_term_capture_due(struct g5p_ru_terminal *t, int kind) ++{ ++ return kind >= G5P_RU_TERM_TIMEOUT && cmpxchg(&t->captured, 0, 1) == 0; ++} ++ ++/* Recovery's terminal encoding: 1 complete, 2 timeout (a fault is a no-completion exit: 2). */ ++static inline int g5p_ru_td_kind(int kind) ++{ ++ return kind == G5P_RU_TERM_COMPLETE ? 1 : 2; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-touch.h b/drivers/gpu/apple-g5p/render-user-touch.h +new file mode 100644 +index 0000000000..145bdbf2d4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-touch.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 RAM-touch latency A/B (RC4a). Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. Include render-user-wait.h first. ++ * ++ * Question (docs/gpu/experiments/latency101-device-analysis.md section 9): is the host's bulk read ++ * of the receipt pages what releases a stalled phase (the +6 ms atom after the job-1 receipt)? ++ * ++ * render_user_touch_ms = T, default 0 = off. On, every phase of every job records the first wait ++ * sample at or after publish + T that left the phase undecided (still pending, deadline not ++ * reached). Odd sequences are the touch arm: right after that sample the host reads the ++ * receipt's read set once (RAM only, folded into a sum). Even sequences are the control arm: the ++ * same instant is recorded and nothing is read. Neither arm changes a sample, the sleep, the ++ * deadline, a latch or the result, and neither prints before the job ends. ++ * ++ * Off: g5p_ru_touch_start() leaves the record zero and g5p_ru_touch_due() is never true, so the ++ * wait loop runs the latency101 instructions unchanged. ++ */ ++#ifndef G5P_RENDER_USER_TOUCH_H ++#define G5P_RENDER_USER_TOUCH_H ++ ++#define G5P_RU_TOUCH_MAX_MS \ ++ 20000U /* never at or past the smallest soft bound */ ++ ++struct g5p_ru_touch { ++ u64 at_ns; /* publish + T; 0 = off for this phase */ ++ u64 seen_ns; /* the undecided sample at or after at_ns that took it; 0 = not reached */ ++ u64 done_ns; /* end of the read (touch arm), seen_ns (control arm) */ ++ u32 bytes; /* bytes read: 0 on the control arm */ ++ u32 sum; /* fold of the words read: the loads cannot be elided */ ++ bool sweep; /* this phase is in the touch arm */ ++}; ++ ++/* 0 (off), or 1..G5P_RU_TOUCH_MAX_MS with latency101 on (hard_ms != 0) and strictly before the ++ * effective soft bound (g5p_ru_lat_soft()), so the instant always falls in the plain wait. ++ */ ++static inline bool g5p_ru_touch_valid(u32 touch_ms, u32 soft_ms, u32 hard_ms) ++{ ++ if (!touch_ms) ++ return true; ++ return hard_ms && touch_ms <= G5P_RU_TOUCH_MAX_MS && touch_ms < soft_ms; ++} ++ ++/* Odd sequences are the touch arm, even ones the control. */ ++static inline bool g5p_ru_touch_arm(u32 seq) ++{ ++ return seq & 1; ++} ++ ++static inline void g5p_ru_touch_start(struct g5p_ru_touch *t, u64 start_ns, ++ u32 touch_ms, u32 seq) ++{ ++ *t = (struct g5p_ru_touch){ 0 }; ++ if (!touch_ms) ++ return; ++ t->at_ns = start_ns + (u64)touch_ms * 1000000ULL; ++ t->sweep = g5p_ru_touch_arm(seq); ++} ++ ++/* ++ * Asked after g5p_ru_wait_step() of the sample taken at now_ns: true once, for the first sample ++ * at ++ * or after the instant whose step left the phase undecided and not expired. A completion, a fault ++ * or the deadline in the same sample wins, and then the instant is never taken. ++ */ ++static inline bool g5p_ru_touch_due(const struct g5p_ru_touch *t, u64 now_ns, ++ u32 result, bool expired) ++{ ++ return t->at_ns && !t->seen_ns && !result && !expired && ++ now_ns >= t->at_ns; ++} ++ ++/* The instant was taken at seen_ns (a sample time, never 0); done_ns is when the read ended. */ ++static inline void g5p_ru_touch_mark(struct g5p_ru_touch *t, u64 seen_ns, ++ u64 done_ns, u32 bytes, u32 sum) ++{ ++ t->seen_ns = seen_ns; ++ t->done_ns = done_ns < seen_ns ? seen_ns : done_ns; ++ t->bytes = bytes; ++ t->sum = sum; ++} ++ ++/* Receipt helper: to - from in microseconds, -1 when an end is missing or the order is wrong. */ ++static inline s64 g5p_ru_touch_us(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000); ++} ++ ++/* Receipt: how long after the instant (the end of the read) the phase's deciding sample saw it ++ * complete; -1 when the instant was not taken or the phase did not complete. ++ */ ++static inline s64 g5p_ru_touch_after_us(const struct g5p_ru_touch *t, ++ u32 result, u64 end_ns) ++{ ++ if (!t->seen_ns || result != G5P_RU_WAIT_COMPLETE) ++ return -1; ++ return g5p_ru_touch_us(t->done_ns, end_ns); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wait.h b/drivers/gpu/apple-g5p/render-user-wait.h +new file mode 100644 +index 0000000000..b9b5f1c49f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wait.h +@@ -0,0 +1,266 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial wait policy. Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. ++ * ++ * Evidence behind the numbers (r96 raw timelines, 2026-10-30): ++ * - the natural TA completion event arrived +12.0060 s and +12.0062 s after the ++ * publish in the two boots that a 12000 ms cap turned into timeouts; the whole ++ * delay is inside the firmware's start23, and once it returns the hardware and ++ * the retirement take less than ~1 ms; ++ * - nothing the host does shortens the delay, so the wait is RAM-only: after the ++ * DC3 attach the host touches no register until the job has ended. ++ * ++ * Policy: ++ * - one phase (TA, then 3D) waits at most `bound` from its own publish: 20 s by ++ * default, never below 20 s, never above the absolute cap; ++ * - the whole job (TA publish to the end of the 3D phase) never exceeds the ++ * absolute cap of 30 s; ++ * - the deadline is decided on RAM samples. A completion seen in the sample that ++ * reaches the deadline wins; a deadline is only final after one closing sample ++ * taken at or after it, which closes the gap between the last poll and the ++ * expiry; a completion after that is evidence only (`late_after`), never the ++ * result; ++ * - a fault wins over a completion in the same sample and over the deadline. ++ */ ++#ifndef G5P_RENDER_USER_WAIT_H ++#define G5P_RENDER_USER_WAIT_H ++ ++#define G5P_RU_V3_WAIT_MS 20000U /* default and minimum per-phase bound */ ++#define G5P_RU_V3_CAP_MS \ ++ 30000U /* absolute job bound, and the largest per-phase bound */ ++ ++/* ++ * latency101 (default off: render_user_soft_ms = render_user_hard_ms = 0 leaves every number above, ++ * and the behaviour of every function below, exactly as it was). When on, the per-phase deadline ++ * is the HARD bound and the job bound is two hard bounds; the SOFT bound marks a phase "slow" ++ * (RAM-only record, no print, no latch). A silent event stream means a dead firmware from the ++ * phase's publish on (faultiso; before it, only from the soft bound). See ++ * docs/gpu/experiments/prod98c-bimodal-latency.md and docs/gpu/experiments/faultiso.md. ++ */ ++#define G5P_RU_LAT_MIN_SOFT_MS \ ++ 20000U /* never below the proven per-phase bound */ ++#define G5P_RU_LAT_DEF_SOFT_MS 45000U ++#define G5P_RU_LAT_MAX_HARD_MS \ ++ 60000U /* largest hard bound; the job bound is twice this */ ++#define G5P_RU_LAT_DEF_OBS_MS \ ++ 2000U /* RAM-only look after a timeout, pre-receipt */ ++#define G5P_RU_LAT_MAX_OBS_MS 5000U ++#define G5P_RU_LAT_SILENT_MS \ ++ 1000U /* no event decoded for this long in a phase = silent */ ++/* The silence clock runs from the publish (the host suites key on this). */ ++#define G5P_RU_LAT_SILENT_FROM_PUBLISH 1 ++ ++enum g5p_ru_wait_result { ++ G5P_RU_WAIT_PENDING, ++ G5P_RU_WAIT_COMPLETE, ++ G5P_RU_WAIT_TIMEOUT, ++ G5P_RU_WAIT_FAULT, ++}; ++ ++struct g5p_ru_wait { ++ u64 start_ns; /* the publish (producer store) of this phase */ ++ u64 deadline_ns; /* start + bound, clamped to the job's absolute deadline */ ++ u64 end_ns; /* the sample that decided */ ++ u32 result; ++ bool expired; /* a sample at or after the deadline has been seen */ ++ bool late_gap; /* completion first seen by the closing sample */ ++ bool late_after; /* completion seen after a timeout was decided: evidence only */ ++ /* latency101; all zero (and never read) unless soft_ns != 0 */ ++ u64 soft_ns; /* start + soft bound; 0 = the grace is off */ ++ u64 soft_seen_ns; /* the first sample at or after soft_ns */ ++ u64 late_ns; /* the first retired sample after the timeout was decided */ ++ u64 event_ns; /* sample time of the last change of the decoded-event counter */ ++ u64 busy_ns; /* the first sample whose idle words showed the engine busy */ ++ u32 events; /* the decoded-event counter at that sample */ ++ bool slow; /* the phase passed its soft bound and is still pending */ ++ bool silent; /* the event stream stopped during the phase: firmware dead */ ++ bool late_soft; /* completed after the soft bound: a success, never a failure */ ++}; ++ ++/* latency101 parameters. Off is hard = soft = 0. On: hard in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000. The job bound the waiters must outlast is two ++ * hard bounds plus one observation window. ++ */ ++static inline u32 g5p_ru_lat_soft(u32 soft_ms, u32 hard_ms) ++{ ++ if (soft_ms) ++ return soft_ms; ++ return hard_ms > G5P_RU_LAT_DEF_SOFT_MS ? G5P_RU_LAT_DEF_SOFT_MS : ++ hard_ms; ++} ++ ++static inline bool g5p_ru_lat_valid(u32 soft_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!hard_ms) ++ return !soft_ms; ++ soft_ms = g5p_ru_lat_soft(soft_ms, hard_ms); ++ return hard_ms <= G5P_RU_LAT_MAX_HARD_MS && ++ soft_ms >= G5P_RU_LAT_MIN_SOFT_MS && soft_ms <= hard_ms && ++ obs_ms <= G5P_RU_LAT_MAX_OBS_MS; ++} ++ ++static inline u32 g5p_ru_lat_job_ms(u32 hard_ms, u32 obs_ms) ++{ ++ return 2 * hard_ms + obs_ms; ++} ++ ++/* A per-phase bound is valid when it is in [G5P_RU_V3_WAIT_MS, G5P_RU_V3_CAP_MS]. */ ++static inline bool g5p_ru_v3_bound_valid(u32 ms) ++{ ++ return ms >= G5P_RU_V3_WAIT_MS && ms <= G5P_RU_V3_CAP_MS; ++} ++ ++/* The deadline of a phase: its own bound, clamped to the job's absolute deadline. ++ * A job deadline of 0 means "no job deadline yet" (the first phase). ++ */ ++static inline u64 g5p_ru_wait_deadline(u64 start_ns, u32 bound_ms, ++ u64 job_deadline_ns) ++{ ++ u64 d = start_ns + (u64)bound_ms * 1000000ULL; ++ ++ return job_deadline_ns && job_deadline_ns < d ? job_deadline_ns : d; ++} ++ ++static inline void g5p_ru_wait_start(struct g5p_ru_wait *w, u64 start_ns, ++ u32 bound_ms, u64 job_deadline_ns) ++{ ++ *w = (struct g5p_ru_wait){ ++ .start_ns = start_ns, ++ .deadline_ns = g5p_ru_wait_deadline(start_ns, bound_ms, ++ job_deadline_ns), ++ }; ++} ++ ++/* ++ * latency101: hard_ms is the per-phase deadline (clamped to the job's deadline as above), ++ * soft_ms ++ * (never past the deadline) the point where a still-pending phase is called slow. `events` seeds ++ * the silence clock. ++ */ ++static inline void g5p_ru_wait_start_lat(struct g5p_ru_wait *w, u64 start_ns, ++ u32 soft_ms, u32 hard_ms, ++ u64 job_deadline_ns, u32 events) ++{ ++ g5p_ru_wait_start(w, start_ns, hard_ms, job_deadline_ns); ++ w->soft_ns = start_ns + (u64)soft_ms * 1000000ULL; ++ if (w->soft_ns > w->deadline_ns) ++ w->soft_ns = w->deadline_ns; ++ w->events = events; ++ w->event_ns = start_ns; ++} ++ ++/* The first sample whose idle words (G+0x724..) show the engine busy, i.e. the firmware's ++ * start path has run. Recorded once; it splits a late start (busy_ns close to the ++ * completion) from slow hardware (busy_ns early). A busy window shorter than the poll ++ * period can be missed: 0 = not seen. ++ */ ++static inline void g5p_ru_wait_busy(struct g5p_ru_wait *w, u64 now_ns, ++ bool busy) ++{ ++ if (busy && !w->busy_ns) ++ w->busy_ns = now_ns; ++} ++ ++/* One look at the decoded-event counter, taken with every RAM sample, before g5p_ru_wait_step(). ++ * The ALIVE stream runs at about 290 events/s whatever the work does (r96c: 29442 polls, no gap ++ * over 3.4 ms; latency101: 287 events/s through every stall of 0.55-33 s), so a stream that has ++ * been silent for G5P_RU_LAT_SILENT_MS is a dead firmware, not a slow one, at any point of the ++ * phase. Returns true then; the caller reports it as a fault. The silence clock starts at the ++ * publish (vpds rung vk, 2026-11-01: the counter froze 1 s after the TA's publish, and a clock ++ * that only ran from the soft bound called it at 45 s, with every other client held behind it). ++ * The caller drains the ring before each sample, so a host that slept longer than the window ++ * sees the events it missed and is not silent. Never true when latency101 is off. ++ */ ++static inline bool g5p_ru_wait_grace(struct g5p_ru_wait *w, u64 now_ns, ++ u32 events) ++{ ++ if (!w->soft_ns) ++ return false; ++ if (events != w->events) { ++ w->events = events; ++ w->event_ns = now_ns; ++ } ++ if (now_ns >= w->soft_ns && !w->soft_seen_ns) ++ w->soft_seen_ns = now_ns; ++ w->silent = now_ns - w->event_ns >= ++ (u64)G5P_RU_LAT_SILENT_MS * 1000000ULL; ++ return w->silent; ++} ++ ++/* One RAM sample taken at `now_ns`. Returns the result so far (PENDING until ++ * decided). A sample at or after the deadline only marks the wait expired unless ++ * it shows completion or a fault; the caller must then take one closing sample and ++ * call g5p_ru_wait_close(). ++ */ ++static inline u32 g5p_ru_wait_step(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (w->soft_ns && now_ns >= w->soft_ns) ++ w->slow = true; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ w->end_ns = now_ns; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->end_ns = now_ns; ++ w->late_soft = w->slow; ++ } else if (now_ns >= w->deadline_ns) { ++ w->expired = true; ++ } ++ return w->result; ++} ++ ++/* The closing sample, taken after the loop left because of the deadline. This is ++ * the only place a timeout is decided. ++ */ ++static inline u32 g5p_ru_wait_close(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->late_gap = true; ++ w->late_soft = w->slow; ++ } else { ++ w->result = G5P_RU_WAIT_TIMEOUT; ++ } ++ w->end_ns = now_ns; ++ return w->result; ++} ++ ++/* A sample after the result was decided: a completion after a timeout is recorded ++ * and nothing else happens (no 3D, no recovery, no register access). ++ */ ++static inline void g5p_ru_wait_after(struct g5p_ru_wait *w, bool complete) ++{ ++ if (w->result == G5P_RU_WAIT_TIMEOUT && complete) ++ w->late_after = true; ++} ++ ++/* latency101: the RAM-only observation window after a timeout was decided, before the receipt is ++ * printed and the terminal latch is taken. g5p_ru_wait_after() looks once, microseconds after the ++ * decision; a completion that arrives milliseconds later (prod98c seq 3: 5.49 ms) was never ++ * recorded, so `late_after=0/0` was a false negative. Returns true while the window is open. A ++ * retired sample ends it and is recorded in late_after/late_ns; it never changes the result (a ++ * deadline stays final). obs_ms = 0 closes it at once (the legacy behaviour). ++ */ ++static inline bool g5p_ru_wait_observe(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, u32 obs_ms) ++{ ++ if (w->result != G5P_RU_WAIT_TIMEOUT || w->late_after) ++ return false; ++ if (complete) { ++ w->late_after = true; ++ w->late_ns = now_ns; ++ return false; ++ } ++ return now_ns < w->end_ns + (u64)obs_ms * 1000000ULL; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +new file mode 100644 +index 0000000000..c001da4865 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -0,0 +1,273 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 publish-time MTS wake (wake110). Pure functions over plain values; the one ++ * register sequence goes through a callback table (the fender.h pattern), so the host ++ * suite compiles this file as it is and records every access in order. ++ * ++ * Question (docs/gpu/experiments/wake110-apple-mts-dispatch.md): is the 0.5-33 s TA/3D ++ * start latency caused by the missing Apple submit kick? Apple's TA and 3D submit ++ * wrappers (iPad6,11 kernelcache 0xfffffff006ca00a0 / 0xfffffff006ca01f8) do, every time: ++ * ring entry, dmb ish, Fender vt+0x80 (R1+0x14 = 7, read back), Fender vt+0x18 ++ * (R1+0x1000 & 0xf), then R0+0xb00 = 0x10 (TA, DM0|COUNTED) or 0x11 (3D, DM1|COUNTED). ++ * The MTS store is reached from every arm; when the GPC nibble is 0 Apple first runs a ++ * power arm (up to forcePowerCycleAndReboot). Linux V3 publishes job N >= 2's TA and ++ * every 3D with no kick at all. ++ * ++ * render_user_wake (latched per context): 0 off (default: no call has any effect), ++ * 1 the A/B, 2 every eligible phase is a factor (the candidate production fix, for use ++ * only after the A/B proved it). The TA of job 1 is never a phase of this file (the r74g ++ * grace and its fenced fallback bell own it). In the A/B each engine's phases are paired ++ * in publish order, (0,1), (2,3), ...; exactly one of each pair is the factor, which one ++ * is fixed by render_user_wake_seed before the context exists, and bit 31 of the seed ++ * inverts the whole schedule (the mirror stage). The verdict recomputes the arm of every ++ * receipt from the seed. ++ * ++ * At the publish, right after the producer store and its barrier, both arms take the same ++ * lock and read the same RAM state and run the same guards; the control stops there (no ++ * register). The factor then runs the Apple order: R1+0x14 = 7, read back (any bit ++ * outside 7: stop), R1+0x1000 (state & 0x1f != 1, the GPC is not active: stop, never ++ * repaired, never a power arm), dma_wmb(), R0+0xb00 = 0x10 | engine. At most one attempt ++ * per phase. Never after a timeout or fault (the terminal latch's sticky `failed`), never ++ * with the boot failed or stopped, never unpowered, never with an r74f bell in flight, ++ * never once the phase shows any sign of retirement, never on a ring that is not exactly ++ * this publish. A consumer already at the target does not cancel (Apple never checks ++ * it); it is recorded. The natural TA completion (terminal kind COMPLETE, the epoch's ++ * first note) is not a failure: the factor's sequence is the one licensed exception to the ++ * r96 "no register after the TA complete" envelope, default off. ++ */ ++#ifndef G5P_RENDER_USER_WAKE_H ++#define G5P_RENDER_USER_WAKE_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++#define G5P_WK_OFF 0U ++#define G5P_WK_AB 1U ++#define G5P_WK_ALWAYS 2U ++#define G5P_WK_MIRROR 0x80000000U /* seed bit 31: every arm inverted */ ++ ++#define G5P_WK_R1_FENDER 0x14U /* Fender vt+0x80: write 7, read back */ ++#define G5P_WK_R1_GPC 0x1000U /* GPC state: & 0x1f == 1 is active */ ++#define G5P_WK_R0_MTS 0xb00U /* MTS_SCHEDULE, write only */ ++#define G5P_WK_MTS_TA 0x10U /* DM0 | COUNTED */ ++#define G5P_WK_MTS_3D 0x11U /* DM1 | COUNTED */ ++ ++/* wake111: the publish-time power evidence, one bit per RAM fact, sampled under the event lock ++ * and printed per phase (power=TT/DD). Every required bit must be set. reset_transition is ++ * the permanent post-release slave-port lockout (set right before the release, never ++ * cleared), not a live reset state: Wake110 vetoed on it, so every phase skipped. It is ++ * recorded (SLAVE_LOCK) and is neither required nor a veto; the slave-port ban is unchanged. ++ */ ++#define G5P_WK_PWR_CORE 0x01U /* g->powered */ ++#define G5P_WK_PWR_FENDER 0x02U /* g->fender_ready */ ++#define G5P_WK_PWR_RELEASED 0x04U /* b->released */ ++#define G5P_WK_PWR_ALIVE 0x08U /* b->alive */ ++#define G5P_WK_PWR_CONTROL 0x10U /* b->control_complete */ ++#define G5P_WK_PWR_DS108 \ ++ 0x20U /* DS108 claim proof, current generation, no power abort */ ++#define G5P_WK_PWR_SLAVE_LOCK 0x40U /* g->reset_transition: evidence only */ ++#define G5P_WK_PWR_REQUIRED \ ++ (G5P_WK_PWR_CORE | G5P_WK_PWR_FENDER | G5P_WK_PWR_RELEASED | \ ++ G5P_WK_PWR_ALIVE | G5P_WK_PWR_CONTROL | G5P_WK_PWR_DS108) ++ ++enum g5p_wk_arm { ++ G5P_WK_ARM_NONE, /* wake off */ ++ G5P_WK_ARM_EXCLUDED, /* the TA of job 1 (r74g) */ ++ G5P_WK_ARM_CONTROL, ++ G5P_WK_ARM_FACTOR, ++ G5P_WK_ARMS, ++}; ++ ++enum g5p_wk_did { ++ G5P_WK_DID_NONE, /* off, or never reached */ ++ G5P_WK_DID_RANG, /* factor: fence, GPC active, 0xb00 written */ ++ G5P_WK_DID_SHAM, /* control: every guard passed, no register */ ++ G5P_WK_DID_EXCLUDED, ++ G5P_WK_SKIP_TWICE, /* this phase was already decided */ ++ G5P_WK_SKIP_LICENSE, /* a timeout or fault was noted (terminal failed) */ ++ G5P_WK_SKIP_FAULT, /* boot failed, event failed or stopped */ ++ G5P_WK_SKIP_POWER, /* one of G5P_WK_PWR_REQUIRED is absent */ ++ G5P_WK_SKIP_R74F, /* an r74f bell is published, open or firing */ ++ G5P_WK_SKIP_RING, /* the ring is not exactly this publish */ ++ G5P_WK_SKIP_RETIRED, /* a stamp == seq or a completion event since the publish */ ++ G5P_WK_SKIP_FENCE, /* R1+0x14 read back outside 7: no further register */ ++ G5P_WK_SKIP_GPC, /* GPC not active: no 0xb00 */ ++ G5P_WK_DIDS, ++}; ++ ++static const char *const g5p_wk_arm_names[G5P_WK_ARMS] = { ++ "none", ++ "excluded", ++ "control", ++ "factor", ++}; ++ ++static const char *const g5p_wk_did_names[G5P_WK_DIDS] = { ++ "none", "rang", "sham", "excluded", "skip-twice", ++ "skip-license", "skip-fault", "skip-power", "skip-r74f", "skip-ring", ++ "skip-retired", "skip-fence", "skip-gpc", ++}; ++ ++/* Everything the decision reads, sampled under the event lock right after the producer store. */ ++struct g5p_wk_in { ++ u32 mode, seed, seq, e; ++ bool attempted; /* this phase already went through g5p_wk_decide() */ ++ bool term_failed; /* g5p_ru_term_failed() */ ++ bool boot_failed; /* b->failed || b->event_failed || b->stopped */ ++ u32 power; /* g5p_wk_power_mask(); the slave lock is evidence, not a veto */ ++ bool r74f_pending; /* an r74f bell in PUBLISHED, OPEN or FIRING */ ++ u32 start, target, prod, ++ cons; /* ring: producer before this publish, our target, now */ ++ u32 stamp[2]; /* this engine's stamp words */ ++ u32 type1; /* this engine's completion events decoded since the phase armed */ ++}; ++ ++/* The register access of the factor, in order. The kernel binds them to writel/readl on the ++ * Fender window and the RGX window and to dma_wmb(); the host suite records them. ++ */ ++struct g5p_wk_io { ++ void *ctx; ++ void (*w1)(void *ctx, u32 reg, u32 value); ++ u32 (*r1)(void *ctx, u32 reg); ++ void (*wbarrier)(void *ctx); ++ void (*w0)(void *ctx, u32 reg, u32 value); ++}; ++ ++static inline bool g5p_wk_mode_valid(u32 mode) ++{ ++ return mode <= G5P_WK_ALWAYS; ++} ++ ++static inline u32 g5p_wk_mix(u32 x) ++{ ++ x ^= x >> 16; ++ x *= 0x7feb352dU; ++ x ^= x >> 15; ++ x *= 0x846ca68bU; ++ x ^= x >> 16; ++ return x; ++} ++ ++/* The phase's index among its engine's eligible phases: TA of job seq >= 2 -> seq - 2, ++ * 3D of job seq >= 1 -> seq - 1. ++ */ ++static inline u32 g5p_wk_index(u32 seq, u32 e) ++{ ++ return e ? seq - 1 : seq - 2; ++} ++ ++static inline u32 g5p_wk_arm(u32 mode, u32 seed, u32 seq, u32 e) ++{ ++ u32 i, flip; ++ ++ if (!mode || !g5p_wk_mode_valid(mode) || !seq || e > 1) ++ return G5P_WK_ARM_NONE; ++ if (!e && seq == 1) ++ return G5P_WK_ARM_EXCLUDED; ++ if (mode == G5P_WK_ALWAYS) ++ return G5P_WK_ARM_FACTOR; ++ i = g5p_wk_index(seq, e); ++ flip = g5p_wk_mix(g5p_wk_mix(seed & ~G5P_WK_MIRROR) + (e << 24) + ++ (i >> 1)) & ++ 1; ++ flip ^= !!(seed & G5P_WK_MIRROR); ++ return ((i & 1) ^ flip) ? G5P_WK_ARM_FACTOR : G5P_WK_ARM_CONTROL; ++} ++ ++static inline u32 g5p_wk_mts(u32 e) ++{ ++ return e ? G5P_WK_MTS_3D : G5P_WK_MTS_TA; ++} ++ ++/* wake111: the evidence word, one bit per fact (the caller samples each fact once). */ ++static inline u32 g5p_wk_power_mask(bool powered, bool fender_ready, ++ bool released, bool alive, ++ bool control_complete, bool ds108, ++ bool slave_lock) ++{ ++ return (powered ? G5P_WK_PWR_CORE : 0) | ++ (fender_ready ? G5P_WK_PWR_FENDER : 0) | ++ (released ? G5P_WK_PWR_RELEASED : 0) | ++ (alive ? G5P_WK_PWR_ALIVE : 0) | ++ (control_complete ? G5P_WK_PWR_CONTROL : 0) | ++ (ds108 ? G5P_WK_PWR_DS108 : 0) | ++ (slave_lock ? G5P_WK_PWR_SLAVE_LOCK : 0); ++} ++ ++static inline bool g5p_wk_power_ready(u32 power) ++{ ++ return (power & G5P_WK_PWR_REQUIRED) == G5P_WK_PWR_REQUIRED; ++} ++ ++/* The RAM guards, common to both arms, in priority order. 0 = pass. */ ++static inline u32 g5p_wk_guard(const struct g5p_wk_in *in) ++{ ++ if (in->attempted) ++ return G5P_WK_SKIP_TWICE; ++ if (in->term_failed) ++ return G5P_WK_SKIP_LICENSE; ++ if (in->boot_failed) ++ return G5P_WK_SKIP_FAULT; ++ if (!g5p_wk_power_ready(in->power)) ++ return G5P_WK_SKIP_POWER; ++ if (in->r74f_pending) ++ return G5P_WK_SKIP_R74F; ++ if (in->start > 255 || in->target > 255 || in->prod != in->target || ++ in->cons > 255 || ((in->target - in->start) & 255) != 1 || ++ ((in->cons - in->start) & 255) > 1) ++ return G5P_WK_SKIP_RING; ++ if (in->stamp[0] == in->seq || in->stamp[1] == in->seq || in->type1) ++ return G5P_WK_SKIP_RETIRED; ++ return 0; ++} ++ ++/* The factor's register sequence (Apple's order). The caller set *fence and *gpc to U32_MAX ++ * (not read). ++ */ ++static inline u32 g5p_wk_ring(const struct g5p_wk_io *io, u32 e, u32 *fence, ++ u32 *gpc) ++{ ++ io->w1(io->ctx, G5P_WK_R1_FENDER, 7); ++ *fence = io->r1(io->ctx, G5P_WK_R1_FENDER); ++ if (*fence & ~7U) ++ return G5P_WK_SKIP_FENCE; ++ *gpc = io->r1(io->ctx, G5P_WK_R1_GPC); ++ if (!G5P_GPC_ON(*gpc)) ++ return G5P_WK_SKIP_GPC; ++ /* the publish's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ io->wbarrier(io->ctx); ++ io->w0(io->ctx, G5P_WK_R0_MTS, g5p_wk_mts(e)); ++ return G5P_WK_DID_RANG; ++} ++ ++/* One phase's decision and, for a factor that passed every guard, its register sequence ++ * (the control arm never touches io). ++ */ ++static inline u32 g5p_wk_decide(const struct g5p_wk_in *in, ++ const struct g5p_wk_io *io, u32 *fence, ++ u32 *gpc) ++{ ++ u32 arm = g5p_wk_arm(in->mode, in->seed, in->seq, in->e), skip; ++ ++ *fence = U32_MAX; ++ *gpc = U32_MAX; ++ if (arm == G5P_WK_ARM_NONE) ++ return G5P_WK_DID_NONE; ++ if (arm == G5P_WK_ARM_EXCLUDED) ++ return G5P_WK_DID_EXCLUDED; ++ skip = g5p_wk_guard(in); ++ if (skip) ++ return skip; ++ if (arm == G5P_WK_ARM_CONTROL) ++ return G5P_WK_DID_SHAM; ++ return g5p_wk_ring(io, in->e, fence, gpc); ++} ++ ++/* A decided phase never decides again (the caller stores this before it unlocks). */ ++static inline bool g5p_wk_attempted(u32 did) ++{ ++ return did != G5P_WK_DID_NONE && did != G5P_WK_DID_EXCLUDED; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-window.h b/drivers/gpu/apple-g5p/render-user-window.h +new file mode 100644 +index 0000000000..bfbae628d2 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-window.h +@@ -0,0 +1,74 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The firmware window of a render VM (docs/gpu/multi-client-design.md §6 R4, stage 3). ++ * ++ * The firmware's own GPU memory lives in one GiB, catalogue entry G5P_RU_FIRMWARE_CI: its ++ * image, the root and shared pages (rings, the PA/tag arrays, W), the MMU table alias and the ++ * PIO pages (Fender, AIC). Stage 1 copied that catalogue entry into every render VM, so any ++ * render job could address all of it. A render VM now gets, under that entry, a directory ++ * the kernel built that maps only the pages the GPU's own requesters reach through the ++ * context's slot: the PM control page (S) and the PM list and free list (F). Their leaf ++ * entries are copied from the firmware's tables, so their translation is unchanged. ++ * ++ * render_user_fw_window = 0 (G5P_RU_FW_FULL) the stage 1 mapping: the whole firmware GiB; ++ * render contexts need CAP_SYS_RAWIO ++ * 1 (G5P_RU_FW_WINDOW) the window above (default) ++ * 2 (G5P_RU_FW_NONE) no firmware page at all (an experiment) ++ * ++ * Pure helpers, host-tested (scripts/gpu/multiclient2-host/test_render_window.c). The ++ * caller provides g5p_le32() and the boot layout constants. ++ */ ++#ifndef G5P_RENDER_USER_WINDOW_H ++#define G5P_RENDER_USER_WINDOW_H ++ ++#define G5P_RU_FW_FULL 0U ++#define G5P_RU_FW_WINDOW 1U ++#define G5P_RU_FW_NONE 2U ++#define G5P_RU_FW_MODES 3U ++ ++#define G5P_RU_FW_PAGES 3U ++/* The pages of the window, all in the firmware GiB. */ ++static const u64 g5p_ru_fw_window_va[G5P_RU_FW_PAGES] = { ++ G5P_SHARED_VA + G5P_R22_PM, /* PM control and S */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE, /* PM list */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, /* PM free list F */ ++}; ++ ++static inline u32 g5p_ru_fw_di(u64 va) ++{ ++ return (va >> 21) & 511; ++} ++ ++static inline u32 g5p_ru_fw_ti(u64 va) ++{ ++ return (va >> 14) & 127; ++} ++ ++/* The firmware's 64-bit leaf entry for `va`, from its leaf pages and their directory ++ * indices; 0 when the firmware does not map it. ++ */ ++static inline u64 g5p_ru_fw_leaf(u8 *const *leaf, const u32 *leaf_di, ++ u32 leaves, u64 va) ++{ ++ for (u32 i = 0; i < leaves; i++) { ++ const u8 *e = leaf[i] + g5p_ru_fw_ti(va) * 8; ++ ++ if (leaf_di[i] == g5p_ru_fw_di(va)) ++ return (u64)g5p_le32(e + 4) << 32 | g5p_le32(e); ++ } ++ return 0; ++} ++ ++/* The window's leaf page index for `va` among `n` distinct directory indices so far, or n ++ * (a new one) when none has its directory index. ++ */ ++static inline u32 g5p_ru_fw_slot(const u32 *di, u32 n, u64 va) ++{ ++ u32 i; ++ ++ for (i = 0; i < n && di[i] != g5p_ru_fw_di(va); i++) ++ ; ++ return i; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-variants-r25.h b/drivers/gpu/apple-g5p/render-variants-r25.h +new file mode 100644 +index 0000000000..8469a2f4b1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-variants-r25.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_VARIANTS_R25_H ++#define G5P_RENDER_VARIANTS_R25_H ++ ++/* Finite, owned-input experiments. No raw W, register or firmware-patch API. */ ++#define G5P_R25_MAX_BLOCKS 128U ++#define G5P_R25_HEAD 0x3000U ++ ++struct g5p_r25_variants { ++ u32 pm, blocks, context, host, sku, memory, rt, slots; ++}; ++ ++static inline bool g5p_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return v->pm <= 3 && ++ (v->blocks == 0 || v->blocks == 64 || v->blocks == 128) && ++ v->context <= 3 && v->host <= 2 && v->sku <= 3 && ++ v->memory <= 4 && v->rt <= 2 && v->slots <= 1; ++} ++ ++/* Event synthesis, removed context tasks, candidate clocks and the low-address ++ * dialect test cannot qualify rendering, even if firmware stamps retire. ++ */ ++static inline bool g5p_r25_diagnostic(const struct g5p_r25_variants *v) ++{ ++ return v->sku || v->context || v->host || v->rt == 2; ++} ++ ++/* PM payload frames are physical PPNs and have no user PTE attributes. This ++ * changes only PM auxiliary mappings (and, in mode4, other private mappings). ++ * Firmware/shared META mappings and the borrowed sink are never rewritten. ++ */ ++static inline u32 g5p_r25_attributes(const struct g5p_r25_variants *v, u64 va) ++{ ++ if (!v->memory) ++ return 0xc; ++ if (v->memory != 4 && ++ (va < G5P_R22_AUX || va >= G5P_R22_AUX + 5 * G5P_GPU_PAGE)) ++ return 0xc; ++ return v->memory == 2 ? 4 : v->memory == 3 ? 8 : 0; ++} ++ ++/* Preserve the original constructor exactly for the default pool. The extra ++ * blocks fit the existing 16KiB list and free allocations, including F's last ++ * dword at128 blocks. Validate every PA before mutating either allocation. ++ */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/retire108.h b/drivers/gpu/apple-g5p/retire108.h +new file mode 100644 +index 0000000000..e944b92c91 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/retire108.h +@@ -0,0 +1,434 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RETIRE108_H ++#define G5P_RETIRE108_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P outgoing retire layer (series gpu-recovery108, patch 0006). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 3. ++ * ++ * Wraps the recovery98 teardown ladder (teardown.h, g5p_td_run()) without ++ * changing it: the TD98 lines of a run are byte for byte the ones of the ++ * same run without this layer. It adds, around a run: ++ * ++ * ADMIT atomically close job admission (one cmpxchg word), then a ++ * bounded RAM-only wait for submitters already inside the ++ * admission window; still inside at the deadline = explicit ++ * ADMIT_OPEN flag, never a block. ++ * CLASSIFY the epoch class from RAM inputs only (struct g5p_r8_in). ++ * RECEIPT one TR108 line with the class and the generation, printed ++ * before the ladder makes its first AIC or R1 access. ++ * (ladder) g5p_td_run(FINAL) through a callback, unchanged. ++ * SLOTS three named post-ladder slots. Each runs only with a glue op ++ * present AND its own parameter on; every skip is named: ++ * fw-sleep validated firmware sleep before stable-off (G2: ++ * no recovered command; the glue has no op) ++ * stable-off one R1 read of isStableOff after a verified ++ * Fender stop leg (G6: default off) ++ * pmgr-gate outgoing PMGR power gate (G1: not implemented, ++ * there is no op and no parameter) ++ * RETIRE the retired latch: from here on g5p_r8_mmio_ok() refuses R0, ++ * R1 and every write except the always-on AIC ack of a late ++ * handler entry. The latch is set whatever the verdict. ++ * ++ * Generations: gen counts real runs (CAPTURE and FINAL) since boot and is ++ * monotonic; a REENTRY (a second FINAL after retire, e.g. notifier then ++ * .shutdown then remove) prints a line but does not advance it. ++ * ++ * Pure policy, like teardown.h: no register access, no kernel API except ++ * cmpxchg(), READ_ONCE()/WRITE_ONCE(), scnprintf() and the errno names. The ++ * same file is compiled by the kernel glue (teardown.inc) and by the offline ++ * tests (patches/gpu-recovery108/tests). Needs teardown.h first. ++ */ ++ ++#define G5P_R8_CLOSED 0x80000000u /* admission word: closed bit */ ++#define G5P_R8_INSIDE 0x7fffffffu /* admission word: submitters inside */ ++#define G5P_R8_POLL_US 100 ++#define G5P_R8_MAX_STEPS 512 /* ADMIT wait cap, independent of the clock */ ++#define G5P_R8_ADMIT_US 20000 /* ADMIT wait budget */ ++ ++/* Admission: one word, closed bit + count of submitters inside the window. */ ++struct g5p_r8_admit { ++ u32 word; ++}; ++ ++/* A submitter enters before its first check of driver state and leaves on ++ * every return path. False once admission is closed: the submit must fail ++ * with -ESHUTDOWN and touch nothing. ++ */ ++static inline bool g5p_r8_admit_enter(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if ((cur & G5P_R8_CLOSED) || ++ (cur & G5P_R8_INSIDE) == G5P_R8_INSIDE) ++ return false; ++ old = cmpxchg(&a->word, cur, cur + 1); ++ if (old == cur) ++ return true; ++ cur = old; ++ } ++} ++ ++static inline void g5p_r8_admit_exit(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if (!(cur & G5P_R8_INSIDE)) ++ return; /* unbalanced exit: never wrap into the closed bit */ ++ old = cmpxchg(&a->word, cur, cur - 1); ++ if (old == cur) ++ return; ++ cur = old; ++ } ++} ++ ++/* Close (idempotent). Returns the submitters inside at the instant of closing. */ ++static inline u32 g5p_r8_admit_close(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ old = cmpxchg(&a->word, cur, cur | G5P_R8_CLOSED); ++ if (old == cur) ++ return cur & G5P_R8_INSIDE; ++ cur = old; ++ } ++} ++ ++static inline u32 g5p_r8_admit_inside(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_INSIDE; ++} ++ ++static inline bool g5p_r8_admit_closed(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_CLOSED; ++} ++ ++/* Epoch classes. The dirty ones are never reported as clean (H10). */ ++enum g5p_r8_class { ++ G5P_R8C_NEVER_BOOTED, /* shared pages never mapped */ ++ G5P_R8C_UNRELEASED, /* mapped, firmware never released */ ++ G5P_R8C_IDLE, /* released and ALIVE, no context ever claimed (not-run) */ ++ G5P_R8C_CLEAN, /* claimed, every job signalled, no holder left */ ++ G5P_R8C_HELD, /* clean, but a process still holds the claimed context */ ++ G5P_R8C_PARKED, /* GPC=0 already verified by a park */ ++ G5P_R8C_IN_FLIGHT, /* a job accepted and not signalled, or a worker inside a wait */ ++ G5P_R8C_TIMEOUT, /* terminal latch = timeout (a late completion stays a timeout) */ ++ G5P_R8C_FAULT, /* failed, event failure or a fault terminal */ ++ G5P_R8C_ABORTED, /* bootstrap power abort latched (the r87b/C3 epoch) */ ++ G5P_R8C_UNKNOWN, /* released, never ALIVE; or an unverified earlier stop */ ++ G5P_R8C_N, ++}; ++ ++static const char *const g5p_r8_class_name[G5P_R8C_N] = { ++ "never-booted", "unreleased", "idle", "clean", "held", "parked", ++ "in-flight", "timeout", "fault", "aborted", "unknown", ++}; ++ ++static inline bool g5p_r8_class_dirty(enum g5p_r8_class c) ++{ ++ return c >= G5P_R8C_IN_FLIGHT; ++} ++ ++/* RAM-only inputs, read by the glue before the run (never from a register). */ ++struct g5p_r8_in { ++ bool booted; /* b->maps_ready */ ++ bool released; /* b->released */ ++ bool alive; /* b->alive before FREEZE, or ALIVE was proven this boot */ ++ bool failed; /* b->failed */ ++ bool fault_event; /* b->event_failed */ ++ int terminal; /* teardown terminal latch: 0 none, 1 complete, 2 timeout/fault */ ++ bool claimed; /* a context claimed the device (DRM quarantined_context) */ ++ bool held; /* ... and a render node file still references it */ ++ u32 queued; /* accepted render jobs not yet signalled */ ++ bool worker_busy; /* the command lock was busy: a worker may be inside a wait */ ++ bool parked; /* b->parked */ ++ bool stopped_unparked; /* b->stopped && !b->parked before this run */ ++ bool power_abort; /* r73b early_power_abort */ ++}; ++ ++static inline u32 g5p_r8_in_bits(const struct g5p_r8_in *i) ++{ ++ return i->booted | i->released << 1 | i->alive << 2 | i->failed << 3 | ++ i->fault_event << 4 | i->claimed << 5 | i->held << 6 | ++ (i->queued != 0) << 7 | i->worker_busy << 8 | i->parked << 9 | ++ i->stopped_unparked << 10 | i->power_abort << 11; ++} ++ ++/* First match wins; the order is the precedence (docs section 3.2). */ ++static inline enum g5p_r8_class g5p_r8_classify(const struct g5p_r8_in *i) ++{ ++ if (!i->booted) ++ return G5P_R8C_NEVER_BOOTED; ++ if (!i->released) ++ return G5P_R8C_UNRELEASED; ++ if (i->stopped_unparked) ++ return G5P_R8C_UNKNOWN; ++ if (i->power_abort) ++ return G5P_R8C_ABORTED; ++ if (i->failed || i->fault_event) ++ return G5P_R8C_FAULT; ++ if (i->terminal == 2) ++ return G5P_R8C_TIMEOUT; ++ if (i->queued || i->worker_busy) ++ return G5P_R8C_IN_FLIGHT; ++ if (i->parked) ++ return G5P_R8C_PARKED; ++ if (!i->alive) ++ return G5P_R8C_UNKNOWN; ++ if (i->claimed) ++ return i->held ? G5P_R8C_HELD : G5P_R8C_CLEAN; ++ return G5P_R8C_IDLE; ++} ++ ++enum g5p_r8_slot { ++ G5P_R8S_FW_SLEEP, ++ G5P_R8S_STABLE_OFF, ++ G5P_R8S_PMGR_GATE, ++ G5P_R8S_N, ++}; ++ ++enum g5p_r8_skip { ++ G5P_R8K_RAN, /* the slot ran (see its result) */ ++ G5P_R8K_NO_MECHANISM, /* the glue has no op for it */ ++ G5P_R8K_OFF, /* its parameter is off (default) */ ++ G5P_R8K_NOT_FINAL, /* CAPTURE never runs a slot */ ++ G5P_R8K_NEEDS_RT1, /* needs a verified Fender stop leg in this run */ ++ G5P_R8K_DIRTY, /* the ladder ended dirty: no register access */ ++ G5P_R8K_N, ++}; ++ ++static const char *const g5p_r8_skip_name[G5P_R8K_N] = { ++ "ran", "no-mechanism", "off", "not-final", "needs-rt1", "dirty", ++}; ++ ++#define G5P_R8F_ADMIT_OPEN \ ++ BIT(0) /* submitters still inside at the ADMIT deadline */ ++#define G5P_R8F_ADMIT_LATE BIT(1) /* ... or the step cap ended the wait */ ++#define G5P_R8F_REENTRY BIT(2) /* answered after retire: nothing ran */ ++#define G5P_R8F_BUSY BIT(3) /* a concurrent entrant was inside */ ++#define G5P_R8F_SLOT_FAIL BIT(4) /* a slot ran and failed */ ++#define G5P_R8F_STABLE_OFF BIT(5) /* stable-off slot read isStableOff == 1 */ ++/* jobs still queued at DETACH (abandoned to their worker) */ ++#define G5P_R8F_DETACH_OPEN BIT(6) ++ ++/* ++ * Ops. td_run runs the recovery98 ladder and returns its rc; td_flags/td_verdict ++ * read its result. slot[] may hold NULL (no mechanism). Only slot ops touch a ++ * register, and only after the ladder, never after RETIRE. ++ */ ++struct g5p_r8_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); ++ int (*td_run)(void *ctx, enum g5p_td_mode mode); ++ u32 (*td_flags)(void *ctx); ++ enum g5p_td_verdict (*td_verdict)(void *ctx); ++ int (*slot[G5P_R8S_N])(void *ctx, u32 *value); ++ void (*emit)(void *ctx, const char *line); ++}; ++ ++struct g5p_r8 { ++ struct g5p_r8_admit admit; ++ u32 admit_us; /* ADMIT budget */ ++ bool slot_on[G5P_R8S_N]; /* the slots' parameters (default all off) */ ++ /* latches */ ++ int running; ++ int retired; /* 0, or the generation that retired the device */ ++ u32 gen; /* monotonic: real runs since boot */ ++ /* last run */ ++ enum g5p_td_mode mode; ++ enum g5p_r8_class cls; ++ u32 in_bits, flags, inside0, inside1, steps, detach_queued; ++ enum g5p_r8_skip skip[G5P_R8S_N]; ++ int slot_rc[G5P_R8S_N]; ++ u32 slot_val[G5P_R8S_N]; ++ int rc; ++ enum g5p_td_verdict verdict; ++ u32 lines; /* TR108 lines emitted since boot */ ++}; ++ ++static inline void g5p_r8_init(struct g5p_r8 *s) ++{ ++ *s = (struct g5p_r8){ .admit_us = G5P_R8_ADMIT_US }; ++} ++ ++/* ++ * The predicate for every register access outside the ladder: after RETIRE only ++ * RAM, a PMGR read and the AIC (the always-on swirq ack of a late handler entry) ++ * remain. R0 and R1 are refused forever, whatever the verdict. ++ */ ++static inline bool g5p_r8_mmio_ok(const struct g5p_r8 *s, enum g5p_td_dom d) ++{ ++ if (d == G5P_TDD_RAM || d == G5P_TDD_PMGR || d == G5P_TDD_AIC) ++ return true; ++ return !READ_ONCE(s->retired); ++} ++ ++static inline bool g5p_r8_retired(const struct g5p_r8 *s) ++{ ++ return READ_ONCE(s->retired) != 0; ++} ++ ++/* The DRM frontend's DETACH callback reports what it left to the job workers. */ ++static inline void g5p_r8_note_detach(struct g5p_r8 *s, u32 queued) ++{ ++ WRITE_ONCE(s->detach_queued, queued); ++ if (queued) ++ s->flags |= G5P_R8F_DETACH_OPEN; ++} ++ ++static inline void g5p_r8_line(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ const char *line) ++{ ++ s->lines++; ++ if (o->emit) ++ o->emit(o->ctx, line); ++} ++ ++/* Bounded RAM-only wait for submitters inside the admission window. */ ++static inline void g5p_r8_admit_wait(struct g5p_r8 *s, ++ const struct g5p_r8_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)s->admit_us * 1000; ++ ++ s->inside0 = g5p_r8_admit_close(&s->admit); ++ s->inside1 = s->inside0; ++ s->steps = 0; ++ while (s->inside1) { ++ if (s->steps >= G5P_R8_MAX_STEPS) { ++ s->flags |= G5P_R8F_ADMIT_LATE; ++ break; ++ } ++ s->steps++; ++ if (o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_R8_POLL_US); ++ s->inside1 = g5p_r8_admit_inside(&s->admit); ++ } ++ if (s->inside1) ++ s->flags |= G5P_R8F_ADMIT_OPEN; ++} ++ ++static inline void g5p_r8_slots(struct g5p_r8 *s, const struct g5p_r8_ops *o) ++{ ++ u32 tdf = o->td_flags ? o->td_flags(o->ctx) : 0; ++ unsigned int i; ++ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ s->slot_rc[i] = 0; ++ s->slot_val[i] = 0; ++ if (s->mode != G5P_TD_FINAL) ++ s->skip[i] = G5P_R8K_NOT_FINAL; ++ else if (!o->slot[i]) ++ s->skip[i] = G5P_R8K_NO_MECHANISM; ++ else if (!s->slot_on[i]) ++ s->skip[i] = G5P_R8K_OFF; ++ else if (s->verdict == G5P_TDV_DIRTY) ++ s->skip[i] = G5P_R8K_DIRTY; ++ else if (!(tdf & G5P_TDF_RT1_OK)) ++ s->skip[i] = G5P_R8K_NEEDS_RT1; ++ else ++ s->skip[i] = G5P_R8K_RAN; ++ } ++ /* One attempt each, in slot order, no retry; a failed slot ends the slots. */ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ if (s->skip[i] != G5P_R8K_RAN) ++ continue; ++ if (s->flags & G5P_R8F_SLOT_FAIL) { ++ s->skip[i] = G5P_R8K_DIRTY; ++ continue; ++ } ++ s->slot_rc[i] = o->slot[i](o->ctx, &s->slot_val[i]); ++ if (s->slot_rc[i]) ++ s->flags |= G5P_R8F_SLOT_FAIL; ++ else if (i == G5P_R8S_STABLE_OFF && s->slot_val[i] == 1) ++ s->flags |= G5P_R8F_STABLE_OFF; ++ } ++} ++ ++/* ++ * One run. mode is the teardown mode the glue chose (teardown=1 CAPTURE, ++ * teardown=2 FINAL). Returns the ladder's rc, -EBUSY for a concurrent entrant, ++ * and the cached rc after retire. Lines: ++ * TR108 gen= phase=classify why= mode= class= dirty= in= inside= after= flags= admit_us= ++ * (TD98 lines of the ladder, unchanged) ++ * TR108 gen= phase=retired|captured verdict= class= flags= fw-sleep= stable-off= pmgr-gate= ++ * so= detach_queued= rc= ++ * TR108 gen= phase=reentry why= retired_gen= rc= (after retire; no gen step) ++ */ ++static inline int g5p_r8_run(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ enum g5p_td_mode mode, const struct g5p_r8_in *in, ++ const char *why) ++{ ++ char line[256]; ++ u64 t0; ++ int rc; ++ ++ if (cmpxchg(&s->running, 0, 1)) { ++ /* A concurrent entrant touches nothing; it does not reach the ladder either. */ ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=busy why=%s mode=%s rc=%d", ++ READ_ONCE(s->gen), why, g5p_td_mode_name[mode], ++ -EBUSY); ++ if (o->emit) ++ o->emit(o->ctx, line); ++ return -EBUSY; ++ } ++ if (READ_ONCE(s->retired)) { ++ /* The ladder answers from its cache too (REENTRY); nothing else runs. */ ++ rc = o->td_run(o->ctx, mode); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=reentry why=%s mode=%s retired_gen=%d rc=%d", ++ s->gen, why, g5p_td_mode_name[mode], ++ READ_ONCE(s->retired), rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++ } ++ s->gen++; ++ s->mode = mode; ++ s->flags = 0; ++ s->inside0 = 0; ++ s->inside1 = 0; ++ s->steps = 0; ++ WRITE_ONCE(s->detach_queued, 0); ++ t0 = o->now_ns(o->ctx); ++ if (mode == G5P_TD_FINAL) ++ g5p_r8_admit_wait(s, o); ++ s->cls = g5p_r8_classify(in); ++ s->in_bits = g5p_r8_in_bits(in); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=classify why=%s mode=%s class=%s dirty=%u in=%03x term=%d queued=%u inside=%u after=%u flags=%02x admit_us=%u", ++ s->gen, why, g5p_td_mode_name[mode], g5p_r8_class_name[s->cls], ++ g5p_r8_class_dirty(s->cls), s->in_bits, in->terminal, ++ in->queued, s->inside0, s->inside1, s->flags, ++ (u32)((o->now_ns(o->ctx) - t0) / 1000)); ++ /* The receipt is out before the ladder's first AIC or R1 access. */ ++ g5p_r8_line(s, o, line); ++ s->rc = o->td_run(o->ctx, mode); ++ s->verdict = o->td_verdict ? o->td_verdict(o->ctx) : G5P_TDV_NONE; ++ g5p_r8_slots(s, o); ++ if (mode == G5P_TD_FINAL) ++ WRITE_ONCE(s->retired, (int)s->gen); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=%s verdict=%s class=%s flags=%02x fw-sleep=%s stable-off=%s pmgr-gate=%s so=%u detach_queued=%u rc=%d", ++ s->gen, mode == G5P_TD_FINAL ? "retired" : "captured", ++ g5p_td_verdict_name[s->verdict], g5p_r8_class_name[s->cls], ++ s->flags, g5p_r8_skip_name[s->skip[G5P_R8S_FW_SLEEP]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_STABLE_OFF]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_PMGR_GATE]], ++ s->slot_val[G5P_R8S_STABLE_OFF], READ_ONCE(s->detach_queued), ++ s->rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.h b/drivers/gpu/apple-g5p/teardown.h +new file mode 100644 +index 0000000000..b536f428cd +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.h +@@ -0,0 +1,740 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_TEARDOWN_H ++#define G5P_TEARDOWN_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P GPU teardown state machine (series gpu-recovery98). ++ * ++ * Design and evidence: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md. ++ * ++ * Pure policy: no register access, no kernel API except cmpxchg(), scnprintf() ++ * and the errno names. Every hardware or firmware-visible access goes through ++ * struct g5p_td_ops, and every call is checked against g5p_td_allow[phase][domain] ++ * before it is made. The same file is compiled by the kernel glue (teardown.inc) ++ * and by the offline fault-injection tests (patches/gpu-recovery/tests). ++ * ++ * Two modes: ++ * CAPTURE timeout / fault / attach failure, and (gpu-recovery99) any number of looks at a ++ * live epoch. Observation only: reads RAM rings, ++ * changes nothing the host could still want to export. Never writes ++ * driver or firmware-shared state, never masks the IRQ, never touches ++ * a register. Arms the quiet latch for diagnostics (g5p_td_mmio_ok()). ++ * FINAL shutdown / reboot notifier / remove. Runs the whole ladder and leaves ++ * a known state for the next kernel. ++ * ++ * Domains: RAM (DMA pages + driver structs), AIC (swirq 0x20e104000 ack and the ++ * IRQ-chip mask), R1 (Fender wrapper window), R0 (core register window), PMGR ++ * (0x20e080340 read-only). R0 has no op at all and is forbidden in every phase. ++ * PMGR is read-only in every phase. R1 is reachable only in the RESET phase. ++ * ++ * Nothing here frees or reuses memory the GPU might still access. ++ */ ++ ++#define G5P_TD_NCMD 4 /* command rings: shared objects 0x60 0x64 0x68 0x6c */ ++#define G5P_TD_NRING (G5P_TD_NCMD + 1) /* + event ring (object 0x70) */ ++#define G5P_TD_MAX_RCPT 12 ++#define G5P_TD_MAX_STEPS 4096 /* hard stop independent of the clock */ ++#define G5P_TD_POLL_US 100 ++#define G5P_TD_EV_SLACK \ ++ 16 /* events produced during the phase above this = storm */ ++ ++enum g5p_td_mode { ++ G5P_TD_CAPTURE = 1, ++ G5P_TD_FINAL = 2, ++}; ++ ++enum g5p_td_phase { ++ G5P_TDP_FREEZE, /* stopped=1 alive=0, watchdog disarmed (RAM) */ ++ G5P_TDP_ACCOUNT, /* RAM-only ring/event accounting */ ++ G5P_TDP_DRAIN, /* bounded RAM-only wait for the firmware to self-drain */ ++ G5P_TDP_EVENTS, /* bounded RAM-only event-ring quiesce */ ++ G5P_TDP_IRQ, /* AIC ack, IRQ mask, settle */ ++ G5P_TDP_DETACH, /* host detach, RAM only */ ++ G5P_TDP_RESET, /* RT1 Fender stop leg when eligible; PMGR read */ ++ G5P_TDP_DONE, ++ G5P_TDP_N, ++}; ++ ++enum g5p_td_dom { ++ G5P_TDD_RAM, ++ G5P_TDD_AIC, ++ G5P_TDD_R1, ++ G5P_TDD_R0, ++ G5P_TDD_PMGR, ++ G5P_TDD_N, ++}; ++ ++#define G5P_TDA_R 1 ++#define G5P_TDA_W 2 ++#define G5P_TDA_RW 3 ++ ++/* Hard stop boundaries. A phase may only use the domains marked here. */ ++static const unsigned char g5p_td_allow[G5P_TDP_N][G5P_TDD_N] = { ++ /* RAM AIC R1 R0 PMGR */ ++ [G5P_TDP_FREEZE] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_ACCOUNT] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_DRAIN] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_EVENTS] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_IRQ] = { G5P_TDA_R, G5P_TDA_W, 0, 0, 0 }, ++ [G5P_TDP_DETACH] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_RESET] = { G5P_TDA_R, 0, G5P_TDA_RW, 0, G5P_TDA_R }, ++ [G5P_TDP_DONE] = { G5P_TDA_R, 0, 0, 0, 0 }, ++}; ++ ++#define G5P_TDF_NOT_BOOTED \ ++ BIT(0) /* shared pages not mapped: nothing to tear down */ ++#define G5P_TDF_UNRELEASED BIT(1) /* firmware never released */ ++#define G5P_TDF_PROTOCOL \ ++ BIT(2) /* ring index > 255, read error, producer moved */ ++#define G5P_TDF_TERMINAL BIT(3) /* TA complete/timeout latched before entry */ ++#define G5P_TDF_LATE_TERMINAL BIT(4) /* ... or during the teardown (race) */ ++#define G5P_TDF_ABANDON BIT(5) /* >=1 ring pending after the drain */ ++#define G5P_TDF_SELF_DRAINED \ ++ BIT(6) /* >=1 ring consumed by the firmware, no bell */ ++#define G5P_TDF_FAULT_EVENT \ ++ BIT(7) /* event type other than 0/1 left unconsumed */ ++#define G5P_TDF_EV_UNQUIET \ ++ BIT(8) /* event ring still non-empty after the budget */ ++#define G5P_TDF_IRQ_UNSETTLED BIT(9) /* IRQ count still moving after the mask */ ++#define G5P_TDF_RT1_OK BIT(10) /* Fender stop leg verified GPC=0 */ ++#define G5P_TDF_RT1_FAIL BIT(11) /* stop leg attempted, not verified */ ++#define G5P_TDF_RT1_SKIPPED BIT(12) /* no register access; see skip reason */ ++#define G5P_TDF_OVERRUN BIT(13) /* step or time budget exhausted */ ++#define G5P_TDF_VIOLATION BIT(14) /* a gate refused an access (machine bug) */ ++#define G5P_TDF_CAPTURE_ONLY BIT(15) /* CAPTURE run: nothing was changed */ ++#define G5P_TDF_REENTRY BIT(16) /* second entry answered from the cache */ ++ ++enum g5p_td_skip { ++ G5P_TDS_NONE, ++ G5P_TDS_NOT_BOOTED, ++ G5P_TDS_NOT_RELEASED, ++ G5P_TDS_NOT_POWERED, ++ G5P_TDS_ALREADY_PARKED, /* clean: GPC=0 already verified by park */ ++ G5P_TDS_STOPPED_UNPARKED, /* dirty: an earlier stop leg did not verify */ ++ G5P_TDS_POWER_ABORT, ++ G5P_TDS_FW_POLICY, /* meta+0x6c8 != 0: firmware also writes R1+0x1000 */ ++ G5P_TDS_LOCK_BUSY, ++ G5P_TDS_CAPTURE_MODE, ++ G5P_TDS_BAD_STATE, /* protocol/overrun/unsettled IRQ: no register access */ ++ G5P_TDS_N, ++}; ++ ++enum g5p_td_verdict { ++ G5P_TDV_NONE, ++ G5P_TDV_OBSERVED, /* CAPTURE: nothing changed */ ++ G5P_TDV_UNTOUCHED, /* never booted or never released: nothing to do */ ++ G5P_TDV_QUIESCED, /* rings clear, events clear, IRQ masked, GPC=0 verified */ ++ G5P_TDV_ABANDONED, /* as QUIESCED, but >=1 command abandoned after the drain */ ++ G5P_TDV_RAM_ONLY, /* IRQ masked, no register access: GPU left as found */ ++ G5P_TDV_DIRTY, /* a step failed: warm continue forbidden, cold DFU */ ++ G5P_TDV_N, ++}; ++ ++static const char *const g5p_td_phase_name[G5P_TDP_N] = { ++ "freeze", "account", "drain", "events", ++ "irq", "detach", "reset", "done", ++}; ++ ++static const char *const g5p_td_mode_name[3] = { "none", "capture", "final" }; ++ ++static const char *const g5p_td_verdict_name[G5P_TDV_N] = { ++ "none", "observed", "untouched", "quiesced", ++ "abandoned", "ram-only", "dirty", ++}; ++ ++struct g5p_td_budget { ++ u32 drain_us; /* self-drain wait; DC3 took 2.34 ms (r87b) */ ++ u32 events_us; ++ u32 settle_us; /* IRQ count must stay constant this long after the mask */ ++ u32 events_pass_max; /* entries per pass; the ring has 256 slots */ ++ u32 events_passes; ++ u32 total_us; /* RESET is skipped when the RAM phases already overran this */ ++}; ++ ++#define G5P_TD_BUDGET_DEFAULT { 20000, 20000, 2000, 256, 4, 100000 } ++ ++/* Inputs the glue reads from RAM before the run (never from a register). */ ++struct g5p_td_pre { ++ bool booted; /* maps_ready: shared pages exist */ ++ bool released; /* firmware was released */ ++ bool powered; /* ps_gfx held on by the retained PM reference */ ++ bool fender_ready; ++ bool already_parked; /* b->parked: GPC=0 verified by park */ ++ bool stopped_unparked; /* b->stopped && !b->parked: an earlier stop did not verify */ ++ bool policy_host_owned; /* init meta+0x6c8 == 0 */ ++ bool power_abort; /* r73b early_power_abort latched */ ++ bool lock_held; /* the glue holds g->lock: no worker is inside a hardware wait */ ++}; ++ ++/* ++ * Every op is bounded by its own contract; the machine additionally bounds how ++ * often and for how long it calls them. Ops marked RAM never touch a register. ++ */ ++struct g5p_td_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); /* sleep, no register access */ ++ void (*freeze)(void *ctx); /* RAM: stopped=1 alive=0, disarm */ ++ /* RAM: idx 0..NCMD-1 command rings, NCMD = event ring. -ENODEV: not mapped. */ ++ int (*ring_read)(void *ctx, u32 idx, u32 *cons, u32 *prod); ++ /* RAM: consume at most max entries of type 0/1, stop at the first other type. */ ++ int (*event_pass)(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault); ++ bool (*terminal_seen)(void *ctx); /* RAM: TA complete/timeout latch */ ++ void (*detach)(void *ctx); /* RAM: signal/forget host jobs */ ++ void (*aic_ack)(void *ctx); /* AIC: swirq+0x94 = 0x4000 */ ++ void (*irq_mask)(void *ctx); /* AIC: ack stays, line masked */ ++ u32 (*irq_total)(void *ctx); /* RAM: handler entry counter */ ++ int (*fender_stop)(void *ctx); /* R1: g5p_fender_idle() */ ++ int (*pmgr_actual)(void *ctx, u32 *target, ++ u32 *actual); /* PMGR read; may be NULL */ ++ void (*emit)(void *ctx, const char *line); /* printk; may be NULL */ ++}; ++ ++struct g5p_td_receipt { ++ unsigned char seq, phase, mode, verdict; ++ s32 rc; ++ u32 us, steps, flags; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [dom][0]=reads [dom][1]=writes, op level */ ++ u32 a, b; /* phase specific, see g5p_td_run() */ ++}; ++ ++struct g5p_td { ++ struct g5p_td_budget budget; ++ /* latches shared with the render path; touched by cmpxchg only */ ++ int running; /* 0 idle, 1 inside g5p_td_run() */ ++ int terminal; /* 0, 1 complete, 2 timeout: first writer wins */ ++ int quiet; /* diagnostics MMIO forbidden from here on */ ++ /* 0 idle, 1 captured (informational), 2 finalized: only 2 answers from the cache */ ++ int state; ++ u32 snaps; /* CAPTURE runs since init: the n= of the TD99 snap line */ ++ /* per run */ ++ enum g5p_td_mode mode; ++ enum g5p_td_phase phase; ++ enum g5p_td_skip skip; ++ enum g5p_td_verdict verdict; ++ bool eligible; ++ u32 flags, steps, violations, seq, nrcpt; ++ u64 t0_ns; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [enum g5p_td_dom][0=reads,1=writes], op level */ ++ u32 pending_mask, pending_initial, abandoned, drained; ++ u32 ring_cons[G5P_TD_NRING], ring_prod[G5P_TD_NRING]; ++ u32 ev_consumed, ev_left, ev_initial, irq_before, irq_after; ++ u32 us_total; ++ u32 pmgr_target, pmgr_actual; ++ int rc; ++ struct g5p_td_receipt rcpt[G5P_TD_MAX_RCPT]; ++}; ++ ++static inline void g5p_td_init(struct g5p_td *t) ++{ ++ const struct g5p_td_budget b = G5P_TD_BUDGET_DEFAULT; ++ ++ *t = (struct g5p_td){ .budget = b }; ++} ++ ++/* One latch for "the TA reached a terminal state"; the first writer wins. */ ++static inline bool g5p_td_note_terminal(struct g5p_td *t, int kind) ++{ ++ if (cmpxchg(&t->terminal, 0, kind) != 0) ++ return false; ++ WRITE_ONCE(t->quiet, 1); ++ return true; ++} ++ ++/* ++ * The single predicate for diagnostic register access (R0 snapshots, end banks, ++ * exports): false once the epoch is terminal or a teardown has started. Nothing ++ * in this series calls a diagnostic; this exists so the render path can. ++ */ ++ ++/* RT1 (Fender stop leg) eligibility. Pure function of RAM-derived inputs. */ ++static inline bool g5p_td_eligible(const struct g5p_td_pre *p, ++ enum g5p_td_mode mode, enum g5p_td_skip *why) ++{ ++ enum g5p_td_skip s = G5P_TDS_NONE; ++ ++ if (!p->booted) ++ s = G5P_TDS_NOT_BOOTED; ++ else if (!p->released) ++ s = G5P_TDS_NOT_RELEASED; ++ else if (!p->powered || !p->fender_ready) ++ s = G5P_TDS_NOT_POWERED; ++ else if (p->already_parked) ++ s = G5P_TDS_ALREADY_PARKED; ++ else if (p->stopped_unparked) ++ s = G5P_TDS_STOPPED_UNPARKED; ++ else if (p->power_abort) ++ s = G5P_TDS_POWER_ABORT; ++ else if (!p->policy_host_owned) ++ s = G5P_TDS_FW_POLICY; ++ else if (!p->lock_held) ++ s = G5P_TDS_LOCK_BUSY; ++ else if (mode != G5P_TD_FINAL) ++ s = G5P_TDS_CAPTURE_MODE; ++ if (why) ++ *why = s; ++ return s == G5P_TDS_NONE; ++} ++ ++static inline int g5p_td_format(const struct g5p_td_receipt *r, char *buf, ++ size_t n) ++{ ++ return scnprintf(buf, n, ++ "TD98 seq=%u phase=%s mode=%s rc=%d us=%u steps=%u flags=%08x aic=%u/%u r1=%u/%u r0=%u/%u pmgr=%u/%u a=%u b=%u verdict=%s", ++ r->seq, g5p_td_phase_name[r->phase], g5p_td_mode_name[r->mode], ++ r->rc, r->us, r->steps, r->flags, r->cnt[G5P_TDD_AIC][0], ++ r->cnt[G5P_TDD_AIC][1], r->cnt[G5P_TDD_R1][0], ++ r->cnt[G5P_TDD_R1][1], r->cnt[G5P_TDD_R0][0], ++ r->cnt[G5P_TDD_R0][1], r->cnt[G5P_TDD_PMGR][0], ++ r->cnt[G5P_TDD_PMGR][1], r->a, r->b, ++ g5p_td_verdict_name[r->verdict]); ++} ++ ++static inline u32 g5p_td_us_since(const struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ u64 d = o->now_ns(o->ctx) - t->t0_ns; ++ ++ return d > 0xffffffffULL * 1000 ? 0xffffffffu : (u32)(d / 1000); ++} ++ ++static inline void g5p_td_receipt(struct g5p_td *t, const struct g5p_td_ops *o, ++ int rc, u32 a, u32 b) ++{ ++ struct g5p_td_receipt *r = ++ &t->rcpt[t->nrcpt < G5P_TD_MAX_RCPT ? t->nrcpt : ++ G5P_TD_MAX_RCPT - 1]; ++ char line[224]; ++ unsigned int d; ++ ++ r->seq = (unsigned char)t->seq++; ++ r->phase = t->phase; ++ r->mode = t->mode; ++ r->verdict = t->verdict; ++ r->rc = rc; ++ r->us = g5p_td_us_since(t, o); ++ r->steps = t->steps; ++ r->flags = t->flags; ++ for (d = 0; d < G5P_TDD_N; d++) { ++ r->cnt[d][0] = t->cnt[d][0]; ++ r->cnt[d][1] = t->cnt[d][1]; ++ } ++ r->a = a; ++ r->b = b; ++ if (t->nrcpt < G5P_TD_MAX_RCPT) ++ t->nrcpt++; ++ if (o->emit) { ++ g5p_td_format(r, line, sizeof(line)); ++ o->emit(o->ctx, line); ++ } ++} ++ ++/* Count and check one access. A refused access is a bug in the machine itself. */ ++static inline int g5p_td_gate(struct g5p_td *t, enum g5p_td_dom d, ++ unsigned int acc) ++{ ++ if ((g5p_td_allow[t->phase][d] & acc) != acc) { ++ t->violations++; ++ t->flags |= G5P_TDF_VIOLATION; ++ return -EACCES; ++ } ++ if ((acc & G5P_TDA_R) && t->cnt[d][0] < 255) ++ t->cnt[d][0]++; ++ if ((acc & G5P_TDA_W) && t->cnt[d][1] < 255) ++ t->cnt[d][1]++; ++ return 0; ++} ++ ++/* One loop iteration or hardware op. False once the hard step cap is hit. */ ++static inline bool g5p_td_step(struct g5p_td *t) ++{ ++ if (t->steps >= G5P_TD_MAX_STEPS) { ++ t->flags |= G5P_TDF_OVERRUN; ++ return false; ++ } ++ t->steps++; ++ return true; ++} ++ ++static inline bool g5p_td_ring_ok(u32 cons, u32 prod) ++{ ++ return cons <= 255 && prod <= 255; ++} ++ ++static inline void g5p_td_note_late(struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ if (o->terminal_seen && o->terminal_seen(o->ctx) && ++ !(t->flags & G5P_TDF_TERMINAL)) ++ t->flags |= G5P_TDF_LATE_TERMINAL; ++} ++ ++static inline int g5p_td_account(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 i, bad = 0, pending = 0, cons, prod; ++ int rc; ++ ++ for (i = 0; i < G5P_TD_NRING; i++) { ++ if (!g5p_td_step(t)) ++ return -ETIMEDOUT; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return -EACCES; ++ rc = o->ring_read(o->ctx, i, &cons, &prod); ++ if (rc == -ENODEV) { ++ t->flags |= G5P_TDF_NOT_BOOTED; ++ break; ++ } ++ if (rc || !g5p_td_ring_ok(cons, prod)) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ bad++; ++ continue; ++ } ++ t->ring_cons[i] = cons; ++ t->ring_prod[i] = prod; ++ if (i < G5P_TD_NCMD && cons != prod) { ++ t->pending_mask |= 1u << i; ++ pending++; ++ } ++ } ++ t->pending_initial = pending; ++ t->ev_left = (t->ring_prod[G5P_TD_NCMD] - t->ring_cons[G5P_TD_NCMD]) & ++ 255; ++ t->ev_initial = t->ev_left; ++ g5p_td_receipt(t, o, bad ? -EPROTO : 0, pending, bad); ++ return 0; ++} ++ ++/* ++ * gpu-recovery99 0004: one extra line per CAPTURE run, after the ACCOUNT receipt, so a live ++ * epoch can be looked at ++ * several times (S1a). RAM only: it formats what ACCOUNT stored in struct g5p_td and reads the ++ * handler entry counter ++ * (RAM) and the clock; no register, no state change. The prefix is not TD98 on purpose: receipt ++ * parsers keyed on ++ * "TD98 seq=" / "TD98 begin" ignore it. ++ * n CAPTURE runs since init (1 based) ++ * t_ns ops->now_ns(), for the IRQ rate between two lines ++ * irq handler entry counter (ops->irq_total), 0 if there is no such op ++ * term terminal latch (0 none, 1 complete, 2 timeout); quiet = the diagnostics latch ++ * own bit0 booted 1 released 2 powered 3 fender_ready 4 already_parked 5 stopped_unparked ++ * 6 policy_host_owned 7 power_abort 8 lock_held ++ * cN/pN command ring N consumer / producer; ce/pe the event ring. Zero when the shared pages ++ * are not mapped. ++ */ ++static inline void g5p_td_snap(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ char line[224]; ++ u32 own, irq = 0; ++ ++ t->snaps++; ++ if (!o->emit) ++ return; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R) && o->irq_total) ++ irq = o->irq_total(o->ctx); ++ own = pre->booted | pre->released << 1 | pre->powered << 2 | ++ pre->fender_ready << 3 | pre->already_parked << 4 | ++ pre->stopped_unparked << 5 | pre->policy_host_owned << 6 | ++ pre->power_abort << 7 | pre->lock_held << 8; ++ scnprintf(line, sizeof(line), ++ "TD99 snap n=%u t_ns=%llu irq=%u term=%d quiet=%d own=%03x c0=%u p0=%u c1=%u p1=%u c2=%u p2=%u c3=%u p3=%u ce=%u pe=%u", ++ t->snaps, (unsigned long long)o->now_ns(o->ctx), irq, ++ READ_ONCE(t->terminal), READ_ONCE(t->quiet), own, ++ t->ring_cons[0], t->ring_prod[0], t->ring_cons[1], ++ t->ring_prod[1], t->ring_cons[2], t->ring_prod[2], ++ t->ring_cons[3], t->ring_prod[3], t->ring_cons[G5P_TD_NCMD], ++ t->ring_prod[G5P_TD_NCMD]); ++ o->emit(o->ctx, line); ++} ++ ++static inline void g5p_td_drain(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.drain_us * 1000; ++ u32 i, cons, prod; ++ ++ while (t->pending_mask) { ++ if (!g5p_td_step(t)) ++ break; ++ if (o->now_ns(o->ctx) >= deadline) { ++ t->flags |= G5P_TDF_ABANDON; ++ break; ++ } ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ for (i = 0; i < G5P_TD_NCMD; i++) { ++ if (!(t->pending_mask & (1u << i))) ++ continue; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return; ++ if (o->ring_read(o->ctx, i, &cons, &prod) || ++ !g5p_td_ring_ok(cons, prod) || ++ prod != t->ring_prod[i]) { ++ /* The producer is host-written and the host is frozen. */ ++ t->flags |= G5P_TDF_PROTOCOL; ++ t->pending_mask &= ~(1u << i); ++ continue; ++ } ++ if (cons == prod) { ++ t->pending_mask &= ~(1u << i); ++ t->flags |= G5P_TDF_SELF_DRAINED; ++ } ++ } ++ } ++ for (i = 0; i < G5P_TD_NCMD; i++) ++ if (t->pending_mask & (1u << i)) ++ t->abandoned++; ++ if (t->abandoned) ++ t->flags |= G5P_TDF_ABANDON; ++ t->drained = t->pending_initial - t->abandoned; ++} ++ ++static inline void g5p_td_events(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.events_us * 1000; ++ u32 pass, consumed, left = t->ev_left; ++ bool fault = false; ++ int rc; ++ ++ for (pass = 0; pass < t->budget.events_passes && left; pass++) { ++ if (!g5p_td_step(t)) ++ break; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_RW)) ++ return; ++ consumed = 0; ++ rc = o->event_pass(o->ctx, t->budget.events_pass_max, &consumed, ++ &left, &fault); ++ t->ev_consumed += consumed; ++ if (rc) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ break; ++ } ++ if (fault) { ++ /* Retained for the host; stop consuming, the IRQ is masked next. */ ++ t->flags |= G5P_TDF_FAULT_EVENT; ++ break; ++ } ++ if (!left || o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ } ++ t->ev_left = left; ++ /* ++ * A ring holds at most 255 entries, so consuming more than the backlog plus a ++ * few ALIVE ticks means the producer outran us: an empty ring at the last ++ * look is a snapshot, not a quiesce. ++ */ ++ if (!fault && (left || (s32)(t->ev_consumed + left - t->ev_initial) > ++ G5P_TD_EV_SLACK)) ++ t->flags |= G5P_TDF_EV_UNQUIET; ++} ++ ++static inline void g5p_td_irq(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 tries, n0, n1; ++ ++ if (g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W) || ++ g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W)) ++ return; ++ t->irq_before = o->irq_total(o->ctx); ++ /* Same pair as the IRQ-cap path: ack the software source, then mask the line. */ ++ o->aic_ack(o->ctx); ++ o->irq_mask(o->ctx); ++ for (tries = 0; tries < 3; tries++) { ++ if (!g5p_td_step(t)) ++ break; ++ n0 = o->irq_total(o->ctx); ++ o->relax_us(o->ctx, t->budget.settle_us); ++ n1 = o->irq_total(o->ctx); ++ t->irq_after = n1; ++ if (n0 == n1) ++ return; ++ } ++ t->flags |= G5P_TDF_IRQ_UNSETTLED; ++} ++ ++static inline void g5p_td_reset(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ enum g5p_td_skip why = G5P_TDS_NONE; ++ bool bad = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_VIOLATION); ++ int rc = 0; ++ ++ t->eligible = g5p_td_eligible(pre, t->mode, &why); ++ if (t->eligible && ++ (bad || g5p_td_us_since(t, o) > t->budget.total_us)) { ++ t->eligible = false; ++ why = G5P_TDS_BAD_STATE; ++ } ++ t->skip = why; ++ if (!t->eligible) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ } else if (!g5p_td_step(t) || g5p_td_gate(t, G5P_TDD_R1, G5P_TDA_RW)) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ t->skip = G5P_TDS_BAD_STATE; ++ } else { ++ /* ++ * Apple stopFirmwareForPowerDown 6c9fa10, R1 only; flown here after a DC ++ * timeout (round 10). Never retried, even on failure. ++ */ ++ rc = o->fender_stop(o->ctx); ++ t->flags |= rc ? G5P_TDF_RT1_FAIL : G5P_TDF_RT1_OK; ++ } ++ if (o->pmgr_actual && !g5p_td_gate(t, G5P_TDD_PMGR, G5P_TDA_R)) ++ o->pmgr_actual(o->ctx, &t->pmgr_target, &t->pmgr_actual); ++ g5p_td_receipt(t, o, rc, t->eligible ? 0 : (u32)t->skip, ++ t->pmgr_actual); ++} ++ ++static inline void g5p_td_decide(struct g5p_td *t, const struct g5p_td_pre *pre) ++{ ++ bool dirty = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_EV_UNQUIET | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_RT1_FAIL | ++ G5P_TDF_OVERRUN | G5P_TDF_VIOLATION) || ++ t->skip == G5P_TDS_STOPPED_UNPARKED || ++ t->skip == G5P_TDS_BAD_STATE; ++ ++ if (t->mode == G5P_TD_CAPTURE) ++ t->verdict = (t->flags & G5P_TDF_VIOLATION) ? G5P_TDV_DIRTY : ++ G5P_TDV_OBSERVED; ++ else if (dirty) ++ t->verdict = G5P_TDV_DIRTY; ++ else if (t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED)) ++ t->verdict = G5P_TDV_UNTOUCHED; ++ else if (t->flags & G5P_TDF_RT1_OK || t->skip == G5P_TDS_ALREADY_PARKED) ++ t->verdict = (t->flags & G5P_TDF_ABANDON) ? G5P_TDV_ABANDONED : ++ G5P_TDV_QUIESCED; ++ else ++ t->verdict = G5P_TDV_RAM_ONLY; ++ (void)pre; ++} ++ ++/* ++ * Run the teardown once per mode. Returns 0 for QUIESCED/ABANDONED/UNTOUCHED/OBSERVED, ++ * -EAGAIN for RAM_ONLY (clean, but the GPU was left as found), -EIO for DIRTY, ++ * -EBUSY if another caller is inside the machine. Every phase ends in a receipt. ++ * Receipt a/b: account pending/bad, drain drained/abandoned, events consumed/left, ++ * irq before/after, reset skip-reason/pmgr-actual, done skip-reason/abandoned. ++ * Counters in the receipts are op-level (one count per gate): the AIC pair is ++ * 2 writes, the whole Fender stop leg is one R1 read+write op. ++ */ ++static inline int g5p_td_run(struct g5p_td *t, const struct g5p_td_ops *o, ++ enum g5p_td_mode mode, ++ const struct g5p_td_pre *pre) ++{ ++ if (cmpxchg(&t->running, 0, 1)) ++ return -EBUSY; ++ WRITE_ONCE(t->quiet, 1); ++ if (t->state == 2) { ++ /* Answered from the cache (after FINAL only; CAPTURE re-runs, gpu- ++ * recovery99 0004): no phase runs, nothing is accessed. ++ */ ++ u32 saved_flags = t->flags; ++ ++ t->t0_ns = o->now_ns(o->ctx) - (u64)t->us_total * 1000; ++ t->flags |= G5P_TDF_REENTRY; ++ t->phase = G5P_TDP_DONE; ++ g5p_td_receipt(t, o, t->rc, t->flags, t->steps); ++ t->flags = saved_flags; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++ } ++ t->mode = mode; ++ t->phase = G5P_TDP_FREEZE; ++ t->t0_ns = o->now_ns(o->ctx); ++ t->seq = 0; ++ t->nrcpt = 0; ++ t->flags = mode == G5P_TD_CAPTURE ? G5P_TDF_CAPTURE_ONLY : 0; ++ t->steps = 0; ++ t->violations = 0; ++ t->abandoned = 0; ++ t->drained = 0; ++ t->pending_mask = 0; ++ t->pending_initial = 0; ++ t->ev_consumed = 0; ++ t->ev_left = 0; ++ t->ev_initial = 0; ++ t->irq_before = 0; ++ t->irq_after = 0; ++ t->skip = G5P_TDS_NONE; ++ t->verdict = G5P_TDV_NONE; ++ memset(t->cnt, 0, sizeof(t->cnt)); ++ /* a CAPTURE may now run many times: nothing of the previous look may leak into this one */ ++ memset(t->ring_cons, 0, sizeof(t->ring_cons)); ++ memset(t->ring_prod, 0, sizeof(t->ring_prod)); ++ if (READ_ONCE(t->terminal)) ++ t->flags |= G5P_TDF_TERMINAL; ++ if (!pre->released) ++ t->flags |= G5P_TDF_UNRELEASED; ++ ++ if (mode == G5P_TD_FINAL) { ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->freeze(o->ctx); ++ g5p_td_receipt(t, o, 0, pre->released, pre->policy_host_owned); ++ } ++ t->phase = G5P_TDP_ACCOUNT; ++ if (g5p_td_account(t, o)) ++ t->flags |= G5P_TDF_OVERRUN; ++ g5p_td_note_late(t, o); ++ if (mode == G5P_TD_CAPTURE) ++ g5p_td_snap(t, o, pre); ++ ++ if (mode == G5P_TD_FINAL && ++ !(t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED | ++ G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN))) { ++ t->phase = G5P_TDP_DRAIN; ++ g5p_td_drain(t, o); ++ g5p_td_note_late(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_ABANDON) ? -ETIMEDOUT : 0, ++ t->drained, t->abandoned); ++ ++ t->phase = G5P_TDP_EVENTS; ++ g5p_td_events(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_EV_UNQUIET) ? -ETIMEDOUT : 0, ++ t->ev_consumed, t->ev_left); ++ } ++ if (mode == G5P_TD_FINAL) { ++ /* The IRQ is masked even after an overrun: ack + mask is the safe exit. */ ++ t->phase = G5P_TDP_IRQ; ++ if (!(t->flags & G5P_TDF_NOT_BOOTED)) ++ g5p_td_irq(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_IRQ_UNSETTLED) ? -ETIMEDOUT : ++ 0, ++ t->irq_before, t->irq_after); ++ ++ t->phase = G5P_TDP_DETACH; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->detach(o->ctx); ++ g5p_td_receipt(t, o, 0, 0, 0); ++ ++ t->phase = G5P_TDP_RESET; ++ g5p_td_reset(t, o, pre); ++ } ++ g5p_td_note_late(t, o); ++ t->phase = G5P_TDP_DONE; ++ g5p_td_decide(t, pre); ++ switch (t->verdict) { ++ case G5P_TDV_DIRTY: ++ t->rc = -EIO; ++ break; ++ case G5P_TDV_RAM_ONLY: ++ t->rc = -EAGAIN; ++ break; ++ default: ++ t->rc = 0; ++ } ++ t->us_total = g5p_td_us_since(t, o); ++ g5p_td_receipt(t, o, t->rc, (u32)t->skip, t->abandoned); ++ t->state = mode == G5P_TD_FINAL ? 2 : 1; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +new file mode 100644 +index 0000000000..6070978986 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -0,0 +1,692 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the GPU teardown state machine (teardown.h). ++ * Design, evidence and device plan: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md ++ * ++ * Included from core.inc after lifecycle.inc, so it may use g5p_dc_root_word(), ++ * g5p_fender_write()/poll() and g5p_r73b_disarm(). No state is added to struct ++ * g5p_boot or struct g5p_diag: one GPU, one file-static teardown instance. ++ * ++ * Compiled with W=1 against a9/tri96 (patches/gpu-recovery/README.md); NOT run on hardware. ++ * ++ * gpu-recovery108 (0006): every run goes through the retire layer (retire108.h): ++ * admission close, RAM-only epoch class and a TR108 receipt before the ladder, the ++ * named post-ladder slots, and the retired latch after a FINAL. The TD98 lines are ++ * unchanged. docs/gpu/experiments/recovery108-driver.md. ++ */ ++#include ++#include ++#include ++#include "teardown.h" ++#include "retire108.h" ++#include "render-user-bind.h" ++#include "render-user-quiesce.h" ++ ++/* 0 off (default; the notifier and .shutdown do nothing), 1 observe-only receipts ++ * (CAPTURE: RAM reads, no change), 2 full ladder (FINAL). ++ */ ++ ++/* gpu-recovery108 G6 slot: one R1 read of isStableOff after a verified Fender stop leg. */ ++ ++/* Defined in render-r96.inc (same translation unit, included later by render-fixed.inc). */ ++ ++static struct g5p_td g5p_td_state; ++static struct g5p_diag *g5p_td_dev; ++static void (*g5p_td_detach_cb)(void *cookie); ++static void *g5p_td_detach_cookie; ++ ++/* gpu-recovery108: the retire layer around the ladder (retire108.h), one GPU, file static. */ ++static struct g5p_r8 g5p_r8_state = { .admit_us = G5P_R8_ADMIT_US }; ++static void (*g5p_r8_epoch_cb)(void *cookie, struct g5p_r8_in *in); ++static void *g5p_r8_epoch_cookie; ++ ++/* The DRM frontend reports its share of the epoch inputs (claim, holders, queued jobs), RAM only. */ ++static void __maybe_unused g5p_r8_register_epoch(void (*cb)(void *, ++ struct g5p_r8_in *), ++ void *cookie) ++{ ++ g5p_r8_epoch_cookie = cookie; ++ smp_store_release(&g5p_r8_epoch_cb, cb); ++} ++ ++/* Job admission: a submitter enters before its first look at driver state and ++ * leaves on every return; false once a FINAL teardown closed admission. ++ */ ++static bool __maybe_unused g5p_r8_enter(void) ++{ ++ return g5p_r8_admit_enter(&g5p_r8_state.admit); ++} ++ ++static void __maybe_unused g5p_r8_exit(void) ++{ ++ g5p_r8_admit_exit(&g5p_r8_state.admit); ++} ++ ++/* The post-retire register predicate for the R1 helpers outside the ladder. */ ++static bool __maybe_unused g5p_r8_r1_allowed(void) ++{ ++ return g5p_r8_mmio_ok(&g5p_r8_state, G5P_TDD_R1); ++} ++ ++static bool __maybe_unused g5p_r8_is_retired(void) ++{ ++ return g5p_r8_retired(&g5p_r8_state); ++} ++ ++/* The DRM frontend registers the callback that fails its queued/in-flight job ++ * fences (exactly once each, -ENODEV) without touching hardware. ++ */ ++static void __maybe_unused g5p_td_register_detach(void (*cb)(void *), ++ void *cookie) ++{ ++ g5p_td_detach_cookie = cookie; ++ smp_store_release(&g5p_td_detach_cb, cb); ++} ++ ++static u64 g5p_td_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_td_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++static void g5p_td_op_freeze(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ struct g5p_boot *b = g->boot; ++ ++ if (!b) ++ return; ++ /* The IRQ-side plateau banks (g5p_r80_irq_bank/g5p_r85_irq_bank) read R0 until this ++ * latch is set; arm it before anything else (RAM only, no-op without an r96 config). ++ */ ++ /* Same freeze as g5p_boot_park(): nothing may publish after this. */ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++} ++ ++static int g5p_td_op_ring_read(void *ctx, u32 idx, u32 *cons, u32 *prod) ++{ ++ static const u32 slots[G5P_TD_NRING] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *ctl; ++ ++ if (!b || !READ_ONCE(b->maps_ready)) ++ return -ENODEV; ++ ctl = (u32 *)g5p_boot_shared(b, slots[idx]); ++ *cons = READ_ONCE(ctl[0]); ++ *prod = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ return 0; ++} ++ ++/* RAM only. Consumes ALIVE (0) and stamp (1) entries; the first other type is ++ * left in place for the host, as g5p_boot_drain() does. ++ */ ++static int g5p_td_op_event_pass(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n = 0; ++ unsigned long flags; ++ int ret = 0; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ while (n < max && consumer != producer) { ++ u32 off = b->layout.offset[0x74] + consumer * 44; ++ ++ if (g5p_le32(b->shared[off / G5P_GPU_PAGE].cpu + ++ off % G5P_GPU_PAGE) > 1) { ++ *fault = true; ++ break; ++ } ++ consumer = (consumer + 1) & 255; ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ n++; ++ } ++ producer = READ_ONCE(control[4]); ++ *left = producer > 255 ? 255 : (producer - consumer) & 255; ++out: ++ *consumed = n; ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* The render path latches terminal states through g5p_td_note_terminal(); until ++ * prod98 wires that call this reports "not seen" and the race is only visible as ++ * ring state. ++ */ ++static bool g5p_td_op_terminal_seen(void *ctx) ++{ ++ return READ_ONCE(g5p_td_state.terminal) != 0; ++} ++ ++static void g5p_td_op_detach(void *ctx) ++{ ++ void (*cb)(void *) = smp_load_acquire(&g5p_td_detach_cb); ++ ++ if (cb) ++ cb(g5p_td_detach_cookie); ++} ++ ++static void g5p_td_op_aic_ack(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* The same software-source ACK the IRQ-cap path uses. */ ++ if (b && b->irq_ready && b->swirq) ++ writel(0x4000, b->swirq + 0x94); ++} ++ ++static void g5p_td_op_irq_mask(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* Line masked at the interrupt controller; the handler stays registered for ++ * a late entry, which still ACKs. Same call as the IRQ-cap path. ++ */ ++ if (b && b->irq_ready && b->irq > 0) ++ disable_irq_nosync(b->irq); ++} ++ ++static u32 g5p_td_op_irq_total(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ return b ? (u32)atomic_read(&b->irqs) : 0; ++} ++ ++static int g5p_td_op_fender_stop(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ ++ /* Apple stopFirmwareForPowerDown 6c9fa10; no retry, no PMGR off. */ ++ return g5p_fender_idle(&io); ++} ++ ++static int g5p_td_op_pmgr(void *ctx, u32 *target, u32 *actual) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) ++ return -ENODEV; ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (!ret) { ++ *target = raw & 0xf; ++ *actual = (raw >> 4) & 0xf; ++ } ++ return ret; ++} ++ ++/* dev_emerg so the receipt reaches netconsole at any console loglevel. */ ++static void g5p_td_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_diag *)ctx)->dev, "%s\n", line); ++} ++ ++static void g5p_td_pre_fill(struct g5p_diag *g, struct g5p_td_pre *p, ++ bool locked) ++{ ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(p, 0, sizeof(*p)); ++ p->lock_held = locked; ++ p->powered = READ_ONCE(g->powered); ++ p->fender_ready = READ_ONCE(g->fender_ready); ++ if (!b) ++ return; ++ p->booted = READ_ONCE(b->maps_ready); ++ p->released = READ_ONCE(b->released); ++ p->already_parked = READ_ONCE(b->parked); ++ p->stopped_unparked = READ_ONCE(b->stopped) && !READ_ONCE(b->parked); ++ p->power_abort = b->r73b && READ_ONCE(b->r73b->early_power_abort); ++ /* RAM-only read of the host-written init word (same as g5p_dc_snapshot()). */ ++ p->policy_host_owned = ++ p->booted && ++ !g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8); ++} ++ ++/* gpu-recovery108: the ladder as the retire layer's td_run op. The TD98 begin line ++ * and the ladder's own lines are exactly the recovery98/99 ones. ++ */ ++struct g5p_r8_ctx { ++ struct g5p_diag *g; ++ const struct g5p_td_ops *ops; ++ struct g5p_td_pre *pre; ++ const char *why; ++ bool locked; ++}; ++ ++static int g5p_r8_op_td_run(void *ctx, enum g5p_td_mode mode) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ dev_emerg(c->g->dev, ++ "TD98 begin why=%s mode=%s teardown=%u locked=%u\n", c->why, ++ g5p_td_mode_name[mode], (0), c->locked); ++ return g5p_td_run(&g5p_td_state, c->ops, mode, c->pre); ++} ++ ++static u32 g5p_r8_op_td_flags(void *ctx) ++{ ++ return g5p_td_state.flags; ++} ++ ++static enum g5p_td_verdict g5p_r8_op_td_verdict(void *ctx) ++{ ++ return g5p_td_state.verdict; ++} ++ ++static u64 g5p_r8_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_r8_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++/* G6: R1+0x0 bit 30 after a verified stop leg (the slot runs only then, before RETIRE). */ ++static int g5p_r8_op_stable_off(void *ctx, u32 *value) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ *value = readl(c->g->window[1]) >> 30; ++ return 0; ++} ++ ++static void g5p_r8_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_r8_ctx *)ctx)->g->dev, "%s\n", line); ++} ++ ++static void g5p_r8_in_fill(struct g5p_diag *g, struct g5p_r8_in *in, ++ bool locked, enum g5p_td_mode mode) ++{ ++ void (*cb)(void *, struct g5p_r8_in *) = ++ smp_load_acquire(&g5p_r8_epoch_cb); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(in, 0, sizeof(*in)); ++ in->terminal = READ_ONCE(g5p_td_state.terminal); ++ /* FINAL without the lock: a worker (or a sysfs writer) holds it, maybe inside a wait. */ ++ in->worker_busy = mode == G5P_TD_FINAL && !locked; ++ if (b) { ++ in->booted = READ_ONCE(b->maps_ready); ++ in->released = READ_ONCE(b->released); ++ in->alive = READ_ONCE(b->alive); ++ in->failed = READ_ONCE(b->failed); ++ in->fault_event = READ_ONCE(b->event_failed); ++ in->parked = READ_ONCE(b->parked); ++ in->stopped_unparked = READ_ONCE(b->stopped) && ++ !READ_ONCE(b->parked); ++ in->power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ if (cb) ++ cb(g5p_r8_epoch_cookie, in); ++} ++ ++static int g5p_td_execute(struct g5p_diag *g, enum g5p_td_mode mode, ++ const char *why) ++{ ++ const struct g5p_td_ops ops = { ++ .ctx = g, ++ .now_ns = g5p_td_op_now, ++ .relax_us = g5p_td_op_relax, ++ .freeze = g5p_td_op_freeze, ++ .ring_read = g5p_td_op_ring_read, ++ .event_pass = g5p_td_op_event_pass, ++ .terminal_seen = g5p_td_op_terminal_seen, ++ .detach = g5p_td_op_detach, ++ .aic_ack = g5p_td_op_aic_ack, ++ .irq_mask = g5p_td_op_irq_mask, ++ .irq_total = g5p_td_op_irq_total, ++ .fender_stop = g5p_td_op_fender_stop, ++ .pmgr_actual = g5p_td_op_pmgr, ++ .emit = g5p_td_op_emit, ++ }; ++ struct g5p_td_pre pre; ++ struct g5p_r8_in in; ++ struct g5p_r8_ctx c = { .g = g, .ops = &ops, .pre = &pre, .why = why }; ++ /* fw-sleep (G2) and pmgr-gate (G1) have no mechanism: their ops stay NULL. */ ++ const struct g5p_r8_ops r8ops = { ++ .ctx = &c, ++ .now_ns = g5p_r8_op_now, ++ .relax_us = g5p_r8_op_relax, ++ .td_run = g5p_r8_op_td_run, ++ .td_flags = g5p_r8_op_td_flags, ++ .td_verdict = g5p_r8_op_td_verdict, ++ .slot = { [G5P_R8S_STABLE_OFF] = g5p_r8_op_stable_off }, ++ .emit = g5p_r8_op_emit, ++ }; ++ bool locked = false; ++ int ret; ++ ++ /* Never queue behind a worker or sysfs writer that may be inside MMIO. CAPTURE ++ * reads RAM only and needs no lock. ++ */ ++ if (mode == G5P_TD_FINAL) ++ locked = mutex_trylock(&g->lock); ++ c.locked = locked; ++ g5p_td_pre_fill(g, &pre, locked); ++ g5p_r8_in_fill(g, &in, locked, mode); ++ g5p_r8_state.slot_on[G5P_R8S_STABLE_OFF] = (0); ++ ret = g5p_r8_run(&g5p_r8_state, &r8ops, mode, &in, why); ++ if (g5p_td_state.flags & G5P_TDF_RT1_OK && g->boot) ++ WRITE_ONCE(g->boot->parked, true); ++ if (locked) ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Observation only: arms the quiet predicate, changes nothing else. */ ++static int __maybe_unused g5p_td_capture(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_CAPTURE, why); ++} ++ ++static int g5p_td_final(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_FINAL, why); ++} ++ ++/* ++ * The quiesce before a kexec, reboot or power-off (render-user-quiesce.h). It runs from the ++ * reboot notifier whatever `teardown` is, once per boot; .shutdown and the `td-kexec` command ++ * reach the same run. It leaves the GPU powered with its firmware stopped and the boot state ++ * frozen: the next kernel boots it as after any warm kexec. ++ */ ++ ++#define G5P_KQ_LOCK_MS 2000U ++#define G5P_KQ_ADMIT_US 20000U ++#define G5P_KQ_OFF_MS 10U ++ ++/* The render frontend's share: its TYPED_V3 epoch inputs and the RAM-only unbind of slots. */ ++struct g5p_kq_frontend { ++ void (*state)(void *cookie, struct g5p_ru_kq_in *in); ++ int (*detach)(void *cookie, const u32 *slots, u32 n); ++}; ++ ++static const struct g5p_kq_frontend *g5p_kq_fe; ++static void *g5p_kq_fe_cookie; ++static atomic_t g5p_kq_ran = ATOMIC_INIT(0); ++ ++struct g5p_kq_ctx { ++ struct g5p_diag *g; ++ u32 slots[G5P_RU_USER_SLOTS]; ++ u32 held; ++}; ++ ++static void __maybe_unused g5p_kq_register(const struct g5p_kq_frontend *fe, ++ void *cookie) ++{ ++ g5p_kq_fe_cookie = cookie; ++ smp_store_release(&g5p_kq_fe, fe); ++} ++ ++static u32 g5p_kq_admit_close(void *ctx) ++{ ++ struct g5p_r8_admit *a = &g5p_r8_state.admit; ++ u64 deadline = ktime_get_ns() + (u64)G5P_KQ_ADMIT_US * NSEC_PER_USEC; ++ u32 inside = g5p_r8_admit_close(a); ++ ++ while (inside && ktime_get_ns() < deadline) { ++ usleep_range(G5P_R8_POLL_US, 2 * G5P_R8_POLL_US); ++ inside = g5p_r8_admit_inside(a); ++ } ++ return inside; ++} ++ ++/* Never queue behind a holder that may be inside MMIO: bounded, like g5p_render_run(). */ ++static bool g5p_kq_lock(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_KQ_LOCK_MS); ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ return true; ++} ++ ++static void g5p_kq_unlock(void *ctx) ++{ ++ mutex_unlock(&((struct g5p_kq_ctx *)ctx)->g->lock); ++} ++ ++/* RAM only: the boot state, the held user slots and the DC ring, then the frontend's inputs. */ ++static void g5p_kq_inputs(void *ctx, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ struct g5p_diag *g = c->g; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ u32 *ctl; ++ ++ in->pmgr = !!g->pmgr; ++ if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released) || ++ READ_ONCE(b->parked)) ++ return; ++ in->booted = true; ++ in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ dma_rmb(); ++ in->cons = READ_ONCE(ctl[0]); ++ in->prod = READ_ONCE(ctl[4]); ++ in->ring_idle = in->prod <= 255 && in->cons == in->prod; ++ c->held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), ++ g5p_boot_shared(b, 0x84), c->slots, ++ ARRAY_SIZE(c->slots)); ++ in->held = c->held; ++ if (fe) { ++ in->frontend = true; ++ fe->state(g5p_kq_fe_cookie, in); ++ } ++} ++ ++static int g5p_kq_detach(void *ctx) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ ++ return fe ? fe->detach(g5p_kq_fe_cookie, c->slots, c->held) : -ENODEV; ++} ++ ++static void g5p_kq_freeze(void *ctx) ++{ ++ g5p_td_op_freeze(((struct g5p_kq_ctx *)ctx)->g); ++} ++ ++/* The IRQ-cap path's AIC ack and mask, then wait out a handler already running (process ++ * context; the threaded handler drains RAM and takes no lock the quiesce holds). ++ */ ++static void g5p_kq_irq_off(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ struct g5p_boot *b = g->boot; ++ ++ g5p_td_op_aic_ack(g); ++ g5p_td_op_irq_mask(g); ++ if (b && b->irq_ready && b->irq > 0) ++ synchronize_irq(b->irq); ++} ++ ++/* PMGR only: the bounded gate cycle of the r33/dirty-start path (refuses a gate that is not ++ * plainly on), which ends with the gate requested on. ++ */ ++static int g5p_kq_cycle(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ ++ g5p_mmio_gate(g); ++ return g5p_r33_gfx_off(g->dev, g->pmgr, G5P_KQ_OFF_MS); ++} ++ ++static int g5p_kq_run(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_ru_kq_ops o = { ++ .admit_close = g5p_kq_admit_close, ++ .lock = g5p_kq_lock, ++ .unlock = g5p_kq_unlock, ++ .inputs = g5p_kq_inputs, ++ .detach = g5p_kq_detach, ++ .freeze = g5p_kq_freeze, ++ .irq_off = g5p_kq_irq_off, ++ .cycle = g5p_kq_cycle, ++ }; ++ struct g5p_ru_kq_out out; ++ struct g5p_kq_ctx *c; ++ u64 t0 = ktime_get_ns(); ++ ++ if (atomic_xchg(&g5p_kq_ran, 1)) { ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s already ran this boot; nothing done end=kq\n", ++ why); ++ return -EALREADY; ++ } ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ if (!c) { ++ dev_emerg(g->dev, "GPU-KQ why=%s ret=%d end=kq\n", why, ++ -ENOMEM); ++ return -ENOMEM; ++ } ++ c->g = g; ++ o.ctx = c; ++ g5p_ru_kq_run(&o, min((G5P_RU_KQ_FULL), G5P_RU_KQ_FULL), &out); ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s mode=%u inside=%u locked=%u booted=%u failed=%u wedged=%u " ++ "ram_only=%u exec=%u dc=%u/%u held=%u plan=%s%s skip=%#x detach=%d froze=%u " ++ "cycle=%d ms=%llu end=kq\n", ++ why, out.in.mode, out.inside, out.in.locked, out.in.booted, ++ out.in.failed, out.in.term_failed, out.in.ram_only, ++ out.in.executing, out.in.cons, out.in.prod, out.in.held, ++ out.plan & G5P_RU_KQ_DETACH ? "detach" : "-", ++ out.plan & G5P_RU_KQ_CYCLE ? "+cycle" : "", out.why, ++ out.detach_ret, out.froze, out.cycle_ret, ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC); ++ kfree(c); ++ return out.detach_ret ?: out.cycle_ret; ++} ++ ++/* sysfs `command`: td-capture / td-final / td-kexec. Runs before the command mutex, like ++ * r73b-export, so a wedged stage cannot block it. -EAGAIN (clean, GPU left as ++ * found) is reported as success; the verdict is in the TD98 lines. td-kexec is the ++ * reboot quiesce run by hand (the GPU is finished for this boot afterwards). ++ */ ++static int g5p_td_command(struct g5p_diag *g, const char *command) ++{ ++ int ret; ++ ++ if (sysfs_streq(command, "td-capture")) ++ ret = g5p_td_capture(g, "sysfs"); ++ else if (sysfs_streq(command, "td-final")) ++ ret = g5p_td_final(g, "sysfs"); ++ else if (sysfs_streq(command, "td-kexec")) ++ ret = g5p_kq_run(g, "sysfs"); ++ else ++ return -EINVAL; ++ return ret == -EAGAIN ? 0 : ret; ++} ++ ++/* Runs from kernel_restart_prepare() for reboot, halt, power-off and kexec, i.e. ++ * before device_shutdown() and its unbounded wait_for_device_probe(). Panic kexec ++ * skips both and is not covered. ++ */ ++static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, ++ void *data) ++{ ++ struct g5p_diag *g = READ_ONCE(g5p_td_dev); ++ ++ if (!g) ++ return NOTIFY_DONE; ++ if (action != SYS_RESTART && action != SYS_HALT && ++ action != SYS_POWER_OFF) ++ return NOTIFY_DONE; ++ /* The quiesce needs a live firmware for its detach, so it runs before the ladder freezes. */ ++ g5p_kq_run(g, "reboot-notifier"); ++ { ++ return NOTIFY_DONE; ++ } ++} ++ ++static struct notifier_block g5p_td_nb = { ++ .notifier_call = g5p_td_reboot_notify, ++ .priority = INT_MAX, /* before every other reboot notifier */ ++}; ++ ++/* g is devm-allocated and the DRM frontend's probe error path unwinds without ++ * g5p_remove(): the notifier must not outlive g. ++ */ ++static void g5p_td_devres_release(void *data) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ WRITE_ONCE(g5p_td_dev, NULL); ++} ++ ++static void g5p_td_attach(struct g5p_diag *g) ++{ ++ g5p_td_init(&g5p_td_state); ++ /* A probe after a retire (not reachable on the built-in driver) never reopens admission. */ ++ if (!g5p_r8_retired(&g5p_r8_state)) { ++ u32 gen = g5p_r8_state.gen; ++ ++ g5p_r8_init(&g5p_r8_state); ++ g5p_r8_state.gen = gen; /* monotonic across a re-probe */ ++ } ++ WRITE_ONCE(g5p_td_dev, g); ++ register_reboot_notifier(&g5p_td_nb); ++ /* On failure the action has already run: no notifier, teardown stays inert. */ ++ if (devm_add_action_or_reset(g->dev, g5p_td_devres_release, g)) ++ dev_warn(g->dev, "teardown not attached\n"); ++} ++ ++/* Returns true when the PM reference must be kept: a PMGR power-off was never ++ * validated on the outgoing side and the stop leg is not isStableOff. ++ */ ++static bool g5p_td_remove(struct g5p_diag *g) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ ; ++ ; ++ WRITE_ONCE(g5p_td_dev, NULL); ++ return (0) != 0; ++} ++ ++/* The .shutdown platform callback: second chance after the notifier. */ ++static void __maybe_unused g5p_td_shutdown(struct g5p_diag *g) ++{ ++ /* A second chance after the notifier (a no-op once it ran). */ ++ if (!atomic_read(&g5p_kq_ran)) ++ g5p_kq_run(g, "shutdown"); ++ { ++ ; ++ ; ++ }; ++} +diff --git a/drivers/gpu/apple-g5p/transport-budget.inc b/drivers/gpu/apple-g5p/transport-budget.inc +new file mode 100644 +index 0000000000..d70e3c6e13 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/transport-budget.inc +@@ -0,0 +1,6 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include "vm-mutation.h" ++static u32 g5p_transport_event_limit(void) ++{ ++ return 256 + 131072 * 4; ++} +diff --git a/drivers/gpu/apple-g5p/vm-mutation.h b/drivers/gpu/apple-g5p/vm-mutation.h +new file mode 100644 +index 0000000000..4645240bdf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/vm-mutation.h +@@ -0,0 +1,36 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_VM_MUTATION_H ++#define G5P_VM_MUTATION_H ++struct g5p_vm_mutation { ++ bool generations; ++ u32 generation[3]; ++}; ++ ++/* Count actual changed descriptors, including clears, not dispatches. Repeated ++ * parent writes of identical bytes do not advance a generation. Packed fields ++ * follow iOS6cabf40. Publication is serialized by the existing command lock. ++ */ ++static void g5p_vm_entry(struct g5p_vm_mutation *c, void *at, u64 value, ++ u32 level) ++{ ++ u64 old = g5p_le32(at); ++ ++ if (level) ++ old |= (u64)g5p_le32(at + 4) << 32; ++ if (old == value) ++ return; ++ if (level) ++ g5p_put64(at, value); ++ else ++ g5p_put32(at, value); ++ c->generation[level]++; ++} ++ ++static u32 g5p_vm_generation(struct g5p_vm_mutation *c) ++{ ++ if (!c->generations) ++ return 0; ++ return ((c->generation[0] & 0x3ff) << 22) | ++ ((c->generation[1] & 0x3ff) << 12) | (c->generation[2] & 0xfff); ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/work-layout.h b/drivers/gpu/apple-g5p/work-layout.h +new file mode 100644 +index 0000000000..b5dc73725a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/work-layout.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_WORK_LAYOUT_H ++#define G5P_WORK_LAYOUT_H ++ ++/* Experimental first CL queue probe: c89420/c88840/c8a808/c980a0. ++ * Tag 4 is emitted for dependencies by c8a808. Treating an already satisfied ++ * dependency as a NOP is inferred and must be validated after G2. ++ */ ++#define G5P_WORK_OFFSET 0x54000U ++#define G5P_WORK_BYTES 0x400U ++#define G5P_WORK_CTL 0x100U ++#define G5P_WORK_RING 0x200U ++#define G5P_WORK_CMD 0x300U ++#define G5P_WORK_STAMP 0x340U ++#define G5P_WORK_SLOTS 16U ++ ++/* Isolated visibility hypothesis, not the recovered normal host alias. */ ++ ++/* Firmware10004e94 copies notification word2 into ctl+30. */ ++ ++#endif diff --git a/kernel/patches/drivers/drm/0010-drm-apple-run-TYPED_V3-jobs-for-the-merged-G5P-Mesa-.patch b/kernel/patches/drivers/drm/0010-drm-apple-run-TYPED_V3-jobs-for-the-merged-G5P-Mesa-.patch new file mode 100644 index 00000000..533e7160 --- /dev/null +++ b/kernel/patches/drivers/drm/0010-drm-apple-run-TYPED_V3-jobs-for-the-merged-G5P-Mesa-.patch @@ -0,0 +1,8888 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 4 Nov 2026 21:55:57 +0000 +Subject: [PATCH] drm/apple: run TYPED_V3 jobs for the merged G5P Mesa winsys + +Provide render-node GEM, PRIME, VM, targets and syncobj submission with +context isolation, recovery and bounded retirement. Default to the +proven event wake and range sync. Trim retired user mappings when a +context closes while retaining the PM late-write route, following the +device-proven close-BO fix 6824230ee54c. No trim toggle is exposed. + +[Orchard: from brkosebay/linux 407b7bf2cbbe1df2f93926bbfc0f642ffffbe71e; preserve existing Orchard DRM build hooks.] + +Assisted-by: LLM + +Co-authored-by: Claude Opus 5.5 + +Co-authored-by: GPT-6.1 Sol +--- + drivers/gpu/drm/Kconfig | 2 + + drivers/gpu/drm/Makefile | 2 + + drivers/gpu/drm/apple-g5p/Kconfig | 13 + + drivers/gpu/drm/apple-g5p/Makefile | 2 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 704 ++++ + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 294 ++ + drivers/gpu/drm/apple-g5p/drm-prime.inc | 114 + + .../gpu/drm/apple-g5p/drm-render-sched.inc | 203 ++ + .../gpu/drm/apple-g5p/drm-render-unbind.inc | 488 +++ + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 62 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 3033 +++++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-render.inc | 2482 ++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-submit.inc | 29 + + drivers/gpu/drm/apple-g5p/drm-vm.inc | 914 +++++ + .../gpu/drm/apple-g5p/render-user-validate.h | 374 ++ + drivers/gpu/drm/apple-g5p/vm-ranges.h | 21 + + 16 files changed, 8737 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/Kconfig + create mode 100644 drivers/gpu/drm/apple-g5p/Makefile + create mode 100644 drivers/gpu/drm/apple-g5p/a9-gpu-drm.c + create mode 100644 drivers/gpu/drm/apple-g5p/drm-latency123.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-prime.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-sched.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-unbind.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-submit.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-vm.inc + create mode 100644 drivers/gpu/drm/apple-g5p/render-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/vm-ranges.h + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 4c74c15014..eaf71742f9 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -371,3 +371,5 @@ endif + # Separate option because drm_panel_orientation_quirks.c is shared with fbdev + config DRM_PANEL_ORIENTATION_QUIRKS + tristate ++ ++source "drivers/gpu/drm/apple-g5p/Kconfig" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index d75eba3997..1315aa3781 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -257,3 +257,5 @@ quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + + $(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) ++ ++obj-$(CONFIG_DRM_APPLE_G5P) += apple-g5p/ +diff --git a/drivers/gpu/drm/apple-g5p/Kconfig b/drivers/gpu/drm/apple-g5p/Kconfig +new file mode 100644 +index 0000000000..e55c0f1de2 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Kconfig +@@ -0,0 +1,13 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_APPLE_G5P ++ bool "Apple A9/G5P render driver" ++ depends on DRM=y && ARCH_APPLE && OF && PM && ARM64_16K_PAGES ++ select DRM_GEM_SHMEM_HELPER ++ select FW_LOADER ++ select CRYPTO_LIB_SHA256 ++ help ++ Render-node driver for the S8003 G5P GPU using the unmodified META ++ firmware from the device. Supports TYPED_V3 jobs, GEM buffers, PRIME, ++ virtual address spaces and syncobj synchronization for the merged ++ Mesa winsys. Register with a9_gpu_drm.enable=1. Built-in only while ++ reset and active-DMA teardown require the platform boot sequence. +diff --git a/drivers/gpu/drm/apple-g5p/Makefile b/drivers/gpu/drm/apple-g5p/Makefile +new file mode 100644 +index 0000000000..843d56a040 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Makefile +@@ -0,0 +1,2 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++obj-$(CONFIG_DRM_APPLE_G5P) += a9-gpu-drm.o +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +new file mode 100644 +index 0000000000..a434242c37 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -0,0 +1,704 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* Experimental render frontend; default disabled, no implicit firmware boot. */ ++#include "../../apple-g5p/core.inc" ++#include "../../apple-g5p/drm-validate.h" ++#include "../../apple-g5p/render-user-evwait.h" ++#include "../../apple-g5p/render-user-sync.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define G5P_MAX_VMS 8 ++#define G5P_MAX_CONTEXTS 32 ++#define G5P_MAX_MAPS 4096 ++#define G5P_MAX_PT_PAGES 512 ++#define G5P_MAX_SYNCS 17 ++ ++struct g5p_bo { ++ struct drm_gem_shmem_object shmem; ++ u32 roles; ++ u32 vm_pins; /* VMs that map it, any file; counted once in pinned_bytes (pin_lock) */ ++ /* release8 memory accounting (drm-vm.inc g5p_bo_live): the live list, its creator */ ++ struct list_head live; ++ pid_t pid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++struct g5p_pt { ++ struct list_head link; ++ struct g5p_dma_page page; ++ u32 key; ++ u32 level; ++}; ++ ++struct g5p_mapping { ++ struct list_head link; ++ struct drm_mm_node node; ++ struct drm_gem_object *obj; ++ u64 offset; ++ u64 retire; /* unmapped beside queued jobs: freed once this fence seqno has signalled */ ++}; ++ ++struct g5p_vm { ++ struct g5p_vm_mutation mutation; ++ bool bounded, frozen, serial_context; ++ bool zero_leaf_flags; /* render context latched r25 memory variant 4 */ ++ /* Its render context's handle is gone (g5p_render_close()): no VM_BIND, no new ++ * context; the context's close item and release own the tables. ++ */ ++ bool resident_gone; ++ struct g5p_context *resident; ++ struct kref ref; ++ struct device *dev; ++ struct g5p_drm *gpu; ++ struct drm_mm mm; ++ struct drm_mm_node reserved; ++ struct list_head maps; ++ struct xarray objs; /* object -> its mappings in this VM (xa_mk_value) */ ++ struct list_head tables; ++ struct g5p_dma_page root; ++ u64 mapped_bytes, pinned_bytes; ++ u32 table_count, map_count, contexts; ++}; ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rdev; ++ ++struct g5p_context { ++ struct g5p_drm *gpu; ++ ++ bool live, bad; ++ bool serial, pending; ++ ++ struct g5p_render *render; /* render contexts only, see drm-render.inc */ ++ struct kref ref; ++ struct g5p_vm *vm; ++}; ++ ++struct g5p_file { ++ /* Serializes VM/context handles, mappings and the one-shot submit state. */ ++ struct mutex lock; ++ struct xarray vms; ++ struct xarray contexts; ++ struct xarray results; ++ struct xarray targets; /* render targets; each holds its context */ ++ u32 render_results; ++ bool submitted; ++ u64 job_id; ++}; ++ ++struct g5p_drm { ++ struct drm_device drm; ++ struct g5p_diag *core; ++ /* Protects cached IRQ event/status fields; never held for hardware IO. */ ++ spinlock_t status_lock; ++ u32 last_event[11], notifications[4], first_fault[11]; ++ u64 event_serial, fault_serial; ++ /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ ++ wait_queue_head_t ev_wq; ++ /* TYPED_V3 3D completion-event latch (render-user-retire.h): G5P_RV3_TD_WANT | endpoint ++ * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. ++ */ ++ u32 td_want; ++ u64 td_hit_ns; ++ /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ ++ bool ta_want; ++ u64 ta_hit_ns; ++ /* wake110: type-1 completion events decoded per engine (word 1 bit 0 TA, bit 1 3D) since ++ * load, and the decode time of the first one since the executor last cleared it. ++ */ ++ u32 t1_count[2]; ++ u64 t1_first_ns[2]; ++ /* perf60 bell gates (drm-render-v3.inc): the previous TYPED_V3 job's end and context. */ ++ u64 bell_prev_end_ns; ++ const void *bell_prev_ctx; ++ u64 last_job; ++ int last_error, job_error; ++ u32 job_state; ++ ++ atomic_t accepted; ++ atomic64_t job_sequence; ++ u64 fence_context; ++ /* Attached catalogues survive fd close until proven unbind/reset. */ ++ struct g5p_context *quarantined_context; ++ /* The first accepted render job claimed the device for render, for the rest of the boot. */ ++ bool render_claimed; ++ struct g5p_rdev *rdev; /* device-wide render state, see drm-render.inc */ ++ struct workqueue_struct ++ *render_wq; /* ordered: the render scheduler's worker */ ++ /* Orders render acceptance, dma_resv fence attach and queueing. */ ++ struct mutex render_submit_lock; ++ /* recovery108 epoch inputs, RAM only: open render-node files, render jobs not yet ++ * signalled. ++ */ ++ atomic_t open_files, r8_queued; ++ /* Memory mapped into any VM, each object once (render_pinned_mib). */ ++ struct mutex pin_lock; ++ u64 pinned_bytes; ++}; ++ ++static inline struct g5p_drm *to_g5p(struct drm_device *drm) ++{ ++ return container_of(drm, struct g5p_drm, drm); ++} ++ ++static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) ++{ ++ return container_of(obj, struct g5p_bo, shmem.base); ++} ++ ++#include "vm-ranges.h" ++ ++static int g5p_resident_idle(struct g5p_context *c, bool live); ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m); ++static int g5p_resident_invalidate(struct g5p_context *c); ++static void g5p_render_release(struct g5p_context *c); ++static int g5p_render_destroy(struct g5p_context *c); ++static void g5p_render_close(struct g5p_context *c); ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a); ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags); ++ ++#include "drm-vm.inc" ++#include "drm-submit.inc" ++#include "drm-render.inc" ++#include "drm-prime.inc" ++ ++static int g5p_get_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_info *a = data; ++ u64 caps = DRM_G5P_CAP_BO_CPU | DRM_G5P_CAP_VM_PREPARE | ++ DRM_G5P_CAP_SYNCOBJ | DRM_G5P_CAP_JOB_STATUS | ++ DRM_G5P_CAP_CONTEXT_STATUS | DRM_G5P_CAP_FAULT_INFO; ++ ++ if (a->size != sizeof(*a) || a->flags || a->reserved[0] || ++ a->reserved[1]) ++ return -EINVAL; ++ ; ++ ; ++ ; ++ ; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ a->version = DRM_G5P_UAPI_VERSION; ++ a->gpu_page_shift = 14; ++ ; ++ ; ++ ; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER | DRM_G5P_CAP_BOUNDED_RENDER | ++ DRM_G5P_CAP_TYPED_RENDER | DRM_G5P_CAP_VM_LIVE | ++ DRM_G5P_CAP_CONTEXT_LIVE | ++ DRM_G5P_CAP_CODE_BASE_VERIFIED; ++ a->code_base = 0x300000000ULL; ++ }; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY; ++ }; ++ if ((1) && g5p_render_msaa_enabled()) ++ caps |= DRM_G5P_CAP_RENDER_MSAA; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_TYPED_RENDER_V3 | ++ DRM_G5P_CAP_RENDER_MULTI_CONTEXT | ++ DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH; ++ }; ++ { ++ caps |= DRM_G5P_CAP_RENDER_UNPRIVILEGED; ++ }; ++ ; ++ ; ++ caps |= DRM_G5P_CAP_PRIME; ++ a->capabilities = caps; ++ a->va_start = G5P_USER_VA_START; ++ a->va_end = G5P_USER_VA_END; ++ a->reserved_start = G5P_USER_RESERVED_START; ++ a->reserved_end = G5P_USER_RESERVED_END; ++ a->max_bo_size = G5P_MAX_BO_SIZE; ++ a->bvnc[0] = 9; ++ a->bvnc[1] = 15; ++ a->bvnc[2] = 6; ++ a->bvnc[3] = 2; ++ return 0; ++} ++ ++/* The gated dialect uses the common B proven by the r19c numerical store. */ ++static int g5p_get_exec_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_exec_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || a->pad || ++ memchr_inv(a->reserved, 0, sizeof(a->reserved))) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ { ++ a->base_scope = DRM_G5P_BASE_VM; ++ a->usc_base = 0x300000000ULL; ++ a->pds_base = a->usc_base; ++ a->window_size = 0x34000; ++ a->allocation_alignment = G5P_GPU_PAGE; ++ a->usc_offset_bits = 32; ++ a->pds_offset_bits = 32; ++ } ++ return 0; ++} ++ ++static int g5p_get_job_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_job_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ bool owned; ++ ++ if (a->flags || a->pad || !a->job_id || a->reset_generation) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ if (xa_load(&f->results, a->job_id)) { ++ struct g5p_fence *fence = xa_load(&f->results, a->job_id); ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = fence->state; ++ a->error = fence->error; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ mutex_unlock(&f->lock); ++ return 0; ++ } ++ owned = f->job_id == a->job_id; ++ mutex_unlock(&f->lock); ++ if (!owned) ++ return -ENOENT; ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = d->job_state; ++ a->error = d->job_error; ++ /* Owner acknowledges terminal status before another fixed submission. ++ * This avoids waking a fence waiter before the device admission reopens. ++ */ ++ ; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return 0; ++} ++ ++/* c99788 classifies event4/7/8. Preserve opaque fields, never invent a VA. */ ++static int g5p_get_fault_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_fault_info *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ ++ if (a->flags || a->reserved) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->event_serial = d->fault_serial; ++ memcpy(a->raw, d->first_fault, sizeof(a->raw)); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!a->event_serial) ++ return 0; ++ switch (a->raw[0]) { ++ case 4: ++ a->kind = DRM_G5P_FAULT_RECOVERY; ++ break; ++ case 7: ++ a->kind = DRM_G5P_FAULT_PAGE_REQUEST; ++ break; ++ case 8: ++ a->kind = DRM_G5P_FAULT_CHANNEL; ++ a->valid = DRM_G5P_FAULT_VALID_CHANNEL; ++ a->channel_class = a->raw[1]; ++ a->channel_token = a->raw[3]; ++ a->channel_stamp = a->raw[4]; ++ break; ++ default: ++ a->kind = DRM_G5P_FAULT_UNKNOWN; ++ break; ++ } ++ return 0; ++} ++ ++static int g5p_get_context_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_context_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(to_g5p(drm)->core->boot); ++ struct g5p_context *c; ++ bool failed; ++ int ret = 0; ++ ++ if (a->flags || a->state || a->map_count || a->mapped_bytes || ++ a->reserved) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c) { ++ ret = -ENOENT; ++ goto out; ++ } ++ a->state = DRM_G5P_CONTEXT_HOST_ONLY; ++ if (READ_ONCE(c->live)) ++ a->state = DRM_G5P_CONTEXT_LIVE; ++ /* a failure noted mid-job is not this context's until the executor rules */ ++ failed = b && READ_ONCE(b->failed) && ++ !g5p_rdev_ruling(to_g5p(drm)->rdev); ++ if (READ_ONCE(c->bad) || failed || (b && READ_ONCE(b->stopped))) ++ a->state = DRM_G5P_CONTEXT_LOST; ++ a->map_count = c->vm->map_count; ++ a->mapped_bytes = c->vm->mapped_bytes; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static void g5p_drm_event(void *cookie, const u32 words[11]) ++{ ++ struct g5p_drm *d = cookie; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ memcpy(d->last_event, words, sizeof(d->last_event)); ++ d->event_serial++; ++ if (words[0] == 1) ++ for (u32 i = 0; i < 4; i++) ++ d->notifications[i] |= words[i + 1]; ++ if (d->td_want && !d->td_hit_ns && ++ g5p_ru_rt_event_match(words, d->td_want & 0xff)) ++ WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); ++ if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) ++ WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); ++ for (u32 e = 0; words[0] == 1 && e < 2; e++) { ++ if (!(words[1] & BIT(e))) ++ continue; ++ WRITE_ONCE(d->t1_count[e], d->t1_count[e] + 1); ++ if (!d->t1_first_ns[e]) ++ WRITE_ONCE(d->t1_first_ns[e], ktime_get_ns()); ++ } ++ if (words[0] != 0 && words[0] != 1) { ++ d->last_error = -EIO; ++ if (!d->fault_serial) { ++ d->fault_serial = d->event_serial; ++ memcpy(d->first_fault, words, sizeof(d->first_fault)); ++ } ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ wake_up(&d->ev_wq); ++} ++ ++static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = kzalloc(sizeof(*f), GFP_KERNEL); ++ ++ if (!f) ++ return -ENOMEM; ++ mutex_init(&f->lock); ++ xa_init_flags(&f->vms, XA_FLAGS_ALLOC1); ++ xa_init_flags(&f->contexts, XA_FLAGS_ALLOC1); ++ xa_init(&f->results); ++ xa_init_flags(&f->targets, XA_FLAGS_ALLOC1); ++ file->driver_priv = f; ++ atomic_inc(&to_g5p(drm)->open_files); ++ return 0; ++} ++ ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_vm *vm; ++ unsigned long id; ++ struct g5p_rt *rt; ++ ++ atomic_dec(&to_g5p(drm)->open_files); ++ /* Render-target handles hold their context; drop them first. */ ++ xa_for_each(&f->targets, id, rt) ++ g5p_context_put(rt->ctx); ++ xa_destroy(&f->targets); ++ /* Jobs hold contexts, contexts hold VMs, mappings hold GEM objects. ++ * Closing the fd drops handle ownership, never a queued job's references. ++ * Host-only tables may retire at job completion: no live GPU attachment. ++ */ ++ xa_for_each(&f->contexts, id, c) { ++ /* A render context's firmware binding outlives the handle until its close ++ * item, queued behind its jobs, has unbound it. ++ */ ++ if (c->render) ++ g5p_render_close(c); ++ g5p_context_put(c); ++ } ++ xa_destroy(&f->contexts); ++ { ++ struct g5p_fence *fence; ++ ++ xa_for_each(&f->results, id, fence) ++ dma_fence_put(&fence->base); ++ xa_destroy(&f->results); ++ } ++ xa_for_each(&f->vms, id, vm) ++ g5p_vm_put(vm); ++ xa_destroy(&f->vms); ++ mutex_destroy(&f->lock); ++ kfree(f); ++ /* The file's handles are gone (drm_gem_release() ran first): what stays live is held ++ * by another file's handle, a dma-buf, a VM or a job (debugfs g5p_bos names it). ++ */ ++ dev_info(drm->dev, ++ "G5P file close live_bos=%u live_bo_bytes=%lluK device_pinned=%lluK\n", ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++} ++ ++static const struct drm_ioctl_desc g5p_ioctls[] = { ++ DRM_IOCTL_DEF_DRV(G5P_GET_INFO, g5p_get_info, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_BO, g5p_create_bo, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_MMAP_OFFSET, g5p_bo_mmap_offset, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_SYNC, g5p_bo_sync, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_VM, g5p_create_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_VM, g5p_destroy_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_VM_BIND, g5p_vm_bind, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_CONTEXT, g5p_create_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_CONTEXT, g5p_destroy_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_EXEC_INFO, g5p_get_exec_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_JOB_STATUS, g5p_get_job_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_CONTEXT_STATUS, g5p_get_context_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_FAULT_INFO, g5p_get_fault_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_RENDER_TARGET, g5p_create_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_RENDER_TARGET, g5p_destroy_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_RENDER_INFO, g5p_get_render_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_RENDER_TYPED, g5p_submit_render_typed, ++ DRM_RENDER_ALLOW), ++}; ++ ++DEFINE_DRM_GEM_FOPS(g5p_drm_fops); ++ ++static const struct drm_driver g5p_drm_driver = { ++ .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | ++ DRIVER_SYNCOBJ_TIMELINE, ++ .open = g5p_drm_open, ++ .postclose = g5p_drm_postclose, ++ .fops = &g5p_drm_fops, ++ .ioctls = g5p_ioctls, ++ .num_ioctls = ARRAY_SIZE(g5p_ioctls), ++ .gem_create_object = g5p_gem_create_object, ++ .prime_handle_to_fd = g5p_prime_handle_to_fd, ++ .prime_fd_to_handle = g5p_prime_fd_to_handle, ++ .gem_prime_import = g5p_gem_prime_import, ++ .gem_prime_import_sg_table = g5p_prime_import_sg_table, ++ .name = "apple_g5p", ++ .desc = "Apple A9 G5P experimental render driver", ++ .major = 0, ++ .minor = DRM_G5P_UAPI_VERSION, ++}; ++ ++/* recovery108: the frontend's epoch inputs for the retire layer. RAM only and lock free: ++ * the claimed context is compared, never dereferenced (it may be dropped concurrently). ++ */ ++static void g5p_drm_r8_epoch(void *cookie, struct g5p_r8_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ in->claimed = READ_ONCE(d->render_claimed) || ++ READ_ONCE(d->quarantined_context); ++ in->held = in->claimed && atomic_read(&d->open_files) > 0; ++ in->queued = queued > 0 ? queued : 0; ++} ++ ++/* The teardown DETACH phase. No fence is signalled here: with admission closed and the ++ * device frozen, the render scheduler fails each queued job exactly once (-ESHUTDOWN, ++ * without waiting for its inputs), and an in-flight job ends on its own RAM-only ++ * deadline. The count left to the workers is reported as the explicit abandonment. ++ */ ++static void g5p_drm_td_detach(void *cookie) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ g5p_r8_note_detach(&g5p_r8_state, queued > 0 ? queued : 0); ++ g5p_rsched_kick(d); ++} ++ ++static int g5p_drm_probe(struct platform_device *pdev) ++{ ++ struct g5p_drm *d; ++ struct g5p_diag *g; ++ int ret; ++ ++ ; ++ ; ++ ret = g5p_probe(pdev); ++ if (ret) ++ return ret; ++ g = platform_get_drvdata(pdev); ++ d = devm_drm_dev_alloc(&pdev->dev, &g5p_drm_driver, struct g5p_drm, ++ drm); ++ if (IS_ERR(d)) { ++ ret = PTR_ERR(d); ++ goto err_pm; ++ } ++ d->core = g; ++ d->fence_context = dma_fence_context_alloc(1); ++ spin_lock_init(&d->status_lock); ++ init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->accepted, 0); ++ atomic64_set(&d->job_sequence, 0); ++ d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", 0); ++ if (IS_ERR_OR_NULL(d->render_wq)) { ++ ret = d->render_wq ? PTR_ERR(d->render_wq) : -ENOMEM; ++ goto err_pm; ++ } ++ ret = drmm_mutex_init(&d->drm, &d->render_submit_lock); ++ if (!ret) ++ ret = drmm_mutex_init(&d->drm, &d->pin_lock); ++ if (ret) ++ goto err_pm; ++ ret = g5p_render_init(d); ++ if (ret) ++ goto err_pm; ++ g->event_cookie = d; ++ g->event_notify = g5p_drm_event; ++ atomic_set(&d->open_files, 0); ++ atomic_set(&d->r8_queued, 0); ++ drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); ++ ret = drm_dev_register(&d->drm, 0); ++ if (ret) ++ goto err_pm; ++ g5p_r8_register_epoch(g5p_drm_r8_epoch, d); ++ g5p_td_register_detach(g5p_drm_td_detach, d); ++ g5p_kq_register(&g5p_kq_drm, d); ++ dev_info(&pdev->dev, ++ "render node registered; no GPU access, COMPUTE/RENDER capabilities absent\n"); ++ return 0; ++err_pm: ++ g->event_notify = NULL; ++ pm_runtime_disable(&pdev->dev); ++ return ret; ++} ++ ++static void g5p_drm_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g->event_cookie; ++ ++ drm_dev_unplug(&d->drm); ++ g5p_remove(pdev); ++ /* recovery108: the remove-time FINAL above was the callbacks' last use; d is ++ * devres-freed next. ++ */ ++ g5p_td_register_detach(NULL, NULL); ++ g5p_r8_register_epoch(NULL, NULL); ++ g5p_kq_register(NULL, NULL); ++} ++ ++static const struct of_device_id g5p_drm_match[] = { ++ { .compatible = "gpu,s8003" }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, g5p_drm_match); ++ ++static struct platform_driver g5p_drm_platform = { ++ .probe = g5p_drm_probe, ++ .remove = g5p_drm_remove, ++ .shutdown = g5p_shutdown, ++ .driver = { ++ .name = "apple-g5p-drm", ++ .of_match_table = g5p_drm_match, ++ .dev_groups = g5p_groups, ++ .suppress_bind_attrs = true, ++ }, ++}; ++ ++static bool enable; ++static bool drm_initialized; ++static bool drm_registered; ++ ++static int g5p_drm_enable_set(const char *value, const struct kernel_param *kp) ++{ ++ bool requested; ++ int ret = kstrtobool(value, &requested); ++ ++ if (ret) ++ return ret; ++ if (!drm_initialized) { ++ enable = requested; ++ return 0; ++ } ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (!requested && drm_registered) ++ return -EBUSY; ++ if (requested && !drm_registered) { ++ ret = g5p_frontend_available(); ++ if (ret) ++ return ret; ++ ret = platform_driver_register(&g5p_drm_platform); ++ if (ret) ++ return ret; ++ drm_registered = true; ++ } ++ enable = requested; ++ return 0; ++} ++ ++static const struct kernel_param_ops g5p_drm_enable_ops = { ++ .set = g5p_drm_enable_set, ++ .get = param_get_bool, ++}; ++module_param_cb(enable, &g5p_drm_enable_ops, &enable, 0600); ++MODULE_PARM_DESC(enable, ++ "Explicitly register G5P DRM frontend; conflicts with diagnostic binding, default false"); ++ ++static int __init g5p_drm_init(void) ++{ ++ drm_initialized = true; ++ if (!enable) { ++ pr_info("apple-g5p-drm: inert; explicit a9_gpu_drm.enable=1 required\n"); ++ return 0; ++ } ++ return g5p_drm_enable_set("1", NULL); ++} ++module_init(g5p_drm_init); ++ ++/* Built-in only until reset can prove all DMA references are quiescent. */ ++MODULE_DESCRIPTION("Default-disabled Apple G5P render/GEM/VM experimental driver"); ++MODULE_LICENSE("Dual MIT/GPL"); ++MODULE_FIRMWARE(G5P_FW_NAME); +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +new file mode 100644 +index 0000000000..085e766504 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -0,0 +1,294 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: runtime A/B knobs for the V3 render path, all default off and all writable ++ * through /sys/module/a9_gpu_drm/parameters on a live boot. Included from drm-render.inc ++ * before drm-render-v3.inc. The executor is serial and holds the core lock, so the state ++ * below needs no lock of its own. ++ * ++ * fw_tick_mask (boot.inc) ctl+0x04, re-applied before every job (RAM) ++ * slc_evict_us while a job is in flight, one pass over slc_evict_mib MiB of ++ * cacheable kernel memory at most every N us (the waiter's ++ * 1-2 ms sample cadence bounds it from below); slc_evict_ops: ++ * 1 read pass, 2 write pass, 4 Normal-NC read of the firmware's ++ * root+shared pages, 8 TLBI VMALLE1IS, 16 DSB SY + SEV ++ * cpu_sync 1 dma_sync every job BO (device before, CPU after), 2 clean ++ * root+shared to PoC before the publish, 4 clean+invalidate the ++ * polled pages before every sample ++ * lat123_log one RENDER-L123 line per job (times in ms from the submit ioctl) ++ * pstate_probe 1 PMGR gfx word at job start/end (always-on block), 2 also one ++ * read of Fender +0x1000 at the first sample that shows a fresh ++ * engine busy rise (engine running, so the block is powered) ++ * ++ * No other register access. Temporary: none of this is the fix. ++ */ ++ ++#include ++#include ++ ++#define G5P_L123_OP_READ BIT(0) ++#define G5P_L123_OP_WRITE BIT(1) ++#define G5P_L123_OP_FW_READ BIT(2) ++#define G5P_L123_OP_TLBI BIT(3) ++#define G5P_L123_OP_SEV BIT(4) ++#define G5P_L123_SYNC_BO BIT(0) ++#define G5P_L123_SYNC_REGIONS BIT(1) ++#define G5P_L123_SYNC_POLL BIT(2) ++#define G5P_L123_LINE 64 ++ ++static struct { ++ void *buf; ++ size_t size; ++ u64 sink; ++ /* per job */ ++ u64 exec_ns, next_ns, evict_ns, ps_ns; ++ u32 us, mib, ops, sync, passes, tick, ps; ++ u32 pmgr[2]; ++ bool seen[2], idle[2], probed; ++ /* render_user_event_wake: pauses ended by an event, the timeout, a post-event poll, legacy */ ++ u32 ew[4]; ++ u64 ew_ns; ++} g5p_l123; ++ ++/* perf60-fillrate.md §11.3 cut 1 (render-user-evwait.h): the phase wait's pause between RAM samples. */ ++static unsigned int render_user_event_wake = 2000; /* release8 default */ ++module_param(render_user_event_wake, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_event_wake, ++ "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " ++ "until the next firmware event (the IRQ thread's decode wakes the waiter), at " ++ "most N us (100..20000) as the safety net, and poll every 100 us once the " ++ "phase's own completion event is latched; read live"); ++ ++/* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ ++static u8 *g5p_l123_word(struct g5p_boot *b, u32 slot, u32 field) ++{ ++ u32 off = b->layout.offset[slot] + field; ++ struct g5p_dma_page *page = b->layout.group[slot] ? b->shared : b->root; ++ ++ return page[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_l123_pages(struct g5p_dma_page *page, u32 count, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ for (u32 i = 0; i < count; i++) ++ if (page[i].cpu) ++ op((unsigned long)page[i].cpu, ++ (unsigned long)page[i].cpu + G5P_GPU_PAGE); ++} ++ ++static void g5p_l123_page_of(const void *p, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ unsigned long start = (unsigned long)p & ++ ~(unsigned long)(G5P_GPU_PAGE - 1); ++ ++ op(start, start + G5P_GPU_PAGE); ++} ++ ++/* cpu_sync 4: the pages the waiter samples (rings, TA/3D/PM pages, the idle words). */ ++static void g5p_l123_poll_pages(struct g5p_boot *b, u32 e) ++{ ++ g5p_l123_page_of(g5p_boot_shared(b, 0x60), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_boot_shared(b, 0x64), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 0), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 1), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_pm_cpu(b), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_global(b, e) + 0x724, dcache_clean_inval_poc); ++ dsb(sy); ++} ++ ++static void g5p_l123_bos(struct g5p_drm *d, struct g5p_rjob *job, bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct sg_table *sgt = ++ to_drm_gem_shmem_obj(job->objs[i].obj)->sgt; ++ ++ if (!sgt) ++ continue; ++ if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ } ++} ++ ++static u32 g5p_l123_pmgr(struct g5p_diag *g) ++{ ++ u32 raw = U32_MAX; ++ ++ if (!((0) & 1) || !g->pmgr || regmap_read(g->pmgr, 0x80340, &raw)) ++ return U32_MAX; ++ return raw; ++} ++ ++/* At the executor's entry: this job's knob values and counters. */ ++static void g5p_l123_job_reset(struct g5p_diag *g) ++{ ++ g5p_l123.exec_ns = ktime_get_ns(); ++ g5p_l123.next_ns = 0; ++ g5p_l123.evict_ns = 0; ++ g5p_l123.ps_ns = 0; ++ g5p_l123.passes = 0; ++ g5p_l123.ps = U32_MAX; ++ g5p_l123.probed = false; ++ memset(g5p_l123.ew, 0, sizeof(g5p_l123.ew)); ++ g5p_l123.ew_ns = 0; ++ g5p_l123.seen[0] = false; ++ g5p_l123.seen[1] = false; ++ g5p_l123.us = (0); ++ g5p_l123.mib = clamp((16), 1U, 64U); ++ g5p_l123.ops = (1); ++ g5p_l123.sync = (0); ++ g5p_l123.pmgr[0] = g5p_l123_pmgr(g); ++ g5p_l123.pmgr[1] = U32_MAX; ++} ++ ++/* After the job's records are written, before they are cleaned and published. May sleep. */ ++static void g5p_l123_job_prepare(struct g5p_drm *d, struct g5p_rjob *job) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u8 *tick = g5p_l123_word(b, G5P_L123_CTL, 4); ++ u32 mask = (0); ++ size_t size = (size_t)g5p_l123.mib << 20; ++ ++ if (mask <= G5P_L123_TICK_MASK && g5p_le32(tick) != mask) { ++ g5p_put32(tick, mask); ++ dma_wmb(); ++ } ++ g5p_l123.tick = g5p_le32(tick); ++ ++ if (!g5p_l123.us || ++ !(g5p_l123.ops & (G5P_L123_OP_READ | G5P_L123_OP_WRITE))) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = NULL; ++ g5p_l123.size = 0; ++ } else if (g5p_l123.size != size) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = vmalloc(size); ++ g5p_l123.size = g5p_l123.buf ? size : 0; ++ if (g5p_l123.buf) ++ memset(g5p_l123.buf, 0x5a, size); ++ } ++ ++ if (g5p_l123.sync & G5P_L123_SYNC_BO) ++ g5p_l123_bos(d, job, false); ++ if (g5p_l123.sync & G5P_L123_SYNC_REGIONS) { ++ g5p_l123_pages(b->root, ARRAY_SIZE(b->root), dcache_clean_poc); ++ g5p_l123_pages(b->shared, ARRAY_SIZE(b->shared), ++ dcache_clean_poc); ++ dsb(sy); ++ } ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(b, 0); ++} ++ ++/* One streaming pass and the other per-pass operations. */ ++static void g5p_l123_evict(struct g5p_boot *b) ++{ ++ u64 t0 = ktime_get_ns(), sum = 0; ++ u32 ops = g5p_l123.ops; ++ ++ if ((ops & G5P_L123_OP_READ) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ sum += READ_ONCE(p[i]); ++ } ++ if ((ops & G5P_L123_OP_WRITE) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ WRITE_ONCE(p[i], i); ++ } ++ if (ops & G5P_L123_OP_FW_READ) { ++ struct g5p_dma_page *regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; ++ regions[r][i].cpu && at < G5P_GPU_PAGE; ++ at += G5P_L123_LINE) ++ sum += READ_ONCE(*( ++ const u32 *)(regions[r][i].cpu + ++ at)); ++ } ++ if (ops & G5P_L123_OP_TLBI) ++ flush_tlb_all(); ++ if (ops & G5P_L123_OP_SEV) { ++ dsb(sy); ++ sev(); ++ } ++ WRITE_ONCE(g5p_l123.sink, g5p_l123.sink + sum); ++ g5p_l123.passes++; ++ g5p_l123.evict_ns += ktime_get_ns() - t0; ++} ++ ++/* Replaces the waiter's sleep between two RAM samples of engine e. seen: d->event_serial as ++ * read after the drain and before this sample (render-user-evwait.h). ++ */ ++static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, ++ const struct g5p_rv3_ev *s, u64 seen) ++{ ++ u64 now = s->ns, t0; ++ enum g5p_ew_act act; ++ u32 wait_us; ++ ++ if (((0) & 2) && !g5p_l123.probed && g5p_l123.seen[e] && ++ g5p_l123.idle[e] && !s->idle_ok && !s->failed && !s->proto && ++ !g5p_ru_term_failed(&g5p_rv3_term) && !READ_ONCE(g->boot->failed)) { ++ struct g5p_mmio_ticket t; ++ ++ /* an R1-only ticket of the register licence (mmio.inc), never a bare read */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ g5p_l123.ps = t.gpc; ++ g5p_l123.ps_ns = ktime_get_ns(); ++ g5p_l123.probed = true; ++ } ++ g5p_l123.seen[e] = true; ++ g5p_l123.idle[e] = s->idle_ok; ++ if (g5p_l123.us && now >= g5p_l123.next_ns) { ++ g5p_l123_evict(g->boot); ++ g5p_l123.next_ns = now + (u64)g5p_l123.us * NSEC_PER_USEC; ++ } ++ act = g5p_ew_plan(READ_ONCE(render_user_event_wake), s->event, ++ &wait_us); ++ t0 = ktime_get_ns(); ++ if (act == G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ g5p_l123.ew[3]++; ++ } else if (act == G5P_EW_SHORT) { ++ usleep_range(wait_us, wait_us + wait_us / 2); ++ g5p_l123.ew[2]++; ++ } else if (wait_event_hrtimeout( ++ d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us))) { ++ g5p_l123.ew[1]++; /* -ETIME: the safety net */ ++ } else { ++ g5p_l123.ew[0]++; ++ } ++ g5p_l123.ew_ns += ktime_get_ns() - t0; ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(g->boot, e); ++} ++ ++/* After the receipt. One line per job while lat123_log is set. */ ++static void g5p_l123_job_end(struct g5p_drm *d, struct g5p_rjob *job, u32 seq, ++ int ret) ++{ ++ struct g5p_diag *g = d->core; ++ ++ if (!ret && (g5p_l123.sync & G5P_L123_SYNC_BO)) ++ g5p_l123_bos(d, job, true); ++ g5p_l123.pmgr[1] = g5p_l123_pmgr(g); ++ { ++ return; ++ } ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-prime.inc b/drivers/gpu/drm/apple-g5p/drm-prime.inc +new file mode 100644 +index 0000000000..bd513b9695 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-prime.inc +@@ -0,0 +1,114 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* PRIME for shmem GEM BOs. ++ * ++ * Export: Mesa WSI/client buffers and kmsro bookkeeping. The dma-buf adds ++ * begin/end_cpu_access, so an importer that CPU-reads (simpledrm's blit, ++ * llvmpipe) gets cache maintenance on the G5P DMA mapping. vmap/mmap reuse ++ * the shmem helpers (write-combined when the device is not DMA-coherent). ++ * ++ * Import: foreign dma-bufs, e.g. simpledrm dumb buffers that the GPU renders ++ * into for scanout. The attachment stays mapped (pinned) for the life of the ++ * GEM object; VM_BIND maps its sg_table. Render jobs listing an imported BO ++ * get whole-buffer cache maintenance and dma_resv fences (drm-render.inc). ++ */ ++ ++static int g5p_dmabuf_begin_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ /* No device mapping yet means no device-owned cache lines either. */ ++ if (sgt) ++ dma_sync_sgtable_for_cpu(obj->dev->dev, sgt, DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static int g5p_dmabuf_end_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ if (sgt) ++ dma_sync_sgtable_for_device(obj->dev->dev, sgt, ++ DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static const struct dma_buf_ops g5p_dmabuf_ops = { ++ .attach = drm_gem_map_attach, ++ .detach = drm_gem_map_detach, ++ .map_dma_buf = drm_gem_map_dma_buf, ++ .unmap_dma_buf = drm_gem_unmap_dma_buf, ++ .release = drm_gem_dmabuf_release, ++ .mmap = drm_gem_dmabuf_mmap, ++ .vmap = drm_gem_dmabuf_vmap, ++ .vunmap = drm_gem_dmabuf_vunmap, ++ .begin_cpu_access = g5p_dmabuf_begin_cpu_access, ++ .end_cpu_access = g5p_dmabuf_end_cpu_access, ++}; ++ ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags) ++{ ++ struct dma_buf_export_info info = { ++ .exp_name = KBUILD_MODNAME, ++ .owner = obj->dev->driver->fops->owner, ++ .ops = &g5p_dmabuf_ops, ++ .size = obj->size, ++ .flags = flags, ++ .priv = obj, ++ .resv = obj->resv, ++ }; ++ ++ return drm_gem_dmabuf_export(obj->dev, &info); ++} ++ ++static int g5p_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file, ++ u32 handle, u32 flags, int *fd) ++{ ++ return drm_gem_prime_handle_to_fd(dev, file, handle, flags, fd); ++} ++ ++static int g5p_prime_fd_to_handle(struct drm_device *dev, struct drm_file *file, ++ int fd, u32 *handle) ++{ ++ return drm_gem_prime_fd_to_handle(dev, file, fd, handle); ++} ++ ++/* Our own export comes back as the same GEM object (the core's identity check ++ * only knows its default ops); anything else is attached and mapped. ++ */ ++static struct drm_gem_object *g5p_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *buf) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ ++ if (buf->ops == &g5p_dmabuf_ops && obj->dev == dev) { ++ drm_gem_object_get(obj); ++ return obj; ++ } ++ return drm_gem_prime_import(dev, buf); ++} ++ ++/* Foreign buffers are GPU READ|WRITE data, never executable, and must be ++ * whole 16KiB GPU pages within the BO size limit. ++ */ ++static struct drm_gem_object * ++g5p_prime_import_sg_table(struct drm_device *dev, ++ struct dma_buf_attachment *attach, ++ struct sg_table *sgt) ++{ ++ struct drm_gem_object *obj; ++ ++ if (attach->dmabuf->size & (G5P_GPU_PAGE - 1) || ++ attach->dmabuf->size > G5P_MAX_BO_SIZE) ++ return ERR_PTR(-EINVAL); ++ obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); ++ if (!IS_ERR(obj)) { ++ to_g5p_bo(obj)->roles = DRM_G5P_BO_READ | DRM_G5P_BO_WRITE; ++ g5p_bo_track(obj); ++ } ++ return obj; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +new file mode 100644 +index 0000000000..21dc64d89a +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -0,0 +1,203 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * The render scheduler (docs/gpu/multi-client-design.md "Stage 2/3 as built"): one worker on ++ * the ordered render_wq serves the contexts round-robin (core/render-user-sched.h). Jobs wait ++ * for their inputs on dma-fence callbacks, not in the worker, so one client's unsignalled ++ * fence never holds up another client. Each picked job runs to completion on the one shared ++ * channel (one credit), exactly as a work item did before; a context's close item runs after ++ * its last job. Included from drm-render.inc after drm-render-v3.inc. ++ * ++ * The shape is drm_sched's (entity FIFO, run-list RR, dependency callbacks, entity teardown ++ * after the last job), so stage 4 can replace it with drm_sched without a UAPI change. ++ */ ++ ++static struct dma_fence *g5p_rjob_dep(struct g5p_rjob *job, u32 i) ++{ ++ return i < job->input_count ? job->inputs[i] : ++ job->deps[i - job->input_count]; ++} ++ ++/* A signalled input: its error, or that of any fence it wraps (array, chain), else 0. */ ++static int g5p_render_fence_status(struct dma_fence *fence) ++{ ++ struct dma_fence_unwrap cursor; ++ struct dma_fence *part; ++ u32 unwrapped = 0; ++ int ret = dma_fence_get_status(fence); ++ ++ if (ret < 0) ++ return ret; ++ dma_fence_unwrap_for_each(part, &cursor, fence) { ++ ret = dma_fence_get_status(part); ++ if (ret <= 0 || ++unwrapped > 4096) { ++ dma_fence_put(cursor.chain); ++ return ret < 0 ? ret : -ETIMEDOUT; ++ } ++ } ++ return 0; ++} ++ ++/* Fence callback (any context, the fence lock held): the head job's input signalled. */ ++static void g5p_rjob_dep_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++{ ++ struct g5p_rjob *job = container_of(cb, struct g5p_rjob, cb); ++ ++ WRITE_ONCE(job->cb_fired, true); ++ mod_delayed_work(job->gpu->render_wq, &job->gpu->rdev->sched_work, 0); ++} ++ ++/* Scheduler lock held: no callback of this job stays armed. */ ++static void g5p_rjob_disarm(struct g5p_rjob *job) ++{ ++ if (job->cb_fence) ++ dma_fence_remove_callback(job->cb_fence, &job->cb); ++ job->cb_fence = NULL; ++} ++ ++/* ++ * Scheduler lock held: may the head job of its context run now? Walks its explicit inputs, ++ * then its implicit dependencies, in order; at the first unsignalled one it arms a callback ++ * and reports the job's input deadline. A failed input, the deadline (G5P_RU_INPUT_MS after ++ * the submit, as the old in-worker wait) and closed admission make it ready with an error, so ++ * that it fails before the hardware and its fence still signals in order. ++ */ ++static bool g5p_rsched_ready(struct g5p_rs_job *node, void *cookie, ++ unsigned long *deadline) ++{ ++ struct g5p_rjob *job = container_of(node, struct g5p_rjob, node); ++ ++ if (job->setup_error || job->dep_error) ++ return true; ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ESHUTDOWN; ++ return true; ++ } ++ if (job->cb_fence) { ++ if (!READ_ONCE(job->cb_fired)) ++ goto pending; ++ job->cb_fence = NULL; ++ } ++ while (job->dep_next < job->input_count + job->dep_count) { ++ struct dma_fence *fence = g5p_rjob_dep(job, job->dep_next); ++ ++ if (dma_fence_is_signaled(fence)) { ++ job->dep_error = g5p_render_fence_status(fence); ++ if (job->dep_error) ++ return true; ++ job->dep_next++; ++ continue; ++ } ++ WRITE_ONCE(job->cb_fired, false); ++ if (dma_fence_add_callback(fence, &job->cb, g5p_rjob_dep_cb)) ++ continue; /* -ENOENT: signalled meanwhile */ ++ job->cb_fence = fence; ++ goto pending; ++ } ++ return true; ++pending: ++ if (time_after_eq(jiffies, job->deadline)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ETIMEDOUT; ++ return true; ++ } ++ *deadline = job->deadline; ++ return false; ++} ++ ++static void g5p_rsched_kick(struct g5p_drm *d) ++{ ++ mod_delayed_work(d->render_wq, &d->rdev->sched_work, 0); ++} ++ ++/* Submit, render_submit_lock held: an accepted job joins its context's FIFO. */ ++static void g5p_rsched_push(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ g5p_rs_push(&d->rdev->sched, &r->entity, &job->node); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ g5p_rsched_kick(d); ++} ++ ++/* The close item joins the context's entity behind its jobs; false if one is queued. */ ++static bool g5p_rsched_close(struct g5p_drm *d, struct g5p_render *r) ++{ ++ unsigned long flags; ++ bool queued; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ queued = g5p_rs_close(&d->rdev->sched, &r->entity); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ if (queued) ++ g5p_rsched_kick(d); ++ return queued; ++} ++ ++static void g5p_render_run(struct g5p_rjob *job); ++static void g5p_render_close_run(struct g5p_render *r); ++static void g5p_render_close_deferred(struct g5p_drm *d); ++ ++/* ++ * The worker: pick, run to completion, repeat; then sleep until a callback, a submit or the ++ * earliest input deadline wakes it. render_wq is ordered, so this never runs twice at once, ++ * and it is the only writer of the device's job state, as each job's work item was. ++ */ ++static void g5p_rsched_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = container_of(to_delayed_work(work), ++ struct g5p_rdev, sched_work); ++ struct g5p_drm *d = rd->drm; ++ ++ /* Each item drops the references it held; the device must outlive this loop. */ ++ drm_dev_get(&d->drm); ++ for (;;) { ++ struct g5p_rs_entity *e; ++ struct g5p_rs_job *node; ++ struct g5p_render *r; ++ struct g5p_context *c; ++ unsigned long flags, wake = 0; ++ bool has_wake; ++ int what; ++ ++ /* A lost job blocks the channel: hold every job until the drain is decided, and ++ * until a recovery the drain or the executor asked for has run. ++ */ ++ if (READ_ONCE(rd->stuck) || READ_ONCE(rd->recovering)) { ++ if (rd->stuck ? g5p_rv3_drain(d) : g5p_rv3_recover(d)) ++ continue; ++ queue_delayed_work(d->render_wq, &rd->sched_work, ++ msecs_to_jiffies(100)); ++ break; ++ } ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ what = g5p_rs_pick(&rd->sched, g5p_rsched_ready, d, &e, &node, ++ &wake, &has_wake); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ if (what == G5P_RS_IDLE) { ++ if (has_wake) ++ queue_delayed_work( ++ d->render_wq, &rd->sched_work, ++ max_t(long, (long)(wake - jiffies), 1)); ++ break; ++ } ++ if (what == G5P_RS_JOB) { ++ g5p_render_run( ++ container_of(node, struct g5p_rjob, node)); ++ g5p_render_close_deferred(d); ++ continue; ++ } ++ r = container_of(e, struct g5p_render, entity); ++ c = r->ctx; ++ g5p_render_close_run(r); ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ g5p_rs_closed(&rd->sched, e); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ /* The close item's own reference (g5p_render_close()); r may go with it. */ ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ drm_dev_put(&d->drm); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +new file mode 100644 +index 0000000000..36550c4432 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -0,0 +1,488 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Unbinding firmware slots (docs/gpu/multi-client-design.md §4.5): a closed context's pair ++ * (stage 2, core/render-user-close.h) and the render frontend's share of the reboot quiesce ++ * (core/render-user-quiesce.h, docs/gpu/experiments/mcfold2-kexec.md). ++ * Included from drm-render.inc after drm-render-v3.inc. ++ */ ++ ++/* The RAM words of `slot` and of its unbind record at ring entry `index`. */ ++static void g5p_render_slot_mem(struct g5p_boot *b, u32 slot, u32 index, ++ struct g5p_ru_slot_mem *m) ++{ ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + (index & 255) * 20 + j * 4; ++ ++ m->rec[j] = ++ b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ } ++ m->pa = g5p_boot_shared(b, 0x86) + slot * 8; ++ m->host = g5p_boot_shared(b, 0x84) + slot * 4; ++ m->cached = g5p_boot_shared(b, 0x85) + slot * 4; ++} ++ ++/* ++ * Unbind `n` firmware slots in one RAM-only DC3 batch: PA and host tag of each to zero, one ++ * record {3, slot} per slot, one producer store, then the RAM-only wait of a bind and a check ++ * that the firmware's DC3 handler copied the zero tag of every slot. No doorbell and no ++ * register access, so it is safe after a job retired and while the GPU may be idle-gated. ++ * ++ * The caller holds the core lock (the DC ring has one writer) and has retired every use of ++ * these slots: no job of theirs is queued on the firmware or in flight. On success the ++ * firmware holds, per slot, PA 0, host and cached tag 0 and CBASE 0, with the slot's PC level ++ * invalidated; the slot may be bound again by a later DC3. Its translations stay allocated: ++ * freeing them is the caller's step, after this returns 0. A batch that is not consumed in ++ * time keeps later binds and unbinds refused until the firmware catches up (dc_pending). ++ */ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_slot_sample s = {}; ++ struct g5p_ru_slot_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, cons, target, bad = 0, verified = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!g5p_ru_unbind_fits(n)) ++ return -EINVAL; ++ for (u32 i = 0; i < n; i++) ++ if (slots[i] < G5P_RU_SLOT_MIN || slots[i] > G5P_RU_SLOT_MAX) ++ return -EINVAL; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ cons = READ_ONCE(ctl[0]); ++ if (start > 255 || cons != start || ++ (rd->dc_pending && cons != rd->dc_wait)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s refused slots=%u dc=%u/%u pending=%u: ring busy; no " ++ "hardware touched end=v3\n", ++ why, n, cons, start, rd->dc_pending); ++ return -EBUSY; ++ } ++ rd->dc_pending = false; ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_unbind_publish(&m, slots[i]); ++ } ++ target = (start + n) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, why); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ for (u32 i = 0; i < n; i++) { ++ u32 one; ++ ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_slot_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ one = g5p_ru_unbind_check(&s, slots[i], target); ++ bad |= one; ++ verified += !one; ++ } ++ if (!ret && bad) ++ ret = -EUCLEAN; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s slots=%u verified=%u ret=%d consumed_us=%lld " ++ "dc=%u->%u/%u bad=%#x first=%u last=%u mode=ram-dc3 end=v3\n", ++ why, n, verified, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ s.cons, s.prod, bad, slots[0], slots[n - 1]); ++ return ret; ++} ++ ++/* ++ * render_user_close_park: bind a closed context's just-unbound pair to the park catalogue, one ++ * DC3 pair as a context's bind (g5p_render_vm_prepare()), checked the same way. The catalogue ++ * holds the firmware window's entry and the borrowed sink entry, both stable within an epoch, so ++ * rewriting them for a later park never clears what an earlier parked pair resolves through. ++ * Core lock held, between jobs (the close item), RAM only and no register: as the unbind. ++ */ ++static int g5p_rv3_park(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, target, entry, sink, bad = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!rd->park.cpu) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->park); ++ if (ret) ++ return ret; ++ memset(rd->park.cpu, 0, G5P_GPU_PAGE); ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (start > 255 || rd->dc_pending || g5p_ru_bind_busy(&s)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park refused slots=%u/%u dc=%u/%u pending=%u busy=%#x: the pair stays " ++ "unbound end=v3\n", ++ r->slot[0], r->slot[1], s.cons, start, rd->dc_pending, ++ g5p_ru_bind_busy(&s)); ++ return -EBUSY; ++ } ++ sink = b->sink_ready ? g5p_le32(b->catalogue.cpu) : 0; ++ if (g5p_le32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4) != entry) ++ g5p_put32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4, entry); ++ if (g5p_le32(rd->park.cpu) != sink) ++ g5p_put32(rd->park.cpu, sink); ++ g5p_table_clean_page(g, &rd->park); ++ dsb(sy); ++ g5p_ru_bind_publish(&m, r->slot, rd->park.dma, G5P_RU_PARK_TAG); ++ target = (start + 2) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "park"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, rd->park.dma, ++ G5P_RU_PARK_TAG, target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ rd->parks += !ret; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park slots=%u/%u ret=%d consumed_us=%lld dc=%u->%u/%u bad=%#x " ++ "parks=%u pc=%pad end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ start, s.cons, s.prod, bad, rd->parks, &rd->park.dma); ++ return ret; ++} ++ ++/* Drop what the claim of the device took (g5p_render_device_claim()); the device stays ++ * claimed for render. Returns whether a context reference was dropped. ++ */ ++static bool g5p_render_unclaim(struct g5p_drm *d, struct g5p_render *r) ++{ ++ if (!r->claimant) ++ return false; ++ r->claimant = false; ++ if (d->quarantined_context == r->ctx) ++ d->quarantined_context = NULL; ++ return true; ++} ++ ++/* ++ * The close item of a TYPED_V3 context (multi-client-design.md "Stage 2/3 as built"): the ++ * scheduler runs it after every job of the context, so every use of the pair has retired or ++ * failed. RAM only: the unbind is g5p_rv3_unbind_slots() for this pair under the core lock. ++ * After a verified unbind the binding's references go, and the last one frees the VM tables, ++ * render targets and mappings (unpinning the BOs) and returns the slots ++ * (g5p_render_release()). Anything the firmware may still hold is retained until reboot. ++ */ ++/* perf60 (docs/gpu/experiments/perf60-clock.md), corrected by the release8 bench (bench-gaps3, ++ * render-user-pmset.h): the PM writes a previous PM context back (its free list F, through that ++ * context's TA slot) not once at the switch but on the later jobs of the next context, until a ++ * third context runs. A closed context therefore keeps its slots bound and its set and pages alive ++ * while it is the last job's context or the previous distinct one; its unbind and release run once ++ * it is neither. At most those two wait. (The first version held only the last job's context and ++ * released it after one other job; 2 held one job longer, by job records: both released it while ++ * the PM still wrote to it.) RAM only: bookkeeping and the close item's references. ++ */ ++/* release8 default */ ++ ++/* ++ * A close item follows every job of this context on the ordered render worker. ++ * resident_gone, set under the file lock before queueing it, rejects all later ++ * VM_BIND and context creation. Thus the close item owns vm->maps exclusively. ++ * ++ * Idle VM_BIND UNMAP already drops a user mapping's pages after retirement. ++ * Do that here without releasing the PM's write-back route: no unbind, no ++ * table pruning, no PM set/pool/aux/RT release, no pm_last/pm_prev change. The ++ * PM free list is in the firmware window, never in vm->maps. All user mappings ++ * must pass the same range check as VM_BIND before any PTE is touched. ++ * ++ * No new kick of this detached context can occur. Clean the cleared user PTEs ++ * to PoC, but do not publish a tag, invalidate a slot or access R0. The PM can ++ * keep using its unchanged translations and pages, including cached ones. ++ * Failed/queued/terminal/shutdown epochs keep every BO, as before. ++ */ ++static void g5p_render_close_trim(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_boot *b; ++ struct g5p_ru_close_in in; ++ struct g5p_mapping *m, *tmp; ++ struct g5p_pt *pt; ++ u64 pinned; ++ u32 maps, why; ++ ++ if (!vm->map_count || !vm->resident_gone || !r->closing || ++ !r->detached || !mutex_trylock(&g->lock)) ++ return; ++ b = g->boot; ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* Match close_run: a successfully drained lost job has no user-memory use left. */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .dc_pending = d->rdev->dc_pending, ++ .queued = atomic_read(&r->queued), ++ }; ++ if (g5p_ru_close_plan(&in, &why) != G5P_RU_CLOSE_UNBIND) ++ goto out; ++ list_for_each_entry(m, &vm->maps, link) ++ if (!g5p_user_range_valid(m->node.start, m->node.size) || ++ (m->node.start < G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE && ++ G5P_RU_PRIVATE_VA < m->node.start + m->node.size)) ++ goto out; ++ ++ pinned = vm->pinned_bytes; ++ maps = vm->map_count; ++ list_for_each_entry(m, &vm->maps, link) ++ g5p_vm_clear(vm, d->drm.dev, m->node.start, m->node.size); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ list_for_each_entry_safe(m, tmp, &vm->maps, link) { ++ vm->mapped_bytes -= m->node.size; ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-trim slots=%u/%u maps=%u unpinned=%lluK remaining=%lluK " ++ "device_pinned=%lluK pm=retained end=v3\n", ++ r->slot[0], r->slot[1], maps, (pinned - vm->pinned_bytes) >> 10, ++ vm->pinned_bytes >> 10, READ_ONCE(d->pinned_bytes) >> 10); ++out: ++ mutex_unlock(&g->lock); ++} ++ ++/* render_user_close_defer: may the firmware's PM still write this context's state back? */ ++static bool g5p_render_pm_holds(struct g5p_rdev *rd, struct g5p_render *r, ++ u32 mode) ++{ ++ return mode && g5p_pmset_holds(rd->pm_last, rd->pm_prev, r); ++} ++ ++static void g5p_render_close_run(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_close_in in; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 puts = 0, why; ++ bool locked = false; ++ int plan, ret = 0; ++ u32 mode, k; ++ ++ mode = (1); ++ k = d->rdev->close_deferred[0] ? 1 : 0; ++ if (mode && !r->close_defer_run && r->bound && ++ g5p_render_pm_holds(d->rdev, r, mode) && ++ k < ARRAY_SIZE(d->rdev->close_deferred)) { ++ g5p_render_close_trim(r); ++ /* the deferred close's own references, dropped by g5p_render_close_deferred() */ ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->rdev->close_deferred[k] = r; ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer slots=%u/%u jobs=%u: the PM may still write its state " ++ "back; unbind once it has moved past it end=v3\n", ++ r->slot[0], r->slot[1], r->jobs); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer2 slots=%u/%u pm=%s pending=%u end=v3\n", ++ r->slot[0], r->slot[1], ++ d->rdev->pm_last == r ? "last" : "previous", k + 1); ++ return; ++ } ++ ++ for (;;) { ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* a lost job that drained left the firmware nothing of this context */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ /* as g5p_rv3_unbind_slots() refuses it */ ++ .dc_pending = b && d->rdev->dc_pending && ++ READ_ONCE(((u32 *)g5p_boot_shared( ++ b, 0x6c))[0]) != d->rdev->dc_wait, ++ .queued = atomic_read(&r->queued), ++ }; ++ plan = g5p_ru_close_plan(&in, &why); ++ if (plan != G5P_RU_CLOSE_UNBIND || locked) ++ break; ++ /* Bounded, as a job takes it: never wait forever behind a command in MMIO. */ ++ while (!(locked = mutex_trylock(&g->lock)) && ++ time_before(jiffies, deadline)) ++ msleep(5); ++ if (!locked) { ++ plan = G5P_RU_CLOSE_RETAIN; ++ why = G5P_RU_CLOSE_LOCK; ++ break; ++ } ++ /* Plan again under the lock: the reboot quiesce or a failure may have run. */ ++ } ++ if (plan == G5P_RU_CLOSE_UNBIND) { ++ ret = g5p_rv3_unbind_slots(d, r->slot, 2, "close"); ++ if (!ret) { ++ r->bound = false; ++ r->retained = false; ++ WRITE_ONCE(c->live, false); ++ puts++; ++ /* the PM may still write this context's free list back through its TA slot */ ++ { ++ g5p_rv3_park(d, r); ++ }; ++ } ++ } ++ if (plan == G5P_RU_CLOSE_RELEASE || ++ (plan == G5P_RU_CLOSE_UNBIND && !ret)) ++ puts += g5p_render_unclaim(d, r); ++ if (locked) ++ mutex_unlock(&g->lock); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close slots=%u/%u plan=%s why=%#x ret=%d jobs=%u claimant=%u " ++ "retained=%u end=v3\n", ++ r->slot[0], r->slot[1], g5p_ru_close_name(plan), why, ret, ++ r->jobs, r->claimant, r->retained); ++ while (puts--) { ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++} ++ ++/* After a job: a deferred close runs once the last job was another context's. */ ++static void g5p_render_close_deferred(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ u32 mode = (1); ++ ++ for (u32 k = 0; k < ARRAY_SIZE(rd->close_deferred); k++) { ++ struct g5p_render *r = rd->close_deferred[k]; ++ struct g5p_context *c; ++ ++ /* mode 0 now: a pending close still waits until the PM has moved past it */ ++ if (!r || g5p_render_pm_holds(rd, r, mode ?: 1)) ++ continue; ++ rd->close_deferred[k] = NULL; ++ c = r->ctx; ++ r->close_defer_run = true; ++ dev_info( ++ d->core->dev, ++ "RENDER-V3 close-deferred slots=%u/%u: neither the PM's last nor its previous " ++ "context end=v3\n", ++ r->slot[0], r->slot[1]); ++ g5p_render_close_run(r); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ /* keep a pending close in [0]: close_run fills [1] only behind it */ ++ if (!rd->close_deferred[0] && rd->close_deferred[1]) { ++ rd->close_deferred[0] = rd->close_deferred[1]; ++ rd->close_deferred[1] = NULL; ++ } ++} ++ ++/* ++ * The handle of a TYPED_V3 context is gone (DESTROY_CONTEXT, or its file closed): stop VM and ++ * render-target edits and queue the close item behind the context's jobs (g5p_rsched_close()). ++ * File lock held, or the file is closing. A context that never submitted is released by its ++ * last reference. ++ */ ++static void g5p_render_close(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ ++ if (!r->v3_accepted || r->closing) ++ return; ++ r->closing = true; ++ r->detached = true; ++ c->vm->resident_gone = true; ++ /* The close item's references, dropped by the scheduler once it ran. */ ++ kref_get(&c->ref); ++ drm_dev_get(&c->gpu->drm); ++ if (!g5p_rsched_close(c->gpu, r)) { ++ g5p_context_put(c); ++ drm_dev_put(&c->gpu->drm); ++ } ++} ++ ++/* Reboot quiesce, RAM only and lock free: the TYPED_V3 latch and whether its executor runs. ++ * The executor is RAM-only once the latch is quiet, unless the wake110 experiment (doorbells ++ * after retirement, off in every desktop tuple) is configured. Called after admission closed; ++ * the scheduler is kicked so that jobs waiting on an input fail now, not at their deadline. ++ */ ++static void g5p_kq_drm_state(void *cookie, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ ++ /* Admission is closed by now: queued jobs blocked on an input fail at once. */ ++ g5p_rsched_kick(d); ++ ++ /* A channel blocked behind a lost job is a wedged epoch for the quiesce. */ ++ in->term_failed = g5p_ru_term_failed(&g5p_rv3_term) || ++ READ_ONCE(d->rdev->stuck) || ++ READ_ONCE(d->rdev->recovering); ++ in->ram_only = READ_ONCE(d->rdev->v3) && ++ !g5p_ru_mmio_ok(&g5p_rv3_term) && !(2); ++ in->executing = READ_ONCE(d->rdev->executing); ++} ++ ++/* Reboot quiesce, core lock held: every held user slot, the first claimant's pair included. ++ * Nothing is freed and no context changes state: admission is closed and the boot state is ++ * frozen right after, so no job runs again on this boot. ++ */ ++static int g5p_kq_drm_detach(void *cookie, const u32 *slots, u32 n) ++{ ++ return g5p_rv3_unbind_slots(cookie, slots, n, "kexec"); ++} ++ ++static const struct g5p_kq_frontend g5p_kq_drm = { ++ .state = g5p_kq_drm_state, ++ .detach = g5p_kq_drm_detach, ++}; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +new file mode 100644 +index 0000000000..ac4dd841e3 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_V3 serial executor: per-job state (receipt data). Types only; code: drm-render-v3.inc. */ ++#ifndef G5P_DRM_RENDER_V3_H ++#define G5P_DRM_RENDER_V3_H ++ ++enum g5p_rv3_phase { ++ G5P_RV3_IDLE, ++ G5P_RV3_ATTACH, ++ G5P_RV3_BIND, ++ G5P_RV3_TA_PUBLISH, ++ G5P_RV3_TA_WAIT, ++ G5P_RV3_TD_PUBLISH, ++ G5P_RV3_TD_WAIT, ++ G5P_RV3_DONE, ++ G5P_RV3_PHASES, ++}; ++ ++/* g5p_drm.td_want: the 3D completion-event latch is armed for the endpoint in the low byte. */ ++#define G5P_RV3_TD_WANT 0x100U ++ ++/* One RAM sample of one engine: ring, stamps, channel watermarks, idle words, PM state, and ++ * (3D) whether the matching completion event has been latched. ++ */ ++struct g5p_rv3_ev { ++ u64 ns; ++ u32 prod, cons, stamp[2], ctl[5], idle[6], pm80; ++ bool proto, failed, consumed, retired, idle_ok, pm_ok, event; ++}; ++ ++struct g5p_rv3 { ++ struct g5p_ru_wait ta, td; ++ struct g5p_ru_touch ++ touch[2]; /* RC4a A/B (render-user-touch.h); zero when off */ ++ struct g5p_ru_retire rt; /* the 3D predicate (render-user-retire.h) */ ++ struct g5p_ru_ta tr; /* the TA predicate (render-user-retire.h) */ ++ struct g5p_rv3_ev last[2]; ++ u64 td_event_ns; /* decode time of the matching 3D event; 0 = none */ ++ u64 ta_event_ns; /* decode time of the first TA event since the publish */ ++ u64 attach_ns, job_deadline_ns, ta_pub_ns, td_pub_ns, ta_consumed_ns, ++ td_consumed_ns; ++ u32 phase, grace_end, bell; ++ u32 seq; /* the device sequence this job got; 0 = none */ ++ /* wake110 (render-user-wake.h), per engine: the decision, the power evidence word it saw ++ * (wake111), the two register words it read (U32_MAX = not read), the consumer it saw, the ++ * type-1 event counts at the phase's arm and at the job's end, the lock window and the ++ * first type-1 decode time. Zero when off. ++ */ ++ u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], ++ wk_t1_pub[2], wk_t1_end[2]; ++ u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ ++ /* The bind of this job's context published its DC3 pair and then failed: that context is ++ * lost, the device and the contexts already bound are not. ++ */ ++ bool bind_failed; ++ /* The job was lost alone (render-user-loss.h): its context is dead, the device is not; the ++ * channel stays blocked behind its stuck phase until the scheduler's drain clears it. ++ */ ++ bool lost; ++}; ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +new file mode 100644 +index 0000000000..f6080f49a3 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -0,0 +1,3033 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial executor: the control flow that completed the natural TA and the ++ * natural 3D in r96 (docs/gpu/experiments/production-typed-render-kernel.md), on typed ++ * state. Included from drm-render.inc after g5p_render_execute(). ++ * ++ * What it does, in order, for each job a V3 context runs (strictly serial across every context, ++ * render-user-serial.h; only the epoch's job 1 attaches and has hardware evidence): ++ * 0. bind, the first job of every later context (multi-client-design.md §4.2): its own DC3 ++ * pair, published in RAM only, no doorbell, a RAM-only wait for the firmware to consume ++ * it. With the TEMPORARY render_user_spare_pairs instead: a pair job 1 pre-bound, taken ++ * over through the tag path. ++ * 1. attach (job 1): the typed preparation (PM, channels, catalogue entries), then the ++ * DC3 pair is published and its doorbell is owned by the r74f machinery (r87b ++ * self-drain: a closed gate is never repaired by the host). The firmware installs ++ * the page-table root and the generation tags itself (r74g); the host does NO MMU pass ++ * and checks the result in RAM only. ++ * 2. TA: record, WORK_CTL and producer, no doorbell; the r74g grace (four ALIVEs or ++ * 20 ms) lets the firmware drain it, and only if it is still pending afterwards the ++ * first ALIVE decoded owns one fenced fallback doorbell. ++ * 3. TA wait: RAM only (drain, sample, sleep), 20 s per phase by default, 30 s for the ++ * whole job (render-user-wait.h decides the deadline and closes the race with a ++ * completion). tahang1: complete on a coherent target with the PM saved and either ++ * the idle words clear, the TA completion event, or a 500 ms coherent hold ++ * (render-user-retire.h); a phase pending 1 s prints one RAM-only `slow` line. From ++ * the TA complete on, and after any timeout or fault, the host touches no register. ++ * 4. 3D: record, WORK_CTL and producer, NO doorbell (r96 stage 1: the firmware consumed ++ * it 2 ms later), RAM-only wait as above. ++ * RC4a (render_user_touch_ms, default 0 = off): at publish + T of each phase, odd jobs read ++ * the receipt pages once and even jobs only record the instant (render-user-touch.h); RAM only. ++ * repeat109: the 3D phase is also complete when its target state is coherent and its completion ++ * event arrived, even if the engine idle words never clear (render-user-retire.h; the latched ++ * render_user_idle_settle_ms, default 0, adds a minimum coherent run); one `RENDER-V3 retire` ++ * receipt line splits the firmware's retirement from the host's decision. RAM only. ++ * wake110 (render_user_wake, default 0 = off): right after the producer store of every phase but ++ * job 1's TA, the A/B's factor phases run Apple's submit kick (R1+0x14 = 7, GPC active, R0+0xb00 = ++ * 0x10 / 0x11) and its control phases the same guards without a register (render-user-wake.h); ++ * one `RENDER-V3 wake` line per job. The only register access after the TA complete, licensed ++ * while no timeout or fault was noted. ++ * Jobs >= 2 (a compositor's next frame, or another context's): no attach, the PM pool is ++ * rebuilt, and both rings are published with no doorbell at all. ++ * Not in this path on purpose: the r74h / r80 / r85-r96 root-cause instrumentation (stream ++ * markers, PM canary, plateau and end banks, region/target dumps), host MMU passes, cache ++ * maintenance MMIO, selector reads, snapshots (they read R0), and any recovery. A timeout ++ * or fault retains everything; the context is frozen and kept until reboot. ++ * ++ * Terminal state (render-user-terminal.h): the TA complete, a timeout and a fault go through one ++ * first-writer-wins latch, g5p_rv3_terminal(). From then on g5p_ru_mmio_ok() is false, the admission ++ * refuses every job that would start behind a failure (-EIO), and the legacy typed paths that read or ++ * write registers (snapshot, invalidate, run, destroy) refuse. g5p_rv3_td_hook() is the optional ++ * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. ++ */ ++ ++static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { ++ "idle", "attach", "bind", "ta-publish", ++ "ta-wait", "3d-publish", "3d-wait", "done", ++}; ++static const char *const g5p_rv3_results[] = { "pending", "complete", "timeout", ++ "fault" }; ++static const char *const g5p_rv3_how[] = { "none", "idle-clear", "idle-stuck", ++ "hold" }; ++ ++#include "drm-latency123.inc" ++ ++/* perf60 stage 4 (render-user-pmset.h): per-context PM records, the iOS model (one ParamBuffer ++ * per work queue, 6cb9afc/6cb2378). Latched at the epoch's first attach, with boot_pm_sets: ++ * 1 every context bound after the epoch's first gets its own PM set (control + S, list, F in ++ * the firmware GiB), its own pool pages and its own aux pages at G5P_RU_PRIVATE_VA in its ++ * VM; the epoch's first context keeps the device PM. PM ID 0 everywhere. ++ * 2 as 1, and set s carries PM ID s + 1 (TA W+0x10, TA start, 3D start, tag6). ++ * RAM only, by the executor between jobs. ++ */ ++/* release8 default */ ++ ++/* The PM a context's jobs use: its set, or the device PM. */ ++static u8 *g5p_rv3_pm_cpu(struct g5p_boot *b, const struct g5p_render *r) ++{ ++ if (r->pm_set >= 0) ++ return b->pm_set[r->pm_set * G5P_PMSET_PAGES + G5P_PMSET_CONTROL] ++ .cpu; ++ return g5p_ru_pm_cpu(b); ++} ++ ++static u32 g5p_rv3_pm_meta(const struct g5p_render *r) ++{ ++ return g5p_meta_address( ++ r->pm_set >= 0 ? g5p_pmset_va(r->pm_set, G5P_PMSET_CONTROL) : ++ G5P_SHARED_VA + G5P_R22_PM); ++} ++ ++static u32 g5p_rv3_pm_id(const struct g5p_rdev *rd, const struct g5p_render *r) ++{ ++ return rd->pm_ctx >= 2 && r->pm_set >= 0 ? g5p_pmset_id(r->pm_set) : 0; ++} ++ ++static struct g5p_dma_page *g5p_rv3_pm_aux(struct g5p_rdev *rd, ++ struct g5p_render *r) ++{ ++ return r->pm_aux_mapped ? r->pm_aux : rd->aux; ++} ++ ++/* The boot tuple the V3 path is proven under (r96 params.txt), minus the instrumentation: ++ * r73b/r73c/r73e/r73f, r74 with the raised IRQ cap, r74f, r74g, r87b self-drain; no r74h, ++ * no r80..r96 (their IRQ banks read registers), no r75/r79 firmware control, and no fixed ++ * render engine (it is exclusive with the DRM one). ++ */ ++static bool g5p_rv3_tuple(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ struct g5p_r73b *x = b ? b->r73b : NULL; ++ ++ return x && g5p_r74g_on(b) && g5p_r74b_ram_only(g) && ++ x->r87b_attach == G5P_R87B_ATTACH_SELF_DRAIN; ++} ++ ++/* The admission state of a context and the device, as plain values for g5p_ru_v3_admit(). */ ++static struct g5p_ru_v3_state g5p_rv3_state(struct g5p_diag *g, ++ struct g5p_render *r) ++{ ++ const struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ ++ return (struct g5p_ru_v3_state){ ++ .tuple = g5p_rv3_tuple(g), ++ .v3_accepted = r->v3_accepted, ++ .bound = r->bound, ++ .jobs = r->jobs, ++ .device_up = READ_ONCE(rd->up), ++ .device_v3 = READ_ONCE(rd->v3), ++ .sequence = READ_ONCE(rd->sequence), ++ .slot_ta = r->slot[0], ++ .slot_3d = r->slot[1], ++ .variant = 0 || r->after_ta || 0 || r->v.pm || r->v.memory || ++ r->v.rt || r->v.slots, ++ .wait_ms = r->wait_ms, ++ .quiet = !g5p_ru_mmio_ok(&g5p_rv3_term), ++ .terminal_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ }; ++} ++ ++/* A terminal epoch is a device state (-EIO), the others are about this request. */ ++static int g5p_rv3_refusal_errno(int why) ++{ ++ if (why == G5P_RU_V3_TERMINAL) ++ return -EIO; ++ return why == G5P_RU_V3_OVERFLOW ? -EOVERFLOW : -EOPNOTSUPP; ++} ++ ++/* Submit-time admission: every V3 job refused here has touched nothing. */ ++static int g5p_rv3_admit_submit(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_ru_v3_state st; ++ int why; ++ ++ /* The epoch is being replaced: the executor admits the job again under the lock. */ ++ if (READ_ONCE(d->rdev->recovering)) ++ return 0; ++ st = g5p_rv3_state(g, r); ++ why = g5p_ru_v3_admit(&st, false); ++ ++ if (!why) ++ return 0; ++ /* per submit while admission is closed and a client retries: ratelimited */ ++ dev_info_ratelimited( ++ g->dev, "RENDER-V3 refused: %s; no hardware touched end=v3\n", ++ g5p_ru_v3_why[why]); ++ return g5p_rv3_refusal_errno(why); ++} ++ ++/* Recovery hooks (patches/gpu-recovery), compiled only when that series is applied: core.inc then includes ++ * teardown.inc before this file, and teardown.h defines G5P_TEARDOWN_H. The CAPTURE entry is observation only ++ * (RAM reads, no register access) and runs only with the recovery module parameter set, once per epoch, on a ++ * timeout or fault. Without the series both are nothing, and the prod98 kernel does not depend on it. ++ */ ++#ifdef G5P_TEARDOWN_H ++static void g5p_rv3_td_hook(struct g5p_diag *g, int kind, const char *why) ++{ ++ g5p_td_note_terminal(&g5p_td_state, g5p_ru_td_kind(kind)); ++ if ((0) && g5p_ru_term_capture_due(&g5p_rv3_term, kind)) ++ g5p_td_capture(g, why); ++} ++#else ++static inline void g5p_rv3_td_hook(struct g5p_diag *g, int kind, ++ const char *why) ++{ ++} ++#endif ++ ++/* The terminal state of the epoch: the TA complete, and any timeout or fault. First writer wins; from the first ++ * note on g5p_ru_mmio_ok() is false and the host touches no register (R0, R1, AIC, doorbell) and no bank. The r80 ++ * flag is set when an r80 block exists, so that a bank that might be latched cannot run behind this path (the V3 ++ * tuple forbids r80, so there is none today); the latch above does not depend on it. ++ */ ++static void g5p_rv3_terminal(struct g5p_diag *g, int kind, const char *why) ++{ ++ bool failed = g5p_ru_term_failed(&g5p_rv3_term); ++ ++ if (g5p_ru_term_note(&g5p_rv3_term, kind)) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s mmio=off r0=off export=ram-only end=v3\n", ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)], ++ why); ++ else if (!failed && g5p_ru_term_failed(&g5p_rv3_term)) ++ /* The epoch's TA complete took the latch long ago (job 1): a later timeout or fault ++ * only makes it sticky-failed, which closes admission for every client. Say so once. ++ */ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s latched=%s failed=1 admission=closed end=v3\n", ++ g5p_ru_term_names ++ [kind > G5P_RU_TERM_NONE && ++ kind < G5P_RU_TERM_KINDS ? ++ kind : ++ G5P_RU_TERM_FAULT], ++ why, ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)]); ++ g5p_rv3_td_hook(g, kind, why); ++} ++ ++/* "Retained boundary" of the r74f machinery -> the errno a job reports. -EINPROGRESS is ++ * never a job error (the UAPI uses it for "pending"). ++ */ ++static int g5p_rv3_errno(struct g5p_diag *g, int ret) ++{ ++ if (ret != -EINPROGRESS) ++ return ret; ++ switch (READ_ONCE(g->boot->r73b->r74f_stop)) { ++ case G5P_R74F_STOP_ATTACH_NOT_KICKED: ++ case G5P_R74F_STOP_ATTACH_NOT_CONSUMED: ++ case G5P_R74F_STOP_TA_NOT_KICKED: ++ case G5P_R74G_STOP_TA_NOT_CONSUMED: ++ case G5P_R74G_STOP_TA_WINDOW_MISSED: ++ return -ETIMEDOUT; ++ default: ++ return -EIO; ++ } ++} ++ ++/* ++ * Interim: on iOS the firmware writes back its own lines with the cache maintenance it runs on ++ * every tick (meta+0x6d4); that needs init words still being recovered (latency123.md), and ++ * this read stands in for it until then. ++ * ++ * While a phase is in flight the firmware does not see its own progress in the root and ++ * shared pages until a coherent read of those lines reaches the fabric: without one, the TA ++ * and 3D stages wait for the firmware tick, for seconds (latency123: key to frame 7.9 s ++ * median; with a Normal-NC read of both regions every 1-2 ms, 0.27 s). One 32-bit load per ++ * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU ++ * has no sampler, so it costs nothing then. ++ */ ++static void g5p_rv3_fw_touch(struct g5p_boot *b) ++{ ++ struct g5p_dma_page *const regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE( ++ *(const u32 *)(regions[r][i].cpu + at)); ++ /* perf60 stage 4: the per-context PM sets are firmware pages like the shared ones */ ++ for (u32 i = 0; i < b->pm_sets * G5P_PMSET_PAGES; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(b->pm_set[i].cpu + at)); ++} ++ ++/* perf60 bell gates, A/B knobs for the multi-context 3D fault on the bell path (render_user_wake=2 only; ++ * 0600, read at every bell, all 0 = no gate). They run between a phase's producer store and its bell; ++ * in policy state 0 the firmware does not drain a ring on its own, so a gate holds the phase back. ++ * RAM reads, sleeps and barriers only; no register access. ++ * bell_gap_us (a) the TA bell waits until this long after the previous job (any context) ended ++ * bell_idle_ms (a) the TA bell first waits, up to this, for the 3D engine idle words to read zero ++ * bell_3d_delay_us (c) mb() and this delay between the 3D producer store and the 3D bell ++ * bell_ctx_gap_us (d) mb() and this delay before the TA bell of a job of another context than the last ++ * bell_touch (c) the Normal-NC read of the firmware pages (g5p_rv3_fw_touch) right before each bell ++ */ ++ ++/* perf60 job-start gate: the same waits before the job's own host writes (the PM pool rebuild and ++ * the shared TA/3D work records, g5p_rv3_pm_rebuild() and g5p_ru_job()), which happen before any ++ * publish. The bell gates above run after them, so they cannot separate "the host rewrites shared ++ * firmware pages while the firmware still finishes the previous job" from a doorbell race. ++ * job_gap_us the job's first host write no sooner than this after the previous job ended ++ * job_idle_ms then wait up to this (RAM) for the TA and 3D engine idle words to read zero ++ */ ++/* perf60 layout (latched at the epoch's first attach, default 0 = the r22 layout): ++ * render_user_job_slots 2: job seq writes its command, W record and microsequence into slot ++ * (seq - 1) % 2 of the channel page, so a job never rewrites the record the firmware used for ++ * the job before it; the pointer ring carries each job's addresses (render-user-layout.h). ++ * render_user_pm_once 1: the PM pool, list and F are built by the epoch's first attach only and ++ * never rewritten by the host between jobs; the TA's tag6 record has the firmware rebuild F, ++ * as on iOS (PM created per queue once; docs/gpu/experiments/freelist-lifetime-production-fix.md). ++ */ ++/* release8 default */ ++/* release8 default */ ++ ++/* The record slot of job seq on this epoch's layout. */ ++static u32 g5p_rv3_js(const struct g5p_rdev *rd, u32 seq) ++{ ++ return rd->job_slots == 2 ? (seq - 1) & 1 : 0; ++} ++ ++ ++ ++ ++ ++ ++ ++ ++ ++/* RAM only. */ ++static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_ru_page(b, e), *global = g5p_ru_global(b, e), ++ *pm = g5p_rv3_pm_cpu(b, r); ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ for (u32 i = 0; i < ARRAY_SIZE(s->stamp); i++) ++ s->stamp[i] = g5p_le32(p + G5P_R22_STAMP + 4 * i); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < ARRAY_SIZE(s->idle); i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->pm80 = g5p_le32(pm + 0x80); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != r->target[e] || s->cons > 255 || ++ !g5p_ru_stamp_sane(p, seq); ++ s->consumed = s->cons == r->target[e]; ++ /* The predicates of the flown r74b (TA) and r96 (3D) waits; the 3D's PM serial is a receipt. */ ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2] && ++ (!e || (!s->idle[4] && !s->idle[5])); ++ s->retired = g5p_ru_retired_at(p, g5p_ru_meta(e), e, seq, ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ s->pm_ok = e ? s->pm80 == r->ctx->gpu->rdev->pm_serial + 1 : ++ g5p_r22_pm_saved(pm + 0x80); ++ s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns) : ++ READ_ONCE(r->ctx->gpu->ta_hit_ns)); ++} ++ ++/* TA: one step of render-user-retire.h's TA rules when ta is given, else the flown r74b ++ * predicate. 3D: one step of its 3D rules when rt is given, else the r96 predicate. ++ */ ++static bool g5p_rv3_done(u32 e, const struct g5p_rv3_ev *s, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta) ++{ ++ if (e && rt) ++ return g5p_ru_rt_step(rt, s->ns, s->consumed, s->retired, ++ s->idle_ok, ++ s->event) != G5P_RU_RT_PENDING; ++ if (!e && ta) ++ return g5p_ru_ta_step(ta, s->ns, s->consumed, s->retired, ++ s->pm_ok, s->idle_ok, ++ s->event) != G5P_RU_TA_PENDING; ++ return s->consumed && s->idle_ok && s->retired && (e || s->pm_ok); ++} ++ ++/* The TA clauses in the bits of render-user-retire.h, for the receipt. */ ++static u32 g5p_rv3_ta_unsat(const struct g5p_rv3_ev *s) ++{ ++ return (s->consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (s->retired ? 0 : G5P_RU_RT_RETIRED) | ++ (s->idle_ok ? 0 : G5P_RU_RT_IDLE) | ++ (s->pm_ok ? 0 : G5P_RU_RT_PM); ++} ++ ++/* Arm engine e's completion-event latch of g5p_drm_event() (3D: for this job's endpoint), before ++ * the phase's producer store, so an event decoded right after the publish is not missed. The ++ * latch is set under status_lock by whichever CPU decodes the event, and the wait samples it ++ * level-triggered every 1-2 ms, so there is no wakeup to lose. RAM only. ++ */ ++static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) ++{ ++ unsigned long flags; ++ ++ BUILD_BUG_ON(G5P_RU_RT_GRID != G5P_WORK_SLOTS); ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = G5P_RV3_TD_WANT | endpoint; ++ WRITE_ONCE(d->td_hit_ns, 0); ++ } else { ++ d->ta_want = true; ++ WRITE_ONCE(d->ta_hit_ns, 0); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) ++{ ++ unsigned long flags; ++ u64 hit; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = 0; ++ hit = d->td_hit_ns; ++ } else { ++ d->ta_want = false; ++ hit = d->ta_hit_ns; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return hit; ++} ++ ++/* RC4a: the receipt's read set (g5p_rv3_receipt()'s blobs: PM-control, then W, channels and the ++ * stamp window of the TA and of the 3D page), read once with 32-bit loads, the access size of the ++ * receipt's hex dump. The words are folded, not copied: no allocation, no store to shared memory. ++ * RAM only: no register, no doorbell, no print, no latch. ++ */ ++static u32 g5p_rv3_touch_sweep(struct g5p_boot *b, u32 *bytes) ++{ ++ u8 *ta = g5p_ru_page(b, 0), *td = g5p_ru_page(b, 1); ++ const u8 *const src[] = { ++ g5p_ru_pm_cpu(b), ta + G5P_R22_W, ta, ++ ta + G5P_R22_TQ, td + G5P_R22_W, td, ++ td + G5P_R22_TQ, ++ }; ++ static const u32 len[] = { ++ 0x140, G5P_RU_TA_W_BYTES, 0x340, 0x80, G5P_RU_3D_W_BYTES, 0x340, ++ 0x80, ++ }; ++ u32 sum = 0, n = 0; ++ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(src); i++) { ++ for (u32 at = 0; at < len[i]; at += 4) ++ sum = (sum << 5 | sum >> 27) ^ ++ READ_ONCE(*(const u32 *)(src[i] + at)); ++ n += len[i]; ++ } ++ *bytes = n; ++ return sum; ++} ++ ++/* RC4a: the instant of one phase, taken at the sample at now_ns. The touch arm reads, the control ++ * arm only records; both record the same instant. ++ */ ++static void g5p_rv3_touch(struct g5p_diag *g, struct g5p_ru_touch *t, ++ u64 now_ns) ++{ ++ u32 bytes = 0, sum = 0; ++ ++ if (t->sweep) ++ sum = g5p_rv3_touch_sweep(g->boot, &bytes); ++ g5p_ru_touch_mark(t, now_ns, t->sweep ? ktime_get_ns() : now_ns, bytes, ++ sum); ++} ++ ++/* tahang1: a phase still pending this long after its publish prints its RAM state once. */ ++#define G5P_RV3_SLOW_MS 1000U ++ ++/* The one `slow` line of a phase: the deciding sample's RAM words and the waiter's own state, so ++ * a stall that later times out is described from the first second on. RAM only. ++ */ ++static void g5p_rv3_slow(struct g5p_diag *g, u32 e, u32 seq, ++ const struct g5p_ru_wait *w, ++ const struct g5p_rv3_ev *s, u32 unsat, u64 hit_ns) ++{ ++ dev_warn( ++ g->dev, ++ "RENDER-V3 slow seq=%u engine=%u ms=%lld ring=%u/%u stamp=%08x/%08x " ++ "ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x,%08x,%08x,%08x pm80=%08x " ++ "consumed=%u retired=%u pm=%u event_ms=%lld unsat=%#x busy_ms=%lld events=%u " ++ "end=v3\n", ++ seq, e, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ s->idle[3], s->idle[4], s->idle[5], s->pm80, s->consumed, ++ s->retired, s->pm_ok, g5p_ru_rt_ms(w->start_ns, hit_ns), unsat, ++ g5p_ru_rt_ms(w->start_ns, w->busy_ns), ++ READ_ONCE(g->boot->events)); ++ g5p_np_policy(g, "slow", seq, e); ++} ++ ++/* nopatch: the policy record at a few phase ends (log-spaced) and at every timeout. RAM only. */ ++static void g5p_rv3_np_end(struct g5p_diag *g, u32 e, u32 seq, u32 result) ++{ ++ if (seq == 1 || seq == 10 || seq == 100 || seq == 1000 || ++ seq == 10000 || result == G5P_RU_WAIT_TIMEOUT) ++ g5p_np_policy(g, ++ result == G5P_RU_WAIT_TIMEOUT ? "phase-timeout" : ++ "phase-end", ++ seq, e); ++} ++ ++/* One phase's RAM-only wait. g5p_ru_wait_step()/close() decide; this loop only samples. RC4a: the ++ * touch instant is taken after a step that left the phase undecided, before the sleep (off: never). ++ * The predicate is level-triggered: every sample, including the one right after each sleep and the ++ * closing one at the deadline, re-reads the whole RAM state (rt: the 3D, ta: the TA). ++ */ ++static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, struct g5p_ru_touch *t, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta, ++ struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) ++{ ++ struct g5p_drm *d = r->ctx->gpu; ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE( ++ d->event_serial); /* after the drain, before the sample */ ++ bool fault, silent, done; ++ ++ g5p_rv3_sample(g, r, e, seq, s); ++ /* latency101: a silent event stream during the phase is a dead firmware (never off). */ ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed) ++ *err = -EIO; ++ else if (silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = g5p_rv3_done(e, s, rt, ta); ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (rt) ++ g5p_ru_rt_note(rt, w->result); ++ if (ta) ++ g5p_ru_ta_note(ta, w->result); ++ if (w->result) { ++ g5p_rv3_np_end(g, e, seq, w->result); ++ return w->result; ++ } ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_RV3_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ g5p_rv3_slow(g, e, seq, w, s, ++ rt ? rt->unsat : ++ ta ? ta->unsat : ++ 0, ++ e ? READ_ONCE(d->td_hit_ns) : ++ READ_ONCE(d->ta_hit_ns)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ if (g5p_ru_touch_due(t, s->ns, w->result, w->expired)) ++ g5p_rv3_touch(g, t, s->ns); ++ g5p_l123_pause(d, g, e, s, seen); ++ } ++} ++ ++/* One phase's deadline: the legacy single bound, or (latency101) the soft/hard pair. */ ++static void g5p_rv3_wait_start(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_ru_wait *w, u64 pub_ns, ++ u64 job_deadline_ns) ++{ ++ if (r->hard_ms) ++ g5p_ru_wait_start_lat(w, pub_ns, r->soft_ms, r->hard_ms, ++ job_deadline_ns, ++ READ_ONCE(g->boot->events)); ++ else ++ g5p_ru_wait_start(w, pub_ns, r->wait_ms, job_deadline_ns); ++} ++ ++/* After a timeout (or fault) was decided: the one RAM look of the legacy path, or (latency101, ++ * timeout only) a RAM-only observation window that keeps draining and sampling at the wait cadence ++ * and prints nothing, so a completion that lands milliseconds after the decision is recorded ++ * (late_after/late_ns) before the receipt line and the terminal latch. It never changes the ++ * result and never touches a register. The 3D look steps a copy of the predicate's record, so the ++ * receipt keeps the deciding sample's. ++ */ ++static void g5p_rv3_observe(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, ++ const struct g5p_ru_retire *rt, ++ const struct g5p_ru_ta *ta, struct g5p_rv3_ev *late) ++{ ++ struct g5p_ru_retire copy, *c = NULL; ++ struct g5p_ru_ta tcopy, *tc = NULL; ++ ++ if (rt) { ++ copy = *rt; ++ c = © ++ } ++ if (ta) { ++ tcopy = *ta; ++ tc = &tcopy; ++ } ++ if (!r->obs_ms || w->result != G5P_RU_WAIT_TIMEOUT) { ++ g5p_rv3_sample(g, r, e, seq, late); ++ g5p_ru_wait_after(w, g5p_rv3_done(e, late, c, tc)); ++ return; ++ } ++ do { ++ g5p_boot_drain( ++ g); /* a fault latches b->failed, the result stays */ ++ g5p_rv3_sample(g, r, e, seq, late); ++ if (!g5p_ru_wait_observe(w, late->ns, ++ g5p_rv3_done(e, late, c, tc), ++ r->obs_ms)) ++ break; ++ usleep_range(1000, 2000); ++ } while (1); ++} ++ ++/* wake110: render-user-wake.h's register access, on the Fender and RGX windows. */ ++static void g5p_rv3_wk_w1(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->window[1] + reg); ++} ++ ++/* The GPC state read of g5p_wk_ring() comes right after its R1+0x14 = 7 wake: in bell mode ++ * (render_user_wake=2) it waits up to G5P_RV3_WK_SETTLE_US, re-reading every 10 us, for a settled ++ * powered pstate (bit 4 clear, nibble nonzero), as Apple's setGPCPerformanceState waits for bit 4. ++ * Reads only; the ring's own check of the final value is unchanged. ++ */ ++#define G5P_RV3_WK_SETTLE_US 2000U ++ ++static u32 g5p_rv3_wk_r1(void *ctx, u32 reg) ++{ ++ void __iomem *r1 = ((struct g5p_diag *)ctx)->window[1]; ++ u32 raw = readl(r1 + reg); ++ u64 t0; ++ ++ if (reg != G5P_WK_R1_GPC || (2) != G5P_WK_ALWAYS || ++ (!(raw & 0x10) && (raw & 0xf))) ++ return raw; ++ t0 = ktime_get_mono_fast_ns(); ++ while (((raw & 0x10) || !(raw & 0xf)) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_WK_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(r1 + reg); ++ } ++ return raw; ++} ++ ++static void g5p_rv3_wk_wmb(void *ctx) ++{ ++ dma_wmb(); ++} ++ ++static void g5p_rv3_wk_w0(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->rgx + reg); ++} ++ ++/* wake110: an r74f bell that still owns its ring (the ALIVE decoder may fire it). */ ++static bool g5p_rv3_wk_r74f_busy(struct g5p_r73b *x) ++{ ++ u32 k = READ_ONCE(x->r74f_active); ++ int s; ++ ++ if (!k || k > G5P_R74F_KINDS) ++ return false; ++ s = atomic_read_acquire(&x->r74f_bell[k - 1].state); ++ return s == G5P_R74F_PUBLISHED || s == G5P_R74F_OPEN || ++ s == G5P_R74F_FIRING; ++} ++ ++/* wake111: DS108's claim proof re-read at the publish: ALIVE proven in the current reset ++ * generation, the claim admitted in it, nothing latched failed since, no R74E power abort. ++ * Called under event_lock (interrupts off); g5p_ds_lock is a leaf (g5p_ds_event() is pure). ++ */ ++static bool g5p_rv3_wk_ds108(struct g5p_boot *b) ++{ ++ bool ok; ++ ++ spin_lock(&g5p_ds_lock); ++ ok = g5p_ds_proven(&g5p_ds_state) && ++ g5p_ds_ok(&g5p_ds_state, G5P_DSE_CLAIM); ++ spin_unlock(&g5p_ds_lock); ++ return ok && !(b->r73b && READ_ONCE(b->r73b->early_power_abort)); ++} ++ ++/* wake110: before the phase's producer store, the engine's type-1 count and a cleared first-decode ++ * time, so an event decoded right after the publish is counted. RAM only. ++ */ ++static void g5p_rv3_wake_mark(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ v->wk_t1_pub[e] = d->t1_count[e]; ++ WRITE_ONCE(d->t1_first_ns[e], 0); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* wake110: at the job's end, the type-1 counts and the first decode of each published phase. */ ++static void g5p_rv3_wake_end(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ for (u32 e = 0; e < 2; e++) { ++ v->wk_t1_end[e] = d->t1_count[e]; ++ v->wk_first_ns[e] = d->t1_first_ns[e]; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (v->phase < G5P_RV3_TA_PUBLISH) ++ v->wk_first_ns[0] = 0; ++ if (v->phase < G5P_RV3_TD_PUBLISH) ++ v->wk_first_ns[1] = 0; ++} ++ ++/* wake110: microseconds / milliseconds from a phase's publish to a later instant; -1 = none. */ ++static s64 g5p_rv3_wk_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++static s64 g5p_rv3_wk_ms(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_MSEC : ++ -1; ++} ++ ++/* wake110: one phase's publish-time decision, right after its producer store and barrier. Both ++ * arms take the event lock (the r74f owner's serialization against the decoder) and sample the ++ * same RAM; only a factor that passed every guard runs g5p_wk_ring(). Nothing is printed under ++ * the lock. Off: returns at once. ++ */ ++static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e, u32 start, u32 seq) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ struct g5p_drm *d = r->ctx->gpu; ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ u8 *p = g5p_ru_page(b, e); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&b->event_lock, flags); ++ v->wk_start_ns[e] = ktime_get_ns(); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = r->wake, ++ .seed = 0, ++ .seq = seq, ++ .e = e, ++ .attempted = g5p_wk_attempted(v->wk_did[e]), ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = r->target[e], ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4) }, ++ .type1 = READ_ONCE(d->t1_count[e]) - v->wk_t1_pub[e], ++ }; ++ v->wk_cons[e] = in.cons; ++ v->wk_power[e] = in.power; ++ v->wk_did[e] = g5p_wk_decide(&in, &io, &v->wk_fence[e], &v->wk_gpc[e]); ++ v->wk_done_ns[e] = ktime_get_ns(); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (v->wk_did[e] == G5P_WK_DID_RANG) ++ r->wake_rang++; ++} ++ ++/* The epoch's first job: the device (PM, channels) and this context's DC3 records, shared with ++ * g5p_render_attach(), then the r74f/r74g protocol of the fixed r74f_attach() on typed state. ++ */ ++static int g5p_rv3_attach(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline; ++ u32 start, tag, bad; ++ int ret; ++ ++ ret = g5p_r74f_admit(g, G5P_R74F_DC3); ++ if (ret) ++ return ret; ++ ret = g5p_render_attach_prepare(d, r, &start, &tag); ++ if (ret) ++ return ret; ++ r->dc_target = (start + 2) & 255; ++ r->bound = true; ++ rd->up = true; ++ rd->v3 = true; ++ rd->job_slots = (2) == 2 ? 2 : 1; ++ rd->pm_once = (1) == 1; ++ rd->pm_ctx = b->pm_sets ? min((3), 3U) : 0; ++ r->pm_first = r->pm_set < 0; ++ dev_info( ++ g->dev, ++ "RENDER-V3 layout job_slots=%u pm_once=%u slot1=%#x..%#x pm_ctx=%u pm_sets=%u " ++ "end=v3\n", ++ rd->job_slots, rd->pm_once, G5P_RU_SLOT1, G5P_RU_SLOT1_END, ++ rd->pm_ctx, b->pm_sets); ++ g5p_render_retain(r); ++ v->touched = true; ++ WRITE_ONCE(r->ctx->live, true); ++ dma_wmb(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ v->attach_ns = ktime_get_ns(); ++ g5p_r74_mark(g, G5P_R74_ATTACH_PUBLISHED, r->dc_target); ++ /* The producer is final from here; only the doorbell may be pending. */ ++ ret = g5p_r74f_bell(g, G5P_R74F_DC3, start, r->dc_target); ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || READ_ONCE(b->failed)) ++ return ret ?: -EIO; ++ dma_rmb(); ++ if (READ_ONCE(ctl[4]) != r->dc_target || ++ READ_ONCE(ctl[0]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(ctl[0]) == r->dc_target) { ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ bad = g5p_r74g_check(g, 0, r->slot, vm->root.dma, tag, ++ start); ++ g5p_r74_mark(g, G5P_R74_ATTACH_CONSUMED, r->dc_target); ++ g5p_r74g_dc3_line(g, 0, false); ++ if (bad) ++ return g5p_rv3_errno(g, ++ g5p_r74g_refuse(g, bad)); ++ for (u32 e = 0; e < 2; e++) ++ g5p_mmu_record( ++ g, ++ "RENDER tags slot=%u host=%08x cached=%08x expected=%08x\n", ++ r->slot[e], ++ g5p_le32(g5p_boot_shared(b, 0x84) + ++ r->slot[e] * 4), ++ g5p_le32(g5p_boot_shared(b, 0x85) + ++ r->slot[e] * 4), ++ tag); ++ g5p_mmu_record( ++ g, ++ "RENDER-USER attached TA=%u 3D=%u PC=%pad tag=%08x blocks=%u; RAM-only, firmware " ++ "DC3 installed CBASE, host MMU pass skipped\n", ++ r->slot[0], r->slot[1], &vm->root.dma, tag, ++ rd->blocks); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } while (time_before(jiffies, deadline)); ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ return g5p_rv3_errno(g, ++ g5p_r74f_stop(g, G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ READ_ONCE(ctl[0]))); ++} ++ ++/* perf60 bell mode (render_user_wake=2): a DC publication (bind, spare pairs, unbind) gets Apple's ++ * submit kick too. With the policy held in state 0 the firmware no longer drains the DC ring on ++ * its own (perf60clk-9ec288b868: a second client's RAM-only DC3 bind was never consumed). The ++ * wake path's guards (event lock, power proof, no fault or terminal, no r74f bell owning a ring, ++ * the ring holding exactly this publication), then Apple's order: R1+0x14 = 7 and its read-back, ++ * R1+0x1000 re-read every 10 us until a settled powered pstate (g5p_gpc_ok(): bit 4 clear, ++ * nibble nonzero; Apple's setGPCPerformanceState waits for bit 4 the same way), at most ++ * G5P_RV3_DC_SETTLE_US, and R0+0xb00 = 0x10 only then. A GPC that never reads on is not written: ++ * no R0 access, the publication waits as before. Off (render_user_wake != 2): returns at once. ++ */ ++#define G5P_RV3_DC_SETTLE_US 2000U ++ ++static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, ++ const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 fence = U32_MAX, gpc = U32_MAX, power, cons, n = 0; ++ const char *did = "skip-guard"; ++ unsigned long flags; ++ u64 t0; ++ ++ ; ++ ; ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ cons = READ_ONCE(ctl[0]); ++ power = g5p_wk_power_mask(READ_ONCE(g->powered), ++ READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), ++ g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ !g5p_wk_power_ready(power) || g5p_rv3_wk_r74f_busy(b->r73b) || ++ cons != start || READ_ONCE(ctl[4]) != target) ++ goto out; ++ writel(7, g->window[1] + 0x14); ++ fence = readl(g->window[1] + 0x14); ++ if (fence & ~7U) { ++ did = "skip-fence"; ++ goto out; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ gpc = readl(g->window[1] + 0x1000); ++ while (!g5p_gpc_ok(gpc) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ gpc = readl(g->window[1] + 0x1000); ++ n++; ++ } ++ if (!g5p_gpc_ok(gpc)) { ++ did = "skip-gpc"; ++ goto out; ++ } ++ /* the publication's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ wmb(); ++ writel(0x10, g->rgx + 0xb00); ++ did = "rang"; ++out: ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 dc-kick why=%s did=%s dc=%u->%u power=%02x fence=%08x gpc=%08x " ++ "settle=%u end=v3\n", ++ why, did, cons, target, power, fence, gpc, n); ++} ++ ++/* RAM only: drain the event stream, read the firmware pages (interim, see g5p_rv3_fw_touch) and ++ * sample the DC ring until its consumer reaches `target` ++ * or G5P_RU_BIND_MS pass. No doorbell and no register, whether or not a job has retired. ++ */ ++static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_RU_BIND_MS); ++ ++ for (;;) { ++ bool expired = time_after_eq(jiffies, deadline); ++ int ret = g5p_boot_drain(g); ++ ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ g5p_rv3_fw_touch(b); ++ dma_rmb(); ++ switch (g5p_ru_bind_step(READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), ++ target, expired)) { ++ case G5P_RU_BIND_CONSUMED: ++ *consumed_ns = ktime_get_ns(); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } ++} ++ ++static s64 g5p_rv3_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++/* ++ * The first job of a context on a device that is already up binds the context's slot pair ++ * with a DC3 pair published in RAM only (multi-client-design.md §4.2). No doorbell and no ++ * register access of any kind, so it is safe after a job retired and while the GPU may be ++ * idle-gated: the firmware services the DC ring in its scheduler pass, before TA and 3D, and ++ * the host waits on RAM. Core lock held, between jobs. ++ * ++ * Refused before the producer store, it touched nothing. From the store on, the firmware may ++ * hold the pair: the context is retained, and on failure it is lost (v->bind_failed) but the ++ * device and the contexts already bound are not. A pair not consumed in time keeps every later ++ * bind refused until the firmware catches up. ++ */ ++/* ++ * render_user_close_park (release8 quiet2, docs/gpu/experiments/release8-quiet2.md). The PM writes ++ * a closed context's free list back through that context's TA slot after the context is gone, and ++ * not only while it is the last or the previous distinct context (render-user-pmset.h): two bench ++ * faults wrote the free list of a context two switches back, through its slot, on the 2nd job of a ++ * third context, after the pm-prev rule had unbound it (page-invalid level 2, its PC level empty). ++ * iOS never leaves the PM's target unresolvable: its PM is the device's (shared by the work queues, ++ * AGXWorkQueue_initAndAttachPM), in the kernel's GPU address space, and a GART keeps its slots until ++ * the address space is destroyed (AGXGart_detachSlotsAndDestroyAddressSpace). Here, a closed ++ * context's pair is bound again, right after its unbind (between jobs, core lock held), to the park ++ * catalogue: a kernel page that maps only the firmware window, where the PM sets live. A late ++ * write-back then lands in the closed context's own set page (released, reused last) instead of ++ * faulting. A later bind of the pair unbinds it first. Both steps are the proven DC3 ++ * transitions (PA to zero, zero to PA); a park that fails leaves the pair unbound, as before. ++ */ ++/* release8 default */ ++ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why); ++ ++/* Before a bind, core lock held: a pair a close parked (RAM: park PA and tag on both slots) is ++ * unbound first, by its own DC3. 0 when the pair was not parked or the unbind verified. ++ */ ++static int g5p_rv3_unpark(struct g5p_drm *d, const u32 *slot) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ ++ if (!d->rdev->park.cpu) ++ return 0; ++ dma_rmb(); ++ g5p_render_bind_mem(b, slot, READ_ONCE(ctl[4]), &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!g5p_ru_bind_parked(&s, d->rdev->park.dma)) ++ return 0; ++ return g5p_rv3_unbind_slots(d, slot, 2, "unpark"); ++} ++ ++static int g5p_rv3_bind(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, tag, bad = 0; ++ int ret; ++ ++ dma_rmb(); ++ if (rd->dc_pending && READ_ONCE(ctl[0]) != rd->dc_wait) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u dc=%u/%u wait=%u: an earlier bind is " ++ "pending; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4]), rd->dc_wait); ++ return -EIO; ++ } ++ rd->dc_pending = false; ++ ret = g5p_rv3_unpark(d, r->slot); ++ if (ret) ++ return ret; ++ ret = g5p_render_vm_prepare(d, r, &start, &tag); ++ if (ret) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u ret=%d dc=%u/%u: ring busy or slot in use; " ++ "no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], ret, READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4])); ++ return ret; ++ } ++ r->dc_target = (start + 2) & 255; ++ g5p_render_retain(r); ++ v->bind_failed = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, r->dc_target, "bind"); ++ ret = g5p_rv3_dc_wait(g, r->dc_target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = r->dc_target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, vm->root.dma, tag, ++ r->dc_target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=%lld tag=%08x host=%08x/%08x " ++ "cached=%08x/%08x dc=%u->%u/%u bad=%#x pc=%pad mode=ram-dc3 end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ tag, s.host[0], s.host[1], s.cached[0], s.cached[1], start, ++ s.cons, s.prod, bad, &vm->root.dma); ++ if (ret) ++ return ret; ++ v->bind_failed = false; ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ return 0; ++} ++ ++/* ++ * TEMPORARY fallback (render_user_spare_pairs, read at the epoch's first job), for a firmware ++ * that never drains a DC3 published after a job retired. Before job 1's attach, bind that ++ * many extra slot pairs, each to a zeroed kernel-owned catalogue, in one RAM-only batch with no ++ * doorbell. The firmware drains it on its own like the self-drained attach of this boot tuple ++ * (r87b); job 1's own attach, doorbell and r74g checks then run unchanged behind it ++ * (g5p_r74f_ring() refuses a ring that holds more than that attach's pair). Later contexts ++ * take a pair over with g5p_rv3_adopt(). A pair that does not check out is never used. Core ++ * lock held, device not up, so register access is still allowed (none is made here). ++ */ ++static int g5p_rv3_spares(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 want = min((0), G5P_RU_MAX_SPARES); ++ u32 start = READ_ONCE(ctl[4]), target, n, ok = 0; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ unsigned long flags; ++ int ret; ++ ++ if (!want || rd->spares) ++ return 0; ++ rd->spares = kcalloc(want, sizeof(*rd->spares), GFP_KERNEL); ++ if (!rd->spares) ++ return -ENOMEM; ++ for (n = 0; n < want; n++) { ++ struct g5p_rspare *sp = &rd->spares[n]; ++ bool pair; ++ ++ spin_lock_irqsave(&g->slot_lock, flags); ++ pair = g5p_ru_slot_pair(g->user_slots, false, sp->slot); ++ if (pair) { ++ __set_bit(sp->slot[0], g->user_slots); ++ __set_bit(sp->slot[1], g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ if (!pair) ++ break; ++ g5p_render_bind_mem(b, sp->slot, start + 2 * n, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (!g5p_ru_bind_busy(&s) && ++ !g5p_vm_alloc_page(d->drm.dev, &sp->root)) { ++ g5p_table_clean_page(g, &sp->root); ++ continue; ++ } ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(sp->slot[0], g->user_slots); ++ __clear_bit(sp->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ break; ++ } ++ if (!n) ++ return 0; ++ dsb(sy); ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_bind_mem(b, rd->spares[i].slot, start + 2 * i, &m); ++ g5p_ru_bind_publish(&m, rd->spares[i].slot, ++ rd->spares[i].root.dma, G5P_RU_SPARE_TAG); ++ } ++ rd->spare_count = n; ++ target = (start + 2 * n) & 255; ++ v->touched = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "spares"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ for (u32 i = 0; !ret && i < n; i++) { ++ struct g5p_rspare *sp = &rd->spares[i]; ++ ++ g5p_render_bind_mem(b, sp->slot, start + 2 * i, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ sp->ok = !g5p_ru_bind_check(&s, sp->slot, sp->root.dma, ++ G5P_RU_SPARE_TAG, target); ++ ok += sp->ok; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 spares pairs=%u ok=%u ret=%d consumed_us=%lld dc=%u->%u first=%u/%u " ++ "last=%u/%u end=v3\n", ++ n, ok, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ READ_ONCE(ctl[0]), rd->spares[0].slot[0], rd->spares[0].slot[1], ++ rd->spares[n - 1].slot[0], rd->spares[n - 1].slot[1]); ++ return ret; ++} ++ ++/* ++ * TEMPORARY fallback: the first job of a context on a device whose job 1 bound spare pairs. ++ * Takes the next pair that checked out, makes its catalogue page this VM's root (a copy of the ++ * current root) and publishes the VM's tag through the proven RAM-only tag path: CBASE already ++ * names the page, and the changed tag makes the firmware invalidate the slots' PC level at this ++ * context's first job. No DC3, no register. Core lock held; the context has a job queued, so ++ * no VM edit of it runs concurrently. ++ */ ++static int g5p_rv3_adopt(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_rspare *sp = NULL; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ struct g5p_dma_page old; ++ unsigned long flags; ++ struct g5p_rt *rt; ++ u32 entry; ++ int ret; ++ ++ for (u32 i = 0; i < rd->spare_count && !sp; i++) ++ if (rd->spares[i].ok && !rd->spares[i].used) ++ sp = &rd->spares[i]; ++ if (!sp) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare: none left of %u; no hardware " ++ "touched end=v3\n", ++ r->slot[0], r->slot[1], rd->spare_count); ++ return -ENOSPC; ++ } ++ /* RAM: the pair still names its page, with the tag job 1 checked. */ ++ dma_rmb(); ++ g5p_render_bind_mem(b, sp->slot, 0, &m); ++ g5p_ru_bind_read(&m, 0, 0, &s); ++ for (u32 e = 0; e < 2; e++) { ++ if (s.pa[e] == sp->root.dma && s.host[e] == G5P_RU_SPARE_TAG && ++ s.cached[e] == G5P_RU_SPARE_TAG) ++ continue; ++ sp->ok = false; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare pair=%u/%u host=%08x cached=%08x: " ++ "pair changed; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], sp->slot[0], sp->slot[1], ++ s.host[e], s.cached[e]); ++ return -EIO; ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ memcpy(sp->root.cpu, vm->root.cpu, G5P_GPU_PAGE); ++ old = vm->root; ++ vm->root = sp->root; ++ sp->root = old; /* the VM's former root, never published */ ++ g5p_ru_free_page(d->drm.dev, &sp->root); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(r->slot[0], g->user_slots); ++ __clear_bit(r->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ r->slot[0] = sp->slot[0]; ++ r->slot[1] = sp->slot[1]; ++ sp->used = true; ++ g5p_render_retain(r); ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ /* Root and tables to PoC, then a tag unlike the cached one in both slots. */ ++ ret = g5p_render_invalidate(r->ctx); ++ v->bind_failed = ret != 0; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=0 tag=%08x cached=%08x/%08x " ++ "pc=%pad mode=spare end=v3\n", ++ r->slot[0], r->slot[1], ret, ++ g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[0] * 4), ++ s.cached[0], s.cached[1], &vm->root.dma); ++ return ret; ++} ++ ++/* r74f_publish_ta() on typed state: record, WORK_CTL, producer; then (job 1 only) the r74g grace ++ * and, only if the ring is still pending, the one ALIVE-owned fallback doorbell. Jobs >= 2 ++ * publish and return: no grace, no bell (the r74f/r74g machinery is one-shot). No canary, no marker. ++ */ ++static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *x = b->r73b; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_r74g_dc3 *dc = x->r74g_dc3; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x60); ++ u8 *p = g5p_ru_page(b, 0); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ bool first = seq == 1; ++ u32 bad; ++ int ret; ++ ++ if (!r->bound || r->submitted[0] || start > 255 || ++ READ_ONCE(ring[0]) != start || ++ g5p_le32(p + G5P_WORK_CTL) != previous || ++ g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) ++ return -EBUSY; ++ if (first) { ++ ret = g5p_r74f_admit(g, G5P_R74F_TA); ++ if (ret) ++ return ret; ++ /* g5p_r74g_ta_preflight() on typed state: DC3 still complete with the same records, ++ * tag and backing, and no page-table mutation since the attach. ++ */ ++ bad = g5p_r74g_check(g, 1, r->slot, dc[0].expected_pa, ++ dc[0].tag, dc[0].start); ++ if (g5p_vm_generation(&vm->mutation) != dc[0].tag || ++ vm->root.dma != dc[0].expected_pa || ++ r->dc_target != dc[0].target) { ++ bad |= G5P_R74G_BAD_MUTATION; ++ dc[1].bad = bad; ++ } ++ g5p_r74g_dc3_line(g, 1, false); ++ if (bad) ++ return g5p_rv3_errno(g, g5p_r74g_refuse(g, bad)); ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x61] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 0 : ++ j == 1 ? g5p_ru_meta(0) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[0] = (start + 1) & 255; ++ r->submitted[0] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_PUBLISH, start); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], r->target[0]); ++ dma_wmb(); ++ if (first) { ++ ret = g5p_r74g_ta(g, start, r->target[0]); ++ v->grace_end = READ_ONCE(x->r74g_grace_end); ++ v->bell = ret == 0; ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ } ++ /* wake110: jobs >= 2 reach this at once after the producer store (job 1: excluded). */ ++ if (!first) ++ 0; ++ g5p_rv3_wake(g, r, v, 0, start, seq); ++ /* The line the userspace lane parses (a9c_parse_publish): same format as the fixed path's. */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 0, r->slot[0], endpoint, first ? 0x10 : 0, ++ g5p_ru_meta(0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 0); ++ return 0; ++} ++ ++/* The RAM preconditions of the 3D publish, one bit per clause (0 = admitted). RAM reads only; the bits are ++ * what the refusal line prints, so a second blocker is visible in the run that meets it: ++ * 0 TA not retired 1 TA not consumed 2 3D already submitted 3 producer > 255 4 consumer != producer ++ * 5 WORK_CTL 6 WORK_CTL+0x40 7 command tag != 4 8 dependency word 9 dependency value ++ * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) ++ */ ++static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, ++ const u32 *ring, u32 start, u32 previous, u32 seq) ++{ ++ const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ ++ return (u32)!r->retired[0] | (u32)!r->consumed[0] << 1 | ++ (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | ++ (u32)(READ_ONCE(ring[0]) != start) << 4 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 5 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 6 | ++ (u32)(g5p_le32(cmd) != 4) << 7 | ++ (u32)(g5p_le32(cmd + 4) != g5p_ru_meta(0) + G5P_R22_STAMP + 4) ++ << 8 | ++ (u32)(g5p_le32(cmd + 8) != seq) << 9 | ++ (u32)(g5p_le32(p + G5P_R22_STAMP) != seq - 1 || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq - 1) ++ << 10; ++} ++ ++/* The 3D after the TA retired: record, WORK_CTL, producer, and NO doorbell. */ ++static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x64); ++ u8 *p = g5p_ru_page(b, 1); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ u32 why; ++ ++ /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one ++ * fails, and the refusal names the clauses and the raw values it read. ++ */ ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq); ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 3d-refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x cmd=%08x/%08x/%08x " ++ "stamp=%08x/%08x end=v3\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ seq))), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 4), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 8), ++ g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x65] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 1 : ++ j == 1 ? g5p_ru_meta(1) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[1] = (start + 1) & 255; ++ r->submitted[1] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->td_pub_ns = ktime_get_ns(); ++ smp_store_release(&ring[4], r->target[1]); ++ dma_wmb(); ++ 0; ++ g5p_rv3_wake(g, r, v, 1, start, seq); ++ /* The 3D twin of the TA's publish line: engine 1, kick 0 (no doorbell). */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 1, r->slot[1], endpoint, 0, ++ g5p_ru_meta(1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 1); ++ return 0; ++} ++ ++static const char *const g5p_rv3_ta_how[G5P_RU_TA_HOWS] = { "none", ++ "idle-clear", ++ "event", "hold" }; ++ ++/* tahang1: the TA decisions since load, by rule (core lock held). A TA that completed with an ++ * idle word still set prints one line with the running totals: rate-limited for rule (b), which ++ * is routine, always for rule (c), which means the event was not seen. RAM only. ++ */ ++static u32 g5p_rv3_ta_rules[G5P_RU_TA_HOWS]; ++static DEFINE_RATELIMIT_STATE(g5p_rv3_ta_rs, 5 * HZ, 4); ++ ++static void g5p_rv3_ta_count(struct g5p_diag *g, struct g5p_drm *d, ++ const struct g5p_rv3 *v) ++{ ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ u32 how = min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1); ++ u64 hit = READ_ONCE(d->ta_hit_ns); ++ ++ g5p_rv3_ta_rules[how]++; ++ if (how == G5P_RU_TA_IDLE_CLEAR) ++ return; ++ if (how == G5P_RU_TA_EVENT && !__ratelimit(&g5p_rv3_ta_rs)) ++ return; ++ dev_warn( ++ g->dev, ++ "RENDER-V3 ta-done seq=%u how=%s idle=%08x,%08x,%08x pm80=%08x coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld busy_ms=%lld rules=%u/%u/%u end=v3\n", ++ v->seq, g5p_rv3_ta_how[how], s->idle[0], s->idle[1], s->idle[2], ++ s->pm80, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, hit), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_ta_rules[G5P_RU_TA_IDLE_CLEAR], ++ g5p_rv3_ta_rules[G5P_RU_TA_EVENT], ++ g5p_rv3_ta_rules[G5P_RU_TA_HOLD]); ++} ++ ++/* A receipt line: a failed job's at KERN_EMERG as before (it survives a freeze), a completed job's at ++ * job_log (wake every job, end/lat/retire the first 8 of a context: routine at desktop rates). ++ */ ++#define g5p_rv3_receipt_printk(g, ret, fmt, ...) \ ++ do { \ ++ if (ret) \ ++ dev_emerg((g)->dev, fmt, ##__VA_ARGS__); \ ++ else \ ++ g5p_job_printk((g)->dev, fmt, ##__VA_ARGS__); \ ++ } while (0) ++ ++/* The end of a job, whatever it was: one receipt line on the console (it survives a freeze) and the ++ * RAM-only raw state in the blob. Reads RAM only. ++ */ ++static void g5p_rv3_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_r73b *x = g->boot->r73b; ++ const struct g5p_rv3_ev *t = &v->last[0], *d = &v->last[1]; ++ const struct g5p_ru_touch *k = v->touch; ++ ++ /* RC4a: every job while the A/B is on, so both arms are complete past the verbose jobs. */ ++ if (r->touch_ms) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 touch seq=%u ret=%d arm=%s touch_ms=%u ta=%s/%lld td=%s/%lld " ++ "seen_us=%lld/%lld read_us=%lld/%lld after_us=%lld/%lld bytes=%u/%u " ++ "sum=%08x/%08x end=v3\n", ++ v->seq, ret, ++ g5p_ru_touch_arm(v->seq) ? "touch" : "control", ++ r->touch_ms, g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? (s64)(v->ta.end_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? (s64)(v->td.end_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_ru_touch_us(v->ta_pub_ns, k[0].seen_ns), ++ g5p_ru_touch_us(v->td_pub_ns, k[1].seen_ns), ++ g5p_ru_touch_us(k[0].seen_ns, k[0].done_ns), ++ g5p_ru_touch_us(k[1].seen_ns, k[1].done_ns), ++ g5p_ru_touch_after_us(&k[0], v->ta.result, ++ v->ta.end_ns), ++ g5p_ru_touch_after_us(&k[1], v->td.result, ++ v->td.end_ns), ++ k[0].bytes, k[1].bytes, k[0].sum, k[1].sum); ++ /* wake110: every job while the wake is on (the end/lat/retire lines stop at seq 8). */ ++ if (r->wake) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 wake seq=%u ret=%d mode=%u seed=%08x arm=%s/%s did=%s/%s " ++ "power=%02x/%02x fence=%08x/%08x gpc=%08x/%08x cons_at=%u/%u bell_us=%lld/%lld " ++ "lock_us=%lld/%lld ta=%s/%lld td=%s/%lld ev_ms=%lld/%lld consumed_ms=%lld/%lld " ++ "busy_ms=%lld/%lld t1_pub=%u/%u t1_end=%u/%u endpoint=%u ta_stamp=%08x/%08x " ++ "td_stamp=%08x/%08x ta_ring=%u/%u td_ring=%u/%u rang=%u end=v3\n", ++ v->seq, ret, r->wake, r->wake_seed, ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 0)], ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[0], ++ G5P_WK_DIDS - 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[1], ++ G5P_WK_DIDS - 1)], ++ v->wk_power[0], v->wk_power[1], v->wk_fence[0], ++ v->wk_fence[1], v->wk_gpc[0], v->wk_gpc[1], ++ v->wk_cons[0], v->wk_cons[1], ++ g5p_rv3_wk_us(v->ta_pub_ns, v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->td_pub_ns, v->wk_done_ns[1]), ++ g5p_rv3_wk_us(v->wk_start_ns[0], v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->wk_start_ns[1], v->wk_done_ns[1]), ++ g5p_rv3_results[v->ta.result], ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_rv3_results[v->td.result], ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->wk_first_ns[0]), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->wk_first_ns[1]), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td_consumed_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.busy_ns), ++ v->wk_t1_pub[0], v->wk_t1_pub[1], v->wk_t1_end[0], ++ v->wk_t1_end[1], g5p_ru_endpoint(v->seq), t->stamp[0], ++ t->stamp[1], d->stamp[0], d->stamp[1], t->prod, t->cons, ++ d->prod, d->cons, r->wake_rang); ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 end seq=%u ret=%d phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " ++ "cap_ms=%u ta_consumed_ms=%lld td_consumed_ms=%lld late_gap=%u/%u " ++ "late_after=%u/%u grace=%u bells=%d ta_ring=%u/%u td_ring=%u/%u " ++ "ta_stamp=%08x/%08x td_stamp=%08x/%08x pm80=%08x r0_after_ta_publish=bell-only " ++ "end=v3\n", ++ v->seq, ret, ++ g5p_rv3_phase_names[min_t(u32, v->phase, G5P_RV3_PHASES - 1)], ++ g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? ++ (s64)(v->ta.end_ns - v->ta_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? ++ (s64)(v->td.end_ns - v->td_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ r->wait_ms, G5P_RU_V3_CAP_MS, ++ v->ta_consumed_ns ? (s64)(v->ta_consumed_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td_consumed_ns ? (s64)(v->td_consumed_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_gap, v->td.late_gap, v->ta.late_after, ++ v->td.late_after, v->grace_end, atomic_read(&x->r74f_bells), ++ t->prod, t->cons, d->prod, d->cons, t->stamp[0], t->stamp[1], ++ d->stamp[0], d->stamp[1], t->pm80 ? t->pm80 : d->pm80); ++ if (r->hard_ms) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 lat seq=%u soft_ms=%u hard_ms=%u obs_ms=%u slow=%u/%u late_soft=%u/%u " ++ "silent=%u/%u soft_seen_ms=%lld/%lld busy_ms=%lld/%lld late_after_ms=%lld/%lld " ++ "events=%u end=v3\n", ++ v->seq, r->soft_ms, r->hard_ms, r->obs_ms, v->ta.slow, ++ v->td.slow, v->ta.late_soft, v->td.late_soft, ++ v->ta.silent, v->td.silent, ++ v->ta.soft_seen_ns ? ++ (s64)(v->ta.soft_seen_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.soft_seen_ns ? ++ (s64)(v->td.soft_seen_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.busy_ns ? (s64)(v->ta.busy_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.busy_ns ? (s64)(v->td.busy_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_ns ? (s64)(v->ta.late_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.late_ns ? (s64)(v->td.late_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ READ_ONCE(g->boot->events)); ++ /* repeat109: the 3D's firmware retirement (coherent_ms, event_ms) apart from the host's decision ++ * (decided_ms; held_ms = how long a coherent target waited for it), the clauses still open at the ++ * deciding or closing sample (unsat=TA/3D, render-user-retire.h bits) and that sample's 3D idle words. ++ */ ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 retire seq=%u ret=%d td=%s how=%s settle_ms=%u coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld held_ms=%lld unsat=%#x/%#x " ++ "idle=%08x,%08x,%08x,%08x,%08x,%08x run=%u breaks=%u busy_ms=%lld clear_ms=%lld " ++ "ta_how=%s ta_unsat=%#x ta_coherent_ms=%lld ta_event_ms=%lld " ++ "ta_idle=%08x,%08x,%08x end=v3\n", ++ v->seq, ret, g5p_rv3_results[v->td.result], ++ g5p_rv3_how[min_t(u32, v->rt.decided, 3)], r->settle_ms, ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td_event_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_ru_rt_ms(v->rt.coherent_ns, v->td.end_ns), ++ g5p_rv3_ta_unsat(t), v->rt.unsat, d->idle[0], d->idle[1], ++ d->idle[2], d->idle[3], d->idle[4], d->idle[5], v->rt.run, ++ v->rt.breaks, g5p_ru_rt_ms(v->td_pub_ns, v->rt.busy_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.clear_ns), ++ g5p_rv3_ta_how[min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1)], ++ v->tr.unsat, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta_event_ns), t->idle[0], ++ t->idle[1], t->idle[2]); ++ /* The raw rows once per context (its first job) and for every failure: the blob is not a ++ * frame log. ++ */ ++ if (!v->touched || (r->jobs > 1 && !ret)) ++ return; ++ g5p_blob_hex(g, "RENDER-V3 PM-control", g5p_rv3_pm_cpu(g->boot, r), ++ 0x140); ++ g5p_blob_hex(g, "RENDER-V3 TA-W", ++ g5p_ru_page(g->boot, 0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, v->seq)), ++ G5P_RU_TA_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 TA-CH", g5p_ru_page(g->boot, 0), 0x340); ++ /* The stamp words (+0x1840/+0x1844) live in this 0x80-byte window; no earlier blob held one. */ ++ g5p_blob_hex(g, "RENDER-V3 TA-STAMP", ++ g5p_ru_page(g->boot, 0) + G5P_R22_TQ, 0x80); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) { ++ g5p_blob_hex(g, "RENDER-V3 3D-W", ++ g5p_ru_page(g->boot, 1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ v->seq)), ++ G5P_RU_3D_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 3D-CH", g5p_ru_page(g->boot, 1), ++ 0x340); ++ g5p_blob_hex(g, "RENDER-V3 3D-STAMP", ++ g5p_ru_page(g->boot, 1) + G5P_R22_TQ, 0x80); ++ } ++} ++ ++/* Jobs >= 2: the legacy job>=2 PM step with policy 0 (pool, list and F rebuilt from the device ++ * pool, state S kept), whichever context the job belongs to. ++ */ ++static int g5p_rv3_pm_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks); ++ mutex_unlock(&rd->lock); ++ kvfree(physical); ++ return ret; ++} ++ ++/* render-user-loss.h for a phase that did not complete. */ ++static int g5p_rv3_loss(struct g5p_drm *d, struct g5p_render *r, bool first, ++ u32 res, const struct g5p_ru_wait *w) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 drain_ms = (45000); ++ struct g5p_ru_loss_in in = { ++ .result = res, ++ .first = first, ++ .isolate = drain_ms && ++ g5p_ru_drain_valid(drain_ms, r->hard_ms, r->obs_ms), ++ .silent = w->silent, ++ .fault = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ }; ++ ++ return g5p_ru_loss(&in); ++} ++ ++/* ++ * The phase on engine `e` of this job timed out with the firmware alive: lose the job and its ++ * context, not the device. No terminal note and no device failure: the job ends -ETIMEDOUT ++ * (g5p_render_run() marks the context bad, so its client gets -EIO from now on and its pair and ++ * memory stay retained), and the channel is blocked behind the stuck phase until the scheduler's ++ * drain (g5p_rv3_drain()) clears it. RAM only. Fails (-ENOMEM) only if the record cannot be ++ * allocated; the caller then ends the epoch as before. ++ */ ++static int g5p_rv3_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ const struct g5p_ru_wait *w = e ? &v->td : &v->ta; ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = e; ++ s->drain_ms = (45000); ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u: context " ++ "lost, device kept, channel blocked end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++/* A job that stopped the firmware may be recovered from in this boot (core recover.inc, ++ * render-user-recover.h). A refusal is one line: why the epoch ends. ++ */ ++static bool g5p_rv3_recoverable(struct g5p_drm *d, bool first) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_ru_recover_in in = { ++ .first = first, ++ .retired = g5p_r8_is_retired(), ++ .closed = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .recoveries = rd->recoveries, ++ .streak = rd->recover_streak, ++ .budget = (1), ++ .streak_max = (3), ++ }; ++ ++ if (g5p_ru_recoverable(&in)) ++ return true; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 no-recover why=%s recoveries=%u streak=%u recover=%u streak_max=%u " ++ "boot_max=%u first=%u end=v3\n", ++ g5p_ru_recover_refusal(&in), in.recoveries, in.streak, ++ in.budget, in.streak_max, G5P_RU_RECOVER_BOOT_MAX, first); ++ return false; ++} ++ ++/* The job stopped the firmware (silent stream, a fault): lose it and its context, not the ++ * device; the worker recovers the GPU before it picks anything (g5p_rv3_recover()). RAM only. ++ */ ++static void g5p_rv3_lose_recover(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e, ++ const char *why) ++{ ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_fw_recovery rec = b->fw_recovery; ++ u32 recs = READ_ONCE(b->fw_recoveries); ++ ++ v->lost = true; ++ WRITE_ONCE(d->rdev->recovering, true); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u why=%s: context lost, device " ++ "recovers end=v3\n", ++ seq, e, r->slot[0], r->slot[1], why); ++ /* The guilty job is the one waited on (one job in flight, core lock); the packet's slot ++ * names the phase the firmware stopped in (render-user-recover.h). RAM only. ++ */ ++ if (recs) ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-attrib seq=%u engine=%u packet_slot=%u match=%u code=%08x " ++ "w20=%08x w24=%08x packets=%u end=v3\n", ++ seq, e, rec.slot, rec.slot == e, rec.code, rec.w20, ++ rec.w24, recs); ++} ++ ++/* render_user_test_fault_job: this job's TA reads its control stream (W+0x98, g5p_ru_fill_ta()) ++ * from the page the context reserved in its VM and never maps, so the GPU faults in that VM. A ++ * RAM edit of the job's own W copy before it is staged; the firmware sees an ordinary job. ++ */ ++static void g5p_rv3_test_fault(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, u32 seq) ++{ ++ g5p_put64(job->ta_w + 0x98, r->fault_node.start); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-fault seq=%u job=%u slots=%u/%u va=%#llx: TA control stream " ++ "unmapped end=v3\n", ++ seq, r->jobs + 1, r->slot[0], r->slot[1], r->fault_node.start); ++} ++ ++/* One V3 job: [attach, the epoch's job 1 only | bind, a later context's first job,] TA, 3D. ++ * Core lock held. Returns 0, -ECANCELED (a latched stage ended it), -ETIMEDOUT (a phase ran out ++ * its deadline, or the bind's DC3 was not consumed), or a fault. ++ */ ++/* perf60 MMU arm, bits 1/3 (render_user_mmu_inval): before a bound context's job publishes ++ * anything, move its slots' host tag to a fresh PC field, so the firmware's tag compare at this ++ * job's TA and 3D kicks takes its PC-level invalidate (e138 = (S << 3) | 4) for each slot. RAM ++ * only. vm->mutation is left alone: an edit on another thread (VM_BIND of a render VM takes no ++ * core lock) recomputes its own tag and checks it only against the cached tags, as before. ++ */ ++static void g5p_rv3_mmu_retag(struct g5p_diag *g, struct g5p_render *r, u32 seq, ++ bool switched) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 host[2], cached[2], tag; ++ ++ dma_rmb(); ++ for (u32 e = 0; e < 2; e++) { ++ host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4); ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ } ++ tag = g5p_ru_mmu_pc_tag(host[0] ?: cached[0], cached[0], cached[1]); ++ if (tag) { ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, ++ tag); ++ dma_wmb(); ++ } ++ g5p_mmu_record( ++ g, ++ "RENDER-V3 mmu-retag seq=%u slots=%u/%u why=%s host=%08x/%08x cached=%08x/%08x " ++ "tag=%08x end=v3\n", ++ seq, r->slot[0], r->slot[1], switched ? "switch" : "every", ++ host[0], host[1], cached[0], cached[1], tag); ++} ++ ++/* perf60 A/B: the park that the never-park envelope removed, back around a context switch only. ++ * Before a bound context's job that follows another context's job, ctrl+0x1040 = 50 makes the ++ * firmware's next pass compute x < 100 (policy state 2: it parks the GPC when idle, as release7); ++ * after the window the pin goes back and the job runs in state 0 with its bells as usual. RAM ++ * only: the firmware reloads policy+0x6f4 from ctrl+0x1040 on every pass (perf-envelope.h). ++ */ ++ ++ ++ ++/* perf60: the incoming context's render targets on a switch (lat123_log), RAM. */ ++static void g5p_rv3_switch_line(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ char buf[160]; ++ struct g5p_rt *rt; ++ u32 n = 0, len = 0; ++ ++ buf[0] = 0; ++ list_for_each_entry(rt, &r->targets, link) ++ if (n++ < 4) ++ len += scnprintf(buf + len, sizeof(buf) - len, ++ " %ux%u/s%u/p%u%s", rt->width, ++ rt->height, rt->samples ?: 1, ++ rt->page_count, ++ rt->destroyed ? "/d" : ""); ++ dev_info(g->dev, ++ "RENDER-V3 switch seq=%u to=%u/%u jobs=%u rts=%u%s end=v3\n", ++ seq, r->slot[0], r->slot[1], r->jobs, n, buf); ++} ++ ++/* perf60 per-context PM state (render_user_pm_switch, default 0). The PM record's selected state ++ * S (control+0x80, 0xc0 bytes: the backing pointers a0/a8/b0 the fresh TA loads the PM context ++ * from, the lanes TA finish saves and the 3D start restores, the bc accumulator) and the five ++ * aux pages it points into (the PM context store) are built once at attach and then shared by ++ * every context; the host never rewrites them. iOS builds one PM record per work queue ++ * (6cb2378: S explicit zeros plus the four backing pointers) and keeps it per queue. ++ * 1: a context's job after another context's job starts from attach's fresh state (S rebuilt, ++ * aux zeroed); ++ * 2: each context keeps its own S + aux: saved after each of its completed jobs, restored when ++ * it follows another context's job, fresh the first time. ++ * RAM only, between jobs (the previous job retired, this one has published nothing). ++ */ ++ ++/* perf60 stage 4: (re)build context r's PM set for this epoch: grow its pool to the device's ++ * block count, zero its aux pages, build its control + S, list and F (g5p_ru_pm(), as attach ++ * builds the device PM) and clean them to PoC. rd->lock (the pool's size) held. ++ */ ++static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ const char *how) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_dma_page *page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ u32 blocks = rd->blocks; ++ u64 *physical, list_va, free_va; ++ int ret = 0; ++ ++ lockdep_assert_held(&rd->lock); ++ if (blocks > r->pm_blocks) { ++ struct g5p_dma_page *pool = ++ kvcalloc(blocks * 8, sizeof(*pool), GFP_KERNEL); ++ ++ if (!pool) ++ return -ENOMEM; ++ if (r->pm_pool) ++ memcpy(pool, r->pm_pool, ++ r->pm_blocks * 8 * sizeof(*pool)); ++ for (u32 i = r->pm_blocks * 8; !ret && i < blocks * 8; i++) ++ ret = g5p_vm_alloc_page(d->drm.dev, &pool[i]); ++ if (ret) { ++ for (u32 i = r->pm_blocks * 8; i < blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &pool[i]); ++ kvfree(pool); ++ return ret; ++ } ++ kvfree(r->pm_pool); ++ r->pm_pool = pool; ++ r->pm_blocks = blocks; ++ } ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ for (u32 i = 0; i < r->pm_blocks * 8; i++) { ++ physical[i] = r->pm_pool[i].dma; ++ memset(r->pm_pool[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm_pool[i]); ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ struct g5p_dma_page *aux = &g5p_rv3_pm_aux(rd, r)[i]; ++ ++ memset(aux->cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, aux); ++ } ++ list_va = g5p_pmset_va(r->pm_set, G5P_PMSET_LIST); ++ free_va = g5p_pmset_va(r->pm_set, G5P_PMSET_FREE); ++ ret = g5p_ru_pm(page[G5P_PMSET_CONTROL].cpu, page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, list_va, free_va, physical, ++ r->pm_blocks, G5P_RU_PRIVATE_VA); ++ kvfree(physical); ++ for (u32 k = 0; k < G5P_PMSET_PAGES; k++) ++ g5p_table_clean_page(g, &page[k]); ++ dma_wmb(); ++ if (!ret) ++ r->pm_epoch = rd->recoveries + 1; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u meta=%#x " ++ "ret=%d end=v3\n", ++ seq, r->slot[0], r->slot[1], how, r->pm_set, ++ g5p_rv3_pm_id(rd, r), r->pm_blocks, g5p_rv3_pm_meta(r), ret); ++ return ret; ++} ++ ++/* Before a context's bind (its tables are published by the bind that follows): take a free set, ++ * allocate its aux pages, point G5P_RU_PRIVATE_VA in its VM at them, and build the set. On any ++ * failure the context keeps the device PM and the shared aux pages, as before stage 4. ++ */ ++static void g5p_rv3_pmctx_assign(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ int set, ret = 0; ++ ++ if (r->pm_set >= 0) ++ return; ++ set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next); ++ for (u32 i = 0; set >= 0 && !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!r->pm_aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &r->pm_aux[i]); ++ if (set < 0 || ret) { ++ dev_warn( ++ d->core->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=none set=%d ret=%d: the device PM end=v3\n", ++ seq, r->slot[0], r->slot[1], set, ret); ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "new"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ return; ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ u64 va = G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE; ++ u8 *pte = g5p_vm_pte(vm, d->drm.dev, va, false); ++ ++ u64 leaf = g5p_leaf_entry(r->pm_aux[i].dma, ++ g5p_r25_attributes(&r->v, va)); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, leaf, 2); ++ } ++ dma_wmb(); ++ r->pm_aux_mapped = true; ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* Before each job of a context with a set: a new epoch or a grown device pool rebuilds the set; ++ * without pm_once the pool is rebuilt per job, as g5p_rv3_pm_rebuild() does the device's. ++ */ ++static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_dma_page *page; ++ u64 *physical; ++ int ret = 0; ++ ++ if (r->pm_set < 0) ++ return 0; ++ page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ mutex_lock(&rd->lock); ++ if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { ++ ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); ++ } else if (!rd->pm_once) { ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), ++ GFP_KERNEL); ++ ret = physical ? 0 : -ENOMEM; ++ for (u32 i = 0; !ret && i < r->pm_blocks * 8; i++) ++ physical[i] = r->pm_pool[i].dma; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ page[G5P_PMSET_CONTROL].cpu, ++ page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, ++ g5p_pmset_va(r->pm_set, G5P_PMSET_LIST), ++ g5p_pmset_va(r->pm_set, G5P_PMSET_FREE), ++ physical, r->pm_blocks); ++ kvfree(physical); ++ } ++ mutex_unlock(&rd->lock); ++ return ret; ++} ++ ++/* render_user_pm_ctx 3: the context that brought the epoch up (the device PM for its attach job) ++ * takes its own set before its next job; the device aux pages its VM already maps become the ++ * set's aux (no other context maps them once every later one has its own), so no PTE changes. ++ */ ++static void g5p_rv3_pmctx_take(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ int set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next), ++ ret; ++ ++ if (set < 0) { ++ r->pm_first = false; /* stays on the device PM; not retried */ ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "take"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ r->pm_first = false; ++ return; ++ } ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* perf60 stage 4 diagnostics (render_user_fault_read): whose page each PM lane in S points at ++ * after the TA saved them: the device pool or aux, a set page, a context's pool or aux. ++ */ ++ ++static void g5p_rv3_pm_lanes(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ { ++ return; ++ } ++} ++ ++/* The context is gone (g5p_render_release()): its set is free again; its pages go. */ ++static void g5p_rv3_pmctx_release(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ ++ if (r->pm_set >= 0) ++ rd->pm_set_used[r->pm_set] = false; ++ r->pm_set = -1; ++ for (u32 i = 0; r->pm_pool && i < r->pm_blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_pool[i]); ++ kvfree(r->pm_pool); ++ r->pm_pool = NULL; ++ r->pm_blocks = 0; ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_aux[i]); ++} ++ ++#define G5P_RV3_PM_S 0x80U ++#define G5P_RV3_PM_S_LEN 0xc0U ++#define G5P_RV3_PM_SAVE (G5P_RV3_PM_S_LEN + G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ ++/* perf60 fault address: PM state snapshots (S, PM control, the aux windows S points at, each aux ++ * page's head) at three phases of every job, in a ring and per slot pair; printed at a fault read. ++ * Only the executor writes them. RAM only. ++ */ ++enum { G5P_PMSNAP_START, G5P_PMSNAP_TA_DONE, G5P_PMSNAP_END }; ++struct g5p_pmsnap { ++ u64 ns, auxva[3]; ++ u32 seq, phase, slot, valid; ++ u8 s[G5P_RV3_PM_S_LEN], ctl[0x80], win[3][64], ++ head[G5P_RU_AUX_PAGES][64]; ++}; ++#define G5P_PMSNAP_RING 9U ++static struct g5p_pmsnap g5p_pmsnap_ring[G5P_PMSNAP_RING]; ++static struct g5p_pmsnap g5p_pmsnap_slot[16][3]; ++static u32 g5p_pmsnap_head; ++ ++static void g5p_rv3_pmsnap(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 phase) ++{ ++ { ++ return; ++ } ++} ++ ++static void g5p_rv3_pmsnap_print(struct g5p_diag *g, u32 seq, const char *set, ++ const struct g5p_pmsnap *x) ++{ ++ u64 now = ktime_get_ns(); ++ ++ if (!x->valid) ++ return; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pmsnap seq=%u set=%s job=%u slot=%u phase=%s ago_us=%llu " ++ "aux_va=%#llx/%#llx/%#llx end=v3\n", ++ seq, set, x->seq, x->slot, ++ x->phase == G5P_PMSNAP_START ? "start" : ++ x->phase == G5P_PMSNAP_TA_DONE ? "ta-done" : ++ "end", ++ (now - x->ns) / NSEC_PER_USEC, x->auxva[0], x->auxva[1], ++ x->auxva[2]); ++ for (u32 o = 0; o < sizeof(x->s); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap S+%02x %*ph\n", o, 64, ++ x->s + o); ++ for (u32 o = 0; o < sizeof(x->ctl); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap C+%02x %*ph\n", o, 64, ++ x->ctl + o); ++ for (u32 k = 0; k < 3; k++) ++ dev_info(g->dev, "RENDER-V3 pmsnap W%u %*ph\n", k, 64, ++ x->win[k]); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ dev_info(g->dev, "RENDER-V3 pmsnap A%u %*ph\n", i, 64, ++ x->head[i]); ++} ++ ++static void g5p_rv3_pmsnap_dump(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ for (u32 p = 0; p < 3; p++) ++ g5p_rv3_pmsnap_print(g, seq, "slot-last", ++ &g5p_pmsnap_slot[r->slot[0] & 15][p]); ++ for (u32 i = G5P_PMSNAP_RING; i; i--) ++ g5p_rv3_pmsnap_print(g, seq, "ring", ++ &g5p_pmsnap_ring[(g5p_pmsnap_head - i) % ++ G5P_PMSNAP_RING]); ++} ++ ++/* perf60 fault address: the VM edit log entries that cover the fault VA (any VM, newest first), ++ * then the faulting job's VM's latest edits. Copied under the lock, printed outside it. ++ */ ++#define G5P_VMLOG_HITS 24U ++#define G5P_VMLOG_TAIL 16U ++static void g5p_vmlog_dump(struct g5p_diag *g, u32 seq, const void *vm, u64 va, ++ bool have_va) ++{ ++ struct g5p_vmlog_ent *out = kmalloc_array( ++ G5P_VMLOG_HITS + G5P_VMLOG_TAIL, sizeof(*out), GFP_KERNEL); ++ u32 hits = 0, tail = 0, n, head; ++ unsigned long flags; ++ u64 now = ktime_get_ns(); ++ ++ if (!out) ++ return; ++ spin_lock_irqsave(&g5p_vmlog.lock, flags); ++ head = g5p_vmlog.head; ++ n = min(head, G5P_VMLOG_N); ++ for (u32 i = 1; i <= n; i++) { ++ const struct g5p_vmlog_ent *x = ++ &g5p_vmlog.e[(head - i) % G5P_VMLOG_N]; ++ ++ if (have_va && hits < G5P_VMLOG_HITS && ++ g5p_ru_mmu_fault_in(va, x->va, x->size)) ++ out[hits++] = *x; ++ else if (x->vm == vm && tail < G5P_VMLOG_TAIL) ++ out[G5P_VMLOG_HITS + tail++] = *x; ++ } ++ spin_unlock_irqrestore(&g5p_vmlog.lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u va=%#llx edits=%u covering=%u vm_tail=%u end=v3\n", ++ seq, have_va ? va : 0, head, hits, tail); ++ for (u32 i = 0; i < hits + tail; i++) { ++ const struct g5p_vmlog_ent *x = ++ &out[i < hits ? i : G5P_VMLOG_HITS + i - hits]; ++ ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u %s ago_ms=%llu op=%s pid=%d comm=%s vm=%s/%u va=%#llx " ++ "size=%#llx handle=%#x off=%#llx end=v3\n", ++ seq, i < hits ? "cover" : "tail", ++ (now - x->ns) / NSEC_PER_MSEC, ++ g5p_vmlog_ops[min_t(u32, x->op, ++ ARRAY_SIZE(g5p_vmlog_ops) - 1)], ++ x->tgid, x->comm, x->vm == vm ? "this" : "other", ++ x->vm_id, x->va, x->size, x->handle, x->offset); ++ } ++ kfree(out); ++} ++ ++/* perf60 fault address (render_user_fault_read): the MMU fault banks, read once at a failed ++ * phase, the way iOS's fault handler (6ca5c14) reads them; the firmware keeps no copy. Bell mode ++ * only. Under the event lock with the wake path's power proof (the fault already latched ++ * b->failed, so that flag is not a guard here), then the GPU register licence (mmio.inc, ++ * mmio-access.md §4): no wake (Apple's reads never write R1+0x14, G6), a refusal retried outside ++ * the locks every 20-40 us for at most G5P_RV3_DC_SETTLE_US, and the four bank reads only under a ++ * held ticket. A parked GPC records skip-parked, as Apple's report has no bif fault entry then. ++ * At most G5P_RV3_FAULT_READS per boot. It runs before the loss decision and the terminal ++ * latch, while the faulted job is still the GPU's current work. ++ */ ++#define G5P_RV3_FAULT_READS G5P_RU_RECOVER_BOOT_MAX ++static atomic_t g5p_rv3_fault_reads = ATOMIC_INIT(0); ++ ++/* faultrec: a fault VA in the PM-set window (render-user-pmset.h) names a set and its part; say ++ * which context owns that set now and how it stands against the two job records and the deferred ++ * closes (render_user_close_defer). RAM only; rdev lock for the context list. ++ */ ++static void g5p_rv3_fault_pmset(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq, u32 bank, ++ const struct g5p_ru_mmu_fault *f) ++{ ++ static const char *const parts[G5P_PMSET_PAGES] = { "control", "list", ++ "free" }; ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_render *o, *owner = NULL; ++ u32 page, set, own[2] = { U32_MAX, U32_MAX }, df[2][2]; ++ bool last = false, prev = false, closing = false, bound = false; ++ ++ if (f->va < G5P_PMSET_VA || f->va >= G5P_PMSET_END) ++ return; ++ page = (f->va - G5P_PMSET_VA) / G5P_GPU_PAGE; ++ set = page / G5P_PMSET_PAGES; ++ mutex_lock(&rd->lock); ++ list_for_each_entry(o, &rd->contexts, rdev_link) ++ if (o->pm_set == (int)set) { ++ owner = o; ++ break; ++ } ++ if (owner) { ++ own[0] = owner->slot[0]; ++ own[1] = owner->slot[1]; ++ last = rd->pm_last == owner; ++ prev = rd->pm_prev == owner; ++ closing = owner->closing; ++ bound = owner->bound; ++ } ++ for (u32 k = 0; k < 2; k++) { ++ o = rd->close_deferred[k]; ++ df[k][0] = o ? o->slot[0] : U32_MAX; ++ df[k][1] = o ? o->slot[1] : U32_MAX; ++ } ++ mutex_unlock(&rd->lock); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-pmset seq=%u bank=%u mmu_context=%u set=%u part=%s owner=%d/%d " ++ "bound=%u closing=%u last=%u prev=%u job=%u/%u deferred=%d/%d,%d/%d job_slots=%u " ++ "end=v3\n", ++ seq, bank, f->context, set, parts[page % G5P_PMSET_PAGES], ++ (int)own[0], (int)own[1], bound, closing, last, prev, ++ r->slot[0], r->slot[1], (int)df[0][0], (int)df[0][1], ++ (int)df[1][0], (int)df[1][1], rd->job_slots); ++} ++ ++static void g5p_rv3_fault_read(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 e, int res, bool silent) ++{ ++ u32 full = (0), w[4] = { 0 }; ++ u32 mode = full | ((1) ? 1 : 0); ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_mmu_fault f[2]; ++ struct g5p_mmio_ticket t = { .r1_0 = U32_MAX, ++ .gpc = U32_MAX, ++ .r1_0_end = U32_MAX, ++ .gpc_end = U32_MAX }; ++ const char *did = "skip-guard"; ++ u32 power = 0, tries = 0; ++ unsigned long flags; ++ bool fault = res == G5P_RU_WAIT_FAULT && !silent, ticket = false; ++ u64 t0; ++ int ret; ++ ++ if ((2) != G5P_WK_ALWAYS || !(mode & (fault ? 1 : 2))) ++ return; ++ if (atomic_inc_return(&g5p_rv3_fault_reads) > G5P_RV3_FAULT_READS) { ++ did = "skip-cap"; ++ goto print; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ for (;;) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->stopped) || !g5p_wk_power_ready(power)) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ break; ++ } ++ tries++; ++ ticket = true; ++ ret = g5p_mmio_hold(g, &t, 0); ++ if (!ret) { ++ w[0] = g5p_mmio_r0_read(g, &t, 0xe150); ++ w[1] = g5p_mmio_r0_read(g, &t, 0xe154); ++ w[2] = g5p_mmio_r0_read(g, &t, 0xe158); ++ w[3] = g5p_mmio_r0_read(g, &t, 0xe15c); ++ g5p_mmio_release(g, &t); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ did = g5p_mmio_did(&t); ++ if (ret != -EAGAIN || ++ ktime_get_mono_fast_ns() - t0 >= ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) ++ break; ++ usleep_range(20, 40); ++ } ++ if (ticket) ++ g5p_mmio_sample_line(g, G5P_MMS_FAULT, &t, w, t.ops ? 4 : 0, ++ true); ++print: ++ f[0] = g5p_ru_mmu_fault_decode(w[0], w[1]); ++ f[1] = g5p_ru_mmu_fault_decode(w[2], w[3]); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-read seq=%u engine=%u slots=%u/%u why=%s did=%s power=%02x " ++ "fence=none r1_0=%08x gpc=%08x tries=%u e150=%08x:%08x e158=%08x:%08x end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ fault ? "fault" : ++ silent ? "silent" : ++ "timeout", ++ did, power, t.r1_0, t.gpc, tries, w[1], w[0], w[3], w[2]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-decode seq=%u bank=%u va=%#llx %s type=%s level=%u " ++ "requester=%u/%s sideband=%#x context=%u job_slots=%u/%u end=v3\n", ++ seq, k, f[k].va, f[k].read ? "read" : "write", ++ g5p_ru_mmu_fault_types[f[k].type], f[k].level, ++ f[k].requester, ++ g5p_ru_mmu_requester(f[k].requester), ++ f[k].sideband, f[k].context, r->slot[0], ++ r->slot[1]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ g5p_rv3_fault_pmset(d, r, seq, k, &f[k]); ++ g5p_rv3_switch_line(g, r, seq); ++ if (!full) ++ return; ++ g5p_vmlog_dump(g, seq, r->ctx->vm, f[0].valid ? f[0].va : f[1].va, ++ f[0].valid || f[1].valid); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ g5p_rv3_pmsnap_dump(g, r, seq); ++} ++ ++/* GPU overclock thermal stop (core boot.inc, boot_gpu_oc_mhz): every 500 ms while the epoch runs ++ * state 5, the three SoC zones; at or above boot_gpu_oc_temp_mc (or unreadable) the overclock is off ++ * for the rest of the boot and the render scheduler recovers the GPU (g5p_rv3_recover(): gate cycle, ++ * firmware boot) between jobs, so the new epoch powers on at state 4 with record 5 cleared: iOS's ++ * park -> reboot -> new pstate, never a pstate change under a live firmware. Every 10 s one status ++ * line: the temperatures, the real Fender state (an R1-only licence ticket) and the jobs run. ++ */ ++#define G5P_OC_POLL_MS 500 ++static void g5p_rsched_kick(struct g5p_drm *d); ++ ++static void g5p_oc_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = ++ container_of(to_delayed_work(work), struct g5p_rdev, oc_work); ++ struct g5p_drm *d = rd->drm; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ u32 lim = (90000), zones; ++ const char *zone; ++ int t; ++ ++ if (!b || !0 || READ_ONCE(g5p_oc_tripped)) { ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ zones = g5p_oc_temp(&t, &zone); ++ if (zones != ARRAY_SIZE(g5p_oc_zone_names) || t >= (int)lim) { ++ WRITE_ONCE(g5p_oc_tripped, true); ++ dev_emerg( ++ g->dev, ++ "GPUPERF oc thermal-stop temp=%d zone=%s zones=%u limit=%u mhz=%u jobs=%u: the " ++ "GPU recovers at state 4 (723 MHz) end=gpuperf\n", ++ t, zone, zones, lim, b->gpu_oc_mhz, rd->oc_jobs); ++ WRITE_ONCE(rd->recovering, true); ++ g5p_rsched_kick(d); ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ if (!(++rd->oc_ticks % (10000 / G5P_OC_POLL_MS))) { ++ struct g5p_mmio_ticket tk; ++ ++ if (!g5p_mmio_hold(g, &tk, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &tk); ++ dev_info( ++ g->dev, ++ "GPUPERF oc status mhz=%u temp=%d zone=%s limit=%u r1=%s r1_1000=%08x jobs=%u " ++ "end=gpuperf\n", ++ b->gpu_oc_mhz, t, zone, lim, g5p_mmio_did(&tk), tk.gpc, ++ rd->oc_jobs); ++ } ++ queue_delayed_work(system_wq, &rd->oc_work, ++ msecs_to_jiffies(G5P_OC_POLL_MS)); ++} ++ ++ ++ ++static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ bool first = !rd->up, bound_before = r->bound; ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (r->stage_done) ++ return -ECANCELED; ++ g5p_l123_job_reset(g); ++ /* The submit-time admission again, now under the core lock and with the bind state: still nothing touched. */ ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ seq = rd->sequence + 1; ++ ; ++ 0; ++ if (bound_before && d->bell_prev_ctx && d->bell_prev_ctx != r->ctx) { ++ u32 ms = (0); ++ ++ ; ++ ; ++ if (ms && b->gpu_perf) ++ 0; ++ } ++ 0; ++ if (g5p_ru_v3_verbose(r->jobs + 1, 0)) ++ g5p_job_printk( ++ g->dev, ++ "RENDER-V3 start seq=%u bound_ms=%u cap_ms=%u slots=%u/%u attach=%s " ++ "ta_doorbell=%s 3d_doorbell=none wait=ram-only after_timeout=quiet end=v3\n", ++ seq, r->wait_ms, G5P_RU_V3_CAP_MS, r->slot[1], ++ r->slot[0], ++ first ? "firmware-dc3" : ++ r->bound ? "kept" : ++ rd->spare_count ? "spare" : ++ "ram-dc3", ++ first ? "grace-owned" : "none"); ++ if (first) { ++ v->phase = G5P_RV3_ATTACH; ++ ret = g5p_rv3_spares(d, v); ++ if (!ret) ++ ret = g5p_rv3_attach(d, r, v); ++ /* perf60 stage 4: a context with a set that brings a new epoch up */ ++ if (!ret) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ if (ret) ++ goto out; ++ ; ++ } else { ++ if (!r->bound) { ++ /* perf60 stage 4: its own PM set and aux pages before the bind ++ * publishes its tables ++ */ ++ if (rd->pm_ctx) ++ g5p_rv3_pmctx_assign(d, r, seq); ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ ; ++ } ++ /* The firmware owns the rings and the channels from here on: any failure quarantines. */ ++ v->touched = true; ++ if (rd->pm_ctx == 3 && r->pm_first && r->pm_set < 0) ++ g5p_rv3_pmctx_take(d, r, seq); ++ if (r->pm_set >= 0) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ else ++ ret = rd->pm_once ? 0 : g5p_rv3_pm_rebuild(d); ++ if (ret) ++ goto out; ++ ; ++ ; ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_START); ++ if (g5p_ru_mmu_retag_due((0), bound_before, ++ d->bell_prev_ctx != r->ctx)) ++ g5p_rv3_mmu_retag(g, r, seq, ++ d->bell_prev_ctx != r->ctx); ++ } ++ rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu(b, r) + 0x80); ++ if (g5p_ru_fault_due(r->fault_job, r->jobs, first)) ++ g5p_rv3_test_fault(g, r, job, seq); ++ for (u32 e = 0; e < 2; e++) { ++ g5p_ru_job_at( ++ g5p_ru_page(b, e), g5p_ru_meta(e), ++ g5p_meta_address(g5p_object_va(&b->layout, 0x51 + e)), ++ g5p_rv3_pm_meta(r), r->slot[e], e, seq, ++ e ? job->td_w : job->ta_w, ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r)); ++ r->submitted[e] = false; ++ r->consumed[e] = false; ++ r->retired[e] = false; ++ } ++ g5p_l123_job_prepare(d, job); ++ g5p_render_clean_job(g); ++ rd->sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_rv3_wake_mark(d, r, v, 0); ++ ret = g5p_rv3_publish_ta(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ if (r->stall_job && r->jobs == r->stall_job && !first) { ++ /* render_user_test_stall_job: the host gives up, the firmware runs on */ ++ g5p_ru_wait_start(&v->ta, v->ta_pub_ns, 1, v->job_deadline_ns); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-stall seq=%u job=%u slots=%u/%u: TA bound 1 ms end=v3\n", ++ seq, r->jobs, r->slot[0], r->slot[1]); ++ } else { ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, ++ v->job_deadline_ns); ++ } ++ g5p_ru_touch_start(&v->touch[0], v->ta_pub_ns, r->touch_ms, seq); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, seq, &v->ta, &v->touch[0], NULL, &v->tr, ++ &v->last[0], &v->ta_consumed_ns, &err); ++ r->consumed[0] = v->ta_consumed_ns != 0; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ /* Timeout or fault: (latency101: the RAM-only observation window first,) terminal ++ * latch, one RAM sample for the record (a completion that arrives now is evidence, ++ * never the result), no export, no register access. ++ */ ++ g5p_rv3_fault_read(d, r, seq, 0, res, v->ta.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 0)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 0, ++ v->ta.silent ? "ta-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "ta-timeout" : ++ "ta-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "ta-timeout" : "ta-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_TIMEOUT, ret); ++ goto out; ++ } ++ g5p_rv3_ta_count(g, d, v); ++ r->retired[0] = true; ++ g5p_rv3_terminal(g, G5P_RU_TERM_COMPLETE, "ta-complete"); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_TA_DONE); ++ g5p_rv3_pm_lanes(d, r, seq); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, ++ g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); ++ ; ++ v->phase = G5P_RV3_TD_PUBLISH; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample( ++ g, r, 1, seq, ++ &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ ++ goto out; ++ } ++ v->phase = G5P_RV3_TD_WAIT; ++ g5p_rv3_wait_start(g, r, &v->td, v->td_pub_ns, v->job_deadline_ns); ++ g5p_ru_touch_start(&v->touch[1], v->td_pub_ns, r->touch_ms, seq); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, ++ &v->last[1], &v->td_consumed_ns, &err); ++ r->consumed[1] = v->td_consumed_ns != 0; ++ /* mmio_selftest bit 2 (rung 5, DFU-risk on a park tuple): one licensed ticket at the 3D ++ * completion decision; always: the idle clock of bit 3. ++ */ ++ ; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ g5p_rv3_fault_read(d, r, seq, 1, res, v->td.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->td); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 1)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 1, ++ v->td.silent ? "3d-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "3d-timeout" : ++ "3d-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "3d-timeout" : "3d-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ /* Rule (c) decided: the matching event was not seen. Always one line (RAM only). */ ++ if (v->rt.decided == G5P_RU_RT_HOLD) ++ dev_warn( ++ g->dev, ++ "RENDER-V3 3d-hold seq=%u idle=%08x,%08x,%08x,%08x,%08x,%08x coherent_ms=%lld " ++ "decided_ms=%lld end=v3\n", ++ seq, v->last[1].idle[0], v->last[1].idle[1], ++ v->last[1].idle[2], v->last[1].idle[3], ++ v->last[1].idle[4], v->last[1].idle[5], ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns)); ++ r->retired[1] = true; ++ v->phase = G5P_RV3_DONE; ++ rd->recover_streak = ++ 0; /* a job finished: a later fault starts a new streak */ ++ ret = 0; ++out: ++ if (v->phase >= G5P_RV3_TA_PUBLISH) ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ g5p_rv3_wake_end(d, r, v); ++ /* Any other failure after the hardware was touched (attach, publish, drain, protocol): the same ++ * criterion g5p_render_run() quarantines the context by. A stage stop (-ECANCELED) is not one. ++ */ ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), ++ g5p_rv3_phase_names[min_t( ++ u32, v->phase, G5P_RV3_PHASES - 1)]); ++ g5p_rv3_receipt(g, r, job, ret); ++ g5p_l123_job_end(d, job, seq, ret); ++ if (!ret) { ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ ; ++ ; ++ } ++ /* render-user-pmset.h: the PM's write-back targets */ ++ if (g5p_pmset_switched(rd->pm_last, r)) { ++ rd->pm_prev = rd->pm_last; ++ rd->pm_last = r; ++ } ++ /* perf60 bell gates: the next job's gap and context switch are measured from here. */ ++ d->bell_prev_end_ns = ktime_get_ns(); ++ d->bell_prev_ctx = r->ctx; ++ return ret; ++} ++ ++/* ++ * The channel is blocked behind a lost job (g5p_rv3_lose()): wait for its stuck phase, RAM only, ++ * with the firmware's event stream watched from the first sample (silent for 1 s = dead). A late ++ * TA gets the job's 3D published (its W is staged since the job's start; no doorbell for a job ++ * other than the epoch's first), so the 3D channel's sequence and the PM state stay as the ++ * firmware expects; a late 3D clears the channel. Then the held jobs run. A silent stream, a ++ * fault, a refused 3D or the drain bound end the epoch as a timeout or fault always did, and the ++ * held jobs then fail at admission. Called by the scheduler's worker before any pick; returns ++ * false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_rv3_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 st = s->engine ? G5P_RU_DRAIN_TD : G5P_RU_DRAIN_TA, ++ res = G5P_RU_WAIT_PENDING; ++ struct g5p_ru_wait w = {}; ++ bool armed = false; ++ int act, err = -EIO; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ for (;;) { ++ u32 e = st == G5P_RU_DRAIN_TA ? 0 : 1; ++ u64 now = ktime_get_ns(); ++ ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->stopped)) { ++ res = G5P_RU_WAIT_FAULT; ++ } else if (!e) { ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], ++ NULL, &v->tr, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ } else { ++ if (!armed) ++ g5p_rv3_event_arm(d, 1, ++ g5p_ru_endpoint(s->seq)); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], ++ &v->rt, NULL, &v->last[1], ++ &v->td_consumed_ns, &err); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ } ++ act = g5p_ru_drain_step(&st, res); ++ if (act != G5P_RU_DRAIN_PUBLISH_TD) ++ break; ++ r->retired[0] = true; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); ++ armed = true; ++ if (g5p_rv3_publish_3d(g, r, v, s->seq)) { ++ g5p_rv3_event_disarm(d, 1); ++ res = G5P_RU_WAIT_FAULT; ++ act = g5p_ru_drain_step(&st, res); ++ break; ++ } ++ } ++ if (act == G5P_RU_DRAIN_UNBLOCK) { ++ r->retired[1] = true; ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drained seq=%u engine=%u slots=%u/%u lost_ms=%lld: channel clear, " ++ "held jobs resume end=v3\n", ++ s->seq, s->engine, r->slot[0], r->slot[1], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drain dead seq=%u engine=%u phase=%s why=%s err=%d lost_ms=%lld: the " ++ "device recovers if it may, else the epoch ends end=v3\n", ++ s->seq, s->engine, ++ st == G5P_RU_DRAIN_DEAD && !v->td_pub_ns ? "ta" : "3d", ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal( ++ g, ++ fault ? G5P_RU_TERM_FAULT : G5P_RU_TERM_TIMEOUT, ++ fault ? "drain-fault" : "drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++/* The firmware window's leaves again from the new epoch's tables, in the same pages, so every ++ * render VM's root entry stays valid (g5p_render_fw_entry()). Core lock held. ++ */ ++static void g5p_render_fw_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_boot *b = d->core->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ ++ if (!w->built) ++ return; ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ w->pages = 0; ++ w->missing = 0; ++ for (u32 i = 0; i < G5P_RU_FW_PAGES; i++) { ++ u64 va = g5p_ru_fw_window_va[i]; ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (k == w->leaves) ++ continue; ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, ++ leaf & 1 ? leaf : 0); ++ w->pages += leaf & 1; ++ w->missing += !(leaf & 1); ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(d->core, &w->leaf[i]); ++} ++ ++/* ++ * A lost job stopped the firmware (g5p_rv3_lose_recover(), or a drain that died): cycle the GPU ++ * and boot the firmware again (core recover.inc), then start the render device over as at boot. ++ * Every context survives except the lost one: its bindings belonged to the old epoch, which the ++ * cycle ended, so each is unbound in RAM, its binding's reference dropped, and its next job binds ++ * again (the first job after the recovery runs the epoch's attach). The aux pages and the PM ++ * pool are zeroed as at their allocation. The held jobs run after this. Returns false, having ++ * done nothing, when the core lock is busy. A failed recovery leaves the device failed. ++ */ ++static bool g5p_rv3_recover(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ unsigned long deadline = jiffies + 2 * HZ, flags; ++ struct g5p_render **drop = NULL; ++ struct g5p_render *r; ++ u32 n = 0, count = 0, dropped = 0, stage; ++ int ret; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ count++; ++ } ++ mutex_unlock(&rd->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ rd->recoveries++; ++ rd->recover_streak++; ++ ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; ++ if (ret) { ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", ++ ret, rd->recoveries); ++ return true; ++ } ++ g5p_ru_term_reset(&g5p_rv3_term); ++ /* the new firmware's PM has written nothing back yet: no context is its last or previous */ ++ rd->pm_last = NULL; ++ rd->pm_prev = NULL; ++ rd->up = false; ++ rd->v3 = false; ++ rd->sequence = 0; ++ rd->pm_serial = 0; ++ rd->dc_pending = false; ++ rd->dc_wait = 0; ++ if (rd->spares) { ++ spin_lock_irqsave(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ if (!rd->spares[i].used) { ++ __clear_bit(rd->spares[i].slot[0], ++ g->user_slots); ++ __clear_bit(rd->spares[i].slot[1], ++ g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rd->spares[i].root); ++ kfree(rd->spares); ++ rd->spares = NULL; ++ rd->spare_count = 0; ++ } ++ g5p_render_fw_rebuild(d); ++ mutex_lock(&rd->lock); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ if (rd->aux[i].cpu) { ++ memset(rd->aux[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->aux[i]); ++ } ++ for (u32 i = 0; i < rd->blocks * 8; i++) { ++ memset(rd->physical[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->physical[i]); ++ } ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ r->jobs = 0; ++ r->dc_target = 0; ++ WRITE_ONCE(r->live_maps, false); ++ memset(r->submitted, 0, sizeof(r->submitted)); ++ memset(r->consumed, 0, sizeof(r->consumed)); ++ memset(r->retired, 0, sizeof(r->retired)); ++ /* nothing of the old epoch holds this context's memory any more */ ++ r->lost_drained = true; ++ WRITE_ONCE(r->ctx->live, false); ++ if (r->retained && n < count) { ++ r->retained = false; ++ drop[n++] = r; ++ } ++ } ++ mutex_unlock(&rd->lock); ++ spin_lock_irqsave(&d->status_lock, flags); ++ d->last_error = 0; ++ d->job_error = 0; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ for (u32 i = 0; i < n; i++) { ++ struct g5p_context *c = drop[i]->ctx; ++ ++ dropped++; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recovered n=%u contexts=%u unbound=%u streak=%u: held jobs resume, " ++ "the next job attaches end=v3\n", ++ rd->recoveries, count, dropped, rd->recover_streak); ++ return true; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +new file mode 100644 +index 0000000000..55334c5996 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -0,0 +1,2482 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Userspace TA/3D render: render contexts, render targets and SUBMIT_RENDER. ++ * Builds on the r22 constructors through render-user-layout.h (byte-equal for ++ * the round16 inputs). Default off. The first accepted job claims the device ++ * for render; TYPED_V3 contexts then share it, each with its own VM and slot ++ * pair (docs/gpu/multi-client-design.md), the other profiles stay with that ++ * first context. Jobs of all contexts are serialized; no reset/recovery: any ++ * hardware failure retains everything. ++ */ ++#include "../../apple-g5p/render-user-layout.h" ++#include "../../apple-g5p/render-user-wait.h" ++#include "../../apple-g5p/render-user-touch.h" ++#include "../../apple-g5p/render-user-retire.h" ++#include "../../apple-g5p/render-user-wake.h" ++#include "../../apple-g5p/render-user-serial.h" ++#include "../../apple-g5p/render-user-terminal.h" ++#include "../../apple-g5p/render-user-bind.h" ++#include "../../apple-g5p/render-user-close.h" ++#include "../../apple-g5p/render-user-sched.h" ++#include "../../apple-g5p/render-user-window.h" ++#include "../../apple-g5p/render-user-msaa.h" ++#include "../../apple-g5p/render-user-loss.h" ++#include "../../apple-g5p/render-user-fault.h" ++#include "../../apple-g5p/render-user-mmu.h" ++#include "../../apple-g5p/render-user-faultreg.h" ++#include "render-user-validate.h" ++#include "drm-render-v3.h" ++ ++/* The terminal state of the epoch (one GPU): from the first terminal note on, no register access. */ ++static struct g5p_ru_terminal g5p_rv3_term; ++ ++/* Defaults of the latched knobs. GET_RENDER_INFO reports VARIANT_ACTIVE when ++ * any knob differs, so the declarations below and that comparison share them. ++ */ ++#define G5P_RU_DEF_CACHE_VARIANT 3 ++#define G5P_RU_DEF_MMU_VARIANT 1 ++/* 20 s: the natural TA completion took 12.006 s in the r96 boots a 12 s cap turned into ++ * timeouts (render-user-wait.h); it is also the minimum of a TYPED_V3 job's per-phase bound. ++ */ ++#define G5P_RU_DEF_WAIT_MS G5P_RU_V3_WAIT_MS ++#define G5P_RU_DEF_PROFILE1_VDM_CONTROL 0x280 ++ ++static unsigned int render_user_wait_ms = 20000; ++module_param(render_user_wait_ms, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_wait_ms, ++ "Latched per-phase TA/3D bound, 3000..30000 ms (default 20000; TYPED_V3 needs " ++ "20000..30000, absolute job cap 30000)"); ++/* latency101 (default off): host-side TYPED_V3 wait policy; nothing the firmware sees depends on it. */ ++static unsigned int render_user_hard_ms = 60000; ++static unsigned int render_user_soft_ms = 45000; ++static unsigned int render_user_obs_ms = 2000; ++module_param(render_user_hard_ms, uint, 0600); ++module_param(render_user_soft_ms, uint, 0600); ++module_param(render_user_obs_ms, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_hard_ms, ++ "Latched TYPED_V3 hard per-phase bound soft..60000 ms, the job bound is twice " ++ "it; 0 = off, render_user_wait_ms and the 30000 job cap apply (default 0)"); ++MODULE_PARM_DESC( ++ render_user_soft_ms, ++ "Latched TYPED_V3 soft per-phase bound 20000..hard ms, 0 = 45000; needs " ++ "render_user_hard_ms"); ++MODULE_PARM_DESC( ++ render_user_obs_ms, ++ "Latched RAM-only observation window after a TYPED_V3 timeout, 0..5000 ms " ++ "(default 2000; needs render_user_hard_ms)"); ++/* RC4a (default off): RAM-only touch A/B of the TYPED_V3 wait (render-user-touch.h). */ ++ ++/* repeat109: the TYPED_V3 3D completion without the idle words (render-user-retire.h). */ ++ ++/* wake110 (default off): Apple's submit kick at the TYPED_V3 publish (render-user-wake.h). */ ++/* release8 default */ ++ ++/* TEMPORARY (multi-client stage 1): the fallback for a firmware that never drains a DC3 ++ * published after a job retired. Remove once the device has settled that question. ++ */ ++ ++/* Stage 2 fault isolation (render-user-loss.h): how long the scheduler holds every other job ++ * while it waits, RAM only, for a lost job's stuck phase. Read at the loss. ++ */ ++ ++/* Stage 2 in-boot recovery (core recover.inc): a job that stopped the firmware loses only its ++ * context, and the GPU is power-cycled and its firmware booted again while the other contexts' ++ * jobs are held (each leaves the old epoch's firmware pages allocated); 0 = such a job ends the ++ * epoch, as before. How many: render_user_recover_streak. Off by default; the release tuple turns ++ * it on (faultiso F1-F4, 2026-11-02: two recoveries in one boot on hardware). ++ */ ++ ++/* render-user-recover.h: Apple restarts the GPU on every recovery packet. Nonzero: recover every ++ * time, but end the epoch after this many recoveries in a row with no job completing between ++ * them. 0 = render_user_recover counts recoveries per boot, as before (a boot's second fault then ++ * ends the epoch, however long the GPU ran in between). ++ */ ++ ++/* Fault-isolation test (host side only: the firmware runs the job as any other). A context ++ * created while this is N gives up on its N-th job's TA after 1 ms instead of its bound, so the ++ * host loses that job the way it loses a stuck one, while the firmware completes it late: the ++ * context is lost and the channel drained behind it. 0 = off (default). ++ */ ++ ++/* Fault-isolation test with a real GPU fault (faultiso). A context created by a CAP_SYS_RAWIO ++ * process while this is N reserves one page of its VM that is never mapped ++ * (G5P_RU_TEST_FAULT_VA), and its N-th job's TA fetches its control stream from that page ++ * instead of the validated stream: the TA reads an unmapped VA of the context's own VM. What ++ * the firmware does then (a fault event, a silent stream, or a stuck engine) is what the test ++ * records; each ends that job and loses its context. 0 = off (default). ++ */ ++ ++/* render_user_test_fault_job's page: the last GiB below the hardware VA limit, far from every ++ * kernel-private range (low sink, aux, PDS window, firmware window). ++ */ ++#define G5P_RU_TEST_FAULT_VA (G5P_RU_HW_VA_LIMIT - 0x40000000ULL) ++ ++/* Stage 3 (render-user-window.h): which firmware memory a render VM maps. Latched per context ++ * at creation. The kill switch 0 restores the stage 1 mapping and then needs CAP_SYS_RAWIO. ++ */ ++ ++/* The latency101 knobs are valid when all are 0 (off), or hard is in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000 and the dialect is TYPED_V3. The RC4a touch ++ * is 0, or 1..20000 below the effective soft bound with latency101 on. The 3D idle settle is ++ * 0..5000. The wake110 mode is 0, or 1..2 with latency101 on and the touch off (one A/B per ++ * context). Anything else refuses the context. ++ */ ++static bool g5p_ru_lat_params_valid(void) ++{ ++ u32 hard = READ_ONCE(render_user_hard_ms), ++ soft = READ_ONCE(render_user_soft_ms); ++ ++ return g5p_ru_lat_valid(soft, hard, READ_ONCE(render_user_obs_ms)) && ++ g5p_ru_touch_valid((0), g5p_ru_lat_soft(soft, hard), hard) && ++ g5p_ru_rt_settle_valid((G5P_RU_RT_DEF_SETTLE_MS)) && ++ g5p_wk_mode_valid((2)) && (!(2) || (hard && !(0))) && ++ (!hard || (1)); ++} ++/* r25 frame-capable selections, same values as the fixed render_*_variant. */ ++ ++#define G5P_RU_MAX_QUEUED 16 ++#define G5P_RU_MAX_TARGETS 32 ++#define G5P_RU_RESULTS 1024 ++#define G5P_RU_INPUT_MS 10000 ++ ++struct g5p_rt { ++ struct list_head link; ++ struct g5p_context *ctx; ++ struct g5p_ru_rt geometry; ++ struct g5p_ru_rt_sizes sizes; ++ struct drm_mm_node node; ++ struct g5p_dma_page *pages; ++ u32 width, height, page_count; ++ u32 samples; /* 1 or 4 (CREATE_RENDER_TARGET); 0 = 1 */ ++ bool destroyed; ++}; ++ ++/* A pair job 1 bound to a kernel-owned catalogue (render_user_spare_pairs, TEMPORARY). */ ++struct g5p_rspare { ++ u32 slot[2]; /* TA, 3D */ ++ struct g5p_dma_page root; /* the catalogue CBASE points at */ ++ bool ok, used; ++}; ++ ++/* ++ * Device-wide render state (docs/gpu/multi-client-design.md §4.1): the one TA/3D channel pair ++ * and the one PM record with its pool and aux pages, shared by every render context. Jobs of ++ * all contexts run one at a time on render_wq with the core lock held, and only they write ++ * up, v3, sequence, pm_serial and the spares. `lock` guards the pool and the aux pages, which ++ * render-target creation grows while another context's job may run. The scheduler's worker ++ * is the only executor: it picks the jobs of all contexts round-robin. ++ */ ++/* The kernel-built firmware window shared by every render VM (render-user-window.h): built ++ * once from the firmware's tables, never edited, kept until reboot. ++ */ ++struct g5p_rwin { ++ bool built; ++ u32 pages, missing, leaves; ++ struct g5p_dma_page dir, leaf[G5P_RU_FW_PAGES]; ++ u32 leaf_di[G5P_RU_FW_PAGES]; ++}; ++ ++struct g5p_rdev { ++ struct mutex lock; ++ bool up, v3; /* PM and channels built; by the TYPED_V3 attach */ ++ bool dc_pending; /* a bind's DC3 pair was not consumed in time ... */ ++ u32 dc_wait; /* ... and this is the consumer it waits for */ ++ u32 sequence, pm_serial; ++ u32 blocks; ++ /* perf60, latched at the epoch's first TYPED_V3 attach: 2 = jobs alternate two record slots ++ * (render-user-layout.h), else 1; pm_once = the PM pool is built at that attach only. ++ */ ++ u32 job_slots; ++ bool pm_once; ++ /* perf60 stage 4, latched there too: render_user_pm_ctx (0 off, 1 per-context PM records, ++ * 2 also per-context PM IDs) and the boot-reserved sets in use (core lock). ++ */ ++ u32 pm_ctx, pm_set_next; ++ bool pm_set_used[G5P_PMSET_MAX]; ++ /* the context of the last job the executor ran and the previous distinct one, the PM's ++ * write-back targets (render-user-pmset.h; compared, never dereferenced), and the closed ++ * contexts whose unbind waits until they are neither (render_user_close_defer) ++ */ ++ struct g5p_render *pm_last, *pm_prev, *close_deferred[2]; ++ struct g5p_dma_page *physical; ++ struct g5p_dma_page aux[G5P_RU_AUX_PAGES]; ++ struct g5p_rspare *spares; ++ u32 spare_count; ++ /* render_user_close_park: the park catalogue a closed pair is bound to (allocated at the ++ * first park, kept for the device's life; rebuilt at every park) ++ */ ++ struct g5p_dma_page park; ++ u32 parks; ++ bool executing; /* a TYPED_V3 job is in g5p_rv3_execute() (reboot quiesce) */ ++ /* The scheduler (drm-render-sched.inc): its run list under sched_lock, and the one ++ * worker on render_wq that runs every job and close item. ++ */ ++ struct g5p_drm *drm; ++ spinlock_t sched_lock; ++ struct g5p_rs sched; ++ struct delayed_work sched_work; ++ /* boot_gpu_oc_mhz: the thermal stop's poll (system_wq, so a busy render_wq never starves it) */ ++ struct delayed_work oc_work; ++ unsigned long oc_armed; ++ u32 oc_ticks, oc_jobs; ++ struct g5p_rwin ++ window; /* written once, by the executor, core lock held */ ++ struct g5p_rstuck ++ *stuck; /* a lost job blocks the channel (drain), worker only */ ++ /* A lost job stopped the firmware: the worker recovers before any pick. Set by the ++ * executor, cleared by the worker; submissions and the claim pass meanwhile. ++ */ ++ bool recovering; ++ u32 recoveries; ++ u32 recover_streak; /* recoveries since a job completed (core lock) */ ++ struct list_head ++ contexts; /* every render context (lock), for the recovery */ ++}; ++ ++/* render-user-fault.h's ruling window on the device state: a failure noted while a TYPED_V3 job or ++ * a drain runs, or a recovery is pending, waits for the executor. RAM only. ++ */ ++static bool g5p_rdev_ruling(struct g5p_rdev *rd) ++{ ++ bool executing = ++ smp_load_acquire(&rd->executing); /* before `recovering` */ ++ ++ return g5p_ru_ruling(executing, READ_ONCE(rd->recovering)); ++} ++ ++/* A render context. Its firmware state is the slot pair (slot[], bound), owned here the way ++ * an Asahi queue owns its firmware context; the channels and the PM record are the device's. ++ */ ++struct g5p_render { ++ struct g5p_context *ctx; ++ u32 slot[2]; /* engine0 TA, engine1 normal3D */ ++ u32 jobs; /* jobs that started on this context */ ++ u32 dc_target; /* DC ring consumer after this context's binding */ ++ struct drm_mm private_mm; ++ struct drm_mm_node aux_node, low_node, firmware_node, private_node; ++ struct drm_mm_node ++ fault_node; /* render_user_test_fault_job's unmapped page */ ++ struct list_head targets; ++ struct g5p_rt *profile1_rt; ++ u32 target_count; ++ u64 fence_context, fence_seqno; ++ atomic_t queued; ++ /* bound: the slot pair is published (for a RAM-only bind: consumed and checked). ++ * retained: the binding holds a context and device reference until its unbind. ++ * closing: the handle is gone; the close item (g5p_render_close_run()) is queued. ++ * live_maps: a job retired on the binding, so VM_BIND MAP may run beside queued jobs. ++ */ ++ bool claimant, bound, retained, aux_mapped, detached, stage_done, ++ unhooked; ++ bool closing, live_maps; ++ bool lost_drained; /* a lost job of it drained: the firmware holds no use of it */ ++ struct g5p_rs_entity ++ entity; /* its place in the scheduler (drm-render-sched.inc) */ ++ struct list_head rdev_link; /* in rdev->contexts */ ++ /* Mappings unmapped beside queued jobs, freed as those retire (g5p_render_reap()). */ ++ spinlock_t zlock; ++ struct list_head zombies; ++ /* Latched at creation; mirrors whatever qualifies the fixed path. */ ++ u32 stage, cache_variant, mmu_variant, timestamp_variant, wait_ms; ++ u32 soft_ms, hard_ms, obs_ms; /* latency101; hard_ms == 0: off */ ++ u32 touch_ms; /* RC4a; 0: off (always 0 when hard_ms is) */ ++ u32 settle_ms; /* repeat109 minimum coherent run before the 3D event completes */ ++ u32 stall_job; /* render_user_test_stall_job; 0: off */ ++ u32 fault_job; /* render_user_test_fault_job; 0: off */ ++ u32 wake, wake_seed; /* wake110 (render-user-wake.h); wake 0: off */ ++ u32 wake_rang; /* wake110: 0xb00 writes of this context */ ++ u32 pm_policy, vdm_control; ++ u32 fw_window; /* render_user_fw_window, G5P_RU_FW_* */ ++ bool after_ta, cache_repeat; ++ struct g5p_r25_variants v; /* pm, memory, rt<=1, slots; others 0 */ ++ u32 target[2], status[2], selector[2]; ++ bool submitted[2], consumed[2], retired[2]; ++ /* TYPED_V3 (drm-render-v3.inc): serial jobs, the context is retained from its binding on. */ ++ bool v3_accepted; ++ /* perf60 render_user_pm_switch 2: this context's own PM state (S + aux) after its last job */ ++ u8 *pm_save; ++ ++ /* perf60 stage 4 (render_user_pm_ctx): its own PM set (-1: the device PM), pool pages, ++ * aux pages mapped at G5P_RU_PRIVATE_VA in its VM, and the epoch they were built for. ++ */ ++ int pm_set; ++ u32 pm_blocks, pm_epoch; ++ struct g5p_dma_page *pm_pool; ++ struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; ++ bool pm_aux_mapped; ++ bool pm_first; /* it brought the epoch up (attach): the device aux is its own */ ++ bool close_defer_run; /* render_user_close_defer: its deferred close is running */ ++}; ++ ++/* The lost job the channel waits behind (render-user-loss.h): its context (a reference held), ++ * its per-job state as the executor left it, and the drain bound read at the loss. ++ */ ++struct g5p_rstuck { ++ struct g5p_render *r; ++ struct g5p_rv3 v; ++ u32 seq, engine, drain_ms; ++ u64 lost_ns; ++}; ++ ++/* A listed BO that needs implicit fences and/or kernel cache maintenance. */ ++struct g5p_robj { ++ struct drm_gem_object *obj; /* reference held by the job */ ++ u32 access; ++ /* ACCESS_CACHE window (render_user_sync_range bit 1: the resource's range; else whole BO) */ ++ u64 lo, hi; ++}; ++ ++struct g5p_rjob { ++ struct g5p_rs_job node; /* in its context's FIFO until picked */ ++ struct g5p_drm *gpu; ++ struct g5p_context *ctx; ++ struct g5p_fence *fence; ++ struct dma_fence *inputs[G5P_MAX_SYNCS - 1]; ++ u32 input_count; ++ struct g5p_robj *objs; ++ u32 obj_count; ++ struct dma_fence **deps; /* implicit dependencies from dma_resv */ ++ u32 dep_count; ++ int setup_error; /* post-acceptance setup failure: fail without hardware */ ++ /* Inputs then deps, in order: dep_next is the first not yet seen signalled, cb_fence the ++ * one whose callback is armed (scheduler lock), dep_error a failed input or the deadline. ++ */ ++ u32 dep_next; ++ struct dma_fence *cb_fence; ++ struct dma_fence_cb cb; ++ bool cb_fired; ++ int dep_error; ++ u64 id; ++ unsigned long deadline; ++ u8 ta_w[G5P_RU_TA_W_BYTES], td_w[G5P_RU_3D_W_BYTES]; ++ bool v3; ++ struct g5p_rv3 rv3; ++ u64 submit_ns; /* latency123: the submit ioctl */ ++}; ++ ++static u8 *g5p_ru_page(struct g5p_boot *b, u32 engine) ++{ ++ return b->shared[(engine ? G5P_R22_3D : G5P_R22_TA) / G5P_GPU_PAGE].cpu; ++} ++ ++static u32 g5p_ru_meta(u32 engine) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + ++ (engine ? G5P_R22_3D : G5P_R22_TA)); ++} ++ ++static u8 *g5p_ru_pm_cpu(struct g5p_boot *b) ++{ ++ return b->shared[G5P_R22_PM / G5P_GPU_PAGE].cpu; ++} ++ ++static u8 *g5p_ru_global(struct g5p_boot *b, u32 engine) ++{ ++ u32 off = b->layout.offset[0x47 + engine]; ++ ++ return b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_ru_free_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ if (p->cpu) ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++} ++ ++/* Probe: the device render state and its scheduler. Its pages are allocated on demand and ++ * kept until reboot. ++ */ ++static void g5p_rsched_work(struct work_struct *work); ++static void g5p_oc_work(struct work_struct *work); ++ ++static int g5p_render_init(struct g5p_drm *d) ++{ ++ d->rdev = drmm_kzalloc(&d->drm, sizeof(*d->rdev), GFP_KERNEL); ++ if (!d->rdev) ++ return -ENOMEM; ++ d->rdev->drm = d; ++ INIT_LIST_HEAD(&d->rdev->contexts); ++ spin_lock_init(&d->rdev->sched_lock); ++ g5p_rs_init(&d->rdev->sched); ++ INIT_DELAYED_WORK(&d->rdev->sched_work, g5p_rsched_work); ++ INIT_DELAYED_WORK(&d->rdev->oc_work, g5p_oc_work); ++ return drmm_mutex_init(&d->drm, &d->rdev->lock); ++} ++ ++/* rdev->lock held: the aux pages (once) and a PM pool of at least `blocks`. The pool never ++ * shrinks; every page added reaches PoC like the bring-up pool, whether or not the device's ++ * bring-up has cleaned the pool yet. The pool, list and F are rebuilt from it before every ++ * job, so growth takes effect at the next job, and the job in flight keeps its pages. ++ */ ++static int g5p_rdev_grow(struct g5p_drm *d, u32 blocks) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_dma_page *physical; ++ u32 have = rd->blocks * 8, i; ++ int ret = 0; ++ ++ for (i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!rd->aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->aux[i]); ++ if (ret || blocks <= rd->blocks) ++ return ret; ++ if (blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ physical = kvcalloc(blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ if (have) ++ memcpy(physical, rd->physical, have * sizeof(*physical)); ++ for (i = have; !ret && i < blocks * 8; i++) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &physical[i]); ++ if (!ret) ++ g5p_table_clean_page(d->core, &physical[i]); ++ } ++ if (ret) { ++ while (i-- > have) ++ g5p_ru_free_page(d->drm.dev, &physical[i]); ++ kvfree(physical); ++ return ret; ++ } ++ kvfree(rd->physical); ++ rd->physical = physical; ++ rd->blocks = blocks; ++ return 0; ++} ++ ++/* rdev->lock held: the pool's page addresses, for the PM builders. */ ++static u64 *g5p_rdev_addresses(struct g5p_rdev *rd) ++{ ++ u64 *physical = ++ kvmalloc_array(rd->blocks * 8, sizeof(*physical), GFP_KERNEL); ++ ++ for (u32 i = 0; physical && i < rd->blocks * 8; i++) ++ physical[i] = rd->physical[i].dma; ++ return physical; ++} ++ ++/* From the moment the firmware may hold a context's slot pair, the context and everything it ++ * owns stay alive whatever its files do, until its close item has unbound the pair ++ * (g5p_render_close_run()); a pair that cannot be unbound keeps them until reboot. ++ */ ++static void g5p_render_retain(struct g5p_render *r) ++{ ++ if (r->retained) ++ return; ++ kref_get(&r->ctx->ref); ++ drm_dev_get(&r->ctx->gpu->drm); ++ r->retained = true; ++} ++ ++/* The RAM words of the binding of `slot` through the DC records at start and start + 1. */ ++static void g5p_render_bind_mem(struct g5p_boot *b, const u32 *slot, u32 start, ++ struct g5p_ru_bind_mem *m) ++{ ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + ++ ((start + i) & 255) * 20 + j * 4; ++ ++ m->rec[i][j] = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ } ++ for (u32 e = 0; e < 2; e++) { ++ m->pa[e] = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ m->host[e] = g5p_boot_shared(b, 0x84) + slot[e] * 4; ++ m->cached[e] = g5p_boot_shared(b, 0x85) + slot[e] * 4; ++ } ++} ++ ++/* ++ * Called with the file lock. Idle: no accepted job is queued or running. A VM_BIND (`live`) ++ * of a context with live_maps may also run beside its queued jobs, the way an Asahi VM_BIND ++ * does not wait for the queue: a compositor imports and releases client buffers while its ++ * previous frame is in flight. Its tag reaches the firmware at the next kick of the slots. A ++ * MAP only adds entries for VAs no accepted job references; an UNMAP clears its entries now but ++ * keeps the object pinned until the queued jobs have retired (g5p_resident_defer()), so a job ++ * that still used the range faults on the hardware instead of reaching freed memory. Not ++ * before a job has retired on the binding: the epoch's first job checks that the tables did not ++ * change since its attach, and a bind in flight computes its tag from them. Render-target ++ * edits still need the context idle. ++ */ ++static int g5p_render_vm_idle(struct g5p_context *c, bool live) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_boot *b = c->gpu->core->boot; ++ /* During a recovery no epoch may receive a tag. */ ++ bool busy = (atomic_read(&r->queued) && ++ !(live && READ_ONCE(r->live_maps))) || ++ READ_ONCE(c->gpu->rdev->recovering); ++ ++ if (c->bad || r->detached || busy) { ++ dev_warn_ratelimited( ++ c->gpu->drm.dev, ++ "G5P VM_BIND busy=render-idle bad=%u detached=%u queued=%d bound=%u\n", ++ c->bad, r->detached, atomic_read(&r->queued), r->bound); ++ return -EBUSY; ++ } ++ if (r->bound && (!b || READ_ONCE(b->failed) || READ_ONCE(b->stopped))) ++ return g5p_rdev_ruling(c->gpu->rdev) ? -EBUSY : -EIO; ++ return 0; ++} ++ ++/* perf60 MMU invalidation arm, default 0 (render-user-mmu.h): bit 0 every edit publication asks ++ * the firmware for a PC-level invalidate, bit 1 a bound context's job after another context's ++ * job retags its slots (PC), bit 2 NC-reads every table line before the tag store, bit 3 retags ++ * on every job. RAM only, like the tag path itself. ++ */ ++ ++/* An NC read of every 64-byte line of the VM's tables (g5p_rv3_fw_touch()'s visibility read). */ ++static void g5p_render_table_touch(struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt; ++ ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(vm->root.cpu + at)); ++ list_for_each_entry(pt, &vm->tables, link) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(pt->page.cpu + at)); ++} ++ ++/* File lock held, context idle (or a MAP with live_maps). After a PTE edit on a bound render ++ * VM: tables to PoC, then the packed tag to both of its slots. RAM only: no register access ++ * is safe after retirement, and the firmware reads these words only at this context's next ++ * kick, which then invalidates the slots' PC level. ++ */ ++static int g5p_render_invalidate(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_diag *g = c->gpu->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_pt *pt; ++ u32 tag, cached[2], bumps = 0, mode = (0); ++ ++ if (!READ_ONCE(r->bound) || r->detached) ++ return 0; ++ if (!b || c->bad || b->failed || b->stopped) ++ return -EIO; ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ if (mode & G5P_RU_MMU_TOUCH) ++ g5p_render_table_touch(vm); ++ tag = g5p_vm_generation(&vm->mutation); ++ for (u32 e = 0; e < 2; e++) ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ while (tag == cached[0] || tag == cached[1]) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ bumps++; ++ } ++ if (bumps) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER generation collision cached=%08x/%08x bumps=%u tag=%08x\n", ++ cached[0], cached[1], bumps, tag); ++ if (mode & G5P_RU_MMU_EDIT) { ++ /* Advance the VM's own PC counter to the PC field the helper picked. */ ++ u32 pc = g5p_ru_mmu_pc_tag(tag, cached[0], cached[1]); ++ ++ if (pc) { ++ vm->mutation.generation[0] += ++ ((pc >> 22) - (tag >> 22)) & 0x3ff; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ } ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, tag); ++ dma_wmb(); ++ if (mode) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick mmu_inval=%x " ++ "cached=%08x/%08x\n", ++ tag, r->slot[0], r->slot[1], mode, cached[0], ++ cached[1]); ++ else ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick\n", ++ tag, r->slot[0], r->slot[1]); ++ return 0; ++} ++ ++/* DC3 consumption precedes this CBASE check and selected-slot invalidation; ++ * the r22 diagnostic sequence with the user catalogue instead of a private PC. ++ */ ++ ++/* CPU-written firmware objects of one job: TA/3D pages, PM control and the ++ * list/free pages. Coherent allocations plus explicit PoC, as r22 does. ++ */ ++static void g5p_render_clean_job(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ for (u32 off = G5P_R22_TA; off <= G5P_R22_PM; off += G5P_GPU_PAGE) ++ g5p_table_clean_page(g, &b->shared[off / G5P_GPU_PAGE]); ++ for (u32 i = 0; i < 2; i++) ++ g5p_table_clean_page(g, &b->render_meta[i]); ++ dsb(sy); ++ dma_wmb(); ++} ++ ++/* RAM only: the DC ring is idle and neither slot of this context has a PA or a tag. */ ++static int g5p_render_vm_ready(struct g5p_boot *b, struct g5p_render *r) ++{ ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u32 start = READ_ONCE(ctl[4]); ++ ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ return g5p_ru_bind_busy(&s) ? -EBUSY : 0; ++} ++ ++/* The epoch's first job, core lock held, device not up: the PM record from the device pool ++ * and both channels, and every page they reach cleaned to PoC. Doing this again with the ++ * device up would reset the channels and the PM state that every context shares. ++ */ ++static int g5p_render_device_prepare(struct g5p_drm *d, ++ const struct g5p_r25_variants *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm(g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, ++ G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks, G5P_RU_PRIVATE_VA); ++ if (!ret) ++ g5p_ru_pm_variant(g5p_ru_pm_cpu(b), b->render_meta[1].cpu, ++ physical, rd->blocks, v->pm); ++ kvfree(physical); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ g5p_table_clean_page(g, &rd->aux[i]); ++ for (u32 i = 0; !ret && i < rd->blocks * 8; i++) ++ g5p_table_clean_page(g, &rd->physical[i]); ++ mutex_unlock(&rd->lock); ++ if (ret) ++ return ret; ++ rd->pm_serial = 0; ++ for (u32 e = 0; e < 2; e++) ++ g5p_ru_channel_init(g5p_ru_page(b, e), g5p_ru_meta(e)); ++ g5p_render_clean_job(g); ++ return 0; ++} ++ ++/* ++ * Core lock held, firmware up: the root entry of the firmware GiB for a VM in mode `fw_window`. ++ * The window is built at its first use from the firmware's leaf entries of the pages it keeps ++ * (render-user-window.h), and then shared, unchanged, by every render VM. RAM only. ++ */ ++static int g5p_render_fw_entry(struct g5p_drm *d, struct g5p_render *r, ++ u32 *entry) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ int ret = 0; ++ ++ *entry = 0; ++ if (r->fw_window == G5P_RU_FW_FULL) ++ *entry = g5p_le32(b->catalogue.cpu + G5P_RU_FIRMWARE_CI * 4); ++ if (r->fw_window != G5P_RU_FW_WINDOW) ++ return 0; ++ if (w->built) { ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ return 0; ++ } ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->dir); ++ /* perf60 stage 4: the per-context PM sets reserved at boot, after the device PM's pages */ ++ for (u32 i = 0; ++ !ret && i < G5P_RU_FW_PAGES + b->pm_sets * G5P_PMSET_PAGES; i++) { ++ u32 j = i - G5P_RU_FW_PAGES; ++ u64 va = i < G5P_RU_FW_PAGES ? ++ g5p_ru_fw_window_va[i] : ++ g5p_pmset_va(j / G5P_PMSET_PAGES, ++ j % G5P_PMSET_PAGES); ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (!(leaf & 1)) { ++ w->missing++; ++ continue; ++ } ++ if (k == w->leaves) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->leaf[k]); ++ if (ret) ++ break; ++ w->leaf_di[k] = g5p_ru_fw_di(va); ++ g5p_put64(w->dir.cpu + w->leaf_di[k] * 8, ++ g5p_directory_entry(w->leaf[k].dma)); ++ w->leaves++; ++ } ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, leaf); ++ w->pages++; ++ } ++ if (ret) { ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_ru_free_page(d->drm.dev, &w->leaf[i]); ++ g5p_ru_free_page(d->drm.dev, &w->dir); ++ memset(w, 0, sizeof(*w)); ++ return ret; ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(g, &w->leaf[i]); ++ g5p_table_clean_page(g, &w->dir); ++ w->built = true; ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ dev_info( ++ g->dev, ++ "RENDER-V3 fw-window pages=%u missing=%u leaves=%u dir=%pad end=v3\n", ++ w->pages, w->missing, w->leaves, &w->dir.dma); ++ return 0; ++} ++ ++/* Core lock held, between jobs: the DC3 pair that points this context's slots at its own page ++ * catalogue. Maps the firmware window (g5p_render_fw_entry()) and borrows the sink catalogue ++ * entry, cleans the tables and this ++ * context's render targets, then writes the PA and tag of both slots and the two records. ++ * RAM only, nothing is written when the ring is busy or a slot is in use; the caller stores ++ * the producer. ++ */ ++static int g5p_render_vm_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 start = READ_ONCE(ctl[4]), tag, entry; ++ struct g5p_ru_bind_mem m; ++ struct g5p_pt *pt; ++ struct g5p_rt *rt; ++ int ret; ++ ++ ret = g5p_render_vm_ready(b, r); ++ if (!ret) ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ dsb(sy); ++ tag = g5p_vm_generation(&vm->mutation); ++ /* A zero tag would read as unbound: the unbind check takes a zero cached tag as proof ++ * that the DC3 handler ran. ++ */ ++ while (!tag && vm->mutation.generations) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_publish(&m, r->slot, vm->root.dma, tag); ++ *start_out = start; ++ *tag_out = tag; ++ return 0; ++} ++ ++/* The epoch's first job, core lock held: build the device, then bind this context's pair. ++ * Any failure leaves every allocation and binding retained. ++ */ ++static int g5p_render_attach_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ int ret = g5p_render_vm_ready(d->core->boot, r); ++ ++ if (!ret) ++ ret = g5p_render_device_prepare(d, &r->v); ++ return ret ?: g5p_render_vm_prepare(d, r, start_out, tag_out); ++} ++ ++/* The legacy profiles' attach: the epoch's first job, with a doorbell and a host MMU pass. */ ++ ++/* Pointer contents, channel producer, global producer and kick, each after a ++ * barrier. The channel must have retired the previous job's endpoint. ++ */ ++ ++/* Same predicates as g5p_r22_observe() for sequence `seq`. */ ++ ++#include "drm-render-v3.inc" ++#include "drm-render-sched.inc" ++#include "drm-render-unbind.inc" ++ ++static void g5p_render_job_free(struct g5p_rjob *job) ++{ ++ for (u32 i = 0; i < job->input_count; i++) ++ dma_fence_put(job->inputs[i]); ++ for (u32 i = 0; i < job->dep_count; i++) ++ dma_fence_put(job->deps[i]); ++ kfree(job->deps); ++ for (u32 i = 0; i < job->obj_count; i++) ++ drm_gem_object_put(job->objs[i].obj); ++ kvfree(job->objs); ++ if (job->fence) ++ dma_fence_put(&job->fence->base); ++ if (job->ctx) ++ g5p_context_put(job->ctx); ++ kfree(job); ++} ++ ++/* CPU cache maintenance on the G5P mapping (for an imported BO, the attachment's sg_table): clean ++ * before the job, invalidate after retirement. Whole BO, or (render_user_sync_range bit 1, latched ++ * at submit) the resource's BO range, the same window both ways. ++ */ ++static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, ++ bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct g5p_robj *o = &job->objs[i]; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(o->obj)->sgt; ++ ++ if (!(o->access & DRM_G5P_RENDER_ACCESS_CACHE) || !sgt) ++ continue; ++ /* A buffer the job only reads (a sampled client dma-buf or wl_shm udmabuf) ++ * was cleaned before the job and the GPU wrote none of it: no invalidate, ++ * which could discard what the CPU side wrote meanwhile into lines of the ++ * pages the buffer shares (a udmabuf covers whole 16 KiB pages). ++ */ ++ if (to_cpu && !(o->access & DRM_G5P_RENDER_ACCESS_WRITE)) ++ continue; ++ if (o->hi && (o->lo || o->hi != o->obj->size)) ++ g5p_sync_sgt_range(d->drm.dev, sgt, o->lo, o->hi, ++ to_cpu); ++ else if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ } ++} ++ ++/* After acceptance, render_submit_lock held, no file lock: one dma_resv at a ++ * time, so no ww ordering is needed. Our own context's earlier fences are ++ * already ordered by the queue. A failure fails the job before hardware. ++ */ ++static int g5p_render_implicit(struct g5p_rjob *job, u64 own_context) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct dma_resv *resv = job->objs[i].obj->resv; ++ bool write = job->objs[i].access & DRM_G5P_RENDER_ACCESS_WRITE; ++ struct dma_fence **fences = NULL, **grown; ++ unsigned int count = 0; ++ int ret; ++ ++ if (!(job->objs[i].access & DRM_G5P_RENDER_ACCESS_IMPLICIT)) ++ continue; ++ dma_resv_lock(resv, NULL); ++ ret = dma_resv_get_fences(resv, ++ write ? DMA_RESV_USAGE_READ : ++ DMA_RESV_USAGE_WRITE, ++ &count, &fences); ++ if (!ret) ++ ret = dma_resv_reserve_fences(resv, 1); ++ if (!ret) ++ dma_resv_add_fence(resv, &job->fence->base, ++ write ? DMA_RESV_USAGE_WRITE : ++ DMA_RESV_USAGE_READ); ++ dma_resv_unlock(resv); ++ grown = ret || !count ? ++ NULL : ++ krealloc_array(job->deps, ++ job->dep_count + count, ++ sizeof(*grown), GFP_KERNEL); ++ if (!ret && count && !grown) ++ ret = -ENOMEM; ++ if (grown) ++ job->deps = grown; ++ for (u32 j = 0; j < count; j++) { ++ if (!ret && fences[j]->context != own_context) ++ job->deps[job->dep_count++] = fences[j]; ++ else ++ dma_fence_put(fences[j]); ++ } ++ kfree(fences); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++/* Free the mappings unmapped beside jobs up to fence seqno `seqno` (all: every one). */ ++static void g5p_render_reap(struct g5p_render *r, u64 seqno, bool all) ++{ ++ struct g5p_mapping *m, *t; ++ unsigned long flags; ++ LIST_HEAD(done); ++ ++ spin_lock_irqsave(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &r->zombies, link) ++ if (all || m->retire <= seqno) ++ list_move_tail(&m->link, &done); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &done, link) { ++ list_del(&m->link); ++ g5p_vm_obj_put(r->ctx->vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++} ++ ++/* One job, picked by the scheduler once its inputs signalled (or failed, or timed out). */ ++static void g5p_render_run(struct g5p_rjob *job) ++{ ++ struct g5p_drm *d = job->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_render *r = job->ctx->render; ++ unsigned long lock_deadline, flags; ++ bool hardware = false; ++ int ret; ++ ++ /* Admission closes only for good (reboot quiesce, FINAL teardown): a job queued before ++ * that fails here, without waiting for its inputs and without competing for the command ++ * lock. ++ */ ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) ++ ret = -ESHUTDOWN; ++ else ++ ret = job->setup_error ?: job->dep_error; ++ if (!ret && (job->ctx->bad || READ_ONCE(g->boot->failed))) ++ ret = -EIO; ++ if (!ret) { ++ /* Bounded: never queue forever behind a command stuck in MMIO. */ ++ lock_deadline = jiffies + 2 * HZ; ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, lock_deadline)) { ++ ret = -EBUSY; ++ break; ++ } ++ msleep(5); ++ } ++ } ++ if (!ret) { ++ if (g->boot->failed || g->boot->stopped) { ++ ret = -EIO; ++ } else if (!1 && !g5p_ru_mmio_ok(&g5p_rv3_term)) { ++ /* A V3 epoch is terminal: the legacy path would read and write registers. */ ++ ret = -EIO; ++ } else { ++ /* RAM-only from the DC3 attach on: no snapshot (it reads R0), no ++ * export, no register access of any kind after a timeout. ++ */ ++ g5p_render_cache(d, job, false); ++ WRITE_ONCE(d->rdev->executing, true); ++ ret = g5p_rv3_execute(d, r, job); ++ /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ ++ smp_store_release(&d->rdev->executing, false); ++ hardware = job->rv3.touched || job->rv3.bind_failed; ++ if (!ret || ret == -ECANCELED) ++ g5p_render_cache( ++ d, job, ++ true); /* before status and fence */ ++ if (!ret && r->bound) ++ WRITE_ONCE(r->live_maps, true); ++ } ++ g->last_result = ret; ++ mutex_unlock(&g->lock); ++ } ++ spin_lock_irqsave(&d->status_lock, flags); ++ /* A failed device fails every later job; one context's lost job or failed bind does not. */ ++ if (READ_ONCE(g->boot->failed) || READ_ONCE(g->boot->event_failed)) ++ ret = ret ?: -EIO; ++ job->fence->state = DRM_G5P_JOB_COMPLETE; ++ job->fence->error = ret < 0 ? ret : 0; ++ d->last_job = job->id; ++ d->last_error = job->fence->error; ++ d->job_error = job->fence->error; ++ d->job_state = DRM_G5P_JOB_COMPLETE; ++ if (ret < 0 && ret != -ECANCELED && hardware) { ++ /* No reset protocol: quarantine the context and the device. A TYPED_V3 ++ * timeout is one exception for the device: nothing is wrong with the ++ * protocol, the firmware may still complete, and the drain must keep ++ * decoding (RAM only) so that a late completion is recorded. A failed ++ * bind is another: the DC ring is not on the path of the contexts ++ * already bound, so only this context is lost. A lost job is the last: ++ * the executor already ruled (the drain, or the recovery that replaces ++ * this epoch), and only its context is lost. ++ */ ++ WRITE_ONCE(job->ctx->bad, true); ++ if (g5p_ru_job_fails_device(1, ret, job->rv3.bind_failed, ++ job->rv3.lost)) ++ WRITE_ONCE(g->boot->failed, true); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (ret < 0) ++ dma_fence_set_error(&job->fence->base, ret); ++ atomic_dec(&r->queued); ++ atomic_dec(&d->r8_queued); ++ dma_fence_signal(&job->fence->base); ++ /* A job that may still run on the hardware (a bad context) keeps what it might use. */ ++ if (!READ_ONCE(job->ctx->bad)) ++ g5p_render_reap(r, job->fence->base.seqno, false); ++ /* a failed job (complete=-N) always prints; a completed one is routine: job_log */ ++ if (ret) ++ dev_info( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ else ++ g5p_job_printk( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ g5p_render_job_free(job); ++ drm_dev_put(&d->drm); ++} ++ ++/* Private PM-aux/RT page with the latched r25 leaf flags (c by default). */ ++static int g5p_render_map_page(struct g5p_drm *d, struct g5p_render *r, u64 va, ++ struct g5p_dma_page *page) ++{ ++ u8 *pte = g5p_vm_pte(r->ctx->vm, d->drm.dev, va, true); ++ ++ if (IS_ERR_OR_NULL(pte)) ++ return pte ? PTR_ERR(pte) : -ENOMEM; ++ if (g5p_le32(pte) || g5p_le32(pte + 4)) ++ return -EBUSY; ++ g5p_vm_entry(&r->ctx->vm->mutation, pte, ++ g5p_leaf_entry(page->dma, g5p_r25_attributes(&r->v, va)), ++ 2); ++ return 0; ++} ++ ++/* clear: unmap first (file lock held, VM still hooked). */ ++static void g5p_rt_release_memory(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rt *rt, bool clear) ++{ ++ if (clear && rt->page_count) ++ g5p_vm_clear(r->ctx->vm, d->drm.dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rt->pages[i]); ++ kvfree(rt->pages); ++ rt->pages = NULL; ++ rt->page_count = 0; ++ if (drm_mm_node_allocated(&rt->node)) ++ drm_mm_remove_node(&rt->node); ++} ++ ++/* File lock held, context idle; RAM only. Main ring then secondary in one ++ * RT_ALIGN node: the first 64x64 target lands exactly on r22's ++ * 101020000/101030000. ++ */ ++static int g5p_rt_create(struct g5p_drm *d, struct g5p_render *r, ++ const struct g5p_ru_rt *geometry, u32 width, ++ u32 height, struct g5p_rt **out) ++{ ++ struct g5p_rt *rt; ++ u64 bytes; ++ int ret; ++ ++ if (r->target_count >= G5P_RU_MAX_TARGETS) ++ return -ENOSPC; ++ rt = kzalloc(sizeof(*rt), GFP_KERNEL); ++ if (!rt) ++ return -ENOMEM; ++ rt->ctx = r->ctx; ++ rt->geometry = *geometry; ++ rt->width = width; ++ rt->height = height; ++ ret = g5p_ru_geometry_check(geometry, width, height, &rt->sizes); ++ if (ret) ++ goto err_free; ++ bytes = rt->sizes.main + rt->sizes.secondary_map; ++ ret = drm_mm_insert_node_in_range( ++ &r->private_mm, &rt->node, bytes, G5P_RU_RT_ALIGN, 0, ++ G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE, ++ DRM_MM_INSERT_LOW); ++ if (ret) ++ goto err_free; ++ rt->pages = ++ kvcalloc(bytes / G5P_GPU_PAGE, sizeof(*rt->pages), GFP_KERNEL); ++ if (!rt->pages) { ++ ret = -ENOMEM; ++ goto err_node; ++ } ++ for (; rt->page_count < bytes / G5P_GPU_PAGE; rt->page_count++) { ++ struct g5p_dma_page *p = &rt->pages[rt->page_count]; ++ ++ ret = g5p_vm_alloc_page(d->drm.dev, p); ++ if (!ret) ++ ret = g5p_render_map_page( ++ d, r, ++ rt->node.start + rt->page_count * G5P_GPU_PAGE, ++ p); ++ if (ret) { ++ g5p_ru_free_page(d->drm.dev, p); ++ goto err_pages; ++ } ++ } ++ if (r->bound) { ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(d->core, &rt->pages[i]); ++ ret = g5p_render_invalidate(r->ctx); ++ if (ret) { ++ /* Hardware may hold the new tag: retain the target. */ ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ return ret; ++ } ++ } ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ *out = rt; ++ g5p_vmlog_add(G5P_VMLOG_RT_NEW, r->ctx->vm, 0, rt->node.start, bytes, ++ rt->width << 16 | rt->height, rt->samples); ++ return 0; ++err_pages: ++ g5p_rt_release_memory(d, r, rt, true); ++ goto err_free; ++err_node: ++ drm_mm_remove_node(&rt->node); ++err_free: ++ kfree(rt); ++ return ret; ++} ++ ++/* Detach every VM-side trace: private PTEs and the reserved VA nodes. File ++ * lock held, or the file is closing (no concurrent VM ioctl can exist). ++ */ ++static void g5p_render_unhook(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct device *dev = c->gpu->drm.dev; ++ struct g5p_rt *rt; ++ ++ if (r->unhooked) ++ return; ++ list_for_each_entry(rt, &r->targets, link) ++ if (rt->page_count) ++ g5p_vm_clear(c->vm, dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ if (r->aux_mapped) ++ g5p_vm_clear(c->vm, dev, G5P_RU_PRIVATE_VA, ++ G5P_RU_AUX_PAGES * G5P_GPU_PAGE); ++ for (u32 i = 0; i < 4; i++) { ++ struct drm_mm_node *n = i == 0 ? &r->low_node : ++ i == 1 ? &r->firmware_node : ++ i == 2 ? &r->private_node : ++ &r->fault_node; ++ ++ if (drm_mm_node_allocated(n)) ++ drm_mm_remove_node(n); ++ } ++ if (c->vm->resident == c) ++ c->vm->resident = NULL; ++ r->unhooked = true; ++} ++ ++/* Last context reference: only reached unbound or after a successful DC3 ++ * unbind, so no firmware or GPU reference to this memory remains. The aux ++ * pages and the PM pool are the device's. ++ */ ++static void g5p_render_release(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_rt *rt, *tmp; ++ unsigned long flags; ++ ++ g5p_render_unhook(c); ++ g5p_render_reap(r, 0, true); ++ if (!list_empty(&r->rdev_link)) { ++ mutex_lock(&d->rdev->lock); ++ list_del_init(&r->rdev_link); ++ mutex_unlock(&d->rdev->lock); ++ } ++ list_for_each_entry_safe(rt, tmp, &r->targets, link) { ++ g5p_rt_release_memory(d, r, rt, false); ++ list_del(&rt->link); ++ kfree(rt); ++ } ++ if (drm_mm_node_allocated(&r->aux_node)) ++ drm_mm_remove_node(&r->aux_node); ++ drm_mm_takedown(&r->private_mm); ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ for (u32 e = 0; e < 2; e++) ++ if (r->slot[e]) ++ __clear_bit(r->slot[e], d->core->user_slots); ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ { ++ u32 aux = 0; ++ ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ aux += !!r->pm_aux[i].cpu; ++ /* release8 accounting: what this context frees, and the device's live GEM total */ ++ dev_info( ++ d->drm.dev, ++ "RENDER-V3 release slots=%u/%u jobs=%u set=%d pool_pages=%u aux_pages=%u " ++ "vm_pinned=%lluK live_bos=%u live_bo_bytes=%lluK end=v3\n", ++ r->slot[0], r->slot[1], r->jobs, r->pm_set, ++ r->pm_pool ? r->pm_blocks * 8 : 0, aux, ++ c->vm ? c->vm->pinned_bytes >> 10 : 0, ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10); ++ } ++ g5p_rv3_pmctx_release(d, r); ++ if (d->rdev->pm_last == r) ++ d->rdev->pm_last = NULL; ++ if (d->rdev->pm_prev == r) ++ d->rdev->pm_prev = NULL; ++ kvfree(r->pm_save); ++ kfree(r); ++ c->render = NULL; ++} ++ ++/* CREATE_CONTEXT type RENDER. Exclusive VM: reserves [64KiB, 1GiB) for the ++ * borrowed VA0 sink, the whole firmware GiB and the private RT/PM window. ++ */ ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_r25_variants v = { ++ .pm = (0), ++ .memory = (0), ++ .rt = (0), ++ .slots = (0), ++ }; ++ unsigned long flags; ++ u32 id, fw_window; ++ int ret; ++ ++ BUILD_BUG_ON(G5P_USER_RESERVED_START & ((1ULL << 30) - 1)); ++ if (a->flags || a->handle) ++ return -EINVAL; ++ ; ++ ; ++ fw_window = (G5P_RU_FW_WINDOW); ++ /* Unprivileged unless the VM would map the whole firmware GiB (the kill switch). */ ++ if (fw_window >= G5P_RU_FW_MODES || ++ (fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO))) ++ return fw_window >= G5P_RU_FW_MODES ? -EINVAL : -EPERM; ++ if ((0) > 2 || ++ ((G5P_RU_DEF_CACHE_VARIANT) != 0 && ++ (G5P_RU_DEF_CACHE_VARIANT) != 3) || ++ (G5P_RU_DEF_MMU_VARIANT) > 1 || (0) > 1 || (0) > 1 || ++ READ_ONCE(render_user_wait_ms) < ((1) ? G5P_RU_V3_WAIT_MS : 3000) || ++ READ_ONCE(render_user_wait_ms) > G5P_RU_V3_CAP_MS || ++ !g5p_ru_lat_params_valid() || !(32) || (32) > G5P_RU_MAX_BLOCKS || ++ ((G5P_RU_DEF_PROFILE1_VDM_CONTROL) & 0xf) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) > G5P_RU_MAX_VDM || ++ !g5p_ru_r25_valid(&v)) ++ return -EINVAL; ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ r = kzalloc(sizeof(*r), GFP_KERNEL); ++ if (!c || !r) { ++ kfree(c); ++ kfree(r); ++ return -ENOMEM; ++ } ++ r->pm_set = -1; ++ c->gpu = d; ++ c->render = r; ++ r->ctx = c; ++ INIT_LIST_HEAD(&r->targets); ++ INIT_LIST_HEAD(&r->rdev_link); ++ g5p_rs_entity_init(&r->entity); ++ spin_lock_init(&r->zlock); ++ INIT_LIST_HEAD(&r->zombies); ++ atomic_set(&r->queued, 0); ++ drm_mm_init(&r->private_mm, G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_SIZE); ++ r->stage = (0); ++ r->cache_variant = (G5P_RU_DEF_CACHE_VARIANT); ++ r->mmu_variant = (G5P_RU_DEF_MMU_VARIANT); ++ r->timestamp_variant = (0); ++ r->wait_ms = READ_ONCE(render_user_wait_ms); ++ r->hard_ms = READ_ONCE(render_user_hard_ms); ++ r->soft_ms = r->hard_ms ? ++ g5p_ru_lat_soft(READ_ONCE(render_user_soft_ms), ++ r->hard_ms) : ++ 0; ++ r->obs_ms = r->hard_ms ? READ_ONCE(render_user_obs_ms) : 0; ++ r->touch_ms = r->hard_ms ? (0) : 0; ++ r->settle_ms = (G5P_RU_RT_DEF_SETTLE_MS); ++ r->stall_job = (0); ++ r->fault_job = capable(CAP_SYS_RAWIO) ? (0) : 0; ++ r->wake = r->hard_ms && !r->touch_ms ? (2) : 0; ++ r->wake = g5p_wk_mode_valid(r->wake) ? r->wake : 0; ++ r->wake_seed = (0); ++ r->after_ta = (0); ++ r->pm_policy = (0); ++ r->cache_repeat = (0); ++ r->vdm_control = (G5P_RU_DEF_PROFILE1_VDM_CONTROL); ++ r->fw_window = fw_window; ++ r->v = v; ++ r->fence_context = dma_fence_context_alloc(1); ++ mutex_lock(&f->lock); ++ c->vm = xa_load(&f->vms, a->vm); ++ if (!c->vm) { ++ ret = -ENOENT; ++ goto err_unlock; ++ } ++ if (c->vm->contexts || c->vm->resident || c->vm->resident_gone || ++ c->vm->frozen || c->vm->serial_context) { ++ ret = -EBUSY; ++ goto err_unlock; ++ } ++ /* Memory mode 4 covers every render leaf, so no BO may be mapped yet. */ ++ if (v.memory == 4 && c->vm->map_count) { ++ ret = -EINVAL; ++ goto err_unlock; ++ } ++ r->low_node.start = G5P_USER_VA_START; ++ r->low_node.size = G5P_RU_USER_VA_MIN - G5P_USER_VA_START; ++ r->private_node.start = G5P_RU_PRIVATE_VA; ++ r->private_node.size = G5P_RU_PRIVATE_SIZE; ++ r->firmware_node.start = G5P_USER_RESERVED_END; ++ r->firmware_node.size = ++ ((G5P_RU_FIRMWARE_CI + 1) << 30) - G5P_USER_RESERVED_END; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->low_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->private_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->firmware_node) || ++ g5p_le32(c->vm->root.cpu + G5P_RU_FIRMWARE_CI * 4) || ++ g5p_le32(c->vm->root.cpu)) { ++ ret = -EINVAL; /* VM already maps a range this context needs */ ++ goto err_unlock; ++ } ++ if (r->fault_job) { ++ r->fault_node.start = G5P_RU_TEST_FAULT_VA; ++ r->fault_node.size = G5P_GPU_PAGE; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->fault_node)) { ++ dev_warn( ++ d->drm.dev, ++ "RENDER-V3 test-fault off: va=%#llx already used in this VM end=v3\n", ++ G5P_RU_TEST_FAULT_VA); ++ r->fault_job = 0; ++ } ++ } ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ if (g5p_ru_slot_pair(d->core->user_slots, v.slots, r->slot)) { ++ __set_bit(r->slot[0], d->core->user_slots); ++ __set_bit(r->slot[1], d->core->user_slots); ++ } else { ++ r->slot[0] = 0; ++ r->slot[1] = 0; ++ } ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ if (!r->slot[0]) { ++ ret = -ENOSPC; ++ goto err_unlock; ++ } ++ /* The device's PM pool (at least this context's latched size) and aux pages, mapped at ++ * the same VA in every render VM, so the PM state each TA reloads is the one the ++ * previous job saved, whichever VM ran it. ++ */ ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, (32)); ++ mutex_unlock(&d->rdev->lock); ++ r->aux_node.start = G5P_RU_PRIVATE_VA; ++ r->aux_node.size = G5P_RU_AUX_PAGES * G5P_GPU_PAGE; ++ if (!ret) ++ ret = drm_mm_reserve_node(&r->private_mm, &r->aux_node); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ ret = g5p_render_map_page(d, r, ++ G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE, ++ &d->rdev->aux[i]); ++ r->aux_mapped = true; ++ } ++ if (ret) ++ goto err_unlock; ++ ret = xa_alloc(&f->contexts, &id, c, XA_LIMIT(1, G5P_MAX_CONTEXTS), ++ GFP_KERNEL); ++ if (ret) ++ goto err_unlock; ++ kref_init(&c->ref); ++ kref_get(&c->vm->ref); ++ mutex_lock(&d->rdev->lock); ++ list_add_tail(&r->rdev_link, &d->rdev->contexts); ++ mutex_unlock(&d->rdev->lock); ++ c->vm->contexts++; ++ c->vm->serial_context = true; ++ c->vm->resident = c; ++ c->vm->zero_leaf_flags = v.memory == 4; ++ a->handle = id; ++ mutex_unlock(&f->lock); ++ return 0; ++err_unlock: ++ if (c->vm) ++ g5p_render_release(c); ++ else ++ kfree(r); ++ mutex_unlock(&f->lock); ++ kfree(c); ++ return ret; ++} ++ ++/* DESTROY_CONTEXT, file lock held. A TYPED_V3 context that submitted: its close item, queued ++ * behind its jobs, unbinds and releases it (g5p_render_close()). Unbound: plain release. A ++ * legacy context that brought the device up, bound and idle: zero PA/tag and consume one DC3 ++ * per slot, as r21 detach does. Failure or a quarantined legacy context retains everything ++ * until reboot. ++ */ ++static int g5p_render_destroy(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl, start, target; ++ unsigned long deadline; ++ int ret; ++ ++ if (r->v3_accepted) { ++ g5p_render_close(c); ++ return 0; ++ } ++ if (atomic_read(&r->queued) || c->bad) ++ return -EBUSY; ++ if (!r->bound) { ++ g5p_render_unhook(c); ++ if (r->claimant) { ++ /* Claimed but never touched hardware: drop the claim's reference. The ++ * device stays claimed for render: other contexts may be queued on it. ++ */ ++ r->claimant = false; ++ d->quarantined_context = NULL; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ return 0; ++ } ++ /* The legacy unbind rings a doorbell: retained once the epoch is terminal. */ ++ if (!r->claimant || !g5p_ru_mmio_ok(&g5p_rv3_term)) ++ return -EBUSY; ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ ret = -EIO; ++ if (b->failed || b->stopped) ++ goto out; ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ start = READ_ONCE(ctl[4]); ++ ret = -EBUSY; ++ if (start > 255 || READ_ONCE(ctl[0]) != start) ++ goto out; ++ for (u32 e = 0; e < 2; e++) { ++ u32 index = (start + e) & 255; ++ ++ g5p_put64(g5p_boot_shared(b, 0x86) + r->slot[e] * 8, 0); ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, 0); ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + index * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE, ++ j == 0 ? 3 : ++ j == 1 ? r->slot[e] : ++ 0); ++ } ++ } ++ target = (start + 2) & 255; ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], target); ++ dma_wmb(); ++ ret = g5p_prepare_kick(g, "RENDER-USER-unbind"); ++ if (ret) ++ goto fail; ++ writel(0x10, g->rgx + 0xb00); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || b->failed) { ++ ret = ret ?: -EIO; ++ goto fail; ++ } ++ dma_rmb(); ++ if (READ_ONCE(ctl[0]) == target) ++ break; ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (READ_ONCE(ctl[0]) != target) { ++ ret = -ETIMEDOUT; ++ goto fail; ++ } ++ r->detached = true; ++ r->bound = false; ++ r->claimant = false; ++ WRITE_ONCE(c->live, false); ++ g5p_render_unhook(c); ++ /* One render attachment per boot: accepted stays held, so neither a new ++ * render owner nor compute can reuse the shared channel pages. ++ */ ++ d->quarantined_context = NULL; ++ g5p_mmu_record( ++ g, ++ "RENDER-USER detached slots=%u/%u; resources freed on release\n", ++ r->slot[0], r->slot[1]); ++ mutex_unlock(&g->lock); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ return 0; ++fail: ++ c->bad = true; ++ b->failed = true; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, ++ const struct drm_g5p_sync *syncs, u32 count, ++ struct drm_syncobj **signal, u64 *point, ++ struct dma_fence_chain **chain) ++{ ++ u32 signal_handle = 0; ++ int ret; ++ ++ if (!count || count > G5P_MAX_SYNCS) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) { ++ for (u32 j = 0; j < i; j++) ++ if (syncs[i].handle == syncs[j].handle && ++ syncs[i].point == syncs[j].point && ++ syncs[i].flags == syncs[j].flags) ++ return -EINVAL; ++ if (syncs[i].flags & ~DRM_G5P_SYNC_SIGNAL) ++ return -EINVAL; ++ if (syncs[i].flags) { ++ if (*signal) ++ return -EINVAL; ++ *signal = drm_syncobj_find(file, syncs[i].handle); ++ if (!*signal) ++ return -ENOENT; ++ *point = syncs[i].point; ++ signal_handle = syncs[i].handle; ++ continue; ++ } ++ if (job->input_count == ARRAY_SIZE(job->inputs)) ++ return -E2BIG; ++ ret = drm_syncobj_find_fence(file, syncs[i].handle, ++ syncs[i].point, 0, ++ &job->inputs[job->input_count]); ++ if (ret) ++ return ret; ++ job->input_count++; ++ } ++ if (!*signal) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) ++ if (!syncs[i].flags && syncs[i].handle == signal_handle && ++ (!*point || !syncs[i].point || syncs[i].point >= *point)) ++ return -EINVAL; ++ if (*point) { ++ struct dma_fence *old = drm_syncobj_fence_get(*signal); ++ ++ ret = old && (!to_dma_fence_chain(old) || ++ old->seqno >= *point) ? ++ -EINVAL : ++ 0; ++ dma_fence_put(old); ++ if (ret) ++ return ret; ++ *chain = dma_fence_chain_alloc(); ++ if (!*chain) ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++/* File lock held. One mapping of the listed BO must contain each range at a ++ * consistent BO offset; access is a subset of the BO's immutable roles. ++ */ ++static int g5p_render_resolve(struct drm_file *file, struct g5p_vm *vm, ++ const struct drm_g5p_render_resource *res, ++ u32 count, struct g5p_ru_range *ranges, ++ struct g5p_robj *objs, u32 *obj_count) ++{ ++ u32 known = DRM_G5P_RENDER_ACCESS_READ | DRM_G5P_RENDER_ACCESS_WRITE | ++ DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ ++ for (u32 i = 0; i < count; i++) { ++ struct drm_gem_object *obj; ++ struct g5p_mapping *m, *found = NULL; ++ u32 roles, access; ++ ++ for (u32 j = 0; j < i; j++) ++ if (res[i].handle == res[j].handle) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, res[i].handle); ++ if (!obj) ++ return -ENOENT; ++ roles = to_g5p_bo(obj)->roles; ++ { ++ access = res[i].kind; ++ if (!(access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE)) || ++ access & ~known || ++ ((access & G5P_RU_ACCESS_READ) && ++ !(roles & DRM_G5P_BO_READ)) || ++ ((access & G5P_RU_ACCESS_WRITE) && ++ !(roles & DRM_G5P_BO_WRITE))) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ /* A foreign buffer always gets cache maintenance and fences. */ ++ if (drm_gem_is_imported(obj)) ++ access |= DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ } ++ list_for_each_entry(m, &vm->maps, link) ++ if (m->obj == obj && res[i].va >= m->node.start && ++ res[i].size && ++ res[i].va - m->node.start < m->node.size && ++ res[i].size <= m->node.size - (res[i].va - ++ m->node.start) && ++ res[i].offset == ++ m->offset + (res[i].va - m->node.start)) ++ found = m; ++ if (!found) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ if (access & (DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE)) { ++ struct g5p_robj *o = &objs[*obj_count]; ++ ++ o->obj = obj; /* the job inherits this reference */ ++ o->access = access; ++ o->lo = 0; ++ o->hi = obj->size; ++ if ((READ_ONCE(render_user_sync_range) & 2) && ++ !g5p_sync_window(res[i].offset, res[i].size, ++ obj->size, &o->lo, &o->hi)) { ++ o->lo = 0; ++ o->hi = obj->size; ++ } ++ (*obj_count)++; ++ } else { ++ drm_gem_object_put(obj); ++ } ++ ranges[i].va = res[i].va; ++ ranges[i].size = res[i].size; ++ ranges[i].access = access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE); ++ /* The BO's EXEC role, for TYPED_V3 only; an imported BO never has it. */ ++ if ((roles & DRM_G5P_BO_EXEC)) ++ ranges[i].access |= G5P_RU_ACCESS_EXEC; ++ } ++ return 0; ++} ++ ++/* LINEAR64_V1 admits only the exact round16 immutable inputs. Writes stay ++ * privileged and caller-serialized, as for the r20 first-store dialect. ++ */ ++ ++/* File lock held. Copy `len` bytes at GPU VA `va` out of the BO the VM maps ++ * there, through the BO's own pages, as g5p_render_profile1_bytes() does. The ++ * mapping is found again by VA and its own (VM-referenced) object is read, so ++ * a handle closed or replaced since g5p_render_resolve() cannot redirect the ++ * read. Only a native BO has shmem pages: an imported one (never EXEC, so ++ * never a TA program) is refused for the stream too. ++ */ ++static int g5p_render_read_va(struct drm_device *drm, struct g5p_vm *vm, u64 va, ++ void *dst, u32 len) ++{ ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); ++ struct g5p_mapping *m, *found = NULL; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ u64 off; ++ int ret; ++ ++ list_for_each_entry(m, &vm->maps, link) ++ if (va >= m->node.start && va - m->node.start < m->node.size && ++ len <= m->node.size - (va - m->node.start)) ++ found = m; ++ if (!found) ++ return -ENOENT; ++ obj = found->obj; ++ off = found->offset + (va - found->node.start); ++ if (drm_gem_is_imported(obj) || off > obj->size || ++ len > obj->size - off) ++ return -EINVAL; ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ ret = IS_ERR(sgt) ? PTR_ERR(sgt) : drm_gem_vmap(obj, &map); ++ if (ret) ++ return ret; ++ dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ if (map.is_iomem) ++ ret = -EINVAL; ++ else ++ memcpy(dst, (const u8 *)map.vaddr + off, len); ++ dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ drm_gem_vunmap(obj, &map); ++ return ret; ++} ++ ++static void g5p_render_pass_from_uapi(struct g5p_ru_pass *p, ++ const struct drm_g5p_render_pass *u) ++{ ++ p->vdm_va = u->vdm_va; ++ p->vdm_size = u->vdm_size; ++ p->vdm_control = u->vdm_control; ++ p->scratch_va = u->scratch_va; ++ p->scratch_size = u->scratch_size; ++ memcpy(p->scratch_ptr, u->scratch_ptr, sizeof(p->scratch_ptr)); ++ p->pool7_va = u->pool7_va; ++ p->pool7_size = u->pool7_size; ++ p->pool8_va = u->pool8_va; ++ p->pool8_size = u->pool8_size; ++ memcpy(p->context_pds, u->context_pds, sizeof(p->context_pds)); ++ memcpy(p->resume_pds, u->resume_pds, sizeof(p->resume_pds)); ++ memcpy(p->ta_pds, u->ta_pds, sizeof(p->ta_pds)); ++ memcpy(p->background, u->background, sizeof(p->background)); ++ memcpy(p->eot, u->eot, sizeof(p->eot)); ++ memcpy(p->eot_resume, u->eot_resume, sizeof(p->eot_resume)); ++ memcpy(p->restore, u->restore, sizeof(p->restore)); ++} ++ ++static void g5p_render_pass2_from_uapi(struct g5p_ru_pass2 *e, ++ const struct drm_g5p_render_pass_ext *u) ++{ ++ *e = (struct g5p_ru_pass2){ ++ .scissor_va = u->scissor_va, ++ .dbias_va = u->dbias_va, ++ .background_texture = u->background_texture, ++ .scissor_count = u->scissor_count, ++ .dbias_count = u->dbias_count, ++ .bgobj_depth = u->bgobj_depth, ++ .bgobj_vals = u->bgobj_vals, ++ .pixel_width = u->pixel_output_width, ++ .zls_mask = u->zls_address_mask, ++ }; ++ memcpy(e->zls, u->zls, sizeof(e->zls)); ++} ++ ++static void g5p_render_pass3_from_uapi(struct g5p_ru_pass3 *v, ++ const struct drm_g5p_render_pass_v3 *u) ++{ ++ *v = (struct g5p_ru_pass3){ ++ .ta_program_va = u->ta_program_va, ++ .ta_program_size = u->ta_program_size, ++ .vdm_end = u->vdm_end, ++ .pixel_width = u->pixel_output_width, ++ .pad = u->pad, ++ }; ++} ++ ++static_assert(offsetof(struct drm_g5p_render_pass, samples) == 296); ++static_assert(offsetof(struct drm_g5p_render_pass, sample_positions) == 304); ++static_assert(offsetof(struct drm_g5p_render_pass, reserved) == 312); ++static_assert(DRM_G5P_CAP_RENDER_MSAA == BIT_ULL(31)); ++static_assert(DRM_G5P_RENDER_MSAA_RESOLVE == G5P_RU_MSAA_RESOLVE); ++ ++/* CAP_RENDER_MSAA: the target's geometry is kernel-derived (it holds the sample ++ * count), and only TYPED_V3 carries the pass fields. ++ */ ++static bool g5p_render_msaa_enabled(void) ++{ ++ return (1) && (1); ++} ++ ++/* The base pass's MSAA fields: zero unless TYPED_V3 with the capability. */ ++static int g5p_render_msaa_from_uapi(struct g5p_ru_msaa *m, ++ const struct drm_g5p_render_pass *u, ++ bool v3) ++{ ++ *m = (struct g5p_ru_msaa){ ++ .samples = u->samples, ++ .flags = u->msaa_flags, ++ .positions = u->sample_positions, ++ }; ++ if ((m->samples || m->flags || m->positions) && ++ (!v3 || !g5p_render_msaa_enabled())) ++ return -EINVAL; ++ return 0; ++} ++ ++/* Render submit lock held. The first accepted job claims the device for render until reboot; ++ * same exclusions as compute/r22. Every later job, of any context, passes while the device is ++ * neither failed nor stopped, without the core lock (a job may hold it for its whole run). The ++ * claimant keeps the reference d->quarantined_context has always held. The IRQ-test term is ++ * the r74 one (render-fixed.inc, render-r72.inc): the r74 boots refuse the IRQ test, so ++ * irq_tested must stay 0 there and must be 1 on every other boot. ++ */ ++static int g5p_render_device_claim(struct g5p_drm *d, struct g5p_context *c) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_render *r = c->render; ++ int ret = -EBUSY; ++ ++ if (READ_ONCE(d->render_claimed)) { ++ /* A recovery swaps the epoch, and a failure noted mid-job may lead to one: the job ++ * waits for the ruling in the scheduler, which fails it if the epoch ends. ++ */ ++ if (g5p_rdev_ruling(d->rdev)) ++ return 0; ++ b = READ_ONCE(g->boot); ++ return !b || READ_ONCE(b->failed) || READ_ONCE(b->stopped) ? ++ -EIO : ++ 0; ++ } ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ if (b->failed || b->stopped || !b->control_complete || b->irq_tested || ++ b->user_compute_owned || d->quarantined_context || ++ !b->render_meta[0].cpu || !b->render_meta[1].cpu || ++ b->layout.used[1] > G5P_R22_TA || ++ /* recovery108: expose the GPU only with a current-generation ALIVE proof. */ ++ g5p_ds_hook_claim(g) || atomic_cmpxchg(&d->accepted, 0, 1)) ++ goto out; ++ b->user_compute_owned = true; /* blocks diagnostic commands, latched */ ++ r->claimant = true; ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->quarantined_context = c; ++ WRITE_ONCE(d->render_claimed, true); ++ ret = 0; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Evict the oldest completed result beyond the per-file bound. */ ++static int g5p_render_result_slot(struct g5p_file *f, struct g5p_drm *d) ++{ ++ unsigned long index = 0, flags; ++ struct g5p_fence *old; ++ bool done; ++ ++ if (f->render_results < G5P_RU_RESULTS) ++ return 0; ++ old = xa_find(&f->results, &index, ULONG_MAX, XA_PRESENT); ++ if (!old) ++ return 0; ++ spin_lock_irqsave(&d->status_lock, flags); ++ done = old->state == DRM_G5P_JOB_COMPLETE; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!done) ++ return -EBUSY; ++ xa_erase(&f->results, index); ++ dma_fence_put(&old->base); ++ f->render_results--; ++ return 0; ++} ++ ++static int g5p_render_submit_admitted(struct drm_device *drm, ++ struct drm_file *file, u32 context, ++ u32 profile, u32 rt_handle, u64 res_ptr, ++ u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, ++ u32 pass_size, u64 *job_id) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_render_pass_v3 *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_ru_pass pass; ++ struct g5p_ru_pass2 pass2 = {}; ++ struct g5p_ru_pass3 pass3 = {}; ++ struct g5p_ru_v3_loc loc = {}; ++ struct g5p_ru_msaa msaa = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ struct g5p_rt *rt = NULL; ++ const bool v2 = false; ++ const bool v3 = true; ++ u64 point = 0; ++ int ret; ++ ++ ; ++ ; ++ /* TYPED_V3 is unprivileged (checked per context below); the legacy profiles touch ++ * registers and stay with CAP_SYS_RAWIO. ++ */ ++ ; ++ ; ++ /* During a recovery the epoch is being replaced: accept, the scheduler holds the job. */ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ /* a failure noted mid-job waits for the executor's ruling (g5p_rdev_ruling()) */ ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(res_ptr), res_count, ++ sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(sync_ptr), sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ { ++ /* pass_size is the exact V1, V2 or V3 size; ext exists for V2 and V3, ++ * the program fields for V3 only. ++ */ ++ upass = memdup_user(u64_to_user_ptr(pass_ptr), pass_size); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (upass->base.size != pass_size || upass->base.flags || ++ memchr_inv(upass->base.reserved, 0, ++ sizeof(upass->base.reserved)) || ++ (true && memchr_inv(upass->ext.reserved, 0, ++ sizeof(upass->ext.reserved))) || ++ (v3 && ++ memchr_inv(upass->reserved, 0, sizeof(upass->reserved)))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_render_pass_from_uapi(&pass, &upass->base); ++ ret = g5p_render_msaa_from_uapi(&msaa, &upass->base, v3); ++ if (ret) ++ goto out; ++ { ++ g5p_render_pass2_from_uapi(&pass2, &upass->ext); ++ }; ++ { ++ g5p_render_pass3_from_uapi(&pass3, upass); ++ }; ++ } ++ ranges = kvcalloc(res_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(res_count, sizeof(*job->objs), GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ /* Acceptance, implicit fence attach and queueing happen in one order. */ ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ { ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, res_count, ranges, job->objs, ++ &job->obj_count); ++ if (ret) ++ goto unlock; ++ { ++ rt = xa_load(&f->targets, rt_handle); ++ if (!rt || rt->ctx != c || rt->destroyed) { ++ ret = -ENOENT; ++ goto unlock; ++ } ++ } ++ { ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ u8 head[G5P_RU_PROGRAM_MIN], tail[4]; ++ ++ /* Structure first; the program head and the stream's last dword are ++ * read only once every range is proven backed, READ and (program) EXEC. ++ */ ++ ret = g5p_ru_pass3_check(&pass, &pass2, &pass3, &lim, ranges, ++ res_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass3.ta_program_va, head, ++ sizeof(head)); ++ if (!ret) ++ ret = g5p_render_read_va( ++ drm, c->vm, pass.vdm_va + pass3.vdm_end - 4, ++ tail, sizeof(tail)); ++ if (!ret) ++ ret = g5p_ru_pass3_content(head, tail); ++ } ++ /* Every profile: a one-sample pass needs a one-sample target, and back. */ ++ if (!ret) ++ ret = g5p_ru_msaa_check(&msaa, rt->samples ? rt->samples : 1); ++ if (ret) ++ goto unlock; ++ g5p_ru_fill_ta(job->ta_w, &pass); ++ { ++ g5p_ru_fill_ta_v3(job->ta_w, loc.program_off); ++ }; ++ g5p_ru_fill_3d(job->td_w, &pass); ++ /* TYPED_V3 with an all-zero ext is the V1 dialect: the V1 3D words, plus ++ * the output width words (width 2 rewrites the frozen dialect-1 values). ++ */ ++ if (v2 || loc.dialect2) ++ g5p_ru_fill_3d_v2(job->td_w, &pass2); ++ else { ++ g5p_ru_fill_3d_width(job->td_w, loc.pixel_width); ++ }; ++ /* r25 RT variant 1 moves only TA.W28/3D.W20 one slice into the ring. */ ++ g5p_ru_rt_fill(job->ta_w, job->td_w, &rt->geometry, ++ rt->node.start + (r->v.rt == 1 ? rt->sizes.slice : 0), ++ rt->node.start + rt->sizes.main); ++ { ++ g5p_ru_fill_msaa(job->ta_w, job->td_w, &msaa, &rt->geometry); ++ }; ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ /* Unpublished, never signalled: still a clean rejection. */ ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = v3; ++ { ++ r->v3_accepted = true; ++ }; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); /* only ever an uninitialized fence here */ ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++/* recovery108: a submitter is inside the admission window for its whole call; once a ++ * FINAL teardown closed admission it is refused before it looks at anything. ++ */ ++static int g5p_render_submit(struct drm_device *drm, struct drm_file *file, ++ u32 context, u32 profile, u32 rt_handle, ++ u64 res_ptr, u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, u32 pass_size, ++ u64 *job_id) ++{ ++ int ret; ++ ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_render_submit_admitted(drm, file, context, profile, rt_handle, ++ res_ptr, res_count, sync_ptr, ++ sync_count, pass_ptr, pass_size, ++ job_id); ++ g5p_r8_exit(); ++ return ret; ++} ++ ++/* The layout the profile dispatch (pass_size selects the profile) and ++ * GET_RENDER_INFO rely on; a header edit that moves any of it fails here. ++ */ ++static_assert(sizeof(struct drm_g5p_render_pass) == 360); ++static_assert(offsetof(struct drm_g5p_render_pass, vdm_control) == 20); ++static_assert(sizeof(struct drm_g5p_render_pass_ext) == 128); ++static_assert(sizeof(struct drm_g5p_render_pass_v2) == 488); ++static_assert(sizeof(struct drm_g5p_render_pass_v3) == 536); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ext) == ++ sizeof(struct drm_g5p_render_pass)); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_va) == 488); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_size) == 496); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, vdm_end) == 500); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, pixel_output_width) == ++ 504); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, reserved) == 512); ++static_assert(sizeof(struct drm_g5p_submit_render_typed) == 80); ++static_assert(sizeof(struct drm_g5p_get_render_info) == 120); ++static_assert(offsetof(struct drm_g5p_get_render_info, hw_va_limit) == 104); ++static_assert(offsetof(struct drm_g5p_get_render_info, reserved1) == 112); ++static_assert(DRM_G5P_CAP_TYPED_RENDER_V3 == BIT_ULL(26)); ++static_assert(DRM_G5P_CAP_RENDER_MULTI_CONTEXT == BIT_ULL(27)); ++static_assert(DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH == BIT_ULL(29)); ++static_assert(DRM_G5P_RENDER_PROFILE_TYPED_V3 == 4); ++ ++static int g5p_submit_render_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_render_typed *a = data; ++ u64 id = 0; ++ int ret; ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || a->profile != DRM_G5P_RENDER_PROFILE_TYPED_V3 || ++ !a->render_target || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_render_pass_v3)) ++ return -EINVAL; ++ ret = g5p_render_submit(drm, file, a->context, ++ DRM_G5P_RENDER_PROFILE_TYPED_V3, ++ a->render_target, a->resources, ++ a->resource_count, a->syncs, a->sync_count, ++ a->pass, a->pass_size, &id); ++ a->job_id = id; ++ return ret; ++} ++ ++static void g5p_rt_geometry_to_uapi(struct drm_g5p_rt_geometry *u, ++ const struct g5p_ru_rt *k) ++{ ++ u->header_bytes = k->header; ++ u->region_bytes = k->region; ++ u->packed_state = k->packed; ++ u->state_word = k->state; ++ u->padded_dims = k->padded; ++ memcpy(u->scale, k->scale, sizeof(u->scale)); ++ memcpy(u->ta_state, k->ta_state, sizeof(u->ta_state)); ++ memcpy(u->ta_tiling, k->ta_tiling, sizeof(u->ta_tiling)); ++ u->ta_tiling_word = k->ta_tiling_word; ++ u->tile_count = k->tiles; ++ u->secondary_bytes = k->secondary; ++ u->secondary_align = k->secondary_align; ++} ++ ++static int g5p_create_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ const struct drm_g5p_rt_geometry *u = &a->geometry; ++ struct g5p_ru_rt geometry; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ bool user = a->flags & DRM_G5P_RT_USER_GEOMETRY; ++ bool kernel = (1); ++ u64 positions = a->samples == 4 ? G5P_RU_POSITIONS_4X : ++ G5P_RU_POSITIONS_1X; ++ u32 id, blocks = 0; ++ int ret; ++ ++ ; ++ ; ++ if (a->flags & ~DRM_G5P_RT_USER_GEOMETRY || a->handle || a->pad || ++ a->main_va || a->main_bytes || a->secondary_va || ++ a->secondary_bytes || ++ (a->samples != 1 && ++ (a->samples != 4 || !g5p_render_msaa_enabled())) || ++ a->layers != 1 || a->pm_blocks || a->pad2 || a->reserved || ++ u->reserved[0] || u->reserved[1]) ++ return -EINVAL; ++ if (user) { ++ { ++ return -EOPNOTSUPP; ++ } ++ } else if (kernel) { ++ /* 6ca6d5c in integers; equal to the leaf for every size. */ ++ if (memchr_inv(u, 0, sizeof(*u)) || ++ g5p_ru_rt_derive_samples(a->width, a->height, a->samples, ++ positions, &geometry)) ++ return -EINVAL; ++ } else { ++ /* Kernel table: only the offline-executed 6ca6d5c result. */ ++ if (a->width != 64 || a->height != 64 || ++ memchr_inv(u, 0, sizeof(*u))) ++ return -EOPNOTSUPP; ++ geometry = g5p_ru_rt64; ++ } ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto out; ++ } ++ r = c->render; ++ ret = g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ /* No core lock: another context's job may hold it for its whole run, and nothing here ++ * touches the hardware. The device pool only grows; it serves every target of every ++ * context. ++ */ ++ if (kernel) { ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, g5p_ru_pm_blocks_for(geometry.tiles, ++ d->rdev->blocks)); ++ blocks = d->rdev->blocks; ++ mutex_unlock(&d->rdev->lock); ++ } ++ if (!ret) ++ ret = g5p_rt_create(d, r, &geometry, a->width, a->height, &rt); ++ if (ret) ++ goto out; ++ rt->samples = a->samples; ++ ret = xa_alloc(&f->targets, &id, rt, ++ XA_LIMIT(1, G5P_RU_MAX_TARGETS * 8), GFP_KERNEL); ++ if (ret) { ++ /* Mapped but unpublished to a job: leave it owned by the context. */ ++ rt->destroyed = true; ++ goto out; ++ } ++ kref_get(&c->ref); ++ a->handle = id; ++ a->main_va = rt->node.start; ++ a->main_bytes = rt->sizes.main; ++ a->secondary_va = rt->node.start + rt->sizes.main; ++ a->secondary_bytes = rt->sizes.secondary_map; ++ if (!user) ++ g5p_rt_geometry_to_uapi(&a->geometry, &geometry); ++ if (kernel) ++ a->pm_blocks = blocks; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_destroy_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c = NULL; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ int ret; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ rt = xa_load(&f->targets, a->handle); ++ if (!rt) { ++ ret = -ENOENT; ++ goto out; ++ } ++ c = rt->ctx; ++ r = c->render; ++ ret = r->detached ? 0 : g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ g5p_vmlog_add(G5P_VMLOG_RT_DEL, c->vm, 0, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map, ++ rt->width << 16 | rt->height, rt->samples); ++ /* RAM only, no core lock: see g5p_create_render_target(). */ ++ if (r->bound) { ++ g5p_vm_clear(c->vm, drm->dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ ret = g5p_render_invalidate(c); ++ if (ret) ++ goto out; ++ } ++ if (!r->detached) { ++ g5p_rt_release_memory(d, r, rt, true); ++ list_del(&rt->link); ++ r->target_count--; ++ if (r->profile1_rt == rt) ++ r->profile1_rt = NULL; ++ kfree(rt); ++ } else { ++ rt->destroyed = true; ++ } ++ xa_erase(&f->targets, a->handle); ++out: ++ mutex_unlock(&f->lock); ++ if (!ret && c) ++ g5p_context_put(c); ++ return ret; ++} ++ ++/* True when a context created now would latch a hypothesis knob that differs ++ * from its default: GET_RENDER_INFO has no context to report on. ++ */ ++static bool g5p_render_variant_active(void) ++{ ++ return (0) || (G5P_RU_DEF_CACHE_VARIANT) != G5P_RU_DEF_CACHE_VARIANT || ++ (G5P_RU_DEF_MMU_VARIANT) != G5P_RU_DEF_MMU_VARIANT || (0) || ++ READ_ONCE(render_user_wait_ms) != G5P_RU_DEF_WAIT_MS || (0) || ++ (0) || (0) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) != ++ G5P_RU_DEF_PROFILE1_VDM_CONTROL || ++ (0) || (0) || (0) || (0); ++} ++ ++static int g5p_get_render_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_render_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || ++ memchr_inv(&a->profiles, 0, ++ sizeof(*a) - offsetof(struct drm_g5p_get_render_info, ++ profiles))) ++ return -EINVAL; ++ ; ++ ; ++ if (g5p_render_variant_active()) ++ a->flags = DRM_G5P_RENDER_INFO_VARIANT_ACTIVE; ++ a->profiles = BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ ; ++ ; ++ { ++ a->profiles |= BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ }; ++ { ++ a->max_render_width = G5P_RU_MAX_DIM; ++ a->max_render_height = G5P_RU_MAX_DIM; ++ a->max_pm_blocks = G5P_RU_MAX_BLOCKS; ++ } ++ a->max_render_targets = G5P_RU_MAX_TARGETS; ++ a->private_va_start = G5P_RU_PRIVATE_VA; ++ a->private_va_end = G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE; ++ a->user_va_min = G5P_RU_USER_VA_MIN; ++ a->usc_base = G5P_RU_PDS_BASE; ++ a->pds_base = a->usc_base; ++ a->code_window = G5P_RU_CODE_WINDOW; ++ a->hw_va_limit = G5P_RU_HW_VA_LIMIT; ++ a->pm_blocks = (32); ++ a->pm_block_bytes = G5P_RU_BLOCK_BYTES; ++ a->max_queued_jobs = G5P_RU_MAX_QUEUED; ++ a->max_vdm_bytes = DRM_G5P_RENDER_MAX_VDM_BYTES; ++ /* TYPED_V3: what a waiter must outlast is the job's absolute bound (TA publish to the end of the ++ * 3D, render-user-wait.h), not the per-phase module value: a TA that takes its whole phase leaves the ++ * 3D the rest of the job cap. ++ */ ++ { ++ if (READ_ONCE(render_user_hard_ms)) ++ a->job_timeout_ms = g5p_ru_lat_job_ms( ++ READ_ONCE(render_user_hard_ms), ++ READ_ONCE(render_user_obs_ms)); ++ else ++ a->job_timeout_ms = G5P_RU_V3_CAP_MS; ++ }; ++ a->retained_results = G5P_RU_RESULTS; ++ return 0; ++} ++ ++/* VM_BIND hooks for the context resident on a VM (serial compute or render). */ ++static int g5p_resident_idle(struct g5p_context *c, bool live) ++{ ++ return g5p_render_vm_idle(c, live); ++} ++ ++/* VM_BIND UNMAP, file lock held, PTEs already cleared and the tag published: a render ++ * context with queued jobs keeps the mapping's object (its pages and its pin) until those ++ * jobs have retired (g5p_render_reap()). False: free it now. ++ */ ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m) ++{ ++ struct g5p_render *r = c->render; ++ unsigned long flags; ++ ++ if (!r || !atomic_read(&r->queued)) ++ return false; ++ m->retire = r->fence_seqno; ++ spin_lock_irqsave(&r->zlock, flags); ++ list_add_tail(&m->link, &r->zombies); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ return true; ++} ++ ++static int g5p_resident_invalidate(struct g5p_context *c) ++{ ++ return g5p_render_invalidate(c); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-submit.inc b/drivers/gpu/drm/apple-g5p/drm-submit.inc +new file mode 100644 +index 0000000000..f36998e4d5 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-submit.inc +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Asynchronous privileged probes; payloads and firmware VM are kernel-owned. */ ++struct g5p_fence { ++ struct dma_fence base; ++ /* Owned by dma_fence; independent of queue/status locks. */ ++ spinlock_t lock; ++ u32 state; ++ int error; ++}; ++ ++static const char *g5p_fence_driver_name(struct dma_fence *f) ++{ ++ return "apple_g5p"; ++} ++ ++static const char *g5p_fence_timeline_name(struct dma_fence *f) ++{ ++ return "g5p-probe"; ++} ++ ++/* Default release uses dma_fence_free()/RCU; base is the first member. */ ++static const struct dma_fence_ops g5p_fence_ops = { ++ .get_driver_name = g5p_fence_driver_name, ++ .get_timeline_name = g5p_fence_timeline_name, ++}; ++ ++/* recovery108: the compute/probe submitters share the render path's admission window. */ ++ ++/* Legacy JOB_COMPUTE remains unsupported: v1 lacks a bounded stream length. */ +diff --git a/drivers/gpu/drm/apple-g5p/drm-vm.inc b/drivers/gpu/drm/apple-g5p/drm-vm.inc +new file mode 100644 +index 0000000000..e72cacecf5 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-vm.inc +@@ -0,0 +1,914 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* 16 KiB VM preparation; accepted user VMs freeze until reboot. */ ++ ++/* release8 memory accounting: every live G5P GEM object (native and imported) with the task ++ * that created it. One line per context release and per file close prints the totals; ++ * debugfs g5p_bos lists each object (size, references, handles, exported/imported, VM pins, ++ * creator) and each open file's handles, so a page held after its client exited is named. ++ */ ++static LIST_HEAD(g5p_bo_live); ++static DEFINE_SPINLOCK(g5p_bo_lock); ++static u32 g5p_bo_count; ++static u64 g5p_bo_bytes; ++ ++static void g5p_gem_free(struct drm_gem_object *obj); ++ ++static const struct drm_gem_object_funcs g5p_gem_funcs = { ++ .free = g5p_gem_free, ++ .print_info = drm_gem_shmem_object_print_info, ++ .pin = drm_gem_shmem_object_pin, ++ .unpin = drm_gem_shmem_object_unpin, ++ .get_sg_table = drm_gem_shmem_object_get_sg_table, ++ .vmap = drm_gem_shmem_object_vmap, ++ .vunmap = drm_gem_shmem_object_vunmap, ++ .mmap = drm_gem_shmem_object_mmap, ++ .export = g5p_gem_prime_export, ++ .vm_ops = &drm_gem_shmem_vm_ops, ++}; ++ ++/* perf60 fault address (render_user_fault_read, drm-render-v3.inc): bit 0 reads the MMU fault ++ * banks at a TA/3D fault event, bit 1 also at a silent or timed-out phase; any nonzero value also ++ * records every VM edit (MAP, UNMAP, render-target create/destroy) in a device-wide ring, so the ++ * fault address can be named. The ring holds an opaque VM pointer, never dereferenced. ++ */ ++ ++/* faultrec: the bank read and its decode alone at a TA/3D fault event (no per-job PM lanes, no ++ * PM snapshots, no VM edit log): one licensed read per fault, at most 32 per boot. ++ */ ++ ++/* perf60-fillrate.md §11.3 cut 5, kernel half (render-user-sync.h): cache maintenance by range. */ ++static unsigned int render_user_sync_range = 3; /* release8 default */ ++module_param(render_user_sync_range, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_sync_range, ++ "perf60 default0 (release8: whole-BO maintenance); bit0 BO_SYNC honours " ++ "offset/size; bit1 a job's ACCESS_CACHE clean/invalidate covers each resource's " ++ "BO range [offset, offset+size), page widened, instead of the whole BO (latched " ++ "per job at submit); BO_SYNC read live"); ++ ++/* CPU cache maintenance of [lo, hi) of a BO's DMA mapping, walked over its DMA segments. */ ++static void g5p_sync_sgt_range(struct device *dev, struct sg_table *sgt, u64 lo, ++ u64 hi, bool to_cpu) ++{ ++ struct scatterlist *sg; ++ u64 pos = 0, start, n; ++ unsigned int i; ++ ++ for_each_sgtable_dma_sg(sgt, sg, i) { ++ if (pos >= hi) ++ break; ++ if (g5p_sync_clip(pos, sg_dma_len(sg), lo, hi, &start, &n)) { ++ if (to_cpu) ++ dma_sync_single_for_cpu( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_single_for_device( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ } ++ pos += sg_dma_len(sg); ++ } ++} ++ ++#define G5P_VMLOG_N 1024U ++enum { G5P_VMLOG_MAP = 1, G5P_VMLOG_UNMAP, G5P_VMLOG_RT_NEW, G5P_VMLOG_RT_DEL }; ++static const char *const g5p_vmlog_ops[] = { "?", "map", "unmap", "rt-new", ++ "rt-del" }; ++ ++struct g5p_vmlog_ent { ++ u64 ns, va, size, offset; ++ const void *vm; ++ u32 op, handle, vm_id; ++ pid_t tgid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++static struct { ++ spinlock_t lock; /* head and the entries */ ++ u32 head; ++ struct g5p_vmlog_ent e[G5P_VMLOG_N]; ++} g5p_vmlog = { .lock = __SPIN_LOCK_UNLOCKED(g5p_vmlog.lock) }; ++ ++static void g5p_vmlog_add(u32 op, const void *vm, u32 vm_id, u64 va, u64 size, ++ u32 handle, u64 offset) ++{ ++ { ++ return; ++ } ++} ++ ++/* Latency123: leaf attributes of new user mappings (0 = Apple's CC | SLC bypass). */ ++ ++/* Stage 3: the device-wide pinned-memory budget. */ ++static unsigned int render_pinned_mib = 1024; ++module_param(render_pinned_mib, uint, 0600); ++MODULE_PARM_DESC( ++ render_pinned_mib, ++ "MiB of BO memory all GPU VMs together may map; an object mapped in several VMs " ++ "or files (PRIME) counts once (default 1024)"); ++ ++static u32 g5p_pte_attrs(void) ++{ ++ static const u32 attrs[] = { 0xc, 0x4, 0x8, 0x0 }; ++ ++ return attrs[min_t(u32, (0), ARRAY_SIZE(attrs) - 1)]; ++} ++ ++static struct drm_gem_object *g5p_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct g5p_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); ++ ++ if (!bo) ++ return ERR_PTR(-ENOMEM); ++ bo->shmem.base.funcs = &g5p_gem_funcs; ++ INIT_LIST_HEAD(&bo->live); ++ return &bo->shmem.base; ++} ++ ++/* A fully created object enters the live list (an object the core frees on a failed init ++ * never does). ++ */ ++static void g5p_bo_track(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ bo->pid = current->tgid; ++ strscpy(bo->comm, current->comm, sizeof(bo->comm)); ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_add_tail(&bo->live, &g5p_bo_live); ++ g5p_bo_count++; ++ g5p_bo_bytes += obj->size; ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++} ++ ++static void g5p_gem_free(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ if (!list_empty(&bo->live)) { ++ list_del_init(&bo->live); ++ g5p_bo_count--; ++ g5p_bo_bytes -= obj->size; ++ } ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ drm_gem_shmem_object_free(obj); ++} ++ ++static int g5p_bos_show(struct seq_file *m, void *data) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct drm_device *drm = entry->dev; ++ u32 exported = 0, imported = 0, nohandle = 0; ++ u64 nohandle_bytes = 0; ++ struct drm_file *file; ++ struct g5p_bo *bo; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_for_each_entry(bo, &g5p_bo_live, live) { ++ struct drm_gem_object *obj = &bo->shmem.base; ++ bool imp = drm_gem_is_imported(obj); ++ ++ exported += !!READ_ONCE(obj->dma_buf); ++ imported += imp; ++ if (!READ_ONCE(obj->handle_count)) { ++ nohandle++; ++ nohandle_bytes += obj->size; ++ } ++ seq_printf( ++ m, ++ "bo size=%zuK refs=%u handles=%u exported=%c imported=%c vm_pins=%u by=%s[%d]\n", ++ obj->size >> 10, kref_read(&obj->refcount), ++ READ_ONCE(obj->handle_count), ++ READ_ONCE(obj->dma_buf) ? 'y' : 'n', imp ? 'y' : 'n', ++ bo->vm_pins, bo->comm, bo->pid); ++ } ++ seq_printf( ++ m, ++ "live=%u bytes=%lluK exported=%u imported=%u no_handle=%u/%lluK device_pinned=%lluK\n", ++ g5p_bo_count, g5p_bo_bytes >> 10, exported, imported, nohandle, ++ nohandle_bytes >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ mutex_lock(&drm->filelist_mutex); ++ list_for_each_entry(file, &drm->filelist, lhead) { ++ struct drm_gem_object *obj; ++ struct task_struct *task; ++ u64 bytes = 0; ++ u32 handles = 0; ++ struct pid *pid; ++ int id; ++ ++ spin_lock(&file->table_lock); ++ idr_for_each_entry(&file->object_idr, obj, id) { ++ handles++; ++ bytes += obj->size; ++ } ++ spin_unlock(&file->table_lock); ++ rcu_read_lock(); ++ pid = rcu_dereference(file->pid); ++ task = pid_task(pid, PIDTYPE_TGID); ++ seq_printf(m, "file %s[%d] handles=%u bytes=%lluK\n", ++ task ? task->comm : "?", pid_vnr(pid), handles, ++ bytes >> 10); ++ rcu_read_unlock(); ++ } ++ mutex_unlock(&drm->filelist_mutex); ++ return 0; ++} ++ ++static int g5p_create_bo(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_bo *a = data; ++ struct drm_gem_shmem_object *shmem; ++ int ret; ++ ++ if (!a->size || a->size > G5P_MAX_BO_SIZE || ++ (a->size & (G5P_GPU_PAGE - 1)) || a->handle || !a->flags || ++ (a->flags & ++ ~(DRM_G5P_BO_READ | DRM_G5P_BO_WRITE | DRM_G5P_BO_EXEC))) ++ return -EINVAL; ++ shmem = drm_gem_shmem_create(drm, a->size); ++ if (IS_ERR(shmem)) ++ return PTR_ERR(shmem); ++ /* Match the DMA API's coherent/noncoherent CPU mapping choice. The GPU ++ * cache contract remains an explicit COMPUTE capability prerequisite. ++ */ ++ shmem->map_wc = !dev_is_dma_coherent(drm->dev); ++ to_g5p_bo(&shmem->base)->roles = a->flags; ++ g5p_bo_track(&shmem->base); ++ ret = drm_gem_handle_create(file, &shmem->base, &a->handle); ++ drm_gem_object_put(&shmem->base); ++ return ret; ++} ++ ++static int g5p_bo_mmap_offset(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_mmap_offset *a = data; ++ struct drm_gem_object *obj; ++ int ret; ++ ++ if (a->flags || a->offset) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ ret = drm_gem_create_mmap_offset(obj); ++ if (!ret) ++ a->offset = drm_vma_node_offset_addr(&obj->vma_node); ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_bo_sync(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_sync *a = data; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *ctx; ++ unsigned long id; ++ int ret = 0; ++ ++ if (a->op != DRM_G5P_BO_SYNC_TO_DEVICE && ++ a->op != DRM_G5P_BO_SYNC_TO_CPU) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ mutex_lock(&f->lock); ++ xa_for_each(&f->contexts, id, ctx) ++ if (ctx->serial && ++ (ctx->bad || smp_load_acquire(&ctx->pending))) { ++ mutex_unlock(&f->lock); ++ ret = -EBUSY; ++ goto out; ++ } ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ mutex_unlock(&f->lock); ++ goto out; ++ } ++ /* render_user_sync_range bit 0: the requested range; else the whole BO (stronger, allowed). */ ++ if (READ_ONCE(render_user_sync_range) & 1) { ++ u64 lo, hi; ++ ++ if (g5p_sync_window(a->offset, a->size, obj->size, &lo, &hi)) ++ g5p_sync_sgt_range(drm->dev, sgt, lo, hi, ++ a->op == DRM_G5P_BO_SYNC_TO_CPU); ++ } else if (a->op == DRM_G5P_BO_SYNC_TO_DEVICE) { ++ dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } else { ++ dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ mutex_unlock(&f->lock); ++out: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_alloc_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ p->cpu = dma_alloc_coherent(dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ if (!g5p_user_pa_valid(p->dma) || dma_to_phys(dev, p->dma) != p->dma) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ return 0; ++} ++ ++static struct g5p_pt *g5p_vm_table(struct g5p_vm *vm, struct device *dev, ++ u32 level, u32 key, bool create) ++{ ++ struct g5p_pt *pt; ++ int ret; ++ ++ list_for_each_entry(pt, &vm->tables, link) ++ if (pt->level == level && pt->key == key) ++ return pt; ++ if (!create) ++ return NULL; ++ if (vm->table_count >= G5P_MAX_PT_PAGES) ++ return ERR_PTR(-ENOSPC); ++ pt = kzalloc(sizeof(*pt), GFP_KERNEL); ++ if (!pt) ++ return ERR_PTR(-ENOMEM); ++ ret = g5p_vm_alloc_page(dev, &pt->page); ++ if (ret) { ++ kfree(pt); ++ return ERR_PTR(ret); ++ } ++ pt->level = level; ++ pt->key = key; ++ list_add_tail(&pt->link, &vm->tables); ++ vm->table_count++; ++ return pt; ++} ++ ++static u8 *g5p_vm_pte(struct g5p_vm *vm, struct device *dev, u64 va, ++ bool create) ++{ ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ struct g5p_pt *dir, *leaf; ++ ++ dir = g5p_vm_table(vm, dev, 1, ci, create); ++ if (IS_ERR_OR_NULL(dir)) ++ return (void *)dir; ++ leaf = g5p_vm_table(vm, dev, 2, (ci << 9) | di, create); ++ if (IS_ERR_OR_NULL(leaf)) ++ return (void *)leaf; ++ if (create) { ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + ci * 4, ++ g5p_catalogue_entry(dir->page.dma), 0); ++ g5p_vm_entry(&vm->mutation, dir->page.cpu + di * 8, ++ g5p_directory_entry(leaf->page.dma), 1); ++ } ++ return leaf->page.cpu + ((va >> 14) & 127) * 8; ++} ++ ++static void g5p_vm_clear(struct g5p_vm *vm, struct device *dev, u64 va, ++ u64 size) ++{ ++ for (u64 off = 0; off < size; off += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, dev, va + off, false); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, 0, 2); ++ } ++ dma_wmb(); ++} ++ ++static bool g5p_vm_table_empty(struct g5p_pt *pt) ++{ ++ u32 count = pt->level == 2 ? 128 : 512; ++ ++ for (u32 i = 0; i < count; i++) ++ if (g5p_le32(pt->page.cpu + i * 8) & 1) ++ return false; ++ return true; ++} ++ ++static void g5p_vm_prune(struct device *dev, struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt, *tmp, *dir; ++ ++ /* These tables are mutable only before firmware attachment. ++ * Remove empty leaves before directories, including failed MAP rollback. ++ */ ++ for (u32 level = 2; level; level--) ++ list_for_each_entry_safe(pt, tmp, &vm->tables, link) { ++ if (pt->level != level || !g5p_vm_table_empty(pt)) ++ continue; ++ if (level == 2) { ++ dir = g5p_vm_table(vm, dev, 1, pt->key >> 9, ++ false); ++ if (dir) ++ g5p_vm_entry(&vm->mutation, ++ dir->page.cpu + (pt->key & ++ 511) * 8, ++ 0, 1); ++ } else { ++ g5p_vm_entry(&vm->mutation, ++ vm->root.cpu + pt->key * 4, 0, 0); ++ } ++ dma_wmb(); ++ list_del(&pt->link); ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ vm->table_count--; ++ kfree(pt); ++ } ++} ++ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj); ++ ++static void g5p_vm_release(struct kref *ref) ++{ ++ struct g5p_vm *vm = container_of(ref, struct g5p_vm, ref); ++ struct device *dev = vm->dev; ++ struct g5p_mapping *m, *mtmp; ++ struct g5p_pt *pt, *ptmp; ++ u64 pinned = vm->pinned_bytes; ++ u32 maps = vm->map_count, tables = vm->table_count; ++ ++ list_for_each_entry_safe(m, mtmp, &vm->maps, link) { ++ drm_mm_remove_node(&m->node); ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ list_del(&m->link); ++ kfree(m); ++ } ++ list_for_each_entry_safe(pt, ptmp, &vm->tables, link) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ list_del(&pt->link); ++ kfree(pt); ++ } ++ dma_free_coherent(dev, G5P_GPU_PAGE, vm->root.cpu, vm->root.dma); ++ xa_destroy(&vm->objs); ++ drm_mm_remove_node(&vm->reserved); ++ drm_mm_takedown(&vm->mm); ++ /* One line per freed VM: the device's pinned total must fall back after a client exits. */ ++ dev_info( ++ dev, ++ "G5P VM release maps=%u tables=%u pinned=%lluK device_pinned=%lluK\n", ++ maps, tables, pinned >> 10, ++ READ_ONCE(vm->gpu->pinned_bytes) >> 10); ++ kfree(vm); ++} ++ ++static void g5p_vm_put(struct g5p_vm *vm) ++{ ++ kref_put(&vm->ref, g5p_vm_release); ++} ++ ++static void g5p_context_release(struct kref *ref) ++{ ++ struct g5p_context *c = container_of(ref, struct g5p_context, ref); ++ g5p_render_release(c); ++ g5p_vm_put(c->vm); ++ kfree(c); ++} ++ ++static void g5p_context_put(struct g5p_context *c) ++{ ++ kref_put(&c->ref, g5p_context_release); ++} ++ ++static int g5p_create_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ u32 id; ++ int ret; ++ ++ if (a->flags || a->handle) ++ return -EINVAL; ++ vm = kzalloc(sizeof(*vm), GFP_KERNEL); ++ if (!vm) ++ return -ENOMEM; ++ kref_init(&vm->ref); ++ vm->dev = drm->dev; ++ vm->gpu = to_g5p(drm); ++ vm->bounded = (0); ++ vm->mutation.generations = true; ++ INIT_LIST_HEAD(&vm->maps); ++ xa_init(&vm->objs); ++ INIT_LIST_HEAD(&vm->tables); ++ drm_mm_init(&vm->mm, G5P_USER_VA_START, ++ G5P_USER_VA_END - G5P_USER_VA_START); ++ vm->reserved.start = G5P_USER_RESERVED_START; ++ vm->reserved.size = G5P_USER_RESERVED_END - G5P_USER_RESERVED_START; ++ ret = drm_mm_reserve_node(&vm->mm, &vm->reserved); ++ if (ret) ++ goto err_mm; ++ ret = g5p_vm_alloc_page(drm->dev, &vm->root); ++ if (ret) ++ goto err_reserved; ++ vm->table_count = 1; ++ mutex_lock(&f->lock); ++ ret = xa_alloc(&f->vms, &id, vm, XA_LIMIT(1, G5P_MAX_VMS), GFP_KERNEL); ++ mutex_unlock(&f->lock); ++ if (ret) { ++ g5p_vm_put(vm); ++ return ret; ++ } ++ a->handle = id; ++ return 0; ++err_reserved: ++ drm_mm_remove_node(&vm->reserved); ++err_mm: ++ drm_mm_takedown(&vm->mm); ++ kfree(vm); ++ return ret; ++} ++ ++static int g5p_destroy_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->handle); ++ if (!vm) ++ ret = -ENOENT; ++ else if (vm->contexts || f->submitted || vm->frozen) ++ ret = -EBUSY; ++ else ++ xa_erase(&f->vms, a->handle); ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_vm_put(vm); ++ return ret; ++} ++ ++static unsigned long g5p_vm_obj_count(struct g5p_vm *vm, ++ struct drm_gem_object *obj) ++{ ++ void *v = xa_load(&vm->objs, (unsigned long)obj); ++ ++ return v ? xa_to_value(v) : 0; ++} ++ ++/* ++ * One more mapping of `obj` in `vm`. Its first one in this VM counts its size against the ++ * VM's pin limit, and its first one in any VM against the device budget (render_pinned_mib). ++ * Our own dma-bufs import back as the same GEM object, in any file, so memory shared between ++ * clients (a client's buffer in the compositor's VM) counts once. ++ */ ++static int g5p_vm_obj_get(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ u64 limit = (u64)READ_ONCE(render_pinned_mib) << 20; ++ unsigned long count; ++ int ret = 0; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (!count && obj->size > G5P_MAX_PINNED_BYTES - vm->pinned_bytes) ++ ret = -ENOSPC; ++ else if (!count && !bo->vm_pins && ++ obj->size > limit - min(limit, d->pinned_bytes)) ++ ret = -ENOSPC; ++ if (!ret) ++ ret = xa_err(xa_store(&vm->objs, (unsigned long)obj, ++ xa_mk_value(count + 1), GFP_KERNEL)); ++ if (!ret && !count) { ++ vm->pinned_bytes += obj->size; ++ if (!bo->vm_pins++) ++ d->pinned_bytes += obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++ return ret; ++} ++ ++/* One mapping of `obj` in `vm` less: g5p_vm_obj_get() undone. Also called by the render ++ * scheduler for an unmap deferred behind queued jobs, hence the lock around the counts. ++ */ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long count; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (count > 1) { ++ /* replaces an existing entry: no allocation */ ++ xa_store(&vm->objs, (unsigned long)obj, xa_mk_value(count - 1), ++ GFP_KERNEL); ++ } else if (!WARN_ON(!count)) { ++ xa_erase(&vm->objs, (unsigned long)obj); ++ vm->pinned_bytes -= obj->size; ++ if (!--bo->vm_pins) ++ d->pinned_bytes -= obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++} ++ ++static int g5p_vm_map_bo(struct drm_device *drm, struct g5p_vm *vm, ++ struct drm_file *file, struct drm_g5p_vm_bind *a) ++{ ++ struct drm_gem_object *obj; ++ struct drm_gem_shmem_object *shmem; ++ struct sg_table *sgt; ++ struct sg_dma_page_iter iter; ++ struct g5p_mapping *m; ++ u64 *addresses, done = 0; ++ u32 count = 0; ++ bool imported; ++ int ret; ++ ++ if (vm->map_count >= G5P_MAX_MAPS || ++ a->size > G5P_MAX_VM_BYTES - vm->mapped_bytes) ++ return -ENOSPC; ++ obj = drm_gem_object_lookup(file, a->bo); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto err_obj; ++ } ++ ret = g5p_vm_obj_get(vm, obj); ++ if (ret) ++ goto err_obj; ++ addresses = kvmalloc_array(obj->size / G5P_GPU_PAGE, sizeof(*addresses), ++ GFP_KERNEL); ++ m = kzalloc(sizeof(*m), GFP_KERNEL); ++ if (!addresses || !m) { ++ ret = -ENOMEM; ++ goto err_alloc; ++ } ++ m->node.start = a->va; ++ m->node.size = a->size; ++ ret = drm_mm_reserve_node(&vm->mm, &m->node); ++ if (ret) { ++ ret = -EINVAL; ++ goto err_alloc; ++ } ++ imported = drm_gem_is_imported(obj); ++ shmem = to_drm_gem_shmem_obj(obj); ++ /* PRIME import mapped this attachment for our device and owns it until ++ * object release. Re-locking the exporter's reservation here can return ++ * -EBUSY after the first V3 submission even though the render VM is idle. ++ */ ++ if (imported) { ++ sgt = READ_ONCE(shmem->sgt); ++ if (!sgt) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=prime-no-sgt vm=%u bo=%u bytes=%llu\n", ++ a->vm, a->bo, a->size); ++ ret = -ENXIO; ++ goto err_node; ++ } ++ } else { ++ sgt = drm_gem_shmem_get_pages_sgt(shmem); ++ } ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=sgt-error vm=%u bo=%u imported=%u err=%d\n", ++ a->vm, a->bo, imported, ret); ++ goto err_node; ++ } ++ for_each_sgtable_dma_page(sgt, &iter, 0) { ++ u64 dma = sg_page_iter_dma_address(&iter); ++ ++ if (count >= obj->size / G5P_GPU_PAGE || ++ !g5p_user_pa_valid(dma) || ++ dma_to_phys(drm->dev, dma) != dma || ++ (!imported && dma_to_phys(drm->dev, dma) != ++ page_to_phys(shmem->pages[count]))) { ++ ret = -ERANGE; ++ goto err_node; ++ } ++ addresses[count++] = dma; ++ } ++ if (count != obj->size / G5P_GPU_PAGE) { ++ ret = -EINVAL; ++ goto err_node; ++ } ++ for (; done < a->size; done += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, drm->dev, a->va + done, true); ++ ++ if (IS_ERR_OR_NULL(pte)) { ++ ret = pte ? PTR_ERR(pte) : -ENOMEM; ++ goto err_clear; ++ } ++ /* Recovered bypass bits; user permission encodings are not claimed. ++ * A render VM latched to r25 memory variant 4 uses flags 0. ++ */ ++ g5p_vm_entry( ++ &vm->mutation, pte, ++ g5p_leaf_entry( ++ addresses[(a->offset + done) / G5P_GPU_PAGE], ++ vm->zero_leaf_flags ? 0 : g5p_pte_attrs()), ++ 2); ++ } ++ dma_wmb(); ++ m->obj = obj; ++ m->offset = a->offset; ++ list_add_tail(&m->link, &vm->maps); ++ vm->mapped_bytes += a->size; ++ vm->map_count++; ++ kvfree(addresses); ++ return 0; ++err_clear: ++ g5p_vm_clear(vm, drm->dev, a->va, done); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++err_node: ++ drm_mm_remove_node(&m->node); ++err_alloc: ++ kvfree(addresses); ++ kfree(m); ++ g5p_vm_obj_put(vm, obj); ++err_obj: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_bind(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_vm_bind *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_mapping *m; ++ struct g5p_vm *vm; ++ int ret = -ENOENT; ++ bool resident = false, locked = false; ++ u32 generation[3]; ++ ++ if (a->flags || !g5p_user_range_valid(a->va, a->size) || ++ (a->op != DRM_G5P_VM_MAP && a->op != DRM_G5P_VM_UNMAP) || ++ (a->op == DRM_G5P_VM_UNMAP && (a->bo || a->offset))) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->vm); ++ if (!vm) ++ goto out; ++ /* Its render context is closing: the close item owns the tables now, and the context ++ * may be freed at any time, so it is not looked at. ++ */ ++ if (vm->resident_gone) { ++ ret = -EBUSY; ++ goto out; ++ } ++ if ((f->submitted || vm->frozen) && !vm->resident) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=no-resident submitted=%u frozen=%u vm=%u op=%u\n", ++ f->submitted, vm->frozen, a->vm, a->op); ++ ret = -EBUSY; ++ goto out; ++ } ++ if (vm->resident) { ++ ret = g5p_resident_idle(vm->resident, true); ++ if (ret) ++ goto out; ++ /* A render context's edit is RAM only and touches its own tables and ++ * slots, which the firmware reads only at that context's next kick; ++ * f->lock plus resident-idle orders it against its submit (a MAP or ++ * UNMAP may pass queued jobs, see g5p_render_vm_idle()). Other contexts' jobs ++ * may hold the core lock meanwhile, so it is not taken. Serialized ++ * compute still uses the global mutex. ++ */ ++ if (!vm->resident->render) { ++ mutex_lock(&to_g5p(drm)->core->lock); ++ locked = true; ++ } ++ resident = true; ++ memcpy(generation, vm->mutation.generation, sizeof(generation)); ++ } ++ if (a->op == DRM_G5P_VM_MAP) { ++ if (vm->bounded && !(vm->resident && vm->resident->render) && ++ g5p_user_private_overlap(a->va, a->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_vm_map_bo(drm, vm, file, a); ++ if (!ret) ++ g5p_vmlog_add(G5P_VMLOG_MAP, vm, a->vm, a->va, a->size, ++ a->bo, a->offset); ++ if (ret == -EBUSY) ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=map vm=%u bo=%u submitted=%u frozen=%u resident=%u render=%u\n", ++ a->vm, a->bo, f->submitted, vm->frozen, ++ !!vm->resident, ++ !!(vm->resident && vm->resident->render)); ++ goto out; ++ } ++ list_for_each_entry(m, &vm->maps, link) { ++ if (m->node.start != a->va || m->node.size != a->size) { ++ if (m->node.start < a->va + a->size && ++ a->va < m->node.start + m->node.size) { ++ ret = -EINVAL; ++ break; ++ } ++ continue; ++ } ++ g5p_vmlog_add(G5P_VMLOG_UNMAP, vm, a->vm, a->va, a->size, 0, 0); ++ g5p_vm_clear(vm, drm->dev, a->va, a->size); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++ else { ++ ret = g5p_resident_invalidate(vm->resident); ++ if (ret) ++ goto out; ++ memcpy(generation, vm->mutation.generation, ++ sizeof(generation)); ++ } ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ vm->mapped_bytes -= a->size; ++ /* Beside queued jobs the PTEs are gone, but the pages stay pinned until ++ * those jobs have retired. ++ */ ++ if (!vm->resident || !g5p_resident_defer(vm->resident, m)) { ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ ret = 0; ++ break; ++ } ++out: ++ if (resident) { ++ int invalidate = 0; ++ ++ if (memcmp(generation, vm->mutation.generation, ++ sizeof(generation))) ++ invalidate = g5p_resident_invalidate(vm->resident); ++ ++ if (invalidate) ++ ret = invalidate; ++ if (locked) ++ mutex_unlock(&to_g5p(drm)->core->lock); ++ } ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_create_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_context *a = data; ++ if (a->type != DRM_G5P_CONTEXT_RENDER) ++ return -EOPNOTSUPP; ++ return g5p_render_create_context(drm, file, a); ++} ++ ++static int g5p_destroy_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_context *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->handle); ++ if (!c) ++ ret = -ENOENT; ++ else ++ ret = g5p_render_destroy(c); ++ if (!ret && c) { ++ c->vm->contexts--; ++ xa_erase(&f->contexts, a->handle); ++ } ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_context_put(c); ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/render-user-validate.h b/drivers/gpu/drm/apple-g5p/render-user-validate.h +new file mode 100644 +index 0000000000..19325a2c31 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/render-user-validate.h +@@ -0,0 +1,374 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_VALIDATE_H ++#define G5P_RENDER_USER_VALIDATE_H ++ ++/* Pure TYPED_V1 checks over already-resolved resource ranges. The caller ++ * proves each range is inside a mapping of the listed BO, in the submitting ++ * context's VM, with BO roles covering `access`. Nothing here is a pointer ++ * the kernel dereferences; GPU addresses are only range-checked. ++ */ ++#define G5P_RU_ACCESS_READ 1U ++#define G5P_RU_ACCESS_WRITE 2U ++#define G5P_RU_MAX_VDM 0x10000U ++#define G5P_RU_MAX_SCRATCH (64ULL << 20) ++ ++/* Hardware facts GET_RENDER_INFO reports and the TYPED_V3 validator enforces; ++ * one definition so the two cannot drift. Code offsets are 32-bit from the ++ * common USC/PDS base. TA/3D reads at or above 2^38 stall silently on A10 ++ * (works at 0x3_, 0x10_ and 0x20_, stalls at 0x40_ and 0x80_, no MMU fault); ++ * not flown on A9, so the limit is a statement, not a proof (see ++ * docs/gpu/experiments/typed-v3-kernel-implementation.md). ++ */ ++#define G5P_RU_PDS_BASE 0x300000000ULL ++#define G5P_RU_CODE_WINDOW 0x100000000ULL /* 2^32 */ ++#define G5P_RU_HW_VA_LIMIT 0x4000000000ULL /* 2^38 */ ++ ++/* Not a requested access: the resolver sets it, for the typed profiles only, ++ * when the listed BO carries DRM_G5P_BO_EXEC. Only TYPED_V3 reads it; every ++ * other check masks on READ/WRITE. It lives in `access` rather than in a new ++ * field so positional g5p_ru_range initialisers keep compiling. ++ */ ++#define G5P_RU_ACCESS_EXEC 16U ++ ++struct g5p_ru_range { ++ u64 va, size; ++ u32 access; ++}; ++ ++/* [va, va + size) inside one listed range that grants every access bit. */ ++static inline bool g5p_ru_in(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 access) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if ((r[i].access & access) == access && va >= r[i].va && ++ va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) ++ return true; ++ return false; ++} ++ ++/* Everything TYPED_V1 and TYPED_V2 share: all fields except the pools. */ ++static inline int g5p_ru_pass_check_common(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || ++ !g5p_ru_in(r, n, p->vdm_va, p->vdm_size, G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ /* Copied to TA W88/R0+470; meaning unresolved (r23), so bounded. */ ++ if ((p->vdm_control & 0xf) || p->vdm_control > G5P_RU_MAX_VDM) ++ return -EINVAL; ++ if ((p->scratch_va & 0xf) || p->scratch_size < 0x10 || ++ p->scratch_size > G5P_RU_MAX_SCRATCH || ++ !g5p_ru_in(r, n, p->scratch_va, p->scratch_size, rw)) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((p->scratch_ptr[i] & 3) || ++ !g5p_ru_in(r, n, p->scratch_ptr[i], 4, rw)) ++ return -EINVAL; ++ /* Three-word descriptors keep their fourth slot zero. */ ++ if (p->ta_pds[3] || p->eot[3] || p->eot_resume[3]) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V1 and LINEAR64_V1: the two allocator pools are page-aligned RW. */ ++static inline int g5p_ru_pass_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret = g5p_ru_pass_check_common(p, r, n); ++ ++ if (ret) ++ return ret; ++ if (((p->pool7_va | p->pool7_size | p->pool8_va | p->pool8_size) & ++ (G5P_GPU_PAGE - 1)) || ++ !g5p_ru_in(r, n, p->pool7_va, p->pool7_size, rw) || ++ !g5p_ru_in(r, n, p->pool8_va, p->pool8_size, rw)) ++ return -EINVAL; ++ return 0; ++} ++ ++#define G5P_RU_MAX_TABLE 65536U ++ ++/* TYPED_V2: the pools are the ISP scissor (8-byte entries) and depth-bias ++ * (12-byte) tables, READ and word aligned; the V1 pool fields stay zero. ++ * Flagged ZLS words are addresses into listed RW ranges; the rest is opaque. ++ */ ++static inline int g5p_ru_pass2_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret; ++ ++ if (p->pool7_va || p->pool7_size || p->pool8_va || p->pool8_size) ++ return -EINVAL; ++ ret = g5p_ru_pass_check_common(p, r, n); ++ if (ret) ++ return ret; ++ if (!e->scissor_count || e->scissor_count > G5P_RU_MAX_TABLE || ++ (e->scissor_va & 3) || ++ !g5p_ru_in(r, n, e->scissor_va, e->scissor_count * 8ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if (!e->dbias_count || e->dbias_count > G5P_RU_MAX_TABLE || ++ (e->dbias_va & 3) || ++ !g5p_ru_in(r, n, e->dbias_va, e->dbias_count * 12ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if ((e->pixel_width != 2 && e->pixel_width != 4 && ++ e->pixel_width != 8) || ++ (e->bgobj_vals & ~0x3ffU) || (e->zls_mask & ~0x3fU)) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((e->zls_mask & (1U << i)) && ++ ((e->zls[i] >> 40) || ++ !g5p_ru_in(r, n, e->zls[i] & ~0xfULL, 16, rw))) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V3: the TA PDS program (W+0x88 -> R0+0x470) is an explicit reference ++ * and the VDM stream length is a separate field that never reaches W. ++ * Contract and evidence: docs/gpu/experiments/typed-v3-kernel-implementation.md. ++ * Everything here is rejected, never masked. Structural rules need only the ++ * resolved ranges; the two content rules take bytes the caller read through ++ * the BOs' own pages (g5p_ru_pass3_content()). The rules are a subset of what ++ * the userspace lane accepts (tools/mesa-a9 a9c_ctx_lint(), a9scene): nothing ++ * here is stricter than a client that passes those lints. ++ * -EINVAL stale vdm_control, malformed program/stream/descriptor fields, ++ * no terminator, or (V1 dialect) the V1 rules fail ++ * -ERANGE outside [pds_base, pds_base + code_window) or past hw_va_limit ++ * -ENOENT no listed range wholly covers the program, stream or task code ++ * -EACCES missing READ, or the program/task code range's BO lacks EXEC ++ * -ENODATA the first 16 program bytes are all zero ++ */ ++#define G5P_RU_PROGRAM_ALIGN 16U ++#define G5P_RU_PROGRAM_MIN 16U ++ ++struct g5p_ru_v3_limits { ++ u64 pds_base, code_window; ++ u64 hw_va_limit; /* 0: no statement */ ++}; ++ ++/* Where the content rules read: indexes into the ranges array. */ ++struct g5p_ru_v3_loc { ++ u64 program_off; /* the exact W+0x88 value */ ++ u32 program_range, vdm_range; ++ u32 pixel_width; /* the pass's output width, 2, 4 or 8, either dialect */ ++ bool dialect2; /* ext was non-zero: V2 3D words; else the V1 dialect */ ++}; ++ ++/* TYPED_V3 carries the V2 ext struct. An all-zero ext selects the V1 dialect ++ * (pools in base, no tables, V1 3D words): the pass tools/mesa-a9 records ++ * today, plus the explicit program. Any non-zero ext field selects V2. ++ */ ++static inline bool g5p_ru_pass2_zero(const struct g5p_ru_pass2 *e) ++{ ++ u64 any = e->scissor_va | e->dbias_va | e->background_texture; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(e->zls); i++) ++ any |= e->zls[i]; ++ any |= e->scissor_count | e->dbias_count | e->bgobj_depth | ++ e->bgobj_vals | e->pixel_width | e->zls_mask; ++ return !any; ++} ++ ++/* First listed range that wholly contains [va, va + size). */ ++static inline bool g5p_ru_cover(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 *idx) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if (va >= r[i].va && va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) { ++ *idx = i; ++ return true; ++ } ++ return false; ++} ++ ++/* [va, va + size) ends at or below the limit; zero limit makes no statement. */ ++static inline bool g5p_ru_below(u64 limit, u64 va, u64 size) ++{ ++ return !limit || (size <= limit && va <= limit - size); ++} ++ ++/* One PDS task reference of a store/resume descriptor: a 32-bit offset from ++ * pds_base, 16-byte aligned like every captured value and W+0x88's mask, ++ * non-zero, inside the code window and the hardware limit, and starting in a ++ * listed READ range (`code`: whose BO also carries EXEC). ++ */ ++static inline int g5p_ru_v3_ref(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, u32 off, ++ bool code) ++{ ++ u64 va; ++ u32 i; ++ ++ if (!off || (off % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (lim->code_window < G5P_RU_PROGRAM_MIN || ++ off > lim->code_window - G5P_RU_PROGRAM_MIN || ++ lim->pds_base > ~0ULL - off) ++ return -ERANGE; ++ va = lim->pds_base + off; ++ if (!g5p_ru_below(lim->hw_va_limit, va, G5P_RU_PROGRAM_MIN)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, va, G5P_RU_PROGRAM_MIN, &i)) ++ return -ENOENT; ++ if (!(r[i].access & G5P_RU_ACCESS_READ) || ++ (code && !(r[i].access & G5P_RU_ACCESS_EXEC))) ++ return -EACCES; ++ return 0; ++} ++ ++/* The TYPED_V3 pixel output width. The V1 dialect takes it from the V3 field ++ * (0 = 2, the V1 dialect exactly), the V2 dialect from ext, where the V3 field ++ * must stay zero. A width above 2 is several colour targets, and a partial ++ * render must then resume with the pass's own EOT: a one-target resume EOT ++ * would store only the first target. The EOT itself is not decoded. ++ */ ++static inline int g5p_ru_pass3_width(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ bool dialect2, u32 *width) ++{ ++ u32 w = v->pixel_width; ++ ++ if (v->pad || (w && w != 2 && w != 4 && w != 8) || (dialect2 && w)) ++ return -EINVAL; ++ if (dialect2) ++ w = e->pixel_width; /* 2, 4 or 8: g5p_ru_pass2_check() */ ++ else if (!w) ++ w = 2; ++ if (w > 2 && memcmp(p->eot, p->eot_resume, sizeof(p->eot))) ++ return -EINVAL; ++ *width = w; ++ return 0; ++} ++ ++/* The V3 rules, then the TYPED_V1 rules (all-zero ext) or the TYPED_V2 rules ++ * (any ext field set; pools zero), then the output width. base.vdm_control is ++ * checked first. On success loc names the ranges to read, the dialect and the ++ * width. ++ */ ++static inline int g5p_ru_pass3_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_ru_v3_loc *loc) ++{ ++ u32 pi, vi; ++ u64 off; ++ int ret; ++ ++ if (p->vdm_control) ++ return -EINVAL; /* a stream length here is the r36/r88 defect */ ++ if (v->ta_program_size < G5P_RU_PROGRAM_MIN || ++ v->ta_program_size > DRM_G5P_RENDER_MAX_PROGRAM_BYTES || ++ (v->ta_program_size % G5P_RU_PROGRAM_ALIGN) || ++ (v->ta_program_va % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || v->vdm_end < 4 || ++ v->vdm_end > p->vdm_size || (v->vdm_end & 3)) ++ return -EINVAL; ++ if (v->ta_program_va < lim->pds_base) ++ return -ERANGE; ++ off = v->ta_program_va - lim->pds_base; ++ if (lim->code_window < v->ta_program_size || ++ off > lim->code_window - v->ta_program_size) ++ return -ERANGE; ++ if (!g5p_ru_below(lim->hw_va_limit, v->ta_program_va, ++ v->ta_program_size) || ++ !g5p_ru_below(lim->hw_va_limit, p->vdm_va, p->vdm_size)) ++ return -ERANGE; ++ /* Every listed range, not only the two above: on A10 a buffer moved past ++ * the limit stalls the TA without a fault, and all buffers moved together. ++ */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, v->ta_program_va, v->ta_program_size, &pi)) ++ return -ENOENT; ++ if (!(r[pi].access & G5P_RU_ACCESS_READ) || ++ !(r[pi].access & G5P_RU_ACCESS_EXEC)) ++ return -EACCES; ++ if (!g5p_ru_cover(r, n, p->vdm_va, p->vdm_size, &vi)) ++ return -ENOENT; ++ if (!(r[vi].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ /* Context store [0] / load [1]: code at word 2 (READ and EXEC), STREAM_OUT2 ++ * at word 5 (READ). Resume tasks: word 2 (READ only: R0+0xf060/0xf070, ++ * named stream-out by the userspace lane and aux program by r92 2.4, so ++ * its role is unresolved and EXEC is not demanded). Words 0, 1, 3 and 4 ++ * stay opaque. The spacing (store - program = 0x280, resume - store = ++ * 0x340) and a program that equals a task entry (r63 flew the store ++ * program; the role is not detectable structurally) are not enforced. ++ */ ++ for (u32 i = 0; i < 2; i++) { ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][2], true); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][5], ++ false); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->resume_pds[i][2], ++ false); ++ if (ret) ++ return ret; ++ } ++ loc->dialect2 = !g5p_ru_pass2_zero(e); ++ ret = loc->dialect2 ? g5p_ru_pass2_check(p, e, r, n) : ++ g5p_ru_pass_check(p, r, n); ++ if (!ret) ++ ret = g5p_ru_pass3_width(p, e, v, loc->dialect2, ++ &loc->pixel_width); ++ if (ret) ++ return ret; ++ loc->program_off = off; ++ loc->program_range = pi; ++ loc->vdm_range = vi; ++ return 0; ++} ++ ++/* The first 16 program bytes are not all zero (r88 published 0x370, which ++ * pointed at zeros) and the last stream dword is the terminator. `head` is ++ * G5P_RU_PROGRAM_MIN bytes, `tail` the 4 stream bytes at vdm_va + vdm_end - 4. ++ */ ++static inline int g5p_ru_pass3_content(const u8 *head, const u8 *tail) ++{ ++ u8 any = 0; ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ for (u32 i = 0; i < G5P_RU_PROGRAM_MIN; i++) ++ any |= head[i]; ++ if (!any) ++ return -ENODATA; ++ if (word != DRM_G5P_RENDER_STREAM_TERMINATE) ++ return -EINVAL; ++ return 0; ++} ++ ++/* AGXRenderTargetMemoryG5 output sanity for userspace geometry: exact one ++ * sample and the sizes the kernel will allocate. Opaque fields unchecked. ++ */ ++static inline int g5p_ru_geometry_check(const struct g5p_ru_rt *rt, u32 width, ++ u32 height, ++ struct g5p_ru_rt_sizes *sizes) ++{ ++ if (!width || !height || width > 16384 || height > 16384) ++ return -EINVAL; ++ if (rt->tiles > ((width + 31) / 32) * ((height + 31) / 32)) ++ return -EINVAL; ++ return g5p_ru_rt_sizes(rt, sizes); ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/vm-ranges.h b/drivers/gpu/drm/apple-g5p/vm-ranges.h +new file mode 100644 +index 0000000000..25bd98ee98 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/vm-ranges.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_H ++#define G5P_COMPUTE_USER_H ++ ++/* Public geometry is fixed across VMs. The private ranges never alias a BO. */ ++ ++static const u64 g5p_private_va[] = { ++ 0x300040000ULL, 0x300014000ULL, 0x300028000ULL, 0x300044000ULL, 0, ++ 0x100080000ULL, 0x100084000ULL, 0x1000a0000ULL, 0x1000a4000ULL, ++}; ++ ++static inline bool g5p_user_private_overlap(u64 va, u64 size) ++{ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_private_va); i++) ++ if (g5p_private_va[i] && va < g5p_private_va[i] + 0x4000 && ++ g5p_private_va[i] < va + size) ++ return true; ++ return false; ++} ++ ++#endif diff --git a/kernel/patches/drivers/drm/0011-dt-bindings-gpu-describe-the-S8003-G5P-resources.patch b/kernel/patches/drivers/drm/0011-dt-bindings-gpu-describe-the-S8003-G5P-resources.patch new file mode 100644 index 00000000..dad4ef46 --- /dev/null +++ b/kernel/patches/drivers/drm/0011-dt-bindings-gpu-describe-the-S8003-G5P-resources.patch @@ -0,0 +1,46 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 5 Nov 2026 09:01:41 +0000 +Subject: [PATCH] dt-bindings: gpu: describe the S8003 G5P resources + +Describe the inherited compatible and the two register windows, five IRQs and graphics power domain. The legacy vendor spelling remains a review concern. + +Assisted-by: LLM +--- + .../bindings/gpu/apple,s8003-g5p.yaml | 27 +++++++++++++++++++ + 1 file changed, 27 insertions(+) + create mode 100644 Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml + +diff --git a/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +new file mode 100644 +index 0000000000..699c3492fa +--- /dev/null ++++ b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +@@ -0,0 +1,27 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/gpu/apple,s8003-g5p.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++title: Apple S8003 G5P GPU ++maintainers: ++ - Berkay Kösebay ++description: | ++ G5P GPU with META firmware on S8003. The compatible is Apple's inherited ++ ADT spelling used by the qualified driver; no new vendor prefix is implied. ++properties: ++ compatible: ++ const: gpu,s8003 ++ reg: ++ maxItems: 2 ++ reg-names: ++ items: ++ - const: rgx ++ - const: fender ++ interrupts: ++ minItems: 5 ++ maxItems: 5 ++ power-domains: ++ maxItems: 1 ++required: [compatible, reg, reg-names, interrupts, power-domains] ++additionalProperties: false diff --git a/src/orchard/apple/agx_g5p.py b/src/orchard/apple/agx_g5p.py new file mode 100644 index 00000000..c46ca6a7 --- /dev/null +++ b/src/orchard/apple/agx_g5p.py @@ -0,0 +1,19 @@ +"""Extract the unmodified G5P C0 META LDR from the user's prelinked kernelcache.""" +import hashlib +from orchard.apple.prelink import Kernelcache, named + +BUNDLE = 'com.apple.AGXFirmwareKextG5P' +CONST = ('__TEXT', '__const') +SIZE = 119112 +SHA256 = '33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca' + + +def extract(blob): + kernel = Kernelcache(blob) + section = named(kernel.kext_sections(BUNDLE, 'G5P firmware'), CONST)[CONST] + if section['size'] != SIZE: + raise ValueError('Unsupported G5P C0 firmware size') + data = kernel.read(section['addr'], section['size']) + if hashlib.sha256(data).hexdigest() != SHA256: + raise ValueError('Unsupported G5P C0 firmware digest') + return data diff --git a/src/orchard/recipe.py b/src/orchard/recipe.py index eb30079b..05dfb2f1 100644 --- a/src/orchard/recipe.py +++ b/src/orchard/recipe.py @@ -8,7 +8,7 @@ import json import plistlib -from orchard.apple import bluetool, d5500, smartio, z2fw, z2packet +from orchard.apple import agx_g5p, bluetool, d5500, smartio, z2fw, z2packet from orchard.apple.osserialize import apple_xml, binary @@ -30,6 +30,10 @@ def _smartio_kext(step, data, expected): return smartio.extract(data, step['kext'])[0] +def _agx_g5p(step, data, expected): + return agx_g5p.extract(data) + + def _vxd(step, data, expected): return d5500.extract(data)[0] @@ -82,6 +86,7 @@ def _z2fw_mtfw(step, data, expected): # step@version -> (implementation, {parameter besides 'step' and 'input': type}) STEPS = { + 'agx-g5p-ldr-from-kernelcache@1': (_agx_g5p, {}), 'copy@1': (_copy, {}), 'bluetool-hcd@1': (_bluetool_hcd, {}), # T8010 SmartIO kext (iPad 6/7, iPhone 7). diff --git a/tests/test_agx_g5p.py b/tests/test_agx_g5p.py new file mode 100644 index 00000000..f048ef01 --- /dev/null +++ b/tests/test_agx_g5p.py @@ -0,0 +1,29 @@ +import hashlib +import unittest +from unittest.mock import Mock, patch +from orchard.apple import agx_g5p + +class G5PFirmware(unittest.TestCase): + def test_original_bytes_and_bundle(self): + data = b'synthetic firmware, not Apple data' + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=len(data))] + kernel.read.return_value = data + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), patch.object(agx_g5p, 'SIZE', len(data)), patch.object(agx_g5p, 'SHA256', hashlib.sha256(data).hexdigest()): + self.assertIs(agx_g5p.extract(b'input'), data) + kernel.kext_sections.assert_called_once_with(agx_g5p.BUNDLE, 'G5P firmware') + kernel.read.assert_called_once_with(4096, len(data)) + + def test_wrong_size_is_rejected_before_read(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=1)] + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'size'): + agx_g5p.extract(b'input') + kernel.read.assert_not_called() + + def test_wrong_digest_is_rejected(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=agx_g5p.SIZE)] + kernel.read.return_value = bytes(agx_g5p.SIZE) + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'digest'): + agx_g5p.extract(b'input') From cfa4c8deb7f32af821f02e54f0abcf5583ea4f9d Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 02/33] G5P render, autoboot, META71C and boot-origin policy kernel/configs/apple-16k/config: CONFIG_DRM_APPLE_G5P=y These options provide g5p and its Kconfig dependencies on both page sizes. Assisted-by: LLM --- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + devices/j71t/firmware.json | 10 + docs/firmware-recipes.md | 16 + kernel/configs/apple-16k/config | 1 + .../s8000/0007-g5p-implementation.patch | 49 + .../drivers/drm/0008-g5p-bindings.patch | 45 + .../drivers/drm/0009-g5p-implementation.patch | 25381 ++++++++++++++++ .../drivers/drm/0010-g5p-implementation.patch | 80 + src/orchard/apple/agx_g5p.py | 19 + src/orchard/recipe.py | 7 +- tests/test_agx_g5p.py | 29 + 12 files changed, 25642 insertions(+), 1 deletion(-) create mode 100644 kernel/patches/devices/s8000/0007-g5p-implementation.patch create mode 100644 kernel/patches/drivers/drm/0008-g5p-bindings.patch create mode 100644 kernel/patches/drivers/drm/0009-g5p-implementation.patch create mode 100644 kernel/patches/drivers/drm/0010-g5p-implementation.patch create mode 100644 src/orchard/apple/agx_g5p.py create mode 100644 tests/test_agx_g5p.py diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..eca49fad 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -15,3 +15,6 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "J71t on hardware" stages = "remote_boot two-stage CPID list" + +# Local capability delta: g5p. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/g5p.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..03cbaa69 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -54,3 +54,6 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" + +# Local capability delta: g5p. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/g5p.md. diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index 3321e246..442cabbd 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -279,6 +279,16 @@ "calibration_size": 1024 } ] + }, + { + "destination": "lib/firmware/apple/agx-g5p-c0.ldr", + "sha256": "33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca", + "steps": [ + { + "step": "agx-g5p-ldr-from-kernelcache@1", + "input": "kernelcache" + } + ] } ] } diff --git a/docs/firmware-recipes.md b/docs/firmware-recipes.md index b0127701..cd1ec35f 100644 --- a/docs/firmware-recipes.md +++ b/docs/firmware-recipes.md @@ -236,3 +236,19 @@ The decompressed N61 kernel's SHA-256 is These fields are explicit recipe parameters, including zero-valued overrides. Physical qualification of firmware loading and input events remains separate from converting this shared firmware file. + +## S8003 G5P META firmware + +`agx-g5p-ldr-from-kernelcache@1` reads the unique `__TEXT,__const` section +of `com.apple.AGXFirmwareKextG5P` in the user's own legacy prelinked +kernelcache. It rejects any size other than 119112 bytes or any digest other +than `33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca`. +The existing J71t kernelcache input names the active device kernelcache and +the pinned restore-image fallback. The normal recipe installer writes the +verified output to `/lib/firmware/apple/agx-g5p-c0.ldr`. No payload patch, +calibration, firmware bytes or captured job is stored in the source tree. +Other C0 revisions and other boards require separate digest qualification. + +The J71t G5P driver brings the GPU up automatically once the extracted +`apple/agx-g5p-c0.ldr` is readable on the root filesystem (up to 120 seconds). +No captured render or compute job fixtures are part of this recipe. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index aae562f8..82a17860 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -3936,6 +3936,7 @@ CONFIG_DRM_SIMPLEDRM=y # CONFIG_DRM_VKMS is not set # CONFIG_DRM_VMWGFX is not set CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DRM_APPLE_G5P=y # # Frame buffer Devices diff --git a/kernel/patches/devices/s8000/0007-g5p-implementation.patch b/kernel/patches/devices/s8000/0007-g5p-implementation.patch new file mode 100644 index 00000000..df4e76a4 --- /dev/null +++ b/kernel/patches/devices/s8000/0007-g5p-implementation.patch @@ -0,0 +1,49 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: G5P render, autoboot, META71C and boot-origin policy + +Carry the original-firmware G5P render/compute ABI and extraction recipe into the system kernel, with typed query/compute follow-ups and qualified WC cache handling. Start firmware at probe, install META71C 340 on every firmware image, select dirty-start mode 3 from the incoming kexec DT marker (cold mode 1), and quiet recurring diagnostics. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..7e4d67d7f 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -24,3 +24,7 @@ &touchscreen { + &wifi { + brcm,board-type = "apple,feara"; + }; ++ ++&gpu_diag { ++ status = "okay"; ++}; +diff --git a/arch/arm64/boot/dts/apple/s8003.dtsi b/arch/arm64/boot/dts/apple/s8003.dtsi +index b51588941..67cdd227a 100644 +--- a/arch/arm64/boot/dts/apple/s8003.dtsi ++++ b/arch/arm64/boot/dts/apple/s8003.dtsi +@@ -336,3 +336,20 @@ map0 { + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/* G5P resources; enable only on a board qualified with this driver. */ ++&{/soc} { ++ gpu_diag: gpu@201000000 { ++ status = "disabled"; ++ compatible = "gpu,s8003"; ++ reg = <0x2 0x01000000 0x0 0x100000>, ++ <0x2 0x01e00000 0x0 0x100000>; ++ reg-names = "rgx", "fender"; ++ power-domains = <&ps_gfx>; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ }; ++}; diff --git a/kernel/patches/drivers/drm/0008-g5p-bindings.patch b/kernel/patches/drivers/drm/0008-g5p-bindings.patch new file mode 100644 index 00000000..23f8f1ad --- /dev/null +++ b/kernel/patches/drivers/drm/0008-g5p-bindings.patch @@ -0,0 +1,45 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] dt-bindings: drm: G5P render, autoboot, META71C and boot-origin policy + +Carry the original-firmware G5P render/compute ABI and extraction recipe into the system kernel, with typed query/compute follow-ups and qualified WC cache handling. Start firmware at probe, install META71C 340 on every firmware image, select dirty-start mode 3 from the incoming kexec DT marker (cold mode 1), and quiet recurring diagnostics. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +new file mode 100644 +index 000000000..699c3492f +--- /dev/null ++++ b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +@@ -0,0 +1,27 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/gpu/apple,s8003-g5p.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++title: Apple S8003 G5P GPU ++maintainers: ++ - Berkay Kösebay ++description: | ++ G5P GPU with META firmware on S8003. The compatible is Apple's inherited ++ ADT spelling used by the qualified driver; no new vendor prefix is implied. ++properties: ++ compatible: ++ const: gpu,s8003 ++ reg: ++ maxItems: 2 ++ reg-names: ++ items: ++ - const: rgx ++ - const: fender ++ interrupts: ++ minItems: 5 ++ maxItems: 5 ++ power-domains: ++ maxItems: 1 ++required: [compatible, reg, reg-names, interrupts, power-domains] ++additionalProperties: false diff --git a/kernel/patches/drivers/drm/0009-g5p-implementation.patch b/kernel/patches/drivers/drm/0009-g5p-implementation.patch new file mode 100644 index 00000000..6c67d3d4 --- /dev/null +++ b/kernel/patches/drivers/drm/0009-g5p-implementation.patch @@ -0,0 +1,25381 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] drm/apple-g5p: G5P render, autoboot, META71C and boot-origin policy + +Carry the original-firmware G5P render/compute ABI and extraction recipe into the system kernel, with typed query/compute follow-ups and qualified WC cache handling. Start firmware at probe, install META71C 340 on every firmware image, select dirty-start mode 3 from the incoming kexec DT marker (cold mode 1), and quiet recurring diagnostics. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/gpu/apple-g5p/bits-compat.h b/drivers/gpu/apple-g5p/bits-compat.h +new file mode 100644 +index 000000000..608e9b8ec +--- /dev/null ++++ b/drivers/gpu/apple-g5p/bits-compat.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_BITS_COMPAT_H ++#define G5P_BITS_COMPAT_H ++ ++#ifdef __KERNEL__ ++#include ++#include ++#else ++#ifndef BIT ++#define BIT(nr) (1UL << (nr)) ++#endif ++#ifndef BIT_ULL ++#define BIT_ULL(nr) (1ULL << (nr)) ++#endif ++#ifndef ARRAY_SIZE ++#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) ++#endif ++#endif ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-layout.h b/drivers/gpu/apple-g5p/boot-layout.h +new file mode 100644 +index 000000000..752e6873f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-layout.h +@@ -0,0 +1,306 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_BOOT_LAYOUT_H ++#define APPLE_G5P_BOOT_LAYOUT_H ++ ++#include "bits-compat.h" ++#include "pack.h" ++ ++#define G5P_GPU_PAGE 0x4000U ++#define G5P_ROOT_SIZE 0x8000U ++#define G5P_SHARED_SIZE 0x60000U ++#define G5P_IMAGE_VA 0xe1c0000000ULL ++#define G5P_SHARED_VA 0xe1c0200000ULL ++#define G5P_ROOT_VA 0xe1c3200000ULL ++#define G5P_PIO_VA 0xe1c4200000ULL ++#define G5P_PIO_SIZE 0x10000U ++ ++/* c95da0 descriptors, sizes at 5940b90; c97b6c packs on 64-byte boundaries. */ ++struct g5p_boot_layout { ++ u32 offset[0x90]; ++ u8 group[0x90]; ++ u32 used[2]; ++}; ++ ++static inline void g5p_put32(u8 *p, u32 value) ++{ ++ p[0] = value; ++ p[1] = value >> 8; ++ p[2] = value >> 16; ++ p[3] = value >> 24; ++} ++ ++static inline void g5p_put64(u8 *p, u64 value) ++{ ++ g5p_put32(p, value); ++ g5p_put32(p + 4, value >> 32); ++} ++ ++static inline u32 g5p_meta_address(u64 va) ++{ ++ /* ca0744: root range 8 bypasses META caches; other shared RAM is global. */ ++ u32 prefix = va >= G5P_ROOT_VA && va < G5P_ROOT_VA + 0x1000000 ? ++ 0x40000000 : ++ 0xc0000000; ++ ++ return ((u32)va + 0x40000000) | prefix; ++} ++ ++static inline int g5p_plan_layout(struct g5p_boot_layout *p) ++{ ++ static const struct { ++ u8 cpu, gpu, group; ++ u32 size; ++ } objects[] = { ++ { 0x44, 0x50, 0, 0x1490 }, ++ { 0x47, 0x51, 0, 0x990 }, ++ { 0x48, 0x52, 0, 0x8e0 }, ++ { 0x49, 0x53, 0, 0x8c0 }, ++ { 0x4a, 0x54, 0, 0x938 }, ++ { 0x4b, 0x55, 0, 0x28 }, ++ { 0x4c, 0x56, 0, 8 }, ++ { 0x4d, 0x57, 0, 0x78c }, ++ { 0x4e, 0x58, 0, 0x84 }, ++ { 0x5e, 0x5f, 1, 0x2058 }, ++ { 0x70, 0x78, 1, 0x20 }, ++ { 0x71, 0x79, 1, 0x80 }, ++ { 0x72, 0x7a, 1, 0x20 }, ++ { 0x73, 0x7b, 1, 0x20 }, ++ { 0x74, 0x7c, 1, 0x2c00 }, ++ { 0x75, 0x7d, 1, 0xb000 }, ++ { 0x4f, 0x59, 1, 0x36000 }, ++ { 0x76, 0x7e, 1, 0x2c00 }, ++ { 0x77, 0x7f, 1, 0x2400 }, ++ { 0x6c, 0x6e, 1, 0x20 }, ++ { 0x6d, 0x6f, 1, 0x1400 }, ++ { 0x60, 0x62, 1, 0x20 }, ++ { 0x64, 0x66, 1, 0x20 }, ++ { 0x68, 0x6a, 1, 0x20 }, ++ { 0x61, 0x63, 1, 0x1400 }, ++ { 0x65, 0x67, 1, 0x1400 }, ++ { 0x69, 0x6b, 1, 0x1400 }, ++ /* Separate c95da0 allocations, placed after the recovered group. */ ++ { 0x5c, 0x5d, 1, 8 }, ++ { 0x85, 0x88, 1, 0x400 }, ++ { 0x84, 0x87, 1, 0x400 }, ++ { 0x86, 0x89, 1, 0x800 }, ++ }; ++ unsigned int i; ++ ++ memset(p, 0, sizeof(*p)); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) { ++ u32 group = objects[i].group; ++ u32 off = (p->used[group] + 63) & ~63U; ++ ++ p->offset[objects[i].cpu] = off; ++ p->offset[objects[i].gpu] = off; ++ p->group[objects[i].cpu] = group; ++ p->group[objects[i].gpu] = group; ++ p->used[group] = off + objects[i].size; ++ } ++ return p->used[0] <= G5P_ROOT_SIZE && p->used[1] <= G5P_SHARED_SIZE ? ++ 0 : ++ -EINVAL; ++} ++ ++static inline u64 g5p_object_va(const struct g5p_boot_layout *p, u32 slot) ++{ ++ return (p->group[slot] ? G5P_SHARED_VA : G5P_ROOT_VA) + p->offset[slot]; ++} ++ ++static inline u8 *g5p_object(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 slot) ++{ ++ return (p->group[slot] ? shared : root) + p->offset[slot]; ++} ++ ++/* ca9b34 + cab44c + cac314: 40-bit PA, 16 KiB GPU pages. */ ++static inline u32 g5p_catalogue_entry(u64 pa) ++{ ++ return (u32)(pa >> 8) | 1; ++} ++ ++static inline u64 g5p_directory_entry(u64 pa) ++{ ++ /* Geometry bytes 3c..3f are 08 05 05 15: size, entry/PA/VA shifts. */ ++ return pa | 3; ++} ++ ++static inline u64 g5p_leaf_entry(u64 pa, u32 attributes) ++{ ++ /* caa6e4 range attributes -> ca9a60: META protect, CC / SLC bypass. */ ++ return (pa & 0xfffffff000ULL) | (attributes & 0xc) | ++ ((u64)(attributes & 0x10) << 58) | 1; ++} ++ ++static inline void g5p_make_boot_config(struct g5p_ldr_image *im, ++ const struct g5p_boot_layout *p) ++{ ++ static const u32 bases[] = { 0x10000000, 0x20000000, 0x40000000, ++ 0x60001000 }; ++ static const u32 sizes[] = { 0x18000, 0x4000, 0x4000, 0x4000 }; ++ static const u32 offsets[] = { 0x4000, 0x1c000, 0, 0x20000 }; ++ u8 *out = im->data + 0x200; ++ unsigned int seg, i; ++ ++ for (seg = 0; seg < 4; seg++) { ++ u64 va = G5P_IMAGE_VA + offsets[seg]; ++ u32 reg = 0x04850000 + seg * 16; ++ u32 values[] = { bases[seg] | 0xf02, sizes[seg] - 1, (u32)va, ++ (u32)(va >> 32) | 0x300000 }; ++ ++ if (seg == 2) { ++ g5p_put32(out, reg + 12); ++ g5p_put32(out + 4, (u32)(va >> 32) | 0x100); ++ out += 8; ++ continue; ++ } ++ for (i = 0; i < 4; i++, out += 8) { ++ g5p_put32(out, reg + i * 4); ++ g5p_put32(out + 4, values[i]); ++ } ++ } ++ for (i = 0; i < im->config_count; i++, out += 8) { ++ g5p_put32(out, im->config[i][0]); ++ g5p_put32(out + 4, im->config[i][1]); ++ } ++ g5p_put64(out, 0); ++ g5p_put32(out + 8, g5p_meta_address(G5P_IMAGE_VA + 0x3ff8)); ++ g5p_put32(im->data + 0x3ff8, g5p_meta_address(g5p_object_va(p, 0x50))); ++ g5p_put32(im->data + 0x3ffc, g5p_meta_address(g5p_object_va(p, 0x5f))); ++} ++ ++/* c96be8/c9edcc/ca3ccc. Unknown platform policy fields remain zero. */ ++static inline void g5p_make_init(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, const struct g5p_ldr_image *im, ++ u64 catalogue, u64 timestamp) ++{ ++ static const u8 roots[] = { ++ 0x62, 0x63, 0x66, 0x67, 0x6a, 0x6b, 0x6e, 0x6f, ++ 0x78, 0x7c, 0x79, 0x7d, 0x7a, 0x7e, 0x7b, 0x7f, ++ }; ++ static const u8 objects[] = { 0x51, 0x52, 0x53, 0x54, 0x55 }; ++ static const u8 tail[] = { 0x58, 0x56, 0x57, 0x88, 0x59 }; ++ static const u64 pio_pa[] = { 0x201e00000ULL, 0x20e100000ULL, ++ 0x20e104000ULL, 0x201000000ULL }; ++ static const u32 pio_size[] = { 0x5000, 0x4000, 0x4000, 0x20000 }; ++ static const u32 pio_offset[] = { 0, 0x8000, 0xc000, 0 }; ++ u8 *init = g5p_object(p, root, shared, 0x44); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *ctrl = g5p_object(p, root, shared, 0x5e); ++ unsigned int i; ++ ++ memset(root, 0, G5P_ROOT_SIZE); ++ memset(shared, 0, G5P_SHARED_SIZE); ++ for (i = 0; i < ARRAY_SIZE(roots); i++) ++ g5p_put32(init + i * 4, ++ g5p_meta_address(g5p_object_va(p, roots[i]))); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) ++ g5p_put32(init + 0x64 + i * 4, ++ g5p_meta_address(g5p_object_va(p, objects[i]))); ++ for (i = 0; i < ARRAY_SIZE(tail); i++) ++ g5p_put32(init + 0x44c + i * 4, ++ g5p_meta_address(g5p_object_va(p, tail[i]))); ++ for (i = 0; i < 3; i++) { ++ u8 *channel = g5p_object(p, root, shared, 0x47 + i); ++ u32 ptr = g5p_meta_address(g5p_object_va(p, 0x51 + i)); ++ ++ g5p_put32(channel + 4, ptr); ++ g5p_put32(channel + 12, ptr + 8); ++ } ++ g5p_put32(g5p_object(p, root, shared, 0x48) + 0x778, 0xffffffff); ++ g5p_put32(init + 0x480, ++ ((u32)(G5P_IMAGE_VA + 0x24000) + 0x40000000) | 0x40000000); ++ g5p_put64(init + 0x484, G5P_IMAGE_VA + 0x24000); ++ g5p_put32(init + 0x48c, 0x55dc); ++ g5p_put64(meta, 0x600000000ULL); ++ g5p_put64(meta + 8, 0xffc00000); ++ g5p_put64(meta + 0x10, 0x300000000ULL); ++ g5p_put64(meta + 0x18, 0x300000000ULL); ++ for (i = 0; i < ARRAY_SIZE(pio_pa); i++) { ++ u8 *entry = meta + 0x630 + i * 16; ++ ++ g5p_put64(entry, pio_pa[i]); ++ g5p_put32(entry + 8, i == 3 ? 0 : ++ g5p_meta_address(G5P_PIO_VA + ++ pio_offset[i])); ++ g5p_put32(entry + 12, pio_size[i]); ++ } ++ g5p_put32(meta + 0x6c0, 1); ++ g5p_put32(meta + 0x6c8, 1); ++ g5p_put64(meta + 0x6d8, 0x100000000ULL); ++ g5p_put32(meta + 0x6e0, 1); ++ g5p_put32(meta + 0x6e4, 1); ++ /* readChipInfo version 15 -> tables 5942650/5942674 -> 1,5; N=6. */ ++ g5p_put32(meta + 0x704, 1); ++ g5p_put32(meta + 0x708, 5); ++ g5p_put32(meta + 0x70c, 6); ++ g5p_put32(meta + 0x758, 0x14); ++ g5p_put32(meta + 0x75c, 0x4000); ++ g5p_put32(meta + 0x760, 0x1020); ++ g5p_put32(meta + 0x764, im->t1_pc); ++ g5p_put32(meta + 0x768, im->t1_sp); ++ g5p_put64(meta + 0x774, G5P_IMAGE_VA); ++ g5p_put64(meta + 0x77c, 0x300000); ++ g5p_put64(meta + 0x784, timestamp); ++ /* Fixed portion of c974f0; dynamic power-policy fields remain zero. */ ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 8, 0x400000000ULL); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x10, 0x3f800000); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x28, 0x100000001ULL); ++ g5p_put32(g5p_object(p, root, shared, 0x4a) + 0x38, 100); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x8f8, timestamp); ++ g5p_put32(ctrl + 0x24, 0x78); ++ g5p_put32(ctrl + 0x30, 1); ++ g5p_put32(ctrl + 0x1044, g5p_meta_address(g5p_object_va(p, 0x87))); ++ g5p_put32(ctrl + 0x1048, g5p_meta_address(g5p_object_va(p, 0x89))); ++ g5p_put32(ctrl + 0x104c, 1); ++ g5p_put64(g5p_object(p, root, shared, 0x86), catalogue); ++} ++ ++/* Explicit boot-time hypotheses; never modify a live root table. */ ++static inline void g5p_dc_init_variant(const struct g5p_boot_layout *p, ++ u8 *root, u8 *shared, u32 variant) ++{ ++ /* ca0744 uses 40 aliases for cache bypass. Applying that alias to DC ++ * is an unverified hypothesis, not recovered normal host behavior. ++ */ ++ if (variant == 1) ++ g5p_put32(root + 0x18, g5p_le32(root + 0x18) & ~0x80000000U); ++ if (variant == 1 || variant == 6) ++ g5p_put32(root + 0x1c, g5p_le32(root + 0x1c) & ~0x80000000U); ++ /* c96be8 chooses this policy from host+1af8 bits; both values exist. ++ * Whether zero is appropriate for our incomplete policy is unknown. ++ */ ++ if (variant == 5) ++ g5p_put32(shared + p->offset[0x5e] + 0x30, 0); ++ /* MINIM 10001ac8 enters policy1000c0d8 only when meta6c0/6c8 ++ * are both nonzero. Experimental pre-MTS bypass; no live length writes. ++ */ ++ if (variant == 7) ++ g5p_put32(g5p_object(p, root, shared, 0x4d) + 0x6c8, 0); ++ ++ /* Round13: recovered iOS inputs; variants 8/9 retain meta6c8=1. ++ * Neither is a complete DVFS policy (table+3c remains absent). ++ */ ++ if (variant == 8 || variant == 9) { ++ u8 *policy = g5p_object(p, root, shared, 0x4a); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *control = g5p_object(p, root, shared, 0x5e); ++ static const u32 mhz[] = { 0, 340, 474, 550, 723 }; ++ ++ g5p_put32(policy + 4, 16); ++ if (variant == 9) { ++ g5p_put32(meta + 0x710, 4); ++ g5p_put32(meta + 0x714, 5); ++ for (u32 i = 0; i < 16; i++) ++ g5p_put32(meta + 0x718 + 4 * i, ++ i < ARRAY_SIZE(mhz) ? mhz[i] : 0); ++ g5p_put32(control + 0x1028, 3348); ++ g5p_put32(control + 0x102c, 400); ++ g5p_put32(control + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++ } ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +new file mode 100644 +index 000000000..2407ec142 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -0,0 +1,1180 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R73b crash-first IRQ174 diagnostics. Included by boot.inc after struct ++ * g5p_boot. Case zero allocates nothing and keeps the r73 IRQ order. ++ * Rows live in CPU RAM only; they are formatted after the epoch, never in the ++ * IRQ service loop. The first 128 rows are retained, the last 128 roll. ++ */ ++#include ++#include ++#include ++#include ++#include "r87b-attach.h" ++#include "r74e112.h" ++#include "heartbeat116.h" ++ ++#define G5P_R73B_ROWS 256 ++#define G5P_R73B_HEAD 128 ++#define G5P_R73B_IRQ_CAP 8192 ++#define G5P_R73B_PANIC_ROWS 32 ++#define G5P_R73B_TRACE_ROWS 128 ++#define G5P_R73B_TRACE_WORDS 9 ++#define G5P_R74_IRQ_CAP 262144 ++#define G5P_R74_IRQ_CAP_MAX 1048576 ++#define G5P_R74_EVENT_FLOOR (256 + 4 * 65536) ++#define G5P_R74_STAGES 8 ++#define G5P_R74_MILESTONES 64 ++#define G5P_R74_MAX_TICKS 16 ++#define G5P_R74_RGX 9 ++#define G5P_R74_WORD1 8 ++#define G5P_R74F_KINDS 3 ++#define G5P_R74F_OFF_US 1000 ++#define G5P_R74F_ON_US 1800 ++#define G5P_R74F_BELL_MS 250 ++#define G5P_R74F_MMU_SLOTS 2 ++#define G5P_R74F_MMU_US 1000 ++#define G5P_R74G_GRACE_ALIVES 4 ++#define G5P_R74G_GRACE_US 20000 ++#define G5P_R74G_COMPANION 0x3980 ++#define G5P_R74G_RECOVERIES 0x3994 ++#define G5P_R74G_CLASS 0x3998 ++#define G5P_R74G_BAD_RING BIT(0) ++#define G5P_R74G_BAD_RECORD BIT(1) ++#define G5P_R74G_BAD_PA BIT(2) ++#define G5P_R74G_BAD_HOST BIT(3) ++#define G5P_R74G_BAD_CACHED BIT(4) ++#define G5P_R74G_BAD_FAILED BIT(5) ++#define G5P_R74G_BAD_COMPANION BIT(6) ++#define G5P_R74G_BAD_MUTATION BIT(7) ++#define G5P_R74H_TAIL 128 ++#define G5P_R74H_PHYS_MAX 1024 ++#define G5P_R74H_PTES 10 ++ ++/* First-ALIVE submit: wait (reads only) for a GPC request in flight to finish. */ ++static unsigned int boot_alive_settle_us = 2000; ++module_param(boot_alive_settle_us, uint, 0600); ++MODULE_PARM_DESC( ++ boot_alive_settle_us, ++ "default0 (single read, as before); 1..5000: when the first-ALIVE submit reads " ++ "R1+0x1000 with bit 4 set (a GPC performance-state request in flight), re-read " ++ "it every 10 us for up to this long before the unchanged state==1 check, as " ++ "Apple's setGPCPerformanceState waits for bit 4; R1 reads only; latched at map"); ++ ++enum g5p_r73b_phase { ++ G5P_R73B_RELEASE_ENTRY = 1, ++ G5P_R73B_RELEASE_FIFO, ++ G5P_R73B_RELEASE_GPC, ++ G5P_R73B_RELEASE_EXIT, ++ G5P_R73B_IRQ, ++ G5P_R73B_DRAIN_BEFORE, ++ G5P_R73B_DRAIN_AFTER, ++ G5P_R73B_DRAIN_BUSY, ++ G5P_R73B_CAP, ++ G5P_R73B_EVENTS_EXIT, ++ G5P_R73B_DC_PUBLISHED, ++ G5P_R73B_DC_KICKED, ++ G5P_R73B_WAIT_EXIT, ++ G5P_R73B_EXPORT, ++ G5P_R73B_PHASES, ++}; ++ ++static const char *const g5p_r73b_phase_names[G5P_R73B_PHASES] = { ++ "unknown", "release-entry", "release-fifo", "release-gpc", ++ "release-exit", "irq", "drain-before", "drain-after", ++ "drain-busy", "cap", "events-exit", "dc-published", ++ "dc-kicked", "wait-exit", "export", ++}; ++ ++enum g5p_r73b_owner { ++ G5P_R73B_NONE, ++ G5P_R73B_OWNER_IRQ, ++ G5P_R73B_OWNER_EVENTS, ++ G5P_R73B_OWNER_PREPARE, ++ G5P_R73B_OWNER_PUBLISH, ++ G5P_R73B_OWNER_KICK, ++ G5P_R73B_OWNER_WAIT, ++}; ++ ++/* Row columns: event, log lane 0, statistics, trace, device control. */ ++static const u8 g5p_r73b_row_slots[5] = { 0x70, 0x71, 0x72, 0x73, 0x6c }; ++ ++struct g5p_r73b_row { ++ u32 seq; /* sequence + 1, published last; zero means empty/torn */ ++ u32 phase; ++ u64 t_ns; ++ u32 irq, ack, released, owner, cap_seen; ++ s32 result; ++ u32 cp[5][2]; ++}; ++ ++enum g5p_r74_step { ++ G5P_R74_IRQ_TEST_REFUSED, ++ G5P_R74_PREPARE_BEGIN, ++ G5P_R74_PREPARE_END, ++ G5P_R74_ATTACH_BEGIN, ++ G5P_R74_ATTACH_PUBLISHED, ++ G5P_R74_ATTACH_DOORBELL_BEGIN, ++ G5P_R74_ATTACH_DOORBELL_END, ++ G5P_R74_ATTACH_CONSUMED, ++ G5P_R74_ATTACH_END, ++ G5P_R74_TA_BEGIN, ++ G5P_R74_TA_PUBLISH, ++ G5P_R74_TA_DOORBELL_RETURNED, ++ G5P_R74_OBSERVE, ++ G5P_R74_WAIT_TICK, ++ G5P_R74_WAIT_EXIT, ++ G5P_R74_REPORT_END, ++ G5P_R74_CMD8_CONSUMED, ++ G5P_R74_TA_CONSUMED, ++ G5P_R74_TA_RAM_COMPLETE, ++ G5P_R74_TA_RAM_TIMEOUT, ++ G5P_R74F_ATTACH_ARMED, ++ G5P_R74F_ATTACH_REACTIVATED, ++ G5P_R74F_ATTACH_BELL, ++ G5P_R74F_MMU_CLOSED, ++ G5P_R74F_TA_ARMED, ++ G5P_R74F_TA_REACTIVATED, ++ G5P_R74F_TA_BELL, ++ G5P_R74F_STOP, ++ G5P_R74G_FW_MMU, ++ G5P_R74G_ACK_SKIPPED, ++ G5P_R74G_TA_GRACE, ++ G5P_R74G_TA_FALLBACK, ++ G5P_R74_STEPS, ++}; ++ ++static const char *const g5p_r74_steps[G5P_R74_STEPS] = { ++ "irq-test-refused", ++ "prepare-begin", ++ "prepare-end", ++ "attach-begin", ++ "attach-dc3-published", ++ "attach-doorbell-begin", ++ "attach-doorbell-end", ++ "attach-consumed", ++ "attach-end", ++ "ta-begin", ++ "ta-publish", ++ "ta-doorbell-returned", ++ "observe", ++ "wait-tick", ++ "wait-exit", ++ "report-end", ++ "cmd8-consumed", ++ "ta-consumed", ++ "ta-ram-complete", ++ "ta-ram-timeout", ++ "attach-bell-armed", ++ "attach-reactivated", ++ "attach-bell", ++ "first-mmu-window-closed", ++ "ta-bell-armed", ++ "ta-reactivated", ++ "ta-bell", ++ "r74f-stop", ++ "firmware-mmu-complete", ++ "r31-ack-skipped", ++ "ta-grace-end", ++ "ta-fallback", ++}; ++ ++/* R74 read-only witnesses; every address is already read elsewhere in-tree. */ ++ ++struct g5p_r74_milestone { ++ u64 t_ns; ++ u32 step, irqs, events; ++ s32 ret; ++ u32 dc[2], ta[2]; ++}; ++ ++enum g5p_r74f_kind { ++ G5P_R74F_DC3, ++ G5P_R74F_TA, ++ G5P_R74F_3D, /* r96: ring 0x64, doorbell 0x11, owned by the ALIVE decoder like the TA's */ ++}; ++ ++/* idle -> published (gate closed) -> open (reactivated) -> firing -> ++ * rang; published/open -> cancelled. FIRING lives under event_lock. ++ */ ++enum g5p_r74f_state { ++ G5P_R74F_IDLE, ++ G5P_R74F_PUBLISHED, ++ G5P_R74F_OPEN, ++ G5P_R74F_FIRING, ++ G5P_R74F_RANG, ++ G5P_R74F_CANCELLED, ++ G5P_R74F_STATES, ++}; ++ ++enum g5p_r74f_cancel { ++ G5P_R74F_CANCEL_NONE, ++ G5P_R74F_CANCEL_REACTIVATION, ++ G5P_R74F_CANCEL_NO_ALIVE, ++ G5P_R74F_CANCEL_CONSUMED, ++ G5P_R74F_CANCEL_PROTOCOL, ++ G5P_R74F_CANCEL_ALIVE_ADVANCED, ++ G5P_R74F_CANCEL_SELF_DRAIN, ++ G5P_R74F_CANCELS, ++}; ++ ++enum g5p_r74f_stop { ++ G5P_R74F_STOP_NONE, ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ G5P_R74F_STOP_MMU_CLOSED, ++ G5P_R74F_STOP_TA_REACTIVATION, ++ G5P_R74F_STOP_TA_NOT_KICKED, ++ G5P_R74G_STOP_DC3_TAGS, ++ G5P_R74G_STOP_DC3_RAM, ++ G5P_R74G_STOP_COMPANION, ++ G5P_R74G_STOP_TA_NOT_CONSUMED, ++ G5P_R74G_STOP_TA_WINDOW_MISSED, ++ G5P_R74F_STOPS, ++}; ++ ++enum g5p_r74f_path { ++ G5P_R74F_PATH_NONE, ++ G5P_R74F_PATH_IMMEDIATE, ++ G5P_R74F_PATH_DEFERRED, ++ G5P_R74F_PATHS, ++}; ++ ++/* One generation. Its owner writes the receipt, then releases state. */ ++struct g5p_r74f_bell { ++ atomic_t state; ++ u32 kind, generation, ring, start, target, path, cancel; ++ u32 arm_alive, open_alive, bell_alive; ++ u32 first_fence, first_state, fence, bell_state; ++ s32 first_ret, bell_ret, power_ret; ++ u32 reactivated, core_fence, gpc_fence, off_raw, on_raw; ++ u32 attempted, kicked, complete; ++ u32 consumer_before, consumer_after, producer_after; ++ u32 alive_done; ++ u64 arm_ns, power_start_ns, off_ns, on_ns, open_ns; ++ u64 bell_start_ns, bell_ns, done_ns; ++}; ++ ++enum g5p_r74g_grace { ++ G5P_R74G_GRACE_NONE, ++ G5P_R74G_GRACE_CONSUMED, ++ G5P_R74G_GRACE_ALIVES_END, ++ G5P_R74G_GRACE_WALL, ++ G5P_R74G_GRACE_FAULT, ++ G5P_R74G_GRACES, ++}; ++ ++/* The RAM-visible results of the firmware DC3 handler: phase 0 at the DC3 ++ * consume, phase 1 again just before the TA producer moves. ++ */ ++struct g5p_r74g_dc3 { ++ u64 pa[2], expected_pa, t_ns; ++ u32 slot[2], host[2], cached[2], rec[2][5]; ++ u32 start, target, consumer, producer, tag; ++ u32 failed, event_failed, companion, recoveries, classification, bad; ++}; ++ ++struct g5p_r73b { ++ struct device *dev; ++ struct g5p_boot *boot; ++ u32 case_id, watchdog_ms, dc_variant, meta6c8, r73e_case; ++ bool ack_first, aux, fast_enabled, fast_window; ++ bool early_release_ready, early_power_abort; ++ atomic_t early_once, early_busy; ++ u64 early_irq_ns, early_power_start_ns; ++ u64 early_off_ns, early_on_ns; ++ u32 early_state, early_after_state, early_power_raw; ++ s32 early_ret, early_power_ret; ++ /* wake112 (r74e112.h): the R74E proof deferred to the first-ALIVE owner. */ ++ atomic_t early_proof; ++ u32 early_proof_gen, early_proof_gen_now, early_proof_why; ++ s32 early_defer_ret; ++ /* boot_alive_settle_us: limit, first in-flight value seen, re-reads, time spent. */ ++ u32 settle_us, settle_first, settle_samples; ++ u64 settle_ns; ++ u64 fast_release_ns, fast_alive_ns, fast_submit_ns; ++ u32 fast_state; ++ s32 fast_ret; ++ /* nopatch: Apple's unmodified image; policy record +0/+4/+8/+c/+34 taken before the first bell. */ ++ bool nopatch; ++ u32 np_pre[5]; ++ atomic_t seq, acks, cap_seen, busy, drains, drain_errors, exported, ++ trace_saved; ++ atomic_t watchdog_state; /* 0 off, 1 armed, 2 disarmed, 3 fired, 4 terminal */ ++ int watchdog_cpu; ++ u32 owner; ++ struct mutex drain_lock; /* single auxiliary-ring owner */ ++ struct hrtimer watchdog; ++ struct g5p_r73b_row rows[G5P_R73B_ROWS]; ++ u32 trace_rows[G5P_R73B_TRACE_ROWS][G5P_R73B_TRACE_WORDS]; ++ u32 irq_cap, r74_case, r74_mask, r74_ticks, r74_word1_count; ++ u32 r74_word1[G5P_R74_WORD1]; ++ atomic_t r74_seq, r74_irq_refused; ++ struct g5p_r74_milestone r74_milestones[G5P_R74_MILESTONES]; ++ ++ /* r74f_alive and FIRING are written under event_lock only. */ ++ u32 r74f_case, r74f_alive, r74f_active, r74f_generation, r74f_stop; ++ ++ u32 r87b_attach; ++ s32 r74f_stop_value; ++ atomic_t r74f_bells; ++ struct g5p_r74f_bell r74f_bell[G5P_R74F_KINDS]; ++ ++ /* R74g: no host MMU pass, publish-only TA grace, one fallback bell. */ ++ u32 r74g_case, r74g_ack_skipped, r74g_grace_end, r74g_opened; ++ u32 r74g_grace_alive; ++ u64 r74g_grace_ns; ++ struct g5p_r74g_dc3 r74g_dc3[2]; ++ /* R74h: RAM-only trace tail and checkpoints; NULL when off. */ ++ u32 r74h_case; ++ ++ /* R80: firmware trace observer; NULL when off. */ ++ ++ /* Heartbeat116 (heartbeat116.h): the R74 window detector; IRQ thread only. */ ++ bool hb_on; ++ u32 hb_seed, hb_empty, hb_cross; ++ atomic_t hb_late; ++ struct g5p_hb_cfg hb_cfg; ++ struct g5p_hb hb; ++}; ++ ++/* The ctrl+0 word installed before release: the r80 stage's, else the r78 control's. */ ++static u32 g5p_boot_ctrl0(struct g5p_boot *b) ++{ ++ return b->r73b && false ? 0 : b->r78_ctrl0; ++} ++ ++/* boot-r80.inc, included by boot.inc after boot-r74h.inc. */ ++ ++/* render-r96.inc: the receipts at export (RAM only) */ ++ ++/* render-r94.inc: the RAM state of the record and the code BO at export, read only */ ++ ++/* boot-r80.inc: the two IRQ-owned plateau banks, read-only */ ++ ++/* render-r84b.inc: the late raw dump and the CPU visibility step, RAM only */ ++ ++static int g5p_boot_drain(struct g5p_diag *g); ++static u32 g5p_transport_event_limit(void); ++static int g5p_r73c_fast_control(struct g5p_diag *g); ++static int g5p_r73e_prepare(struct g5p_diag *g); ++static int g5p_r73e_irq_submit(struct g5p_diag *g); ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner); ++ ++static const char *g5p_r73b_name(u32 id) ++{ ++ return id == 1 ? "meta1-bounded-all-rings" : ++ id == 2 ? "meta0-bounded-all-rings-control" : ++ id == 3 ? "meta1-event-only-reference" : ++ "off"; ++} ++ ++static bool g5p_r73b_sampled(u32 n) ++{ ++ return n <= 16 || is_power_of_2(n); ++} ++ ++static u32 *g5p_r73b_ptr(struct g5p_boot *b, u32 off) ++{ ++ return (u32 *)(b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r73b_word(struct g5p_boot *b, u32 off) ++{ ++ return READ_ONCE(*g5p_r73b_ptr(b, off)); ++} ++ ++static u32 g5p_r73b_slot(u32 seq) ++{ ++ if (seq < G5P_R73B_HEAD) ++ return seq; ++ return G5P_R73B_HEAD + ++ (seq - G5P_R73B_HEAD) % (G5P_R73B_ROWS - G5P_R73B_HEAD); ++} ++ ++/* Lockless: a writer owns its slot by sequence; seq is published last. */ ++static void g5p_r73b_crumb(struct g5p_boot *b, u32 phase, u32 owner, int result) ++{ ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r73b_row *row; ++ u32 seq; ++ ++ if (!r) ++ return; ++ seq = atomic_inc_return(&r->seq) - 1; ++ row = &r->rows[g5p_r73b_slot(seq)]; ++ WRITE_ONCE(row->seq, 0); ++ /* Invalidate before rewriting so a reader never pairs old seq/new data. */ ++ smp_wmb(); ++ row->phase = phase; ++ row->t_ns = ktime_get_mono_fast_ns(); ++ row->irq = atomic_read(&b->irqs); ++ row->ack = atomic_read(&r->acks); ++ row->released = READ_ONCE(b->released); ++ row->owner = owner; ++ row->result = result; ++ row->cap_seen = atomic_read(&r->cap_seen); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r73b_row_slots); i++) { ++ u32 ctl = b->layout.offset[g5p_r73b_row_slots[i]]; ++ ++ row->cp[i][0] = g5p_r73b_word(b, ctl); ++ row->cp[i][1] = g5p_r73b_word(b, ctl + 0x10); ++ } ++ /* Publish the row contents before its sequence. */ ++ smp_wmb(); ++ WRITE_ONCE(row->seq, seq + 1); ++} ++ ++static const char *g5p_r74_name(u32 id) ++{ ++ return id == 1 ? "forced-ready-meta1-natural-ta" : ++ id == 2 ? "meta0-unpatched-natural-ta-control" : ++ "off"; ++} ++ ++static const char *g5p_r74_step_name(u32 step) ++{ ++ return step < G5P_R74_STEPS ? g5p_r74_steps[step] : "unknown"; ++} ++ ++static u32 g5p_r74_case(struct g5p_boot *b) ++{ ++ return b && b->r73b ? 1 : 0; ++} ++ ++/* R74b case 1: after cmd8 retirement, optional host MMIO must not race the ++ * firmware policy loop power-gating RGX. Functional setup/kick sites opt in ++ * explicitly; this predicate gates observations only. ++ */ ++static bool g5p_r74b_ram_only(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ ++ return g5p_r74_case(b) == 1 && b && READ_ONCE(b->released) && ++ READ_ONCE(b->r72_meta6c8); ++} ++ ++/* Process context only, never per IRQ: one console line per r74 step, so a ++ * whole-SoC freeze is localized by the last line netconsole delivered. ++ */ ++static void g5p_r74_mark(struct g5p_diag *g, u32 step, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_r74_milestone m; ++ u32 seq; ++ ++ if (!r || !1) ++ return; ++ m.t_ns = ktime_get_ns(); ++ m.step = step; ++ m.ret = ret; ++ m.irqs = atomic_read(&b->irqs); ++ m.events = READ_ONCE(b->events); ++ m.dc[0] = g5p_r73b_word(b, b->layout.offset[0x6c]); ++ m.dc[1] = g5p_r73b_word(b, b->layout.offset[0x6c] + 0x10); ++ m.ta[0] = g5p_r73b_word(b, b->layout.offset[0x60]); ++ m.ta[1] = g5p_r73b_word(b, b->layout.offset[0x60] + 0x10); ++ seq = atomic_inc_return(&r->r74_seq) - 1; ++ if (seq < G5P_R74_MILESTONES) ++ r->r74_milestones[seq] = m; ++ dev_emerg( ++ g->dev, ++ "R74 milestone seq=%u step=%s ret=%d irqs=%u events=%u dc=%u/%u ta=%u/%u " ++ "t_ns=%llu end=r74\n", ++ seq, g5p_r74_step_name(step), ret, m.irqs, m.events, m.dc[0], ++ m.dc[1], m.ta[0], m.ta[1], m.t_ns); ++} ++ ++/* R74 keeps repeated ALIVEs quiet whatever their word1: each new value is ++ * printed once (eight at most), so a moved loop cannot flood the console. ++ */ ++static bool g5p_r74_word1(struct g5p_boot *b, u32 word1) ++{ ++ struct g5p_r73b *r = b->r73b; ++ ++ ; ++ ; ++ for (u32 i = 0; i < r->r74_word1_count; i++) ++ if (r->r74_word1[i] == word1) ++ return false; ++ if (r->r74_word1_count == G5P_R74_WORD1) ++ return false; ++ r->r74_word1[r->r74_word1_count++] = word1; ++ return true; ++} ++ ++static bool g5p_r73b_copy_row(struct g5p_r73b *r, u32 seq, ++ struct g5p_r73b_row *out) ++{ ++ struct g5p_r73b_row *row = &r->rows[g5p_r73b_slot(seq)]; ++ ++ if (READ_ONCE(row->seq) != seq + 1) ++ return false; ++ /* Pairs with the writer's publication barrier. */ ++ smp_rmb(); ++ *out = *row; ++ /* Re-check that the slot was not recycled while copying. */ ++ smp_rmb(); ++ return READ_ONCE(row->seq) == seq + 1; ++} ++ ++static int g5p_r73b_format(char *buf, size_t len, ++ const struct g5p_r73b_row *row) ++{ ++ u32 phase = row->phase < G5P_R73B_PHASES ? row->phase : 0; ++ ++ return scnprintf( ++ buf, len, ++ "R73B crumb seq=%u t_ns=%llu phase=%s irq=%u ack=%u released=%u event=%u/%u " ++ "log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d cap=%u end=r73b\n", ++ row->seq - 1, row->t_ns, g5p_r73b_phase_names[phase], row->irq, ++ row->ack, row->released, row->cp[0][0], row->cp[0][1], ++ row->cp[1][0], row->cp[1][1], row->cp[2][0], row->cp[2][1], ++ row->cp[3][0], row->cp[3][1], row->cp[4][0], row->cp[4][1], ++ row->owner, row->result, row->cap_seen); ++} ++ ++/* Snapshot the producer once and consume at most that modular distance ++ * (<= 255). One consumer store returns the whole batch; a producer that ++ * advances meanwhile is left for the next pass, never chased. ++ */ ++static int g5p_r73b_drain_ring(struct g5p_boot *b, u32 ctl_slot, u32 lane, ++ u32 data_slot, u32 stride, u32 ring, ++ u32 *consumed) ++{ ++ u32 ctl = b->layout.offset[ctl_slot] + lane * 0x20; ++ u32 data = b->layout.offset[data_slot] + ++ (ctl_slot == 0x71 ? lane * 0x2c00 : 0); ++ u32 consumer = g5p_r73b_word(b, ctl); ++ u32 producer = g5p_r73b_word(b, ctl + 0x10); ++ u32 count, hash = b->r73_aux_hash[ring] ?: 2166136261U; ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ count = (producer - consumer) & 255; ++ if (!count) ++ return 0; ++ /* Producer index before the slot payloads it covers. */ ++ dma_rmb(); ++ for (u32 n = 0; n < count; n++) { ++ u32 words[G5P_R73B_TRACE_WORDS] = { 0 }; ++ u32 off = data + consumer * stride; ++ ++ for (u32 i = 0; i < stride / 4; i++) { ++ u32 value = g5p_r73b_word(b, off + i * 4); ++ ++ if (i < ARRAY_SIZE(words)) ++ words[i] = value; ++ hash = (hash ^ value) * 16777619U; ++ } ++ if (ring == 5 && ++ atomic_read(&b->r73b->trace_saved) < G5P_R73B_TRACE_ROWS) { ++ u32 saved = atomic_read(&b->r73b->trace_saved); ++ ++ memcpy(b->r73b->trace_rows[saved], words, ++ sizeof(words)); ++ /* Publish this complete row before increasing the visible count. */ ++ smp_wmb(); ++ atomic_inc(&b->r73b->trace_saved); ++ }; ++ ; ++ if (!b->r73_aux_entries[ring]) ++ memcpy(b->r73_aux_first[ring], words, ++ sizeof(b->r73_aux_first[ring])); ++ memcpy(b->r73_aux_last[ring], words, ++ sizeof(b->r73_aux_last[ring])); ++ b->r73_aux_entries[ring]++; ++ consumer = (consumer + 1) & 255; ++ } ++ b->r73_aux_hash[ring] = hash; ++ /* dma_wmb() ordering plus the slot loads above: Apple returns the ++ * consumer only after its reads, with a full barrier. ++ */ ++ dma_mb(); ++ WRITE_ONCE(*g5p_r73b_ptr(b, ctl), consumer); ++ *consumed += count; ++ return 0; ++} ++ ++/* Apple order after the ACK: four log lanes, events, statistics, trace. */ ++static int g5p_r73b_aux_pass(struct g5p_diag *g, u32 *consumed) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret = 0; ++ ++ for (u32 lane = 0; lane < 4 && !ret; lane++) ++ ret = g5p_r73b_drain_ring(b, 0x71, lane, 0x75, 44, lane, ++ consumed); ++ if (!ret) ++ ret = g5p_boot_drain(g); ++ /* R80: ring 4 has one consumer, the drain thread (retains every record). */ ++ if (!ret && !false) ++ ret = g5p_r73b_drain_ring(b, 0x72, 0, 0x76, 44, 4, consumed); ++ if (!ret) ++ ret = g5p_r73b_drain_ring(b, 0x73, 0, 0x77, 36, 5, consumed); ++ return ret; ++} ++ ++static bool g5p_r73b_pending(struct g5p_boot *b, bool aux) ++{ ++ static const u8 slots[] = { 0x70, 0x71, 0x72, 0x73 }; ++ ++ for (u32 i = 0; i < (aux ? ARRAY_SIZE(slots) : 1); i++) { ++ for (u32 lane = 0; lane < (slots[i] == 0x71 ? 4 : 1); lane++) { ++ u32 ctl = b->layout.offset[slots[i]] + lane * 0x20; ++ ++ if (g5p_r73b_word(b, ctl) != ++ g5p_r73b_word(b, ctl + 0x10)) ++ return true; ++ } ++ } ++ return false; ++} ++ ++/* The IRQ thread only trylocks the auxiliary rings; a miss still runs the ++ * spinlock-protected event decoder. Process-context paths lock normally. ++ * Drains with work, errors or sampled entries record before/after rows. ++ */ ++static int g5p_r73b_service(struct g5p_diag *g, u32 owner, bool sampled) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 consumed = 0, before; ++ bool rows, aux; ++ int ret; ++ ++ if (!r) ++ return g5p_boot_drain(g); ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* The atomic r73c window services only the event ring. Auxiliary ++ * traffic is retained until cmd8 is either published or refused. */ ++ aux = 1 && !READ_ONCE(r->fast_window); ++ if (aux && owner == G5P_R73B_OWNER_IRQ && ++ !mutex_trylock(&r->drain_lock)) { ++ atomic_inc(&r->busy); ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BUSY, READ_ONCE(r->owner), ++ -EBUSY); ++ ret = g5p_boot_drain(g); ++ goto out; ++ } ++ if (aux && owner != G5P_R73B_OWNER_IRQ) ++ mutex_lock(&r->drain_lock); ++ if (aux) ++ WRITE_ONCE(r->owner, owner); ++ atomic_inc(&r->drains); ++ rows = sampled || g5p_r73b_pending(b, aux); ++ if (rows) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BEFORE, owner, 0); ++ before = READ_ONCE(b->events); ++ ret = aux ? g5p_r73b_aux_pass(g, &consumed) : g5p_boot_drain(g); ++ consumed += READ_ONCE(b->events) - before; ++ if (rows || ret) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_AFTER, owner, ret ?: consumed); ++ if (aux) { ++ WRITE_ONCE(r->owner, G5P_R73B_NONE); ++ mutex_unlock(&r->drain_lock); ++ } ++out: ++ if (ret) { ++ atomic_inc(&r->drain_errors); ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return ret; ++} ++ ++/* Heartbeat116: an R74 epoch bounds IRQ174 and its ring entries per window. */ ++static bool g5p_hb116_on(struct g5p_boot *b) ++{ ++ return b->r73b && b->r73b->hb_on; ++} ++ ++/* Auxiliary-ring entries consumed so far (RAM, monotonic, may wrap). */ ++static u32 g5p_hb116_aux(struct g5p_boot *b) ++{ ++ u32 sum = 0; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(b->r73_aux_entries); i++) ++ sum += READ_ONCE(b->r73_aux_entries[i]); ++ return sum; ++} ++ ++static u32 g5p_hb116_progress(struct g5p_boot *b) ++{ ++ return READ_ONCE(b->events) + g5p_hb116_aux(b); ++} ++ ++/* The lifetime cap's terminal stop, once: final row, cap_seen, failed, one disable. */ ++static void g5p_hb116_trip(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb *h = &r->hb; ++ u32 prior = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (atomic_cmpxchg(&r->cap_seen, 0, 1)) ++ return; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "HB116 trip why=%s prior_failed=%u windows=%u stalled=%u ms=%llu irqs=%u " ++ "events=%u acks=%u empty=%u aux=%u total_irqs=%u total_events=%u t_ns=%llu " ++ "stop=masked late=ack-only end=hb116\n", ++ g5p_hb_why_name[h->why < G5P_HBW_N ? h->why : 0], prior, ++ h->windows, h->stalled, h->trip.ns / NSEC_PER_MSEC, ++ h->trip.irqs, h->trip.events, h->trip.acks, h->trip.empty, ++ h->trip.aux, n, READ_ONCE(b->events), h->trip_ns); ++} ++ ++/* Receipts only: the first window past each Wake112 lifetime bound; nothing trips here. */ ++static void g5p_hb116_cross(struct g5p_diag *g, u32 irqs, u32 events) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ u32 limit = g5p_transport_event_limit(); ++ ++ if (!(r->hb_cross & 1) && irqs > r->irq_cap) { ++ r->hb_cross |= 1; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=irq-cap value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ r->irq_cap, irqs, events, r->hb.windows); ++ } ++ if (!(r->hb_cross & 2) && events >= limit) { ++ r->hb_cross |= 2; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=event-limit value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ limit, irqs, events, r->hb.windows); ++ } ++} ++ ++/* After the entry's ACK and service: account it, close a window, or trip. */ ++static void g5p_hb116_irq(struct g5p_diag *g, int irq, u32 n, u32 before) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb_win *w = &r->hb.last; ++ u32 seed = r->hb_seed, events = READ_ONCE(b->events); ++ struct g5p_hb_in in; ++ ++ if (g5p_hb116_progress(b) == before) ++ r->hb_empty++; ++ in.now_ns = ktime_get_mono_fast_ns(); ++ in.irqs = n + seed; ++ in.events = events + seed; ++ in.acks = atomic_read(&r->acks) + seed; ++ in.empty = r->hb_empty + seed; ++ in.aux = g5p_hb116_aux(b) + seed; ++ switch (g5p_hb_step(&r->hb, &r->hb_cfg, &in)) { ++ case G5P_HBR_OPEN: ++ case G5P_HBR_CLOSED: ++ break; ++ case G5P_HBR_TRIP: ++ g5p_hb116_trip(g, irq, n); ++ return; ++ } ++ g5p_hb116_cross(g, n, events); ++ if (g5p_hb_report(&r->hb)) ++ dev_info( ++ g->dev, ++ "HB116 window n=%u t_ns=%llu ms=%llu irqs=%u events=%u acks=%u empty=%u aux=%u " ++ "stalled=%u peak_irqs=%u peak_events=%u total_irqs=%u total_events=%u " ++ "total_acks=%d end=hb116\n", ++ r->hb.windows, in.now_ns, w->ns / NSEC_PER_MSEC, ++ w->irqs, w->events, w->acks, w->empty, w->aux, ++ r->hb.stalled, r->hb.peak_irqs, r->hb.peak_events, n, ++ events, atomic_read(&r->acks)); ++} ++ ++/* Every entry ACKs. Over the cap: ACK, final row, cap_seen, one disable; ++ * the kernel stays alive so the runner can export the record. ++ * Heartbeat116: an R74 epoch has no lifetime cap. The window detector runs ++ * after the ACK and the service; after its trip an entry only ACKs. ++ */ ++static irqreturn_t g5p_r73b_irq(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ bool sampled = g5p_r73b_sampled(n); ++ u32 before = 0; ++ ++ if (r->hb_on && atomic_read(&r->cap_seen)) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ atomic_inc(&r->hb_late); ++ return IRQ_HANDLED; ++ } ++ if (!r->hb_on && n > r->irq_cap) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ if (!atomic_read(&r->cap_seen)) { ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (!atomic_cmpxchg(&r->cap_seen, 0, 1)) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ } ++ } ++ return IRQ_HANDLED; ++ } ++ { ++ /* KPF level6G: Apple ACKs the AIC meta interrupt before ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ } ++ if (sampled) ++ g5p_r73b_crumb(b, G5P_R73B_IRQ, G5P_R73B_OWNER_IRQ, n); ++ if (r->hb_on) ++ before = g5p_hb116_progress(b); ++ if (READ_ONCE(b->released)) ++ g5p_r73b_service(g, G5P_R73B_OWNER_IRQ, sampled); ++ if (1 && READ_ONCE(b->alive)) ++ g5p_r73e_alive_submit(g, G5P_R73B_OWNER_IRQ); ++ ; ++ ; ++ ; ++ ; ++ ; ++ if (r->hb_on) ++ g5p_hb116_irq(g, irq, n, before); ++ return IRQ_HANDLED; ++} ++ ++static void g5p_r73b_disarm(struct g5p_r73b *r) ++{ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 2) == 1) ++ hrtimer_cancel(&r->watchdog); ++} ++ ++static void g5p_r73b_stop(void *data) ++{ ++ g5p_r73b_disarm(data); ++} ++ ++/* Cached CPU RAM only: no MMIO, no locks, then a deliberate panic so a ++ * silent hang becomes a pstore/ramoops record. ++ */ ++static enum hrtimer_restart g5p_r73b_watchdog_fire(struct hrtimer *timer) ++{ ++ struct g5p_r73b *r = container_of(timer, struct g5p_r73b, watchdog); ++ struct g5p_boot *b = r->boot; ++ struct g5p_r73b_row row; ++ u32 total = atomic_read(&r->seq), first, last; ++ char line[320]; ++ bool found = false; ++ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 3) != 1) ++ return HRTIMER_NORESTART; ++ /* A contained cap or a latched failure is terminal: the IRQ only ACKs, ++ * service returns -EIO and the runner sees failed=1. Keep that host ++ * alive for the export instead of turning it into a deliberate panic. ++ */ ++ if (atomic_read(&r->cap_seen) || READ_ONCE(b->failed)) { ++ atomic_set(&r->watchdog_state, 4); ++ dev_emerg( ++ r->dev, ++ "R73B watchdog terminal case=%u irqs=%d acks=%d cap_seen=%d failed=%u rows=%u " ++ "no-panic end=r73b\n", ++ r->case_id, atomic_read(&b->irqs), ++ atomic_read(&r->acks), atomic_read(&r->cap_seen), ++ READ_ONCE(b->failed), total); ++ return HRTIMER_NORESTART; ++ } ++ dev_emerg( ++ r->dev, ++ "R73B receipt case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u release=%s end=r73b\n", ++ r->case_id, g5p_r73b_name(r->case_id), r->ack_first, r->aux, ++ r->irq_cap, r->watchdog_ms, r->dc_variant, r->meta6c8, ++ init_utsname()->release); ++ first = total > G5P_R73B_PANIC_ROWS ? total - G5P_R73B_PANIC_ROWS : 0; ++ for (u32 seq = first; seq < total; seq++) { ++ if (!g5p_r73b_copy_row(r, seq, &row)) ++ continue; ++ g5p_r73b_format(line, sizeof(line), &row); ++ dev_emerg(r->dev, "%s", line); ++ } ++ for (last = total; last > first && !found; last--) ++ found = g5p_r73b_copy_row(r, last - 1, &row); ++ if (!found) ++ memset(&row, 0, sizeof(row)); ++ panic("G5P R73B watchdog: release epoch incomplete after %ums case=%u irqs=%d acks=%d " ++ "cap_seen=%d busy=%d rows=%u last seq=%u phase=%s irq=%u ack=%u released=%u " ++ "event=%u/%u log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d " ++ "cap=%u end=r73b\n", ++ 0, 1, atomic_read(&b->irqs), atomic_read(&r->acks), ++ atomic_read(&r->cap_seen), atomic_read(&r->busy), total, ++ row.seq ? row.seq - 1 : 0, ++ g5p_r73b_phase_names[row.phase < G5P_R73B_PHASES ? row.phase : 0], ++ row.irq, row.ack, row.released, row.cp[0][0], row.cp[0][1], ++ row.cp[1][0], row.cp[1][1], row.cp[2][0], row.cp[2][1], ++ row.cp[3][0], row.cp[3][1], row.cp[4][0], row.cp[4][1], row.owner, ++ row.result, row.cap_seen); ++ return HRTIMER_NORESTART; ++} ++ ++static void g5p_r73b_release_entry(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ ++ if (!r) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_ENTRY, G5P_R73B_NONE, 0); ++ { ++ return; ++ } ++} ++ ++static void g5p_r73b_release_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_EXIT, G5P_R73B_NONE, ret); ++ if (ret) ++ g5p_r73b_disarm(b->r73b); ++} ++ ++/* The release epoch ends when the host has returned from the ALIVE wait. */ ++static void g5p_r73b_events_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_EVENTS_EXIT, G5P_R73B_OWNER_EVENTS, ret); ++ g5p_r73b_disarm(b->r73b); ++} ++ ++static int g5p_r73b_latch(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const u32 id = (1), ms = (0); ++ const u32 fast = (1); ++ const u32 early = (1); ++ const u32 r74 = (1), cap = (1048576); ++ const u32 r74f = (1); ++ const u32 r74g = (1); ++ const u32 r74h = (0); ++ ++ struct g5p_r73b *r; ++ int ret; ++ ++ ; ++ ; ++ ; ++ ; ++ ; ++ ; ++ /* nopatch replaces r73f case 1 under the whole r74/r74f/r74g path, and nothing else. */ ++ /* nopatch's receipts and its r74 gate are the TYPED_V3 path's; any other path (r74 0) runs ++ * Apple's image anyway (only r73f patches it) and gets neither. ++ */ ++ if (1 != g5p_fw_unpatched_want((1), 0, 0, 0) || ++ (1 && r74 == 1 && (0 || r74f != 1 || r74g != 1))) { ++ dev_err(g->dev, ++ "NOPATCH reject phase=map unpatched=%u/%u r73f_case=%u path=%u/%u/%u end=nopatch\n", ++ 1, (1), 0, r74, r74f, r74g); ++ return -EINVAL; ++ } ++ /* R74 rides only on an exact r73f case 1 (or nopatch) or 4 with the raised limits. */ ++ if (r74 && (!id || r74 > 2 || 0 != (r74 == 1 ? (1 ? 0 : 1) : 4) || ++ 1 != (r74 == 1) || !(1) || cap < G5P_R74_IRQ_CAP || ++ cap > G5P_R74_IRQ_CAP_MAX || ++ g5p_transport_event_limit() < G5P_R74_EVENT_FLOOR)) { ++ dev_err(g->dev, ++ "R74 reject phase=map case=%u r73b_case=%u r73f_case=%u meta6c8=%u " ++ "render_engine=%u irq_cap=%u event_limit=%u end=r74\n", ++ r74, id, 0, b->r72_meta6c8, (1), cap, ++ g5p_transport_event_limit()); ++ return -EINVAL; ++ } ++ ++ /* R74f rides only on the exact r74 case 1 forced-ready epoch. */ ++ ; ++ ++ /* R74g rides only on the exact r74f case 1 epoch. */ ++ ; ++ ++ /* R74h observes only the exact r74g case 1 epoch. */ ++ ; ++ ++ /* The bounded serialized drain supersedes r73's IRQ-context drain. */ ++ ; ++ ; ++ ; ++ ; ++ if (id > 3 || fast > 1 || (fast && id == 3) || early > 2 || ++ (early && (!fast || id != early)) || b->r73_aux_drain || 7 != 7 || ++ 1 != (id == 2 ? 0 : 1) || (ms && (ms < 1000 || ms > 30000))) ++ return -EINVAL; ++ r = devm_kzalloc(g->dev, sizeof(*r), GFP_KERNEL); ++ if (!r) ++ return -ENOMEM; ++ r->dev = g->dev; ++ r->boot = b; ++ r->case_id = id; ++ r->watchdog_ms = ms; ++ r->dc_variant = 7; ++ r->meta6c8 = 1; ++ r->aux = id != 3; ++ r->ack_first = true; ++ r->fast_enabled = fast; ++ r->r73e_case = early; ++ r->r74_case = r74; ++ r->r74f_case = r74f; ++ r->r74g_case = r74g; ++ r->r74h_case = r74h; ++ r->settle_us = min_t(u32, READ_ONCE(boot_alive_settle_us), 5000); ++ r->nopatch = 1 && r74 == 1; ++ r->irq_cap = r74 ? cap : G5P_R73B_IRQ_CAP; ++ /* Heartbeat116: in an R74 epoch the window detector replaces both lifetime bounds. */ ++ r->hb_on = r74 != 0; ++ r->hb_seed = r74 ? (0) : 0; ++ g5p_hb_cfg_default(&r->hb_cfg); ++ atomic_set(&r->early_once, 0); ++ atomic_set(&r->early_busy, 0); ++ atomic_set(&r->early_proof, G5P_PX_NONE); ++ r->watchdog_cpu = -1; ++ mutex_init(&r->drain_lock); ++ hrtimer_setup(&r->watchdog, g5p_r73b_watchdog_fire, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL_PINNED_HARD); ++ ret = devm_add_action_or_reset(g->dev, g5p_r73b_stop, r); ++ if (ret) ++ return ret; ++ b->r73_aux_drain = 1; ++ b->r73b = r; ++ dev_info( ++ g->dev, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), r->ack_first, r->aux, r->irq_cap, ms, ++ r->dc_variant, r->meta6c8, G5P_R73B_ROWS, ++ init_utsname()->release); ++ g5p_blob_printf( ++ g, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), 1, 1, r->irq_cap, ms, 7, 1, ++ G5P_R73B_ROWS, init_utsname()->release); ++ g5p_blob_printf(g, "R73C latched fast=%u case=%u meta6c8=%u end=r73c\n", ++ 1, id, 1); ++ dev_info(g->dev, "R73E latched case=%u meta6c8=%u fast=%u end=r73e\n", ++ early, r->meta6c8, r->fast_enabled); ++ ; ++ ; ++ if (r->nopatch) ++ dev_info( ++ g->dev, ++ "NOPATCH latched fw=apple-original r73f_case=0 meta6c8=%u r73e_case=%u r74=%u " ++ "r74f=%u r74g=%u force_policy_ready=0 end=nopatch\n", ++ r->meta6c8, early, r74, r74f, r74g); ++ { ++ dev_info( ++ g->dev, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, r->meta6c8, ++ r->irq_cap, g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, 1, r->irq_cap, ++ g5p_transport_event_limit()); ++ dev_info( ++ g->dev, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ g5p_blob_printf( ++ g, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ g5p_blob_printf( ++ g, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ }; ++ /* TYPED_V3 production path: the self-drain DC3 attach policy of r87b rides on r74f + r74g ++ * alone, with none of the r74h / r80 instrumentation. r74g has the firmware own the first ++ * MMU pass, which is what makes self-drain safe (r87b-attach.h); r74h only observes. Every ++ * other r8x / r9x knob is still refused below without boot_r80_case. ++ */ ++ { ++ r->r87b_attach = G5P_R87B_ATTACH_SELF_DRAIN; ++ dev_info( ++ g->dev, ++ "R87B latched policy=1 name=self-drain r74f=%u r74g=%u r74h=%u r80=0 " ++ "production_path=1 end=r87b\n", ++ r74f, r74g, r74h); ++ } ++ { ++ { ++ { ++ { ++ { ++ if ((1) && !r->r87b_attach) { ++ dev_err(g->dev, ++ "R87B reject phase=map r87b knob without boot_r80_case policy=%u end=r87b\n", ++ (1)); ++ return -EBADMSG; ++ } else { ++ { ++ { ++ ; ++ ; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ } ++ return 0; ++} ++ ++/* Process context, cached CPU RAM only; never waits for the command lock. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +new file mode 100644 +index 000000000..71cf79dbe +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -0,0 +1,532 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74f deferred doorbell. Included by boot.inc before the event decoder. ++ * Diagnostic bypass on the r74 forced-ready epoch only; not power policy. ++ * ++ * A DC3 pair or a natural TA is published once, append-only, before this ++ * code runs; its producer is never rewound, republished or torn down. If ++ * the immediate functional gate finds GPC inactive, that generation stays ++ * pending (never b->failed), the host repeats the r74e release transition ++ * once, and the first type-0 ALIVE decoded afterwards owns one doorbell ++ * attempt. The owner runs under event_lock: producer confirm, R1+14 fence, ++ * active check, dma_wmb(), doorbell 0x10. Receipts stay in RAM; console ++ * lines are printed afterwards from process context. ++ */ ++ ++static const char *const g5p_r74f_kinds[G5P_R74F_KINDS] = { "dc3", "ta", "3d" }; ++ ++static const char *const g5p_r74f_states[G5P_R74F_STATES] = { ++ "idle", "published", "open", "firing", "rang", "cancelled", ++}; ++ ++static const char *const g5p_r74f_cancels[G5P_R74F_CANCELS] = { ++ "none", "reactivation-failed", ++ "no-alive", "consumed-before-bell", ++ "protocol", "alive-during-reactivation", ++ "self-drain", ++}; ++ ++static const char *const g5p_r74f_stops[G5P_R74F_STOPS] = { ++ "none", ++ "attach-reactivation-failed", ++ "attach-published-not-kicked", ++ "attach-kicked-not-consumed", ++ "first-mmu-window-closed", ++ "ta-reactivation-failed", ++ "ta-published-not-kicked", ++ "firmware-dc3-tag-mismatch", ++ "dc3-ram-validation-refused", ++ "firmware-companion-recovery", ++ "ta-published-not-consumed", ++ "ta-window-missed", ++}; ++ ++static const char *const g5p_r74f_paths[G5P_R74F_PATHS] = { ++ "none", ++ "immediate", ++ "deferred", ++}; ++ ++static bool g5p_r74f_on(struct g5p_boot *b) ++{ ++ return b && b->r73b && 1; ++} ++ ++static u32 *g5p_r74f_ctl(struct g5p_boot *b, u32 kind) ++{ ++ return (u32 *)g5p_boot_shared(b, kind == G5P_R74F_TA ? 0x60 : ++ kind == G5P_R74F_3D ? 0x64 : ++ 0x6c); ++} ++ ++/* Owner only, event_lock held with interrupts off. Between the active check ++ * and the doorbell there is only dma_wmb(): no console, no other MMIO. ++ */ ++static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ u32 *fence, u32 *state) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 consumer = READ_ONCE(ctl[0]); ++ u32 producer = smp_load_acquire(&ctl[4]); ++ u32 queued = (f->target - f->start) & 255; ++ u32 progressed = (consumer - f->start) & 255; ++ u32 expected = f->kind == G5P_R74F_DC3 ? 2 : 1; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* Do not turn corrupted modulo-256 ring state into a doorbell. */ ++ if (consumer > 255 || producer > 255 || queued != expected || ++ progressed > queued) ++ return -EPROTO; ++ /* Acquire: pairs with the producer store published before the state. */ ++ if (producer != f->target) ++ return -EPROTO; ++ if (consumer == f->target) ++ return -EALREADY; ++ writel(7, g->window[1] + 0x14); ++ *fence = readl(g->window[1] + 0x14); ++ if (*fence & ~7U) ++ return -EPROTO; ++ *state = readl(g->window[1] + 0x1000); ++ /* perf60 (gpc_any_pstate): a GPC request in flight or a not-yet-awake GPC after the wake ++ * above; re-read for up to 2 ms, as Apple's setGPCPerformanceState waits for bit 4. Reads only. ++ */ ++ { ++ u64 t0 = ktime_get_mono_fast_ns(); ++ ++ while (!g5p_gpc_ok(*state) && ktime_get_mono_fast_ns() - t0 < ++ 2000ULL * NSEC_PER_USEC) { ++ udelay(10); ++ *state = readl(g->window[1] + 0x1000); ++ } ++ } ++ if (!g5p_gpc_ok(*state)) ++ return -EHOSTDOWN; ++ dma_wmb(); ++ writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); ++ return 0; ++} ++ ++/* Owner only: record the attempt, then publish it with the state release. */ ++static void g5p_r74f_rang(struct g5p_r73b *r, struct g5p_r74f_bell *f, int ret, ++ u32 fence, u32 state, u32 path) ++{ ++ f->bell_ns = ktime_get_mono_fast_ns(); ++ f->path = path; ++ f->fence = fence; ++ f->bell_state = state; ++ f->bell_ret = ret; ++ if (!ret) { ++ f->kicked = 1; ++ atomic_inc(&r->r74f_bells); ++ } else if (ret == -EALREADY) { ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ } else if (ret != -EHOSTDOWN) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ } ++ atomic_set_release(&f->state, G5P_R74F_RANG); ++} ++ ++/* g5p_boot_drain(), event_lock held: one type-0 ALIVE was just decoded. */ ++static void g5p_r74f_alive(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ struct g5p_r74f_bell *f; ++ u32 kind, fence = U32_MAX, state = U32_MAX; ++ int ret; ++ ++ WRITE_ONCE(r->r74f_alive, r->r74f_alive + 1); ++ kind = READ_ONCE(r->r74f_active); ++ if (!kind) ++ return; ++ f = &r->r74f_bell[kind - 1]; ++ if (atomic_read_acquire(&f->state) != G5P_R74F_OPEN || ++ r->r74f_alive <= f->open_alive) ++ return; ++ /* Sole owner: exactly one decoder moves OPEN to FIRING. */ ++ if (atomic_cmpxchg(&f->state, G5P_R74F_OPEN, G5P_R74F_FIRING) != ++ G5P_R74F_OPEN) ++ return; ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_DEFERRED); ++} ++ ++static int g5p_r74f_poll(struct g5p_diag *g, u32 alive, u32 expected, ++ u32 timeout_us, u64 *elapsed_ns, u32 *raw) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u64 start = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ *raw = U32_MAX; ++ for (;;) { ++ if (READ_ONCE(g->boot->failed)) { ++ ret = -EIO; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ /* Serialize the ordinal check and exactly one R1 read with the ++ * decoder. Once ALIVE advances, no further R1 MMIO is licensed. ++ */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ break; ++ } ++ *raw = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if ((*raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ return ret; ++} ++ ++/* Absolute freshness budget from the DC3 doorbell. The first MMU pass may ++ * touch R0 only while the same active window is still demonstrably fresh. ++ */ ++ ++/* The r74e release transition, once per generation, on r74f receipts and ++ * without console output: core request clear, GPC 0 within 1.0 ms, GPC 1 ++ * within 1.8 ms. Every R1 access is serialized against ALIVE decoding; ++ * ordinal advance cancels the generation before any further R1 MMIO. ++ */ ++static int g5p_r74f_reactivate(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u32 alive = f->arm_alive; ++ int ret; ++ ++ f->reactivated = 1; ++ f->power_start_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ f->power_ret = -EHOSTDOWN; ++ return -EHOSTDOWN; ++ } ++ writel(0, g->window[1] + 0x20000); ++ f->core_fence = readl(g->window[1] + 0x20000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, 0, G5P_R74F_OFF_US, &f->off_ns, ++ &f->off_raw); ++ if (!ret) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ goto out; ++ } ++ /* boot_gpu_perf_seed: the firmware believes pstate q; never drop the GPC under it */ ++ writel(0x10 | (b->perf_poweron ?: ++ b->perf_seed ?: ++ 1), ++ g->window[1] + 0x1000); ++ f->gpc_fence = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, ++ b->perf_poweron ?: ++ b->perf_seed ?: ++ 1, ++ G5P_R74F_ON_US, &f->on_ns, &f->on_raw); ++ } ++out: ++ f->power_ret = ret; ++ return ret; ++} ++ ++/* Process context, before any publication: no overlap with a pending ring. */ ++static int g5p_r74f_admit(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *dc = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ ++ if (!g5p_r74b_ram_only(g) || READ_ONCE(b->failed) || ++ READ_ONCE(r->r74f_stop) || ++ atomic_read(&r->r74f_bell[kind].state) != G5P_R74F_IDLE) ++ return -EPERM; ++ if (kind == G5P_R74F_DC3) ++ return READ_ONCE(r->r74f_active) ? -EBUSY : 0; ++ /* M7: the TA wakeup never shares a doorbell with a pending DC3 pair. */ ++ dma_rmb(); ++ if (READ_ONCE(r->r74f_active) != G5P_R74F_DC3 + 1 || ++ !r->r74f_bell[G5P_R74F_DC3].complete || ++ READ_ONCE(dc[0]) != READ_ONCE(dc[4])) ++ return -EBUSY; ++ return 0; ++} ++ ++static void g5p_r74f_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 state = atomic_read_acquire(&f->state); ++ char line[3][512]; ++ ++ dma_rmb(); ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74F bell kind=%s generation=%u ring=%02x start=%u target=%u state=%s path=%s " ++ "first_ret=%d first_fence=%08x first_state=%08x attempted=%u fence=%08x " ++ "bell_state=%08x bell_ret=%d kicked=%u bells=%d cancel=%s complete=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->ring, f->start, ++ f->target, g5p_r74f_states[state < G5P_R74F_STATES ? state : 0], ++ g5p_r74f_paths[f->path], f->first_ret, f->first_fence, ++ f->first_state, f->attempted, f->fence, f->bell_state, ++ f->bell_ret, f->kicked, atomic_read(&b->r73b->r74f_bells), ++ g5p_r74f_cancels[f->cancel], f->complete); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74F ring kind=%s generation=%u arm_alive=%u open_alive=%u bell_alive=%u " ++ "alive_now=%u consumer_before=%u consumer_after=%u producer_after=%u " ++ "consumer_now=%u producer_now=%u failed=%u event_failed=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->arm_alive, ++ f->open_alive, f->bell_alive, READ_ONCE(b->r73b->r74f_alive), ++ f->consumer_before, f->consumer_after, f->producer_after, ++ READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), READ_ONCE(b->failed), ++ READ_ONCE(b->event_failed)); ++ scnprintf( ++ line[2], sizeof(line[2]), ++ "R74F power kind=%s generation=%u reactivated=%u core_fence=%08x gpc_fence=%08x " ++ "off_ns=%llu on_ns=%llu off_raw=%08x on_raw=%08x power_ret=%d arm_ns=%llu " ++ "power_start_ns=%llu open_ns=%llu bell_start_ns=%llu bell_ns=%llu done_ns=%llu " ++ "end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->reactivated, ++ f->core_fence, f->gpc_fence, f->off_ns, f->on_ns, f->off_raw, ++ f->on_raw, f->power_ret, f->arm_ns, f->power_start_ns, ++ f->open_ns, f->bell_start_ns, f->bell_ns, f->done_ns); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++/* r87b receipt: the DC3 attach policy and what the host did after the publish. ++ * RAM reads only; one line per DC3 generation (console at the consume/timeout ++ * point, blob at export). ++ */ ++static void g5p_r87b_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 policy = READ_ONCE(b->r73b->r87b_attach); ++ char line[640]; ++ ++ dma_rmb(); ++ scnprintf( ++ line, sizeof(line), ++ "R87B attach policy=%u name=%s generation=%u gate_first_ret=%d " ++ "gate_first_state=%08x gate_closed=%u path=%s cancel=%s bell_rung=%u bells=%d " ++ "reactivated=%u cons_before=%u prod_before=%u prod_pub=%u cons_after=%u " ++ "prod_after=%u consumed=%u arm_alive=%u alive_done=%u alive_delta=%u " ++ "pub_to_done_ns=%llu irqs=%d events=%u end=r87b\n", ++ policy, policy ? "self-drain" : "original", f->generation, ++ f->first_ret, f->first_state, f->first_ret == -EHOSTDOWN, ++ g5p_r74f_paths[f->path], g5p_r74f_cancels[f->cancel], f->kicked, ++ atomic_read(&b->r73b->r74f_bells), f->reactivated, ++ f->consumer_before, f->start, f->target, f->consumer_after, ++ f->producer_after, f->complete, f->arm_alive, f->alive_done, ++ f->alive_done - f->arm_alive, ++ f->done_ns >= f->arm_ns ? f->done_ns - f->arm_ns : 0, ++ atomic_read(&b->irqs), READ_ONCE(b->events)); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* A retained boundary: the publication stays, nothing is failed. */ ++static int g5p_r74f_stop(struct g5p_diag *g, u32 stop, int value) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ ++ r->r74f_stop_value = value; ++ WRITE_ONCE(r->r74f_stop, stop); ++ g5p_r74_mark(g, G5P_R74F_STOP, stop); ++ dev_emerg(g->dev, "R74F stop reason=%s value=%d failed=%u end=r74f\n", ++ g5p_r74f_stops[stop], value, READ_ONCE(g->boot->failed)); ++ return -EINPROGRESS; ++} ++ ++/* After the reactivation: open the generation, wait (bounded) for the ++ * ALIVE owner while the event drain stays alive, then cancel if unrung. ++ */ ++static int g5p_r74f_open(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags, deadline; ++ int ret; ++ ++ ret = g5p_r74f_reactivate(g, f); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (ret) { ++ f->cancel = ret == -EHOSTDOWN ? G5P_R74F_CANCEL_ALIVE_ADVANCED : ++ G5P_R74F_CANCEL_REACTIVATION; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ f->open_alive = r->r74f_alive; ++ f->open_ns = ktime_get_mono_fast_ns(); ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (ret) ++ return 0; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ /* The IRQ path is the sole event consumer and bell owner. */ ++ if (READ_ONCE(b->failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* Process context, command mutex held, producer already published. Returns ++ * 0 after one doorbell, 1 if consumed without one, -EINPROGRESS when the ++ * publication is retained unrung, else a fatal protocol error. ++ */ ++static int g5p_r74f_bell(struct g5p_diag *g, u32 kind, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(b, kind); ++ u32 fence = U32_MAX, state = U32_MAX, now; ++ unsigned long flags; ++ int ret, drained = 0; ++ bool ta = kind == G5P_R74F_TA; ++ bool self_drain; ++ ++ if (kind == G5P_R74F_3D) ++ return -EINVAL; /* the 3D bell is armed by g5p_r96_3d_arm(): never the DC3 / self-drain path */ ++ f->kind = kind; ++ f->ring = ta ? 0x60 : 0x6c; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ /* The immediate functional gate r74e used, without console output. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, kind + 1); ++ atomic_set(&f->state, G5P_R74F_FIRING); ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ f->first_ret = ret; ++ f->first_fence = fence; ++ f->first_state = state; ++ /* r87b self-drain: a closed DC3 gate is only a phase sample of the ++ * firmware's GPC window. The host never repairs it: no R1+20000 clear, no ++ * GPC off/on, no bell. The caller waits RAM-only for the firmware to ++ * consume the pair. Only the DC3 kind; an active gate rang above. ++ */ ++ self_drain = ret == -EHOSTDOWN && kind == G5P_R74F_DC3 && ++ READ_ONCE(r->r87b_attach) == G5P_R87B_ATTACH_SELF_DRAIN; ++ if (self_drain) { ++ f->cancel = G5P_R74F_CANCEL_SELF_DRAIN; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (ret == -EHOSTDOWN) { ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ } else { ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_IMMEDIATE); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (self_drain) { ++ drained = 1; ++ g5p_r74_mark(g, G5P_R74F_ATTACH_ARMED, state); ++ } else if (ret == -EHOSTDOWN) { ++ drained = g5p_r74f_open(g, f); ++ /* Functional reactivation precedes all console/milestone output. */ ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_ARMED : G5P_R74F_ATTACH_ARMED, ++ state); ++ g5p_r74_mark(g, ++ ta ? G5P_R74F_TA_REACTIVATED : ++ G5P_R74F_ATTACH_REACTIVATED, ++ f->power_ret); ++ } ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_BELL : G5P_R74F_ATTACH_BELL, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ if (drained) ++ return drained; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_REACTIVATION || ++ f->cancel == G5P_R74F_CANCEL_ALIVE_ADVANCED) ++ return g5p_r74f_stop( ++ g, ++ ta ? G5P_R74F_STOP_TA_REACTIVATION : ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ f->power_ret); ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* RAM only: the consumer this generation reached, at consume or timeout. */ ++static void g5p_r74f_consumed(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_r74f_bell *f = &g->boot->r73b->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(g->boot, kind); ++ ++ dma_rmb(); ++ f->consumer_after = READ_ONCE(ctl[0]); ++ f->producer_after = READ_ONCE(ctl[4]); ++ f->complete = f->consumer_after == f->target; ++ f->done_ns = ktime_get_mono_fast_ns(); ++ f->alive_done = READ_ONCE(g->boot->r73b->r74f_alive); ++ if (kind == G5P_R74F_DC3) ++ g5p_r87b_line(g, f, false); ++} ++ ++/* g5p_r22_mmu(): attach's first pass only; records the gate it already read. */ ++ ++/* g5p_r22_command(): an r74f boundary keeps the epoch, drain and backing. */ ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74g.inc b/drivers/gpu/apple-g5p/boot-r74g.inc +new file mode 100644 +index 000000000..6ff269317 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74g.inc +@@ -0,0 +1,288 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74g. Included by boot.inc after boot-r74f.inc; r74f case 1 only. It is a ++ * diagnostic bypass on the forced-ready epoch, not power policy. ++ * ++ * The firmware DC3 handler (0x1000218c..0x10002218) writes e140/e148 for ++ * slot N, copies host_generation[N] to cached_generation[N], then issues ++ * the e138 (N << 3) | 4 catalogue invalidation and waits for it. The DC ++ * dispatcher advances the consumer only after the handler returns, so a ++ * consumed DC3 pair is the completion receipt. r74g checks the RAM-visible ++ * results and issues no host MMU MMIO. The natural TA is then published ++ * without a doorbell. A RAM-only grace (four type-0 ALIVEs, 20 ms cap) ++ * lets the firmware drain it on its own. If it stays pending, the r74f ++ * ALIVE owner gets one fenced fallback doorbell. Receipts print afterwards. ++ */ ++ ++/* perf60: with the policy held in state 0 the firmware is quiet (no ALIVE stream, no self-drain), so ++ * job 1's TA bell, owned by "the first ALIVE after the grace", would never ring. ++ */ ++/* release8 default */ ++ ++static const char *const g5p_r74g_graces[G5P_R74G_GRACES] = { ++ "none", "consumed", "alives", "wall", "fault", ++}; ++ ++static bool g5p_r74g_on(struct g5p_boot *b) ++{ ++ return g5p_r74f_on(b) && 1; ++} ++ ++static u32 g5p_r74g_root(struct g5p_boot *b, u32 off) ++{ ++ return g5p_le32(b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r74g_shared(struct g5p_boot *b, u32 at) ++{ ++ return g5p_le32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE); ++} ++ ++/* RAM only: the DC3 ring, both records in attach order (3D, then TA), the ++ * PA[N] entries, host and cached tags, both failure latches and the ++ * root+3980 companion word, whose value 2 is firmware recovery/escape. ++ */ ++static u32 g5p_r74g_check(struct g5p_diag *g, u32 phase, const u32 *slot, ++ u64 pa, u32 tag, u32 start) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r74g_dc3 *d = &b->r73b->r74g_dc3[phase]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ u32 bad = 0; ++ ++ dma_rmb(); ++ d->start = start; ++ d->target = (start + 2) & 255; ++ d->tag = tag; ++ d->expected_pa = pa; ++ d->consumer = READ_ONCE(ctl[0]); ++ d->producer = READ_ONCE(ctl[4]); ++ if (d->consumer != d->target || d->producer != d->target) ++ bad |= G5P_R74G_BAD_RING; ++ for (u32 i = 0; i < 2; i++) { ++ u32 at = b->layout.offset[0x6d] + ((start + i) & 255) * 20; ++ ++ for (u32 j = 0; j < 5; j++) { ++ d->rec[i][j] = g5p_r74g_shared(b, at + j * 4); ++ if (d->rec[i][j] != (j == 0 ? 3 : ++ j == 1 ? slot[1 - i] : ++ 0)) ++ bad |= G5P_R74G_BAD_RECORD; ++ } ++ } ++ for (u32 e = 0; e < 2; e++) { ++ u8 *p = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ ++ d->slot[e] = slot[e]; ++ d->pa[e] = (u64)g5p_le32(p + 4) << 32 | g5p_le32(p); ++ d->host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + slot[e] * 4); ++ d->cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + slot[e] * 4); ++ if (d->pa[e] != pa) ++ bad |= G5P_R74G_BAD_PA; ++ if (d->host[e] != tag) ++ bad |= G5P_R74G_BAD_HOST; ++ if (d->cached[e] != tag) ++ bad |= G5P_R74G_BAD_CACHED; ++ } ++ d->failed = READ_ONCE(b->failed); ++ d->event_failed = READ_ONCE(b->event_failed); ++ if (d->failed || d->event_failed) ++ bad |= G5P_R74G_BAD_FAILED; ++ d->companion = g5p_r74g_root(b, G5P_R74G_COMPANION); ++ d->recoveries = g5p_r74g_root(b, G5P_R74G_RECOVERIES); ++ d->classification = g5p_r74g_root(b, G5P_R74G_CLASS); ++ if (d->companion == 2) ++ bad |= G5P_R74G_BAD_COMPANION; ++ d->bad = bad; ++ d->t_ns = ktime_get_mono_fast_ns(); ++ return bad; ++} ++ ++/* A refused check is retained, never b->failed: companion 2 is firmware ++ * recovery, a cached-tag-only mismatch is the firmware tag copy, and ++ * anything else (a host tag included) is a refused RAM validation. ++ */ ++static int g5p_r74g_refuse(struct g5p_diag *g, u32 bad) ++{ ++ if (bad & G5P_R74G_BAD_COMPANION) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_COMPANION, bad); ++ if (bad == G5P_R74G_BAD_CACHED) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_TAGS, bad); ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_RAM, bad); ++} ++ ++/* Two bounded lines: every field at full width still ends in end=r74g. */ ++static void g5p_r74g_dc3_line(struct g5p_diag *g, u32 phase, bool blob) ++{ ++ const struct g5p_r74g_dc3 *d = &g->boot->r73b->r74g_dc3[phase]; ++ const char *name = phase ? "ta-preflight" : "dc3-consume"; ++ char line[2][512]; ++ ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74G dc3 phase=%s bad=%02x start=%u target=%u consumer=%u producer=%u tag=%08x " ++ "slot=%u/%u host=%08x/%08x cached=%08x/%08x failed=%u event_failed=%u " ++ "companion=%u recoveries=%u class=%u host_mmu=skipped t_ns=%llu end=r74g\n", ++ name, d->bad, d->start, d->target, d->consumer, d->producer, ++ d->tag, d->slot[0], d->slot[1], d->host[0], d->host[1], ++ d->cached[0], d->cached[1], d->failed, d->event_failed, ++ d->companion, d->recoveries, d->classification, d->t_ns); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74G dc3-ram phase=%s pa=%llx/%llx expected_pa=%llx rec0=%x,%x,%x,%x,%x " ++ "rec1=%x,%x,%x,%x,%x end=r74g\n", ++ name, d->pa[0], d->pa[1], d->expected_pa, d->rec[0][0], ++ d->rec[0][1], d->rec[0][2], d->rec[0][3], d->rec[0][4], ++ d->rec[1][0], d->rec[1][1], d->rec[1][2], d->rec[1][3], ++ d->rec[1][4]); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++static void g5p_r74g_ta_line(struct g5p_diag *g, bool blob) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ const struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 end = r->r74g_grace_end; ++ char line[256]; ++ ++ scnprintf( ++ line, sizeof(line), ++ "R74G ta grace_end=%s arm_alive=%u grace_alive=%u grace_ns=%llu opened=%u " ++ "open_alive=%u ack_skipped=%u end=r74g\n", ++ g5p_r74g_graces[end < G5P_R74G_GRACES ? end : 0], f->arm_alive, ++ r->r74g_grace_alive, r->r74g_grace_ns - f->arm_ns, ++ r->r74g_opened, f->open_alive, r->r74g_ack_skipped); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* Process context, command mutex held, TA producer already published once. ++ * Grace: RAM reads and sleeps only, no MMIO and no doorbell, while the IRQ ++ * thread keeps decoding. It ends at the consume, four type-0 ALIVEs after ++ * the arm, or the 20 ms wall cap. If the TA is still pending, the ++ * generation opens, and the first ALIVE decoded after that owns the single ++ * r74f fallback attempt. Returns 0 after one doorbell, 1 if consumed without ++ * one, -EINPROGRESS for a retained boundary, else a fatal error. ++ */ ++static int g5p_r74g_ta(struct g5p_diag *g, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_TA); ++ unsigned long flags, deadline; ++ u32 end = G5P_R74G_GRACE_NONE, now; ++ int ret = 0; ++ ++ f->kind = G5P_R74F_TA; ++ f->ring = 0x60; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, G5P_R74F_TA + 1); ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ while (end == G5P_R74G_GRACE_NONE) { ++ ; ++ dma_rmb(); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) ++ end = G5P_R74G_GRACE_FAULT; ++ else if (READ_ONCE(ctl[0]) == target) ++ end = G5P_R74G_GRACE_CONSUMED; ++ else if (READ_ONCE(r->r74f_alive) - f->arm_alive >= ++ G5P_R74G_GRACE_ALIVES) ++ end = G5P_R74G_GRACE_ALIVES_END; ++ else if (ktime_get_mono_fast_ns() - f->arm_ns >= ++ (u64)G5P_R74G_GRACE_US * 1000) ++ end = G5P_R74G_GRACE_WALL; ++ else ++ usleep_range(100, 200); ++ } ++ spin_lock_irqsave(&b->event_lock, flags); ++ r->r74g_grace_end = end; ++ r->r74g_grace_alive = r->r74f_alive; ++ r->r74g_grace_ns = ktime_get_mono_fast_ns(); ++ dma_rmb(); ++ if (end == G5P_R74G_GRACE_FAULT) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (READ_ONCE(ctl[0]) == target) { ++ /* Drained by the firmware policy loop: never ring. */ ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ r->r74g_opened = 1; ++ f->open_alive = r->r74f_alive; ++ f->open_ns = r->r74g_grace_ns; ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ /* ta1_host_bell: the host owns it instead, in the same lock section a decoder would. */ ++ if ((1) && atomic_cmpxchg(&f->state, G5P_R74F_OPEN, ++ G5P_R74F_FIRING) == G5P_R74F_OPEN) { ++ u32 fence = U32_MAX, state = U32_MAX; ++ int rung; ++ ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ rung = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, rung, fence, state, ++ G5P_R74F_PATH_IMMEDIATE); ++ } ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (end == G5P_R74G_GRACE_FAULT) ++ ret = -EIO; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, G5P_R74G_TA_GRACE, end); ++ if (r->r74g_opened) ++ g5p_r74_mark(g, G5P_R74G_TA_FALLBACK, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ g5p_r74g_ta_line(g, false); ++ if (ret) ++ return ret; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_CONSUMED) ++ return 1; ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_NOT_CONSUMED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_WINDOW_MISSED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +new file mode 100644 +index 000000000..d5bca5a4e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -0,0 +1,1663 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Experimental G2 stages. Included after the bounded META slave primitives. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "boot-layout.h" ++#include "render-user-pmset.h" ++#include "render-user-recover.h" ++#include "fender.h" ++#include "readback.h" ++#include "meta71c114.h" ++#include "latency123.h" ++#include "perf-envelope.h" ++ ++/* Boot deltas are selected before map; no live render-time reconfiguration. */ ++ ++/* R46 (diag-fable51 T7): the first SOFT_RESET write of the host reset FIFO ++ * also asserts bits 56-59, VBS/VCE/TE3/TILING_CORE in the PBE2_IN_XE ++ * layout. Apple's mask 00e7fffffffffc1d is exactly the other layout's ++ * MASKFULL, in which those bits do not exist and TE/GPP/UVS/TLA/PM/VDM are ++ * already asserted; expected to be a no-op. The unchanged second write ++ * releases them. The FIFO forbids a readback here. ++ */ ++#define G5P_BOOT_RESET1 0x00e7fffffffffc1dULL ++#define G5P_BOOT_R46_RESET_BITS (0xfULL << 56) ++ ++/* R72b: isolate Apple meta+0x6c8 while retaining dc7 transport. */ ++ ++/* The event ring's stamp events (type 1) dump once per epoch, then a count at each power of two. */ ++ ++/* R73: mirror Apple's auxiliary firmware-ring returns on IRQ174. */ ++ ++/* Latency114: the pstate-1 frequency word meta+0x71c (meta71c114.h). */ ++ ++/* Latency123 (latency123.h): arm A boot words, and the live ctl+0x04 tick-service mask. */ ++ ++/* perf60 clock (perf-envelope.h): the firmware's own pstate request, pinned before release. */ ++/* release8 default */ ++/* GPU overclock (gpu-oc.h, docs/power/ds-gpu-dvfs.md §9 R5): PSINFO record 5 = stock record 4 ++ * with only the PLL multiplier moved (no voltage change), written before release while the GPC is at ++ * state 1, the firmware's table extended to state 5, the release powering on at 5. Needs ++ * boot_gpu_perf 500 and boot_gpu_perf_poweron 1. A thermal stop (drm-render-v3.inc) reboots the ++ * firmware at state 4 when a SoC zone reaches boot_gpu_oc_temp_mc. Record 5 is volatile PMGR ++ * state (LLB rewrites the table only at SoC reset; iBoot reads it once to build the ADT before us) ++ * and is cleared on every boot that does not overclock. ++ */ ++ ++static bool ++ g5p_oc_tripped; /* the thermal stop fired: every later epoch of this boot runs state 4 */ ++ ++static const char *const g5p_oc_zone_names[] = { "apple-a9-soc0", ++ "apple-a9-soc1", ++ "apple-a9-tcc2" }; ++ ++/* The hottest of the three SoC zones (millidegrees); the number of zones read. */ ++static u32 g5p_oc_temp(int *max_mc, const char **zone) ++{ ++ u32 n = 0; ++ ++ *max_mc = INT_MIN; ++ *zone = "none"; ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_zone_names); i++) { ++ struct thermal_zone_device *tz = ++ thermal_zone_get_zone_by_name(g5p_oc_zone_names[i]); ++ int t; ++ ++ if (IS_ERR_OR_NULL(tz) || thermal_zone_get_temp(tz, &t)) ++ continue; ++ n++; ++ if (t > *max_mc) { ++ *max_mc = t; ++ *zone = g5p_oc_zone_names[i]; ++ } ++ } ++ return n; ++} ++ ++static int g5p_oc_read(struct g5p_diag *g, u32 n, u32 w[4]) ++{ ++ for (u32 i = 0; i < 4; i++) ++ if (regmap_read(g->pmgr, G5P_PSINFO_REC(n) + 4 * i, &w[i])) ++ return -EIO; ++ return 0; ++} ++ ++/* Write record 5 from the live record 4 and read it back; the GPC is at state 1 (map time). ++ * Slot 5 is read first: nonzero and not a record this driver writes -> refused, left untouched. ++ */ ++ ++/* A boot without the overclock clears a record 5 this driver wrote; anything else stays. */ ++static void g5p_oc_clear(struct g5p_diag *g) ++{ ++ u32 r4[4], r5[4]; ++ ++ if (!g->pmgr || g5p_oc_read(g, 4, r4) || ++ g5p_oc_read(g, G5P_OC_STATE, r5) || ++ !(r5[0] || r5[1] || r5[2] || r5[3])) ++ return; ++ if (!g5p_oc_is_ours(r4, r5)) { ++ dev_info( ++ g->dev, ++ "GPUPERF oc slot 5 not ours, left: %08x %08x %08x %08x end=gpuperf\n", ++ r5[0], r5[1], r5[2], r5[3]); ++ return; ++ } ++ for (u32 i = 0; i < 4; i++) ++ regmap_write(g->pmgr, G5P_PSINFO_REC(G5P_OC_STATE) + 4 * i, 0); ++ g5p_oc_read(g, G5P_OC_STATE, r5); ++ dev_info( ++ g->dev, ++ "GPUPERF oc record 5 cleared (no overclock this epoch) word0=%08x end=gpuperf\n", ++ r5[0]); ++} ++ ++/* perf60 pin >100, the host's own route: power the GPC on at q (R1+0x1000 = 0x10 | q) in the release, ++ * as the GPC1 request the kext and we already make, with q in place of 1; the policy record carries ++ * the seed-1 words (+0x28 = +0x2c = q) and stays "never sampled" (+0 = 0), so the boot sampler only ++ * initialises; it derives +0x34 and its clock entry priv+0x2c4 from Fender (0x1001c838). ++ */ ++/* release8 default */ ++/* perf60 pin >100 (perf-envelope.h g5p_perf_seed): the boot sampler applies the pin before ALIVE. */ ++ ++/* perf60 stage 4 (render-user-pmset.h): firmware-GiB pages for per-context PM records. */ ++/* release8 default */ ++/* perf60 A/B (docs/gpu/experiments/perf60-clock.md): the firmware's per-tick full flush alone. */ ++ ++/* Latched by map, before firmware sees any pointers. Zero preserves round 9. */ ++ ++/* Explicit decoded VA input only: zero is a valid address when valid=true. ++ * Never infer an address or context from revision-dependent e150 bits. ++ */ ++static unsigned long long mmu_fault_va; ++ ++static const bool boot_experiment = 1; ++ ++struct g5p_dma_page { ++ u8 *cpu; ++ dma_addr_t dma; ++}; ++ ++struct g5p_boot { ++ u32 r27_variant, r27_mode_writes, r46_reset, r50_mode, r50_writes; ++ ++ bool table_clean, preinstall_context, compute_boot_vm; ++ u32 compute_slot; ++ ++ bool compute_numerical, user_compute_owned; ++ ++ struct g5p_boot_layout layout; ++ struct g5p_dma_page image[G5P_IMAGE_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page root[G5P_ROOT_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page shared[G5P_SHARED_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page render_meta[2]; ++ /* perf60 stage 4 (render-user-pmset.h): boot_pm_sets per-context PM sets, 3 pages each */ ++ u32 pm_sets; ++ struct g5p_dma_page pm_set[G5P_PMSET_MAX * G5P_PMSET_PAGES]; ++ struct g5p_dma_page catalogue, directory, zero, leaf[4]; ++ struct g5p_dma_page sink_directory, sink_leaf, sink_page; ++ u32 setup_variant; ++ bool sink_ready, sink_dumped; ++ u32 leaf_index[4]; ++ u32 leaves, pages, mapped, fifo, events, alive_word; ++ u32 fender_raw[ARRAY_SIZE(g5p_fender_regs)]; ++ u32 rgx_raw[ARRAY_SIZE(g5p_rgx_regs)]; ++ const char *snapshot_stage; ++ bool snapshot_valid, rgx_valid; ++ u32 rgx_count; ++ int fifo_error; ++ int irq; ++ atomic_t irqs; ++ spinlock_t event_lock; ++ void __iomem *swirq; ++ bool maps_ready, armed, copied, released, alive, failed, irq_ready; ++ bool release_ack, release_ack_attempted; ++ bool fault_va_valid; ++ u64 fault_va; ++ bool work_ready, work_submitted, work_complete; ++ u32 work_target, control_target; ++ u32 dc_variant, work_variant; ++ u32 r72_meta6c8; ++ u32 meta71c; ++ u32 fw_power_words; ++ u32 gpu_perf; ++ u32 perf_seed; /* boot_gpu_perf_seed: the seeded pstate q, 0 = not seeded */ ++ u32 perf_seed_mode; /* boot_gpu_perf_seed as latched (1..5) */ ++ u32 perf_poweron; /* boot_gpu_perf_poweron: the pstate q the release requests, 0 = 1 as before */ ++ u32 gpu_oc_mhz; /* boot_gpu_oc_mhz as applied (state 5), 0 = no overclock */ ++ /* record 5's voltage codes */ ++ u64 seed_stamp, ++ seed_gpc1; /* seed 3/4: the stamp and the GPC1 write (arch timer) */ ++ u32 meta6d4; ++ u32 r78_ctrl0; ++ u32 r73_aux_drain; ++ u32 r73_aux_entries[6]; ++ u32 r73_aux_hash[6]; ++ u32 r73_aux_first[6][3]; ++ u32 r73_aux_last[6][3]; ++ struct g5p_r73b *r73b; ++ bool irq_capped; ++ bool dc_observe; ++ ++ bool control_ready, control_submitted, control_complete; ++ bool irq_tested, stopped, parked; ++ bool control_timed_out, work_timed_out, event_failed; ++ bool context_submitted, context_complete, context_timed_out; ++ u32 context_target; ++ /* The epoch's last recovery packet (render-user-recover.h), under event_lock. */ ++ struct g5p_fw_recovery fw_recovery; ++ u32 fw_recoveries; ++ u32 event_n[10]; /* events drained this epoch by type (9: 9 and above) */ ++}; ++ ++#include "boot-r73b.inc" ++ ++/* The fault-status subset is read-only, including while META reset is held. ++ * Never access R0 with GPC off/busy; no slave-port transaction is involved. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage); ++ ++static int g5p_boot_alloc_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ p->cpu = dmam_alloc_coherent(g->dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ /* There is no GPU bus translation in the recovered GART path. */ ++ if ((p->dma & (G5P_GPU_PAGE - 1)) || p->dma >= BIT_ULL(40) || ++ dma_to_phys(g->dev, p->dma) != p->dma) { ++ dev_err(g->dev, ++ "unsupported DMA translation/alignment at %pad\n", ++ &p->dma); ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ g->boot->pages++; ++ return 0; ++} ++ ++/* Deliberately map one16KiB page only, not the iOS all-address fallback. ++ * Own distinct directory/leaf/backing; do not alias firmware tables or e160. ++ * No live table surgery: this helper runs only during first map construction. ++ */ ++static int g5p_boot_sink_init(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ; ++ ; ++ if (!b->catalogue.cpu || b->maps_ready || b->armed || b->released || ++ b->sink_ready || b->sink_directory.cpu || b->sink_leaf.cpu || ++ b->sink_page.cpu) ++ return -EPERM; ++ if (g5p_le32(b->catalogue.cpu)) ++ return -EEXIST; ++ ret = g5p_boot_alloc_page(g, &b->sink_directory); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_leaf); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_page); ++ if (ret) ++ return ret; ++ /* Round12 iOS optional dummy-tree leaf flags0: valid writable PA|1. ++ * Only cat[0]/dir[0]/leaf[0] are installed; every other hole stays absent. ++ */ ++ g5p_put64(b->sink_leaf.cpu, g5p_leaf_entry(b->sink_page.dma, 0)); ++ g5p_put64(b->sink_directory.cpu, g5p_directory_entry(b->sink_leaf.dma)); ++ dma_wmb(); ++ g5p_put32(b->catalogue.cpu, g5p_catalogue_entry(b->sink_directory.dma)); ++ dma_wmb(); ++ b->mapped++; ++ b->sink_ready = true; ++ g5p_mmu_record( ++ g, ++ "MMU sink-installed VA=0 bytes=4000 context=0 leaf_flags=0; pre-release " ++ "containment experiment\n"); ++ g5p_blob_printf( ++ g, ++ "MMU sink PC=%pad cat0=%08x dir=%pad dir0=%016llx leaf=%pad pte0=%016llx " ++ "backing=%pad; initially zero\n", ++ &b->catalogue.dma, g5p_catalogue_entry(b->sink_directory.dma), ++ &b->sink_directory.dma, g5p_directory_entry(b->sink_leaf.dma), ++ &b->sink_leaf.dma, g5p_leaf_entry(b->sink_page.dma, 0), ++ &b->sink_page.dma); ++ return 0; ++} ++ ++static int g5p_boot_map_page(struct g5p_diag *g, u64 va, u64 pa, u32 attributes) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ u32 ti = (va >> 14) & 127, i; ++ int ret; ++ ++ if ((va | pa) & (G5P_GPU_PAGE - 1)) ++ return -EINVAL; ++ /* This experiment only owns the recovered firmware heap's catalogue slot. */ ++ if (ci != ((G5P_IMAGE_VA >> 30) & 1023)) ++ return -ERANGE; ++ for (i = 0; i < b->leaves; i++) ++ if (b->leaf_index[i] == di) ++ break; ++ if (i == b->leaves) { ++ if (i == ARRAY_SIZE(b->leaf)) ++ return -ENOSPC; ++ ret = g5p_boot_alloc_page(g, &b->leaf[i]); ++ if (ret) ++ return ret; ++ b->leaf_index[i] = di; ++ g5p_put64(b->directory.cpu + di * 8, ++ g5p_directory_entry(b->leaf[i].dma)); ++ b->leaves++; ++ } ++ if (g5p_le32(b->leaf[i].cpu + ti * 8)) ++ return -EEXIST; ++ g5p_put64(b->leaf[i].cpu + ti * 8, g5p_leaf_entry(pa, attributes)); ++ g5p_put32(b->catalogue.cpu + ci * 4, ++ g5p_catalogue_entry(b->directory.dma)); ++ b->mapped++; ++ return 0; ++} ++ ++#include "mmu-attrs.inc" ++#include "mmu-tables.inc" ++ ++static int g5p_boot_region(struct g5p_diag *g, struct g5p_dma_page *p, u32 size, ++ u64 va) ++{ ++ u32 offset; ++ u32 attributes = va == G5P_IMAGE_VA ? 0x10 : 0x14; ++ int ret; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) { ++ ret = g5p_boot_alloc_page(g, p); ++ if (ret) ++ return ret; ++ ret = g5p_boot_map_page(g, va + offset, p->dma, attributes); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void g5p_boot_copy(struct g5p_dma_page *p, const u8 *data, u32 size) ++{ ++ u32 offset; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) ++ memcpy(p->cpu, data + offset, G5P_GPU_PAGE); ++} ++ ++static u8 *g5p_boot_shared(struct g5p_boot *b, u32 slot) ++{ ++ u32 off = b->layout.offset[slot]; ++ ++ return b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static int g5p_boot_maps(struct g5p_diag *g) ++{ ++ struct g5p_boot *b; ++ u8 *root = NULL, *shared = NULL; ++ bool m71c_wrote; ++ u32 i, m71c, seed; ++ int ret; ++ ++ if (!g->loaded || !g->id_checked || PAGE_SIZE != G5P_GPU_PAGE) ++ return -EPERM; ++ if (g->core_id != 0x011b0602 || g->core_revision != 0x00090f00) ++ return -ENODEV; ++ if (g->boot) ++ return g->boot->maps_ready ? 0 : -EIO; ++ b = devm_kzalloc(g->dev, sizeof(*b), GFP_KERNEL); ++ if (!b) ++ return -ENOMEM; ++ g->boot = b; ++ b->r50_mode = (0); ++ ; ++ b->r27_variant = (0); ++ ; ++ b->r46_reset = (0); ++ ; ++ b->r72_meta6c8 = (1); ++ b->r78_ctrl0 = (0); ++ if (b->r78_ctrl0 && b->r78_ctrl0 != 0x5f) { ++ b->failed = true; ++ return -EINVAL; ++ }; ++ b->r73_aux_drain = (0); ++ if (b->r73_aux_drain > 1 || (b->r73_aux_drain && !1)) { ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->release_ack = (0); ++ b->table_clean = (1); ++ b->compute_boot_vm = (0); ++ b->compute_slot = 0 ? 0 : (7); ++ b->preinstall_context = (0) || 0; ++ ; ++ b->setup_variant = (1); ++ ; ++ b->fault_va = READ_ONCE(mmu_fault_va); ++ b->fault_va_valid = (0); ++ ; ++ b->dc_variant = (7); ++ b->work_variant = (0); ++ b->dc_observe = (0); ++ ; ++ /* iOS: meta+0x71c = 340, the pstate-1 frequency in MHz. */ ++ b->meta71c = (340); ++ m71c = g5p_m71c_check(b->meta71c, 7); ++ if (m71c != G5P_M71C_OK) { ++ dev_err(g->dev, ++ "META71C refused requested=%u variant=%u why=%s before_release=1 end=meta71c\n", ++ b->meta71c, b->dc_variant, g5p_m71c_whys[m71c]); ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->fw_power_words = (0); ++ b->gpu_perf = (400); ++ b->meta6d4 = (0); ++ seed = (0); ++ b->gpu_oc_mhz = 0; ++ ; ++ b->perf_poweron = ++ (1) ? (0 ? G5P_OC_STATE : g5p_perf_seed_q(b->gpu_perf)) : 0; ++ b->perf_seed = seed && !b->perf_poweron ? g5p_perf_seed_q(b->gpu_perf) : ++ 0; ++ b->perf_seed_mode = b->perf_seed ? seed : 0; ++ if (!(g5p_perf_valid(b->gpu_perf) || ++ (0 && b->gpu_perf == G5P_OC_PIN)) || ++ (b->gpu_perf && b->fw_power_words) || seed > 5 || (1) > 1 || ++ (b->gpu_perf > 100 && !(1)) || b->meta6d4 > 1 || ++ (b->meta6d4 && !b->gpu_perf)) { ++ dev_err(g->dev, ++ "GPUPERF refused pin=%u power_words=%u gpc_any_pstate=%u meta6d4=%u seed=%u " ++ "before_release=1 end=gpuperf\n", ++ b->gpu_perf, b->fw_power_words, (1), b->meta6d4, seed); ++ b->failed = true; ++ return -EINVAL; ++ } ++ if (b->fw_power_words > G5P_L123_FULL || (0) > G5P_L123_TICK_MASK) { ++ dev_err(g->dev, ++ "L123 refused power_words=%u tick_mask=%u before_release=1 end=l123\n", ++ b->fw_power_words, (0)); ++ b->failed = true; ++ return -EINVAL; ++ } ++ ret = g5p_r73b_latch(g); ++ if (ret) { ++ b->failed = true; ++ return ret; ++ }; ++ dev_info( ++ g->dev, ++ "DC matrix latched variant=%u observe=%u (changes require reboot)\n", ++ b->dc_variant, b->dc_observe); ++ dev_info( ++ g->dev, ++ "COMPUTE VM latched slot=%u boot_vm=%u preinstall=%u (changes require reboot)\n", ++ b->compute_slot, b->compute_boot_vm, b->preinstall_context); ++ spin_lock_init(&b->event_lock); ++ atomic_set(&b->irqs, 0); ++ ret = g5p_plan_layout(&b->layout); ++ if (ret) ++ goto out; ++ root = kvzalloc(G5P_ROOT_SIZE, GFP_KERNEL); ++ shared = kvzalloc(G5P_SHARED_SIZE, GFP_KERNEL); ++ if (!root || !shared) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_boot_alloc_page(g, &b->catalogue); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->directory); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->zero); ++ if (ret) ++ goto out; ++ ret = g5p_boot_sink_init(g); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->image, G5P_IMAGE_SIZE, G5P_IMAGE_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->root, G5P_ROOT_SIZE, G5P_ROOT_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->shared, G5P_SHARED_SIZE, G5P_SHARED_VA); ++ if (ret) ++ goto out; ++ /* R22 metadata mappings are installed before release, only behind its ++ * default-off gate. Compute/boot defaults allocate no additional pages. ++ */ ++ { ++ ret = g5p_boot_region(g, b->render_meta, 2 * G5P_GPU_PAGE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE); ++ if (ret) ++ goto out; ++ b->pm_sets = min((8), G5P_PMSET_MAX); ++ if (b->pm_sets) { ++ ret = g5p_boot_region(g, b->pm_set, ++ b->pm_sets * G5P_PMSET_PAGES * ++ G5P_GPU_PAGE, ++ G5P_PMSET_VA); ++ if (ret) ++ goto out; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-sets boot sets=%u va=%#llx..%#llx before_release=1 end=v3\n", ++ b->pm_sets, G5P_PMSET_VA, ++ g5p_pmset_va(b->pm_sets, 0)); ++ } ++ } ++ ret = 0; ++ if (ret) ++ goto out; ++ /* c80758: Fender (2 pages), AIC timer, AIC software interrupt. */ ++ for (i = 0; i < 4; i++) { ++ u64 pa = i < 2 ? 0x201e00000ULL + i * G5P_GPU_PAGE : ++ i == 2 ? 0x20e100000ULL : ++ 0x20e104000ULL; ++ ++ ret = g5p_boot_map_page(g, G5P_PIO_VA + i * G5P_GPU_PAGE, pa, ++ 0x18); ++ if (ret) ++ goto out; ++ } ++ g5p_make_boot_config(&g->image, &b->layout); ++ g5p_make_init(&b->layout, root, shared, &g->image, b->catalogue.dma, ++ arch_timer_read_counter()); ++ g5p_dc_init_variant(&b->layout, root, shared, 7); ++ /* meta+0x784 is the abstime OFFSET the firmware adds to its timebase (0x100221e8 = PIO-1 ++ * counter + priv+688, loaded from meta+0x784 at 0x1001ff58); iOS writes ++ * ml_get_abstime_offset(). Linux's timebase is the counter itself: 0. (make_init's absolute ++ * stamp put the firmware's clock ~uptime ahead of every host stamp in policy+0x8f8.) ++ */ ++ g5p_put64(g5p_object(&b->layout, root, shared, 0x4d) + 0x784, 0); ++ if (g5p_boot_ctrl0(b)) ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x5e), ++ g5p_boot_ctrl0(b)); ++ dev_info( ++ g->dev, ++ "R78 ctrl0 requested=%08x installed=%08x before_release=1 end=r78\n", ++ g5p_boot_ctrl0(b), ++ g5p_le32(g5p_object(&b->layout, root, shared, 0x5e))); ++ ; ++ ; ++ { ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x4d) + 0x6c8, ++ 1); ++ }; ++ m71c = g5p_m71c_install(&b->layout, root, shared, b->meta71c, ++ &m71c_wrote); ++ dev_info( ++ g->dev, ++ "META71C requested=%u installed=%08x variant=%u wrote=%u before_release=1 " ++ "end=meta71c\n", ++ b->meta71c, m71c, b->dc_variant, m71c_wrote); ++ if (m71c != b->meta71c) { ++ ret = -EIO; ++ goto out; ++ } ++ if (b->fw_power_words) ++ g5p_l123_arm_a(&b->layout, root, shared, b->fw_power_words); ++ if (b->gpu_perf) { ++ u8 *pol = g5p_object(&b->layout, root, shared, G5P_PERF_POLICY); ++ u8 *mt = g5p_object(&b->layout, root, shared, G5P_PERF_META); ++ u8 *ct = g5p_object(&b->layout, root, shared, G5P_PERF_CTL); ++ ++ g5p_perf_install(&b->layout, root, shared, b->gpu_perf); ++ { ++ g5p_oc_clear(g); ++ } ++ if (b->meta6d4) ++ g5p_put32(mt + 0x6d4, 1); ++ if (b->perf_poweron) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_poweron, ++ false); ++ dev_info( ++ g->dev, ++ "GPUPERF poweron q=%u policy28=%u/%u policy0=%u policy4=%u seed=%u ignored " ++ "before_release=1 end=gpuperf\n", ++ b->perf_poweron, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), seed); ++ } else { ++ dev_info( ++ g->dev, ++ "GPUPERF poweron ignored pin=%u (pins 200..400 only) before_release=1 end=gpuperf\n", ++ b->gpu_perf); ++ } ++ if (seed && !b->perf_seed && !b->perf_poweron) ++ dev_info( ++ g->dev, ++ "GPUPERF seed ignored pin=%u (seeds pins 200..400 only) before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf); ++ if (b->perf_seed) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_seed, ++ seed >= 2); ++ /* seed 5: meta+0x784 is the abstime OFFSET the firmware adds to its timebase ++ * (0x100221e8 = PIO-1 counter + priv+688, loaded from meta+0x784 at 0x1001ff58); ++ * iOS writes ml_get_abstime_offset(), make_init the absolute counter, so every ++ * policy+0x8f8 stamp lay ~counter in the firmware's past. Linux has no offset. ++ */ ++ if (seed == 5) ++ g5p_put64(mt + 0x784, 0); ++ dev_info( ++ g->dev, ++ "GPUPERF seed mode=%u q=%u policy28=%u/%u policy30=%u policy8=%u policy0=%u " ++ "policy4=%u meta784=%llx before_release=1 end=gpuperf\n", ++ seed, b->perf_seed, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0x30), ++ g5p_le32(pol + 8), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), ++ (u64)g5p_le32(mt + 0x784) | ++ (u64)g5p_le32(mt + 0x788) << 32); ++ } ++ dev_info( ++ g->dev, ++ "GPUPERF boot pin=%u table=%u/%u/%u,%u,%u,%u,%u limits=%u/%u/%u ovr=%u/%u " ++ "policy688=%u/%u policy6f0=%u/%u policy4=%u policy8=%u/%u q=%u mhz=%u " ++ "gpc_any_pstate=%u seed=%u/%u policy0=%u policy28=%u/%u before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf, g5p_le32(mt + 0x710), g5p_le32(mt + 0x714), ++ g5p_le32(mt + 0x718), g5p_le32(mt + 0x71c), ++ g5p_le32(mt + 0x720), g5p_le32(mt + 0x724), ++ g5p_le32(mt + 0x728), g5p_le32(ct + 0x1028), ++ g5p_le32(ct + 0x102c), g5p_le32(ct + 0x1030), ++ g5p_le32(ct + 0x1038), g5p_le32(ct + 0x1040), ++ g5p_le32(pol + 0x688), g5p_le32(pol + 0x68c), ++ g5p_le32(pol + 0x6f0), g5p_le32(pol + 0x6f4), ++ g5p_le32(pol + 4), g5p_le32(pol + 8), ++ g5p_le32(pol + 0xc), b->gpu_perf / 100, ++ b->gpu_perf / 100 < ARRAY_SIZE(g5p_perf_mhz) ? ++ g5p_perf_mhz[b->gpu_perf / 100] : ++ b->gpu_oc_mhz, ++ (1), seed, b->perf_seed, g5p_le32(pol + 0), ++ g5p_le32(pol + 0x28), g5p_le32(pol + 0x2c)); ++ } ++ g5p_put32(g5p_object(&b->layout, root, shared, G5P_L123_CTL) + 4, (0)); ++ { ++ u8 *meta = g5p_object(&b->layout, root, shared, G5P_L123_META); ++ ++ dev_info( ++ g->dev, ++ "L123 boot power_words=%u meta6d4=%u table=%u/%u/%u,%u,%u,%u,%u policy4=%u " ++ "ctl04=%u limits=%u/%u/%u before_release=1 end=l123\n", ++ b->fw_power_words, g5p_le32(meta + 0x6d4), ++ g5p_le32(meta + 0x710), g5p_le32(meta + 0x714), ++ g5p_le32(meta + 0x718), g5p_le32(meta + 0x71c), ++ g5p_le32(meta + 0x720), g5p_le32(meta + 0x724), ++ g5p_le32(meta + 0x728), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_POLICY) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1028), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x102c), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1030)); ++ /* the accessor's licence bound (mmio-access.md §4.2 step 4): meta+0x710 as written */ ++ WRITE_ONCE(g->mmio_max_pstate, ++ g5p_mmio_max_pstate(g5p_le32(meta + 0x710))); ++ }; ++ ; ++ g5p_boot_copy(b->image, g->image.data, G5P_IMAGE_SIZE); ++ g5p_boot_copy(b->root, root, G5P_ROOT_SIZE); ++ g5p_boot_copy(b->shared, shared, G5P_SHARED_SIZE); ++ dma_wmb(); ++ ret = 0; ++ if (ret) ++ goto out; ++ g5p_tables_clean(g, "boot-before-release", NULL, 0); ++ b->maps_ready = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 maps: %u x 16KiB allocations; %u PTEs, %u leaves, PC=%pad\n", ++ b->pages, b->mapped, b->leaves, &b->catalogue.dma); ++ dev_info( ++ g->dev, ++ "G2 init: root=%llx control=%llx event-ctl=%llx event-data=%llx\n", ++ g5p_object_va(&b->layout, 0x50), ++ g5p_object_va(&b->layout, 0x5f), ++ g5p_object_va(&b->layout, 0x78), ++ g5p_object_va(&b->layout, 0x7c)); ++ dev_info( ++ g->dev, ++ "G2 experimental zero policy fields; fault observation only in map\n"); ++out: ++ kvfree(root); ++ kvfree(shared); ++ if (ret) ++ b->failed = true; ++ g5p_ds_hook_maps(g, ret); ++ return ret; ++} ++ ++#include "boot-r74f.inc" ++#include "boot-r74g.inc" ++ ++static u32 g5p_transport_event_limit(void); ++ ++static int g5p_boot_drain(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ dev_err(g->dev, "G2 invalid event indices %u/%u\n", consumer, ++ producer); ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ for (n = 0; n < 16 && consumer != producer; n++) { ++ u32 words[11], off = b->layout.offset[0x74] + consumer * 44, t, ++ n; ++ u8 *src; ++ unsigned int i; ++ bool want; ++ ++ /* Heartbeat116: an R74 epoch bounds ring entries per window ++ * in the IRQ thread instead (heartbeat116.h EVENTS). ++ */ ++ if (!g5p_hb116_on(b) && ++ b->events >= g5p_transport_event_limit()) { ++ ret = -EOVERFLOW; ++ goto out; ++ } ++ /* Entries may straddle DMA pages. */ ++ for (i = 0; i < ARRAY_SIZE(words); i++) { ++ u32 at = off + i * 4; ++ ++ src = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ words[i] = g5p_le32(src); ++ } ++ /* A healthy meta6c8 setup loop can emit thousands of identical ++ * ALIVEs. Keep its first four packets and the exact aggregate, ++ * without making printk throughput part of the experiment. ++ * R74 also quiets a moved loop: each new word1 prints once. ++ */ ++ want = !b->r73b || words[0] || b->events < 4 || ++ (words[1] != 0x8000a120 && g5p_r74_word1(b, words[1])); ++ /* Stamp events (type 1, two per job) dump once per epoch, then only count: ++ * at a few jobs per frame they were most of the log and evicted the faults ++ * (release8 bench, 306 k lines). Every other type dumps as before. ++ */ ++ t = min_t(u32, words[0], ARRAY_SIZE(b->event_n) - 1); ++ n = ++b->event_n[t]; ++ if (words[0] == 1 && n > 1 && !(0)) { ++ if (is_power_of_2(n)) ++ dev_info( ++ g->dev, ++ "G2 events type=1 n=%u: stamp events counted, dumped once (boot_event_dump 1: each)\n", ++ n); ++ want = false; ++ } ++ /* Alive (0) and stamp (1) events are routine, one or more per job: job_log. Every other ++ * type (recovery, page request, channel error, fault) prints as before. ++ */ ++ if (want && words[0] > 1) { ++ dev_info(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ print_hex_dump(KERN_INFO, ++ "G2 event raw: ", DUMP_PREFIX_NONE, 16, ++ 4, words, sizeof(words), false); ++ } else if (want) { ++ g5p_job_printk(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ g5p_job_hex("G2 event raw: ", words, sizeof(words)); ++ }; ++ if (g->event_notify) ++ g->event_notify(g->event_cookie, words); ++ /* c99788 routes these to recovery/page-request/channel-error paths. */ ++ if (words[0] != 0 && words[0] != 1) { ++ WRITE_ONCE(b->failed, true); ++ switch (words[0]) { ++ case G5P_FW_EVENT_RECOVERY: { ++ /* The kext's fields (render-user-recover.h). */ ++ struct g5p_fw_recovery rec = ++ g5p_fw_recovery_decode(words); ++ ++ b->fw_recovery = rec; ++ b->fw_recoveries++; ++ dev_err(g->dev, ++ "G2 recovery event slot=%u flag=%u code=%08x w18=%08x w20=%08x w24=%08x n=%u: " ++ "the firmware stopped; iOS restarts the GPU\n", ++ rec.slot, rec.flag, rec.code, rec.w18, ++ rec.w20, rec.w24, b->fw_recoveries); ++ break; ++ } ++ case 7: ++ dev_err(g->dev, ++ "G2 page-request event payload=%08x/%08x/%08x; address encoding unresolved\n", ++ words[1], words[2], words[3]); ++ break; ++ case 8: ++ dev_err(g->dev, ++ "G2 channel-error event kind=%u engine=%u token=%08x stamp=%08x; no " ++ "acknowledgment command guessed\n", ++ words[1], words[2], words[3], words[4]); ++ break; ++ } ++ dev_err(g->dev, ++ "G2 unsupported/fault event=%u; epoch failed, entry retained (the render device " ++ "recovers it if it may)\n", ++ words[0]); ++ ret = -EIO; ++ break; ++ } ++ if (words[0] == 0 && READ_ONCE(b->released) && ++ !READ_ONCE(b->stopped)) { ++ b->alive_word = words[1]; ++ WRITE_ONCE(b->alive, true); ++ } ++ b->events++; ++ consumer = (consumer + 1) & 255; ++ /* c99788 uses a full barrier: finish slot reads before returning it. */ ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ /* Return the ALIVE slot before ringing work. No logging or ++ * other MMIO may be inserted between these two operations. ++ */ ++ if (!words[0] && g5p_r74f_on(b) && ++ READ_ONCE(b->r73b->owner) == G5P_R73B_OWNER_IRQ && ++ READ_ONCE(b->released) && !READ_ONCE(b->stopped)) ++ g5p_r74f_alive(g); ++ } ++out: ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++static irqreturn_t g5p_boot_irq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ struct g5p_boot *b = g->boot; ++ u32 n = atomic_inc_return(&b->irqs); ++ int ret = 0; ++ ++ if (b->r73b) ++ return g5p_r73b_irq(g, irq, n); ++ if (n > g5p_transport_event_limit()) { ++ /* Never leave the software source asserted, even when capped. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (!b->irq_capped) { ++ b->irq_capped = true; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "G2 IRQ174 storm capped; reboot required\n"); ++ } ++ return IRQ_HANDLED; ++ } ++ /* KPF level6G: Apple uses this AIC ACK before all ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (READ_ONCE(b->released)) ++ ret = g5p_boot_drain(g); ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return IRQ_HANDLED; ++} ++ ++static int g5p_boot_irq_setup(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b->irq_ready) ++ return 0; ++ b->swirq = devm_ioremap(g->dev, 0x20e104000ULL, 0x100); ++ if (!b->swirq) ++ return -ENOMEM; ++ b->irq = platform_get_irq(to_platform_device(g->dev), 4); ++ if (b->irq < 0) ++ return b->irq; ++ ret = devm_request_threaded_irq(g->dev, b->irq, NULL, g5p_boot_irq, ++ IRQF_ONESHOT, "g5p-meta", g); ++ if (ret) ++ return ret; ++ b->irq_ready = true; ++ dev_info(g->dev, "G2 IRQ174 registered as Linux IRQ %d\n", b->irq); ++ return 0; ++} ++ ++/* Cache first, then report: sysfs status never issues GPU MMIO. */ ++static void g5p_boot_snapshot(struct g5p_diag *g, const char *stage, ++ enum g5p_readback_state mode) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 i, state; ++ ++ b->snapshot_stage = stage; ++ b->snapshot_valid = false; ++ b->rgx_valid = false; ++ b->rgx_count = 0; ++ g5p_power_snapshot(g, stage); ++ dev_info(g->dev, "G2 snapshot %s: R1 first\n", stage); ++ for (i = 0; i < ARRAY_SIZE(g5p_fender_regs); i++) { ++ b->fender_raw[i] = readl(g->window[1] + g5p_fender_regs[i]); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x\n", g5p_fender_regs[i], ++ b->fender_raw[i]); ++ } ++ b->snapshot_valid = true; ++ state = b->fender_raw[4]; ++ /* GPC=1 did not make META a18 safe while bit32 was held. */ ++ b->rgx_count = g5p_readback_rgx_count(state, mode); ++ if (!b->rgx_count) { ++ dev_info(g->dev, "G2 R0 snapshot skipped: R1+1000=%08x\n", ++ state); ++ return; ++ } ++ for (i = 0; i < b->rgx_count; i++) { ++ dev_info(g->dev, "G2 R R0+%05x\n", g5p_rgx_regs[i]); ++ b->rgx_raw[i] = readl(g->rgx + g5p_rgx_regs[i]); ++ dev_info(g->dev, "G2 raw R0+%05x=%08x\n", g5p_rgx_regs[i], ++ b->rgx_raw[i]); ++ } ++ b->rgx_valid = true; ++} ++ ++static int g5p_fender_write(void *ctx, u32 reg, u32 value) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ ++ /* recovery108: never an R1 access after a FINAL teardown retired the device. */ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 W R1+%05x=%08x\n", reg, value); ++ writel(value, g->window[1] + reg); ++ raw = readl(g->window[1] + reg); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x (write fence)\n", reg, raw); ++ return 0; ++} ++ ++static int g5p_fender_poll(void *ctx, u32 reg, u32 mask, u32 expected) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 poll R1+%05x mask=%08x expected=%08x (100ms)\n", ++ reg, mask, expected); ++ ret = readl_poll_timeout(g->window[1] + reg, raw, ++ (raw & mask) == expected, 10, 100000); ++ dev_info(g->dev, "G2 poll raw R1+%05x=%08x ret=%d\n", reg, raw, ret); ++ return ret; ++} ++ ++static int g5p_r74e_power_poll(struct g5p_diag *g, u32 expected, u32 timeout_us, ++ u64 *elapsed_ns, u32 *samples) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u64 start = ktime_get_mono_fast_ns(); ++ u32 raw = U32_MAX, n = 0; ++ int ret; ++ ++ for (;;) { ++ if (READ_ONCE(b->alive)) { ++ ret = -EHOSTDOWN; ++ break; ++ } ++ raw = readl(g->window[1] + 0x1000); ++ n++; ++ if ((raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ /* wake112: the reads taken; 0 means raw is still the U32_MAX initializer, not a read. */ ++ *samples = n; ++ r->early_power_raw = raw; ++ return ret; ++} ++ ++/* wake112 (r74e112.h): a failed deferred proof is the cold power abort, absorbing. */ ++static void g5p_r74e_fail(struct g5p_r73b *r, u32 why) ++{ ++ r->early_proof_why = why; ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ atomic_set_release(&r->early_proof, G5P_PX_FAILED); ++} ++ ++/* The once-owner takes the deferred proof right before its one R1+0x1000 read. ++ * false: that read must not happen, and the proof has failed. ++ */ ++static bool g5p_r74e_take(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_px_pre p; ++ u32 why; ++ ++ if (atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, G5P_PX_TAKEN) != ++ G5P_PX_DEFERRED) ++ return false; ++ p = (struct g5p_px_pre){ ++ .gen = r->early_proof_gen, ++ .gen_now = g5p_ds_gen(), ++ .alive = READ_ONCE(b->alive), ++ .released = READ_ONCE(b->released), ++ .stopped = READ_ONCE(b->stopped), ++ .failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ .r1_allowed = g5p_r8_r1_allowed(), ++ }; ++ r->early_proof_gen_now = p.gen_now; ++ why = g5p_px_check(&p); ++ if (why) ++ g5p_r74e_fail(r, why); ++ return !why; ++} ++ ++/* The owner's read decides: only the state field 1 licenses the release. */ ++static void g5p_r74e_settle(struct g5p_r73b *r, u32 raw) ++{ ++ if (!g5p_px_on(raw)) { ++ g5p_r74e_fail(r, G5P_PXW_STATE); ++ return; ++ } ++ WRITE_ONCE(r->early_release_ready, true); ++ atomic_set_release(&r->early_proof, G5P_PX_PROVEN); ++} ++ ++/* No ALIVE wait ends with the proof still owed: close it, no register access. ++ * true: it was still owed. ++ */ ++static bool g5p_r74e_close(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ ++ if (!r || atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, ++ G5P_PX_TAKEN) != G5P_PX_DEFERRED) ++ return false; ++ g5p_r74e_fail(r, G5P_PXW_UNTAKEN); ++ return true; ++} ++ ++/* boot_gpu_perf_seed 3/4: the sampler's last-sample time policy+0x8f8 = now. Seed 3 stamps at the ++ * start of the release (flown 2026-11-03: 38 ms before the stream's end, the window was missed); ++ * seed 4 stamps as the last host write before the one that lets META run, the GPC1 request ++ * R1+0x1000 = 0x11 (writel orders the RAM store before it). ++ */ ++static void g5p_perf_seed_stamp(struct g5p_boot *b, u32 mode) ++{ ++ u32 o; ++ ++ if (mode == 4 ? b->perf_seed_mode < 4 : b->perf_seed_mode != mode) ++ return; /* the GPC1 stamp serves seeds 4 and 5 */ ++ o = b->layout.offset[G5P_PERF_POLICY] + 0x8f8; ++ b->seed_stamp = arch_timer_read_counter(); ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)b->seed_stamp)); ++ o += 4; ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)(b->seed_stamp >> 32))); ++ wmb(); ++} ++ ++static int g5p_r74e_fender_start(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxp latch; ++ bool on_ran = false; ++ u32 samples = ++ 0; /* wake112: the reads of the poll that ran last (the printed raw) */ ++ int ret; ++ ++ r->early_power_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_r74e_power_poll(g, 0, 1000, &r->early_off_ns, ++ &samples); ++ if (!ret) { ++ g5p_perf_seed_stamp(b, 4); ++ ret = g5p_fender_write(g, 0x1000, ++ 0x10 | (b->perf_poweron ?: 1)); ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ if (!ret) { ++ on_ran = true; ++ /* poweron: the first request above pstate 1, a voltage step; a longer bound */ ++ ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, ++ b->perf_poweron ? 10000 : 1800, ++ &r->early_on_ns, &samples); ++ if (b->perf_poweron) ++ dev_emerg( ++ g->dev, ++ "GPUPERF poweron q=%u r1_1000=%08x on_ns=%llu ret=%d end=gpuperf\n", ++ b->perf_poweron, r->early_power_raw, ++ r->early_on_ns, ret); ++ } ++ /* wake112: only ALIVE preempting the on-poll before its first read defers; the ++ * first-ALIVE owner owes the proof. A sampled on-poll (0x11, ...) aborts as before. ++ */ ++ latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples); ++ if (latch == G5P_PXP_DEFER) { ++ r->early_defer_ret = ret; ++ r->early_proof_gen = g5p_ds_gen(); ++ ret = 0; ++ atomic_set_release(&r->early_proof, G5P_PX_DEFERRED); ++ } ++ r->early_power_ret = ret; ++ if (latch == G5P_PXP_ABORT) { ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ } else if (latch == G5P_PXP_READY) { ++ WRITE_ONCE(r->early_release_ready, true); ++ } ++ dev_emerg( ++ g->dev, ++ "R74E power off_ns=%llu on_ns=%llu raw=%08x ret=%d release_ready=%u alive=%u " ++ "sampled=%u end=r74e\n", ++ r->early_off_ns, r->early_on_ns, r->early_power_raw, ret, ++ r->early_release_ready, READ_ONCE(b->alive), samples); ++ if (latch == G5P_PXP_DEFER) ++ dev_emerg( ++ g->dev, ++ "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u end=r74e112\n", ++ r->early_proof_gen, r->early_defer_ret, ++ r->early_power_raw, READ_ONCE(b->alive)); ++ return ret; ++} ++ ++static void g5p_fifo_word(struct g5p_diag *g, u32 word) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 raw; ++ ++ if (b->fifo_error) ++ return; ++ if (b->fifo < 0x10000 || b->fifo >= 0x10200) { ++ b->fifo_error = -EOVERFLOW; ++ return; ++ } ++ writel(word, g->window[1] + b->fifo); ++ raw = readl(g->window[1] + b->fifo); ++ if (raw != word) { ++ dev_err(g->dev, ++ "G2 FIFO verify R1+%05x actual=%08x expected=%08x\n", ++ b->fifo, raw, word); ++ b->fifo_error = -EIO; ++ } ++ b->fifo += 4; ++} ++ ++static void g5p_fifo32(struct g5p_diag *g, u32 reg, u32 value, u32 flags) ++{ ++ dev_info(g->dev, "G2 FIFO R0+%05x=%08x flags=%02x\n", reg, value, ++ flags); ++ g5p_fifo_word(g, (reg & 0xff00ffff) | (flags & 0xff) << 16); ++ g5p_fifo_word(g, value); ++} ++ ++static void g5p_fifo64(struct g5p_diag *g, u32 reg, u64 value) ++{ ++ g5p_fifo32(g, reg + 4, value >> 32, 0); ++ g5p_fifo32(g, reg, value, 0xff); ++} ++ ++static int g5p_boot_fifo_dump(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off, raw, end = min(b->fifo, 0x10200U); ++ ++ if (end <= 0x10000) ++ return -ENODATA; ++ dev_info(g->dev, "G2 FIFO dump R1+10000..%05x (<=128 words)\n", ++ end - 4); ++ for (off = 0x10000; off < end; off += 4) { ++ raw = readl(g->window[1] + off); ++ dev_info(g->dev, "G2 FIFO raw R1+%05x=%08x\n", off, raw); ++ } ++ return 0; ++} ++ ++static u64 g5p_snapshot64(struct g5p_boot *b, u32 reg) ++{ ++ u32 i; ++ ++ for (i = 0; i + 1 < b->rgx_count; i++) ++ if (g5p_rgx_regs[i] == reg && g5p_rgx_regs[i + 1] == reg + 4) ++ return (u64)b->rgx_raw[i + 1] << 32 | b->rgx_raw[i]; ++ return 0; ++} ++ ++static void g5p_release_observe(void *ctx, const char *stage) ++{ ++ ; ++ ; ++} ++ ++/* A diagnostic notification ACK only, never engine repair or a retry loop. ++ * Snapshot first, require the active wrapper, ACK only200 once, fence and ++ * re-sample. A cleared event alone is not evidence the underlying fault healed. ++ */ ++static int g5p_release_ack(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 state, event, after; ++ ++ if (!b->release_ack) ++ return 0; ++ if (!b->released || b->stopped || b->release_ack_attempted) ++ return -EPERM; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_fault_snapshot(g, "release-ack-before"); ++ g5p_mmu_progress(g, "release-ack-event", 0x130); ++ event = readl(g->rgx + 0x130); ++ b->release_ack_attempted = true; ++ if (event & 0x200) { ++ g5p_mmu_progress(g, "release-ack-write", 0x138); ++ writel(0x200, g->rgx + 0x138); ++ /* Ordered same-device read fences the single write. */ ++ g5p_mmu_progress(g, "release-ack-fence", 0x130); ++ after = readl(g->rgx + 0x130); ++ g5p_mmu_record( ++ g, ++ "MMU release-ack mask=00000200 before=%08x after=%08x; notification only\n", ++ event, after); ++ } else { ++ g5p_mmu_record(g, ++ "MMU release-ack skipped=no-event before=%08x\n", ++ event); ++ } ++ g5p_fault_snapshot(g, "release-ack-after"); ++ return 0; ++} ++ ++static int g5p_boot_program_stream(struct g5p_diag *g, bool release) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ u64 reset; ++ int ret; ++ ++ if (b->armed && !release) ++ return -EALREADY; ++ ret = g5p_boot_irq_setup(g); ++ if (ret) ++ return ret; ++ g5p_boot_snapshot(g, release ? "before-release" : "before-arm", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ ; ++ ret = 0; ++ if (ret) ++ goto out; ++ /* Slave-port access remains forbidden for the rest of this boot. */ ++ g->reset_transition = true; ++ /* c9fa10: core request false, GPC 0. Establish an actual state edge; ++ * wake left GPC at 1, so writing 1 again did not prove FIFO execution. ++ * No R0 accesses between this point and the successful active-state poll. ++ */ ++ dev_info(g->dev, "G2 Fender park: core=0, GPC=0\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ goto out; ++ /* c9fc10: core request true before prepareBoot c9feb4. */ ++ dev_info(g->dev, "G2 Fender prepare: core=20, require GPC=0\n"); ++ ret = g5p_fender_hold(&io); ++ if (ret) ++ goto out; ++ dev_info(g->dev, ++ "G2 init stream: master boot, release=%u; FIFO reset\n", ++ release); ++ ret = g5p_fender_write(g, 0x10, 0); ++ if (ret) ++ goto out; ++ /* setMode(3), after Fender prepare cleared +10, before firmware runs. ++ * Both arm and release execute prepare; never reapply after ALIVE. ++ */ ++ ; ++ ; ++ ; ++ g5p_fender_write(g, 0, 0x40000000); ++ ; ++ ; ++ b->fifo = 0x10000; ++ b->fifo_error = 0; ++ ; ++ ; ++ g5p_fifo64(g, 0x100, ++ G5P_BOOT_RESET1 | (0 ? G5P_BOOT_R46_RESET_BITS : 0)); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x00e7fe071ffffc1dULL); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x100000000ULL); ++ ; ++ ; ++ g5p_fifo32(g, 0xbf8, 1, 0xff); ++ ; ++ ; ++ g5p_fifo32(g, 0xe140, 0, 0xff); ++ ; ++ ; ++ g5p_fifo64(g, 0xe148, b->catalogue.dma >> 12); ++ ; ++ ; ++ g5p_fifo64(g, 0xe200, 3); ++ ; ++ ; ++ g5p_fifo64(g, 0xe208, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fifo64(g, 0xe210, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fender_write(g, 0x2010, 0xbf); ++ ; ++ ; ++ g5p_fender_write(g, 0x2014, 0x233); ++ ; ++ ; ++ g5p_fifo64(g, 0xe160, b->zero.dma); ++ ; ++ ; ++ g5p_fifo64(g, 0xbb8, 0); ++ ; ++ ; ++ if (release) { ++ g5p_fifo64(g, 0x100, 0); ++ ; ++ } ++ g5p_fifo_word(g, 0xffff0000); ++ ; ++ ; ++ ret = b->fifo_error; ++ if (ret) ++ goto out; ++ dma_wmb(); ++ dev_info( ++ g->dev, ++ "G2 FIFO stored and verified: %u words (not execution proof)\n", ++ (b->fifo - 0x10000) / 4); ++ if (release) { ++ WRITE_ONCE(b->released, true); ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_FIFO, G5P_R73B_NONE, 0); ++ ret = 0; ++ if (ret) ++ goto out; ++ } ++ /* c9f954: core request false, then GPC 1. */ ++ dev_info(g->dev, "G2 Fender execute: core=0, GPC=1 (from 0)\n"); ++ if (release && !(b->r73b && 1)) { ++ g5p_perf_seed_stamp( ++ b, ++ 4); /* the non-r74e start: as close as this path gets */ ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ ret = release && b->r73b && 1 ? g5p_r74e_fender_start(g) : ++ release ? g5p_fender_start_observed_q(&io, g5p_release_observe, ++ b->perf_poweron ?: 1) : ++ g5p_fender_start(&io); ++ if (release) ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_GPC, G5P_R73B_NONE, ret); ++ if (ret) ++ goto out; ++ /* r74e: after GPC1 is active, wait for ALIVE without optional MMIO. */ ++ if (release && b->r73b && 1) ++ return 0; ++ usleep_range(10, 20); ++ g5p_boot_snapshot(g, release ? "after-release" : "after-arm", ++ release ? G5P_READBACK_WRAPPER_ONLY : ++ G5P_READBACK_RESET_HELD); ++ ; ++ /* Only the fault-status subset above is sampled after release. */ ++ if (release) ++ return g5p_release_ack(g); ++ if (!b->rgx_valid) ++ return -EIO; ++ reset = g5p_snapshot64(b, 0x100); ++ dev_info(g->dev, "G2 verify reset actual=%016llx expected=%016llx\n", ++ reset, 0x100000000ULL); ++ dev_info( ++ g->dev, ++ "G2 expected catalogue=%llx scratch=%llx SLC=3 (not read: state safety)\n", ++ (u64)b->catalogue.dma >> 12, (u64)b->zero.dma); ++ if (reset != 0x100000000ULL) ++ return -EIO; ++ dev_info( ++ g->dev, ++ "G2 reset-held gate verified; remaining FIFO effects await firmware event\n"); ++ b->armed = true; ++ return 0; ++out: ++ /* A slow r74e epoch is scoreable without another unsafe MMIO read. */ ++ if (release && b->r73b && 1 && READ_ONCE(b->r73b->early_power_abort)) ++ return ret; ++ /* Wrapper-only snapshot if GPC remains off/busy. */ ++ g5p_boot_snapshot(g, "transition-or-FIFO-failed", ++ G5P_READBACK_WRAPPER_ONLY); ++ return ret; ++} ++ ++static int g5p_boot_arm(struct g5p_diag *g) ++{ ++ return g5p_boot_program_stream(g, false); ++} ++ ++static int g5p_boot_load(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off; ++ ++ /* c9f1bc/ca43b0: segment and LDR configuration pairs are supplied in ++ * mapped RAM. Polling MSLVCTRL1 in the reset-held state locks this SoC. ++ */ ++ for (off = 0; off < G5P_IMAGE_SIZE; off += G5P_GPU_PAGE) ++ if (memcmp(b->image[off / G5P_GPU_PAGE].cpu, ++ g->image.data + off, G5P_GPU_PAGE)) ++ return -EBADMSG; ++ dma_wmb(); ++ b->copied = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 RAM load verified bytes=%x segment_pairs=13 LDR_pairs=27; no slave-port MMIO\n", ++ G5P_IMAGE_SIZE); ++ return 0; ++} ++ ++static int g5p_boot_release(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u64 seed_ts = 0; ++ int ret; ++ ++ if (b->released) ++ return -EALREADY; ++ /* recovery108: fail closed before the release touches anything. */ ++ ret = g5p_ds_hook_release_gate(g); ++ if (ret) ++ return ret; ++ g5p_r73b_release_entry(g); ++ if (b->r73b && 1) { ++ ret = g5p_r73e_prepare(g); ++ if (ret) ++ return ret; ++ } ++ dev_info(g->dev, ++ "G2 release: META master boot; no GPU job submitted\n"); ++ /* Reproduce the complete original stream, including its final reset release. ++ * Do not assume the experimental arm stage preserves a FIFO read cursor. ++ */ ++ /* boot_gpu_perf_seed 3: the sampler's last-sample time = now, as close to the release as the ++ * host gets (perf-envelope.h); the firmware divides (now - policy+0x8f8) by 24000 * policy+4. ++ */ ++ if (b->perf_seed_mode >= 3) { ++ seed_ts = arch_timer_read_counter(); ++ g5p_perf_seed_stamp(b, 3); ++ } ++ ret = g5p_boot_program_stream(g, true); ++ if (seed_ts) { ++ u64 end = arch_timer_read_counter(), ++ rate = arch_timer_get_rate(); ++ ++ /* always printed on a seed 3/4 boot, before ALIVE is awaited */ ++ dev_emerg( ++ g->dev, ++ "GPUPERF seed mode=%u stamp=%llx release_to_gpc1_us=%llu stamp_to_gpc1_us=%llu " ++ "gpc1_to_stream_end_us=%llu stamp_to_stream_end_us=%llu period_us=16000 " ++ "before_alive=1 end=gpuperf\n", ++ b->perf_seed_mode, b->seed_stamp, ++ b->seed_gpc1 ? ++ div_u64((b->seed_gpc1 - seed_ts) * 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? div_u64((b->seed_gpc1 - b->seed_stamp) * ++ 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? ++ div_u64((end - b->seed_gpc1) * 1000000ULL, ++ rate) : ++ 0, ++ div_u64((end - b->seed_stamp) * 1000000ULL, rate)); ++ } ++ g5p_r73b_release_exit(g, ret); ++ g5p_ds_hook_release(g, ret); ++ if (ret) ++ return ret; ++ if (b->r73b && 1) ++ return g5p_r73c_fast_control(g); ++ return 0; ++} ++ ++static int g5p_boot_events_once(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 3 * HZ; ++ u64 submit_wait_start = 0; ++ u32 pass = 0, submit_wait_passes = 0; ++ int ret; ++ ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_EVENTS, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->alive)) { ++ if (b->r73b && 1) { ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ if (ret == -EAGAIN) ++ submit_wait_start = ++ ktime_get_mono_fast_ns(); ++ while (ret == -EAGAIN && ++ ktime_get_mono_fast_ns() - ++ submit_wait_start < ++ 100ULL * NSEC_PER_MSEC) { ++ usleep_range(50, 100); ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ submit_wait_passes++; ++ } ++ if (submit_wait_passes) ++ dev_info( ++ g->dev, ++ "R73C alive-submit synchronize wait_ns=%llu passes=%u ret=%d submitted=%u end=r73c\n", ++ ktime_get_mono_fast_ns() - ++ submit_wait_start, ++ submit_wait_passes, ret, ++ READ_ONCE( ++ b->control_submitted)); ++ if (ret) ++ return ret; ++ } ++ dev_info( ++ g->dev, ++ "G2 ALIVE type=0 word1=%08x events=%u IRQ174=%d\n", ++ b->alive_word, b->events, ++ atomic_read(&b->irqs)); ++ g5p_r41_bridge_firmware(g); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, ++ "G2 no alive event: events=%u IRQ174=%d; preserve baseline\n", ++ b->events, atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++/* recovery108: every ALIVE wait (release fast path and the events command) is an ALIVE event. */ ++static int g5p_boot_events(struct g5p_diag *g) ++{ ++ int ret = g5p_boot_events_once(g); ++ ++ /* wake112: no ALIVE wait ends with an R74E proof still owed (r74e112.h). */ ++ if (g5p_r74e_close(g)) { ++ dev_emerg( ++ g->dev, ++ "R74E112 close gen=%u why=untaken ret=%d end=r74e112\n", ++ g->boot->r73b->early_proof_gen, ret); ++ if (!ret) ++ ret = -EHOSTDOWN; ++ } ++ g5p_ds_hook_alive(g, ret); ++ return ret; ++} ++ ++static int g5p_boot_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b && b->stopped) ++ return -ESHUTDOWN; ++ if (sysfs_streq(command, "map")) ++ return g5p_boot_maps(g); ++ /* Read-only diagnostics remain available after a latched boot failure. */ ++ if (b && g->powered && g->fender_ready && g->id_checked) { ++ if (sysfs_streq(command, "snapshot")) { ++ g5p_boot_snapshot(g, "explicit", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ return 0; ++ } ++ if (sysfs_streq(command, "fifo")) ++ return g5p_boot_fifo_dump(g); ++ } ++ if (!boot_experiment || !b || !b->maps_ready || b->failed) ++ return -EPERM; ++ if (sysfs_streq(command, "arm")) { ++ ret = g5p_boot_arm(g); ++ } else if (sysfs_streq(command, "load") && b->armed && !b->released && ++ !b->copied) { ++ ret = g5p_boot_load(g); ++ g5p_ds_hook_load(g, ret); ++ } else if (sysfs_streq(command, "release") && b->copied) { ++ ret = g5p_boot_release(g); ++ } else if (sysfs_streq(command, "events") && b->released) { ++ ret = g5p_boot_events(g); ++ g5p_r73b_events_exit(g, ret); ++ } else { ++ return -EPERM; ++ } ++ if (ret && ret != -EALREADY) ++ b->failed = true; ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/compute-user-layout.h b/drivers/gpu/apple-g5p/compute-user-layout.h +new file mode 100644 +index 000000000..78b459832 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/compute-user-layout.h +@@ -0,0 +1,207 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CDM channel and per-job firmware objects. Layout helpers perform no I/O. ++ * The kernel owns the channel, control watermarks, pointer ring, work record, ++ * completion microsequence, timestamp queue, stamps and dependency object. ++ * GPU program instructions are provided by userspace, never embedded here. ++ * ++ * Compute uses the context's TA MMU slot. The scheduler serializes TA and CDM ++ * work across contexts, and a lost CDM job blocks the channel until retirement ++ * or recovery. Stamp identity zero is safe because the TA event latch is armed ++ * only while executing a TA phase; completion also requires coherent stamps. ++ */ ++#ifndef G5P_COMPUTE_USER_LAYOUT_H ++#define G5P_COMPUTE_USER_LAYOUT_H ++ ++#define G5P_CU_ENGINE 2U ++#define G5P_CU_OFFSET (G5P_R22_3D + 0x3000U) /* in the shared region */ ++#define G5P_CU_BYTES 0x1000U ++#define G5P_CU_W 0x400U ++#define G5P_CU_W_BYTES 0x290U /* through the timestamps at W+0x280..0x288 */ ++#define G5P_CU_SKU 0x800U ++#define G5P_CU_SKU_BYTES 0xc0U ++#define G5P_CU_TQ 0x900U ++#define G5P_CU_START 0x940U /* W258: the retirement mirror */ ++#define G5P_CU_END 0x944U /* W25c: finish33's stamp */ ++#define G5P_CU_DEP 0x980U ++#define G5P_CU_DEP_BYTES 0x280U ++#define G5P_CU_IDENTITY 0U ++#define G5P_CU_SEQ_MAX 0x7fffffffU ++ ++/* Firmware configuration W+0x244 and compute record type. */ ++#define G5P_CU_W244 0x00a00004U ++#define G5P_CU_TAG 3U ++ ++/* The validated register images of one job (compute-user-validate.h fills it). */ ++struct g5p_cu_regs { ++ u64 stream; /* W+0x70 -> R0+0x480 */ ++ u64 terminate; /* the stream's terminate word: W+0x214, W+0x23c */ ++ u64 state[4]; /* W+0x1ec..0x204 */ ++ u64 tail; /* W+0x60 -> R0+0x498, W+0x20c */ ++ u32 config[2]; /* W+0x10, W+0x14 -> R0+0x4c0, 0x4c8 */ ++ /* {code | data << 32}: W+0x21c, W+0x22c, W+0x68 -> R0+0x4b8 */ ++ u64 store, resume, term; ++ u32 store1, resume1; /* W+0x224 -> R0+0x4b0, W+0x234 */ ++}; ++ ++/* Job `seq` (1-based) occupies pointer-ring entry (seq - 1) % 16 and publishes seq % 16. */ ++static inline u32 g5p_cu_ring_index(u32 seq) ++{ ++ return (seq - 1) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_cu_endpoint(u32 seq) ++{ ++ return seq % G5P_WORK_SLOTS; ++} ++ ++/* The channel at its epoch start: header, ring capacity, timestamp queue, dependency object ++ * (g5p_compute_dependency()); watermarks, ring and stamps zero. `p` is the channel's ++ * G5P_CU_BYTES, `meta` its META address. ++ */ ++static inline void g5p_cu_channel_init(u8 *p, u32 meta) ++{ ++ u8 *d = p + G5P_CU_DEP; ++ ++ memset(p, 0, G5P_CU_BYTES); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_CU_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_CU_TQ + 4, meta + G5P_CU_TQ); ++ g5p_put64(d + 0x30, 1); ++ g5p_put64(d + 0x48, ~0ULL); ++ g5p_put32(d + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(d + off + 8, 0xf); ++} ++ ++/* The SKU microsequence of job `seq`: start32, timestamp, wait COMPUTE_FINISHED (event 4), ++ * wait (R0+0x4a0 & 1) == 1, timestamp, finish33 (W25c = W260), the backlink, end. ++ * Each sequence reference uses the current job number. ++ */ ++static inline u32 g5p_cu_sku(u8 *sku, u32 meta, u32 global, u32 slot, u32 seq) ++{ ++ u32 w = meta + G5P_CU_W, end = meta + G5P_CU_END; ++ const u32 words[] = { ++ 0x32, ++ w + 0x10, ++ w + 0x60, ++ global + 0x720, ++ meta, ++ slot, ++ 0, ++ 0, ++ 0, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ 0x80000018, ++ w + 0x278, ++ w + 0x280, ++ w + 0x280, ++ seq, ++ 0x101, ++ 0x004a000a, ++ 1, ++ 1, ++ 0x18, ++ w + 0x278, ++ w + 0x280, ++ w + 0x284, ++ seq, ++ 0x33, ++ global + 0x720, ++ meta, ++ slot, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ end, ++ seq, ++ (u32)-0x6c, ++ 0x40000017, ++ }; ++ ++ memset(sku, 0, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(words); i++) ++ g5p_put32(sku + i * 4, words[i]); ++ return ARRAY_SIZE(words) * 4; ++} ++ ++/* Every per-job object of job `seq` on slot `slot`: W, SKU and its pointer-ring entry. The ++ * channel header, the dependency object and the timestamp queue stay as initialised. ++ */ ++static inline void g5p_cu_job(u8 *p, u32 meta, u32 global, u32 slot, u32 seq, ++ const struct g5p_cu_regs *k) ++{ ++ u8 *w = p + G5P_CU_W; ++ ++ memset(w, 0, G5P_CU_W_BYTES); ++ g5p_put32(w, G5P_CU_TAG); ++ g5p_put32(w + 8, slot); ++ g5p_put32(w + 0xc, meta + G5P_CU_DEP); ++ g5p_put32(w + 0x10, k->config[0]); ++ g5p_put32(w + 0x14, k->config[1]); ++ g5p_put64(w + 0x60, k->tail); ++ g5p_put64(w + 0x68, k->term); ++ g5p_put64(w + 0x70, k->stream); ++ /* The six selected-resource entries at W+0x78..0xa7 stay zero (disabled). */ ++ g5p_put32(w + 0x1e0, meta + G5P_CU_SKU); ++ g5p_put32(w + 0x1e4, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x1ec + i * 8, k->state[i]); ++ g5p_put64(w + 0x20c, k->tail); ++ g5p_put64(w + 0x214, k->terminate); ++ g5p_put64(w + 0x21c, k->store); ++ g5p_put32(w + 0x224, k->store1); ++ g5p_put64(w + 0x22c, k->resume); ++ g5p_put32(w + 0x234, k->resume1); ++ g5p_put64(w + 0x23c, k->terminate); ++ g5p_put32(w + 0x244, G5P_CU_W244); ++ g5p_put32(w + 0x258, meta + G5P_CU_START); ++ g5p_put32(w + 0x25c, meta + G5P_CU_END); ++ g5p_put32(w + 0x260, seq); ++ g5p_put32(w + 0x264, G5P_CU_IDENTITY); ++ g5p_put32(w + 0x270, seq); ++ g5p_cu_sku(p + G5P_CU_SKU, meta, global, slot, seq); ++ g5p_put32(p + G5P_WORK_RING + g5p_cu_ring_index(seq) * 4, ++ meta + G5P_CU_W); ++} ++ ++/* Retirement of job `seq` (the r19d predicate, RAM only): W still names our stamps and ++ * sequence, finish33's stamp and the retirement mirror both at seq, all five channel ++ * watermarks at its endpoint. ++ */ ++static inline bool g5p_cu_retired(const u8 *p, u32 meta, u32 seq) ++{ ++ const u8 *w = p + G5P_CU_W; ++ u32 endpoint = g5p_cu_endpoint(seq); ++ ++ if (g5p_le32(w + 0x258) != meta + G5P_CU_START || ++ g5p_le32(w + 0x25c) != meta + G5P_CU_END || ++ g5p_le32(w + 0x260) != seq || g5p_le32(p + G5P_CU_START) != seq || ++ g5p_le32(p + G5P_CU_END) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* The stamps are the previous job's or ours, and no watermark runs past our endpoint's ++ * predecessor: anything else is a protocol error (another writer, or the wrong channel). ++ */ ++static inline bool g5p_cu_stamp_sane(const u8 *p, u32 seq) ++{ ++ u32 start = g5p_le32(p + G5P_CU_START), end = g5p_le32(p + G5P_CU_END); ++ ++ return (start == seq - 1 || start == seq) && ++ (end == seq - 1 || end == seq); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +new file mode 100644 +index 000000000..5bfa441e1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -0,0 +1,583 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared private bring-up implementation for mutually exclusive frontends. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++/* perf60 clock: which GPC states license an R0 access (a bell, a TLB pass, a register sample). */ ++/* release8 default */ ++ ++/* The GPC is powered and not mid-transition: state 1 only, or (gpc_any_pstate) any nonzero ++ * pstate nibble with no request in flight. A parked GPC (nibble 0) never qualifies. ++ */ ++static inline bool g5p_gpc_ok(u32 raw) ++{ ++ ; ++ ; ++ return !(raw & 0x10) && (raw & 0xf); ++} ++ ++/* The pure headers (readback.h, r74e112.h, render-user-wake.h, dirtystart108.h) judge the GPC state ++ * through G5P_GPC_ON(); their default is the old == 1, for the host tests. The kernel takes the rule. ++ */ ++#define G5P_GPC_ON(raw) g5p_gpc_ok(raw) ++ ++#include "pack.h" ++#include "work-layout.h" ++#include "mmio-licence.h" ++#include "gpu-oc.h" ++#include ++#include "readback.h" ++ ++/* nopatch: the TYPED_V3 boot path (r74/r74f/r74g, meta6c8=1) on Apple's original image. The release8 ++ * default since its device pass (2026-11-03, release8-nopatch a10a8556b9): an explicit firmware patch ++ * request (boot_r73f_case 1..4, or the r75/r79 cases that ride on it) takes the patched path instead, ++ * so the patched fallback is boot_r73f_case 1 alone (g5p_fw_unpatched_want()). ++ */ ++/* release8 default */ ++ ++/* The effective nopatch latch: the parameter, unless a patched image is asked for. */ ++static u32 g5p_fw_unpatched_want(u32 param, u32 r73f, u32 r75, u32 r79) ++{ ++ return param == 1 && !r73f && !r75 && !r79; ++} ++ ++#define G5P_MSLVDATAX 0x0a00 ++#define G5P_MSLVDATAT 0x0a08 ++#define G5P_MSLVCTRL0 0x0a10 ++#define G5P_MSLVCTRL1 0x0a18 ++#define G5P_MSLVIRQSTATUS 0x0ac8 ++#define G5P_SP_READY (BIT(24) | BIT(26)) ++#define G5P_FW_NAME "apple/agx-g5p-c0.ldr" ++ ++struct g5p_boot; ++ ++struct g5p_diag { ++ struct g5p_diagnostics *diagnostics; ++ struct g5p_boot *boot; ++ ++ spinlock_t slot_lock; ++ DECLARE_BITMAP(user_slots, 256); ++ struct device *dev; ++ void __iomem *window[2]; ++ void __iomem *rgx; ++ struct regmap *pmgr; ++ /* Serializes commands and all hardware access. */ ++ struct mutex lock; ++ bool powered; ++ bool loaded; ++ ++ struct g5p_ldr_image image; ++ bool fender_ready; ++ bool id_checked; ++ int last_result; ++ u32 words_written; ++ u32 core_id, core_revision; ++ u32 last_sp_addr; ++ bool reset_transition; ++ void (*event_notify)(void *cookie, const u32 words[11]); ++ void *event_cookie; ++ /* The GPU register licence (mmio.inc, docs/gpu/experiments/mmio-access.md §4). mmio_lock is ++ * a leaf: taken inside event_lock where both are held, never the reverse, nothing inside. ++ * mmio_off forbids R1/R0 through the accessor from before any PMGR gfx gate until Fender is ++ * up again (g5p_mmio_gate(), cleared by g5p_fender_wake_once()). ++ */ ++ spinlock_t mmio_lock; ++ bool mmio_off; ++ u32 mmio_max_pstate; ++ struct g5p_mmio_self *mmio_self; ++}; ++ ++/* Before every PMGR gfx gate (boot R33 off, dirty-start, kexec quiesce and in-boot recovery ++ * cycles): no accessor MMIO from here until g5p_fender_wake_once() brings Fender up again. ++ * Apple clears fender+0xc inside setPlatformPower(0), before its gate (G5). ++ */ ++static void g5p_mmio_gate(struct g5p_diag *g) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = true; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++} ++ ++/* gpu-recovery108: retire predicates (teardown.inc) and dirty-start hooks (dirtystart108.inc). */ ++static bool g5p_r8_r1_allowed(void); ++static bool g5p_r8_is_retired(void); ++static void g5p_ds_hook_probe(struct g5p_diag *g); ++static void g5p_ds_hook_power(struct g5p_diag *g); ++static int g5p_ds_hook_reset(struct g5p_diag *g); ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret); ++static int g5p_ds_hook_release_gate(struct g5p_diag *g); ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret); ++/* wake112: the reset generation of a deferred R74E proof (dirtystart108.inc). */ ++static u32 g5p_ds_gen(void); ++ ++#include "diagnostics.inc" ++ ++/* The GPU ADT clock/power gate124 resolves to ps-regs[4]+index8*8. ++ * Read via its existing syscon, sharing the provider mapping without writes. ++ */ ++static void g5p_power_observer_init(struct g5p_diag *g) ++{ ++ struct device_node *domain, *parent; ++ u32 offset; ++ ++ domain = of_parse_phandle(g->dev->of_node, "power-domains", 0); ++ if (!domain) ++ return; ++ parent = of_get_parent(domain); ++ if (of_device_is_compatible(domain, "apple,s8000-pmgr-pwrstate") && ++ !of_property_read_u32(domain, "reg", &offset) && ++ offset == 0x80340 && parent && ++ of_device_is_compatible(parent, "apple,s8000-pmgr")) { ++ g->pmgr = syscon_node_to_regmap(parent); ++ if (IS_ERR(g->pmgr)) ++ g->pmgr = NULL; ++ } ++ of_node_put(parent); ++ of_node_put(domain); ++} ++ ++/* pmgr-pwrstate.c masks; history flags are sticky, not current clock state. */ ++static void g5p_power_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) { ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx unavailable: provider not resolved\n", ++ stage); ++ return; ++ } ++ dev_info(g->dev, "POWER %s reading PMGR gfx20e080340 gate124\n", stage); ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (ret) { ++ dev_info(g->dev, "POWER %s PMGR gfx read error=%d\n", stage, ++ ret); ++ return; ++ } ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx20e080340=%08x target=%x actual=%x auto=%u auto_state=%x " ++ "min=%x reset=%u parent_off=%u disabled=%u; current outer-domain fields only\n", ++ stage, raw, raw & 0xf, (raw >> 4) & 0xf, (raw >> 28) & 1, ++ (raw >> 24) & 0xf, (raw >> 16) & 0xf, raw >> 31, ++ (raw >> 11) & 1, (raw >> 10) & 1); ++ dev_info( ++ g->dev, ++ "POWER %s PMGR history was_clkgated=%u was_pwrgated=%u; sticky history, not " ++ "current state\n", ++ stage, (raw >> 9) & 1, (raw >> 8) & 1); ++} ++ ++static int g5p_power(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->powered) ++ return 0; ++ dev_info(g->dev, "request gfx power domain\n"); ++ ret = pm_runtime_resume_and_get(g->dev); ++ if (ret < 0) ++ return ret; ++ g->powered = true; ++ dev_info(g->dev, "gfx runtime PM active\n"); ++ g5p_ds_hook_power(g); ++ return 0; ++} ++ ++#include "power-r33.inc" ++#include "power-r41.inc" ++ ++static bool boot_gfx_off_done; ++ ++static int g5p_fender_wake_once(struct g5p_diag *g) ++{ ++ void __iomem *fender = g->window[1]; ++ u32 value; ++ int ret; ++ ++ ret = g5p_power(g); ++ if (ret) ++ return ret; ++ /* recovery108 mode 3: the one bounded incoming cycle, before any GPU MMIO. */ ++ ret = g5p_ds_hook_reset(g); ++ if (ret) ++ return ret; ++ /* R33: once per boot, before the first Fender write (no GPU MMIO yet). */ ++ ; ++ ret = g5p_r41_bridge_prepare(g); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "W R1+10000=ffff0000 (Apple ...6c93fe8)\n"); ++ writel(0xffff0000, fender + 0x10000); ++ dev_info(g->dev, "W R1+00000=40000000 (Apple ...6c93ff0)\n"); ++ writel(0x40000000, fender); ++ dev_info(g->dev, "W R1+01000=00000011 (Apple ...6c93ff8)\n"); ++ writel(0x11, fender + 0x1000); ++ dev_info(g->dev, "poll R1+01000 bit4 clear\n"); ++ ret = readl_poll_timeout(fender + 0x1000, value, !(value & BIT(4)), 10, ++ 100000); ++ dev_info(g->dev, "Fender wake status=%08x result=%d\n", value, ret); ++ if (ret) ++ return ret; ++ usleep_range(10, 20); ++ g->fender_ready = true; ++ { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = false; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++ } ++ return 0; ++} ++ ++static int g5p_fender_wake(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->fender_ready) ++ return 0; ++ ret = g5p_fender_wake_once(g); ++ g5p_ds_hook_wake(g, ret); ++ return ret; ++} ++ ++static int g5p_sp_wait(struct g5p_diag *g) ++{ ++ u32 value; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = readl_poll_timeout(g->rgx + G5P_MSLVCTRL1, value, ++ (value & G5P_SP_READY) == G5P_SP_READY, 10, ++ 100000); ++ if (ret) ++ dev_err(g->dev, "slave port timeout CTRL1=%08x words=%u\n", ++ value, g->words_written); ++ if (ret) ++ dev_err(g->dev, "slave target=%08x\n", g->last_sp_addr); ++ return ret; ++} ++ ++static int g5p_meta_read(struct g5p_diag *g, u32 addr, u32 *out) ++{ ++ int ret; ++ ++ g->last_sp_addr = addr; ++ ret = g5p_sp_wait(g); ++ ++ if (ret) ++ return ret; ++ writel(addr | 1, g->rgx + G5P_MSLVCTRL0); ++ readl(g->rgx + G5P_MSLVCTRL0); ++ ret = g5p_sp_wait(g); ++ if (ret) ++ return ret; ++ *out = readl(g->rgx + G5P_MSLVDATAX); ++ return 0; ++} ++ ++static int g5p_firmware(struct g5p_diag *g, bool prepare) ++{ ++ static const u8 expected[SHA256_DIGEST_SIZE] = { ++ 0x33, 0xae, 0x20, 0xcb, 0xbe, 0x24, 0x98, 0xe4, ++ 0xf1, 0x19, 0xba, 0x4e, 0x9a, 0x55, 0x46, 0x7e, ++ 0x4e, 0x01, 0x92, 0xd4, 0xc8, 0x6d, 0x5b, 0xf1, ++ 0x90, 0x3a, 0xdc, 0xe6, 0x3f, 0xfa, 0x5d, 0xca, ++ }; ++ /* SHA-256 of the packed image of that LDR, unmodified (nopatch's receipt) */ ++ static const u8 g5p_fw_packed_sha[SHA256_DIGEST_SIZE] = { ++ 0x48, 0xea, 0x03, 0x14, 0x08, 0x23, 0x96, 0x23, ++ 0xd8, 0x6f, 0x79, 0xb8, 0xa5, 0xda, 0x55, 0x7e, ++ 0x25, 0x3d, 0xb1, 0x6f, 0xb6, 0x23, 0x47, 0x95, ++ 0xe8, 0xed, 0xba, 0xa5, 0xcd, 0x9b, 0xb9, 0x1a, ++ }; ++ const struct firmware *fw; ++ u8 digest[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ ret = request_firmware_direct(&fw, G5P_FW_NAME, g->dev); ++ if (ret) ++ return ret; ++ sha256(fw->data, fw->size, digest); ++ if (fw->size != 119112 || memcmp(digest, expected, sizeof(expected))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, NULL, NULL); ++ if (ret) ++ goto out; ++ if (prepare && !g->loaded) { ++ g->image.data = kvzalloc(G5P_IMAGE_SIZE, GFP_KERNEL); ++ if (!g->image.data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, g5p_pack_record, ++ &g->image); ++ if (!ret && g->image.config_count != G5P_CONFIG_WRITES) ++ ret = -EINVAL; ++ if (!ret) { ++ sha256(g->image.data, G5P_IMAGE_SIZE, digest); ++ if (memcmp(digest, g5p_fw_packed_sha, sizeof(digest))) ++ ret = -EBADMSG; ++ } ++ if (ret) { ++ kvfree(g->image.data); ++ memset(&g->image, 0, sizeof(g->image)); ++ goto out; ++ } ++ g->loaded = true; ++ dev_info( ++ g->dev, ++ "NOPATCH packed apple_original=1 packed_sha256=%32phN end=nopatch\n", ++ digest); ++ } ++out: ++ release_firmware(fw); ++ return ret; ++} ++ ++static int g5p_sample(struct g5p_diag *g) ++{ ++ u32 value; ++ unsigned int thread, reg; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ if (!g->rgx) ++ return -EINVAL; ++ dev_info(g->dev, "RGX CTRL1=%08x IRQSTATUS=%08x\n", ++ readl(g->rgx + G5P_MSLVCTRL1), ++ readl(g->rgx + G5P_MSLVIRQSTATUS)); ++ for (thread = 0; thread < 2; thread++) { ++ for (reg = 0; reg <= 0x10; reg += 0x10) { ++ ret = g5p_meta_read( ++ g, 0x04800000 + thread * 0x1000 + reg, &value); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "META T%u %s=%08x\n", thread, ++ reg ? "STATUS" : "ENABLE", value); ++ } ++ } ++ return 0; ++} ++ ++static int g5p_read_id(struct g5p_diag *g) ++{ ++ u32 words[2], version; ++ static const u32 offsets[] = { 0x20, 0x18 }; ++ unsigned int i; ++ ++ if (!g->fender_ready || READ_ONCE(g->reset_transition)) ++ return -EPERM; ++ g->id_checked = false; ++ for (i = 0; i < ARRAY_SIZE(offsets); i++) { ++ dev_info(g->dev, "R R0+%05x\n", offsets[i]); ++ words[i] = readl(g->rgx + offsets[i]); ++ dev_info(g->dev, "R0+%05x=%08x\n", offsets[i], words[i]); ++ } ++ g->core_revision = words[0]; ++ g->core_id = words[1]; ++ if ((!words[0] && !words[1]) || words[0] == ~0U || words[1] == ~0U) ++ return -ENODEV; ++ /* Match Apple readChipInfo at ...6ca644c before deeper META accesses. */ ++ version = (words[0] >> 8) & 0xff; ++ if ((words[0] & 0xff0000) != 0x90000 || ++ (version != 7 && version != 10 && version != 13 && version != 15)) ++ return -ENODEV; ++ g->id_checked = true; ++ return 0; ++} ++ ++#include "boot.inc" ++#include "mmio.inc" ++#include "kick.inc" ++#include "lifecycle.inc" ++#include "teardown.inc" ++#include "dirtystart108.inc" ++#include "recover.inc" ++#include "transport-budget.inc" ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++ ++static ssize_t result_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ return sysfs_emit( ++ buf, ++ "last=%d loaded=%u maps=%u armed=%u copied=%u released=%u alive=%u " ++ "control_complete=%u failed=%u\n", ++ READ_ONCE(g->last_result), READ_ONCE(g->loaded), ++ b && b->maps_ready, b && b->armed, b && b->copied, ++ b && b->released, b && b->alive, b && b->control_complete, ++ b && b->failed); ++} ++static DEVICE_ATTR_RO(result); ++ ++/* Cached raw words from the last explicit/automatic snapshot: no GPU MMIO. */ ++ ++/* One bring-up phase, as the command file and the probe-time bring-up run it. *ran is set ++ * when the phase itself ran (and its result was recorded). ++ */ ++static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) ++{ ++ int ret; ++ ++ *ran = false; ++ if (g5p_r8_is_retired()) ++ return -ESHUTDOWN; ++ guard(mutex) ++ (&g->lock); ++ if (g->boot && (g->boot->stopped || g->boot->user_compute_owned)) ++ return -EBUSY; ++ if (sysfs_streq(buf, "validate") || sysfs_streq(buf, "prepare")) ++ ret = g5p_firmware(g, sysfs_streq(buf, "prepare")); ++ else if (sysfs_streq(buf, "wake")) ++ ret = g5p_fender_wake(g); ++ else if (sysfs_streq(buf, "id")) ++ ret = g5p_read_id(g); ++ else if (sysfs_streq(buf, "sample")) ++ ret = g->id_checked ? g5p_sample(g) : -EPERM; ++ else if (sysfs_streq(buf, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(buf, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(buf, "control-wait")) ++ ret = g5p_control_wait(g); ++ else if (sysfs_streq(buf, "map") || sysfs_streq(buf, "arm") || ++ sysfs_streq(buf, "load") || sysfs_streq(buf, "release") || ++ sysfs_streq(buf, "events")) ++ ret = g5p_boot_command(g, buf); ++ else ++ return -EOPNOTSUPP; ++ g->last_result = ret; ++ *ran = true; ++ return ret; ++} ++ ++static ssize_t command_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ bool ran; ++ int ret; ++ ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (sysfs_streq(buf, "td-kexec")) ++ return g5p_td_command(g, buf) ?: count; ++ ret = g5p_run_phase(g, buf, &ran); ++ if (ran) ++ dev_info(dev, "command %.*s result=%d\n", (int)count, buf, ret); ++ return ret ? ret : count; ++} ++static DEVICE_ATTR_WO(command); ++ ++static struct attribute *g5p_attrs[] = { ++ &dev_attr_command.attr, ++ &dev_attr_result.attr, ++ &dev_attr_recovery_status.attr, ++ NULL, ++}; ++static const struct attribute_group g5p_group = { .attrs = g5p_attrs }; ++__ATTRIBUTE_GROUPS(g5p); ++ ++static int g5p_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct g5p_diag *g; ++ unsigned int i; ++ ++ g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL); ++ if (!g) ++ return -ENOMEM; ++ g->dev = dev; ++ if (g5p_diagnostics_init(g)) ++ return -ENOMEM; ++ mutex_init(&g->lock); ++ spin_lock_init(&g->slot_lock); ++ spin_lock_init(&g->mmio_lock); ++ g->mmio_max_pstate = 1; ++ bitmap_set(g->user_slots, 0, 7); ++ for (i = 0; i < 2; i++) { ++ g->window[i] = devm_platform_ioremap_resource(pdev, i); ++ if (IS_ERR(g->window[i])) ++ return PTR_ERR(g->window[i]); ++ } ++ g->rgx = g->window[0]; ++ g5p_power_observer_init(g); ++ platform_set_drvdata(pdev, g); ++ if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40))) ++ return -ENODEV; ++ pm_runtime_enable(dev); ++ g5p_td_attach(g); ++ g5p_ds_hook_probe(g); ++ ; ++ dev_info(dev, "mapped both ADT windows; awaiting explicit command\n"); ++ return 0; ++} ++ ++static void g5p_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ /* With teardown on, keep the PM reference: an outgoing PMGR off is unvalidated. */ ++ bool keep_power; ++ ++ ; ++ keep_power = g5p_td_remove(g); ++ if (g->powered && !keep_power) { ++ g5p_mmio_gate(g); ++ pm_runtime_put_sync(g->dev); ++ } ++ pm_runtime_disable(g->dev); ++ kvfree(g->image.data); ++} ++ ++/* Registration itself must not claim a device owned by another frontend. */ ++static int g5p_frontend_available(void) ++{ ++ struct device *dev; ++ int ret; ++ ++ if (!of_machine_is_compatible("apple,j71t")) ++ return -ENODEV; ++ dev = bus_find_device_by_name(&platform_bus_type, NULL, ++ "201000000.gpu"); ++ if (!dev) ++ return -ENODEV; ++ device_lock(dev); ++ ret = dev->driver ? -EBUSY : 0; ++ device_unlock(dev); ++ put_device(dev); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/diagnostics.inc b/drivers/gpu/apple-g5p/diagnostics.inc +new file mode 100644 +index 000000000..184ce954b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/diagnostics.inc +@@ -0,0 +1,181 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Cached append-only text. Readers never take the command lock or touch MMIO. ++ * Preserve earliest records on overflow, expose dropped bytes explicitly. ++ */ ++#define G5P_BLOB_SIZE SZ_4M ++#define G5P_MMU_HISTORY_SIZE SZ_32K ++ ++struct g5p_diagnostics { ++ spinlock_t lock; ++ u32 used, history_used; ++ u64 dropped, history_dropped; ++ char stage[40]; ++ u32 next_register; ++ bool reading; ++ char history[G5P_MMU_HISTORY_SIZE]; ++ char blob[G5P_BLOB_SIZE]; ++}; ++ ++static void g5p_diagnostics_free(void *data) ++{ ++ kvfree(data); ++} ++ ++static int g5p_diagnostics_init(struct g5p_diag *g) ++{ ++ g->diagnostics = kvzalloc(sizeof(*g->diagnostics), GFP_KERNEL); ++ if (!g->diagnostics) ++ return -ENOMEM; ++ spin_lock_init(&g->diagnostics->lock); ++ return devm_add_action_or_reset(g->dev, g5p_diagnostics_free, ++ g->diagnostics); ++} ++ ++static void g5p_diagnostic_append(struct g5p_diag *g, bool history, ++ const char *text, size_t len) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ u32 *used = history ? &d->history_used : &d->used; ++ u64 *dropped = history ? &d->history_dropped : &d->dropped; ++ u32 capacity = history ? G5P_MMU_HISTORY_SIZE : G5P_BLOB_SIZE; ++ char *dst = history ? d->history : d->blob; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ /* Keep complete lines; once full, never replace earlier evidence. */ ++ if (*dropped || len > capacity - *used) { ++ *dropped += len; ++ } else { ++ memcpy(dst + *used, text, len); ++ *used += len; ++ } ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_blob_printf(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ char line[512]; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, false, line, len); ++} ++ ++static void g5p_blob_hex(struct g5p_diag *g, const char *name, const u8 *data, ++ u32 len) ++{ ++ for (u32 off = 0; off < len; off += 16) { ++ char text[80]; ++ ++ hex_dump_to_buffer(data + off, min_t(u32, len - off, 16), 16, 4, ++ text, sizeof(text), false); ++ g5p_blob_printf(g, "%s: %04x: %s\n", name, off, text); ++ } ++} ++ ++/* R1+1000 uses synthetic address11000; other offsets denote R0. */ ++static void g5p_mmu_progress(struct g5p_diag *g, const char *stage, u32 next) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ strscpy(d->stage, stage, sizeof(d->stage)); ++ d->next_register = next; ++ d->reading = true; ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_mmu_record(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ dev_info(g->dev, "%s", line); ++} ++ ++/* The routine per-job and per-event lines (a job's publish, wake and success receipt, the firmware's ++ * alive and stamp events). At desktop job rates they are hundreds of lines a second, more than netconsole ++ * can send: its queue grew to 700 MB of unreclaimable slab (release8, 55 min). Faults, losses, ++ * recoveries, terminal latches and every failed job keep their own levels and always print. ++ * job_log 0 dev_dbg (dynamic debug: off unless enabled in dynamic_debug/control) ++ * 1 KERN_DEBUG: in dmesg for the parsers (RENDER-USER job=, RENDER publish), below the ++ * default console loglevel, so not on a console or netconsole ++ * 2 KERN_INFO as before (short netconsole traces only: floods at desktop job rates) ++ */ ++static unsigned int job_log = 0; ++module_param(job_log, uint, 0600); ++MODULE_PARM_DESC( ++ job_log, ++ "default0 routine per-job/per-event lines are dev_dbg; 1 KERN_DEBUG (dmesg, no " ++ "console); 2 KERN_INFO as before (floods netconsole)"); ++ ++#define g5p_job_printk(dev, fmt, ...) \ ++ do { \ ++ switch (READ_ONCE(job_log)) { \ ++ case 0: \ ++ dev_dbg(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ case 1: \ ++ dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ default: \ ++ dev_info(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ } \ ++ } while (0) ++ ++/* g5p_mmu_record for a routine per-job line: the blob keeps it, the console sees it only at job_log 2. */ ++static __printf(2, 3) __maybe_unused ++ void g5p_mmu_record_job(struct g5p_diag *g, const char *fmt, ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ g5p_job_printk(g->dev, "%s", line); ++} ++ ++/* A routine event's raw words at the job_log level. */ ++static __maybe_unused void g5p_job_hex(const char *prefix, const void *buf, ++ size_t len) ++{ ++ switch (READ_ONCE(job_log)) { ++ case 0: ++ print_hex_dump_debug(prefix, DUMP_PREFIX_NONE, 16, 4, buf, len, ++ false); ++ break; ++ case 1: ++ print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ default: ++ print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ } ++} +diff --git a/drivers/gpu/apple-g5p/dirtystart108.h b/drivers/gpu/apple-g5p/dirtystart108.h +new file mode 100644 +index 000000000..fcf7432fa +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.h +@@ -0,0 +1,454 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DIRTYSTART108_H ++#define G5P_DIRTYSTART108_H ++ ++#include "bits-compat.h" ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * G5P incoming dirty-start state machine (series gpu-recovery108, patch 0007). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 4. ++ * ++ * The bootstrap is stepped by userspace (map, load, release, events, ++ * control), so this is an event-driven machine, not one run: the glue feeds ++ * one event per driver step with the step's result and RAM-only inputs, and ++ * asks the machine at two gates (before release, at the DRM claim). It ++ * assumes nothing pristine: the origin of the power state is classified from ++ * the kexec flag, the requested incoming PMGR cycle and the PMGR word, and ++ * every step's own result is evidence. ++ * ++ * PROBE origin inputs: booted from kexec, kexec_gpu_cycle requested ++ * POWER the PMGR gfx word, read once after the PM reference (PMGR read) ++ * RESET the bounded incoming power cycle (g5p_r33_gfx_off(), one attempt ++ * per boot, only in mode 3, only before the first wrapper wake) ++ * WAKE the Fender wake result ++ * MAPS fresh shared state: every ring control word in range (after the wake: ++ * map needs the core id, read through the woken wrapper) ++ * LOAD the firmware image memcmp of the RAM copy (b->copied) ++ * RELEASE the reset-stream/release result, the R74E power result ++ * ALIVE the first ALIVE wait result and, when read, the first-ALIVE ++ * GPC word (r73e); 0x10/0x11 is the C3 closed gate ++ * CLAIM the DRM claim asks; answers from the ALIVE proof ++ * ++ * Modes (parameter dirty_start): ++ * 0 observe: receipts only, both gates open (combined107 behaviour) ++ * 1 gate (default): fail closed on evidence the driver itself produced (a ++ * failed or out-of-order step, a power abort, a GPC word other than ++ * 0x01 at the first ALIVE, a ring out of range); a claim needs ALIVE ++ * proven in the current reset generation. Equal to combined107 on every ++ * bootstrap that succeeds. ++ * 2 strict: also refuse release and claim for an origin that is not cold ++ * and not a witnessed cycle (warm-unproven, warm-uncycled, unknown), ++ * unless a reset succeeded in this generation ++ * 3 strict + reset: as 2, and the glue may spend the one RESET attempt ++ * on such an origin before the wrapper wake (destructive: PMGR write) ++ * ++ * Generations: gen starts at 1 at PROBE and steps once per RESET attempt. A ++ * reset voids every proof of the previous generation; ALIVE proven in an ++ * earlier generation never exposes. gen never decreases. ++ * ++ * Pure policy: no register access, no kernel API except scnprintf(). Each ++ * event emits exactly one DS108 line through the emit callback. ++ */ ++ ++enum g5p_ds_ev { ++ G5P_DSE_PROBE, ++ G5P_DSE_POWER, ++ G5P_DSE_RESET, ++ G5P_DSE_WAKE, ++ G5P_DSE_MAPS, ++ G5P_DSE_LOAD, ++ G5P_DSE_RELEASE, ++ G5P_DSE_ALIVE, ++ G5P_DSE_CLAIM, ++ G5P_DSE_RECOVER, /* an in-boot recovery cycled the gate after a proven epoch */ ++ G5P_DSE_N, ++}; ++ ++static const char *const g5p_ds_ev_name[G5P_DSE_N] = { ++ "probe", "power", "reset", "wake", "maps", ++ "load", "release", "alive", "claim", "recover", ++}; ++ ++enum g5p_ds_origin { ++ G5P_DSO_UNKNOWN, /* not probed, or the PMGR word could not be read */ ++ G5P_DSO_COLD, /* not booted from kexec */ ++ G5P_DSO_WARM_CYCLED, /* kexec, incoming cycle requested and witnessed (was_pwrgated) */ ++ G5P_DSO_WARM_UNPROVEN, /* kexec, cycle requested, no witness in the PMGR word */ ++ G5P_DSO_WARM_UNCYCLED, /* kexec, no incoming cycle requested */ ++ G5P_DSO_N, ++}; ++ ++static const char *const g5p_ds_origin_name[G5P_DSO_N] = { ++ "unknown", "cold", "warm-cycled", "warm-unproven", "warm-uncycled", ++}; ++ ++enum g5p_ds_why { ++ G5P_DSW_NONE, ++ G5P_DSW_PROTOCOL, /* a step out of order (its predecessor not ok in this gen) */ ++ G5P_DSW_WAKE, /* the wrapper wake failed */ ++ G5P_DSW_MAPS, /* map failed or a ring control word out of range */ ++ G5P_DSW_LOAD, /* the RAM image is not the firmware (memcmp) */ ++ G5P_DSW_RELEASE, /* release / reset stream failed */ ++ G5P_DSW_POWER_ABORT, /* the R74E power poll or the first-ALIVE gate aborted */ ++ G5P_DSW_GPC, /* first-ALIVE GPC word not 0x01 (0x10/0x11: C3) */ ++ G5P_DSW_ALIVE, /* no ALIVE, or the ALIVE wait failed */ ++ G5P_DSW_RESET, /* the incoming power cycle failed: power state unknown */ ++ G5P_DSW_N, ++}; ++ ++static const char *const g5p_ds_why_name[G5P_DSW_N] = { ++ "none", "protocol", "wake", "maps", "load", ++ "release", "power-abort", "gpc", "alive", "reset", ++}; ++ ++/* Gate answers (not latched): why a gate is closed right now. */ ++enum g5p_ds_gate { ++ G5P_DSG_OPEN, ++ G5P_DSG_FAILED, /* the fail-closed latch */ ++ G5P_DSG_NOT_PROVEN, /* no ALIVE proof in the current generation */ ++ G5P_DSG_ORIGIN, /* strict: origin neither cold nor a witnessed cycle */ ++ G5P_DSG_STATE, /* the claim snapshot contradicts the proof */ ++ G5P_DSG_N, ++}; ++ ++static const char *const g5p_ds_gate_name[G5P_DSG_N] = { ++ "open", "failed", "not-proven", "origin", "state", ++}; ++ ++#define G5P_DS_MODE_MAX 3 ++#define G5P_DS_GPC_ON 0x01u ++#define G5P_DS_PMGR_PWRGATED \ ++ BIT(8) /* sticky was_pwrgated history bit of 0x80340 */ ++ ++#define G5P_DSF_RINGS_BUSY \ ++ BIT(0) /* a ring not empty at map (data, not a failure) */ ++#define G5P_DSF_ALIVE_LATE \ ++ BIT(1) /* ALIVE first seen at the claim (no ALIVE event) */ ++#define G5P_DSF_RESET_RAN BIT(2) /* a RESET attempt ran this boot */ ++#define G5P_DSF_RESET_OK \ ++ BIT(3) /* ... and succeeded (gen of the success = gen) */ ++/* a second attempt or one after the wake was refused */ ++#define G5P_DSF_RESET_REFUSED BIT(4) ++#define G5P_DSF_DUP BIT(5) /* an event repeated (recorded, not a failure) */ ++#define G5P_DSF_PMGR_UNREAD BIT(6) /* the PMGR word could not be read */ ++ ++/* One flat input record; each event reads only its own fields (RAM or step results). */ ++struct g5p_ds_in { ++ bool kexec; /* PROBE: chosen linux,booted-from-kexec */ ++ bool cycle_requested; /* PROBE: apple_pmgr.kexec_gpu_cycle=1 on the cmdline */ ++ bool pmgr_ok; /* POWER: the PMGR read succeeded */ ++ u32 pmgr_raw; /* POWER: the gfx word */ ++ int ret; /* RESET/WAKE/MAPS/LOAD/RELEASE/ALIVE: the step's return */ ++ u32 rings_bad; /* MAPS: control words > 255 */ ++ u32 rings_busy; /* MAPS: rings not empty */ ++ int power_ret; /* RELEASE: R74E power result */ ++ u32 power_raw; /* RELEASE: its last R1+0x1000 sample */ ++ bool power_abort; /* RELEASE/ALIVE/CLAIM: r73b early_power_abort */ ++ bool gpc_valid; /* ALIVE: the r73e first-ALIVE word was read */ ++ u32 gpc; /* ALIVE: that word */ ++ bool alive, released, failed, ++ control_complete; /* ALIVE/CLAIM snapshot */ ++}; ++ ++struct g5p_ds { ++ u32 gen; /* reset generation, monotonic, 0 before PROBE */ ++ u32 seen, ok; /* event bitmasks, current generation */ ++ u32 flags; ++ enum g5p_ds_origin origin; ++ bool kexec, cycle; /* PROBE inputs, kept for POWER */ ++ u32 pmgr_raw; ++ u32 resets; /* RESET attempts this boot (at most one runs) */ ++ u32 reset_ok_gen; /* gen in which a RESET succeeded, 0 never */ ++ u32 alive_gen; /* gen in which ALIVE was proven, 0 never */ ++ bool failed; /* fail-closed latch (absorbing) */ ++ enum g5p_ds_why why; ++ u32 events, lines, claims, refused; ++ enum g5p_ds_ev last; ++ enum g5p_ds_gate last_gate; ++}; ++ ++static inline bool g5p_ds_ok(const struct g5p_ds *d, enum g5p_ds_ev e) ++{ ++ return d->ok & (1u << e); ++} ++ ++static inline void g5p_ds_fail(struct g5p_ds *d, enum g5p_ds_why why) ++{ ++ if (!d->failed) ++ d->why = why; ++ d->failed = true; ++} ++ ++static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool cycle, ++ bool pmgr_ok, u32 raw) ++{ ++ if (!pmgr_ok) ++ return G5P_DSO_UNKNOWN; ++ if (!kexec) ++ return G5P_DSO_COLD; ++ if (!cycle) ++ return G5P_DSO_WARM_UNCYCLED; ++ return (raw & G5P_DS_PMGR_PWRGATED) ? G5P_DSO_WARM_CYCLED : ++ G5P_DSO_WARM_UNPROVEN; ++} ++ ++/* The origin is trusted (cold, witnessed cycle) or a reset succeeded in this generation. */ ++static inline bool g5p_ds_origin_trusted(const struct g5p_ds *d) ++{ ++ return d->origin == G5P_DSO_COLD || d->origin == G5P_DSO_WARM_CYCLED || ++ (d->reset_ok_gen && d->reset_ok_gen == d->gen); ++} ++ ++/* Should the glue spend the one RESET attempt now (mode 3, before the wake)? */ ++static inline bool g5p_ds_reset_wanted(const struct g5p_ds *d, u32 mode) ++{ ++ return mode >= 3 && !d->failed && !d->resets && ++ !(d->seen & (1u << G5P_DSE_WAKE)) && ++ (d->seen & (1u << G5P_DSE_POWER)) && !g5p_ds_origin_trusted(d); ++} ++ ++/* The release gate (asked by the glue right before the reset stream runs). */ ++static inline enum g5p_ds_gate g5p_ds_release_gate(const struct g5p_ds *d, ++ u32 mode) ++{ ++ if (!mode) ++ return G5P_DSG_OPEN; ++ if (d->failed) ++ return G5P_DSG_FAILED; ++ if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ return G5P_DSG_ORIGIN; ++ return G5P_DSG_OPEN; ++} ++ ++/* The ALIVE proof the claim needs: proven in this generation, nothing failed since. */ ++static inline bool g5p_ds_proven(const struct g5p_ds *d) ++{ ++ return !d->failed && d->alive_gen && d->alive_gen == d->gen && ++ g5p_ds_ok(d, G5P_DSE_RELEASE) && g5p_ds_ok(d, G5P_DSE_LOAD); ++} ++ ++static inline int g5p_ds_line(struct g5p_ds *d, char *buf, size_t n, ++ enum g5p_ds_ev e, int ret, u32 a, u32 b) ++{ ++ d->lines++; ++ return scnprintf(buf, n, ++ "DS108 gen=%u ev=%s ret=%d origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x pmgr=%08x a=%x b=%x gate=%s", ++ d->gen, g5p_ds_ev_name[e], ret, g5p_ds_origin_name[d->origin], ++ d->failed, g5p_ds_why_name[d->why], d->seen, d->ok, d->flags, ++ d->pmgr_raw, a, b, g5p_ds_gate_name[d->last_gate]); ++} ++ ++/* ++ * Feed one event. Returns, for CLAIM, 0 (expose) or -EAGAIN/-EIO (refuse); ++ * for every other event 0. mode is the current dirty_start parameter; at mode 0 ++ * the claim answers 0 whatever the state (the legacy driver decides alone). ++ * line/n receive the event's receipt (always written). ++ */ ++static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in, u32 mode, char *line, ++ size_t n) ++{ ++ u32 bit = 1u << e, a = 0, b = 0; ++ int ret = 0; ++ ++ if (mode > G5P_DS_MODE_MAX) ++ mode = G5P_DS_MODE_MAX; ++ d->events++; ++ d->last = e; ++ if ((d->seen & bit) && e != G5P_DSE_CLAIM) ++ d->flags |= G5P_DSF_DUP; ++ switch (e) { ++ case G5P_DSE_PROBE: ++ /* The only event that may start a generation: probe runs once per boot. */ ++ if (!d->gen) ++ d->gen = 1; ++ d->origin = G5P_DSO_UNKNOWN; ++ a = in->kexec; ++ b = in->cycle_requested; ++ d->kexec = in->kexec; ++ d->cycle = in->cycle_requested; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_POWER: ++ if (!(d->seen & (1u << G5P_DSE_PROBE))) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->origin = g5p_ds_origin_of(d->kexec, d->cycle, in->pmgr_ok, ++ in->pmgr_raw); ++ if (!in->pmgr_ok) ++ d->flags |= G5P_DSF_PMGR_UNREAD; ++ d->pmgr_raw = in->pmgr_ok ? in->pmgr_raw : 0; ++ a = in->pmgr_ok; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RESET: ++ a = in->ret; ++ if (d->resets || (d->seen & (1u << G5P_DSE_WAKE))) { ++ /* The glue must not have run it; record a refused attempt and fail ++ * closed. ++ */ ++ d->flags |= G5P_DSF_RESET_REFUSED; ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->resets++; ++ d->flags |= G5P_DSF_RESET_RAN; ++ /* A reset voids the previous generation's proofs (power epoch boundary). */ ++ d->gen++; ++ /* Unreachable today (a reset runs before the wake, so before every later ++ * step); kept so that a reordered glue cannot carry a proof across it. ++ */ ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->flags |= G5P_DSF_RESET_OK; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_WAKE: ++ d->seen |= bit; ++ if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_WAKE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_MAPS: ++ d->seen |= bit; ++ a = in->rings_bad; ++ b = in->rings_busy; ++ if (in->rings_busy) ++ d->flags |= G5P_DSF_RINGS_BUSY; ++ /* map needs the core id, read through the woken wrapper */ ++ if (!g5p_ds_ok(d, G5P_DSE_WAKE)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret || in->rings_bad) ++ g5p_ds_fail(d, G5P_DSW_MAPS); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_LOAD: ++ d->seen |= bit; ++ if (!g5p_ds_ok(d, G5P_DSE_MAPS)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_LOAD); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RELEASE: ++ d->seen |= bit; ++ a = (u32)in->power_ret; ++ b = in->power_raw; ++ if (!g5p_ds_ok(d, G5P_DSE_LOAD)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->power_abort || in->power_ret) ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_RELEASE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_ALIVE: ++ d->seen |= bit; ++ a = in->gpc_valid; ++ b = in->gpc; ++ if (!g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ } else if (in->power_abort) { ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ } else if (in->gpc_valid && !G5P_GPC_ON(in->gpc)) { ++ g5p_ds_fail(d, G5P_DSW_GPC); ++ } else if (in->ret || !in->alive || in->failed) { ++ g5p_ds_fail(d, G5P_DSW_ALIVE); ++ } else { ++ d->ok |= bit; ++ d->alive_gen = d->gen; ++ } ++ break; ++ case G5P_DSE_RECOVER: ++ /* The render frontend's in-boot recovery (recover.inc): the gate was cycled ++ * after this generation's release, with the firmware frozen. Like a RESET it ++ * starts a generation that voids every proof of the last one, and it is a ++ * witnessed cycle for the origin gate; the bootstrap's steps then run again ++ * and must each succeed in the new generation. Not after a failure, and not ++ * before a release: there is no epoch to recover then. ++ */ ++ a = in->ret; ++ if (d->failed || !g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->gen++; ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ /* The old ALIVE proof is void; the claim's late path may prove the new one. */ ++ d->alive_gen = 0; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_CLAIM: ++ d->claims++; ++ d->seen |= bit; ++ a = in->alive | in->released << 1 | in->failed << 2 | ++ in->control_complete << 3 | in->power_abort << 4; ++ /* ALIVE reached through a path without the ALIVE event: accept the RAM ++ * latch as the proof only after a successful release of this generation. ++ */ ++ if (!d->failed && !d->alive_gen && in->alive && !in->failed && ++ !in->power_abort && g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ d->alive_gen = d->gen; ++ d->ok |= 1u << G5P_DSE_ALIVE; ++ d->flags |= G5P_DSF_ALIVE_LATE; ++ } ++ if (!mode) ++ d->last_gate = G5P_DSG_OPEN; ++ else if (d->failed) ++ d->last_gate = G5P_DSG_FAILED; ++ else if (!g5p_ds_proven(d)) ++ d->last_gate = G5P_DSG_NOT_PROVEN; ++ else if (!in->alive || !in->released || in->failed || ++ in->power_abort || !in->control_complete) ++ d->last_gate = G5P_DSG_STATE; ++ else if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ d->last_gate = G5P_DSG_ORIGIN; ++ else ++ d->last_gate = G5P_DSG_OPEN; ++ if (d->last_gate == G5P_DSG_OPEN) { ++ d->ok |= bit; ++ } else { ++ d->refused++; ++ ret = d->last_gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++ } ++ break; ++ default: ++ break; ++ } ++ g5p_ds_line(d, line, n, e, e == G5P_DSE_CLAIM ? ret : in->ret, a, b); ++ return ret; ++} ++ ++ ++ ++static inline void g5p_ds_init(struct g5p_ds *d) ++{ ++ *d = (struct g5p_ds){ .last = G5P_DSE_N }; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +new file mode 100644 +index 000000000..037e5c56e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -0,0 +1,273 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the incoming dirty-start machine (dirtystart108.h), series ++ * gpu-recovery108 (0007). Design: docs/gpu/experiments/recovery108-driver.md. ++ * ++ * Included from core.inc after teardown.inc, so it may use the teardown ring ++ * read (RAM), g5p_r33_gfx_off() and the retire layer's state. Patch 0008 calls ++ * the hooks from the driver steps; until then nothing here runs. ++ * ++ * Register access added by this file: one PMGR read of the gfx word after the ++ * PM reference (POWER, the same read g5p_power_snapshot() makes), and in mode 3 ++ * only one bounded PMGR cycle through g5p_r33_gfx_off() before the first ++ * wrapper wake. Every other input is RAM or a step's own return value. ++ */ ++#include ++#include "dirtystart108.h" ++ ++static struct g5p_ds g5p_ds_state = { .last = G5P_DSE_N }; ++static DEFINE_SPINLOCK(g5p_ds_lock); ++ ++static u32 g5p_ds_mode(void) ++{ ++ return min_t(u32, (1), G5P_DS_MODE_MAX); ++} ++ ++/* One event, one DS108 line (dev_emerg: netconsole at any loglevel). */ ++static int g5p_ds_feed(struct g5p_diag *g, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in) ++{ ++ unsigned long flags; ++ char line[224]; ++ int ret; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ ret = g5p_ds_event(&g5p_ds_state, e, in, g5p_ds_mode(), line, ++ sizeof(line)); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "%s\n", line); ++ return ret; ++} ++ ++/* wake112: the current reset generation, recorded with a deferred R74E proof (RAM, leaf lock). */ ++static u32 g5p_ds_gen(void) ++{ ++ unsigned long flags; ++ u32 gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return gen; ++} ++ ++/* The last apple_pmgr.kexec_gpu_cycle= token of the chosen bootargs (the one the kernel obeys). */ ++static bool g5p_ds_cycle_requested(struct device_node *chosen) ++{ ++ static const char key[] = "apple_pmgr.kexec_gpu_cycle="; ++ const char *args, *p, *last = NULL; ++ char tok[8]; ++ bool v = false; ++ size_t n; ++ ++ if (!chosen || of_property_read_string(chosen, "bootargs", &args)) ++ return false; ++ for (p = strstr(args, key); p; p = strstr(p + 1, key)) ++ if (p == args || p[-1] == ' ') ++ last = p; ++ if (!last) ++ return false; ++ last += sizeof(key) - 1; ++ n = strcspn(last, " "); ++ if (!n || n >= sizeof(tok)) ++ return false; ++ memcpy(tok, last, n); ++ tok[n] = 0; ++ return !kstrtobool(tok, &v) && v; ++} ++ ++static void g5p_ds_hook_probe(struct g5p_diag *g) ++{ ++ struct device_node *chosen = of_find_node_by_path("/chosen"); ++ struct g5p_ds_in in = {}; ++ ++ in.kexec = chosen && ++ of_property_read_bool(chosen, "linux,booted-from-kexec"); ++ in.cycle_requested = g5p_ds_cycle_requested(chosen); ++ of_node_put(chosen); ++ g5p_ds_feed(g, G5P_DSE_PROBE, &in); ++} ++ ++/* After the PM reference is taken: one PMGR read (the gfx word), no write. */ ++static void g5p_ds_hook_power(struct g5p_diag *g) ++{ ++ struct g5p_ds_in in = {}; ++ u32 raw = 0; ++ ++ in.pmgr_ok = g->pmgr && !regmap_read(g->pmgr, 0x80340, &raw); ++ in.pmgr_raw = raw; ++ g5p_ds_feed(g, G5P_DSE_POWER, &in); ++} ++ ++/* Before the first wrapper wake. Mode 3 only, one attempt per boot, through the ++ * r33 helper (its own refusal rules: min/reset/auto or not on; every exit leaves ++ * the gate requested on). Returns the cycle's error; the wake then fails closed. ++ */ ++static int g5p_ds_hook_reset(struct g5p_diag *g) ++{ ++ u32 ms = clamp_t(u32, (10), 1, G5P_R33_GFX_OFF_MAX_MS); ++ struct g5p_ds_in in = {}; ++ unsigned long flags; ++ bool want; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ want = g5p_ds_reset_wanted(&g5p_ds_state, g5p_ds_mode()); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ if (!want) ++ return 0; ++ g5p_mmio_gate(g); ++ in.ret = g5p_r33_gfx_off(g->dev, g->pmgr, ms); ++ g5p_ds_feed(g, G5P_DSE_RESET, &in); ++ boot_gfx_off_done = true; /* one PMGR cycle per boot, whoever runs it */ ++ return in.ret; ++} ++ ++/* The legacy boot_gfx_off_ms cycle (R33) is the same mechanism: record it as the reset. */ ++ ++/* An in-boot recovery's gate cycle (recover.inc): a new generation, or a failed machine. */ ++static int g5p_ds_hook_recover(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ unsigned long flags; ++ bool failed; ++ ++ g5p_ds_feed(g, G5P_DSE_RECOVER, &in); ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ failed = g5p_ds_state.failed; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return failed ? -EIO : 0; ++} ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_WAKE, &in); ++} ++ ++/* Fresh shared state: every ring control word in range (RAM, the teardown ring read). */ ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ u32 i, cons, prod; ++ ++ for (i = 0; !ret && i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ in.rings_bad++; ++ continue; ++ } ++ if (cons > 255 || prod > 255) ++ in.rings_bad++; ++ else if (cons != prod) ++ in.rings_busy++; ++ } ++ g5p_ds_feed(g, G5P_DSE_MAPS, &in); ++} ++ ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_LOAD, &in); ++} ++ ++/* Asked before the release touches anything. -EIO: the fail-closed latch; -EAGAIN: strict origin. */ ++static int g5p_ds_hook_release_gate(struct g5p_diag *g) ++{ ++ enum g5p_ds_gate gate; ++ unsigned long flags; ++ u32 mode = g5p_ds_mode(), gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gate = g5p_ds_release_gate(&g5p_ds_state, mode); ++ g5p_ds_state.last_gate = gate; ++ if (gate != G5P_DSG_OPEN) ++ g5p_ds_state.refused++; ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "DS108 gen=%u gate=release mode=%u answer=%s\n", gen, ++ mode, g5p_ds_gate_name[gate]); ++ if (gate == G5P_DSG_OPEN) ++ return 0; ++ return gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++} ++ ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (r) { ++ in.power_ret = r->early_power_ret; ++ in.power_raw = r->early_power_raw; ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ } ++ g5p_ds_feed(g, G5P_DSE_RELEASE, &in); ++} ++ ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed); ++ } ++ if (r) { ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ /* r73e reads R1+0x1000 at the first ALIVE; 0 means not read. */ ++ in.gpc = READ_ONCE(r->early_state); ++ in.gpc_valid = 1 && in.gpc; ++ } ++ g5p_ds_feed(g, G5P_DSE_ALIVE, &in); ++} ++ ++/* The DRM claim: 0 expose, -EAGAIN not proven / strict origin, -EIO the fail-closed latch. */ ++static int __maybe_unused g5p_ds_hook_claim(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_ds_in in = {}; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ in.control_complete = READ_ONCE(b->control_complete); ++ in.power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ return g5p_ds_feed(g, G5P_DSE_CLAIM, &in); ++} ++ ++/* RAM only: the retire layer and the dirty-start machine, for capture without a log. */ ++static ssize_t recovery_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct g5p_r8 *s = &g5p_r8_state; ++ unsigned long flags; ++ struct g5p_ds d; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ d = g5p_ds_state; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return sysfs_emit( ++ buf, ++ "r8 gen=%u retired=%d mode=%s class=%s verdict=%s flags=%02x admit_closed=%u " ++ "inside=%u detach_queued=%u lines=%u\n" ++ "ds gen=%u mode=%u origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x " ++ "pmgr=%08x alive_gen=%u resets=%u reset_ok_gen=%u events=%u claims=%u refused=%u " ++ "gate=%s\n", ++ READ_ONCE(s->gen), READ_ONCE(s->retired), ++ g5p_td_mode_name[s->mode], g5p_r8_class_name[s->cls], ++ g5p_td_verdict_name[s->verdict], s->flags, ++ g5p_r8_admit_closed(&s->admit), g5p_r8_admit_inside(&s->admit), ++ READ_ONCE(s->detach_queued), s->lines, d.gen, g5p_ds_mode(), ++ g5p_ds_origin_name[d.origin], d.failed, g5p_ds_why_name[d.why], ++ d.seen, d.ok, d.flags, d.pmgr_raw, d.alive_gen, d.resets, ++ d.reset_ok_gen, d.events, d.claims, d.refused, ++ g5p_ds_gate_name[d.last_gate]); ++} ++static DEVICE_ATTR_RO(recovery_status); +diff --git a/drivers/gpu/apple-g5p/drm-validate.h b/drivers/gpu/apple-g5p/drm-validate.h +new file mode 100644 +index 000000000..671e13ab8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/drm-validate.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DRM_VALIDATE_H ++#define G5P_DRM_VALIDATE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_USER_VA_START 0x10000ULL ++#define G5P_USER_VA_END 0x10000000000ULL ++#define G5P_USER_RESERVED_START G5P_IMAGE_VA ++#define G5P_USER_RESERVED_END (G5P_PIO_VA + G5P_PIO_SIZE) ++#define G5P_MAX_BO_SIZE (64ULL << 20) ++/* VA bookkeeping: the sum of mapping sizes in one VM. A GEM bound twice ++ * (GBM alloc, then the EGL import of its own dma-buf) counts twice here. ++ */ ++#define G5P_MAX_VM_BYTES BIT_ULL(30) ++/* Memory behind one VM's mappings, counted once per GEM object. */ ++#define G5P_MAX_PINNED_BYTES (768ULL << 20) ++ ++static inline bool g5p_user_pa_valid(u64 pa) ++{ ++ return pa < 0x10000000000ULL && !(pa & (G5P_GPU_PAGE - 1)); ++} ++ ++static inline bool g5p_user_range_valid(u64 va, u64 size) ++{ ++ if (!size || ((va | size) & (G5P_GPU_PAGE - 1)) || ++ va < G5P_USER_VA_START || va >= G5P_USER_VA_END || ++ size > G5P_USER_VA_END - va) ++ return false; ++ return va >= G5P_USER_RESERVED_END || ++ va + size <= G5P_USER_RESERVED_START; ++} ++ ++static inline bool g5p_bo_range_valid(u64 offset, u64 size, u64 bo_size) ++{ ++ return size && !((offset | size) & (G5P_GPU_PAGE - 1)) && ++ offset < bo_size && size <= bo_size - offset; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/fender.h b/drivers/gpu/apple-g5p/fender.h +new file mode 100644 +index 000000000..cbdf203c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/fender.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_FENDER_H ++#define G5P_FENDER_H ++ ++/* Every poll callback must enforce a deadline and retain its final raw word. ++ * Apple 20H350: stop c9fa10, prepare c9fc10, boot c9f954; ++ * Fender helpers ca4ccc (GPC state) and ca6544 (core-power request). ++ * Hardware interpretation of the 0 -> 1 transition remains experimental. ++ */ ++struct g5p_fender_io { ++ void *ctx; ++ int (*write)(void *ctx, u32 reg, u32 value); ++ int (*poll)(void *ctx, u32 reg, u32 mask, u32 expected); ++}; ++ ++static inline int g5p_fender_idle(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x1000, 0x10, 0); ++ if (ret) ++ return ret; ++ ret = io->write(io->ctx, 0x1000, 0x10); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++static inline int g5p_fender_hold(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0x20); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x20000, 0x20, 0x20); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++/* q: the pstate the GPC is requested at (R1+0x1000 = 0x10 | q, poll for q); 1 as before, ++ * or boot_gpu_perf_poweron's pin (boot.inc). ++ */ ++static inline int ++g5p_fender_start_observed_q(const struct g5p_fender_io *io, ++ void (*observe)(void *, const char *), u32 q) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-core-clear"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc0-verified"); ++ ret = io->write(io->ctx, 0x1000, 0x10 | q); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc1-requested"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, q); ++ if (!ret && observe) ++ observe(io->ctx, "release-gpc1-active"); ++ return ret; ++} ++ ++static inline int g5p_fender_start_observed(const struct g5p_fender_io *io, ++ void (*observe)(void *, ++ const char *)) ++{ ++ return g5p_fender_start_observed_q(io, observe, 1); ++} ++ ++static inline int g5p_fender_start(const struct g5p_fender_io *io) ++{ ++ return g5p_fender_start_observed(io, NULL); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/gpu-oc.h b/drivers/gpu/apple-g5p/gpu-oc.h +new file mode 100644 +index 000000000..733aba9d4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/gpu-oc.h +@@ -0,0 +1,149 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * GPU overclock record (boot_gpu_oc_mhz): docs/power/ds-gpu-dvfs.md §3-4, §9 R5; ++ * docs/gpu/experiments/clock-mechanisms.md §2. Host test: scripts/gpu/perf60-host/test_oc.c. ++ * ++ * The hardware applies PSINFO record n (PMGR+0x70000 + 0x10*n, written by LLB 180395fd8 for ++ * slots 0-4 once per SoC reset, volatile) when Fender is asked for pstate n. Word 0 = core-rail ++ * DAC code [31:24] | PLL valid [21] | multiplier [20:12] | predivider [8:4] | postdivider-1 [3:0]; ++ * word 1 [7:0] = buck8 code; word 2 = timing selectors. ++ * ++ * PLL: VCO = 24 MHz * mult / prediv, f = VCO / postdiv. Apple's GPU-domain (0x17) descriptors ++ * (LLB 0x1803bc718, all three chip sections) run VCOs from 800 to 2160 MHz (2160 = id 0x20 of ++ * section 0x2f0001ff, 90/1/3); the live states use 948..1650. The overclock keeps postdiv 2 (every ++ * live state at or above 474 MHz uses 2 or 3) and the VCO inside Apple's range: f <= 1080 MHz. ++ * Apple's own descriptors at 804 (67/1/2), 850 (425/6/2), 900 (75/1/2); elsewhere record 4's ++ * dividers (prediv 4, postdiv 2: f = 3 MHz * mult) on a 3 MHz grid. ++ * ++ * Voltage: record 5 keeps record 4's timing word; its core (rail 1, PMIC 0x11, 450 mV + 3.125 mV * ++ * code, GPU only) and buck8 (rail 8, PMIC 0x18, 600 mV + 3.125 mV * code) codes default to record ++ * 4's (138 = 881.25 mV, 124 = 987.5 mV), never lower, raised only on request in whole 3.125 mV ++ * codes, ++ * and never above the ceilings: the boot parameters (default state 4 + 50 mV, the CPU overclock's ++ * rail-offset cap) bounded by the compiled maxima below (state 4 + 150 mV; raising those needs a ++ * rebuild). The encoding itself reaches 1296 mV on both rails; that is not an operating point. ++ */ ++#ifndef G5P_GPU_OC_H ++#define G5P_GPU_OC_H ++ ++#include "bits-compat.h" ++ ++#define G5P_PSINFO_BASE 0x70000U /* PMGR syscon offset */ ++#define G5P_PSINFO_REC(n) (G5P_PSINFO_BASE + 0x10U * (n)) ++#define G5P_OC_STATE 5U ++#define G5P_OC_PIN 500U /* boot_gpu_perf for state 5 */ ++#define G5P_OC_STOCK4 \ ++ 0x8a2f1041U /* record 4 word 0 as LLB writes it on this chip */ ++#define G5P_OC_STOCK4_BUCK8 124U /* record 4 word 1 [7:0] */ ++#define G5P_OC_MIN_MHZ 726U /* the first grid step above 723 */ ++#define G5P_OC_VCO_MAX 2160U /* Apple's highest GPU-domain VCO */ ++#define G5P_OC_MAX_MHZ (G5P_OC_VCO_MAX / 2) /* postdiv 2: 1080 */ ++#define G5P_OC_VCO_STOCK_MAX \ ++ 1650U /* the highest VCO of a live state (550 MHz) */ ++#define G5P_OC_VALID BIT(21) ++#define G5P_OC_PLL_MASK 0x001ff1ffU /* mult, prediv, postdiv-1 */ ++#define G5P_OC_UV_STEP 3125U ++#define G5P_OC_CORE_BASE_UV 450000U ++#define G5P_OC_BUCK8_BASE_UV 600000U ++/* 881250 */ ++#define G5P_OC_CORE_STOCK_UV \ ++ (G5P_OC_CORE_BASE_UV + G5P_OC_UV_STEP * (G5P_OC_STOCK4 >> 24)) ++/* 987500 */ ++#define G5P_OC_BUCK8_STOCK_UV \ ++ (G5P_OC_BUCK8_BASE_UV + G5P_OC_UV_STEP * G5P_OC_STOCK4_BUCK8) ++/* the boot parameters' default ceiling: P4 + 50 mV */ ++#define G5P_OC_DEFAULT_HEADROOM_UV 50000U ++#define G5P_OC_ABS_HEADROOM_UV \ ++ 150000U /* compiled: P4 + 150 mV, no parameter goes above */ ++ ++struct g5p_oc_pll { ++ u32 mhz, mult, prediv, postdiv; ++ bool llb; /* Apple's own descriptor */ ++}; ++ ++static const struct g5p_oc_pll g5p_oc_llb[] = { ++ { 804, 67, 1, 2, true }, /* LLB id 0x22, all three GPU sections */ ++ { 850, 425, 6, 2, true }, /* LLB id 0x23, the live section */ ++ { 900, 75, 1, 2, true }, /* LLB id 0x24, the live section */ ++}; ++ ++/* ++ * The PLL for a requested MHz: Apple's descriptor at 804/850/900 (the request or its 3 MHz ++ * rounding), ++ * else the 3 MHz grid (rounded down) with record 4's dividers; false outside 726..1080. ++ */ ++static inline bool g5p_oc_pll(u32 want, struct g5p_oc_pll *p) ++{ ++ u32 mhz = want / 3 * 3; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_llb); i++) ++ if (want == g5p_oc_llb[i].mhz || mhz == g5p_oc_llb[i].mhz) { ++ *p = g5p_oc_llb[i]; ++ return true; ++ } ++ if (mhz < G5P_OC_MIN_MHZ || mhz > G5P_OC_MAX_MHZ) ++ return false; ++ *p = (struct g5p_oc_pll){ mhz, mhz / 3, 4, 2, false }; ++ return true; ++} ++ ++/* A rail voltage as a code: exact multiples of 3.125 mV above the base only. */ ++static inline bool g5p_oc_code(u32 uv, u32 base, u32 *code) ++{ ++ if (uv < base || (uv - base) % G5P_OC_UV_STEP || ++ (uv - base) / G5P_OC_UV_STEP > 0xff) ++ return false; ++ *code = (uv - base) / G5P_OC_UV_STEP; ++ return true; ++} ++ ++/* The voltage codes for a request (0 = state 4's), bounded below by state 4 and above by the ++ * ceilings (each already clamped to the compiled maximum by g5p_oc_ceiling()). ++ */ ++ ++/* Record 5's words 0 and 1 from the live record 4 (stock only), a PLL and the voltage codes. */ ++static inline bool g5p_oc_words(const u32 r4[4], u32 want, u32 core, u32 buck8, ++ u32 w[4], struct g5p_oc_pll *p) ++{ ++ if (r4[0] != G5P_OC_STOCK4 || (r4[1] & 0xff) != G5P_OC_STOCK4_BUCK8 || ++ !g5p_oc_pll(want, p) || core < (G5P_OC_STOCK4 >> 24) || ++ core > 0xff || buck8 < G5P_OC_STOCK4_BUCK8 || buck8 > 0xff) ++ return false; ++ w[0] = core << 24 | G5P_OC_VALID | p->mult << 12 | p->prediv << 4 | ++ (p->postdiv - 1); ++ w[1] = (r4[1] & ~0xffU) | buck8; ++ w[2] = r4[2]; ++ w[3] = r4[3]; ++ return true; ++} ++ ++/* Hz of a record word 0, as iBoot 87001c3ac decodes it (0 if the predivider is 0). */ ++static inline u64 g5p_oc_hz(u32 w0) ++{ ++ u32 mult = (w0 >> 12) & 0x1ff, prediv = (w0 >> 4) & 0x1f, ++ postdiv = (w0 & 0xf) + 1; ++ ++ return prediv ? 24000000ULL * mult / prediv / postdiv : 0; ++} ++ ++/* A nonzero record 5 that this driver writes: record 4's timing and word-3 words, valid, a PLL ++ * g5p_oc_pll() produces, voltage codes between state 4's and the compiled maxima. Anything else in ++ * slot 5 is not ours: never overwritten or cleared. ++ */ ++static inline bool g5p_oc_is_ours(const u32 r4[4], const u32 r5[4]) ++{ ++ u32 core = r5[0] >> 24, buck8 = r5[1] & 0xff, w[4], cmax, bmax; ++ struct g5p_oc_pll p; ++ u64 hz = g5p_oc_hz(r5[0]); ++ ++ if (!g5p_oc_code(G5P_OC_CORE_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_CORE_BASE_UV, &cmax) || ++ !g5p_oc_code(G5P_OC_BUCK8_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_BUCK8_BASE_UV, &bmax) || ++ core > cmax || buck8 > bmax || !hz || hz % 1000000ULL || ++ !g5p_oc_words(r4, (u32)(hz / 1000000ULL), core, buck8, w, &p)) ++ return false; ++ return w[0] == r5[0] && w[1] == r5[1] && w[2] == r5[2] && w[3] == r5[3]; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/heartbeat116.h b/drivers/gpu/apple-g5p/heartbeat116.h +new file mode 100644 +index 000000000..428146764 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/heartbeat116.h +@@ -0,0 +1,175 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_HEARTBEAT116_H ++#define G5P_HEARTBEAT116_H ++ ++/* ++ * Heartbeat116: the R74 IRQ174 storm bound as a window rate, not a lifetime count. ++ * ++ * A healthy R74 epoch takes IRQ174 at about 575/s and consumes about 290 event-ring ++ * entries/s, idle or not: one type-0 ALIVE per firmware scheduler iteration, and two IRQ ++ * entries per ring entry (irqs = 2 * events - 18 over the Warm113 C4-prep R74 milestones; ++ * r81 measured 577/289 per s). Wake112 bounded both counts over the epoch's lifetime: ++ * IRQ entries against boot_r74_irq_cap (1048576) and ring entries against the compute event ++ * limit (524544). Either bound latched failed/event_failed, so an idle, healthy GPU faulted ++ * after about 30 minutes. ++ * ++ * Here both bounds are rates. A window opens at the first IRQ entry and closes at the first ++ * entry at least window_ns later, then re-bases on the totals of that entry. Totals are u32 ++ * and every in-window count is a u32 difference, so a wrap of a total is harmless. ++ * ++ * RATE more than irq_ceiling IRQ entries in the open window (checked on every entry) ++ * EVENTS more than event_ceiling ring entries in the open window (checked on every entry) ++ * STALL stall_windows consecutive closed windows in which ++ * irqs > ratio * events + slack. Healthy traffic is 2:1. A latch that re-fires ++ * without ring progress, or a heartbeat after a fault stopped the drain, is not. ++ * ++ * A trip is absorbing. Nothing here clears a failure; the glue adds the same terminal stop ++ * as the lifetime cap had (cap_seen, failed, event_failed, one disable_irq_nosync). ++ * ++ * Pure policy: no register access, no lock, no kernel API. The caller serializes (the R73B ++ * IRQ thread is the only caller). ++ */ ++ ++#define G5P_HB_WINDOW_NS 1000000000ULL ++#define G5P_HB_IRQ_CEILING 16384U ++#define G5P_HB_EVENT_CEILING 8192U ++#define G5P_HB_RATIO 8U ++#define G5P_HB_SLACK 512U ++#define G5P_HB_STALL_WINDOWS 3U ++ ++enum g5p_hb_why { ++ G5P_HBW_NONE, ++ G5P_HBW_RATE, ++ G5P_HBW_EVENTS, ++ G5P_HBW_STALL, ++ G5P_HBW_N, ++}; ++ ++static const char *const g5p_hb_why_name[G5P_HBW_N] = { ++ "none", ++ "rate", ++ "events", ++ "stall", ++}; ++ ++enum g5p_hb_res { ++ G5P_HBR_OPEN, /* counted in the open window */ ++ G5P_HBR_CLOSED, /* this entry closed a window (h->last) */ ++ G5P_HBR_TRIP, /* tripped now or earlier */ ++}; ++ ++struct g5p_hb_cfg { ++ u64 window_ns; ++ u32 irq_ceiling, event_ceiling, ratio, slack, stall_windows; ++}; ++ ++/* The totals the IRQ thread samples after servicing one entry. */ ++struct g5p_hb_in { ++ u64 now_ns; ++ u32 irqs; /* IRQ174 entries (b->irqs) */ ++ u32 events; /* event-ring entries consumed (b->events) */ ++ u32 acks; /* AIC ACKs written by the R73B handler */ ++ u32 empty; /* entries whose service consumed no ring entry */ ++ u32 aux; /* auxiliary-ring entries consumed */ ++}; ++ ++/* One window: its start and length and its in-window counts. */ ++struct g5p_hb_win { ++ u64 start_ns, ns; ++ u32 irqs, events, acks, empty, aux; ++}; ++ ++struct g5p_hb { ++ u32 open; /* zero until the first entry (kzalloc) */ ++ u32 why; /* enum g5p_hb_why, written once */ ++ u32 windows; /* windows closed */ ++ u32 stalled; /* consecutive unproductive closed windows */ ++ u32 peak_irqs, peak_events; ++ u64 start_ns; ++ struct g5p_hb_in base; /* totals at the window's first entry */ ++ struct g5p_hb_win last; /* the last closed window */ ++ struct g5p_hb_win trip; /* the open window at the trip */ ++ u64 trip_ns; ++}; ++ ++static inline void g5p_hb_cfg_default(struct g5p_hb_cfg *c) ++{ ++ c->window_ns = G5P_HB_WINDOW_NS; ++ c->irq_ceiling = G5P_HB_IRQ_CEILING; ++ c->event_ceiling = G5P_HB_EVENT_CEILING; ++ c->ratio = G5P_HB_RATIO; ++ c->slack = G5P_HB_SLACK; ++ c->stall_windows = G5P_HB_STALL_WINDOWS; ++} ++ ++static inline void g5p_hb_count(const struct g5p_hb *h, ++ const struct g5p_hb_in *in, ++ struct g5p_hb_win *w) ++{ ++ w->start_ns = h->start_ns; ++ w->ns = in->now_ns >= h->start_ns ? in->now_ns - h->start_ns : 0; ++ w->irqs = in->irqs - h->base.irqs; ++ w->events = in->events - h->base.events; ++ w->acks = in->acks - h->base.acks; ++ w->empty = in->empty - h->base.empty; ++ w->aux = in->aux - h->base.aux; ++} ++ ++static inline bool g5p_hb_unproductive(const struct g5p_hb_cfg *c, ++ const struct g5p_hb_win *w) ++{ ++ return (u64)w->irqs > (u64)c->ratio * w->events + c->slack; ++} ++ ++static inline enum g5p_hb_res ++g5p_hb_trip(struct g5p_hb *h, u32 why, const struct g5p_hb_win *w, u64 now_ns) ++{ ++ h->why = why; ++ h->trip = *w; ++ h->trip_ns = now_ns; ++ return G5P_HBR_TRIP; ++} ++ ++/* One serviced IRQ entry. */ ++static inline enum g5p_hb_res g5p_hb_step(struct g5p_hb *h, ++ const struct g5p_hb_cfg *c, ++ const struct g5p_hb_in *in) ++{ ++ struct g5p_hb_win w; ++ ++ if (h->why) ++ return G5P_HBR_TRIP; ++ if (!h->open) { ++ h->open = 1; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ return G5P_HBR_OPEN; ++ } ++ g5p_hb_count(h, in, &w); ++ if (w.irqs > c->irq_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_RATE, &w, in->now_ns); ++ if (w.events > c->event_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_EVENTS, &w, in->now_ns); ++ if (w.ns < c->window_ns) ++ return G5P_HBR_OPEN; ++ h->last = w; ++ h->windows++; ++ if (w.irqs > h->peak_irqs) ++ h->peak_irqs = w.irqs; ++ if (w.events > h->peak_events) ++ h->peak_events = w.events; ++ h->stalled = g5p_hb_unproductive(c, &w) ? h->stalled + 1 : 0; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ if (h->stalled >= c->stall_windows) ++ return g5p_hb_trip(h, G5P_HBW_STALL, &w, in->now_ns); ++ return G5P_HBR_CLOSED; ++} ++ ++/* Receipts: the first four closed windows, then every 64th. */ ++static inline bool g5p_hb_report(const struct g5p_hb *h) ++{ ++ return h->windows <= 4 || !(h->windows % 64); ++} ++ ++#endif /* G5P_HEARTBEAT116_H */ +diff --git a/drivers/gpu/apple-g5p/kick.inc b/drivers/gpu/apple-g5p/kick.inc +new file mode 100644 +index 000000000..6ef7395c6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/kick.inc +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared pre-kick step recovered from ca0358/ca05d4 and ca6644. */ ++static int g5p_prepare_kick(struct g5p_diag *g, const char *name) ++{ ++ u32 raw, state; ++ ++ if (gfx_bridge_replay && !strcmp(name, "RENDER-TA")) ++ g5p_r41_bridge_ta(g); ++ /* ca05d4 always invokes Fender +80 before inspecting GPC state. ++ * ca6644: W R1+14=7, then read fence. Hardware bit names unknown. ++ * This is distinct from acknowledging the host AIC interrupt. ++ */ ++ dev_info(g->dev, "%s Fender W R1+00014=00000007 before MTS kick\n", ++ name); ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ dev_info(g->dev, "%s Fender raw R1+00014=%08x (write fence)\n", name, ++ raw); ++ if (raw & ~7U) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ if (READ_ONCE(g->boot->failed)) ++ return -EIO; ++ return 0; ++} +diff --git a/drivers/gpu/apple-g5p/latency123.h b/drivers/gpu/apple-g5p/latency123.h +new file mode 100644 +index 000000000..164dc8e68 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/latency123.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: firmware words a powered iOS GPU has and our bootstrap leaves zero ++ * (docs/gpu/firmware-re/wake-latency-re.md sections 3.4 and 4.2). ++ * ++ * Arm A. The firmware copies meta+0x6c0..0x758 into its private block once, at init ++ * (MINIM 0x100009ac -> 0x10003ca4), so these words only act from the boot image: ++ * meta+0x6d4 = 1 per-tick cache maintenance 0x1001b638 (688/1720/e230/e138) ++ * meta+0x710/0x714 pstate table max index / count ++ * meta+0x718.. pstate MHz; the tick period is table[Fender+0x1000 & 0xf] ++ * policy+0x4 = 16 governor sample period, ms (the governor reads it live) ++ * 1 = capped: table { 0, 340 }, max index 1. The firmware's own pstate setter ++ * (0x1001c88c, Fender+0x1000 = index | 0x10) can then only ask for pstate 1, the state ++ * the host already requests, so the clock does not move. ++ * 2 = full: table { 0, 340, 474, 550, 723 }, max index 4, plus the limits README rank 2 ++ * pairs with it (control+0x1028/0x102c/0x1030 = 3348/400/400, policy+0x7cc/0x6fc = ++ * 400). The firmware may then raise the GPU clock itself; nothing on Linux programs ++ * a GPU voltage table, so this arm runs after the capped one. ++ * ++ * Arm B. ctl+0x04 is the per-engine tick-service mask (iOS 7). Every tick re-copies ++ * ctl+0x00..0x20 (0x10001328), so the word is live; the boot image gets the value the ++ * parameter holds at map and the render path re-applies it before each job. ++ * ++ * RAM only. Temporary A/B code: the winning words become unconditional init writes. ++ */ ++#ifndef APPLE_G5P_LATENCY123_H ++#define APPLE_G5P_LATENCY123_H ++ ++#include "bits-compat.h" ++#define G5P_L123_POLICY 0x4aU ++#define G5P_L123_META 0x4dU ++#define G5P_L123_CTL 0x5eU ++#define G5P_L123_TICK_MASK 7U ++#define G5P_L123_CAPPED 1U ++#define G5P_L123_FULL 2U ++ ++static const u32 g5p_l123_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline void g5p_l123_arm_a(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 arm) ++{ ++ u8 *meta = g5p_object(p, root, shared, G5P_L123_META); ++ u8 *policy = g5p_object(p, root, shared, G5P_L123_POLICY); ++ u8 *ctl = g5p_object(p, root, shared, G5P_L123_CTL); ++ u32 count = arm == G5P_L123_FULL ? ARRAY_SIZE(g5p_l123_mhz) : 2; ++ u32 i; ++ ++ g5p_put32(meta + 0x6d4, 1); ++ g5p_put32(meta + 0x710, count - 1); ++ g5p_put32(meta + 0x714, count); ++ for (i = 0; i < count; i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_l123_mhz[i]); ++ g5p_put32(policy + 4, 16); ++ if (arm != G5P_L123_FULL) ++ return; ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, 400); ++ g5p_put32(ctl + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/ldr.h b/drivers/gpu/apple-g5p/ldr.h +new file mode 100644 +index 000000000..165aacf85 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/ldr.h +@@ -0,0 +1,154 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Deliberately small Apple G5P META LDR reader. No allocation or MMIO here. */ ++#ifndef APPLE_G5P_LDR_H ++#define APPLE_G5P_LDR_H ++ ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_LDR_MAX_RECORDS 128 ++#define G5P_LDR_MAX_BYTES (256 * 1024) ++ ++enum g5p_ldr_op { G5P_LOAD, G5P_ZERO, G5P_CONFIG, G5P_START }; ++ ++typedef int (*g5p_ldr_emit)(void *ctx, enum g5p_ldr_op op, u32 address, ++ u32 size, const u8 *data); ++ ++static inline u32 g5p_le32(const u8 *p) ++{ ++ return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; ++} ++ ++static inline u16 g5p_le16(const u8 *p) ++{ ++ return (u16)p[0] | (u16)p[1] << 8; ++} ++ ++static inline bool g5p_ldr_bounds(size_t len, u32 off, u32 size) ++{ ++ return off <= len && size <= len - off; ++} ++ ++static inline int g5p_ldr_l2(const u8 *fw, size_t len, u32 off, u16 tag, ++ const u8 **data, u32 *size) ++{ ++ u16 bytes; ++ ++ if (!g5p_ldr_bounds(len, off, 4)) ++ return -EINVAL; ++ bytes = g5p_le16(fw + off + 2); ++ if (bytes < 6 || !g5p_ldr_bounds(len, off, bytes) || ++ g5p_le16(fw + off) != tag) ++ return -EINVAL; ++ *data = fw + off + 4; ++ *size = bytes - 6; ++ return 0; ++} ++ ++/* Validate in a first pass (emit=NULL); only then use a hardware callback. */ ++static inline int g5p_ldr_walk(const u8 *fw, size_t len, g5p_ldr_emit emit, ++ void *ctx) ++{ ++ u32 pos, seen[G5P_LDR_MAX_RECORDS], total = 0; ++ unsigned int nr = 0, starts = 0; ++ ++ if (len != 119112 || g5p_le32(fw) != 0x01aa5500 || ++ !g5p_ldr_bounds(len, g5p_le32(fw + 4), 1)) ++ return -EINVAL; ++ pos = g5p_le32(fw + 8); ++ while (pos != 0xffffffff) { ++ const u8 *p, *data; ++ u32 command, bytes, addr = 0, size = 0, l2, next; ++ enum g5p_ldr_op op; ++ unsigned int i; ++ int ret; ++ ++ if (nr == G5P_LDR_MAX_RECORDS || !g5p_ldr_bounds(len, pos, 12)) ++ return -EINVAL; ++ for (i = 0; i < nr; i++) ++ if (seen[i] == pos) ++ return -EINVAL; ++ seen[nr++] = pos; ++ p = fw + pos; ++ command = g5p_le16(p); ++ bytes = g5p_le16(p + 2); ++ next = g5p_le32(p + 4); ++ if (bytes < 12 || (bytes & 3) || ++ !g5p_ldr_bounds(len, pos, bytes)) ++ return -EINVAL; ++ if (command & 0x10) { ++ if (command != 0x10 || bytes != 16) ++ return -EINVAL; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), 0x8010, ++ &data, &size); ++ if (ret) ++ return ret; ++ goto next_record; ++ } ++ switch (command) { ++ case 0: ++ if (bytes != 20) ++ return -EINVAL; ++ op = G5P_LOAD; ++ addr = g5p_le32(p + 8); ++ l2 = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, l2, 0x8000, &data, &size); ++ if (ret) ++ return ret; ++ break; ++ case 4: ++ if (bytes != 24) ++ return -EINVAL; ++ op = G5P_ZERO; ++ addr = g5p_le32(p + 8); ++ size = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 16), 0x8004, ++ &data, &l2); ++ if (ret || l2) ++ return -EINVAL; ++ break; ++ case 5: ++ case 3: ++ if (bytes != 16) ++ return -EINVAL; ++ op = command == 5 ? G5P_CONFIG : G5P_START; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), ++ 0x8000 | command, &data, &size); ++ if (ret || !size || size % (command == 5 ? 12 : 16)) ++ return -EINVAL; ++ if (command == 5) { ++ for (i = 0; i < size; i += 12) ++ if (g5p_le32(data + i) != 2 || ++ (g5p_le32(data + i + 4) & 3)) ++ return -EINVAL; ++ } else { ++ if (++starts != 1 || size != 32 || ++ next != 0xffffffff || g5p_le32(data) != 1 || ++ g5p_le32(data + 16) != 0) ++ return -EINVAL; ++ } ++ break; ++ default: ++ return -EINVAL; ++ } ++ if (op == G5P_LOAD || op == G5P_ZERO) { ++ if ((addr & 3) || (size & 3) || !size || ++ size > G5P_LDR_MAX_BYTES || ++ addr > (u32)~0U - size || ++ total > G5P_LDR_MAX_BYTES - size) ++ return -EINVAL; ++ total += size; ++ } ++ if (emit) { ++ ret = emit(ctx, op, addr, size, data); ++ if (ret) ++ return ret; ++ } ++next_record: ++ pos = next; ++ } ++ return nr == 94 && starts == 1 ? 0 : -EINVAL; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +new file mode 100644 +index 000000000..9a896569c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -0,0 +1,1032 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Post-alive transport and park experiments; never implicit recovery. */ ++static const bool control_experiment = 1; ++static const bool irq_experiment = 0; ++static const bool teardown_experiment = 1; ++ ++/* A complete sparse image: first sample omits only zero words, later samples ++ * emit every changed word (including transitions to zero). All allocated ++ * root/shared memory is covered, including unknown status/trace buffers. ++ * These are CPU coherent RAM observations, never proof of META cache contents. ++ * IRQs can update memory concurrently; this is deliberately not an atomic dump. ++ */ ++static u32 g5p_dc_root_word(struct g5p_boot *b, u32 offset) ++{ ++ return READ_ONCE(*(u32 *)(b->root[offset / G5P_GPU_PAGE].cpu + ++ offset % G5P_GPU_PAGE)); ++} ++ ++/* nopatch: the policy record (slot 0x4a) words 0x1000d044 decides readiness from: +8 state, ++ * +0xc substate, +0x34 pstate mirror; +0 and +4 (sample period) for context. iOS keeps +8 at 0. ++ */ ++static const u32 g5p_np_words[5] = { 0, 4, 8, 0xc, 0x34 }; ++ ++static void g5p_np_read(struct g5p_boot *b, u32 *w) ++{ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_np_words); i++) ++ w[i] = g5p_dc_root_word(b, b->layout.offset[0x4a] + ++ g5p_np_words[i]); ++} ++ ++static void g5p_np_print(struct g5p_diag *g, const char *stage, u32 seq, u32 e, ++ const u32 *w) ++{ ++ dev_info( ++ g->dev, ++ "NOPATCH policy stage=%s seq=%u engine=%u +0=%08x +4=%08x +8=%08x +c=%08x " ++ "+34=%08x events=%u end=nopatch\n", ++ stage, seq, e, w[0], w[1], w[2], w[3], w[4], ++ READ_ONCE(g->boot->events)); ++} ++ ++/* RAM only; nothing unless the unpatched image is latched. */ ++static void g5p_np_policy(struct g5p_diag *g, const char *stage, u32 seq, u32 e) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 w[ARRAY_SIZE(g5p_np_words)]; ++ ++ if (!b || !b->r73b || !b->r73b->nopatch) ++ return; ++ g5p_np_read(b, w); ++ g5p_np_print(g, stage, seq, e, w); ++} ++ ++/* Read fixed allocated policy objects, never follow firmware-owned pointers. ++ * 1000d044/1000d504 test these states;1001c900 stores requested pstate. ++ * These are sampled boundary predicates, NOT entry/return breadcrumbs or PC. ++ */ ++ ++/* Firmware MMU recovery uses event mask200 and reads e150/e158. ++ * Read raw qword halves without ACK/clear; concurrent updates can tear them. ++ * An absent sampled event does not exclude an earlier firmware-handled fault. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 event, lo, hi, lo2, hi2; ++ ++ g5p_mmu_progress(g, stage, 0x130); ++ event = readl(g->rgx + 0x130); ++ g5p_mmu_progress(g, stage, 0xe150); ++ lo = readl(g->rgx + 0xe150); ++ g5p_mmu_progress(g, stage, 0xe154); ++ hi = readl(g->rgx + 0xe154); ++ g5p_mmu_progress(g, stage, 0xe158); ++ lo2 = readl(g->rgx + 0xe158); ++ g5p_mmu_progress(g, stage, 0xe15c); ++ hi2 = readl(g->rgx + 0xe15c); ++ g5p_mmu_record( ++ g, ++ "MMU %s event130=%08x pending=%u e150=%08x:%08x e158=%08x:%08x raw-split read-only\n", ++ stage, event, !!(event & 0x200), hi, lo, hi2, lo2); ++} ++ ++/* Same active-wrapper gate as CDM observations. Never write/ACK a register. ++ * 1001c9b8 samples R0+164/160/164; bound host sampling even on rollover. ++ * A split sample with different high words is explicitly unstable. ++ */ ++ ++static void g5p_dc_snapshot(struct g5p_diag *g, const char *stage, bool first) ++{ ++ { ++ return; ++ } ++} ++ ++static int g5p_control_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 *work = (u32 *)g5p_boot_shared(b, 0x68); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ u32 ctl_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6e)); ++ u32 data_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6f)); ++ u8 *init = b->root[0].cpu; ++ u8 *handoff = b->image[0].cpu + 0x3ff8; ++ u32 *events = (u32 *)g5p_boot_shared(b, 0x70); ++ ++ /* c9f868/c9f884: validate the loaded boot handoff, not just its builder. */ ++ if (g5p_le32(handoff) != g5p_meta_address(G5P_ROOT_VA) || ++ g5p_le32(handoff + 4) != ++ g5p_meta_address(g5p_object_va(&b->layout, 0x5f))) ++ return -EPROTO; ++ /* Drain under event_lock before checking that ALIVE has been returned. */ ++ if (g5p_r73b_service(g, G5P_R73B_OWNER_PREPARE, true)) ++ return -EIO; ++ if (READ_ONCE(events[0]) > 255 || READ_ONCE(events[4]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(events[0]) != READ_ONCE(events[4])) ++ return -EBUSY; ++ dev_info(g->dev, ++ "DC boot handoff verified; event consumer=%u producer=%u\n", ++ READ_ONCE(events[0]), READ_ONCE(events[4])); ++ ++ ; ++ ; ++ ; ++ ; ++ /* c96be8: init+18/+1c are META addresses, never CPU/DMA pointers. */ ++ if (g5p_le32(init + 0x18) != ctl_meta || ++ g5p_le32(init + 0x1c) != data_meta) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC mapping ctl GPU=%llx META=%08x data GPU=%llx META=%08x; init verified\n", ++ g5p_object_va(&b->layout, 0x6e), ctl_meta, ++ g5p_object_va(&b->layout, 0x6f), data_meta); ++ ++ if (b->work_ready) ++ return -EBUSY; ++ if (b->control_ready) ++ return b->r73b && 1 ? 0 : -EALREADY; ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (consumer != producer || READ_ONCE(work[0]) != READ_ONCE(work[4]) || ++ (b->work_submitted && !b->work_complete)) ++ return -EBUSY; ++ /* c993f8, size=0: no payload copy, ctrl+1020=0, command={8,0,0,0,0}. ++ * No claim that this is an opcode-independent firmware NOP. ++ */ ++ g5p_put32(g5p_boot_shared(b, 0x5e) + 0x1020, 0); ++ b->control_target = (producer + 1) & 255; ++ b->control_ready = true; ++ dev_info( ++ g->dev, ++ "DC prepare cmd=8 payload_words=0 consumer=%u producer=%u target=%u; RAM only\n", ++ consumer, producer, b->control_target); ++ return 0; ++} ++ ++static int g5p_r73e_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ret = g5p_control_prepare(g); ++ if (!ret) ++ WRITE_ONCE(b->r73b->fast_window, true); ++ dev_info( ++ g->dev, ++ "R73E cmd8 prepared-before-release ret=%d target=%u end=r73e\n", ++ ret, b->control_target); ++ return ret; ++} ++ ++/* Apple's GPC precondition (AGXMetaFender_setGPCPerformanceState, ...6ca4ccc): a request in ++ * flight shows R1+0x1000 bit 4; wait for it to clear before judging the state. R1 reads only, ++ * bounded by boot_alive_settle_us; with 0 the first read stands, as before. ++ */ ++static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw) ++{ ++ u64 start; ++ ++ if (!r->settle_us || !(raw & 0x10)) ++ return raw; ++ if (!r->settle_first) ++ r->settle_first = raw; ++ start = ktime_get_mono_fast_ns(); ++ while ((raw & 0x10) && ktime_get_mono_fast_ns() - start < ++ (u64)r->settle_us * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(g->window[1] + 0x1000); ++ r->settle_samples++; ++ } ++ r->settle_ns += ktime_get_mono_fast_ns() - start; ++ return raw; ++} ++ ++static int g5p_r73e_irq_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 off, raw; ++ enum g5p_pxg gate; ++ ++ if (!b->control_ready || b->control_submitted) ++ return -EPERM; ++ /* wake112: release_ready, or a deferred R74E proof that the read below settles. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate != G5P_PXG_READY && gate != G5P_PXG_PROVE) ++ return -EHOSTDOWN; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ if (gate == G5P_PXG_PROVE && !g5p_r74e_take(g)) ++ return -EHOSTDOWN; ++ r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (gate == G5P_PXG_PROVE) ++ g5p_r74e_settle(r, r->early_state); ++ if (!g5p_gpc_ok(r->early_state)) ++ return -EHOSTDOWN; ++ /* nopatch: five RAM loads, the state the readiness gate will see before this bell. */ ++ if (r->nopatch) ++ g5p_np_read(b, r->np_pre); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_IRQ, 0); ++ /* Functional pre-kick sequence only; no optional diagnostics here. */ ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ if (raw & ~7U) ++ return -EPROTO; ++ r->early_after_state = ++ g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (!g5p_gpc_ok(r->early_after_state)) ++ return -EHOSTDOWN; ++ dev_emerg( ++ g->dev, ++ "R73E doorbell begin first-ALIVE W0+00b00 value=00000010 end=r73e\n"); ++ writel(0x10, g->rgx + 0xb00); ++ dev_emerg(g->dev, "R73E doorbell end first-ALIVE W0+00b00 end=r73e\n"); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_IRQ, 0); ++ return 0; ++} ++ ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxg gate; ++ u32 px; ++ int ret; ++ ++ /* wake112: a deferred R74E proof admits the once-owner as release_ready does. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate == G5P_PXG_REFUSE) ++ return -EHOSTDOWN; ++ if (gate == G5P_PXG_WAIT) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_busy, 0, 1)) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_once, 0, 1)) { ++ ret = READ_ONCE(r->early_ret); ++ goto out; ++ } ++ r->early_irq_ns = ktime_get_mono_fast_ns(); ++ r->early_ret = g5p_r73e_irq_submit(g); ++ ret = r->early_ret; ++ WRITE_ONCE(r->fast_window, false); ++ if (ret == -EHOSTDOWN) ++ WRITE_ONCE(r->early_power_abort, true); ++ dev_emerg( ++ g->dev, ++ "R74E ALIVE submit case=%u owner=%u state=%08x after=%08x ret=%d " ++ "release_ready=%u power_abort=%u submitted=%u end=r74e\n", ++ r->r73e_case, owner, r->early_state, r->early_after_state, ret, ++ r->early_release_ready, r->early_power_abort, ++ READ_ONCE(b->control_submitted)); ++ if (r->settle_first) ++ dev_emerg( ++ g->dev, ++ "R74E settle first=%08x state=%08x after=%08x samples=%u ns=%llu limit_us=%u " ++ "ret=%d end=r74e\n", ++ r->settle_first, r->early_state, r->early_after_state, ++ r->settle_samples, r->settle_ns, r->settle_us, ret); ++ if (r->nopatch && READ_ONCE(b->control_submitted)) ++ g5p_np_print(g, "pre-bell", 0, 0, r->np_pre); ++ px = atomic_read(&r->early_proof); ++ /* raw: the proof read, or U32_MAX when it did not happen (a refusal or no take). */ ++ if (px != G5P_PX_NONE) ++ dev_emerg( ++ g->dev, ++ "R74E112 proof owner=%u state=%s why=%s gen=%u/%u raw=%08x release_ready=%u " ++ "power_abort=%u end=r74e112\n", ++ owner, g5p_px_name[min_t(u32, px, G5P_PX_N - 1)], ++ g5p_px_why_name[min_t(u32, r->early_proof_why, ++ G5P_PXW_N - 1)], ++ r->early_proof_gen, r->early_proof_gen_now, ++ px == G5P_PX_PROVEN || ++ r->early_proof_why == G5P_PXW_STATE ? ++ r->early_state : ++ U32_MAX, ++ r->early_release_ready, r->early_power_abort); ++out: ++ atomic_set(&r->early_busy, 0); ++ return ret; ++} ++ ++static int g5p_control_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 state, off; ++ int ret; ++ ++ if (b->control_submitted) ++ return b->r73b && 1 ? 0 : -EPERM; ++ if (!b->control_ready) ++ return -EPERM; ++ /* R73 synchronous pre-publish pass closes the race where trace records ++ * do not raise a final IRQ. A negative result is scoreable only when the ++ * factor reaches cmd8 with every outbound auxiliary consumer caught up. ++ */ ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_PUBLISH, true); ++ if (ret) ++ return ret; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ dev_info(g->dev, "DC raw R1+01000=%08x (require active before kick)\n", ++ state); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_dc_snapshot(g, "before-publish", true); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ /* ca05d4: full dmb ish after producer, before Fender and MTS. */ ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_PUBLISH, 0); ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *src = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ if (g5p_le32(src) != (i ? 0 : 8)) ++ return -EBADMSG; ++ } ++ if (READ_ONCE(ctl[4]) != b->control_target) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC slot verified words=00000008/0/0/0/0 producer=%u; full barrier\n", ++ b->control_target); ++ ret = g5p_prepare_kick(g, "DC"); ++ if (ret) ++ return ret; ++ ; ++ if (b->r73_aux_drain) { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_KICK, true); ++ if (ret) ++ return ret; ++ } ++ g5p_dc_snapshot(g, "before-kick", false); ++ dev_info( ++ g->dev, ++ "DC publish cmd=8 slot=%u target=%u; W R0+00b00=%08x variant=%u\n", ++ producer, b->control_target, b->dc_variant == 4 ? 0x12 : 0x10, ++ b->dc_variant); ++ /* 0x12 is verified for CL, NOT for DC: variant 4 tests dispatch routing. */ ++ writel(7 == 4 ? 0x12 : 0x10, g->rgx + 0xb00); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_KICK, 0); ++ g5p_dc_snapshot(g, "after-kick", false); ++ return 0; ++} ++ ++/* perf60 pin >100 (seed 1-5, poweron): the policy record as the firmware left it, from RAM, plus the ++ * real Fender state through an R1-only ticket of the register licence (any pstate, bit 29 allowed). ++ * The sampler 0x1000aa80 stores its own "now" at policy+0x918 on every call (0x1000ab44) and the ++ * current state at +0x34 (0x1000ab18), so fw_now against the host's arch timer names the ++ * firmware's clock, and +0x34 / +0 / +0x8f8 name the branch it took. ++ */ ++static void g5p_perf_readback(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 pol = b->layout.offset[0x4a], meta = b->layout.offset[0x4d]; ++ u64 host = arch_timer_read_counter(), rate = arch_timer_get_rate(); ++ u64 p8f8, p918, p920, m784; ++ struct g5p_mmio_ticket t; ++ ++ if (!b->gpu_perf || (!b->perf_seed && !b->perf_poweron)) ++ return; ++ dma_rmb(); ++ p8f8 = g5p_dc_root_word(b, pol + 0x8f8) | ++ (u64)g5p_dc_root_word(b, pol + 0x8fc) << 32; ++ p918 = g5p_dc_root_word(b, pol + 0x918) | ++ (u64)g5p_dc_root_word(b, pol + 0x91c) << 32; ++ p920 = g5p_dc_root_word(b, pol + 0x920) | ++ (u64)g5p_dc_root_word(b, pol + 0x924) << 32; ++ m784 = g5p_dc_root_word(b, meta + 0x784) | ++ (u64)g5p_dc_root_word(b, meta + 0x788) << 32; ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ dev_emerg( ++ g->dev, ++ "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " ++ "pol28=%u pol2c=%u pol30=%u pol34=%u p8f8=%llx fw_now918=%llx p920=%llx " ++ "meta784=%llx host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " ++ "host_minus_fw_counter_us=%lld r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", ++ why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, ++ g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30), ++ g5p_dc_root_word(b, pol + 0x34), p8f8, p918, p920, m784, host, ++ b->seed_stamp, ++ b->seed_stamp ? ++ div_s64((s64)(p918 - m784 - b->seed_stamp) * 1000000LL, ++ rate) : ++ 0, ++ div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate), ++ g5p_mmio_did(&t), t.gpc, t.r1_0); ++} ++ ++static int g5p_r73c_fast_control(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ int ret; ++ ++ r->fast_release_ns = ktime_get_mono_fast_ns(); ++ WRITE_ONCE(r->fast_window, true); ++ ret = g5p_boot_events(g); ++ r->fast_alive_ns = ktime_get_mono_fast_ns(); ++ if (!ret) ++ ret = g5p_control_prepare(g); ++ if (!ret) { ++ r->fast_state = readl(g->window[1] + 0x1000); ++ ret = g5p_control_submit(g); ++ } ++ r->fast_submit_ns = ktime_get_mono_fast_ns(); ++ r->fast_ret = ret; ++ WRITE_ONCE(r->fast_window, false); ++ g5p_r73b_events_exit(g, ret); ++ dev_info( ++ g->dev, ++ "R73C fast release_to_alive_ns=%llu release_to_submit_ns=%llu state=%08x ret=%d " ++ "control_submitted=%u end=r73c\n", ++ r->fast_alive_ns - r->fast_release_ns, ++ r->fast_submit_ns - r->fast_release_ns, r->fast_state, ret, ++ READ_ONCE(b->control_submitted)); ++ g5p_perf_readback(g, ret ? "alive-timeout" : "alive"); ++ return ret; ++} ++ ++/* perf60 pin 200 (docs/gpu/experiments/perf60-clock.md "Pin 200: the guarded drain read" and ++ * "Clock-change drain RE" §4): at a boot control timeout with boot_gpu_perf set (before any job, ++ * the firmware alive), every read goes through the GPU register licence (mmio.inc) and nothing ++ * wakes the GPU (no R1+0x14 write, mmio-access.md G6): ++ * r1: R1 +0, +8, +0x14, +0x1000, +0x1004, +0x20000 (one R1-only ticket, any pstate) ++ * drain: 11f00 688 1720 1728 e230 e238 e138, then the clock words R0+0x0/0x4/0x8/0xc ++ * timer: R0+0x160/0x164, twice, 1 ms apart ++ * tx: T0/T1 TXENABLE/TXSTATUS (0x04800000/10, 0x04801000/10) through the META slave port ++ * pc: (bit 1 only) T0 PC: RQ 0x0480fff8 = 0x10005, poll bit 31, read 0x0480fff0; x8, 100 us ++ * apart. This is the driver's first slave-port write (CTRL0 = addr, DATAT = value). ++ * One ticket per group or slave-port op (<= G5P_MMIO_MAX_OPS), a refusal retried <= 2 ms. ++ */ ++ ++#define G5P_GPP_BUDGET_US 2000U ++ ++/* One META slave-port read in one ticket: ready, CTRL0 = addr | 1, read back, ready (<= 4 ++ * polls), DATAX: 8 ops. -EBUSY when the port is not ready within the ticket. ++ */ ++static int g5p_mmio_sp_read(struct g5p_diag *g, u32 addr, u32 *out, u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1 = 0; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr | 1)) { ++ g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL0); ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ *out = g5p_mmio_r0_read(g, &t, G5P_MSLVDATAX); ++ ret = 0; ++ break; ++ } ++ } ++ } ++ if (!g5p_mmio_release(g, &t) && !ret) ++ ret = -ESTALE; ++ return ret; ++} ++ ++/* One META slave-port write in one ticket: ready, CTRL0 = addr, DATAT = value, ready (<= 4). */ ++static int g5p_mmio_sp_write(struct g5p_diag *g, u32 addr, u32 value, ++ u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr) && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVDATAT, value)) { ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ ret = 0; ++ break; ++ } ++ } ++ } ++ g5p_mmio_release(g, &t); ++ return ret; ++} ++ ++/* A group of plain R0 reads (<= G5P_MMIO_MAX_OPS) under one ticket; returns the ticket's did. */ ++static const char *g5p_gpp_r0(struct g5p_diag *g, const u32 *off, u32 *v, u32 n, ++ u32 *tries, u32 *gpc) ++{ ++ struct g5p_mmio_ticket t; ++ ++ if (g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries)) { ++ *gpc = t.gpc; ++ return g5p_mmio_did(&t); ++ } ++ for (u32 i = 0; i < n; i++) ++ v[i] = g5p_mmio_r0_read(g, &t, off[i]); ++ g5p_mmio_release(g, &t); ++ *gpc = t.gpc; ++ g5p_mmio_sample_line(g, G5P_MMS_PROBE, &t, v, n, true); ++ return g5p_mmio_did(&t); ++} ++ ++/* What a slave-port op's return means, for the dc-probe lines. */ ++static const char *g5p_sp_why(int ret) ++{ ++ switch (ret) { ++ case 0: ++ return "ok"; ++ case -ENODATA: ++ return "not-run"; ++ case -EPERM: ++ return "forbidden-after-release"; /* g->reset_transition: boot.inc's post-release ban */ ++ case -ENODEV: ++ return "licence-off"; ++ case -EAGAIN: ++ return "licence-refused"; ++ case -EBUSY: ++ return "port-not-ready"; ++ case -ESTALE: ++ return "stale"; ++ case -ETIMEDOUT: ++ return "rq-no-ready"; ++ default: ++ return "error"; ++ } ++} ++ ++static void g5p_gpu_perf_probe(struct g5p_diag *g) ++{ ++ static const u32 drain[] = { 0x11f00, 0x688, 0x1720, 0x1728, ++ 0xe230, 0xe238, 0xe138 }; ++ static const u32 clk[] = { 0x0, 0x4, 0x8, 0xc }; ++ static const u32 timer[] = { 0x160, 0x164 }; ++ static const u32 r1off[] = { 0x0, 0x8, 0x14, 0x1000, 0x1004, 0x20000 }; ++ static const u32 tx[] = { 0x04800000, 0x04800010, 0x04801000, ++ 0x04801010 }; ++ u32 mode = (0); ++ struct g5p_boot *b = g->boot; ++ u32 vd[ARRAY_SIZE(drain)], vc[ARRAY_SIZE(clk)], vt[2][2], ++ r1[ARRAY_SIZE(r1off)]; ++ u32 vx[ARRAY_SIZE(tx)], pc[8], pcrq[8], gate = U32_MAX, tries = 0, ++ gpc = U32_MAX, gpc_x; ++ int txr[ARRAY_SIZE(tx)], pcr[8]; ++ const char *d_r1 = "skip-guard", *d_drain = "skip-guard", ++ *d_clk = "skip-guard"; ++ const char *d_t0 = "skip-guard", *d_t1 = "skip-guard"; ++ struct g5p_mmio_ticket t; ++ ++ if (!mode || !b->gpu_perf) ++ return; ++ memset(vd, 0xff, sizeof(vd)); ++ memset(vc, 0xff, sizeof(vc)); ++ memset(vt, 0xff, sizeof(vt)); ++ memset(r1, 0xff, sizeof(r1)); ++ memset(vx, 0xff, sizeof(vx)); ++ memset(pc, 0xff, sizeof(pc)); ++ memset(pcrq, 0xff, sizeof(pcrq)); ++ for (u32 i = 0; i < ARRAY_SIZE(txr); i++) ++ txr[i] = -ENODATA; ++ for (u32 i = 0; i < ARRAY_SIZE(pcr); i++) ++ pcr[i] = -ENODATA; ++ if (g->pmgr && regmap_read(g->pmgr, 0x80340, &gate)) ++ gate = U32_MAX; ++ if (READ_ONCE(b->failed)) ++ goto print; ++ /* r1: before anything else, R1-only (valid at any pstate and with bit 29 set) */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) { ++ for (u32 i = 0; i < ARRAY_SIZE(r1off); i++) ++ r1[i] = g5p_mmio_r1_read(g, &t, r1off[i]); ++ g5p_mmio_release(g, &t); ++ } ++ d_r1 = g5p_mmio_did(&t); ++ /* gpc: the drain group's licence read; every group's ticket also has its own sample line */ ++ d_drain = g5p_gpp_r0(g, drain, vd, ARRAY_SIZE(drain), &tries, &gpc); ++ d_clk = g5p_gpp_r0(g, clk, vc, ARRAY_SIZE(clk), &tries, &gpc_x); ++ d_t0 = g5p_gpp_r0(g, timer, vt[0], 2, &tries, &gpc_x); ++ usleep_range(1000, 1100); ++ d_t1 = g5p_gpp_r0(g, timer, vt[1], 2, &tries, &gpc_x); ++ for (u32 i = 0; i < ARRAY_SIZE(tx); i++) ++ txr[i] = g5p_mmio_sp_read(g, tx[i], &vx[i], &tries); ++ if (mode & 2) { ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) { ++ pcr[i] = g5p_mmio_sp_write(g, 0x0480fff8, 0x10005, ++ &tries); ++ for (u32 k = 0; !pcr[i] && k < 4; k++) { ++ pcr[i] = g5p_mmio_sp_read(g, 0x0480fff8, ++ &pcrq[i], &tries); ++ if (pcr[i] || (pcrq[i] & BIT(31))) ++ break; ++ udelay(10); ++ } ++ if (!pcr[i]) ++ pcr[i] = pcrq[i] & BIT(31) ? ++ g5p_mmio_sp_read(g, 0x0480fff0, ++ &pc[i], ++ &tries) : ++ -ETIMEDOUT; ++ udelay(100); ++ } ++ } ++print: ++ dev_err(g->dev, ++ "GPUPERF dc-probe did=%s gate=%08x r1=%s r1_0=%08x r1_8=%08x r1_14=%08x " ++ "r1_1000=%08x r1_1004=%08x r1_20000=%08x gpc=%08x tries=%u end=gpuperf\n", ++ d_drain, gate, d_r1, r1[0], r1[1], r1[2], r1[3], r1[4], r1[5], ++ gpc, tries); ++ dev_err(g->dev, ++ "GPUPERF dc-probe drain=%s 11f00=%08x 688=%08x 1720=%08x 1728=%08x e230=%08x " ++ "e238=%08x e138=%08x clk=%s clk=%08x:%08x clkstat=%08x:%08x end=gpuperf\n", ++ d_drain, vd[0], vd[1], vd[2], vd[3], vd[4], vd[5], vd[6], d_clk, ++ vc[1], vc[0], vc[3], vc[2]); ++ dev_err(g->dev, ++ "GPUPERF dc-probe timer=%s/%s 160=%08x/%08x 164=%08x/%08x gap_us=1000 " ++ "txen0=%08x/%s txst0=%08x/%s txen1=%08x/%s txst1=%08x/%s end=gpuperf\n", ++ d_t0, d_t1, vt[0][0], vt[1][0], vt[0][1], vt[1][1], vx[0], ++ g5p_sp_why(txr[0]), vx[1], g5p_sp_why(txr[1]), vx[2], ++ g5p_sp_why(txr[2]), vx[3], g5p_sp_why(txr[3])); ++ if (mode & 2) ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) ++ dev_err(g->dev, ++ "GPUPERF dc-probe pc i=%u ret=%s rq=%08x t0pc=%08x end=gpuperf\n", ++ i, g5p_sp_why(pcr[i]), pcrq[i], pc[i]); ++} ++ ++static int g5p_control_wait(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long start = jiffies, deadline = start + 3 * HZ; ++ bool sampled = false, rekicked = false; ++ u32 consumer, producer, pass = 0; ++ int ret; ++ ++ if (!b->control_submitted || b->control_complete) ++ return -EPERM; ++ do { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_WAIT, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ consumer = READ_ONCE(ctl[0]); ++ producer = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ if (consumer > 255 || producer != b->control_target || ++ (consumer != b->control_target && ++ consumer != ((b->control_target - 1) & 255))) ++ return -EPROTO; ++ if (consumer == b->control_target) { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (g5p_r74_case(b) == 1) { ++ WRITE_ONCE(b->control_complete, true); ++ g5p_r74_mark(g, G5P_R74_CMD8_CONSUMED, ++ consumer); ++ dev_emerg( ++ g->dev, ++ "R74B cmd8-consumed consumer=%u producer=%u IRQ174=%d ram_only=1 end=r74b\n", ++ consumer, producer, ++ atomic_read(&b->irqs)); ++ g5p_np_policy(g, "cmd8-consumed", consumer, 0); ++ return 0; ++ } ++ g5p_dc_snapshot(g, "consumed", false); ++ b->control_complete = true; ++ ; ++ dev_info( ++ g->dev, ++ "DC consumed cmd=8 consumer=%u producer=%u IRQ174=%d; transport only\n", ++ consumer, producer, atomic_read(&b->irqs)); ++ return 0; ++ } ++ if (!sampled && !time_before(jiffies, start + HZ / 10)) { ++ sampled = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "100ms", false); ++ g5p_np_policy(g, "100ms", consumer, 0); ++ } ++ if (7 == 3 && !rekicked && !time_before(jiffies, start + HZ)) { ++ /* Re-notify the SAME slot once; never publish a second command. */ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_prepare_kick(g, "DC re-kick"); ++ if (ret) ++ return ret; ++ mb(); ++ writel(0x10, g->rgx + 0xb00); ++ rekicked = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "re-kick", false); ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "timeout", false); ++ g5p_np_policy(g, "timeout", consumer, 0); ++ dev_err(g->dev, ++ "DC timeout consumer=%u producer=%u target=%u; retain RAM, no retry\n", ++ consumer, producer, b->control_target); ++ /* perf60 (boot_gpu_perf): the policy-state words at the stall, RAM only. pol34 = the pstate the ++ * firmware last requested (its setter stores it before the Fender write): q means it reached the ++ * setter, 1 that it stopped before (the clock-change drain 0x1001cec8 -> 0x1001b638). ++ */ ++ if (b->gpu_perf) { ++ u32 pol = b->layout.offset[0x4a]; ++ ++ dma_rmb(); ++ dev_err(g->dev, ++ "GPUPERF dc-timeout pin=%u pol8=%u/%u q=%u x=%u pol34=%u req=%u/%u ceil=%u/%u " ++ "seed=%u pol0=%u pol4=%u pol28=%u pol2c=%u pol30=%u end=gpuperf\n", ++ b->gpu_perf, g5p_dc_root_word(b, pol + 8), ++ g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x38), ++ g5p_dc_root_word(b, pol + 0x34), ++ g5p_dc_root_word(b, pol + 0x6f0), ++ g5p_dc_root_word(b, pol + 0x6f4), ++ g5p_dc_root_word(b, pol + 0x688), ++ g5p_dc_root_word(b, pol + 0x68c), b->perf_seed, ++ g5p_dc_root_word(b, pol + 0), ++ g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30)); ++ g5p_gpu_perf_probe(g); ++ g5p_perf_readback(g, "dc-timeout"); ++ } ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ b->control_timed_out = true; ++ return -ETIMEDOUT; ++} ++ ++static int g5p_irq_test(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + HZ / 10; ++ int before = atomic_read(&b->irqs), ret; ++ ++ if (b->irq_tested || !b->irq_ready || g5p_r74_case(b)) ++ return -EPERM; ++ /* ADT set/ack addresses, not a GPU slave-port transaction. Firmware may ++ * also interrupt concurrently: this tests delivery, not event provenance. ++ */ ++ dev_info(g->dev, "IRQ test baseline=%d W AIC+014=00004000\n", before); ++ writel(0x4000, b->swirq + 0x14); ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (atomic_read(&b->irqs) > before) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ b->irq_tested = true; ++ ; ++ dev_info( ++ g->dev, ++ "IRQ delivery observed before=%d after=%d; no shader completion implied\n", ++ before, atomic_read(&b->irqs)); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, "IRQ delivery timeout before=%d after=%d\n", before, ++ atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++static int g5p_boot_park(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ static const u32 slots[] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ bool failed_control = READ_ONCE(b->failed) && b->control_timed_out; ++ bool failed_work = READ_ONCE(b->failed) && b->work_timed_out; ++ bool failed_context = READ_ONCE(b->failed) && b->context_timed_out; ++ bool failed_timeout = failed_control || failed_work || failed_context; ++ int ret; ++ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ if (READ_ONCE(b->event_failed) || ++ (b->control_submitted && !b->control_complete && !failed_control) || ++ (b->work_submitted && !b->work_complete && !failed_work) || ++ (b->context_submitted && !b->context_complete && !failed_context)) ++ return -EBUSY; ++ /* A failed drain must never be bypassed. On the sole permitted failure ++ * path require the event ring empty below without consuming anything. ++ */ ++ if (!failed_timeout) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(slots); i++) { ++ u32 *ctl = (u32 *)g5p_boot_shared(b, slots[i]); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (((failed_control || failed_context) && slots[i] == 0x6c) || ++ (failed_work && slots[i] == 0x68)) { ++ u32 target = failed_context && slots[i] == 0x6c ? ++ b->context_target : ++ slots[i] == 0x6c ? b->control_target : ++ b->work_target; ++ ++ if (producer != target || ++ (consumer != producer && ++ consumer != ((producer - 1) & 255))) ++ return -EPROTO; ++ } else if (consumer != producer) { ++ return -EBUSY; ++ } ++ } ++ if (READ_ONCE(b->event_failed)) ++ return -EIO; ++ if (failed_timeout) ++ dev_info( ++ g->dev, ++ "PARK failed %s timeout; abandon command, preserve failure and RAM\n", ++ failed_context ? "CONTEXT" : ++ failed_work ? "CL" : ++ "DC"); ++ /* Freeze commands before the first transition; no retry even if it fails. ++ * c9fa10 enabled-Fender branch: core=0, GPC=0, no a18 access. ++ */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ dev_info( ++ g->dev, ++ "PARK begin core=0 GPC=0; wrapper only, DMA memory retained\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ return ret; ++ /* Keep the IRQ/ack handler and every allocation alive until reboot. ++ * GPC=0 alone is not proof that outstanding bus traffic has drained. ++ */ ++ b->parked = true; ++ dev_info( ++ g->dev, ++ "PARK GPC=0 verified; quarantine until reboot, no PMGR off or DMA free\n"); ++ return 0; ++} ++ ++static int g5p_lifecycle_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (!boot_experiment || !b || !b->alive || b->stopped) ++ return -EPERM; ++ if (b->failed && ++ !(sysfs_streq(command, "park") && teardown_experiment && ++ (b->control_timed_out || b->work_timed_out || ++ b->context_timed_out))) ++ return -EPERM; ++ if (!strncmp(command, "control-", 8) && control_experiment) { ++ if (sysfs_streq(command, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(command, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(command, "control-wait")) ++ ret = g5p_control_wait(g); ++ else ++ return -EINVAL; ++ } else if (sysfs_streq(command, "irq-test") && ++ (g5p_r74_case(b) || (1))) { ++ /* R74 never runs the AIC self-test r73f case 01 stopped in. */ ++ if (b->r73b) ++ atomic_inc(&b->r73b->r74_irq_refused); ++ g5p_r74_mark(g, G5P_R74_IRQ_TEST_REFUSED, -EPERM); ++ dev_err(g->dev, ++ "R74 refuse irq-test case=%u irq_tested=%u end=r74\n", ++ g5p_r74_case(b), b->irq_tested); ++ return -EPERM; ++ } else if (sysfs_streq(command, "irq-test") && irq_experiment) { ++ ret = g5p_irq_test(g); ++ } else if (sysfs_streq(command, "park") && teardown_experiment) { ++ ret = g5p_boot_park(g); ++ } else { ++ return -EPERM; ++ } ++ if (sysfs_streq(command, "control-wait")) ++ g5p_r73b_crumb(b, G5P_R73B_WAIT_EXIT, G5P_R73B_OWNER_WAIT, ret); ++ if (ret && ret != -EALREADY && ret != -EPERM) ++ WRITE_ONCE(b->failed, true); ++ return ret; ++} ++ ++/* Outgoing-side kexec/reboot quiesce; see docs/gpu/experiments/gpu-reset-recovery-production.md ++ * section 6.10. NOT a reset: Fender core=0/GPC=0 only, no PMGR off, no DMA free and ++ * no R0 access. It acts only when the command lock is free (no job or sysfs command is ++ * inside a hardware wait), the firmware was released and the pstate policy is ++ * host-owned (init meta+0x6c8 == 0); every other state is left alone. Unlike ++ * g5p_boot_park() it does not require drained rings: round 10 flew park after an ++ * unconsumed DC command, and at shutdown there is nothing left to drain for. ++ * GPC=0 is not a proof that bus traffic drained (see g5p_boot_park). ++ */ ++ ++static void g5p_td_shutdown(struct g5p_diag *g); ++ ++static void g5p_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ struct g5p_boot *b; ++ int ret; ++ ++ if (!g) ++ return; ++ /* Idempotent second chance after the reboot notifier; a no-op unless teardown != 0. */ ++ g5p_td_shutdown(g); ++ { ++ return; ++ }; ++ /* A worker or sysfs writer stuck in MMIO holds g->lock: never queue behind it. */ ++ if (!mutex_trylock(&g->lock)) { ++ dev_warn(g->dev, "SHUTDOWN park skipped: command lock busy\n"); ++ return; ++ } ++ b = g->boot; ++ /* R1 is touched only with ps_gfx held on (the retained runtime-PM reference), ++ * the wrapper woken and the firmware released. ++ */ ++ if (!b || !g->powered || !g->fender_ready || !b->maps_ready || ++ !b->released || b->stopped || b->parked) { ++ dev_info(g->dev, "SHUTDOWN park not needed\n"); ++ goto out; ++ } ++ /* With firmware-owned policy (init meta+0x6c8 != 0) the firmware also writes ++ * R1+0x1000, so a host write would race it (dsx-r74b). RAM-only read of the ++ * host-written init word, the same one g5p_dc_snapshot() reports. ++ */ ++ if (g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8)) { ++ dev_info( ++ g->dev, ++ "SHUTDOWN park refused: firmware-owned policy (meta+0x6c8 != 0)\n"); ++ goto out; ++ } ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ /* Freeze commands before the first transition; no retry even if it fails. */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ ret = g5p_fender_idle(&io); ++ if (!ret) ++ WRITE_ONCE(b->parked, true); ++ dev_info( ++ g->dev, ++ "SHUTDOWN park core=0 GPC=0 result=%d; DMA memory retained, no PMGR off\n", ++ ret); ++out: ++ mutex_unlock(&g->lock); ++} +diff --git a/drivers/gpu/apple-g5p/meta71c114.h b/drivers/gpu/apple-g5p/meta71c114.h +new file mode 100644 +index 000000000..cc84dbd82 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/meta71c114.h +@@ -0,0 +1,65 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency114: one RAM dword, meta+0x71c, in the bootstrap image. ++ * ++ * meta (object 0x4d) +0x718 is the firmware's pstate frequency table in MHz. ++ * Its init copies the table to private+0x26c at boot (MINIM 0x1000122c..127c), ++ * and the scheduler loop reloads its timer from entry 1 on both branches ++ * (0x8000a1a4 via private+0x2c4 at pstate 1, 0x8000a338 via private+0x270). ++ * Linux leaves the table zero; Apple has { 0, 340, 474, 550, 723 }. The A/B ++ * installs entry 1 only: 0 is the control (nothing is stored, variant 7 leaves ++ * the word zero), 340 the factor. Everything else is refused at map, before ++ * any store. The store goes into the staging buffer, before g5p_boot_copy() ++ * publishes the image and long before the firmware is released. There is no ++ * register access here. ++ */ ++#ifndef APPLE_G5P_META71C114_H ++#define APPLE_G5P_META71C114_H ++ ++#define G5P_M71C_SLOT 0x4dU ++#define G5P_M71C_OFFSET 0x71cU ++#define G5P_M71C_CONTROL 0U ++#define G5P_M71C_FACTOR 340U ++#define G5P_M71C_VARIANT 7U ++ ++enum { ++ G5P_M71C_OK, ++ G5P_M71C_BAD_VALUE, ++ G5P_M71C_BAD_VARIANT, ++ G5P_M71C_WHYS, ++}; ++ ++static const char *const g5p_m71c_whys[G5P_M71C_WHYS] = { ++ "none", ++ "value", ++ "variant", ++}; ++ ++/* Latched at map: only the two pre-registered arms, the factor on dc7 only. */ ++static inline u32 g5p_m71c_check(u32 requested, u32 variant) ++{ ++ if (requested != G5P_M71C_CONTROL && requested != G5P_M71C_FACTOR) ++ return G5P_M71C_BAD_VALUE; ++ if (requested == G5P_M71C_FACTOR && variant != G5P_M71C_VARIANT) ++ return G5P_M71C_BAD_VARIANT; ++ return G5P_M71C_OK; ++} ++ ++/* ++ * One store into the staging image (the factor only) and its read-back. The ++ * caller fails the map unless the read-back equals the request. ++ */ ++static inline u32 g5p_m71c_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 requested, bool *wrote) ++{ ++ u8 *word = g5p_object(p, root, shared, G5P_M71C_SLOT) + G5P_M71C_OFFSET; ++ ++ *wrote = false; ++ if (requested == G5P_M71C_FACTOR) { ++ g5p_put32(word, requested); ++ *wrote = true; ++ } ++ return g5p_le32(word); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio-licence.h b/drivers/gpu/apple-g5p/mmio-licence.h +new file mode 100644 +index 000000000..d9d23ca2a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio-licence.h +@@ -0,0 +1,148 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU register licence (docs/gpu/experiments/mmio-access.md §4), pure so the host test ++ * (scripts/gpu/perf60-host/test_mmio.c) drives the same code. iOS's ++ * AGXMetaFirmware_isGPURegisterAccessible (fffffff006ca10ec) licenses an R0 access when PMGR gfx ++ * is on and the wrapper is up (fender+0xc), R1+0 bit 29 is clear (the firmware's power-down / ++ * clock-change routine 0x1001cec8 raises it before the GPC goes to 0) and the GPC pstate nibble is ++ * 1..meta+0x710. We are stricter: R1+0x1000 bit 4 (a transition in flight) must be clear and ++ * R1+0 == 0 (a dead wrapper, Apple's ca05d4 vt+0x70 branch) refuses too. ++ * ++ * Order, and nothing else touches the bus: the RAM terms (no MMIO when any fails), then R1+0, then ++ * R1+0x1000, then at most G5P_MMIO_MAX_OPS R0 accesses, then R1+0x1000 and R1+0 again at release ++ * to account a sample whose licence changed under it (stale: data loss, not exclusion). ++ * A refusal never waits and never wakes (no R1+0x14 write): the caller decides whether to retry. ++ */ ++#ifndef G5P_MMIO_LICENCE_H ++#define G5P_MMIO_LICENCE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_MMIO_MAX_OPS 8U /* R0 accesses per ticket */ ++#define G5P_MMIO_R1_CTRL \ ++ 0x0U /* R1+0: bit 29 = firmware power-down in progress */ ++#define G5P_MMIO_R1_GPC \ ++ 0x1000U /* R1+0x1000: bit 4 transition, nibble = pstate */ ++#define G5P_MMIO_BIT29 BIT(29) ++ ++enum g5p_mmio_why { ++ G5P_MMIO_OK, ++ G5P_MMIO_OFF, /* RAM: power epoch not readable (no MMIO done) -> -ENODEV */ ++ G5P_MMIO_BUSY, /* the lock was held (try mode, IRQ census) -> -EAGAIN, no MMIO */ ++ G5P_MMIO_DEAD, /* R1+0 == 0 */ ++ G5P_MMIO_FWOFF, /* R1+0 bit 29 */ ++ G5P_MMIO_TRANS, /* R1+0x1000 bit 4 */ ++ G5P_MMIO_PARKED, /* nibble 0 */ ++ G5P_MMIO_OVERMAX, /* nibble > max pstate */ ++ G5P_MMIO_PARKEPOCH, /* R0 in a park epoch (g5p_mmio_r0_regime_ok) */ ++ G5P_MMIO_NR, ++}; ++ ++static const char *const g5p_mmio_why_names[G5P_MMIO_NR] = { ++ "read", "skip-off", "skip-busy", ++ "skip-dead", "skip-bit29", "skip-transition", ++ "skip-parked", "skip-overmax", "skip-park-epoch", ++}; ++ ++/* The RAM preconditions (§4.2 step 2). b->failed is deliberately not one: the fault read runs ++ * after it latches, as Apple's does. ++ */ ++struct g5p_mmio_ram { ++ bool powered, fender_ready, mmio_off, have_boot, released, stopped; ++}; ++ ++static inline bool g5p_mmio_ram_ok(const struct g5p_mmio_ram *r) ++{ ++ return r->powered && r->fender_ready && !r->mmio_off && r->have_boot && ++ r->released && !r->stopped; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_r1_0(u32 r1_0) ++{ ++ if (!r1_0) ++ return G5P_MMIO_DEAD; ++ if (r1_0 & G5P_MMIO_BIT29) ++ return G5P_MMIO_FWOFF; ++ return G5P_MMIO_OK; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_gpc(u32 gpc, u32 max) ++{ ++ if (gpc & 0x10) ++ return G5P_MMIO_TRANS; ++ if (!(gpc & 0xf)) ++ return G5P_MMIO_PARKED; ++ if ((gpc & 0xf) > max) ++ return G5P_MMIO_OVERMAX; ++ return G5P_MMIO_OK; ++} ++ ++/* ++ * R0 only where the firmware cannot park under it. The governor's park does not raise R1+0 ++ * bit 29 (mmio-access.md §2.3: only the power-down routine 0x1001cec8 does, a few calls before it ++ * drops the GPC), so against a park the licence's R1 checks are a race: a check that passes can be ++ * followed by the GPC going to 0 before the R0 load. The never-park envelope (boot_gpu_perf >= ++ * 100, latched at map: the policy writer's x >= 100, q >= 1, the governor never selects pstate 0) ++ * has no such window; at pin 400/500 the 3D-completion and clock samples saw bit 29 0 times, while ++ * the park tuple's rung 5 (2026-11-03) saw it on 749 of 983 3D completions and then froze the SoC. ++ * iOS reads R0 only at a fault event, never at idle. `override` = mmio_r0_park (DFU-class rungs ++ * only). ++ */ ++static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool override) ++{ ++ return override || gpu_perf >= 100; ++} ++ ++/* ++ * meta+0x710 as written into the boot image; 0 (no table) means the firmware stays at pstate 1. ++ */ ++static inline u32 g5p_mmio_max_pstate(u32 meta710) ++{ ++ return meta710 < 1 ? 1 : meta710 > 15 ? 15 : meta710; ++} ++ ++typedef u32 (*g5p_mmio_rd_t)(void *ctx, u32 off); ++ ++/* §4.2 steps 2-4 under the caller's lock: no bus access unless the RAM terms pass, R1+0 before ++ * R1+0x1000, nothing after a refusal. *r1_0 / *gpc stay U32_MAX when not read. ++ */ ++static inline enum g5p_mmio_why g5p_mmio_licence(const struct g5p_mmio_ram *ram, ++ u32 max, g5p_mmio_rd_t rd, ++ void *ctx, u32 *r1_0, u32 *gpc) ++{ ++ enum g5p_mmio_why why; ++ ++ *r1_0 = 0xffffffffU; ++ *gpc = 0xffffffffU; ++ if (!g5p_mmio_ram_ok(ram)) ++ return G5P_MMIO_OFF; ++ *r1_0 = rd(ctx, G5P_MMIO_R1_CTRL); ++ why = g5p_mmio_check_r1_0(*r1_0); ++ if (why) ++ return why; ++ *gpc = rd(ctx, G5P_MMIO_R1_GPC); ++ return g5p_mmio_check_gpc(*gpc, max); ++} ++ ++/* An R0 op is allowed while ops < G5P_MMIO_MAX_OPS; the counter counts refusals too. */ ++static inline bool g5p_mmio_op_take(u32 *ops) ++{ ++ return (*ops)++ < G5P_MMIO_MAX_OPS; ++} ++ ++/* §4.3 release: the licence fields (nibble, bit 4, bit 29) unchanged and still licensing. */ ++static inline bool g5p_mmio_stale(u32 r1_0, u32 gpc, u32 r1_0_end, u32 gpc_end, ++ u32 max) ++{ ++ if ((gpc & 0x1f) != (gpc_end & 0x1f) || ++ (r1_0 & G5P_MMIO_BIT29) != (r1_0_end & G5P_MMIO_BIT29)) ++ return true; ++ return g5p_mmio_check_r1_0(r1_0_end) || ++ g5p_mmio_check_gpc(gpc_end, max); ++} ++ ++/* mmio_selftest rung 3/5 expectations: CORE_REVISION and CORE_ID as read at bring-up. */ ++#define G5P_MMIO_EXPECT_20 0x00090f00U ++#define G5P_MMIO_EXPECT_18 0x011b0602U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio.inc b/drivers/gpu/apple-g5p/mmio.inc +new file mode 100644 +index 000000000..b8c25bd8d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio.inc +@@ -0,0 +1,295 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The one GPU register accessor (docs/gpu/experiments/mmio-access.md §4), Apple's licence from ++ * AGXMetaFirmware_isGPURegisterAccessible (mmio-licence.h) under g->mmio_lock: ++ * ++ * if (!g5p_mmio_hold(g, &t, 0)) { ++ * v = g5p_mmio_r0_read(g, &t, off); // <= G5P_MMIO_MAX_OPS, nothing else in between ++ * g5p_mmio_release(g, &t); // false: the licence changed under the sample ++ * } ++ * ++ * hold takes mmio_lock with IRQs off and returns with it held on success. It refuses instead of ++ * waiting and never wakes the GPU (no R1+0x14 write): -ENODEV when the power epoch is not ++ * readable (RAM only, no MMIO done), -EAGAIN when parked, bit 29, in transition, above the ++ * pstate table, or (G5P_MMIO_TRY) when the lock is taken; -EPERM for an R0 ticket in an epoch ++ * the firmware may park (boot_gpu_perf < 100, unless mmio_r0_park). Callable from hard IRQ, ++ * threaded IRQ and process context, inside event_lock and g->lock. Between hold and release: no ++ * printk, no other MMIO, no sleep, no allocation. ++ * ++ * mmio_selftest (0600, default 0, live) drives the proof rungs of §6: ++ * bit 0 rung 1/1b: 1 kHz hrtimer census, R1+0 and R1+0x1000 only (no R0) ++ * bit 1 rung 3/4: every 100 ms hold -> R0+0x20, R0+0x18 -> release (DFU-risk with parks: rung 4) ++ * bit 2 rung 5: the same at every 3D completion decision (drm side; DFU-risk, park tuple) ++ * bit 3 idle check: after 5, 30 and 120 s with no job, the PMGR gfx word, then (gfx on only) ++ * R1+0x1000 and R1+0; never R0 ++ * Lines: GPU-MMIO sample (per sample, at most one per source per second unless the outcome ++ * changes or the sample is stale/bad), GPU-MMIO idle, GPU-MMIO stats (every 10 s while on and ++ * once when it goes back to 0). ++ */ ++#define G5P_MMIO_TRY \ ++ BIT(0) /* trylock: refuse busy instead of spinning (the IRQ census) */ ++/* An R1-only ticket, counted in the census class: held whenever the RAM terms pass (R1 is ++ * readable while PMGR gfx is on, mmio-access.md §1.3), R1+0 and R1+0x1000 both read even when the ++ * licence refuses; R0 only if the licence passed (t->r0_ok). ++ */ ++#define G5P_MMIO_CENSUS BIT(1) ++ ++struct g5p_mmio_ticket { ++ u32 r1_0, gpc, r1_0_end, gpc_end, flags, ops; ++ u64 t_ns, t_end_ns; ++ unsigned long irq; ++ enum g5p_mmio_why why; ++ bool held, stale, r0_ok; ++}; ++ ++enum { ++ G5P_MMS_CENSUS, ++ G5P_MMS_IDLE, ++ G5P_MMS_3D, ++ G5P_MMS_FAULT, ++ G5P_MMS_PROBE, ++ G5P_MMS_NR ++}; ++static const char *const g5p_mms_names[G5P_MMS_NR] = { ++ "census", "idle-100ms", "3d-done", "fault-read", "dc-probe", ++}; ++ ++#define G5P_MMIO_RING 32 ++struct g5p_mmio_rec { ++ u64 t_ns; ++ u32 r1_0, gpc, us; ++ u8 why, stale; ++}; ++ ++struct g5p_mmio_self { ++ struct g5p_diag *g; ++ struct hrtimer timer; ++ struct delayed_work work; ++ bool census_on, was_on; ++ u64 stats_ns; ++ /* counters: [0] R1-only census tickets, [1] R0 tickets */ ++ atomic_t n[2][G5P_MMIO_NR]; ++ atomic_t stale[2], r0_ops, bad; ++ /* census transitions, written by the hrtimer, drained by the work */ ++ spinlock_t ring_lock; ++ struct g5p_mmio_rec ring[G5P_MMIO_RING]; ++ u32 ring_head, ring_tail, ring_drop; ++ struct g5p_mmio_rec census_last; ++ u8 census_prev_why, census_prev_stale; ++ /* per-source print limiter */ ++ u64 printed_ns[G5P_MMS_NR]; ++ u8 printed_why[G5P_MMS_NR]; ++ /* idle check */ ++ u64 job_ns; ++ u32 idle_done; ++ /* bit 4, the GPU clock: per Fender pstate nibble, the highest rate seen and the sample counts */ ++ u32 clk_max_khz[16], clk_cr_khz[16], clk_n[16], clk_busy[16], clk_reset; ++ u64 clk_printed_ns; ++}; ++ ++/* mmio_selftest bit 4: the gap between the two tickets of one clock sample */ ++ ++/* R0 tickets in a park epoch (boot_gpu_perf < 100): refused unless 1 (g5p_mmio_r0_regime_ok). */ ++ ++static u32 g5p_mmio_rd_r1(void *ctx, u32 off) ++{ ++ return readl((void __iomem *)ctx + off); ++} ++ ++static void g5p_mmio_count(struct g5p_diag *g, const struct g5p_mmio_ticket *t) ++{ ++ struct g5p_mmio_self *s = READ_ONCE(g->mmio_self); ++ u32 cls = t->flags & G5P_MMIO_CENSUS ? 0 : 1; ++ ++ if (!s) ++ return; ++ atomic_inc(&s->n[cls][t->why]); ++ if (t->stale) ++ atomic_inc(&s->stale[cls]); ++ if (t->ops && cls) ++ atomic_add(min_t(u32, t->ops, G5P_MMIO_MAX_OPS), &s->r0_ops); ++} ++ ++static int g5p_mmio_hold(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags) ++{ ++ struct g5p_mmio_ram ram = { 0 }; ++ struct g5p_boot *b; ++ unsigned long irq; ++ ++ memset(t, 0, sizeof(*t)); ++ t->flags = flags; ++ t->r1_0 = t->gpc = t->r1_0_end = t->gpc_end = U32_MAX; ++ if (flags & G5P_MMIO_TRY) { ++ if (!spin_trylock_irqsave(&g->mmio_lock, irq)) { ++ t->why = G5P_MMIO_BUSY; ++ goto refused; ++ } ++ } else { ++ spin_lock_irqsave(&g->mmio_lock, irq); ++ } ++ t->t_ns = ktime_get_mono_fast_ns(); ++ ram.mmio_off = g->mmio_off; ++ ram.powered = READ_ONCE(g->powered); ++ ram.fender_ready = READ_ONCE(g->fender_ready); ++ /* mmio_off first: a boot struct behind a gate is never dereferenced */ ++ b = ram.mmio_off ? NULL : READ_ONCE(g->boot); ++ if (b) { ++ ram.have_boot = true; ++ ram.released = READ_ONCE(b->released); ++ ram.stopped = READ_ONCE(b->stopped); ++ } ++ /* before any bus access: an R0 ticket in a park epoch is refused outright */ ++ if (!(flags & G5P_MMIO_CENSUS) && b && ++ !g5p_mmio_r0_regime_ok(b->gpu_perf, (0))) { ++ t->why = G5P_MMIO_PARKEPOCH; ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ t->why = g5p_mmio_licence(&ram, READ_ONCE(g->mmio_max_pstate), ++ g5p_mmio_rd_r1, g->window[1], &t->r1_0, ++ &t->gpc); ++ if (t->why && (!(flags & G5P_MMIO_CENSUS) || t->why == G5P_MMIO_OFF)) { ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ if (t->gpc == U32_MAX) ++ t->gpc = readl( ++ g->window[1] + ++ G5P_MMIO_R1_GPC); /* census: after an R1+0 refusal */ ++ t->r0_ok = !t->why; ++ t->irq = irq; ++ t->held = true; ++ return 0; ++refused: ++ g5p_mmio_count(g, t); ++ /* a park epoch does not change by waiting: -EPERM, never retried (g5p_mmio_hold_wait) */ ++ return t->why == G5P_MMIO_OFF ? -ENODEV : ++ t->why == G5P_MMIO_PARKEPOCH ? -EPERM : ++ -EAGAIN; ++} ++ ++/* Only between a successful hold and its release; past G5P_MMIO_MAX_OPS no bus access. */ ++static u32 g5p_mmio_r0_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->rgx + off); ++} ++ ++/* R0 writes count as ops too (the slave-port request words of the pin-200 probe). */ ++static bool g5p_mmio_r0_write(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off, u32 v) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return false; ++ writel(v, g->rgx + off); ++ return true; ++} ++ ++/* Any R1 word under a held ticket (census or licensed); counts as an op. */ ++static u32 g5p_mmio_r1_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->window[1] + off); ++} ++ ++/* Process context: refusals (-EAGAIN) retried outside the lock every 20-40 us, <= budget_us. */ ++static int g5p_mmio_hold_wait(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags, u32 budget_us, u32 *tries) ++{ ++ u64 t0 = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ might_sleep(); ++ for (;;) { ++ (*tries)++; ++ ret = g5p_mmio_hold(g, t, flags); ++ if (ret != -EAGAIN || ktime_get_mono_fast_ns() - t0 >= ++ (u64)budget_us * NSEC_PER_USEC) ++ return ret; ++ usleep_range(20, 40); ++ } ++} ++ ++static bool g5p_mmio_release(struct g5p_diag *g, struct g5p_mmio_ticket *t) ++{ ++ if (!t->held) ++ return false; ++ t->gpc_end = readl(g->window[1] + G5P_MMIO_R1_GPC); ++ t->r1_0_end = readl(g->window[1] + G5P_MMIO_R1_CTRL); ++ t->stale = t->r0_ok && ++ g5p_mmio_stale(t->r1_0, t->gpc, t->r1_0_end, t->gpc_end, ++ READ_ONCE(g->mmio_max_pstate)); ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ t->held = false; ++ spin_unlock_irqrestore(&g->mmio_lock, t->irq); ++ g5p_mmio_count(g, t); ++ return !t->stale; ++} ++ ++static const char *g5p_mmio_did(const struct g5p_mmio_ticket *t) ++{ ++ return t->stale ? "stale" : g5p_mmio_why_names[t->why]; ++} ++ ++static u32 g5p_mmio_us(const struct g5p_mmio_ticket *t) ++{ ++ return t->t_end_ns > t->t_ns ? ++ (u32)div_u64(t->t_end_ns - t->t_ns, NSEC_PER_USEC) : ++ 0; ++} ++ ++/* The per-sample line, after release. force: print regardless of the per-source limiter. */ ++static void g5p_mmio_sample_line(struct g5p_diag *g, u32 src, ++ const struct g5p_mmio_ticket *t, u32 *v, ++ u32 nv, bool force) ++{ ++ struct g5p_mmio_self *s = g->mmio_self; ++ u64 now = ktime_get_mono_fast_ns(); ++ char vs[9 * G5P_MMIO_MAX_OPS + 1] = "-"; ++ u32 len = 0; ++ ++ if (s && !force && !t->stale && s->printed_why[src] == t->why && ++ now - s->printed_ns[src] < NSEC_PER_SEC) ++ return; ++ if (s) { ++ s->printed_ns[src] = now; ++ s->printed_why[src] = t->why; ++ } ++ for (u32 i = 0; i < nv && i < G5P_MMIO_MAX_OPS; i++) ++ len += scnprintf(vs + len, sizeof(vs) - len, "%s%08x", ++ i ? "," : "", v[i]); ++ dev_info( ++ g->dev, ++ "GPU-MMIO sample why=%s did=%s r1_0=%08x gpc=%08x r1_0_end=%08x gpc_end=%08x " ++ "max=%u v=%s us=%u end=mmio\n", ++ g5p_mms_names[src], g5p_mmio_did(t), t->r1_0, t->gpc, ++ t->r1_0_end, t->gpc_end, READ_ONCE(g->mmio_max_pstate), vs, ++ g5p_mmio_us(t)); ++} ++ ++/* Rung 3/5: one licensed ticket, CORE_REVISION and CORE_ID. */ ++ ++/* drm side: rung 5 at the 3D completion decision, and the idle clock for bit 3. */ ++ ++/* Bit 4: the GPU core clock. CR_TIMER (R0+0x160) ticks once per 256 core clocks whether or not a ++ * phase is open (r3clk: 2823 kHz at pstate 4 = 722.7 MHz, 3703 kHz at 948 MHz), so CR_TIMER over ++ * host time x 256 is the clock at any moment. The Rogue PERF_TA_CYCLE / PERF_3D_CYCLE / ++ * PERF_TA_OR_3D_CYCLE counters (R0+0x6020/0x6028/0x6038, 32 bit) advance at the core clock while a ++ * TA or 3D phase is open (340.0 MHz through open TA phases, docs/gpu/experiments/46-device-tests.md): ++ * over host time, the highest rate per pstate (a window busy all through) is the clock again. One ++ * sample = two licensed tickets, mmio_clock_gap_us apart, each reading the three counters and ++ * CR_TIMER's low word back to back. A cycle rate above 2 GHz is a firmware counter clear ++ * (R0+0x6000 = 1, 0x1001ab00) inside the window and is dropped. ++ */ ++ ++/* Bit 3: does the GPC power down at full idle in state 0? The PMGR word is always readable; ++ * R1 only while the gate reads on (actual == target == 0xf), through a census ticket so the RAM ++ * terms and the lock still apply. A refused licence still returns the R1 words it read. ++ */ +diff --git a/drivers/gpu/apple-g5p/mmu-attrs.inc b/drivers/gpu/apple-g5p/mmu-attrs.inc +new file mode 100644 +index 000000000..694552016 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-attrs.inc +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include ++#include ++ ++/* Read the actual stable kernel mapping of a retained DMA allocation. ++ * DMA memory may be a vmalloc PTE or a cached linear-map block. ++ */ ++static u64 g5p_table_cpu_descriptor(unsigned long va) ++{ ++ pgd_t *pgd = pgd_offset_k(va); ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ ++ if (pgd_none(READ_ONCE(*pgd)) || pgd_bad(READ_ONCE(*pgd))) ++ return 0; ++ p4d = p4d_offset(pgd, va); ++ if (p4d_none(READ_ONCE(*p4d)) || p4d_bad(READ_ONCE(*p4d))) ++ return 0; ++ pud = pud_offset(p4d, va); ++ if (pud_leaf(READ_ONCE(*pud))) ++ return pud_val(READ_ONCE(*pud)); ++ if (pud_none(READ_ONCE(*pud)) || pud_bad(READ_ONCE(*pud))) ++ return 0; ++ pmd = pmd_offset(pud, va); ++ if (pmd_leaf(READ_ONCE(*pmd))) ++ return pmd_val(READ_ONCE(*pmd)); ++ if (pmd_none(READ_ONCE(*pmd)) || pmd_bad(READ_ONCE(*pmd))) ++ return 0; ++ return pte_val(READ_ONCE(*pte_offset_kernel(pmd, va))); ++} ++ ++static void g5p_table_attrs(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *p) ++{ ++ u64 desc, mair = read_sysreg(mair_el1); ++ u32 index, attr; ++ ++ if (!p->cpu) ++ return; ++ desc = g5p_table_cpu_descriptor((unsigned long)p->cpu); ++ index = (desc & PTE_ATTRINDX_MASK) >> 2; ++ attr = (mair >> (index * 8)) & 0xff; ++ g5p_mmu_record( ++ g, ++ "MMU allocation %s DMA=%pad coherent=%u cpu_desc=%016llx AttrIndx=%u " ++ "MAIR=%016llx attr=%02x table_memtype=%s\n", ++ stage, &p->dma, dev_is_dma_coherent(g->dev), desc, index, mair, ++ attr, ++ !desc ? "unknown" : ++ attr == 0xff ? "Normal-WB" : ++ attr == 0x44 ? "Normal-NC" : ++ "other"); ++} +diff --git a/drivers/gpu/apple-g5p/mmu-tables.inc b/drivers/gpu/apple-g5p/mmu-tables.inc +new file mode 100644 +index 000000000..94ae6ed8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-tables.inc +@@ -0,0 +1,57 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Fixed, pre-release aliases only. No firmware patch or live table surgery. */ ++#define G5P_TABLE_ALIAS_VA (G5P_ROOT_VA + 0x100000ULL) ++#define G5P_TABLE_COOKIE 0x3ffcU ++ ++static void g5p_table_clean_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ unsigned long start; ++ ++ if (!p->cpu) ++ return; ++ start = (unsigned long)p->cpu; ++ /* Explicit architecture PoC operation, even if DT calls this coherent. ++ * dma_sync_single_for_device is not an API for coherent allocations. ++ */ ++ dcache_clean_poc(start, start + G5P_GPU_PAGE); ++} ++ ++static void g5p_tables_clean(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *user, u32 count) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ g5p_mmu_record( ++ g, ++ "MMU tables %s clean_poc=%u dma_coherent=%u dma_pgprot=%016llx; coherent DMA " ++ "API, extra clean is experimental\n", ++ stage, 1, dev_is_dma_coherent(g->dev), ++ (u64)pgprot_val(dma_pgprot(g->dev, PAGE_KERNEL, 0))); ++ for (u32 i = 0; i < count; i++) ++ g5p_table_attrs(g, stage, &user[i]); ++ g5p_table_attrs(g, stage, &b->catalogue); ++ g5p_table_attrs(g, stage, &b->directory); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_attrs(g, stage, &b->leaf[i]); ++ ; ++ ; ++ for (u32 i = 0; i < count; i++) ++ g5p_table_clean_page(g, &user[i]); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_clean_page(g, &b->leaf[i]); ++ g5p_table_clean_page(g, &b->sink_leaf); ++ g5p_table_clean_page(g, &b->sink_directory); ++ g5p_table_clean_page(g, &b->directory); ++ g5p_table_clean_page(g, &b->catalogue); ++ dsb(sy); ++ dma_wmb(); ++ g5p_mmu_record( ++ g, ++ "MMU tables %s PoC complete user_pages=%u boot_leaves=%u; dsb sy before publication\n", ++ stage, count, b->leaves); ++} ++ ++/* Deliberately separate from g5p_meta_read(): the generic slave guard stays ++ * latched after reset. This opt-in reader can address only the preinstalled ++ * table aliases and one immutable root control word, never private META RAM. ++ */ +diff --git a/drivers/gpu/apple-g5p/pack.h b/drivers/gpu/apple-g5p/pack.h +new file mode 100644 +index 000000000..942b44ba3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/pack.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_PACK_H ++#define APPLE_G5P_PACK_H ++ ++#include "bits-compat.h" ++#include "ldr.h" ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_IMAGE_SIZE 0x30000 ++#define G5P_CONFIG_WRITES 27 ++ ++struct g5p_ldr_image { ++ u8 *data; ++ u32 config[G5P_CONFIG_WRITES][2]; ++ u32 config_count; ++ u32 t1_pc; ++ u32 t1_sp; ++}; ++ ++/* Exact C0 ranges derived from AGXMetaFirmware_layoutAndLoadLDR ...6c9f1bc. ++ * GPU virtual mappings and the boot config at +0x200 are intentionally separate. ++ */ ++static inline int g5p_image_offset(u32 addr, u32 size, u32 *offset) ++{ ++ static const struct { ++ u32 base, size, packed; ++ } segments[] = { ++ { 0x10000000, 0x18000, 0x04000 }, ++ { 0x20000000, 0x04000, 0x1c000 }, ++ { 0xc0000000, 0x04000, 0x00000 }, ++ { 0xe0001000, 0x04000, 0x20000 }, ++ { 0x80000080, 0x055dc, 0x24000 }, ++ }; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(segments); i++) { ++ if (addr >= segments[i].base && size <= segments[i].size && ++ addr - segments[i].base <= segments[i].size - size) { ++ *offset = segments[i].packed + addr - segments[i].base; ++ return 0; ++ } ++ } ++ return -EINVAL; ++} ++ ++static inline int g5p_pack_record(void *ctx, enum g5p_ldr_op op, u32 addr, ++ u32 size, const u8 *data) ++{ ++ struct g5p_ldr_image *image = ctx; ++ u32 i, offset; ++ int ret; ++ ++ switch (op) { ++ case G5P_LOAD: ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memcpy(image->data + offset, data, size); ++ break; ++ case G5P_ZERO: ++ /* Apple does not apply the local/core-memory ZeroMem records here. */ ++ if (addr >= 0x80000000 && addr < 0x88000000) ++ break; ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memset(image->data + offset, 0, size); ++ break; ++ case G5P_CONFIG: ++ for (i = 0; i < size; i += 12) { ++ if (image->config_count == G5P_CONFIG_WRITES) ++ return -EINVAL; ++ image->config[image->config_count][0] = ++ g5p_le32(data + i + 4); ++ image->config[image->config_count++][1] = ++ g5p_le32(data + i + 8); ++ } ++ break; ++ case G5P_START: ++ image->t1_sp = g5p_le32(data + 4); ++ image->t1_pc = g5p_le32(data + 8); ++ break; ++ } ++ return 0; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/perf-envelope.h b/drivers/gpu/apple-g5p/perf-envelope.h +new file mode 100644 +index 000000000..57387bcba +--- /dev/null ++++ b/drivers/gpu/apple-g5p/perf-envelope.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 clock: the inputs of the firmware's policy-state writer (0x10008218) and of its governor, ++ * so the firmware itself moves the GPC from pstate 1 to q = pin / 100. Both compute ++ * x = min(policy+0x6f0, policy+0x688, ctrl+0x102c) ++ * (META CS is unsigned lower; iOS's live policy+0x38 equals this min in every sample), store ++ * policy+8 = 0 when x % 100 == 0 (2 when x < 100), and request pstate q = x / 100 (policy+0x28). ++ * +0x6f0 and +0x688 are recomputed by the firmware from its coefficient block unless their ++ * overrides +0x6f4 / +0x68c are nonzero, and the firmware itself reloads those overrides from ++ * the control object on every pass, whatever the record held: +0x6f4 = ctrl+0x1040 (0x1000a1e4, ++ * 0x1000b388, 0x1000c568) and +0x68c = ctrl+0x1038 (0x1000a74c, 0x1000b10c, 0x1000c2fc). iOS ++ * leaves both control words 0 and arms the sampler (policy+4 = 16); this boot pins them ++ * (ctrl+0x1040 = pin, ctrl+0x1038 = 400) and leaves the sampler off (policy+4 = 0), so there is ++ * no idle parking at pstate 0. The pstate table and limits are iOS's (live j71t capture, ++ * 2026-11-01): meta+0x710/0x714 = 4/5, MHz {0, 340, 474, 550, 723}, ctrl+0x1028/0x1030 = ++ * 3348/400; the floor ctrl+0x102c is the pin. RAM only, before release. ++ */ ++#ifndef APPLE_G5P_PERF_ENVELOPE_H ++#define APPLE_G5P_PERF_ENVELOPE_H ++ ++#include "bits-compat.h" ++#define G5P_PERF_POLICY 0x4aU ++#define G5P_PERF_META 0x4dU ++#define G5P_PERF_CTL 0x5eU ++#define G5P_PERF_TOP 400U ++ ++static const u32 g5p_perf_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline bool g5p_perf_valid(u32 pin) ++{ ++ return pin <= G5P_PERF_TOP && !(pin % 100); ++} ++ ++static inline void g5p_perf_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 pin) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ u8 *meta = g5p_object(p, root, shared, G5P_PERF_META); ++ u8 *ctl = g5p_object(p, root, shared, G5P_PERF_CTL); ++ u32 i; ++ ++ g5p_put32(meta + 0x710, ARRAY_SIZE(g5p_perf_mhz) - 1); ++ g5p_put32(meta + 0x714, ARRAY_SIZE(g5p_perf_mhz)); ++ for (i = 0; i < ARRAY_SIZE(g5p_perf_mhz); i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_perf_mhz[i]); ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, pin); ++ g5p_put32(ctl + 0x1030, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1038, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1040, pin); ++ g5p_put32(policy + 0x688, G5P_PERF_TOP); ++ g5p_put32(policy + 0x68c, G5P_PERF_TOP); ++ g5p_put32(policy + 0x6f0, pin); ++ g5p_put32(policy + 0x6f4, pin); ++} ++ ++/* boot_gpu_oc_mhz (gpu-oc.h): state 5 = the overclock record; the firmware's table, ceilings and ++ * policy limits extended to index 5 so the pin 500 and the power-on request of state 5 are in range ++ * (the firmware derives its clock entry priv+0x2c4 = meta+0x718[state] from Fender, 0x1001c838). ++ */ ++ ++/* ++ * boot_gpu_perf_seed (perf60-clock.md "Clock-change drain RE" §3): in policy state 0 the ++ * firmware ++ * never changes pstate while powered; only its boot sampler (0x1000aa80, before ALIVE) moves the ++ * GPC, when policy+0x28 != Fender & 0xf. Seeding the policy record with the firmware's own ++ * steady-state output for the pin, +0x28 = +0x2c = q, +0x30 = 0, +8 = 0, makes that sampler ++ * request q through its setter (0x1001c9b8 -> 0x1001c88c: pol34 = q, R1+0x1000 = q | 0x10), ++ * which is ++ * iOS's park -> reboot -> sampler path. q = pin / 100 for pins 200..400, else 0 (no seed). ++ */ ++static inline u32 g5p_perf_seed_q(u32 pin) ++{ ++ return pin >= 200 && pin <= G5P_PERF_TOP && !(pin % 100) ? pin / 100 : ++ 0; ++} ++ ++/* Seed 2 adds what the sampler needs to get past its first-call init: 0x1000aa80 at 0x1000abb0 ++ * takes policy+0 == 0 as "never sampled" and only initialises (0x1000ae88: +0 = 1, its time words) ++ * without reaching the target compare at 0x1000ada4 or the setter; its sampling path divides by ++ * policy+4 (0x10022f88 at 0x1000ac28). The boot call at 0x10000b54 is its only caller, so a fresh ++ * record never moves Fender. iOS's record persists across firmware reboots: +0 = 1 and +4 = 16 in ++ * every sample of the 2026-11-01 capture (idle and load). Seed 2 writes those two as iOS has them. ++ * Seed 2 on hardware (2026-11-03): no ALIVE, Fender 0. With +0 = 1 the sampler takes its sampling ++ * path when (now - policy+0x8f8) / 24000 / policy+4 >= 1 (0x1000ac10..ac5c): +0x8f8 is the host's ++ * map-time stamp, ~3 s old at release, so it runs the load governor (0x1000afa4 -> 0x1000b050, busy ++ * accumulators policy+0x62c..0x68c, all 0 on a fresh record) and lands on pstate 0. Seed 3 ++ * (boot.inc ++ * g5p_boot_release) restamps +0x8f8 right before the release stream: < 1 period elapsed, the load ++ * update is skipped (0x1000ac64 -> 0x1000acb0 -> 0x1000ada4) and the compare against +0x28 = q ++ * reaches the setter (0x1000ae68 -> 0x1001c9b8). ++ */ ++static inline void g5p_perf_seed(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 q, bool ios_record) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ ++ g5p_put64(policy + 0x28, (u64)q << 32 | q); ++ g5p_put32(policy + 0x30, 0); ++ g5p_put32(policy + 8, 0); ++ if (ios_record) { ++ g5p_put32(policy + 0, 1); ++ g5p_put32(policy + 4, 16); ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/power-r33.inc b/drivers/gpu/apple-g5p/power-r33.inc +new file mode 100644 +index 000000000..ce8a18f3c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r33.inc +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R33 (ds-r33-b knob A): the kexec GFX cycle holds ps_gfx off for ~10 us, and ++ * R0+0x418/0x41c (no reset) keeps the previous boot's value through it. At the ++ * first Fender wake, before any GPU MMIO and before any GPU IRQ is requested, ++ * hold the same PMGR gate (0x80340, gate124) off for boot_gfx_off_ms, then ++ * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset, ++ * auto). Every exit leaves the gate requested on. ++ */ ++#define G5P_R33_GFX_PS 0x80340U ++#define G5P_R33_GFX_OFF_MAX_MS 2000U ++#define G5P_R33_GFX_POLL_US 10000U ++ ++static inline bool g5p_r33_gfx_on(u32 raw) ++{ ++ return (raw & 0xf) == 0xf && ((raw >> 4) & 0xf) == 0xf; ++} ++ ++/* Only a plain, fully-on, software-controlled gate may be cycled. */ ++static inline bool g5p_r33_gfx_cyclable(u32 raw) ++{ ++ return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31) && ++ !((raw >> 28) & 1); ++} ++ ++static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) ++{ ++ u32 before, held, v = 0, after = 0; ++ u64 t0; ++ int ret, on; ++ ++ if (!ms) ++ return 0; ++ if (ms > G5P_R33_GFX_OFF_MAX_MS || !pmgr) ++ return -EINVAL; ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &before); ++ if (ret) ++ return ret; ++ if (!g5p_r33_gfx_cyclable(before)) { ++ dev_info( ++ dev, ++ "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset/auto or not on\n", ++ ms, before); ++ return -EBUSY; ++ } ++ ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0); ++ if (!ret) ++ ret = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, v, ++ !((v >> 4) & 0xf), 10, ++ G5P_R33_GFX_POLL_US); ++ t0 = ktime_get_ns(); ++ if (!ret) ++ msleep(ms); ++ held = (u32)((ktime_get_ns() - t0) / 1000); ++ if (!ret) ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &v); ++ /* Never leave the GPU domain off, whatever happened above. */ ++ on = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0xf); ++ if (!on) ++ on = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, after, ++ g5p_r33_gfx_on(after), 10, ++ G5P_R33_GFX_POLL_US); ++ dev_info( ++ dev, ++ "POWER gfx-off ms=%u held_us=%u before=%08x end=%08x after=%08x off_ret=%d " ++ "on_ret=%d\n", ++ ms, held, before, v, after, ret, on); ++ if (!ret && (v >> 4) & 0xf) ++ ret = -EIO; ++ return ret ?: on; ++} +diff --git a/drivers/gpu/apple-g5p/power-r41.inc b/drivers/gpu/apple-g5p/power-r41.inc +new file mode 100644 +index 000000000..9e5c1e668 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r41.inc +@@ -0,0 +1,135 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* J71t live ADT + ApplePMGR_restoreDeviceBridgeSettings (5a39918). ++ * Volatile fabric MMIO only. GFX selects bridge 8, then bridge 9; the two ++ * version-0 counter controls follow bridge 8's table, before bridge 9. ++ * Keep devm_ioremap/readl/writel identical to boot.inc's swirq mapping. ++ */ ++static const unsigned int gfx_bridge_replay = 0; ++ ++struct g5p_r41_write { ++ u32 bridge, offset, value; ++}; ++ ++static const struct g5p_r41_write g5p_r41_writes[] = { ++ { 8, 0x0000, 0x00000120 }, { 8, 0x0018, 0x80010005 }, ++ { 8, 0x001c, 0x00010001 }, { 8, 0x410c, 0x00000003 }, ++ { 8, 0x4100, 0x00000003 }, { 9, 0x0000, 0x00697101 }, ++ { 9, 0x0004, 0x00697101 }, { 9, 0x0008, 0x00697101 }, ++ { 9, 0x000c, 0x00697101 }, { 9, 0x0010, 0x00697101 }, ++ { 9, 0x0014, 0x00697101 }, { 9, 0x0018, 0x00697101 }, ++ { 9, 0x001c, 0x00697101 }, { 9, 0x0020, 0x00697101 }, ++ { 9, 0x0024, 0x00697101 }, { 9, 0x0028, 0x00697101 }, ++ { 9, 0x002c, 0x00697101 }, { 9, 0x0030, 0x00697101 }, ++ { 9, 0x0034, 0x00697101 }, { 9, 0x0038, 0x00697101 }, ++ { 9, 0x003c, 0x00697101 }, { 9, 0x0040, 0x00697101 }, ++ { 9, 0x0044, 0x00697101 }, { 9, 0x0048, 0x00697101 }, ++ { 9, 0x004c, 0x00697101 }, { 9, 0x0050, 0x00697101 }, ++ { 9, 0x0054, 0x00697101 }, { 9, 0x0058, 0x00697101 }, ++ { 9, 0x005c, 0x00697101 }, { 9, 0x0060, 0x00697101 }, ++ { 9, 0x0064, 0x00697101 }, { 9, 0x0068, 0x00697101 }, ++ { 9, 0x006c, 0x00697101 }, { 9, 0x0070, 0x00697101 }, ++ { 9, 0x0074, 0x00697101 }, { 9, 0x0078, 0x00697101 }, ++ { 9, 0x007c, 0x00697101 }, ++}; ++ ++static struct { ++ void __iomem *window[2]; ++ unsigned int mode; ++ int result; ++ bool attempted, ready, firmware_logged, ta_logged; ++} g5p_r41_bridge; ++ ++/* Reject anything outside the exact write allowlist, including unaligned ++ * offsets and other registers inside the two mapped ADT ranges. ++ */ ++static bool g5p_r41_valid(const struct g5p_r41_write *w) ++{ ++ if (w->offset & 3) ++ return false; ++ if (w->bridge == 8) ++ return w->offset < 0x5000 && ++ (w->offset == 0 || w->offset == 0x18 || ++ w->offset == 0x1c || w->offset == 0x410c || ++ w->offset == 0x4100); ++ return w->bridge == 9 && w->offset <= 0x7c; ++} ++ ++static void g5p_r41_bridge_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ u32 value = ++ readl(g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ ++ dev_info( ++ g->dev, ++ "BRIDGE r41 stage=%s mode=%u index=%u bridge=%u off=%04x value=%08x want=%08x\n", ++ stage, g5p_r41_bridge.mode, i, w->bridge, w->offset, ++ value, w->value); ++ } ++} ++ ++/* Called under g->lock after ACTIVE (and any r33 off/on), before Fender. ++ * Latch errors too: never retry a partial experiment in the same boot. ++ */ ++static int g5p_r41_bridge_prepare(struct g5p_diag *g) ++{ ++ unsigned int i, mode = gfx_bridge_replay; ++ ++ if (!mode) ++ return 0; ++ if (g5p_r41_bridge.attempted) ++ return g5p_r41_bridge.result; ++ g5p_r41_bridge.attempted = true; ++ g5p_r41_bridge.mode = mode; ++ g5p_r41_bridge.result = -EINVAL; ++ if (mode > 4) ++ return -EINVAL; ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) ++ if (!g5p_r41_valid(&g5p_r41_writes[i])) ++ return -EINVAL; ++ g5p_r41_bridge.result = -ENOMEM; ++ g5p_r41_bridge.window[0] = devm_ioremap(g->dev, 0x201f00000ULL, 0x5000); ++ if (!g5p_r41_bridge.window[0]) ++ return -ENOMEM; ++ g5p_r41_bridge.window[1] = devm_ioremap(g->dev, 0x201f80000ULL, 0x1000); ++ if (!g5p_r41_bridge.window[1]) ++ return -ENOMEM; ++ /* Read all 37 even in split modes; never read unmapped/partial state. */ ++ g5p_r41_bridge_snapshot(g, "before"); ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ ++ if (mode == 1 || (mode == 3 && w->bridge != 8) || ++ (mode == 4 && w->bridge != 9)) ++ continue; ++ writel(w->value, ++ g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ dev_info( ++ g->dev, ++ "BRIDGE r41 write mode=%u index=%u bridge=%u off=%04x value=%08x\n", ++ mode, i, w->bridge, w->offset, w->value); ++ } ++ g5p_r41_bridge_snapshot(g, "after"); ++ g5p_r41_bridge.ready = true; ++ g5p_r41_bridge.result = 0; ++ return 0; ++} ++ ++static void g5p_r41_bridge_firmware(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.firmware_logged) { ++ g5p_r41_bridge.firmware_logged = true; ++ g5p_r41_bridge_snapshot(g, "firmware-alive"); ++ } ++} ++ ++static void g5p_r41_bridge_ta(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.ta_logged) { ++ g5p_r41_bridge.ta_logged = true; ++ g5p_r41_bridge_snapshot(g, "first-TA-kick"); ++ } ++} +diff --git a/drivers/gpu/apple-g5p/r74e112.h b/drivers/gpu/apple-g5p/r74e112.h +new file mode 100644 +index 000000000..cacac73ac +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r74e112.h +@@ -0,0 +1,137 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R74E112_H ++#define G5P_R74E112_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * Wake112: the R74E GPC1 power proof when ALIVE preempts the on-poll. ++ * ++ * R74E (boot.inc) requests GPC=1 and polls R1+0x1000 until the state field reads 1. ++ * The poll stops as soon as ALIVE is decoded: after ALIVE only the first-ALIVE owner ++ * may read R1. On a warm, kexec-cycled boot the firmware can reach ALIVE before the ++ * poll's first read (Wake111 20261031T201022Z: off_ns=1500 on_ns=375 raw=ffffffff, ++ * the U32_MAX "not read" sentinel). Wake111 then latched a power abort although the ++ * state was never sampled. ++ * ++ * Wake112 defers that one proof to the owner instead, but only when the on-poll took ++ * no sample: the poll counts its reads, and the U32_MAX raw is never taken as "not read". ++ * The owner's existing first-ALIVE read of R1+0x1000 (g5p_r73e_irq_submit, before any ++ * register write) is the proof. State 1 sets release_ready and the unchanged cold ++ * sequence continues. Any other value (0, 0x10, 0x11, ffffffff, ...) is an absorbing ++ * power abort. An on-poll that reads state 1 never defers. An on-poll that sampled ++ * before ALIVE (0x11 in rounds 74e..79b, or any other word) and every other R74E failure ++ * (off-poll, timeout, GPC1 write refused) abort exactly as before. ++ * ++ * States (r73b->early_proof): ++ * NONE no deferral: the poll decided, or the release has not run ++ * DEFERRED ALIVE preempted the on-poll; one owner read is owed (generation recorded) ++ * TAKEN the once-owner claimed it; it settles with no exit in between ++ * PROVEN that read had state 1: release_ready ++ * FAILED any other value, another generation, a lost precondition, or an ALIVE ++ * wait that ended with the proof still owed: power abort, absorbing ++ * ++ * Pure policy: no register access, no lock, no kernel API. ++ */ ++ ++#define G5P_PX_STATE_MASK 0x1fU ++#define G5P_PX_STATE_ON 0x01U ++ ++enum g5p_px { ++ G5P_PX_NONE, ++ G5P_PX_DEFERRED, ++ G5P_PX_TAKEN, ++ G5P_PX_PROVEN, ++ G5P_PX_FAILED, ++ G5P_PX_N, ++}; ++ ++static const char *const g5p_px_name[G5P_PX_N] = { ++ "none", "deferred", "taken", "proven", "failed", ++}; ++ ++/* Why a deferred proof failed (NONE: it did not). */ ++enum g5p_px_why { ++ G5P_PXW_NONE, ++ G5P_PXW_STATE, /* the read's state field is not 1 */ ++ G5P_PXW_STALE, /* deferred in another reset generation */ ++ G5P_PXW_LOST, /* ALIVE or the release lost, stopped or failed since */ ++ G5P_PXW_RETIRED, /* a FINAL teardown forbids R1 */ ++ G5P_PXW_UNTAKEN, /* the ALIVE wait ended with the proof still owed */ ++ G5P_PXW_N, ++}; ++ ++static const char *const g5p_px_why_name[G5P_PXW_N] = { ++ "none", "state", "stale", "lost", "retired", "untaken", ++}; ++ ++/* What the R74E release path latches. */ ++enum g5p_pxp { ++ G5P_PXP_READY, /* the on-poll read state 1: release_ready (cold) */ ++ G5P_PXP_DEFER, /* ALIVE preempted the on-poll before its first read */ ++ G5P_PXP_ABORT, /* anything else: power abort (cold) */ ++}; ++ ++/* ++ * on_ran: the core request, the off-poll and the GPC1 write succeeded and the on-poll ++ * ran. ret: the last step's result. preempted: the on-poll stopped on ALIVE. ++ * samples: the on-poll's R1+0x1000 reads. Only a preemption with no read defers (no ++ * abort, no release_ready); a sampled one saw the state and aborts. ++ */ ++static inline enum g5p_pxp g5p_px_poll(bool on_ran, int ret, bool preempted, ++ u32 samples) ++{ ++ if (!on_ran) ++ return G5P_PXP_ABORT; ++ if (!ret) ++ return G5P_PXP_READY; ++ return preempted && !samples ? G5P_PXP_DEFER : G5P_PXP_ABORT; ++} ++ ++/* The first-ALIVE owner's admission. */ ++enum g5p_pxg { ++ G5P_PXG_READY, /* release_ready: the cold owner path */ ++ G5P_PXG_PROVE, /* a deferred proof is owed: the once-owner's read decides */ ++ G5P_PXG_WAIT, /* nothing proven yet, or another owner holds the proof */ ++ G5P_PXG_REFUSE, /* power abort or a failed proof: no register access */ ++}; ++ ++static inline enum g5p_pxg g5p_px_gate(bool power_abort, bool release_ready, ++ u32 px) ++{ ++ if (power_abort || px == G5P_PX_FAILED) ++ return G5P_PXG_REFUSE; ++ if (release_ready) ++ return G5P_PXG_READY; ++ if (px == G5P_PX_DEFERRED) ++ return G5P_PXG_PROVE; ++ return G5P_PXG_WAIT; ++} ++ ++/* RAM facts the once-owner samples after taking the proof, before its one read. */ ++struct g5p_px_pre { ++ u32 gen, gen_now; /* DS108 generation at the deferral and now */ ++ bool alive, released, stopped, failed; ++ bool r1_allowed; /* g5p_r8_r1_allowed() */ ++}; ++ ++static inline enum g5p_px_why g5p_px_check(const struct g5p_px_pre *p) ++{ ++ if (!p->gen || p->gen != p->gen_now) ++ return G5P_PXW_STALE; ++ if (!p->alive || !p->released || p->stopped || p->failed) ++ return G5P_PXW_LOST; ++ if (!p->r1_allowed) ++ return G5P_PXW_RETIRED; ++ return G5P_PXW_NONE; ++} ++ ++/* The proof: only the state field 1 is power-on. */ ++static inline bool g5p_px_on(u32 raw) ++{ ++ return G5P_GPC_ON(raw); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/r87b-attach.h b/drivers/gpu/apple-g5p/r87b-attach.h +new file mode 100644 +index 000000000..81600ac06 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r87b-attach.h +@@ -0,0 +1,34 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R87B_ATTACH_H ++#define G5P_R87B_ATTACH_H ++/* R87b DIAGNOSTIC ONLY, never production-safe: the DC3 attach policy selector. ++ * (docs/gpu/experiments/87b-build-report.md) ++ * ++ * Two campaigns (r85 stage 2, r87 stage 1) never reached GPU work: the DC3 attach pair was ++ * published, the immediate gate (R1+0x1000 & 0x1f == 1) read 0x10/0x11, the host GPC off/on ++ * repair timed out (-110) and attach stopped with -115 before the RAM-only consume wait ran, ++ * after which the machine froze. Every retained instance of that failure shows the firmware ++ * consuming the pair on its own (dc=3/3 at export, no bell rung). boot_r87b_attach selects: ++ * 0 the original attach policy, exactly r87: a closed gate is repaired by the host (R1+0x20000 ++ * clear, GPC off/on, ALIVE-owned doorbell). The matched control. ++ * 1 self-drain (the r82b design, docs/gpu/experiments/82-attach-reactivation-triage.md): a ++ * closed DC3 gate is only a phase sample of the firmware's GPC window. The host does no ++ * repair and rings no bell; it keeps the RAM-only 3 s consume wait of render-r74f.inc and ++ * continues through the firmware-MMU attach path (r74g) when the firmware consumed the pair. ++ * An active gate still rings the bell at once (the unchanged 80 % path). The TA kind is ++ * never affected. ++ * The selector is latched at map and read only through the latched copy. A non-zero value needs ++ * the r80 epoch (which implies r74h, r74g and r74f case 1): self-drain is only safe because r74g ++ * has the firmware own the first MMU pass, so no host R0 access is licensed by a rung bell. ++ * The TYPED_V3 production path latches policy 1 with boot_r74f_case = boot_r74g_case = 1 and NO ++ * r80 / r74h (boot-r73b.inc, "production path"); g5p_r87b_attach_valid() describes the r80 ++ * epoch only. ++ */ ++#include ++#include ++ ++#define G5P_R87B_ATTACH_ORIGINAL 0U ++#define G5P_R87B_ATTACH_SELF_DRAIN 1U ++#define G5P_R87B_ATTACH_MAX 1U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/readback.h b/drivers/gpu/apple-g5p/readback.h +new file mode 100644 +index 000000000..fdd71ab30 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/readback.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_READBACK_H ++#define G5P_READBACK_H ++ ++#include "bits-compat.h" ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++enum g5p_readback_state { ++ G5P_READBACK_WRAPPER_ONLY, ++ G5P_READBACK_BASELINE, ++ G5P_READBACK_RESET_HELD, ++}; ++ ++/* Returning wrapper registers in exp03; never a META slave-port register. */ ++static const u32 g5p_fender_regs[] = { ++ 0, 8, 0x10, 0x14, 0x1000, 0x1004, 0x2010, 0x2014, 0x20000, ++}; ++ ++/* Reset halves first: the only post-arm R0 readback needed by the gate. */ ++static const u32 g5p_rgx_regs[] = { ++ 0x100, 0x104, 0, 4, 8, 0xc, 0x18, 0x20, 0x80, 0x84, 0x88, 0x8c, ++}; ++ ++static inline u32 g5p_readback_rgx_count(u32 gpc, enum g5p_readback_state state) ++{ ++ if (!G5P_GPC_ON(gpc)) ++ return 0; ++ if (state == G5P_READBACK_RESET_HELD) ++ return 2; ++ return state == G5P_READBACK_BASELINE ? ARRAY_SIZE(g5p_rgx_regs) : 0; ++} ++ ++static inline bool g5p_slave_access_allowed(bool reset_transition) ++{ ++ return !reset_transition; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/recover.inc b/drivers/gpu/apple-g5p/recover.inc +new file mode 100644 +index 000000000..f550d1010 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/recover.inc +@@ -0,0 +1,101 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * In-boot GPU recovery (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault isolation). ++ * ++ * A render job that stopped the firmware (its event stream went silent, or a fault) blocks the ++ * one TA/3D channel for every client, and the firmware has no abort. iOS ends that state with a ++ * firmware stop, a platform power cycle and a new firmware boot (kext 6c9e070, firmware-re README ++ * "Fault recovery"). This is the same, built from two proven pieces: ++ * - the reboot quiesce's freeze, IRQ mask and PMGR gfx gate cycle (teardown.inc g5p_kq_*), which ++ * on hardware cycled a wedged, failed device safely (deskfold reboot notifier: plan=-+cycle, ++ * then the next kernel came up warm-cycled); ++ * - the bootstrap's own stage functions (wake, map, arm, load, release, events, control), the ++ * same code the sysfs commands run after the next kernel's cycle on every warm hop. ++ * The old struct g5p_boot is frozen and left allocated (devm; a few hundred KiB per recovery), ++ * so a stale pointer anywhere still reads valid memory; a new one is built by the stages. id and ++ * sample are not rerun (they refuse once the boot saw a reset transition; map needs only the ++ * cached core id), nor is irq-test (refused in an r74 epoch). The firmware image is reused. ++ * ++ * Core lock held, no job in flight (the render scheduler's worker). Returns 0 with a live new ++ * epoch, or an error with the device left frozen and failed. ++ */ ++ ++/* The bootstrap's stages after the wake, as the sysfs runner orders them. */ ++static const char *const g5p_recover_stages[] = { ++ "map", ++ "arm", ++ "load", ++ "release", ++ "events", ++ "control-prepare", ++ "control-submit", ++ "control-wait", ++}; ++ ++static int __maybe_unused g5p_recover(struct g5p_diag *g, const char *why, ++ u32 *stage_out) ++{ ++ struct g5p_boot *old = g->boot; ++ struct g5p_kq_ctx kq = { .g = g }; ++ u64 t0 = ktime_get_ns(); ++ u32 stage = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ *stage_out = 0; ++ if (!old || !READ_ONCE(old->released) || g5p_r8_is_retired() || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || !g->pmgr || ++ !g->id_checked) ++ return -EPERM; ++ /* The quiesce's freeze and IRQ mask, then the line is given back for the new epoch. */ ++ g5p_kq_freeze(&kq); ++ g5p_kq_irq_off(&kq); ++ if (old->irq_ready && old->irq > 0) ++ devm_free_irq(g->dev, old->irq, g); ++ /* mmio-access.md G5: the GPU is marked unusable before its power is cut (Apple clears ++ * fender+0xc inside setPlatformPower(0), before the gate); g5p_kq_cycle() also sets mmio_off. ++ */ ++ WRITE_ONCE(g->fender_ready, false); ++ ret = g5p_kq_cycle(&kq); ++ ret = g5p_ds_hook_recover(g, ret) ?: ret; ++ if (ret) ++ goto out; ++ /* A new epoch: the stages build a new struct g5p_boot from here. */ ++ WRITE_ONCE(g->boot, NULL); ++ stage = 1; ++ ret = g5p_fender_wake(g); ++ for (u32 i = 0; !ret && i < ARRAY_SIZE(g5p_recover_stages); i++) { ++ const char *s = g5p_recover_stages[i]; ++ ++ stage = i + 2; ++ ret = strncmp(s, "control-", 8) ? g5p_boot_command(g, s) : ++ g5p_lifecycle_command(g, s); ++ if (ret == -EALREADY) ++ ret = 0; ++ } ++ if (!ret && (!g->boot || !READ_ONCE(g->boot->alive) || ++ !READ_ONCE(g->boot->control_complete))) ++ ret = -EIO; ++ /* The claim gate again: the held jobs need ALIVE proven in the new generation. */ ++ if (!ret) ++ ret = g5p_ds_hook_claim(g); ++ if (!ret) { ++ /* The render claim is kept: no diagnostic command may reset this epoch. */ ++ WRITE_ONCE(g->boot->user_compute_owned, true); ++ stage = 0; ++ } ++out: ++ if (ret && g->boot) ++ WRITE_ONCE(g->boot->failed, true); ++ g->last_result = ret; ++ *stage_out = stage; ++ dev_emerg( ++ g->dev, ++ "GPU-RECOVER why=%s ret=%d stage=%s ms=%llu gen=%u end=recover\n", ++ why, ret, ++ !stage ? (ret ? "cycle" : "done") : ++ stage == 1 ? "wake" : ++ g5p_recover_stages[stage - 2], ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC, g5p_ds_gen()); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/render-layout-r22.h b/drivers/gpu/apple-g5p/render-layout-r22.h +new file mode 100644 +index 000000000..8391a8320 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout-r22.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_R22_H ++#define G5P_RENDER_LAYOUT_R22_H ++ ++#include "render-layout.h" ++ ++#define G5P_R22_TA 0x54000U ++#define G5P_R22_3D 0x58000U ++#define G5P_R22_PM 0x5c000U ++#define G5P_R22_W 0x400U ++#define G5P_R22_TA_SKU 0xa00U ++#define G5P_R22_3D_SKU 0xd00U ++#define G5P_R22_TQ 0x1800U ++#define G5P_R22_STAMP 0x1840U ++#define G5P_R22_DEP 0x1900U ++#define G5P_R22_BLOCKS 32U ++#define G5P_R22_PHYS_PAGES (G5P_R22_BLOCKS * 8) ++#define G5P_R22_AUX 0x101000000ULL ++#define G5P_R22_RT 0x101020000ULL ++#define G5P_R22_RT_AUX 0x101030000ULL ++ ++/* 6cb124c/13f4/14dc: PA PPNs, GPU F and META links are distinct. ++ * Caller provides complete 16KiB control/list/F allocations. Check the whole ++ * physical input before writing anything. No synthesized PM catalogue encoder. ++ */ ++ ++/* 6ca6d5c arithmetic oracle + 6cba480/6c7b64c. Native legacy mapping ++ * dialect has bit36 clear: L=H, Laux=Haux. No PA substitution or float math. ++ */ ++ ++static const u32 g5p_r22_wait_words[] = { ++ 0x201, 0x138204, 0x10, 0x8388204, 0, 0x104000a, 0x1f, ++ 0x1f, 0x8388204, 1, 0x104000a, 0x1f, 0x1f, 0xf48008, ++ 0x10000015, 4, 0, 0x14, 0x109f, 0x13000a, 0x4000, ++ 0x4000, 0x138204, 0x4000, 0x8388204, 0, 0x104000a, 8, ++ 8, 0x8388204, 1, 0x104000a, 8, 8, ++}; ++ ++/* Full normal timestamped recipes, independent stamp identities TA0/3D1. ++ * k is dependency+4 (zero on the proven fresh initializer), not MMU tag. ++ */ ++ ++/* PM bank0 save writes runtime qwords before the a0/a8/b0 backing anchors. */ ++static inline bool g5p_r22_pm_saved(const u8 *state) ++{ ++ for (u32 off = 8; off < 0xa0; off += 4) ++ if (g5p_le32(state + off)) ++ return true; ++ return false; ++} ++ ++/* Never trust a firmware pointer until it equals our retained stamp word. */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-layout.h b/drivers/gpu/apple-g5p/render-layout.h +new file mode 100644 +index 000000000..f2a0373da +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_H ++#define G5P_RENDER_LAYOUT_H ++ ++#include "boot-layout.h" ++ ++/* Offline construction primitives only. No TA/3D publication path exists. ++ * 6caf9f8/6cb2378: caller must recover N/S1/S2/S3, not infer them from USC ++ * count or target dimensions. Sizes are bounded to u32 allocation extents. ++ */ ++struct g5p_pm_sizes { ++ u32 control, page_lists, backing; ++}; ++ ++static inline int g5p_pm_sizes(u32 n, u32 blocks, u64 s1, u64 s2, u64 s3, ++ struct g5p_pm_sizes *out) ++{ ++ u64 limit = 0xffffffffULL; ++ u64 a, b; ++ ++ if (!n || !blocks || !s1 || !s2 || !s3) ++ return -EINVAL; ++ if (n > (limit - 0x80) / 0xc0 || blocks > (limit - 0x44) / 0x24 || ++ s1 > limit / n || s2 > limit / n || s3 > limit) ++ return -EOVERFLOW; ++ a = s1 * n; ++ b = s2 * n; ++ if (a > limit - b || a + b > limit - s3) ++ return -EOVERFLOW; ++ out->control = 0x80 + n * 0xc0; ++ out->page_lists = 0x44 + blocks * 0x24; ++ out->backing = a + b + s3; ++ return 0; ++} ++ ++/* Only four recovered backing-pointer writes, not a complete PM record. ++ * Caller owns a fully initialized0xc0 record and validates GPU mappings. ++ */ ++static inline int g5p_pm_record_backing(u8 *record, u32 index, u32 n, u64 base, ++ u64 s1, u64 s2, u64 s3, u32 meta) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (index >= n) ++ return -EINVAL; ++ ret = g5p_pm_sizes(n, 1, s1, s2, s3, &sizes); ++ if (ret) ++ return ret; ++ if (base > ~0ULL - sizes.backing) ++ return -EOVERFLOW; ++ g5p_put64(record + 0xa8, base + index * s1); ++ g5p_put64(record + 0xa0, base + n * s1 + index * s2); ++ g5p_put64(record + 0xb0, base + n * s1 + n * s2); ++ g5p_put32(record + 0xb8, meta); ++ return 0; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-bind.h b/drivers/gpu/apple-g5p/render-user-bind.h +new file mode 100644 +index 000000000..3bbbb1e0f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-bind.h +@@ -0,0 +1,285 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Firmware slot pairs of render contexts (docs/gpu/multi-client-design.md, stage 1). ++ * ++ * Each render context owns a TA and a 3D firmware slot in 7..255, both pointing at its own ++ * page catalogue. The pair is bound once by two DC3 records {3, slot, 0, 0, 0}, 3D first, ++ * then TA, with the catalogue PA in 0x86[slot] and the host tag in 0x84[slot]. The firmware's ++ * DC3 handler installs CBASE and copies the host tag to 0x85[slot]. ++ * ++ * These helpers only read and write those RAM words, so they cannot reach a register and are ++ * safe after a job has retired. Pure functions, host-tested ++ * (scripts/gpu/multiclient1-host/test_render_bind.c). The caller provides g5p_le32(), ++ * g5p_put32() and g5p_put64(). ++ */ ++#ifndef G5P_RENDER_USER_BIND_H ++#define G5P_RENDER_USER_BIND_H ++ ++#define G5P_RU_SLOT_MIN 7U ++#define G5P_RU_SLOT_MAX 255U ++/* RAM-only wait for the firmware to consume a published DC3 batch. */ ++#define G5P_RU_BIND_MS 3000U ++/* Host tag of a pre-bound spare pair (render_user_spare_pairs): never zero. */ ++#define G5P_RU_SPARE_TAG 0x00400000U ++/* Host tag of a parked pair (render_user_close_park): never zero, never a spare's. */ ++#define G5P_RU_PARK_TAG 0x00500000U ++#define G5P_RU_MAX_SPARES 32U ++ ++/* The words one pair's binding touches. Pointers, because the DC records may straddle the ++ * shared pages. Index [e]: 0 TA slot, 1 3D slot. ++ */ ++struct g5p_ru_bind_mem { ++ u8 *rec[2][5]; /* the records at start and start + 1 */ ++ u8 *pa[2]; /* 0x86[slot], 64 bit */ ++ u8 *host[2]; /* 0x84[slot] */ ++ u8 *cached[2]; /* 0x85[slot] */ ++}; ++ ++/* One RAM sample of those words and of the DC ring control. */ ++struct g5p_ru_bind_sample { ++ u32 cons, prod; ++ u32 rec[2][5]; ++ u64 pa[2]; ++ u32 host[2], cached[2]; ++}; ++ ++/* g5p_ru_bind_busy() */ ++#define G5P_RU_BIND_BUSY_RING 0x1U /* the DC ring is not idle */ ++#define G5P_RU_BIND_BUSY_SLOT 0x2U /* a slot has a PA or a host tag */ ++ ++/* g5p_ru_bind_check(), the r74g criteria */ ++#define G5P_RU_BIND_BAD_RING 0x1U ++#define G5P_RU_BIND_BAD_RECORD 0x2U ++#define G5P_RU_BIND_BAD_PA 0x4U ++#define G5P_RU_BIND_BAD_HOST 0x8U ++#define G5P_RU_BIND_BAD_CACHED 0x10U ++ ++enum g5p_ru_bind_wait { ++ G5P_RU_BIND_PENDING, ++ G5P_RU_BIND_CONSUMED, ++ G5P_RU_BIND_PROTO, ++ G5P_RU_BIND_TIMEOUT, ++}; ++ ++static inline bool g5p_ru_slot_used(const unsigned long *map, u32 slot) ++{ ++ return (map[slot / BITS_PER_LONG] >> (slot % BITS_PER_LONG)) & 1; ++} ++ ++/* The slot pair of a new render context: the two lowest free slots in 7..255. The lower one ++ * is the 3D slot, or the TA slot with ta_first (render_user_slot_variant 1). False when fewer ++ * than two are free. slot[0] is TA, slot[1] is 3D. ++ */ ++static inline bool g5p_ru_slot_pair(const unsigned long *map, bool ta_first, ++ u32 *slot) ++{ ++ u32 pick[2], n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < 2; s++) ++ if (!g5p_ru_slot_used(map, s)) ++ pick[n++] = s; ++ if (n < 2) ++ return false; ++ slot[0] = pick[ta_first ? 0 : 1]; ++ slot[1] = pick[ta_first ? 1 : 0]; ++ return true; ++} ++ ++/* Word j of record i of a pair: {3, slot, 0, 0, 0}, the 3D slot's record first. */ ++static inline u32 g5p_ru_bind_record(const u32 *slot, u32 i, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot[1 - i] : 0; ++} ++ ++static inline void g5p_ru_bind_read(const struct g5p_ru_bind_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_bind_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ s->rec[i][j] = g5p_le32(m->rec[i][j]); ++ for (u32 e = 0; e < 2; e++) { ++ s->pa[e] = (u64)g5p_le32(m->pa[e] + 4) << 32 | ++ g5p_le32(m->pa[e]); ++ s->host[e] = g5p_le32(m->host[e]); ++ s->cached[e] = g5p_le32(m->cached[e]); ++ } ++} ++ ++/* Before a publish: the ring must be idle and both slots unused (zero PA and host tag). */ ++static inline u32 g5p_ru_bind_busy(const struct g5p_ru_bind_sample *s) ++{ ++ u32 busy = 0; ++ ++ if (s->prod > 255 || s->cons != s->prod) ++ busy |= G5P_RU_BIND_BUSY_RING; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] || s->host[e]) ++ busy |= G5P_RU_BIND_BUSY_SLOT; ++ return busy; ++} ++ ++/* ++ * A parked pair (render_user_close_park, release8 quiet2): a closed context's slots, bound to the ++ * kernel's park catalogue (it maps only the firmware window) with the park tag, so that the PM's ++ * late write-back of that context's free list resolves instead of faulting on an empty PC level. ++ * Both slots on `park`, both host tags the park tag. park 0: nothing is parked. ++ */ ++static inline bool g5p_ru_bind_parked(const struct g5p_ru_bind_sample *s, ++ u64 park) ++{ ++ if (!park) ++ return false; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] != park || s->host[e] != G5P_RU_PARK_TAG) ++ return false; ++ return true; ++} ++ ++/* PA and tag of both slots, then the two records. The caller cleans the catalogue first and ++ * stores the producer afterwards, each behind a barrier. ++ */ ++static inline void g5p_ru_bind_publish(const struct g5p_ru_bind_mem *m, ++ const u32 *slot, u64 root, u32 tag) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ u32 e = 1 - i; ++ ++ g5p_put64(m->pa[e], root); ++ g5p_put32(m->host[e], tag); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[i][j], g5p_ru_bind_record(slot, i, j)); ++ } ++} ++ ++/* After the ring reached `target`: the records intact, both slots on `root`, and the firmware ++ * tag equal to the host tag (the DC3 handler ran for both). ++ */ ++static inline u32 g5p_ru_bind_check(const struct g5p_ru_bind_sample *s, ++ const u32 *slot, u64 root, u32 tag, ++ u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[i][j] != g5p_ru_bind_record(slot, i, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ for (u32 e = 0; e < 2; e++) { ++ if (s->pa[e] != root) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host[e] != tag) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached[e] != tag) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ } ++ return bad; ++} ++ ++/* One sample of a published batch. The caller takes `expired` before reading the ring, so ++ * the sample taken at the deadline still counts. ++ */ ++static inline int g5p_ru_bind_step(u32 cons, u32 prod, u32 target, bool expired) ++{ ++ if (prod != target || cons > 255) ++ return G5P_RU_BIND_PROTO; ++ if (cons == target) ++ return G5P_RU_BIND_CONSUMED; ++ return expired ? G5P_RU_BIND_TIMEOUT : G5P_RU_BIND_PENDING; ++} ++ ++/* ++ * Unbind (iOS AGXGart_detachSlotsAndDestroyAddressSpace, firmware-re README round 15 §1): PA ++ * and host tag of a slot go to zero and one DC3 record {3, slot, 0, 0, 0} follows. The DC3 ++ * handler then installs CBASE 0, copies the zero tag to 0x85[slot] and invalidates the slot's ++ * PC level. The slots of one batch take consecutive ring entries, one record each, behind a ++ * single producer store. The caller must have retired every use of the slots first. ++ */ ++#define G5P_RU_USER_SLOTS (G5P_RU_SLOT_MAX - G5P_RU_SLOT_MIN + 1) ++ ++struct g5p_ru_slot_mem { ++ u8 *rec[5]; /* the slot's record in the batch */ ++ u8 *pa, *host, ++ *cached; /* 0x86[slot] (64 bit), 0x84[slot], 0x85[slot] */ ++}; ++ ++struct g5p_ru_slot_sample { ++ u32 cons, prod; ++ u32 rec[5]; ++ u64 pa; ++ u32 host, cached; ++}; ++ ++/* The user slots the firmware may hold: those with a PA or a host tag. At most `max`. */ ++static inline u32 g5p_ru_held_slots(const u8 *pa, const u8 *host, u32 *slots, ++ u32 max) ++{ ++ u32 n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < max; s++) ++ if (g5p_le32(pa + s * 8) || g5p_le32(pa + s * 8 + 4) || ++ g5p_le32(host + s * 4)) ++ slots[n++] = s; ++ return n; ++} ++ ++/* A batch of n records fits the 256-entry ring behind an idle consumer. */ ++static inline bool g5p_ru_unbind_fits(u32 n) ++{ ++ return n && n <= G5P_RU_USER_SLOTS; ++} ++ ++static inline u32 g5p_ru_unbind_record(u32 slot, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot : 0; ++} ++ ++static inline void g5p_ru_slot_read(const struct g5p_ru_slot_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_slot_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 j = 0; j < 5; j++) ++ s->rec[j] = g5p_le32(m->rec[j]); ++ s->pa = (u64)g5p_le32(m->pa + 4) << 32 | g5p_le32(m->pa); ++ s->host = g5p_le32(m->host); ++ s->cached = g5p_le32(m->cached); ++} ++ ++/* PA and host tag to zero, then the record. The caller stores the producer behind a barrier. */ ++static inline void g5p_ru_unbind_publish(const struct g5p_ru_slot_mem *m, ++ u32 slot) ++{ ++ g5p_put64(m->pa, 0); ++ g5p_put32(m->host, 0); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[j], g5p_ru_unbind_record(slot, j)); ++} ++ ++/* After the ring reached `target`: the record intact, PA and host tag still zero, and the ++ * firmware's tag zero too, which only its DC3 handler writes (the slot was bound with a ++ * non-zero tag). The G5P_RU_BIND_BAD_* bits. ++ */ ++static inline u32 g5p_ru_unbind_check(const struct g5p_ru_slot_sample *s, ++ u32 slot, u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[j] != g5p_ru_unbind_record(slot, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ if (s->pa) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ return bad; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-close.h b/drivers/gpu/apple-g5p/render-user-close.h +new file mode 100644 +index 000000000..2aa461bce +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-close.h +@@ -0,0 +1,91 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Release of a closed render context (docs/gpu/multi-client-design.md, stage 2). ++ * ++ * A TYPED_V3 context whose handle is gone (DESTROY_CONTEXT or file close) gets one close item. ++ * It runs after the context's last job, in the job order, and decides from plain values what ++ * the firmware may still hold: ++ * ++ * RELEASE the firmware never held the pair: drop the references, free everything; ++ * UNBIND the pair is bound and every use retired: RAM-only DC3 unbind ++ * (g5p_rv3_unbind_slots()), then free on success, retain on failure; ++ * RETAIN the firmware may still use the pair or its memory: keep the context, its ++ * slots and its memory until reboot (the kexec quiesce detaches them then). ++ * ++ * The rule follows iOS AGXGart_detachSlotsAndDestroyAddressSpace (firmware-re README round 15 ++ * §1, "Linux sequence" step 8): stop new jobs, wait for every engine use, publish PA and tag ++ * zero, consume the DC3, and only then recycle the slots and free the translations. A slot ++ * returns to the allocator only after its unbind was consumed and verified, so an old unbind ++ * can never clear a new owner's PA, and the lowest free pair may be reused at once (iOS pops ++ * a freed slot off its free stack first). ++ * ++ * Pure function, host-tested (scripts/gpu/multiclient2-host/test_render_close.c). ++ */ ++#ifndef G5P_RENDER_USER_CLOSE_H ++#define G5P_RENDER_USER_CLOSE_H ++ ++enum g5p_ru_close_plan { ++ G5P_RU_CLOSE_RELEASE, ++ G5P_RU_CLOSE_UNBIND, ++ G5P_RU_CLOSE_RETAIN, ++}; ++ ++/* Why a bound or published pair is retained (bits). */ ++#define G5P_RU_CLOSE_UNVERIFIED \ ++ 0x01U /* a bind was published but never checked out */ ++#define G5P_RU_CLOSE_BAD \ ++ 0x02U /* a job of this context failed on the hardware */ ++#define G5P_RU_CLOSE_TERMINAL 0x04U /* the epoch noted a timeout or fault */ ++#define G5P_RU_CLOSE_FAILED 0x08U /* the device failed */ ++#define G5P_RU_CLOSE_STOPPED 0x10U /* the device stopped (teardown, quiesce) */ ++#define G5P_RU_CLOSE_SHUTDOWN \ ++ 0x20U /* admission closed for good (reboot quiesce) */ ++#define G5P_RU_CLOSE_QUEUED 0x40U /* a job of this context is still queued */ ++#define G5P_RU_CLOSE_PENDING 0x80U /* an earlier DC batch was never consumed */ ++#define G5P_RU_CLOSE_LOCK 0x100U /* the command lock could not be taken */ ++ ++struct g5p_ru_close_in { ++ bool retained; /* the firmware may hold the pair: a DC3 bind was published */ ++ bool bound; /* that bind was consumed and checked */ ++ bool bad, term_failed, dev_failed, stopped, shutdown, dc_pending; ++ u32 queued; ++}; ++ ++static inline const char *g5p_ru_close_name(int plan) ++{ ++ return plan == G5P_RU_CLOSE_RELEASE ? "release" : ++ plan == G5P_RU_CLOSE_UNBIND ? "unbind" : ++ "retain"; ++} ++ ++/* The plan, and in *why the G5P_RU_CLOSE_* bits that made it RETAIN (0 otherwise). */ ++static inline int g5p_ru_close_plan(const struct g5p_ru_close_in *in, u32 *why) ++{ ++ u32 w = 0; ++ ++ *why = 0; ++ if (!in->retained) ++ return in->queued ? (*why = G5P_RU_CLOSE_QUEUED, ++ G5P_RU_CLOSE_RETAIN) : ++ G5P_RU_CLOSE_RELEASE; ++ if (!in->bound) ++ w |= G5P_RU_CLOSE_UNVERIFIED; ++ if (in->bad) ++ w |= G5P_RU_CLOSE_BAD; ++ if (in->term_failed) ++ w |= G5P_RU_CLOSE_TERMINAL; ++ if (in->dev_failed) ++ w |= G5P_RU_CLOSE_FAILED; ++ if (in->stopped) ++ w |= G5P_RU_CLOSE_STOPPED; ++ if (in->shutdown) ++ w |= G5P_RU_CLOSE_SHUTDOWN; ++ if (in->queued) ++ w |= G5P_RU_CLOSE_QUEUED; ++ if (in->dc_pending) ++ w |= G5P_RU_CLOSE_PENDING; ++ *why = w; ++ return w ? G5P_RU_CLOSE_RETAIN : G5P_RU_CLOSE_UNBIND; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +new file mode 100644 +index 000000000..ccb7d2c8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The phase wait's pause between two RAM samples (perf60-fillrate.md §11.3 cut 1): woken by the ++ * firmware event, not a fixed 1-2 ms sleep. Host test: scripts/gpu/perf60-host/test_evwait.c. ++ * ++ * Protocol (g5p_rv3_wait): drain -> seen = event_serial -> sample RAM -> decide -> pause. ++ * The IRQ thread's drain decodes each firmware event, bumps event_serial under status_lock and ++ * wakes the queue. The pause sleeps until event_serial != seen or the safety timeout, so an ++ * event decoded at any point after `seen` was read (before, during or after the sample) ends the ++ * pause at once: no lost wakeup. The timeout is the safety net for a RAM state that changes ++ * without an event; the predicate stays level-triggered on every sample, as before. ++ * Once the phase's own completion event is latched (s->event) but the RAM rules still wait ++ * (idle words, retire stamp, PM receipt land a little after the event), the pause is a short ++ * G5P_EW_POST_US poll: there is no further event to wait for. ++ */ ++#ifndef G5P_RENDER_USER_EVWAIT_H ++#define G5P_RENDER_USER_EVWAIT_H ++ ++#define G5P_EW_POST_US \ ++ 100U /* after the phase's own event: completion is imminent */ ++#define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ ++ ++enum g5p_ew_act { ++ G5P_EW_LEGACY, /* render_user_event_wake 0: usleep_range(1000, 2000), release8 */ ++ G5P_EW_SHORT, /* the phase's event seen: sleep G5P_EW_POST_US */ ++ G5P_EW_EVENT, /* sleep until any event since `seen`, at most *wait_us */ ++}; ++ ++static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, ++ u32 *wait_us) ++{ ++ if (!knob_us) { ++ *wait_us = 0; ++ return G5P_EW_LEGACY; ++ } ++ if (phase_event) { ++ *wait_us = G5P_EW_POST_US; ++ return G5P_EW_SHORT; ++ } ++ *wait_us = knob_us < G5P_EW_POST_US ? G5P_EW_POST_US : ++ knob_us > G5P_EW_MAX_US ? G5P_EW_MAX_US : ++ knob_us; ++ return G5P_EW_EVENT; ++} ++ ++/* The wait condition: an event was decoded since `seen` was read. */ ++static inline bool g5p_ew_woken(u64 seen, u64 serial) ++{ ++ return serial != seen; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-fault.h b/drivers/gpu/apple-g5p/render-user-fault.h +new file mode 100644 +index 000000000..2af9f7513 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-fault.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * A client job that faults (docs/gpu/experiments/faultiso.md). Pure decisions, host-tested ++ * (scripts/gpu/faultiso-host/test_fault_iso.c). ++ * ++ * On this firmware a GPU page fault is not survivable in place. The firmware acknowledges the ++ * MMU fault itself (R0+0x138), reports it as event 4 and parks (firmware-re README "Fault ++ * ownership"); iOS answers event 4 only with its restart tail (kext 6c81dac -> 6c834a0: firmware ++ * stop, context and dependency reset, shared state rebuilt, firmware boot), and has no ++ * per-context ACK, channel reset or slot unbind for it. The no-reset path iOS does have, the ++ * firmware's own job kill (event 8, then device-control command 0xa), is not a page fault. So a ++ * job that stops the firmware, by a fault event or by a silent event stream, loses its context ++ * and the GPU is recovered in the boot (render_user_recover), or the epoch ends. ++ */ ++#ifndef G5P_RENDER_USER_FAULT_H ++#define G5P_RENDER_USER_FAULT_H ++ ++/* The errno a lost job reports: a timeout's own, else the fault's (-EIO for a fault event or a ++ * silent stream, -EPROTO, or the drain's error). `result` is the phase's G5P_RU_WAIT_*. ++ */ ++static inline int g5p_ru_lost_errno(u32 result, int err) ++{ ++ return result == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++} ++ ++/* ++ * Does a job that failed after touching the hardware fail the device too? The legacy profiles ++ * always do. A TYPED_V3 job does not when it timed out (nothing is wrong with the protocol and ++ * the firmware may still finish), when its bind failed (the DC ring is not on the path of the ++ * contexts already bound), or when it was lost (the executor ruled: the channel drains, or a ++ * recovery replaces the epoch). Its context is quarantined in every case. ++ */ ++static inline bool g5p_ru_job_fails_device(bool v3, int ret, bool bind_failed, ++ bool lost) ++{ ++ return !v3 || (ret != -ETIMEDOUT && !bind_failed && !lost); ++} ++ ++/* ++ * A device failure noted while a job or a drain runs (a fault event decoded mid-phase), or while ++ * a recovery is pending, is not yet ruled on: the executor either loses that job's context and ++ * recovers, or ends the epoch. Until then the other contexts wait (submit accepted and held, ++ * VM_BIND busy, context status not lost) instead of failing. The executor sets `recovering` ++ * before it clears `executing`, so a reader that loads `executing` with acquire and then ++ * `recovering` always sees one of them through a recovery. ++ */ ++static inline bool g5p_ru_ruling(bool executing, bool recovering) ++{ ++ return executing || recovering; ++} ++ ++/* render_user_test_fault_job: the context's job about to start (it has run `jobs` before) gets ++ * the unmapped control stream. Never the epoch's first job: its attach and bell are special. ++ */ ++static inline bool g5p_ru_fault_due(u32 fault_job, u32 jobs, bool first) ++{ ++ return fault_job && jobs + 1 == fault_job && !first; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-faultreg.h b/drivers/gpu/apple-g5p/render-user-faultreg.h +new file mode 100644 +index 000000000..83cac51f7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-faultreg.h +@@ -0,0 +1,124 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 fault address (render_user_fault_read, drm-render-v3.inc): decode of a GPU MMU fault ++ * status bank, R0+0xe150/e154 (bank 0) or R0+0xe158/e15c (bank 1), as high32:low32. ++ * ++ * The firmware's fault path (source 4 -> 1001a494) reads both banks with a 64-bit load, keeps ++ * only a class in its companion record and posts event 4 without the status, so the address ++ * exists only in these registers. iOS reads them itself in its fault handler (kext 6ca5c14, ++ * accelerator virtual+740) with exactly these fields (docs/gpu/firmware-re/README.md "Complete ++ * e150 decode"): bit 0 valid, 2:1 type (0 page invalid, 2 read-only, 3 PM/META protected), ++ * 3 read (0 write), 5:4 level (3 = catalogue), 11:6 requester (iOS table 6ff5ed8), 19:12 ++ * sideband, 27:20 MMU context (the slot), VA = (status >> 24) & 0xfffffffff0 (16-byte precision). ++ * Pure, host-tested (scripts/gpu/perf60-host/test_fault_decode.c). ++ */ ++#ifndef G5P_RENDER_USER_FAULTREG_H ++#define G5P_RENDER_USER_FAULTREG_H ++ ++#include "bits-compat.h" ++struct g5p_ru_mmu_fault { ++ bool valid, read; ++ u8 type, level, requester, sideband, context; ++ u64 va; ++}; ++ ++static inline struct g5p_ru_mmu_fault g5p_ru_mmu_fault_decode(u32 lo, u32 hi) ++{ ++ u64 s = (u64)hi << 32 | lo; ++ struct g5p_ru_mmu_fault f = { ++ .valid = s & 1, ++ .type = (s >> 1) & 3, ++ .read = (s >> 3) & 1, ++ .level = (s >> 4) & 3, ++ .requester = (s >> 6) & 0x3f, ++ .sideband = (s >> 12) & 0xff, ++ .context = (s >> 20) & 0xff, ++ .va = (s >> 24) & 0xfffffffff0ULL, ++ }; ++ ++ return f; ++} ++ ++static const char *const g5p_ru_mmu_fault_types[4] = { ++ "page-invalid", ++ "type1", ++ "read-only", ++ "pm-meta-protected", ++}; ++ ++/* Requester names, iOS kext table 6ff5ed8 (16-byte {name, class} entries; read from the j71t ++ * 16.7.10 kernelcache, 2026-11-03). 56 entries; a larger index prints its number only. ++ */ ++static const char *const g5p_ru_mmu_requesters[] = { ++ "META", ++ "TLA", ++ "HOST", ++ "VDMM", ++ "CDM", ++ "IPP", ++ "PM", ++ "TILING", ++ "MCU_TEXAS0", ++ "PDS_TEXAS0", ++ "PBE0_TEXAS0", ++ "PBE1_TEXAS0", ++ "VDMS_BLACKPEARL0", ++ "IPF_BLACKPEARL0", ++ "ISP_BLACKPEARL0", ++ "TPF_BLACKPEARL0", ++ "USCS_BLACKPEARL0", ++ "PPP_BLACKPEARL0", ++ "VCE_BLACKPEARL0", ++ "FBCDC_BLACKPEARL0", ++ "MCU_TEXAS1", ++ "PDS_TEXAS1", ++ "PBE0_TEXAS1", ++ "PBE1_TEXAS1", ++ "MCU_TEXAS2", ++ "PDS_TEXAS2", ++ "PBE0_TEXAS2", ++ "PBE1_TEXAS2", ++ "VDMS_BLACKPEARL1", ++ "IPF_BLACKPEARL1", ++ "ISP_BLACKPEARL1", ++ "TPF_BLACKPEARL1", ++ "USCS_BLACKPEARL1", ++ "PPP_BLACKPEARL1", ++ "VCE_BLACKPEARL1", ++ "FBCDC_BLACKPEARL1", ++ "MCU_TEXAS3", ++ "PDS_TEXAS3", ++ "PBE0_TEXAS3", ++ "PBE1_TEXAS3", ++ "MCU_TEXAS4", ++ "PDS_TEXAS4", ++ "PBE0_TEXAS4", ++ "PBE1_TEXAS4", ++ "VDMS_BLACKPEARL2", ++ "IPF_BLACKPEARL2", ++ "ISP_BLACKPEARL2", ++ "TPF_BLACKPEARL2", ++ "USCS_BLACKPEARL2", ++ "PPP_BLACKPEARL2", ++ "VCE_BLACKPEARL2", ++ "FBCDC_BLACKPEARL2", ++ "MCU_TEXAS5", ++ "PDS_TEXAS5", ++ "PBE0_TEXAS5", ++ "PBE1_TEXAS5", ++}; ++ ++static inline const char *g5p_ru_mmu_requester(u32 requester) ++{ ++ if (requester >= ARRAY_SIZE(g5p_ru_mmu_requesters)) ++ return "?"; ++ return g5p_ru_mmu_requesters[requester]; ++} ++ ++/* The range test of the VM edit log: does [va, va + size) hold the fault address? */ ++static inline bool g5p_ru_mmu_fault_in(u64 fault_va, u64 va, u64 size) ++{ ++ return fault_va >= va && fault_va - va < size; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-geometry.h b/drivers/gpu/apple-g5p/render-user-geometry.h +new file mode 100644 +index 000000000..2117d567b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-geometry.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_GEOMETRY_H ++#define G5P_RENDER_USER_GEOMETRY_H ++ ++/* Kernel-derived render-target geometry: AGXRenderTargetMemoryG5's arithmetic ++ * leaf 6ca6d5c (firmware round16 section 3) for one sample at the pixel ++ * centre (8,8) or four samples (docs/gpu/experiments/msaa.md), in integer ++ * arithmetic only. It is Rogue's non-simple ++ * parameter format: 32x32 tiles, 4x4 macrotiles of ++ * ALIGN(DIV_ROUND_UP(tiles, 4), 4) tiles per axis, five-byte region headers ++ * per padded tile, a tail pointer cache of next_pow2(max(padded tiles))^2 ++ * eight-byte entries. The scale pairs are the leaf's IEEE singles ++ * 0x3e89309c / n, 0x3fddb3d9 / n (fdiv, round to nearest even) and n * 16. ++ * tools/gpu/test_rt_oracle.c compares every output with the unmodified leaf ++ * for every width and height in 1..G5P_RU_MAX_DIM. ++ * ++ * G5P_RU_MAX_DIM is rogue_get_render_size_max() for this tiling class; it ++ * is a memory bound (16 MiB private window, 8 MiB main ring), not a field ++ * limit: the 9-bit TE macrotile and 10-bit padded-size fields would hold ++ * twice as much. ++ */ ++#define G5P_RU_MAX_DIM 8192U ++#define G5P_RU_SCALE_A 0x3e89309cU ++#define G5P_RU_SCALE_B 0x3fddb3d9U ++ ++static inline u32 g5p_ru_fls(u32 v) ++{ ++ return v ? 32 - __builtin_clz(v) : 0; ++} ++ ++/* ucvtf of 1 <= n < 2^24: exact. */ ++static inline u32 g5p_ru_f32(u32 n) ++{ ++ u32 bits = g5p_ru_fls(n); ++ ++ return ((126 + bits) << 23) | ((n << (24 - bits)) & 0x7fffff); ++} ++ ++/* fdiv of a normal positive single by 1 <= n < 2^24, round to nearest even. ++ * The quotient q of the 24-bit significand shifted by fls(n) + 3 has 27 or ++ * 28 bits; s of them are rounding bits, the remainder is the sticky bit. ++ * Results stay normal for the two leaf constants and n <= G5P_RU_MAX_DIM. ++ */ ++static inline u32 g5p_ru_fdiv(u32 a, u32 n) ++{ ++ u32 shift = g5p_ru_fls(n) + 3; ++ u64 num = (u64)((a & 0x7fffff) | 0x800000) << shift; ++ u32 q = num / n, sticky = num % n != 0; ++ u32 s = g5p_ru_fls(q) - 24, m = q >> s, low = q & ((1U << s) - 1); ++ ++ if (low > 1U << (s - 1) || ++ (low == 1U << (s - 1) && (sticky || (m & 1)))) ++ m++; ++ if (m >> 24) { ++ m >>= 1; ++ s++; ++ } ++ return ((((a >> 23) & 0xff) - shift + s) << 23) | (m & 0x7fffff); ++} ++ ++/* Sample positions, one byte per sample: x in [3:0], y in [7:4], 1/16 px. ++ * One sample sits at the pixel centre; four use the standard Vulkan/D3D ++ * pattern, which is also Metal's (record+0x69c..0x6b8 of the iOS ++ * msaa4-resolve/-store captures: (6,2) (14,6) (2,10) (10,14)). ++ */ ++#define G5P_RU_POSITIONS_1X 0x88ULL ++#define G5P_RU_POSITIONS_4X 0xeaa26e26ULL ++ ++/* The leaf for one or four samples (6ca6d5c's 1, 2, 4, 8 table, of which ++ * Linux uses 1 and 4). Four samples are 2x2 per pixel: the region headers ++ * and the tail pointer cache are sized for twice the tiles in x and y, the ++ * ISP macrotile size (Rbc) doubles in y, Rb8 is log2(samples) (5940f40) ++ * and the TE word Rf0 is 5. Rb0 and Rf8 are the positions word, packed by ++ * the leaf as x0 | y0 << 4 | x1 << 8 | ... | y3 << 28. The scales, the TE ++ * tiling words and the tile count do not depend on the samples. ++ */ ++static inline int g5p_ru_rt_derive_samples(u32 width, u32 height, u32 samples, ++ u64 positions, struct g5p_ru_rt *rt) ++{ ++ u32 tx = (width + 31) / 32, ty = (height + 31) / 32; ++ u32 mx = (((tx + 3) >> 2) + 3) & ~3U, my = (((ty + 3) >> 2) + 3) & ~3U; ++ u32 s = samples == 4 ? 2 : 1; ++ u32 px = mx * s * 4, py = my * s * 4, tpc; ++ ++ if (!width || !height || width > G5P_RU_MAX_DIM || ++ height > G5P_RU_MAX_DIM || (samples != 1 && samples != 4) || ++ positions >> (samples * 8)) ++ return -EINVAL; ++ tpc = 1U << g5p_ru_fls((px > py ? px : py) - 1); ++ *rt = (struct g5p_ru_rt) { ++ .header = px * py * 5, ++ .region = mx * my * s * s * 5, ++ .packed = positions, ++ .state = s - 1 ? 2 : 0, ++ .padded = (mx & 0x3ff) << 16 | ((my * s) & 0x3ff), ++ .scale = { ++ g5p_ru_fdiv(G5P_RU_SCALE_A, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_A, height) << 32, ++ g5p_ru_fdiv(G5P_RU_SCALE_B, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_B, height) << 32, ++ (g5p_ru_f32(width) + (4U << 23)) | ++ (u64)(g5p_ru_f32(height) + (4U << 23)) << 32, ++ }, ++ .ta_state = { s - 1 ? 5 : 0, positions }, ++ .ta_tiling = { ++ (u64)((ty - 1) << 12 | (tx - 1)) << 32 | ++ ((height - 1) << 16 | (width - 1)), ++ (u64)(my << 18 | my << 10 | 3 * my) << 32 | ++ (mx << 18 | mx << 10 | 3 * mx), ++ }, ++ .ta_tiling_word = mx * my, ++ .tiles = tx * ty, ++ .secondary = (u64)tpc * tpc * 8, ++ .secondary_align = 0x40, ++ }; ++ return 0; ++} ++ ++/* g5p_ru_rt64 is this function's result for 64x64. */ ++ ++/* PM pool for a kernel-derived target. Linux policy, not hardware: Rogue's ++ * minimum free list (50 4KiB pages, non-XE cores) plus one 4KiB PM page per ++ * tile, in 128KiB blocks, never below the latched pool and at most one F ++ * page. 64x64 (4 tiles) needs 2, so the latched 32 stays. There is no PM ++ * growth during a job: a frame that exhausts the pool fails like any OOM. ++ */ ++static inline u32 g5p_ru_pm_blocks_for(u32 tiles, u32 latched) ++{ ++ u64 need = ((50ULL + tiles) * 0x1000 + G5P_RU_BLOCK_BYTES - 1) / ++ G5P_RU_BLOCK_BYTES; ++ ++ if (need < latched) ++ need = latched; ++ return need > G5P_RU_MAX_BLOCKS ? G5P_RU_MAX_BLOCKS : need; ++} ++ ++ ++ ++static inline int g5p_ru_rt_derive(u32 width, u32 height, struct g5p_ru_rt *rt) ++{ ++ return g5p_ru_rt_derive_samples(width, height, 1, G5P_RU_POSITIONS_1X, rt); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +new file mode 100644 +index 000000000..822709ed8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -0,0 +1,651 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_LAYOUT_H ++#define G5P_RENDER_USER_LAYOUT_H ++ ++#include "bits-compat.h" ++/* Parameterized forms of the r22 fixed constructors for userspace render. ++ * Pure byte builders: no allocation, MMIO, locking or firmware publication. ++ * With r22's inputs every builder reproduces render-layout-r22.h exactly; ++ * tools/gpu/test_render_user.c checks that equivalence byte for byte. ++ */ ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++#include "render-user-template.h" ++ ++/* Kernel-private window in every render VM. r22's aux/RT addresses are the ++ * first allocations, so a 64x64 target keeps the fixed path's W anchors. ++ * Root entry 0 carries the boot VA0 sink, so user maps start at 1 GiB. ++ */ ++#define G5P_RU_PRIVATE_VA G5P_R22_AUX ++#define G5P_RU_PRIVATE_SIZE 0x1000000ULL ++#define G5P_RU_USER_VA_MIN 0x40000000ULL ++/* Catalogue entry of the firmware GiB, borrowed from the boot tables. */ ++#define G5P_RU_FIRMWARE_CI ((G5P_IMAGE_VA >> 30) & 1023) ++#define G5P_RU_AUX_BYTES 0x10680U ++#define G5P_RU_AUX_PAGES 5U ++#define G5P_RU_RT_ALIGN 0x10000ULL ++#define G5P_RU_MAX_BLOCKS 128U ++#define G5P_RU_BLOCK_BYTES 0x20000U ++#define G5P_RU_MAX_HEADER 0x80000U ++#define G5P_RU_MAX_SECONDARY 0x100000ULL ++#define G5P_RU_TA_W_BYTES 0x560U ++#define G5P_RU_3D_W_BYTES 0x8c0U ++#define G5P_RU_CMD_BYTES 0x40U ++ ++/* Kernel mirror of drm_g5p_rt_geometry (firmware round16 R-offsets). */ ++struct g5p_ru_rt { ++ u32 header, region; /* R10, R20 */ ++ u64 packed; /* Rb0 */ ++ u32 state, padded; /* Rb8, Rbc */ ++ u64 scale[3]; /* Rc0, Rc8, Rd0 */ ++ u64 ta_state[2]; /* Rf0, Rf8 */ ++ u64 ta_tiling[2]; /* R100, R108 */ ++ u32 ta_tiling_word, tiles; /* R110, R118 */ ++ u64 secondary, secondary_align; /* R138, R178 */ ++}; ++ ++/* 6ca6d5c executed offline for 64,64,1,8,8 (round16 section 3). */ ++static const struct g5p_ru_rt g5p_ru_rt64 = { ++ .header = 0x500, ++ .region = 0x50, ++ .packed = 0x88, ++ .state = 0, ++ .padded = 0x40004, ++ .scale = { 0x3b89309c3b89309cULL, 0x3cddb3d93cddb3d9ULL, ++ 0x4480000044800000ULL }, ++ .ta_state = { 0, 0x88 }, ++ .ta_tiling = { 0x00001001003f003fULL, 0x0010100c0010100cULL }, ++ .ta_tiling_word = 0x10, ++ .tiles = 4, ++ .secondary = 0x800, ++ .secondary_align = 0x40, ++}; ++ ++#include "render-user-geometry.h" ++ ++struct g5p_ru_rt_sizes { ++ u64 slice, main, secondary_map; ++}; ++ ++/* 6cb3d68: slice=R10 rounded to 4KiB; main ring is the power-of-two capacity ++ * for sixteen slices. 6cb40e0 places the secondary extent at offset 0 of ++ * its own mapping. Bounds are Linux allocation policy, not hardware limits. ++ */ ++static inline int g5p_ru_rt_sizes(const struct g5p_ru_rt *rt, ++ struct g5p_ru_rt_sizes *out) ++{ ++ u64 main = 0x1000; ++ ++ if (!rt->header || rt->header > G5P_RU_MAX_HEADER || !rt->secondary || ++ rt->secondary > G5P_RU_MAX_SECONDARY || !rt->secondary_align || ++ (rt->secondary_align & (rt->secondary_align - 1)) || ++ rt->secondary_align > G5P_GPU_PAGE || !rt->tiles) ++ return -EINVAL; ++ out->slice = ((u64)rt->header + 0xfff) & ~0xfffULL; ++ while (main < out->slice * 16) ++ main <<= 1; ++ out->main = main < G5P_GPU_PAGE ? G5P_GPU_PAGE : main; ++ out->secondary_map = (rt->secondary + G5P_GPU_PAGE - 1) & ++ ~(u64)(G5P_GPU_PAGE - 1); ++ return 0; ++} ++ ++/* 6cba480/6c7b64c anchors. Native mapping dialect: L=H, Laux=Haux. ++ * Equal to g5p_r22_rt() for g5p_ru_rt64 at the r22 addresses. 3D W58 and ++ * Wa8 are the importer's literal Rb0 and R20>>2 for the verified geometry. ++ */ ++static inline void g5p_ru_rt_fill(u8 *ta, u8 *td, const struct g5p_ru_rt *rt, ++ u64 main_va, u64 secondary_va) ++{ ++ g5p_put64(ta + 0x28, main_va); ++ g5p_put64(ta + 0x30, secondary_va); ++ g5p_put64(ta + 0x38, rt->ta_state[0]); ++ g5p_put64(ta + 0x40, rt->ta_state[1]); ++ g5p_put64(ta + 0x444, secondary_va); ++ g5p_put32(ta + 0x44c, rt->secondary); ++ g5p_put64(ta + 0x3ac, rt->ta_tiling[0]); ++ g5p_put64(ta + 0x3b4, rt->ta_tiling[1]); ++ g5p_put32(ta + 0x3bc, rt->ta_tiling_word); ++ g5p_put64(td + 0x20, main_va); ++ g5p_put64(td + 0x28, rt->packed); ++ g5p_put32(td + 0x30, rt->state); ++ g5p_put32(td + 0x34, rt->padded); ++ g5p_put32(td + 0x38, rt->region); ++ g5p_put64(td + 0x3c, rt->scale[0]); ++ g5p_put64(td + 0x44, rt->scale[1]); ++ g5p_put64(td + 0x4c, rt->scale[2]); ++ g5p_put64(td + 0x58, rt->packed); ++ g5p_put32(td + 0xa8, rt->region >> 2); ++ g5p_put32(td + 0x3a0, rt->padded); ++ g5p_put64(td + 0x3d0, rt->scale[0]); ++ g5p_put64(td + 0x3d8, rt->scale[1]); ++ g5p_put32(td + 0x3e0, rt->scale[2]); ++ g5p_put32(td + 0x3e4, rt->scale[0]); ++} ++ ++/* Validated typed render inputs, already range-checked by the caller. */ ++struct g5p_ru_pass { ++ u64 vdm_va; ++ u32 vdm_size, vdm_control; ++ u64 scratch_va, scratch_size, scratch_ptr[6]; ++ u64 pool7_va, pool7_size, pool8_va, pool8_size; ++ u32 context_pds[2][6], resume_pds[2][4], ta_pds[4]; ++ u32 background[8], eot[4], eot_resume[4], restore[4]; ++}; ++ ++/* Importer chains 6cba8a0/6c7b100/6c7b34c -> 6ca9710/6ca7ae0 (offline ++ * differential map in make-render-user-template.py). Every user VA field ++ * and code-relative descriptor lands here; the rest of W is the frozen ++ * dialect-1 scalar table or a kernel RT/channel field. ++ */ ++static inline void g5p_ru_fill_ta(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_TA_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_ta_fixed); i++) ++ g5p_put32(w + g5p_ru_ta_fixed[i][0], g5p_ru_ta_fixed[i][1]); ++ g5p_put64(w + 0x48, p->scratch_ptr[4]); ++ g5p_put64(w + 0x50, p->scratch_ptr[5]); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put32(w + 0x58 + i * 4, p->context_pds[0][i]); ++ g5p_put32(w + 0x70 + i * 4, p->context_pds[1][i]); ++ } ++ g5p_put64(w + 0x88, p->vdm_control & ~0xfULL); ++ g5p_put64(w + 0x90, p->scratch_va & ~7ULL); ++ g5p_put64(w + 0x98, p->vdm_va & ~3ULL); ++ for (u32 i = 0; i < 4; i++) { ++ g5p_put32(w + 0xa0 + i * 4, p->resume_pds[0][i]); ++ g5p_put32(w + 0xb0 + i * 4, p->resume_pds[1][i]); ++ } ++ for (u32 i = 0; i < 3; i++) ++ g5p_put32(w + 0xc0 + i * 4, p->ta_pds[i]); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x468 + i * 8, p->scratch_ptr[i]); ++ g5p_put64(w + 0x488, p->scratch_ptr[4]); ++} ++ ++static inline void g5p_ru_fill_3d(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_3D_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_3d_fixed); i++) ++ g5p_put32(w + g5p_ru_3d_fixed[i][0], g5p_ru_3d_fixed[i][1]); ++ g5p_put64(w + 0x60, p->pool8_va & ~3ULL); ++ g5p_put64(w + 0x68, p->pool7_va & ~3ULL); ++ g5p_put64(w + 0x490, p->pool7_va); ++ g5p_put64(w + 0x4a0, p->pool8_va); ++ for (u32 i = 0; i < 8; i++) ++ g5p_put32(w + 0x5b8 + i * 4, p->background[i]); ++ g5p_put32(w + 0xb0, p->background[0]); ++ g5p_put32(w + 0xb4, p->background[1]); ++ for (u32 i = 4; i < 8; i++) ++ g5p_put32(w + 0xc0 + (i - 4) * 4, p->background[i]); ++ /* 6ca7ae0 swaps the first EOT qword's halves in its B upload. */ ++ g5p_put32(w + 0x3b4, p->eot[1]); ++ g5p_put32(w + 0x3b8, p->eot[0]); ++ g5p_put32(w + 0x3bc, p->eot[2]); ++ for (u32 i = 0; i < 3; i++) { ++ g5p_put32(w + 0x680 + i * 4, p->eot[i]); ++ g5p_put32(w + 0x694 + i * 4, p->eot_resume[i]); ++ } ++ g5p_put32(w + 0x600, p->restore[0]); ++ g5p_put32(w + 0x604, p->restore[1]); ++ g5p_put32(w + 0x610, p->restore[2]); ++ g5p_put32(w + 0x614, p->restore[3]); ++} ++ ++/* TYPED_V2 additions, already validated by g5p_ru_pass2_check(). */ ++struct g5p_ru_pass2 { ++ u64 scissor_va, dbias_va, background_texture, zls[6]; ++ u64 occlusion_query_va, occlusion_query_size; ++ u32 scissor_count, dbias_count, bgobj_depth, bgobj_vals; ++ u32 pixel_width, zls_mask; ++}; ++ ++/* Dialect-2 words over g5p_ru_fill_3d() (TYPED_V2 has zero pools). Each ++ * register image goes to its B-upload word and to the normal builder's ++ * copy (6c7a06c); make-render-user-dialect2.py proves every position by ++ * perturbing record 2d8/2e0 (pools), 2f0/310/338/360/388/308 (ZLS), 508 ++ * (background texture) and 4d0 (clears) through the r22 importer. ++ * Pixel output: 6ca7ae0 for D2f8 = width, one sample (D6e0 = 0), D630 = ++ * D6b2 = 0, six cores. Widths 2/4/8 keep two or more partitions, so W3c0 ++ * and W3c4 keep their dialect-1 values; width 2 is dialect 1 exactly. ++ */ ++static inline void g5p_ru_fill_3d_width(u8 *w, u32 width) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by D2f8 - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u32 count = 0x4000 / (width << 10); ++ u32 code = width_code[width - 2]; ++ ++ g5p_put32(w + 0x3c8, count & 0x1f); ++ g5p_put32(w + 0x3cc, code >> 4 | 0xf8); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(w + 0x108 + i * 8, code | (count - 1) << 7 | 0x43); ++ g5p_put32(w + 0x6b0, width); ++} ++ ++static inline void g5p_ru_fill_3d_v2(u8 *w, const struct g5p_ru_pass2 *e) ++{ ++ static const u16 zls_w[6][2] = { ++ { 0x80, 0x618 }, { 0x88, 0x628 }, { 0x90, 0x630 }, ++ { 0x98, 0x648 }, { 0xa0, 0x650 }, { 0x138, 0x6d0 }, ++ }; ++ ++ g5p_put64(w + 0x60, e->scissor_va & ~3ULL); ++ g5p_put64(w + 0x4a0, e->scissor_va); ++ g5p_put64(w + 0x68, e->dbias_va & ~3ULL); ++ g5p_put64(w + 0x490, e->dbias_va); ++ /* ++ * D318 is copied to both partition records. The normal 3D upload masks ++ * the low nibble before programming ISP_OCLQRY_BASE. ++ */ ++ g5p_put64(w + 0x70, e->occlusion_query_va); ++ g5p_put64(w + 0x78, e->occlusion_query_va); ++ g5p_put64(w + 0xb8, e->background_texture); ++ g5p_put64(w + 0x5f0, e->background_texture); ++ g5p_put32(w + 0x3a4, e->bgobj_depth); ++ g5p_put32(w + 0x6a8, e->bgobj_depth); ++ g5p_put32(w + 0x3a8, e->bgobj_vals); ++ g5p_put32(w + 0x6ac, e->bgobj_vals); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put64(w + zls_w[i][0], e->zls[i]); ++ g5p_put64(w + zls_w[i][1], e->zls[i]); ++ } ++ g5p_ru_fill_3d_width(w, e->pixel_width); ++} ++ ++/* TYPED_V3 additions, already validated by g5p_ru_pass3_check(). */ ++struct g5p_ru_pass3 { ++ u64 ta_program_va; ++ u32 ta_program_size, vdm_end; ++ u32 pixel_width, pad; /* V1 dialect output width, 0 = 2 */ ++}; ++ ++/* Dialect-3 word over g5p_ru_fill_ta(): W+0x88 (-> R0+0x470) is the exact ++ * byte offset of the TA PDS program from pds_base, a plain 64-bit store with ++ * no mask. g5p_ru_fill_ta() leaves it zero for a V3 pass (vdm_control is ++ * zero), so V1/V2 bytes are untouched; the stream length never reaches W. ++ */ ++static inline void g5p_ru_fill_ta_v3(u8 *w, u64 program_off) ++{ ++ g5p_put64(w + 0x88, program_off); ++} ++ ++/* 6cb2378 selected state S at control+80: explicit zeros plus the four ++ * recovered backing pointers. N=1, PM ID0/record0. aux_va covers 10680. ++ */ ++static inline int g5p_ru_pm_state(u8 *control, u64 aux_va, u64 list_va) ++{ ++ memset(control + 0x80, 0, 0xc0); ++ return g5p_pm_record_backing(control + 0x80, 0, 1, aux_va, 0x80, ++ 0x10000, 0x600, ++ g5p_meta_address(list_va + 0x40)); ++} ++ ++/* 6cb124c/13f4/14dc pool for `blocks` resident = maximum blocks. Rewrites ++ * control+0..7f, the whole list and F; S at control+80 is not touched. ++ * physical[] holds blocks*8 16KiB PAs. Validate everything before writing. ++ */ ++static inline int g5p_ru_pm_check(u64 list_va, u64 free_va, const u64 *physical, ++ u32 blocks) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (!blocks || blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ ret = g5p_pm_sizes(1, blocks, 0x80, 0x10000, 0x600, &sizes); ++ if (ret) ++ return ret; ++ if ((list_va | free_va) & (G5P_GPU_PAGE - 1) || ++ list_va >= (1ULL << 40) || free_va >= (1ULL << 40) || ++ 0x80 + blocks * 0x24 > G5P_GPU_PAGE || ++ blocks * 32 * 4 > G5P_GPU_PAGE) ++ return -EINVAL; ++ for (u32 i = 0; i < blocks * 8; i++) ++ if ((physical[i] & (G5P_GPU_PAGE - 1)) || ++ physical[i] >= (1ULL << 40)) ++ return -ERANGE; ++ return 0; ++} ++ ++static inline int g5p_ru_pm_pool(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks) ++{ ++ u32 previous = 0; ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ memset(control, 0, 0x80); ++ memset(list, 0, G5P_GPU_PAGE); ++ memset(free, 0, G5P_GPU_PAGE); ++ for (u32 j = 0; j < blocks; j++) { ++ u32 at = 0x80 + j * 0x24; ++ ++ g5p_put32(list + at, previous); ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[j * 8 + k] >> 12; ++ ++ g5p_put32(list + at + 4 + k * 4, ppn); ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * 4, ++ ppn + p); ++ } ++ previous = g5p_meta_address(list_va + at); ++ } ++ g5p_put32(list + 0x60, 1); ++ g5p_put32(control + 0x10, previous); ++ g5p_put64(control + 0x14, free_va); ++ g5p_put32(control + 0x1c, blocks * 32 * 4); ++ g5p_put32(control + 0x20, blocks * 32); ++ g5p_put32(control + 0x24, blocks * 32 - 1); ++ g5p_put32(control + 0x34, G5P_GPU_PAGE); ++ g5p_put32(control + 0x40, g5p_meta_address(list_va)); ++ g5p_put32(control + 0x54, blocks * 32); ++ g5p_put32(control + 0x58, blocks * 32); ++ return 0; ++} ++ ++/* Complete fresh PM object: g5p_r22_pm() when blocks=32, aux=G5P_R22_AUX. */ ++static inline int g5p_ru_pm(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks, ++ u64 aux_va) ++{ ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ if ((aux_va & (G5P_GPU_PAGE - 1)) || ++ aux_va > (1ULL << 40) - G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ return -EINVAL; ++ memset(control, 0, G5P_GPU_PAGE); ++ ret = g5p_ru_pm_pool(control, list, free, list_va, free_va, physical, ++ blocks); ++ return ret ?: g5p_ru_pm_state(control, aux_va, list_va); ++} ++ ++/* The r25 latched selections that can still yield a frame. Context/host/SKU ++ * probes and the heap-relative RT dialect are diagnostic-only in the fixed ++ * path (pair rejected) and are not offered here; blocks is pm_blocks. ++ */ ++static inline bool g5p_ru_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return g5p_r25_valid(v) && !g5p_r25_diagnostic(v) && !v->blocks; ++} ++ ++/* r25 render_pm_variant 2/3 on a pool from g5p_ru_pm()/g5p_ru_pm_pool(): ++ * reverse only F's block order, or report F's whole mapped page as its ++ * extent. Equal to g5p_r25_pool() for 32, 64 and 128 blocks. ++ */ ++static inline void g5p_ru_pm_variant(u8 *control, u8 *free, const u64 *physical, ++ u32 blocks, u32 variant) ++{ ++ if (variant == 2) ++ for (u32 j = 0; j < blocks; j++) ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[(blocks - 1 - j) * 8 + k] >> ++ 12; ++ ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * ++ 4, ++ ppn + p); ++ } ++ if (variant == 3) ++ g5p_put32(control + 0x1c, G5P_GPU_PAGE); ++} ++ ++/* r25 render_pm_variant 1, after g5p_ru_job() built the TA page: tag6+10 ++ * names an owned holder with the linked head (outside CH/ring/W/SKU/TQ/ ++ * dependency); F/count/head start clear and last=ffffffff, so 10005368 ++ * rebuilds them. Equal to g5p_r25_inputs() for the same pool. ++ */ ++ ++/* Persistent per-channel objects (6c89420 layout): written once at attach. ++ * Control indices, TQ and the dependency object stay firmware-owned after. ++ */ ++static inline void g5p_ru_channel_init(u8 *p, u32 meta) ++{ ++ memset(p, 0, G5P_GPU_PAGE); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_R22_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_R22_TQ + 4, meta + G5P_R22_TQ); ++ g5p_put64(p + G5P_R22_DEP + 0x30, 1); ++ g5p_put64(p + G5P_R22_DEP + 0x48, ~0ULL); ++ g5p_put32(p + G5P_R22_DEP + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(p + G5P_R22_DEP + off + 8, 0xf); ++} ++ ++/* Pointer-ring positions of job `seq` (1-based): two entries per job. */ ++static inline u32 g5p_ru_ring_index(u32 seq) ++{ ++ return (2 * (seq - 1)) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_ru_endpoint(u32 seq) ++{ ++ return (2 * seq) % G5P_WORK_SLOTS; ++} ++ ++/* perf60 job slots (render_user_job_slots, drm-render-v3.inc). A job's command, W record and ++ * microsequence ("SKU") live in slot 0 (the r22 offsets) or slot 1, in the channel page's free ++ * tail; the firmware reaches both through the job's two pointer-ring entries. The channel header, ++ * control, pointer ring, TQ, stamps and dependency object stay shared. Slot 1 clears TQ/STAMP/DEP ++ * (0x1800..0x1b70) and the r25 holder (G5P_R25_HEAD, 0x3000). ++ */ ++#define G5P_RU_SLOT1 0x1c00U ++#define G5P_RU_SLOT1_END \ ++ (G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES + 0x180U) ++ ++static inline u32 g5p_ru_cmd_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 : G5P_WORK_CMD; ++} ++ ++static inline u32 g5p_ru_w_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 + G5P_RU_CMD_BYTES : G5P_R22_W; ++} ++ ++static inline u32 g5p_ru_sku_at(u32 js, bool td) ++{ ++ if (js) ++ return G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES; ++ return td ? G5P_R22_3D_SKU : G5P_R22_TA_SKU; ++} ++ ++/* Per-job objects for sequence `seq`: W image (fixed+typed+RT), channel W ++ * fields, the full timestamped microsequence, the tag6 PM bootstrap (TA) or ++ * tag4 TA-finish dependency (3D) and both pointer-ring entries. With seq=1 ++ * this equals g5p_r22_channel(). Stamp target and sequence are both seq. ++ * ta_finish is M(TA stamp word addressed by TA.W530), used by 3D only. ++ */ ++static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, ++ bool td, u32 seq, const u8 *image, ++ u32 ta_finish, bool zero_timestamps, u32 js, ++ u32 pmid) ++{ ++ u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; ++ u32 finish = meta + G5P_R22_STAMP + 4; ++ u32 sku = g5p_ru_sku_at(js, td), cmd = g5p_ru_cmd_at(js); ++ u32 ring = g5p_ru_ring_index(seq); ++ u8 *work = p + g5p_ru_w_at(js); ++ u32 cursor = sku, starts, finishes; ++ ++ memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); ++ memset(p + sku, 0, td ? 0x180 : 0xc0); ++ g5p_put32(work, td ? 1 : 0); ++ g5p_put32(work + 4, slot); ++ g5p_put32(work + (td ? 0x14 : 0xc), meta + G5P_R22_DEP); ++ g5p_put32(work + (td ? 0x18 : 0x14), pm); ++ if (!td && pmid) ++ g5p_put32(work + 0x10, ++ pmid); /* TA W+10: the PM identity (iOS Dcb0) */ ++ g5p_put32(work + (td ? 0x1c : 0x18), state); ++ g5p_put32(work + (td ? 0xc : 0x454), meta + sku); ++ g5p_put32(work + (td ? 0x10 : 0x458), td ? 0x180 : 0xc0); ++ g5p_put32(work + (td ? 0x888 : 0x52c), finish - 4); ++ g5p_put32(work + (td ? 0x88c : 0x530), finish); ++ g5p_put32(work + (td ? 0x890 : 0x534), seq); ++ g5p_put32(work + (td ? 0x894 : 0x538), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x898 : 0x53c), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x8a0 : 0x544), seq); ++ { ++ u32 ta_start[] = { ++ 0x23, ++ w + 0x3a4, ++ w + 0x20, ++ state, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ pmid, ++ 0, ++ 0, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ 0, ++ seq, ++ }; ++ u32 td_start[] = { ++ 0x25, ++ w + 0x3a0, ++ w + 0x58, ++ state, ++ global + 0x724, ++ w + 0x7f0, ++ w + 0x7f4, ++ w + 0x878, ++ meta, ++ w, ++ slot, ++ 0, ++ pmid, ++ g5p_le32(work + 0x800), ++ 0, ++ 1, ++ w + 0x6d8, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ 0, ++ seq, ++ }; ++ u32 ta_finish[] = { ++ 0x24, ++ state, ++ pm, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ seq, ++ finish, ++ seq, ++ (u32)-0x70, ++ }; ++ u32 td_finish[] = { ++ 0x26, ++ seq, ++ finish, ++ seq, ++ state, ++ pm, ++ 1, ++ global + 0x724, ++ w + 0x878, ++ w + 0x7f0, ++ meta, ++ w, ++ slot, ++ w + 0x6d8, ++ 0, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ (u32)-0x104, ++ }; ++ u32 result = w + (td ? 0x8ac : 0x550); ++ u32 timestamp = w + (td ? 0x8b4 : 0x558); ++ u32 ts0[] = { 0x80000018, result, timestamp, timestamp, seq }; ++ u32 ts1[] = { 0x18, result, timestamp, timestamp + 4, seq }; ++ ++ starts = td ? ARRAY_SIZE(td_start) : ARRAY_SIZE(ta_start); ++ finishes = td ? ARRAY_SIZE(td_finish) : ARRAY_SIZE(ta_finish); ++ for (u32 i = 0; i < starts; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_start[i] : ta_start[i]); ++ for (u32 i = 0; i < ARRAY_SIZE(ts0); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts0[i]); ++ if (td) { ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r22_wait_words); ++ i++, cursor += 4) ++ g5p_put32(p + cursor, g5p_r22_wait_words[i]); ++ } else { ++ g5p_put32(p + cursor, 0x801); ++ cursor += 4; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(ts1); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts1[i]); ++ for (u32 i = 0; i < finishes; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_finish[i] : ta_finish[i]); ++ g5p_put32(p + cursor, 0x40000017); ++ } ++ memset(p + cmd, 0, G5P_RU_CMD_BYTES); ++ g5p_put32(p + cmd, td ? 4 : 6); ++ g5p_put32(p + cmd + 4, td ? ta_finish : slot); ++ g5p_put32(p + cmd + 8, ++ td ? seq : pmid); /* tag6 {6, STA, PMid, M(control)} */ ++ g5p_put32(p + cmd + 12, td ? seq & 0x7fffffff : pm); ++ g5p_put32(p + G5P_WORK_RING + ring * 4, meta + cmd); ++ g5p_put32(p + G5P_WORK_RING + ((ring + 1) % G5P_WORK_SLOTS) * 4, w); ++} ++ ++/* Retirement of job `seq`: both W pointers still ours, finish word and ++ * retirement mirror at seq, all five channel watermarks at its endpoint. ++ * g5p_r22_retired() is the seq=1 case. ++ */ ++static inline bool g5p_ru_retired_at(const u8 *p, u32 meta, bool td, u32 seq, ++ u32 js) ++{ ++ const u8 *w = p + g5p_ru_w_at(js); ++ u32 endpoint = g5p_ru_endpoint(seq); ++ ++ if (g5p_le32(w + (td ? 0x888 : 0x52c)) != meta + G5P_R22_STAMP || ++ g5p_le32(w + (td ? 0x88c : 0x530)) != meta + G5P_R22_STAMP + 4 || ++ g5p_le32(w + (td ? 0x890 : 0x534)) != seq || ++ g5p_le32(p + G5P_R22_STAMP) != seq || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* A stamp beyond the published target, or a watermark not on the even ++ * two-entry grid, is a protocol error rather than "not yet retired". ++ */ ++static inline bool g5p_ru_stamp_sane(const u8 *p, u32 seq) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ s32 ahead = (s32)(g5p_le32(p + G5P_R22_STAMP + i * 4) - seq); ++ ++ if (ahead > 0) ++ return false; ++ } ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) >= G5P_WORK_SLOTS) ++ return false; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-loss.h b/drivers/gpu/apple-g5p/render-user-loss.h +new file mode 100644 +index 000000000..2aad49d88 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-loss.h +@@ -0,0 +1,120 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context fault isolation (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault ++ * isolation). Pure decisions, host-tested (scripts/gpu/multiclient2-host/test_render_loss.c). ++ * ++ * The hardware gives no way to abandon a job: the firmware has no abort (iOS ends a ++ * non-progressing engine with a firmware stop, a platform power cycle and a new firmware boot, ++ * kext 6c9e070, firmware-re README "Fault recovery"), the TA and 3D channels are FIFO, and the ++ * one TA and one 3D engine serve every context. So one context's job can be *lost* (its client ++ * gets an error, its context is dead) without ending the device only while the firmware is ++ * alive, and only if the stuck phase still finishes: until then no other job may be published ++ * behind it (it would wait behind the same engine and be lost in turn). ++ * ++ * loss at the end of a phase that did not complete: ++ * NONE it completed; ++ * CONTEXT a phase timeout on a job other than the epoch's first, with the firmware ++ * alive (its event stream never went silent, no fault event, no protocol ++ * error) and isolation on: the job and its context are lost, the device is ++ * not; the channel is blocked behind the stuck phase; ++ * DEVICE anything else: the epoch is terminal, as before. ++ * drain while blocked, the scheduler runs no job and waits, RAM only, for the stuck phase: ++ * a late TA gets its 3D published (the channel and PM state then stay consistent), ++ * a late 3D ends it; the channel is then clear and every held job runs. A silent ++ * event stream, a fault, a refused 3D or the drain bound make the epoch terminal. ++ */ ++#ifndef G5P_RENDER_USER_LOSS_H ++#define G5P_RENDER_USER_LOSS_H ++ ++enum g5p_ru_loss { ++ G5P_RU_LOSS_NONE, ++ G5P_RU_LOSS_CONTEXT, ++ G5P_RU_LOSS_DEVICE, ++}; ++ ++struct g5p_ru_loss_in { ++ u32 result; /* G5P_RU_WAIT_* of the phase */ ++ bool first; /* the epoch's first job (its attach and bell) */ ++ bool isolate; /* render_user_drain_ms != 0 */ ++ bool silent; /* the event stream stopped: the firmware is dead */ ++ bool fault; /* a fault event, a failed boot, a protocol or drain error */ ++}; ++ ++static inline int g5p_ru_loss(const struct g5p_ru_loss_in *in) ++{ ++ if (in->result == G5P_RU_WAIT_COMPLETE) ++ return G5P_RU_LOSS_NONE; ++ if (in->result != G5P_RU_WAIT_TIMEOUT || in->first || !in->isolate || ++ in->silent || in->fault) ++ return G5P_RU_LOSS_DEVICE; ++ return G5P_RU_LOSS_CONTEXT; ++} ++ ++/* The drain of a blocked channel. */ ++enum g5p_ru_drain_state { ++ G5P_RU_DRAIN_TA, /* waiting for the stuck TA */ ++ G5P_RU_DRAIN_TD, /* waiting for the stuck job's 3D */ ++ G5P_RU_DRAIN_CLEAR, /* done: the channel is clear */ ++ G5P_RU_DRAIN_DEAD, /* the epoch is terminal */ ++}; ++ ++/* What the drain does next. */ ++enum g5p_ru_drain_action { ++ G5P_RU_DRAIN_WAIT, /* sample again */ ++ G5P_RU_DRAIN_PUBLISH_TD, /* publish the stuck job's 3D, wait for it */ ++ G5P_RU_DRAIN_UNBLOCK, /* release the held jobs */ ++ G5P_RU_DRAIN_TERMINAL, /* note the epoch terminal, release the held jobs (they fail) */ ++}; ++ ++/* One step from the result of the phase wait (G5P_RU_WAIT_*) in state *st. A refused 3D ++ * publish is reported by the caller as G5P_RU_WAIT_FAULT on the TD state. ++ */ ++static inline int g5p_ru_drain_step(u32 *st, u32 result) ++{ ++ if (*st >= G5P_RU_DRAIN_CLEAR) ++ return *st == G5P_RU_DRAIN_CLEAR ? G5P_RU_DRAIN_UNBLOCK : ++ G5P_RU_DRAIN_TERMINAL; ++ if (result == G5P_RU_WAIT_PENDING) ++ return G5P_RU_DRAIN_WAIT; ++ if (result != G5P_RU_WAIT_COMPLETE) { ++ *st = G5P_RU_DRAIN_DEAD; ++ return G5P_RU_DRAIN_TERMINAL; ++ } ++ if (*st == G5P_RU_DRAIN_TA) { ++ *st = G5P_RU_DRAIN_TD; ++ return G5P_RU_DRAIN_PUBLISH_TD; ++ } ++ *st = G5P_RU_DRAIN_CLEAR; ++ return G5P_RU_DRAIN_UNBLOCK; ++} ++ ++/* ++ * The drain bound must leave the held clients' own wait intact. A client waits for its job up to ++ * the job_timeout_ms GET_RENDER_INFO reports plus a margin (the ICD: + 3 s, so 2 * hard + obs + 3 s ++ * = 125 s with latency101 on, 33 s with it off), and a held job has already waited out the stuck ++ * phase (up to the hard bound, or render_user_wait_ms) before the drain starts, then runs after ++ * it. Valid: 0 (isolation off), or from 1000 up to hard + obs - 10000 with latency101 on (hard ++ * 60000, obs 2000: 52000), and up to 8000 with it off. An invalid value at the time of a loss ++ * turns isolation off for it. ++ */ ++static inline bool g5p_ru_drain_valid(u32 drain_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!drain_ms) ++ return true; ++ if (drain_ms < 1000) ++ return false; ++ if (!hard_ms) ++ return drain_ms <= 8000; ++ return hard_ms + obs_ms >= 11000 && ++ drain_ms <= hard_ms + obs_ms - 10000; ++} ++ ++ ++ ++static inline const char *g5p_ru_loss_name(int loss) ++{ ++ return loss == G5P_RU_LOSS_NONE ? "none" : ++ loss == G5P_RU_LOSS_CONTEXT ? "context" : "device"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-mmu.h b/drivers/gpu/apple-g5p/render-user-mmu.h +new file mode 100644 +index 000000000..5a05dccbf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-mmu.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 MMU invalidation arm (render_user_mmu_inval, drm-render-v3.inc), pure helpers. ++ * ++ * The only invalidation of a user MMU context besides DC3 is the firmware's per-kick tag compare ++ * (10017ac4): diff = host_tag[S] ^ cached_tag[S]; bits 31:22 set -> e138 = (S << 3) | 4 (PC), ++ * else bits 21:12 -> | 2 (PD), else bits 11:0 -> | 1 (PT); then cached = host. One invalidate per ++ * kick, the highest changed field (docs/gpu/firmware-re/README.md "generation producer"). A PC ++ * field change therefore asks for the firmware's PC-level invalidate of that context, the level ++ * Apple's own producer (6cabe70) and DC3 (1000218c) use as the strongest one. ++ * ++ * g5p_ru_mmu_pc_tag(): a host tag whose PC field differs from both slots' cached PC fields, so ++ * the next kick of each slot takes the PC branch. The PD and PT fields are the given tag's. ++ * Never 0 (0 is the unbound tag) and never equal to either cached tag. Host-tested ++ * (scripts/gpu/perf60-host/test_mmu_inval.c). ++ */ ++#ifndef G5P_RENDER_USER_MMU_H ++#define G5P_RENDER_USER_MMU_H ++ ++#define G5P_RU_MMU_PC_MASK 0xffc00000U ++#define G5P_RU_MMU_PC_ONE 0x00400000U ++ ++/* render_user_mmu_inval bits */ ++#define G5P_RU_MMU_EDIT \ ++ 0x1U /* every VM edit publication (and spare adoption) asks for PC */ ++#define G5P_RU_MMU_SWITCH \ ++ 0x2U /* a bound context's job after another context's job: PC retag */ ++#define G5P_RU_MMU_TOUCH \ ++ 0x4U /* NC read of every table line before the tag store */ ++#define G5P_RU_MMU_EVERY \ ++ 0x8U /* every job of a bound context: PC retag (superset of SWITCH) */ ++ ++static inline u32 g5p_ru_mmu_pc_tag(u32 tag, u32 cached0, u32 cached1) ++{ ++ u32 pc = (tag & G5P_RU_MMU_PC_MASK) + G5P_RU_MMU_PC_ONE; ++ ++ /* At most two values are excluded, so three candidates always contain a free one. */ ++ for (u32 i = 0; i < 3; i++, pc += G5P_RU_MMU_PC_ONE) { ++ u32 t = (pc & G5P_RU_MMU_PC_MASK) | (tag & ~G5P_RU_MMU_PC_MASK); ++ ++ if (((t ^ cached0) & G5P_RU_MMU_PC_MASK) && ++ ((t ^ cached1) & G5P_RU_MMU_PC_MASK) && t) ++ return t; ++ } ++ return 0; /* unreachable: kept for the host test */ ++} ++ ++/* ++ * Retag on a job: the published host tag, or the cached one if none, moved to a fresh PC field. ++ */ ++static inline bool g5p_ru_mmu_retag_due(u32 mode, bool bound_before, ++ bool switched) ++{ ++ if (!bound_before) ++ return false; /* this job's DC3 or adoption publishes its own tag */ ++ return (mode & G5P_RU_MMU_EVERY) || ++ ((mode & G5P_RU_MMU_SWITCH) && switched); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-msaa.h b/drivers/gpu/apple-g5p/render-user-msaa.h +new file mode 100644 +index 000000000..f27f197a6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-msaa.h +@@ -0,0 +1,88 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_MSAA_H ++#define G5P_RENDER_USER_MSAA_H ++ ++#include "bits-compat.h" ++/* TYPED_V3 4x multisampling: the per-job words a pass with a 4-sample render ++ * target needs on top of the V1/V2 3D words and the target's geometry ++ * (g5p_ru_rt_derive_samples()). Pure byte builders over the kernel's own ++ * work records: no allocation, MMIO, locking or firmware publication. ++ * ++ * Evidence (docs/gpu/experiments/msaa.md): the iOS gles3/gles3b captures ++ * 20261101T175626Z, msaa4-resolve, msaa4-store, g3b-a2c and g3b-samplemask, ++ * against g3-base, and the importer's 3D/TA fillers 6ca7ae0/6ca9710 executed ++ * with the 4-sample descriptor (build/gpu/msaa/filler-msaa.c): ++ * record+0x4d8 0xf -> 0x7 D6a8 -> 3D W3c0[3:0] (ISP_CTL; the draws' ++ * ISPCTL.upass drops 0xf -> 0x7 with it) ++ * record+0x640 0 -> 1 D6ad -> 3D W3c0 bit 17 and W86c (resolve only; ++ * bit 17 is ISP_CTL.PROCESS_EMPTY_TILES, which ++ * Linux's template sets for every pass since ++ * 0e05f993cf, so only W86c is new) ++ * RT Rb8 0 -> 2 D6e0 -> 3D Wd0 (AA mode, CR f30), and the ++ * pixel-output shift 10 -> 11 in W3c0 [21], ++ * W3c4, W3c8 and the six W108 words ++ * RT Rb0 / Rf8 3D W58 (CR fa8) / TA W40 (PPP_MULTISAMPLECTL) ++ * The rest of the target's geometry reaches W through g5p_ru_rt_fill(). ++ */ ++#define G5P_RU_MSAA_RESOLVE 1U /* DRM_G5P_RENDER_MSAA_RESOLVE */ ++ ++struct g5p_ru_msaa { ++ u32 samples, flags; /* 0 = 1 */ ++ u64 positions; /* 0 = the target's */ ++}; ++ ++/* samples 0 or 1 is the one-sample pass and must use a one-sample target; ++ * 4 must use a four-sample target. Positions are one byte per sample, so a ++ * word with bits above the sample count is malformed; zero takes the ++ * target's (the standard pattern). Resolve exists only with four samples. ++ * Nothing here is a hardware hazard: positions are a 4-bit grid and the ++ * resolve bit changes what the PBE writes, not where. ++ */ ++static inline int g5p_ru_msaa_check(const struct g5p_ru_msaa *m, u32 rt_samples) ++{ ++ u32 s = m->samples ? m->samples : 1; ++ ++ if ((s != 1 && s != 4) || s != rt_samples || ++ m->flags & ~G5P_RU_MSAA_RESOLVE || (m->flags && s == 1) || ++ m->positions >> (s * 8)) ++ return -EINVAL; ++ return 0; ++} ++ ++/* After g5p_ru_fill_3d(), the dialect's width words and g5p_ru_rt_fill(). ++ * A one-sample pass is left as it is, unless it names its position. ++ */ ++static inline void g5p_ru_fill_msaa(u8 *ta, u8 *td, const struct g5p_ru_msaa *m, ++ const struct g5p_ru_rt *rt) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by width - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u64 positions = m->positions ? m->positions : rt->packed; ++ bool resolve = m->flags & G5P_RU_MSAA_RESOLVE; ++ u32 width, count, mode, w3c0; ++ ++ if (m->samples != 4 && !m->positions) ++ return; ++ g5p_put64(ta + 0x40, positions); ++ g5p_put64(td + 0x58, positions); ++ if (m->samples != 4) ++ return; ++ /* The width the dialect wrote (2, 4 or 8); 6ca7ae0 shifts by 11 with an ++ * AA mode, so each width has half the output partitions. ++ */ ++ width = g5p_le32(td + 0x6b0); ++ count = 0x4000 / (width << 11); ++ mode = count > 1 ? 2 : count; ++ w3c0 = g5p_le32(td + 0x3c0) & ~(0xfU | BIT(21)); ++ g5p_put32(td + 0x3c0, ++ w3c0 | 0x7 | (resolve ? BIT(17) : 0) | (mode - 1) << 21); ++ g5p_put32(td + 0x3c4, mode * 0x204); ++ g5p_put32(td + 0x3c8, count & 0x1f); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(td + 0x108 + i * 8, ++ width_code[width - 2] | (count - 1) << 7 | 0x43); ++ g5p_put64(td + 0xd0, rt->state); ++ g5p_put32(td + 0x86c, resolve); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-pmset.h b/drivers/gpu/apple-g5p/render-user-pmset.h +new file mode 100644 +index 000000000..170738f1f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-pmset.h +@@ -0,0 +1,77 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context PM records (perf60 stage 4, docs/gpu/multi-client-design.md risk R3). ++ * ++ * iOS acquires a ParamBuffer ID per prepare from a 127-entry pool (6cb9afc) and builds a PM ++ * control/state/aux object and its block list per PM (6cb2378); one PM is never shared by two ++ * work queues. Linux built one device PM (control+S at shared+G5P_R22_PM, list and F in the ++ * two render_meta pages, aux at G5P_RU_PRIVATE_VA) for every render context; with the GPU ++ * never parked, one context's job then started from the PM state another context's job left. ++ * ++ * boot_pm_sets reserves N sets in the firmware GiB at boot, right after render_meta: set s is ++ * three pages {control + S, list, F} at G5P_PMSET_VA + s * 3 pages, mapped like the shared ++ * pages (they are firmware objects the firmware and the PM requesters address by meta VA). ++ * A render context takes one set, its own pool pages and its own aux pages. PM ID: 0 stays ++ * the device PM's; set s may carry s + 1. ++ * ++ * Pure helpers, host-tested (scripts/gpu/perf60-host/test_pmset.c). ++ */ ++#ifndef G5P_RENDER_USER_PMSET_H ++#define G5P_RENDER_USER_PMSET_H ++ ++#define G5P_PMSET_MAX 8U ++#define G5P_PMSET_PAGES 3U /* control + S, list, F */ ++#define G5P_PMSET_VA (G5P_SHARED_VA + G5P_SHARED_SIZE + 2 * G5P_GPU_PAGE) ++#define G5P_PMSET_END \ ++ (G5P_PMSET_VA + G5P_PMSET_MAX * G5P_PMSET_PAGES * G5P_GPU_PAGE) ++ ++enum { G5P_PMSET_CONTROL, G5P_PMSET_LIST, G5P_PMSET_FREE }; ++ ++static inline u64 g5p_pmset_va(u32 set, u32 part) ++{ ++ return G5P_PMSET_VA + ++ ((u64)set * G5P_PMSET_PAGES + part) * G5P_GPU_PAGE; ++} ++ ++static inline u32 g5p_pmset_id(u32 set) ++{ ++ return set + 1; ++} ++ ++/* ++ * The PM's write-back target (release8 bench-gaps3, 2026-11-03). After the GPU switches from ++ * context A's jobs to context B's, the firmware's PM keeps writing A's free list F back, through ++ * A's TA slot (its MMU context), on B's later jobs, until a job of a third context runs: two ++ * faults, one on the 2nd and one on the 3rd job of B after A's last, both the first job after A ++ * was unbound (page-invalid level 2 in A's MMU context, A's set F page; with A still bound the ++ * same writes landed). So a context's slots and set stay bound while it is the context of the ++ * last job or the previous *distinct* one: g5p_pmset_track() at every job end, g5p_pmset_holds() ++ * at close. That bounds when the slots may go, not when the PM stops: two later faults (release8 ++ * quiet2, seq 858 and 893) wrote the free list of a context two switches back, after this rule had ++ * released it. So a released pair is parked rather than left empty (render_user_close_park, ++ * drm-render-v3.inc), and a released set is reused last (g5p_pmset_pick()). ++ */ ++static inline bool g5p_pmset_switched(const void *last, const void *job) ++{ ++ return last != job; ++} ++ ++static inline bool g5p_pmset_holds(const void *last, const void *prev, ++ const void *ctx) ++{ ++ return ctx && (ctx == last || ctx == prev); ++} ++ ++/* A free set, round-robin from `next` (a released set is reused last). -1: none free. */ ++static inline int g5p_pmset_pick(const bool *used, u32 n, u32 next) ++{ ++ for (u32 i = 0; i < n; i++) { ++ u32 s = (next + i) % n; ++ ++ if (!used[s]) ++ return s; ++ } ++ return -1; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-quiesce.h b/drivers/gpu/apple-g5p/render-user-quiesce.h +new file mode 100644 +index 000000000..3671e415e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-quiesce.h +@@ -0,0 +1,145 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU quiesce before a kexec, reboot or power-off (docs/gpu/experiments/mcfold2-kexec.md). ++ * ++ * Without it the firmware keeps running from the outgoing kernel's RAM across the relocation: ++ * it still writes its event rings, and the MMU still holds a CBASE for every bound render slot ++ * pointing at page tables the next kernel reuses, until the next kernel's PMGR GFX cycle about ++ * 0.4 s into its boot. The quiesce ends that window on the outgoing side, in this order: ++ * ++ * 1 close job admission and wait (bounded) for submitters inside; ++ * 2 take the command lock (bounded trylock), so no job or command is in flight; ++ * 3 DETACH: every user slot the firmware may hold is unbound by a RAM-only DC3 batch ++ * (render-user-bind.h), only when the lock is ours, the firmware is healthy, no timeout ++ * or fault was noted and the DC ring is idle; ++ * 4 freeze the boot state and mask the GPU IRQ (no host register access after this); ++ * 5 CYCLE: one bounded PMGR GFX off/on, the same gate cycle the next kernel performs with ++ * apple_pmgr.kexec_gpu_cycle=1, only when no host thread can be inside a GPU register ++ * access: the lock is ours, or its holder is the TYPED_V3 executor in its RAM-only regime. ++ * ++ * Pure functions over the ops below, host-tested (scripts/gpu/mcfold2-host/test_kexec_quiesce.c). ++ */ ++#ifndef G5P_RENDER_USER_QUIESCE_H ++#define G5P_RENDER_USER_QUIESCE_H ++ ++/* kexec_quiesce module parameter */ ++#define G5P_RU_KQ_OFF 0U ++#define G5P_RU_KQ_RAM 1U /* detach only, no PMGR access */ ++#define G5P_RU_KQ_FULL 2U /* detach and cycle (default) */ ++ ++/* Plan */ ++#define G5P_RU_KQ_DETACH 0x1U ++#define G5P_RU_KQ_CYCLE 0x2U ++ ++/* Why a step was skipped */ ++#define G5P_RU_KQ_WHY_OFF 0x001U /* kexec_quiesce=0 */ ++#define G5P_RU_KQ_WHY_DOWN 0x002U /* the firmware never ran on this boot */ ++#define G5P_RU_KQ_WHY_BUSY 0x004U /* the command lock is not ours: no detach */ ++#define G5P_RU_KQ_WHY_FAILED \ ++ 0x008U /* fault or protocol failure: firmware state unknown */ ++#define G5P_RU_KQ_WHY_WEDGED \ ++ 0x010U /* a TYPED_V3 timeout or fault: a job may hold its slots */ ++#define G5P_RU_KQ_WHY_RING 0x020U /* the DC ring is not idle */ ++#define G5P_RU_KQ_WHY_NONE 0x040U /* no user slot is held */ ++#define G5P_RU_KQ_WHY_HOLDER \ ++ 0x080U /* unknown lock holder, maybe inside MMIO: no cycle */ ++#define G5P_RU_KQ_WHY_NO_PMGR 0x100U ++#define G5P_RU_KQ_WHY_RAM 0x200U /* kexec_quiesce=1: no cycle */ ++#define G5P_RU_KQ_WHY_NO_DRM 0x400U /* no render frontend: nobody can detach */ ++ ++struct g5p_ru_kq_in { ++ u32 mode; /* kexec_quiesce */ ++ bool booted; /* the firmware was released on this boot */ ++ bool locked; /* the command lock is ours */ ++ bool failed; /* b->failed or b->event_failed */ ++ bool frontend; /* a render frontend that can detach is registered */ ++ bool term_failed; /* the TYPED_V3 latch noted a timeout or fault */ ++ bool ram_only; /* the TYPED_V3 latch is quiet: its executor touches no register */ ++ bool executing; /* the TYPED_V3 executor is inside a job */ ++ bool ring_idle; /* DC consumer == producer <= 255 */ ++ bool pmgr; /* the PMGR regmap is present */ ++ u32 held; /* user slots with a PA or a host tag */ ++ u32 cons, prod; /* the DC ring when sampled */ ++}; ++ ++static inline u32 g5p_ru_kq_plan(const struct g5p_ru_kq_in *in, u32 *why) ++{ ++ u32 plan = 0, w = 0; ++ ++ if (in->mode == G5P_RU_KQ_OFF || !in->booted) { ++ *why = in->mode == G5P_RU_KQ_OFF ? G5P_RU_KQ_WHY_OFF : ++ G5P_RU_KQ_WHY_DOWN; ++ return 0; ++ } ++ if (!in->locked) ++ w |= G5P_RU_KQ_WHY_BUSY; ++ if (in->failed) ++ w |= G5P_RU_KQ_WHY_FAILED; ++ if (in->term_failed) ++ w |= G5P_RU_KQ_WHY_WEDGED; ++ if (!in->ring_idle) ++ w |= G5P_RU_KQ_WHY_RING; ++ if (!in->held) ++ w |= G5P_RU_KQ_WHY_NONE; ++ if (!in->frontend) ++ w |= G5P_RU_KQ_WHY_NO_DRM; ++ if (!w) ++ plan |= G5P_RU_KQ_DETACH; ++ if (in->mode < G5P_RU_KQ_FULL) ++ w |= G5P_RU_KQ_WHY_RAM; ++ else if (!in->pmgr) ++ w |= G5P_RU_KQ_WHY_NO_PMGR; ++ else if (!in->locked && !(in->executing && in->ram_only)) ++ w |= G5P_RU_KQ_WHY_HOLDER; ++ else ++ plan |= G5P_RU_KQ_CYCLE; ++ *why = w; ++ return plan; ++} ++ ++struct g5p_ru_kq_ops { ++ void *ctx; ++ u32 (*admit_close)(void *ctx); /* close admission; submitters still inside */ ++ bool (*lock)(void *ctx); /* bounded trylock of the command lock */ ++ void (*unlock)(void *ctx); ++ void (*inputs)(void *ctx, struct g5p_ru_kq_in *in); /* RAM only */ ++ int (*detach)(void *ctx); /* RAM only */ ++ void (*freeze)(void *ctx); /* RAM only */ ++ void (*irq_off)(void *ctx); /* AIC ack, mask, wait for the handler */ ++ int (*cycle)(void *ctx); /* PMGR only */ ++}; ++ ++struct g5p_ru_kq_out { ++ struct g5p_ru_kq_in in; ++ u32 inside, plan, why; ++ int detach_ret, cycle_ret; ++ bool froze; ++}; ++ ++/* One quiesce. detach_ret/cycle_ret stay 0 for a step that did not run (see plan). */ ++static inline void g5p_ru_kq_run(const struct g5p_ru_kq_ops *o, u32 mode, ++ struct g5p_ru_kq_out *out) ++{ ++ *out = (struct g5p_ru_kq_out){ .in.mode = mode }; ++ if (mode == G5P_RU_KQ_OFF) { ++ out->why = G5P_RU_KQ_WHY_OFF; ++ return; ++ } ++ out->inside = o->admit_close(o->ctx); ++ out->in.locked = o->lock(o->ctx); ++ o->inputs(o->ctx, &out->in); ++ out->plan = g5p_ru_kq_plan(&out->in, &out->why); ++ if (out->plan & G5P_RU_KQ_DETACH) ++ out->detach_ret = o->detach(o->ctx); ++ if (out->in.booted) { ++ o->freeze(o->ctx); ++ o->irq_off(o->ctx); ++ out->froze = true; ++ } ++ if (out->plan & G5P_RU_KQ_CYCLE) ++ out->cycle_ret = o->cycle(o->ctx); ++ if (out->in.locked) ++ o->unlock(o->ctx); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-recover.h b/drivers/gpu/apple-g5p/render-user-recover.h +new file mode 100644 +index 000000000..51982c5c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-recover.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * In-boot GPU recovery: the firmware's recovery packet and the recovery decision. Pure, ++ * host-tested (scripts/gpu/perf60-host/test_recover.c). ++ * ++ * The recovery packet is firmware event type 4 (44 bytes). The AGX kext's event drain ++ * (AGXFirmware_drainFirmwareEventRing 6c99788, case 4) reads +0x0c (the faulting slot, 0x80 for ++ * -1), +0x10 (a flag, accel+0x1af8 bit 3), +0x18, +0x20 and +0x24 (accel+0x1a1c/0x1af4/0x1af0), ++ * marks a recovery pending (accel+0x1a24 = 2) and calls IOGPUScheduler_signalHardwareError(sched, ++ * 1) (6c67b4c): "GPURestartSignaled", the scheduler enqueues one GPU restart (a second signal ++ * before it runs is "Redundant hardware error"). The restart is ++ * AGXMetaFirmware_stopForRecoveryOptionalPowerCycle (6ca1e40) and ++ * AGXMetaFirmware_rebootAfterRecovery (6ca1efc), which panics only if the firmware does not boot ++ * again. There is no per-boot limit: every recovery packet restarts the GPU. The ++ * kext does not read +0x04. ++ * ++ * On A9/Linux the slot matches the phase the host waited in: 0 on every TA fault, 1 on the 3D fault ++ * of cf7 (2026-11-02); +0x20 = 8 and +0x24 = 0x81818105 (TA) / 0x81810581 (3D) so far. ++ */ ++#ifndef G5P_RENDER_USER_RECOVER_H ++#define G5P_RENDER_USER_RECOVER_H ++ ++#define G5P_FW_EVENT_RECOVERY 4 ++#define G5P_FW_RECOVERY_NOSLOT 0x80 ++ ++struct g5p_fw_recovery { ++ u32 code; /* +0x04, not read by the kext */ ++ u32 slot; /* +0x0c, G5P_FW_RECOVERY_NOSLOT for -1 */ ++ u32 flag; /* +0x10 != 0 */ ++ u32 w18, w20, w24; ++}; ++ ++/* `w` is the 11-word event entry, type 4. */ ++static inline struct g5p_fw_recovery g5p_fw_recovery_decode(const u32 *w) ++{ ++ struct g5p_fw_recovery r = { ++ .code = w[1], ++ .slot = w[3] == 0xffffffffU ? G5P_FW_RECOVERY_NOSLOT : w[3], ++ .flag = w[4] != 0, ++ .w18 = w[6], ++ .w20 = w[8], ++ .w24 = w[9], ++ }; ++ ++ return r; ++} ++ ++/* ++ * May a job that stopped the firmware (a recovery packet, a silent event stream, a hard timeout) ++ * be recovered from in this boot, the job's context lost and every other kept? ++ * ++ * budget == 0 never (render_user_recover 0: the epoch ends, as before); ++ * retired/closed never (the boot is shutting the GPU down); ++ * streak_max == 0 the old rule: `budget` recoveries per boot, never on the epoch's first job; ++ * streak_max != 0 Apple's rule, every time, with two guards: at most streak_max recoveries in a ++ * row with no job completing between them (a GPU that faults again before it ++ * finishes anything is not recovering), and at most G5P_RU_RECOVER_BOOT_MAX per ++ * boot (recover.inc leaves each old epoch allocated, ~0.8 MiB: 50 x 16 KiB pages ++ * and its struct, so a stale pointer still reads valid memory; this bounds that ++ * to ~26 MiB). The boot's first epoch's first job stays terminal (a failed ++ * bring-up, not a client); the first job of a later epoch (the attach after a ++ * recovery) is recoverable like any other. ++ */ ++#define G5P_RU_RECOVER_BOOT_MAX 32 ++ ++struct g5p_ru_recover_in { ++ bool first; /* the job runs its epoch's attach */ ++ bool retired; /* the GPU was retired (kexec / reboot quiesce) */ ++ bool closed; /* admission closed */ ++ u32 recoveries; /* recoveries this boot */ ++ u32 streak; /* recoveries since the last job completed */ ++ u32 budget; /* render_user_recover */ ++ u32 streak_max; /* render_user_recover_streak */ ++}; ++ ++static inline bool g5p_ru_recoverable(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget || in->retired || in->closed) ++ return false; ++ if (!in->streak_max) ++ return !in->first && in->recoveries < in->budget; ++ if (in->first && !in->recoveries) ++ return false; ++ return in->streak < in->streak_max && ++ in->recoveries < G5P_RU_RECOVER_BOOT_MAX; ++} ++ ++/* Why g5p_ru_recoverable() said no (for the terminal line), or "-" when it said yes. */ ++static inline const char * ++g5p_ru_recover_refusal(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget) ++ return "recover-off"; ++ if (in->retired || in->closed) ++ return "shutdown"; ++ if (!in->streak_max) ++ return in->first ? "first-job" : ++ in->recoveries >= in->budget ? "per-boot-budget" : ++ "-"; ++ if (in->first && !in->recoveries) ++ return "first-job"; ++ if (in->streak >= in->streak_max) ++ return "streak"; ++ return in->recoveries >= G5P_RU_RECOVER_BOOT_MAX ? "boot-max" : "-"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-retire.h b/drivers/gpu/apple-g5p/render-user-retire.h +new file mode 100644 +index 000000000..6e5729003 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-retire.h +@@ -0,0 +1,248 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 3D completion: target coherence and engine idle, decided separately. Pure ++ * functions, no kernel services beyond the integer types, so the host suite compiles this ++ * file as it is. Include render-user-wait.h first. ++ * ++ * combined107 P4R seq 4 (build/gpu/combined107/raw/20261031T145710Z): the 3D target state ++ * was coherent (ring consumer = producer = target 4, both stamps 4, W+0x888/0x88c ours, ++ * W+0x890 = 4, all five channel watermarks at endpoint 8, and the firmware's type-1 3D ++ * completion event with the endpoint pair 7/8 decoded at publish + 19999 ms), yet the ++ * wait timed out at 60002 ms. The wait is a 1-2 ms RAM poll (no waitqueue, no wake), so ++ * this was not a lost edge: the done predicate also required the 3D global's idle words ++ * (G+0x724/0x728/0x72c/0x734/0x738) to be zero, and they never cleared. ++ * ++ * The 3D phase is complete when either ++ * (a) the target is coherent and the idle words are zero (the flown r96 predicate), or ++ * (b) the target is coherent and the matching 3D completion event was decoded after this ++ * job's 3D publish (idle_stuck). The event is the firmware's completion report, the ++ * signal a fence follows on Asahi too; the idle words lag it by an arbitrary time. ++ * settle_ms (default 0) only adds a minimum length of the coherent run, or ++ * (c) the target has been coherent, unbroken, for G5P_RU_RT_HOLD_MS (at least settle_ms): ++ * tahang1, as the TA's rule (c), so a lost or late event does not hold a job whose ++ * stamps landed. The PM state needs no clause here: the TA phase, which the 3D ++ * publish waits for, required it saved. ++ * Target coherent = ring consumer == target && g5p_ru_retired(): both stamps == seq, the W ++ * stamp pointers ours, the W sequence word == seq, every channel watermark == endpoint. ++ * The stamps carry the identity (31-bit sequence, one per job); the watermarks and the ++ * event's endpoint pair repeat every eight jobs and only corroborate. The TA phase has its ++ * own rules (tahang1, below). ++ */ ++#ifndef G5P_RENDER_USER_RETIRE_H ++#define G5P_RENDER_USER_RETIRE_H ++ ++#define G5P_RU_RT_DEF_SETTLE_MS 0U /* default of render_user_idle_settle_ms */ ++#define G5P_RU_RT_MAX_SETTLE_MS \ ++ 5000U /* far below the smallest per-phase bound (20 s) */ ++#define G5P_RU_RT_GRID 16U /* the channel watermark grid, G5P_WORK_SLOTS */ ++#define G5P_RU_RT_HOLD_MS \ ++ 500U /* rule (c), raised to settle_ms when that is longer */ ++ ++/* Clauses of the 3D predicate, as the receipt's `unsat` bits (set = not satisfied). */ ++#define G5P_RU_RT_CONSUMED 0x1U /* ring consumer != target */ ++#define G5P_RU_RT_RETIRED 0x2U /* g5p_ru_retired() false */ ++#define G5P_RU_RT_IDLE 0x4U /* an idle word is non-zero */ ++#define G5P_RU_RT_EVENT 0x8U /* no matching completion event latched */ ++#define G5P_RU_RT_SETTLE \ ++ 0x10U /* coherent, run shorter than the settle or hold window */ ++#define G5P_RU_RT_PM 0x20U /* TA receipt only: the PM state is not saved */ ++ ++enum g5p_ru_rt_how { ++ G5P_RU_RT_PENDING, ++ G5P_RU_RT_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_RT_IDLE_STUCK, /* rule (b) */ ++ G5P_RU_RT_HOLD, /* rule (c) */ ++}; ++ ++struct g5p_ru_retire { ++ u64 settle_ns; /* minimum coherent run before rule (b); 0 = none */ ++ u64 hold_ns; /* rule (c): max(settle_ns, G5P_RU_RT_HOLD_MS) */ ++ u64 run_ns; /* first sample of the current coherent run; 0 = none */ ++ u64 coherent_ns; /* first coherent sample of the phase */ ++ u64 event_ns; /* first sample that saw the matching event latched */ ++ u64 busy_ns; /* first sample with an idle word set */ ++ u64 clear_ns; /* first all-clear sample after busy_ns */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample */ ++ u32 how; /* enum g5p_ru_rt_how of the last sample */ ++ u32 decided; /* enum g5p_ru_rt_how of the sample that decided COMPLETE */ ++}; ++ ++static inline bool g5p_ru_rt_settle_valid(u32 settle_ms) ++{ ++ return settle_ms <= G5P_RU_RT_MAX_SETTLE_MS; ++} ++ ++static inline void g5p_ru_rt_start(struct g5p_ru_retire *t, u32 settle_ms) ++{ ++ u32 hold_ms = settle_ms > G5P_RU_RT_HOLD_MS ? settle_ms : ++ G5P_RU_RT_HOLD_MS; ++ ++ *t = (struct g5p_ru_retire){ ++ .settle_ns = (u64)settle_ms * 1000000ULL, ++ .hold_ns = (u64)hold_ms * 1000000ULL, ++ }; ++} ++ ++/* The firmware's 3D completion is a type-1 event with notification bit 1 (engine 1) whose ++ * words 8/9 are the channel endpoint pair (2 seq - 1, 2 seq): 1/2, 3/4, 5/6, 7/8 for ++ * combined107 seq 1..4. The grid wraps at G5P_RU_RT_GRID, so the pair is compared modulo ++ * the grid whether the firmware reports it wrapped or as a running count: endpoint 0 ++ * (seq 8) matches 15/0 and 15/16 alike. ++ */ ++static inline bool g5p_ru_rt_event_match(const u32 words[11], u32 endpoint) ++{ ++ return words[0] == 1 && (words[1] & 2) && ++ words[9] % G5P_RU_RT_GRID == endpoint % G5P_RU_RT_GRID && ++ (words[9] - words[8]) % G5P_RU_RT_GRID == 1; ++} ++ ++/* One RAM sample of the 3D phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_rt_how); the caller passes `!= G5P_RU_RT_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_rt_step(struct g5p_ru_retire *t, u64 now_ns, ++ bool consumed, bool retired, bool idle, ++ bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ if (!idle && !t->busy_ns) ++ t->busy_ns = now_ns; ++ if (idle && t->busy_ns && !t->clear_ns) ++ t->clear_ns = now_ns; ++ t->how = G5P_RU_RT_PENDING; ++ if (!consumed || !retired) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ t->run_ns = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_RT_IDLE_CLEAR; ++ else if (event && now_ns - t->run_ns >= t->settle_ns) ++ t->how = G5P_RU_RT_IDLE_STUCK; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_RT_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_rt_note(struct g5p_ru_retire *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++/* ms from `from_ns` to `to_ns` for the receipt; -1 when either is missing or out of order. */ ++static inline s64 g5p_ru_rt_ms(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000000ULL); ++} ++ ++/* ++ * tahang1: the TA phase. mcfold1 desktop seq 251 (after T1) and seq 339 (one client): ring ++ * consumed at 2 ms, the firmware's type-1 TA completion event decoded at 7.7-8.4 ms, both ++ * stamps, the W words and the watermarks at the job and the PM state saved, yet a TA idle word ++ * (G+0x724/0x728/0x72c) set from 7-8 ms on never cleared, and the r74b predicate (which needs ++ * the idle words zero) timed out at 60 s (`unsat=0x4`). On 49 of 241 good TAs of the same ++ * boot that word was set at 7-12 ms and cleared 2 ms later. The 3D engine shows the same lag ++ * (rule (b) above). ++ * ++ * The TA phase is complete when the target is coherent (ring consumer == target and ++ * g5p_ru_retired()) with the PM state saved, and ++ * (a) the idle words are zero (the flown r74b predicate), or ++ * (b) a type-1 TA completion event was decoded after this job's TA publish, or ++ * (c) the target has stayed coherent, unbroken, for G5P_RU_TA_HOLD_MS. ++ * The stamps carry the job's identity; the TA event does not (its words are the same every ++ * job), it only reports that the firmware finished a TA after the publish. Rule (c) keeps a ++ * lost or late event from holding a job whose stamps already landed. ++ */ ++#define G5P_RU_TA_HOLD_MS 500U ++ ++enum g5p_ru_ta_how { ++ G5P_RU_TA_PENDING, ++ G5P_RU_TA_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_TA_EVENT, /* rule (b) */ ++ G5P_RU_TA_HOLD, /* rule (c) */ ++ G5P_RU_TA_HOWS, ++}; ++ ++struct g5p_ru_ta { ++ u64 hold_ns; /* rule (c) */ ++ u64 run_ns; /* first sample of the current coherent run */ ++ u64 coherent_ns; /* first coherent sample of the phase; 0 = none */ ++ u64 event_ns; /* first sample that saw the event latched; 0 = none */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample (SETTLE: run below the hold) */ ++ u32 how; /* enum g5p_ru_ta_how of the last sample */ ++ u32 decided; /* enum g5p_ru_ta_how of the sample that decided COMPLETE */ ++}; ++ ++static inline void g5p_ru_ta_start(struct g5p_ru_ta *t) ++{ ++ *t = (struct g5p_ru_ta){ .hold_ns = ++ (u64)G5P_RU_TA_HOLD_MS * 1000000ULL }; ++} ++ ++/* The firmware's TA completion: a type-1 event with notification bit 0 (engine 0). */ ++static inline bool g5p_ru_ta_event_match(const u32 words[11]) ++{ ++ return words[0] == 1 && (words[1] & 1); ++} ++ ++/* One RAM sample of the TA phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_ta_how); the caller passes `!= G5P_RU_TA_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_ta_step(struct g5p_ru_ta *t, u64 now_ns, bool consumed, ++ bool retired, bool pm, bool idle, bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT) | ++ (pm ? 0 : G5P_RU_RT_PM); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ t->how = G5P_RU_TA_PENDING; ++ if (!consumed || !retired || !pm) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_TA_IDLE_CLEAR; ++ else if (event) ++ t->how = G5P_RU_TA_EVENT; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_TA_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_ta_note(struct g5p_ru_ta *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sched.h b/drivers/gpu/apple-g5p/render-user-sched.h +new file mode 100644 +index 000000000..174a29e23 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sched.h +@@ -0,0 +1,155 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Render job scheduling across contexts (docs/gpu/multi-client-design.md, stage 2), shaped ++ * like drm_sched so that stage 4 can swap drm_sched in without a UAPI or structural change: ++ * ++ * entity = one render context (drm_sched_entity): a FIFO of its accepted jobs, and at ++ * most one close item, which runs after its last job (drm_sched_entity_destroy()); ++ * run list = the entities with work, served round-robin: each pick takes the first entity ++ * whose head job is ready and moves it to the tail (drm_sched's RR policy); ++ * ready = every dependency of the head job has signalled, it failed before the hardware, ++ * its input deadline passed, or admission closed for good (the job then fails ++ * without the hardware). Readiness is the caller's predicate; the kernel's adds ++ * dma-fence callbacks (drm_sched_job_add_dependency()) so that an unsignalled ++ * input of one client never blocks another client's jobs. ++ * ++ * One credit: the single shared TA/3D channel and PM record run one job at a time, so the ++ * caller runs each picked item to completion before the next pick. Only the head of an ++ * entity is looked at, so each context's fences signal in submission order. ++ * ++ * The lists are plain list_heads; the caller holds its scheduler lock around every call. ++ * Host-tested (scripts/gpu/multiclient2-host/test_render_sched.c). ++ */ ++#ifndef G5P_RENDER_USER_SCHED_H ++#define G5P_RENDER_USER_SCHED_H ++ ++struct g5p_rs_job { ++ struct list_head link; ++}; ++ ++struct g5p_rs_entity { ++ struct list_head link; /* in g5p_rs.run while it has work */ ++ struct list_head jobs; /* FIFO of g5p_rs_job */ ++ bool in_run, close_pending; ++}; ++ ++struct g5p_rs { ++ struct list_head run; ++}; ++ ++/* g5p_rs_pick() result */ ++enum g5p_rs_pick { ++ G5P_RS_IDLE, /* nothing ready; *wake holds the earliest deadline if any job waits */ ++ G5P_RS_JOB, /* *job was popped from *entity */ ++ G5P_RS_CLOSE, /* *entity has no job left and its close item is due */ ++}; ++ ++/* Head-job predicate: true when it may run now. *deadline is the time at which it becomes ++ * ready on its own (its input deadline); the predicate leaves it alone if there is none. ++ */ ++typedef bool (*g5p_rs_ready_fn)(struct g5p_rs_job *job, void *cookie, ++ unsigned long *deadline); ++ ++static inline void g5p_rs_init(struct g5p_rs *s) ++{ ++ INIT_LIST_HEAD(&s->run); ++} ++ ++static inline void g5p_rs_entity_init(struct g5p_rs_entity *e) ++{ ++ INIT_LIST_HEAD(&e->link); ++ INIT_LIST_HEAD(&e->jobs); ++ e->in_run = false; ++ e->close_pending = false; ++} ++ ++static inline void g5p_rs_enlist(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (!e->in_run) { ++ list_add_tail(&e->link, &s->run); ++ e->in_run = true; ++ } ++} ++ ++/* An accepted job joins the end of its entity's FIFO. */ ++static inline void g5p_rs_push(struct g5p_rs *s, struct g5p_rs_entity *e, ++ struct g5p_rs_job *job) ++{ ++ list_add_tail(&job->link, &e->jobs); ++ g5p_rs_enlist(s, e); ++} ++ ++/* The entity's close item: due once its FIFO is empty. False if one is already queued. */ ++static inline bool g5p_rs_close(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (e->close_pending) ++ return false; ++ e->close_pending = true; ++ g5p_rs_enlist(s, e); ++ return true; ++} ++ ++/* After the close item ran: the entity leaves the scheduler for good. */ ++static inline void g5p_rs_closed(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ (void)s; ++ e->close_pending = false; ++ if (e->in_run) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } ++} ++ ++/* ++ * The next item to run, round-robin over the entities with work. A picked entity moves to ++ * the tail; one left without a job and without a close item leaves the run list. Entities ++ * whose head is not ready keep their place. `has_wake` reports whether *wake is set. ++ */ ++static inline int g5p_rs_pick(struct g5p_rs *s, g5p_rs_ready_fn ready, ++ void *cookie, struct g5p_rs_entity **entity, ++ struct g5p_rs_job **job, unsigned long *wake, ++ bool *has_wake) ++{ ++ struct g5p_rs_entity *e; ++ ++ *has_wake = false; ++ list_for_each_entry(e, &s->run, link) { ++ struct g5p_rs_job *head; ++ unsigned long deadline; ++ bool timed = false; ++ ++ if (list_empty(&e->jobs)) { ++ /* Without a close item it left the run list at its last pick. */ ++ if (!e->close_pending) ++ continue; ++ *entity = e; ++ *job = NULL; ++ list_move_tail(&e->link, &s->run); ++ return G5P_RS_CLOSE; ++ } ++ head = list_first_entry(&e->jobs, struct g5p_rs_job, link); ++ deadline = 0; ++ if (!ready(head, cookie, &deadline)) { ++ timed = deadline != 0; ++ if (timed && ++ (!*has_wake || (long)(deadline - *wake) < 0)) { ++ *wake = deadline; ++ *has_wake = true; ++ } ++ continue; ++ } ++ list_del_init(&head->link); ++ *entity = e; ++ *job = head; ++ if (list_empty(&e->jobs) && !e->close_pending) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } else { ++ list_move_tail(&e->link, &s->run); ++ } ++ return G5P_RS_JOB; ++ } ++ return G5P_RS_IDLE; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-serial.h b/drivers/gpu/apple-g5p/render-user-serial.h +new file mode 100644 +index 000000000..e5cffa4e3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-serial.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial jobs of every render context (a compositor and its clients: one TA+3D job ++ * per frame each), on the device's one TA/3D channel pair. ++ * Pure functions, host-tested (scripts/gpu/multiclient1-host/test_render_admit.c). ++ * ++ * Semantics: ++ * - jobs of all contexts run strictly one after another: the queue is FIFO, each job is ++ * TA publish -> TA complete -> 3D publish -> 3D complete before the next job starts, and ++ * there is never a TA(N+1) in flight with a 3D(N); ++ * - the epoch's first job brings the device up: PM, channels and its context's DC3 attach ++ * with the r74f/r74g bells (one-shot). Every later context binds its own slot pair at its ++ * first job with a DC3 pair published in RAM only (render-user-bind.h), no doorbell; ++ * - every job after the first only rebuilds the PM pool, builds its pages for the device's ++ * sequence number, publishes with NO doorbell and waits RAM-only; ++ * - the first job of a context that does not complete (timeout, fault, protocol error) ++ * marks that context bad: every job still queued behind it fails with -EIO without ++ * touching the hardware, and every later submit is refused (-EIO); ++ * - TYPED_V3 never mixes with the other profiles, on a context or on the device; ++ * - after any timeout or fault (render-user-terminal.h `failed`) no job starts at all, and ++ * once the epoch is quiet the device cannot be brought up (that rings a doorbell); binding ++ * another context is RAM only and stays allowed; ++ * - the sequence number is the device's, 31 bit; the channel watermark grid wraps ++ * (g5p_ru_endpoint). ++ */ ++#ifndef G5P_RENDER_USER_SERIAL_H ++#define G5P_RENDER_USER_SERIAL_H ++ ++enum g5p_ru_v3_refusal { ++ G5P_RU_V3_OK, ++ G5P_RU_V3_NO_TUPLE, /* boot tuple (r74g + r87b self-drain, no r74h/r80) absent */ ++ G5P_RU_V3_MIXED, /* another profile ran on this context or brought the device up */ ++ G5P_RU_V3_OVERFLOW, /* sequence space exhausted */ ++ G5P_RU_V3_SLOTS, /* not two distinct slots in 7..255 */ ++ G5P_RU_V3_VARIANT, /* r22/r25 variant, 3d_after_ta, timestamps or pm_policy */ ++ G5P_RU_V3_BOUND, /* per-phase bound outside 20000..30000 ms */ ++ G5P_RU_V3_STATE, /* executor only: bind state and job counts disagree */ ++ G5P_RU_V3_TERMINAL, /* terminal epoch: timeout/fault, or quiet and the device down */ ++ G5P_RU_V3_REFUSALS, ++}; ++ ++static const char *const g5p_ru_v3_why[G5P_RU_V3_REFUSALS] = { ++ "", ++ "transport is not ready for TYPED_V3 rendering", ++ "another profile ran on this context or brought the device up", ++ "sequence space exhausted", ++ "slots are not two distinct slots in 7..255", ++ "an r22/r25 variant, 3d_after_ta, timestamps or pm_policy is latched", ++ "per-phase bound outside 20000..30000 ms", ++ "bind state and job counts disagree", ++ "terminal epoch (timeout or fault, or quiet and the device not up): no hardware access", ++}; ++ ++/* Everything the admission needs, as plain values. */ ++struct g5p_ru_v3_state { ++ bool tuple; /* boot tuple present */ ++ bool v3_accepted; /* a V3 job was accepted on this context */ ++ bool bound; /* this context's slot pair was published */ ++ u32 jobs; /* jobs that started on this context */ ++ bool device_up; /* the epoch's first job built PM and channels */ ++ bool device_v3; /* ... through the TYPED_V3 attach */ ++ u32 sequence; /* the device's last job that started (0 = none) */ ++ u32 slot_ta, slot_3d; ++ bool variant; ++ u32 wait_ms; ++ bool quiet; /* a terminal note was made: mmio_ok is false */ ++ bool terminal_failed; /* ... and it was, or a later one is, a timeout or fault */ ++}; ++ ++/* Any two distinct user slots: unbinds leave holes, so a pair need not be adjacent. */ ++static inline bool g5p_ru_v3_slots_valid(u32 slot_ta, u32 slot_3d) ++{ ++ return slot_ta >= 7 && slot_3d >= 7 && slot_ta <= 255 && ++ slot_3d <= 255 && slot_ta != slot_3d; ++} ++ ++#define G5P_RU_V3_SEQ_MAX 0x7fffffffU ++ ++/* At submit time and again in the executor (`executing`): a refusal here touched nothing. */ ++static inline int g5p_ru_v3_admit(const struct g5p_ru_v3_state *s, ++ bool executing) ++{ ++ if (!s->tuple) ++ return G5P_RU_V3_NO_TUPLE; ++ /* After any timeout or fault nothing new starts (closes the window between a failing job's ++ * unlock and the context being marked bad). Once quiet, the device cannot be brought up, ++ * because that rings a doorbell; a job on an up device publishes with no doorbell and ++ * touches no register, and so does the bind of a new context. ++ */ ++ if (s->terminal_failed || (s->quiet && !s->device_up)) ++ return G5P_RU_V3_TERMINAL; ++ /* A job of another profile ran on this context, or another profile brought the device ++ * up: only V3 may follow V3. ++ */ ++ if ((s->jobs && !s->v3_accepted) || (s->device_up && !s->device_v3)) ++ return G5P_RU_V3_MIXED; ++ if (s->sequence >= G5P_RU_V3_SEQ_MAX) ++ return G5P_RU_V3_OVERFLOW; ++ if (!g5p_ru_v3_slots_valid(s->slot_ta, s->slot_3d)) ++ return G5P_RU_V3_SLOTS; ++ if (s->variant) ++ return G5P_RU_V3_VARIANT; ++ if (!g5p_ru_v3_bound_valid(s->wait_ms)) ++ return G5P_RU_V3_BOUND; ++ /* The executor: the device is up exactly when a job has started on it, and a context is ++ * bound exactly when it has run a job, always on an up device. ++ */ ++ if (executing && (s->device_up != !!s->sequence || ++ s->bound != !!s->jobs || (s->bound && !s->device_up))) ++ return G5P_RU_V3_STATE; ++ return G5P_RU_V3_OK; ++} ++ ++/* Verbose receipts for each context's first jobs and every failure; a compositor prints no ++ * line per frame. ++ */ ++static inline bool g5p_ru_v3_verbose(u32 jobs, int ret) ++{ ++ return jobs <= 8 || ret; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sync.h b/drivers/gpu/apple-g5p/render-user-sync.h +new file mode 100644 +index 000000000..d212d3199 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sync.h +@@ -0,0 +1,50 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CPU cache maintenance by range (perf60-fillrate.md §11.3 cut 5, kernel half). Host test: ++ * scripts/gpu/perf60-host/test_sync_range.c. ++ * ++ * render_user_sync_range bit 0: BO_SYNC honours offset/size (already validated page aligned by ++ * g5p_bo_range_valid) instead of syncing the whole BO. Bit 1: a job's ACCESS_CACHE maintenance ++ * (clean before, invalidate after retirement) covers the resource's BO range [offset, offset + ++ * size), the bytes the job can reach through that resource, instead of the whole BO. The window ++ * is widened to whole GPU pages and clamped to the BO. Off: the whole BO, as release8. ++ * The window is walked over the sg_table's DMA segments in order (their lengths concatenate to ++ * the BO's bytes); each segment gets the intersection, relative to the segment. ++ */ ++#ifndef G5P_RENDER_USER_SYNC_H ++#define G5P_RENDER_USER_SYNC_H ++ ++#define G5P_SYNC_ALIGN 0x4000ULL ++ ++/* ++ * [lo, hi) = [off, off + size) widened to pages, clamped to the BO; false if empty or outside. ++ */ ++static inline bool g5p_sync_window(u64 off, u64 size, u64 bo_size, u64 *lo, ++ u64 *hi) ++{ ++ u64 end; ++ ++ if (!size || off >= bo_size) ++ return false; ++ end = size > bo_size - off ? bo_size : off + size; ++ *lo = off & ~(G5P_SYNC_ALIGN - 1); ++ *hi = (end + G5P_SYNC_ALIGN - 1) & ~(G5P_SYNC_ALIGN - 1); ++ if (*hi > bo_size) ++ *hi = bo_size; ++ return *hi > *lo; ++} ++ ++/* The part of segment [pos, pos + len) inside [lo, hi): *start relative to the segment. */ ++static inline bool g5p_sync_clip(u64 pos, u64 len, u64 lo, u64 hi, u64 *start, ++ u64 *n) ++{ ++ u64 a = pos > lo ? pos : lo, b = pos + len < hi ? pos + len : hi; ++ ++ if (b <= a) ++ return false; ++ *start = a - pos; ++ *n = b - a; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-template.h b/drivers/gpu/apple-g5p/render-user-template.h +new file mode 100644 +index 000000000..92bd1604a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-template.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Generated by make-render-user-template.py from the hash-pinned r22 importer. */ ++#ifndef G5P_RENDER_USER_TEMPLATE_H ++#define G5P_RENDER_USER_TEMPLATE_H ++ ++/* Frozen dialect-1 W scalars {offset, value}: color-only, one sample, ++ * normal 3D, no depth/resource arrays. Typed and kernel fields excluded. ++ */ ++static const u32 g5p_ru_ta_fixed[][2] = { ++ { 0x020, 0x1e3ce508 }, { 0x024, 0x1e3ce508 }, { 0x3a4, 0x00000088 }, ++ { 0x3a8, 0x00000002 }, { 0x3c0, 0x00020000 }, { 0x514, 0x08002358 }, ++ { 0x528, 0x00000001 }, ++}; ++ ++static const u32 g5p_ru_3d_fixed[][2] = { ++ { 0x108, 0x000003c3 }, { 0x110, 0x000003c3 }, { 0x118, 0x000003c3 }, ++ { 0x120, 0x000003c3 }, { 0x128, 0x000003c3 }, { 0x130, 0x000003c3 }, ++ { 0x3a4, 0x3f800000 }, { 0x3a8, 0x00000300 }, { 0x3b0, 0x0000001f }, ++ { 0x3c0, 0x0023000f }, { 0x3c4, 0x00000408 }, { 0x3c8, 0x00000008 }, ++ { 0x3cc, 0x000000f8 }, { 0x6a8, 0x3f800000 }, { 0x6ac, 0x00000300 }, ++ { 0x6b0, 0x00000002 }, { 0x868, 0x08002358 }, ++}; ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-terminal.h b/drivers/gpu/apple-g5p/render-user-terminal.h +new file mode 100644 +index 000000000..4ced26eaf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-terminal.h +@@ -0,0 +1,103 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 terminal state of the epoch. Pure functions over one small struct; the ++ * kernel supplies READ_ONCE/WRITE_ONCE/cmpxchg and the host suite shims them, so the ++ * host compiles this file as it is. ++ * ++ * One latch, first writer wins. The first terminal event of the epoch (the natural TA ++ * complete, a timeout, a fault or a protocol error) fixes `kind`; from then on `quiet` ++ * is set and never cleared, and g5p_ru_mmio_ok() is false: the host touches no register ++ * (R0, R1, AIC, doorbell) and no diagnostic bank. RAM stays accessible. A later ++ * failure (the 3D after a completed TA, job N+1 after job N) cannot change `kind` but ++ * sets the sticky `failed`, which is what the admission uses: after any timeout or fault ++ * no new job starts, from any queue, even one that passed the pre-lock context checks ++ * before the failing job marked the context bad. ++ * ++ * The recovery series (patches/gpu-recovery) is independent of this file. Its ++ * g5p_td_note_terminal() takes {1 complete, 2 timeout}; g5p_ru_td_kind() maps ours. ++ */ ++#ifndef G5P_RENDER_USER_TERMINAL_H ++#define G5P_RENDER_USER_TERMINAL_H ++ ++enum g5p_ru_term_kind { ++ G5P_RU_TERM_NONE, ++ G5P_RU_TERM_COMPLETE, /* the natural TA completed (RAM) */ ++ G5P_RU_TERM_TIMEOUT, /* a phase ran out its deadline */ ++ G5P_RU_TERM_FAULT, /* protocol error, drain error, attach or publish failure */ ++ G5P_RU_TERM_KINDS, ++}; ++ ++static const char *const g5p_ru_term_names[G5P_RU_TERM_KINDS] = { ++ "none", ++ "complete", ++ "timeout", ++ "fault", ++}; ++ ++struct g5p_ru_terminal { ++ int kind; /* first writer wins */ ++ int quiet; /* set with the first note, never cleared */ ++ int failed; /* sticky: any timeout or fault was noted */ ++ int captured; /* the CAPTURE entry (recovery hook) was taken, once */ ++}; ++ ++/* A new firmware epoch (the in-boot recovery, recover.inc) starts a new latch; nothing else ++ * clears it. ++ */ ++static inline void g5p_ru_term_reset(struct g5p_ru_terminal *t) ++{ ++ WRITE_ONCE(t->kind, 0); ++ WRITE_ONCE(t->quiet, 0); ++ WRITE_ONCE(t->failed, 0); ++ WRITE_ONCE(t->captured, 0); ++} ++ ++/* The kind a job's return value stands for (only called for ret < 0, ret != -ECANCELED). */ ++static inline int g5p_ru_term_from_ret(int ret) ++{ ++ return ret == -ETIMEDOUT ? G5P_RU_TERM_TIMEOUT : G5P_RU_TERM_FAULT; ++} ++ ++/* Returns true for the first note of the epoch. */ ++static inline bool g5p_ru_term_note(struct g5p_ru_terminal *t, int kind) ++{ ++ bool first; ++ ++ if (kind <= G5P_RU_TERM_NONE || kind >= G5P_RU_TERM_KINDS) ++ kind = G5P_RU_TERM_FAULT; ++ first = cmpxchg(&t->kind, G5P_RU_TERM_NONE, kind) == G5P_RU_TERM_NONE; ++ if (kind >= G5P_RU_TERM_TIMEOUT) ++ WRITE_ONCE(t->failed, 1); ++ WRITE_ONCE(t->quiet, 1); ++ return first; ++} ++ ++static inline int g5p_ru_term_kind(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->kind); ++} ++ ++static inline bool g5p_ru_term_failed(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->failed); ++} ++ ++/* The single predicate for register access: false from the first terminal note on. */ ++static inline bool g5p_ru_mmio_ok(const struct g5p_ru_terminal *t) ++{ ++ return !READ_ONCE(t->quiet); ++} ++ ++/* True once per epoch for a timeout or fault: the caller may take the recovery CAPTURE entry. */ ++static inline bool g5p_ru_term_capture_due(struct g5p_ru_terminal *t, int kind) ++{ ++ return kind >= G5P_RU_TERM_TIMEOUT && cmpxchg(&t->captured, 0, 1) == 0; ++} ++ ++/* Recovery's terminal encoding: 1 complete, 2 timeout (a fault is a no-completion exit: 2). */ ++static inline int g5p_ru_td_kind(int kind) ++{ ++ return kind == G5P_RU_TERM_COMPLETE ? 1 : 2; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-touch.h b/drivers/gpu/apple-g5p/render-user-touch.h +new file mode 100644 +index 000000000..145bdbf2d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-touch.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 RAM-touch latency A/B (RC4a). Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. Include render-user-wait.h first. ++ * ++ * Question (docs/gpu/experiments/latency101-device-analysis.md section 9): is the host's bulk read ++ * of the receipt pages what releases a stalled phase (the +6 ms atom after the job-1 receipt)? ++ * ++ * render_user_touch_ms = T, default 0 = off. On, every phase of every job records the first wait ++ * sample at or after publish + T that left the phase undecided (still pending, deadline not ++ * reached). Odd sequences are the touch arm: right after that sample the host reads the ++ * receipt's read set once (RAM only, folded into a sum). Even sequences are the control arm: the ++ * same instant is recorded and nothing is read. Neither arm changes a sample, the sleep, the ++ * deadline, a latch or the result, and neither prints before the job ends. ++ * ++ * Off: g5p_ru_touch_start() leaves the record zero and g5p_ru_touch_due() is never true, so the ++ * wait loop runs the latency101 instructions unchanged. ++ */ ++#ifndef G5P_RENDER_USER_TOUCH_H ++#define G5P_RENDER_USER_TOUCH_H ++ ++#define G5P_RU_TOUCH_MAX_MS \ ++ 20000U /* never at or past the smallest soft bound */ ++ ++struct g5p_ru_touch { ++ u64 at_ns; /* publish + T; 0 = off for this phase */ ++ u64 seen_ns; /* the undecided sample at or after at_ns that took it; 0 = not reached */ ++ u64 done_ns; /* end of the read (touch arm), seen_ns (control arm) */ ++ u32 bytes; /* bytes read: 0 on the control arm */ ++ u32 sum; /* fold of the words read: the loads cannot be elided */ ++ bool sweep; /* this phase is in the touch arm */ ++}; ++ ++/* 0 (off), or 1..G5P_RU_TOUCH_MAX_MS with latency101 on (hard_ms != 0) and strictly before the ++ * effective soft bound (g5p_ru_lat_soft()), so the instant always falls in the plain wait. ++ */ ++static inline bool g5p_ru_touch_valid(u32 touch_ms, u32 soft_ms, u32 hard_ms) ++{ ++ if (!touch_ms) ++ return true; ++ return hard_ms && touch_ms <= G5P_RU_TOUCH_MAX_MS && touch_ms < soft_ms; ++} ++ ++/* Odd sequences are the touch arm, even ones the control. */ ++static inline bool g5p_ru_touch_arm(u32 seq) ++{ ++ return seq & 1; ++} ++ ++static inline void g5p_ru_touch_start(struct g5p_ru_touch *t, u64 start_ns, ++ u32 touch_ms, u32 seq) ++{ ++ *t = (struct g5p_ru_touch){ 0 }; ++ if (!touch_ms) ++ return; ++ t->at_ns = start_ns + (u64)touch_ms * 1000000ULL; ++ t->sweep = g5p_ru_touch_arm(seq); ++} ++ ++/* ++ * Asked after g5p_ru_wait_step() of the sample taken at now_ns: true once, for the first sample ++ * at ++ * or after the instant whose step left the phase undecided and not expired. A completion, a fault ++ * or the deadline in the same sample wins, and then the instant is never taken. ++ */ ++static inline bool g5p_ru_touch_due(const struct g5p_ru_touch *t, u64 now_ns, ++ u32 result, bool expired) ++{ ++ return t->at_ns && !t->seen_ns && !result && !expired && ++ now_ns >= t->at_ns; ++} ++ ++/* The instant was taken at seen_ns (a sample time, never 0); done_ns is when the read ended. */ ++static inline void g5p_ru_touch_mark(struct g5p_ru_touch *t, u64 seen_ns, ++ u64 done_ns, u32 bytes, u32 sum) ++{ ++ t->seen_ns = seen_ns; ++ t->done_ns = done_ns < seen_ns ? seen_ns : done_ns; ++ t->bytes = bytes; ++ t->sum = sum; ++} ++ ++/* Receipt helper: to - from in microseconds, -1 when an end is missing or the order is wrong. */ ++static inline s64 g5p_ru_touch_us(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000); ++} ++ ++/* Receipt: how long after the instant (the end of the read) the phase's deciding sample saw it ++ * complete; -1 when the instant was not taken or the phase did not complete. ++ */ ++static inline s64 g5p_ru_touch_after_us(const struct g5p_ru_touch *t, ++ u32 result, u64 end_ns) ++{ ++ if (!t->seen_ns || result != G5P_RU_WAIT_COMPLETE) ++ return -1; ++ return g5p_ru_touch_us(t->done_ns, end_ns); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wait.h b/drivers/gpu/apple-g5p/render-user-wait.h +new file mode 100644 +index 000000000..b9b5f1c49 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wait.h +@@ -0,0 +1,266 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial wait policy. Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. ++ * ++ * Evidence behind the numbers (r96 raw timelines, 2026-10-30): ++ * - the natural TA completion event arrived +12.0060 s and +12.0062 s after the ++ * publish in the two boots that a 12000 ms cap turned into timeouts; the whole ++ * delay is inside the firmware's start23, and once it returns the hardware and ++ * the retirement take less than ~1 ms; ++ * - nothing the host does shortens the delay, so the wait is RAM-only: after the ++ * DC3 attach the host touches no register until the job has ended. ++ * ++ * Policy: ++ * - one phase (TA, then 3D) waits at most `bound` from its own publish: 20 s by ++ * default, never below 20 s, never above the absolute cap; ++ * - the whole job (TA publish to the end of the 3D phase) never exceeds the ++ * absolute cap of 30 s; ++ * - the deadline is decided on RAM samples. A completion seen in the sample that ++ * reaches the deadline wins; a deadline is only final after one closing sample ++ * taken at or after it, which closes the gap between the last poll and the ++ * expiry; a completion after that is evidence only (`late_after`), never the ++ * result; ++ * - a fault wins over a completion in the same sample and over the deadline. ++ */ ++#ifndef G5P_RENDER_USER_WAIT_H ++#define G5P_RENDER_USER_WAIT_H ++ ++#define G5P_RU_V3_WAIT_MS 20000U /* default and minimum per-phase bound */ ++#define G5P_RU_V3_CAP_MS \ ++ 30000U /* absolute job bound, and the largest per-phase bound */ ++ ++/* ++ * latency101 (default off: render_user_soft_ms = render_user_hard_ms = 0 leaves every number above, ++ * and the behaviour of every function below, exactly as it was). When on, the per-phase deadline ++ * is the HARD bound and the job bound is two hard bounds; the SOFT bound marks a phase "slow" ++ * (RAM-only record, no print, no latch). A silent event stream means a dead firmware from the ++ * phase's publish on (faultiso; before it, only from the soft bound). See ++ * docs/gpu/experiments/prod98c-bimodal-latency.md and docs/gpu/experiments/faultiso.md. ++ */ ++#define G5P_RU_LAT_MIN_SOFT_MS \ ++ 20000U /* never below the proven per-phase bound */ ++#define G5P_RU_LAT_DEF_SOFT_MS 45000U ++#define G5P_RU_LAT_MAX_HARD_MS \ ++ 60000U /* largest hard bound; the job bound is twice this */ ++#define G5P_RU_LAT_DEF_OBS_MS \ ++ 2000U /* RAM-only look after a timeout, pre-receipt */ ++#define G5P_RU_LAT_MAX_OBS_MS 5000U ++#define G5P_RU_LAT_SILENT_MS \ ++ 1000U /* no event decoded for this long in a phase = silent */ ++/* The silence clock runs from the publish (the host suites key on this). */ ++#define G5P_RU_LAT_SILENT_FROM_PUBLISH 1 ++ ++enum g5p_ru_wait_result { ++ G5P_RU_WAIT_PENDING, ++ G5P_RU_WAIT_COMPLETE, ++ G5P_RU_WAIT_TIMEOUT, ++ G5P_RU_WAIT_FAULT, ++}; ++ ++struct g5p_ru_wait { ++ u64 start_ns; /* the publish (producer store) of this phase */ ++ u64 deadline_ns; /* start + bound, clamped to the job's absolute deadline */ ++ u64 end_ns; /* the sample that decided */ ++ u32 result; ++ bool expired; /* a sample at or after the deadline has been seen */ ++ bool late_gap; /* completion first seen by the closing sample */ ++ bool late_after; /* completion seen after a timeout was decided: evidence only */ ++ /* latency101; all zero (and never read) unless soft_ns != 0 */ ++ u64 soft_ns; /* start + soft bound; 0 = the grace is off */ ++ u64 soft_seen_ns; /* the first sample at or after soft_ns */ ++ u64 late_ns; /* the first retired sample after the timeout was decided */ ++ u64 event_ns; /* sample time of the last change of the decoded-event counter */ ++ u64 busy_ns; /* the first sample whose idle words showed the engine busy */ ++ u32 events; /* the decoded-event counter at that sample */ ++ bool slow; /* the phase passed its soft bound and is still pending */ ++ bool silent; /* the event stream stopped during the phase: firmware dead */ ++ bool late_soft; /* completed after the soft bound: a success, never a failure */ ++}; ++ ++/* latency101 parameters. Off is hard = soft = 0. On: hard in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000. The job bound the waiters must outlast is two ++ * hard bounds plus one observation window. ++ */ ++static inline u32 g5p_ru_lat_soft(u32 soft_ms, u32 hard_ms) ++{ ++ if (soft_ms) ++ return soft_ms; ++ return hard_ms > G5P_RU_LAT_DEF_SOFT_MS ? G5P_RU_LAT_DEF_SOFT_MS : ++ hard_ms; ++} ++ ++static inline bool g5p_ru_lat_valid(u32 soft_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!hard_ms) ++ return !soft_ms; ++ soft_ms = g5p_ru_lat_soft(soft_ms, hard_ms); ++ return hard_ms <= G5P_RU_LAT_MAX_HARD_MS && ++ soft_ms >= G5P_RU_LAT_MIN_SOFT_MS && soft_ms <= hard_ms && ++ obs_ms <= G5P_RU_LAT_MAX_OBS_MS; ++} ++ ++static inline u32 g5p_ru_lat_job_ms(u32 hard_ms, u32 obs_ms) ++{ ++ return 2 * hard_ms + obs_ms; ++} ++ ++/* A per-phase bound is valid when it is in [G5P_RU_V3_WAIT_MS, G5P_RU_V3_CAP_MS]. */ ++static inline bool g5p_ru_v3_bound_valid(u32 ms) ++{ ++ return ms >= G5P_RU_V3_WAIT_MS && ms <= G5P_RU_V3_CAP_MS; ++} ++ ++/* The deadline of a phase: its own bound, clamped to the job's absolute deadline. ++ * A job deadline of 0 means "no job deadline yet" (the first phase). ++ */ ++static inline u64 g5p_ru_wait_deadline(u64 start_ns, u32 bound_ms, ++ u64 job_deadline_ns) ++{ ++ u64 d = start_ns + (u64)bound_ms * 1000000ULL; ++ ++ return job_deadline_ns && job_deadline_ns < d ? job_deadline_ns : d; ++} ++ ++static inline void g5p_ru_wait_start(struct g5p_ru_wait *w, u64 start_ns, ++ u32 bound_ms, u64 job_deadline_ns) ++{ ++ *w = (struct g5p_ru_wait){ ++ .start_ns = start_ns, ++ .deadline_ns = g5p_ru_wait_deadline(start_ns, bound_ms, ++ job_deadline_ns), ++ }; ++} ++ ++/* ++ * latency101: hard_ms is the per-phase deadline (clamped to the job's deadline as above), ++ * soft_ms ++ * (never past the deadline) the point where a still-pending phase is called slow. `events` seeds ++ * the silence clock. ++ */ ++static inline void g5p_ru_wait_start_lat(struct g5p_ru_wait *w, u64 start_ns, ++ u32 soft_ms, u32 hard_ms, ++ u64 job_deadline_ns, u32 events) ++{ ++ g5p_ru_wait_start(w, start_ns, hard_ms, job_deadline_ns); ++ w->soft_ns = start_ns + (u64)soft_ms * 1000000ULL; ++ if (w->soft_ns > w->deadline_ns) ++ w->soft_ns = w->deadline_ns; ++ w->events = events; ++ w->event_ns = start_ns; ++} ++ ++/* The first sample whose idle words (G+0x724..) show the engine busy, i.e. the firmware's ++ * start path has run. Recorded once; it splits a late start (busy_ns close to the ++ * completion) from slow hardware (busy_ns early). A busy window shorter than the poll ++ * period can be missed: 0 = not seen. ++ */ ++static inline void g5p_ru_wait_busy(struct g5p_ru_wait *w, u64 now_ns, ++ bool busy) ++{ ++ if (busy && !w->busy_ns) ++ w->busy_ns = now_ns; ++} ++ ++/* One look at the decoded-event counter, taken with every RAM sample, before g5p_ru_wait_step(). ++ * The ALIVE stream runs at about 290 events/s whatever the work does (r96c: 29442 polls, no gap ++ * over 3.4 ms; latency101: 287 events/s through every stall of 0.55-33 s), so a stream that has ++ * been silent for G5P_RU_LAT_SILENT_MS is a dead firmware, not a slow one, at any point of the ++ * phase. Returns true then; the caller reports it as a fault. The silence clock starts at the ++ * publish (vpds rung vk, 2026-11-01: the counter froze 1 s after the TA's publish, and a clock ++ * that only ran from the soft bound called it at 45 s, with every other client held behind it). ++ * The caller drains the ring before each sample, so a host that slept longer than the window ++ * sees the events it missed and is not silent. Never true when latency101 is off. ++ */ ++static inline bool g5p_ru_wait_grace(struct g5p_ru_wait *w, u64 now_ns, ++ u32 events) ++{ ++ if (!w->soft_ns) ++ return false; ++ if (events != w->events) { ++ w->events = events; ++ w->event_ns = now_ns; ++ } ++ if (now_ns >= w->soft_ns && !w->soft_seen_ns) ++ w->soft_seen_ns = now_ns; ++ w->silent = now_ns - w->event_ns >= ++ (u64)G5P_RU_LAT_SILENT_MS * 1000000ULL; ++ return w->silent; ++} ++ ++/* One RAM sample taken at `now_ns`. Returns the result so far (PENDING until ++ * decided). A sample at or after the deadline only marks the wait expired unless ++ * it shows completion or a fault; the caller must then take one closing sample and ++ * call g5p_ru_wait_close(). ++ */ ++static inline u32 g5p_ru_wait_step(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (w->soft_ns && now_ns >= w->soft_ns) ++ w->slow = true; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ w->end_ns = now_ns; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->end_ns = now_ns; ++ w->late_soft = w->slow; ++ } else if (now_ns >= w->deadline_ns) { ++ w->expired = true; ++ } ++ return w->result; ++} ++ ++/* The closing sample, taken after the loop left because of the deadline. This is ++ * the only place a timeout is decided. ++ */ ++static inline u32 g5p_ru_wait_close(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->late_gap = true; ++ w->late_soft = w->slow; ++ } else { ++ w->result = G5P_RU_WAIT_TIMEOUT; ++ } ++ w->end_ns = now_ns; ++ return w->result; ++} ++ ++/* A sample after the result was decided: a completion after a timeout is recorded ++ * and nothing else happens (no 3D, no recovery, no register access). ++ */ ++static inline void g5p_ru_wait_after(struct g5p_ru_wait *w, bool complete) ++{ ++ if (w->result == G5P_RU_WAIT_TIMEOUT && complete) ++ w->late_after = true; ++} ++ ++/* latency101: the RAM-only observation window after a timeout was decided, before the receipt is ++ * printed and the terminal latch is taken. g5p_ru_wait_after() looks once, microseconds after the ++ * decision; a completion that arrives milliseconds later (prod98c seq 3: 5.49 ms) was never ++ * recorded, so `late_after=0/0` was a false negative. Returns true while the window is open. A ++ * retired sample ends it and is recorded in late_after/late_ns; it never changes the result (a ++ * deadline stays final). obs_ms = 0 closes it at once (the legacy behaviour). ++ */ ++static inline bool g5p_ru_wait_observe(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, u32 obs_ms) ++{ ++ if (w->result != G5P_RU_WAIT_TIMEOUT || w->late_after) ++ return false; ++ if (complete) { ++ w->late_after = true; ++ w->late_ns = now_ns; ++ return false; ++ } ++ return now_ns < w->end_ns + (u64)obs_ms * 1000000ULL; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +new file mode 100644 +index 000000000..64a51eaf4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -0,0 +1,273 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 publish-time MTS wake (wake110). Pure functions over plain values; the one ++ * register sequence goes through a callback table (the fender.h pattern), so the host ++ * suite compiles this file as it is and records every access in order. ++ * ++ * Question (docs/gpu/experiments/wake110-apple-mts-dispatch.md): is the 0.5-33 s TA/3D ++ * start latency caused by the missing Apple submit kick? Apple's TA and 3D submit ++ * wrappers (iPad6,11 kernelcache 0xfffffff006ca00a0 / 0xfffffff006ca01f8) do, every time: ++ * ring entry, dmb ish, Fender vt+0x80 (R1+0x14 = 7, read back), Fender vt+0x18 ++ * (R1+0x1000 & 0xf), then R0+0xb00 = 0x10 (TA, DM0|COUNTED) or 0x11 (3D, DM1|COUNTED). ++ * The MTS store is reached from every arm; when the GPC nibble is 0 Apple first runs a ++ * power arm (up to forcePowerCycleAndReboot). Linux V3 publishes job N >= 2's TA and ++ * every 3D with no kick at all. ++ * ++ * render_user_wake (latched per context): 0 off (default: no call has any effect), ++ * 1 the A/B, 2 every eligible phase is a factor (the candidate production fix, for use ++ * only after the A/B proved it). The TA of job 1 is never a phase of this file (the r74g ++ * grace and its fenced fallback bell own it). In the A/B each engine's phases are paired ++ * in publish order, (0,1), (2,3), ...; exactly one of each pair is the factor, which one ++ * is fixed by render_user_wake_seed before the context exists, and bit 31 of the seed ++ * inverts the whole schedule (the mirror stage). The verdict recomputes the arm of every ++ * receipt from the seed. ++ * ++ * At the publish, right after the producer store and its barrier, both arms take the same ++ * lock and read the same RAM state and run the same guards; the control stops there (no ++ * register). The factor then runs the Apple order: R1+0x14 = 7, read back (any bit ++ * outside 7: stop), R1+0x1000 (state & 0x1f != 1, the GPC is not active: stop, never ++ * repaired, never a power arm), dma_wmb(), R0+0xb00 = 0x10 | engine. At most one attempt ++ * per phase. Never after a timeout or fault (the terminal latch's sticky `failed`), never ++ * with the boot failed or stopped, never unpowered, never with an r74f bell in flight, ++ * never once the phase shows any sign of retirement, never on a ring that is not exactly ++ * this publish. A consumer already at the target does not cancel (Apple never checks ++ * it); it is recorded. The natural TA completion (terminal kind COMPLETE, the epoch's ++ * first note) is not a failure: the factor's sequence is the one licensed exception to the ++ * r96 "no register after the TA complete" envelope, default off. ++ */ ++#ifndef G5P_RENDER_USER_WAKE_H ++#define G5P_RENDER_USER_WAKE_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++#define G5P_WK_OFF 0U ++#define G5P_WK_AB 1U ++#define G5P_WK_ALWAYS 2U ++#define G5P_WK_MIRROR 0x80000000U /* seed bit 31: every arm inverted */ ++ ++#define G5P_WK_R1_FENDER 0x14U /* Fender vt+0x80: write 7, read back */ ++#define G5P_WK_R1_GPC 0x1000U /* GPC state: & 0x1f == 1 is active */ ++#define G5P_WK_R0_MTS 0xb00U /* MTS_SCHEDULE, write only */ ++#define G5P_WK_MTS_TA 0x10U /* DM0 | COUNTED */ ++#define G5P_WK_MTS_3D 0x11U /* DM1 | COUNTED */ ++ ++/* wake111: the publish-time power evidence, one bit per RAM fact, sampled under the event lock ++ * and printed per phase (power=TT/DD). Every required bit must be set. reset_transition is ++ * the permanent post-release slave-port lockout (set right before the release, never ++ * cleared), not a live reset state: Wake110 vetoed on it, so every phase skipped. It is ++ * recorded (SLAVE_LOCK) and is neither required nor a veto; the slave-port ban is unchanged. ++ */ ++#define G5P_WK_PWR_CORE 0x01U /* g->powered */ ++#define G5P_WK_PWR_FENDER 0x02U /* g->fender_ready */ ++#define G5P_WK_PWR_RELEASED 0x04U /* b->released */ ++#define G5P_WK_PWR_ALIVE 0x08U /* b->alive */ ++#define G5P_WK_PWR_CONTROL 0x10U /* b->control_complete */ ++#define G5P_WK_PWR_DS108 \ ++ 0x20U /* DS108 claim proof, current generation, no power abort */ ++#define G5P_WK_PWR_SLAVE_LOCK 0x40U /* g->reset_transition: evidence only */ ++#define G5P_WK_PWR_REQUIRED \ ++ (G5P_WK_PWR_CORE | G5P_WK_PWR_FENDER | G5P_WK_PWR_RELEASED | \ ++ G5P_WK_PWR_ALIVE | G5P_WK_PWR_CONTROL | G5P_WK_PWR_DS108) ++ ++enum g5p_wk_arm { ++ G5P_WK_ARM_NONE, /* wake off */ ++ G5P_WK_ARM_EXCLUDED, /* the TA of job 1 (r74g) */ ++ G5P_WK_ARM_CONTROL, ++ G5P_WK_ARM_FACTOR, ++ G5P_WK_ARMS, ++}; ++ ++enum g5p_wk_did { ++ G5P_WK_DID_NONE, /* off, or never reached */ ++ G5P_WK_DID_RANG, /* factor: fence, GPC active, 0xb00 written */ ++ G5P_WK_DID_SHAM, /* control: every guard passed, no register */ ++ G5P_WK_DID_EXCLUDED, ++ G5P_WK_SKIP_TWICE, /* this phase was already decided */ ++ G5P_WK_SKIP_LICENSE, /* a timeout or fault was noted (terminal failed) */ ++ G5P_WK_SKIP_FAULT, /* boot failed, event failed or stopped */ ++ G5P_WK_SKIP_POWER, /* one of G5P_WK_PWR_REQUIRED is absent */ ++ G5P_WK_SKIP_R74F, /* an r74f bell is published, open or firing */ ++ G5P_WK_SKIP_RING, /* the ring is not exactly this publish */ ++ G5P_WK_SKIP_RETIRED, /* a stamp == seq or a completion event since the publish */ ++ G5P_WK_SKIP_FENCE, /* R1+0x14 read back outside 7: no further register */ ++ G5P_WK_SKIP_GPC, /* GPC not active: no 0xb00 */ ++ G5P_WK_DIDS, ++}; ++ ++static const char *const g5p_wk_arm_names[G5P_WK_ARMS] = { ++ "none", ++ "excluded", ++ "control", ++ "factor", ++}; ++ ++static const char *const g5p_wk_did_names[G5P_WK_DIDS] = { ++ "none", "rang", "sham", "excluded", "skip-twice", ++ "skip-license", "skip-fault", "skip-power", "skip-r74f", "skip-ring", ++ "skip-retired", "skip-fence", "skip-gpc", ++}; ++ ++/* Everything the decision reads, sampled under the event lock right after the producer store. */ ++struct g5p_wk_in { ++ u32 mode, seed, seq, e; ++ bool attempted; /* this phase already went through g5p_wk_decide() */ ++ bool term_failed; /* g5p_ru_term_failed() */ ++ bool boot_failed; /* b->failed || b->event_failed || b->stopped */ ++ u32 power; /* g5p_wk_power_mask(); the slave lock is evidence, not a veto */ ++ bool r74f_pending; /* an r74f bell in PUBLISHED, OPEN or FIRING */ ++ u32 start, target, prod, ++ cons; /* ring: producer before this publish, our target, now */ ++ u32 stamp[2]; /* this engine's stamp words */ ++ u32 type1; /* this engine's completion events decoded since the phase armed */ ++}; ++ ++/* The register access of the factor, in order. The kernel binds them to writel/readl on the ++ * Fender window and the RGX window and to dma_wmb(); the host suite records them. ++ */ ++struct g5p_wk_io { ++ void *ctx; ++ void (*w1)(void *ctx, u32 reg, u32 value); ++ u32 (*r1)(void *ctx, u32 reg); ++ void (*wbarrier)(void *ctx); ++ void (*w0)(void *ctx, u32 reg, u32 value); ++}; ++ ++static inline bool g5p_wk_mode_valid(u32 mode) ++{ ++ return mode <= G5P_WK_ALWAYS; ++} ++ ++static inline u32 g5p_wk_mix(u32 x) ++{ ++ x ^= x >> 16; ++ x *= 0x7feb352dU; ++ x ^= x >> 15; ++ x *= 0x846ca68bU; ++ x ^= x >> 16; ++ return x; ++} ++ ++/* The phase's index among its engine's eligible phases: TA of job seq >= 2 -> seq - 2, ++ * 3D of job seq >= 1 -> seq - 1. ++ */ ++static inline u32 g5p_wk_index(u32 seq, u32 e) ++{ ++ return e ? seq - 1 : seq - 2; ++} ++ ++static inline u32 g5p_wk_arm(u32 mode, u32 seed, u32 seq, u32 e) ++{ ++ u32 i, flip; ++ ++ if (!mode || !g5p_wk_mode_valid(mode) || !seq || e > 1) ++ return G5P_WK_ARM_NONE; ++ if (!e && seq == 1) ++ return G5P_WK_ARM_EXCLUDED; ++ if (mode == G5P_WK_ALWAYS) ++ return G5P_WK_ARM_FACTOR; ++ i = g5p_wk_index(seq, e); ++ flip = g5p_wk_mix(g5p_wk_mix(seed & ~G5P_WK_MIRROR) + (e << 24) + ++ (i >> 1)) & ++ 1; ++ flip ^= !!(seed & G5P_WK_MIRROR); ++ return ((i & 1) ^ flip) ? G5P_WK_ARM_FACTOR : G5P_WK_ARM_CONTROL; ++} ++ ++static inline u32 g5p_wk_mts(u32 e) ++{ ++ return G5P_WK_MTS_TA | e; ++} ++ ++/* wake111: the evidence word, one bit per fact (the caller samples each fact once). */ ++static inline u32 g5p_wk_power_mask(bool powered, bool fender_ready, ++ bool released, bool alive, ++ bool control_complete, bool ds108, ++ bool slave_lock) ++{ ++ return (powered ? G5P_WK_PWR_CORE : 0) | ++ (fender_ready ? G5P_WK_PWR_FENDER : 0) | ++ (released ? G5P_WK_PWR_RELEASED : 0) | ++ (alive ? G5P_WK_PWR_ALIVE : 0) | ++ (control_complete ? G5P_WK_PWR_CONTROL : 0) | ++ (ds108 ? G5P_WK_PWR_DS108 : 0) | ++ (slave_lock ? G5P_WK_PWR_SLAVE_LOCK : 0); ++} ++ ++static inline bool g5p_wk_power_ready(u32 power) ++{ ++ return (power & G5P_WK_PWR_REQUIRED) == G5P_WK_PWR_REQUIRED; ++} ++ ++/* The RAM guards, common to both arms, in priority order. 0 = pass. */ ++static inline u32 g5p_wk_guard(const struct g5p_wk_in *in) ++{ ++ if (in->attempted) ++ return G5P_WK_SKIP_TWICE; ++ if (in->term_failed) ++ return G5P_WK_SKIP_LICENSE; ++ if (in->boot_failed) ++ return G5P_WK_SKIP_FAULT; ++ if (!g5p_wk_power_ready(in->power)) ++ return G5P_WK_SKIP_POWER; ++ if (in->r74f_pending) ++ return G5P_WK_SKIP_R74F; ++ if (in->start > 255 || in->target > 255 || in->prod != in->target || ++ in->cons > 255 || ((in->target - in->start) & 255) != 1 || ++ ((in->cons - in->start) & 255) > 1) ++ return G5P_WK_SKIP_RING; ++ if (in->stamp[0] == in->seq || in->stamp[1] == in->seq || in->type1) ++ return G5P_WK_SKIP_RETIRED; ++ return 0; ++} ++ ++/* The factor's register sequence (Apple's order). The caller set *fence and *gpc to U32_MAX ++ * (not read). ++ */ ++static inline u32 g5p_wk_ring(const struct g5p_wk_io *io, u32 e, u32 *fence, ++ u32 *gpc) ++{ ++ io->w1(io->ctx, G5P_WK_R1_FENDER, 7); ++ *fence = io->r1(io->ctx, G5P_WK_R1_FENDER); ++ if (*fence & ~7U) ++ return G5P_WK_SKIP_FENCE; ++ *gpc = io->r1(io->ctx, G5P_WK_R1_GPC); ++ if (!G5P_GPC_ON(*gpc)) ++ return G5P_WK_SKIP_GPC; ++ /* the publish's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ io->wbarrier(io->ctx); ++ io->w0(io->ctx, G5P_WK_R0_MTS, g5p_wk_mts(e)); ++ return G5P_WK_DID_RANG; ++} ++ ++/* One phase's decision and, for a factor that passed every guard, its register sequence ++ * (the control arm never touches io). ++ */ ++static inline u32 g5p_wk_decide(const struct g5p_wk_in *in, ++ const struct g5p_wk_io *io, u32 *fence, ++ u32 *gpc) ++{ ++ u32 arm = g5p_wk_arm(in->mode, in->seed, in->seq, in->e), skip; ++ ++ *fence = U32_MAX; ++ *gpc = U32_MAX; ++ if (arm == G5P_WK_ARM_NONE) ++ return G5P_WK_DID_NONE; ++ if (arm == G5P_WK_ARM_EXCLUDED) ++ return G5P_WK_DID_EXCLUDED; ++ skip = g5p_wk_guard(in); ++ if (skip) ++ return skip; ++ if (arm == G5P_WK_ARM_CONTROL) ++ return G5P_WK_DID_SHAM; ++ return g5p_wk_ring(io, in->e, fence, gpc); ++} ++ ++/* A decided phase never decides again (the caller stores this before it unlocks). */ ++static inline bool g5p_wk_attempted(u32 did) ++{ ++ return did != G5P_WK_DID_NONE && did != G5P_WK_DID_EXCLUDED; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-window.h b/drivers/gpu/apple-g5p/render-user-window.h +new file mode 100644 +index 000000000..bfbae628d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-window.h +@@ -0,0 +1,74 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The firmware window of a render VM (docs/gpu/multi-client-design.md §6 R4, stage 3). ++ * ++ * The firmware's own GPU memory lives in one GiB, catalogue entry G5P_RU_FIRMWARE_CI: its ++ * image, the root and shared pages (rings, the PA/tag arrays, W), the MMU table alias and the ++ * PIO pages (Fender, AIC). Stage 1 copied that catalogue entry into every render VM, so any ++ * render job could address all of it. A render VM now gets, under that entry, a directory ++ * the kernel built that maps only the pages the GPU's own requesters reach through the ++ * context's slot: the PM control page (S) and the PM list and free list (F). Their leaf ++ * entries are copied from the firmware's tables, so their translation is unchanged. ++ * ++ * render_user_fw_window = 0 (G5P_RU_FW_FULL) the stage 1 mapping: the whole firmware GiB; ++ * render contexts need CAP_SYS_RAWIO ++ * 1 (G5P_RU_FW_WINDOW) the window above (default) ++ * 2 (G5P_RU_FW_NONE) no firmware page at all (an experiment) ++ * ++ * Pure helpers, host-tested (scripts/gpu/multiclient2-host/test_render_window.c). The ++ * caller provides g5p_le32() and the boot layout constants. ++ */ ++#ifndef G5P_RENDER_USER_WINDOW_H ++#define G5P_RENDER_USER_WINDOW_H ++ ++#define G5P_RU_FW_FULL 0U ++#define G5P_RU_FW_WINDOW 1U ++#define G5P_RU_FW_NONE 2U ++#define G5P_RU_FW_MODES 3U ++ ++#define G5P_RU_FW_PAGES 3U ++/* The pages of the window, all in the firmware GiB. */ ++static const u64 g5p_ru_fw_window_va[G5P_RU_FW_PAGES] = { ++ G5P_SHARED_VA + G5P_R22_PM, /* PM control and S */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE, /* PM list */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, /* PM free list F */ ++}; ++ ++static inline u32 g5p_ru_fw_di(u64 va) ++{ ++ return (va >> 21) & 511; ++} ++ ++static inline u32 g5p_ru_fw_ti(u64 va) ++{ ++ return (va >> 14) & 127; ++} ++ ++/* The firmware's 64-bit leaf entry for `va`, from its leaf pages and their directory ++ * indices; 0 when the firmware does not map it. ++ */ ++static inline u64 g5p_ru_fw_leaf(u8 *const *leaf, const u32 *leaf_di, ++ u32 leaves, u64 va) ++{ ++ for (u32 i = 0; i < leaves; i++) { ++ const u8 *e = leaf[i] + g5p_ru_fw_ti(va) * 8; ++ ++ if (leaf_di[i] == g5p_ru_fw_di(va)) ++ return (u64)g5p_le32(e + 4) << 32 | g5p_le32(e); ++ } ++ return 0; ++} ++ ++/* The window's leaf page index for `va` among `n` distinct directory indices so far, or n ++ * (a new one) when none has its directory index. ++ */ ++static inline u32 g5p_ru_fw_slot(const u32 *di, u32 n, u64 va) ++{ ++ u32 i; ++ ++ for (i = 0; i < n && di[i] != g5p_ru_fw_di(va); i++) ++ ; ++ return i; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-variants-r25.h b/drivers/gpu/apple-g5p/render-variants-r25.h +new file mode 100644 +index 000000000..8469a2f4b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-variants-r25.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_VARIANTS_R25_H ++#define G5P_RENDER_VARIANTS_R25_H ++ ++/* Finite, owned-input experiments. No raw W, register or firmware-patch API. */ ++#define G5P_R25_MAX_BLOCKS 128U ++#define G5P_R25_HEAD 0x3000U ++ ++struct g5p_r25_variants { ++ u32 pm, blocks, context, host, sku, memory, rt, slots; ++}; ++ ++static inline bool g5p_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return v->pm <= 3 && ++ (v->blocks == 0 || v->blocks == 64 || v->blocks == 128) && ++ v->context <= 3 && v->host <= 2 && v->sku <= 3 && ++ v->memory <= 4 && v->rt <= 2 && v->slots <= 1; ++} ++ ++/* Event synthesis, removed context tasks, candidate clocks and the low-address ++ * dialect test cannot qualify rendering, even if firmware stamps retire. ++ */ ++static inline bool g5p_r25_diagnostic(const struct g5p_r25_variants *v) ++{ ++ return v->sku || v->context || v->host || v->rt == 2; ++} ++ ++/* PM payload frames are physical PPNs and have no user PTE attributes. This ++ * changes only PM auxiliary mappings (and, in mode4, other private mappings). ++ * Firmware/shared META mappings and the borrowed sink are never rewritten. ++ */ ++static inline u32 g5p_r25_attributes(const struct g5p_r25_variants *v, u64 va) ++{ ++ if (!v->memory) ++ return 0xc; ++ if (v->memory != 4 && ++ (va < G5P_R22_AUX || va >= G5P_R22_AUX + 5 * G5P_GPU_PAGE)) ++ return 0xc; ++ return v->memory == 2 ? 4 : v->memory == 3 ? 8 : 0; ++} ++ ++/* Preserve the original constructor exactly for the default pool. The extra ++ * blocks fit the existing 16KiB list and free allocations, including F's last ++ * dword at128 blocks. Validate every PA before mutating either allocation. ++ */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/retire108.h b/drivers/gpu/apple-g5p/retire108.h +new file mode 100644 +index 000000000..e944b92c9 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/retire108.h +@@ -0,0 +1,434 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RETIRE108_H ++#define G5P_RETIRE108_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P outgoing retire layer (series gpu-recovery108, patch 0006). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 3. ++ * ++ * Wraps the recovery98 teardown ladder (teardown.h, g5p_td_run()) without ++ * changing it: the TD98 lines of a run are byte for byte the ones of the ++ * same run without this layer. It adds, around a run: ++ * ++ * ADMIT atomically close job admission (one cmpxchg word), then a ++ * bounded RAM-only wait for submitters already inside the ++ * admission window; still inside at the deadline = explicit ++ * ADMIT_OPEN flag, never a block. ++ * CLASSIFY the epoch class from RAM inputs only (struct g5p_r8_in). ++ * RECEIPT one TR108 line with the class and the generation, printed ++ * before the ladder makes its first AIC or R1 access. ++ * (ladder) g5p_td_run(FINAL) through a callback, unchanged. ++ * SLOTS three named post-ladder slots. Each runs only with a glue op ++ * present AND its own parameter on; every skip is named: ++ * fw-sleep validated firmware sleep before stable-off (G2: ++ * no recovered command; the glue has no op) ++ * stable-off one R1 read of isStableOff after a verified ++ * Fender stop leg (G6: default off) ++ * pmgr-gate outgoing PMGR power gate (G1: not implemented, ++ * there is no op and no parameter) ++ * RETIRE the retired latch: from here on g5p_r8_mmio_ok() refuses R0, ++ * R1 and every write except the always-on AIC ack of a late ++ * handler entry. The latch is set whatever the verdict. ++ * ++ * Generations: gen counts real runs (CAPTURE and FINAL) since boot and is ++ * monotonic; a REENTRY (a second FINAL after retire, e.g. notifier then ++ * .shutdown then remove) prints a line but does not advance it. ++ * ++ * Pure policy, like teardown.h: no register access, no kernel API except ++ * cmpxchg(), READ_ONCE()/WRITE_ONCE(), scnprintf() and the errno names. The ++ * same file is compiled by the kernel glue (teardown.inc) and by the offline ++ * tests (patches/gpu-recovery108/tests). Needs teardown.h first. ++ */ ++ ++#define G5P_R8_CLOSED 0x80000000u /* admission word: closed bit */ ++#define G5P_R8_INSIDE 0x7fffffffu /* admission word: submitters inside */ ++#define G5P_R8_POLL_US 100 ++#define G5P_R8_MAX_STEPS 512 /* ADMIT wait cap, independent of the clock */ ++#define G5P_R8_ADMIT_US 20000 /* ADMIT wait budget */ ++ ++/* Admission: one word, closed bit + count of submitters inside the window. */ ++struct g5p_r8_admit { ++ u32 word; ++}; ++ ++/* A submitter enters before its first check of driver state and leaves on ++ * every return path. False once admission is closed: the submit must fail ++ * with -ESHUTDOWN and touch nothing. ++ */ ++static inline bool g5p_r8_admit_enter(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if ((cur & G5P_R8_CLOSED) || ++ (cur & G5P_R8_INSIDE) == G5P_R8_INSIDE) ++ return false; ++ old = cmpxchg(&a->word, cur, cur + 1); ++ if (old == cur) ++ return true; ++ cur = old; ++ } ++} ++ ++static inline void g5p_r8_admit_exit(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if (!(cur & G5P_R8_INSIDE)) ++ return; /* unbalanced exit: never wrap into the closed bit */ ++ old = cmpxchg(&a->word, cur, cur - 1); ++ if (old == cur) ++ return; ++ cur = old; ++ } ++} ++ ++/* Close (idempotent). Returns the submitters inside at the instant of closing. */ ++static inline u32 g5p_r8_admit_close(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ old = cmpxchg(&a->word, cur, cur | G5P_R8_CLOSED); ++ if (old == cur) ++ return cur & G5P_R8_INSIDE; ++ cur = old; ++ } ++} ++ ++static inline u32 g5p_r8_admit_inside(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_INSIDE; ++} ++ ++static inline bool g5p_r8_admit_closed(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_CLOSED; ++} ++ ++/* Epoch classes. The dirty ones are never reported as clean (H10). */ ++enum g5p_r8_class { ++ G5P_R8C_NEVER_BOOTED, /* shared pages never mapped */ ++ G5P_R8C_UNRELEASED, /* mapped, firmware never released */ ++ G5P_R8C_IDLE, /* released and ALIVE, no context ever claimed (not-run) */ ++ G5P_R8C_CLEAN, /* claimed, every job signalled, no holder left */ ++ G5P_R8C_HELD, /* clean, but a process still holds the claimed context */ ++ G5P_R8C_PARKED, /* GPC=0 already verified by a park */ ++ G5P_R8C_IN_FLIGHT, /* a job accepted and not signalled, or a worker inside a wait */ ++ G5P_R8C_TIMEOUT, /* terminal latch = timeout (a late completion stays a timeout) */ ++ G5P_R8C_FAULT, /* failed, event failure or a fault terminal */ ++ G5P_R8C_ABORTED, /* bootstrap power abort latched (the r87b/C3 epoch) */ ++ G5P_R8C_UNKNOWN, /* released, never ALIVE; or an unverified earlier stop */ ++ G5P_R8C_N, ++}; ++ ++static const char *const g5p_r8_class_name[G5P_R8C_N] = { ++ "never-booted", "unreleased", "idle", "clean", "held", "parked", ++ "in-flight", "timeout", "fault", "aborted", "unknown", ++}; ++ ++static inline bool g5p_r8_class_dirty(enum g5p_r8_class c) ++{ ++ return c >= G5P_R8C_IN_FLIGHT; ++} ++ ++/* RAM-only inputs, read by the glue before the run (never from a register). */ ++struct g5p_r8_in { ++ bool booted; /* b->maps_ready */ ++ bool released; /* b->released */ ++ bool alive; /* b->alive before FREEZE, or ALIVE was proven this boot */ ++ bool failed; /* b->failed */ ++ bool fault_event; /* b->event_failed */ ++ int terminal; /* teardown terminal latch: 0 none, 1 complete, 2 timeout/fault */ ++ bool claimed; /* a context claimed the device (DRM quarantined_context) */ ++ bool held; /* ... and a render node file still references it */ ++ u32 queued; /* accepted render jobs not yet signalled */ ++ bool worker_busy; /* the command lock was busy: a worker may be inside a wait */ ++ bool parked; /* b->parked */ ++ bool stopped_unparked; /* b->stopped && !b->parked before this run */ ++ bool power_abort; /* r73b early_power_abort */ ++}; ++ ++static inline u32 g5p_r8_in_bits(const struct g5p_r8_in *i) ++{ ++ return i->booted | i->released << 1 | i->alive << 2 | i->failed << 3 | ++ i->fault_event << 4 | i->claimed << 5 | i->held << 6 | ++ (i->queued != 0) << 7 | i->worker_busy << 8 | i->parked << 9 | ++ i->stopped_unparked << 10 | i->power_abort << 11; ++} ++ ++/* First match wins; the order is the precedence (docs section 3.2). */ ++static inline enum g5p_r8_class g5p_r8_classify(const struct g5p_r8_in *i) ++{ ++ if (!i->booted) ++ return G5P_R8C_NEVER_BOOTED; ++ if (!i->released) ++ return G5P_R8C_UNRELEASED; ++ if (i->stopped_unparked) ++ return G5P_R8C_UNKNOWN; ++ if (i->power_abort) ++ return G5P_R8C_ABORTED; ++ if (i->failed || i->fault_event) ++ return G5P_R8C_FAULT; ++ if (i->terminal == 2) ++ return G5P_R8C_TIMEOUT; ++ if (i->queued || i->worker_busy) ++ return G5P_R8C_IN_FLIGHT; ++ if (i->parked) ++ return G5P_R8C_PARKED; ++ if (!i->alive) ++ return G5P_R8C_UNKNOWN; ++ if (i->claimed) ++ return i->held ? G5P_R8C_HELD : G5P_R8C_CLEAN; ++ return G5P_R8C_IDLE; ++} ++ ++enum g5p_r8_slot { ++ G5P_R8S_FW_SLEEP, ++ G5P_R8S_STABLE_OFF, ++ G5P_R8S_PMGR_GATE, ++ G5P_R8S_N, ++}; ++ ++enum g5p_r8_skip { ++ G5P_R8K_RAN, /* the slot ran (see its result) */ ++ G5P_R8K_NO_MECHANISM, /* the glue has no op for it */ ++ G5P_R8K_OFF, /* its parameter is off (default) */ ++ G5P_R8K_NOT_FINAL, /* CAPTURE never runs a slot */ ++ G5P_R8K_NEEDS_RT1, /* needs a verified Fender stop leg in this run */ ++ G5P_R8K_DIRTY, /* the ladder ended dirty: no register access */ ++ G5P_R8K_N, ++}; ++ ++static const char *const g5p_r8_skip_name[G5P_R8K_N] = { ++ "ran", "no-mechanism", "off", "not-final", "needs-rt1", "dirty", ++}; ++ ++#define G5P_R8F_ADMIT_OPEN \ ++ BIT(0) /* submitters still inside at the ADMIT deadline */ ++#define G5P_R8F_ADMIT_LATE BIT(1) /* ... or the step cap ended the wait */ ++#define G5P_R8F_REENTRY BIT(2) /* answered after retire: nothing ran */ ++#define G5P_R8F_BUSY BIT(3) /* a concurrent entrant was inside */ ++#define G5P_R8F_SLOT_FAIL BIT(4) /* a slot ran and failed */ ++#define G5P_R8F_STABLE_OFF BIT(5) /* stable-off slot read isStableOff == 1 */ ++/* jobs still queued at DETACH (abandoned to their worker) */ ++#define G5P_R8F_DETACH_OPEN BIT(6) ++ ++/* ++ * Ops. td_run runs the recovery98 ladder and returns its rc; td_flags/td_verdict ++ * read its result. slot[] may hold NULL (no mechanism). Only slot ops touch a ++ * register, and only after the ladder, never after RETIRE. ++ */ ++struct g5p_r8_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); ++ int (*td_run)(void *ctx, enum g5p_td_mode mode); ++ u32 (*td_flags)(void *ctx); ++ enum g5p_td_verdict (*td_verdict)(void *ctx); ++ int (*slot[G5P_R8S_N])(void *ctx, u32 *value); ++ void (*emit)(void *ctx, const char *line); ++}; ++ ++struct g5p_r8 { ++ struct g5p_r8_admit admit; ++ u32 admit_us; /* ADMIT budget */ ++ bool slot_on[G5P_R8S_N]; /* the slots' parameters (default all off) */ ++ /* latches */ ++ int running; ++ int retired; /* 0, or the generation that retired the device */ ++ u32 gen; /* monotonic: real runs since boot */ ++ /* last run */ ++ enum g5p_td_mode mode; ++ enum g5p_r8_class cls; ++ u32 in_bits, flags, inside0, inside1, steps, detach_queued; ++ enum g5p_r8_skip skip[G5P_R8S_N]; ++ int slot_rc[G5P_R8S_N]; ++ u32 slot_val[G5P_R8S_N]; ++ int rc; ++ enum g5p_td_verdict verdict; ++ u32 lines; /* TR108 lines emitted since boot */ ++}; ++ ++static inline void g5p_r8_init(struct g5p_r8 *s) ++{ ++ *s = (struct g5p_r8){ .admit_us = G5P_R8_ADMIT_US }; ++} ++ ++/* ++ * The predicate for every register access outside the ladder: after RETIRE only ++ * RAM, a PMGR read and the AIC (the always-on swirq ack of a late handler entry) ++ * remain. R0 and R1 are refused forever, whatever the verdict. ++ */ ++static inline bool g5p_r8_mmio_ok(const struct g5p_r8 *s, enum g5p_td_dom d) ++{ ++ if (d == G5P_TDD_RAM || d == G5P_TDD_PMGR || d == G5P_TDD_AIC) ++ return true; ++ return !READ_ONCE(s->retired); ++} ++ ++static inline bool g5p_r8_retired(const struct g5p_r8 *s) ++{ ++ return READ_ONCE(s->retired) != 0; ++} ++ ++/* The DRM frontend's DETACH callback reports what it left to the job workers. */ ++static inline void g5p_r8_note_detach(struct g5p_r8 *s, u32 queued) ++{ ++ WRITE_ONCE(s->detach_queued, queued); ++ if (queued) ++ s->flags |= G5P_R8F_DETACH_OPEN; ++} ++ ++static inline void g5p_r8_line(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ const char *line) ++{ ++ s->lines++; ++ if (o->emit) ++ o->emit(o->ctx, line); ++} ++ ++/* Bounded RAM-only wait for submitters inside the admission window. */ ++static inline void g5p_r8_admit_wait(struct g5p_r8 *s, ++ const struct g5p_r8_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)s->admit_us * 1000; ++ ++ s->inside0 = g5p_r8_admit_close(&s->admit); ++ s->inside1 = s->inside0; ++ s->steps = 0; ++ while (s->inside1) { ++ if (s->steps >= G5P_R8_MAX_STEPS) { ++ s->flags |= G5P_R8F_ADMIT_LATE; ++ break; ++ } ++ s->steps++; ++ if (o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_R8_POLL_US); ++ s->inside1 = g5p_r8_admit_inside(&s->admit); ++ } ++ if (s->inside1) ++ s->flags |= G5P_R8F_ADMIT_OPEN; ++} ++ ++static inline void g5p_r8_slots(struct g5p_r8 *s, const struct g5p_r8_ops *o) ++{ ++ u32 tdf = o->td_flags ? o->td_flags(o->ctx) : 0; ++ unsigned int i; ++ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ s->slot_rc[i] = 0; ++ s->slot_val[i] = 0; ++ if (s->mode != G5P_TD_FINAL) ++ s->skip[i] = G5P_R8K_NOT_FINAL; ++ else if (!o->slot[i]) ++ s->skip[i] = G5P_R8K_NO_MECHANISM; ++ else if (!s->slot_on[i]) ++ s->skip[i] = G5P_R8K_OFF; ++ else if (s->verdict == G5P_TDV_DIRTY) ++ s->skip[i] = G5P_R8K_DIRTY; ++ else if (!(tdf & G5P_TDF_RT1_OK)) ++ s->skip[i] = G5P_R8K_NEEDS_RT1; ++ else ++ s->skip[i] = G5P_R8K_RAN; ++ } ++ /* One attempt each, in slot order, no retry; a failed slot ends the slots. */ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ if (s->skip[i] != G5P_R8K_RAN) ++ continue; ++ if (s->flags & G5P_R8F_SLOT_FAIL) { ++ s->skip[i] = G5P_R8K_DIRTY; ++ continue; ++ } ++ s->slot_rc[i] = o->slot[i](o->ctx, &s->slot_val[i]); ++ if (s->slot_rc[i]) ++ s->flags |= G5P_R8F_SLOT_FAIL; ++ else if (i == G5P_R8S_STABLE_OFF && s->slot_val[i] == 1) ++ s->flags |= G5P_R8F_STABLE_OFF; ++ } ++} ++ ++/* ++ * One run. mode is the teardown mode the glue chose (teardown=1 CAPTURE, ++ * teardown=2 FINAL). Returns the ladder's rc, -EBUSY for a concurrent entrant, ++ * and the cached rc after retire. Lines: ++ * TR108 gen= phase=classify why= mode= class= dirty= in= inside= after= flags= admit_us= ++ * (TD98 lines of the ladder, unchanged) ++ * TR108 gen= phase=retired|captured verdict= class= flags= fw-sleep= stable-off= pmgr-gate= ++ * so= detach_queued= rc= ++ * TR108 gen= phase=reentry why= retired_gen= rc= (after retire; no gen step) ++ */ ++static inline int g5p_r8_run(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ enum g5p_td_mode mode, const struct g5p_r8_in *in, ++ const char *why) ++{ ++ char line[256]; ++ u64 t0; ++ int rc; ++ ++ if (cmpxchg(&s->running, 0, 1)) { ++ /* A concurrent entrant touches nothing; it does not reach the ladder either. */ ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=busy why=%s mode=%s rc=%d", ++ READ_ONCE(s->gen), why, g5p_td_mode_name[mode], ++ -EBUSY); ++ if (o->emit) ++ o->emit(o->ctx, line); ++ return -EBUSY; ++ } ++ if (READ_ONCE(s->retired)) { ++ /* The ladder answers from its cache too (REENTRY); nothing else runs. */ ++ rc = o->td_run(o->ctx, mode); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=reentry why=%s mode=%s retired_gen=%d rc=%d", ++ s->gen, why, g5p_td_mode_name[mode], ++ READ_ONCE(s->retired), rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++ } ++ s->gen++; ++ s->mode = mode; ++ s->flags = 0; ++ s->inside0 = 0; ++ s->inside1 = 0; ++ s->steps = 0; ++ WRITE_ONCE(s->detach_queued, 0); ++ t0 = o->now_ns(o->ctx); ++ if (mode == G5P_TD_FINAL) ++ g5p_r8_admit_wait(s, o); ++ s->cls = g5p_r8_classify(in); ++ s->in_bits = g5p_r8_in_bits(in); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=classify why=%s mode=%s class=%s dirty=%u in=%03x term=%d queued=%u inside=%u after=%u flags=%02x admit_us=%u", ++ s->gen, why, g5p_td_mode_name[mode], g5p_r8_class_name[s->cls], ++ g5p_r8_class_dirty(s->cls), s->in_bits, in->terminal, ++ in->queued, s->inside0, s->inside1, s->flags, ++ (u32)((o->now_ns(o->ctx) - t0) / 1000)); ++ /* The receipt is out before the ladder's first AIC or R1 access. */ ++ g5p_r8_line(s, o, line); ++ s->rc = o->td_run(o->ctx, mode); ++ s->verdict = o->td_verdict ? o->td_verdict(o->ctx) : G5P_TDV_NONE; ++ g5p_r8_slots(s, o); ++ if (mode == G5P_TD_FINAL) ++ WRITE_ONCE(s->retired, (int)s->gen); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=%s verdict=%s class=%s flags=%02x fw-sleep=%s stable-off=%s pmgr-gate=%s so=%u detach_queued=%u rc=%d", ++ s->gen, mode == G5P_TD_FINAL ? "retired" : "captured", ++ g5p_td_verdict_name[s->verdict], g5p_r8_class_name[s->cls], ++ s->flags, g5p_r8_skip_name[s->skip[G5P_R8S_FW_SLEEP]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_STABLE_OFF]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_PMGR_GATE]], ++ s->slot_val[G5P_R8S_STABLE_OFF], READ_ONCE(s->detach_queued), ++ s->rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.h b/drivers/gpu/apple-g5p/teardown.h +new file mode 100644 +index 000000000..b536f428c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.h +@@ -0,0 +1,740 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_TEARDOWN_H ++#define G5P_TEARDOWN_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P GPU teardown state machine (series gpu-recovery98). ++ * ++ * Design and evidence: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md. ++ * ++ * Pure policy: no register access, no kernel API except cmpxchg(), scnprintf() ++ * and the errno names. Every hardware or firmware-visible access goes through ++ * struct g5p_td_ops, and every call is checked against g5p_td_allow[phase][domain] ++ * before it is made. The same file is compiled by the kernel glue (teardown.inc) ++ * and by the offline fault-injection tests (patches/gpu-recovery/tests). ++ * ++ * Two modes: ++ * CAPTURE timeout / fault / attach failure, and (gpu-recovery99) any number of looks at a ++ * live epoch. Observation only: reads RAM rings, ++ * changes nothing the host could still want to export. Never writes ++ * driver or firmware-shared state, never masks the IRQ, never touches ++ * a register. Arms the quiet latch for diagnostics (g5p_td_mmio_ok()). ++ * FINAL shutdown / reboot notifier / remove. Runs the whole ladder and leaves ++ * a known state for the next kernel. ++ * ++ * Domains: RAM (DMA pages + driver structs), AIC (swirq 0x20e104000 ack and the ++ * IRQ-chip mask), R1 (Fender wrapper window), R0 (core register window), PMGR ++ * (0x20e080340 read-only). R0 has no op at all and is forbidden in every phase. ++ * PMGR is read-only in every phase. R1 is reachable only in the RESET phase. ++ * ++ * Nothing here frees or reuses memory the GPU might still access. ++ */ ++ ++#define G5P_TD_NCMD 4 /* command rings: shared objects 0x60 0x64 0x68 0x6c */ ++#define G5P_TD_NRING (G5P_TD_NCMD + 1) /* + event ring (object 0x70) */ ++#define G5P_TD_MAX_RCPT 12 ++#define G5P_TD_MAX_STEPS 4096 /* hard stop independent of the clock */ ++#define G5P_TD_POLL_US 100 ++#define G5P_TD_EV_SLACK \ ++ 16 /* events produced during the phase above this = storm */ ++ ++enum g5p_td_mode { ++ G5P_TD_CAPTURE = 1, ++ G5P_TD_FINAL = 2, ++}; ++ ++enum g5p_td_phase { ++ G5P_TDP_FREEZE, /* stopped=1 alive=0, watchdog disarmed (RAM) */ ++ G5P_TDP_ACCOUNT, /* RAM-only ring/event accounting */ ++ G5P_TDP_DRAIN, /* bounded RAM-only wait for the firmware to self-drain */ ++ G5P_TDP_EVENTS, /* bounded RAM-only event-ring quiesce */ ++ G5P_TDP_IRQ, /* AIC ack, IRQ mask, settle */ ++ G5P_TDP_DETACH, /* host detach, RAM only */ ++ G5P_TDP_RESET, /* RT1 Fender stop leg when eligible; PMGR read */ ++ G5P_TDP_DONE, ++ G5P_TDP_N, ++}; ++ ++enum g5p_td_dom { ++ G5P_TDD_RAM, ++ G5P_TDD_AIC, ++ G5P_TDD_R1, ++ G5P_TDD_R0, ++ G5P_TDD_PMGR, ++ G5P_TDD_N, ++}; ++ ++#define G5P_TDA_R 1 ++#define G5P_TDA_W 2 ++#define G5P_TDA_RW 3 ++ ++/* Hard stop boundaries. A phase may only use the domains marked here. */ ++static const unsigned char g5p_td_allow[G5P_TDP_N][G5P_TDD_N] = { ++ /* RAM AIC R1 R0 PMGR */ ++ [G5P_TDP_FREEZE] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_ACCOUNT] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_DRAIN] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_EVENTS] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_IRQ] = { G5P_TDA_R, G5P_TDA_W, 0, 0, 0 }, ++ [G5P_TDP_DETACH] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_RESET] = { G5P_TDA_R, 0, G5P_TDA_RW, 0, G5P_TDA_R }, ++ [G5P_TDP_DONE] = { G5P_TDA_R, 0, 0, 0, 0 }, ++}; ++ ++#define G5P_TDF_NOT_BOOTED \ ++ BIT(0) /* shared pages not mapped: nothing to tear down */ ++#define G5P_TDF_UNRELEASED BIT(1) /* firmware never released */ ++#define G5P_TDF_PROTOCOL \ ++ BIT(2) /* ring index > 255, read error, producer moved */ ++#define G5P_TDF_TERMINAL BIT(3) /* TA complete/timeout latched before entry */ ++#define G5P_TDF_LATE_TERMINAL BIT(4) /* ... or during the teardown (race) */ ++#define G5P_TDF_ABANDON BIT(5) /* >=1 ring pending after the drain */ ++#define G5P_TDF_SELF_DRAINED \ ++ BIT(6) /* >=1 ring consumed by the firmware, no bell */ ++#define G5P_TDF_FAULT_EVENT \ ++ BIT(7) /* event type other than 0/1 left unconsumed */ ++#define G5P_TDF_EV_UNQUIET \ ++ BIT(8) /* event ring still non-empty after the budget */ ++#define G5P_TDF_IRQ_UNSETTLED BIT(9) /* IRQ count still moving after the mask */ ++#define G5P_TDF_RT1_OK BIT(10) /* Fender stop leg verified GPC=0 */ ++#define G5P_TDF_RT1_FAIL BIT(11) /* stop leg attempted, not verified */ ++#define G5P_TDF_RT1_SKIPPED BIT(12) /* no register access; see skip reason */ ++#define G5P_TDF_OVERRUN BIT(13) /* step or time budget exhausted */ ++#define G5P_TDF_VIOLATION BIT(14) /* a gate refused an access (machine bug) */ ++#define G5P_TDF_CAPTURE_ONLY BIT(15) /* CAPTURE run: nothing was changed */ ++#define G5P_TDF_REENTRY BIT(16) /* second entry answered from the cache */ ++ ++enum g5p_td_skip { ++ G5P_TDS_NONE, ++ G5P_TDS_NOT_BOOTED, ++ G5P_TDS_NOT_RELEASED, ++ G5P_TDS_NOT_POWERED, ++ G5P_TDS_ALREADY_PARKED, /* clean: GPC=0 already verified by park */ ++ G5P_TDS_STOPPED_UNPARKED, /* dirty: an earlier stop leg did not verify */ ++ G5P_TDS_POWER_ABORT, ++ G5P_TDS_FW_POLICY, /* meta+0x6c8 != 0: firmware also writes R1+0x1000 */ ++ G5P_TDS_LOCK_BUSY, ++ G5P_TDS_CAPTURE_MODE, ++ G5P_TDS_BAD_STATE, /* protocol/overrun/unsettled IRQ: no register access */ ++ G5P_TDS_N, ++}; ++ ++enum g5p_td_verdict { ++ G5P_TDV_NONE, ++ G5P_TDV_OBSERVED, /* CAPTURE: nothing changed */ ++ G5P_TDV_UNTOUCHED, /* never booted or never released: nothing to do */ ++ G5P_TDV_QUIESCED, /* rings clear, events clear, IRQ masked, GPC=0 verified */ ++ G5P_TDV_ABANDONED, /* as QUIESCED, but >=1 command abandoned after the drain */ ++ G5P_TDV_RAM_ONLY, /* IRQ masked, no register access: GPU left as found */ ++ G5P_TDV_DIRTY, /* a step failed: warm continue forbidden, cold DFU */ ++ G5P_TDV_N, ++}; ++ ++static const char *const g5p_td_phase_name[G5P_TDP_N] = { ++ "freeze", "account", "drain", "events", ++ "irq", "detach", "reset", "done", ++}; ++ ++static const char *const g5p_td_mode_name[3] = { "none", "capture", "final" }; ++ ++static const char *const g5p_td_verdict_name[G5P_TDV_N] = { ++ "none", "observed", "untouched", "quiesced", ++ "abandoned", "ram-only", "dirty", ++}; ++ ++struct g5p_td_budget { ++ u32 drain_us; /* self-drain wait; DC3 took 2.34 ms (r87b) */ ++ u32 events_us; ++ u32 settle_us; /* IRQ count must stay constant this long after the mask */ ++ u32 events_pass_max; /* entries per pass; the ring has 256 slots */ ++ u32 events_passes; ++ u32 total_us; /* RESET is skipped when the RAM phases already overran this */ ++}; ++ ++#define G5P_TD_BUDGET_DEFAULT { 20000, 20000, 2000, 256, 4, 100000 } ++ ++/* Inputs the glue reads from RAM before the run (never from a register). */ ++struct g5p_td_pre { ++ bool booted; /* maps_ready: shared pages exist */ ++ bool released; /* firmware was released */ ++ bool powered; /* ps_gfx held on by the retained PM reference */ ++ bool fender_ready; ++ bool already_parked; /* b->parked: GPC=0 verified by park */ ++ bool stopped_unparked; /* b->stopped && !b->parked: an earlier stop did not verify */ ++ bool policy_host_owned; /* init meta+0x6c8 == 0 */ ++ bool power_abort; /* r73b early_power_abort latched */ ++ bool lock_held; /* the glue holds g->lock: no worker is inside a hardware wait */ ++}; ++ ++/* ++ * Every op is bounded by its own contract; the machine additionally bounds how ++ * often and for how long it calls them. Ops marked RAM never touch a register. ++ */ ++struct g5p_td_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); /* sleep, no register access */ ++ void (*freeze)(void *ctx); /* RAM: stopped=1 alive=0, disarm */ ++ /* RAM: idx 0..NCMD-1 command rings, NCMD = event ring. -ENODEV: not mapped. */ ++ int (*ring_read)(void *ctx, u32 idx, u32 *cons, u32 *prod); ++ /* RAM: consume at most max entries of type 0/1, stop at the first other type. */ ++ int (*event_pass)(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault); ++ bool (*terminal_seen)(void *ctx); /* RAM: TA complete/timeout latch */ ++ void (*detach)(void *ctx); /* RAM: signal/forget host jobs */ ++ void (*aic_ack)(void *ctx); /* AIC: swirq+0x94 = 0x4000 */ ++ void (*irq_mask)(void *ctx); /* AIC: ack stays, line masked */ ++ u32 (*irq_total)(void *ctx); /* RAM: handler entry counter */ ++ int (*fender_stop)(void *ctx); /* R1: g5p_fender_idle() */ ++ int (*pmgr_actual)(void *ctx, u32 *target, ++ u32 *actual); /* PMGR read; may be NULL */ ++ void (*emit)(void *ctx, const char *line); /* printk; may be NULL */ ++}; ++ ++struct g5p_td_receipt { ++ unsigned char seq, phase, mode, verdict; ++ s32 rc; ++ u32 us, steps, flags; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [dom][0]=reads [dom][1]=writes, op level */ ++ u32 a, b; /* phase specific, see g5p_td_run() */ ++}; ++ ++struct g5p_td { ++ struct g5p_td_budget budget; ++ /* latches shared with the render path; touched by cmpxchg only */ ++ int running; /* 0 idle, 1 inside g5p_td_run() */ ++ int terminal; /* 0, 1 complete, 2 timeout: first writer wins */ ++ int quiet; /* diagnostics MMIO forbidden from here on */ ++ /* 0 idle, 1 captured (informational), 2 finalized: only 2 answers from the cache */ ++ int state; ++ u32 snaps; /* CAPTURE runs since init: the n= of the TD99 snap line */ ++ /* per run */ ++ enum g5p_td_mode mode; ++ enum g5p_td_phase phase; ++ enum g5p_td_skip skip; ++ enum g5p_td_verdict verdict; ++ bool eligible; ++ u32 flags, steps, violations, seq, nrcpt; ++ u64 t0_ns; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [enum g5p_td_dom][0=reads,1=writes], op level */ ++ u32 pending_mask, pending_initial, abandoned, drained; ++ u32 ring_cons[G5P_TD_NRING], ring_prod[G5P_TD_NRING]; ++ u32 ev_consumed, ev_left, ev_initial, irq_before, irq_after; ++ u32 us_total; ++ u32 pmgr_target, pmgr_actual; ++ int rc; ++ struct g5p_td_receipt rcpt[G5P_TD_MAX_RCPT]; ++}; ++ ++static inline void g5p_td_init(struct g5p_td *t) ++{ ++ const struct g5p_td_budget b = G5P_TD_BUDGET_DEFAULT; ++ ++ *t = (struct g5p_td){ .budget = b }; ++} ++ ++/* One latch for "the TA reached a terminal state"; the first writer wins. */ ++static inline bool g5p_td_note_terminal(struct g5p_td *t, int kind) ++{ ++ if (cmpxchg(&t->terminal, 0, kind) != 0) ++ return false; ++ WRITE_ONCE(t->quiet, 1); ++ return true; ++} ++ ++/* ++ * The single predicate for diagnostic register access (R0 snapshots, end banks, ++ * exports): false once the epoch is terminal or a teardown has started. Nothing ++ * in this series calls a diagnostic; this exists so the render path can. ++ */ ++ ++/* RT1 (Fender stop leg) eligibility. Pure function of RAM-derived inputs. */ ++static inline bool g5p_td_eligible(const struct g5p_td_pre *p, ++ enum g5p_td_mode mode, enum g5p_td_skip *why) ++{ ++ enum g5p_td_skip s = G5P_TDS_NONE; ++ ++ if (!p->booted) ++ s = G5P_TDS_NOT_BOOTED; ++ else if (!p->released) ++ s = G5P_TDS_NOT_RELEASED; ++ else if (!p->powered || !p->fender_ready) ++ s = G5P_TDS_NOT_POWERED; ++ else if (p->already_parked) ++ s = G5P_TDS_ALREADY_PARKED; ++ else if (p->stopped_unparked) ++ s = G5P_TDS_STOPPED_UNPARKED; ++ else if (p->power_abort) ++ s = G5P_TDS_POWER_ABORT; ++ else if (!p->policy_host_owned) ++ s = G5P_TDS_FW_POLICY; ++ else if (!p->lock_held) ++ s = G5P_TDS_LOCK_BUSY; ++ else if (mode != G5P_TD_FINAL) ++ s = G5P_TDS_CAPTURE_MODE; ++ if (why) ++ *why = s; ++ return s == G5P_TDS_NONE; ++} ++ ++static inline int g5p_td_format(const struct g5p_td_receipt *r, char *buf, ++ size_t n) ++{ ++ return scnprintf(buf, n, ++ "TD98 seq=%u phase=%s mode=%s rc=%d us=%u steps=%u flags=%08x aic=%u/%u r1=%u/%u r0=%u/%u pmgr=%u/%u a=%u b=%u verdict=%s", ++ r->seq, g5p_td_phase_name[r->phase], g5p_td_mode_name[r->mode], ++ r->rc, r->us, r->steps, r->flags, r->cnt[G5P_TDD_AIC][0], ++ r->cnt[G5P_TDD_AIC][1], r->cnt[G5P_TDD_R1][0], ++ r->cnt[G5P_TDD_R1][1], r->cnt[G5P_TDD_R0][0], ++ r->cnt[G5P_TDD_R0][1], r->cnt[G5P_TDD_PMGR][0], ++ r->cnt[G5P_TDD_PMGR][1], r->a, r->b, ++ g5p_td_verdict_name[r->verdict]); ++} ++ ++static inline u32 g5p_td_us_since(const struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ u64 d = o->now_ns(o->ctx) - t->t0_ns; ++ ++ return d > 0xffffffffULL * 1000 ? 0xffffffffu : (u32)(d / 1000); ++} ++ ++static inline void g5p_td_receipt(struct g5p_td *t, const struct g5p_td_ops *o, ++ int rc, u32 a, u32 b) ++{ ++ struct g5p_td_receipt *r = ++ &t->rcpt[t->nrcpt < G5P_TD_MAX_RCPT ? t->nrcpt : ++ G5P_TD_MAX_RCPT - 1]; ++ char line[224]; ++ unsigned int d; ++ ++ r->seq = (unsigned char)t->seq++; ++ r->phase = t->phase; ++ r->mode = t->mode; ++ r->verdict = t->verdict; ++ r->rc = rc; ++ r->us = g5p_td_us_since(t, o); ++ r->steps = t->steps; ++ r->flags = t->flags; ++ for (d = 0; d < G5P_TDD_N; d++) { ++ r->cnt[d][0] = t->cnt[d][0]; ++ r->cnt[d][1] = t->cnt[d][1]; ++ } ++ r->a = a; ++ r->b = b; ++ if (t->nrcpt < G5P_TD_MAX_RCPT) ++ t->nrcpt++; ++ if (o->emit) { ++ g5p_td_format(r, line, sizeof(line)); ++ o->emit(o->ctx, line); ++ } ++} ++ ++/* Count and check one access. A refused access is a bug in the machine itself. */ ++static inline int g5p_td_gate(struct g5p_td *t, enum g5p_td_dom d, ++ unsigned int acc) ++{ ++ if ((g5p_td_allow[t->phase][d] & acc) != acc) { ++ t->violations++; ++ t->flags |= G5P_TDF_VIOLATION; ++ return -EACCES; ++ } ++ if ((acc & G5P_TDA_R) && t->cnt[d][0] < 255) ++ t->cnt[d][0]++; ++ if ((acc & G5P_TDA_W) && t->cnt[d][1] < 255) ++ t->cnt[d][1]++; ++ return 0; ++} ++ ++/* One loop iteration or hardware op. False once the hard step cap is hit. */ ++static inline bool g5p_td_step(struct g5p_td *t) ++{ ++ if (t->steps >= G5P_TD_MAX_STEPS) { ++ t->flags |= G5P_TDF_OVERRUN; ++ return false; ++ } ++ t->steps++; ++ return true; ++} ++ ++static inline bool g5p_td_ring_ok(u32 cons, u32 prod) ++{ ++ return cons <= 255 && prod <= 255; ++} ++ ++static inline void g5p_td_note_late(struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ if (o->terminal_seen && o->terminal_seen(o->ctx) && ++ !(t->flags & G5P_TDF_TERMINAL)) ++ t->flags |= G5P_TDF_LATE_TERMINAL; ++} ++ ++static inline int g5p_td_account(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 i, bad = 0, pending = 0, cons, prod; ++ int rc; ++ ++ for (i = 0; i < G5P_TD_NRING; i++) { ++ if (!g5p_td_step(t)) ++ return -ETIMEDOUT; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return -EACCES; ++ rc = o->ring_read(o->ctx, i, &cons, &prod); ++ if (rc == -ENODEV) { ++ t->flags |= G5P_TDF_NOT_BOOTED; ++ break; ++ } ++ if (rc || !g5p_td_ring_ok(cons, prod)) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ bad++; ++ continue; ++ } ++ t->ring_cons[i] = cons; ++ t->ring_prod[i] = prod; ++ if (i < G5P_TD_NCMD && cons != prod) { ++ t->pending_mask |= 1u << i; ++ pending++; ++ } ++ } ++ t->pending_initial = pending; ++ t->ev_left = (t->ring_prod[G5P_TD_NCMD] - t->ring_cons[G5P_TD_NCMD]) & ++ 255; ++ t->ev_initial = t->ev_left; ++ g5p_td_receipt(t, o, bad ? -EPROTO : 0, pending, bad); ++ return 0; ++} ++ ++/* ++ * gpu-recovery99 0004: one extra line per CAPTURE run, after the ACCOUNT receipt, so a live ++ * epoch can be looked at ++ * several times (S1a). RAM only: it formats what ACCOUNT stored in struct g5p_td and reads the ++ * handler entry counter ++ * (RAM) and the clock; no register, no state change. The prefix is not TD98 on purpose: receipt ++ * parsers keyed on ++ * "TD98 seq=" / "TD98 begin" ignore it. ++ * n CAPTURE runs since init (1 based) ++ * t_ns ops->now_ns(), for the IRQ rate between two lines ++ * irq handler entry counter (ops->irq_total), 0 if there is no such op ++ * term terminal latch (0 none, 1 complete, 2 timeout); quiet = the diagnostics latch ++ * own bit0 booted 1 released 2 powered 3 fender_ready 4 already_parked 5 stopped_unparked ++ * 6 policy_host_owned 7 power_abort 8 lock_held ++ * cN/pN command ring N consumer / producer; ce/pe the event ring. Zero when the shared pages ++ * are not mapped. ++ */ ++static inline void g5p_td_snap(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ char line[224]; ++ u32 own, irq = 0; ++ ++ t->snaps++; ++ if (!o->emit) ++ return; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R) && o->irq_total) ++ irq = o->irq_total(o->ctx); ++ own = pre->booted | pre->released << 1 | pre->powered << 2 | ++ pre->fender_ready << 3 | pre->already_parked << 4 | ++ pre->stopped_unparked << 5 | pre->policy_host_owned << 6 | ++ pre->power_abort << 7 | pre->lock_held << 8; ++ scnprintf(line, sizeof(line), ++ "TD99 snap n=%u t_ns=%llu irq=%u term=%d quiet=%d own=%03x c0=%u p0=%u c1=%u p1=%u c2=%u p2=%u c3=%u p3=%u ce=%u pe=%u", ++ t->snaps, (unsigned long long)o->now_ns(o->ctx), irq, ++ READ_ONCE(t->terminal), READ_ONCE(t->quiet), own, ++ t->ring_cons[0], t->ring_prod[0], t->ring_cons[1], ++ t->ring_prod[1], t->ring_cons[2], t->ring_prod[2], ++ t->ring_cons[3], t->ring_prod[3], t->ring_cons[G5P_TD_NCMD], ++ t->ring_prod[G5P_TD_NCMD]); ++ o->emit(o->ctx, line); ++} ++ ++static inline void g5p_td_drain(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.drain_us * 1000; ++ u32 i, cons, prod; ++ ++ while (t->pending_mask) { ++ if (!g5p_td_step(t)) ++ break; ++ if (o->now_ns(o->ctx) >= deadline) { ++ t->flags |= G5P_TDF_ABANDON; ++ break; ++ } ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ for (i = 0; i < G5P_TD_NCMD; i++) { ++ if (!(t->pending_mask & (1u << i))) ++ continue; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return; ++ if (o->ring_read(o->ctx, i, &cons, &prod) || ++ !g5p_td_ring_ok(cons, prod) || ++ prod != t->ring_prod[i]) { ++ /* The producer is host-written and the host is frozen. */ ++ t->flags |= G5P_TDF_PROTOCOL; ++ t->pending_mask &= ~(1u << i); ++ continue; ++ } ++ if (cons == prod) { ++ t->pending_mask &= ~(1u << i); ++ t->flags |= G5P_TDF_SELF_DRAINED; ++ } ++ } ++ } ++ for (i = 0; i < G5P_TD_NCMD; i++) ++ if (t->pending_mask & (1u << i)) ++ t->abandoned++; ++ if (t->abandoned) ++ t->flags |= G5P_TDF_ABANDON; ++ t->drained = t->pending_initial - t->abandoned; ++} ++ ++static inline void g5p_td_events(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.events_us * 1000; ++ u32 pass, consumed, left = t->ev_left; ++ bool fault = false; ++ int rc; ++ ++ for (pass = 0; pass < t->budget.events_passes && left; pass++) { ++ if (!g5p_td_step(t)) ++ break; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_RW)) ++ return; ++ consumed = 0; ++ rc = o->event_pass(o->ctx, t->budget.events_pass_max, &consumed, ++ &left, &fault); ++ t->ev_consumed += consumed; ++ if (rc) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ break; ++ } ++ if (fault) { ++ /* Retained for the host; stop consuming, the IRQ is masked next. */ ++ t->flags |= G5P_TDF_FAULT_EVENT; ++ break; ++ } ++ if (!left || o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ } ++ t->ev_left = left; ++ /* ++ * A ring holds at most 255 entries, so consuming more than the backlog plus a ++ * few ALIVE ticks means the producer outran us: an empty ring at the last ++ * look is a snapshot, not a quiesce. ++ */ ++ if (!fault && (left || (s32)(t->ev_consumed + left - t->ev_initial) > ++ G5P_TD_EV_SLACK)) ++ t->flags |= G5P_TDF_EV_UNQUIET; ++} ++ ++static inline void g5p_td_irq(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 tries, n0, n1; ++ ++ if (g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W) || ++ g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W)) ++ return; ++ t->irq_before = o->irq_total(o->ctx); ++ /* Same pair as the IRQ-cap path: ack the software source, then mask the line. */ ++ o->aic_ack(o->ctx); ++ o->irq_mask(o->ctx); ++ for (tries = 0; tries < 3; tries++) { ++ if (!g5p_td_step(t)) ++ break; ++ n0 = o->irq_total(o->ctx); ++ o->relax_us(o->ctx, t->budget.settle_us); ++ n1 = o->irq_total(o->ctx); ++ t->irq_after = n1; ++ if (n0 == n1) ++ return; ++ } ++ t->flags |= G5P_TDF_IRQ_UNSETTLED; ++} ++ ++static inline void g5p_td_reset(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ enum g5p_td_skip why = G5P_TDS_NONE; ++ bool bad = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_VIOLATION); ++ int rc = 0; ++ ++ t->eligible = g5p_td_eligible(pre, t->mode, &why); ++ if (t->eligible && ++ (bad || g5p_td_us_since(t, o) > t->budget.total_us)) { ++ t->eligible = false; ++ why = G5P_TDS_BAD_STATE; ++ } ++ t->skip = why; ++ if (!t->eligible) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ } else if (!g5p_td_step(t) || g5p_td_gate(t, G5P_TDD_R1, G5P_TDA_RW)) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ t->skip = G5P_TDS_BAD_STATE; ++ } else { ++ /* ++ * Apple stopFirmwareForPowerDown 6c9fa10, R1 only; flown here after a DC ++ * timeout (round 10). Never retried, even on failure. ++ */ ++ rc = o->fender_stop(o->ctx); ++ t->flags |= rc ? G5P_TDF_RT1_FAIL : G5P_TDF_RT1_OK; ++ } ++ if (o->pmgr_actual && !g5p_td_gate(t, G5P_TDD_PMGR, G5P_TDA_R)) ++ o->pmgr_actual(o->ctx, &t->pmgr_target, &t->pmgr_actual); ++ g5p_td_receipt(t, o, rc, t->eligible ? 0 : (u32)t->skip, ++ t->pmgr_actual); ++} ++ ++static inline void g5p_td_decide(struct g5p_td *t, const struct g5p_td_pre *pre) ++{ ++ bool dirty = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_EV_UNQUIET | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_RT1_FAIL | ++ G5P_TDF_OVERRUN | G5P_TDF_VIOLATION) || ++ t->skip == G5P_TDS_STOPPED_UNPARKED || ++ t->skip == G5P_TDS_BAD_STATE; ++ ++ if (t->mode == G5P_TD_CAPTURE) ++ t->verdict = (t->flags & G5P_TDF_VIOLATION) ? G5P_TDV_DIRTY : ++ G5P_TDV_OBSERVED; ++ else if (dirty) ++ t->verdict = G5P_TDV_DIRTY; ++ else if (t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED)) ++ t->verdict = G5P_TDV_UNTOUCHED; ++ else if (t->flags & G5P_TDF_RT1_OK || t->skip == G5P_TDS_ALREADY_PARKED) ++ t->verdict = (t->flags & G5P_TDF_ABANDON) ? G5P_TDV_ABANDONED : ++ G5P_TDV_QUIESCED; ++ else ++ t->verdict = G5P_TDV_RAM_ONLY; ++ (void)pre; ++} ++ ++/* ++ * Run the teardown once per mode. Returns 0 for QUIESCED/ABANDONED/UNTOUCHED/OBSERVED, ++ * -EAGAIN for RAM_ONLY (clean, but the GPU was left as found), -EIO for DIRTY, ++ * -EBUSY if another caller is inside the machine. Every phase ends in a receipt. ++ * Receipt a/b: account pending/bad, drain drained/abandoned, events consumed/left, ++ * irq before/after, reset skip-reason/pmgr-actual, done skip-reason/abandoned. ++ * Counters in the receipts are op-level (one count per gate): the AIC pair is ++ * 2 writes, the whole Fender stop leg is one R1 read+write op. ++ */ ++static inline int g5p_td_run(struct g5p_td *t, const struct g5p_td_ops *o, ++ enum g5p_td_mode mode, ++ const struct g5p_td_pre *pre) ++{ ++ if (cmpxchg(&t->running, 0, 1)) ++ return -EBUSY; ++ WRITE_ONCE(t->quiet, 1); ++ if (t->state == 2) { ++ /* Answered from the cache (after FINAL only; CAPTURE re-runs, gpu- ++ * recovery99 0004): no phase runs, nothing is accessed. ++ */ ++ u32 saved_flags = t->flags; ++ ++ t->t0_ns = o->now_ns(o->ctx) - (u64)t->us_total * 1000; ++ t->flags |= G5P_TDF_REENTRY; ++ t->phase = G5P_TDP_DONE; ++ g5p_td_receipt(t, o, t->rc, t->flags, t->steps); ++ t->flags = saved_flags; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++ } ++ t->mode = mode; ++ t->phase = G5P_TDP_FREEZE; ++ t->t0_ns = o->now_ns(o->ctx); ++ t->seq = 0; ++ t->nrcpt = 0; ++ t->flags = mode == G5P_TD_CAPTURE ? G5P_TDF_CAPTURE_ONLY : 0; ++ t->steps = 0; ++ t->violations = 0; ++ t->abandoned = 0; ++ t->drained = 0; ++ t->pending_mask = 0; ++ t->pending_initial = 0; ++ t->ev_consumed = 0; ++ t->ev_left = 0; ++ t->ev_initial = 0; ++ t->irq_before = 0; ++ t->irq_after = 0; ++ t->skip = G5P_TDS_NONE; ++ t->verdict = G5P_TDV_NONE; ++ memset(t->cnt, 0, sizeof(t->cnt)); ++ /* a CAPTURE may now run many times: nothing of the previous look may leak into this one */ ++ memset(t->ring_cons, 0, sizeof(t->ring_cons)); ++ memset(t->ring_prod, 0, sizeof(t->ring_prod)); ++ if (READ_ONCE(t->terminal)) ++ t->flags |= G5P_TDF_TERMINAL; ++ if (!pre->released) ++ t->flags |= G5P_TDF_UNRELEASED; ++ ++ if (mode == G5P_TD_FINAL) { ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->freeze(o->ctx); ++ g5p_td_receipt(t, o, 0, pre->released, pre->policy_host_owned); ++ } ++ t->phase = G5P_TDP_ACCOUNT; ++ if (g5p_td_account(t, o)) ++ t->flags |= G5P_TDF_OVERRUN; ++ g5p_td_note_late(t, o); ++ if (mode == G5P_TD_CAPTURE) ++ g5p_td_snap(t, o, pre); ++ ++ if (mode == G5P_TD_FINAL && ++ !(t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED | ++ G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN))) { ++ t->phase = G5P_TDP_DRAIN; ++ g5p_td_drain(t, o); ++ g5p_td_note_late(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_ABANDON) ? -ETIMEDOUT : 0, ++ t->drained, t->abandoned); ++ ++ t->phase = G5P_TDP_EVENTS; ++ g5p_td_events(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_EV_UNQUIET) ? -ETIMEDOUT : 0, ++ t->ev_consumed, t->ev_left); ++ } ++ if (mode == G5P_TD_FINAL) { ++ /* The IRQ is masked even after an overrun: ack + mask is the safe exit. */ ++ t->phase = G5P_TDP_IRQ; ++ if (!(t->flags & G5P_TDF_NOT_BOOTED)) ++ g5p_td_irq(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_IRQ_UNSETTLED) ? -ETIMEDOUT : ++ 0, ++ t->irq_before, t->irq_after); ++ ++ t->phase = G5P_TDP_DETACH; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->detach(o->ctx); ++ g5p_td_receipt(t, o, 0, 0, 0); ++ ++ t->phase = G5P_TDP_RESET; ++ g5p_td_reset(t, o, pre); ++ } ++ g5p_td_note_late(t, o); ++ t->phase = G5P_TDP_DONE; ++ g5p_td_decide(t, pre); ++ switch (t->verdict) { ++ case G5P_TDV_DIRTY: ++ t->rc = -EIO; ++ break; ++ case G5P_TDV_RAM_ONLY: ++ t->rc = -EAGAIN; ++ break; ++ default: ++ t->rc = 0; ++ } ++ t->us_total = g5p_td_us_since(t, o); ++ g5p_td_receipt(t, o, t->rc, (u32)t->skip, t->abandoned); ++ t->state = mode == G5P_TD_FINAL ? 2 : 1; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +new file mode 100644 +index 000000000..607097898 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -0,0 +1,692 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the GPU teardown state machine (teardown.h). ++ * Design, evidence and device plan: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md ++ * ++ * Included from core.inc after lifecycle.inc, so it may use g5p_dc_root_word(), ++ * g5p_fender_write()/poll() and g5p_r73b_disarm(). No state is added to struct ++ * g5p_boot or struct g5p_diag: one GPU, one file-static teardown instance. ++ * ++ * Compiled with W=1 against a9/tri96 (patches/gpu-recovery/README.md); NOT run on hardware. ++ * ++ * gpu-recovery108 (0006): every run goes through the retire layer (retire108.h): ++ * admission close, RAM-only epoch class and a TR108 receipt before the ladder, the ++ * named post-ladder slots, and the retired latch after a FINAL. The TD98 lines are ++ * unchanged. docs/gpu/experiments/recovery108-driver.md. ++ */ ++#include ++#include ++#include ++#include "teardown.h" ++#include "retire108.h" ++#include "render-user-bind.h" ++#include "render-user-quiesce.h" ++ ++/* 0 off (default; the notifier and .shutdown do nothing), 1 observe-only receipts ++ * (CAPTURE: RAM reads, no change), 2 full ladder (FINAL). ++ */ ++ ++/* gpu-recovery108 G6 slot: one R1 read of isStableOff after a verified Fender stop leg. */ ++ ++/* Defined in render-r96.inc (same translation unit, included later by render-fixed.inc). */ ++ ++static struct g5p_td g5p_td_state; ++static struct g5p_diag *g5p_td_dev; ++static void (*g5p_td_detach_cb)(void *cookie); ++static void *g5p_td_detach_cookie; ++ ++/* gpu-recovery108: the retire layer around the ladder (retire108.h), one GPU, file static. */ ++static struct g5p_r8 g5p_r8_state = { .admit_us = G5P_R8_ADMIT_US }; ++static void (*g5p_r8_epoch_cb)(void *cookie, struct g5p_r8_in *in); ++static void *g5p_r8_epoch_cookie; ++ ++/* The DRM frontend reports its share of the epoch inputs (claim, holders, queued jobs), RAM only. */ ++static void __maybe_unused g5p_r8_register_epoch(void (*cb)(void *, ++ struct g5p_r8_in *), ++ void *cookie) ++{ ++ g5p_r8_epoch_cookie = cookie; ++ smp_store_release(&g5p_r8_epoch_cb, cb); ++} ++ ++/* Job admission: a submitter enters before its first look at driver state and ++ * leaves on every return; false once a FINAL teardown closed admission. ++ */ ++static bool __maybe_unused g5p_r8_enter(void) ++{ ++ return g5p_r8_admit_enter(&g5p_r8_state.admit); ++} ++ ++static void __maybe_unused g5p_r8_exit(void) ++{ ++ g5p_r8_admit_exit(&g5p_r8_state.admit); ++} ++ ++/* The post-retire register predicate for the R1 helpers outside the ladder. */ ++static bool __maybe_unused g5p_r8_r1_allowed(void) ++{ ++ return g5p_r8_mmio_ok(&g5p_r8_state, G5P_TDD_R1); ++} ++ ++static bool __maybe_unused g5p_r8_is_retired(void) ++{ ++ return g5p_r8_retired(&g5p_r8_state); ++} ++ ++/* The DRM frontend registers the callback that fails its queued/in-flight job ++ * fences (exactly once each, -ENODEV) without touching hardware. ++ */ ++static void __maybe_unused g5p_td_register_detach(void (*cb)(void *), ++ void *cookie) ++{ ++ g5p_td_detach_cookie = cookie; ++ smp_store_release(&g5p_td_detach_cb, cb); ++} ++ ++static u64 g5p_td_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_td_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++static void g5p_td_op_freeze(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ struct g5p_boot *b = g->boot; ++ ++ if (!b) ++ return; ++ /* The IRQ-side plateau banks (g5p_r80_irq_bank/g5p_r85_irq_bank) read R0 until this ++ * latch is set; arm it before anything else (RAM only, no-op without an r96 config). ++ */ ++ /* Same freeze as g5p_boot_park(): nothing may publish after this. */ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++} ++ ++static int g5p_td_op_ring_read(void *ctx, u32 idx, u32 *cons, u32 *prod) ++{ ++ static const u32 slots[G5P_TD_NRING] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *ctl; ++ ++ if (!b || !READ_ONCE(b->maps_ready)) ++ return -ENODEV; ++ ctl = (u32 *)g5p_boot_shared(b, slots[idx]); ++ *cons = READ_ONCE(ctl[0]); ++ *prod = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ return 0; ++} ++ ++/* RAM only. Consumes ALIVE (0) and stamp (1) entries; the first other type is ++ * left in place for the host, as g5p_boot_drain() does. ++ */ ++static int g5p_td_op_event_pass(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n = 0; ++ unsigned long flags; ++ int ret = 0; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ while (n < max && consumer != producer) { ++ u32 off = b->layout.offset[0x74] + consumer * 44; ++ ++ if (g5p_le32(b->shared[off / G5P_GPU_PAGE].cpu + ++ off % G5P_GPU_PAGE) > 1) { ++ *fault = true; ++ break; ++ } ++ consumer = (consumer + 1) & 255; ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ n++; ++ } ++ producer = READ_ONCE(control[4]); ++ *left = producer > 255 ? 255 : (producer - consumer) & 255; ++out: ++ *consumed = n; ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* The render path latches terminal states through g5p_td_note_terminal(); until ++ * prod98 wires that call this reports "not seen" and the race is only visible as ++ * ring state. ++ */ ++static bool g5p_td_op_terminal_seen(void *ctx) ++{ ++ return READ_ONCE(g5p_td_state.terminal) != 0; ++} ++ ++static void g5p_td_op_detach(void *ctx) ++{ ++ void (*cb)(void *) = smp_load_acquire(&g5p_td_detach_cb); ++ ++ if (cb) ++ cb(g5p_td_detach_cookie); ++} ++ ++static void g5p_td_op_aic_ack(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* The same software-source ACK the IRQ-cap path uses. */ ++ if (b && b->irq_ready && b->swirq) ++ writel(0x4000, b->swirq + 0x94); ++} ++ ++static void g5p_td_op_irq_mask(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* Line masked at the interrupt controller; the handler stays registered for ++ * a late entry, which still ACKs. Same call as the IRQ-cap path. ++ */ ++ if (b && b->irq_ready && b->irq > 0) ++ disable_irq_nosync(b->irq); ++} ++ ++static u32 g5p_td_op_irq_total(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ return b ? (u32)atomic_read(&b->irqs) : 0; ++} ++ ++static int g5p_td_op_fender_stop(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ ++ /* Apple stopFirmwareForPowerDown 6c9fa10; no retry, no PMGR off. */ ++ return g5p_fender_idle(&io); ++} ++ ++static int g5p_td_op_pmgr(void *ctx, u32 *target, u32 *actual) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) ++ return -ENODEV; ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (!ret) { ++ *target = raw & 0xf; ++ *actual = (raw >> 4) & 0xf; ++ } ++ return ret; ++} ++ ++/* dev_emerg so the receipt reaches netconsole at any console loglevel. */ ++static void g5p_td_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_diag *)ctx)->dev, "%s\n", line); ++} ++ ++static void g5p_td_pre_fill(struct g5p_diag *g, struct g5p_td_pre *p, ++ bool locked) ++{ ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(p, 0, sizeof(*p)); ++ p->lock_held = locked; ++ p->powered = READ_ONCE(g->powered); ++ p->fender_ready = READ_ONCE(g->fender_ready); ++ if (!b) ++ return; ++ p->booted = READ_ONCE(b->maps_ready); ++ p->released = READ_ONCE(b->released); ++ p->already_parked = READ_ONCE(b->parked); ++ p->stopped_unparked = READ_ONCE(b->stopped) && !READ_ONCE(b->parked); ++ p->power_abort = b->r73b && READ_ONCE(b->r73b->early_power_abort); ++ /* RAM-only read of the host-written init word (same as g5p_dc_snapshot()). */ ++ p->policy_host_owned = ++ p->booted && ++ !g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8); ++} ++ ++/* gpu-recovery108: the ladder as the retire layer's td_run op. The TD98 begin line ++ * and the ladder's own lines are exactly the recovery98/99 ones. ++ */ ++struct g5p_r8_ctx { ++ struct g5p_diag *g; ++ const struct g5p_td_ops *ops; ++ struct g5p_td_pre *pre; ++ const char *why; ++ bool locked; ++}; ++ ++static int g5p_r8_op_td_run(void *ctx, enum g5p_td_mode mode) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ dev_emerg(c->g->dev, ++ "TD98 begin why=%s mode=%s teardown=%u locked=%u\n", c->why, ++ g5p_td_mode_name[mode], (0), c->locked); ++ return g5p_td_run(&g5p_td_state, c->ops, mode, c->pre); ++} ++ ++static u32 g5p_r8_op_td_flags(void *ctx) ++{ ++ return g5p_td_state.flags; ++} ++ ++static enum g5p_td_verdict g5p_r8_op_td_verdict(void *ctx) ++{ ++ return g5p_td_state.verdict; ++} ++ ++static u64 g5p_r8_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_r8_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++/* G6: R1+0x0 bit 30 after a verified stop leg (the slot runs only then, before RETIRE). */ ++static int g5p_r8_op_stable_off(void *ctx, u32 *value) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ *value = readl(c->g->window[1]) >> 30; ++ return 0; ++} ++ ++static void g5p_r8_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_r8_ctx *)ctx)->g->dev, "%s\n", line); ++} ++ ++static void g5p_r8_in_fill(struct g5p_diag *g, struct g5p_r8_in *in, ++ bool locked, enum g5p_td_mode mode) ++{ ++ void (*cb)(void *, struct g5p_r8_in *) = ++ smp_load_acquire(&g5p_r8_epoch_cb); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(in, 0, sizeof(*in)); ++ in->terminal = READ_ONCE(g5p_td_state.terminal); ++ /* FINAL without the lock: a worker (or a sysfs writer) holds it, maybe inside a wait. */ ++ in->worker_busy = mode == G5P_TD_FINAL && !locked; ++ if (b) { ++ in->booted = READ_ONCE(b->maps_ready); ++ in->released = READ_ONCE(b->released); ++ in->alive = READ_ONCE(b->alive); ++ in->failed = READ_ONCE(b->failed); ++ in->fault_event = READ_ONCE(b->event_failed); ++ in->parked = READ_ONCE(b->parked); ++ in->stopped_unparked = READ_ONCE(b->stopped) && ++ !READ_ONCE(b->parked); ++ in->power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ if (cb) ++ cb(g5p_r8_epoch_cookie, in); ++} ++ ++static int g5p_td_execute(struct g5p_diag *g, enum g5p_td_mode mode, ++ const char *why) ++{ ++ const struct g5p_td_ops ops = { ++ .ctx = g, ++ .now_ns = g5p_td_op_now, ++ .relax_us = g5p_td_op_relax, ++ .freeze = g5p_td_op_freeze, ++ .ring_read = g5p_td_op_ring_read, ++ .event_pass = g5p_td_op_event_pass, ++ .terminal_seen = g5p_td_op_terminal_seen, ++ .detach = g5p_td_op_detach, ++ .aic_ack = g5p_td_op_aic_ack, ++ .irq_mask = g5p_td_op_irq_mask, ++ .irq_total = g5p_td_op_irq_total, ++ .fender_stop = g5p_td_op_fender_stop, ++ .pmgr_actual = g5p_td_op_pmgr, ++ .emit = g5p_td_op_emit, ++ }; ++ struct g5p_td_pre pre; ++ struct g5p_r8_in in; ++ struct g5p_r8_ctx c = { .g = g, .ops = &ops, .pre = &pre, .why = why }; ++ /* fw-sleep (G2) and pmgr-gate (G1) have no mechanism: their ops stay NULL. */ ++ const struct g5p_r8_ops r8ops = { ++ .ctx = &c, ++ .now_ns = g5p_r8_op_now, ++ .relax_us = g5p_r8_op_relax, ++ .td_run = g5p_r8_op_td_run, ++ .td_flags = g5p_r8_op_td_flags, ++ .td_verdict = g5p_r8_op_td_verdict, ++ .slot = { [G5P_R8S_STABLE_OFF] = g5p_r8_op_stable_off }, ++ .emit = g5p_r8_op_emit, ++ }; ++ bool locked = false; ++ int ret; ++ ++ /* Never queue behind a worker or sysfs writer that may be inside MMIO. CAPTURE ++ * reads RAM only and needs no lock. ++ */ ++ if (mode == G5P_TD_FINAL) ++ locked = mutex_trylock(&g->lock); ++ c.locked = locked; ++ g5p_td_pre_fill(g, &pre, locked); ++ g5p_r8_in_fill(g, &in, locked, mode); ++ g5p_r8_state.slot_on[G5P_R8S_STABLE_OFF] = (0); ++ ret = g5p_r8_run(&g5p_r8_state, &r8ops, mode, &in, why); ++ if (g5p_td_state.flags & G5P_TDF_RT1_OK && g->boot) ++ WRITE_ONCE(g->boot->parked, true); ++ if (locked) ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Observation only: arms the quiet predicate, changes nothing else. */ ++static int __maybe_unused g5p_td_capture(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_CAPTURE, why); ++} ++ ++static int g5p_td_final(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_FINAL, why); ++} ++ ++/* ++ * The quiesce before a kexec, reboot or power-off (render-user-quiesce.h). It runs from the ++ * reboot notifier whatever `teardown` is, once per boot; .shutdown and the `td-kexec` command ++ * reach the same run. It leaves the GPU powered with its firmware stopped and the boot state ++ * frozen: the next kernel boots it as after any warm kexec. ++ */ ++ ++#define G5P_KQ_LOCK_MS 2000U ++#define G5P_KQ_ADMIT_US 20000U ++#define G5P_KQ_OFF_MS 10U ++ ++/* The render frontend's share: its TYPED_V3 epoch inputs and the RAM-only unbind of slots. */ ++struct g5p_kq_frontend { ++ void (*state)(void *cookie, struct g5p_ru_kq_in *in); ++ int (*detach)(void *cookie, const u32 *slots, u32 n); ++}; ++ ++static const struct g5p_kq_frontend *g5p_kq_fe; ++static void *g5p_kq_fe_cookie; ++static atomic_t g5p_kq_ran = ATOMIC_INIT(0); ++ ++struct g5p_kq_ctx { ++ struct g5p_diag *g; ++ u32 slots[G5P_RU_USER_SLOTS]; ++ u32 held; ++}; ++ ++static void __maybe_unused g5p_kq_register(const struct g5p_kq_frontend *fe, ++ void *cookie) ++{ ++ g5p_kq_fe_cookie = cookie; ++ smp_store_release(&g5p_kq_fe, fe); ++} ++ ++static u32 g5p_kq_admit_close(void *ctx) ++{ ++ struct g5p_r8_admit *a = &g5p_r8_state.admit; ++ u64 deadline = ktime_get_ns() + (u64)G5P_KQ_ADMIT_US * NSEC_PER_USEC; ++ u32 inside = g5p_r8_admit_close(a); ++ ++ while (inside && ktime_get_ns() < deadline) { ++ usleep_range(G5P_R8_POLL_US, 2 * G5P_R8_POLL_US); ++ inside = g5p_r8_admit_inside(a); ++ } ++ return inside; ++} ++ ++/* Never queue behind a holder that may be inside MMIO: bounded, like g5p_render_run(). */ ++static bool g5p_kq_lock(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_KQ_LOCK_MS); ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ return true; ++} ++ ++static void g5p_kq_unlock(void *ctx) ++{ ++ mutex_unlock(&((struct g5p_kq_ctx *)ctx)->g->lock); ++} ++ ++/* RAM only: the boot state, the held user slots and the DC ring, then the frontend's inputs. */ ++static void g5p_kq_inputs(void *ctx, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ struct g5p_diag *g = c->g; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ u32 *ctl; ++ ++ in->pmgr = !!g->pmgr; ++ if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released) || ++ READ_ONCE(b->parked)) ++ return; ++ in->booted = true; ++ in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ dma_rmb(); ++ in->cons = READ_ONCE(ctl[0]); ++ in->prod = READ_ONCE(ctl[4]); ++ in->ring_idle = in->prod <= 255 && in->cons == in->prod; ++ c->held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), ++ g5p_boot_shared(b, 0x84), c->slots, ++ ARRAY_SIZE(c->slots)); ++ in->held = c->held; ++ if (fe) { ++ in->frontend = true; ++ fe->state(g5p_kq_fe_cookie, in); ++ } ++} ++ ++static int g5p_kq_detach(void *ctx) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ ++ return fe ? fe->detach(g5p_kq_fe_cookie, c->slots, c->held) : -ENODEV; ++} ++ ++static void g5p_kq_freeze(void *ctx) ++{ ++ g5p_td_op_freeze(((struct g5p_kq_ctx *)ctx)->g); ++} ++ ++/* The IRQ-cap path's AIC ack and mask, then wait out a handler already running (process ++ * context; the threaded handler drains RAM and takes no lock the quiesce holds). ++ */ ++static void g5p_kq_irq_off(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ struct g5p_boot *b = g->boot; ++ ++ g5p_td_op_aic_ack(g); ++ g5p_td_op_irq_mask(g); ++ if (b && b->irq_ready && b->irq > 0) ++ synchronize_irq(b->irq); ++} ++ ++/* PMGR only: the bounded gate cycle of the r33/dirty-start path (refuses a gate that is not ++ * plainly on), which ends with the gate requested on. ++ */ ++static int g5p_kq_cycle(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ ++ g5p_mmio_gate(g); ++ return g5p_r33_gfx_off(g->dev, g->pmgr, G5P_KQ_OFF_MS); ++} ++ ++static int g5p_kq_run(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_ru_kq_ops o = { ++ .admit_close = g5p_kq_admit_close, ++ .lock = g5p_kq_lock, ++ .unlock = g5p_kq_unlock, ++ .inputs = g5p_kq_inputs, ++ .detach = g5p_kq_detach, ++ .freeze = g5p_kq_freeze, ++ .irq_off = g5p_kq_irq_off, ++ .cycle = g5p_kq_cycle, ++ }; ++ struct g5p_ru_kq_out out; ++ struct g5p_kq_ctx *c; ++ u64 t0 = ktime_get_ns(); ++ ++ if (atomic_xchg(&g5p_kq_ran, 1)) { ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s already ran this boot; nothing done end=kq\n", ++ why); ++ return -EALREADY; ++ } ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ if (!c) { ++ dev_emerg(g->dev, "GPU-KQ why=%s ret=%d end=kq\n", why, ++ -ENOMEM); ++ return -ENOMEM; ++ } ++ c->g = g; ++ o.ctx = c; ++ g5p_ru_kq_run(&o, min((G5P_RU_KQ_FULL), G5P_RU_KQ_FULL), &out); ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s mode=%u inside=%u locked=%u booted=%u failed=%u wedged=%u " ++ "ram_only=%u exec=%u dc=%u/%u held=%u plan=%s%s skip=%#x detach=%d froze=%u " ++ "cycle=%d ms=%llu end=kq\n", ++ why, out.in.mode, out.inside, out.in.locked, out.in.booted, ++ out.in.failed, out.in.term_failed, out.in.ram_only, ++ out.in.executing, out.in.cons, out.in.prod, out.in.held, ++ out.plan & G5P_RU_KQ_DETACH ? "detach" : "-", ++ out.plan & G5P_RU_KQ_CYCLE ? "+cycle" : "", out.why, ++ out.detach_ret, out.froze, out.cycle_ret, ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC); ++ kfree(c); ++ return out.detach_ret ?: out.cycle_ret; ++} ++ ++/* sysfs `command`: td-capture / td-final / td-kexec. Runs before the command mutex, like ++ * r73b-export, so a wedged stage cannot block it. -EAGAIN (clean, GPU left as ++ * found) is reported as success; the verdict is in the TD98 lines. td-kexec is the ++ * reboot quiesce run by hand (the GPU is finished for this boot afterwards). ++ */ ++static int g5p_td_command(struct g5p_diag *g, const char *command) ++{ ++ int ret; ++ ++ if (sysfs_streq(command, "td-capture")) ++ ret = g5p_td_capture(g, "sysfs"); ++ else if (sysfs_streq(command, "td-final")) ++ ret = g5p_td_final(g, "sysfs"); ++ else if (sysfs_streq(command, "td-kexec")) ++ ret = g5p_kq_run(g, "sysfs"); ++ else ++ return -EINVAL; ++ return ret == -EAGAIN ? 0 : ret; ++} ++ ++/* Runs from kernel_restart_prepare() for reboot, halt, power-off and kexec, i.e. ++ * before device_shutdown() and its unbounded wait_for_device_probe(). Panic kexec ++ * skips both and is not covered. ++ */ ++static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, ++ void *data) ++{ ++ struct g5p_diag *g = READ_ONCE(g5p_td_dev); ++ ++ if (!g) ++ return NOTIFY_DONE; ++ if (action != SYS_RESTART && action != SYS_HALT && ++ action != SYS_POWER_OFF) ++ return NOTIFY_DONE; ++ /* The quiesce needs a live firmware for its detach, so it runs before the ladder freezes. */ ++ g5p_kq_run(g, "reboot-notifier"); ++ { ++ return NOTIFY_DONE; ++ } ++} ++ ++static struct notifier_block g5p_td_nb = { ++ .notifier_call = g5p_td_reboot_notify, ++ .priority = INT_MAX, /* before every other reboot notifier */ ++}; ++ ++/* g is devm-allocated and the DRM frontend's probe error path unwinds without ++ * g5p_remove(): the notifier must not outlive g. ++ */ ++static void g5p_td_devres_release(void *data) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ WRITE_ONCE(g5p_td_dev, NULL); ++} ++ ++static void g5p_td_attach(struct g5p_diag *g) ++{ ++ g5p_td_init(&g5p_td_state); ++ /* A probe after a retire (not reachable on the built-in driver) never reopens admission. */ ++ if (!g5p_r8_retired(&g5p_r8_state)) { ++ u32 gen = g5p_r8_state.gen; ++ ++ g5p_r8_init(&g5p_r8_state); ++ g5p_r8_state.gen = gen; /* monotonic across a re-probe */ ++ } ++ WRITE_ONCE(g5p_td_dev, g); ++ register_reboot_notifier(&g5p_td_nb); ++ /* On failure the action has already run: no notifier, teardown stays inert. */ ++ if (devm_add_action_or_reset(g->dev, g5p_td_devres_release, g)) ++ dev_warn(g->dev, "teardown not attached\n"); ++} ++ ++/* Returns true when the PM reference must be kept: a PMGR power-off was never ++ * validated on the outgoing side and the stop leg is not isStableOff. ++ */ ++static bool g5p_td_remove(struct g5p_diag *g) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ ; ++ ; ++ WRITE_ONCE(g5p_td_dev, NULL); ++ return (0) != 0; ++} ++ ++/* The .shutdown platform callback: second chance after the notifier. */ ++static void __maybe_unused g5p_td_shutdown(struct g5p_diag *g) ++{ ++ /* A second chance after the notifier (a no-op once it ran). */ ++ if (!atomic_read(&g5p_kq_ran)) ++ g5p_kq_run(g, "shutdown"); ++ { ++ ; ++ ; ++ }; ++} +diff --git a/drivers/gpu/apple-g5p/transport-budget.inc b/drivers/gpu/apple-g5p/transport-budget.inc +new file mode 100644 +index 000000000..d70e3c6e1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/transport-budget.inc +@@ -0,0 +1,6 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include "vm-mutation.h" ++static u32 g5p_transport_event_limit(void) ++{ ++ return 256 + 131072 * 4; ++} +diff --git a/drivers/gpu/apple-g5p/vm-mutation.h b/drivers/gpu/apple-g5p/vm-mutation.h +new file mode 100644 +index 000000000..4645240bd +--- /dev/null ++++ b/drivers/gpu/apple-g5p/vm-mutation.h +@@ -0,0 +1,36 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_VM_MUTATION_H ++#define G5P_VM_MUTATION_H ++struct g5p_vm_mutation { ++ bool generations; ++ u32 generation[3]; ++}; ++ ++/* Count actual changed descriptors, including clears, not dispatches. Repeated ++ * parent writes of identical bytes do not advance a generation. Packed fields ++ * follow iOS6cabf40. Publication is serialized by the existing command lock. ++ */ ++static void g5p_vm_entry(struct g5p_vm_mutation *c, void *at, u64 value, ++ u32 level) ++{ ++ u64 old = g5p_le32(at); ++ ++ if (level) ++ old |= (u64)g5p_le32(at + 4) << 32; ++ if (old == value) ++ return; ++ if (level) ++ g5p_put64(at, value); ++ else ++ g5p_put32(at, value); ++ c->generation[level]++; ++} ++ ++static u32 g5p_vm_generation(struct g5p_vm_mutation *c) ++{ ++ if (!c->generations) ++ return 0; ++ return ((c->generation[0] & 0x3ff) << 22) | ++ ((c->generation[1] & 0x3ff) << 12) | (c->generation[2] & 0xfff); ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/work-layout.h b/drivers/gpu/apple-g5p/work-layout.h +new file mode 100644 +index 000000000..b5dc73725 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/work-layout.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_WORK_LAYOUT_H ++#define G5P_WORK_LAYOUT_H ++ ++/* Experimental first CL queue probe: c89420/c88840/c8a808/c980a0. ++ * Tag 4 is emitted for dependencies by c8a808. Treating an already satisfied ++ * dependency as a NOP is inferred and must be validated after G2. ++ */ ++#define G5P_WORK_OFFSET 0x54000U ++#define G5P_WORK_BYTES 0x400U ++#define G5P_WORK_CTL 0x100U ++#define G5P_WORK_RING 0x200U ++#define G5P_WORK_CMD 0x300U ++#define G5P_WORK_STAMP 0x340U ++#define G5P_WORK_SLOTS 16U ++ ++/* Isolated visibility hypothesis, not the recovered normal host alias. */ ++ ++/* Firmware10004e94 copies notification word2 into ctl+30. */ ++ ++#endif +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 4c74c1501..eaf71742f 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -371,3 +371,5 @@ endif + # Separate option because drm_panel_orientation_quirks.c is shared with fbdev + config DRM_PANEL_ORIENTATION_QUIRKS + tristate ++ ++source "drivers/gpu/drm/apple-g5p/Kconfig" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index d75eba399..1315aa378 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -257,3 +257,5 @@ quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + + $(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) ++ ++obj-$(CONFIG_DRM_APPLE_G5P) += apple-g5p/ +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..24c76fef9 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -653,11 +653,36 @@ static int adbe_dumb_create(struct drm_file *file, struct drm_device *drm, + return drm_gem_dma_dumb_create_internal(file, drm, args); + } + ++/* Native allocations: the DMA helper's defaults plus the uncached-export ++ * contract. Imported objects never make that promise. ++ */ ++static const struct drm_gem_object_funcs adbe_gem_funcs = { ++ .free = drm_gem_dma_object_free, ++ .print_info = drm_gem_dma_object_print_info, ++ .get_sg_table = drm_gem_dma_object_get_sg_table, ++ .vmap = drm_gem_dma_object_vmap, ++ .mmap = drm_gem_dma_object_mmap, ++ .vm_ops = &drm_gem_dma_vm_ops, ++ .is_cpu_uncached = drm_gem_dma_object_is_cpu_uncached, ++}; ++ ++static struct drm_gem_object *adbe_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct drm_gem_dma_object *dma_obj = kzalloc_obj(*dma_obj); ++ ++ if (!dma_obj) ++ return ERR_PTR(-ENOMEM); ++ dma_obj->base.funcs = &adbe_gem_funcs; ++ return &dma_obj->base; ++} ++ + DEFINE_DRM_GEM_DMA_FOPS(adbe_fops); + + static const struct drm_driver adbe_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .fops = &adbe_fops, ++ .gem_create_object = adbe_gem_create_object, + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(adbe_dumb_create), + DRM_FBDEV_DMA_DRIVER_OPS, + .name = "adbe", +diff --git a/drivers/gpu/drm/apple-g5p/Kconfig b/drivers/gpu/drm/apple-g5p/Kconfig +new file mode 100644 +index 000000000..00620b99c +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Kconfig +@@ -0,0 +1,15 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_APPLE_G5P ++ bool "Apple A9/G5P render driver" ++ depends on DRM=y && ARCH_APPLE && OF && PM && ARM64_16K_PAGES ++ select DRM_GEM_SHMEM_HELPER ++ select FW_LOADER ++ select CRYPTO_LIB_SHA256 ++ help ++ Render-node driver for the S8003 G5P GPU using the unmodified META ++ firmware from the device. Supports TYPED_V3 jobs, GEM buffers, PRIME, ++ virtual address spaces and syncobj synchronization for the merged ++ Mesa winsys. The GPU is brought up at probe once the user's ++ apple/agx-g5p-c0.ldr is readable (a9_gpu_drm.autoboot=0 or ++ a9_gpu_drm.enable=0 to opt out). Built-in only while reset and ++ active-DMA teardown require the platform boot sequence. +diff --git a/drivers/gpu/drm/apple-g5p/Makefile b/drivers/gpu/drm/apple-g5p/Makefile +new file mode 100644 +index 000000000..843d56a04 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Makefile +@@ -0,0 +1,2 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++obj-$(CONFIG_DRM_APPLE_G5P) += a9-gpu-drm.o +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +new file mode 100644 +index 000000000..ed2b769d7 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -0,0 +1,773 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* Experimental render frontend; default disabled, no implicit firmware boot. */ ++#include "../../apple-g5p/core.inc" ++#include "../../apple-g5p/drm-validate.h" ++#include "../../apple-g5p/render-user-evwait.h" ++#include "../../apple-g5p/render-user-sync.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define G5P_MAX_VMS 8 ++#define G5P_MAX_CONTEXTS 32 ++#define G5P_MAX_MAPS 4096 ++#define G5P_MAX_PT_PAGES 512 ++#define G5P_MAX_SYNCS 17 ++ ++struct g5p_bo { ++ struct drm_gem_shmem_object shmem; ++ u32 roles; ++ u32 vm_pins; /* VMs that map it, any file; counted once in pinned_bytes (pin_lock) */ ++ /* release8 memory accounting (drm-vm.inc g5p_bo_live): the live list, its creator */ ++ struct list_head live; ++ pid_t pid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++struct g5p_pt { ++ struct list_head link; ++ struct g5p_dma_page page; ++ u32 key; ++ u32 level; ++}; ++ ++struct g5p_mapping { ++ struct list_head link; ++ struct drm_mm_node node; ++ struct drm_gem_object *obj; ++ u64 offset; ++ u64 retire; /* unmapped beside queued jobs: freed once this fence seqno has signalled */ ++}; ++ ++struct g5p_vm { ++ struct g5p_vm_mutation mutation; ++ bool bounded, frozen, serial_context; ++ bool zero_leaf_flags; /* render context latched r25 memory variant 4 */ ++ /* Its render context's handle is gone (g5p_render_close()): no VM_BIND, no new ++ * context; the context's close item and release own the tables. ++ */ ++ bool resident_gone; ++ struct g5p_context *resident; ++ struct kref ref; ++ struct device *dev; ++ struct g5p_drm *gpu; ++ struct drm_mm mm; ++ struct drm_mm_node reserved; ++ struct list_head maps; ++ struct xarray objs; /* object -> its mappings in this VM (xa_mk_value) */ ++ struct list_head tables; ++ struct g5p_dma_page root; ++ u64 mapped_bytes, pinned_bytes; ++ u32 table_count, map_count, contexts; ++}; ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rdev; ++ ++struct g5p_context { ++ struct g5p_drm *gpu; ++ ++ bool live, bad; ++ bool serial, pending; ++ ++ struct g5p_render *render; /* render contexts only, see drm-render.inc */ ++ struct kref ref; ++ struct g5p_vm *vm; ++}; ++ ++struct g5p_file { ++ /* Serializes VM/context handles, mappings and the one-shot submit state. */ ++ struct mutex lock; ++ struct xarray vms; ++ struct xarray contexts; ++ struct xarray results; ++ struct xarray targets; /* render targets; each holds its context */ ++ u32 render_results; ++ bool submitted; ++ u64 job_id; ++}; ++ ++struct g5p_drm { ++ struct drm_device drm; ++ struct g5p_diag *core; ++ /* Protects cached IRQ event/status fields; never held for hardware IO. */ ++ spinlock_t status_lock; ++ u32 last_event[11], notifications[4], first_fault[11]; ++ u64 event_serial, fault_serial; ++ struct delayed_work autoboot; /* probe-time bring-up */ ++ unsigned long autoboot_deadline; ++ u32 autoboot_phase; ++ /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ ++ wait_queue_head_t ev_wq; ++ /* TYPED_V3 3D completion-event latch (render-user-retire.h): G5P_RV3_TD_WANT | endpoint ++ * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. ++ */ ++ u32 td_want; ++ u64 td_hit_ns; ++ /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ ++ bool ta_want; ++ u64 ta_hit_ns; ++ /* wake110: type-1 completion events decoded per engine (word 1 bit 0 TA, bit 1 3D) since ++ * load, and the decode time of the first one since the executor last cleared it. ++ */ ++ u32 t1_count[2]; ++ u64 t1_first_ns[2]; ++ /* perf60 bell gates (drm-render-v3.inc): the previous TYPED_V3 job's end and context. */ ++ u64 bell_prev_end_ns; ++ const void *bell_prev_ctx; ++ u64 last_job; ++ int last_error, job_error; ++ u32 job_state; ++ ++ atomic_t accepted; ++ atomic64_t job_sequence; ++ u64 fence_context; ++ /* Attached catalogues survive fd close until proven unbind/reset. */ ++ struct g5p_context *quarantined_context; ++ /* The first accepted render job claimed the device for render, for the rest of the boot. */ ++ bool render_claimed; ++ struct g5p_rdev *rdev; /* device-wide render state, see drm-render.inc */ ++ struct workqueue_struct ++ *render_wq; /* ordered: the render scheduler's worker */ ++ /* Orders render acceptance, dma_resv fence attach and queueing. */ ++ struct mutex render_submit_lock; ++ /* recovery108 epoch inputs, RAM only: open render-node files, render jobs not yet ++ * signalled. ++ */ ++ atomic_t open_files, r8_queued; ++ /* Memory mapped into any VM, each object once (render_pinned_mib). */ ++ struct mutex pin_lock; ++ u64 pinned_bytes; ++}; ++ ++static inline struct g5p_drm *to_g5p(struct drm_device *drm) ++{ ++ return container_of(drm, struct g5p_drm, drm); ++} ++ ++static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) ++{ ++ return container_of(obj, struct g5p_bo, shmem.base); ++} ++ ++#include "vm-ranges.h" ++ ++static int g5p_resident_idle(struct g5p_context *c, bool live); ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m); ++static int g5p_resident_invalidate(struct g5p_context *c); ++static void g5p_render_release(struct g5p_context *c); ++static int g5p_render_destroy(struct g5p_context *c); ++static void g5p_render_close(struct g5p_context *c); ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a); ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags); ++ ++#include "drm-vm.inc" ++#include "drm-submit.inc" ++#include "drm-render.inc" ++#include "drm-prime.inc" ++#include "drm-compute-v1.inc" ++ ++static int g5p_get_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_info *a = data; ++ u64 caps = DRM_G5P_CAP_BO_CPU | DRM_G5P_CAP_VM_PREPARE | ++ DRM_G5P_CAP_SYNCOBJ | DRM_G5P_CAP_JOB_STATUS | ++ DRM_G5P_CAP_CONTEXT_STATUS | DRM_G5P_CAP_FAULT_INFO; ++ ++ if (a->size != sizeof(*a) || a->flags || a->reserved[0] || ++ a->reserved[1]) ++ return -EINVAL; ++ ; ++ ; ++ ; ++ ; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ a->version = DRM_G5P_UAPI_VERSION; ++ a->gpu_page_shift = 14; ++ ; ++ ; ++ ; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER | DRM_G5P_CAP_BOUNDED_RENDER | ++ DRM_G5P_CAP_TYPED_RENDER | DRM_G5P_CAP_VM_LIVE | ++ DRM_G5P_CAP_CONTEXT_LIVE | ++ DRM_G5P_CAP_CODE_BASE_VERIFIED; ++ a->code_base = 0x300000000ULL; ++ }; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY; ++ }; ++ if ((1) && g5p_render_msaa_enabled()) ++ caps |= DRM_G5P_CAP_RENDER_MSAA; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_TYPED_RENDER_V3 | ++ DRM_G5P_CAP_RENDER_MULTI_CONTEXT | ++ DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH | ++ DRM_G5P_CAP_RENDER_OCCLUSION_QUERY; ++ }; ++ { ++ caps |= DRM_G5P_CAP_RENDER_UNPRIVILEGED; ++ }; ++ ; ++ ; ++ caps |= DRM_G5P_CAP_PRIME; ++ caps |= DRM_G5P_CAP_TYPED_COMPUTE; ++ a->capabilities = caps; ++ a->va_start = G5P_USER_VA_START; ++ a->va_end = G5P_USER_VA_END; ++ a->reserved_start = G5P_USER_RESERVED_START; ++ a->reserved_end = G5P_USER_RESERVED_END; ++ a->max_bo_size = G5P_MAX_BO_SIZE; ++ a->bvnc[0] = 9; ++ a->bvnc[1] = 15; ++ a->bvnc[2] = 6; ++ a->bvnc[3] = 2; ++ return 0; ++} ++ ++/* The gated dialect uses the common B proven by the r19c numerical store. */ ++static int g5p_get_exec_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_exec_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || a->pad || ++ memchr_inv(a->reserved, 0, sizeof(a->reserved))) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ { ++ a->base_scope = DRM_G5P_BASE_VM; ++ a->usc_base = 0x300000000ULL; ++ a->pds_base = a->usc_base; ++ a->window_size = 0x34000; ++ a->allocation_alignment = G5P_GPU_PAGE; ++ a->usc_offset_bits = 32; ++ a->pds_offset_bits = 32; ++ } ++ return 0; ++} ++ ++static int g5p_get_job_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_job_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ bool owned; ++ ++ if (a->flags || a->pad || !a->job_id || a->reset_generation) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ if (xa_load(&f->results, a->job_id)) { ++ struct g5p_fence *fence = xa_load(&f->results, a->job_id); ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = fence->state; ++ a->error = fence->error; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ mutex_unlock(&f->lock); ++ return 0; ++ } ++ owned = f->job_id == a->job_id; ++ mutex_unlock(&f->lock); ++ if (!owned) ++ return -ENOENT; ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = d->job_state; ++ a->error = d->job_error; ++ /* Owner acknowledges terminal status before another fixed submission. ++ * This avoids waking a fence waiter before the device admission reopens. ++ */ ++ ; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return 0; ++} ++ ++/* c99788 classifies event4/7/8. Preserve opaque fields, never invent a VA. */ ++static int g5p_get_fault_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_fault_info *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ ++ if (a->flags || a->reserved) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->event_serial = d->fault_serial; ++ memcpy(a->raw, d->first_fault, sizeof(a->raw)); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!a->event_serial) ++ return 0; ++ switch (a->raw[0]) { ++ case 4: ++ a->kind = DRM_G5P_FAULT_RECOVERY; ++ break; ++ case 7: ++ a->kind = DRM_G5P_FAULT_PAGE_REQUEST; ++ break; ++ case 8: ++ a->kind = DRM_G5P_FAULT_CHANNEL; ++ a->valid = DRM_G5P_FAULT_VALID_CHANNEL; ++ a->channel_class = a->raw[1]; ++ a->channel_token = a->raw[3]; ++ a->channel_stamp = a->raw[4]; ++ break; ++ default: ++ a->kind = DRM_G5P_FAULT_UNKNOWN; ++ break; ++ } ++ return 0; ++} ++ ++static int g5p_get_context_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_context_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(to_g5p(drm)->core->boot); ++ struct g5p_context *c; ++ bool failed; ++ int ret = 0; ++ ++ if (a->flags || a->state || a->map_count || a->mapped_bytes || ++ a->reserved) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c) { ++ ret = -ENOENT; ++ goto out; ++ } ++ a->state = DRM_G5P_CONTEXT_HOST_ONLY; ++ if (READ_ONCE(c->live)) ++ a->state = DRM_G5P_CONTEXT_LIVE; ++ /* a failure noted mid-job is not this context's until the executor rules */ ++ failed = b && READ_ONCE(b->failed) && ++ !g5p_rdev_ruling(to_g5p(drm)->rdev); ++ if (READ_ONCE(c->bad) || failed || (b && READ_ONCE(b->stopped))) ++ a->state = DRM_G5P_CONTEXT_LOST; ++ a->map_count = c->vm->map_count; ++ a->mapped_bytes = c->vm->mapped_bytes; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static void g5p_drm_event(void *cookie, const u32 words[11]) ++{ ++ struct g5p_drm *d = cookie; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ memcpy(d->last_event, words, sizeof(d->last_event)); ++ d->event_serial++; ++ if (words[0] == 1) ++ for (u32 i = 0; i < 4; i++) ++ d->notifications[i] |= words[i + 1]; ++ if (d->td_want && !d->td_hit_ns && ++ g5p_ru_rt_event_match(words, d->td_want & 0xff)) ++ WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); ++ if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) ++ WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); ++ for (u32 e = 0; words[0] == 1 && e < 2; e++) { ++ if (!(words[1] & BIT(e))) ++ continue; ++ WRITE_ONCE(d->t1_count[e], d->t1_count[e] + 1); ++ if (!d->t1_first_ns[e]) ++ WRITE_ONCE(d->t1_first_ns[e], ktime_get_ns()); ++ } ++ if (words[0] != 0 && words[0] != 1) { ++ d->last_error = -EIO; ++ if (!d->fault_serial) { ++ d->fault_serial = d->event_serial; ++ memcpy(d->first_fault, words, sizeof(d->first_fault)); ++ } ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ wake_up(&d->ev_wq); ++} ++ ++static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = kzalloc(sizeof(*f), GFP_KERNEL); ++ ++ if (!f) ++ return -ENOMEM; ++ mutex_init(&f->lock); ++ xa_init_flags(&f->vms, XA_FLAGS_ALLOC1); ++ xa_init_flags(&f->contexts, XA_FLAGS_ALLOC1); ++ xa_init(&f->results); ++ xa_init_flags(&f->targets, XA_FLAGS_ALLOC1); ++ file->driver_priv = f; ++ atomic_inc(&to_g5p(drm)->open_files); ++ return 0; ++} ++ ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_vm *vm; ++ unsigned long id; ++ struct g5p_rt *rt; ++ ++ atomic_dec(&to_g5p(drm)->open_files); ++ /* Render-target handles hold their context; drop them first. */ ++ xa_for_each(&f->targets, id, rt) ++ g5p_context_put(rt->ctx); ++ xa_destroy(&f->targets); ++ /* Jobs hold contexts, contexts hold VMs, mappings hold GEM objects. ++ * Closing the fd drops handle ownership, never a queued job's references. ++ * Host-only tables may retire at job completion: no live GPU attachment. ++ */ ++ xa_for_each(&f->contexts, id, c) { ++ /* A render context's firmware binding outlives the handle until its close ++ * item, queued behind its jobs, has unbound it. ++ */ ++ if (c->render) ++ g5p_render_close(c); ++ g5p_context_put(c); ++ } ++ xa_destroy(&f->contexts); ++ { ++ struct g5p_fence *fence; ++ ++ xa_for_each(&f->results, id, fence) ++ dma_fence_put(&fence->base); ++ xa_destroy(&f->results); ++ } ++ xa_for_each(&f->vms, id, vm) ++ g5p_vm_put(vm); ++ xa_destroy(&f->vms); ++ mutex_destroy(&f->lock); ++ kfree(f); ++ /* The file's handles are gone (drm_gem_release() ran first): what stays live is held ++ * by another file's handle, a dma-buf, a VM or a job (debugfs g5p_bos names it). ++ */ ++ dev_info(drm->dev, ++ "G5P file close live_bos=%u live_bo_bytes=%lluK device_pinned=%lluK\n", ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++} ++ ++static const struct drm_ioctl_desc g5p_ioctls[] = { ++ DRM_IOCTL_DEF_DRV(G5P_GET_INFO, g5p_get_info, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_BO, g5p_create_bo, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_MMAP_OFFSET, g5p_bo_mmap_offset, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_SYNC, g5p_bo_sync, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_VM, g5p_create_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_VM, g5p_destroy_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_VM_BIND, g5p_vm_bind, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_CONTEXT, g5p_create_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_CONTEXT, g5p_destroy_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_EXEC_INFO, g5p_get_exec_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_JOB_STATUS, g5p_get_job_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_CONTEXT_STATUS, g5p_get_context_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_FAULT_INFO, g5p_get_fault_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_RENDER_TARGET, g5p_create_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_RENDER_TARGET, g5p_destroy_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_RENDER_INFO, g5p_get_render_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_RENDER_TYPED, g5p_submit_render_typed, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_COMPUTE_TYPED, g5p_submit_compute_typed, ++ DRM_RENDER_ALLOW), ++}; ++ ++DEFINE_DRM_GEM_FOPS(g5p_drm_fops); ++ ++static const struct drm_driver g5p_drm_driver = { ++ .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | ++ DRIVER_SYNCOBJ_TIMELINE, ++ .open = g5p_drm_open, ++ .postclose = g5p_drm_postclose, ++ .fops = &g5p_drm_fops, ++ .ioctls = g5p_ioctls, ++ .num_ioctls = ARRAY_SIZE(g5p_ioctls), ++ .gem_create_object = g5p_gem_create_object, ++ .prime_handle_to_fd = g5p_prime_handle_to_fd, ++ .prime_fd_to_handle = g5p_prime_fd_to_handle, ++ .gem_prime_import = g5p_gem_prime_import, ++ .gem_prime_import_sg_table = g5p_prime_import_sg_table, ++ .name = "apple_g5p", ++ .desc = "Apple A9 G5P experimental render driver", ++ .major = 0, ++ .minor = DRM_G5P_UAPI_VERSION, ++}; ++ ++/* recovery108: the frontend's epoch inputs for the retire layer. RAM only and lock free: ++ * the claimed context is compared, never dereferenced (it may be dropped concurrently). ++ */ ++static void g5p_drm_r8_epoch(void *cookie, struct g5p_r8_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ in->claimed = READ_ONCE(d->render_claimed) || ++ READ_ONCE(d->quarantined_context); ++ in->held = in->claimed && atomic_read(&d->open_files) > 0; ++ in->queued = queued > 0 ? queued : 0; ++} ++ ++/* The teardown DETACH phase. No fence is signalled here: with admission closed and the ++ * device frozen, the render scheduler fails each queued job exactly once (-ESHUTDOWN, ++ * without waiting for its inputs), and an in-flight job ends on its own RAM-only ++ * deadline. The count left to the workers is reported as the explicit abandonment. ++ */ ++static void g5p_drm_td_detach(void *cookie) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ g5p_r8_note_detach(&g5p_r8_state, queued > 0 ? queued : 0); ++ g5p_rsched_kick(d); ++} ++ ++static bool autoboot = true; ++module_param(autoboot, bool, 0400); ++MODULE_PARM_DESC(autoboot, "bring the GPU up at probe with the command file's phase sequence (default true)"); ++static unsigned int autoboot_wait_s = 120; ++module_param(autoboot_wait_s, uint, 0400); ++MODULE_PARM_DESC(autoboot_wait_s, ++ "seconds the probe-time bring-up waits for " G5P_FW_NAME " to become readable (default 120)"); ++ ++/* The bootstrap's order: the one-time firmware, wake and identity steps, then the stages ++ * recovery repeats (recover.inc). ++ */ ++static const char *const g5p_autoboot_phases[] = { ++ "validate", "prepare", "wake", "id", "sample", "map", "arm", "load", "release", ++ "events", "control-prepare", "control-submit", "control-wait", ++}; ++ ++static void g5p_autoboot_work(struct work_struct *work) ++{ ++ struct g5p_drm *d = container_of(to_delayed_work(work), struct g5p_drm, autoboot); ++ struct g5p_diag *g = d->core; ++ bool ran; ++ int ret; ++ ++ while (d->autoboot_phase < ARRAY_SIZE(g5p_autoboot_phases)) { ++ const char *phase = g5p_autoboot_phases[d->autoboot_phase]; ++ ++ ret = g5p_run_phase(g, phase, &ran); ++ /* Built in: the firmware is on the root file system, which may not be mounted yet. */ ++ if (ret == -ENOENT && !d->autoboot_phase && ++ time_before(jiffies, d->autoboot_deadline)) { ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 2 * HZ); ++ return; ++ } ++ dev_info(g->dev, "autoboot phase=%s result=%d\n", phase, ret); ++ if (ret) { ++ dev_warn(g->dev, "autoboot stopped at %s (%d); the command file can continue\n", ++ phase, ret); ++ return; ++ } ++ d->autoboot_phase++; ++ } ++ dev_info(g->dev, "autoboot complete: GPU up\n"); ++} ++ ++static int g5p_drm_probe(struct platform_device *pdev) ++{ ++ struct g5p_drm *d; ++ struct g5p_diag *g; ++ int ret; ++ ++ ; ++ ; ++ ret = g5p_probe(pdev); ++ if (ret) ++ return ret; ++ g = platform_get_drvdata(pdev); ++ d = devm_drm_dev_alloc(&pdev->dev, &g5p_drm_driver, struct g5p_drm, ++ drm); ++ if (IS_ERR(d)) { ++ ret = PTR_ERR(d); ++ goto err_pm; ++ } ++ d->core = g; ++ d->fence_context = dma_fence_context_alloc(1); ++ spin_lock_init(&d->status_lock); ++ init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->accepted, 0); ++ atomic64_set(&d->job_sequence, 0); ++ d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", 0); ++ if (IS_ERR_OR_NULL(d->render_wq)) { ++ ret = d->render_wq ? PTR_ERR(d->render_wq) : -ENOMEM; ++ goto err_pm; ++ } ++ ret = drmm_mutex_init(&d->drm, &d->render_submit_lock); ++ if (!ret) ++ ret = drmm_mutex_init(&d->drm, &d->pin_lock); ++ if (ret) ++ goto err_pm; ++ ret = g5p_render_init(d); ++ if (ret) ++ goto err_pm; ++ g->event_cookie = d; ++ g->event_notify = g5p_drm_event; ++ atomic_set(&d->open_files, 0); ++ atomic_set(&d->r8_queued, 0); ++ drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); ++ drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); ++ ret = drm_dev_register(&d->drm, 0); ++ if (ret) ++ goto err_pm; ++ g5p_r8_register_epoch(g5p_drm_r8_epoch, d); ++ g5p_td_register_detach(g5p_drm_td_detach, d); ++ g5p_kq_register(&g5p_kq_drm, d); ++ dev_info(&pdev->dev, ++ "render node registered; no GPU access, COMPUTE/RENDER capabilities absent\n"); ++ INIT_DELAYED_WORK(&d->autoboot, g5p_autoboot_work); ++ if (READ_ONCE(autoboot)) { ++ d->autoboot_deadline = jiffies + secs_to_jiffies(autoboot_wait_s); ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 0); ++ } ++ return 0; ++err_pm: ++ g->event_notify = NULL; ++ pm_runtime_disable(&pdev->dev); ++ return ret; ++} ++ ++static void g5p_drm_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g->event_cookie; ++ ++ cancel_delayed_work_sync(&d->autoboot); ++ drm_dev_unplug(&d->drm); ++ g5p_remove(pdev); ++ /* recovery108: the remove-time FINAL above was the callbacks' last use; d is ++ * devres-freed next. ++ */ ++ g5p_td_register_detach(NULL, NULL); ++ g5p_r8_register_epoch(NULL, NULL); ++ g5p_kq_register(NULL, NULL); ++} ++ ++static void g5p_drm_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g ? g->event_cookie : NULL; ++ ++ if (d) ++ cancel_delayed_work_sync(&d->autoboot); ++ g5p_shutdown(pdev); ++} ++ ++static const struct of_device_id g5p_drm_match[] = { ++ { .compatible = "gpu,s8003" }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, g5p_drm_match); ++ ++static struct platform_driver g5p_drm_platform = { ++ .probe = g5p_drm_probe, ++ .remove = g5p_drm_remove, ++ .shutdown = g5p_drm_shutdown, ++ .driver = { ++ .name = "apple-g5p-drm", ++ .of_match_table = g5p_drm_match, ++ .dev_groups = g5p_groups, ++ .suppress_bind_attrs = true, ++ }, ++}; ++ ++static bool enable = true; ++static bool drm_initialized; ++static bool drm_registered; ++ ++static int g5p_drm_enable_set(const char *value, const struct kernel_param *kp) ++{ ++ bool requested; ++ int ret = kstrtobool(value, &requested); ++ ++ if (ret) ++ return ret; ++ if (!drm_initialized) { ++ enable = requested; ++ return 0; ++ } ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (!requested && drm_registered) ++ return -EBUSY; ++ if (requested && !drm_registered) { ++ ret = g5p_frontend_available(); ++ if (ret) ++ return ret; ++ ret = platform_driver_register(&g5p_drm_platform); ++ if (ret) ++ return ret; ++ drm_registered = true; ++ } ++ enable = requested; ++ return 0; ++} ++ ++static const struct kernel_param_ops g5p_drm_enable_ops = { ++ .set = g5p_drm_enable_set, ++ .get = param_get_bool, ++}; ++module_param_cb(enable, &g5p_drm_enable_ops, &enable, 0600); ++MODULE_PARM_DESC(enable, ++ "Register the G5P DRM frontend (default true); conflicts with diagnostic binding"); ++ ++static int __init g5p_drm_init(void) ++{ ++ drm_initialized = true; ++ if (!enable) { ++ pr_info("apple-g5p-drm: inert; explicit a9_gpu_drm.enable=1 required\n"); ++ return 0; ++ } ++ return g5p_drm_enable_set("1", NULL); ++} ++module_init(g5p_drm_init); ++ ++/* Built-in only until reset can prove all DMA references are quiescent. */ ++MODULE_DESCRIPTION("Default-disabled Apple G5P render/GEM/VM experimental driver"); ++MODULE_LICENSE("Dual MIT/GPL"); ++MODULE_FIRMWARE(G5P_FW_NAME); +diff --git a/drivers/gpu/drm/apple-g5p/cache-policy.h b/drivers/gpu/drm/apple-g5p/cache-policy.h +new file mode 100644 +index 000000000..40f5cdfe1 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-policy.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* No caller may infer CPU caching from BO size or GPU use alone. */ ++#ifndef G5P_CACHE_POLICY_H ++#define G5P_CACHE_POLICY_H ++static inline int g5p_cache_class(int imported, int exported, int wc) ++{ ++ return imported ? 2 : (!exported && wc ? 0 : 1); ++} ++static inline int g5p_cache_omit(int enabled, int imported, int exported, int wc) ++{ ++ return enabled && g5p_cache_class(imported, exported, wc) == 0; ++} ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/cache-receipt.inc b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +new file mode 100644 +index 000000000..c791dd260 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +@@ -0,0 +1,46 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Bounded RAM-only records. No register accesses, default off. */ ++#include ++#include "cache-policy.h" ++/* Qualified by kernel-cost2b and combined-bench2c; conservative off switches remain. */ ++static bool render_user_cache_import_wc = true; ++module_param(render_user_cache_import_wc, bool, 0600); ++MODULE_PARM_DESC(render_user_cache_import_wc, "Omit per-job maintenance only for explicitly CPU-uncached native GEM exporters; default true"); ++static bool render_user_cache_read_va = true; ++module_param(render_user_cache_read_va, bool, 0600); ++MODULE_PARM_DESC(render_user_cache_read_va, "Validate native WC command bytes without redundant sync; range-sync conservative mappings; default true"); ++static bool render_user_cache_private_wc = true; ++module_param(render_user_cache_private_wc, bool, 0600); ++MODULE_PARM_DESC(render_user_cache_private_wc, "Candidate: omit repeated maintenance only for native private write-combined BOs; imports/exports unchanged; default true"); ++static bool render_user_cache_receipts; ++module_param(render_user_cache_receipts, bool, 0600); ++MODULE_PARM_DESC(render_user_cache_receipts, "Bounded RAM-only per-job CPU/cache receipts, default false"); ++struct g5p_cache_receipt { ++ u64 job, start_cpu, start_wall, total_cpu, total_wall; ++ u64 requested[2][3], maintained[2][3], cpu[2][3], wall[2][3]; ++ u32 pid, serial; ++ bool enabled, policy, import_policy; ++}; ++#define G5P_CACHE_RECORDS 128 ++static struct g5p_cache_receipt g5p_cache_ring[G5P_CACHE_RECORDS]; ++static DEFINE_SPINLOCK(g5p_cache_ring_lock); ++static u32 g5p_cache_head; ++static int g5p_cache_show(struct seq_file *m, void *data) ++{ ++ struct g5p_cache_receipt r; ++ u32 head = READ_ONCE(g5p_cache_head), begin = head > G5P_CACHE_RECORDS ? head - G5P_CACHE_RECORDS : 0; ++ unsigned long flags; ++ seq_puts(m, "v=1 classes=private-wc,native-other,import time=task-runtime-ns wall=elapsed-ns\n"); ++ for (u32 n = begin; n < head; n++) { ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ r = g5p_cache_ring[n % G5P_CACHE_RECORDS]; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ if (r.serial != n + 1) continue; /* overwritten during this debugfs read */ ++ seq_printf(m, "job=%llu pid=%u policy=%u total_cpu_ns=%llu total_wall_ns=%llu", r.job, r.pid, r.policy, r.total_cpu, r.total_wall); ++ for (u32 phase = 0; phase < 2; phase++) ++ for (u32 c = 0; c < 3; c++) ++ seq_printf(m, " p%uc%u=%llu,%llu,%llu,%llu", phase, c, r.requested[phase][c], r.maintained[phase][c], r.cpu[phase][c], r.wall[phase][c]); ++ seq_putc(m, '\n'); ++ } ++ return 0; ++} +diff --git a/drivers/gpu/drm/apple-g5p/compute-user-validate.h b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +new file mode 100644 +index 000000000..ac16d4d0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +@@ -0,0 +1,141 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_VALIDATE_H ++#define G5P_COMPUTE_USER_VALIDATE_H ++ ++/* ++ * TYPED_COMPUTE pass checks over already-resolved resource ++ * ranges, as render-user-validate.h does for TYPED_V3. Pure, host-tested ++ * (scripts/gpu/compute-host/test_compute.c). Nothing here is a pointer the kernel ++ * dereferences; the stream's last word is read by the caller through the BO's own pages ++ * (g5p_cu_content()). Like TYPED_V3 the kernel does not decode the CDM records, the PDS ++ * programs or the USC code they name; it bounds every address the firmware or the CDM ++ * starts from and leaves the rest to the per-context VM and fault isolation. ++ * -EINVAL flags, alignment, sizes, a non-zero terminate control word, no terminator ++ * -ERANGE a PDS reference outside [pds_base, pds_base + code_window), or a range past ++ * hw_va_limit ++ * -ENOENT no listed range wholly covers the stream, a state word or a PDS reference ++ * -EACCES the range lacks READ (stream, PDS data), WRITE (state) or EXEC (PDS code) ++ */ ++#define G5P_CU_MAX_STREAM 0x10000U ++#define G5P_CU_STREAM_MIN 4U ++#define G5P_CU_TERMINATE 0x80000000U ++ ++/* The pass as the UAPI carries it, kernel copy. */ ++struct g5p_cu_pass { ++ u64 cdm_va; ++ u32 cdm_size, cdm_end; ++ u64 state[4]; ++ u64 tail; ++ u32 config[2]; ++ u32 store[3], resume[3], term[3]; /* code, data, control */ ++}; ++ ++/* Where the content rule reads: the range holding the stream. */ ++struct g5p_cu_loc { ++ u32 stream_range; ++}; ++ ++/* One {code, data} PDS pair: code in a READ|EXEC range, data in a READ range (g5p_ru_v3_ref()). */ ++static inline int g5p_cu_pds(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ const u32 *pds) ++{ ++ int ret = g5p_ru_v3_ref(lim, r, n, pds[0], true); ++ ++ return ret ?: g5p_ru_v3_ref(lim, r, n, pds[1], false); ++} ++ ++static inline int g5p_cu_pass_check(const struct g5p_cu_pass *p, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_cu_loc *loc) ++{ ++ const u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ u32 si; ++ int ret; ++ ++ if (p->cdm_size < G5P_CU_STREAM_MIN || ++ p->cdm_size > G5P_CU_MAX_STREAM || ++ ((p->cdm_va | p->cdm_size | p->cdm_end) & 3) || ++ p->cdm_end < G5P_CU_STREAM_MIN || p->cdm_end > p->cdm_size) ++ return -EINVAL; ++ if ((p->tail & 0xf) || p->term[2]) ++ return -EINVAL; ++ for (u32 i = 0; i < 4; i++) ++ if (p->state[i] & 3) ++ return -EINVAL; ++ /* Every listed range, as TYPED_V3: the CDM and its tasks read and write past them. */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, p->cdm_va, p->cdm_size, &si)) ++ return -ENOENT; ++ if (!(r[si].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ for (u32 i = 0; i < 5; i++) { ++ u64 va = i < 4 ? p->state[i] : p->tail; ++ u32 k; ++ ++ if (!g5p_ru_cover(r, n, va, i < 4 ? 4 : 16, &k)) ++ return -ENOENT; ++ if ((r[k].access & rw) != rw) ++ return -EACCES; ++ } ++ ret = g5p_cu_pds(lim, r, n, p->store); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->resume); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->term); ++ if (ret) ++ return ret; ++ loc->stream_range = si; ++ return 0; ++} ++ ++/* The last stream dword, at cdm_va + cdm_end - 4, is the CDM terminate word. */ ++static inline int g5p_cu_content(const u8 *tail) ++{ ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ return word == G5P_CU_TERMINATE ? 0 : -EINVAL; ++} ++ ++/* The firmware's register images of a checked pass. */ ++static inline void g5p_cu_regs_of(struct g5p_cu_regs *k, ++ const struct g5p_cu_pass *p) ++{ ++ k->stream = p->cdm_va; ++ k->terminate = p->cdm_va + p->cdm_end - 4; ++ for (u32 i = 0; i < 4; i++) ++ k->state[i] = p->state[i]; ++ k->tail = p->tail; ++ k->config[0] = p->config[0]; ++ k->config[1] = p->config[1]; ++ k->store = p->store[0] | (u64)p->store[1] << 32; ++ k->resume = p->resume[0] | (u64)p->resume[1] << 32; ++ k->term = p->term[0] | (u64)p->term[1] << 32; ++ k->store1 = p->store[2]; ++ k->resume1 = p->resume[2]; ++} ++ ++/* ++ * Submit-time admission of a compute job, on top of g5p_ru_v3_admit(): the CDM rides an epoch ++ * a render job brought up (the attach rings the TA bell; a compute job never does), and its ++ * own sequence is bounded like the render one. ++ */ ++enum g5p_cu_refusal { ++ G5P_CU_OK, ++ G5P_CU_NO_EPOCH, /* no render job has brought the device up in this epoch */ ++ G5P_CU_OVERFLOW, /* the CDM sequence is exhausted */ ++}; ++ ++static inline int g5p_cu_admit(bool device_up, u32 cu_sequence) ++{ ++ if (!device_up) ++ return G5P_CU_NO_EPOCH; ++ if (cu_sequence >= G5P_CU_SEQ_MAX) ++ return G5P_CU_OVERFLOW; ++ return G5P_CU_OK; ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.h b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +new file mode 100644 +index 000000000..3531cefb7 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_COMPUTE (drm-compute-v1.inc): what the TYPED_V3 executor, the scheduler's drain and the ++ * render device's bring-up call before the file is included. ++ */ ++#ifndef G5P_DRM_COMPUTE_V1_H ++#define G5P_DRM_COMPUTE_V1_H ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rjob; ++ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job); ++static bool g5p_cu_drain(struct g5p_drm *d); ++static void g5p_cu_device_init(struct g5p_drm *d); ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +new file mode 100644 +index 000000000..dcbe34c0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -0,0 +1,720 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * TYPED_COMPUTE: compute jobs on TYPED_V3 render contexts. ++ * ++ * A compute job is a TYPED_V3 job of its render context with one phase on the device's CDM ++ * channel instead of a TA and a 3D: it is queued, scheduled, fenced, quarantined and lost ++ * exactly like a render job (g5p_render_run(), drm-render-sched.inc, render-user-fault.h). ++ * It binds the context's slot pair the RAM-only way if the context has not run a job yet, ++ * builds the CDM job (compute-user-layout.h) from the pass the submit validated ++ * (compute-user-validate.h), publishes it on the CL ring with a guarded DM2 wake, and waits RAM only ++ * for coherent channel watermarks and both completion stamps. It never brings the device up (that rings the TA bell): ++ * an epoch without a render job refuses it before the hardware. ++ * ++ * Only the licensed publish-time wake touches registers; validation, waits, ++ * samples, retirement and recovery decisions are RAM-only. ++ */ ++ ++static u8 *g5p_cu_page(struct g5p_boot *b) ++{ ++ return b->shared[G5P_CU_OFFSET / G5P_GPU_PAGE].cpu + ++ G5P_CU_OFFSET % G5P_GPU_PAGE; ++} ++ ++static u32 g5p_cu_meta(void) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + G5P_CU_OFFSET); ++} ++ ++/* The CDM's global object (the boot layout's engine-2 object, R0 group, CPU 0x49 / GPU 0x53). */ ++static u32 g5p_cu_global_meta(struct g5p_boot *b) ++{ ++ return g5p_meta_address( ++ g5p_object_va(&b->layout, 0x51 + G5P_CU_ENGINE)); ++} ++ ++/* The render device's bring-up (g5p_render_device_prepare()), after the TA/3D channel init that ++ * clears the 3D page: the CDM channel at its epoch start. RAM only. ++ */ ++static void g5p_cu_device_init(struct g5p_drm *d) ++{ ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_R22_DEP + 0x280); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_RU_SLOT1_END); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE + G5P_CU_BYTES > ++ G5P_GPU_PAGE); ++ BUILD_BUG_ON(G5P_CU_DEP + G5P_CU_DEP_BYTES > G5P_CU_BYTES); ++ BUILD_BUG_ON(G5P_CU_W + G5P_CU_W_BYTES > G5P_CU_SKU); ++ g5p_cu_channel_init(g5p_cu_page(d->core->boot), g5p_cu_meta()); ++ d->rdev->cu_sequence = 0; ++ d->rdev->cu_target = 0; ++} ++ ++/* One RAM sample of the CDM channel for job `seq` (the slots of struct g5p_rv3_ev: stamp[0] = ++ * W258 mirror, stamp[1] = W25c, ctl = the five watermarks, idle[0..2] = the CDM global's ++ * G724/728/72c). RAM only. ++ */ ++static void g5p_cu_sample(struct g5p_diag *g, u32 seq, u32 target, ++ struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_cu_page(b), *global = g5p_ru_global(b, G5P_CU_ENGINE); ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ s->stamp[0] = g5p_le32(p + G5P_CU_START); ++ s->stamp[1] = g5p_le32(p + G5P_CU_END); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < 3; i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != target || s->cons > 255 || ++ !g5p_cu_stamp_sane(p, seq); ++ s->consumed = s->cons == target; ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2]; ++ s->retired = g5p_cu_retired(p, g5p_cu_meta(), seq); ++ s->pm_ok = true; ++} ++ ++#define G5P_CU_SLOW_MS 1000U ++ ++/* The CDM phase's RAM-only wait, the TYPED_V3 wait (g5p_rv3_wait()) for engine 2: the silence ++ * clock, the fault/protocol rules and the closing sample are the same; complete is the ring ++ * consumed plus the retirement predicate (the idle words are recorded, not required: render path). ++ */ ++/* Event serial is sampled before RAM, so an IRQ between sample and sleep ++ * cannot be lost. Any firmware event wakes us; only the full RAM retirement ++ * predicate may complete the job. No render-only engine-indexed observers. ++ */ ++static void g5p_cu_pause(struct g5p_drm *d, u64 seen) ++{ ++ u32 wait_us; ++ ++ if (g5p_ew_plan(READ_ONCE(render_user_event_wake), false, &wait_us) == ++ G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ return; ++ } ++ wait_event_hrtimeout(d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us)); ++} ++ ++static u32 g5p_cu_wait(struct g5p_drm *d, struct g5p_diag *g, u32 seq, ++ u32 target, struct g5p_ru_wait *w, struct g5p_rv3_ev *s, ++ u64 *consumed_ns, int *err) ++{ ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE(d->event_serial); ++ bool fault, silent, done; ++ ++ g5p_cu_sample(g, seq, target, s); ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed || silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = s->consumed && s->retired; ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (w->result) ++ return w->result; ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_CU_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ dev_warn( ++ g->dev, ++ "compute slow seq=%u ms=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x consumed=%u retired=%u events=%u\n", ++ seq, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, ++ s->cons, s->stamp[0], s->stamp[1], s->ctl[0], ++ s->ctl[1], s->ctl[2], s->ctl[3], s->ctl[4], ++ s->idle[0], s->idle[1], s->idle[2], s->consumed, ++ s->retired, READ_ONCE(g->boot->events)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ g5p_cu_pause(d, seen); ++ } ++} ++ ++/* The guarded publish-time wake, identical ordering to TA/3D with DM2. ++ * Core lock held; only this helper may touch registers in the compute path. ++ * The event lock closes the reset/fault race. Never called from wait/drain. ++ */ ++static int g5p_cu_wake(struct g5p_diag *g, u32 seq, u32 start, u32 target) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ u32 why, fence = U32_MAX, gpc = U32_MAX; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = G5P_WK_ALWAYS, ++ .e = G5P_CU_ENGINE, ++ .seq = seq, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = target, ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_CU_START), ++ g5p_le32(p + G5P_CU_END) }, ++ }; ++ why = g5p_wk_guard(&in); ++ if (!why) ++ why = g5p_wk_ring(&io, G5P_CU_ENGINE, &fence, &gpc); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_dbg(g->dev, ++ "compute wake seq=%u did=%u power=%#x fence=%#x gpc=%#x\n", seq, ++ why, in.power, fence, gpc); ++ return why == G5P_WK_DID_RANG ? 0 : -EHOSTDOWN; ++} ++ ++/* The CL publish of job `seq`: the global ring entry {type 2, channel, endpoint}, the channel's ++ * admission watermark, then the producer, followed by the guarded DM2 wake. Refused, with nothing written, unless the ++ * ring is idle and the channel stands at the previous job's endpoint and stamps. RAM only. ++ */ ++static int g5p_cu_publish(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_cu_endpoint(seq - 1); ++ u32 endpoint = g5p_cu_endpoint(seq), why; ++ ++ why = (u32)(start > 255) | (u32)(READ_ONCE(ring[0]) != start) << 1 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 2 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 3 | ++ (u32)(g5p_le32(p + G5P_CU_START) != seq - 1 || ++ g5p_le32(p + G5P_CU_END) != seq - 1) ++ << 4; ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "compute refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x stamp=%08x/%08x: nothing published\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + G5P_CU_START), g5p_le32(p + G5P_CU_END)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x69] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 2 : ++ j == 1 ? g5p_cu_meta() : ++ j == 2 ? endpoint : ++ 0); ++ } ++ rd->cu_target = (start + 1) & 255; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], rd->cu_target); ++ dma_wmb(); ++ if (g5p_ru_v3_verbose(r->jobs, 0)) ++ dev_dbg(g->dev, ++ "compute publish slot=%u endpoint=%u ring=%u W=%08x sequence=%u kick=dm2\n", ++ r->slot[0], endpoint, start, g5p_cu_meta() + G5P_CU_W, ++ seq); ++ return g5p_cu_wake(g, seq, start, rd->cu_target); ++} ++ ++/* The CDM phase timed out with the firmware alive: lose the job and its context, block the ++ * channel behind it until the scheduler's drain (g5p_cu_drain()) sees it retire. RAM only. ++ */ ++static int g5p_cu_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = G5P_CU_ENGINE; ++ s->drain_ms = 45000; ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "compute lost seq=%u slot=%u phase_ms=%lld drain_ms=%u: context lost, device kept, channel blocked\n", ++ seq, r->slot[0], ++ g5p_rv3_us(v->ta.start_ns, v->ta.end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++static void g5p_cu_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ const struct g5p_rv3 *v = &job->rv3; ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ dev_dbg(g->dev, ++ "compute job=%llu seq=%u ret=%d slot=%u consumed_us=%lld done_us=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x lost=%u bind_failed=%u\n", ++ job->id, v->seq, ret, r->slot[0], ++ g5p_rv3_us(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_us(v->ta_pub_ns, v->ta.end_ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ v->lost, v->bind_failed); ++} ++ ++/* One compute job, core lock held (g5p_render_run() via g5p_rv3_execute()). Returns 0, -EAGAIN ++ * (no epoch: nothing touched), -ETIMEDOUT, or a fault. ++ */ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (r->stage_done) ++ return -ECANCELED; ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ why = g5p_cu_admit(READ_ONCE(rd->up), rd->cu_sequence); ++ if (why) ++ return why == G5P_CU_NO_EPOCH ? -EAGAIN : -EOVERFLOW; ++ seq = rd->cu_sequence + 1; ++ if (!r->bound) { ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ } ++ /* The firmware owns the CL ring and the channel from here on: any failure quarantines. */ ++ v->touched = true; ++ g5p_cu_job(g5p_cu_page(b), g5p_cu_meta(), g5p_cu_global_meta(b), ++ r->slot[0], seq, &job->cu); ++ g5p_render_clean_job(g); ++ rd->cu_sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ ret = g5p_cu_publish(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, v->job_deadline_ns); ++ res = g5p_cu_wait(d, g, seq, rd->cu_target, &v->ta, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ /* The TA/3D paths' MMU fault-bank read, for the CDM (engine 2). */ ++ g5p_rv3_fault_read(d, r, seq, G5P_CU_ENGINE, res, v->ta.silent); ++ loss = g5p_rv3_loss(d, r, false, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && !g5p_cu_lose(d, r, v, seq)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, false)) { ++ g5p_rv3_lose_recover(d, r, v, seq, G5P_CU_ENGINE, ++ v->ta.silent ? "cdm-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "cdm-timeout" : ++ "cdm-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal(g, ++ res == G5P_RU_WAIT_TIMEOUT ? ++ G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "cdm-timeout" : ++ "cdm-fault"); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ if (!v->last[0].idle_ok) ++ dev_warn( ++ g->dev, ++ "compute retired seq=%u with idle=%08x,%08x,%08x set\n", ++ seq, v->last[0].idle[0], v->last[0].idle[1], ++ v->last[0].idle[2]); ++ v->phase = G5P_RV3_DONE; ++ ret = 0; ++out: ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), "cdm"); ++ g5p_cu_receipt(g, r, job, ret); ++ return ret; ++} ++ ++/* The scheduler's drain for a lost compute job (g5p_rv3_drain() hands engine 2 here): wait, RAM ++ * only and for the drain bound, for the stuck job to retire. It did: the channel is clear and ++ * the held jobs run. It did not (silent, fault, bound): the device recovers if it may, else the ++ * epoch ends. Returns false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_cu_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ struct g5p_ru_wait w = {}; ++ u32 res = G5P_RU_WAIT_FAULT; ++ int err = -EIO; ++ u64 now; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ now = ktime_get_ns(); ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (!READ_ONCE(b->failed) && !READ_ONCE(b->stopped)) ++ res = g5p_cu_wait(d, g, s->seq, rd->cu_target, &w, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res == G5P_RU_WAIT_COMPLETE) { ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "compute drained seq=%u slot=%u lost_ms=%lld: channel clear, held jobs resume\n", ++ s->seq, r->slot[0], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "compute drain dead seq=%u why=%s err=%d lost_ms=%lld: the device recovers if it may, else the epoch ends\n", ++ s->seq, ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal(g, ++ fault ? G5P_RU_TERM_FAULT : ++ G5P_RU_TERM_TIMEOUT, ++ fault ? "cdm-drain-fault" : ++ "cdm-drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++static void g5p_cu_pass_from_uapi(struct g5p_cu_pass *p, ++ const struct drm_g5p_compute_pass *u) ++{ ++ *p = (struct g5p_cu_pass){ ++ .cdm_va = u->cdm_va, ++ .cdm_size = u->cdm_size, ++ .cdm_end = u->cdm_end, ++ .tail = u->state_tail_va, ++ .config = { u->config[0], u->config[1] }, ++ .store = { u->store.code, u->store.data, u->store.control }, ++ .resume = { u->resume.code, u->resume.data, u->resume.control }, ++ .term = { u->terminate.code, u->terminate.data, ++ u->terminate.control }, ++ }; ++ for (u32 i = 0; i < 4; i++) ++ p->state[i] = u->state_va[i]; ++} ++ ++static bool g5p_cu_uapi_clean(const struct drm_g5p_compute_pass *u) ++{ ++ return u->size == sizeof(*u) && !u->flags && !u->store.pad && ++ !u->resume.pad && !u->terminate.pad && ++ !memchr_inv(u->reserved, 0, sizeof(u->reserved)); ++} ++ ++/* Acceptance, as g5p_render_submit_admitted() for TYPED_V3 minus the render target: every error ++ * precedes the fence replacement and the queue. ++ */ ++static int g5p_cu_submit_admitted(struct drm_device *drm, struct drm_file *file, ++ const struct drm_g5p_submit_compute_typed *a, ++ u64 *job_id) ++{ ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_compute_pass *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_cu_pass pass; ++ struct g5p_cu_loc loc = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ u8 tail[4]; ++ u64 point = 0; ++ int ret; ++ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(a->resources), ++ a->resource_count, sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(a->syncs), a->sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ upass = memdup_user(u64_to_user_ptr(a->pass), sizeof(*upass)); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (!g5p_cu_uapi_clean(upass)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_cu_pass_from_uapi(&pass, upass); ++ ranges = kvcalloc(a->resource_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(a->resource_count, sizeof(*job->objs), ++ GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, a->sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ /* Compute rides an epoch a render job brought up; a recovery in progress is held. */ ++ if (!READ_ONCE(d->rdev->recovering) && ++ g5p_cu_admit(READ_ONCE(d->rdev->up), ++ READ_ONCE(d->rdev->cu_sequence))) { ++ ret = READ_ONCE(d->rdev->up) ? -EOVERFLOW : -EAGAIN; ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, a->resource_count, ranges, ++ job->objs, &job->obj_count); ++ if (ret) ++ goto unlock; ++ /* Structure first; the stream's last dword is read only once its range is proven backed. */ ++ ret = g5p_cu_pass_check(&pass, &lim, ranges, a->resource_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass.cdm_va + pass.cdm_end - 4, tail, ++ sizeof(tail)); ++ if (!ret) ++ ret = g5p_cu_content(tail); ++ if (ret) ++ goto unlock; ++ g5p_cu_regs_of(&job->cu, &pass); ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = true; ++ job->compute = true; ++ r->v3_accepted = true; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++static_assert(sizeof(struct drm_g5p_compute_pds) == 16); ++static_assert(sizeof(struct drm_g5p_compute_pass) == 152); ++static_assert(offsetof(struct drm_g5p_compute_pass, state_va) == 24); ++static_assert(offsetof(struct drm_g5p_compute_pass, config) == 64); ++static_assert(offsetof(struct drm_g5p_compute_pass, store) == 72); ++static_assert(offsetof(struct drm_g5p_compute_pass, reserved) == 120); ++static_assert(sizeof(struct drm_g5p_submit_compute_typed) == 72); ++static_assert(DRM_G5P_CAP_TYPED_COMPUTE == BIT_ULL(30)); ++static_assert(DRM_G5P_COMPUTE_MAX_STREAM_BYTES == G5P_CU_MAX_STREAM); ++static_assert(DRM_G5P_COMPUTE_STREAM_TERMINATE == G5P_CU_TERMINATE); ++ ++static int g5p_submit_compute_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_compute_typed *a = data; ++ u64 id = 0; ++ int ret; ++ ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_compute_pass)) ++ return -EINVAL; ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_cu_submit_admitted(drm, file, a, &id); ++ g5p_r8_exit(); ++ a->job_id = id; ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +new file mode 100644 +index 000000000..085e76650 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -0,0 +1,294 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: runtime A/B knobs for the V3 render path, all default off and all writable ++ * through /sys/module/a9_gpu_drm/parameters on a live boot. Included from drm-render.inc ++ * before drm-render-v3.inc. The executor is serial and holds the core lock, so the state ++ * below needs no lock of its own. ++ * ++ * fw_tick_mask (boot.inc) ctl+0x04, re-applied before every job (RAM) ++ * slc_evict_us while a job is in flight, one pass over slc_evict_mib MiB of ++ * cacheable kernel memory at most every N us (the waiter's ++ * 1-2 ms sample cadence bounds it from below); slc_evict_ops: ++ * 1 read pass, 2 write pass, 4 Normal-NC read of the firmware's ++ * root+shared pages, 8 TLBI VMALLE1IS, 16 DSB SY + SEV ++ * cpu_sync 1 dma_sync every job BO (device before, CPU after), 2 clean ++ * root+shared to PoC before the publish, 4 clean+invalidate the ++ * polled pages before every sample ++ * lat123_log one RENDER-L123 line per job (times in ms from the submit ioctl) ++ * pstate_probe 1 PMGR gfx word at job start/end (always-on block), 2 also one ++ * read of Fender +0x1000 at the first sample that shows a fresh ++ * engine busy rise (engine running, so the block is powered) ++ * ++ * No other register access. Temporary: none of this is the fix. ++ */ ++ ++#include ++#include ++ ++#define G5P_L123_OP_READ BIT(0) ++#define G5P_L123_OP_WRITE BIT(1) ++#define G5P_L123_OP_FW_READ BIT(2) ++#define G5P_L123_OP_TLBI BIT(3) ++#define G5P_L123_OP_SEV BIT(4) ++#define G5P_L123_SYNC_BO BIT(0) ++#define G5P_L123_SYNC_REGIONS BIT(1) ++#define G5P_L123_SYNC_POLL BIT(2) ++#define G5P_L123_LINE 64 ++ ++static struct { ++ void *buf; ++ size_t size; ++ u64 sink; ++ /* per job */ ++ u64 exec_ns, next_ns, evict_ns, ps_ns; ++ u32 us, mib, ops, sync, passes, tick, ps; ++ u32 pmgr[2]; ++ bool seen[2], idle[2], probed; ++ /* render_user_event_wake: pauses ended by an event, the timeout, a post-event poll, legacy */ ++ u32 ew[4]; ++ u64 ew_ns; ++} g5p_l123; ++ ++/* perf60-fillrate.md §11.3 cut 1 (render-user-evwait.h): the phase wait's pause between RAM samples. */ ++static unsigned int render_user_event_wake = 2000; /* release8 default */ ++module_param(render_user_event_wake, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_event_wake, ++ "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " ++ "until the next firmware event (the IRQ thread's decode wakes the waiter), at " ++ "most N us (100..20000) as the safety net, and poll every 100 us once the " ++ "phase's own completion event is latched; read live"); ++ ++/* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ ++static u8 *g5p_l123_word(struct g5p_boot *b, u32 slot, u32 field) ++{ ++ u32 off = b->layout.offset[slot] + field; ++ struct g5p_dma_page *page = b->layout.group[slot] ? b->shared : b->root; ++ ++ return page[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_l123_pages(struct g5p_dma_page *page, u32 count, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ for (u32 i = 0; i < count; i++) ++ if (page[i].cpu) ++ op((unsigned long)page[i].cpu, ++ (unsigned long)page[i].cpu + G5P_GPU_PAGE); ++} ++ ++static void g5p_l123_page_of(const void *p, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ unsigned long start = (unsigned long)p & ++ ~(unsigned long)(G5P_GPU_PAGE - 1); ++ ++ op(start, start + G5P_GPU_PAGE); ++} ++ ++/* cpu_sync 4: the pages the waiter samples (rings, TA/3D/PM pages, the idle words). */ ++static void g5p_l123_poll_pages(struct g5p_boot *b, u32 e) ++{ ++ g5p_l123_page_of(g5p_boot_shared(b, 0x60), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_boot_shared(b, 0x64), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 0), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 1), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_pm_cpu(b), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_global(b, e) + 0x724, dcache_clean_inval_poc); ++ dsb(sy); ++} ++ ++static void g5p_l123_bos(struct g5p_drm *d, struct g5p_rjob *job, bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct sg_table *sgt = ++ to_drm_gem_shmem_obj(job->objs[i].obj)->sgt; ++ ++ if (!sgt) ++ continue; ++ if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ } ++} ++ ++static u32 g5p_l123_pmgr(struct g5p_diag *g) ++{ ++ u32 raw = U32_MAX; ++ ++ if (!((0) & 1) || !g->pmgr || regmap_read(g->pmgr, 0x80340, &raw)) ++ return U32_MAX; ++ return raw; ++} ++ ++/* At the executor's entry: this job's knob values and counters. */ ++static void g5p_l123_job_reset(struct g5p_diag *g) ++{ ++ g5p_l123.exec_ns = ktime_get_ns(); ++ g5p_l123.next_ns = 0; ++ g5p_l123.evict_ns = 0; ++ g5p_l123.ps_ns = 0; ++ g5p_l123.passes = 0; ++ g5p_l123.ps = U32_MAX; ++ g5p_l123.probed = false; ++ memset(g5p_l123.ew, 0, sizeof(g5p_l123.ew)); ++ g5p_l123.ew_ns = 0; ++ g5p_l123.seen[0] = false; ++ g5p_l123.seen[1] = false; ++ g5p_l123.us = (0); ++ g5p_l123.mib = clamp((16), 1U, 64U); ++ g5p_l123.ops = (1); ++ g5p_l123.sync = (0); ++ g5p_l123.pmgr[0] = g5p_l123_pmgr(g); ++ g5p_l123.pmgr[1] = U32_MAX; ++} ++ ++/* After the job's records are written, before they are cleaned and published. May sleep. */ ++static void g5p_l123_job_prepare(struct g5p_drm *d, struct g5p_rjob *job) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u8 *tick = g5p_l123_word(b, G5P_L123_CTL, 4); ++ u32 mask = (0); ++ size_t size = (size_t)g5p_l123.mib << 20; ++ ++ if (mask <= G5P_L123_TICK_MASK && g5p_le32(tick) != mask) { ++ g5p_put32(tick, mask); ++ dma_wmb(); ++ } ++ g5p_l123.tick = g5p_le32(tick); ++ ++ if (!g5p_l123.us || ++ !(g5p_l123.ops & (G5P_L123_OP_READ | G5P_L123_OP_WRITE))) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = NULL; ++ g5p_l123.size = 0; ++ } else if (g5p_l123.size != size) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = vmalloc(size); ++ g5p_l123.size = g5p_l123.buf ? size : 0; ++ if (g5p_l123.buf) ++ memset(g5p_l123.buf, 0x5a, size); ++ } ++ ++ if (g5p_l123.sync & G5P_L123_SYNC_BO) ++ g5p_l123_bos(d, job, false); ++ if (g5p_l123.sync & G5P_L123_SYNC_REGIONS) { ++ g5p_l123_pages(b->root, ARRAY_SIZE(b->root), dcache_clean_poc); ++ g5p_l123_pages(b->shared, ARRAY_SIZE(b->shared), ++ dcache_clean_poc); ++ dsb(sy); ++ } ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(b, 0); ++} ++ ++/* One streaming pass and the other per-pass operations. */ ++static void g5p_l123_evict(struct g5p_boot *b) ++{ ++ u64 t0 = ktime_get_ns(), sum = 0; ++ u32 ops = g5p_l123.ops; ++ ++ if ((ops & G5P_L123_OP_READ) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ sum += READ_ONCE(p[i]); ++ } ++ if ((ops & G5P_L123_OP_WRITE) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ WRITE_ONCE(p[i], i); ++ } ++ if (ops & G5P_L123_OP_FW_READ) { ++ struct g5p_dma_page *regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; ++ regions[r][i].cpu && at < G5P_GPU_PAGE; ++ at += G5P_L123_LINE) ++ sum += READ_ONCE(*( ++ const u32 *)(regions[r][i].cpu + ++ at)); ++ } ++ if (ops & G5P_L123_OP_TLBI) ++ flush_tlb_all(); ++ if (ops & G5P_L123_OP_SEV) { ++ dsb(sy); ++ sev(); ++ } ++ WRITE_ONCE(g5p_l123.sink, g5p_l123.sink + sum); ++ g5p_l123.passes++; ++ g5p_l123.evict_ns += ktime_get_ns() - t0; ++} ++ ++/* Replaces the waiter's sleep between two RAM samples of engine e. seen: d->event_serial as ++ * read after the drain and before this sample (render-user-evwait.h). ++ */ ++static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, ++ const struct g5p_rv3_ev *s, u64 seen) ++{ ++ u64 now = s->ns, t0; ++ enum g5p_ew_act act; ++ u32 wait_us; ++ ++ if (((0) & 2) && !g5p_l123.probed && g5p_l123.seen[e] && ++ g5p_l123.idle[e] && !s->idle_ok && !s->failed && !s->proto && ++ !g5p_ru_term_failed(&g5p_rv3_term) && !READ_ONCE(g->boot->failed)) { ++ struct g5p_mmio_ticket t; ++ ++ /* an R1-only ticket of the register licence (mmio.inc), never a bare read */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ g5p_l123.ps = t.gpc; ++ g5p_l123.ps_ns = ktime_get_ns(); ++ g5p_l123.probed = true; ++ } ++ g5p_l123.seen[e] = true; ++ g5p_l123.idle[e] = s->idle_ok; ++ if (g5p_l123.us && now >= g5p_l123.next_ns) { ++ g5p_l123_evict(g->boot); ++ g5p_l123.next_ns = now + (u64)g5p_l123.us * NSEC_PER_USEC; ++ } ++ act = g5p_ew_plan(READ_ONCE(render_user_event_wake), s->event, ++ &wait_us); ++ t0 = ktime_get_ns(); ++ if (act == G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ g5p_l123.ew[3]++; ++ } else if (act == G5P_EW_SHORT) { ++ usleep_range(wait_us, wait_us + wait_us / 2); ++ g5p_l123.ew[2]++; ++ } else if (wait_event_hrtimeout( ++ d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us))) { ++ g5p_l123.ew[1]++; /* -ETIME: the safety net */ ++ } else { ++ g5p_l123.ew[0]++; ++ } ++ g5p_l123.ew_ns += ktime_get_ns() - t0; ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(g->boot, e); ++} ++ ++/* After the receipt. One line per job while lat123_log is set. */ ++static void g5p_l123_job_end(struct g5p_drm *d, struct g5p_rjob *job, u32 seq, ++ int ret) ++{ ++ struct g5p_diag *g = d->core; ++ ++ if (!ret && (g5p_l123.sync & G5P_L123_SYNC_BO)) ++ g5p_l123_bos(d, job, true); ++ g5p_l123.pmgr[1] = g5p_l123_pmgr(g); ++ { ++ return; ++ } ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-prime.inc b/drivers/gpu/drm/apple-g5p/drm-prime.inc +new file mode 100644 +index 000000000..bd513b969 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-prime.inc +@@ -0,0 +1,114 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* PRIME for shmem GEM BOs. ++ * ++ * Export: Mesa WSI/client buffers and kmsro bookkeeping. The dma-buf adds ++ * begin/end_cpu_access, so an importer that CPU-reads (simpledrm's blit, ++ * llvmpipe) gets cache maintenance on the G5P DMA mapping. vmap/mmap reuse ++ * the shmem helpers (write-combined when the device is not DMA-coherent). ++ * ++ * Import: foreign dma-bufs, e.g. simpledrm dumb buffers that the GPU renders ++ * into for scanout. The attachment stays mapped (pinned) for the life of the ++ * GEM object; VM_BIND maps its sg_table. Render jobs listing an imported BO ++ * get whole-buffer cache maintenance and dma_resv fences (drm-render.inc). ++ */ ++ ++static int g5p_dmabuf_begin_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ /* No device mapping yet means no device-owned cache lines either. */ ++ if (sgt) ++ dma_sync_sgtable_for_cpu(obj->dev->dev, sgt, DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static int g5p_dmabuf_end_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ if (sgt) ++ dma_sync_sgtable_for_device(obj->dev->dev, sgt, ++ DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static const struct dma_buf_ops g5p_dmabuf_ops = { ++ .attach = drm_gem_map_attach, ++ .detach = drm_gem_map_detach, ++ .map_dma_buf = drm_gem_map_dma_buf, ++ .unmap_dma_buf = drm_gem_unmap_dma_buf, ++ .release = drm_gem_dmabuf_release, ++ .mmap = drm_gem_dmabuf_mmap, ++ .vmap = drm_gem_dmabuf_vmap, ++ .vunmap = drm_gem_dmabuf_vunmap, ++ .begin_cpu_access = g5p_dmabuf_begin_cpu_access, ++ .end_cpu_access = g5p_dmabuf_end_cpu_access, ++}; ++ ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags) ++{ ++ struct dma_buf_export_info info = { ++ .exp_name = KBUILD_MODNAME, ++ .owner = obj->dev->driver->fops->owner, ++ .ops = &g5p_dmabuf_ops, ++ .size = obj->size, ++ .flags = flags, ++ .priv = obj, ++ .resv = obj->resv, ++ }; ++ ++ return drm_gem_dmabuf_export(obj->dev, &info); ++} ++ ++static int g5p_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file, ++ u32 handle, u32 flags, int *fd) ++{ ++ return drm_gem_prime_handle_to_fd(dev, file, handle, flags, fd); ++} ++ ++static int g5p_prime_fd_to_handle(struct drm_device *dev, struct drm_file *file, ++ int fd, u32 *handle) ++{ ++ return drm_gem_prime_fd_to_handle(dev, file, fd, handle); ++} ++ ++/* Our own export comes back as the same GEM object (the core's identity check ++ * only knows its default ops); anything else is attached and mapped. ++ */ ++static struct drm_gem_object *g5p_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *buf) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ ++ if (buf->ops == &g5p_dmabuf_ops && obj->dev == dev) { ++ drm_gem_object_get(obj); ++ return obj; ++ } ++ return drm_gem_prime_import(dev, buf); ++} ++ ++/* Foreign buffers are GPU READ|WRITE data, never executable, and must be ++ * whole 16KiB GPU pages within the BO size limit. ++ */ ++static struct drm_gem_object * ++g5p_prime_import_sg_table(struct drm_device *dev, ++ struct dma_buf_attachment *attach, ++ struct sg_table *sgt) ++{ ++ struct drm_gem_object *obj; ++ ++ if (attach->dmabuf->size & (G5P_GPU_PAGE - 1) || ++ attach->dmabuf->size > G5P_MAX_BO_SIZE) ++ return ERR_PTR(-EINVAL); ++ obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); ++ if (!IS_ERR(obj)) { ++ to_g5p_bo(obj)->roles = DRM_G5P_BO_READ | DRM_G5P_BO_WRITE; ++ g5p_bo_track(obj); ++ } ++ return obj; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +new file mode 100644 +index 000000000..21dc64d89 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -0,0 +1,203 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * The render scheduler (docs/gpu/multi-client-design.md "Stage 2/3 as built"): one worker on ++ * the ordered render_wq serves the contexts round-robin (core/render-user-sched.h). Jobs wait ++ * for their inputs on dma-fence callbacks, not in the worker, so one client's unsignalled ++ * fence never holds up another client. Each picked job runs to completion on the one shared ++ * channel (one credit), exactly as a work item did before; a context's close item runs after ++ * its last job. Included from drm-render.inc after drm-render-v3.inc. ++ * ++ * The shape is drm_sched's (entity FIFO, run-list RR, dependency callbacks, entity teardown ++ * after the last job), so stage 4 can replace it with drm_sched without a UAPI change. ++ */ ++ ++static struct dma_fence *g5p_rjob_dep(struct g5p_rjob *job, u32 i) ++{ ++ return i < job->input_count ? job->inputs[i] : ++ job->deps[i - job->input_count]; ++} ++ ++/* A signalled input: its error, or that of any fence it wraps (array, chain), else 0. */ ++static int g5p_render_fence_status(struct dma_fence *fence) ++{ ++ struct dma_fence_unwrap cursor; ++ struct dma_fence *part; ++ u32 unwrapped = 0; ++ int ret = dma_fence_get_status(fence); ++ ++ if (ret < 0) ++ return ret; ++ dma_fence_unwrap_for_each(part, &cursor, fence) { ++ ret = dma_fence_get_status(part); ++ if (ret <= 0 || ++unwrapped > 4096) { ++ dma_fence_put(cursor.chain); ++ return ret < 0 ? ret : -ETIMEDOUT; ++ } ++ } ++ return 0; ++} ++ ++/* Fence callback (any context, the fence lock held): the head job's input signalled. */ ++static void g5p_rjob_dep_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++{ ++ struct g5p_rjob *job = container_of(cb, struct g5p_rjob, cb); ++ ++ WRITE_ONCE(job->cb_fired, true); ++ mod_delayed_work(job->gpu->render_wq, &job->gpu->rdev->sched_work, 0); ++} ++ ++/* Scheduler lock held: no callback of this job stays armed. */ ++static void g5p_rjob_disarm(struct g5p_rjob *job) ++{ ++ if (job->cb_fence) ++ dma_fence_remove_callback(job->cb_fence, &job->cb); ++ job->cb_fence = NULL; ++} ++ ++/* ++ * Scheduler lock held: may the head job of its context run now? Walks its explicit inputs, ++ * then its implicit dependencies, in order; at the first unsignalled one it arms a callback ++ * and reports the job's input deadline. A failed input, the deadline (G5P_RU_INPUT_MS after ++ * the submit, as the old in-worker wait) and closed admission make it ready with an error, so ++ * that it fails before the hardware and its fence still signals in order. ++ */ ++static bool g5p_rsched_ready(struct g5p_rs_job *node, void *cookie, ++ unsigned long *deadline) ++{ ++ struct g5p_rjob *job = container_of(node, struct g5p_rjob, node); ++ ++ if (job->setup_error || job->dep_error) ++ return true; ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ESHUTDOWN; ++ return true; ++ } ++ if (job->cb_fence) { ++ if (!READ_ONCE(job->cb_fired)) ++ goto pending; ++ job->cb_fence = NULL; ++ } ++ while (job->dep_next < job->input_count + job->dep_count) { ++ struct dma_fence *fence = g5p_rjob_dep(job, job->dep_next); ++ ++ if (dma_fence_is_signaled(fence)) { ++ job->dep_error = g5p_render_fence_status(fence); ++ if (job->dep_error) ++ return true; ++ job->dep_next++; ++ continue; ++ } ++ WRITE_ONCE(job->cb_fired, false); ++ if (dma_fence_add_callback(fence, &job->cb, g5p_rjob_dep_cb)) ++ continue; /* -ENOENT: signalled meanwhile */ ++ job->cb_fence = fence; ++ goto pending; ++ } ++ return true; ++pending: ++ if (time_after_eq(jiffies, job->deadline)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ETIMEDOUT; ++ return true; ++ } ++ *deadline = job->deadline; ++ return false; ++} ++ ++static void g5p_rsched_kick(struct g5p_drm *d) ++{ ++ mod_delayed_work(d->render_wq, &d->rdev->sched_work, 0); ++} ++ ++/* Submit, render_submit_lock held: an accepted job joins its context's FIFO. */ ++static void g5p_rsched_push(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ g5p_rs_push(&d->rdev->sched, &r->entity, &job->node); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ g5p_rsched_kick(d); ++} ++ ++/* The close item joins the context's entity behind its jobs; false if one is queued. */ ++static bool g5p_rsched_close(struct g5p_drm *d, struct g5p_render *r) ++{ ++ unsigned long flags; ++ bool queued; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ queued = g5p_rs_close(&d->rdev->sched, &r->entity); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ if (queued) ++ g5p_rsched_kick(d); ++ return queued; ++} ++ ++static void g5p_render_run(struct g5p_rjob *job); ++static void g5p_render_close_run(struct g5p_render *r); ++static void g5p_render_close_deferred(struct g5p_drm *d); ++ ++/* ++ * The worker: pick, run to completion, repeat; then sleep until a callback, a submit or the ++ * earliest input deadline wakes it. render_wq is ordered, so this never runs twice at once, ++ * and it is the only writer of the device's job state, as each job's work item was. ++ */ ++static void g5p_rsched_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = container_of(to_delayed_work(work), ++ struct g5p_rdev, sched_work); ++ struct g5p_drm *d = rd->drm; ++ ++ /* Each item drops the references it held; the device must outlive this loop. */ ++ drm_dev_get(&d->drm); ++ for (;;) { ++ struct g5p_rs_entity *e; ++ struct g5p_rs_job *node; ++ struct g5p_render *r; ++ struct g5p_context *c; ++ unsigned long flags, wake = 0; ++ bool has_wake; ++ int what; ++ ++ /* A lost job blocks the channel: hold every job until the drain is decided, and ++ * until a recovery the drain or the executor asked for has run. ++ */ ++ if (READ_ONCE(rd->stuck) || READ_ONCE(rd->recovering)) { ++ if (rd->stuck ? g5p_rv3_drain(d) : g5p_rv3_recover(d)) ++ continue; ++ queue_delayed_work(d->render_wq, &rd->sched_work, ++ msecs_to_jiffies(100)); ++ break; ++ } ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ what = g5p_rs_pick(&rd->sched, g5p_rsched_ready, d, &e, &node, ++ &wake, &has_wake); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ if (what == G5P_RS_IDLE) { ++ if (has_wake) ++ queue_delayed_work( ++ d->render_wq, &rd->sched_work, ++ max_t(long, (long)(wake - jiffies), 1)); ++ break; ++ } ++ if (what == G5P_RS_JOB) { ++ g5p_render_run( ++ container_of(node, struct g5p_rjob, node)); ++ g5p_render_close_deferred(d); ++ continue; ++ } ++ r = container_of(e, struct g5p_render, entity); ++ c = r->ctx; ++ g5p_render_close_run(r); ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ g5p_rs_closed(&rd->sched, e); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ /* The close item's own reference (g5p_render_close()); r may go with it. */ ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ drm_dev_put(&d->drm); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +new file mode 100644 +index 000000000..36550c443 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -0,0 +1,488 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Unbinding firmware slots (docs/gpu/multi-client-design.md §4.5): a closed context's pair ++ * (stage 2, core/render-user-close.h) and the render frontend's share of the reboot quiesce ++ * (core/render-user-quiesce.h, docs/gpu/experiments/mcfold2-kexec.md). ++ * Included from drm-render.inc after drm-render-v3.inc. ++ */ ++ ++/* The RAM words of `slot` and of its unbind record at ring entry `index`. */ ++static void g5p_render_slot_mem(struct g5p_boot *b, u32 slot, u32 index, ++ struct g5p_ru_slot_mem *m) ++{ ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + (index & 255) * 20 + j * 4; ++ ++ m->rec[j] = ++ b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ } ++ m->pa = g5p_boot_shared(b, 0x86) + slot * 8; ++ m->host = g5p_boot_shared(b, 0x84) + slot * 4; ++ m->cached = g5p_boot_shared(b, 0x85) + slot * 4; ++} ++ ++/* ++ * Unbind `n` firmware slots in one RAM-only DC3 batch: PA and host tag of each to zero, one ++ * record {3, slot} per slot, one producer store, then the RAM-only wait of a bind and a check ++ * that the firmware's DC3 handler copied the zero tag of every slot. No doorbell and no ++ * register access, so it is safe after a job retired and while the GPU may be idle-gated. ++ * ++ * The caller holds the core lock (the DC ring has one writer) and has retired every use of ++ * these slots: no job of theirs is queued on the firmware or in flight. On success the ++ * firmware holds, per slot, PA 0, host and cached tag 0 and CBASE 0, with the slot's PC level ++ * invalidated; the slot may be bound again by a later DC3. Its translations stay allocated: ++ * freeing them is the caller's step, after this returns 0. A batch that is not consumed in ++ * time keeps later binds and unbinds refused until the firmware catches up (dc_pending). ++ */ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_slot_sample s = {}; ++ struct g5p_ru_slot_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, cons, target, bad = 0, verified = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!g5p_ru_unbind_fits(n)) ++ return -EINVAL; ++ for (u32 i = 0; i < n; i++) ++ if (slots[i] < G5P_RU_SLOT_MIN || slots[i] > G5P_RU_SLOT_MAX) ++ return -EINVAL; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ cons = READ_ONCE(ctl[0]); ++ if (start > 255 || cons != start || ++ (rd->dc_pending && cons != rd->dc_wait)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s refused slots=%u dc=%u/%u pending=%u: ring busy; no " ++ "hardware touched end=v3\n", ++ why, n, cons, start, rd->dc_pending); ++ return -EBUSY; ++ } ++ rd->dc_pending = false; ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_unbind_publish(&m, slots[i]); ++ } ++ target = (start + n) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, why); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ for (u32 i = 0; i < n; i++) { ++ u32 one; ++ ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_slot_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ one = g5p_ru_unbind_check(&s, slots[i], target); ++ bad |= one; ++ verified += !one; ++ } ++ if (!ret && bad) ++ ret = -EUCLEAN; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s slots=%u verified=%u ret=%d consumed_us=%lld " ++ "dc=%u->%u/%u bad=%#x first=%u last=%u mode=ram-dc3 end=v3\n", ++ why, n, verified, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ s.cons, s.prod, bad, slots[0], slots[n - 1]); ++ return ret; ++} ++ ++/* ++ * render_user_close_park: bind a closed context's just-unbound pair to the park catalogue, one ++ * DC3 pair as a context's bind (g5p_render_vm_prepare()), checked the same way. The catalogue ++ * holds the firmware window's entry and the borrowed sink entry, both stable within an epoch, so ++ * rewriting them for a later park never clears what an earlier parked pair resolves through. ++ * Core lock held, between jobs (the close item), RAM only and no register: as the unbind. ++ */ ++static int g5p_rv3_park(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, target, entry, sink, bad = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!rd->park.cpu) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->park); ++ if (ret) ++ return ret; ++ memset(rd->park.cpu, 0, G5P_GPU_PAGE); ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (start > 255 || rd->dc_pending || g5p_ru_bind_busy(&s)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park refused slots=%u/%u dc=%u/%u pending=%u busy=%#x: the pair stays " ++ "unbound end=v3\n", ++ r->slot[0], r->slot[1], s.cons, start, rd->dc_pending, ++ g5p_ru_bind_busy(&s)); ++ return -EBUSY; ++ } ++ sink = b->sink_ready ? g5p_le32(b->catalogue.cpu) : 0; ++ if (g5p_le32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4) != entry) ++ g5p_put32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4, entry); ++ if (g5p_le32(rd->park.cpu) != sink) ++ g5p_put32(rd->park.cpu, sink); ++ g5p_table_clean_page(g, &rd->park); ++ dsb(sy); ++ g5p_ru_bind_publish(&m, r->slot, rd->park.dma, G5P_RU_PARK_TAG); ++ target = (start + 2) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "park"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, rd->park.dma, ++ G5P_RU_PARK_TAG, target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ rd->parks += !ret; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park slots=%u/%u ret=%d consumed_us=%lld dc=%u->%u/%u bad=%#x " ++ "parks=%u pc=%pad end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ start, s.cons, s.prod, bad, rd->parks, &rd->park.dma); ++ return ret; ++} ++ ++/* Drop what the claim of the device took (g5p_render_device_claim()); the device stays ++ * claimed for render. Returns whether a context reference was dropped. ++ */ ++static bool g5p_render_unclaim(struct g5p_drm *d, struct g5p_render *r) ++{ ++ if (!r->claimant) ++ return false; ++ r->claimant = false; ++ if (d->quarantined_context == r->ctx) ++ d->quarantined_context = NULL; ++ return true; ++} ++ ++/* ++ * The close item of a TYPED_V3 context (multi-client-design.md "Stage 2/3 as built"): the ++ * scheduler runs it after every job of the context, so every use of the pair has retired or ++ * failed. RAM only: the unbind is g5p_rv3_unbind_slots() for this pair under the core lock. ++ * After a verified unbind the binding's references go, and the last one frees the VM tables, ++ * render targets and mappings (unpinning the BOs) and returns the slots ++ * (g5p_render_release()). Anything the firmware may still hold is retained until reboot. ++ */ ++/* perf60 (docs/gpu/experiments/perf60-clock.md), corrected by the release8 bench (bench-gaps3, ++ * render-user-pmset.h): the PM writes a previous PM context back (its free list F, through that ++ * context's TA slot) not once at the switch but on the later jobs of the next context, until a ++ * third context runs. A closed context therefore keeps its slots bound and its set and pages alive ++ * while it is the last job's context or the previous distinct one; its unbind and release run once ++ * it is neither. At most those two wait. (The first version held only the last job's context and ++ * released it after one other job; 2 held one job longer, by job records: both released it while ++ * the PM still wrote to it.) RAM only: bookkeeping and the close item's references. ++ */ ++/* release8 default */ ++ ++/* ++ * A close item follows every job of this context on the ordered render worker. ++ * resident_gone, set under the file lock before queueing it, rejects all later ++ * VM_BIND and context creation. Thus the close item owns vm->maps exclusively. ++ * ++ * Idle VM_BIND UNMAP already drops a user mapping's pages after retirement. ++ * Do that here without releasing the PM's write-back route: no unbind, no ++ * table pruning, no PM set/pool/aux/RT release, no pm_last/pm_prev change. The ++ * PM free list is in the firmware window, never in vm->maps. All user mappings ++ * must pass the same range check as VM_BIND before any PTE is touched. ++ * ++ * No new kick of this detached context can occur. Clean the cleared user PTEs ++ * to PoC, but do not publish a tag, invalidate a slot or access R0. The PM can ++ * keep using its unchanged translations and pages, including cached ones. ++ * Failed/queued/terminal/shutdown epochs keep every BO, as before. ++ */ ++static void g5p_render_close_trim(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_boot *b; ++ struct g5p_ru_close_in in; ++ struct g5p_mapping *m, *tmp; ++ struct g5p_pt *pt; ++ u64 pinned; ++ u32 maps, why; ++ ++ if (!vm->map_count || !vm->resident_gone || !r->closing || ++ !r->detached || !mutex_trylock(&g->lock)) ++ return; ++ b = g->boot; ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* Match close_run: a successfully drained lost job has no user-memory use left. */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .dc_pending = d->rdev->dc_pending, ++ .queued = atomic_read(&r->queued), ++ }; ++ if (g5p_ru_close_plan(&in, &why) != G5P_RU_CLOSE_UNBIND) ++ goto out; ++ list_for_each_entry(m, &vm->maps, link) ++ if (!g5p_user_range_valid(m->node.start, m->node.size) || ++ (m->node.start < G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE && ++ G5P_RU_PRIVATE_VA < m->node.start + m->node.size)) ++ goto out; ++ ++ pinned = vm->pinned_bytes; ++ maps = vm->map_count; ++ list_for_each_entry(m, &vm->maps, link) ++ g5p_vm_clear(vm, d->drm.dev, m->node.start, m->node.size); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ list_for_each_entry_safe(m, tmp, &vm->maps, link) { ++ vm->mapped_bytes -= m->node.size; ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-trim slots=%u/%u maps=%u unpinned=%lluK remaining=%lluK " ++ "device_pinned=%lluK pm=retained end=v3\n", ++ r->slot[0], r->slot[1], maps, (pinned - vm->pinned_bytes) >> 10, ++ vm->pinned_bytes >> 10, READ_ONCE(d->pinned_bytes) >> 10); ++out: ++ mutex_unlock(&g->lock); ++} ++ ++/* render_user_close_defer: may the firmware's PM still write this context's state back? */ ++static bool g5p_render_pm_holds(struct g5p_rdev *rd, struct g5p_render *r, ++ u32 mode) ++{ ++ return mode && g5p_pmset_holds(rd->pm_last, rd->pm_prev, r); ++} ++ ++static void g5p_render_close_run(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_close_in in; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 puts = 0, why; ++ bool locked = false; ++ int plan, ret = 0; ++ u32 mode, k; ++ ++ mode = (1); ++ k = d->rdev->close_deferred[0] ? 1 : 0; ++ if (mode && !r->close_defer_run && r->bound && ++ g5p_render_pm_holds(d->rdev, r, mode) && ++ k < ARRAY_SIZE(d->rdev->close_deferred)) { ++ g5p_render_close_trim(r); ++ /* the deferred close's own references, dropped by g5p_render_close_deferred() */ ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->rdev->close_deferred[k] = r; ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer slots=%u/%u jobs=%u: the PM may still write its state " ++ "back; unbind once it has moved past it end=v3\n", ++ r->slot[0], r->slot[1], r->jobs); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer2 slots=%u/%u pm=%s pending=%u end=v3\n", ++ r->slot[0], r->slot[1], ++ d->rdev->pm_last == r ? "last" : "previous", k + 1); ++ return; ++ } ++ ++ for (;;) { ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* a lost job that drained left the firmware nothing of this context */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ /* as g5p_rv3_unbind_slots() refuses it */ ++ .dc_pending = b && d->rdev->dc_pending && ++ READ_ONCE(((u32 *)g5p_boot_shared( ++ b, 0x6c))[0]) != d->rdev->dc_wait, ++ .queued = atomic_read(&r->queued), ++ }; ++ plan = g5p_ru_close_plan(&in, &why); ++ if (plan != G5P_RU_CLOSE_UNBIND || locked) ++ break; ++ /* Bounded, as a job takes it: never wait forever behind a command in MMIO. */ ++ while (!(locked = mutex_trylock(&g->lock)) && ++ time_before(jiffies, deadline)) ++ msleep(5); ++ if (!locked) { ++ plan = G5P_RU_CLOSE_RETAIN; ++ why = G5P_RU_CLOSE_LOCK; ++ break; ++ } ++ /* Plan again under the lock: the reboot quiesce or a failure may have run. */ ++ } ++ if (plan == G5P_RU_CLOSE_UNBIND) { ++ ret = g5p_rv3_unbind_slots(d, r->slot, 2, "close"); ++ if (!ret) { ++ r->bound = false; ++ r->retained = false; ++ WRITE_ONCE(c->live, false); ++ puts++; ++ /* the PM may still write this context's free list back through its TA slot */ ++ { ++ g5p_rv3_park(d, r); ++ }; ++ } ++ } ++ if (plan == G5P_RU_CLOSE_RELEASE || ++ (plan == G5P_RU_CLOSE_UNBIND && !ret)) ++ puts += g5p_render_unclaim(d, r); ++ if (locked) ++ mutex_unlock(&g->lock); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close slots=%u/%u plan=%s why=%#x ret=%d jobs=%u claimant=%u " ++ "retained=%u end=v3\n", ++ r->slot[0], r->slot[1], g5p_ru_close_name(plan), why, ret, ++ r->jobs, r->claimant, r->retained); ++ while (puts--) { ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++} ++ ++/* After a job: a deferred close runs once the last job was another context's. */ ++static void g5p_render_close_deferred(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ u32 mode = (1); ++ ++ for (u32 k = 0; k < ARRAY_SIZE(rd->close_deferred); k++) { ++ struct g5p_render *r = rd->close_deferred[k]; ++ struct g5p_context *c; ++ ++ /* mode 0 now: a pending close still waits until the PM has moved past it */ ++ if (!r || g5p_render_pm_holds(rd, r, mode ?: 1)) ++ continue; ++ rd->close_deferred[k] = NULL; ++ c = r->ctx; ++ r->close_defer_run = true; ++ dev_info( ++ d->core->dev, ++ "RENDER-V3 close-deferred slots=%u/%u: neither the PM's last nor its previous " ++ "context end=v3\n", ++ r->slot[0], r->slot[1]); ++ g5p_render_close_run(r); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ /* keep a pending close in [0]: close_run fills [1] only behind it */ ++ if (!rd->close_deferred[0] && rd->close_deferred[1]) { ++ rd->close_deferred[0] = rd->close_deferred[1]; ++ rd->close_deferred[1] = NULL; ++ } ++} ++ ++/* ++ * The handle of a TYPED_V3 context is gone (DESTROY_CONTEXT, or its file closed): stop VM and ++ * render-target edits and queue the close item behind the context's jobs (g5p_rsched_close()). ++ * File lock held, or the file is closing. A context that never submitted is released by its ++ * last reference. ++ */ ++static void g5p_render_close(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ ++ if (!r->v3_accepted || r->closing) ++ return; ++ r->closing = true; ++ r->detached = true; ++ c->vm->resident_gone = true; ++ /* The close item's references, dropped by the scheduler once it ran. */ ++ kref_get(&c->ref); ++ drm_dev_get(&c->gpu->drm); ++ if (!g5p_rsched_close(c->gpu, r)) { ++ g5p_context_put(c); ++ drm_dev_put(&c->gpu->drm); ++ } ++} ++ ++/* Reboot quiesce, RAM only and lock free: the TYPED_V3 latch and whether its executor runs. ++ * The executor is RAM-only once the latch is quiet, unless the wake110 experiment (doorbells ++ * after retirement, off in every desktop tuple) is configured. Called after admission closed; ++ * the scheduler is kicked so that jobs waiting on an input fail now, not at their deadline. ++ */ ++static void g5p_kq_drm_state(void *cookie, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ ++ /* Admission is closed by now: queued jobs blocked on an input fail at once. */ ++ g5p_rsched_kick(d); ++ ++ /* A channel blocked behind a lost job is a wedged epoch for the quiesce. */ ++ in->term_failed = g5p_ru_term_failed(&g5p_rv3_term) || ++ READ_ONCE(d->rdev->stuck) || ++ READ_ONCE(d->rdev->recovering); ++ in->ram_only = READ_ONCE(d->rdev->v3) && ++ !g5p_ru_mmio_ok(&g5p_rv3_term) && !(2); ++ in->executing = READ_ONCE(d->rdev->executing); ++} ++ ++/* Reboot quiesce, core lock held: every held user slot, the first claimant's pair included. ++ * Nothing is freed and no context changes state: admission is closed and the boot state is ++ * frozen right after, so no job runs again on this boot. ++ */ ++static int g5p_kq_drm_detach(void *cookie, const u32 *slots, u32 n) ++{ ++ return g5p_rv3_unbind_slots(cookie, slots, n, "kexec"); ++} ++ ++static const struct g5p_kq_frontend g5p_kq_drm = { ++ .state = g5p_kq_drm_state, ++ .detach = g5p_kq_drm_detach, ++}; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +new file mode 100644 +index 000000000..ac4dd841e +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_V3 serial executor: per-job state (receipt data). Types only; code: drm-render-v3.inc. */ ++#ifndef G5P_DRM_RENDER_V3_H ++#define G5P_DRM_RENDER_V3_H ++ ++enum g5p_rv3_phase { ++ G5P_RV3_IDLE, ++ G5P_RV3_ATTACH, ++ G5P_RV3_BIND, ++ G5P_RV3_TA_PUBLISH, ++ G5P_RV3_TA_WAIT, ++ G5P_RV3_TD_PUBLISH, ++ G5P_RV3_TD_WAIT, ++ G5P_RV3_DONE, ++ G5P_RV3_PHASES, ++}; ++ ++/* g5p_drm.td_want: the 3D completion-event latch is armed for the endpoint in the low byte. */ ++#define G5P_RV3_TD_WANT 0x100U ++ ++/* One RAM sample of one engine: ring, stamps, channel watermarks, idle words, PM state, and ++ * (3D) whether the matching completion event has been latched. ++ */ ++struct g5p_rv3_ev { ++ u64 ns; ++ u32 prod, cons, stamp[2], ctl[5], idle[6], pm80; ++ bool proto, failed, consumed, retired, idle_ok, pm_ok, event; ++}; ++ ++struct g5p_rv3 { ++ struct g5p_ru_wait ta, td; ++ struct g5p_ru_touch ++ touch[2]; /* RC4a A/B (render-user-touch.h); zero when off */ ++ struct g5p_ru_retire rt; /* the 3D predicate (render-user-retire.h) */ ++ struct g5p_ru_ta tr; /* the TA predicate (render-user-retire.h) */ ++ struct g5p_rv3_ev last[2]; ++ u64 td_event_ns; /* decode time of the matching 3D event; 0 = none */ ++ u64 ta_event_ns; /* decode time of the first TA event since the publish */ ++ u64 attach_ns, job_deadline_ns, ta_pub_ns, td_pub_ns, ta_consumed_ns, ++ td_consumed_ns; ++ u32 phase, grace_end, bell; ++ u32 seq; /* the device sequence this job got; 0 = none */ ++ /* wake110 (render-user-wake.h), per engine: the decision, the power evidence word it saw ++ * (wake111), the two register words it read (U32_MAX = not read), the consumer it saw, the ++ * type-1 event counts at the phase's arm and at the job's end, the lock window and the ++ * first type-1 decode time. Zero when off. ++ */ ++ u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], ++ wk_t1_pub[2], wk_t1_end[2]; ++ u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ ++ /* The bind of this job's context published its DC3 pair and then failed: that context is ++ * lost, the device and the contexts already bound are not. ++ */ ++ bool bind_failed; ++ /* The job was lost alone (render-user-loss.h): its context is dead, the device is not; the ++ * channel stays blocked behind its stuck phase until the scheduler's drain clears it. ++ */ ++ bool lost; ++}; ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +new file mode 100644 +index 000000000..34fbad607 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -0,0 +1,3038 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial executor: the control flow that completed the natural TA and the ++ * natural 3D in r96 (docs/gpu/experiments/production-typed-render-kernel.md), on typed ++ * state. Included from drm-render.inc after g5p_render_execute(). ++ * ++ * What it does, in order, for each job a V3 context runs (strictly serial across every context, ++ * render-user-serial.h; only the epoch's job 1 attaches and has hardware evidence): ++ * 0. bind, the first job of every later context (multi-client-design.md §4.2): its own DC3 ++ * pair, published in RAM only, no doorbell, a RAM-only wait for the firmware to consume ++ * it. With the TEMPORARY render_user_spare_pairs instead: a pair job 1 pre-bound, taken ++ * over through the tag path. ++ * 1. attach (job 1): the typed preparation (PM, channels, catalogue entries), then the ++ * DC3 pair is published and its doorbell is owned by the r74f machinery (r87b ++ * self-drain: a closed gate is never repaired by the host). The firmware installs ++ * the page-table root and the generation tags itself (r74g); the host does NO MMU pass ++ * and checks the result in RAM only. ++ * 2. TA: record, WORK_CTL and producer, no doorbell; the r74g grace (four ALIVEs or ++ * 20 ms) lets the firmware drain it, and only if it is still pending afterwards the ++ * first ALIVE decoded owns one fenced fallback doorbell. ++ * 3. TA wait: RAM only (drain, sample, sleep), 20 s per phase by default, 30 s for the ++ * whole job (render-user-wait.h decides the deadline and closes the race with a ++ * completion). tahang1: complete on a coherent target with the PM saved and either ++ * the idle words clear, the TA completion event, or a 500 ms coherent hold ++ * (render-user-retire.h); a phase pending 1 s prints one RAM-only `slow` line. From ++ * the TA complete on, and after any timeout or fault, the host touches no register. ++ * 4. 3D: record, WORK_CTL and producer, NO doorbell (r96 stage 1: the firmware consumed ++ * it 2 ms later), RAM-only wait as above. ++ * RC4a (render_user_touch_ms, default 0 = off): at publish + T of each phase, odd jobs read ++ * the receipt pages once and even jobs only record the instant (render-user-touch.h); RAM only. ++ * repeat109: the 3D phase is also complete when its target state is coherent and its completion ++ * event arrived, even if the engine idle words never clear (render-user-retire.h; the latched ++ * render_user_idle_settle_ms, default 0, adds a minimum coherent run); one `RENDER-V3 retire` ++ * receipt line splits the firmware's retirement from the host's decision. RAM only. ++ * wake110 (render_user_wake, default 0 = off): right after the producer store of every phase but ++ * job 1's TA, the A/B's factor phases run Apple's submit kick (R1+0x14 = 7, GPC active, R0+0xb00 = ++ * 0x10 / 0x11) and its control phases the same guards without a register (render-user-wake.h); ++ * one `RENDER-V3 wake` line per job. The only register access after the TA complete, licensed ++ * while no timeout or fault was noted. ++ * Jobs >= 2 (a compositor's next frame, or another context's): no attach, the PM pool is ++ * rebuilt, and both rings are published with no doorbell at all. ++ * Not in this path on purpose: the r74h / r80 / r85-r96 root-cause instrumentation (stream ++ * markers, PM canary, plateau and end banks, region/target dumps), host MMU passes, cache ++ * maintenance MMIO, selector reads, snapshots (they read R0), and any recovery. A timeout ++ * or fault retains everything; the context is frozen and kept until reboot. ++ * ++ * Terminal state (render-user-terminal.h): the TA complete, a timeout and a fault go through one ++ * first-writer-wins latch, g5p_rv3_terminal(). From then on g5p_ru_mmio_ok() is false, the admission ++ * refuses every job that would start behind a failure (-EIO), and the legacy typed paths that read or ++ * write registers (snapshot, invalidate, run, destroy) refuse. g5p_rv3_td_hook() is the optional ++ * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. ++ */ ++ ++static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { ++ "idle", "attach", "bind", "ta-publish", ++ "ta-wait", "3d-publish", "3d-wait", "done", ++}; ++static const char *const g5p_rv3_results[] = { "pending", "complete", "timeout", ++ "fault" }; ++static const char *const g5p_rv3_how[] = { "none", "idle-clear", "idle-stuck", ++ "hold" }; ++ ++#include "drm-latency123.inc" ++ ++/* perf60 stage 4 (render-user-pmset.h): per-context PM records, the iOS model (one ParamBuffer ++ * per work queue, 6cb9afc/6cb2378). Latched at the epoch's first attach, with boot_pm_sets: ++ * 1 every context bound after the epoch's first gets its own PM set (control + S, list, F in ++ * the firmware GiB), its own pool pages and its own aux pages at G5P_RU_PRIVATE_VA in its ++ * VM; the epoch's first context keeps the device PM. PM ID 0 everywhere. ++ * 2 as 1, and set s carries PM ID s + 1 (TA W+0x10, TA start, 3D start, tag6). ++ * RAM only, by the executor between jobs. ++ */ ++/* release8 default */ ++ ++/* The PM a context's jobs use: its set, or the device PM. */ ++static u8 *g5p_rv3_pm_cpu(struct g5p_boot *b, const struct g5p_render *r) ++{ ++ if (r->pm_set >= 0) ++ return b->pm_set[r->pm_set * G5P_PMSET_PAGES + G5P_PMSET_CONTROL] ++ .cpu; ++ return g5p_ru_pm_cpu(b); ++} ++ ++static u32 g5p_rv3_pm_meta(const struct g5p_render *r) ++{ ++ return g5p_meta_address( ++ r->pm_set >= 0 ? g5p_pmset_va(r->pm_set, G5P_PMSET_CONTROL) : ++ G5P_SHARED_VA + G5P_R22_PM); ++} ++ ++static u32 g5p_rv3_pm_id(const struct g5p_rdev *rd, const struct g5p_render *r) ++{ ++ return rd->pm_ctx >= 2 && r->pm_set >= 0 ? g5p_pmset_id(r->pm_set) : 0; ++} ++ ++static struct g5p_dma_page *g5p_rv3_pm_aux(struct g5p_rdev *rd, ++ struct g5p_render *r) ++{ ++ return r->pm_aux_mapped ? r->pm_aux : rd->aux; ++} ++ ++/* The boot tuple the V3 path is proven under (r96 params.txt), minus the instrumentation: ++ * r73b/r73c/r73e/r73f, r74 with the raised IRQ cap, r74f, r74g, r87b self-drain; no r74h, ++ * no r80..r96 (their IRQ banks read registers), no r75/r79 firmware control, and no fixed ++ * render engine (it is exclusive with the DRM one). ++ */ ++static bool g5p_rv3_tuple(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ struct g5p_r73b *x = b ? b->r73b : NULL; ++ ++ return x && g5p_r74g_on(b) && g5p_r74b_ram_only(g) && ++ x->r87b_attach == G5P_R87B_ATTACH_SELF_DRAIN; ++} ++ ++/* The admission state of a context and the device, as plain values for g5p_ru_v3_admit(). */ ++static struct g5p_ru_v3_state g5p_rv3_state(struct g5p_diag *g, ++ struct g5p_render *r) ++{ ++ const struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ ++ return (struct g5p_ru_v3_state){ ++ .tuple = g5p_rv3_tuple(g), ++ .v3_accepted = r->v3_accepted, ++ .bound = r->bound, ++ .jobs = r->jobs, ++ .device_up = READ_ONCE(rd->up), ++ .device_v3 = READ_ONCE(rd->v3), ++ .sequence = READ_ONCE(rd->sequence), ++ .slot_ta = r->slot[0], ++ .slot_3d = r->slot[1], ++ .variant = 0 || r->after_ta || 0 || r->v.pm || r->v.memory || ++ r->v.rt || r->v.slots, ++ .wait_ms = r->wait_ms, ++ .quiet = !g5p_ru_mmio_ok(&g5p_rv3_term), ++ .terminal_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ }; ++} ++ ++/* A terminal epoch is a device state (-EIO), the others are about this request. */ ++static int g5p_rv3_refusal_errno(int why) ++{ ++ if (why == G5P_RU_V3_TERMINAL) ++ return -EIO; ++ return why == G5P_RU_V3_OVERFLOW ? -EOVERFLOW : -EOPNOTSUPP; ++} ++ ++/* Submit-time admission: every V3 job refused here has touched nothing. */ ++static int g5p_rv3_admit_submit(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_ru_v3_state st; ++ int why; ++ ++ /* The epoch is being replaced: the executor admits the job again under the lock. */ ++ if (READ_ONCE(d->rdev->recovering)) ++ return 0; ++ st = g5p_rv3_state(g, r); ++ why = g5p_ru_v3_admit(&st, false); ++ ++ if (!why) ++ return 0; ++ /* per submit while admission is closed and a client retries: ratelimited */ ++ dev_info_ratelimited( ++ g->dev, "RENDER-V3 refused: %s; no hardware touched end=v3\n", ++ g5p_ru_v3_why[why]); ++ return g5p_rv3_refusal_errno(why); ++} ++ ++/* Recovery hooks (patches/gpu-recovery), compiled only when that series is applied: core.inc then includes ++ * teardown.inc before this file, and teardown.h defines G5P_TEARDOWN_H. The CAPTURE entry is observation only ++ * (RAM reads, no register access) and runs only with the recovery module parameter set, once per epoch, on a ++ * timeout or fault. Without the series both are nothing, and the prod98 kernel does not depend on it. ++ */ ++#ifdef G5P_TEARDOWN_H ++static void g5p_rv3_td_hook(struct g5p_diag *g, int kind, const char *why) ++{ ++ g5p_td_note_terminal(&g5p_td_state, g5p_ru_td_kind(kind)); ++ if ((0) && g5p_ru_term_capture_due(&g5p_rv3_term, kind)) ++ g5p_td_capture(g, why); ++} ++#else ++static inline void g5p_rv3_td_hook(struct g5p_diag *g, int kind, ++ const char *why) ++{ ++} ++#endif ++ ++/* The terminal state of the epoch: the TA complete, and any timeout or fault. First writer wins; from the first ++ * note on g5p_ru_mmio_ok() is false and the host touches no register (R0, R1, AIC, doorbell) and no bank. The r80 ++ * flag is set when an r80 block exists, so that a bank that might be latched cannot run behind this path (the V3 ++ * tuple forbids r80, so there is none today); the latch above does not depend on it. ++ */ ++static void g5p_rv3_terminal(struct g5p_diag *g, int kind, const char *why) ++{ ++ bool failed = g5p_ru_term_failed(&g5p_rv3_term); ++ ++ if (g5p_ru_term_note(&g5p_rv3_term, kind)) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s mmio=off r0=off export=ram-only end=v3\n", ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)], ++ why); ++ else if (!failed && g5p_ru_term_failed(&g5p_rv3_term)) ++ /* The epoch's TA complete took the latch long ago (job 1): a later timeout or fault ++ * only makes it sticky-failed, which closes admission for every client. Say so once. ++ */ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s latched=%s failed=1 admission=closed end=v3\n", ++ g5p_ru_term_names ++ [kind > G5P_RU_TERM_NONE && ++ kind < G5P_RU_TERM_KINDS ? ++ kind : ++ G5P_RU_TERM_FAULT], ++ why, ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)]); ++ g5p_rv3_td_hook(g, kind, why); ++} ++ ++/* "Retained boundary" of the r74f machinery -> the errno a job reports. -EINPROGRESS is ++ * never a job error (the UAPI uses it for "pending"). ++ */ ++static int g5p_rv3_errno(struct g5p_diag *g, int ret) ++{ ++ if (ret != -EINPROGRESS) ++ return ret; ++ switch (READ_ONCE(g->boot->r73b->r74f_stop)) { ++ case G5P_R74F_STOP_ATTACH_NOT_KICKED: ++ case G5P_R74F_STOP_ATTACH_NOT_CONSUMED: ++ case G5P_R74F_STOP_TA_NOT_KICKED: ++ case G5P_R74G_STOP_TA_NOT_CONSUMED: ++ case G5P_R74G_STOP_TA_WINDOW_MISSED: ++ return -ETIMEDOUT; ++ default: ++ return -EIO; ++ } ++} ++ ++/* ++ * Interim: on iOS the firmware writes back its own lines with the cache maintenance it runs on ++ * every tick (meta+0x6d4); that needs init words still being recovered (latency123.md), and ++ * this read stands in for it until then. ++ * ++ * While a phase is in flight the firmware does not see its own progress in the root and ++ * shared pages until a coherent read of those lines reaches the fabric: without one, the TA ++ * and 3D stages wait for the firmware tick, for seconds (latency123: key to frame 7.9 s ++ * median; with a Normal-NC read of both regions every 1-2 ms, 0.27 s). One 32-bit load per ++ * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU ++ * has no sampler, so it costs nothing then. ++ */ ++static void g5p_rv3_fw_touch(struct g5p_boot *b) ++{ ++ struct g5p_dma_page *const regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE( ++ *(const u32 *)(regions[r][i].cpu + at)); ++ /* perf60 stage 4: the per-context PM sets are firmware pages like the shared ones */ ++ for (u32 i = 0; i < b->pm_sets * G5P_PMSET_PAGES; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(b->pm_set[i].cpu + at)); ++} ++ ++/* perf60 bell gates, A/B knobs for the multi-context 3D fault on the bell path (render_user_wake=2 only; ++ * 0600, read at every bell, all 0 = no gate). They run between a phase's producer store and its bell; ++ * in policy state 0 the firmware does not drain a ring on its own, so a gate holds the phase back. ++ * RAM reads, sleeps and barriers only; no register access. ++ * bell_gap_us (a) the TA bell waits until this long after the previous job (any context) ended ++ * bell_idle_ms (a) the TA bell first waits, up to this, for the 3D engine idle words to read zero ++ * bell_3d_delay_us (c) mb() and this delay between the 3D producer store and the 3D bell ++ * bell_ctx_gap_us (d) mb() and this delay before the TA bell of a job of another context than the last ++ * bell_touch (c) the Normal-NC read of the firmware pages (g5p_rv3_fw_touch) right before each bell ++ */ ++ ++/* perf60 job-start gate: the same waits before the job's own host writes (the PM pool rebuild and ++ * the shared TA/3D work records, g5p_rv3_pm_rebuild() and g5p_ru_job()), which happen before any ++ * publish. The bell gates above run after them, so they cannot separate "the host rewrites shared ++ * firmware pages while the firmware still finishes the previous job" from a doorbell race. ++ * job_gap_us the job's first host write no sooner than this after the previous job ended ++ * job_idle_ms then wait up to this (RAM) for the TA and 3D engine idle words to read zero ++ */ ++/* perf60 layout (latched at the epoch's first attach, default 0 = the r22 layout): ++ * render_user_job_slots 2: job seq writes its command, W record and microsequence into slot ++ * (seq - 1) % 2 of the channel page, so a job never rewrites the record the firmware used for ++ * the job before it; the pointer ring carries each job's addresses (render-user-layout.h). ++ * render_user_pm_once 1: the PM pool, list and F are built by the epoch's first attach only and ++ * never rewritten by the host between jobs; the TA's tag6 record has the firmware rebuild F, ++ * as on iOS (PM created per queue once; docs/gpu/experiments/freelist-lifetime-production-fix.md). ++ */ ++/* release8 default */ ++/* release8 default */ ++ ++/* The record slot of job seq on this epoch's layout. */ ++static u32 g5p_rv3_js(const struct g5p_rdev *rd, u32 seq) ++{ ++ return rd->job_slots == 2 ? (seq - 1) & 1 : 0; ++} ++ ++ ++ ++ ++ ++ ++ ++ ++ ++/* RAM only. */ ++static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_ru_page(b, e), *global = g5p_ru_global(b, e), ++ *pm = g5p_rv3_pm_cpu(b, r); ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ for (u32 i = 0; i < ARRAY_SIZE(s->stamp); i++) ++ s->stamp[i] = g5p_le32(p + G5P_R22_STAMP + 4 * i); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < ARRAY_SIZE(s->idle); i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->pm80 = g5p_le32(pm + 0x80); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != r->target[e] || s->cons > 255 || ++ !g5p_ru_stamp_sane(p, seq); ++ s->consumed = s->cons == r->target[e]; ++ /* The predicates of the flown r74b (TA) and r96 (3D) waits; the 3D's PM serial is a receipt. */ ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2] && ++ (!e || (!s->idle[4] && !s->idle[5])); ++ s->retired = g5p_ru_retired_at(p, g5p_ru_meta(e), e, seq, ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ s->pm_ok = e ? s->pm80 == r->ctx->gpu->rdev->pm_serial + 1 : ++ g5p_r22_pm_saved(pm + 0x80); ++ s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns) : ++ READ_ONCE(r->ctx->gpu->ta_hit_ns)); ++} ++ ++/* TA: one step of render-user-retire.h's TA rules when ta is given, else the flown r74b ++ * predicate. 3D: one step of its 3D rules when rt is given, else the r96 predicate. ++ */ ++static bool g5p_rv3_done(u32 e, const struct g5p_rv3_ev *s, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta) ++{ ++ if (e && rt) ++ return g5p_ru_rt_step(rt, s->ns, s->consumed, s->retired, ++ s->idle_ok, ++ s->event) != G5P_RU_RT_PENDING; ++ if (!e && ta) ++ return g5p_ru_ta_step(ta, s->ns, s->consumed, s->retired, ++ s->pm_ok, s->idle_ok, ++ s->event) != G5P_RU_TA_PENDING; ++ return s->consumed && s->idle_ok && s->retired && (e || s->pm_ok); ++} ++ ++/* The TA clauses in the bits of render-user-retire.h, for the receipt. */ ++static u32 g5p_rv3_ta_unsat(const struct g5p_rv3_ev *s) ++{ ++ return (s->consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (s->retired ? 0 : G5P_RU_RT_RETIRED) | ++ (s->idle_ok ? 0 : G5P_RU_RT_IDLE) | ++ (s->pm_ok ? 0 : G5P_RU_RT_PM); ++} ++ ++/* Arm engine e's completion-event latch of g5p_drm_event() (3D: for this job's endpoint), before ++ * the phase's producer store, so an event decoded right after the publish is not missed. The ++ * latch is set under status_lock by whichever CPU decodes the event, and the wait samples it ++ * level-triggered every 1-2 ms, so there is no wakeup to lose. RAM only. ++ */ ++static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) ++{ ++ unsigned long flags; ++ ++ BUILD_BUG_ON(G5P_RU_RT_GRID != G5P_WORK_SLOTS); ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = G5P_RV3_TD_WANT | endpoint; ++ WRITE_ONCE(d->td_hit_ns, 0); ++ } else { ++ d->ta_want = true; ++ WRITE_ONCE(d->ta_hit_ns, 0); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) ++{ ++ unsigned long flags; ++ u64 hit; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = 0; ++ hit = d->td_hit_ns; ++ } else { ++ d->ta_want = false; ++ hit = d->ta_hit_ns; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return hit; ++} ++ ++/* RC4a: the receipt's read set (g5p_rv3_receipt()'s blobs: PM-control, then W, channels and the ++ * stamp window of the TA and of the 3D page), read once with 32-bit loads, the access size of the ++ * receipt's hex dump. The words are folded, not copied: no allocation, no store to shared memory. ++ * RAM only: no register, no doorbell, no print, no latch. ++ */ ++static u32 g5p_rv3_touch_sweep(struct g5p_boot *b, u32 *bytes) ++{ ++ u8 *ta = g5p_ru_page(b, 0), *td = g5p_ru_page(b, 1); ++ const u8 *const src[] = { ++ g5p_ru_pm_cpu(b), ta + G5P_R22_W, ta, ++ ta + G5P_R22_TQ, td + G5P_R22_W, td, ++ td + G5P_R22_TQ, ++ }; ++ static const u32 len[] = { ++ 0x140, G5P_RU_TA_W_BYTES, 0x340, 0x80, G5P_RU_3D_W_BYTES, 0x340, ++ 0x80, ++ }; ++ u32 sum = 0, n = 0; ++ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(src); i++) { ++ for (u32 at = 0; at < len[i]; at += 4) ++ sum = (sum << 5 | sum >> 27) ^ ++ READ_ONCE(*(const u32 *)(src[i] + at)); ++ n += len[i]; ++ } ++ *bytes = n; ++ return sum; ++} ++ ++/* RC4a: the instant of one phase, taken at the sample at now_ns. The touch arm reads, the control ++ * arm only records; both record the same instant. ++ */ ++static void g5p_rv3_touch(struct g5p_diag *g, struct g5p_ru_touch *t, ++ u64 now_ns) ++{ ++ u32 bytes = 0, sum = 0; ++ ++ if (t->sweep) ++ sum = g5p_rv3_touch_sweep(g->boot, &bytes); ++ g5p_ru_touch_mark(t, now_ns, t->sweep ? ktime_get_ns() : now_ns, bytes, ++ sum); ++} ++ ++/* tahang1: a phase still pending this long after its publish prints its RAM state once. */ ++#define G5P_RV3_SLOW_MS 1000U ++ ++/* The one `slow` line of a phase: the deciding sample's RAM words and the waiter's own state, so ++ * a stall that later times out is described from the first second on. RAM only. ++ */ ++static void g5p_rv3_slow(struct g5p_diag *g, u32 e, u32 seq, ++ const struct g5p_ru_wait *w, ++ const struct g5p_rv3_ev *s, u32 unsat, u64 hit_ns) ++{ ++ dev_warn( ++ g->dev, ++ "RENDER-V3 slow seq=%u engine=%u ms=%lld ring=%u/%u stamp=%08x/%08x " ++ "ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x,%08x,%08x,%08x pm80=%08x " ++ "consumed=%u retired=%u pm=%u event_ms=%lld unsat=%#x busy_ms=%lld events=%u " ++ "end=v3\n", ++ seq, e, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ s->idle[3], s->idle[4], s->idle[5], s->pm80, s->consumed, ++ s->retired, s->pm_ok, g5p_ru_rt_ms(w->start_ns, hit_ns), unsat, ++ g5p_ru_rt_ms(w->start_ns, w->busy_ns), ++ READ_ONCE(g->boot->events)); ++ g5p_np_policy(g, "slow", seq, e); ++} ++ ++/* nopatch: the policy record at a few phase ends (log-spaced) and at every timeout. RAM only. */ ++static void g5p_rv3_np_end(struct g5p_diag *g, u32 e, u32 seq, u32 result) ++{ ++ if (seq == 1 || seq == 10 || seq == 100 || seq == 1000 || ++ seq == 10000 || result == G5P_RU_WAIT_TIMEOUT) ++ g5p_np_policy(g, ++ result == G5P_RU_WAIT_TIMEOUT ? "phase-timeout" : ++ "phase-end", ++ seq, e); ++} ++ ++/* One phase's RAM-only wait. g5p_ru_wait_step()/close() decide; this loop only samples. RC4a: the ++ * touch instant is taken after a step that left the phase undecided, before the sleep (off: never). ++ * The predicate is level-triggered: every sample, including the one right after each sleep and the ++ * closing one at the deadline, re-reads the whole RAM state (rt: the 3D, ta: the TA). ++ */ ++static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, struct g5p_ru_touch *t, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta, ++ struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) ++{ ++ struct g5p_drm *d = r->ctx->gpu; ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE( ++ d->event_serial); /* after the drain, before the sample */ ++ bool fault, silent, done; ++ ++ g5p_rv3_sample(g, r, e, seq, s); ++ /* latency101: a silent event stream during the phase is a dead firmware (never off). */ ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed) ++ *err = -EIO; ++ else if (silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = g5p_rv3_done(e, s, rt, ta); ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (rt) ++ g5p_ru_rt_note(rt, w->result); ++ if (ta) ++ g5p_ru_ta_note(ta, w->result); ++ if (w->result) { ++ g5p_rv3_np_end(g, e, seq, w->result); ++ return w->result; ++ } ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_RV3_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ g5p_rv3_slow(g, e, seq, w, s, ++ rt ? rt->unsat : ++ ta ? ta->unsat : ++ 0, ++ e ? READ_ONCE(d->td_hit_ns) : ++ READ_ONCE(d->ta_hit_ns)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ if (g5p_ru_touch_due(t, s->ns, w->result, w->expired)) ++ g5p_rv3_touch(g, t, s->ns); ++ g5p_l123_pause(d, g, e, s, seen); ++ } ++} ++ ++/* One phase's deadline: the legacy single bound, or (latency101) the soft/hard pair. */ ++static void g5p_rv3_wait_start(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_ru_wait *w, u64 pub_ns, ++ u64 job_deadline_ns) ++{ ++ if (r->hard_ms) ++ g5p_ru_wait_start_lat(w, pub_ns, r->soft_ms, r->hard_ms, ++ job_deadline_ns, ++ READ_ONCE(g->boot->events)); ++ else ++ g5p_ru_wait_start(w, pub_ns, r->wait_ms, job_deadline_ns); ++} ++ ++/* After a timeout (or fault) was decided: the one RAM look of the legacy path, or (latency101, ++ * timeout only) a RAM-only observation window that keeps draining and sampling at the wait cadence ++ * and prints nothing, so a completion that lands milliseconds after the decision is recorded ++ * (late_after/late_ns) before the receipt line and the terminal latch. It never changes the ++ * result and never touches a register. The 3D look steps a copy of the predicate's record, so the ++ * receipt keeps the deciding sample's. ++ */ ++static void g5p_rv3_observe(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, ++ const struct g5p_ru_retire *rt, ++ const struct g5p_ru_ta *ta, struct g5p_rv3_ev *late) ++{ ++ struct g5p_ru_retire copy, *c = NULL; ++ struct g5p_ru_ta tcopy, *tc = NULL; ++ ++ if (rt) { ++ copy = *rt; ++ c = © ++ } ++ if (ta) { ++ tcopy = *ta; ++ tc = &tcopy; ++ } ++ if (!r->obs_ms || w->result != G5P_RU_WAIT_TIMEOUT) { ++ g5p_rv3_sample(g, r, e, seq, late); ++ g5p_ru_wait_after(w, g5p_rv3_done(e, late, c, tc)); ++ return; ++ } ++ do { ++ g5p_boot_drain( ++ g); /* a fault latches b->failed, the result stays */ ++ g5p_rv3_sample(g, r, e, seq, late); ++ if (!g5p_ru_wait_observe(w, late->ns, ++ g5p_rv3_done(e, late, c, tc), ++ r->obs_ms)) ++ break; ++ usleep_range(1000, 2000); ++ } while (1); ++} ++ ++/* wake110: render-user-wake.h's register access, on the Fender and RGX windows. */ ++static void g5p_rv3_wk_w1(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->window[1] + reg); ++} ++ ++/* The GPC state read of g5p_wk_ring() comes right after its R1+0x14 = 7 wake: in bell mode ++ * (render_user_wake=2) it waits up to G5P_RV3_WK_SETTLE_US, re-reading every 10 us, for a settled ++ * powered pstate (bit 4 clear, nibble nonzero), as Apple's setGPCPerformanceState waits for bit 4. ++ * Reads only; the ring's own check of the final value is unchanged. ++ */ ++#define G5P_RV3_WK_SETTLE_US 2000U ++ ++static u32 g5p_rv3_wk_r1(void *ctx, u32 reg) ++{ ++ void __iomem *r1 = ((struct g5p_diag *)ctx)->window[1]; ++ u32 raw = readl(r1 + reg); ++ u64 t0; ++ ++ if (reg != G5P_WK_R1_GPC || (2) != G5P_WK_ALWAYS || ++ (!(raw & 0x10) && (raw & 0xf))) ++ return raw; ++ t0 = ktime_get_mono_fast_ns(); ++ while (((raw & 0x10) || !(raw & 0xf)) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_WK_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(r1 + reg); ++ } ++ return raw; ++} ++ ++static void g5p_rv3_wk_wmb(void *ctx) ++{ ++ dma_wmb(); ++} ++ ++static void g5p_rv3_wk_w0(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->rgx + reg); ++} ++ ++/* wake110: an r74f bell that still owns its ring (the ALIVE decoder may fire it). */ ++static bool g5p_rv3_wk_r74f_busy(struct g5p_r73b *x) ++{ ++ u32 k = READ_ONCE(x->r74f_active); ++ int s; ++ ++ if (!k || k > G5P_R74F_KINDS) ++ return false; ++ s = atomic_read_acquire(&x->r74f_bell[k - 1].state); ++ return s == G5P_R74F_PUBLISHED || s == G5P_R74F_OPEN || ++ s == G5P_R74F_FIRING; ++} ++ ++/* wake111: DS108's claim proof re-read at the publish: ALIVE proven in the current reset ++ * generation, the claim admitted in it, nothing latched failed since, no R74E power abort. ++ * Called under event_lock (interrupts off); g5p_ds_lock is a leaf (g5p_ds_event() is pure). ++ */ ++static bool g5p_rv3_wk_ds108(struct g5p_boot *b) ++{ ++ bool ok; ++ ++ spin_lock(&g5p_ds_lock); ++ ok = g5p_ds_proven(&g5p_ds_state) && ++ g5p_ds_ok(&g5p_ds_state, G5P_DSE_CLAIM); ++ spin_unlock(&g5p_ds_lock); ++ return ok && !(b->r73b && READ_ONCE(b->r73b->early_power_abort)); ++} ++ ++/* wake110: before the phase's producer store, the engine's type-1 count and a cleared first-decode ++ * time, so an event decoded right after the publish is counted. RAM only. ++ */ ++static void g5p_rv3_wake_mark(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ v->wk_t1_pub[e] = d->t1_count[e]; ++ WRITE_ONCE(d->t1_first_ns[e], 0); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* wake110: at the job's end, the type-1 counts and the first decode of each published phase. */ ++static void g5p_rv3_wake_end(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ for (u32 e = 0; e < 2; e++) { ++ v->wk_t1_end[e] = d->t1_count[e]; ++ v->wk_first_ns[e] = d->t1_first_ns[e]; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (v->phase < G5P_RV3_TA_PUBLISH) ++ v->wk_first_ns[0] = 0; ++ if (v->phase < G5P_RV3_TD_PUBLISH) ++ v->wk_first_ns[1] = 0; ++} ++ ++/* wake110: microseconds / milliseconds from a phase's publish to a later instant; -1 = none. */ ++static s64 g5p_rv3_wk_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++static s64 g5p_rv3_wk_ms(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_MSEC : ++ -1; ++} ++ ++/* wake110: one phase's publish-time decision, right after its producer store and barrier. Both ++ * arms take the event lock (the r74f owner's serialization against the decoder) and sample the ++ * same RAM; only a factor that passed every guard runs g5p_wk_ring(). Nothing is printed under ++ * the lock. Off: returns at once. ++ */ ++static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e, u32 start, u32 seq) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ struct g5p_drm *d = r->ctx->gpu; ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ u8 *p = g5p_ru_page(b, e); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&b->event_lock, flags); ++ v->wk_start_ns[e] = ktime_get_ns(); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = r->wake, ++ .seed = 0, ++ .seq = seq, ++ .e = e, ++ .attempted = g5p_wk_attempted(v->wk_did[e]), ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = r->target[e], ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4) }, ++ .type1 = READ_ONCE(d->t1_count[e]) - v->wk_t1_pub[e], ++ }; ++ v->wk_cons[e] = in.cons; ++ v->wk_power[e] = in.power; ++ v->wk_did[e] = g5p_wk_decide(&in, &io, &v->wk_fence[e], &v->wk_gpc[e]); ++ v->wk_done_ns[e] = ktime_get_ns(); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (v->wk_did[e] == G5P_WK_DID_RANG) ++ r->wake_rang++; ++} ++ ++/* The epoch's first job: the device (PM, channels) and this context's DC3 records, shared with ++ * g5p_render_attach(), then the r74f/r74g protocol of the fixed r74f_attach() on typed state. ++ */ ++static int g5p_rv3_attach(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline; ++ u32 start, tag, bad; ++ int ret; ++ ++ ret = g5p_r74f_admit(g, G5P_R74F_DC3); ++ if (ret) ++ return ret; ++ ret = g5p_render_attach_prepare(d, r, &start, &tag); ++ if (ret) ++ return ret; ++ r->dc_target = (start + 2) & 255; ++ r->bound = true; ++ rd->up = true; ++ rd->v3 = true; ++ rd->job_slots = (2) == 2 ? 2 : 1; ++ rd->pm_once = (1) == 1; ++ rd->pm_ctx = b->pm_sets ? min((3), 3U) : 0; ++ r->pm_first = r->pm_set < 0; ++ dev_info( ++ g->dev, ++ "RENDER-V3 layout job_slots=%u pm_once=%u slot1=%#x..%#x pm_ctx=%u pm_sets=%u " ++ "end=v3\n", ++ rd->job_slots, rd->pm_once, G5P_RU_SLOT1, G5P_RU_SLOT1_END, ++ rd->pm_ctx, b->pm_sets); ++ g5p_render_retain(r); ++ v->touched = true; ++ WRITE_ONCE(r->ctx->live, true); ++ dma_wmb(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ v->attach_ns = ktime_get_ns(); ++ g5p_r74_mark(g, G5P_R74_ATTACH_PUBLISHED, r->dc_target); ++ /* The producer is final from here; only the doorbell may be pending. */ ++ ret = g5p_r74f_bell(g, G5P_R74F_DC3, start, r->dc_target); ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || READ_ONCE(b->failed)) ++ return ret ?: -EIO; ++ dma_rmb(); ++ if (READ_ONCE(ctl[4]) != r->dc_target || ++ READ_ONCE(ctl[0]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(ctl[0]) == r->dc_target) { ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ bad = g5p_r74g_check(g, 0, r->slot, vm->root.dma, tag, ++ start); ++ g5p_r74_mark(g, G5P_R74_ATTACH_CONSUMED, r->dc_target); ++ g5p_r74g_dc3_line(g, 0, false); ++ if (bad) ++ return g5p_rv3_errno(g, ++ g5p_r74g_refuse(g, bad)); ++ for (u32 e = 0; e < 2; e++) ++ g5p_mmu_record( ++ g, ++ "RENDER tags slot=%u host=%08x cached=%08x expected=%08x\n", ++ r->slot[e], ++ g5p_le32(g5p_boot_shared(b, 0x84) + ++ r->slot[e] * 4), ++ g5p_le32(g5p_boot_shared(b, 0x85) + ++ r->slot[e] * 4), ++ tag); ++ g5p_mmu_record( ++ g, ++ "RENDER-USER attached TA=%u 3D=%u PC=%pad tag=%08x blocks=%u; RAM-only, firmware " ++ "DC3 installed CBASE, host MMU pass skipped\n", ++ r->slot[0], r->slot[1], &vm->root.dma, tag, ++ rd->blocks); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } while (time_before(jiffies, deadline)); ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ return g5p_rv3_errno(g, ++ g5p_r74f_stop(g, G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ READ_ONCE(ctl[0]))); ++} ++ ++/* perf60 bell mode (render_user_wake=2): a DC publication (bind, spare pairs, unbind) gets Apple's ++ * submit kick too. With the policy held in state 0 the firmware no longer drains the DC ring on ++ * its own (perf60clk-9ec288b868: a second client's RAM-only DC3 bind was never consumed). The ++ * wake path's guards (event lock, power proof, no fault or terminal, no r74f bell owning a ring, ++ * the ring holding exactly this publication), then Apple's order: R1+0x14 = 7 and its read-back, ++ * R1+0x1000 re-read every 10 us until a settled powered pstate (g5p_gpc_ok(): bit 4 clear, ++ * nibble nonzero; Apple's setGPCPerformanceState waits for bit 4 the same way), at most ++ * G5P_RV3_DC_SETTLE_US, and R0+0xb00 = 0x10 only then. A GPC that never reads on is not written: ++ * no R0 access, the publication waits as before. Off (render_user_wake != 2): returns at once. ++ */ ++#define G5P_RV3_DC_SETTLE_US 2000U ++ ++static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, ++ const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 fence = U32_MAX, gpc = U32_MAX, power, cons, n = 0; ++ const char *did = "skip-guard"; ++ unsigned long flags; ++ u64 t0; ++ ++ ; ++ ; ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ cons = READ_ONCE(ctl[0]); ++ power = g5p_wk_power_mask(READ_ONCE(g->powered), ++ READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), ++ g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ !g5p_wk_power_ready(power) || g5p_rv3_wk_r74f_busy(b->r73b) || ++ cons != start || READ_ONCE(ctl[4]) != target) ++ goto out; ++ writel(7, g->window[1] + 0x14); ++ fence = readl(g->window[1] + 0x14); ++ if (fence & ~7U) { ++ did = "skip-fence"; ++ goto out; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ gpc = readl(g->window[1] + 0x1000); ++ while (!g5p_gpc_ok(gpc) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ gpc = readl(g->window[1] + 0x1000); ++ n++; ++ } ++ if (!g5p_gpc_ok(gpc)) { ++ did = "skip-gpc"; ++ goto out; ++ } ++ /* the publication's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ wmb(); ++ writel(0x10, g->rgx + 0xb00); ++ did = "rang"; ++out: ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 dc-kick why=%s did=%s dc=%u->%u power=%02x fence=%08x gpc=%08x " ++ "settle=%u end=v3\n", ++ why, did, cons, target, power, fence, gpc, n); ++} ++ ++/* RAM only: drain the event stream, read the firmware pages (interim, see g5p_rv3_fw_touch) and ++ * sample the DC ring until its consumer reaches `target` ++ * or G5P_RU_BIND_MS pass. No doorbell and no register, whether or not a job has retired. ++ */ ++static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_RU_BIND_MS); ++ ++ for (;;) { ++ bool expired = time_after_eq(jiffies, deadline); ++ int ret = g5p_boot_drain(g); ++ ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ g5p_rv3_fw_touch(b); ++ dma_rmb(); ++ switch (g5p_ru_bind_step(READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), ++ target, expired)) { ++ case G5P_RU_BIND_CONSUMED: ++ *consumed_ns = ktime_get_ns(); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } ++} ++ ++static s64 g5p_rv3_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++/* ++ * The first job of a context on a device that is already up binds the context's slot pair ++ * with a DC3 pair published in RAM only (multi-client-design.md §4.2). No doorbell and no ++ * register access of any kind, so it is safe after a job retired and while the GPU may be ++ * idle-gated: the firmware services the DC ring in its scheduler pass, before TA and 3D, and ++ * the host waits on RAM. Core lock held, between jobs. ++ * ++ * Refused before the producer store, it touched nothing. From the store on, the firmware may ++ * hold the pair: the context is retained, and on failure it is lost (v->bind_failed) but the ++ * device and the contexts already bound are not. A pair not consumed in time keeps every later ++ * bind refused until the firmware catches up. ++ */ ++/* ++ * render_user_close_park (release8 quiet2, docs/gpu/experiments/release8-quiet2.md). The PM writes ++ * a closed context's free list back through that context's TA slot after the context is gone, and ++ * not only while it is the last or the previous distinct context (render-user-pmset.h): two bench ++ * faults wrote the free list of a context two switches back, through its slot, on the 2nd job of a ++ * third context, after the pm-prev rule had unbound it (page-invalid level 2, its PC level empty). ++ * iOS never leaves the PM's target unresolvable: its PM is the device's (shared by the work queues, ++ * AGXWorkQueue_initAndAttachPM), in the kernel's GPU address space, and a GART keeps its slots until ++ * the address space is destroyed (AGXGart_detachSlotsAndDestroyAddressSpace). Here, a closed ++ * context's pair is bound again, right after its unbind (between jobs, core lock held), to the park ++ * catalogue: a kernel page that maps only the firmware window, where the PM sets live. A late ++ * write-back then lands in the closed context's own set page (released, reused last) instead of ++ * faulting. A later bind of the pair unbinds it first. Both steps are the proven DC3 ++ * transitions (PA to zero, zero to PA); a park that fails leaves the pair unbound, as before. ++ */ ++/* release8 default */ ++ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why); ++ ++/* Before a bind, core lock held: a pair a close parked (RAM: park PA and tag on both slots) is ++ * unbound first, by its own DC3. 0 when the pair was not parked or the unbind verified. ++ */ ++static int g5p_rv3_unpark(struct g5p_drm *d, const u32 *slot) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ ++ if (!d->rdev->park.cpu) ++ return 0; ++ dma_rmb(); ++ g5p_render_bind_mem(b, slot, READ_ONCE(ctl[4]), &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!g5p_ru_bind_parked(&s, d->rdev->park.dma)) ++ return 0; ++ return g5p_rv3_unbind_slots(d, slot, 2, "unpark"); ++} ++ ++static int g5p_rv3_bind(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, tag, bad = 0; ++ int ret; ++ ++ dma_rmb(); ++ if (rd->dc_pending && READ_ONCE(ctl[0]) != rd->dc_wait) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u dc=%u/%u wait=%u: an earlier bind is " ++ "pending; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4]), rd->dc_wait); ++ return -EIO; ++ } ++ rd->dc_pending = false; ++ ret = g5p_rv3_unpark(d, r->slot); ++ if (ret) ++ return ret; ++ ret = g5p_render_vm_prepare(d, r, &start, &tag); ++ if (ret) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u ret=%d dc=%u/%u: ring busy or slot in use; " ++ "no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], ret, READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4])); ++ return ret; ++ } ++ r->dc_target = (start + 2) & 255; ++ g5p_render_retain(r); ++ v->bind_failed = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, r->dc_target, "bind"); ++ ret = g5p_rv3_dc_wait(g, r->dc_target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = r->dc_target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, vm->root.dma, tag, ++ r->dc_target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=%lld tag=%08x host=%08x/%08x " ++ "cached=%08x/%08x dc=%u->%u/%u bad=%#x pc=%pad mode=ram-dc3 end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ tag, s.host[0], s.host[1], s.cached[0], s.cached[1], start, ++ s.cons, s.prod, bad, &vm->root.dma); ++ if (ret) ++ return ret; ++ v->bind_failed = false; ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ return 0; ++} ++ ++/* ++ * TEMPORARY fallback (render_user_spare_pairs, read at the epoch's first job), for a firmware ++ * that never drains a DC3 published after a job retired. Before job 1's attach, bind that ++ * many extra slot pairs, each to a zeroed kernel-owned catalogue, in one RAM-only batch with no ++ * doorbell. The firmware drains it on its own like the self-drained attach of this boot tuple ++ * (r87b); job 1's own attach, doorbell and r74g checks then run unchanged behind it ++ * (g5p_r74f_ring() refuses a ring that holds more than that attach's pair). Later contexts ++ * take a pair over with g5p_rv3_adopt(). A pair that does not check out is never used. Core ++ * lock held, device not up, so register access is still allowed (none is made here). ++ */ ++static int g5p_rv3_spares(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 want = min((0), G5P_RU_MAX_SPARES); ++ u32 start = READ_ONCE(ctl[4]), target, n, ok = 0; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ unsigned long flags; ++ int ret; ++ ++ if (!want || rd->spares) ++ return 0; ++ rd->spares = kcalloc(want, sizeof(*rd->spares), GFP_KERNEL); ++ if (!rd->spares) ++ return -ENOMEM; ++ for (n = 0; n < want; n++) { ++ struct g5p_rspare *sp = &rd->spares[n]; ++ bool pair; ++ ++ spin_lock_irqsave(&g->slot_lock, flags); ++ pair = g5p_ru_slot_pair(g->user_slots, false, sp->slot); ++ if (pair) { ++ __set_bit(sp->slot[0], g->user_slots); ++ __set_bit(sp->slot[1], g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ if (!pair) ++ break; ++ g5p_render_bind_mem(b, sp->slot, start + 2 * n, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (!g5p_ru_bind_busy(&s) && ++ !g5p_vm_alloc_page(d->drm.dev, &sp->root)) { ++ g5p_table_clean_page(g, &sp->root); ++ continue; ++ } ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(sp->slot[0], g->user_slots); ++ __clear_bit(sp->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ break; ++ } ++ if (!n) ++ return 0; ++ dsb(sy); ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_bind_mem(b, rd->spares[i].slot, start + 2 * i, &m); ++ g5p_ru_bind_publish(&m, rd->spares[i].slot, ++ rd->spares[i].root.dma, G5P_RU_SPARE_TAG); ++ } ++ rd->spare_count = n; ++ target = (start + 2 * n) & 255; ++ v->touched = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "spares"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ for (u32 i = 0; !ret && i < n; i++) { ++ struct g5p_rspare *sp = &rd->spares[i]; ++ ++ g5p_render_bind_mem(b, sp->slot, start + 2 * i, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ sp->ok = !g5p_ru_bind_check(&s, sp->slot, sp->root.dma, ++ G5P_RU_SPARE_TAG, target); ++ ok += sp->ok; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 spares pairs=%u ok=%u ret=%d consumed_us=%lld dc=%u->%u first=%u/%u " ++ "last=%u/%u end=v3\n", ++ n, ok, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ READ_ONCE(ctl[0]), rd->spares[0].slot[0], rd->spares[0].slot[1], ++ rd->spares[n - 1].slot[0], rd->spares[n - 1].slot[1]); ++ return ret; ++} ++ ++/* ++ * TEMPORARY fallback: the first job of a context on a device whose job 1 bound spare pairs. ++ * Takes the next pair that checked out, makes its catalogue page this VM's root (a copy of the ++ * current root) and publishes the VM's tag through the proven RAM-only tag path: CBASE already ++ * names the page, and the changed tag makes the firmware invalidate the slots' PC level at this ++ * context's first job. No DC3, no register. Core lock held; the context has a job queued, so ++ * no VM edit of it runs concurrently. ++ */ ++static int g5p_rv3_adopt(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_rspare *sp = NULL; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ struct g5p_dma_page old; ++ unsigned long flags; ++ struct g5p_rt *rt; ++ u32 entry; ++ int ret; ++ ++ for (u32 i = 0; i < rd->spare_count && !sp; i++) ++ if (rd->spares[i].ok && !rd->spares[i].used) ++ sp = &rd->spares[i]; ++ if (!sp) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare: none left of %u; no hardware " ++ "touched end=v3\n", ++ r->slot[0], r->slot[1], rd->spare_count); ++ return -ENOSPC; ++ } ++ /* RAM: the pair still names its page, with the tag job 1 checked. */ ++ dma_rmb(); ++ g5p_render_bind_mem(b, sp->slot, 0, &m); ++ g5p_ru_bind_read(&m, 0, 0, &s); ++ for (u32 e = 0; e < 2; e++) { ++ if (s.pa[e] == sp->root.dma && s.host[e] == G5P_RU_SPARE_TAG && ++ s.cached[e] == G5P_RU_SPARE_TAG) ++ continue; ++ sp->ok = false; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare pair=%u/%u host=%08x cached=%08x: " ++ "pair changed; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], sp->slot[0], sp->slot[1], ++ s.host[e], s.cached[e]); ++ return -EIO; ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ memcpy(sp->root.cpu, vm->root.cpu, G5P_GPU_PAGE); ++ old = vm->root; ++ vm->root = sp->root; ++ sp->root = old; /* the VM's former root, never published */ ++ g5p_ru_free_page(d->drm.dev, &sp->root); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(r->slot[0], g->user_slots); ++ __clear_bit(r->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ r->slot[0] = sp->slot[0]; ++ r->slot[1] = sp->slot[1]; ++ sp->used = true; ++ g5p_render_retain(r); ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ /* Root and tables to PoC, then a tag unlike the cached one in both slots. */ ++ ret = g5p_render_invalidate(r->ctx); ++ v->bind_failed = ret != 0; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=0 tag=%08x cached=%08x/%08x " ++ "pc=%pad mode=spare end=v3\n", ++ r->slot[0], r->slot[1], ret, ++ g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[0] * 4), ++ s.cached[0], s.cached[1], &vm->root.dma); ++ return ret; ++} ++ ++/* r74f_publish_ta() on typed state: record, WORK_CTL, producer; then (job 1 only) the r74g grace ++ * and, only if the ring is still pending, the one ALIVE-owned fallback doorbell. Jobs >= 2 ++ * publish and return: no grace, no bell (the r74f/r74g machinery is one-shot). No canary, no marker. ++ */ ++static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *x = b->r73b; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_r74g_dc3 *dc = x->r74g_dc3; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x60); ++ u8 *p = g5p_ru_page(b, 0); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ bool first = seq == 1; ++ u32 bad; ++ int ret; ++ ++ if (!r->bound || r->submitted[0] || start > 255 || ++ READ_ONCE(ring[0]) != start || ++ g5p_le32(p + G5P_WORK_CTL) != previous || ++ g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) ++ return -EBUSY; ++ if (first) { ++ ret = g5p_r74f_admit(g, G5P_R74F_TA); ++ if (ret) ++ return ret; ++ /* g5p_r74g_ta_preflight() on typed state: DC3 still complete with the same records, ++ * tag and backing, and no page-table mutation since the attach. ++ */ ++ bad = g5p_r74g_check(g, 1, r->slot, dc[0].expected_pa, ++ dc[0].tag, dc[0].start); ++ if (g5p_vm_generation(&vm->mutation) != dc[0].tag || ++ vm->root.dma != dc[0].expected_pa || ++ r->dc_target != dc[0].target) { ++ bad |= G5P_R74G_BAD_MUTATION; ++ dc[1].bad = bad; ++ } ++ g5p_r74g_dc3_line(g, 1, false); ++ if (bad) ++ return g5p_rv3_errno(g, g5p_r74g_refuse(g, bad)); ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x61] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 0 : ++ j == 1 ? g5p_ru_meta(0) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[0] = (start + 1) & 255; ++ r->submitted[0] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_PUBLISH, start); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], r->target[0]); ++ dma_wmb(); ++ if (first) { ++ ret = g5p_r74g_ta(g, start, r->target[0]); ++ v->grace_end = READ_ONCE(x->r74g_grace_end); ++ v->bell = ret == 0; ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ } ++ /* wake110: jobs >= 2 reach this at once after the producer store (job 1: excluded). */ ++ if (!first) ++ 0; ++ g5p_rv3_wake(g, r, v, 0, start, seq); ++ /* The line the userspace lane parses (a9c_parse_publish): same format as the fixed path's. */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 0, r->slot[0], endpoint, first ? 0x10 : 0, ++ g5p_ru_meta(0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 0); ++ return 0; ++} ++ ++/* The RAM preconditions of the 3D publish, one bit per clause (0 = admitted). RAM reads only; the bits are ++ * what the refusal line prints, so a second blocker is visible in the run that meets it: ++ * 0 TA not retired 1 TA not consumed 2 3D already submitted 3 producer > 255 4 consumer != producer ++ * 5 WORK_CTL 6 WORK_CTL+0x40 7 command tag != 4 8 dependency word 9 dependency value ++ * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) ++ */ ++static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, ++ const u32 *ring, u32 start, u32 previous, u32 seq) ++{ ++ const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ ++ return (u32)!r->retired[0] | (u32)!r->consumed[0] << 1 | ++ (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | ++ (u32)(READ_ONCE(ring[0]) != start) << 4 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 5 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 6 | ++ (u32)(g5p_le32(cmd) != 4) << 7 | ++ (u32)(g5p_le32(cmd + 4) != g5p_ru_meta(0) + G5P_R22_STAMP + 4) ++ << 8 | ++ (u32)(g5p_le32(cmd + 8) != seq) << 9 | ++ (u32)(g5p_le32(p + G5P_R22_STAMP) != seq - 1 || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq - 1) ++ << 10; ++} ++ ++/* The 3D after the TA retired: record, WORK_CTL, producer, and NO doorbell. */ ++static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x64); ++ u8 *p = g5p_ru_page(b, 1); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ u32 why; ++ ++ /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one ++ * fails, and the refusal names the clauses and the raw values it read. ++ */ ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq); ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 3d-refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x cmd=%08x/%08x/%08x " ++ "stamp=%08x/%08x end=v3\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ seq))), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 4), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 8), ++ g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x65] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 1 : ++ j == 1 ? g5p_ru_meta(1) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[1] = (start + 1) & 255; ++ r->submitted[1] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->td_pub_ns = ktime_get_ns(); ++ smp_store_release(&ring[4], r->target[1]); ++ dma_wmb(); ++ 0; ++ g5p_rv3_wake(g, r, v, 1, start, seq); ++ /* The 3D twin of the TA's publish line: engine 1, kick 0 (no doorbell). */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 1, r->slot[1], endpoint, 0, ++ g5p_ru_meta(1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 1); ++ return 0; ++} ++ ++static const char *const g5p_rv3_ta_how[G5P_RU_TA_HOWS] = { "none", ++ "idle-clear", ++ "event", "hold" }; ++ ++/* tahang1: the TA decisions since load, by rule (core lock held). A TA that completed with an ++ * idle word still set prints one line with the running totals: rate-limited for rule (b), which ++ * is routine, always for rule (c), which means the event was not seen. RAM only. ++ */ ++static u32 g5p_rv3_ta_rules[G5P_RU_TA_HOWS]; ++static DEFINE_RATELIMIT_STATE(g5p_rv3_ta_rs, 5 * HZ, 4); ++ ++static void g5p_rv3_ta_count(struct g5p_diag *g, struct g5p_drm *d, ++ const struct g5p_rv3 *v) ++{ ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ u32 how = min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1); ++ u64 hit = READ_ONCE(d->ta_hit_ns); ++ ++ g5p_rv3_ta_rules[how]++; ++ if (how == G5P_RU_TA_IDLE_CLEAR) ++ return; ++ if (how == G5P_RU_TA_EVENT && !__ratelimit(&g5p_rv3_ta_rs)) ++ return; ++ dev_warn( ++ g->dev, ++ "RENDER-V3 ta-done seq=%u how=%s idle=%08x,%08x,%08x pm80=%08x coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld busy_ms=%lld rules=%u/%u/%u end=v3\n", ++ v->seq, g5p_rv3_ta_how[how], s->idle[0], s->idle[1], s->idle[2], ++ s->pm80, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, hit), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_ta_rules[G5P_RU_TA_IDLE_CLEAR], ++ g5p_rv3_ta_rules[G5P_RU_TA_EVENT], ++ g5p_rv3_ta_rules[G5P_RU_TA_HOLD]); ++} ++ ++/* A receipt line: a failed job's at KERN_EMERG as before (it survives a freeze), a completed job's at ++ * job_log (wake every job, end/lat/retire the first 8 of a context: routine at desktop rates). ++ */ ++#define g5p_rv3_receipt_printk(g, ret, fmt, ...) \ ++ do { \ ++ if (ret) \ ++ dev_emerg((g)->dev, fmt, ##__VA_ARGS__); \ ++ else \ ++ g5p_job_printk((g)->dev, fmt, ##__VA_ARGS__); \ ++ } while (0) ++ ++/* The end of a job, whatever it was: one receipt line on the console (it survives a freeze) and the ++ * RAM-only raw state in the blob. Reads RAM only. ++ */ ++static void g5p_rv3_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_r73b *x = g->boot->r73b; ++ const struct g5p_rv3_ev *t = &v->last[0], *d = &v->last[1]; ++ const struct g5p_ru_touch *k = v->touch; ++ ++ /* RC4a: every job while the A/B is on, so both arms are complete past the verbose jobs. */ ++ if (r->touch_ms) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 touch seq=%u ret=%d arm=%s touch_ms=%u ta=%s/%lld td=%s/%lld " ++ "seen_us=%lld/%lld read_us=%lld/%lld after_us=%lld/%lld bytes=%u/%u " ++ "sum=%08x/%08x end=v3\n", ++ v->seq, ret, ++ g5p_ru_touch_arm(v->seq) ? "touch" : "control", ++ r->touch_ms, g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? (s64)(v->ta.end_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? (s64)(v->td.end_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_ru_touch_us(v->ta_pub_ns, k[0].seen_ns), ++ g5p_ru_touch_us(v->td_pub_ns, k[1].seen_ns), ++ g5p_ru_touch_us(k[0].seen_ns, k[0].done_ns), ++ g5p_ru_touch_us(k[1].seen_ns, k[1].done_ns), ++ g5p_ru_touch_after_us(&k[0], v->ta.result, ++ v->ta.end_ns), ++ g5p_ru_touch_after_us(&k[1], v->td.result, ++ v->td.end_ns), ++ k[0].bytes, k[1].bytes, k[0].sum, k[1].sum); ++ /* wake110: every job while the wake is on (the end/lat/retire lines stop at seq 8). */ ++ if (r->wake) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 wake seq=%u ret=%d mode=%u seed=%08x arm=%s/%s did=%s/%s " ++ "power=%02x/%02x fence=%08x/%08x gpc=%08x/%08x cons_at=%u/%u bell_us=%lld/%lld " ++ "lock_us=%lld/%lld ta=%s/%lld td=%s/%lld ev_ms=%lld/%lld consumed_ms=%lld/%lld " ++ "busy_ms=%lld/%lld t1_pub=%u/%u t1_end=%u/%u endpoint=%u ta_stamp=%08x/%08x " ++ "td_stamp=%08x/%08x ta_ring=%u/%u td_ring=%u/%u rang=%u end=v3\n", ++ v->seq, ret, r->wake, r->wake_seed, ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 0)], ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[0], ++ G5P_WK_DIDS - 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[1], ++ G5P_WK_DIDS - 1)], ++ v->wk_power[0], v->wk_power[1], v->wk_fence[0], ++ v->wk_fence[1], v->wk_gpc[0], v->wk_gpc[1], ++ v->wk_cons[0], v->wk_cons[1], ++ g5p_rv3_wk_us(v->ta_pub_ns, v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->td_pub_ns, v->wk_done_ns[1]), ++ g5p_rv3_wk_us(v->wk_start_ns[0], v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->wk_start_ns[1], v->wk_done_ns[1]), ++ g5p_rv3_results[v->ta.result], ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_rv3_results[v->td.result], ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->wk_first_ns[0]), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->wk_first_ns[1]), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td_consumed_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.busy_ns), ++ v->wk_t1_pub[0], v->wk_t1_pub[1], v->wk_t1_end[0], ++ v->wk_t1_end[1], g5p_ru_endpoint(v->seq), t->stamp[0], ++ t->stamp[1], d->stamp[0], d->stamp[1], t->prod, t->cons, ++ d->prod, d->cons, r->wake_rang); ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 end seq=%u ret=%d phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " ++ "cap_ms=%u ta_consumed_ms=%lld td_consumed_ms=%lld late_gap=%u/%u " ++ "late_after=%u/%u grace=%u bells=%d ta_ring=%u/%u td_ring=%u/%u " ++ "ta_stamp=%08x/%08x td_stamp=%08x/%08x pm80=%08x r0_after_ta_publish=bell-only " ++ "end=v3\n", ++ v->seq, ret, ++ g5p_rv3_phase_names[min_t(u32, v->phase, G5P_RV3_PHASES - 1)], ++ g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? ++ (s64)(v->ta.end_ns - v->ta_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? ++ (s64)(v->td.end_ns - v->td_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ r->wait_ms, G5P_RU_V3_CAP_MS, ++ v->ta_consumed_ns ? (s64)(v->ta_consumed_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td_consumed_ns ? (s64)(v->td_consumed_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_gap, v->td.late_gap, v->ta.late_after, ++ v->td.late_after, v->grace_end, atomic_read(&x->r74f_bells), ++ t->prod, t->cons, d->prod, d->cons, t->stamp[0], t->stamp[1], ++ d->stamp[0], d->stamp[1], t->pm80 ? t->pm80 : d->pm80); ++ if (r->hard_ms) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 lat seq=%u soft_ms=%u hard_ms=%u obs_ms=%u slow=%u/%u late_soft=%u/%u " ++ "silent=%u/%u soft_seen_ms=%lld/%lld busy_ms=%lld/%lld late_after_ms=%lld/%lld " ++ "events=%u end=v3\n", ++ v->seq, r->soft_ms, r->hard_ms, r->obs_ms, v->ta.slow, ++ v->td.slow, v->ta.late_soft, v->td.late_soft, ++ v->ta.silent, v->td.silent, ++ v->ta.soft_seen_ns ? ++ (s64)(v->ta.soft_seen_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.soft_seen_ns ? ++ (s64)(v->td.soft_seen_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.busy_ns ? (s64)(v->ta.busy_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.busy_ns ? (s64)(v->td.busy_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_ns ? (s64)(v->ta.late_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.late_ns ? (s64)(v->td.late_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ READ_ONCE(g->boot->events)); ++ /* repeat109: the 3D's firmware retirement (coherent_ms, event_ms) apart from the host's decision ++ * (decided_ms; held_ms = how long a coherent target waited for it), the clauses still open at the ++ * deciding or closing sample (unsat=TA/3D, render-user-retire.h bits) and that sample's 3D idle words. ++ */ ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 retire seq=%u ret=%d td=%s how=%s settle_ms=%u coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld held_ms=%lld unsat=%#x/%#x " ++ "idle=%08x,%08x,%08x,%08x,%08x,%08x run=%u breaks=%u busy_ms=%lld clear_ms=%lld " ++ "ta_how=%s ta_unsat=%#x ta_coherent_ms=%lld ta_event_ms=%lld " ++ "ta_idle=%08x,%08x,%08x end=v3\n", ++ v->seq, ret, g5p_rv3_results[v->td.result], ++ g5p_rv3_how[min_t(u32, v->rt.decided, 3)], r->settle_ms, ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td_event_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_ru_rt_ms(v->rt.coherent_ns, v->td.end_ns), ++ g5p_rv3_ta_unsat(t), v->rt.unsat, d->idle[0], d->idle[1], ++ d->idle[2], d->idle[3], d->idle[4], d->idle[5], v->rt.run, ++ v->rt.breaks, g5p_ru_rt_ms(v->td_pub_ns, v->rt.busy_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.clear_ns), ++ g5p_rv3_ta_how[min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1)], ++ v->tr.unsat, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta_event_ns), t->idle[0], ++ t->idle[1], t->idle[2]); ++ /* The raw rows once per context (its first job) and for every failure: the blob is not a ++ * frame log. ++ */ ++ if (!v->touched || (r->jobs > 1 && !ret)) ++ return; ++ g5p_blob_hex(g, "RENDER-V3 PM-control", g5p_rv3_pm_cpu(g->boot, r), ++ 0x140); ++ g5p_blob_hex(g, "RENDER-V3 TA-W", ++ g5p_ru_page(g->boot, 0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, v->seq)), ++ G5P_RU_TA_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 TA-CH", g5p_ru_page(g->boot, 0), 0x340); ++ /* The stamp words (+0x1840/+0x1844) live in this 0x80-byte window; no earlier blob held one. */ ++ g5p_blob_hex(g, "RENDER-V3 TA-STAMP", ++ g5p_ru_page(g->boot, 0) + G5P_R22_TQ, 0x80); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) { ++ g5p_blob_hex(g, "RENDER-V3 3D-W", ++ g5p_ru_page(g->boot, 1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ v->seq)), ++ G5P_RU_3D_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 3D-CH", g5p_ru_page(g->boot, 1), ++ 0x340); ++ g5p_blob_hex(g, "RENDER-V3 3D-STAMP", ++ g5p_ru_page(g->boot, 1) + G5P_R22_TQ, 0x80); ++ } ++} ++ ++/* Jobs >= 2: the legacy job>=2 PM step with policy 0 (pool, list and F rebuilt from the device ++ * pool, state S kept), whichever context the job belongs to. ++ */ ++static int g5p_rv3_pm_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks); ++ mutex_unlock(&rd->lock); ++ kvfree(physical); ++ return ret; ++} ++ ++/* render-user-loss.h for a phase that did not complete. */ ++static int g5p_rv3_loss(struct g5p_drm *d, struct g5p_render *r, bool first, ++ u32 res, const struct g5p_ru_wait *w) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 drain_ms = (45000); ++ struct g5p_ru_loss_in in = { ++ .result = res, ++ .first = first, ++ .isolate = drain_ms && ++ g5p_ru_drain_valid(drain_ms, r->hard_ms, r->obs_ms), ++ .silent = w->silent, ++ .fault = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ }; ++ ++ return g5p_ru_loss(&in); ++} ++ ++/* ++ * The phase on engine `e` of this job timed out with the firmware alive: lose the job and its ++ * context, not the device. No terminal note and no device failure: the job ends -ETIMEDOUT ++ * (g5p_render_run() marks the context bad, so its client gets -EIO from now on and its pair and ++ * memory stay retained), and the channel is blocked behind the stuck phase until the scheduler's ++ * drain (g5p_rv3_drain()) clears it. RAM only. Fails (-ENOMEM) only if the record cannot be ++ * allocated; the caller then ends the epoch as before. ++ */ ++static int g5p_rv3_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ const struct g5p_ru_wait *w = e ? &v->td : &v->ta; ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = e; ++ s->drain_ms = (45000); ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u: context " ++ "lost, device kept, channel blocked end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++/* A job that stopped the firmware may be recovered from in this boot (core recover.inc, ++ * render-user-recover.h). A refusal is one line: why the epoch ends. ++ */ ++static bool g5p_rv3_recoverable(struct g5p_drm *d, bool first) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_ru_recover_in in = { ++ .first = first, ++ .retired = g5p_r8_is_retired(), ++ .closed = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .recoveries = rd->recoveries, ++ .streak = rd->recover_streak, ++ .budget = (1), ++ .streak_max = (3), ++ }; ++ ++ if (g5p_ru_recoverable(&in)) ++ return true; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 no-recover why=%s recoveries=%u streak=%u recover=%u streak_max=%u " ++ "boot_max=%u first=%u end=v3\n", ++ g5p_ru_recover_refusal(&in), in.recoveries, in.streak, ++ in.budget, in.streak_max, G5P_RU_RECOVER_BOOT_MAX, first); ++ return false; ++} ++ ++/* The job stopped the firmware (silent stream, a fault): lose it and its context, not the ++ * device; the worker recovers the GPU before it picks anything (g5p_rv3_recover()). RAM only. ++ */ ++static void g5p_rv3_lose_recover(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e, ++ const char *why) ++{ ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_fw_recovery rec = b->fw_recovery; ++ u32 recs = READ_ONCE(b->fw_recoveries); ++ ++ v->lost = true; ++ WRITE_ONCE(d->rdev->recovering, true); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u why=%s: context lost, device " ++ "recovers end=v3\n", ++ seq, e, r->slot[0], r->slot[1], why); ++ /* The guilty job is the one waited on (one job in flight, core lock); the packet's slot ++ * names the phase the firmware stopped in (render-user-recover.h). RAM only. ++ */ ++ if (recs) ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-attrib seq=%u engine=%u packet_slot=%u match=%u code=%08x " ++ "w20=%08x w24=%08x packets=%u end=v3\n", ++ seq, e, rec.slot, rec.slot == e, rec.code, rec.w20, ++ rec.w24, recs); ++} ++ ++/* render_user_test_fault_job: this job's TA reads its control stream (W+0x98, g5p_ru_fill_ta()) ++ * from the page the context reserved in its VM and never maps, so the GPU faults in that VM. A ++ * RAM edit of the job's own W copy before it is staged; the firmware sees an ordinary job. ++ */ ++static void g5p_rv3_test_fault(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, u32 seq) ++{ ++ g5p_put64(job->ta_w + 0x98, r->fault_node.start); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-fault seq=%u job=%u slots=%u/%u va=%#llx: TA control stream " ++ "unmapped end=v3\n", ++ seq, r->jobs + 1, r->slot[0], r->slot[1], r->fault_node.start); ++} ++ ++/* One V3 job: [attach, the epoch's job 1 only | bind, a later context's first job,] TA, 3D. ++ * Core lock held. Returns 0, -ECANCELED (a latched stage ended it), -ETIMEDOUT (a phase ran out ++ * its deadline, or the bind's DC3 was not consumed), or a fault. ++ */ ++/* perf60 MMU arm, bits 1/3 (render_user_mmu_inval): before a bound context's job publishes ++ * anything, move its slots' host tag to a fresh PC field, so the firmware's tag compare at this ++ * job's TA and 3D kicks takes its PC-level invalidate (e138 = (S << 3) | 4) for each slot. RAM ++ * only. vm->mutation is left alone: an edit on another thread (VM_BIND of a render VM takes no ++ * core lock) recomputes its own tag and checks it only against the cached tags, as before. ++ */ ++static void g5p_rv3_mmu_retag(struct g5p_diag *g, struct g5p_render *r, u32 seq, ++ bool switched) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 host[2], cached[2], tag; ++ ++ dma_rmb(); ++ for (u32 e = 0; e < 2; e++) { ++ host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4); ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ } ++ tag = g5p_ru_mmu_pc_tag(host[0] ?: cached[0], cached[0], cached[1]); ++ if (tag) { ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, ++ tag); ++ dma_wmb(); ++ } ++ g5p_mmu_record( ++ g, ++ "RENDER-V3 mmu-retag seq=%u slots=%u/%u why=%s host=%08x/%08x cached=%08x/%08x " ++ "tag=%08x end=v3\n", ++ seq, r->slot[0], r->slot[1], switched ? "switch" : "every", ++ host[0], host[1], cached[0], cached[1], tag); ++} ++ ++/* perf60 A/B: the park that the never-park envelope removed, back around a context switch only. ++ * Before a bound context's job that follows another context's job, ctrl+0x1040 = 50 makes the ++ * firmware's next pass compute x < 100 (policy state 2: it parks the GPC when idle, as release7); ++ * after the window the pin goes back and the job runs in state 0 with its bells as usual. RAM ++ * only: the firmware reloads policy+0x6f4 from ctrl+0x1040 on every pass (perf-envelope.h). ++ */ ++ ++ ++ ++/* perf60: the incoming context's render targets on a switch (lat123_log), RAM. */ ++static void g5p_rv3_switch_line(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ char buf[160]; ++ struct g5p_rt *rt; ++ u32 n = 0, len = 0; ++ ++ buf[0] = 0; ++ list_for_each_entry(rt, &r->targets, link) ++ if (n++ < 4) ++ len += scnprintf(buf + len, sizeof(buf) - len, ++ " %ux%u/s%u/p%u%s", rt->width, ++ rt->height, rt->samples ?: 1, ++ rt->page_count, ++ rt->destroyed ? "/d" : ""); ++ dev_info(g->dev, ++ "RENDER-V3 switch seq=%u to=%u/%u jobs=%u rts=%u%s end=v3\n", ++ seq, r->slot[0], r->slot[1], r->jobs, n, buf); ++} ++ ++/* perf60 per-context PM state (render_user_pm_switch, default 0). The PM record's selected state ++ * S (control+0x80, 0xc0 bytes: the backing pointers a0/a8/b0 the fresh TA loads the PM context ++ * from, the lanes TA finish saves and the 3D start restores, the bc accumulator) and the five ++ * aux pages it points into (the PM context store) are built once at attach and then shared by ++ * every context; the host never rewrites them. iOS builds one PM record per work queue ++ * (6cb2378: S explicit zeros plus the four backing pointers) and keeps it per queue. ++ * 1: a context's job after another context's job starts from attach's fresh state (S rebuilt, ++ * aux zeroed); ++ * 2: each context keeps its own S + aux: saved after each of its completed jobs, restored when ++ * it follows another context's job, fresh the first time. ++ * RAM only, between jobs (the previous job retired, this one has published nothing). ++ */ ++ ++/* perf60 stage 4: (re)build context r's PM set for this epoch: grow its pool to the device's ++ * block count, zero its aux pages, build its control + S, list and F (g5p_ru_pm(), as attach ++ * builds the device PM) and clean them to PoC. rd->lock (the pool's size) held. ++ */ ++static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ const char *how) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_dma_page *page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ u32 blocks = rd->blocks; ++ u64 *physical, list_va, free_va; ++ int ret = 0; ++ ++ lockdep_assert_held(&rd->lock); ++ if (blocks > r->pm_blocks) { ++ struct g5p_dma_page *pool = ++ kvcalloc(blocks * 8, sizeof(*pool), GFP_KERNEL); ++ ++ if (!pool) ++ return -ENOMEM; ++ if (r->pm_pool) ++ memcpy(pool, r->pm_pool, ++ r->pm_blocks * 8 * sizeof(*pool)); ++ for (u32 i = r->pm_blocks * 8; !ret && i < blocks * 8; i++) ++ ret = g5p_vm_alloc_page(d->drm.dev, &pool[i]); ++ if (ret) { ++ for (u32 i = r->pm_blocks * 8; i < blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &pool[i]); ++ kvfree(pool); ++ return ret; ++ } ++ kvfree(r->pm_pool); ++ r->pm_pool = pool; ++ r->pm_blocks = blocks; ++ } ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ for (u32 i = 0; i < r->pm_blocks * 8; i++) { ++ physical[i] = r->pm_pool[i].dma; ++ memset(r->pm_pool[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm_pool[i]); ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ struct g5p_dma_page *aux = &g5p_rv3_pm_aux(rd, r)[i]; ++ ++ memset(aux->cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, aux); ++ } ++ list_va = g5p_pmset_va(r->pm_set, G5P_PMSET_LIST); ++ free_va = g5p_pmset_va(r->pm_set, G5P_PMSET_FREE); ++ ret = g5p_ru_pm(page[G5P_PMSET_CONTROL].cpu, page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, list_va, free_va, physical, ++ r->pm_blocks, G5P_RU_PRIVATE_VA); ++ kvfree(physical); ++ for (u32 k = 0; k < G5P_PMSET_PAGES; k++) ++ g5p_table_clean_page(g, &page[k]); ++ dma_wmb(); ++ if (!ret) ++ r->pm_epoch = rd->recoveries + 1; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u meta=%#x " ++ "ret=%d end=v3\n", ++ seq, r->slot[0], r->slot[1], how, r->pm_set, ++ g5p_rv3_pm_id(rd, r), r->pm_blocks, g5p_rv3_pm_meta(r), ret); ++ return ret; ++} ++ ++/* Before a context's bind (its tables are published by the bind that follows): take a free set, ++ * allocate its aux pages, point G5P_RU_PRIVATE_VA in its VM at them, and build the set. On any ++ * failure the context keeps the device PM and the shared aux pages, as before stage 4. ++ */ ++static void g5p_rv3_pmctx_assign(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ int set, ret = 0; ++ ++ if (r->pm_set >= 0) ++ return; ++ set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next); ++ for (u32 i = 0; set >= 0 && !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!r->pm_aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &r->pm_aux[i]); ++ if (set < 0 || ret) { ++ dev_warn( ++ d->core->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=none set=%d ret=%d: the device PM end=v3\n", ++ seq, r->slot[0], r->slot[1], set, ret); ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "new"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ return; ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ u64 va = G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE; ++ u8 *pte = g5p_vm_pte(vm, d->drm.dev, va, false); ++ ++ u64 leaf = g5p_leaf_entry(r->pm_aux[i].dma, ++ g5p_r25_attributes(&r->v, va)); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, leaf, 2); ++ } ++ dma_wmb(); ++ r->pm_aux_mapped = true; ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* Before each job of a context with a set: a new epoch or a grown device pool rebuilds the set; ++ * without pm_once the pool is rebuilt per job, as g5p_rv3_pm_rebuild() does the device's. ++ */ ++static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_dma_page *page; ++ u64 *physical; ++ int ret = 0; ++ ++ if (r->pm_set < 0) ++ return 0; ++ page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ mutex_lock(&rd->lock); ++ if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { ++ ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); ++ } else if (!rd->pm_once) { ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), ++ GFP_KERNEL); ++ ret = physical ? 0 : -ENOMEM; ++ for (u32 i = 0; !ret && i < r->pm_blocks * 8; i++) ++ physical[i] = r->pm_pool[i].dma; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ page[G5P_PMSET_CONTROL].cpu, ++ page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, ++ g5p_pmset_va(r->pm_set, G5P_PMSET_LIST), ++ g5p_pmset_va(r->pm_set, G5P_PMSET_FREE), ++ physical, r->pm_blocks); ++ kvfree(physical); ++ } ++ mutex_unlock(&rd->lock); ++ return ret; ++} ++ ++/* render_user_pm_ctx 3: the context that brought the epoch up (the device PM for its attach job) ++ * takes its own set before its next job; the device aux pages its VM already maps become the ++ * set's aux (no other context maps them once every later one has its own), so no PTE changes. ++ */ ++static void g5p_rv3_pmctx_take(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ int set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next), ++ ret; ++ ++ if (set < 0) { ++ r->pm_first = false; /* stays on the device PM; not retried */ ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "take"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ r->pm_first = false; ++ return; ++ } ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* perf60 stage 4 diagnostics (render_user_fault_read): whose page each PM lane in S points at ++ * after the TA saved them: the device pool or aux, a set page, a context's pool or aux. ++ */ ++ ++static void g5p_rv3_pm_lanes(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ { ++ return; ++ } ++} ++ ++/* The context is gone (g5p_render_release()): its set is free again; its pages go. */ ++static void g5p_rv3_pmctx_release(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ ++ if (r->pm_set >= 0) ++ rd->pm_set_used[r->pm_set] = false; ++ r->pm_set = -1; ++ for (u32 i = 0; r->pm_pool && i < r->pm_blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_pool[i]); ++ kvfree(r->pm_pool); ++ r->pm_pool = NULL; ++ r->pm_blocks = 0; ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_aux[i]); ++} ++ ++#define G5P_RV3_PM_S 0x80U ++#define G5P_RV3_PM_S_LEN 0xc0U ++#define G5P_RV3_PM_SAVE (G5P_RV3_PM_S_LEN + G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ ++/* perf60 fault address: PM state snapshots (S, PM control, the aux windows S points at, each aux ++ * page's head) at three phases of every job, in a ring and per slot pair; printed at a fault read. ++ * Only the executor writes them. RAM only. ++ */ ++enum { G5P_PMSNAP_START, G5P_PMSNAP_TA_DONE, G5P_PMSNAP_END }; ++struct g5p_pmsnap { ++ u64 ns, auxva[3]; ++ u32 seq, phase, slot, valid; ++ u8 s[G5P_RV3_PM_S_LEN], ctl[0x80], win[3][64], ++ head[G5P_RU_AUX_PAGES][64]; ++}; ++#define G5P_PMSNAP_RING 9U ++static struct g5p_pmsnap g5p_pmsnap_ring[G5P_PMSNAP_RING]; ++static struct g5p_pmsnap g5p_pmsnap_slot[16][3]; ++static u32 g5p_pmsnap_head; ++ ++static void g5p_rv3_pmsnap(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 phase) ++{ ++ { ++ return; ++ } ++} ++ ++static void g5p_rv3_pmsnap_print(struct g5p_diag *g, u32 seq, const char *set, ++ const struct g5p_pmsnap *x) ++{ ++ u64 now = ktime_get_ns(); ++ ++ if (!x->valid) ++ return; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pmsnap seq=%u set=%s job=%u slot=%u phase=%s ago_us=%llu " ++ "aux_va=%#llx/%#llx/%#llx end=v3\n", ++ seq, set, x->seq, x->slot, ++ x->phase == G5P_PMSNAP_START ? "start" : ++ x->phase == G5P_PMSNAP_TA_DONE ? "ta-done" : ++ "end", ++ (now - x->ns) / NSEC_PER_USEC, x->auxva[0], x->auxva[1], ++ x->auxva[2]); ++ for (u32 o = 0; o < sizeof(x->s); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap S+%02x %*ph\n", o, 64, ++ x->s + o); ++ for (u32 o = 0; o < sizeof(x->ctl); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap C+%02x %*ph\n", o, 64, ++ x->ctl + o); ++ for (u32 k = 0; k < 3; k++) ++ dev_info(g->dev, "RENDER-V3 pmsnap W%u %*ph\n", k, 64, ++ x->win[k]); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ dev_info(g->dev, "RENDER-V3 pmsnap A%u %*ph\n", i, 64, ++ x->head[i]); ++} ++ ++static void g5p_rv3_pmsnap_dump(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ for (u32 p = 0; p < 3; p++) ++ g5p_rv3_pmsnap_print(g, seq, "slot-last", ++ &g5p_pmsnap_slot[r->slot[0] & 15][p]); ++ for (u32 i = G5P_PMSNAP_RING; i; i--) ++ g5p_rv3_pmsnap_print(g, seq, "ring", ++ &g5p_pmsnap_ring[(g5p_pmsnap_head - i) % ++ G5P_PMSNAP_RING]); ++} ++ ++/* perf60 fault address: the VM edit log entries that cover the fault VA (any VM, newest first), ++ * then the faulting job's VM's latest edits. Copied under the lock, printed outside it. ++ */ ++#define G5P_VMLOG_HITS 24U ++#define G5P_VMLOG_TAIL 16U ++static void g5p_vmlog_dump(struct g5p_diag *g, u32 seq, const void *vm, u64 va, ++ bool have_va) ++{ ++ struct g5p_vmlog_ent *out = kmalloc_array( ++ G5P_VMLOG_HITS + G5P_VMLOG_TAIL, sizeof(*out), GFP_KERNEL); ++ u32 hits = 0, tail = 0, n, head; ++ unsigned long flags; ++ u64 now = ktime_get_ns(); ++ ++ if (!out) ++ return; ++ spin_lock_irqsave(&g5p_vmlog.lock, flags); ++ head = g5p_vmlog.head; ++ n = min(head, G5P_VMLOG_N); ++ for (u32 i = 1; i <= n; i++) { ++ const struct g5p_vmlog_ent *x = ++ &g5p_vmlog.e[(head - i) % G5P_VMLOG_N]; ++ ++ if (have_va && hits < G5P_VMLOG_HITS && ++ g5p_ru_mmu_fault_in(va, x->va, x->size)) ++ out[hits++] = *x; ++ else if (x->vm == vm && tail < G5P_VMLOG_TAIL) ++ out[G5P_VMLOG_HITS + tail++] = *x; ++ } ++ spin_unlock_irqrestore(&g5p_vmlog.lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u va=%#llx edits=%u covering=%u vm_tail=%u end=v3\n", ++ seq, have_va ? va : 0, head, hits, tail); ++ for (u32 i = 0; i < hits + tail; i++) { ++ const struct g5p_vmlog_ent *x = ++ &out[i < hits ? i : G5P_VMLOG_HITS + i - hits]; ++ ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u %s ago_ms=%llu op=%s pid=%d comm=%s vm=%s/%u va=%#llx " ++ "size=%#llx handle=%#x off=%#llx end=v3\n", ++ seq, i < hits ? "cover" : "tail", ++ (now - x->ns) / NSEC_PER_MSEC, ++ g5p_vmlog_ops[min_t(u32, x->op, ++ ARRAY_SIZE(g5p_vmlog_ops) - 1)], ++ x->tgid, x->comm, x->vm == vm ? "this" : "other", ++ x->vm_id, x->va, x->size, x->handle, x->offset); ++ } ++ kfree(out); ++} ++ ++/* perf60 fault address (render_user_fault_read): the MMU fault banks, read once at a failed ++ * phase, the way iOS's fault handler (6ca5c14) reads them; the firmware keeps no copy. Bell mode ++ * only. Under the event lock with the wake path's power proof (the fault already latched ++ * b->failed, so that flag is not a guard here), then the GPU register licence (mmio.inc, ++ * mmio-access.md §4): no wake (Apple's reads never write R1+0x14, G6), a refusal retried outside ++ * the locks every 20-40 us for at most G5P_RV3_DC_SETTLE_US, and the four bank reads only under a ++ * held ticket. A parked GPC records skip-parked, as Apple's report has no bif fault entry then. ++ * At most G5P_RV3_FAULT_READS per boot. It runs before the loss decision and the terminal ++ * latch, while the faulted job is still the GPU's current work. ++ */ ++#define G5P_RV3_FAULT_READS G5P_RU_RECOVER_BOOT_MAX ++static atomic_t g5p_rv3_fault_reads = ATOMIC_INIT(0); ++ ++/* faultrec: a fault VA in the PM-set window (render-user-pmset.h) names a set and its part; say ++ * which context owns that set now and how it stands against the two job records and the deferred ++ * closes (render_user_close_defer). RAM only; rdev lock for the context list. ++ */ ++static void g5p_rv3_fault_pmset(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq, u32 bank, ++ const struct g5p_ru_mmu_fault *f) ++{ ++ static const char *const parts[G5P_PMSET_PAGES] = { "control", "list", ++ "free" }; ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_render *o, *owner = NULL; ++ u32 page, set, own[2] = { U32_MAX, U32_MAX }, df[2][2]; ++ bool last = false, prev = false, closing = false, bound = false; ++ ++ if (f->va < G5P_PMSET_VA || f->va >= G5P_PMSET_END) ++ return; ++ page = (f->va - G5P_PMSET_VA) / G5P_GPU_PAGE; ++ set = page / G5P_PMSET_PAGES; ++ mutex_lock(&rd->lock); ++ list_for_each_entry(o, &rd->contexts, rdev_link) ++ if (o->pm_set == (int)set) { ++ owner = o; ++ break; ++ } ++ if (owner) { ++ own[0] = owner->slot[0]; ++ own[1] = owner->slot[1]; ++ last = rd->pm_last == owner; ++ prev = rd->pm_prev == owner; ++ closing = owner->closing; ++ bound = owner->bound; ++ } ++ for (u32 k = 0; k < 2; k++) { ++ o = rd->close_deferred[k]; ++ df[k][0] = o ? o->slot[0] : U32_MAX; ++ df[k][1] = o ? o->slot[1] : U32_MAX; ++ } ++ mutex_unlock(&rd->lock); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-pmset seq=%u bank=%u mmu_context=%u set=%u part=%s owner=%d/%d " ++ "bound=%u closing=%u last=%u prev=%u job=%u/%u deferred=%d/%d,%d/%d job_slots=%u " ++ "end=v3\n", ++ seq, bank, f->context, set, parts[page % G5P_PMSET_PAGES], ++ (int)own[0], (int)own[1], bound, closing, last, prev, ++ r->slot[0], r->slot[1], (int)df[0][0], (int)df[0][1], ++ (int)df[1][0], (int)df[1][1], rd->job_slots); ++} ++ ++static void g5p_rv3_fault_read(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 e, int res, bool silent) ++{ ++ u32 full = (0), w[4] = { 0 }; ++ u32 mode = full | ((1) ? 1 : 0); ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_mmu_fault f[2]; ++ struct g5p_mmio_ticket t = { .r1_0 = U32_MAX, ++ .gpc = U32_MAX, ++ .r1_0_end = U32_MAX, ++ .gpc_end = U32_MAX }; ++ const char *did = "skip-guard"; ++ u32 power = 0, tries = 0; ++ unsigned long flags; ++ bool fault = res == G5P_RU_WAIT_FAULT && !silent, ticket = false; ++ u64 t0; ++ int ret; ++ ++ if ((2) != G5P_WK_ALWAYS || !(mode & (fault ? 1 : 2))) ++ return; ++ if (atomic_inc_return(&g5p_rv3_fault_reads) > G5P_RV3_FAULT_READS) { ++ did = "skip-cap"; ++ goto print; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ for (;;) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->stopped) || !g5p_wk_power_ready(power)) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ break; ++ } ++ tries++; ++ ticket = true; ++ ret = g5p_mmio_hold(g, &t, 0); ++ if (!ret) { ++ w[0] = g5p_mmio_r0_read(g, &t, 0xe150); ++ w[1] = g5p_mmio_r0_read(g, &t, 0xe154); ++ w[2] = g5p_mmio_r0_read(g, &t, 0xe158); ++ w[3] = g5p_mmio_r0_read(g, &t, 0xe15c); ++ g5p_mmio_release(g, &t); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ did = g5p_mmio_did(&t); ++ if (ret != -EAGAIN || ++ ktime_get_mono_fast_ns() - t0 >= ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) ++ break; ++ usleep_range(20, 40); ++ } ++ if (ticket) ++ g5p_mmio_sample_line(g, G5P_MMS_FAULT, &t, w, t.ops ? 4 : 0, ++ true); ++print: ++ f[0] = g5p_ru_mmu_fault_decode(w[0], w[1]); ++ f[1] = g5p_ru_mmu_fault_decode(w[2], w[3]); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-read seq=%u engine=%u slots=%u/%u why=%s did=%s power=%02x " ++ "fence=none r1_0=%08x gpc=%08x tries=%u e150=%08x:%08x e158=%08x:%08x end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ fault ? "fault" : ++ silent ? "silent" : ++ "timeout", ++ did, power, t.r1_0, t.gpc, tries, w[1], w[0], w[3], w[2]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-decode seq=%u bank=%u va=%#llx %s type=%s level=%u " ++ "requester=%u/%s sideband=%#x context=%u job_slots=%u/%u end=v3\n", ++ seq, k, f[k].va, f[k].read ? "read" : "write", ++ g5p_ru_mmu_fault_types[f[k].type], f[k].level, ++ f[k].requester, ++ g5p_ru_mmu_requester(f[k].requester), ++ f[k].sideband, f[k].context, r->slot[0], ++ r->slot[1]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ g5p_rv3_fault_pmset(d, r, seq, k, &f[k]); ++ g5p_rv3_switch_line(g, r, seq); ++ if (!full) ++ return; ++ g5p_vmlog_dump(g, seq, r->ctx->vm, f[0].valid ? f[0].va : f[1].va, ++ f[0].valid || f[1].valid); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ g5p_rv3_pmsnap_dump(g, r, seq); ++} ++ ++/* GPU overclock thermal stop (core boot.inc, boot_gpu_oc_mhz): every 500 ms while the epoch runs ++ * state 5, the three SoC zones; at or above boot_gpu_oc_temp_mc (or unreadable) the overclock is off ++ * for the rest of the boot and the render scheduler recovers the GPU (g5p_rv3_recover(): gate cycle, ++ * firmware boot) between jobs, so the new epoch powers on at state 4 with record 5 cleared: iOS's ++ * park -> reboot -> new pstate, never a pstate change under a live firmware. Every 10 s one status ++ * line: the temperatures, the real Fender state (an R1-only licence ticket) and the jobs run. ++ */ ++#define G5P_OC_POLL_MS 500 ++static void g5p_rsched_kick(struct g5p_drm *d); ++ ++static void g5p_oc_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = ++ container_of(to_delayed_work(work), struct g5p_rdev, oc_work); ++ struct g5p_drm *d = rd->drm; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ u32 lim = (90000), zones; ++ const char *zone; ++ int t; ++ ++ if (!b || !0 || READ_ONCE(g5p_oc_tripped)) { ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ zones = g5p_oc_temp(&t, &zone); ++ if (zones != ARRAY_SIZE(g5p_oc_zone_names) || t >= (int)lim) { ++ WRITE_ONCE(g5p_oc_tripped, true); ++ dev_emerg( ++ g->dev, ++ "GPUPERF oc thermal-stop temp=%d zone=%s zones=%u limit=%u mhz=%u jobs=%u: the " ++ "GPU recovers at state 4 (723 MHz) end=gpuperf\n", ++ t, zone, zones, lim, b->gpu_oc_mhz, rd->oc_jobs); ++ WRITE_ONCE(rd->recovering, true); ++ g5p_rsched_kick(d); ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ if (!(++rd->oc_ticks % (10000 / G5P_OC_POLL_MS))) { ++ struct g5p_mmio_ticket tk; ++ ++ if (!g5p_mmio_hold(g, &tk, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &tk); ++ dev_info( ++ g->dev, ++ "GPUPERF oc status mhz=%u temp=%d zone=%s limit=%u r1=%s r1_1000=%08x jobs=%u " ++ "end=gpuperf\n", ++ b->gpu_oc_mhz, t, zone, lim, g5p_mmio_did(&tk), tk.gpc, ++ rd->oc_jobs); ++ } ++ queue_delayed_work(system_wq, &rd->oc_work, ++ msecs_to_jiffies(G5P_OC_POLL_MS)); ++} ++ ++ ++ ++static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ bool first = !rd->up, bound_before = r->bound; ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (job->compute) ++ return g5p_cu_execute(d, r, job); ++ if (r->stage_done) ++ return -ECANCELED; ++ g5p_l123_job_reset(g); ++ /* The submit-time admission again, now under the core lock and with the bind state: still nothing touched. */ ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ seq = rd->sequence + 1; ++ ; ++ 0; ++ if (bound_before && d->bell_prev_ctx && d->bell_prev_ctx != r->ctx) { ++ u32 ms = (0); ++ ++ ; ++ ; ++ if (ms && b->gpu_perf) ++ 0; ++ } ++ 0; ++ if (g5p_ru_v3_verbose(r->jobs + 1, 0)) ++ g5p_job_printk( ++ g->dev, ++ "RENDER-V3 start seq=%u bound_ms=%u cap_ms=%u slots=%u/%u attach=%s " ++ "ta_doorbell=%s 3d_doorbell=none wait=ram-only after_timeout=quiet end=v3\n", ++ seq, r->wait_ms, G5P_RU_V3_CAP_MS, r->slot[1], ++ r->slot[0], ++ first ? "firmware-dc3" : ++ r->bound ? "kept" : ++ rd->spare_count ? "spare" : ++ "ram-dc3", ++ first ? "grace-owned" : "none"); ++ if (first) { ++ v->phase = G5P_RV3_ATTACH; ++ ret = g5p_rv3_spares(d, v); ++ if (!ret) ++ ret = g5p_rv3_attach(d, r, v); ++ /* perf60 stage 4: a context with a set that brings a new epoch up */ ++ if (!ret) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ if (ret) ++ goto out; ++ ; ++ } else { ++ if (!r->bound) { ++ /* perf60 stage 4: its own PM set and aux pages before the bind ++ * publishes its tables ++ */ ++ if (rd->pm_ctx) ++ g5p_rv3_pmctx_assign(d, r, seq); ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ ; ++ } ++ /* The firmware owns the rings and the channels from here on: any failure quarantines. */ ++ v->touched = true; ++ if (rd->pm_ctx == 3 && r->pm_first && r->pm_set < 0) ++ g5p_rv3_pmctx_take(d, r, seq); ++ if (r->pm_set >= 0) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ else ++ ret = rd->pm_once ? 0 : g5p_rv3_pm_rebuild(d); ++ if (ret) ++ goto out; ++ ; ++ ; ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_START); ++ if (g5p_ru_mmu_retag_due((0), bound_before, ++ d->bell_prev_ctx != r->ctx)) ++ g5p_rv3_mmu_retag(g, r, seq, ++ d->bell_prev_ctx != r->ctx); ++ } ++ rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu(b, r) + 0x80); ++ if (g5p_ru_fault_due(r->fault_job, r->jobs, first)) ++ g5p_rv3_test_fault(g, r, job, seq); ++ for (u32 e = 0; e < 2; e++) { ++ g5p_ru_job_at( ++ g5p_ru_page(b, e), g5p_ru_meta(e), ++ g5p_meta_address(g5p_object_va(&b->layout, 0x51 + e)), ++ g5p_rv3_pm_meta(r), r->slot[e], e, seq, ++ e ? job->td_w : job->ta_w, ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r)); ++ r->submitted[e] = false; ++ r->consumed[e] = false; ++ r->retired[e] = false; ++ } ++ g5p_l123_job_prepare(d, job); ++ g5p_render_clean_job(g); ++ rd->sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_rv3_wake_mark(d, r, v, 0); ++ ret = g5p_rv3_publish_ta(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ if (r->stall_job && r->jobs == r->stall_job && !first) { ++ /* render_user_test_stall_job: the host gives up, the firmware runs on */ ++ g5p_ru_wait_start(&v->ta, v->ta_pub_ns, 1, v->job_deadline_ns); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-stall seq=%u job=%u slots=%u/%u: TA bound 1 ms end=v3\n", ++ seq, r->jobs, r->slot[0], r->slot[1]); ++ } else { ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, ++ v->job_deadline_ns); ++ } ++ g5p_ru_touch_start(&v->touch[0], v->ta_pub_ns, r->touch_ms, seq); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, seq, &v->ta, &v->touch[0], NULL, &v->tr, ++ &v->last[0], &v->ta_consumed_ns, &err); ++ r->consumed[0] = v->ta_consumed_ns != 0; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ /* Timeout or fault: (latency101: the RAM-only observation window first,) terminal ++ * latch, one RAM sample for the record (a completion that arrives now is evidence, ++ * never the result), no export, no register access. ++ */ ++ g5p_rv3_fault_read(d, r, seq, 0, res, v->ta.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 0)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 0, ++ v->ta.silent ? "ta-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "ta-timeout" : ++ "ta-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "ta-timeout" : "ta-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_TIMEOUT, ret); ++ goto out; ++ } ++ g5p_rv3_ta_count(g, d, v); ++ r->retired[0] = true; ++ g5p_rv3_terminal(g, G5P_RU_TERM_COMPLETE, "ta-complete"); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_TA_DONE); ++ g5p_rv3_pm_lanes(d, r, seq); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, ++ g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); ++ ; ++ v->phase = G5P_RV3_TD_PUBLISH; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample( ++ g, r, 1, seq, ++ &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ ++ goto out; ++ } ++ v->phase = G5P_RV3_TD_WAIT; ++ g5p_rv3_wait_start(g, r, &v->td, v->td_pub_ns, v->job_deadline_ns); ++ g5p_ru_touch_start(&v->touch[1], v->td_pub_ns, r->touch_ms, seq); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, ++ &v->last[1], &v->td_consumed_ns, &err); ++ r->consumed[1] = v->td_consumed_ns != 0; ++ /* mmio_selftest bit 2 (rung 5, DFU-risk on a park tuple): one licensed ticket at the 3D ++ * completion decision; always: the idle clock of bit 3. ++ */ ++ ; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ g5p_rv3_fault_read(d, r, seq, 1, res, v->td.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->td); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 1)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 1, ++ v->td.silent ? "3d-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "3d-timeout" : ++ "3d-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "3d-timeout" : "3d-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ /* Rule (c) decided: the matching event was not seen. Always one line (RAM only). */ ++ if (v->rt.decided == G5P_RU_RT_HOLD) ++ dev_warn( ++ g->dev, ++ "RENDER-V3 3d-hold seq=%u idle=%08x,%08x,%08x,%08x,%08x,%08x coherent_ms=%lld " ++ "decided_ms=%lld end=v3\n", ++ seq, v->last[1].idle[0], v->last[1].idle[1], ++ v->last[1].idle[2], v->last[1].idle[3], ++ v->last[1].idle[4], v->last[1].idle[5], ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns)); ++ r->retired[1] = true; ++ v->phase = G5P_RV3_DONE; ++ rd->recover_streak = ++ 0; /* a job finished: a later fault starts a new streak */ ++ ret = 0; ++out: ++ if (v->phase >= G5P_RV3_TA_PUBLISH) ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ g5p_rv3_wake_end(d, r, v); ++ /* Any other failure after the hardware was touched (attach, publish, drain, protocol): the same ++ * criterion g5p_render_run() quarantines the context by. A stage stop (-ECANCELED) is not one. ++ */ ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), ++ g5p_rv3_phase_names[min_t( ++ u32, v->phase, G5P_RV3_PHASES - 1)]); ++ g5p_rv3_receipt(g, r, job, ret); ++ g5p_l123_job_end(d, job, seq, ret); ++ if (!ret) { ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ ; ++ ; ++ } ++ /* render-user-pmset.h: the PM's write-back targets */ ++ if (g5p_pmset_switched(rd->pm_last, r)) { ++ rd->pm_prev = rd->pm_last; ++ rd->pm_last = r; ++ } ++ /* perf60 bell gates: the next job's gap and context switch are measured from here. */ ++ d->bell_prev_end_ns = ktime_get_ns(); ++ d->bell_prev_ctx = r->ctx; ++ return ret; ++} ++ ++/* ++ * The channel is blocked behind a lost job (g5p_rv3_lose()): wait for its stuck phase, RAM only, ++ * with the firmware's event stream watched from the first sample (silent for 1 s = dead). A late ++ * TA gets the job's 3D published (its W is staged since the job's start; no doorbell for a job ++ * other than the epoch's first), so the 3D channel's sequence and the PM state stay as the ++ * firmware expects; a late 3D clears the channel. Then the held jobs run. A silent stream, a ++ * fault, a refused 3D or the drain bound end the epoch as a timeout or fault always did, and the ++ * held jobs then fail at admission. Called by the scheduler's worker before any pick; returns ++ * false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_rv3_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 st = s->engine ? G5P_RU_DRAIN_TD : G5P_RU_DRAIN_TA, ++ res = G5P_RU_WAIT_PENDING; ++ struct g5p_ru_wait w = {}; ++ bool armed = false; ++ int act, err = -EIO; ++ ++ /* TYPED_COMPUTE: a lost CDM job has its own one-phase drain. */ ++ if (s->engine == G5P_CU_ENGINE) ++ return g5p_cu_drain(d); ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ for (;;) { ++ u32 e = st == G5P_RU_DRAIN_TA ? 0 : 1; ++ u64 now = ktime_get_ns(); ++ ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->stopped)) { ++ res = G5P_RU_WAIT_FAULT; ++ } else if (!e) { ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], ++ NULL, &v->tr, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ } else { ++ if (!armed) ++ g5p_rv3_event_arm(d, 1, ++ g5p_ru_endpoint(s->seq)); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], ++ &v->rt, NULL, &v->last[1], ++ &v->td_consumed_ns, &err); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ } ++ act = g5p_ru_drain_step(&st, res); ++ if (act != G5P_RU_DRAIN_PUBLISH_TD) ++ break; ++ r->retired[0] = true; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); ++ armed = true; ++ if (g5p_rv3_publish_3d(g, r, v, s->seq)) { ++ g5p_rv3_event_disarm(d, 1); ++ res = G5P_RU_WAIT_FAULT; ++ act = g5p_ru_drain_step(&st, res); ++ break; ++ } ++ } ++ if (act == G5P_RU_DRAIN_UNBLOCK) { ++ r->retired[1] = true; ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drained seq=%u engine=%u slots=%u/%u lost_ms=%lld: channel clear, " ++ "held jobs resume end=v3\n", ++ s->seq, s->engine, r->slot[0], r->slot[1], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drain dead seq=%u engine=%u phase=%s why=%s err=%d lost_ms=%lld: the " ++ "device recovers if it may, else the epoch ends end=v3\n", ++ s->seq, s->engine, ++ st == G5P_RU_DRAIN_DEAD && !v->td_pub_ns ? "ta" : "3d", ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal( ++ g, ++ fault ? G5P_RU_TERM_FAULT : G5P_RU_TERM_TIMEOUT, ++ fault ? "drain-fault" : "drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++/* The firmware window's leaves again from the new epoch's tables, in the same pages, so every ++ * render VM's root entry stays valid (g5p_render_fw_entry()). Core lock held. ++ */ ++static void g5p_render_fw_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_boot *b = d->core->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ ++ if (!w->built) ++ return; ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ w->pages = 0; ++ w->missing = 0; ++ for (u32 i = 0; i < G5P_RU_FW_PAGES; i++) { ++ u64 va = g5p_ru_fw_window_va[i]; ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (k == w->leaves) ++ continue; ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, ++ leaf & 1 ? leaf : 0); ++ w->pages += leaf & 1; ++ w->missing += !(leaf & 1); ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(d->core, &w->leaf[i]); ++} ++ ++/* ++ * A lost job stopped the firmware (g5p_rv3_lose_recover(), or a drain that died): cycle the GPU ++ * and boot the firmware again (core recover.inc), then start the render device over as at boot. ++ * Every context survives except the lost one: its bindings belonged to the old epoch, which the ++ * cycle ended, so each is unbound in RAM, its binding's reference dropped, and its next job binds ++ * again (the first job after the recovery runs the epoch's attach). The aux pages and the PM ++ * pool are zeroed as at their allocation. The held jobs run after this. Returns false, having ++ * done nothing, when the core lock is busy. A failed recovery leaves the device failed. ++ */ ++static bool g5p_rv3_recover(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ unsigned long deadline = jiffies + 2 * HZ, flags; ++ struct g5p_render **drop = NULL; ++ struct g5p_render *r; ++ u32 n = 0, count = 0, dropped = 0, stage; ++ int ret; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ count++; ++ } ++ mutex_unlock(&rd->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ rd->recoveries++; ++ rd->recover_streak++; ++ ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; ++ if (ret) { ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", ++ ret, rd->recoveries); ++ return true; ++ } ++ g5p_ru_term_reset(&g5p_rv3_term); ++ /* the new firmware's PM has written nothing back yet: no context is its last or previous */ ++ rd->pm_last = NULL; ++ rd->pm_prev = NULL; ++ rd->up = false; ++ rd->v3 = false; ++ rd->sequence = 0; ++ rd->pm_serial = 0; ++ rd->dc_pending = false; ++ rd->dc_wait = 0; ++ if (rd->spares) { ++ spin_lock_irqsave(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ if (!rd->spares[i].used) { ++ __clear_bit(rd->spares[i].slot[0], ++ g->user_slots); ++ __clear_bit(rd->spares[i].slot[1], ++ g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rd->spares[i].root); ++ kfree(rd->spares); ++ rd->spares = NULL; ++ rd->spare_count = 0; ++ } ++ g5p_render_fw_rebuild(d); ++ mutex_lock(&rd->lock); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ if (rd->aux[i].cpu) { ++ memset(rd->aux[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->aux[i]); ++ } ++ for (u32 i = 0; i < rd->blocks * 8; i++) { ++ memset(rd->physical[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->physical[i]); ++ } ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ r->jobs = 0; ++ r->dc_target = 0; ++ WRITE_ONCE(r->live_maps, false); ++ memset(r->submitted, 0, sizeof(r->submitted)); ++ memset(r->consumed, 0, sizeof(r->consumed)); ++ memset(r->retired, 0, sizeof(r->retired)); ++ /* nothing of the old epoch holds this context's memory any more */ ++ r->lost_drained = true; ++ WRITE_ONCE(r->ctx->live, false); ++ if (r->retained && n < count) { ++ r->retained = false; ++ drop[n++] = r; ++ } ++ } ++ mutex_unlock(&rd->lock); ++ spin_lock_irqsave(&d->status_lock, flags); ++ d->last_error = 0; ++ d->job_error = 0; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ for (u32 i = 0; i < n; i++) { ++ struct g5p_context *c = drop[i]->ctx; ++ ++ dropped++; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recovered n=%u contexts=%u unbound=%u streak=%u: held jobs resume, " ++ "the next job attaches end=v3\n", ++ rd->recoveries, count, dropped, rd->recover_streak); ++ return true; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +new file mode 100644 +index 000000000..94cf4c673 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -0,0 +1,2553 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Userspace TA/3D render: render contexts, render targets and SUBMIT_RENDER. ++ * Builds on the r22 constructors through render-user-layout.h (byte-equal for ++ * the round16 inputs). Default off. The first accepted job claims the device ++ * for render; TYPED_V3 contexts then share it, each with its own VM and slot ++ * pair (docs/gpu/multi-client-design.md), the other profiles stay with that ++ * first context. Jobs of all contexts are serialized; no reset/recovery: any ++ * hardware failure retains everything. ++ */ ++#include "../../apple-g5p/render-user-layout.h" ++#include "../../apple-g5p/render-user-wait.h" ++#include "../../apple-g5p/render-user-touch.h" ++#include "../../apple-g5p/render-user-retire.h" ++#include "../../apple-g5p/render-user-wake.h" ++#include "../../apple-g5p/render-user-serial.h" ++#include "../../apple-g5p/render-user-terminal.h" ++#include "../../apple-g5p/render-user-bind.h" ++#include "../../apple-g5p/render-user-close.h" ++#include "../../apple-g5p/render-user-sched.h" ++#include "../../apple-g5p/render-user-window.h" ++#include "../../apple-g5p/render-user-msaa.h" ++#include "../../apple-g5p/render-user-loss.h" ++#include "../../apple-g5p/render-user-fault.h" ++#include "../../apple-g5p/render-user-mmu.h" ++#include "../../apple-g5p/render-user-faultreg.h" ++#include "render-user-validate.h" ++#include "drm-render-v3.h" ++#include "../../apple-g5p/compute-user-layout.h" ++#include "compute-user-validate.h" ++#include "drm-compute-v1.h" ++ ++/* The terminal state of the epoch (one GPU): from the first terminal note on, no register access. */ ++static struct g5p_ru_terminal g5p_rv3_term; ++ ++/* Defaults of the latched knobs. GET_RENDER_INFO reports VARIANT_ACTIVE when ++ * any knob differs, so the declarations below and that comparison share them. ++ */ ++#define G5P_RU_DEF_CACHE_VARIANT 3 ++#define G5P_RU_DEF_MMU_VARIANT 1 ++/* 20 s: the natural TA completion took 12.006 s in the r96 boots a 12 s cap turned into ++ * timeouts (render-user-wait.h); it is also the minimum of a TYPED_V3 job's per-phase bound. ++ */ ++#define G5P_RU_DEF_WAIT_MS G5P_RU_V3_WAIT_MS ++#define G5P_RU_DEF_PROFILE1_VDM_CONTROL 0x280 ++ ++static unsigned int render_user_wait_ms = 20000; ++module_param(render_user_wait_ms, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_wait_ms, ++ "Latched per-phase TA/3D bound, 3000..30000 ms (default 20000; TYPED_V3 needs " ++ "20000..30000, absolute job cap 30000)"); ++/* latency101 (default off): host-side TYPED_V3 wait policy; nothing the firmware sees depends on it. */ ++static unsigned int render_user_hard_ms = 60000; ++static unsigned int render_user_soft_ms = 45000; ++static unsigned int render_user_obs_ms = 2000; ++module_param(render_user_hard_ms, uint, 0600); ++module_param(render_user_soft_ms, uint, 0600); ++module_param(render_user_obs_ms, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_hard_ms, ++ "Latched TYPED_V3 hard per-phase bound soft..60000 ms, the job bound is twice " ++ "it; 0 = off, render_user_wait_ms and the 30000 job cap apply (default 0)"); ++MODULE_PARM_DESC( ++ render_user_soft_ms, ++ "Latched TYPED_V3 soft per-phase bound 20000..hard ms, 0 = 45000; needs " ++ "render_user_hard_ms"); ++MODULE_PARM_DESC( ++ render_user_obs_ms, ++ "Latched RAM-only observation window after a TYPED_V3 timeout, 0..5000 ms " ++ "(default 2000; needs render_user_hard_ms)"); ++/* RC4a (default off): RAM-only touch A/B of the TYPED_V3 wait (render-user-touch.h). */ ++ ++/* repeat109: the TYPED_V3 3D completion without the idle words (render-user-retire.h). */ ++ ++/* wake110 (default off): Apple's submit kick at the TYPED_V3 publish (render-user-wake.h). */ ++/* release8 default */ ++ ++/* TEMPORARY (multi-client stage 1): the fallback for a firmware that never drains a DC3 ++ * published after a job retired. Remove once the device has settled that question. ++ */ ++ ++/* Stage 2 fault isolation (render-user-loss.h): how long the scheduler holds every other job ++ * while it waits, RAM only, for a lost job's stuck phase. Read at the loss. ++ */ ++ ++/* Stage 2 in-boot recovery (core recover.inc): a job that stopped the firmware loses only its ++ * context, and the GPU is power-cycled and its firmware booted again while the other contexts' ++ * jobs are held (each leaves the old epoch's firmware pages allocated); 0 = such a job ends the ++ * epoch, as before. How many: render_user_recover_streak. Off by default; the release tuple turns ++ * it on (faultiso F1-F4, 2026-11-02: two recoveries in one boot on hardware). ++ */ ++ ++/* render-user-recover.h: Apple restarts the GPU on every recovery packet. Nonzero: recover every ++ * time, but end the epoch after this many recoveries in a row with no job completing between ++ * them. 0 = render_user_recover counts recoveries per boot, as before (a boot's second fault then ++ * ends the epoch, however long the GPU ran in between). ++ */ ++ ++/* Fault-isolation test (host side only: the firmware runs the job as any other). A context ++ * created while this is N gives up on its N-th job's TA after 1 ms instead of its bound, so the ++ * host loses that job the way it loses a stuck one, while the firmware completes it late: the ++ * context is lost and the channel drained behind it. 0 = off (default). ++ */ ++ ++/* Fault-isolation test with a real GPU fault (faultiso). A context created by a CAP_SYS_RAWIO ++ * process while this is N reserves one page of its VM that is never mapped ++ * (G5P_RU_TEST_FAULT_VA), and its N-th job's TA fetches its control stream from that page ++ * instead of the validated stream: the TA reads an unmapped VA of the context's own VM. What ++ * the firmware does then (a fault event, a silent stream, or a stuck engine) is what the test ++ * records; each ends that job and loses its context. 0 = off (default). ++ */ ++ ++/* render_user_test_fault_job's page: the last GiB below the hardware VA limit, far from every ++ * kernel-private range (low sink, aux, PDS window, firmware window). ++ */ ++#define G5P_RU_TEST_FAULT_VA (G5P_RU_HW_VA_LIMIT - 0x40000000ULL) ++ ++/* Stage 3 (render-user-window.h): which firmware memory a render VM maps. Latched per context ++ * at creation. The kill switch 0 restores the stage 1 mapping and then needs CAP_SYS_RAWIO. ++ */ ++ ++/* The latency101 knobs are valid when all are 0 (off), or hard is in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000 and the dialect is TYPED_V3. The RC4a touch ++ * is 0, or 1..20000 below the effective soft bound with latency101 on. The 3D idle settle is ++ * 0..5000. The wake110 mode is 0, or 1..2 with latency101 on and the touch off (one A/B per ++ * context). Anything else refuses the context. ++ */ ++static bool g5p_ru_lat_params_valid(void) ++{ ++ u32 hard = READ_ONCE(render_user_hard_ms), ++ soft = READ_ONCE(render_user_soft_ms); ++ ++ return g5p_ru_lat_valid(soft, hard, READ_ONCE(render_user_obs_ms)) && ++ g5p_ru_touch_valid((0), g5p_ru_lat_soft(soft, hard), hard) && ++ g5p_ru_rt_settle_valid((G5P_RU_RT_DEF_SETTLE_MS)) && ++ g5p_wk_mode_valid((2)) && (!(2) || (hard && !(0))) && ++ (!hard || (1)); ++} ++/* r25 frame-capable selections, same values as the fixed render_*_variant. */ ++ ++#define G5P_RU_MAX_QUEUED 16 ++#define G5P_RU_MAX_TARGETS 32 ++#define G5P_RU_RESULTS 1024 ++#define G5P_RU_INPUT_MS 10000 ++ ++struct g5p_rt { ++ struct list_head link; ++ struct g5p_context *ctx; ++ struct g5p_ru_rt geometry; ++ struct g5p_ru_rt_sizes sizes; ++ struct drm_mm_node node; ++ struct g5p_dma_page *pages; ++ u32 width, height, page_count; ++ u32 samples; /* 1 or 4 (CREATE_RENDER_TARGET); 0 = 1 */ ++ bool destroyed; ++}; ++ ++/* A pair job 1 bound to a kernel-owned catalogue (render_user_spare_pairs, TEMPORARY). */ ++struct g5p_rspare { ++ u32 slot[2]; /* TA, 3D */ ++ struct g5p_dma_page root; /* the catalogue CBASE points at */ ++ bool ok, used; ++}; ++ ++/* ++ * Device-wide render state (docs/gpu/multi-client-design.md §4.1): the one TA/3D channel pair ++ * and the one PM record with its pool and aux pages, shared by every render context. Jobs of ++ * all contexts run one at a time on render_wq with the core lock held, and only they write ++ * up, v3, sequence, pm_serial and the spares. `lock` guards the pool and the aux pages, which ++ * render-target creation grows while another context's job may run. The scheduler's worker ++ * is the only executor: it picks the jobs of all contexts round-robin. ++ */ ++/* The kernel-built firmware window shared by every render VM (render-user-window.h): built ++ * once from the firmware's tables, never edited, kept until reboot. ++ */ ++struct g5p_rwin { ++ bool built; ++ u32 pages, missing, leaves; ++ struct g5p_dma_page dir, leaf[G5P_RU_FW_PAGES]; ++ u32 leaf_di[G5P_RU_FW_PAGES]; ++}; ++ ++struct g5p_rdev { ++ struct mutex lock; ++ bool up, v3; /* PM and channels built; by the TYPED_V3 attach */ ++ bool dc_pending; /* a bind's DC3 pair was not consumed in time ... */ ++ u32 dc_wait; /* ... and this is the consumer it waits for */ ++ u32 sequence, pm_serial; ++ u32 cu_sequence, cu_target; /* TYPED_COMPUTE: CDM sequence, CL producer target */ ++ u32 blocks; ++ /* perf60, latched at the epoch's first TYPED_V3 attach: 2 = jobs alternate two record slots ++ * (render-user-layout.h), else 1; pm_once = the PM pool is built at that attach only. ++ */ ++ u32 job_slots; ++ bool pm_once; ++ /* perf60 stage 4, latched there too: render_user_pm_ctx (0 off, 1 per-context PM records, ++ * 2 also per-context PM IDs) and the boot-reserved sets in use (core lock). ++ */ ++ u32 pm_ctx, pm_set_next; ++ bool pm_set_used[G5P_PMSET_MAX]; ++ /* the context of the last job the executor ran and the previous distinct one, the PM's ++ * write-back targets (render-user-pmset.h; compared, never dereferenced), and the closed ++ * contexts whose unbind waits until they are neither (render_user_close_defer) ++ */ ++ struct g5p_render *pm_last, *pm_prev, *close_deferred[2]; ++ struct g5p_dma_page *physical; ++ struct g5p_dma_page aux[G5P_RU_AUX_PAGES]; ++ struct g5p_rspare *spares; ++ u32 spare_count; ++ /* render_user_close_park: the park catalogue a closed pair is bound to (allocated at the ++ * first park, kept for the device's life; rebuilt at every park) ++ */ ++ struct g5p_dma_page park; ++ u32 parks; ++ bool executing; /* a TYPED_V3 job is in g5p_rv3_execute() (reboot quiesce) */ ++ /* The scheduler (drm-render-sched.inc): its run list under sched_lock, and the one ++ * worker on render_wq that runs every job and close item. ++ */ ++ struct g5p_drm *drm; ++ spinlock_t sched_lock; ++ struct g5p_rs sched; ++ struct delayed_work sched_work; ++ /* boot_gpu_oc_mhz: the thermal stop's poll (system_wq, so a busy render_wq never starves it) */ ++ struct delayed_work oc_work; ++ unsigned long oc_armed; ++ u32 oc_ticks, oc_jobs; ++ struct g5p_rwin ++ window; /* written once, by the executor, core lock held */ ++ struct g5p_rstuck ++ *stuck; /* a lost job blocks the channel (drain), worker only */ ++ /* A lost job stopped the firmware: the worker recovers before any pick. Set by the ++ * executor, cleared by the worker; submissions and the claim pass meanwhile. ++ */ ++ bool recovering; ++ u32 recoveries; ++ u32 recover_streak; /* recoveries since a job completed (core lock) */ ++ struct list_head ++ contexts; /* every render context (lock), for the recovery */ ++}; ++ ++/* render-user-fault.h's ruling window on the device state: a failure noted while a TYPED_V3 job or ++ * a drain runs, or a recovery is pending, waits for the executor. RAM only. ++ */ ++static bool g5p_rdev_ruling(struct g5p_rdev *rd) ++{ ++ bool executing = ++ smp_load_acquire(&rd->executing); /* before `recovering` */ ++ ++ return g5p_ru_ruling(executing, READ_ONCE(rd->recovering)); ++} ++ ++/* A render context. Its firmware state is the slot pair (slot[], bound), owned here the way ++ * an Asahi queue owns its firmware context; the channels and the PM record are the device's. ++ */ ++struct g5p_render { ++ struct g5p_context *ctx; ++ u32 slot[2]; /* engine0 TA, engine1 normal3D */ ++ u32 jobs; /* jobs that started on this context */ ++ u32 dc_target; /* DC ring consumer after this context's binding */ ++ struct drm_mm private_mm; ++ struct drm_mm_node aux_node, low_node, firmware_node, private_node; ++ struct drm_mm_node ++ fault_node; /* render_user_test_fault_job's unmapped page */ ++ struct list_head targets; ++ struct g5p_rt *profile1_rt; ++ u32 target_count; ++ u64 fence_context, fence_seqno; ++ atomic_t queued; ++ /* bound: the slot pair is published (for a RAM-only bind: consumed and checked). ++ * retained: the binding holds a context and device reference until its unbind. ++ * closing: the handle is gone; the close item (g5p_render_close_run()) is queued. ++ * live_maps: a job retired on the binding, so VM_BIND MAP may run beside queued jobs. ++ */ ++ bool claimant, bound, retained, aux_mapped, detached, stage_done, ++ unhooked; ++ bool closing, live_maps; ++ bool lost_drained; /* a lost job of it drained: the firmware holds no use of it */ ++ struct g5p_rs_entity ++ entity; /* its place in the scheduler (drm-render-sched.inc) */ ++ struct list_head rdev_link; /* in rdev->contexts */ ++ /* Mappings unmapped beside queued jobs, freed as those retire (g5p_render_reap()). */ ++ spinlock_t zlock; ++ struct list_head zombies; ++ /* Latched at creation; mirrors whatever qualifies the fixed path. */ ++ u32 stage, cache_variant, mmu_variant, timestamp_variant, wait_ms; ++ u32 soft_ms, hard_ms, obs_ms; /* latency101; hard_ms == 0: off */ ++ u32 touch_ms; /* RC4a; 0: off (always 0 when hard_ms is) */ ++ u32 settle_ms; /* repeat109 minimum coherent run before the 3D event completes */ ++ u32 stall_job; /* render_user_test_stall_job; 0: off */ ++ u32 fault_job; /* render_user_test_fault_job; 0: off */ ++ u32 wake, wake_seed; /* wake110 (render-user-wake.h); wake 0: off */ ++ u32 wake_rang; /* wake110: 0xb00 writes of this context */ ++ u32 pm_policy, vdm_control; ++ u32 fw_window; /* render_user_fw_window, G5P_RU_FW_* */ ++ bool after_ta, cache_repeat; ++ struct g5p_r25_variants v; /* pm, memory, rt<=1, slots; others 0 */ ++ u32 target[2], status[2], selector[2]; ++ bool submitted[2], consumed[2], retired[2]; ++ /* TYPED_V3 (drm-render-v3.inc): serial jobs, the context is retained from its binding on. */ ++ bool v3_accepted; ++ /* perf60 render_user_pm_switch 2: this context's own PM state (S + aux) after its last job */ ++ u8 *pm_save; ++ ++ /* perf60 stage 4 (render_user_pm_ctx): its own PM set (-1: the device PM), pool pages, ++ * aux pages mapped at G5P_RU_PRIVATE_VA in its VM, and the epoch they were built for. ++ */ ++ int pm_set; ++ u32 pm_blocks, pm_epoch; ++ struct g5p_dma_page *pm_pool; ++ struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; ++ bool pm_aux_mapped; ++ bool pm_first; /* it brought the epoch up (attach): the device aux is its own */ ++ bool close_defer_run; /* render_user_close_defer: its deferred close is running */ ++}; ++ ++/* The lost job the channel waits behind (render-user-loss.h): its context (a reference held), ++ * its per-job state as the executor left it, and the drain bound read at the loss. ++ */ ++struct g5p_rstuck { ++ struct g5p_render *r; ++ struct g5p_rv3 v; ++ u32 seq, engine, drain_ms; ++ u64 lost_ns; ++}; ++ ++#include "cache-receipt.inc" ++ ++/* A listed BO that needs implicit fences and/or kernel cache maintenance. */ ++struct g5p_robj { ++ struct drm_gem_object *obj; /* reference held by the job */ ++ u32 access; ++ /* ACCESS_CACHE window (render_user_sync_range bit 1: the resource's range; else whole BO) */ ++ u64 lo, hi; ++}; ++ ++struct g5p_rjob { ++ struct g5p_cache_receipt cache; ++ struct g5p_rs_job node; /* in its context's FIFO until picked */ ++ struct g5p_drm *gpu; ++ struct g5p_context *ctx; ++ struct g5p_fence *fence; ++ struct dma_fence *inputs[G5P_MAX_SYNCS - 1]; ++ u32 input_count; ++ struct g5p_robj *objs; ++ u32 obj_count; ++ struct dma_fence **deps; /* implicit dependencies from dma_resv */ ++ u32 dep_count; ++ int setup_error; /* post-acceptance setup failure: fail without hardware */ ++ /* Inputs then deps, in order: dep_next is the first not yet seen signalled, cb_fence the ++ * one whose callback is armed (scheduler lock), dep_error a failed input or the deadline. ++ */ ++ u32 dep_next; ++ struct dma_fence *cb_fence; ++ struct dma_fence_cb cb; ++ bool cb_fired; ++ int dep_error; ++ u64 id; ++ unsigned long deadline; ++ u8 ta_w[G5P_RU_TA_W_BYTES], td_w[G5P_RU_3D_W_BYTES]; ++ bool v3; ++ bool compute; /* TYPED_COMPUTE (drm-compute-v1.inc): a CDM job */ ++ struct g5p_cu_regs cu; ++ struct g5p_rv3 rv3; ++ u64 submit_ns; /* latency123: the submit ioctl */ ++}; ++ ++static u8 *g5p_ru_page(struct g5p_boot *b, u32 engine) ++{ ++ return b->shared[(engine ? G5P_R22_3D : G5P_R22_TA) / G5P_GPU_PAGE].cpu; ++} ++ ++static u32 g5p_ru_meta(u32 engine) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + ++ (engine ? G5P_R22_3D : G5P_R22_TA)); ++} ++ ++static u8 *g5p_ru_pm_cpu(struct g5p_boot *b) ++{ ++ return b->shared[G5P_R22_PM / G5P_GPU_PAGE].cpu; ++} ++ ++static u8 *g5p_ru_global(struct g5p_boot *b, u32 engine) ++{ ++ u32 off = b->layout.offset[0x47 + engine]; ++ ++ return b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_ru_free_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ if (p->cpu) ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++} ++ ++/* Probe: the device render state and its scheduler. Its pages are allocated on demand and ++ * kept until reboot. ++ */ ++static void g5p_rsched_work(struct work_struct *work); ++static void g5p_oc_work(struct work_struct *work); ++ ++static int g5p_render_init(struct g5p_drm *d) ++{ ++ d->rdev = drmm_kzalloc(&d->drm, sizeof(*d->rdev), GFP_KERNEL); ++ if (!d->rdev) ++ return -ENOMEM; ++ d->rdev->drm = d; ++ INIT_LIST_HEAD(&d->rdev->contexts); ++ spin_lock_init(&d->rdev->sched_lock); ++ g5p_rs_init(&d->rdev->sched); ++ INIT_DELAYED_WORK(&d->rdev->sched_work, g5p_rsched_work); ++ INIT_DELAYED_WORK(&d->rdev->oc_work, g5p_oc_work); ++ return drmm_mutex_init(&d->drm, &d->rdev->lock); ++} ++ ++/* rdev->lock held: the aux pages (once) and a PM pool of at least `blocks`. The pool never ++ * shrinks; every page added reaches PoC like the bring-up pool, whether or not the device's ++ * bring-up has cleaned the pool yet. The pool, list and F are rebuilt from it before every ++ * job, so growth takes effect at the next job, and the job in flight keeps its pages. ++ */ ++static int g5p_rdev_grow(struct g5p_drm *d, u32 blocks) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_dma_page *physical; ++ u32 have = rd->blocks * 8, i; ++ int ret = 0; ++ ++ for (i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!rd->aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->aux[i]); ++ if (ret || blocks <= rd->blocks) ++ return ret; ++ if (blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ physical = kvcalloc(blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ if (have) ++ memcpy(physical, rd->physical, have * sizeof(*physical)); ++ for (i = have; !ret && i < blocks * 8; i++) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &physical[i]); ++ if (!ret) ++ g5p_table_clean_page(d->core, &physical[i]); ++ } ++ if (ret) { ++ while (i-- > have) ++ g5p_ru_free_page(d->drm.dev, &physical[i]); ++ kvfree(physical); ++ return ret; ++ } ++ kvfree(rd->physical); ++ rd->physical = physical; ++ rd->blocks = blocks; ++ return 0; ++} ++ ++/* rdev->lock held: the pool's page addresses, for the PM builders. */ ++static u64 *g5p_rdev_addresses(struct g5p_rdev *rd) ++{ ++ u64 *physical = ++ kvmalloc_array(rd->blocks * 8, sizeof(*physical), GFP_KERNEL); ++ ++ for (u32 i = 0; physical && i < rd->blocks * 8; i++) ++ physical[i] = rd->physical[i].dma; ++ return physical; ++} ++ ++/* From the moment the firmware may hold a context's slot pair, the context and everything it ++ * owns stay alive whatever its files do, until its close item has unbound the pair ++ * (g5p_render_close_run()); a pair that cannot be unbound keeps them until reboot. ++ */ ++static void g5p_render_retain(struct g5p_render *r) ++{ ++ if (r->retained) ++ return; ++ kref_get(&r->ctx->ref); ++ drm_dev_get(&r->ctx->gpu->drm); ++ r->retained = true; ++} ++ ++/* The RAM words of the binding of `slot` through the DC records at start and start + 1. */ ++static void g5p_render_bind_mem(struct g5p_boot *b, const u32 *slot, u32 start, ++ struct g5p_ru_bind_mem *m) ++{ ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + ++ ((start + i) & 255) * 20 + j * 4; ++ ++ m->rec[i][j] = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ } ++ for (u32 e = 0; e < 2; e++) { ++ m->pa[e] = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ m->host[e] = g5p_boot_shared(b, 0x84) + slot[e] * 4; ++ m->cached[e] = g5p_boot_shared(b, 0x85) + slot[e] * 4; ++ } ++} ++ ++/* ++ * Called with the file lock. Idle: no accepted job is queued or running. A VM_BIND (`live`) ++ * of a context with live_maps may also run beside its queued jobs, the way an Asahi VM_BIND ++ * does not wait for the queue: a compositor imports and releases client buffers while its ++ * previous frame is in flight. Its tag reaches the firmware at the next kick of the slots. A ++ * MAP only adds entries for VAs no accepted job references; an UNMAP clears its entries now but ++ * keeps the object pinned until the queued jobs have retired (g5p_resident_defer()), so a job ++ * that still used the range faults on the hardware instead of reaching freed memory. Not ++ * before a job has retired on the binding: the epoch's first job checks that the tables did not ++ * change since its attach, and a bind in flight computes its tag from them. Render-target ++ * edits still need the context idle. ++ */ ++static int g5p_render_vm_idle(struct g5p_context *c, bool live) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_boot *b = c->gpu->core->boot; ++ /* During a recovery no epoch may receive a tag. */ ++ bool busy = (atomic_read(&r->queued) && ++ !(live && READ_ONCE(r->live_maps))) || ++ READ_ONCE(c->gpu->rdev->recovering); ++ ++ if (c->bad || r->detached || busy) { ++ dev_warn_ratelimited( ++ c->gpu->drm.dev, ++ "G5P VM_BIND busy=render-idle bad=%u detached=%u queued=%d bound=%u\n", ++ c->bad, r->detached, atomic_read(&r->queued), r->bound); ++ return -EBUSY; ++ } ++ if (r->bound && (!b || READ_ONCE(b->failed) || READ_ONCE(b->stopped))) ++ return g5p_rdev_ruling(c->gpu->rdev) ? -EBUSY : -EIO; ++ return 0; ++} ++ ++/* perf60 MMU invalidation arm, default 0 (render-user-mmu.h): bit 0 every edit publication asks ++ * the firmware for a PC-level invalidate, bit 1 a bound context's job after another context's ++ * job retags its slots (PC), bit 2 NC-reads every table line before the tag store, bit 3 retags ++ * on every job. RAM only, like the tag path itself. ++ */ ++ ++/* An NC read of every 64-byte line of the VM's tables (g5p_rv3_fw_touch()'s visibility read). */ ++static void g5p_render_table_touch(struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt; ++ ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(vm->root.cpu + at)); ++ list_for_each_entry(pt, &vm->tables, link) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(pt->page.cpu + at)); ++} ++ ++/* File lock held, context idle (or a MAP with live_maps). After a PTE edit on a bound render ++ * VM: tables to PoC, then the packed tag to both of its slots. RAM only: no register access ++ * is safe after retirement, and the firmware reads these words only at this context's next ++ * kick, which then invalidates the slots' PC level. ++ */ ++static int g5p_render_invalidate(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_diag *g = c->gpu->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_pt *pt; ++ u32 tag, cached[2], bumps = 0, mode = (0); ++ ++ if (!READ_ONCE(r->bound) || r->detached) ++ return 0; ++ if (!b || c->bad || b->failed || b->stopped) ++ return -EIO; ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ if (mode & G5P_RU_MMU_TOUCH) ++ g5p_render_table_touch(vm); ++ tag = g5p_vm_generation(&vm->mutation); ++ for (u32 e = 0; e < 2; e++) ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ while (tag == cached[0] || tag == cached[1]) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ bumps++; ++ } ++ if (bumps) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER generation collision cached=%08x/%08x bumps=%u tag=%08x\n", ++ cached[0], cached[1], bumps, tag); ++ if (mode & G5P_RU_MMU_EDIT) { ++ /* Advance the VM's own PC counter to the PC field the helper picked. */ ++ u32 pc = g5p_ru_mmu_pc_tag(tag, cached[0], cached[1]); ++ ++ if (pc) { ++ vm->mutation.generation[0] += ++ ((pc >> 22) - (tag >> 22)) & 0x3ff; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ } ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, tag); ++ dma_wmb(); ++ if (mode) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick mmu_inval=%x " ++ "cached=%08x/%08x\n", ++ tag, r->slot[0], r->slot[1], mode, cached[0], ++ cached[1]); ++ else ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick\n", ++ tag, r->slot[0], r->slot[1]); ++ return 0; ++} ++ ++/* DC3 consumption precedes this CBASE check and selected-slot invalidation; ++ * the r22 diagnostic sequence with the user catalogue instead of a private PC. ++ */ ++ ++/* CPU-written firmware objects of one job: TA/3D pages, PM control and the ++ * list/free pages. Coherent allocations plus explicit PoC, as r22 does. ++ */ ++static void g5p_render_clean_job(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ for (u32 off = G5P_R22_TA; off <= G5P_R22_PM; off += G5P_GPU_PAGE) ++ g5p_table_clean_page(g, &b->shared[off / G5P_GPU_PAGE]); ++ for (u32 i = 0; i < 2; i++) ++ g5p_table_clean_page(g, &b->render_meta[i]); ++ dsb(sy); ++ dma_wmb(); ++} ++ ++/* RAM only: the DC ring is idle and neither slot of this context has a PA or a tag. */ ++static int g5p_render_vm_ready(struct g5p_boot *b, struct g5p_render *r) ++{ ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u32 start = READ_ONCE(ctl[4]); ++ ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ return g5p_ru_bind_busy(&s) ? -EBUSY : 0; ++} ++ ++/* The epoch's first job, core lock held, device not up: the PM record from the device pool ++ * and both channels, and every page they reach cleaned to PoC. Doing this again with the ++ * device up would reset the channels and the PM state that every context shares. ++ */ ++static int g5p_render_device_prepare(struct g5p_drm *d, ++ const struct g5p_r25_variants *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm(g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, ++ G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks, G5P_RU_PRIVATE_VA); ++ if (!ret) ++ g5p_ru_pm_variant(g5p_ru_pm_cpu(b), b->render_meta[1].cpu, ++ physical, rd->blocks, v->pm); ++ kvfree(physical); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ g5p_table_clean_page(g, &rd->aux[i]); ++ for (u32 i = 0; !ret && i < rd->blocks * 8; i++) ++ g5p_table_clean_page(g, &rd->physical[i]); ++ mutex_unlock(&rd->lock); ++ if (ret) ++ return ret; ++ rd->pm_serial = 0; ++ for (u32 e = 0; e < 2; e++) ++ g5p_ru_channel_init(g5p_ru_page(b, e), g5p_ru_meta(e)); ++ g5p_cu_device_init(d); ++ g5p_render_clean_job(g); ++ return 0; ++} ++ ++/* ++ * Core lock held, firmware up: the root entry of the firmware GiB for a VM in mode `fw_window`. ++ * The window is built at its first use from the firmware's leaf entries of the pages it keeps ++ * (render-user-window.h), and then shared, unchanged, by every render VM. RAM only. ++ */ ++static int g5p_render_fw_entry(struct g5p_drm *d, struct g5p_render *r, ++ u32 *entry) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ int ret = 0; ++ ++ *entry = 0; ++ if (r->fw_window == G5P_RU_FW_FULL) ++ *entry = g5p_le32(b->catalogue.cpu + G5P_RU_FIRMWARE_CI * 4); ++ if (r->fw_window != G5P_RU_FW_WINDOW) ++ return 0; ++ if (w->built) { ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ return 0; ++ } ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->dir); ++ /* perf60 stage 4: the per-context PM sets reserved at boot, after the device PM's pages */ ++ for (u32 i = 0; ++ !ret && i < G5P_RU_FW_PAGES + b->pm_sets * G5P_PMSET_PAGES; i++) { ++ u32 j = i - G5P_RU_FW_PAGES; ++ u64 va = i < G5P_RU_FW_PAGES ? ++ g5p_ru_fw_window_va[i] : ++ g5p_pmset_va(j / G5P_PMSET_PAGES, ++ j % G5P_PMSET_PAGES); ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (!(leaf & 1)) { ++ w->missing++; ++ continue; ++ } ++ if (k == w->leaves) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->leaf[k]); ++ if (ret) ++ break; ++ w->leaf_di[k] = g5p_ru_fw_di(va); ++ g5p_put64(w->dir.cpu + w->leaf_di[k] * 8, ++ g5p_directory_entry(w->leaf[k].dma)); ++ w->leaves++; ++ } ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, leaf); ++ w->pages++; ++ } ++ if (ret) { ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_ru_free_page(d->drm.dev, &w->leaf[i]); ++ g5p_ru_free_page(d->drm.dev, &w->dir); ++ memset(w, 0, sizeof(*w)); ++ return ret; ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(g, &w->leaf[i]); ++ g5p_table_clean_page(g, &w->dir); ++ w->built = true; ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ dev_info( ++ g->dev, ++ "RENDER-V3 fw-window pages=%u missing=%u leaves=%u dir=%pad end=v3\n", ++ w->pages, w->missing, w->leaves, &w->dir.dma); ++ return 0; ++} ++ ++/* Core lock held, between jobs: the DC3 pair that points this context's slots at its own page ++ * catalogue. Maps the firmware window (g5p_render_fw_entry()) and borrows the sink catalogue ++ * entry, cleans the tables and this ++ * context's render targets, then writes the PA and tag of both slots and the two records. ++ * RAM only, nothing is written when the ring is busy or a slot is in use; the caller stores ++ * the producer. ++ */ ++static int g5p_render_vm_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 start = READ_ONCE(ctl[4]), tag, entry; ++ struct g5p_ru_bind_mem m; ++ struct g5p_pt *pt; ++ struct g5p_rt *rt; ++ int ret; ++ ++ ret = g5p_render_vm_ready(b, r); ++ if (!ret) ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ dsb(sy); ++ tag = g5p_vm_generation(&vm->mutation); ++ /* A zero tag would read as unbound: the unbind check takes a zero cached tag as proof ++ * that the DC3 handler ran. ++ */ ++ while (!tag && vm->mutation.generations) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_publish(&m, r->slot, vm->root.dma, tag); ++ *start_out = start; ++ *tag_out = tag; ++ return 0; ++} ++ ++/* The epoch's first job, core lock held: build the device, then bind this context's pair. ++ * Any failure leaves every allocation and binding retained. ++ */ ++static int g5p_render_attach_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ int ret = g5p_render_vm_ready(d->core->boot, r); ++ ++ if (!ret) ++ ret = g5p_render_device_prepare(d, &r->v); ++ return ret ?: g5p_render_vm_prepare(d, r, start_out, tag_out); ++} ++ ++/* The legacy profiles' attach: the epoch's first job, with a doorbell and a host MMU pass. */ ++ ++/* Pointer contents, channel producer, global producer and kick, each after a ++ * barrier. The channel must have retired the previous job's endpoint. ++ */ ++ ++/* Same predicates as g5p_r22_observe() for sequence `seq`. */ ++ ++#include "drm-render-v3.inc" ++#include "drm-render-sched.inc" ++#include "drm-render-unbind.inc" ++ ++static void g5p_render_job_free(struct g5p_rjob *job) ++{ ++ for (u32 i = 0; i < job->input_count; i++) ++ dma_fence_put(job->inputs[i]); ++ for (u32 i = 0; i < job->dep_count; i++) ++ dma_fence_put(job->deps[i]); ++ kfree(job->deps); ++ for (u32 i = 0; i < job->obj_count; i++) ++ drm_gem_object_put(job->objs[i].obj); ++ kvfree(job->objs); ++ if (job->fence) ++ dma_fence_put(&job->fence->base); ++ if (job->ctx) ++ g5p_context_put(job->ctx); ++ kfree(job); ++} ++ ++/* CPU cache maintenance on the G5P mapping (for an imported BO, the attachment's sg_table): clean ++ * before the job, invalidate after retirement. Whole BO, or (render_user_sync_range bit 1, latched ++ * at submit) the resource's BO range, the same window both ways. ++ */ ++static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, ++ bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct g5p_robj *o = &job->objs[i]; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(o->obj)->sgt; ++ ++ if (!(o->access & DRM_G5P_RENDER_ACCESS_CACHE) || !sgt) ++ continue; ++ /* A buffer the job only reads (a sampled client dma-buf or wl_shm udmabuf) ++ * was cleaned before the job and the GPU wrote none of it: no invalidate, ++ * which could discard what the CPU side wrote meanwhile into lines of the ++ * pages the buffer shares (a udmabuf covers whole 16 KiB pages). ++ */ ++ if (to_cpu && !(o->access & DRM_G5P_RENDER_ACCESS_WRITE)) ++ continue; ++ struct g5p_cache_receipt *cr = &job->cache; ++ bool imported = drm_gem_is_imported(o->obj); ++ bool exported = READ_ONCE(o->obj->dma_buf) != NULL; ++ bool wc = to_drm_gem_shmem_obj(o->obj)->map_wc; ++ int cls = g5p_cache_class(imported, exported, wc); ++ u64 bytes = o->hi ? o->hi - o->lo : o->obj->size; ++ u64 cpu = 0, wall = 0; ++ if (cr->enabled) ++ cr->requested[to_cpu][cls] += bytes; ++ /* Native pages are initially DMA-mapped with no SKIP_CPU_SYNC. ++ * All later CPU views of a private map_wc object (GEM mmap/vmap, ++ * including command validation) are write-combined. There is no ++ * cacheable client alias to clean/invalidate. Shared/foreign BOs ++ * remain conservative. Barriers below still order WC accesses. ++ */ ++ bool imported_uncached = imported && ++ drm_gem_dma_buf_is_cpu_uncached(o->obj->import_attach->dmabuf); ++ if ((imported_uncached && cr->import_policy) || g5p_cache_omit(cr->policy, imported, exported, wc)) ++ continue; ++ if (cr->enabled) { cpu = task_sched_runtime(current); wall = ktime_get_ns(); } ++ if (o->hi && (o->lo || o->hi != o->obj->size)) ++ g5p_sync_sgt_range(d->drm.dev, sgt, o->lo, o->hi, ++ to_cpu); ++ else if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ if (cr->enabled) { ++ cr->maintained[to_cpu][cls] += bytes; ++ cr->cpu[to_cpu][cls] += task_sched_runtime(current) - cpu; ++ cr->wall[to_cpu][cls] += ktime_get_ns() - wall; ++ } ++ } ++ if (to_cpu) dma_rmb(); else dma_wmb(); ++} ++ ++/* After acceptance, render_submit_lock held, no file lock: one dma_resv at a ++ * time, so no ww ordering is needed. Our own context's earlier fences are ++ * already ordered by the queue. A failure fails the job before hardware. ++ */ ++static int g5p_render_implicit(struct g5p_rjob *job, u64 own_context) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct dma_resv *resv = job->objs[i].obj->resv; ++ bool write = job->objs[i].access & DRM_G5P_RENDER_ACCESS_WRITE; ++ struct dma_fence **fences = NULL, **grown; ++ unsigned int count = 0; ++ int ret; ++ ++ if (!(job->objs[i].access & DRM_G5P_RENDER_ACCESS_IMPLICIT)) ++ continue; ++ dma_resv_lock(resv, NULL); ++ ret = dma_resv_get_fences(resv, ++ write ? DMA_RESV_USAGE_READ : ++ DMA_RESV_USAGE_WRITE, ++ &count, &fences); ++ if (!ret) ++ ret = dma_resv_reserve_fences(resv, 1); ++ if (!ret) ++ dma_resv_add_fence(resv, &job->fence->base, ++ write ? DMA_RESV_USAGE_WRITE : ++ DMA_RESV_USAGE_READ); ++ dma_resv_unlock(resv); ++ grown = ret || !count ? ++ NULL : ++ krealloc_array(job->deps, ++ job->dep_count + count, ++ sizeof(*grown), GFP_KERNEL); ++ if (!ret && count && !grown) ++ ret = -ENOMEM; ++ if (grown) ++ job->deps = grown; ++ for (u32 j = 0; j < count; j++) { ++ if (!ret && fences[j]->context != own_context) ++ job->deps[job->dep_count++] = fences[j]; ++ else ++ dma_fence_put(fences[j]); ++ } ++ kfree(fences); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++/* Free the mappings unmapped beside jobs up to fence seqno `seqno` (all: every one). */ ++static void g5p_render_reap(struct g5p_render *r, u64 seqno, bool all) ++{ ++ struct g5p_mapping *m, *t; ++ unsigned long flags; ++ LIST_HEAD(done); ++ ++ spin_lock_irqsave(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &r->zombies, link) ++ if (all || m->retire <= seqno) ++ list_move_tail(&m->link, &done); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &done, link) { ++ list_del(&m->link); ++ g5p_vm_obj_put(r->ctx->vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++} ++ ++/* One job, picked by the scheduler once its inputs signalled (or failed, or timed out). */ ++static void g5p_render_run(struct g5p_rjob *job) ++{ ++ struct g5p_drm *d = job->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_render *r = job->ctx->render; ++ unsigned long lock_deadline, flags; ++ bool hardware = false; ++ int ret; ++ ++ job->cache.enabled = READ_ONCE(render_user_cache_receipts); ++ job->cache.policy = READ_ONCE(render_user_cache_private_wc); ++ job->cache.import_policy = READ_ONCE(render_user_cache_import_wc); ++ if (job->cache.enabled) { ++ job->cache.job = job->id; ++ job->cache.start_cpu = task_sched_runtime(current); ++ job->cache.start_wall = ktime_get_ns(); ++ } ++ /* Admission closes only for good (reboot quiesce, FINAL teardown): a job queued before ++ * that fails here, without waiting for its inputs and without competing for the command ++ * lock. ++ */ ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) ++ ret = -ESHUTDOWN; ++ else ++ ret = job->setup_error ?: job->dep_error; ++ if (!ret && (job->ctx->bad || READ_ONCE(g->boot->failed))) ++ ret = -EIO; ++ if (!ret) { ++ /* Bounded: never queue forever behind a command stuck in MMIO. */ ++ lock_deadline = jiffies + 2 * HZ; ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, lock_deadline)) { ++ ret = -EBUSY; ++ break; ++ } ++ msleep(5); ++ } ++ } ++ if (!ret) { ++ if (g->boot->failed || g->boot->stopped) { ++ ret = -EIO; ++ } else if (!1 && !g5p_ru_mmio_ok(&g5p_rv3_term)) { ++ /* A V3 epoch is terminal: the legacy path would read and write registers. */ ++ ret = -EIO; ++ } else { ++ /* RAM-only from the DC3 attach on: no snapshot (it reads R0), no ++ * export, no register access of any kind after a timeout. ++ */ ++ g5p_render_cache(d, job, false); ++ WRITE_ONCE(d->rdev->executing, true); ++ ret = g5p_rv3_execute(d, r, job); ++ /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ ++ smp_store_release(&d->rdev->executing, false); ++ hardware = job->rv3.touched || job->rv3.bind_failed; ++ if (!ret || ret == -ECANCELED) ++ g5p_render_cache( ++ d, job, ++ true); /* before status and fence */ ++ if (!ret && r->bound) ++ WRITE_ONCE(r->live_maps, true); ++ } ++ g->last_result = ret; ++ mutex_unlock(&g->lock); ++ } ++ spin_lock_irqsave(&d->status_lock, flags); ++ /* A failed device fails every later job; one context's lost job or failed bind does not. */ ++ if (READ_ONCE(g->boot->failed) || READ_ONCE(g->boot->event_failed)) ++ ret = ret ?: -EIO; ++ job->fence->state = DRM_G5P_JOB_COMPLETE; ++ job->fence->error = ret < 0 ? ret : 0; ++ d->last_job = job->id; ++ d->last_error = job->fence->error; ++ d->job_error = job->fence->error; ++ d->job_state = DRM_G5P_JOB_COMPLETE; ++ if (ret < 0 && ret != -ECANCELED && hardware) { ++ /* No reset protocol: quarantine the context and the device. A TYPED_V3 ++ * timeout is one exception for the device: nothing is wrong with the ++ * protocol, the firmware may still complete, and the drain must keep ++ * decoding (RAM only) so that a late completion is recorded. A failed ++ * bind is another: the DC ring is not on the path of the contexts ++ * already bound, so only this context is lost. A lost job is the last: ++ * the executor already ruled (the drain, or the recovery that replaces ++ * this epoch), and only its context is lost. ++ */ ++ WRITE_ONCE(job->ctx->bad, true); ++ if (g5p_ru_job_fails_device(1, ret, job->rv3.bind_failed, ++ job->rv3.lost)) ++ WRITE_ONCE(g->boot->failed, true); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (ret < 0) ++ dma_fence_set_error(&job->fence->base, ret); ++ atomic_dec(&r->queued); ++ atomic_dec(&d->r8_queued); ++ dma_fence_signal(&job->fence->base); ++ /* A job that may still run on the hardware (a bad context) keeps what it might use. */ ++ if (!READ_ONCE(job->ctx->bad)) ++ g5p_render_reap(r, job->fence->base.seqno, false); ++ /* a failed job (complete=-N) always prints; a completed one is routine: job_log */ ++ if (ret) ++ dev_info( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ else ++ g5p_job_printk( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ if (job->cache.enabled) { ++ job->cache.total_cpu = task_sched_runtime(current) - job->cache.start_cpu; ++ job->cache.total_wall = ktime_get_ns() - job->cache.start_wall; ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ job->cache.serial = g5p_cache_head + 1; ++ g5p_cache_ring[g5p_cache_head++ % G5P_CACHE_RECORDS] = job->cache; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ } ++ g5p_render_job_free(job); ++ ++ drm_dev_put(&d->drm); ++} ++ ++/* Private PM-aux/RT page with the latched r25 leaf flags (c by default). */ ++static int g5p_render_map_page(struct g5p_drm *d, struct g5p_render *r, u64 va, ++ struct g5p_dma_page *page) ++{ ++ u8 *pte = g5p_vm_pte(r->ctx->vm, d->drm.dev, va, true); ++ ++ if (IS_ERR_OR_NULL(pte)) ++ return pte ? PTR_ERR(pte) : -ENOMEM; ++ if (g5p_le32(pte) || g5p_le32(pte + 4)) ++ return -EBUSY; ++ g5p_vm_entry(&r->ctx->vm->mutation, pte, ++ g5p_leaf_entry(page->dma, g5p_r25_attributes(&r->v, va)), ++ 2); ++ return 0; ++} ++ ++/* clear: unmap first (file lock held, VM still hooked). */ ++static void g5p_rt_release_memory(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rt *rt, bool clear) ++{ ++ if (clear && rt->page_count) ++ g5p_vm_clear(r->ctx->vm, d->drm.dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rt->pages[i]); ++ kvfree(rt->pages); ++ rt->pages = NULL; ++ rt->page_count = 0; ++ if (drm_mm_node_allocated(&rt->node)) ++ drm_mm_remove_node(&rt->node); ++} ++ ++/* File lock held, context idle; RAM only. Main ring then secondary in one ++ * RT_ALIGN node: the first 64x64 target lands exactly on r22's ++ * 101020000/101030000. ++ */ ++static int g5p_rt_create(struct g5p_drm *d, struct g5p_render *r, ++ const struct g5p_ru_rt *geometry, u32 width, ++ u32 height, struct g5p_rt **out) ++{ ++ struct g5p_rt *rt; ++ u64 bytes; ++ int ret; ++ ++ if (r->target_count >= G5P_RU_MAX_TARGETS) ++ return -ENOSPC; ++ rt = kzalloc(sizeof(*rt), GFP_KERNEL); ++ if (!rt) ++ return -ENOMEM; ++ rt->ctx = r->ctx; ++ rt->geometry = *geometry; ++ rt->width = width; ++ rt->height = height; ++ ret = g5p_ru_geometry_check(geometry, width, height, &rt->sizes); ++ if (ret) ++ goto err_free; ++ bytes = rt->sizes.main + rt->sizes.secondary_map; ++ ret = drm_mm_insert_node_in_range( ++ &r->private_mm, &rt->node, bytes, G5P_RU_RT_ALIGN, 0, ++ G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE, ++ DRM_MM_INSERT_LOW); ++ if (ret) ++ goto err_free; ++ rt->pages = ++ kvcalloc(bytes / G5P_GPU_PAGE, sizeof(*rt->pages), GFP_KERNEL); ++ if (!rt->pages) { ++ ret = -ENOMEM; ++ goto err_node; ++ } ++ for (; rt->page_count < bytes / G5P_GPU_PAGE; rt->page_count++) { ++ struct g5p_dma_page *p = &rt->pages[rt->page_count]; ++ ++ ret = g5p_vm_alloc_page(d->drm.dev, p); ++ if (!ret) ++ ret = g5p_render_map_page( ++ d, r, ++ rt->node.start + rt->page_count * G5P_GPU_PAGE, ++ p); ++ if (ret) { ++ g5p_ru_free_page(d->drm.dev, p); ++ goto err_pages; ++ } ++ } ++ if (r->bound) { ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(d->core, &rt->pages[i]); ++ ret = g5p_render_invalidate(r->ctx); ++ if (ret) { ++ /* Hardware may hold the new tag: retain the target. */ ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ return ret; ++ } ++ } ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ *out = rt; ++ g5p_vmlog_add(G5P_VMLOG_RT_NEW, r->ctx->vm, 0, rt->node.start, bytes, ++ rt->width << 16 | rt->height, rt->samples); ++ return 0; ++err_pages: ++ g5p_rt_release_memory(d, r, rt, true); ++ goto err_free; ++err_node: ++ drm_mm_remove_node(&rt->node); ++err_free: ++ kfree(rt); ++ return ret; ++} ++ ++/* Detach every VM-side trace: private PTEs and the reserved VA nodes. File ++ * lock held, or the file is closing (no concurrent VM ioctl can exist). ++ */ ++static void g5p_render_unhook(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct device *dev = c->gpu->drm.dev; ++ struct g5p_rt *rt; ++ ++ if (r->unhooked) ++ return; ++ list_for_each_entry(rt, &r->targets, link) ++ if (rt->page_count) ++ g5p_vm_clear(c->vm, dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ if (r->aux_mapped) ++ g5p_vm_clear(c->vm, dev, G5P_RU_PRIVATE_VA, ++ G5P_RU_AUX_PAGES * G5P_GPU_PAGE); ++ for (u32 i = 0; i < 4; i++) { ++ struct drm_mm_node *n = i == 0 ? &r->low_node : ++ i == 1 ? &r->firmware_node : ++ i == 2 ? &r->private_node : ++ &r->fault_node; ++ ++ if (drm_mm_node_allocated(n)) ++ drm_mm_remove_node(n); ++ } ++ if (c->vm->resident == c) ++ c->vm->resident = NULL; ++ r->unhooked = true; ++} ++ ++/* Last context reference: only reached unbound or after a successful DC3 ++ * unbind, so no firmware or GPU reference to this memory remains. The aux ++ * pages and the PM pool are the device's. ++ */ ++static void g5p_render_release(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_rt *rt, *tmp; ++ unsigned long flags; ++ ++ g5p_render_unhook(c); ++ g5p_render_reap(r, 0, true); ++ if (!list_empty(&r->rdev_link)) { ++ mutex_lock(&d->rdev->lock); ++ list_del_init(&r->rdev_link); ++ mutex_unlock(&d->rdev->lock); ++ } ++ list_for_each_entry_safe(rt, tmp, &r->targets, link) { ++ g5p_rt_release_memory(d, r, rt, false); ++ list_del(&rt->link); ++ kfree(rt); ++ } ++ if (drm_mm_node_allocated(&r->aux_node)) ++ drm_mm_remove_node(&r->aux_node); ++ drm_mm_takedown(&r->private_mm); ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ for (u32 e = 0; e < 2; e++) ++ if (r->slot[e]) ++ __clear_bit(r->slot[e], d->core->user_slots); ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ { ++ u32 aux = 0; ++ ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ aux += !!r->pm_aux[i].cpu; ++ /* release8 accounting: what this context frees, and the device's live GEM total */ ++ dev_info( ++ d->drm.dev, ++ "RENDER-V3 release slots=%u/%u jobs=%u set=%d pool_pages=%u aux_pages=%u " ++ "vm_pinned=%lluK live_bos=%u live_bo_bytes=%lluK end=v3\n", ++ r->slot[0], r->slot[1], r->jobs, r->pm_set, ++ r->pm_pool ? r->pm_blocks * 8 : 0, aux, ++ c->vm ? c->vm->pinned_bytes >> 10 : 0, ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10); ++ } ++ g5p_rv3_pmctx_release(d, r); ++ if (d->rdev->pm_last == r) ++ d->rdev->pm_last = NULL; ++ if (d->rdev->pm_prev == r) ++ d->rdev->pm_prev = NULL; ++ kvfree(r->pm_save); ++ kfree(r); ++ c->render = NULL; ++} ++ ++/* CREATE_CONTEXT type RENDER. Exclusive VM: reserves [64KiB, 1GiB) for the ++ * borrowed VA0 sink, the whole firmware GiB and the private RT/PM window. ++ */ ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_r25_variants v = { ++ .pm = (0), ++ .memory = (0), ++ .rt = (0), ++ .slots = (0), ++ }; ++ unsigned long flags; ++ u32 id, fw_window; ++ int ret; ++ ++ BUILD_BUG_ON(G5P_USER_RESERVED_START & ((1ULL << 30) - 1)); ++ if (a->flags || a->handle) ++ return -EINVAL; ++ ; ++ ; ++ fw_window = (G5P_RU_FW_WINDOW); ++ /* Unprivileged unless the VM would map the whole firmware GiB (the kill switch). */ ++ if (fw_window >= G5P_RU_FW_MODES || ++ (fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO))) ++ return fw_window >= G5P_RU_FW_MODES ? -EINVAL : -EPERM; ++ if ((0) > 2 || ++ ((G5P_RU_DEF_CACHE_VARIANT) != 0 && ++ (G5P_RU_DEF_CACHE_VARIANT) != 3) || ++ (G5P_RU_DEF_MMU_VARIANT) > 1 || (0) > 1 || (0) > 1 || ++ READ_ONCE(render_user_wait_ms) < ((1) ? G5P_RU_V3_WAIT_MS : 3000) || ++ READ_ONCE(render_user_wait_ms) > G5P_RU_V3_CAP_MS || ++ !g5p_ru_lat_params_valid() || !(32) || (32) > G5P_RU_MAX_BLOCKS || ++ ((G5P_RU_DEF_PROFILE1_VDM_CONTROL) & 0xf) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) > G5P_RU_MAX_VDM || ++ !g5p_ru_r25_valid(&v)) ++ return -EINVAL; ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ r = kzalloc(sizeof(*r), GFP_KERNEL); ++ if (!c || !r) { ++ kfree(c); ++ kfree(r); ++ return -ENOMEM; ++ } ++ r->pm_set = -1; ++ c->gpu = d; ++ c->render = r; ++ r->ctx = c; ++ INIT_LIST_HEAD(&r->targets); ++ INIT_LIST_HEAD(&r->rdev_link); ++ g5p_rs_entity_init(&r->entity); ++ spin_lock_init(&r->zlock); ++ INIT_LIST_HEAD(&r->zombies); ++ atomic_set(&r->queued, 0); ++ drm_mm_init(&r->private_mm, G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_SIZE); ++ r->stage = (0); ++ r->cache_variant = (G5P_RU_DEF_CACHE_VARIANT); ++ r->mmu_variant = (G5P_RU_DEF_MMU_VARIANT); ++ r->timestamp_variant = (0); ++ r->wait_ms = READ_ONCE(render_user_wait_ms); ++ r->hard_ms = READ_ONCE(render_user_hard_ms); ++ r->soft_ms = r->hard_ms ? ++ g5p_ru_lat_soft(READ_ONCE(render_user_soft_ms), ++ r->hard_ms) : ++ 0; ++ r->obs_ms = r->hard_ms ? READ_ONCE(render_user_obs_ms) : 0; ++ r->touch_ms = r->hard_ms ? (0) : 0; ++ r->settle_ms = (G5P_RU_RT_DEF_SETTLE_MS); ++ r->stall_job = (0); ++ r->fault_job = capable(CAP_SYS_RAWIO) ? (0) : 0; ++ r->wake = r->hard_ms && !r->touch_ms ? (2) : 0; ++ r->wake = g5p_wk_mode_valid(r->wake) ? r->wake : 0; ++ r->wake_seed = (0); ++ r->after_ta = (0); ++ r->pm_policy = (0); ++ r->cache_repeat = (0); ++ r->vdm_control = (G5P_RU_DEF_PROFILE1_VDM_CONTROL); ++ r->fw_window = fw_window; ++ r->v = v; ++ r->fence_context = dma_fence_context_alloc(1); ++ mutex_lock(&f->lock); ++ c->vm = xa_load(&f->vms, a->vm); ++ if (!c->vm) { ++ ret = -ENOENT; ++ goto err_unlock; ++ } ++ if (c->vm->contexts || c->vm->resident || c->vm->resident_gone || ++ c->vm->frozen || c->vm->serial_context) { ++ ret = -EBUSY; ++ goto err_unlock; ++ } ++ /* Memory mode 4 covers every render leaf, so no BO may be mapped yet. */ ++ if (v.memory == 4 && c->vm->map_count) { ++ ret = -EINVAL; ++ goto err_unlock; ++ } ++ r->low_node.start = G5P_USER_VA_START; ++ r->low_node.size = G5P_RU_USER_VA_MIN - G5P_USER_VA_START; ++ r->private_node.start = G5P_RU_PRIVATE_VA; ++ r->private_node.size = G5P_RU_PRIVATE_SIZE; ++ r->firmware_node.start = G5P_USER_RESERVED_END; ++ r->firmware_node.size = ++ ((G5P_RU_FIRMWARE_CI + 1) << 30) - G5P_USER_RESERVED_END; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->low_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->private_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->firmware_node) || ++ g5p_le32(c->vm->root.cpu + G5P_RU_FIRMWARE_CI * 4) || ++ g5p_le32(c->vm->root.cpu)) { ++ ret = -EINVAL; /* VM already maps a range this context needs */ ++ goto err_unlock; ++ } ++ if (r->fault_job) { ++ r->fault_node.start = G5P_RU_TEST_FAULT_VA; ++ r->fault_node.size = G5P_GPU_PAGE; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->fault_node)) { ++ dev_warn( ++ d->drm.dev, ++ "RENDER-V3 test-fault off: va=%#llx already used in this VM end=v3\n", ++ G5P_RU_TEST_FAULT_VA); ++ r->fault_job = 0; ++ } ++ } ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ if (g5p_ru_slot_pair(d->core->user_slots, v.slots, r->slot)) { ++ __set_bit(r->slot[0], d->core->user_slots); ++ __set_bit(r->slot[1], d->core->user_slots); ++ } else { ++ r->slot[0] = 0; ++ r->slot[1] = 0; ++ } ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ if (!r->slot[0]) { ++ ret = -ENOSPC; ++ goto err_unlock; ++ } ++ /* The device's PM pool (at least this context's latched size) and aux pages, mapped at ++ * the same VA in every render VM, so the PM state each TA reloads is the one the ++ * previous job saved, whichever VM ran it. ++ */ ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, (32)); ++ mutex_unlock(&d->rdev->lock); ++ r->aux_node.start = G5P_RU_PRIVATE_VA; ++ r->aux_node.size = G5P_RU_AUX_PAGES * G5P_GPU_PAGE; ++ if (!ret) ++ ret = drm_mm_reserve_node(&r->private_mm, &r->aux_node); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ ret = g5p_render_map_page(d, r, ++ G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE, ++ &d->rdev->aux[i]); ++ r->aux_mapped = true; ++ } ++ if (ret) ++ goto err_unlock; ++ ret = xa_alloc(&f->contexts, &id, c, XA_LIMIT(1, G5P_MAX_CONTEXTS), ++ GFP_KERNEL); ++ if (ret) ++ goto err_unlock; ++ kref_init(&c->ref); ++ kref_get(&c->vm->ref); ++ mutex_lock(&d->rdev->lock); ++ list_add_tail(&r->rdev_link, &d->rdev->contexts); ++ mutex_unlock(&d->rdev->lock); ++ c->vm->contexts++; ++ c->vm->serial_context = true; ++ c->vm->resident = c; ++ c->vm->zero_leaf_flags = v.memory == 4; ++ a->handle = id; ++ mutex_unlock(&f->lock); ++ return 0; ++err_unlock: ++ if (c->vm) ++ g5p_render_release(c); ++ else ++ kfree(r); ++ mutex_unlock(&f->lock); ++ kfree(c); ++ return ret; ++} ++ ++/* DESTROY_CONTEXT, file lock held. A TYPED_V3 context that submitted: its close item, queued ++ * behind its jobs, unbinds and releases it (g5p_render_close()). Unbound: plain release. A ++ * legacy context that brought the device up, bound and idle: zero PA/tag and consume one DC3 ++ * per slot, as r21 detach does. Failure or a quarantined legacy context retains everything ++ * until reboot. ++ */ ++static int g5p_render_destroy(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl, start, target; ++ unsigned long deadline; ++ int ret; ++ ++ if (r->v3_accepted) { ++ g5p_render_close(c); ++ return 0; ++ } ++ if (atomic_read(&r->queued) || c->bad) ++ return -EBUSY; ++ if (!r->bound) { ++ g5p_render_unhook(c); ++ if (r->claimant) { ++ /* Claimed but never touched hardware: drop the claim's reference. The ++ * device stays claimed for render: other contexts may be queued on it. ++ */ ++ r->claimant = false; ++ d->quarantined_context = NULL; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ return 0; ++ } ++ /* The legacy unbind rings a doorbell: retained once the epoch is terminal. */ ++ if (!r->claimant || !g5p_ru_mmio_ok(&g5p_rv3_term)) ++ return -EBUSY; ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ ret = -EIO; ++ if (b->failed || b->stopped) ++ goto out; ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ start = READ_ONCE(ctl[4]); ++ ret = -EBUSY; ++ if (start > 255 || READ_ONCE(ctl[0]) != start) ++ goto out; ++ for (u32 e = 0; e < 2; e++) { ++ u32 index = (start + e) & 255; ++ ++ g5p_put64(g5p_boot_shared(b, 0x86) + r->slot[e] * 8, 0); ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, 0); ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + index * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE, ++ j == 0 ? 3 : ++ j == 1 ? r->slot[e] : ++ 0); ++ } ++ } ++ target = (start + 2) & 255; ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], target); ++ dma_wmb(); ++ ret = g5p_prepare_kick(g, "RENDER-USER-unbind"); ++ if (ret) ++ goto fail; ++ writel(0x10, g->rgx + 0xb00); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || b->failed) { ++ ret = ret ?: -EIO; ++ goto fail; ++ } ++ dma_rmb(); ++ if (READ_ONCE(ctl[0]) == target) ++ break; ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (READ_ONCE(ctl[0]) != target) { ++ ret = -ETIMEDOUT; ++ goto fail; ++ } ++ r->detached = true; ++ r->bound = false; ++ r->claimant = false; ++ WRITE_ONCE(c->live, false); ++ g5p_render_unhook(c); ++ /* One render attachment per boot: accepted stays held, so neither a new ++ * render owner nor compute can reuse the shared channel pages. ++ */ ++ d->quarantined_context = NULL; ++ g5p_mmu_record( ++ g, ++ "RENDER-USER detached slots=%u/%u; resources freed on release\n", ++ r->slot[0], r->slot[1]); ++ mutex_unlock(&g->lock); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ return 0; ++fail: ++ c->bad = true; ++ b->failed = true; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, ++ const struct drm_g5p_sync *syncs, u32 count, ++ struct drm_syncobj **signal, u64 *point, ++ struct dma_fence_chain **chain) ++{ ++ u32 signal_handle = 0; ++ int ret; ++ ++ if (!count || count > G5P_MAX_SYNCS) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) { ++ for (u32 j = 0; j < i; j++) ++ if (syncs[i].handle == syncs[j].handle && ++ syncs[i].point == syncs[j].point && ++ syncs[i].flags == syncs[j].flags) ++ return -EINVAL; ++ if (syncs[i].flags & ~DRM_G5P_SYNC_SIGNAL) ++ return -EINVAL; ++ if (syncs[i].flags) { ++ if (*signal) ++ return -EINVAL; ++ *signal = drm_syncobj_find(file, syncs[i].handle); ++ if (!*signal) ++ return -ENOENT; ++ *point = syncs[i].point; ++ signal_handle = syncs[i].handle; ++ continue; ++ } ++ if (job->input_count == ARRAY_SIZE(job->inputs)) ++ return -E2BIG; ++ ret = drm_syncobj_find_fence(file, syncs[i].handle, ++ syncs[i].point, 0, ++ &job->inputs[job->input_count]); ++ if (ret) ++ return ret; ++ job->input_count++; ++ } ++ if (!*signal) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) ++ if (!syncs[i].flags && syncs[i].handle == signal_handle && ++ (!*point || !syncs[i].point || syncs[i].point >= *point)) ++ return -EINVAL; ++ if (*point) { ++ struct dma_fence *old = drm_syncobj_fence_get(*signal); ++ ++ ret = old && (!to_dma_fence_chain(old) || ++ old->seqno >= *point) ? ++ -EINVAL : ++ 0; ++ dma_fence_put(old); ++ if (ret) ++ return ret; ++ *chain = dma_fence_chain_alloc(); ++ if (!*chain) ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++/* File lock held. One mapping of the listed BO must contain each range at a ++ * consistent BO offset; access is a subset of the BO's immutable roles. ++ */ ++static int g5p_render_resolve(struct drm_file *file, struct g5p_vm *vm, ++ const struct drm_g5p_render_resource *res, ++ u32 count, struct g5p_ru_range *ranges, ++ struct g5p_robj *objs, u32 *obj_count) ++{ ++ u32 known = DRM_G5P_RENDER_ACCESS_READ | DRM_G5P_RENDER_ACCESS_WRITE | ++ DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ ++ for (u32 i = 0; i < count; i++) { ++ struct drm_gem_object *obj; ++ struct g5p_mapping *m, *found = NULL; ++ u32 roles, access; ++ ++ for (u32 j = 0; j < i; j++) ++ if (res[i].handle == res[j].handle) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, res[i].handle); ++ if (!obj) ++ return -ENOENT; ++ roles = to_g5p_bo(obj)->roles; ++ { ++ access = res[i].kind; ++ if (!(access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE)) || ++ access & ~known || ++ ((access & G5P_RU_ACCESS_READ) && ++ !(roles & DRM_G5P_BO_READ)) || ++ ((access & G5P_RU_ACCESS_WRITE) && ++ !(roles & DRM_G5P_BO_WRITE))) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ /* A foreign buffer always gets cache maintenance and fences. */ ++ if (drm_gem_is_imported(obj)) ++ access |= DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ } ++ list_for_each_entry(m, &vm->maps, link) ++ if (m->obj == obj && res[i].va >= m->node.start && ++ res[i].size && ++ res[i].va - m->node.start < m->node.size && ++ res[i].size <= m->node.size - (res[i].va - ++ m->node.start) && ++ res[i].offset == ++ m->offset + (res[i].va - m->node.start)) ++ found = m; ++ if (!found) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ if (access & (DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE)) { ++ struct g5p_robj *o = &objs[*obj_count]; ++ ++ o->obj = obj; /* the job inherits this reference */ ++ o->access = access; ++ o->lo = 0; ++ o->hi = obj->size; ++ if ((READ_ONCE(render_user_sync_range) & 2) && ++ !g5p_sync_window(res[i].offset, res[i].size, ++ obj->size, &o->lo, &o->hi)) { ++ o->lo = 0; ++ o->hi = obj->size; ++ } ++ (*obj_count)++; ++ } else { ++ drm_gem_object_put(obj); ++ } ++ ranges[i].va = res[i].va; ++ ranges[i].size = res[i].size; ++ ranges[i].access = access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE); ++ /* The BO's EXEC role, for TYPED_V3 only; an imported BO never has it. */ ++ if ((roles & DRM_G5P_BO_EXEC)) ++ ranges[i].access |= G5P_RU_ACCESS_EXEC; ++ } ++ return 0; ++} ++ ++/* LINEAR64_V1 admits only the exact round16 immutable inputs. Writes stay ++ * privileged and caller-serialized, as for the r20 first-store dialect. ++ */ ++ ++/* File lock held. Copy `len` bytes at GPU VA `va` out of the BO the VM maps ++ * there, through the BO's own pages, as g5p_render_profile1_bytes() does. The ++ * mapping is found again by VA and its own (VM-referenced) object is read, so ++ * a handle closed or replaced since g5p_render_resolve() cannot redirect the ++ * read. Only a native BO has shmem pages: an imported one (never EXEC, so ++ * never a TA program) is refused for the stream too. ++ */ ++static int g5p_render_read_va(struct drm_device *drm, struct g5p_vm *vm, u64 va, ++ void *dst, u32 len) ++{ ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); ++ struct g5p_mapping *m, *found = NULL; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ u64 off; ++ int ret; ++ ++ list_for_each_entry(m, &vm->maps, link) ++ if (va >= m->node.start && va - m->node.start < m->node.size && ++ len <= m->node.size - (va - m->node.start)) ++ found = m; ++ if (!found) ++ return -ENOENT; ++ obj = found->obj; ++ off = found->offset + (va - found->node.start); ++ if (drm_gem_is_imported(obj) || off > obj->size || ++ len > obj->size - off) ++ return -EINVAL; ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ ret = IS_ERR(sgt) ? PTR_ERR(sgt) : drm_gem_vmap(obj, &map); ++ if (ret) ++ return ret; ++ /* Keep content validation. Native private WC has no CPU cache alias; ++ * all other supported mappings retain a bounded DMA ownership sync. ++ */ ++ bool range = READ_ONCE(render_user_cache_read_va); ++ bool omit = range && g5p_cache_omit(READ_ONCE(render_user_cache_private_wc), ++ false, READ_ONCE(obj->dma_buf) != NULL, to_drm_gem_shmem_obj(obj)->map_wc); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, true); ++ else dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_rmb(); ++ if (map.is_iomem) ++ ret = -EINVAL; ++ else ++ memcpy(dst, (const u8 *)map.vaddr + off, len); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, false); ++ else dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_wmb(); ++ drm_gem_vunmap(obj, &map); ++ return ret; ++} ++ ++static void g5p_render_pass_from_uapi(struct g5p_ru_pass *p, ++ const struct drm_g5p_render_pass *u) ++{ ++ p->vdm_va = u->vdm_va; ++ p->vdm_size = u->vdm_size; ++ p->vdm_control = u->vdm_control; ++ p->scratch_va = u->scratch_va; ++ p->scratch_size = u->scratch_size; ++ memcpy(p->scratch_ptr, u->scratch_ptr, sizeof(p->scratch_ptr)); ++ p->pool7_va = u->pool7_va; ++ p->pool7_size = u->pool7_size; ++ p->pool8_va = u->pool8_va; ++ p->pool8_size = u->pool8_size; ++ memcpy(p->context_pds, u->context_pds, sizeof(p->context_pds)); ++ memcpy(p->resume_pds, u->resume_pds, sizeof(p->resume_pds)); ++ memcpy(p->ta_pds, u->ta_pds, sizeof(p->ta_pds)); ++ memcpy(p->background, u->background, sizeof(p->background)); ++ memcpy(p->eot, u->eot, sizeof(p->eot)); ++ memcpy(p->eot_resume, u->eot_resume, sizeof(p->eot_resume)); ++ memcpy(p->restore, u->restore, sizeof(p->restore)); ++} ++ ++static void g5p_render_pass2_from_uapi(struct g5p_ru_pass2 *e, ++ const struct drm_g5p_render_pass_ext *u) ++{ ++ *e = (struct g5p_ru_pass2){ ++ .scissor_va = u->scissor_va, ++ .dbias_va = u->dbias_va, ++ .background_texture = u->background_texture, ++ .occlusion_query_va = u->occlusion_query_va, ++ .occlusion_query_size = u->occlusion_query_size, ++ .scissor_count = u->scissor_count, ++ .dbias_count = u->dbias_count, ++ .bgobj_depth = u->bgobj_depth, ++ .bgobj_vals = u->bgobj_vals, ++ .pixel_width = u->pixel_output_width, ++ .zls_mask = u->zls_address_mask, ++ }; ++ memcpy(e->zls, u->zls, sizeof(e->zls)); ++} ++ ++static void g5p_render_pass3_from_uapi(struct g5p_ru_pass3 *v, ++ const struct drm_g5p_render_pass_v3 *u) ++{ ++ *v = (struct g5p_ru_pass3){ ++ .ta_program_va = u->ta_program_va, ++ .ta_program_size = u->ta_program_size, ++ .vdm_end = u->vdm_end, ++ .pixel_width = u->pixel_output_width, ++ .pad = u->pad, ++ }; ++} ++ ++static_assert(offsetof(struct drm_g5p_render_pass, samples) == 296); ++static_assert(offsetof(struct drm_g5p_render_pass, sample_positions) == 304); ++static_assert(offsetof(struct drm_g5p_render_pass, reserved) == 312); ++static_assert(DRM_G5P_CAP_RENDER_MSAA == BIT_ULL(31)); ++static_assert(DRM_G5P_CAP_RENDER_OCCLUSION_QUERY == BIT_ULL(33)); ++static_assert(DRM_G5P_RENDER_MSAA_RESOLVE == G5P_RU_MSAA_RESOLVE); ++ ++/* CAP_RENDER_MSAA: the target's geometry is kernel-derived (it holds the sample ++ * count), and only TYPED_V3 carries the pass fields. ++ */ ++static bool g5p_render_msaa_enabled(void) ++{ ++ return (1) && (1); ++} ++ ++/* The base pass's MSAA fields: zero unless TYPED_V3 with the capability. */ ++static int g5p_render_msaa_from_uapi(struct g5p_ru_msaa *m, ++ const struct drm_g5p_render_pass *u, ++ bool v3) ++{ ++ *m = (struct g5p_ru_msaa){ ++ .samples = u->samples, ++ .flags = u->msaa_flags, ++ .positions = u->sample_positions, ++ }; ++ if ((m->samples || m->flags || m->positions) && ++ (!v3 || !g5p_render_msaa_enabled())) ++ return -EINVAL; ++ return 0; ++} ++ ++/* Render submit lock held. The first accepted job claims the device for render until reboot; ++ * same exclusions as compute/r22. Every later job, of any context, passes while the device is ++ * neither failed nor stopped, without the core lock (a job may hold it for its whole run). The ++ * claimant keeps the reference d->quarantined_context has always held. The IRQ-test term is ++ * the r74 one (render-fixed.inc, render-r72.inc): the r74 boots refuse the IRQ test, so ++ * irq_tested must stay 0 there and must be 1 on every other boot. ++ */ ++static int g5p_render_device_claim(struct g5p_drm *d, struct g5p_context *c) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_render *r = c->render; ++ int ret = -EBUSY; ++ ++ if (READ_ONCE(d->render_claimed)) { ++ /* A recovery swaps the epoch, and a failure noted mid-job may lead to one: the job ++ * waits for the ruling in the scheduler, which fails it if the epoch ends. ++ */ ++ if (g5p_rdev_ruling(d->rdev)) ++ return 0; ++ b = READ_ONCE(g->boot); ++ return !b || READ_ONCE(b->failed) || READ_ONCE(b->stopped) ? ++ -EIO : ++ 0; ++ } ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ if (b->failed || b->stopped || !b->control_complete || b->irq_tested || ++ b->user_compute_owned || d->quarantined_context || ++ !b->render_meta[0].cpu || !b->render_meta[1].cpu || ++ b->layout.used[1] > G5P_R22_TA || ++ /* recovery108: expose the GPU only with a current-generation ALIVE proof. */ ++ g5p_ds_hook_claim(g) || atomic_cmpxchg(&d->accepted, 0, 1)) ++ goto out; ++ b->user_compute_owned = true; /* blocks diagnostic commands, latched */ ++ r->claimant = true; ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->quarantined_context = c; ++ WRITE_ONCE(d->render_claimed, true); ++ ret = 0; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Evict the oldest completed result beyond the per-file bound. */ ++static int g5p_render_result_slot(struct g5p_file *f, struct g5p_drm *d) ++{ ++ unsigned long index = 0, flags; ++ struct g5p_fence *old; ++ bool done; ++ ++ if (f->render_results < G5P_RU_RESULTS) ++ return 0; ++ old = xa_find(&f->results, &index, ULONG_MAX, XA_PRESENT); ++ if (!old) ++ return 0; ++ spin_lock_irqsave(&d->status_lock, flags); ++ done = old->state == DRM_G5P_JOB_COMPLETE; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!done) ++ return -EBUSY; ++ xa_erase(&f->results, index); ++ dma_fence_put(&old->base); ++ f->render_results--; ++ return 0; ++} ++ ++static int g5p_render_submit_admitted(struct drm_device *drm, ++ struct drm_file *file, u32 context, ++ u32 profile, u32 rt_handle, u64 res_ptr, ++ u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, ++ u32 pass_size, u64 *job_id) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_render_pass_v3 *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_ru_pass pass; ++ struct g5p_ru_pass2 pass2 = {}; ++ struct g5p_ru_pass3 pass3 = {}; ++ struct g5p_ru_v3_loc loc = {}; ++ struct g5p_ru_msaa msaa = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ struct g5p_rt *rt = NULL; ++ const bool v2 = false; ++ const bool v3 = true; ++ u64 point = 0; ++ int ret; ++ ++ ; ++ ; ++ /* TYPED_V3 is unprivileged (checked per context below); the legacy profiles touch ++ * registers and stay with CAP_SYS_RAWIO. ++ */ ++ ; ++ ; ++ /* During a recovery the epoch is being replaced: accept, the scheduler holds the job. */ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ /* a failure noted mid-job waits for the executor's ruling (g5p_rdev_ruling()) */ ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(res_ptr), res_count, ++ sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(sync_ptr), sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ { ++ /* pass_size is the exact V1, V2 or V3 size; ext exists for V2 and V3, ++ * the program fields for V3 only. ++ */ ++ upass = memdup_user(u64_to_user_ptr(pass_ptr), pass_size); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (upass->base.size != pass_size || upass->base.flags || ++ memchr_inv(upass->base.reserved, 0, ++ sizeof(upass->base.reserved)) || ++ (true && memchr_inv(upass->ext.reserved, 0, ++ sizeof(upass->ext.reserved))) || ++ (v3 && ++ memchr_inv(upass->reserved, 0, sizeof(upass->reserved)))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_render_pass_from_uapi(&pass, &upass->base); ++ ret = g5p_render_msaa_from_uapi(&msaa, &upass->base, v3); ++ if (ret) ++ goto out; ++ { ++ g5p_render_pass2_from_uapi(&pass2, &upass->ext); ++ }; ++ { ++ g5p_render_pass3_from_uapi(&pass3, upass); ++ }; ++ } ++ ranges = kvcalloc(res_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(res_count, sizeof(*job->objs), GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ /* Acceptance, implicit fence attach and queueing happen in one order. */ ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ { ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, res_count, ranges, job->objs, ++ &job->obj_count); ++ if (ret) ++ goto unlock; ++ { ++ rt = xa_load(&f->targets, rt_handle); ++ if (!rt || rt->ctx != c || rt->destroyed) { ++ ret = -ENOENT; ++ goto unlock; ++ } ++ } ++ { ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ u8 head[G5P_RU_PROGRAM_MIN], tail[4]; ++ ++ /* Structure first; the program head and the stream's last dword are ++ * read only once every range is proven backed, READ and (program) EXEC. ++ */ ++ ret = g5p_ru_pass3_check(&pass, &pass2, &pass3, &lim, ranges, ++ res_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass3.ta_program_va, head, ++ sizeof(head)); ++ if (!ret) ++ ret = g5p_render_read_va( ++ drm, c->vm, pass.vdm_va + pass3.vdm_end - 4, ++ tail, sizeof(tail)); ++ if (!ret) ++ ret = g5p_ru_pass3_content(head, tail); ++ } ++ /* Every profile: a one-sample pass needs a one-sample target, and back. */ ++ if (!ret) ++ ret = g5p_ru_msaa_check(&msaa, rt->samples ? rt->samples : 1); ++ if (ret) ++ goto unlock; ++ g5p_ru_fill_ta(job->ta_w, &pass); ++ { ++ g5p_ru_fill_ta_v3(job->ta_w, loc.program_off); ++ }; ++ g5p_ru_fill_3d(job->td_w, &pass); ++ /* TYPED_V3 with an all-zero ext is the V1 dialect: the V1 3D words, plus ++ * the output width words (width 2 rewrites the frozen dialect-1 values). ++ */ ++ if (v2 || loc.dialect2) ++ g5p_ru_fill_3d_v2(job->td_w, &pass2); ++ else { ++ g5p_ru_fill_3d_width(job->td_w, loc.pixel_width); ++ }; ++ /* r25 RT variant 1 moves only TA.W28/3D.W20 one slice into the ring. */ ++ g5p_ru_rt_fill(job->ta_w, job->td_w, &rt->geometry, ++ rt->node.start + (r->v.rt == 1 ? rt->sizes.slice : 0), ++ rt->node.start + rt->sizes.main); ++ { ++ g5p_ru_fill_msaa(job->ta_w, job->td_w, &msaa, &rt->geometry); ++ }; ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ /* Unpublished, never signalled: still a clean rejection. */ ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = v3; ++ { ++ r->v3_accepted = true; ++ }; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ job->cache.pid = task_tgid_nr(current); ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); /* only ever an uninitialized fence here */ ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++/* recovery108: a submitter is inside the admission window for its whole call; once a ++ * FINAL teardown closed admission it is refused before it looks at anything. ++ */ ++static int g5p_render_submit(struct drm_device *drm, struct drm_file *file, ++ u32 context, u32 profile, u32 rt_handle, ++ u64 res_ptr, u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, u32 pass_size, ++ u64 *job_id) ++{ ++ int ret; ++ ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_render_submit_admitted(drm, file, context, profile, rt_handle, ++ res_ptr, res_count, sync_ptr, ++ sync_count, pass_ptr, pass_size, ++ job_id); ++ g5p_r8_exit(); ++ return ret; ++} ++ ++/* The layout the profile dispatch (pass_size selects the profile) and ++ * GET_RENDER_INFO rely on; a header edit that moves any of it fails here. ++ */ ++static_assert(sizeof(struct drm_g5p_render_pass) == 360); ++static_assert(offsetof(struct drm_g5p_render_pass, vdm_control) == 20); ++static_assert(sizeof(struct drm_g5p_render_pass_ext) == 128); ++static_assert(sizeof(struct drm_g5p_render_pass_v2) == 488); ++static_assert(sizeof(struct drm_g5p_render_pass_v3) == 536); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ext) == ++ sizeof(struct drm_g5p_render_pass)); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_va) == 488); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_size) == 496); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, vdm_end) == 500); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, pixel_output_width) == ++ 504); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, reserved) == 512); ++static_assert(sizeof(struct drm_g5p_submit_render_typed) == 80); ++static_assert(sizeof(struct drm_g5p_get_render_info) == 120); ++static_assert(offsetof(struct drm_g5p_get_render_info, hw_va_limit) == 104); ++static_assert(offsetof(struct drm_g5p_get_render_info, reserved1) == 112); ++static_assert(DRM_G5P_CAP_TYPED_RENDER_V3 == BIT_ULL(26)); ++static_assert(DRM_G5P_CAP_RENDER_MULTI_CONTEXT == BIT_ULL(27)); ++static_assert(DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH == BIT_ULL(29)); ++static_assert(DRM_G5P_RENDER_PROFILE_TYPED_V3 == 4); ++ ++static int g5p_submit_render_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_render_typed *a = data; ++ u64 id = 0; ++ int ret; ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || a->profile != DRM_G5P_RENDER_PROFILE_TYPED_V3 || ++ !a->render_target || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_render_pass_v3)) ++ return -EINVAL; ++ ret = g5p_render_submit(drm, file, a->context, ++ DRM_G5P_RENDER_PROFILE_TYPED_V3, ++ a->render_target, a->resources, ++ a->resource_count, a->syncs, a->sync_count, ++ a->pass, a->pass_size, &id); ++ a->job_id = id; ++ return ret; ++} ++ ++static void g5p_rt_geometry_to_uapi(struct drm_g5p_rt_geometry *u, ++ const struct g5p_ru_rt *k) ++{ ++ u->header_bytes = k->header; ++ u->region_bytes = k->region; ++ u->packed_state = k->packed; ++ u->state_word = k->state; ++ u->padded_dims = k->padded; ++ memcpy(u->scale, k->scale, sizeof(u->scale)); ++ memcpy(u->ta_state, k->ta_state, sizeof(u->ta_state)); ++ memcpy(u->ta_tiling, k->ta_tiling, sizeof(u->ta_tiling)); ++ u->ta_tiling_word = k->ta_tiling_word; ++ u->tile_count = k->tiles; ++ u->secondary_bytes = k->secondary; ++ u->secondary_align = k->secondary_align; ++} ++ ++static int g5p_create_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ const struct drm_g5p_rt_geometry *u = &a->geometry; ++ struct g5p_ru_rt geometry; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ bool user = a->flags & DRM_G5P_RT_USER_GEOMETRY; ++ bool kernel = (1); ++ u64 positions = a->samples == 4 ? G5P_RU_POSITIONS_4X : ++ G5P_RU_POSITIONS_1X; ++ u32 id, blocks = 0; ++ int ret; ++ ++ ; ++ ; ++ if (a->flags & ~DRM_G5P_RT_USER_GEOMETRY || a->handle || a->pad || ++ a->main_va || a->main_bytes || a->secondary_va || ++ a->secondary_bytes || ++ (a->samples != 1 && ++ (a->samples != 4 || !g5p_render_msaa_enabled())) || ++ a->layers != 1 || a->pm_blocks || a->pad2 || a->reserved || ++ u->reserved[0] || u->reserved[1]) ++ return -EINVAL; ++ if (user) { ++ { ++ return -EOPNOTSUPP; ++ } ++ } else if (kernel) { ++ /* 6ca6d5c in integers; equal to the leaf for every size. */ ++ if (memchr_inv(u, 0, sizeof(*u)) || ++ g5p_ru_rt_derive_samples(a->width, a->height, a->samples, ++ positions, &geometry)) ++ return -EINVAL; ++ } else { ++ /* Kernel table: only the offline-executed 6ca6d5c result. */ ++ if (a->width != 64 || a->height != 64 || ++ memchr_inv(u, 0, sizeof(*u))) ++ return -EOPNOTSUPP; ++ geometry = g5p_ru_rt64; ++ } ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto out; ++ } ++ r = c->render; ++ ret = g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ /* No core lock: another context's job may hold it for its whole run, and nothing here ++ * touches the hardware. The device pool only grows; it serves every target of every ++ * context. ++ */ ++ if (kernel) { ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, g5p_ru_pm_blocks_for(geometry.tiles, ++ d->rdev->blocks)); ++ blocks = d->rdev->blocks; ++ mutex_unlock(&d->rdev->lock); ++ } ++ if (!ret) ++ ret = g5p_rt_create(d, r, &geometry, a->width, a->height, &rt); ++ if (ret) ++ goto out; ++ rt->samples = a->samples; ++ ret = xa_alloc(&f->targets, &id, rt, ++ XA_LIMIT(1, G5P_RU_MAX_TARGETS * 8), GFP_KERNEL); ++ if (ret) { ++ /* Mapped but unpublished to a job: leave it owned by the context. */ ++ rt->destroyed = true; ++ goto out; ++ } ++ kref_get(&c->ref); ++ a->handle = id; ++ a->main_va = rt->node.start; ++ a->main_bytes = rt->sizes.main; ++ a->secondary_va = rt->node.start + rt->sizes.main; ++ a->secondary_bytes = rt->sizes.secondary_map; ++ if (!user) ++ g5p_rt_geometry_to_uapi(&a->geometry, &geometry); ++ if (kernel) ++ a->pm_blocks = blocks; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_destroy_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c = NULL; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ int ret; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ rt = xa_load(&f->targets, a->handle); ++ if (!rt) { ++ ret = -ENOENT; ++ goto out; ++ } ++ c = rt->ctx; ++ r = c->render; ++ ret = r->detached ? 0 : g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ g5p_vmlog_add(G5P_VMLOG_RT_DEL, c->vm, 0, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map, ++ rt->width << 16 | rt->height, rt->samples); ++ /* RAM only, no core lock: see g5p_create_render_target(). */ ++ if (r->bound) { ++ g5p_vm_clear(c->vm, drm->dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ ret = g5p_render_invalidate(c); ++ if (ret) ++ goto out; ++ } ++ if (!r->detached) { ++ g5p_rt_release_memory(d, r, rt, true); ++ list_del(&rt->link); ++ r->target_count--; ++ if (r->profile1_rt == rt) ++ r->profile1_rt = NULL; ++ kfree(rt); ++ } else { ++ rt->destroyed = true; ++ } ++ xa_erase(&f->targets, a->handle); ++out: ++ mutex_unlock(&f->lock); ++ if (!ret && c) ++ g5p_context_put(c); ++ return ret; ++} ++ ++/* True when a context created now would latch a hypothesis knob that differs ++ * from its default: GET_RENDER_INFO has no context to report on. ++ */ ++static bool g5p_render_variant_active(void) ++{ ++ return (0) || (G5P_RU_DEF_CACHE_VARIANT) != G5P_RU_DEF_CACHE_VARIANT || ++ (G5P_RU_DEF_MMU_VARIANT) != G5P_RU_DEF_MMU_VARIANT || (0) || ++ READ_ONCE(render_user_wait_ms) != G5P_RU_DEF_WAIT_MS || (0) || ++ (0) || (0) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) != ++ G5P_RU_DEF_PROFILE1_VDM_CONTROL || ++ (0) || (0) || (0) || (0); ++} ++ ++static int g5p_get_render_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_render_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || ++ memchr_inv(&a->profiles, 0, ++ sizeof(*a) - offsetof(struct drm_g5p_get_render_info, ++ profiles))) ++ return -EINVAL; ++ ; ++ ; ++ if (g5p_render_variant_active()) ++ a->flags = DRM_G5P_RENDER_INFO_VARIANT_ACTIVE; ++ a->profiles = BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ ; ++ ; ++ { ++ a->profiles |= BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ }; ++ { ++ a->max_render_width = G5P_RU_MAX_DIM; ++ a->max_render_height = G5P_RU_MAX_DIM; ++ a->max_pm_blocks = G5P_RU_MAX_BLOCKS; ++ } ++ a->max_render_targets = G5P_RU_MAX_TARGETS; ++ a->private_va_start = G5P_RU_PRIVATE_VA; ++ a->private_va_end = G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE; ++ a->user_va_min = G5P_RU_USER_VA_MIN; ++ a->usc_base = G5P_RU_PDS_BASE; ++ a->pds_base = a->usc_base; ++ a->code_window = G5P_RU_CODE_WINDOW; ++ a->hw_va_limit = G5P_RU_HW_VA_LIMIT; ++ a->pm_blocks = (32); ++ a->pm_block_bytes = G5P_RU_BLOCK_BYTES; ++ a->max_queued_jobs = G5P_RU_MAX_QUEUED; ++ a->max_vdm_bytes = DRM_G5P_RENDER_MAX_VDM_BYTES; ++ /* TYPED_V3: what a waiter must outlast is the job's absolute bound (TA publish to the end of the ++ * 3D, render-user-wait.h), not the per-phase module value: a TA that takes its whole phase leaves the ++ * 3D the rest of the job cap. ++ */ ++ { ++ if (READ_ONCE(render_user_hard_ms)) ++ a->job_timeout_ms = g5p_ru_lat_job_ms( ++ READ_ONCE(render_user_hard_ms), ++ READ_ONCE(render_user_obs_ms)); ++ else ++ a->job_timeout_ms = G5P_RU_V3_CAP_MS; ++ }; ++ a->retained_results = G5P_RU_RESULTS; ++ return 0; ++} ++ ++/* VM_BIND hooks for the context resident on a VM (serial compute or render). */ ++static int g5p_resident_idle(struct g5p_context *c, bool live) ++{ ++ return g5p_render_vm_idle(c, live); ++} ++ ++/* VM_BIND UNMAP, file lock held, PTEs already cleared and the tag published: a render ++ * context with queued jobs keeps the mapping's object (its pages and its pin) until those ++ * jobs have retired (g5p_render_reap()). False: free it now. ++ */ ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m) ++{ ++ struct g5p_render *r = c->render; ++ unsigned long flags; ++ ++ if (!r || !atomic_read(&r->queued)) ++ return false; ++ m->retire = r->fence_seqno; ++ spin_lock_irqsave(&r->zlock, flags); ++ list_add_tail(&m->link, &r->zombies); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ return true; ++} ++ ++static int g5p_resident_invalidate(struct g5p_context *c) ++{ ++ return g5p_render_invalidate(c); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-submit.inc b/drivers/gpu/drm/apple-g5p/drm-submit.inc +new file mode 100644 +index 000000000..f36998e4d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-submit.inc +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Asynchronous privileged probes; payloads and firmware VM are kernel-owned. */ ++struct g5p_fence { ++ struct dma_fence base; ++ /* Owned by dma_fence; independent of queue/status locks. */ ++ spinlock_t lock; ++ u32 state; ++ int error; ++}; ++ ++static const char *g5p_fence_driver_name(struct dma_fence *f) ++{ ++ return "apple_g5p"; ++} ++ ++static const char *g5p_fence_timeline_name(struct dma_fence *f) ++{ ++ return "g5p-probe"; ++} ++ ++/* Default release uses dma_fence_free()/RCU; base is the first member. */ ++static const struct dma_fence_ops g5p_fence_ops = { ++ .get_driver_name = g5p_fence_driver_name, ++ .get_timeline_name = g5p_fence_timeline_name, ++}; ++ ++/* recovery108: the compute/probe submitters share the render path's admission window. */ ++ ++/* Legacy JOB_COMPUTE remains unsupported: v1 lacks a bounded stream length. */ +diff --git a/drivers/gpu/drm/apple-g5p/drm-vm.inc b/drivers/gpu/drm/apple-g5p/drm-vm.inc +new file mode 100644 +index 000000000..e72cacecf +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-vm.inc +@@ -0,0 +1,914 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* 16 KiB VM preparation; accepted user VMs freeze until reboot. */ ++ ++/* release8 memory accounting: every live G5P GEM object (native and imported) with the task ++ * that created it. One line per context release and per file close prints the totals; ++ * debugfs g5p_bos lists each object (size, references, handles, exported/imported, VM pins, ++ * creator) and each open file's handles, so a page held after its client exited is named. ++ */ ++static LIST_HEAD(g5p_bo_live); ++static DEFINE_SPINLOCK(g5p_bo_lock); ++static u32 g5p_bo_count; ++static u64 g5p_bo_bytes; ++ ++static void g5p_gem_free(struct drm_gem_object *obj); ++ ++static const struct drm_gem_object_funcs g5p_gem_funcs = { ++ .free = g5p_gem_free, ++ .print_info = drm_gem_shmem_object_print_info, ++ .pin = drm_gem_shmem_object_pin, ++ .unpin = drm_gem_shmem_object_unpin, ++ .get_sg_table = drm_gem_shmem_object_get_sg_table, ++ .vmap = drm_gem_shmem_object_vmap, ++ .vunmap = drm_gem_shmem_object_vunmap, ++ .mmap = drm_gem_shmem_object_mmap, ++ .export = g5p_gem_prime_export, ++ .vm_ops = &drm_gem_shmem_vm_ops, ++}; ++ ++/* perf60 fault address (render_user_fault_read, drm-render-v3.inc): bit 0 reads the MMU fault ++ * banks at a TA/3D fault event, bit 1 also at a silent or timed-out phase; any nonzero value also ++ * records every VM edit (MAP, UNMAP, render-target create/destroy) in a device-wide ring, so the ++ * fault address can be named. The ring holds an opaque VM pointer, never dereferenced. ++ */ ++ ++/* faultrec: the bank read and its decode alone at a TA/3D fault event (no per-job PM lanes, no ++ * PM snapshots, no VM edit log): one licensed read per fault, at most 32 per boot. ++ */ ++ ++/* perf60-fillrate.md §11.3 cut 5, kernel half (render-user-sync.h): cache maintenance by range. */ ++static unsigned int render_user_sync_range = 3; /* release8 default */ ++module_param(render_user_sync_range, uint, 0600); ++MODULE_PARM_DESC( ++ render_user_sync_range, ++ "perf60 default0 (release8: whole-BO maintenance); bit0 BO_SYNC honours " ++ "offset/size; bit1 a job's ACCESS_CACHE clean/invalidate covers each resource's " ++ "BO range [offset, offset+size), page widened, instead of the whole BO (latched " ++ "per job at submit); BO_SYNC read live"); ++ ++/* CPU cache maintenance of [lo, hi) of a BO's DMA mapping, walked over its DMA segments. */ ++static void g5p_sync_sgt_range(struct device *dev, struct sg_table *sgt, u64 lo, ++ u64 hi, bool to_cpu) ++{ ++ struct scatterlist *sg; ++ u64 pos = 0, start, n; ++ unsigned int i; ++ ++ for_each_sgtable_dma_sg(sgt, sg, i) { ++ if (pos >= hi) ++ break; ++ if (g5p_sync_clip(pos, sg_dma_len(sg), lo, hi, &start, &n)) { ++ if (to_cpu) ++ dma_sync_single_for_cpu( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_single_for_device( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ } ++ pos += sg_dma_len(sg); ++ } ++} ++ ++#define G5P_VMLOG_N 1024U ++enum { G5P_VMLOG_MAP = 1, G5P_VMLOG_UNMAP, G5P_VMLOG_RT_NEW, G5P_VMLOG_RT_DEL }; ++static const char *const g5p_vmlog_ops[] = { "?", "map", "unmap", "rt-new", ++ "rt-del" }; ++ ++struct g5p_vmlog_ent { ++ u64 ns, va, size, offset; ++ const void *vm; ++ u32 op, handle, vm_id; ++ pid_t tgid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++static struct { ++ spinlock_t lock; /* head and the entries */ ++ u32 head; ++ struct g5p_vmlog_ent e[G5P_VMLOG_N]; ++} g5p_vmlog = { .lock = __SPIN_LOCK_UNLOCKED(g5p_vmlog.lock) }; ++ ++static void g5p_vmlog_add(u32 op, const void *vm, u32 vm_id, u64 va, u64 size, ++ u32 handle, u64 offset) ++{ ++ { ++ return; ++ } ++} ++ ++/* Latency123: leaf attributes of new user mappings (0 = Apple's CC | SLC bypass). */ ++ ++/* Stage 3: the device-wide pinned-memory budget. */ ++static unsigned int render_pinned_mib = 1024; ++module_param(render_pinned_mib, uint, 0600); ++MODULE_PARM_DESC( ++ render_pinned_mib, ++ "MiB of BO memory all GPU VMs together may map; an object mapped in several VMs " ++ "or files (PRIME) counts once (default 1024)"); ++ ++static u32 g5p_pte_attrs(void) ++{ ++ static const u32 attrs[] = { 0xc, 0x4, 0x8, 0x0 }; ++ ++ return attrs[min_t(u32, (0), ARRAY_SIZE(attrs) - 1)]; ++} ++ ++static struct drm_gem_object *g5p_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct g5p_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); ++ ++ if (!bo) ++ return ERR_PTR(-ENOMEM); ++ bo->shmem.base.funcs = &g5p_gem_funcs; ++ INIT_LIST_HEAD(&bo->live); ++ return &bo->shmem.base; ++} ++ ++/* A fully created object enters the live list (an object the core frees on a failed init ++ * never does). ++ */ ++static void g5p_bo_track(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ bo->pid = current->tgid; ++ strscpy(bo->comm, current->comm, sizeof(bo->comm)); ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_add_tail(&bo->live, &g5p_bo_live); ++ g5p_bo_count++; ++ g5p_bo_bytes += obj->size; ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++} ++ ++static void g5p_gem_free(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ if (!list_empty(&bo->live)) { ++ list_del_init(&bo->live); ++ g5p_bo_count--; ++ g5p_bo_bytes -= obj->size; ++ } ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ drm_gem_shmem_object_free(obj); ++} ++ ++static int g5p_bos_show(struct seq_file *m, void *data) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct drm_device *drm = entry->dev; ++ u32 exported = 0, imported = 0, nohandle = 0; ++ u64 nohandle_bytes = 0; ++ struct drm_file *file; ++ struct g5p_bo *bo; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_for_each_entry(bo, &g5p_bo_live, live) { ++ struct drm_gem_object *obj = &bo->shmem.base; ++ bool imp = drm_gem_is_imported(obj); ++ ++ exported += !!READ_ONCE(obj->dma_buf); ++ imported += imp; ++ if (!READ_ONCE(obj->handle_count)) { ++ nohandle++; ++ nohandle_bytes += obj->size; ++ } ++ seq_printf( ++ m, ++ "bo size=%zuK refs=%u handles=%u exported=%c imported=%c vm_pins=%u by=%s[%d]\n", ++ obj->size >> 10, kref_read(&obj->refcount), ++ READ_ONCE(obj->handle_count), ++ READ_ONCE(obj->dma_buf) ? 'y' : 'n', imp ? 'y' : 'n', ++ bo->vm_pins, bo->comm, bo->pid); ++ } ++ seq_printf( ++ m, ++ "live=%u bytes=%lluK exported=%u imported=%u no_handle=%u/%lluK device_pinned=%lluK\n", ++ g5p_bo_count, g5p_bo_bytes >> 10, exported, imported, nohandle, ++ nohandle_bytes >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ mutex_lock(&drm->filelist_mutex); ++ list_for_each_entry(file, &drm->filelist, lhead) { ++ struct drm_gem_object *obj; ++ struct task_struct *task; ++ u64 bytes = 0; ++ u32 handles = 0; ++ struct pid *pid; ++ int id; ++ ++ spin_lock(&file->table_lock); ++ idr_for_each_entry(&file->object_idr, obj, id) { ++ handles++; ++ bytes += obj->size; ++ } ++ spin_unlock(&file->table_lock); ++ rcu_read_lock(); ++ pid = rcu_dereference(file->pid); ++ task = pid_task(pid, PIDTYPE_TGID); ++ seq_printf(m, "file %s[%d] handles=%u bytes=%lluK\n", ++ task ? task->comm : "?", pid_vnr(pid), handles, ++ bytes >> 10); ++ rcu_read_unlock(); ++ } ++ mutex_unlock(&drm->filelist_mutex); ++ return 0; ++} ++ ++static int g5p_create_bo(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_bo *a = data; ++ struct drm_gem_shmem_object *shmem; ++ int ret; ++ ++ if (!a->size || a->size > G5P_MAX_BO_SIZE || ++ (a->size & (G5P_GPU_PAGE - 1)) || a->handle || !a->flags || ++ (a->flags & ++ ~(DRM_G5P_BO_READ | DRM_G5P_BO_WRITE | DRM_G5P_BO_EXEC))) ++ return -EINVAL; ++ shmem = drm_gem_shmem_create(drm, a->size); ++ if (IS_ERR(shmem)) ++ return PTR_ERR(shmem); ++ /* Match the DMA API's coherent/noncoherent CPU mapping choice. The GPU ++ * cache contract remains an explicit COMPUTE capability prerequisite. ++ */ ++ shmem->map_wc = !dev_is_dma_coherent(drm->dev); ++ to_g5p_bo(&shmem->base)->roles = a->flags; ++ g5p_bo_track(&shmem->base); ++ ret = drm_gem_handle_create(file, &shmem->base, &a->handle); ++ drm_gem_object_put(&shmem->base); ++ return ret; ++} ++ ++static int g5p_bo_mmap_offset(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_mmap_offset *a = data; ++ struct drm_gem_object *obj; ++ int ret; ++ ++ if (a->flags || a->offset) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ ret = drm_gem_create_mmap_offset(obj); ++ if (!ret) ++ a->offset = drm_vma_node_offset_addr(&obj->vma_node); ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_bo_sync(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_sync *a = data; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *ctx; ++ unsigned long id; ++ int ret = 0; ++ ++ if (a->op != DRM_G5P_BO_SYNC_TO_DEVICE && ++ a->op != DRM_G5P_BO_SYNC_TO_CPU) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ mutex_lock(&f->lock); ++ xa_for_each(&f->contexts, id, ctx) ++ if (ctx->serial && ++ (ctx->bad || smp_load_acquire(&ctx->pending))) { ++ mutex_unlock(&f->lock); ++ ret = -EBUSY; ++ goto out; ++ } ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ mutex_unlock(&f->lock); ++ goto out; ++ } ++ /* render_user_sync_range bit 0: the requested range; else the whole BO (stronger, allowed). */ ++ if (READ_ONCE(render_user_sync_range) & 1) { ++ u64 lo, hi; ++ ++ if (g5p_sync_window(a->offset, a->size, obj->size, &lo, &hi)) ++ g5p_sync_sgt_range(drm->dev, sgt, lo, hi, ++ a->op == DRM_G5P_BO_SYNC_TO_CPU); ++ } else if (a->op == DRM_G5P_BO_SYNC_TO_DEVICE) { ++ dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } else { ++ dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ mutex_unlock(&f->lock); ++out: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_alloc_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ p->cpu = dma_alloc_coherent(dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ if (!g5p_user_pa_valid(p->dma) || dma_to_phys(dev, p->dma) != p->dma) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ return 0; ++} ++ ++static struct g5p_pt *g5p_vm_table(struct g5p_vm *vm, struct device *dev, ++ u32 level, u32 key, bool create) ++{ ++ struct g5p_pt *pt; ++ int ret; ++ ++ list_for_each_entry(pt, &vm->tables, link) ++ if (pt->level == level && pt->key == key) ++ return pt; ++ if (!create) ++ return NULL; ++ if (vm->table_count >= G5P_MAX_PT_PAGES) ++ return ERR_PTR(-ENOSPC); ++ pt = kzalloc(sizeof(*pt), GFP_KERNEL); ++ if (!pt) ++ return ERR_PTR(-ENOMEM); ++ ret = g5p_vm_alloc_page(dev, &pt->page); ++ if (ret) { ++ kfree(pt); ++ return ERR_PTR(ret); ++ } ++ pt->level = level; ++ pt->key = key; ++ list_add_tail(&pt->link, &vm->tables); ++ vm->table_count++; ++ return pt; ++} ++ ++static u8 *g5p_vm_pte(struct g5p_vm *vm, struct device *dev, u64 va, ++ bool create) ++{ ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ struct g5p_pt *dir, *leaf; ++ ++ dir = g5p_vm_table(vm, dev, 1, ci, create); ++ if (IS_ERR_OR_NULL(dir)) ++ return (void *)dir; ++ leaf = g5p_vm_table(vm, dev, 2, (ci << 9) | di, create); ++ if (IS_ERR_OR_NULL(leaf)) ++ return (void *)leaf; ++ if (create) { ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + ci * 4, ++ g5p_catalogue_entry(dir->page.dma), 0); ++ g5p_vm_entry(&vm->mutation, dir->page.cpu + di * 8, ++ g5p_directory_entry(leaf->page.dma), 1); ++ } ++ return leaf->page.cpu + ((va >> 14) & 127) * 8; ++} ++ ++static void g5p_vm_clear(struct g5p_vm *vm, struct device *dev, u64 va, ++ u64 size) ++{ ++ for (u64 off = 0; off < size; off += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, dev, va + off, false); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, 0, 2); ++ } ++ dma_wmb(); ++} ++ ++static bool g5p_vm_table_empty(struct g5p_pt *pt) ++{ ++ u32 count = pt->level == 2 ? 128 : 512; ++ ++ for (u32 i = 0; i < count; i++) ++ if (g5p_le32(pt->page.cpu + i * 8) & 1) ++ return false; ++ return true; ++} ++ ++static void g5p_vm_prune(struct device *dev, struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt, *tmp, *dir; ++ ++ /* These tables are mutable only before firmware attachment. ++ * Remove empty leaves before directories, including failed MAP rollback. ++ */ ++ for (u32 level = 2; level; level--) ++ list_for_each_entry_safe(pt, tmp, &vm->tables, link) { ++ if (pt->level != level || !g5p_vm_table_empty(pt)) ++ continue; ++ if (level == 2) { ++ dir = g5p_vm_table(vm, dev, 1, pt->key >> 9, ++ false); ++ if (dir) ++ g5p_vm_entry(&vm->mutation, ++ dir->page.cpu + (pt->key & ++ 511) * 8, ++ 0, 1); ++ } else { ++ g5p_vm_entry(&vm->mutation, ++ vm->root.cpu + pt->key * 4, 0, 0); ++ } ++ dma_wmb(); ++ list_del(&pt->link); ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ vm->table_count--; ++ kfree(pt); ++ } ++} ++ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj); ++ ++static void g5p_vm_release(struct kref *ref) ++{ ++ struct g5p_vm *vm = container_of(ref, struct g5p_vm, ref); ++ struct device *dev = vm->dev; ++ struct g5p_mapping *m, *mtmp; ++ struct g5p_pt *pt, *ptmp; ++ u64 pinned = vm->pinned_bytes; ++ u32 maps = vm->map_count, tables = vm->table_count; ++ ++ list_for_each_entry_safe(m, mtmp, &vm->maps, link) { ++ drm_mm_remove_node(&m->node); ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ list_del(&m->link); ++ kfree(m); ++ } ++ list_for_each_entry_safe(pt, ptmp, &vm->tables, link) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ list_del(&pt->link); ++ kfree(pt); ++ } ++ dma_free_coherent(dev, G5P_GPU_PAGE, vm->root.cpu, vm->root.dma); ++ xa_destroy(&vm->objs); ++ drm_mm_remove_node(&vm->reserved); ++ drm_mm_takedown(&vm->mm); ++ /* One line per freed VM: the device's pinned total must fall back after a client exits. */ ++ dev_info( ++ dev, ++ "G5P VM release maps=%u tables=%u pinned=%lluK device_pinned=%lluK\n", ++ maps, tables, pinned >> 10, ++ READ_ONCE(vm->gpu->pinned_bytes) >> 10); ++ kfree(vm); ++} ++ ++static void g5p_vm_put(struct g5p_vm *vm) ++{ ++ kref_put(&vm->ref, g5p_vm_release); ++} ++ ++static void g5p_context_release(struct kref *ref) ++{ ++ struct g5p_context *c = container_of(ref, struct g5p_context, ref); ++ g5p_render_release(c); ++ g5p_vm_put(c->vm); ++ kfree(c); ++} ++ ++static void g5p_context_put(struct g5p_context *c) ++{ ++ kref_put(&c->ref, g5p_context_release); ++} ++ ++static int g5p_create_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ u32 id; ++ int ret; ++ ++ if (a->flags || a->handle) ++ return -EINVAL; ++ vm = kzalloc(sizeof(*vm), GFP_KERNEL); ++ if (!vm) ++ return -ENOMEM; ++ kref_init(&vm->ref); ++ vm->dev = drm->dev; ++ vm->gpu = to_g5p(drm); ++ vm->bounded = (0); ++ vm->mutation.generations = true; ++ INIT_LIST_HEAD(&vm->maps); ++ xa_init(&vm->objs); ++ INIT_LIST_HEAD(&vm->tables); ++ drm_mm_init(&vm->mm, G5P_USER_VA_START, ++ G5P_USER_VA_END - G5P_USER_VA_START); ++ vm->reserved.start = G5P_USER_RESERVED_START; ++ vm->reserved.size = G5P_USER_RESERVED_END - G5P_USER_RESERVED_START; ++ ret = drm_mm_reserve_node(&vm->mm, &vm->reserved); ++ if (ret) ++ goto err_mm; ++ ret = g5p_vm_alloc_page(drm->dev, &vm->root); ++ if (ret) ++ goto err_reserved; ++ vm->table_count = 1; ++ mutex_lock(&f->lock); ++ ret = xa_alloc(&f->vms, &id, vm, XA_LIMIT(1, G5P_MAX_VMS), GFP_KERNEL); ++ mutex_unlock(&f->lock); ++ if (ret) { ++ g5p_vm_put(vm); ++ return ret; ++ } ++ a->handle = id; ++ return 0; ++err_reserved: ++ drm_mm_remove_node(&vm->reserved); ++err_mm: ++ drm_mm_takedown(&vm->mm); ++ kfree(vm); ++ return ret; ++} ++ ++static int g5p_destroy_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->handle); ++ if (!vm) ++ ret = -ENOENT; ++ else if (vm->contexts || f->submitted || vm->frozen) ++ ret = -EBUSY; ++ else ++ xa_erase(&f->vms, a->handle); ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_vm_put(vm); ++ return ret; ++} ++ ++static unsigned long g5p_vm_obj_count(struct g5p_vm *vm, ++ struct drm_gem_object *obj) ++{ ++ void *v = xa_load(&vm->objs, (unsigned long)obj); ++ ++ return v ? xa_to_value(v) : 0; ++} ++ ++/* ++ * One more mapping of `obj` in `vm`. Its first one in this VM counts its size against the ++ * VM's pin limit, and its first one in any VM against the device budget (render_pinned_mib). ++ * Our own dma-bufs import back as the same GEM object, in any file, so memory shared between ++ * clients (a client's buffer in the compositor's VM) counts once. ++ */ ++static int g5p_vm_obj_get(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ u64 limit = (u64)READ_ONCE(render_pinned_mib) << 20; ++ unsigned long count; ++ int ret = 0; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (!count && obj->size > G5P_MAX_PINNED_BYTES - vm->pinned_bytes) ++ ret = -ENOSPC; ++ else if (!count && !bo->vm_pins && ++ obj->size > limit - min(limit, d->pinned_bytes)) ++ ret = -ENOSPC; ++ if (!ret) ++ ret = xa_err(xa_store(&vm->objs, (unsigned long)obj, ++ xa_mk_value(count + 1), GFP_KERNEL)); ++ if (!ret && !count) { ++ vm->pinned_bytes += obj->size; ++ if (!bo->vm_pins++) ++ d->pinned_bytes += obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++ return ret; ++} ++ ++/* One mapping of `obj` in `vm` less: g5p_vm_obj_get() undone. Also called by the render ++ * scheduler for an unmap deferred behind queued jobs, hence the lock around the counts. ++ */ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long count; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (count > 1) { ++ /* replaces an existing entry: no allocation */ ++ xa_store(&vm->objs, (unsigned long)obj, xa_mk_value(count - 1), ++ GFP_KERNEL); ++ } else if (!WARN_ON(!count)) { ++ xa_erase(&vm->objs, (unsigned long)obj); ++ vm->pinned_bytes -= obj->size; ++ if (!--bo->vm_pins) ++ d->pinned_bytes -= obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++} ++ ++static int g5p_vm_map_bo(struct drm_device *drm, struct g5p_vm *vm, ++ struct drm_file *file, struct drm_g5p_vm_bind *a) ++{ ++ struct drm_gem_object *obj; ++ struct drm_gem_shmem_object *shmem; ++ struct sg_table *sgt; ++ struct sg_dma_page_iter iter; ++ struct g5p_mapping *m; ++ u64 *addresses, done = 0; ++ u32 count = 0; ++ bool imported; ++ int ret; ++ ++ if (vm->map_count >= G5P_MAX_MAPS || ++ a->size > G5P_MAX_VM_BYTES - vm->mapped_bytes) ++ return -ENOSPC; ++ obj = drm_gem_object_lookup(file, a->bo); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto err_obj; ++ } ++ ret = g5p_vm_obj_get(vm, obj); ++ if (ret) ++ goto err_obj; ++ addresses = kvmalloc_array(obj->size / G5P_GPU_PAGE, sizeof(*addresses), ++ GFP_KERNEL); ++ m = kzalloc(sizeof(*m), GFP_KERNEL); ++ if (!addresses || !m) { ++ ret = -ENOMEM; ++ goto err_alloc; ++ } ++ m->node.start = a->va; ++ m->node.size = a->size; ++ ret = drm_mm_reserve_node(&vm->mm, &m->node); ++ if (ret) { ++ ret = -EINVAL; ++ goto err_alloc; ++ } ++ imported = drm_gem_is_imported(obj); ++ shmem = to_drm_gem_shmem_obj(obj); ++ /* PRIME import mapped this attachment for our device and owns it until ++ * object release. Re-locking the exporter's reservation here can return ++ * -EBUSY after the first V3 submission even though the render VM is idle. ++ */ ++ if (imported) { ++ sgt = READ_ONCE(shmem->sgt); ++ if (!sgt) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=prime-no-sgt vm=%u bo=%u bytes=%llu\n", ++ a->vm, a->bo, a->size); ++ ret = -ENXIO; ++ goto err_node; ++ } ++ } else { ++ sgt = drm_gem_shmem_get_pages_sgt(shmem); ++ } ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=sgt-error vm=%u bo=%u imported=%u err=%d\n", ++ a->vm, a->bo, imported, ret); ++ goto err_node; ++ } ++ for_each_sgtable_dma_page(sgt, &iter, 0) { ++ u64 dma = sg_page_iter_dma_address(&iter); ++ ++ if (count >= obj->size / G5P_GPU_PAGE || ++ !g5p_user_pa_valid(dma) || ++ dma_to_phys(drm->dev, dma) != dma || ++ (!imported && dma_to_phys(drm->dev, dma) != ++ page_to_phys(shmem->pages[count]))) { ++ ret = -ERANGE; ++ goto err_node; ++ } ++ addresses[count++] = dma; ++ } ++ if (count != obj->size / G5P_GPU_PAGE) { ++ ret = -EINVAL; ++ goto err_node; ++ } ++ for (; done < a->size; done += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, drm->dev, a->va + done, true); ++ ++ if (IS_ERR_OR_NULL(pte)) { ++ ret = pte ? PTR_ERR(pte) : -ENOMEM; ++ goto err_clear; ++ } ++ /* Recovered bypass bits; user permission encodings are not claimed. ++ * A render VM latched to r25 memory variant 4 uses flags 0. ++ */ ++ g5p_vm_entry( ++ &vm->mutation, pte, ++ g5p_leaf_entry( ++ addresses[(a->offset + done) / G5P_GPU_PAGE], ++ vm->zero_leaf_flags ? 0 : g5p_pte_attrs()), ++ 2); ++ } ++ dma_wmb(); ++ m->obj = obj; ++ m->offset = a->offset; ++ list_add_tail(&m->link, &vm->maps); ++ vm->mapped_bytes += a->size; ++ vm->map_count++; ++ kvfree(addresses); ++ return 0; ++err_clear: ++ g5p_vm_clear(vm, drm->dev, a->va, done); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++err_node: ++ drm_mm_remove_node(&m->node); ++err_alloc: ++ kvfree(addresses); ++ kfree(m); ++ g5p_vm_obj_put(vm, obj); ++err_obj: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_bind(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_vm_bind *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_mapping *m; ++ struct g5p_vm *vm; ++ int ret = -ENOENT; ++ bool resident = false, locked = false; ++ u32 generation[3]; ++ ++ if (a->flags || !g5p_user_range_valid(a->va, a->size) || ++ (a->op != DRM_G5P_VM_MAP && a->op != DRM_G5P_VM_UNMAP) || ++ (a->op == DRM_G5P_VM_UNMAP && (a->bo || a->offset))) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->vm); ++ if (!vm) ++ goto out; ++ /* Its render context is closing: the close item owns the tables now, and the context ++ * may be freed at any time, so it is not looked at. ++ */ ++ if (vm->resident_gone) { ++ ret = -EBUSY; ++ goto out; ++ } ++ if ((f->submitted || vm->frozen) && !vm->resident) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=no-resident submitted=%u frozen=%u vm=%u op=%u\n", ++ f->submitted, vm->frozen, a->vm, a->op); ++ ret = -EBUSY; ++ goto out; ++ } ++ if (vm->resident) { ++ ret = g5p_resident_idle(vm->resident, true); ++ if (ret) ++ goto out; ++ /* A render context's edit is RAM only and touches its own tables and ++ * slots, which the firmware reads only at that context's next kick; ++ * f->lock plus resident-idle orders it against its submit (a MAP or ++ * UNMAP may pass queued jobs, see g5p_render_vm_idle()). Other contexts' jobs ++ * may hold the core lock meanwhile, so it is not taken. Serialized ++ * compute still uses the global mutex. ++ */ ++ if (!vm->resident->render) { ++ mutex_lock(&to_g5p(drm)->core->lock); ++ locked = true; ++ } ++ resident = true; ++ memcpy(generation, vm->mutation.generation, sizeof(generation)); ++ } ++ if (a->op == DRM_G5P_VM_MAP) { ++ if (vm->bounded && !(vm->resident && vm->resident->render) && ++ g5p_user_private_overlap(a->va, a->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_vm_map_bo(drm, vm, file, a); ++ if (!ret) ++ g5p_vmlog_add(G5P_VMLOG_MAP, vm, a->vm, a->va, a->size, ++ a->bo, a->offset); ++ if (ret == -EBUSY) ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=map vm=%u bo=%u submitted=%u frozen=%u resident=%u render=%u\n", ++ a->vm, a->bo, f->submitted, vm->frozen, ++ !!vm->resident, ++ !!(vm->resident && vm->resident->render)); ++ goto out; ++ } ++ list_for_each_entry(m, &vm->maps, link) { ++ if (m->node.start != a->va || m->node.size != a->size) { ++ if (m->node.start < a->va + a->size && ++ a->va < m->node.start + m->node.size) { ++ ret = -EINVAL; ++ break; ++ } ++ continue; ++ } ++ g5p_vmlog_add(G5P_VMLOG_UNMAP, vm, a->vm, a->va, a->size, 0, 0); ++ g5p_vm_clear(vm, drm->dev, a->va, a->size); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++ else { ++ ret = g5p_resident_invalidate(vm->resident); ++ if (ret) ++ goto out; ++ memcpy(generation, vm->mutation.generation, ++ sizeof(generation)); ++ } ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ vm->mapped_bytes -= a->size; ++ /* Beside queued jobs the PTEs are gone, but the pages stay pinned until ++ * those jobs have retired. ++ */ ++ if (!vm->resident || !g5p_resident_defer(vm->resident, m)) { ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ ret = 0; ++ break; ++ } ++out: ++ if (resident) { ++ int invalidate = 0; ++ ++ if (memcmp(generation, vm->mutation.generation, ++ sizeof(generation))) ++ invalidate = g5p_resident_invalidate(vm->resident); ++ ++ if (invalidate) ++ ret = invalidate; ++ if (locked) ++ mutex_unlock(&to_g5p(drm)->core->lock); ++ } ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_create_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_context *a = data; ++ if (a->type != DRM_G5P_CONTEXT_RENDER) ++ return -EOPNOTSUPP; ++ return g5p_render_create_context(drm, file, a); ++} ++ ++static int g5p_destroy_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_context *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->handle); ++ if (!c) ++ ret = -ENOENT; ++ else ++ ret = g5p_render_destroy(c); ++ if (!ret && c) { ++ c->vm->contexts--; ++ xa_erase(&f->contexts, a->handle); ++ } ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_context_put(c); ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/render-user-validate.h b/drivers/gpu/drm/apple-g5p/render-user-validate.h +new file mode 100644 +index 000000000..d5ccfa9c9 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/render-user-validate.h +@@ -0,0 +1,380 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_VALIDATE_H ++#define G5P_RENDER_USER_VALIDATE_H ++ ++/* Pure TYPED_V1 checks over already-resolved resource ranges. The caller ++ * proves each range is inside a mapping of the listed BO, in the submitting ++ * context's VM, with BO roles covering `access`. Nothing here is a pointer ++ * the kernel dereferences; GPU addresses are only range-checked. ++ */ ++#define G5P_RU_ACCESS_READ 1U ++#define G5P_RU_ACCESS_WRITE 2U ++#define G5P_RU_MAX_VDM 0x10000U ++#define G5P_RU_MAX_SCRATCH (64ULL << 20) ++ ++/* Hardware facts GET_RENDER_INFO reports and the TYPED_V3 validator enforces; ++ * one definition so the two cannot drift. Code offsets are 32-bit from the ++ * common USC/PDS base. TA/3D reads at or above 2^38 stall silently on A10 ++ * (works at 0x3_, 0x10_ and 0x20_, stalls at 0x40_ and 0x80_, no MMU fault); ++ * not flown on A9, so the limit is a statement, not a proof (see ++ * docs/gpu/experiments/typed-v3-kernel-implementation.md). ++ */ ++#define G5P_RU_PDS_BASE 0x300000000ULL ++#define G5P_RU_CODE_WINDOW 0x100000000ULL /* 2^32 */ ++#define G5P_RU_HW_VA_LIMIT 0x4000000000ULL /* 2^38 */ ++ ++/* Not a requested access: the resolver sets it, for the typed profiles only, ++ * when the listed BO carries DRM_G5P_BO_EXEC. Only TYPED_V3 reads it; every ++ * other check masks on READ/WRITE. It lives in `access` rather than in a new ++ * field so positional g5p_ru_range initialisers keep compiling. ++ */ ++#define G5P_RU_ACCESS_EXEC 16U ++ ++struct g5p_ru_range { ++ u64 va, size; ++ u32 access; ++}; ++ ++/* [va, va + size) inside one listed range that grants every access bit. */ ++static inline bool g5p_ru_in(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 access) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if ((r[i].access & access) == access && va >= r[i].va && ++ va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) ++ return true; ++ return false; ++} ++ ++/* Everything TYPED_V1 and TYPED_V2 share: all fields except the pools. */ ++static inline int g5p_ru_pass_check_common(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || ++ !g5p_ru_in(r, n, p->vdm_va, p->vdm_size, G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ /* Copied to TA W88/R0+470; meaning unresolved (r23), so bounded. */ ++ if ((p->vdm_control & 0xf) || p->vdm_control > G5P_RU_MAX_VDM) ++ return -EINVAL; ++ if ((p->scratch_va & 0xf) || p->scratch_size < 0x10 || ++ p->scratch_size > G5P_RU_MAX_SCRATCH || ++ !g5p_ru_in(r, n, p->scratch_va, p->scratch_size, rw)) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((p->scratch_ptr[i] & 3) || ++ !g5p_ru_in(r, n, p->scratch_ptr[i], 4, rw)) ++ return -EINVAL; ++ /* Three-word descriptors keep their fourth slot zero. */ ++ if (p->ta_pds[3] || p->eot[3] || p->eot_resume[3]) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V1 and LINEAR64_V1: the two allocator pools are page-aligned RW. */ ++static inline int g5p_ru_pass_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret = g5p_ru_pass_check_common(p, r, n); ++ ++ if (ret) ++ return ret; ++ if (((p->pool7_va | p->pool7_size | p->pool8_va | p->pool8_size) & ++ (G5P_GPU_PAGE - 1)) || ++ !g5p_ru_in(r, n, p->pool7_va, p->pool7_size, rw) || ++ !g5p_ru_in(r, n, p->pool8_va, p->pool8_size, rw)) ++ return -EINVAL; ++ return 0; ++} ++ ++#define G5P_RU_MAX_TABLE 65536U ++ ++/* TYPED_V2: the pools are the ISP scissor (8-byte entries) and depth-bias ++ * (12-byte) tables, READ and word aligned; the V1 pool fields stay zero. ++ * Flagged ZLS words are addresses into listed RW ranges; the rest is opaque. ++ */ ++static inline int g5p_ru_pass2_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret; ++ ++ if (p->pool7_va || p->pool7_size || p->pool8_va || p->pool8_size) ++ return -EINVAL; ++ ret = g5p_ru_pass_check_common(p, r, n); ++ if (ret) ++ return ret; ++ if (!e->scissor_count || e->scissor_count > G5P_RU_MAX_TABLE || ++ (e->scissor_va & 3) || ++ !g5p_ru_in(r, n, e->scissor_va, e->scissor_count * 8ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if (!e->dbias_count || e->dbias_count > G5P_RU_MAX_TABLE || ++ (e->dbias_va & 3) || ++ !g5p_ru_in(r, n, e->dbias_va, e->dbias_count * 12ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if ((e->pixel_width != 2 && e->pixel_width != 4 && ++ e->pixel_width != 8) || ++ (e->bgobj_vals & ~0x3ffU) || (e->zls_mask & ~0x3fU)) ++ return -EINVAL; ++ if (!!e->occlusion_query_va != !!e->occlusion_query_size || ++ (e->occlusion_query_va && ++ ((e->occlusion_query_va & 0xf) || e->occlusion_query_size < sizeof(u32) || ++ !g5p_ru_in(r, n, e->occlusion_query_va, e->occlusion_query_size, rw)))) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((e->zls_mask & (1U << i)) && ++ ((e->zls[i] >> 40) || ++ !g5p_ru_in(r, n, e->zls[i] & ~0xfULL, 16, rw))) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V3: the TA PDS program (W+0x88 -> R0+0x470) is an explicit reference ++ * and the VDM stream length is a separate field that never reaches W. ++ * Contract and evidence: docs/gpu/experiments/typed-v3-kernel-implementation.md. ++ * Everything here is rejected, never masked. Structural rules need only the ++ * resolved ranges; the two content rules take bytes the caller read through ++ * the BOs' own pages (g5p_ru_pass3_content()). The rules are a subset of what ++ * the userspace lane accepts (tools/mesa-a9 a9c_ctx_lint(), a9scene): nothing ++ * here is stricter than a client that passes those lints. ++ * -EINVAL stale vdm_control, malformed program/stream/descriptor fields, ++ * no terminator, or (V1 dialect) the V1 rules fail ++ * -ERANGE outside [pds_base, pds_base + code_window) or past hw_va_limit ++ * -ENOENT no listed range wholly covers the program, stream or task code ++ * -EACCES missing READ, or the program/task code range's BO lacks EXEC ++ * -ENODATA the first 16 program bytes are all zero ++ */ ++#define G5P_RU_PROGRAM_ALIGN 16U ++#define G5P_RU_PROGRAM_MIN 16U ++ ++struct g5p_ru_v3_limits { ++ u64 pds_base, code_window; ++ u64 hw_va_limit; /* 0: no statement */ ++}; ++ ++/* Where the content rules read: indexes into the ranges array. */ ++struct g5p_ru_v3_loc { ++ u64 program_off; /* the exact W+0x88 value */ ++ u32 program_range, vdm_range; ++ u32 pixel_width; /* the pass's output width, 2, 4 or 8, either dialect */ ++ bool dialect2; /* ext was non-zero: V2 3D words; else the V1 dialect */ ++}; ++ ++/* TYPED_V3 carries the V2 ext struct. An all-zero ext selects the V1 dialect ++ * (pools in base, no tables, V1 3D words): the pass tools/mesa-a9 records ++ * today, plus the explicit program. Any non-zero ext field selects V2. ++ */ ++static inline bool g5p_ru_pass2_zero(const struct g5p_ru_pass2 *e) ++{ ++ u64 any = e->scissor_va | e->dbias_va | e->background_texture | ++ e->occlusion_query_va | e->occlusion_query_size; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(e->zls); i++) ++ any |= e->zls[i]; ++ any |= e->scissor_count | e->dbias_count | e->bgobj_depth | ++ e->bgobj_vals | e->pixel_width | e->zls_mask; ++ return !any; ++} ++ ++/* First listed range that wholly contains [va, va + size). */ ++static inline bool g5p_ru_cover(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 *idx) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if (va >= r[i].va && va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) { ++ *idx = i; ++ return true; ++ } ++ return false; ++} ++ ++/* [va, va + size) ends at or below the limit; zero limit makes no statement. */ ++static inline bool g5p_ru_below(u64 limit, u64 va, u64 size) ++{ ++ return !limit || (size <= limit && va <= limit - size); ++} ++ ++/* One PDS task reference of a store/resume descriptor: a 32-bit offset from ++ * pds_base, 16-byte aligned like every captured value and W+0x88's mask, ++ * non-zero, inside the code window and the hardware limit, and starting in a ++ * listed READ range (`code`: whose BO also carries EXEC). ++ */ ++static inline int g5p_ru_v3_ref(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, u32 off, ++ bool code) ++{ ++ u64 va; ++ u32 i; ++ ++ if (!off || (off % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (lim->code_window < G5P_RU_PROGRAM_MIN || ++ off > lim->code_window - G5P_RU_PROGRAM_MIN || ++ lim->pds_base > ~0ULL - off) ++ return -ERANGE; ++ va = lim->pds_base + off; ++ if (!g5p_ru_below(lim->hw_va_limit, va, G5P_RU_PROGRAM_MIN)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, va, G5P_RU_PROGRAM_MIN, &i)) ++ return -ENOENT; ++ if (!(r[i].access & G5P_RU_ACCESS_READ) || ++ (code && !(r[i].access & G5P_RU_ACCESS_EXEC))) ++ return -EACCES; ++ return 0; ++} ++ ++/* The TYPED_V3 pixel output width. The V1 dialect takes it from the V3 field ++ * (0 = 2, the V1 dialect exactly), the V2 dialect from ext, where the V3 field ++ * must stay zero. A width above 2 is several colour targets, and a partial ++ * render must then resume with the pass's own EOT: a one-target resume EOT ++ * would store only the first target. The EOT itself is not decoded. ++ */ ++static inline int g5p_ru_pass3_width(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ bool dialect2, u32 *width) ++{ ++ u32 w = v->pixel_width; ++ ++ if (v->pad || (w && w != 2 && w != 4 && w != 8) || (dialect2 && w)) ++ return -EINVAL; ++ if (dialect2) ++ w = e->pixel_width; /* 2, 4 or 8: g5p_ru_pass2_check() */ ++ else if (!w) ++ w = 2; ++ if (w > 2 && memcmp(p->eot, p->eot_resume, sizeof(p->eot))) ++ return -EINVAL; ++ *width = w; ++ return 0; ++} ++ ++/* The V3 rules, then the TYPED_V1 rules (all-zero ext) or the TYPED_V2 rules ++ * (any ext field set; pools zero), then the output width. base.vdm_control is ++ * checked first. On success loc names the ranges to read, the dialect and the ++ * width. ++ */ ++static inline int g5p_ru_pass3_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_ru_v3_loc *loc) ++{ ++ u32 pi, vi; ++ u64 off; ++ int ret; ++ ++ if (p->vdm_control) ++ return -EINVAL; /* a stream length here is the r36/r88 defect */ ++ if (v->ta_program_size < G5P_RU_PROGRAM_MIN || ++ v->ta_program_size > DRM_G5P_RENDER_MAX_PROGRAM_BYTES || ++ (v->ta_program_size % G5P_RU_PROGRAM_ALIGN) || ++ (v->ta_program_va % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || v->vdm_end < 4 || ++ v->vdm_end > p->vdm_size || (v->vdm_end & 3)) ++ return -EINVAL; ++ if (v->ta_program_va < lim->pds_base) ++ return -ERANGE; ++ off = v->ta_program_va - lim->pds_base; ++ if (lim->code_window < v->ta_program_size || ++ off > lim->code_window - v->ta_program_size) ++ return -ERANGE; ++ if (!g5p_ru_below(lim->hw_va_limit, v->ta_program_va, ++ v->ta_program_size) || ++ !g5p_ru_below(lim->hw_va_limit, p->vdm_va, p->vdm_size)) ++ return -ERANGE; ++ /* Every listed range, not only the two above: on A10 a buffer moved past ++ * the limit stalls the TA without a fault, and all buffers moved together. ++ */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, v->ta_program_va, v->ta_program_size, &pi)) ++ return -ENOENT; ++ if (!(r[pi].access & G5P_RU_ACCESS_READ) || ++ !(r[pi].access & G5P_RU_ACCESS_EXEC)) ++ return -EACCES; ++ if (!g5p_ru_cover(r, n, p->vdm_va, p->vdm_size, &vi)) ++ return -ENOENT; ++ if (!(r[vi].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ /* Context store [0] / load [1]: code at word 2 (READ and EXEC), STREAM_OUT2 ++ * at word 5 (READ). Resume tasks: word 2 (READ only: R0+0xf060/0xf070, ++ * named stream-out by the userspace lane and aux program by r92 2.4, so ++ * its role is unresolved and EXEC is not demanded). Words 0, 1, 3 and 4 ++ * stay opaque. The spacing (store - program = 0x280, resume - store = ++ * 0x340) and a program that equals a task entry (r63 flew the store ++ * program; the role is not detectable structurally) are not enforced. ++ */ ++ for (u32 i = 0; i < 2; i++) { ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][2], true); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][5], ++ false); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->resume_pds[i][2], ++ false); ++ if (ret) ++ return ret; ++ } ++ loc->dialect2 = !g5p_ru_pass2_zero(e); ++ ret = loc->dialect2 ? g5p_ru_pass2_check(p, e, r, n) : ++ g5p_ru_pass_check(p, r, n); ++ if (!ret) ++ ret = g5p_ru_pass3_width(p, e, v, loc->dialect2, ++ &loc->pixel_width); ++ if (ret) ++ return ret; ++ loc->program_off = off; ++ loc->program_range = pi; ++ loc->vdm_range = vi; ++ return 0; ++} ++ ++/* The first 16 program bytes are not all zero (r88 published 0x370, which ++ * pointed at zeros) and the last stream dword is the terminator. `head` is ++ * G5P_RU_PROGRAM_MIN bytes, `tail` the 4 stream bytes at vdm_va + vdm_end - 4. ++ */ ++static inline int g5p_ru_pass3_content(const u8 *head, const u8 *tail) ++{ ++ u8 any = 0; ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ for (u32 i = 0; i < G5P_RU_PROGRAM_MIN; i++) ++ any |= head[i]; ++ if (!any) ++ return -ENODATA; ++ if (word != DRM_G5P_RENDER_STREAM_TERMINATE) ++ return -EINVAL; ++ return 0; ++} ++ ++/* AGXRenderTargetMemoryG5 output sanity for userspace geometry: exact one ++ * sample and the sizes the kernel will allocate. Opaque fields unchecked. ++ */ ++static inline int g5p_ru_geometry_check(const struct g5p_ru_rt *rt, u32 width, ++ u32 height, ++ struct g5p_ru_rt_sizes *sizes) ++{ ++ if (!width || !height || width > 16384 || height > 16384) ++ return -EINVAL; ++ if (rt->tiles > ((width + 31) / 32) * ((height + 31) / 32)) ++ return -EINVAL; ++ return g5p_ru_rt_sizes(rt, sizes); ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/vm-ranges.h b/drivers/gpu/drm/apple-g5p/vm-ranges.h +new file mode 100644 +index 000000000..25bd98ee9 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/vm-ranges.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_H ++#define G5P_COMPUTE_USER_H ++ ++/* Public geometry is fixed across VMs. The private ranges never alias a BO. */ ++ ++static const u64 g5p_private_va[] = { ++ 0x300040000ULL, 0x300014000ULL, 0x300028000ULL, 0x300044000ULL, 0, ++ 0x100080000ULL, 0x100084000ULL, 0x1000a0000ULL, 0x1000a4000ULL, ++}; ++ ++static inline bool g5p_user_private_overlap(u64 va, u64 size) ++{ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_private_va); i++) ++ if (g5p_private_va[i] && va < g5p_private_va[i] + 0x4000 && ++ g5p_private_va[i] < va + size) ++ return true; ++ return false; ++} ++ ++#endif +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index 9b44c78cd..bb959fe40 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -835,6 +835,26 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { + .vunmap = drm_gem_dmabuf_vunmap, + }; + ++/** ++ * drm_gem_dma_buf_is_cpu_uncached - query a trusted native GEM exporter ++ * @dma_buf: exported buffer whose supported CPU aliases are queried ++ * ++ * Returns true only for the explicit native exporter contract. Non-GEM, ++ * re-exported foreign objects and unspecified exporters remain conservative. ++ */ ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf) ++{ ++ struct drm_gem_object *obj; ++ ++ if (!dma_buf || dma_buf->ops != &drm_gem_prime_dmabuf_ops) ++ return false; ++ obj = dma_buf->priv; ++ return !drm_gem_is_imported(obj) && obj->funcs && ++ obj->funcs->is_cpu_uncached && obj->funcs->is_cpu_uncached(obj); ++} ++EXPORT_SYMBOL_GPL(drm_gem_dma_buf_is_cpu_uncached); ++ ++ + /** + * drm_prime_pages_to_sg - converts a page array into an sg list + * @dev: DRM device +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 885244e37..048193377 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -73,6 +73,14 @@ enum drm_gem_object_status { + * struct drm_gem_object_funcs - GEM object functions + */ + struct drm_gem_object_funcs { ++ /** ++ * @is_cpu_uncached: Optional exporter contract. True only when every ++ * supported CPU mapping (kernel and userspace) is uncached or WC and ++ * no cacheable alias exists. Native allocation properties must be ++ * immutable; imported memory must not inherit this promise. Default ++ * NULL is conservative. This does not remove DMA ordering barriers. ++ */ ++ bool (*is_cpu_uncached)(struct drm_gem_object *obj); + /** + * @free: + * +diff --git a/include/drm/drm_gem_dma_helper.h b/include/drm/drm_gem_dma_helper.h +index f2678e7ec..309e86817 100644 +--- a/include/drm/drm_gem_dma_helper.h ++++ b/include/drm/drm_gem_dma_helper.h +@@ -2,6 +2,8 @@ + #ifndef __DRM_GEM_DMA_HELPER_H__ + #define __DRM_GEM_DMA_HELPER_H__ + ++#include ++#include + #include + #include + #include +@@ -133,6 +135,16 @@ static inline int drm_gem_dma_object_mmap(struct drm_gem_object *obj, struct vm_ + return drm_gem_dma_mmap(dma_obj, vma); + } + ++/* Native DMA allocations use dma_alloc_wc and WC mmap/vmap. Noncoherent ++ * allocations and foreign imports cannot make this promise. ++ */ ++static inline bool drm_gem_dma_object_is_cpu_uncached(struct drm_gem_object *obj) ++{ ++ return !drm_gem_is_imported(obj) && ++ !to_drm_gem_dma_obj(obj)->map_noncoherent && ++ !dev_is_dma_coherent(drm_dev_dma_dev(obj->dev)); ++} ++ + /* + * Driver ops + */ +diff --git a/include/drm/drm_prime.h b/include/drm/drm_prime.h +index f50f862f0..dbd4ce818 100644 +--- a/include/drm/drm_prime.h ++++ b/include/drm/drm_prime.h +@@ -99,6 +99,8 @@ struct dma_buf *drm_gem_prime_export(struct drm_gem_object *obj, + + unsigned long drm_prime_get_contiguous_size(struct sg_table *sgt); + ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf); ++ + /* helper functions for importing */ + bool drm_gem_is_prime_exported_dma_buf(struct drm_device *dev, + struct dma_buf *dma_buf); +diff --git a/include/uapi/drm/apple_g5p_drm.h b/include/uapi/drm/apple_g5p_drm.h +new file mode 100644 +index 000000000..5e41ab533 +--- /dev/null ++++ b/include/uapi/drm/apple_g5p_drm.h +@@ -0,0 +1,631 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++#ifndef _UAPI_APPLE_G5P_DRM_H_ ++#define _UAPI_APPLE_G5P_DRM_H_ ++ ++#include "drm.h" ++ ++/* G5P merged winsys ABI. Retired ioctl numbers remain reserved. */ ++#define DRM_G5P_UAPI_VERSION 1 ++#define DRM_G5P_CAP_BO_CPU (1ULL << 0) ++#define DRM_G5P_CAP_VM_PREPARE (1ULL << 1) ++#define DRM_G5P_CAP_SYNCOBJ (1ULL << 2) ++#define DRM_G5P_CAP_RENDER (1ULL << 5) ++#define DRM_G5P_CAP_PRIME (1ULL << 6) ++#define DRM_G5P_CAP_CODE_BASE_VERIFIED (1ULL << 7) ++#define DRM_G5P_CAP_VM_LIVE (1ULL << 8) ++#define DRM_G5P_CAP_CONTEXT_LIVE (1ULL << 9) ++#define DRM_G5P_CAP_JOB_STATUS (1ULL << 10) ++#define DRM_G5P_CAP_CONTEXT_STATUS (1ULL << 11) ++#define DRM_G5P_CAP_FAULT_INFO (1ULL << 13) ++ ++/* LIVE means the owned user catalogue has completed firmware attachment. */ ++#define DRM_G5P_CONTEXT_LIVE 3 ++#define DRM_G5P_CONTEXT_HOST_ONLY 0 ++#define DRM_G5P_CONTEXT_LOST 1 ++ ++struct drm_g5p_get_context_status { ++ __u32 context; ++ __u32 flags; ++ __u32 state; ++ __u32 map_count; ++ __u64 mapped_bytes; ++ __u64 reserved; ++}; ++ ++/* First terminal firmware event, retained independently of last_event. */ ++#define DRM_G5P_FAULT_NONE 0 ++#define DRM_G5P_FAULT_RECOVERY 1 ++#define DRM_G5P_FAULT_PAGE_REQUEST 2 ++#define DRM_G5P_FAULT_CHANNEL 3 ++#define DRM_G5P_FAULT_UNKNOWN 4 ++#define DRM_G5P_FAULT_VALID_CHANNEL (1U << 0) ++struct drm_g5p_get_fault_info { ++ __u32 flags; ++ __u32 kind; ++ __u64 event_serial; /* One-based arrival order; zero if no fault. */ ++ __u32 valid; ++ __u32 channel_class; /* Event8: 0 memory failure, 1 killed, others opaque. */ ++ __u32 channel_token; /* Firmware token, not a DRM context handle. */ ++ __u32 channel_stamp; ++ __u32 raw[11]; /* Includes unknown fields; no decoded fault VA is claimed. */ ++ __u32 reserved; ++}; ++ ++struct drm_g5p_get_info { ++ __u32 size; ++ __u32 flags; ++ __u32 version; ++ __u32 gpu_page_shift; ++ __u64 capabilities; ++ __u64 va_start; ++ __u64 va_end; ++ __u64 reserved_start; ++ __u64 reserved_end; ++ __u64 code_base; /* Zero until CODE_BASE_VERIFIED. */ ++ __u64 max_bo_size; ++ __u32 bvnc[4]; /* Static target, not a new MMIO measurement. */ ++ __u64 reserved[2]; ++}; ++ ++/* GPU access roles; CPU initialization/mmap is permitted for every role. */ ++#define DRM_G5P_BO_READ (1U << 0) ++#define DRM_G5P_BO_WRITE (1U << 1) ++#define DRM_G5P_BO_EXEC (1U << 2) ++struct drm_g5p_create_bo { ++ __u64 size; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_bo_mmap_offset { ++ __u32 handle; ++ __u32 flags; ++ __u64 offset; ++}; ++ ++#define DRM_G5P_BO_SYNC_TO_DEVICE 0 ++#define DRM_G5P_BO_SYNC_TO_CPU 1 ++struct drm_g5p_bo_sync { ++ __u32 handle; ++ __u32 op; ++ __u64 offset; ++ __u64 size; ++}; ++ ++struct drm_g5p_create_vm { ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_vm { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++#define DRM_G5P_VM_MAP 0 ++#define DRM_G5P_VM_UNMAP 1 ++struct drm_g5p_vm_bind { ++ __u32 vm; ++ __u32 bo; ++ __u64 va; ++ __u64 size; ++ __u64 offset; ++ __u32 op; ++ __u32 flags; ++}; ++ ++struct drm_g5p_create_context { ++ __u32 vm; ++ __u32 type; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_context { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* A point of zero uses a binary syncobj; nonzero uses a timeline point. */ ++#define DRM_G5P_SYNC_SIGNAL (1U << 0) ++struct drm_g5p_sync { ++ __u32 handle; ++ __u32 flags; ++ __u64 point; ++}; ++ ++/* Privileged one-shot kernel fixture; all payload/BO fields must be zero. ++ * The empty user context authorizes ownership only; its VM is not attached. ++ * Does not advertise general COMPUTE or attach user-provided GPU addresses. ++ */ ++ ++/* Geometry is all zero/NONE while CODE_BASE_VERIFIED is clear. */ ++#define DRM_G5P_BASE_NONE 0 ++#define DRM_G5P_BASE_DEVICE 1 ++#define DRM_G5P_BASE_VM 2 ++#define DRM_G5P_BASE_CONTEXT 3 ++struct drm_g5p_get_exec_info { ++ __u32 size; ++ __u32 flags; ++ __u32 base_scope; ++ __u32 pad; ++ __u64 usc_base; ++ __u64 pds_base; ++ __u64 window_size; ++ __u64 allocation_alignment; ++ __u32 usc_offset_bits; ++ __u32 pds_offset_bits; ++ __u64 reserved[3]; ++}; ++ ++#define DRM_G5P_JOB_PENDING 0 ++#define DRM_G5P_JOB_COMPLETE 1 ++struct drm_g5p_get_job_status { ++ __u64 job_id; ++ __u32 flags; ++ __u32 state; ++ __s32 error; /* -EINPROGRESS while pending; immutable after completion. */ ++ __u32 pad; ++ __u64 reset_generation; /* Zero: no reset/recovery support. */ ++}; ++ ++#define DRM_G5P_GET_INFO 0x00 ++#define DRM_G5P_CREATE_BO 0x01 ++#define DRM_G5P_BO_MMAP_OFFSET 0x02 ++#define DRM_G5P_BO_SYNC 0x03 ++#define DRM_G5P_CREATE_VM 0x04 ++#define DRM_G5P_DESTROY_VM 0x05 ++#define DRM_G5P_VM_BIND 0x06 ++#define DRM_G5P_CREATE_CONTEXT 0x07 ++#define DRM_G5P_DESTROY_CONTEXT 0x08 ++#define DRM_G5P_GET_EXEC_INFO 0x0b ++#define DRM_G5P_GET_JOB_STATUS 0x0c ++#define DRM_G5P_GET_CONTEXT_STATUS 0x0d ++#define DRM_G5P_GET_FAULT_INFO 0x0e ++ ++#define DRM_IOCTL_G5P_GET_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_INFO, struct drm_g5p_get_info) ++#define DRM_IOCTL_G5P_CREATE_BO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_BO, struct drm_g5p_create_bo) ++#define DRM_IOCTL_G5P_BO_MMAP_OFFSET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_BO_MMAP_OFFSET, \ ++ struct drm_g5p_bo_mmap_offset) ++#define DRM_IOCTL_G5P_BO_SYNC \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_BO_SYNC, struct drm_g5p_bo_sync) ++#define DRM_IOCTL_G5P_CREATE_VM \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_VM, struct drm_g5p_create_vm) ++#define DRM_IOCTL_G5P_DESTROY_VM \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_VM, \ ++ struct drm_g5p_destroy_vm) ++#define DRM_IOCTL_G5P_VM_BIND \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_VM_BIND, struct drm_g5p_vm_bind) ++#define DRM_IOCTL_G5P_CREATE_CONTEXT \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_CONTEXT, \ ++ struct drm_g5p_create_context) ++#define DRM_IOCTL_G5P_DESTROY_CONTEXT \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_CONTEXT, \ ++ struct drm_g5p_destroy_context) ++ ++#define DRM_IOCTL_G5P_GET_EXEC_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_EXEC_INFO, \ ++ struct drm_g5p_get_exec_info) ++#define DRM_IOCTL_G5P_GET_JOB_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_JOB_STATUS, \ ++ struct drm_g5p_get_job_status) ++ ++#define DRM_IOCTL_G5P_GET_CONTEXT_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_CONTEXT_STATUS, \ ++ struct drm_g5p_get_context_status) ++ ++#define DRM_IOCTL_G5P_GET_FAULT_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_FAULT_INFO, \ ++ struct drm_g5p_get_fault_info) ++ ++#define DRM_G5P_CAP_IDLE_VM_BIND (1ULL << 18) ++ ++#define DRM_G5P_CAP_BOUNDED_RENDER (1ULL << 19) ++#define DRM_G5P_CAP_TYPED_RENDER (1ULL << 20) ++#define DRM_G5P_CAP_RENDER_USER_GEOMETRY (1ULL << 21) ++#define DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY (1ULL << 22) ++#define DRM_G5P_CAP_TYPED_RENDER_V2 (1ULL << 23) ++/* Bit 24 is reserved for COMPUTE_BLIT_V1 and bit 25 is FIXED_DATA_B1 on ++ * a9/release, so the next bit free in every branch is 26. ++ */ ++#define DRM_G5P_CAP_TYPED_RENDER_V3 (1ULL << 26) ++ ++/** ++ * DRM_G5P_CAP_RENDER_MULTI_CONTEXT - several render contexts share the GPU ++ * ++ * Reported together with TYPED_RENDER_V3. Any number of render contexts, in any ++ * number of files, may submit TYPED_V3 jobs. Each context has its own VM and its ++ * own firmware slot pair, and the jobs of all contexts run one at a time in ++ * submission order. Without this bit, the first context to submit owns the GPU ++ * until reboot and every other context is refused. The other render profiles ++ * stay single-context. ++ */ ++#define DRM_G5P_CAP_RENDER_MULTI_CONTEXT (1ULL << 27) ++ ++/** ++ * DRM_G5P_CAP_RENDER_UNPRIVILEGED - TYPED_V3 render needs no CAP_SYS_RAWIO ++ * ++ * Reported together with RENDER_MULTI_CONTEXT. Any opener of the render node may ++ * create render contexts and submit TYPED_V3 jobs. A render VM then maps none of ++ * the firmware's memory beyond the kernel-built window the GPU's own requesters ++ * need, and the contexts close cleanly: a closed context's firmware binding is ++ * removed and its memory freed. Pinned memory is limited device-wide. The other ++ * render profiles, compute and the diagnostic ioctls still need CAP_SYS_RAWIO. ++ */ ++#define DRM_G5P_CAP_RENDER_UNPRIVILEGED (1ULL << 28) ++ ++/** ++ * DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH - TYPED_V3 carries pixel_output_width ++ * ++ * Reported together with TYPED_RENDER_V3. struct drm_g5p_render_pass_v3 has ++ * pixel_output_width, so a TYPED_V3 pass in the V1 dialect (all-zero ext) can ++ * write several colour targets. Without this bit the field is reserved and ++ * must be zero. ++ */ ++#define DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH (1ULL << 29) ++ ++/* drm_g5p_create_context.type; flags must be zero. */ ++#define DRM_G5P_CONTEXT_RENDER 1 ++ ++#define DRM_G5P_RENDER_PROFILE_TYPED_V3 4 ++ ++/* LINEAR64_V1: kind is also the resource's position in the canonical array. */ ++ ++#define DRM_G5P_RENDER_ACCESS_READ (1U << 0) ++#define DRM_G5P_RENDER_ACCESS_WRITE (1U << 1) ++#define DRM_G5P_RENDER_ACCESS_IMPLICIT (1U << 2) ++#define DRM_G5P_RENDER_ACCESS_CACHE (1U << 3) ++ ++#define DRM_G5P_RENDER_MAX_RESOURCES 256 ++#define DRM_G5P_RENDER_MAX_SYNCS 17 ++#define DRM_G5P_RENDER_MAX_VDM_BYTES 0x10000 ++ ++/* A GPU VA range [va, va + size) inside one existing mapping of handle's BO ++ * in the context VM, at BO byte offset `offset`. Never a CPU pointer, PA or ++ * META address. LINEAR64_V1 requires offset 0 and the exact whole BO. ++ */ ++struct drm_g5p_render_resource { ++ __u32 handle; ++ __u32 kind; ++ __u64 offset; ++ __u64 size; ++ __u64 va; ++}; ++ ++/** ++ * DRM_G5P_CAP_RENDER_MSAA - 4x multisampled render targets and passes ++ * ++ * Reported with TYPED_RENDER_V3 and RENDER_KERNEL_GEOMETRY. ++ * CREATE_RENDER_TARGET accepts samples = 4 (kernel-derived geometry; with ++ * DRM_G5P_RT_USER_GEOMETRY userspace may only restate it), and a TYPED_V3 ++ * pass carries samples, msaa_flags and sample_positions in struct ++ * drm_g5p_render_pass. Without this bit those fields are reserved. ++ */ ++#define DRM_G5P_CAP_RENDER_MSAA (1ULL << 31) ++ ++/** ++ * DRM_G5P_CAP_RENDER_OCCLUSION_QUERY - TYPED_V3 carries ISP query storage ++ * ++ * The V2 extension names the query-result allocation used by ISP visibility ++ * tests. The kernel validates the complete range and writes its base into the ++ * two per-partition ISP_OCLQRY_BASE work-record slots. A zero VA and size ++ * disable visibility-result storage. ++ */ ++#define DRM_G5P_CAP_RENDER_OCCLUSION_QUERY (1ULL << 33) ++ ++/* drm_g5p_render_pass.msaa_flags: the end of tile averages each pixel's ++ * samples into the (one-sample) surface the EOT's PBE words name. ++ */ ++#define DRM_G5P_RENDER_MSAA_RESOLVE (1U << 0) ++ ++struct drm_g5p_render_pass { ++ __u32 size; /* sizeof(struct drm_g5p_render_pass) */ ++ __u32 flags; /* zero */ ++ __u64 vdm_va; /* VDM control stream, READ, 4-byte aligned */ ++ __u32 vdm_size; /* bytes, multiple of 4, <= MAX_VDM_BYTES */ ++ __u32 vdm_control; /* V1/V2: TA W+0x88 value, multiple of 16; V3: zero */ ++ __u64 scratch_va; /* TA scratch block, WRITE, 16-byte aligned */ ++ __u64 scratch_size; ++ __u64 scratch_ptr[6]; /* TA scratch/marker pointers, in a WRITE range */ ++ __u64 pool7_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool7_size; ++ __u64 pool8_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool8_size; ++ __u32 context_pds[2][6]; /* TA context store/load PDS descriptors */ ++ __u32 resume_pds[2][4]; /* TA resume PDS descriptors */ ++ __u32 ta_pds[4]; /* TA state PDS descriptor; word 3 zero */ ++ __u32 background[8]; /* 3D background (load/clear) object */ ++ __u32 eot[4]; /* 3D end-of-tile object; word 3 zero */ ++ __u32 eot_resume[4]; /* 3D partial-render EOT; word 3 zero */ ++ __u32 restore[4]; /* 3D partial-render restore words */ ++ /* DRM_G5P_CAP_RENDER_MSAA, TYPED_V3 only; zero in every other profile. ++ * samples: 0 or 1 with a one-sample target, 4 with a four-sample one. ++ * sample_positions: one byte per sample, x in [3:0] and y in [7:4] in ++ * 1/16 px (PPP_MULTISAMPLECTL); 0 takes the target's standard pattern. ++ * The kernel writes the AA mode, ISP_CTL and output-partition words ++ * for the sample count. The draws' ISPCTL.upass (0x7 at four samples, ++ * 0xf at one), the PBE/EOT words and the fragment programs are ++ * userspace's, as for one sample; see docs/gpu/experiments/msaa.md. ++ */ ++ __u32 samples; ++ __u32 msaa_flags; /* DRM_G5P_RENDER_MSAA_* */ ++ __u64 sample_positions; ++ __u64 reserved[6]; ++}; ++ ++/* ++ * TYPED_V2 additions. Each field is a 3D register image that the firmware ++ * uploads from the normal-3D work object. The CR offsets are the firmware ++ * store targets (its 3D upload puts W3c0 on ROGUE_CR_ISP_CTL 0xf38); the ++ * Rogue names are inferred, graded in docs/gpu/render-uapi.md. ++ */ ++#define DRM_G5P_RENDER_MAX_TABLE_ENTRIES 65536 /* ISPDBSC 16-bit indices */ ++#define DRM_G5P_RENDER_ZLS_REGS 6 ++ ++struct drm_g5p_render_pass_ext { ++ /* ISP_SCISSOR_BASE (CR fb0): IPF_SCISSOR_WORD_0/1 pairs, 8 bytes each, ++ * indexed by TA_STATE_ISPDBSC.scindex. READ, 4-byte aligned. ++ */ ++ __u64 scissor_va; ++ __u32 scissor_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ __u32 dbias_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ /* ISP_DBIAS_BASE (CR fb8): {constant, slope, clamp} f32 triples, ++ * 12 bytes each, indexed by ISPDBSC.dbindex. READ, 4-byte aligned. ++ */ ++ __u64 dbias_va; ++ /* PDS_BGRND1_BASE (CR 6a8): background object texture/varying data ++ * (B-relative, like background[]); LOAD_OP_LOAD's texture state. ++ */ ++ __u64 background_texture; ++ __u32 bgobj_depth; /* ISP_BGOBJDEPTH (CR f80): depth clear, ZLS format */ ++ __u32 bgobj_vals; /* ISP_BGOBJVALS (CR f88): stencil [7:0], mask [8], bgtag [9] */ ++ /* Pixel output width (2, 4 or 8): the 6ca7ae0 inputs to ISP_CTL's ++ * partition fields and the six per-core output words. 2 for one ++ * 32-bpp colour attachment (the captured value). ++ */ ++ __u32 pixel_output_width; ++ /* Bit i: zls[i] is an address ([39:4]; [63:40] zero; [3:0] flags) ++ * inside a listed READ|WRITE range. The kernel does not decode ZLS. ++ */ ++ __u32 zls_address_mask; ++ /* Opaque ZLS register images, CR f48, f50, f58, f60, f68 and fd0: ++ * Rogue ISP_ZLSCTL, ZLOAD/ZSTORE and STENCIL_LOAD/STORE bases plus ++ * one more; positions proven, order unqualified. Zero: no ZLS. ++ */ ++ __u64 zls[DRM_G5P_RENDER_ZLS_REGS]; ++ __u64 occlusion_query_va; /* ISP_OCLQRY_BASE, 16-byte aligned, READ|WRITE */ ++ __u64 occlusion_query_size; /* bytes in the listed allocation */ ++ __u64 reserved[2]; ++}; ++ ++struct drm_g5p_render_pass_v2 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++}; ++ ++#define DRM_G5P_RENDER_MAX_PROGRAM_BYTES 0x1000 ++#define DRM_G5P_RENDER_STREAM_TERMINATE 0xc0000000U ++ ++struct drm_g5p_render_pass_v3 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++ __u64 ta_program_va; /* -> W+0x88 / R0+0x470 as an offset from pds_base */ ++ __u32 ta_program_size; /* bytes, 16..MAX_PROGRAM_BYTES, multiple of 16 */ ++ __u32 vdm_end; /* bytes from vdm_va past STREAM_TERMINATE, <= vdm_size */ ++ __u32 pixel_output_width; /* V1 dialect: 0 (= 2), 2, 4 or 8 */ ++ __u32 pad; /* zero */ ++ __u64 reserved[3]; ++}; ++ ++struct drm_g5p_submit_render_typed { ++ __u32 context; ++ __u32 flags; /* zero */ ++ __u32 profile; ++ __u32 render_target; /* handle from CREATE_RENDER_TARGET */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[] */ ++ __u64 pass; /* struct drm_g5p_render_pass */ ++ __u32 pass_size; ++ __u32 pad; ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; ++}; ++ ++/* ++ * Render-target memory geometry: the AGXRenderTargetMemoryG5 arithmetic ++ * outputs (firmware round16 section 3, R-offsets in comments). The kernel ++ * computes them for its verified table (64x64x1), or with ++ * RENDER_KERNEL_GEOMETRY for any size; with RENDER_USER_GEOMETRY userspace ++ * supplies them and the kernel bounds the memory they imply (and, with ++ * RENDER_KERNEL_GEOMETRY, requires its own result). ++ */ ++struct drm_g5p_rt_geometry { ++ __u32 header_bytes; /* R10 */ ++ __u32 region_bytes; /* R20 */ ++ __u64 packed_state; /* Rb0 */ ++ __u32 state_word; /* Rb8 */ ++ __u32 padded_dims; /* Rbc */ ++ __u64 scale[3]; /* Rc0, Rc8, Rd0 packed float pairs */ ++ __u64 ta_state[2]; /* Rf0, Rf8 */ ++ __u64 ta_tiling[2]; /* R100, R108 */ ++ __u32 ta_tiling_word; /* R110 */ ++ __u32 tile_count; /* R118 */ ++ __u64 secondary_bytes; /* R138 */ ++ __u64 secondary_align; /* R178 */ ++ __u64 reserved[2]; ++}; ++ ++#define DRM_G5P_RT_USER_GEOMETRY (1U << 0) ++struct drm_g5p_create_render_target { ++ __u32 context; /* render context; RT memory lives in its VM */ ++ __u32 flags; ++ __u32 width; ++ __u32 height; ++ __u32 samples; /* 1, or 4 with DRM_G5P_CAP_RENDER_MSAA */ ++ __u32 layers; /* 1 */ ++ __u32 handle; /* output */ ++ __u32 pad; ++ __u64 main_va; /* output: kernel-private GPU VA */ ++ __u64 main_bytes; /* output */ ++ __u64 secondary_va; /* output */ ++ __u64 secondary_bytes; /* output: mapped bytes */ ++ struct drm_g5p_rt_geometry geometry; /* input if USER, else output */ ++ __u32 pm_blocks; /* output with RENDER_KERNEL_GEOMETRY: context PM pool */ ++ __u32 pad2; ++ __u64 reserved; ++}; ++ ++struct drm_g5p_destroy_render_target { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* GET_RENDER_INFO.flags, output only (input must be zero). ++ * VARIANT_ACTIVE: a render_user_* hypothesis knob (stage, cache/mmu/timestamp ++ * variant, 3d_after_ta, pm_policy, cache_repeat, wait_ms, pm/memory/rt/slot ++ * variant, profile1_vdm_control) differs from its default, so a context ++ * created now latches a non-default job behaviour. GET_RENDER_INFO has no ++ * context: it reports what CREATE_CONTEXT would latch at this moment. ++ */ ++#define DRM_G5P_RENDER_INFO_VARIANT_ACTIVE (1U << 0) ++ ++struct drm_g5p_get_render_info { ++ __u32 size; /* sizeof, exact */ ++ __u32 flags; /* in: zero; out: RENDER_INFO_* */ ++ __u32 profiles; /* bit N: profile N accepted */ ++ __u32 max_render_targets; ++ __u64 private_va_start; /* kernel window in render VMs */ ++ __u64 private_va_end; ++ __u64 user_va_min; /* lowest mappable VA in a render VM */ ++ __u64 usc_base; /* common USC/PDS base, 32-bit offsets */ ++ __u64 pds_base; ++ __u64 code_window; /* bytes above usc_base usable for code */ ++ __u32 pm_blocks; /* parameter buffer blocks per context */ ++ __u32 pm_block_bytes; ++ __u32 max_queued_jobs; ++ __u32 max_vdm_bytes; ++ __u32 job_timeout_ms; ++ __u32 retained_results; ++ /* RENDER_KERNEL_GEOMETRY only, else zero (64x64 only). */ ++ __u32 max_render_width; ++ __u32 max_render_height; ++ __u32 max_pm_blocks; /* PM pool growth bound per context */ ++ __u32 pad; ++ union { ++ __u64 reserved[2]; /* in: zero */ ++ struct { ++ /* TYPED_V3: every GPU VA the TA/3D read ends at or below ++ * this; 0 makes no statement. Out only. ++ */ ++ __u64 hw_va_limit; ++ __u64 reserved1; ++ }; ++ }; ++}; ++ ++#define DRM_G5P_CREATE_RENDER_TARGET 0x11 ++#define DRM_G5P_DESTROY_RENDER_TARGET 0x12 ++#define DRM_G5P_GET_RENDER_INFO 0x13 ++#define DRM_G5P_SUBMIT_RENDER_TYPED 0x14 ++ ++#define DRM_IOCTL_G5P_SUBMIT_RENDER_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_RENDER_TYPED, \ ++ struct drm_g5p_submit_render_typed) ++#define DRM_IOCTL_G5P_CREATE_RENDER_TARGET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_RENDER_TARGET, \ ++ struct drm_g5p_create_render_target) ++#define DRM_IOCTL_G5P_DESTROY_RENDER_TARGET \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_RENDER_TARGET, \ ++ struct drm_g5p_destroy_render_target) ++#define DRM_IOCTL_G5P_GET_RENDER_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_RENDER_INFO, \ ++ struct drm_g5p_get_render_info) ++ ++/** ++ * DRM_G5P_CAP_TYPED_COMPUTE - compute jobs on render contexts (SUBMIT_COMPUTE_TYPED) ++ * ++ * Reported together with TYPED_RENDER_V3. A ++ * render context (DRM_G5P_CONTEXT_RENDER) runs compute jobs beside its TYPED_V3 render jobs, ++ * in the same VM, in submission order, serialized with every other context's jobs. The ++ * kernel builds the firmware's compute work (W, microsequence, stamps, channel, slot); the ++ * job carries the CDM control stream and the register images iOS's userspace hands its ++ * kernel, which the kernel bounds but does not decode. Compute needs an epoch a render job ++ * brought up: on a device no render job has started, SUBMIT_COMPUTE_TYPED fails with EAGAIN. ++ */ ++#define DRM_G5P_CAP_TYPED_COMPUTE (1ULL << 30) ++ ++#define DRM_G5P_COMPUTE_MAX_STREAM_BYTES 0x10000 ++#define DRM_G5P_COMPUTE_STREAM_TERMINATE 0x80000000U ++ ++/* A PDS program reference: code and data offsets from GET_RENDER_INFO.pds_base (16-byte ++ * aligned; code in a READ resource whose BO carries DRM_G5P_BO_EXEC, data in a READ resource) ++ * and its opaque control word (Rogue's PDS1 position: sizes and temps). ++ */ ++struct drm_g5p_compute_pds { ++ __u32 code; ++ __u32 data; ++ __u32 control; ++ __u32 pad; /* zero */ ++}; ++ ++/* ++ * One compute job. Every *_va is a GPU VA inside a listed resource with the stated access. ++ * cdm_va/cdm_size the CDM control stream (W+0x70, R0+0x480): READ, 4-byte aligned, ++ * cdm_size <= MAX_STREAM_BYTES, in a native (not PRIME-imported) BO ++ * cdm_end bytes from cdm_va past the stream's terminate word (the dword at ++ * cdm_va + cdm_end - 4 is STREAM_TERMINATE), <= cdm_size ++ * state_va[4] CDM context-state words (W+0x1ec..0x204), READ|WRITE, 4-byte aligned ++ * state_tail_va CDM context-state tail (W+0x60, R0+0x498; W+0x20c), READ|WRITE, 16 bytes, ++ * 16-byte aligned ++ * config[2] CDM configuration words (W+0x10/0x14, R0+0x4c0/0x4c8), opaque ++ * store context-store program (W+0x21c/0x224, R0+0x4a8/0x4b0) ++ * resume context-resume program (W+0x22c/0x234) ++ * terminate terminate program (W+0x68, R0+0x4b8); control must be zero ++ */ ++struct drm_g5p_compute_pass { ++ __u32 size; /* sizeof(struct drm_g5p_compute_pass) */ ++ __u32 flags; /* zero */ ++ __u64 cdm_va; ++ __u32 cdm_size; ++ __u32 cdm_end; ++ __u64 state_va[4]; ++ __u64 state_tail_va; ++ __u32 config[2]; ++ struct drm_g5p_compute_pds store; ++ struct drm_g5p_compute_pds resume; ++ struct drm_g5p_compute_pds terminate; ++ __u64 reserved[4]; /* zero */ ++}; ++ ++/* Command 0x15. resources lists every BO the job references (1..RENDER_MAX_RESOURCES, ++ * TYPED_V1 access-mask kinds, IMPLICIT and CACHE included); pass_size is exactly ++ * sizeof(struct drm_g5p_compute_pass). Syncs, job ids, results and the error contract are ++ * SUBMIT_RENDER_TYPED's: every error other than EINTR/EFAULT on copy-out precedes fence ++ * replacement and hardware publication. ++ */ ++struct drm_g5p_submit_compute_typed { ++ __u32 context; /* a DRM_G5P_CONTEXT_RENDER context */ ++ __u32 flags; /* zero */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[]: exactly one SIGNAL, at most 16 inputs */ ++ __u64 pass; /* struct drm_g5p_compute_pass */ ++ __u32 pass_size; ++ __u32 pad; /* zero */ ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; /* zero */ ++}; ++ ++#define DRM_G5P_SUBMIT_COMPUTE_TYPED 0x15 ++ ++#define DRM_IOCTL_G5P_SUBMIT_COMPUTE_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_COMPUTE_TYPED, \ ++ struct drm_g5p_submit_compute_typed) ++ ++#endif diff --git a/kernel/patches/drivers/drm/0010-g5p-implementation.patch b/kernel/patches/drivers/drm/0010-g5p-implementation.patch new file mode 100644 index 00000000..8d025000 --- /dev/null +++ b/kernel/patches/drivers/drm/0010-g5p-implementation.patch @@ -0,0 +1,80 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] drm/apple-g5p: G5P render, autoboot, META71C and boot-origin policy + +Carry the original-firmware G5P render/compute ABI and extraction recipe into the system kernel, with typed query/compute follow-ups and qualified WC cache handling. Start firmware at probe, install META71C 340 on every firmware image, select dirty-start mode 3 from the incoming kexec DT marker (cold mode 1), and quiet recurring diagnostics. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +index 2407ec142..749aeef25 100644 +--- a/drivers/gpu/apple-g5p/boot-r73b.inc ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -780,7 +780,7 @@ static void g5p_hb116_irq(struct g5p_diag *g, int irq, u32 n, u32 before) + } + g5p_hb116_cross(g, n, events); + if (g5p_hb_report(&r->hb)) +- dev_info( ++ g5p_job_printk( + g->dev, + "HB116 window n=%u t_ns=%llu ms=%llu irqs=%u events=%u acks=%u empty=%u aux=%u " + "stalled=%u peak_irqs=%u peak_events=%u total_irqs=%u total_events=%u " +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +index d5bca5a4e..8281f3a68 100644 +--- a/drivers/gpu/apple-g5p/boot.inc ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -775,7 +775,7 @@ static int g5p_boot_drain(struct g5p_diag *g) + n = ++b->event_n[t]; + if (words[0] == 1 && n > 1 && !(0)) { + if (is_power_of_2(n)) +- dev_info( ++ g5p_job_printk( + g->dev, + "G2 events type=1 n=%u: stamp events counted, dumped once (boot_event_dump 1: each)\n", + n); +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +index 037e5c56e..be023e7e7 100644 +--- a/drivers/gpu/apple-g5p/dirtystart108.inc ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -18,9 +18,12 @@ + static struct g5p_ds g5p_ds_state = { .last = G5P_DSE_N }; + static DEFINE_SPINLOCK(g5p_ds_lock); + ++/* Latched from /chosen at probe, before POWER or the first wrapper wake. */ ++static u32 g5p_ds_boot_mode = 1; ++ + static u32 g5p_ds_mode(void) + { +- return min_t(u32, (1), G5P_DS_MODE_MAX); ++ return READ_ONCE(g5p_ds_boot_mode); + } + + /* One event, one DS108 line (dev_emerg: netconsole at any loglevel). */ +@@ -85,6 +88,10 @@ static void g5p_ds_hook_probe(struct g5p_diag *g) + of_property_read_bool(chosen, "linux,booted-from-kexec"); + in.cycle_requested = g5p_ds_cycle_requested(chosen); + of_node_put(chosen); ++ /* Retained warm state needs strict origin checks and the bounded reset. */ ++ WRITE_ONCE(g5p_ds_boot_mode, in.kexec ? 3 : 1); ++ dev_info(g->dev, "DS108 boot mode=%u kexec=%u\n", ++ g5p_ds_mode(), in.kexec); + g5p_ds_feed(g, G5P_DSE_PROBE, &in); + } + +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 34fbad607..1d0a0a76d 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -2476,7 +2476,7 @@ static void g5p_oc_work(struct work_struct *work) + + if (!g5p_mmio_hold(g, &tk, G5P_MMIO_CENSUS)) + g5p_mmio_release(g, &tk); +- dev_info( ++ g5p_job_printk( + g->dev, + "GPUPERF oc status mhz=%u temp=%d zone=%s limit=%u r1=%s r1_1000=%08x jobs=%u " + "end=gpuperf\n", diff --git a/src/orchard/apple/agx_g5p.py b/src/orchard/apple/agx_g5p.py new file mode 100644 index 00000000..c46ca6a7 --- /dev/null +++ b/src/orchard/apple/agx_g5p.py @@ -0,0 +1,19 @@ +"""Extract the unmodified G5P C0 META LDR from the user's prelinked kernelcache.""" +import hashlib +from orchard.apple.prelink import Kernelcache, named + +BUNDLE = 'com.apple.AGXFirmwareKextG5P' +CONST = ('__TEXT', '__const') +SIZE = 119112 +SHA256 = '33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca' + + +def extract(blob): + kernel = Kernelcache(blob) + section = named(kernel.kext_sections(BUNDLE, 'G5P firmware'), CONST)[CONST] + if section['size'] != SIZE: + raise ValueError('Unsupported G5P C0 firmware size') + data = kernel.read(section['addr'], section['size']) + if hashlib.sha256(data).hexdigest() != SHA256: + raise ValueError('Unsupported G5P C0 firmware digest') + return data diff --git a/src/orchard/recipe.py b/src/orchard/recipe.py index 8b32889e..c8a16887 100644 --- a/src/orchard/recipe.py +++ b/src/orchard/recipe.py @@ -8,7 +8,7 @@ import json import plistlib -from orchard.apple import bluetool, d5500, sgx, smartio, trx, z2fw, z2packet +from orchard.apple import agx_g5p, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet from orchard.apple.osserialize import apple_xml, binary @@ -39,6 +39,10 @@ def _smartio_kext(step, data, expected): return smartio.extract(data, step['kext'])[0] +def _agx_g5p(step, data, expected): + return agx_g5p.extract(data) + + def _vxd(step, data, expected): return d5500.extract(data)[0] @@ -96,6 +100,7 @@ def _z2fw_mtfw(step, data, expected): # step@version -> (implementation, {parameter besides 'step' and 'input': type}) STEPS = { + 'agx-g5p-ldr-from-kernelcache@1': (_agx_g5p, {}), 'copy@1': (_copy, {}), 'trx-add-nvram@1': (_trx, {'nvram': str, 'placeholder_mac': str}), 'sgx543-from-kernelcache@1': (_sgx, {'build': str}), diff --git a/tests/test_agx_g5p.py b/tests/test_agx_g5p.py new file mode 100644 index 00000000..f048ef01 --- /dev/null +++ b/tests/test_agx_g5p.py @@ -0,0 +1,29 @@ +import hashlib +import unittest +from unittest.mock import Mock, patch +from orchard.apple import agx_g5p + +class G5PFirmware(unittest.TestCase): + def test_original_bytes_and_bundle(self): + data = b'synthetic firmware, not Apple data' + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=len(data))] + kernel.read.return_value = data + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), patch.object(agx_g5p, 'SIZE', len(data)), patch.object(agx_g5p, 'SHA256', hashlib.sha256(data).hexdigest()): + self.assertIs(agx_g5p.extract(b'input'), data) + kernel.kext_sections.assert_called_once_with(agx_g5p.BUNDLE, 'G5P firmware') + kernel.read.assert_called_once_with(4096, len(data)) + + def test_wrong_size_is_rejected_before_read(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=1)] + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'size'): + agx_g5p.extract(b'input') + kernel.read.assert_not_called() + + def test_wrong_digest_is_rejected(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=agx_g5p.SIZE)] + kernel.read.return_value = bytes(agx_g5p.SIZE) + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'digest'): + agx_g5p.extract(b'input') From fd6e7febcf5e4eb9f3752067a029b9c20de3c401 Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 03/33] AOP sensors and certified warm reload kernel/configs/apple-16k/config: CONFIG_APPLE_AOP_GYRO=m CONFIG_APPLE_AOP_PRESSURE=m These options provide aop and its Kconfig dependencies on both page sizes. Assisted-by: LLM --- ...9-AOP-s-boot-arguments-and-sensor-ca.patch | 116 + chips/s8003.toml | 3 + devices/j71t/device.toml | 7 +- devices/j71t/firmware.json | 34 + docs/compatibility.md | 3 +- kernel/configs/apple-16k/config | 2 + .../s8000/0008-aop-implementation.patch | 121 + .../drivers/iio/0005-aop-implementation.patch | 760 ++++ .../pmdomain/0001-aop-implementation.patch | 24 + .../drivers/soc-apple/0014-aop-bindings.patch | 127 + .../soc-apple/0015-aop-implementation.patch | 3285 +++++++++++++++++ tests/test_j71t_data.py | 31 + 12 files changed, 4510 insertions(+), 3 deletions(-) create mode 100644 boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch create mode 100644 kernel/patches/devices/s8000/0008-aop-implementation.patch create mode 100644 kernel/patches/drivers/iio/0005-aop-implementation.patch create mode 100644 kernel/patches/drivers/pmdomain/0001-aop-implementation.patch create mode 100644 kernel/patches/drivers/soc-apple/0014-aop-bindings.patch create mode 100644 kernel/patches/drivers/soc-apple/0015-aop-implementation.patch create mode 100644 tests/test_j71t_data.py diff --git a/boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch b/boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch new file mode 100644 index 00000000..13245cd5 --- /dev/null +++ b/boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 2 Nov 2026 15:07:08 +0100 +Subject: [PATCH] kboot: Pass the A9 AOP's boot arguments and sensor + calibration + +The A9's AOP firmware takes more than the ECID from its loader: the AOP +target number (iop-aop-nub aop-target), the chip's ID (/chosen chip-id) +and revision (/arm-io chip-revision). Its accelerometer, gyroscope and +pressure sensor also send no reports until the driver has given them +the unit's factory calibration, which iBoot resolves from syscfg into +the sensors' nodes below iop-aop-nub. + +Copy the three numbers into the AOP's node (alias aop) as +apple,aop-target, apple,chip-id and apple,chip-revision, and each +sensor's calibration properties into the AOP's child node of the same +name, prefixed with "apple,". Unprovisioned "syscfg/" references and +nodes the device tree doesn't have are skipped. The orientation +properties are not copied: the A9 firmware rejects them and the board's +mount-matrix describes the mounting. + +The values are the unit's own and never leave the device: the kernel +sends them to the AOP and nothing else. + +Assisted-by: LLM +--- + src/kboot.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 68 insertions(+), 1 deletion(-) + +diff --git a/src/kboot.c b/src/kboot.c +index ebd6282..7ce9d72 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -1005,6 +1005,73 @@ static int dt_set_als(void) + return 0; + } + ++/* ++ * The A9's AOP firmware also takes the AOP target number and the chip's ID and ++ * revision in its boot arguments, and each motion sensor's factory ++ * calibration, which iBoot resolves from syscfg into the sensor's node below ++ * iop-aop-nub. Copy them into the AOP's node and its sensor child nodes of the ++ * same name; the calibration keeps its name with an "apple," prefix. ++ */ ++static const char *const aop_sensor_calibrations[] = { ++ "low-temp-accel-offset", ++ "accel-sensitivity-calibration", ++ "accel-range-sensitivity-cal", ++ "accel-interrupt-calibration", ++ "gyro-sensitivity-calibration", ++ "gyro-range-sensitivity-cal", ++ "gyro-interrupt-calibration", ++ "gyro-trim-calibration", ++ "pressure-offset-calibration", ++ "pressure-accel-sensitivity", ++ "pressure-heater-resistance", ++}; ++ ++static int dt_set_aop_sensors(int node) ++{ ++ int nub = adt_path_offset(adt, "/arm-io/aop/iop-aop-nub"); ++ if (nub < 0) ++ return 0; ++ ++ u32 val; ++ if (ADT_GETPROP(adt, nub, "aop-target", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,aop-target", val)) ++ bail("FDT: failed to set apple,aop-target\n"); ++ ++ int chosen = adt_path_offset(adt, "/chosen"); ++ if (chosen >= 0 && ADT_GETPROP(adt, chosen, "chip-id", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-id", val)) ++ bail("FDT: failed to set apple,chip-id\n"); ++ ++ int arm_io = adt_path_offset(adt, "/arm-io"); ++ if (arm_io >= 0 && ADT_GETPROP(adt, arm_io, "chip-revision", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-revision", val)) ++ bail("FDT: failed to set apple,chip-revision\n"); ++ ++ int sensor = nub; ++ ADT_FOREACH_CHILD(adt, sensor) ++ { ++ const char *name = adt_get_name(adt, sensor); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(aop_sensor_calibrations); i++) { ++ u32 len; ++ const u8 *blob = adt_getprop(adt, sensor, aop_sensor_calibrations[i], &len); ++ if (!blob || !len || (len >= 7 && !memcmp(blob, "syscfg/", 7))) ++ continue; ++ ++ /* Earlier properties may have moved the sensor's node. */ ++ int child = fdt_subnode_offset(dt, node, name); ++ if (child < 0) ++ break; ++ ++ char prop[64]; ++ snprintf(prop, sizeof(prop), "apple,%s", aop_sensor_calibrations[i]); ++ if (fdt_setprop(dt, child, prop, blob, len)) ++ bail("FDT: failed to set %s/%s\n", name, prop); ++ } ++ } ++ return 0; ++} ++ + /* + * The AOP's firmware carries the chip's ECID in its boot arguments. Pass it + * only to the AOP's node; the driver removes it from the tree once read. +@@ -1028,7 +1095,7 @@ static int dt_set_aop(void) + + if (fdt_setprop_u64(dt, node, "apple,ecid", ecid)) + bail("FDT: failed to set apple,ecid\n"); +- return 0; ++ return dt_set_aop_sensors(node); + } + + static int dt_set_multitouch(void) diff --git a/chips/s8003.toml b/chips/s8003.toml index eca49fad..fb750ff9 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -18,3 +18,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: g5p. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/g5p.md. + +# Local capability delta: aop. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/aop.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 03cbaa69..b8b78cbb 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -34,8 +34,8 @@ screen_off = "untested" backlight = "untested" touch = "untested" buttons = "untested" -accelerometer = "missing" -gyroscope = "missing" +accelerometer = "untested" +gyroscope = "untested" light_sensor = "untested" touch_id = "missing" storage_read = "untested" @@ -57,3 +57,6 @@ camera = "missing" # Local capability delta: g5p. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/g5p.md. + +# Local capability delta: aop. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/aop.md. diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index 442cabbd..968551c1 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -113,6 +113,12 @@ }, "device": "/usr/share/firmware/multitouch/J71t.mtprops", "sha256": "69f27fe207cc5810a748dac3dfe640b1c5a755709f49513b4b5125660784f0fc" + }, + "aop.im4p": { + "ipsw": { + "zip": "Firmware/AOP/aopfw-ipad6faop.im4p" + }, + "sha256": "5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa" } }, "outputs": [ @@ -289,6 +295,34 @@ "input": "kernelcache" } ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f.bin", + "sha256": "af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP" + } + ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f-ext.bin", + "sha256": "2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP_ext" + } + ] } ] } diff --git a/docs/compatibility.md b/docs/compatibility.md index 52850565..1f8d2a8c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -161,7 +161,7 @@ Boards Orchard builds for. States: working, untested, missing, none (no such har | `n66` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n69u` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j71t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | +| `j71t` | untested | untested | untested | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | | `j72t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | | `n66m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n69` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | @@ -248,6 +248,7 @@ A yes means an enabled device tree node binds a driver in `7.3.0-rc1-orchard-app | Board | Feature | | | --- | --- | --- | +| `j71t` | Accelerometer | untested, but no driver binds | | `d10` | Bluetooth input devices | missing, but a driver binds | | `d10` | Bluetooth audio | missing, but a driver binds | | `d101` | Bluetooth input devices | missing, but a driver binds | diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index 82a17860..8676844b 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -5621,6 +5621,7 @@ CONFIG_APPLE_AOP_ACCEL=m # CONFIG_ADIS16260 is not set # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set +CONFIG_APPLE_AOP_GYRO=m # CONFIG_BMG160 is not set # CONFIG_FXAS21002C is not set # CONFIG_MPU3050_I2C is not set @@ -5856,6 +5857,7 @@ CONFIG_TSL2583=m # CONFIG_HP206C is not set # CONFIG_ZPA2326 is not set # CONFIG_ADP810 is not set +CONFIG_APPLE_AOP_PRESSURE=m # end of Pressure sensors # diff --git a/kernel/patches/devices/s8000/0008-aop-implementation.patch b/kernel/patches/devices/s8000/0008-aop-implementation.patch new file mode 100644 index 00000000..a2182740 --- /dev/null +++ b/kernel/patches/devices/s8000/0008-aop-implementation.patch @@ -0,0 +1,121 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +index 354533f26..338182cf5 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +@@ -11,6 +11,7 @@ + + / { + aliases { ++ aop = &aop; /* Used by loader */ + ausbphy0 = &ausbphy0; + nvme-hmb = &nvme_hmb; /* Used by loader */ + serial0 = &serial0; +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index bec208568..2a0a83383 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -446,6 +446,43 @@ wdt: watchdog@2102b0000 { + interrupt-parent = <&aic>; + interrupts = ; + }; ++ ++ aop: aop@210800000 { ++ compatible = "apple,s8000-aop"; ++ reg = <0x2 0x10800000 0x0 0x4000>, ++ <0x2 0x10000600 0x0 0x4>, ++ <0x2 0x10e00000 0x0 0x9b000>; ++ reg-names = "akf", "control", "sram"; ++ mboxes = <&aop_mbox>; ++ mbox-names = "mbox"; ++ status = "disabled"; ++ ++ aop_accel: accel { ++ compatible = "apple,s8000-aop-accel"; ++ }; ++ ++ aop_gyro: gyro { ++ compatible = "apple,s8000-aop-gyro"; ++ }; ++ ++ aop_pressure: pressure { ++ compatible = "apple,s8000-aop-pressure"; ++ }; ++ }; ++ ++ aop_mbox: mbox@210804000 { ++ compatible = "apple,s8000-a7iop-mailbox", "apple,t8010-a7iop-mailbox"; ++ reg = <0x2 0x10804000 0x0 0x1000>; ++ interrupt-parent = <&aic>; ++ interrupts = , ++ , ++ , ++ ; ++ interrupt-names = "send-empty", "send-not-empty", ++ "recv-empty", "recv-not-empty"; ++ #mbox-cells = <0>; ++ status = "disabled"; ++ }; + }; + + timer { +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 7e4d67d7f..347ec084c 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -16,6 +16,44 @@ / { + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; + }; + ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* Same tail page as release6, never a firmware DMA grant. */ ++ aop_shutdown: aop-shutdown@8583fc000 { ++ reg = <0x8 0x583fc000 0 0x4000>; ++ no-map; ++ }; ++ }; ++}; ++ ++&aop { ++ memory-region = <&aop_shutdown>; ++ apple,observe-domains = <&ps_aop_gpio &ps_aop &ps_aop_busif ++ &ps_aop_cpu &ps_aop_filter>; ++ apple,spi0-domain = <&ps_spi0>; ++ firmware-name = "apple/aop-ipad6f.bin", "apple/aop-ipad6f-ext.bin"; ++ status = "okay"; ++}; ++ ++/* ++ * The AOP reports the accelerometer in its sensor frame, which is the ++ * device frame (x right, y up, z out of the screen) negated: flat, screen up ++ * reads z = -1 g, upright y = -1 g. iio-sensor-proxy takes upright as ++ * negative y and needs z flipped. ++ */ ++&aop_accel { ++ mount-matrix = "1", "0", "0", ++ "0", "1", "0", ++ "0", "0", "-1"; ++}; ++ ++&aop_mbox { ++ status = "okay"; ++}; ++ + &touchscreen { + firmware-name = "apple/j71t-fwstart.z2fw"; + }; diff --git a/kernel/patches/drivers/iio/0005-aop-implementation.patch b/kernel/patches/drivers/iio/0005-aop-implementation.patch new file mode 100644 index 00000000..6d6ff334 --- /dev/null +++ b/kernel/patches/drivers/iio/0005-aop-implementation.patch @@ -0,0 +1,760 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] iio: apple-aop: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig +index b01598194..ed328a590 100644 +--- a/drivers/iio/accel/Kconfig ++++ b/drivers/iio/accel/Kconfig +@@ -213,7 +213,7 @@ config APPLE_AOP_ACCEL + select IIO_KFIFO_BUF + help + Say yes here to get support for the accelerometer that the Always-On +- Processor (AOP) of Apple A10 iPads and iPhones serves. ++ Processor (AOP) of Apple A9 and A10 iPads and iPhones serves. + + To compile this driver as a module, choose M here: the module will be + called apple_aop_accel. +diff --git a/drivers/iio/accel/apple_aop_accel.c b/drivers/iio/accel/apple_aop_accel.c +index 96c901361..e1b0e136d 100644 +--- a/drivers/iio/accel/apple_aop_accel.c ++++ b/drivers/iio/accel/apple_aop_accel.c +@@ -9,9 +9,13 @@ + * + * 0 u32 sequence + * 4 u16 reserved +- * 6 s32 x, y, z in 1/65536 g, in the device frame (x right, y up, +- * z out of the screen; +1 g opposite to gravity) ++ * 6 s32 x, y, z in 1/65536 g + * 18 u32 timestamp ++ * ++ * The T8010's firmware reports in the device frame (x right, y up, z out of ++ * the screen; +1 g opposite to gravity). The S8000's firmware first needs the ++ * unit's factory calibration and reports in the sensor's frame; the board's ++ * mount-matrix maps it. + */ + + #include +@@ -21,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -41,8 +46,16 @@ + /* 9.80665 m/s^2 / 65536 */ + #define AOP_ACCEL_SCALE_NANO 149637 + ++struct apple_aop_accel_hw { ++ const struct apple_aop_calibration *calibration; ++ unsigned int num_calibration; ++ /* No separate raw sample stream (raw interval 0) */ ++ bool no_raw_stream; ++}; ++ + struct apple_aop_accel { + struct device *dev; ++ const struct apple_aop_accel_hw *hw; + const struct apple_aop_service *service; + struct iio_mount_matrix orientation; + +@@ -75,7 +88,9 @@ static int apple_aop_accel_set_interval(struct apple_aop_accel *accel, u32 inter + int i, ret; + + for (i = 0; i < ARRAY_SIZE(tags); i++) { +- ret = apple_aop_accel_set_property(accel, tags[i], interval_us); ++ ret = apple_aop_accel_set_property(accel, tags[i], ++ tags[i] == AOP_PROP_RAW_INTERVAL && ++ accel->hw->no_raw_stream ? 0 : interval_us); + if (ret) + return ret; + } +@@ -230,6 +245,7 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return -ENOMEM; + accel = iio_priv(indio_dev); + accel->dev = dev; ++ accel->hw = device_get_match_data(dev); + accel->service = service; + mutex_init(&accel->lock); + spin_lock_init(&accel->sample_lock); +@@ -252,6 +268,15 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (accel->hw->num_calibration) { ++ ret = apple_aop_calibrate(service, dev, accel->hw->calibration, ++ accel->hw->num_calibration); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to calibrate\n"); ++ } ++ } ++ + ret = apple_aop_accel_set_interval(accel, AOP_ACCEL_DEFAULT_US); + if (ret) { + apple_aop_unlisten(service); +@@ -265,8 +290,25 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return devm_iio_device_register(dev, indio_dev); + } + ++/* Offset at low temperature, sensitivity (either), interrupt calibration */ ++static const struct apple_aop_calibration apple_aop_accel_s8000_calibration[] = { ++ { "apple,low-temp-accel-offset", 0x1a, 1 }, ++ { "apple,accel-sensitivity-calibration", 0x0b, 2 }, ++ { "apple,accel-range-sensitivity-cal", 0x27, 2 }, ++ { "apple,accel-interrupt-calibration", 0x19, 3 }, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_s8000 = { ++ .calibration = apple_aop_accel_s8000_calibration, ++ .num_calibration = ARRAY_SIZE(apple_aop_accel_s8000_calibration), ++ .no_raw_stream = true, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_t8010 = { }; ++ + static const struct of_device_id apple_aop_accel_of_match[] = { +- { .compatible = "apple,t8010-aop-accel" }, ++ { .compatible = "apple,s8000-aop-accel", .data = &apple_aop_accel_s8000 }, ++ { .compatible = "apple,t8010-aop-accel", .data = &apple_aop_accel_t8010 }, + { } + }; + MODULE_DEVICE_TABLE(of, apple_aop_accel_of_match); +diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig +index 3e193ee0f..8f506a65d 100644 +--- a/drivers/iio/gyro/Kconfig ++++ b/drivers/iio/gyro/Kconfig +@@ -63,6 +63,18 @@ config ADXRS450 + This driver can also be built as a module. If so, the module + will be called adxrs450. + ++config APPLE_AOP_GYRO ++ tristate "Apple AOP gyroscope" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the gyroscope that the Always-On ++ Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_gyro. ++ + config BMG160 + tristate "BOSCH BMG160 Gyro Sensor" + depends on (I2C || SPI_MASTER) +diff --git a/drivers/iio/gyro/Makefile b/drivers/iio/gyro/Makefile +index 0319b397d..212bbb814 100644 +--- a/drivers/iio/gyro/Makefile ++++ b/drivers/iio/gyro/Makefile +@@ -10,6 +10,7 @@ obj-$(CONFIG_ADIS16136) += adis16136.o + obj-$(CONFIG_ADIS16260) += adis16260.o + obj-$(CONFIG_ADXRS290) += adxrs290.o + obj-$(CONFIG_ADXRS450) += adxrs450.o ++obj-$(CONFIG_APPLE_AOP_GYRO) += apple_aop_gyro.o + obj-$(CONFIG_BMG160) += bmg160_core.o + obj-$(CONFIG_BMG160_I2C) += bmg160_i2c.o + obj-$(CONFIG_BMG160_SPI) += bmg160_spi.o +diff --git a/drivers/iio/gyro/apple_aop_gyro.c b/drivers/iio/gyro/apple_aop_gyro.c +new file mode 100644 +index 000000000..edbda1f43 +--- /dev/null ++++ b/drivers/iio/gyro/apple_aop_gyro.c +@@ -0,0 +1,291 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Gyroscope served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the gyroscope itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one motion report (EPIC ++ * subtype 0xc1) per report interval, laid out like the accelerometer's: ++ * ++ * 0 u16 sequence ++ * 2 u32 status ++ * 6 s32 x, y, z in 1/65536 degree per second ++ * 18 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_GYRO 0xc1 ++#define AOP_GYRO_LEN 22 ++#define AOP_GYRO_XYZ 6 ++ ++#define AOP_GYRO_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_GYRO_MIN_US 10000 /* 100 Hz */ ++#define AOP_GYRO_MAX_US 1000000 /* 1 Hz */ ++ ++/* pi / 180 rad/s / 65536 = 1171270 / 2^42 rad/s */ ++#define AOP_GYRO_SCALE 1171270 ++#define AOP_GYRO_SCALE_LOG2 42 ++ ++struct apple_aop_gyro { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ struct iio_mount_matrix orientation; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ s32 sample[3]; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(s32, scan, 3); ++}; ++ ++static int apple_aop_gyro_set_u32(struct apple_aop_gyro *gyro, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(gyro->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_gyro_set_interval(struct apple_aop_gyro *gyro, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ gyro->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_gyro_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int i; ++ ++ if (subtype != AOP_REPORT_GYRO || len < AOP_GYRO_LEN) ++ return; ++ ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ for (i = 0; i < 3; i++) ++ gyro->sample[i] = get_unaligned_le32(data + AOP_GYRO_XYZ + 4 * i); ++ gyro->have_sample = true; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ memcpy(gyro->scan, gyro->sample, sizeof(gyro->sample)); ++ iio_push_to_buffers_with_ts(indio_dev, gyro->scan, sizeof(gyro->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_gyro_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ *val = gyro->sample[chan->scan_index]; ++ ret = gyro->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ *val = AOP_GYRO_SCALE; ++ *val2 = AOP_GYRO_SCALE_LOG2; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&gyro->lock); ++ *val = USEC_PER_SEC / gyro->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % gyro->interval_us) * USEC_PER_SEC / ++ gyro->interval_us; ++ mutex_unlock(&gyro->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_gyro_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_GYRO_MIN_US, AOP_GYRO_MAX_US); ++ ++ mutex_lock(&gyro->lock); ++ ret = apple_aop_gyro_set_interval(gyro, interval); ++ mutex_unlock(&gyro->lock); ++ return ret; ++} ++ ++static const struct iio_mount_matrix * ++apple_aop_gyro_get_mount_matrix(const struct iio_dev *indio_dev, ++ const struct iio_chan_spec *chan) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ ++ return &gyro->orientation; ++} ++ ++static const struct iio_chan_spec_ext_info apple_aop_gyro_ext_info[] = { ++ IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, apple_aop_gyro_get_mount_matrix), ++ { } ++}; ++ ++#define APPLE_AOP_GYRO_CHANNEL(axis, index) { \ ++ .type = IIO_ANGL_VEL, \ ++ .modified = 1, \ ++ .channel2 = IIO_MOD_##axis, \ ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ ++ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \ ++ .scan_index = index, \ ++ .scan_type = { \ ++ .sign = 's', \ ++ .realbits = 32, \ ++ .storagebits = 32, \ ++ .endianness = IIO_CPU, \ ++ }, \ ++ .ext_info = apple_aop_gyro_ext_info, \ ++} ++ ++static const struct iio_chan_spec apple_aop_gyro_channels[] = { ++ APPLE_AOP_GYRO_CHANNEL(X, 0), ++ APPLE_AOP_GYRO_CHANNEL(Y, 1), ++ APPLE_AOP_GYRO_CHANNEL(Z, 2), ++ IIO_CHAN_SOFT_TIMESTAMP(3), ++}; ++ ++static const struct iio_info apple_aop_gyro_info = { ++ .read_raw = apple_aop_gyro_read_raw, ++ .write_raw = apple_aop_gyro_write_raw, ++}; ++ ++/* Sensitivity and interrupt calibration make it complete */ ++static const struct apple_aop_calibration apple_aop_gyro_calibration[] = { ++ { "apple,gyro-sensitivity-calibration", 0x0b, 1 }, ++ { "apple,gyro-interrupt-calibration", 0x19, 2 }, ++ { "apple,gyro-range-sensitivity-cal", 0x27, 0 }, ++ { "apple,gyro-trim-calibration", 0x20, 0 }, ++}; ++ ++static void apple_aop_gyro_stop(void *data) ++{ ++ struct apple_aop_gyro *gyro = data; ++ ++ apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(gyro->service); ++} ++ ++static int apple_aop_gyro_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_gyro *gyro; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*gyro)); ++ if (!indio_dev) ++ return -ENOMEM; ++ gyro = iio_priv(indio_dev); ++ gyro->dev = dev; ++ gyro->service = service; ++ mutex_init(&gyro->lock); ++ spin_lock_init(&gyro->sample_lock); ++ ++ ret = iio_read_mount_matrix(dev, &gyro->orientation); ++ if (ret) ++ return ret; ++ ++ indio_dev->name = "apple-aop-gyro"; ++ indio_dev->info = &apple_aop_gyro_info; ++ indio_dev->channels = apple_aop_gyro_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_gyro_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_gyro_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_gyro_calibration, ++ ARRAY_SIZE(apple_aop_gyro_calibration)); ++ if (!ret) ++ ret = apple_aop_gyro_set_interval(gyro, AOP_GYRO_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_gyro_stop, gyro); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_gyro_of_match[] = { ++ { .compatible = "apple,s8000-aop-gyro" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_gyro_of_match); ++ ++static struct platform_driver apple_aop_gyro_driver = { ++ .driver = { ++ .name = "apple-aop-gyro", ++ .of_match_table = apple_aop_gyro_of_match, ++ }, ++ .probe = apple_aop_gyro_probe, ++}; ++module_platform_driver(apple_aop_gyro_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP gyroscope driver"); ++MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig +index 838a8340c..18535ad99 100644 +--- a/drivers/iio/pressure/Kconfig ++++ b/drivers/iio/pressure/Kconfig +@@ -384,4 +384,16 @@ config ADP810 + To compile this driver as a module, choose M here: the module will + be called adp810 + ++config APPLE_AOP_PRESSURE ++ tristate "Apple AOP barometric pressure sensor" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the barometric pressure sensor that ++ the Always-On Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_pressure. ++ + endmenu +diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile +index bc0d11a20..bf3e27ba9 100644 +--- a/drivers/iio/pressure/Makefile ++++ b/drivers/iio/pressure/Makefile +@@ -9,6 +9,7 @@ obj-$(CONFIG_ABP2030PA) += abp2030pa.o + obj-$(CONFIG_ABP2030PA_I2C) += abp2030pa_i2c.o + obj-$(CONFIG_ABP2030PA_SPI) += abp2030pa_spi.o + obj-$(CONFIG_ADP810) += adp810.o ++obj-$(CONFIG_APPLE_AOP_PRESSURE) += apple_aop_pressure.o + obj-$(CONFIG_ROHM_BM1390) += rohm-bm1390.o + obj-$(CONFIG_BMP280) += bmp280.o + bmp280-objs := bmp280-core.o bmp280-regmap.o +diff --git a/drivers/iio/pressure/apple_aop_pressure.c b/drivers/iio/pressure/apple_aop_pressure.c +new file mode 100644 +index 000000000..8cd983c57 +--- /dev/null ++++ b/drivers/iio/pressure/apple_aop_pressure.c +@@ -0,0 +1,257 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Barometric pressure sensor served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the pressure sensor itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one report (EPIC subtype ++ * 0xc4) per report interval: ++ * ++ * 0 u16 sequence ++ * 2 u32 pressure in 1/65536 kPa ++ * 6 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_PRESSURE 0xc4 ++#define AOP_PRESSURE_LEN 10 ++#define AOP_PRESSURE_VALUE 2 ++ ++#define AOP_PRESSURE_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_PRESSURE_MIN_US 10000 /* 100 Hz */ ++#define AOP_PRESSURE_MAX_US 1000000 /* 1 Hz */ ++ ++struct apple_aop_pressure { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ u32 sample; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(u32, scan, 1); ++}; ++ ++static int apple_aop_pressure_set_u32(struct apple_aop_pressure *pressure, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(pressure->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_pressure_set_interval(struct apple_aop_pressure *pressure, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ pressure->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_pressure_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ ++ if (subtype != AOP_REPORT_PRESSURE || len < AOP_PRESSURE_LEN) ++ return; ++ ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ pressure->sample = get_unaligned_le32(data + AOP_PRESSURE_VALUE); ++ pressure->have_sample = true; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ pressure->scan[0] = pressure->sample; ++ iio_push_to_buffers_with_ts(indio_dev, pressure->scan, sizeof(pressure->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_pressure_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ *val = pressure->sample; ++ ret = pressure->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ /* kPa */ ++ *val = 1; ++ *val2 = 16; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&pressure->lock); ++ *val = USEC_PER_SEC / pressure->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % pressure->interval_us) * USEC_PER_SEC / ++ pressure->interval_us; ++ mutex_unlock(&pressure->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_pressure_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_PRESSURE_MIN_US, AOP_PRESSURE_MAX_US); ++ ++ mutex_lock(&pressure->lock); ++ ret = apple_aop_pressure_set_interval(pressure, interval); ++ mutex_unlock(&pressure->lock); ++ return ret; ++} ++ ++static const struct iio_chan_spec apple_aop_pressure_channels[] = { ++ { ++ .type = IIO_PRESSURE, ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), ++ .scan_index = 0, ++ .scan_type = { ++ .sign = 'u', ++ .realbits = 32, ++ .storagebits = 32, ++ .endianness = IIO_CPU, ++ }, ++ }, ++ IIO_CHAN_SOFT_TIMESTAMP(1), ++}; ++ ++static const struct iio_info apple_aop_pressure_info = { ++ .read_raw = apple_aop_pressure_read_raw, ++ .write_raw = apple_aop_pressure_write_raw, ++}; ++ ++/* The offset or the acceleration sensitivity makes it complete */ ++static const struct apple_aop_calibration apple_aop_pressure_calibration[] = { ++ { "apple,pressure-offset-calibration", 0x0b, 1 }, ++ { "apple,pressure-accel-sensitivity", 0xd3, 1 }, ++ { "apple,pressure-heater-resistance", 0xd4, 0 }, ++}; ++ ++static void apple_aop_pressure_stop(void *data) ++{ ++ struct apple_aop_pressure *pressure = data; ++ ++ apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(pressure->service); ++} ++ ++static int apple_aop_pressure_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_pressure *pressure; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*pressure)); ++ if (!indio_dev) ++ return -ENOMEM; ++ pressure = iio_priv(indio_dev); ++ pressure->dev = dev; ++ pressure->service = service; ++ mutex_init(&pressure->lock); ++ spin_lock_init(&pressure->sample_lock); ++ ++ indio_dev->name = "apple-aop-pressure"; ++ indio_dev->info = &apple_aop_pressure_info; ++ indio_dev->channels = apple_aop_pressure_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_pressure_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_pressure_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_pressure_calibration, ++ ARRAY_SIZE(apple_aop_pressure_calibration)); ++ if (!ret) ++ ret = apple_aop_pressure_set_interval(pressure, AOP_PRESSURE_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_pressure_stop, pressure); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_pressure_of_match[] = { ++ { .compatible = "apple,s8000-aop-pressure" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_pressure_of_match); ++ ++static struct platform_driver apple_aop_pressure_driver = { ++ .driver = { ++ .name = "apple-aop-pressure", ++ .of_match_table = apple_aop_pressure_of_match, ++ }, ++ .probe = apple_aop_pressure_probe, ++}; ++module_platform_driver(apple_aop_pressure_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP barometric pressure sensor driver"); ++MODULE_LICENSE("Dual MIT/GPL"); diff --git a/kernel/patches/drivers/pmdomain/0001-aop-implementation.patch b/kernel/patches/drivers/pmdomain/0001-aop-implementation.patch new file mode 100644 index 00000000..2fd722aa --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0001-aop-implementation.patch @@ -0,0 +1,24 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pmdomain: apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..a0559873c 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -417,6 +417,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + goto err_remove; + } + ++ platform_set_drvdata(pdev, ps); + return 0; + err_remove: + if (ps->has_genpd) { diff --git a/kernel/patches/drivers/soc-apple/0014-aop-bindings.patch b/kernel/patches/drivers/soc-apple/0014-aop-bindings.patch new file mode 100644 index 00000000..aaf9d2d5 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0014-aop-bindings.patch @@ -0,0 +1,127 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] dt-bindings: soc-apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +index ba035ff80..ea414a8f4 100644 +--- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml ++++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +@@ -59,6 +59,12 @@ properties: + items: + - const: apple,t8010-a7iop-mailbox + ++ - description: ++ The same A7 IOP mailbox on the A9. ++ items: ++ - const: apple,s8000-a7iop-mailbox ++ - const: apple,t8010-a7iop-mailbox ++ + reg: + maxItems: 1 + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +index 14726afa0..5d2485ac9 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +@@ -4,14 +4,14 @@ + $id: http://devicetree.org/schemas/soc/apple/apple,t8010-aop.yaml# + $schema: http://devicetree.org/meta-schemas/core.yaml# + +-title: Apple T8010 Always-On Processor ++title: Apple S8000 and T8010 Always-On Processor + + maintainers: + - Dalton Alexandre + + description: +- The Always-On Processor (AOP) of the A10 is an ARMv7 coprocessor that +- runs RTKit firmware and serves the motion sensors (accelerometer, ++ The Always-On Processor (AOP) of the A9 and A10 is an ARMv7 coprocessor ++ that runs RTKit firmware and serves the motion sensors (accelerometer, + gyroscope, compass, pressure) and the always-on microphones. It runs from + local SRAM, with a further firmware segment in system memory, and talks + to the application processor through an A7 IOP mailbox. Its firmware is +@@ -19,7 +19,9 @@ description: + + properties: + compatible: +- const: apple,t8010-aop ++ enum: ++ - apple,s8000-aop ++ - apple,t8010-aop + + reg: + items: +@@ -50,6 +52,38 @@ properties: + The chip's unique ID (ECID), filled in by the bootloader. The + firmware's boot arguments carry it. + ++ apple,aop-target: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The AOP target number from the Apple device tree, filled in by the ++ bootloader. The S8000 firmware's boot arguments carry it. ++ ++ apple,chip-id: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip ID (e.g. 0x8003), filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ apple,chip-revision: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip revision, filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ memory-region: ++ maxItems: 1 ++ description: Reserved 16 KiB shutdown transcript page for J71t warm reload. ++ ++ apple,observe-domains: ++ $ref: /schemas/types.yaml#/definitions/phandle-array ++ minItems: 5 ++ maxItems: 5 ++ description: AOP GPIO, AOP, bus interface, CPU and filter in lock order. ++ ++ apple,spi0-domain: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: Read-only SPI0 power-state provider observation. ++ + patternProperties: + "^(accel|gyro|compass|pressure)$": + type: object +@@ -58,6 +92,9 @@ patternProperties: + properties: + compatible: + enum: ++ - apple,s8000-aop-accel ++ - apple,s8000-aop-gyro ++ - apple,s8000-aop-pressure + - apple,t8010-aop-accel + - apple,t8010-aop-gyro + - apple,t8010-aop-compass +@@ -65,6 +102,15 @@ patternProperties: + + mount-matrix: + description: The sensor's mounting rotation in the device. ++ ++ patternProperties: ++ "^apple,(low-temp-accel-offset|(accel|gyro)-(sensitivity|interrupt)-calibration|(accel|gyro)-range-sensitivity-cal|gyro-trim-calibration|pressure-(offset-calibration|accel-sensitivity|heater-resistance))$": ++ $ref: /schemas/types.yaml#/definitions/uint8-array ++ maxItems: 64 ++ description: ++ The unit's factory calibration of the sensor, copied by the ++ bootloader from the property of the same name (without the ++ prefix) in the Apple device tree. The S8000 firmware takes it. + required: + - compatible + additionalProperties: false diff --git a/kernel/patches/drivers/soc-apple/0015-aop-implementation.patch b/kernel/patches/drivers/soc-apple/0015-aop-implementation.patch new file mode 100644 index 00000000..31a196d9 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0015-aop-implementation.patch @@ -0,0 +1,3285 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] soc: apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/MAINTAINERS b/MAINTAINERS +index a4a69643c..12c292825 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -2068,6 +2068,8 @@ APPLE T8010 ALWAYS-ON PROCESSOR + M: Dalton Alexandre + S: Maintained + F: Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++F: drivers/iio/gyro/apple_aop_gyro.c ++F: drivers/iio/pressure/apple_aop_pressure.c + F: drivers/soc/apple/aop.c + F: include/linux/soc/apple/aop.h + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 17c017425..a4f6fbce4 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -479,3 +481,84 @@ MODULE_AUTHOR("Hector Martin "); + MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); + + module_platform_driver(apple_pmgr_ps_driver); ++ ++/* ++ * Read-only observer for bring-up. The caller's nodes follow the same ++ * child -> parent order as genpd power transitions. No domain is attached, ++ * resumed, pinned on, reset, or written by this function. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data) ++{ ++ struct platform_device *providers[5]; ++ struct apple_pmgr_ps *ps[5]; ++ unsigned long flags; ++ unsigned int i, j, found = 0; ++ int ret = 0; ++ bool accessible = true; ++ ++ if (!count || count > ARRAY_SIZE(providers) || !read) ++ return -EINVAL; ++ for (i = 0; i < count; i++) { ++ for (j = 0; j < i; j++) ++ if (nodes[i] == nodes[j]) { ++ ret = -EINVAL; ++ goto put; ++ } ++ providers[i] = of_find_device_by_node(nodes[i]); ++ if (!providers[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ device_lock(&providers[i]->dev); ++ found++; ++ if (!providers[i]->dev.driver || ++ strcmp(providers[i]->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ ps[i] = platform_get_drvdata(providers[i]); ++ if (!ps[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ } ++ ++ /* PMGR domains use GENPD_FLAG_IRQ_SAFE, i.e. genpd.slock. */ ++ local_irq_save(flags); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->genpd.slock, i); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->lock, i); ++ for (i = 0; i < count; i++) { ++ states[i].genpd_status = ps[i]->genpd.status; ++ ret = regmap_read(ps[i]->regmap, ps[i]->offset, &states[i].raw); ++ if (ret) ++ goto unlock; ++ if (states[i].genpd_status != GENPD_STATE_ON || ++ FIELD_GET(APPLE_PMGR_PS_ACTUAL, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ FIELD_GET(APPLE_PMGR_PS_TARGET, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ states[i].raw & (APPLE_PMGR_RESET | APPLE_PMGR_DEV_DISABLE | ++ APPLE_PMGR_PARENT_OFF)) ++ accessible = false; ++ } ++ if (accessible) ++ read(data); ++ else ++ ret = -EHOSTDOWN; ++unlock: ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->lock); ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->genpd.slock); ++ local_irq_restore(flags); ++put: ++ while (found) { ++ found--; ++ device_unlock(&providers[found]->dev); ++ put_device(&providers[found]->dev); ++ } ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_observe); +diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig +index e68a49f1b..c0af67b62 100644 +--- a/drivers/soc/apple/Kconfig ++++ b/drivers/soc/apple/Kconfig +@@ -39,16 +39,16 @@ config APPLE_RTKIT + Say 'y' here if you have an Apple SoC. + + config APPLE_AOP_T8010 +- tristate "Apple T8010 Always-On Processor" ++ tristate "Apple S8000 and T8010 Always-On Processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help +- The Always-On Processor (AOP) of the Apple A10 runs the firmware for +- the motion sensors and the always-on microphones. This driver loads +- and starts that firmware and provides its services to the sensor +- drivers. ++ The Always-On Processor (AOP) of the Apple A9 and A10 runs the ++ firmware for the motion sensors and the always-on microphones. This ++ driver loads and starts that firmware and provides its services to ++ the sensor drivers. + +- Say 'y' or 'm' here if you have an A10-based iPad or iPhone. ++ Say 'y' or 'm' here if you have an A9- or A10-based iPad or iPhone. + + config APPLE_PMP_T8015_V1 + tristate "Apple T8015 PMP" +diff --git a/drivers/soc/apple/aop-warm-record.h b/drivers/soc/apple/aop-warm-record.h +new file mode 100644 +index 000000000..a735809d5 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-record.h +@@ -0,0 +1,64 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#include ++#include ++#define A9_HO_PAGE 0x4000U ++#define A9_HO_LOG_OFFSET 0x800U ++#define A9_SL_MAGIC 0x4c533941U /* "A9SL" */ ++#define A9_SL_VERSION 1U ++ ++enum a9_ho_step { ++ A9_HO_STEP_BEGIN, ++ A9_HO_STEP_RATES, ++ A9_HO_STEP_AFK_SENT, ++ A9_HO_STEP_AFK_ACKED, ++ A9_HO_STEP_DISABLED, ++ A9_HO_STEP_AP_SENT, ++ A9_HO_STEP_AP_ACKED, ++ A9_HO_STEP_IOP_SENT, ++ A9_HO_STEP_IOP_ACKED, ++ A9_HO_STEP_RAM_STABLE, ++ A9_HO_STEP_MAILBOX_IDLE, ++ A9_HO_STEP_HALTED, ++ A9_HO_STEP_DRAINED, ++ A9_HO_STEPS, ++}; ++ ++static const char * const a9_ho_step_names[A9_HO_STEPS] = { ++ "begin", "rates-zero", "afk-sent", "afk-acked", "endpoint-disable", ++ "ap-sent", "ap-ack", "iop-sent", "iop-ack", "ram-stable", "mailbox-idle", ++ "halt", "drain", ++}; ++ ++struct a9_sl_header { ++ u32 magic, version, capacity, length; ++ u32 lines, truncated, checksum, mode; ++ u64 begin_ns; ++ u8 writer[16]; ++ char entry[64]; ++}; ++ ++_Static_assert(sizeof(struct a9_sl_header) == 120, "shutdown log ABI"); ++ ++#define A9_SL_CAPACITY (A9_HO_PAGE - A9_HO_LOG_OFFSET - sizeof(struct a9_sl_header)) ++ ++/* FNV-1a; incremental over appended text. */ ++static inline u32 a9_sl_hash(u32 hash, const char *text, size_t length) ++{ ++ size_t i; ++ ++ for (i = 0; i < length; i++) ++ hash = (hash ^ (u8)text[i]) * 16777619U; ++ return hash; ++} ++ ++#define A9_SL_HASH_SEED 2166136261U ++ ++/* A replayable previous transcript: complete header, bounded, intact. */ ++static inline bool a9_sl_valid(const struct a9_sl_header *h, const char *text) ++{ ++ return h->magic == A9_SL_MAGIC && h->version == A9_SL_VERSION && ++ h->capacity == A9_SL_CAPACITY && h->length <= h->capacity && ++ !h->entry[sizeof(h->entry) - 1] && ++ a9_sl_hash(A9_SL_HASH_SEED, text, h->length) == h->checksum; ++} ++ +diff --git a/drivers/soc/apple/aop-warm-spi0.c.h b/drivers/soc/apple/aop-warm-spi0.c.h +new file mode 100644 +index 000000000..bf883ffe8 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.c.h +@@ -0,0 +1,1340 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Ported from a9/aop-rc1: wake16, release6-d254a9f790. */ ++static void a9_aop_spi0_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_REGS; i++) ++ values[i] = readl(base + a9_spi0_offsets[i]); ++} ++ ++/* ++ * AOPSPIMaster_reset (0102f2f4) under the AOP-domain PMGR guard with IRQs ++ * off and the AOP CPU halted. Its provider power transition 3 is an ++ * AOP-local call with no AP register equivalent and is not replayed. A poll ++ * timeout still restores the saved control word. ++ */ ++static void a9_aop_spi0_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, s->before); ++ s->saved = s->before[A9_SPI0_I_CONTROL]; ++ s->asserted = a9_spi0_base_word(s->saved) | A9_SPI0_RESET; ++ writel(s->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ s->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(s->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(s->saved, control); ++ s->reset_done_ns = ktime_get_ns(); ++ a9_aop_spi0_snapshot(s->base, s->after); ++ if (!ret && (s->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ s->error = ret; ++} ++ ++/* ++ * Wake8 device evidence: SCM was 0 before/after the first controller reset, ++ * then became 3 at the channel-3 stop store. There is no firmware-derived ++ * SCM status write. Replay AOPSPIMaster_reset after the teardown so the ++ * controller reset, rather than an invented SCM write, is the one variable. ++ */ ++static void a9_aop_spi0_post_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, r->before); ++ r->saved = r->before[A9_SPI0_I_CONTROL]; ++ r->asserted = a9_spi0_base_word(r->saved) | A9_SPI0_RESET; ++ writel(r->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ r->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(r->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(r->saved, control); ++ a9_aop_spi0_snapshot(s->base, r->after); ++ if (!ret && (r->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ r->error = ret; ++} ++ ++static void a9_aop_spi0_ch3_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_CH3_REGS; i++) ++ values[i] = readl(base + a9_spi0_ch3_offsets[i]); ++} ++ ++static void a9_aop_spi0_trace_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int channel, i; ++ ++ s->trace.control = readl(s->base + A9_SPI0_CONTROL); ++ s->trace.clock = readl(s->base + A9_SPI0_CLOCK); ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->trace.scm[i] = readl(s->base + a9_spi0_scm_offsets[i]); ++ a9_aop_spi0_ch3_snapshot(s->base, s->trace.ch3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) ++ for (i = 0; i < A9_SPI0_CHANNEL_REGS; i++) ++ s->trace.channel[channel][i] = ++ readl(s->base + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel + ++ a9_spi0_channel_offsets[i]); ++} ++ ++/* Warm path only, CPU halted. Log outside the PMGR guard; no SCM writes. */ ++static int a9_aop_spi0_trace(struct apple_aop *aop, const char *stage) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *c = s->trace.ch3, *m = s->trace.scm; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_trace_read, s); ++ if (ret) { ++ s->intc.failed = "failed-spi0-trace-guard"; ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=%d: guard refused\n", stage, ret); ++ return ret; ++ } ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=0 ctl=%08x clk=%08x scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ stage, s->trace.control, s->trace.clock, m[0], m[1], m[2], m[3], ++ c[0], c[1], c[2], c[3], c[4], c[5], c[6]); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ v = s->trace.channel[channel]; ++ dev_info(aop->dev, "SPI0 SCM channel stage=%s ch=%02u ctl=%08x route=%08x rate=%08x cfg=%08x interval=%08x cursor=%08x desc=%08x pending0=%u pending1=%u\n", ++ stage, channel, v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ !!(c[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(c[A9_CH3_I_8D4] & BIT(2 * channel))); ++ } ++ return 0; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s); ++ ++/* ++ * Wake10 safety gate, strengthened in wake11 to retain every selected-channel ++ * teardown invariant. A fault observed by a copy sample remains terminal even ++ * if it is transient and the final read happens to clear. ++ */ ++static int a9_aop_spi0_gate(struct apple_aop *aop, const char *stage, ++ const char *check, bool include_copy) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *m, *c; ++ int ret; ++ ++ ret = a9_aop_spi0_trace(aop, stage); ++ if (ret) ++ return ret; ++ m = s->trace.scm; ++ c = s->trace.ch3; ++ if (m[0] || m[1]) ++ s->gate_failed = "failed-spi0-scm"; ++ else if (c[A9_CH3_I_8D0] || c[A9_CH3_I_8D4]) ++ s->gate_failed = "failed-spi0-fault"; ++ else if (include_copy && s->copy_faulted) ++ s->gate_failed = "failed-spi0-copy-sample"; ++ else { ++ s->gate_failed = a9_spi0_ch3_verify(c, s->teardown_stable & BIT(3)); ++ if (!s->gate_failed) ++ s->gate_failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = s->gate_failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 %s check errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x copy_faulted=%u result=%s\n", ++ check, ret, m[0], m[1], c[A9_CH3_I_8D0], c[A9_CH3_I_8D4], ++ s->copy_faulted, s->gate_failed ?: "pass"); ++ return ret; ++} ++ ++#define A9_SPI0_COPY_CHUNK 256U ++#define A9_SPI0_COPY_SAMPLE_US 5000U ++ ++struct a9_aop_spi0_copy_probe { ++ void __iomem *base; ++ u32 scm_lo, scm_hi, fault, fault_8d4; ++}; ++ ++static void a9_aop_spi0_copy_probe_read(void *data) ++{ ++ struct a9_aop_spi0_copy_probe *p = data; ++ ++ p->scm_lo = readl(p->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(p->base + A9_SPI0_SCM_HI); ++ p->fault = readl(p->base + A9_SPI0_FAULT); ++ p->fault_8d4 = readl(p->base + A9_SPI0_FAULT_8D4); ++} ++ ++/* ++ * Read-only samples spaced by at least 5 ms across both halves of the SRAM ++ * copy. Once a nonzero state is seen it remains a terminal fact, but the copy ++ * completes so later samples and the full after-copy channel snapshot expose ++ * whether the source is level, transient or moves between channels. ++ */ ++static int a9_aop_spi0_copy_sample(struct apple_aop *aop, const char *phase, ++ size_t offset, size_t size, u64 start_ns, ++ u64 *last_us, unsigned int *sample, ++ bool force) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_copy_probe p = { .base = s->base }; ++ u64 elapsed_us = div_u64(ktime_get_boottime_ns() - start_ns, ++ NSEC_PER_USEC); ++ bool first_fault, initial = !*sample; ++ int ret; ++ ++ if (!force && *sample && elapsed_us - *last_us < A9_SPI0_COPY_SAMPLE_US) ++ return 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_copy_probe_read, &p); ++ first_fault = !s->copy_faulted && ++ (p.scm_lo || p.scm_hi || p.fault || p.fault_8d4); ++ if (!ret && first_fault) ++ s->copy_faulted = true; ++ dev_info(aop->dev, "SPI0 COPY sample=%u phase=%s elapsed_us=%llu offset=0x%zx size=0x%zx scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ *sample, phase, elapsed_us, offset, size, p.scm_lo, p.scm_hi, ++ p.fault, p.fault_8d4, ret ? "guard" : s->copy_faulted ? "fault" : "pass"); ++ *last_us = elapsed_us; ++ (*sample)++; ++ if (ret) ++ return ret; ++ if (first_fault) { ++ ret = a9_aop_spi0_trace(aop, "copy-fault"); ++ if (ret) ++ return ret; ++ } ++ /* The offset-zero sample is the last no-write gate before the copy. */ ++ return !initial || !s->copy_faulted ? 0 : -EIO; ++} ++ ++static int a9_aop_copy_verify_wake10(struct apple_aop *aop, const u8 *src, ++ size_t size) ++{ ++ u32 chunk[A9_SPI0_COPY_CHUNK / 4]; ++ u64 start_ns = ktime_get_boottime_ns(), last_us = 0; ++ unsigned int sample = 0; ++ size_t done, n; ++ int ret; ++ ++ ret = a9_aop_spi0_copy_sample(aop, "write", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __iowrite32_copy(aop->sram + done, src + done, n / 4); ++ ret = a9_aop_spi0_copy_sample(aop, "write", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_copy_sample(aop, "verify", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __ioread32_copy(chunk, aop->sram + done, n / 4); ++ if (memcmp(chunk, src + done, n)) ++ return -EIO; ++ ret = a9_aop_spi0_copy_sample(aop, "verify", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void __iomem *a9_aop_spi0_channel_base(struct a9_aop_spi0 *s, ++ unsigned int channel) ++{ ++ return s->base + A9_SPI0_CHANNEL_BASE + A9_SPI0_CHANNEL_STRIDE * channel; ++} ++ ++/* ++ * Wake15: which other channels were RUN with a counting interval when this ++ * stop was issued, and which of them advanced across the stop. Read only; it ++ * separates a stop that had a ticking peer from the stop of the last live ++ * channel in the log. It never gates anything in-kernel. ++ */ ++static void a9_aop_spi0_peers_begin(struct a9_aop_spi0 *s, unsigned int channel, ++ u32 *interval, u32 *run) ++{ ++ unsigned int i; ++ u32 ctl; ++ ++ *run = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ interval[i] = 0; ++ if (i == channel) ++ continue; ++ ctl = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ interval[i] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ if ((ctl & A9_SPI0_CH3_RUN) && interval[i]) ++ *run |= BIT(i); ++ } ++} ++ ++static void a9_aop_spi0_stop_timing(struct a9_aop_spi0 *s, ++ struct a9_aop_spi0_ch3 *c, ++ const u32 *interval) ++{ ++ unsigned int i; ++ ++ c->peers_ticking = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ if ((c->peers_run & BIT(i)) && ++ readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]) != ++ interval[i]) ++ c->peers_ticking |= BIT(i); ++ } ++ c->reset_to_stop_us = c->stop_ns > s->reset_done_ns ? ++ div_u64(c->stop_ns - s->reset_done_ns, NSEC_PER_USEC) : 0; ++} ++ ++/* ++ * Wake12/13 captured channel 7, and wake13 channel 3, stopped but never idle: ++ * for 20 complete slices the control/progress word, config, interval, cursor, ++ * SCM and both fault banks were unchanged. This predicate is intentionally ++ * narrower than "RUN clear and no progress": the channel must be one of the ++ * two device-observed periodic clients, must carry that client's route, rate ++ * and config before the stop, and every sample must be byte-identical with SCM ++ * 3/0 (every earlier non-idle stop, which followed a channel-3 stop that had ++ * latched it) or 0/0 (a first stop that latches nothing), no fault, no cursor ++ * movement and a retained nonzero interval. ++ * It only permits the callback masks, descriptor clear, config acknowledge and ++ * the established controller reset; every later SCM, fault and copy gate stays. ++ */ ++static const char *a9_aop_spi0_stable_why(struct a9_aop_spi0_ch3 *c, ++ const struct a9_aop_channel_diag_slice *diag, ++ unsigned int channel) ++{ ++ const struct a9_aop_ch3_slice *first; ++ unsigned int i; ++ u32 route; ++ ++ if (channel == A9_SPI0_PHOSPHORUS_CHANNEL) ++ route = A9_SPI0_PHOSPHORUS_ROUTE; ++ else if (channel == A9_SPI0_MPU_ASYNC_CHANNEL) ++ route = A9_SPI0_MPU_ASYNC_ROUTE; ++ else ++ return "channel"; ++ if (c->poll != -ETIMEDOUT || c->slices != A9_SPI0_CH3_SLICES) ++ return "no-timeout"; ++ if (!(c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) || ++ c->before[A9_CH3_I_ROUTE] != route || ++ c->before[A9_CH3_I_RATE] != A9_SPI0_PHOSPHORUS_RATE || ++ c->before[A9_CH3_I_C3C] != A9_SPI0_PHOSPHORUS_CFG || ++ c->before[A9_CH3_I_DESC] || (c->stop & A9_SPI0_CH3_RUN)) ++ return "before-identity"; ++ if (c->before[A9_CH3_I_8D4] || (c->before[A9_CH3_I_8D0] & ~BIT(2 * channel))) ++ return "before-fault"; ++ ++ first = &c->slice[0]; ++ if ((first->c3 & A9_SPI0_STABLE_CTL_MASK) != A9_SPI0_STABLE_CTL_STOPPED || ++ (first->c3 & (A9_SPI0_CH3_RUN | A9_SPI0_CH3_IDLE | A9_SPI0_CH3_IRQ1))) ++ return "stopped-control"; ++ if ((first->scm_lo != 0 && first->scm_lo != 3) || first->scm_hi) ++ return "scm"; ++ if (first->fault || diag[0].fault_8d4) ++ return "sample-fault"; ++ if (diag[0].cfg != A9_SPI0_PHOSPHORUS_CFG || diag[0].cursor || !diag[0].interval) ++ return "cfg-cursor-interval"; ++ ++ for (i = 1; i < A9_SPI0_CH3_SLICES; i++) { ++ const struct a9_aop_ch3_slice *p = &c->slice[i]; ++ ++ if (p->c3 != first->c3 || p->scm_lo != first->scm_lo || ++ p->scm_hi != first->scm_hi || p->fault != first->fault || ++ diag[i].cfg != diag[0].cfg || ++ diag[i].interval != diag[0].interval || ++ diag[i].cursor != diag[0].cursor || ++ diag[i].fault_8d4 != diag[0].fault_8d4) ++ return "progress"; ++ } ++ if (c->slice[A9_SPI0_CH3_SLICES - 1].ns < ++ (u64)A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US * NSEC_PER_USEC) ++ return "short"; ++ return NULL; ++} ++ ++/* ++ * AOPSPIMaster_teardownChannel (0102e4f8) for channel 3, then the fault ++ * handler's acknowledge (0102f20a), under the same guard as the reset with ++ * the CPU halted. Every store is a read-modify-write except the descriptor, ++ * which the firmware zeroes. The idle wait runs in 1 ms slices, up to twice ++ * the firmware's own 10 ms SCMto bound; each slice ends with read-only ++ * samples of C3, the SCM state and +0x8d0 (wake15: and config, interval, ++ * cursor and +0x8d4). An idle timeout stops before the masks unless the ++ * stable-stop predicate holds. The acknowledge bit self-clears; +0x8d0 bit 6 ++ * shows whether it took effect. ++ */ ++static void a9_aop_spi0_ch3_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ void __iomem *c3 = s->base + A9_SPI0_CH3; ++ void __iomem *c3c = s->base + A9_SPI0_CH3_0C; ++ void __iomem *base = a9_aop_spi0_channel_base(s, 3); ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_ch3_snapshot(s->base, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(c3) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, c3); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, 3, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(c3, value, value & A9_SPI0_CH3_IDLE, ++ 1, A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->ch3_diag[c->slices]; ++ d->cfg = readl(c3c); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->ch3_diag, 3); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(3); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(c3) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, c3); ++ c->class1 = readl(c3) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, c3); ++ writel(0, s->base + A9_SPI0_CH3_DESC); ++ c->ack = readl(c3c) | A9_SPI0_CH3_ACK; ++ writel(c->ack, c3c); ++ } ++ a9_aop_spi0_ch3_snapshot(s->base, c->after); ++ c->failed = c->poll && !c->stable ? "failed-ch3-teardown-idle" : ++ a9_spi0_ch3_verify(c->after, c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++ ++static void a9_aop_spi0_channel_snapshot(struct a9_aop_spi0 *s, ++ unsigned int channel, u32 *values) ++{ ++ void __iomem *base = a9_aop_spi0_channel_base(s, channel); ++ ++ values[A9_CH3_I_C3] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ values[A9_CH3_I_ROUTE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_ROUTE]); ++ values[A9_CH3_I_RATE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_RATE]); ++ values[A9_CH3_I_C3C] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]); ++ values[A9_CH3_I_DESC] = readl(base + A9_SPI0_CHANNEL_DESC); ++ values[A9_CH3_I_8D0] = readl(s->base + A9_SPI0_FAULT); ++ values[A9_CH3_I_8D4] = readl(s->base + A9_SPI0_FAULT_8D4); ++} ++ ++static const char *a9_spi0_channel_state_verify(u32 control, u32 desc, ++ u32 fault, u32 fault_8d4, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ u32 pending = BIT(2 * channel); ++ ++ if (control & A9_SPI0_CH3_RUN) ++ return "failed-channel-teardown-run"; ++ if (control & A9_SPI0_CH3_IRQ0) ++ return "failed-channel-teardown-class0"; ++ if (control & A9_SPI0_CH3_IRQ1) ++ return "failed-channel-teardown-class1"; ++ if (!(control & A9_SPI0_CH3_IDLE) && !stable_stop) ++ return "failed-channel-teardown-idle"; ++ if (desc) ++ return "failed-channel-teardown-desc"; ++ if ((fault | fault_8d4) & pending) ++ return "failed-channel-teardown-pending"; ++ return NULL; ++} ++ ++static const char *a9_spi0_channel_verify(const u32 *after, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ return a9_spi0_channel_state_verify(after[A9_CH3_I_C3], ++ after[A9_CH3_I_DESC], ++ after[A9_CH3_I_8D0], ++ after[A9_CH3_I_8D4], channel, ++ stable_stop); ++} ++ ++/* ++ * Wake11 applies the channel-3 firmware sequence unchanged to another slot. ++ * Wake12 only appends read-only scheduler samples to that established poll. ++ */ ++static void a9_aop_spi0_channel_locked(void *data) ++{ ++ struct a9_aop_spi0_channel_op *op = data; ++ struct a9_aop_spi0 *s = op->spi0; ++ struct a9_aop_spi0_ch3 *c = op->state; ++ void __iomem *base = a9_aop_spi0_channel_base(s, op->channel); ++ void __iomem *control = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]; ++ void __iomem *config = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]; ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(control) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, control); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, op->channel, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(control, value, ++ value & A9_SPI0_CH3_IDLE, 1, ++ A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->channel_diag[c->slices]; ++ d->cfg = readl(config); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->channel_diag, op->channel); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(op->channel); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(control) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, control); ++ c->class1 = readl(control) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, control); ++ writel(0, base + A9_SPI0_CHANNEL_DESC); ++ c->ack = readl(config) | A9_SPI0_CH3_ACK; ++ writel(c->ack, config); ++ } ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->after); ++ c->failed = c->poll && !c->stable ? "failed-channel-teardown-idle" : ++ a9_spi0_channel_verify(c->after, op->channel, ++ c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++/* Called only if ps_spi0 is fully on; reads nothing. */ ++static void a9_aop_spi0_pmgr_on(void *data) ++{ ++ *(bool *)data = true; ++} ++ ++/* AP PMGR ps_spi0, read-only: never requested, resumed or written. ++ * 0 = on, -EHOSTDOWN = off (raw still filled), -ENODEV = no node. ++ */ ++static int a9_aop_spi0_pmgr(struct apple_aop *aop, u32 *raw) ++{ ++ struct apple_pmgr_observation state = {}; ++ bool on = false; ++ int ret; ++ ++ if (!aop->spi0_pmgr) ++ return -ENODEV; ++ ret = apple_pmgr_observe(&aop->spi0_pmgr, 1, &state, ++ a9_aop_spi0_pmgr_on, &on); ++ *raw = state.raw; ++ return ret ?: on ? 0 : -EIO; ++} ++ ++/* ++ * After a reset that cleared +0x8d0 bit 6, before the image copy: CPU halted, ++ * AP notifications masked. A refusal sets ch3.failed and nothing is loaded. ++ */ ++static int a9_aop_spi0_ch3_teardown(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *b = c->before, *a = c->after; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ unsigned int i; ++ bool source_progress = false; ++ int ret; ++ ++ aop->experiment_step = "spi0-ch3-teardown"; ++ dev_info(aop->dev, "SPI0 CH3TD begin pa=%llx cpu=halted: first channel-3 store follows\n", ++ A9_SPI0_PA + A9_SPI0_CH3); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, "spi0-ch3-teardown"); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_ch3_locked, s); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH3TD errno=%d ran=0: guard refused, channel 3 not touched\n", ++ ret); ++ return ret; ++ } ++ c->ran = true; ++ dev_info(aop->dev, "SPI0 CH3TD before c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH3TD idle-sample slice=%u at_us=%llu c3=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ i, div_u64(p->ns, NSEC_PER_USEC), p->c3, p->scm_lo, ++ p->scm_hi, p->fault, !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->ch3_diag[i]; ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ i, p->c3, d->cfg, d->interval, d->cursor, p->scm_lo, ++ p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->ch3_diag[0].interval || ++ d->cursor != s->ch3_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->ch3_diag[0].interval, ++ s->ch3_diag[c->slices - 1].interval, ++ s->ch3_diag[0].cursor, ++ s->ch3_diag[c->slices - 1].cursor, source_progress, ++ c->stable ? "stable-stop-reset-only" : "refused"); ++ /* beyond_fw_bound=1: idle, but later than the firmware would wait. */ ++ dev_info(aop->dev, "SPI0 CH3TD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ c->stop, c->idle, c->poll); ++ else if (c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->poll, c->class0, c->class1, c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH3TD after c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH3TD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ /* c3c_bit5 is the pulse's readback, logged only (0 at 19:50). */ ++ dev_info(aop->dev, "SPI0 CH3TD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x c3c_bit5=%u ch3_pending=%u result=%s\n", ++ c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), ++ a[A9_CH3_I_DESC], !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), c->failed ?: ++ c->stable ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static int a9_aop_spi0_channel_teardown(struct apple_aop *aop, ++ unsigned int channel) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->channel; ++ struct a9_aop_spi0_channel_op op = { ++ .spi0 = s, ++ .state = c, ++ .channel = channel, ++ }; ++ const u32 *b, *a; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ char checkpoint[32]; ++ unsigned int i; ++ bool source_progress = false; ++ bool stable_stop; ++ int ret; ++ ++ memset(c, 0, sizeof(*c)); ++ memset(s->channel_diag, 0, sizeof(s->channel_diag)); ++ s->teardown_failed_channel = channel; ++ aop->experiment_step = "spi0-channel-teardown"; ++ snprintf(checkpoint, sizeof(checkpoint), "spi0-ch%u-teardown", channel); ++ dev_info(aop->dev, "SPI0 CH%uTD begin pa=%llx cpu=halted: first channel-%u store follows\n", ++ channel, A9_SPI0_PA + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel, channel); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, checkpoint); ++ if (ret) { ++ c->failed = "failed-channel-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_channel_locked, &op); ++ if (ret) { ++ c->failed = "failed-channel-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH%uTD errno=%d ran=0: guard refused, channel %u not touched\n", ++ channel, ret, channel); ++ return ret; ++ } ++ c->ran = true; ++ stable_stop = s->teardown_stable & BIT(channel); ++ b = c->before; ++ a = c->after; ++ dev_info(aop->dev, "SPI0 CH%uTD before ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD idle-sample slice=%u at_us=%llu ctl=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ channel, i, div_u64(p->ns, NSEC_PER_USEC), p->c3, ++ p->scm_lo, p->scm_hi, p->fault, ++ !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->channel_diag[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ channel, i, p->c3, d->cfg, d->interval, d->cursor, ++ p->scm_lo, p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->channel_diag[0].interval || ++ d->cursor != s->channel_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ channel, c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->channel_diag[0].interval, ++ s->channel_diag[c->slices - 1].interval, ++ s->channel_diag[0].cursor, ++ s->channel_diag[c->slices - 1].cursor, ++ source_progress, ++ stable_stop ? "stable-stop-reset-only" : "refused"); ++ dev_info(aop->dev, "SPI0 CH%uTD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ channel, c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ channel, c->stop, c->idle, c->poll); ++ else if (stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->poll, c->class0, c->class1, ++ c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH%uTD after ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH%uTD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ channel, c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ dev_info(aop->dev, "SPI0 CH%uTD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x cfg_bit5=%u pending0=%u pending1=%u result=%s\n", ++ channel, c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), a[A9_CH3_I_DESC], ++ !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(a[A9_CH3_I_8D4] & BIT(2 * channel)), c->failed ?: ++ stable_stop ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s) ++{ ++ const u32 *v; ++ const char *failed = NULL; ++ unsigned int channel; ++ ++ if (s->teardown_selected != s->teardown_done) ++ return "failed-channel-teardown-incomplete"; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ v = s->trace.channel[channel]; ++ failed = a9_spi0_channel_state_verify(v[A9_CHANNEL_I_CTL], ++ v[A9_CHANNEL_I_DESC], ++ s->trace.ch3[A9_CH3_I_8D0], ++ s->trace.ch3[A9_CH3_I_8D4], channel, ++ s->teardown_stable & BIT(channel)); ++ if (failed) { ++ s->teardown_failed_channel = channel; ++ return failed; ++ } ++ } ++ return NULL; ++} ++ ++/* ++ * Channel 3 remains mandatory and executes first. Wake11 additionally selects ++ * every snapshot slot with run bit 0 (RUN) set and a nonzero interval, which ++ * excludes configured-but-idle slots while selecting any periodic client still ++ * owned by the old firmware. Wake16 returns to wake11's order after wake15's ++ * channel-7-first stop failed: all ten device stops fit "channel 3 idles when ++ * stopped within ~2 ms of the reset (5/5; 0/2 at 11 and 33 ms) and channel 7 ++ * never idles (0/3, at 0.3 ms, 1.9 ms and 15 ms)". Channel 3 goes first and ++ * early; channel 7 is then the stopped-without-idle channel the stable-stop ++ * fingerprint was captured from (wake11/12). ++ */ ++static int a9_aop_spi0_teardown_running(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ s->teardown_selected = BIT(3); ++ s->teardown_done = 0; ++ s->teardown_stable = 0; ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3) ++ continue; ++ v = s->trace.channel[channel]; ++ if ((v[A9_CHANNEL_I_CTL] & A9_SPI0_CH3_RUN) && ++ v[A9_CHANNEL_I_INTERVAL]) ++ s->teardown_selected |= BIT(channel); ++ } ++ dev_info(aop->dev, "SPI0 teardown select stage=after-controller-reset selected=%04x mandatory=0008 rule=ch3-or-run-and-interval order=ch3-then-others\n", ++ s->teardown_selected); ++ ret = a9_aop_spi0_ch3_teardown(aop); ++ if (ret) { ++ s->teardown_failed_channel = 3; ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=3 errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, ret, ++ s->ch3.failed ?: "failed-ch3-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ ret = a9_aop_spi0_channel_teardown(aop, channel); ++ if (ret) { ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=%u errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, channel, ret, ++ s->channel.failed ?: "failed-channel-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(channel); ++ } ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ dev_info(aop->dev, "SPI0 teardown complete selected=%04x done=%04x stable=%04x result=pass\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable); ++ return 0; ++} ++ ++/* ++ * Second copy of the firmware controller reset, after the stop has completed. ++ * The preceding after-ch3-teardown trace is the read-only before snapshot; ++ * after-post-teardown-reset is the matching read-only after snapshot. Refuse ++ * the image copy unless SCM is zero and every channel quiet-state gate holds. ++ */ ++static int a9_aop_spi0_post_reset(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ const u32 *b = r->before, *a = r->after; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.ran || ++ s->ch3.error || s->teardown_selected != s->teardown_done || r->ran) { ++ r->failed = "failed-post-reset-state"; ++ return -EPERM; ++ } ++ aop->experiment_step = "spi0-post-teardown-reset"; ++ dev_info(aop->dev, "SPI0 POSTRST begin pa=%llx reason=ch3-stop-latched-scm cpu=halted\n", ++ A9_SPI0_PA); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-post-teardown-reset"); ++ if (ret) { ++ r->failed = "failed-post-reset-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_post_reset_locked, s); ++ r->ran = !ret; ++ r->error = ret ?: r->error; ++ dev_info(aop->dev, "SPI0 POSTRST errno=%d ran=%d before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ r->error, r->ran, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 POSTRST after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], r->saved, r->asserted, ++ r->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ if (r->error) { ++ r->failed = "failed-post-reset-controller"; ++ return r->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-post-teardown-reset"); ++ if (ret) ++ return ret; ++ if (s->trace.scm[0] || s->trace.scm[1]) ++ r->failed = "failed-post-reset-scm"; ++ else { ++ r->failed = a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3)); ++ if (!r->failed) ++ r->failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = r->failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 POSTRST check errno=%d scm_lo=%08x scm_hi=%08x ch3_pending=%u result=%s\n", ++ ret, s->trace.scm[0], s->trace.scm[1], ++ !!(s->trace.ch3[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), ++ r->failed ?: "pass"); ++ dev_info(aop->dev, "SPI0 POSTRST channels selected=%04x done=%04x stable=%04x result=%s\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable, ++ r->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * The ten INTC words of a9_intc_offsets into state[@at], read only. The ++ * INTC is the AOP wrapper block (0xc1800000 in the firmware, space 2 as ++ * SPI0's verified 0xc1100000 = 0x210100000), which this driver reads ++ * unguarded for the mailbox and CPU control; no separate PMGR domain. ++ */ ++static void a9_aop_intc_snapshot(struct apple_aop *aop, unsigned int at) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ u32 *v = n->state[at]; ++ unsigned int i; ++ ++ for (i = 0; i < A9_INTC_REGS; i++) ++ v[i] = aop_rd(aop, a9_intc_offsets[i]); ++ n->states |= BIT(at); ++ dev_info(aop->dev, "AOP INTC state at=%s reset808=%08x current81c=%08x raise900=%08x ack980=%08x mask_a00=%08x mask_a04=%08x mask_a08=%08x mask_a80=%08x g7_c020=%08x g7_c024=%08x src_masked=%u view=%s\n", ++ a9_intc_at_name(at), v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ v[7], v[8], v[9], !!(v[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(v)); ++} ++ ++/* ++ * Round 4 (ds-aop-spi0-ch3-round4.md): the old session's SPI0 global IRQ ++ * 0x10014 stays latched in the AOP interrupt controller, and fresh firmware ++ * takes it as soon as SPI lifecycle event 1 enables it, before its own SPI0 ++ * reset (SCMErr:0x3,3). With the CPU halted and before the first SPI0 ++ * access, disable that source (group 1 disable bank, 0102c9ce). The bank ++ * reads back the mask, as the firmware's own reads of it show: bit 20 must ++ * read 1 after the store. ++ */ ++static int a9_aop_intc_disable(struct apple_aop *aop) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ const u32 *b = n->state[A9_INTC_AT_PRE_DISABLE]; ++ const u32 *a = n->state[A9_INTC_AT_POST_DISABLE]; ++ int ret; ++ ++ aop->experiment_step = "intc-disable"; ++ dev_info(aop->dev, "AOP INTC disable begin irq=0x10014 pa=%llx value=%08x cpu=halted: first INTC store follows\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "intc-disable"); ++ if (ret) { ++ n->failed = "failed-intc-disable-checkpoint"; ++ return ret; ++ } ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_DISABLE); ++ ret = aop_wr(aop, A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_DISABLE); ++ if (ret) { ++ n->failed = "failed-intc-disable"; ++ } else if (!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM)) { ++ n->failed = "failed-intc-disable-readback"; ++ ret = -EIO; ++ } ++ n->disabled = !ret; ++ dev_info(aop->dev, "AOP INTC disable errno=%d before=%08x value=%08x after=%08x mask_a80=%08x src_masked=%u view=%s result=%s\n", ++ ret, b[A9_INTC_I_MASK0], A9_INTC_SPI0_SCM, a[A9_INTC_I_MASK0], ++ a[A9_INTC_I_ENABLE0], !!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(a), n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Release policy for aop_spi0_reset=1. The reload is device-proven for exactly ++ * one retained state: the previous kernel quiesced and halted the firmware ++ * (wake16, 2026-10-29; decision prev-quiesce-halted). Auto-boot still set, a ++ * previous kernel that did not halt it, or an older kernel in the chain are ++ * different hardware states that no run has reloaded with the final teardown. ++ * Refuse them before the first halt or SPI0 access so the old firmware is ++ * left exactly as found; aop_spi0_untested=1 attempts them anyway. A skip ++ * (DFU cold, bad record, nothing ran) must also find the wrapper at its reset ++ * state: a record-less AOP that something already stopped is refused rather ++ * than cold-loaded. A stale record skips as before. ++ */ ++static int a9_aop_spi0_policy(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ enum a9_spi0_reason reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, ++ &aop->spi0.prev); ++ ++ if (a9_spi0_policy_allows(reason, outbox & AOP_OUTBOX_ENABLE, spi0_untested)) ++ return 0; ++ dev_info(aop->dev, "SPI0 policy decision=refuse reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d: only prev-quiesce-halted reloads and a skip needs the wrapper at its reset state block not touched\n", ++ a9_spi0_reason_name(reason), auto_boot, outbox, ++ a9_spi0_prev_name(aop->spi0.prev.state), aop->spi0.prev.hello, ++ aop->spi0.prev.halted); ++ return -EPERM; ++} ++ ++/* ++ * After the proven halt/drain, before any SRAM write. Only firmware that ran ++ * since the last SoC reset leaves SPI0 state behind: auto-boot 1, or the ++ * previous kernel's transcript bound by +0x4020 (a9_spi0_decide). Without ++ * that evidence the block is never touched. @auto_boot and @outbox are the ++ * boot snapshot from before the stop. Nonzero means: do not load or ++ * release the AOP. ++ */ ++static int a9_aop_spi0_prereset(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *b = s->before, *a = s->after; ++ const char *failed = NULL; ++ bool run; ++ int ret; ++ ++ s->auto_boot = auto_boot; ++ s->outbox = outbox; ++ s->reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, &s->prev); ++ s->decided = true; ++ run = a9_spi0_reason_runs(s->reason); ++ dev_info(aop->dev, "SPI0 reset decision=%s reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d prev_steps=0x%x prev_carry=%d%s\n", ++ run ? "run" : "skip", a9_spi0_reason_name(s->reason), auto_boot, ++ outbox, a9_spi0_prev_name(s->prev.state), s->prev.hello, ++ s->prev.halted, s->prev.steps, s->prev.carried, ++ run ? "" : ": block not touched"); ++ if (!run) { ++ s->skipped = a9_spi0_reason_name(s->reason); ++ return 0; ++ } ++ if (!s->base || !aop->force_halted || !aop->force_drained || ++ !a9_aop_force_stopped(aop)) ++ return -EPERM; ++ ret = a9_aop_intc_disable(aop); ++ if (ret) ++ return ret; ++ aop->experiment_step = "spi0-reset"; ++ s->pmgr_before_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_before); ++ /* Reaches ramoops before the first AP access to the block. */ ++ dev_info(aop->dev, "SPI0 reset begin pa=%llx cpu=halted ps_spi0=%08x errno=%d: first AP access follows\n", ++ A9_SPI0_PA, s->pmgr_before, s->pmgr_before_errno); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-reset"); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_trace(aop, "before-controller-reset"); ++ if (ret) ++ return ret; ++ /* ++ * Wake15: from the controller reset to the last channel stop nothing may ++ * drain a console in this task. Holding the console lock makes every ++ * dev_info() below only append to the log buffer; the flushed spi0-reset ++ * checkpoint above already put the begin line into ramoops, and the lines ++ * are emitted by console_unlock() once the stops are stored. ++ */ ++ console_lock(); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_reset_locked, s); ++ s->ran = !ret; ++ s->error = ret ?: s->error; ++ s->pmgr_after_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_after); ++ dev_info(aop->dev, "SPI0 reset errno=%d ran=%d pa=%llx before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ s->error, s->ran, A9_SPI0_PA, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 reset after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u provider3=not-replayed\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], s->saved, s->asserted, ++ s->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 pmgr ps_spi0 before=%08x errno=%d after=%08x errno=%d (read-only, never powered)\n", ++ s->pmgr_before, s->pmgr_before_errno, s->pmgr_after, ++ s->pmgr_after_errno); ++ if (s->error) { ++ console_unlock(); ++ return s->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-controller-reset"); ++ if (!ret) ++ ret = a9_aop_spi0_teardown_running(aop); ++ console_unlock(); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-ch3-teardown"); ++ if (!ret) ++ failed = a9_aop_spi0_selected_verify(s); ++ if (failed) { ++ s->channel.failed = failed; ++ ret = -EIO; ++ } ++ if (!ret) ++ ret = a9_aop_spi0_post_reset(aop); ++ return ret; ++} ++ ++/* Wake10 samples both SCM words and both fault classes, not only channel 3. */ ++static void a9_aop_spi0_fault_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ ++ s->prerun_scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ s->prerun_scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ s->prerun = readl(s->base + A9_SPI0_FAULT); ++ s->prerun_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ a9_aop_spi0_ch3_snapshot(s->base, s->ch3.reassert); ++} ++ ++/* ++ * Last AP look before RUN, CPU still halted, after the image copy: bit 6 ++ * must stay clear in every one of 31 guarded samples spaced >= 5 ms apart, ++ * spanning >= 150 ms. A reasserted channel-3 bit would crash the fresh ++ * image as before, so the first set sample refuses the release; nothing is ++ * retried. Any refusal sets ch3.failed. ++ */ ++static int a9_aop_spi0_recheck(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *r = c->reassert; ++ u64 start = ktime_get_boottime_ns(); ++ unsigned int i; ++ int ret = 0; ++ ++ for (i = 0; i < A9_SPI0_WINDOW_SAMPLES; i++) { ++ if (i) ++ usleep_range(A9_SPI0_WINDOW_US, A9_SPI0_WINDOW_US + 1000); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_fault_read, s); ++ c->samples = i + 1; ++ c->elapsed_ms = div_u64(ktime_get_boottime_ns() - start, NSEC_PER_MSEC); ++ if (ret) { ++ c->failed = "failed-ch3-window-guard"; ++ break; ++ } ++ if (s->prerun_scm_lo || s->prerun_scm_hi || s->prerun || ++ s->prerun_8d4) { ++ s->reasserted = c->reasserted = true; ++ c->reassert_sample = i; ++ c->reassert_ms = c->elapsed_ms; ++ c->failed = "failed-spi0-window-reasserted"; ++ ret = -EIO; ++ dev_info(aop->dev, "SPI0 WINDOW reasserted sample=%u at_ms=%llu scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ i, c->reassert_ms, s->prerun_scm_lo, s->prerun_scm_hi, ++ s->prerun, s->prerun_8d4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ dev_info(aop->dev, "SPI0 WINDOW channel3 c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ r[0], r[1], r[2], r[3], r[4], r[5], r[6]); ++ break; ++ } ++ } ++ /* usleep_range() cannot return early; this only trips on a bad clock. */ ++ if (!ret && c->elapsed_ms < A9_SPI0_WINDOW_MS) { ++ c->failed = "failed-ch3-window-short"; ++ ret = -EIO; ++ } ++ c->window_done = !ret; ++ dev_info(aop->dev, "SPI0 CH3TD window errno=%d samples=%u/%u elapsed_ms=%llu min_ms=%u result=%s\n", ++ ret, c->samples, A9_SPI0_WINDOW_SAMPLES, c->elapsed_ms, ++ A9_SPI0_WINDOW_MS, ret ? c->failed : "pass"); ++ dev_info(aop->dev, "SPI0 pre-run errno=%d 8d0=%08x ch3_pending=%u cpu=halted\n", ++ ret, s->prerun, !!(s->prerun & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 WINDOW state errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ ret, s->prerun_scm_lo, s->prerun_scm_hi, s->prerun, ++ s->prerun_8d4, ret ? c->failed : "pass"); ++ if (ret && c->reasserted) ++ a9_aop_spi0_trace(aop, "window-fault"); ++ return ret; ++} ++ ++/* SCM state and the channel-3 interval pair, read-only, into scm[scm_at]. */ ++static void a9_aop_spi0_scm_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->intc.scm[s->intc.scm_at][i] = readl(s->base + a9_spi0_scm_offsets[i]); ++} ++ ++/* ++ * After the clean pre-RUN window, CPU still halted: the firmware's group-1 ++ * pre-handler acknowledge for 0x10014 (0102c6de..0102c6f2) discards the ++ * old-session latch now that its SPI0 cause is quiet. Fresh firmware ++ * re-enables the source in event 1. The INTC words, the SCM state and the ++ * C3 interval pair are logged around it and never gated: with 0x10014 ++ * masked, no readable word shows whether it is still pending. ++ */ ++static int a9_aop_intc_ack(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ const u32 *p = n->scm[0], *q = n->scm[1]; ++ const u32 *b = n->state[A9_INTC_AT_PRE_ACK]; ++ const u32 *a = n->state[A9_INTC_AT_POST_ACK]; ++ int ret; ++ ++ aop->experiment_step = "intc-ack"; ++ n->scm_at = 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) { ++ n->failed = "failed-intc-ack-guard"; ++ dev_info(aop->dev, "AOP INTC pre-ack errno=%d: guard refused, no acknowledge\n", ++ ret); ++ return ret; ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_ACK); ++ dev_info(aop->dev, "AOP INTC pre-ack scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x cpu=halted\n", ++ p[0], p[1], p[2], p[3], b[A9_INTC_I_CURRENT]); ++ ret = aop_wr(aop, A9_INTC_ACK, A9_INTC_SPI0_SCM); ++ /* Also the posted-write barrier. */ ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_ACK); ++ n->acked = !ret; ++ dev_info(aop->dev, "AOP INTC ack errno=%d before=%08x value=%08x after=%08x\n", ++ ret, b[A9_INTC_I_ACK], A9_INTC_SPI0_SCM, a[A9_INTC_I_ACK]); ++ if (ret) { ++ n->failed = "failed-intc-ack"; ++ return ret; ++ } ++ n->scm_at = 1; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) ++ n->failed = "failed-intc-post-guard"; ++ dev_info(aop->dev, "AOP INTC post-ack errno=%d scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x result=%s\n", ++ ret, q[0], q[1], q[2], q[3], a[A9_INTC_I_CURRENT], n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Wake7 still took 0x10014 before SPI event 5 despite the targeted ack. ++ * Replay INTC lifecycle event 3 (0102c592/598/59c/5a2): 1/read/0/read at ++ * +0x808. aop_wr includes a posted-write read after each store. This resets ++ * all AOP-private interrupt state; it does not write the SCM status or MPU. ++ * No assumption that SCM must become zero. Only the final reset readback ++ * and the established channel-3 quiet conditions gate RUN. ++ */ ++static int a9_aop_intc_reset(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.window_done || ++ !n->acked || n->reset_done || ++ (aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) || readl(aop->control)) { ++ n->failed = "failed-intc-reset-state"; ++ return -EPERM; ++ } ++ ret = a9_aop_spi0_trace(aop, "pre-intc-reset"); ++ if (ret) ++ return ret; ++ aop->experiment_step = "intc-reset"; ++ dev_info(aop->dev, "AOP INTC reset begin pa=%llx sequence=1/read/0/read cpu=halted\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_RESET); ++ ret = aop_wr(aop, A9_INTC_RESET, 1); ++ n->reset_asserted = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-assert"; ++ return ret; ++ } ++ ret = aop_wr(aop, A9_INTC_RESET, 0); ++ n->reset_after = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-deassert"; ++ return ret; ++ } ++ if (n->reset_after) { ++ n->failed = "failed-intc-reset-readback"; ++ ret = -EIO; ++ } else { ++ ret = a9_aop_spi0_trace(aop, "post-intc-reset"); ++ if (!ret && a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3))) { ++ n->failed = "failed-intc-reset-ch3"; ++ ret = -EIO; ++ } else if (!ret && a9_aop_spi0_selected_verify(s)) { ++ n->failed = "failed-intc-reset-channel"; ++ ret = -EIO; ++ } ++ } ++ n->reset_done = !ret; ++ dev_info(aop->dev, "AOP INTC reset errno=%d asserted=%08x after=%08x result=%s\n", ++ ret, n->reset_asserted, n->reset_after, n->failed ?: "pass"); ++ return ret; ++} ++ +diff --git a/drivers/soc/apple/aop-warm-spi0.h b/drivers/soc/apple/aop-warm-spi0.h +new file mode 100644 +index 000000000..2470165ce +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.h +@@ -0,0 +1,543 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * J71t AOP reload/wake experiments, both opt-in and default off. Constants ++ * and evidence types only; the MMIO lives in apple-a9-aop-active.c. ++ * Evidence: docs/periph/ds-aop-reload-crash-and-wake.md. ++ * ++ * aop_spi0_reset=1: every halt + SRAM reload of a live AOP crashed in ++ * AOPSPIMaster_channelFaultIRQ (0102f252, DFAR 0x4c): the AOP-private SPI0 ++ * controller kept channel 3's +0x8d0 bit 6 while the fresh firmware's ++ * channel table was still empty. With the CPU halted, the AP replays the ++ * firmware's own controller reset AOPSPIMaster_reset (0102f2f4), then the ++ * firmware's own channel-3 teardown, and samples bit 6 for >= 150 ms after ++ * the image copy before it releases RUN once. Round 4: the old session's ++ * SPI0 global IRQ 0x10014 stays latched in the AOP interrupt controller ++ * (SCMErr:0x3,3 at the accel STARTEP), so the AP disables that source before ++ * the reset and acknowledges it after the clean window, just before RUN. ++ * Wake7: the channel-3 idle wait gets the firmware's own 10 ms bound twice ++ * over, sampled per 1 ms slice, and the disable is judged by the INTC mask ++ * readback, which the firmware itself reads. ++ * Wake8: after the targeted ack, replay INTC event 3 (+0x808: 1/read/0/read). ++ * This resets retained interrupt delivery, not the read-only SCM state. ++ * Wake9: wake8 device evidence showed the channel-3 stop itself changes SCM ++ * from 0 to 3. Replay the same SPI controller reset once more after teardown; ++ * require the post-reset SCM state to be zero and the teardown state to hold. ++ * Wake10: the second reset clears SCM, but a halted CPU then saw SCM 3 and ++ * +0x8d0 bit 14 (channel 7) during the firmware copy. Snapshot all 16 channel ++ * slots, sample global SCM/fault state across the copy, and gate the copy and ++ * final RUN on a completely quiet readback. ++ * Wake11: that snapshot proved channels 3 and 7 were both running. Keep the ++ * channel-3 sequence unchanged, then apply the same firmware teardown to every ++ * other run-bit + nonzero-interval channel before the post-teardown reset. ++ * Wake12: channel 7 is the phosphorus/BMP284 async client. Its firmware has no ++ * owner-specific stop path, so do not invent another store when the generic ++ * stop times out. Record the remaining scheduler state read-only and refuse. ++ * Wake13/14: every device stop of channel 3 that idled (wake8..wake12) came ++ * 1.5-2.2 ms after the controller reset; the two that did not (wake13/14) came ++ * 11/33 ms after it (pr_flush in the checkpoint), with class-0 state identical to ++ * wake12's passes. Wake15 closed that window (no pr_flush between the reset and ++ * the stops, console lock held), recorded the reset-to-stop latency and the peers ++ * per stop, and accepted one identity-pinned stopped-without-idle channel (3 or ++ * 7) as wake13 did for channel 7. Wake15's channel-7-first order then showed ++ * channel 7 never idles even at 0.3 ms (it is 0/3 over wake11/12/15), with ++ * nothing ticking. Wake16 therefore returns to channel 3 first (early, the only ++ * stop that ever idled) and then channel 7, which takes the stable-stop path ++ * (SCM 0/0 or 3/0, constant); every stop logs why the fingerprint did or did not ++ * hold (stable_why=). Wake14's class-0 prelude stays removed. ++ * ++ * aop_wake_wrapper=1: before 0x220, the AppleA7IOP status-3 resume wrapper ++ * writes (b4b854..b4b8b4). No CPU RUN, no auto-boot, no NMI. ++ */ ++#ifndef _A9_AOP_RELOAD_H ++#define _A9_AOP_RELOAD_H ++ ++#include ++#include ++#include "aop-warm-record.h" ++ ++/* Firmware device spi0: IOP 0x10100000 (space 2, 0x1000) via /arm-io ranges. */ ++#define A9_SPI0_PA 0x210100000ULL ++#define A9_SPI0_SIZE 0x1000U ++#define A9_SPI0_WRAPPER_PA 0x210800000ULL /* the only layout this applies to */ ++#define A9_SPI0_CONTROL 0x000U ++#define A9_SPI0_CLOCK 0x008U /* 0102f2aa programs 0x01b71b00 here */ ++#define A9_SPI0_FAULT 0x8d0U /* bit 2n = channel n, read by 0102f20a */ ++#define A9_SPI0_FAULT_8D4 0x8d4U ++#define A9_SPI0_CHANNEL_BASE 0x100U ++#define A9_SPI0_CHANNEL_STRIDE 0x080U ++#define A9_SPI0_CHANNELS 16U ++#define A9_SPI0_CHANNEL_REGS 7U ++#define A9_SPI0_CHANNEL_DESC 0x018U ++#define A9_SPI0_CH3 0x280U /* base + 0x100 + 0x80 * 3 */ ++#define A9_SPI0_CH3_0C 0x28cU ++#define A9_SPI0_RESET 0x100U /* control bit 8 */ ++#define A9_SPI0_ENABLE 0x001U /* control bit 0 */ ++#define A9_SPI0_CH3_PENDING 0x040U ++/* The firmware spins without bound; the AP gives each poll 1 ms. */ ++#define A9_SPI0_POLL_US 1000U ++#define A9_SPI0_DELAY_US 100U /* 0102b3f8(100, 0), twice */ ++#define A9_SPI0_REGS 6 ++ ++static const u32 a9_spi0_offsets[A9_SPI0_REGS] = { ++ A9_SPI0_CONTROL, A9_SPI0_CLOCK, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++ A9_SPI0_CH3, A9_SPI0_CH3_0C, ++}; ++ ++/* Snapshot indices into a9_spi0_offsets. */ ++enum { A9_SPI0_I_CONTROL, A9_SPI0_I_CLOCK, A9_SPI0_I_FAULT }; ++ ++/* ++ * Channel-3 teardown after the reset (ds-aop-spi0-ch3-round3.md section 5). ++ * The reset leaves C3 = 0x00010447: idle but still enabled, async-armed and ++ * with the class-0 fault IRQ on, so a level source re-latched +0x8d0 bit 6 ++ * within ~105 ms with the CPU halted. AOPSPIMaster_teardownChannel ++ * (0102e4f8) stops the channel and masks both IRQ classes; the fault ++ * handler (0102f20a) then acknowledges with C3C bit 5. ++ */ ++#define A9_SPI0_CH3_ROUTE 0x284U ++#define A9_SPI0_CH3_RATE 0x288U ++#define A9_SPI0_CH3_DESC 0x298U /* channel +0x18, zeroed on teardown */ ++#define A9_SPI0_CH3_RUN 0x001U /* C3 bit 0, channel run/enable */ ++#define A9_SPI0_CH3_IDLE 0x002U /* C3 bit 1, polled after the stop */ ++#define A9_SPI0_CH3_IRQ0 0x040U /* C3 bit 6, +0x8d0 class-0/fault enable */ ++#define A9_SPI0_CH3_IRQ1 0x100U /* C3 bit 8, +0x8d4 class-1 enable */ ++#define A9_SPI0_CHANNEL_PROGRESS_SHIFT 24U /* hardware-updated, not an IRQ class */ ++#define A9_SPI0_CH3_STOP 0x411U /* clear run bit 0 and bits 4, 10 */ ++#define A9_SPI0_CH3_CLASS0 0x450U /* mask class 0: bits 4, 6, 10 */ ++#define A9_SPI0_CH3_CLASS1 0x510U /* mask class 1: bits 4, 8, 10 */ ++#define A9_SPI0_CH3_ACK 0x020U /* C3C bit 5, ORed by 0102f20a; self-clearing */ ++#define A9_SPI0_PHOSPHORUS_CHANNEL 7U ++#define A9_SPI0_PHOSPHORUS_CTL_MASK 0x00ffffffU ++#define A9_SPI0_PHOSPHORUS_CTL_RUN 0x00010445U ++#define A9_SPI0_PHOSPHORUS_CTL_STOPPED 0x00010444U ++#define A9_SPI0_PHOSPHORUS_ROUTE 0x00000040U ++#define A9_SPI0_PHOSPHORUS_RATE 0x019000c8U ++#define A9_SPI0_PHOSPHORUS_CFG 0x00200009U ++/* Channel 3 is the MPU6900 async FIFO client (route 0x10, same rate/config). */ ++#define A9_SPI0_MPU_ASYNC_CHANNEL 3U ++#define A9_SPI0_MPU_ASYNC_ROUTE 0x00000010U ++/* Stopped control word, any top byte, class-0 enable optional (0x40). */ ++#define A9_SPI0_STABLE_CTL_STOPPED 0x00010404U ++#define A9_SPI0_STABLE_CTL_MASK 0x00ffffbfU ++/* Peer sampling and the "late stop" mark are reported, never gated, in-kernel. */ ++#define A9_SPI0_LATE_STOP_US 3000U ++/* ++ * AOPSPIMaster_setChannelEnabled(false) (0102ef7c) stores C3 & ~0x411 only ++ * if bit 0 was set, then spins on bit 1 until master +0x30, and panics ++ * "SCMto:%u" (0x010398da) past it. Event 1 sets +0x30 to 0x2710 us when it ++ * is 0 (0102e6fa, as in the live-cold object), through timer +0x18 = ++ * us * Hz / 1000000 (0102ff3c): a 10 ms bound. The AP polls in 1 ms slices, ++ * up to twice that, sampling C3, the SCM state and +0x8d0 after each slice. ++ */ ++#define A9_SPI0_CH3_IDLE_FW_US 10000U ++#define A9_SPI0_CH3_SLICE_US 1000U ++#define A9_SPI0_CH3_SLICES 20U /* 20 ms */ ++/* At least 150 ms (the 16:27 re-latch took ~105 ms) of samples before RUN. */ ++#define A9_SPI0_WINDOW_SAMPLES 31U ++#define A9_SPI0_WINDOW_US 5000U /* 30 sleeps >= 5 ms each */ ++#define A9_SPI0_WINDOW_MS 150U /* required first-to-last sample span */ ++#define A9_SPI0_CH3_REGS 7 ++ ++static const u32 a9_spi0_ch3_offsets[A9_SPI0_CH3_REGS] = { ++ A9_SPI0_CH3, A9_SPI0_CH3_ROUTE, A9_SPI0_CH3_RATE, A9_SPI0_CH3_0C, ++ A9_SPI0_CH3_DESC, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++}; ++ ++/* Every firmware-visible channel slot, read only in wake10 and selected in wake11. */ ++static const u32 a9_spi0_channel_offsets[A9_SPI0_CHANNEL_REGS] = { ++ 0x00U, 0x04U, 0x08U, 0x0cU, 0x10U, 0x14U, 0x18U, ++}; ++ ++enum { ++ A9_CHANNEL_I_CTL, A9_CHANNEL_I_ROUTE, A9_CHANNEL_I_RATE, A9_CHANNEL_I_CFG, ++ A9_CHANNEL_I_INTERVAL, A9_CHANNEL_I_CURSOR, A9_CHANNEL_I_DESC, ++}; ++ ++/* Snapshot indices into a9_spi0_ch3_offsets. */ ++enum { ++ A9_CH3_I_C3, A9_CH3_I_ROUTE, A9_CH3_I_RATE, A9_CH3_I_C3C, A9_CH3_I_DESC, ++ A9_CH3_I_8D0, A9_CH3_I_8D4, ++}; ++ ++/* One idle-poll slice: C3 as the slice ended, then SCM state and +0x8d0. */ ++struct a9_aop_ch3_slice { ++ u64 ns; /* since the stop store */ ++ u32 c3, scm_lo, scm_hi, fault; ++}; ++ ++/* Wake12 read-only scheduler evidence for an additional-channel stop poll. */ ++struct a9_aop_channel_diag_slice { ++ u32 cfg, interval, cursor, fault_8d4; ++}; ++ ++struct a9_aop_spi0_ch3 { ++ u32 before[A9_SPI0_CH3_REGS], after[A9_SPI0_CH3_REGS]; ++ u32 stop, idle, class0, class1, ack; /* words written / polled */ ++ int poll; /* idle-bit poll result */ ++ bool run_before; /* C3 bit 0 before the stop */ ++ unsigned int slices; /* idle-poll slices used */ ++ struct a9_aop_ch3_slice slice[A9_SPI0_CH3_SLICES]; ++ u64 idle_us; /* stop store to the idle read */ ++ int error; ++ bool ran; /* the guarded teardown executed */ ++ const char *failed; /* distinct refusal reason, or NULL */ ++ /* Wake15: stop latency after the controller reset and the peers' state. */ ++ u64 stop_ns, reset_to_stop_us; ++ u32 peers_run, peers_ticking; /* masks of the other 15 channels */ ++ bool stable; /* accepted stopped-without-idle identity */ ++ const char *stable_why; /* first failed fingerprint condition, NULL if stable */ ++ /* Pre-RUN window after the image copy; a refusal also sets failed. */ ++ unsigned int samples; ++ u64 elapsed_ms; ++ bool window_done, reasserted; ++ unsigned int reassert_sample; ++ u64 reassert_ms; ++ u32 reassert[A9_SPI0_CH3_REGS]; ++}; ++ ++struct a9_aop_spi0; ++ ++struct a9_aop_spi0_channel_op { ++ struct a9_aop_spi0 *spi0; ++ struct a9_aop_spi0_ch3 *state; ++ unsigned int channel; ++}; ++ ++/* ++ * AOP interrupt controller, wrapper offsets (ds-aop-spi0-ch3-round4.md ++ * section 3). IRQ 0x10014 is group 1, source 0x14: bank 0, bit 20. The ++ * group-1 disable/enable banks are write-one commands whose reads return ++ * the mask, 1 = disabled: the sleep command saves +0xa00..+0xa08 (0102c5ce) ++ * before masking all (0102c5da), the restore writes saved to +0xa00 and ++ * ~saved to +0xa80 (0102c9e6/0102c9f4), and aop_interrupt_write_enabled ++ * reads the target bank for the previous state (0102c9c0). So the disable ++ * is judged by bit 20 of the +0xa00 readback (wake6: ffffffff before the ++ * store, every source already masked by the quiesce). +0x81c only reports ++ * an unmasked pending source: the sleep path masks all, writes group 7 ++ * +0xc020/+0xc024 = 2 and spins until +0x81c reads 0 (0102c654), so ++ * all-ones there means the AP is not reading the INTC. +0x808 ends 0 after ++ * the event-3 pulse. +0x900 is the group-1 raise (0102c7c6) and +0x980 its ++ * pre-handler clear (0102c6f2); the firmware never reads them, so their ++ * readbacks are logged only. No register shows a masked source pending. ++ */ ++#define A9_INTC_RESET 0x808U /* event-3 pulse 1 then 0 (0102c592) */ ++#define A9_INTC_CURRENT 0x81cU /* current unmasked source */ ++#define A9_INTC_RAISE 0x900U /* group 1 raise, 0102c7c6 */ ++#define A9_INTC_ACK 0x980U /* group 1 pre-handler ack/clear, 0102c6f2 */ ++#define A9_INTC_DISABLE 0xa00U /* group 1 disable, store at 0102c9ce */ ++#define A9_INTC_ENABLE 0xa80U /* group 1 enable; reads the same mask */ ++#define A9_INTC_G7_DISABLE 0xc020U /* group 7 disable, 2 after sleep */ ++#define A9_INTC_SPI0_SCM 0x00100000U /* IRQ 0x10014 */ ++#define A9_INTC_ALL_ONES 0xffffffffU ++#define A9_INTC_REGS 10 ++ ++static const u32 a9_intc_offsets[A9_INTC_REGS] = { ++ A9_INTC_RESET, A9_INTC_CURRENT, A9_INTC_RAISE, A9_INTC_ACK, ++ A9_INTC_DISABLE, A9_INTC_DISABLE + 4, A9_INTC_DISABLE + 8, ++ A9_INTC_ENABLE, A9_INTC_G7_DISABLE, A9_INTC_G7_DISABLE + 4, ++}; ++ ++/* Snapshot indices into a9_intc_offsets. */ ++enum { ++ A9_INTC_I_RESET, A9_INTC_I_CURRENT, A9_INTC_I_RAISE, A9_INTC_I_ACK, ++ A9_INTC_I_MASK0, A9_INTC_I_MASK1, A9_INTC_I_MASK2, A9_INTC_I_ENABLE0, ++ A9_INTC_I_G7_0, A9_INTC_I_G7_1, ++}; ++ ++/* When the AP snapshots the INTC words. */ ++enum { ++ A9_INTC_AT_PRE_DISABLE, A9_INTC_AT_POST_DISABLE, A9_INTC_AT_PRE_ACK, ++ A9_INTC_AT_POST_ACK, A9_INTC_AT_N, ++}; ++ ++static inline const char *a9_intc_at_name(unsigned int at) ++{ ++ static const char * const names[] = { ++ "pre-disable", "post-disable", "pre-ack", "post-ack", ++ }; ++ ++ return at < A9_INTC_AT_N ? names[at] : "unknown"; ++} ++ ++/* "all-ones" if +0x81c reads ffffffff, which a live INTC cannot report. */ ++static inline const char *a9_intc_view(const u32 *state) ++{ ++ return state[A9_INTC_I_CURRENT] == A9_INTC_ALL_ONES ? "all-ones" : "intc"; ++} ++/* ++ * Read only, never written: the SCM state (also per idle-poll slice) and, ++ * around the acknowledge, the C3 interval pair. ++ */ ++#define A9_SPI0_SCM_LO 0x00cU ++#define A9_SPI0_SCM_HI 0x010U ++#define A9_SPI0_CH3_IVAL 0x290U ++#define A9_SPI0_CH3_CURSOR 0x294U ++#define A9_SPI0_SCM_REGS 4 ++ ++static const u32 a9_spi0_scm_offsets[A9_SPI0_SCM_REGS] = { ++ A9_SPI0_SCM_LO, A9_SPI0_SCM_HI, A9_SPI0_CH3_IVAL, A9_SPI0_CH3_CURSOR, ++}; ++ ++struct a9_aop_intc { ++ u32 state[A9_INTC_AT_N][A9_INTC_REGS]; /* around each store */ ++ unsigned int states; /* bit n: state[n] was read */ ++ u32 scm[2][A9_SPI0_SCM_REGS]; /* before / after the ack */ ++ unsigned int scm_at; /* scm[] row the guarded read fills */ ++ bool disabled, acked; ++ bool reset_done; ++ u32 reset_asserted, reset_after; ++ const char *failed; /* distinct refusal reason, or NULL */ ++}; ++ ++/* Read-only milestones. Wake10 gates the global SCM/fault words, not channels. */ ++struct a9_aop_spi0_trace { ++ u32 control, clock, scm[A9_SPI0_SCM_REGS], ch3[A9_SPI0_CH3_REGS]; ++ u32 channel[A9_SPI0_CHANNELS][A9_SPI0_CHANNEL_REGS]; ++}; ++ ++/* Wake9's second, post-channel-stop replay of AOPSPIMaster_reset. */ ++struct a9_aop_spi0_post_reset { ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved, asserted, polled; ++ int error; ++ bool ran; ++ const char *failed; ++}; ++ ++/* ++ * First failed readback after the teardown, or NULL if all hold. C3C bit 5 ++ * is a pulse: at 19:50 the write of 0x00200029 read back 0x00200009 while ++ * +0x8d0 bit 6 went 1 -> 0, so the acknowledge is judged by bit 6 alone and ++ * the C3C readback is only logged. ++ */ ++static inline const char *a9_spi0_ch3_verify(const u32 *after, bool stable) ++{ ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) ++ return "failed-ch3-teardown-run"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0) ++ return "failed-ch3-teardown-class0"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1) ++ return "failed-ch3-teardown-class1"; ++ if (!(after[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE) && !stable) ++ return "failed-ch3-teardown-idle"; ++ if (after[A9_CH3_I_DESC]) ++ return "failed-ch3-teardown-desc"; ++ if (after[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING) ++ return "failed-ch3-teardown-pending"; ++ return NULL; ++} ++ ++/* ++ * Has firmware run since the last SoC reset? auto-boot alone cannot say: ++ * the outgoing aop_quiesce=1 halt clears it, exactly like a DFU cold start. ++ * The previous kernel's reserved-RAM shutdown transcript says whether its ++ * firmware ran. That DRAM can outlive a DFU; wrapper +0x4020 outbox enable ++ * cannot: set after reset (12/12 DFU cold snapshots), cleared by the first ++ * store of every Linux stop. No SPI0 access is needed to decide. ++ */ ++enum a9_spi0_prev_state { ++ A9_SPI0_PREV_MISSING, /* no transcript magic, or none reserved */ ++ A9_SPI0_PREV_INVALID, /* magic present, header/checksum rejected */ ++ A9_SPI0_PREV_VALID, ++}; ++ ++struct a9_spi0_prev { ++ enum a9_spi0_prev_state state; ++ bool hello; /* "SHUTDOWN begin ... hello=1": its firmware ran */ ++ bool halted; /* "QUIESCE step=complete ... halted=1" + steps bit */ ++ bool carried; /* "SPI0-CARRY retained=1": ran earlier in the chain */ ++ u32 steps; /* "QUIESCE step=complete ... steps=0x..." */ ++}; ++ ++enum a9_spi0_reason { ++ /* Run the reset. */ ++ A9_SPI0_LIVE_AUTO_BOOT, /* auto-boot 1 before the stop */ ++ A9_SPI0_PREV_HALTED, /* previous kernel quiesced and halted it */ ++ A9_SPI0_PREV_LIVE, /* previous kernel's firmware ran, no halt */ ++ A9_SPI0_PREV_CARRIED, /* an earlier kernel in the chain ran it */ ++ /* Skip: the block is not touched. */ ++ A9_SPI0_FRESH, /* no transcript, auto-boot 0: DFU cold */ ++ A9_SPI0_PREV_BAD, /* transcript present but corrupt */ ++ A9_SPI0_PREV_NO_RUN, /* transcript: nothing ran in the chain */ ++ A9_SPI0_STALE, /* run evidence, wrapper at reset state */ ++}; ++ ++static inline const char *a9_spi0_reason_name(enum a9_spi0_reason reason) ++{ ++ static const char * const names[] = { ++ "live-auto-boot", "prev-quiesce-halted", "prev-live", "prev-carried", ++ "fresh-instance", "prev-transcript-invalid", "prev-no-run", ++ "stale-transcript", ++ }; ++ ++ return reason <= A9_SPI0_STALE ? names[reason] : "unknown"; ++} ++ ++static inline const char *a9_spi0_prev_name(enum a9_spi0_prev_state state) ++{ ++ return state == A9_SPI0_PREV_VALID ? "valid" : ++ state == A9_SPI0_PREV_INVALID ? "invalid" : "missing"; ++} ++ ++static inline bool a9_spi0_reason_runs(enum a9_spi0_reason reason) ++{ ++ return reason <= A9_SPI0_PREV_CARRIED; ++} ++ ++/* ++ * Release policy: a reason that would run the reload is allowed by default ++ * only for prev-quiesce-halted, the one state device-proven end to end. A ++ * skip (fresh, bad record, nothing ran) is allowed only while the wrapper is ++ * still at its reset state; stale records skip as before. Untested lifts both. ++ */ ++static inline bool a9_spi0_policy_allows(enum a9_spi0_reason reason, bool outbox_enabled, ++ bool untested) ++{ ++ if (untested || reason == A9_SPI0_PREV_HALTED || reason == A9_SPI0_STALE) ++ return true; ++ if (a9_spi0_reason_runs(reason)) ++ return false; ++ /* ++ * A skip means "nothing ran since the SoC reset", which only holds if the ++ * wrapper is still at its reset state (+0x4020 outbox enable set, 12/12 DFU ++ * cold starts). Without it some earlier kernel stopped the firmware and its ++ * record is lost: a cold load would land on a halted, never-torn-down AOP. ++ */ ++ return outbox_enabled; ++} ++ ++/* Token search inside one transcript line, which holds no NUL. */ ++static inline bool a9_spi0_line_has(const char *line, size_t length, ++ const char *token, size_t n, ++ const char **at) ++{ ++ size_t i; ++ ++ for (i = 0; n <= length && i <= length - n; i++) { ++ if (!memcmp(line + i, token, n)) { ++ *at = line + i + n; ++ return true; ++ } ++ } ++ return false; ++} ++ ++#define a9_spi0_has(line, length, token, at) \ ++ a9_spi0_line_has(line, length, token, sizeof(token) - 1, at) ++ ++static inline bool a9_spi0_line_starts(const char *line, size_t length, ++ const char *prefix, size_t n) ++{ ++ return length >= n && !memcmp(line, prefix, n); ++} ++ ++#define a9_spi0_starts(line, length, prefix) \ ++ a9_spi0_line_starts(line, length, prefix, sizeof(prefix) - 1) ++ ++static inline u32 a9_spi0_hex(const char *p, const char *end) ++{ ++ u32 value = 0; ++ ++ for (; p < end; p++) { ++ if (*p >= '0' && *p <= '9') ++ value = value << 4 | (u32)(*p - '0'); ++ else if (*p >= 'a' && *p <= 'f') ++ value = value << 4 | (u32)(*p - 'a' + 10); ++ else ++ break; ++ } ++ return value; ++} ++ ++/* Our own checksummed transcript; only whole-line prefixes count. */ ++static inline void a9_spi0_prev_parse(struct a9_spi0_prev *p, ++ const char *text, size_t length) ++{ ++ const char *end = text + length, *line, *next, *at; ++ size_t n; ++ ++ for (line = text; line < end; line = next < end ? next + 1 : end) { ++ next = memchr(line, '\n', end - line); ++ if (!next) ++ next = end; ++ n = next - line; ++ if (a9_spi0_starts(line, n, "SHUTDOWN begin ") && ++ a9_spi0_has(line, n, " hello=1 ", &at)) ++ p->hello = true; ++ if (a9_spi0_starts(line, n, "QUIESCE step=complete ")) { ++ if (a9_spi0_has(line, n, " steps=0x", &at)) ++ p->steps = a9_spi0_hex(at, next); ++ p->halted = a9_spi0_has(line, n, " halted=1 ", &at) && ++ (p->steps & (1U << A9_HO_STEP_HALTED)); ++ } ++ if (a9_spi0_starts(line, n, "SPI0-CARRY retained=1 ")) ++ p->carried = true; ++ } ++} ++ ++/* ++ * Skips unless there is positive evidence: running the reset on a fresh ++ * block would be the first AP access to SPI0 with nothing retained to ++ * clear (and ps_spi0 may gate it), while a skip on retained state only ++ * repeats the known, non-hanging accel STARTEP crash. ++ */ ++static inline enum a9_spi0_reason a9_spi0_decide(bool auto_boot, bool outbox_enabled, ++ const struct a9_spi0_prev *p) ++{ ++ if (auto_boot) ++ return A9_SPI0_LIVE_AUTO_BOOT; ++ if (p->state == A9_SPI0_PREV_MISSING) ++ return A9_SPI0_FRESH; ++ if (p->state != A9_SPI0_PREV_VALID) ++ return A9_SPI0_PREV_BAD; ++ if (!p->hello && !p->carried) ++ return A9_SPI0_PREV_NO_RUN; ++ if (outbox_enabled) ++ return A9_SPI0_STALE; ++ if (p->hello) ++ return p->halted ? A9_SPI0_PREV_HALTED : A9_SPI0_PREV_LIVE; ++ return A9_SPI0_PREV_CARRIED; ++} ++ ++struct a9_aop_spi0 { ++ void __iomem *base; /* mapped only with aop_spi0_reset=1 */ ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved; /* control word before the reset */ ++ u32 asserted; /* saved & ~0x101 | 0x100 */ ++ u32 polled; /* control word when the polls ended */ ++ u32 prerun; /* +0x8d0 just before RUN */ ++ u32 prerun_8d4, prerun_scm_lo, prerun_scm_hi; ++ u32 pmgr_before, pmgr_after; /* AP PMGR ps_spi0 raw, read-only */ ++ int pmgr_before_errno, pmgr_after_errno; ++ int error; ++ bool ran, reasserted, copy_faulted; ++ const char *gate_failed; ++ const char *skipped; ++ struct a9_spi0_prev prev; /* parsed at probe from the transcript */ ++ u32 auto_boot, outbox; /* boot snapshot, before the stop */ ++ enum a9_spi0_reason reason; ++ bool decided; ++ struct a9_aop_spi0_ch3 ch3; /* after a clean reset only */ ++ /* Wake11 reuses one bounded scratch record for each additional channel. */ ++ struct a9_aop_spi0_ch3 channel; ++ u32 teardown_selected, teardown_done; ++ unsigned int teardown_failed_channel; ++ struct a9_aop_spi0_post_reset post_reset; /* after the stop latches SCM */ ++ struct a9_aop_intc intc; /* IRQ 0x10014 around the reset */ ++ struct a9_aop_spi0_trace trace; /* latest read-only milestone */ ++ /* Appended so every pre-wake12 field, including ch3, keeps its layout. */ ++ struct a9_aop_channel_diag_slice channel_diag[A9_SPI0_CH3_SLICES]; ++ /* Wake13: exact phosphorus stopped-without-idle exception, by channel. */ ++ u32 teardown_stable; ++ /* Wake15: channel 3 shares the sampled diagnostics; reset completion time. */ ++ struct a9_aop_channel_diag_slice ch3_diag[A9_SPI0_CH3_SLICES]; ++ u64 reset_done_ns; ++}; ++ ++/* 0102f2f4: bits 8 and 0 clear; the reset write adds bit 8. */ ++static inline u32 a9_spi0_base_word(u32 saved) ++{ ++ return saved & ~(A9_SPI0_RESET | A9_SPI0_ENABLE); ++} ++ ++#endif +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index e8ae90098..8bcab7651 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -1,19 +1,28 @@ + // SPDX-License-Identifier: GPL-2.0-only OR MIT + /* +- * Apple T8010 Always-On Processor (AOP) ++ * Apple S8000 and T8010 Always-On Processor (AOP) + * + * The AOP runs RTKit firmware that the operating system loads: a main image + * into the AOP's SRAM and an external segment into system memory, with boot +- * arguments filled in. Its services talk AFK/EPIC over shared ring buffers on +- * the RTKit application endpoints, as on later SoCs (see the Asahi Linux AOP ++ * arguments filled in. On the S8000 (A9) the AOP sees its SRAM through a window ++ * that has to be set up first, and its firmware takes more boot arguments. ++ * Its services talk AFK/EPIC over shared ring buffers on the RTKit ++ * application endpoints, as on later SoCs (see the Asahi Linux AOP + * driver). Each service with a device tree child node of the same name gets a + * child platform device; its driver receives the service's reports through + * apple_aop_listen() and sends requests with apple_aop_call(). + * +- * The firmware cannot be restarted once stopped, so the AOP starts once per +- * boot and the driver cannot be unbound or unloaded after that. ++ * J71t can reload after a certified quiesce/halt/drain across kexec. Other ++ * retained states are refused. Unbind/unload after starting remains forbidden. + */ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include "aop-warm-spi0.h" + #include + #include + #include +@@ -41,6 +50,14 @@ + #define APPLE_AOP_IRQ_MASK_ALL 0x1111 + #define APPLE_AOP_IRQ_INBOX BIT(0) + #define APPLE_AOP_IRQ_OUTBOX BIT(12) ++/* S8000: the window through which the AOP sees its SRAM */ ++#define APPLE_AOP_REMAP_PA_LO 0x08 ++#define APPLE_AOP_REMAP_PA_HI 0x10 ++#define APPLE_AOP_REMAP_IOVA_LO 0x18 ++#define APPLE_AOP_REMAP_IOVA_HI 0x20 ++#define APPLE_AOP_REMAP_SIZE 0x28 ++#define APPLE_AOP_REMAP_RESERVED 0x30 ++#define APPLE_AOP_REMAP_ENABLE 0x38 + + #define APPLE_AOP_CONTROL_AUTO_BOOT 0x0 + +@@ -54,6 +71,13 @@ + #define APPLE_AOP_BOOTARG_ECID APPLE_AOP_FOURCC('E', 'C', 'I', 'D') + #define APPLE_AOP_BOOTARG_NONC APPLE_AOP_FOURCC('N', 'O', 'N', 'C') + #define APPLE_AOP_BOOTARG_EC0P APPLE_AOP_FOURCC('E', 'C', '0', 'p') ++#define APPLE_AOP_BOOTARG_AOPT APPLE_AOP_FOURCC('A', 'O', 'P', 't') ++#define APPLE_AOP_BOOTARG_NCAL APPLE_AOP_FOURCC('n', 'C', 'a', 'l') ++#define APPLE_AOP_BOOTARG_WRAD APPLE_AOP_FOURCC('W', 'r', 'A', 'd') ++#define APPLE_AOP_BOOTARG_SOC APPLE_AOP_FOURCC('S', 'O', 'C', '_') ++#define APPLE_AOP_BOOTARG_SOCR APPLE_AOP_FOURCC('S', 'O', 'C', 'R') ++#define APPLE_AOP_BOOTARG_STKG APPLE_AOP_FOURCC('S', 'T', 'K', 'G') ++#define APPLE_AOP_MAX_BOOTARGS 10 + + #define APPLE_AOP_HALT_TIMEOUT_US 10000 + +@@ -78,7 +102,9 @@ + #define AFK_OPC_GET_BUF_ACK 0xa1 + #define AFK_OPC_SEND 0xa2 + #define AFK_OPC_START 0xa3 +-#define AFK_OPC_SHUTDOWN_ACK 0xc1 ++#define AFK_OPC_SHUTDOWN 0xa4 ++#define AFK_OPC_SHUTDOWN_ACK 0x87 ++#define AFK_OPC_HIBERNATE_ACK 0xc3 + + #define AFK_BLOCK_STEP 0x40 + #define AFK_MAX_BUFFER SZ_1M +@@ -100,6 +126,12 @@ + #define APPLE_AOP_MAX_CALLS 8 + #define APPLE_AOP_CALL_TIMEOUT msecs_to_jiffies(1000) + ++/* Service properties */ ++#define APPLE_AOP_PROPERTY_MAX 64 ++#define APPLE_AOP_PROP_CALIBRATION 0x3d ++#define APPLE_AOP_CALIBRATION_PARTIAL 1 ++#define APPLE_AOP_CALIBRATION_COMPLETE 2 ++ + struct afk_qe_header { + __le32 magic; + __le32 size; +@@ -160,6 +192,8 @@ struct apple_aop_ep { + + struct mutex lock; /* tx ring, seq and calls */ + u16 seq; ++ bool initialized; ++ struct completion stopped; + struct apple_aop_call *calls[APPLE_AOP_MAX_CALLS]; + }; + +@@ -182,12 +216,25 @@ struct apple_aop_register_work { + struct apple_aop_service service; + }; + ++struct apple_aop_hw { ++ /* Set up the AOP's window onto its SRAM before starting it */ ++ bool sram_window; ++ /* A fresh NONC boot argument */ ++ bool nonce; ++ /* AOPt, SOC_, SOCR, WrAd, STKG and nCal boot arguments */ ++ bool soc_bootargs; ++}; ++ + struct apple_aop { + struct device *dev; ++ const struct apple_aop_hw *hw; + void __iomem *akf; + void __iomem *control; + void __iomem *sram; ++ resource_size_t akf_base; ++ resource_size_t sram_base; + resource_size_t sram_size; ++ u32 target, chip_id, chip_revision; + struct apple_rtkit *rtk; + + void *ext; +@@ -202,6 +249,16 @@ struct apple_aop { + struct mutex children_lock; /* children, removing */ + struct list_head children; + bool removing; ++ u64 accel_reports; ++ int shutdown_error; ++ /* J71t only; all other boards retain their original start/stop path. */ ++ bool warm, hello, iop_crashed, force_halted, force_drained; ++ void __iomem *warm_wrapper; ++ struct device_node *domains[5], *spi0_pmgr; ++ struct a9_aop_spi0 spi0; ++ const char *experiment_step; ++ struct a9_sl_header *shutdown_log; ++ char *shutdown_text; + }; + + static bool apple_aop_is_started(struct apple_aop *aop) +@@ -223,6 +280,409 @@ static bool apple_aop_halt(struct apple_aop *aop) + APPLE_AOP_HALT_TIMEOUT_US); + } + ++static bool a9_aop_ramoops_ready(void) ++{ ++ struct console *con; ++ bool ready = false; ++ int cookie; ++ ++ cookie = console_srcu_read_lock(); ++ for_each_console_srcu(con) { ++ if (!strcmp(con->name, "ramoops") && ++ (console_srcu_read_flags(con) & (CON_ENABLED | CON_SUSPENDED)) == ++ CON_ENABLED) { ++ ready = true; ++ break; ++ } ++ } ++ console_srcu_read_unlock(cookie); ++ return ready; ++} ++ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* J71t warm reload integration. SPI0 machinery is the release6 wake16 code. */ ++#define AOP_DOMAINS 5 ++#define AOP_CPU_CONTROL 0x44 ++#define AOP_CPU_START BIT(4) ++#define AOP_OUTBOX_CONTROL 0x4020 ++#define AOP_INBOX_CONTROL 0x4008 ++#define AOP_OUTBOX_ENABLE BIT(0) ++#define AOP_MBOX_FAULTS GENMASK(19, 18) ++#define cold_defer 1 ++#define spi0_untested false ++ ++static u32 aop_rd(struct apple_aop *aop, u32 offset) ++{ ++ return readl(aop->warm_wrapper + offset); ++} ++ ++static bool a9_aop_force_stopped(struct apple_aop *aop) ++{ ++ return !(aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) && ++ !readl(aop->control) && ++ !(aop_rd(aop, AOP_OUTBOX_CONTROL) & (AOP_OUTBOX_ENABLE | AOP_MBOX_FAULTS)) && ++ !(aop_rd(aop, AOP_INBOX_CONTROL) & AOP_MBOX_FAULTS); ++} ++ ++struct aop_warm_write { void __iomem *target; u32 value; }; ++static void aop_warm_store(void *data) ++{ ++ struct aop_warm_write *w = data; ++ ++ writel(w->value, w->target); ++ readl(w->target); ++} ++ ++static int aop_wr(struct apple_aop *aop, u32 offset, u32 value) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct aop_warm_write w = { aop->warm_wrapper + offset, value }; ++ ++ if ((aop_rd(aop, AOP_INBOX_CONTROL) | aop_rd(aop, AOP_OUTBOX_CONTROL)) & ++ AOP_MBOX_FAULTS) ++ return -EIO; ++ dev_info(aop->dev, "COLD-W step=%s bank=wrapper offset=%04x before=%08x value=%08x\n", ++ aop->experiment_step ?: "warm-stop", offset, readl(w.target), value); ++ { ++ int ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, aop_warm_store, &w); ++ ++ dev_info(aop->dev, "COLD-R offset=%04x after=%08x\n", offset, readl(w.target)); ++ return ret; ++ } ++} ++ ++static int a9_aop_cold_checkpoint_noflush(struct apple_aop *aop, const char *step) ++{ ++ if (!a9_aop_ramoops_ready()) ++ return -ENODEV; ++ dev_info(aop->dev, "COLD checkpoint=%s flush=skipped\n", step); ++ return 0; ++} ++ ++static int a9_aop_cold_checkpoint(struct apple_aop *aop, const char *step) ++{ ++ int ret = a9_aop_cold_checkpoint_noflush(aop, step); ++ ++ if (ret) ++ return ret; ++ if (!pr_flush(1000, false)) ++ return -ETIMEDOUT; ++ mb(); ++ return 0; ++} ++ ++#include "aop-warm-spi0.c.h" ++static int apple_aop_send(struct apple_aop_ep *ep, u8 opcode, u64 payload); ++ ++/* No cmdline override for untested states: default is the release6 policy. */ ++static atomic_t aop_warm_attempted = ATOMIC_INIT(0); ++ ++static int apple_aop_warm_stop(struct apple_aop *aop) ++{ ++ unsigned int fifo, i; ++ u32 offset, val; ++ int ret; ++ ++ ret = aop_wr(aop, 0x4020, aop_rd(aop, 0x4020) & ~BIT(0)); ++ if (!ret) ++ ret = aop_wr(aop, 0x4000, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0xc00, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0x44, aop_rd(aop, 0x44) & ~BIT(4)); ++ if (ret) ++ return ret; ++ writel(0, aop->control); ++ msleep(10); ++ if (!a9_aop_force_stopped(aop)) ++ return -EIO; ++ aop->force_halted = true; ++ for (fifo = 0; fifo < 2; fifo++) { ++ offset = fifo ? 0x4020 : 0x4008; ++ for (i = 0; i < 64; i++) { ++ val = aop_rd(aop, offset); ++ if (!a9_aop_force_stopped(aop) || (val & AOP_MBOX_FAULTS)) ++ return -EIO; ++ if (val & BIT(17)) ++ break; ++ readq(aop->warm_wrapper + (fifo ? 0x4038 : 0x4018)); ++ } ++ if (!(aop_rd(aop, offset) & BIT(17))) ++ return -ETIMEDOUT; ++ } ++ aop->force_drained = true; ++ return 0; ++} ++ ++static void apple_aop_warm_put_node(void *node) ++{ ++ of_node_put(node); ++} ++ ++static void apple_aop_warm_unmap(void *p) ++{ ++ memunmap(p); ++} ++ ++static int apple_aop_warm_prepare(struct apple_aop *aop) ++{ ++ struct device *dev = aop->dev; ++ struct device_node *np = dev->of_node, *mem; ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct a9_sl_header header; ++ struct resource res; ++ char *text; ++ u32 auto_boot, outbox; ++ unsigned int i; ++ int ret; ++ ++ if (!of_machine_is_compatible("apple,j71t") || !aop->hw->sram_window) ++ return 0; ++ /* Missing the new DT is not permission to run a retained instance. */ ++ mem = of_parse_phandle(np, "memory-region", 0); ++ if (!mem) ++ return dev_err_probe(dev, -EINVAL, "warm reload requires the matching J71t DT\n"); ++ ret = of_address_to_resource(mem, 0, &res); ++ of_node_put(mem); ++ if (ret || resource_size(&res) != A9_HO_PAGE || res.start != 0x8583fc000ULL || ++ aop->akf_base != A9_SPI0_WRAPPER_PA) ++ return -EINVAL; ++ aop->shutdown_log = memremap(res.start, A9_HO_PAGE, MEMREMAP_WB); ++ if (!aop->shutdown_log) ++ return -ENOMEM; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_unmap, aop->shutdown_log); ++ if (ret) ++ return ret; ++ /* Keep release6's transcript location and validation ABI. */ ++ aop->shutdown_log = (void *)aop->shutdown_log + A9_HO_LOG_OFFSET; ++ aop->shutdown_text = (char *)(aop->shutdown_log + 1); ++ memcpy(&header, aop->shutdown_log, sizeof(header)); ++ text = devm_kmemdup(dev, aop->shutdown_text, A9_SL_CAPACITY, GFP_KERNEL); ++ if (!text) ++ return -ENOMEM; ++ if (header.magic == A9_SL_MAGIC) { ++ aop->spi0.prev.state = A9_SPI0_PREV_INVALID; ++ if (a9_sl_valid(&header, text)) { ++ aop->spi0.prev.state = A9_SPI0_PREV_VALID; ++ a9_spi0_prev_parse(&aop->spi0.prev, text, header.length); ++ for (i = 0; i < header.length;) { ++ char *next = memchr(text + i, '\n', header.length - i); ++ unsigned int n = next ? next - text - i : header.length - i; ++ ++ dev_info(dev, "AOP-PREV %.*s\n", n, text + i); ++ i += n + 1; ++ } ++ } ++ } ++ aop->warm_wrapper = devm_ioremap(dev, aop->akf_base, 0x1c000); ++ if (!aop->warm_wrapper) ++ return -ENOMEM; ++ for (i = 0; i < AOP_DOMAINS; i++) { ++ aop->domains[i] = of_parse_phandle(np, "apple,observe-domains", i); ++ if (!aop->domains[i]) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->domains[i]); ++ if (ret) ++ return ret; ++ } ++ aop->spi0_pmgr = of_parse_phandle(np, "apple,spi0-domain", 0); ++ if (!aop->spi0_pmgr) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->spi0_pmgr); ++ if (ret) ++ return ret; ++ /* Check all domains before reading the wrapper; no power transitions. */ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, a9_aop_spi0_pmgr_on, &(bool){false}); ++ if (ret) ++ return dev_err_probe(dev, ret, "AOP domain guard refused\n"); ++ auto_boot = readl(aop->control); ++ outbox = aop_rd(aop, AOP_OUTBOX_CONTROL); ++ ret = a9_aop_spi0_policy(aop, auto_boot, outbox); ++ if (ret) ++ return ret; /* Before first halt, mask, SRAM or SPI0 write. */ ++ /* Orchard's old halt leaves +0x4020 enabled. On a known kexec, ++ * that alone cannot certify the cold wrapper baseline. Release6's ++ * wrapper discriminator was tied to its own outgoing stop routine. ++ */ ++ if (of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev) != A9_SPI0_PREV_HALTED) { ++ dev_info(dev, "SPI0 policy decision=refuse reason=uncertified-kexec auto_boot=%u out4020=%08x: block not touched\n", ++ auto_boot, outbox); ++ return -EPERM; ++ } ++ aop->warm = true; ++ aop->spi0.reason = a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev); ++ if (!a9_spi0_reason_runs(aop->spi0.reason)) { ++ /* Original cold path: no halt, drain, reset, or SPI0 access. */ ++ dev_info(dev, "SPI0 reset decision=skip reason=%s: block not touched\n", ++ a9_spi0_reason_name(aop->spi0.reason)); ++ return 0; ++ } ++ if (atomic_xchg(&aop_warm_attempted, 1)) ++ return dev_err_probe(dev, -EPERM, "refused-second-experiment-attempt\n"); ++ if (!a9_aop_ramoops_ready()) ++ return dev_err_probe(dev, -ENODEV, "warm reload requires ramoops console\n"); ++ aop->spi0.base = devm_ioremap(dev, A9_SPI0_PA, A9_SPI0_SIZE); ++ if (!aop->spi0.base) ++ return -ENOMEM; ++ /* Consume certification before the first control write. Reloading the ++ * module after a failure must not reuse it as a second attempt. ++ */ ++ aop->shutdown_log->magic = 0; ++ wmb(); ++ ret = apple_aop_warm_stop(aop); ++ if (!ret) ++ ret = a9_aop_spi0_prereset(aop, auto_boot, outbox); ++ if (ret) ++ dev_err(dev, "warm reload refused errno=%d; no copy/RUN, recover with DFU\n", ret); ++ return ret; ++} ++ ++static int apple_aop_warm_release(struct apple_aop *aop) ++{ ++ int ret; ++ ++ ret = a9_aop_spi0_recheck(aop); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-window"); ++ if (!ret) ++ ret = a9_aop_intc_ack(aop); ++ if (!ret) ++ ret = a9_aop_intc_reset(aop); ++ if (!ret) ++ ret = a9_aop_cold_checkpoint(aop, "before-run"); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "pre-run-final", "FINAL", true); ++ return ret; ++} ++ ++static void apple_aop_warm_hash(struct apple_aop *aop, u8 *digest) ++{ ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ if (aop->ext) ++ sha256_update(&ctx, aop->ext, aop->ext_size); ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) ++ if (aop->eps[i].buf) ++ sha256_update(&ctx, aop->eps[i].buf, aop->eps[i].buf_size); ++ sha256_final(&ctx, digest); ++} ++ ++static __printf(2, 3) void apple_aop_warm_log(struct apple_aop *aop, const char *fmt, ...) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ va_list args; ++ char line[512]; ++ size_t n; ++ ++ va_start(args, fmt); ++ n = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ dev_info(aop->dev, "%s", line); ++ if (n > h->capacity - h->length) { ++ h->truncated++; ++ return; ++ } ++ memcpy(aop->shutdown_text + h->length, line, n); ++ wmb(); ++ h->checksum = a9_sl_hash(h->checksum, line, n); ++ h->length += n; ++ h->lines++; ++ wmb(); ++} ++ ++static void apple_aop_warm_shutdown(struct apple_aop *aop) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ unsigned long deadline = jiffies + msecs_to_jiffies(10000), start; ++ unsigned int i, acked = 0; ++ int ret = -EIO; ++ ++ memset(h, 0, sizeof(*h)); ++ h->version = A9_SL_VERSION; ++ h->capacity = A9_SL_CAPACITY; ++ h->checksum = A9_SL_HASH_SEED; ++ h->mode = 1; ++ h->begin_ns = ktime_get_boottime_ns(); ++ wmb(); ++ h->magic = A9_SL_MAGIC; ++ apple_aop_warm_log(aop, "SHUTDOWN begin mode=quiesce hello=%d phase=stopping iop_crashed=%d\n", ++ aop->hello, aop->iop_crashed); ++ if (!aop->hello || aop->iop_crashed || aop->shutdown_error) ++ goto done; ++ /* Rates-zero has stopped reports. Stop one AFK at a time, max 2s ++ * per endpoint and 10s in total, including send time. Never force halt ++ * after a failed ACK. Keep every grant allocated for the entire hook. ++ */ ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ unsigned long limit; ++ ++ if (!ep->initialized) ++ continue; ++ if (time_after_eq(jiffies, deadline)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ reinit_completion(&ep->stopped); ++ start = jiffies; ++ ret = apple_aop_send(ep, AFK_OPC_SHUTDOWN, 0); ++ if (ret) ++ goto done; ++ limit = time_before(jiffies, deadline) ? min_t(unsigned long, ++ deadline - jiffies, msecs_to_jiffies(2000)) : 0; ++ if (!limit || !wait_for_completion_timeout(&ep->stopped, limit)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ acked++; ++ apple_aop_warm_log(aop, "QUIESCE step=afk-ack ep=0x%x state=stopped ms=%u\n", ++ ep->index, jiffies_to_msecs(jiffies - start)); ++ } ++ /* release6: complete every AFK stop before disabling endpoint routing. */ ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; ++ } ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ ++ if (!ep->initialized) ++ continue; ++ /* MGMT STARTEP bit 0 disables routing, no separate stop ACK. */ ++ ret = apple_rtkit_send_message(aop->rtk, 0, ++ (5ULL << 52) | ((u64)ep->index << 32) | 1, NULL, false); ++ if (ret) ++ goto done; ++ } ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; /* Never certify an incompletely initialized instance. */ ++ } ++ ret = apple_rtkit_shutdown_preserve(aop->rtk); ++ if (ret) ++ goto done; ++ msleep(20); ++ apple_aop_warm_hash(aop, before); ++ msleep(20); ++ apple_aop_warm_hash(aop, after); ++ ret = memcmp(before, after, sizeof(before)) ? -EBUSY : 0; ++ apple_aop_warm_log(aop, "QUIESCE step=ram-stable window_ms=20 status=%d\n", ret); ++ if (ret || aop->iop_crashed) ++ goto done; ++ ret = apple_aop_warm_stop(aop); ++done: ++ apple_aop_warm_log(aop, "QUIESCE step=complete status=%d quiesced=%d halted=%d drained=%d steps=0x%x afk_acks=%u iop_crashed=%d\n", ++ ret, !ret, aop->force_halted, aop->force_drained, ++ !ret ? 0x1bff : 0, acked, aop->iop_crashed); ++ if (ret) ++ dev_err(aop->dev, "quiesce failed; next AOP must refuse, recover with DFU\n"); ++} ++ + /* Firmware loading */ + + struct apple_aop_bootarg { +@@ -232,11 +692,17 @@ struct apple_aop_bootarg { + bool found; + }; + ++static void apple_aop_add_bootarg(struct apple_aop_bootarg *args, unsigned int *count, ++ u32 key, const void *value, u32 size) ++{ ++ args[(*count)++] = (struct apple_aop_bootarg){ .key = key, .value = value, .size = size }; ++} ++ + static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t size, +- struct apple_aop_bootarg *args, int count) ++ struct apple_aop_bootarg *args, unsigned int count) + { + size_t offset, end, pos; +- int i; ++ unsigned int i; + + offset = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_OFFSET); + end = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_SIZE); +@@ -269,13 +735,29 @@ static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t siz + return 0; + } + ++/* S8000: point the AOP's window at its SRAM; the firmware runs from there. */ ++static void apple_aop_map_sram(struct apple_aop *aop) ++{ ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_LO); ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_HI); ++ writel(aop->sram_size & ~0xfffULL, aop->akf + APPLE_AOP_REMAP_SIZE); ++ writel(0, aop->akf + APPLE_AOP_REMAP_RESERVED); ++ writel(lower_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_LO); ++ writel(upper_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_HI); ++ writel(1, aop->akf + APPLE_AOP_REMAP_ENABLE); ++} ++ + static int apple_aop_load(struct apple_aop *aop, u64 ecid) + { ++ struct apple_aop_bootarg args[APPLE_AOP_MAX_BOOTARGS]; + const struct firmware *main_fw, *ext_fw; + const char *main_name, *ext_name; + struct device_node *np = aop->dev->of_node; +- u8 prng[32], nonce[8]; +- __le64 ecid_le = cpu_to_le64(ecid), ext_le; ++ u8 prng[32], nonce[8], guard[4], ncal = 0; ++ __le64 ecid_le = cpu_to_le64(ecid), ext_le, wrapper_le; ++ __le32 target_le, chip_le, revision_le; ++ unsigned int count = 0; ++ size_t offset; + u8 *image; + int ret; + +@@ -316,28 +798,58 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + } + + get_random_bytes(prng, sizeof(prng)); +- get_random_bytes(nonce, sizeof(nonce)); +- { +- struct apple_aop_bootarg args[] = { +- { APPLE_AOP_BOOTARG_PRNG, prng, sizeof(prng) }, +- { APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le) }, +- { APPLE_AOP_BOOTARG_NONC, nonce, sizeof(nonce) }, +- { APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le) }, +- }; +- +- ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, +- ARRAY_SIZE(args)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_PRNG, &prng, sizeof(prng)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le)); ++ if (aop->hw->nonce) { ++ get_random_bytes(nonce, sizeof(nonce)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NONC, &nonce, sizeof(nonce)); ++ } ++ if (aop->hw->soc_bootargs) { ++ /* A random stack guard with one zero byte */ ++ get_random_bytes(guard, sizeof(guard)); ++ guard[get_random_u32_below(sizeof(guard))] = 0; ++ target_le = cpu_to_le32(aop->target); ++ chip_le = cpu_to_le32(aop->chip_id); ++ revision_le = cpu_to_le32(aop->chip_revision); ++ wrapper_le = cpu_to_le64(aop->akf_base); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_AOPT, ++ &target_le, sizeof(target_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NCAL, &ncal, sizeof(ncal)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_WRAD, ++ &wrapper_le, sizeof(wrapper_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOC, ++ &chip_le, sizeof(chip_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOCR, ++ &revision_le, sizeof(revision_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_STKG, &guard, sizeof(guard)); + } ++ ++ ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, count); + if (ret) { + dev_err(aop->dev, "unexpected boot arguments in %s: %d\n", main_name, ret); + } else { +- memcpy_toio(aop->sram, image, main_fw->size); +- memset_io(aop->sram + main_fw->size, 0, aop->sram_size - main_fw->size); ++ /* The SRAM takes 32-bit accesses, as Apple's loader makes them. */ ++ if (aop->spi0.ran) { ++ ret = a9_aop_spi0_gate(aop, "pre-copy-final", "PRECOPY", false); ++ if (!ret) ++ ret = a9_aop_copy_verify_wake10(aop, image, main_fw->size); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "after-copy", "AFTERCOPY", true); ++ if (ret) ++ goto out_image; ++ } else { ++ __iowrite32_copy(aop->sram, image, main_fw->size / 4); ++ } ++ for (offset = main_fw->size; offset + 4 <= aop->sram_size; offset += 4) ++ writel_relaxed(0, aop->sram + offset); + } + ++out_image: + kfree_sensitive(image); + memzero_explicit(prng, sizeof(prng)); + memzero_explicit(nonce, sizeof(nonce)); ++ memzero_explicit(guard, sizeof(guard)); + memzero_explicit(&ecid_le, sizeof(ecid_le)); + out_ext: + release_firmware(ext_fw); +@@ -448,7 +960,8 @@ static void apple_aop_announce(struct apple_aop_ep *ep, u32 channel, const void + const struct epic_service_announce *announce = data; + struct apple_aop_register_work *rw; + +- if (len < sizeof(*announce)) ++ /* Only the name and channel are used; the S8000's announcements are shorter. */ ++ if (len < offsetofend(struct epic_service_announce, channel)) + return; + + rw = kzalloc_obj(*rw); +@@ -475,6 +988,8 @@ static void apple_aop_report(struct apple_aop_ep *ep, u32 channel, u16 subtype, + struct apple_aop_listener *listener; + bool handled = false; + ++ if (ep->index == 0x21 && subtype == 0xc2) ++ aop->accel_reports++; + mutex_lock(&aop->listeners_lock); + list_for_each_entry(listener, &aop->listeners, list) { + if (listener->endpoint == ep->index && listener->channel == channel) { +@@ -631,6 +1146,7 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + + switch (FIELD_GET(AFK_MSG_OPCODE, msg)) { + case AFK_OPC_INIT: ++ ep->initialized = true; + apple_aop_send(ep, AFK_OPC_INIT_ACK, 0); + break; + case AFK_OPC_GET_BUF: +@@ -651,7 +1167,10 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + break; + case AFK_OPC_INIT_UNK: + case AFK_OPC_START_ACK: ++ break; + case AFK_OPC_SHUTDOWN_ACK: ++ case AFK_OPC_HIBERNATE_ACK: ++ complete(&ep->stopped); + break; + default: + dev_dbg(aop->dev, "endpoint 0x%02x: message 0x%llx\n", endpoint, msg); +@@ -662,7 +1181,8 @@ static void apple_aop_crashed(void *cookie, const void *crashlog, size_t crashlo + { + struct apple_aop *aop = cookie; + +- dev_err(aop->dev, "firmware crashed\n"); ++ aop->iop_crashed = true; ++ dev_err(aop->dev, "firmware crashed iop_crashed=1\n"); + } + + static const struct apple_rtkit_ops apple_aop_rtkit_ops = { +@@ -767,6 +1287,8 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + + if (!ep) + return -EINVAL; ++ if (service->aop->warm && READ_ONCE(service->aop->iop_crashed)) ++ return -EIO; + init_completion(&call.done); + + mutex_lock(&ep->lock); +@@ -797,6 +1319,58 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + } + EXPORT_SYMBOL_GPL(apple_aop_call); + ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len) ++{ ++ u8 buf[8 + APPLE_AOP_PROPERTY_MAX]; ++ int ret; ++ ++ if (len > APPLE_AOP_PROPERTY_MAX) ++ return -EINVAL; ++ put_unaligned_le32(0, buf); ++ put_unaligned_le32(key, buf + 4); ++ memcpy(buf + 8, value, len); ++ ++ ret = apple_aop_call(service, APPLE_AOP_SET_PROPERTY, buf, 8 + len, NULL, 0); ++ return ret > 0 ? -EIO : ret; ++} ++EXPORT_SYMBOL_GPL(apple_aop_set_property); ++ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count) ++{ ++ unsigned long needed = 0, applied = 0; ++ __le32 state; ++ unsigned int i; ++ int len, ret; ++ ++ for (i = 0; i < count; i++) { ++ const void *blob; ++ ++ if (cal[i].group) ++ needed |= BIT(cal[i].group); ++ blob = of_get_property(dev->of_node, cal[i].property, &len); ++ if (!blob) ++ continue; ++ ret = apple_aop_set_property(service, cal[i].key, blob, len); ++ if (ret) { ++ dev_warn(dev, "%s not applied: %d\n", cal[i].property, ret); ++ continue; ++ } ++ if (cal[i].group) ++ applied |= BIT(cal[i].group); ++ } ++ ++ if (needed && applied == needed) { ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_COMPLETE); ++ } else { ++ dev_warn(dev, "factory calibration incomplete\n"); ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_PARTIAL); ++ } ++ return apple_aop_set_property(service, APPLE_AOP_PROP_CALIBRATION, &state, sizeof(state)); ++} ++EXPORT_SYMBOL_GPL(apple_aop_calibrate); ++ + /* Driver */ + + static void apple_aop_free_buffers(struct apple_aop *aop) +@@ -825,27 +1399,35 @@ static int apple_aop_probe(struct platform_device *pdev) + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); + mutex_init(&aop->children_lock); + INIT_LIST_HEAD(&aop->children); + +- aop->akf = devm_platform_ioremap_resource_byname(pdev, "akf"); ++ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "akf"); ++ aop->akf = devm_ioremap_resource(dev, res); + if (IS_ERR(aop->akf)) + return PTR_ERR(aop->akf); ++ aop->akf_base = res->start; + aop->control = devm_platform_ioremap_resource_byname(pdev, "control"); + if (IS_ERR(aop->control)) + return PTR_ERR(aop->control); + aop->sram = devm_platform_get_and_ioremap_resource(pdev, 2, &res); + if (IS_ERR(aop->sram)) + return PTR_ERR(aop->sram); ++ aop->sram_base = res->start; + aop->sram_size = resource_size(res); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + +- if (apple_aop_is_started(aop)) ++ ret = apple_aop_warm_prepare(aop); ++ if (ret) ++ return ret; ++ ++ if (!aop->spi0.ran && apple_aop_is_started(aop)) + return dev_err_probe(dev, -EBUSY, + "already started; the AOP starts once per boot\n"); + +@@ -853,6 +1435,17 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + return dev_err_probe(dev, ret, "no apple,ecid from the bootloader\n"); + ++ if (aop->hw->soc_bootargs) { ++ ret = of_property_read_u32(np, "apple,aop-target", &aop->target); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-id", &aop->chip_id); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-revision", &aop->chip_revision); ++ if (ret) ++ return dev_err_probe(dev, ret, ++ "no AOP target or chip ID from the bootloader\n"); ++ } ++ + aop->eps = devm_kcalloc(dev, AFK_ENDPOINT_COUNT, sizeof(*aop->eps), GFP_KERNEL); + if (!aop->eps) + return -ENOMEM; +@@ -860,6 +1453,7 @@ static int apple_aop_probe(struct platform_device *pdev) + aop->eps[i].aop = aop; + aop->eps[i].index = AFK_ENDPOINT_START + i; + mutex_init(&aop->eps[i].lock); ++ init_completion(&aop->eps[i].stopped); + } + + ret = apple_aop_load(aop, ecid); +@@ -867,6 +1461,9 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + goto err_free; + ++ if (aop->hw->sram_window) ++ apple_aop_map_sram(aop); ++ + aop->rtk = apple_rtkit_init(dev, aop, "mbox", 0, &apple_aop_rtkit_ops); + if (IS_ERR(aop->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(aop->rtk), "failed to init RTKit\n"); +@@ -884,10 +1481,24 @@ static int apple_aop_probe(struct platform_device *pdev) + prop = of_find_property(np, "apple,ecid", NULL); + if (prop) + of_remove_property(np, prop); +- writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, +- aop->akf + APPLE_AOP_CPU_CONTROL); ++ if (aop->spi0.ran) { ++ ret = apple_aop_warm_release(aop); ++ if (ret) ++ goto err_halt; ++ } ++ if (aop->spi0.ran) { ++ aop->experiment_step = "warm-run"; ++ ret = aop_wr(aop, APPLE_AOP_CPU_CONTROL, ++ readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN); ++ if (ret) ++ goto err_halt; ++ } else { ++ writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, ++ aop->akf + APPLE_AOP_CPU_CONTROL); ++ } + writel(1, aop->control + APPLE_AOP_CONTROL_AUTO_BOOT); + ++ dev_info(dev, "AOP released warm=%d\n", aop->spi0.ran); + ret = apple_rtkit_wake(aop->rtk); + if (ret) { + dev_err(dev, "firmware did not boot: %d\n", ret); +@@ -900,6 +1511,8 @@ static int apple_aop_probe(struct platform_device *pdev) + dev_warn(dev, "failed to start endpoint 0x%02x: %d\n", i, ret); + } + ++ aop->hello = true; ++ dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); + return 0; + +@@ -929,27 +1542,81 @@ static void apple_aop_shutdown(struct platform_device *pdev) + mutex_lock(&aop->children_lock); + aop->removing = true; + list_for_each_entry_safe(child, tmp, &aop->children, list) { ++ ++ if (aop->warm) { ++ const struct apple_aop_service *service = dev_get_platdata(&child->pdev->dev); ++ static const u32 keys[] = { 0x16, 0x5c, 0 }; ++ ++ /* release6 sets the three accelerometer rates only. */ ++ if (strcmp(service->name, "accel")) ++ continue; ++ __le32 zero = 0; ++ unsigned int i; ++ int ret; ++ ++ for (i = 0; i < ARRAY_SIZE(keys); i++) { ++ ret = apple_aop_set_property(service, keys[i], &zero, sizeof(zero)); ++ if (ret) { ++ aop->shutdown_error = ret; ++ break; ++ } ++ } ++ } ++ /* Shutdown retains listeners, children and DMA grants, as release6 does. ++ * Unregister is remove-time work and can wait in IIO/driver core. ++ */ ++ if (aop->warm) ++ continue; + platform_device_unregister(child->pdev); + list_del(&child->list); + kfree(child); + } + mutex_unlock(&aop->children_lock); + ++ if (aop->warm) { ++ apple_aop_warm_shutdown(aop); ++ return; ++ } + if (!apple_aop_halt(aop)) + dev_err(aop->dev, "failed to halt\n"); + } + ++static const struct apple_aop_hw apple_aop_s8000_hw = { ++ .sram_window = true, ++ .soc_bootargs = true, ++}; ++ ++static const struct apple_aop_hw apple_aop_t8010_hw = { ++ .nonce = true, ++}; ++ + static const struct of_device_id apple_aop_of_match[] = { +- { .compatible = "apple,t8010-aop" }, ++ { .compatible = "apple,s8000-aop", .data = &apple_aop_s8000_hw }, ++ { .compatible = "apple,t8010-aop", .data = &apple_aop_t8010_hw }, + {} + }; + MODULE_DEVICE_TABLE(of, apple_aop_of_match); + ++static ssize_t warm_state_show(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ struct apple_aop *aop = dev_get_drvdata(dev); ++ ++ if (!aop) ++ return sysfs_emit(buf, "owned=0\n"); ++ return sysfs_emit(buf, "owned=%u reload=%u iop_crashed=%u accel_reports=%llu\n", ++ aop->hello, aop->spi0.ran, READ_ONCE(aop->iop_crashed), ++ READ_ONCE(aop->accel_reports)); ++} ++static DEVICE_ATTR_RO(warm_state); ++static struct attribute *apple_aop_attrs[] = { &dev_attr_warm_state.attr, NULL }; ++ATTRIBUTE_GROUPS(apple_aop); ++ + static struct platform_driver apple_aop_driver = { + .driver = { + .name = "apple-aop", + .of_match_table = apple_aop_of_match, + .suppress_bind_attrs = true, ++ .dev_groups = apple_aop_groups, + }, + .probe = apple_aop_probe, + .shutdown = apple_aop_shutdown, +@@ -957,5 +1624,5 @@ static struct platform_driver apple_aop_driver = { + module_platform_driver(apple_aop_driver); + + MODULE_AUTHOR("Dalton Alexandre "); +-MODULE_DESCRIPTION("Apple T8010 Always-On Processor driver"); ++MODULE_DESCRIPTION("Apple S8000 and T8010 Always-On Processor driver"); + MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c +index 725ab3fa7..d7928621a 100644 +--- a/drivers/soc/apple/rtkit.c ++++ b/drivers/soc/apple/rtkit.c +@@ -4,6 +4,8 @@ + * Copyright (C) The Asahi Linux Contributors + */ + ++#include ++#include + #include "rtkit-internal.h" + + enum { +@@ -867,6 +869,54 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) + } + EXPORT_SYMBOL_GPL(apple_rtkit_boot); + ++/* Unlike shutdown(), retain all grants until the caller halts the CPU. ++ * Reinit frees DMA buffers and is unsafe before that point on S8000. ++ */ ++static void apple_rtkit_grants_hash(struct apple_rtkit *rtk, u8 *digest) ++{ ++ struct apple_rtkit_shmem *buffers[] = { ++ &rtk->ioreport_buffer, &rtk->crashlog_buffer, ++ &rtk->oslog_buffer, &rtk->syslog_buffer, ++ }; ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ for (i = 0; i < ARRAY_SIZE(buffers); i++) ++ if (buffers[i]->buffer) ++ sha256_update(&ctx, buffers[i]->buffer, buffers[i]->size); ++ sha256_final(&ctx, digest); ++} ++ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk) ++{ ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ if (rtk->crashed) ++ return -EIO; ++ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_QUIESCED); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=ap-ack state=0x10\n"); ++ ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=iop-ack state=0x201\n"); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, before); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, after); ++ if (rtk->crashed || memcmp(before, after, sizeof(before))) ++ return -EBUSY; ++ /* No more callback can race the caller's halt/drain. Do not reinit. */ ++ apple_mbox_stop(rtk->mbox); ++ flush_workqueue(rtk->wq); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_rtkit_shutdown_preserve); ++ + int apple_rtkit_shutdown(struct apple_rtkit *rtk) + { + int ret; +diff --git a/include/linux/soc/apple/aop.h b/include/linux/soc/apple/aop.h +index 8c8bc71b6..b5188a0b1 100644 +--- a/include/linux/soc/apple/aop.h ++++ b/include/linux/soc/apple/aop.h +@@ -7,6 +7,8 @@ + + #include + ++struct device; ++ + struct apple_aop; + + /* +@@ -40,4 +42,30 @@ void apple_aop_unlisten(const struct apple_aop_service *service); + int apple_aop_call(const struct apple_aop_service *service, u16 subtype, + const void *data, size_t len, void *reply, size_t reply_len); + ++/* The EPIC notification subtype that sets a service property */ ++#define APPLE_AOP_SET_PROPERTY 0x4 ++ ++/* Set property @key of a service to @len (at most 64) bytes at @value. */ ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len); ++ ++/* ++ * A factory calibration blob of a sensor: the bootloader copies it from the ++ * Apple device tree into the sensor's node as @property, and the firmware ++ * takes it as property @key. The calibration is complete once a blob of each ++ * @group (1 to 31) was applied; group 0 blobs are not needed for that. ++ */ ++struct apple_aop_calibration { ++ const char *property; ++ u32 key; ++ u8 group; ++}; ++ ++/* ++ * Send the factory calibration found in @dev's node, then tell the firmware ++ * whether it is complete. Some firmware (S8000) sends no reports before. ++ */ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count); ++ + #endif /* _LINUX_SOC_APPLE_AOP_H_ */ +diff --git a/include/linux/soc/apple/pmgr-observe.h b/include/linux/soc/apple/pmgr-observe.h +new file mode 100644 +index 000000000..477b7fa22 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-observe.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++#define _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++ ++#include ++ ++struct device_node; ++ ++struct apple_pmgr_observation { ++ u32 raw; ++ int genpd_status; ++}; ++ ++/* ++ * Diagnostic only. Nodes must be in child-before-parent lock order, with no ++ * duplicates (maximum five). Takes no power/runtime-PM reference. read() must ++ * not sleep or acquire genpd locks; it runs with IRQs disabled and all domain ++ * locks held, only if every genpd and hardware actual/target state is ON and ++ * reset/disable/parent-off are clear. Hardware auto-PM remains untouched and is ++ * recorded in raw; locks exclude software transitions, not autonomous gating. ++ * -EHOSTDOWN means read() was skipped. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data); ++ ++#endif +diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h +index 1f9ebb618..c8e8df255 100644 +--- a/include/linux/soc/apple/rtkit.h ++++ b/include/linux/soc/apple/rtkit.h +@@ -124,6 +124,8 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); + * Shutdown the co-processor + */ + int apple_rtkit_shutdown(struct apple_rtkit *rtk); ++/* AP 0x10 / IOP 0x201, stable system RAM, stop RX; retain every grant. */ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk); + + /* + * Stop the mailbox of a shut-down co-processor, so that its power domain can +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index 3fcdf4b4e..fcc51b699 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4589,6 +4589,7 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) + { + return __pr_flush(NULL, timeout_ms, reset_on_progress); + } ++EXPORT_SYMBOL_GPL(pr_flush); + + /* + * Delayed printk version, for scheduler-internal messages: diff --git a/tests/test_j71t_data.py b/tests/test_j71t_data.py new file mode 100644 index 00000000..c6d83c08 --- /dev/null +++ b/tests/test_j71t_data.py @@ -0,0 +1,31 @@ +"""J71t firmware recipe: the AOP images.""" +import json +from pathlib import Path +import tomllib +import unittest + +HARDWARE = Path(__file__).resolve().parents[1] +DEVICE = HARDWARE / 'devices/j71t/device.toml' + + +class J71tData(unittest.TestCase): + def setUp(self): + device = tomllib.loads(DEVICE.read_text()) + self.recipe = json.loads((DEVICE.parent / device['firmware']['recipe']).read_text()) + + def test_aop_images_come_from_the_board_firmware(self): + aop = self.recipe['inputs']['aop.im4p'] + self.assertEqual(aop['ipsw'], {'zip': 'Firmware/AOP/aopfw-ipad6faop.im4p'}) + self.assertEqual(aop['sha256'], '5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa') + for name, role, digest in ( + ('aop-ipad6f.bin', 'AOP', 'af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8'), + ('aop-ipad6f-ext.bin', 'AOP_ext', '2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22')): + output, = [o for o in self.recipe['outputs'] if o['destination'] == 'lib/firmware/apple/' + name] + self.assertEqual(output['sha256'], digest) + self.assertEqual(output['steps'], [{'step': 'im4p-payload@1', 'input': 'aop.im4p'}, + {'step': 'aop-image@1', 'coprocessor': role}]) + self.assertFalse((HARDWARE / output['destination']).exists()) + + +if __name__ == '__main__': + unittest.main() From 28c7d41f017a4ef2348bfef80dbf7a222e8c17c7 Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 04/33] PCI/NVMe: retain the stopped ANS handoff across kexec kernel/configs/apple-16k/config: # CONFIG_NVME_APPLE_ANS_EARLY_USB is not set These options provide warm-storage and its Kconfig dependencies on both page sizes. kernel/configs/apple-4k/config: # CONFIG_NVME_APPLE_ANS_EARLY_USB is not set These options provide warm-storage and its Kconfig dependencies on both page sizes. Assisted-by: LLM --- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + kernel/configs/apple-16k/config | 1 + kernel/configs/apple-4k/config | 1 + .../0032-warm-storage-implementation.patch | 1579 +++++++++++++++++ 5 files changed, 1587 insertions(+) create mode 100644 kernel/patches/drivers/pci/0032-warm-storage-implementation.patch diff --git a/chips/s8003.toml b/chips/s8003.toml index fb750ff9..a8dc5c58 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -21,3 +21,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: aop. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/aop.md. + +# Local capability delta: warm-storage. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-storage.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index b8b78cbb..594ff9a9 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -60,3 +60,6 @@ camera = "missing" # Local capability delta: aop. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/aop.md. + +# Local capability delta: warm-storage. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-storage.md. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index 8676844b..8cd1f68e 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -1815,6 +1815,7 @@ CONFIG_BLK_DEV_NBD=y # CONFIG_NVME_CORE=m CONFIG_BLK_DEV_NVME=m +# CONFIG_NVME_APPLE_ANS_EARLY_USB is not set # CONFIG_NVME_MULTIPATH is not set # CONFIG_NVME_VERBOSE_ERRORS is not set # CONFIG_NVME_HWMON is not set diff --git a/kernel/configs/apple-4k/config b/kernel/configs/apple-4k/config index bf332e9c..e10187a1 100644 --- a/kernel/configs/apple-4k/config +++ b/kernel/configs/apple-4k/config @@ -1816,6 +1816,7 @@ CONFIG_BLK_DEV_NBD=y # CONFIG_NVME_CORE=m CONFIG_BLK_DEV_NVME=m +# CONFIG_NVME_APPLE_ANS_EARLY_USB is not set # CONFIG_NVME_MULTIPATH is not set # CONFIG_NVME_VERBOSE_ERRORS is not set # CONFIG_NVME_HWMON is not set diff --git a/kernel/patches/drivers/pci/0032-warm-storage-implementation.patch b/kernel/patches/drivers/pci/0032-warm-storage-implementation.patch new file mode 100644 index 00000000..9f3131e9 --- /dev/null +++ b/kernel/patches/drivers/pci/0032-warm-storage-implementation.patch @@ -0,0 +1,1579 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] PCI: apple-h9p: PCI/NVMe: retain the stopped ANS handoff across kexec + +Pair the S8003 H9P powered handoff with an ANS stop certificate covering SHST, CC/RDY, BME, queue/tag retirement, final NVMMU FIFO acknowledgement and endpoint DART shutdown. Include the final fatal-kexec drain/isolation correction and the modular kexec-state export required by NVMe. Failed stops withhold the certificate; failed powered resume stays under PERST. Real suspend/resume and the Orchard warm ladder remain pending, with no automatic suspend enablement. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 1ee6add55..7906595cd 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -35,6 +35,7 @@ + #include + + #include ++#include + + #include "dma-iommu.h" + +@@ -1009,6 +1010,37 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++int apple_ans_dart_set_active(struct device *dev, bool active) ++{ ++ struct iommu_domain *domain = iommu_get_domain_for_dev(dev); ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ struct apple_dart_domain *d; ++ int i, ret; ++ ++ if (!of_machine_is_compatible("apple,j71t") || !domain || !cfg || ++ !cfg->stream_maps[0].dart || ++ (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) ++ return -ENODEV; ++ d = to_dart_domain(domain); ++ for_each_stream_map(i, cfg, map) { ++ if (map->dart->hw->type != DART_S5L8960X) ++ return -EOPNOTSUPP; ++ if (!active) { ++ apple_dart_hw_disable_dma(map); ++ continue; ++ } ++ ret = apple_dart_setup_translation(d, map); ++ if (ret) ++ return ret; ++ } ++ if (active) ++ WRITE_ONCE(d->flush_error, 0); ++ dev_info(dev, "Apple ANS: DART force_active(%u) complete\n", active); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_set_active); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { +diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig +index 31974c7dd..0504eb40d 100644 +--- a/drivers/nvme/host/Kconfig ++++ b/drivers/nvme/host/Kconfig +@@ -14,6 +14,15 @@ config BLK_DEV_NVME + To compile this driver as a module, choose M here: the + module will be called nvme. + ++config NVME_APPLE_ANS_EARLY_USB ++ bool "Defer S8000/S8003 PCIe/ANS until initramfs USB rescue is ready" ++ depends on BLK_DEV_NVME && PCIE_APPLE_H9P ++ help ++ Default pcie_apple_h9p.probe and nvme.apple_ans_probe to false ++ on S8000/S8003. Requires an initramfs which starts USB rescue, opens ++ both gates and reprobes the host and endpoints. Leave off with ++ the ordinary root initrd. Other boards are unchanged. ++ + config NVME_MULTIPATH + bool "NVMe multipath support" + depends on NVME_CORE +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index 5ba239e47..0db0cb71b 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -84,6 +84,7 @@ struct apple_ans_nvme { + u32 cc; + unsigned long cc_jiffies; + bool cc_written; ++ bool host_fault; + dma_addr_t hmb_dma; + u32 hmb_size; + struct apple_ans_nvme_req req[APPLE_ANS_NVMMU_MAX_REQS]; +@@ -218,13 +219,20 @@ static int nvme_pci_apple_ans_reset_dma(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long flags; +- unsigned int tag; + int ret; + +- for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { +- ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ans->host == &apple_ans_host_s8003) { ++ ret = apple_ans_nvmmu_reset(dev->dev); + if (ret) + return ret; ++ } else { ++ unsigned int tag; ++ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } + } + + spin_lock_irqsave(&ans->req_lock, flags); +@@ -258,6 +266,14 @@ static int nvme_pci_apple_ans_config(struct nvme_dev *dev) + return 0; + } + ++static bool apple_ans_probe = !IS_ENABLED(CONFIG_NVME_APPLE_ANS_EARLY_USB); ++module_param(apple_ans_probe, bool, 0644); ++ ++bool nvme_pci_apple_ans_probe_allowed(struct pci_dev *pdev) ++{ ++ return !nvme_pci_apple_ans_match(pdev) || READ_ONCE(apple_ans_probe); ++} ++ + static int nvme_pci_apple_ans_preinit(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); +@@ -326,10 +342,24 @@ static int nvme_pci_apple_ans_prepare_enable(struct nvme_dev *dev) + return 0; + } + ++static int nvme_pci_apple_ans_check_link(struct nvme_dev *dev) ++{ ++ struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); ++ ++ if (ans && ans->host == &apple_ans_host_s8003 && ++ (ans->host_fault || apple_ans_link_up(dev->dev) <= 0)) { ++ ans->host_fault = true; ++ return -EIO; ++ } ++ return 0; ++} ++ + static int nvme_pci_apple_ans_write_cc(struct nvme_dev *dev, u32 cc) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + writel(cc, dev->bar + NVME_REG_CC); + if (!ans) + return 0; +@@ -349,6 +379,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long deadline; + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + *csts = readl(dev->bar + NVME_REG_CSTS); + if (!ans || !ans->cc_written || !ans->host->rdy_timeout_ms || + (ans->cc & NVME_CC_SHN_MASK)) +@@ -360,6 +392,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + time_after(jiffies, deadline)) { + dev_err(dev->dev, "ANS RDY timeout (CC %#x, CSTS %#x)\n", + ans->cc, *csts); ++ ans->host_fault = true; ++ apple_ans_host_dump(dev->dev, "RDY deadline"); + return -ETIMEDOUT; + } + +@@ -419,6 +453,7 @@ static int nvme_pci_apple_ans_free_req(struct nvme_dev *dev, + dev_err_ratelimited(dev->dev, + "failed to release ANS DMA slot %u: %d\n", + tag, ret); ++ nvme_reset_ctrl(&dev->ctrl); + return ret; + } + +diff --git a/drivers/nvme/host/pci-internal.h b/drivers/nvme/host/pci-internal.h +index a62777abd..ef3ea7b99 100644 +--- a/drivers/nvme/host/pci-internal.h ++++ b/drivers/nvme/host/pci-internal.h +@@ -74,6 +74,8 @@ struct nvme_dev { + struct nvme_ctrl ctrl; + u32 last_ps; + bool hmb; ++ unsigned long apple_ans_flags; ++ atomic_t apple_ans_irq_count; + struct sg_table *hmb_sgt; + mempool_t *dmavec_mempool; + const struct nvme_pci_dma_ops *dma_ops; +@@ -195,6 +197,7 @@ struct nvme_iod { + + extern const struct nvme_pci_dma_ops nvme_pci_apple_ans_ops; + ++bool nvme_pci_apple_ans_probe_allowed(struct pci_dev *pdev); + bool nvme_pci_apple_ans_match(struct pci_dev *pdev); + + #endif /* _NVME_PCI_INTERNAL_H */ +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 9ffe2fb74..32cc6ba89 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -5,6 +5,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -17,11 +18,17 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + #include + #include + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -1559,6 +1566,9 @@ static irqreturn_t nvme_irq(int irq, void *data) + DEFINE_IO_COMP_BATCH(iob); + bool found; + ++ if (!nvmeq->qid && dev->dma_ops == &nvme_pci_apple_ans_ops) ++ atomic_inc(&dev->apple_ans_irq_count); ++ + if (nvme_apple_ans_shared_vector(dev)) + found = nvme_apple_ans_irq_vector(dev, nvmeq->cq_vector, &iob); + else +@@ -1836,6 +1846,40 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) + "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n"); + } + ++/* No endpoint MMIO/config or doorbell reads: safe first evidence for ANS. */ ++static void nvme_apple_ans_dump(struct nvme_dev *dev, const char *reason) ++{ ++ unsigned int i; ++ ++ dev_info(dev->dev, "Apple ANS: snapshot %s state %d online %u allocated %u vectors %u recovery_used %u\n", ++ reason, nvme_ctrl_state(&dev->ctrl), dev->online_queues, ++ dev->ctrl.queue_count, dev->num_vecs, ++ test_bit(0, &dev->apple_ans_flags)); ++ apple_ans_host_dump(dev->dev, reason); ++ for (i = 0; i < dev->ctrl.queue_count; i++) { ++ struct nvme_queue *q = &dev->queues[i]; ++ u16 head = READ_ONCE(q->cq_head); ++ struct nvme_completion *cqe; ++ int irq = -1; ++ ++ if (!q->cqes || head >= q->q_depth) ++ continue; ++ cqe = &q->cqes[head]; ++ if (test_bit(NVMEQ_ENABLED, &q->flags) && ++ !test_bit(NVMEQ_POLLED, &q->flags)) ++ irq = pci_irq_vector(to_pci_dev(dev->dev), q->cq_vector); ++ dev_info(dev->dev, "Apple ANS: q%u flags %#lx depth %u SQ tail %u/%u CQ head %u phase %u next status %#x cid %#x sqhead %u vector %u irq %d ANS admin IRQ actions %u SQ %pad CQ %pad\n", ++ i, READ_ONCE(q->flags), q->q_depth, ++ READ_ONCE(q->sq_tail), READ_ONCE(q->last_sq_tail), ++ head, READ_ONCE(q->cq_phase), ++ le16_to_cpu(READ_ONCE(cqe->status)), ++ le16_to_cpu(READ_ONCE(cqe->command_id)), ++ le16_to_cpu(READ_ONCE(cqe->sq_head)), q->cq_vector, ++ irq, atomic_read(&dev->apple_ans_irq_count), ++ &q->sq_dma_addr, &q->cq_dma_addr); ++ } ++} ++ + static enum blk_eh_timer_return nvme_timeout(struct request *req) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); +@@ -1844,8 +1888,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + struct request *abort_req; + struct nvme_command cmd = { }; + struct pci_dev *pdev = to_pci_dev(dev->dev); +- u32 csts = readl(dev->bar + NVME_REG_CSTS); ++ u32 csts = ~0U; + u8 opcode; ++ bool ans = dev->dma_ops == &nvme_pci_apple_ans_ops && ++ apple_ans_is_s8003(dev->dev); ++ ++ if (ans) { ++ nvme_apple_ans_dump(dev, "timeout before CSTS"); ++ if (dev->dma_ops->read_csts(dev, &csts)) ++ goto disable; ++ } else { ++ csts = readl(dev->bar + NVME_REG_CSTS); ++ } + + /* + * Shutdown the device immediately if we see it is disconnected. This +@@ -1921,6 +1975,11 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + * returned to the driver, or if this is the admin queue. + */ + opcode = nvme_req(req)->cmd->common.opcode; ++ if (ans) { ++ if (test_and_set_bit(0, &dev->apple_ans_flags)) ++ nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); ++ goto disable; ++ } + if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) { + dev_warn(dev->ctrl.device, + "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n", +@@ -2804,7 +2863,32 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, + return a->mode; + } + ++static ssize_t apple_ans_diag_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); ++ u32 cc, csts; ++ ++ if (ndev->dma_ops != &nvme_pci_apple_ans_ops) ++ return -EOPNOTSUPP; ++ if (!sysfs_streq(buf, "host") && !sysfs_streq(buf, "endpoint")) ++ return -EINVAL; ++ mutex_lock(&ndev->shutdown_lock); ++ nvme_apple_ans_dump(ndev, "sysfs"); ++ if (sysfs_streq(buf, "endpoint") && apple_ans_link_up(ndev->dev) > 0) { ++ if (!ndev->dma_ops->read_csts(ndev, &csts)) { ++ cc = readl(ndev->bar + NVME_REG_CC); ++ dev_info(dev, "ANS endpoint CC %#x CSTS %#x\n", cc, csts); ++ } ++ } ++ mutex_unlock(&ndev->shutdown_lock); ++ return count; ++} ++static DEVICE_ATTR_WO(apple_ans_diag); ++ + static struct attribute *nvme_pci_attrs[] = { ++ &dev_attr_apple_ans_diag.attr, + &dev_attr_cmb.attr, + &dev_attr_cmbloc.attr, + &dev_attr_cmbsz.attr, +@@ -3283,11 +3367,69 @@ static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev) + return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY); + } + +-static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static bool apple_ans_sleep_pair = true; ++module_param(apple_ans_sleep_pair, bool, 0644); ++static bool apple_ans_sleep_cc0_ok; ++module_param(apple_ans_sleep_cc0_ok, bool, 0644); ++ ++static int nvme_apple_ans_sleep_precheck(struct notifier_block *nb, ++ unsigned long action, void *unused) ++{ ++ if (action == PM_SUSPEND_TO_IDLE && ++ of_machine_is_compatible("apple,s8003") && ++ !READ_ONCE(apple_ans_sleep_pair)) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_DONE; ++} ++ ++static struct notifier_block nvme_apple_ans_sleep_nb = { ++ .notifier_call = nvme_apple_ans_sleep_precheck, ++}; ++ ++/* SHST alone does not certify a stopped controller. Keep endpoint MMIO before ++ * clearing BME and retiring translations, as in suspend-r1c/r1d. */ ++static int nvme_apple_ans_sleep_disable(struct nvme_dev *dev, int shutdown_ret) ++{ ++ struct pci_dev *pdev = to_pci_dev(dev->dev); ++ u32 csts = ~0U, cc, expected; ++ int ret, err; ++ ++ ret = dev->dma_ops->read_csts(dev, &csts); ++ if (shutdown_ret || ret || (csts & NVME_CSTS_CFS) || ++ (csts & NVME_CSTS_SHST_MASK) != NVME_CSTS_SHST_CMPLT) { ++ pci_clear_master(pdev); ++ return -EBUSY; ++ } ++ dev->ctrl.ctrl_config &= ~(NVME_CC_ENABLE | NVME_CC_SHN_MASK); ++ expected = dev->ctrl.ctrl_config; ++ ret = dev->dma_ops->write_cc(dev, expected); ++ if (!ret) { ++ ret = read_poll_timeout(dev->dma_ops->read_csts, err, ++ err || !(csts & NVME_CSTS_RDY), ++ 1000, 2000000, false, dev, &csts); ++ if (!ret) ++ ret = err; ++ } ++ if (ret) ++ goto block_dma; ++ cc = readl(dev->bar + NVME_REG_CC); ++ if ((!kexec_in_progress && csts) || (csts & NVME_CSTS_CFS) || ++ (cc & (NVME_CC_ENABLE | NVME_CC_SHN_MASK)) || ++ (!kexec_in_progress && cc != expected && ++ !(READ_ONCE(apple_ans_sleep_cc0_ok) && !cc))) ++ ret = -EBUSY; ++block_dma: ++ pci_clear_master(pdev); ++ return ret; ++} ++ ++static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool paired) + { + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); + dead = nvme_pci_ctrl_is_dead(dev); +@@ -3298,15 +3440,26 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown) +- nvme_wait_freeze_timeout(&dev->ctrl); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) ++ panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); + } + + nvme_quiesce_io_queues(&dev->ctrl); + ++ if (paired && (dead || !dev->ctrl.queue_count)) ++ panic("Apple ANS: dead outgoing controller; refusing kexec"); + if (!dead && dev->ctrl.queue_count > 0) { ++ if (paired) { ++ nvme_sync_io_queues(&dev->ctrl); ++ ret = apple_ans_nvmmu_drain(dev->dev); ++ if (ret) ++ panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ } + nvme_delete_io_queues(dev); +- nvme_disable_ctrl(&dev->ctrl, shutdown); ++ dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); ++ ret = nvme_disable_ctrl(&dev->ctrl, shutdown); ++ if (paired) ++ ret = nvme_apple_ans_sleep_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } + nvme_suspend_io_queues(dev); +@@ -3329,7 +3482,15 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) + nvme_unquiesce_admin_queue(&dev->ctrl); + } ++ if (paired && !ret) ++ ret = apple_ans_sleep_stopped(dev->dev); + mutex_unlock(&dev->shutdown_lock); ++ return ret; ++} ++ ++static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++{ ++ nvme_dev_disable_paired(dev, shutdown, false); + } + + static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) +@@ -3775,7 +3936,10 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) + struct nvme_dev *dev; + int result = -ENOMEM; + ++ if (!nvme_pci_apple_ans_probe_allowed(pdev)) ++ return -EPROBE_DEFER; + dev = nvme_pci_alloc_dev(pdev, id); ++ + if (IS_ERR(dev)) + return PTR_ERR(dev); + +@@ -3920,6 +4084,12 @@ static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); + ++ if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) ++ panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + nvme_disable_prepare_reset(dev, true); + } + +@@ -3970,6 +4140,14 @@ static int nvme_resume(struct device *dev) + struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); + struct nvme_ctrl *ctrl = &ndev->ctrl; + ++ if (apple_ans_is_s8003(ndev->dev) && ndev->dma_ops == &nvme_pci_apple_ans_ops && ++ ndev->last_ps == U32_MAX) { ++ int ret = apple_ans_dart_set_active(ndev->dev, true); ++ ++ if (ret) ++ return ret; ++ } ++ + if (ndev->last_ps == U32_MAX || + nvme_set_power_state(ctrl, ndev->last_ps) != 0) + goto reset; +@@ -3989,6 +4167,19 @@ static int nvme_suspend(struct device *dev) + int ret = -EBUSY; + + ndev->last_ps = U32_MAX; ++ if (apple_ans_is_s8003(ndev->dev) && ndev->dma_ops == &nvme_pci_apple_ans_ops) { ++ if (!READ_ONCE(apple_ans_sleep_pair) || apple_ans_sleep_begin(ndev->dev)) ++ return -EBUSY; ++ if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || ++ !nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) ++ return -EBUSY; ++ ret = nvme_dev_disable_paired(ndev, true, true); ++ if (ret) { ++ apple_ans_dart_set_active(ndev->dev, true); ++ nvme_try_sched_reset(ctrl); ++ } ++ return ret; ++ } + + /* + * The platform does not remove power for a kernel managed suspend so +@@ -4356,12 +4547,20 @@ static struct pci_driver nvme_driver = { + + static int __init nvme_init(void) + { ++ int ret; ++ + BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); + BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); + BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); + BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2); + +- return pci_register_driver(&nvme_driver); ++ ret = register_pm_suspend_precheck_notifier(&nvme_apple_ans_sleep_nb); ++ if (ret) ++ return ret; ++ ret = pci_register_driver(&nvme_driver); ++ if (ret) ++ unregister_pm_suspend_precheck_notifier(&nvme_apple_ans_sleep_nb); ++ return ret; + } + + static void __exit nvme_exit(void) +@@ -4369,6 +4568,7 @@ static void __exit nvme_exit(void) + kfree(nvme_pci_quirk_list); + pci_unregister_driver(&nvme_driver); + flush_workqueue(nvme_wq); ++ unregister_pm_suspend_precheck_notifier(&nvme_apple_ans_sleep_nb); + } + + MODULE_AUTHOR("Matthew Wilcox "); +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 15f7a0cbe..52a79fba6 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -15,6 +15,8 @@ + + #include + #include ++#include ++#include + #include + #include + #include +@@ -38,12 +40,20 @@ + #include + #include + #include ++#include + #include + #include + #include ++#include ++#include ++#include ++#include ++#include ++#include + + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -248,7 +258,7 @@ + #define S8003_PORT_LINKCDMSTS_LTSSM GENMASK(14, 9) + #define S8003_LTSSM_L0 0x11 + +-/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (j71t) */ ++/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (S8003) */ + #define S8003_PERST_TO_REFCLK_US 60 + #define S8003_REFCLK_TO_PERST_US 100 + #define S8003_PERST_HOLD_MS 100 +@@ -269,7 +279,7 @@ + + /* + * NVMMU invalidate FIFO status, in the port 0 block (iBoot sets the live ADT +- * nvme-mmu0 invalidate-fifo-workaround = 1 on j71t). iOS (AppleNVMeMMU, ++ * nvme-mmu0 invalidate-fifo-workaround = 1 on S8003). iOS (AppleNVMeMMU, + * 0xfffffff005eebc9c) waits up to 20 ms for it to drain before every + * invalidate and never reads the invalidate register back. + */ +@@ -429,7 +439,7 @@ static const struct apple_h9p_tunable t7000_port_tunables[] = { + }; + + /* +- * S8003 tunables, which iBoot puts in the live ADT; copied from the j71t ++ * S8003 tunables, which iBoot puts in the live ADT; copied from the S8003 + * live ADT (iOS 16.7.10 iBoot), where both bridges carry the same values. + * apcie-phy-tunables and apcie-common-tunables apply after the common init, + * in that order. +@@ -582,6 +592,7 @@ struct apple_h9p_tunables { + struct apple_h9p_nvmmu { + struct apple_h9p_pcie *pcie; + void __iomem *base; ++ bool poisoned; + /* Programmed: the ANS glue may map through it */ + bool ready; + /* Serializes the invalidate FIFO wait and the invalidate write */ +@@ -711,6 +722,8 @@ struct apple_h9p_pcie { + u32 late_ports; + unsigned int late_tries; + struct delayed_work late_work; ++ bool ans_sleep_stopped, ans_sleep_powered; ++ struct notifier_block sleep_precheck; + + struct apple_h9p_nvmmu nvmmu; + +@@ -1554,6 +1567,64 @@ static irqreturn_t apple_h9p_nvmmu_irq(int irq, void *data) + return IRQ_HANDLED; + } + ++static int apple_h9p_nvmmu_program(struct apple_h9p_pcie *pcie) ++{ ++ struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; ++ u32 val; ++ int ret; ++ ++ if (pcie->hw->nvmmu_inv_fifo) { ++ /* ++ * iOS's S8003 order: the invalidate FIFO drained, the old SART ++ * aperture off before its address changes, the zeroed scratch ++ * page and TCBs visible, then scratch page, TCB base, init, SART. ++ */ ++ ret = readl_poll_timeout(pcie->base_port[0] + ++ S8003_PORT_NVMMU_INV_FIFO, val, ++ val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ dma_wmb(); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ } else { ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ } ++ h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, ++ nvmmu->base + H9P_NVMMU_TCB_CTRL); ++ ++ ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, ++ pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ if (ret) ++ return ret; ++ ++ h9p_writel_flush(nvmmu->va_base - 0x80000000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_BASE); ++ h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, ++ H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_END); ++ h9p_writel_flush(nvmmu->pa_base >> 20, ++ nvmmu->base + H9P_NVMMU_SART_PA_BASE); ++ h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ ++ return 0; ++} ++ + static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + { + struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; +@@ -1561,7 +1632,7 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + struct device_node *mem_np; + struct resource res; + resource_size_t size; +- u32 iova, val; ++ u32 iova; + int irq; + int ret; + +@@ -1630,55 +1701,10 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + } + + spin_lock_init(&nvmmu->inv_lock); +- if (pcie->hw->nvmmu_inv_fifo) { +- /* +- * iOS's S8003 order: the invalidate FIFO drained, the old SART +- * aperture off before its address changes, the zeroed scratch +- * page and TCBs visible, then scratch page, TCB base, init, SART. +- */ +- ret = readl_poll_timeout(pcie->base_port[0] + +- S8003_PORT_NVMMU_INV_FIFO, val, +- val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, +- 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); +- if (ret) +- goto out_put_node; +- h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); +- dma_wmb(); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- } else { +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- } +- h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, +- nvmmu->base + H9P_NVMMU_TCB_CTRL); +- +- ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, +- pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ ret = apple_h9p_nvmmu_program(pcie); + if (ret) + goto out_put_node; + +- h9p_writel_flush(nvmmu->va_base - 0x80000000U, +- nvmmu->base + H9P_NVMMU_SART_VA_BASE); +- h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, +- H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, +- nvmmu->base + H9P_NVMMU_SART_VA_END); +- h9p_writel_flush(nvmmu->pa_base >> 20, +- nvmmu->base + H9P_NVMMU_SART_PA_BASE); +- h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); +- + irq = platform_get_irq_byname_optional(pcie->pdev, "nvmmu"); + if (irq > 0) { + ret = devm_request_irq(dev, irq, apple_h9p_nvmmu_irq, 0, +@@ -1726,6 +1752,7 @@ static int apple_h9p_nvmmu_invalidate(struct apple_h9p_pcie *pcie, + spin_unlock_irqrestore(&nvmmu->inv_lock, flags); + + if (ret) { ++ WRITE_ONCE(nvmmu->poisoned, true); + dev_err_ratelimited(pcie->dev, + "NVMMU invalidate FIFO not empty (%#x), tag %u\n", + val, tag); +@@ -1781,6 +1808,8 @@ int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + nvmmu = &pcie->nvmmu; + if (!nvmmu->ready) + return -EOPNOTSUPP; ++ if (READ_ONCE(nvmmu->poisoned)) ++ return -EIO; + + tcb = (u32 *)nvmmu->tcb + tag * H9P_NVMMU_TCB_DWORDS; + sgl = (u32 *)nvmmu->tcb_sgl + tag * H9P_NVMMU_SGL_WORDS; +@@ -2705,6 +2734,8 @@ static int apple_s8003_power_cycle_port(struct apple_h9p_pcie *pcie, + return ret; + } + ++ if (!port && pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); + apple_s8003_port_off(pcie, port); + usleep_range(S8003_POWER_OFF_SETTLE_US, S8003_POWER_OFF_SETTLE_US + 1000); + ret = apple_h9p_wait(rc + S8003_RC_PORT_CTL(port), +@@ -2887,6 +2918,81 @@ static int apple_s8003_start_link(struct apple_h9p_pcie *pcie, + return ret; + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* S8003 adaptation of suspend-r1d/r1e/r1f to Orchard's host driver. */ ++static unsigned int h9p_suspend; ++module_param_named(suspend, h9p_suspend, uint, 0644); ++static bool ans_keep_power; ++module_param(ans_keep_power, bool, 0644); ++static char *ans_boot_mode = "handoff"; ++module_param(ans_boot_mode, charp, 0444); ++static unsigned int ans_sleep_flush = 3; ++module_param(ans_sleep_flush, uint, 0644); ++static unsigned int ans_kexec_flush; ++module_param(ans_kexec_flush, uint, 0644); ++ ++static bool apple_s8003_handoff_enabled(void) ++{ ++ return READ_ONCE(ans_keep_power) && !strcmp(ans_boot_mode, "handoff"); ++} ++ ++static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *rc = pcie->base_rc; ++ ++ return (readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_ENABLE) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_ON) && ++ (readl(rc + S8003_RC_PORT_REFCLK(0)) & S8003_RC_PORT_REFCLK_ON) && ++ !(readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_RESET) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_REFCLK_NOGATE) && ++ (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); ++} ++ ++/* Best effort, bounded, while PERST is asserted. Never cycle port power. */ ++static void apple_s8003_drain_link(struct apple_h9p_pcie *pcie, unsigned int flush) ++{ ++ u32 val; ++ int ret; ++ ++ if (!(flush & 1)) ++ return; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, val, ++ !(val & BIT(0)), 10, 100000); ++ dev_info(pcie->dev, "ANS link drain: status=%#x drained=%u\n", val, !ret); ++} ++ ++static int apple_s8003_flush_link(struct apple_h9p_pcie *pcie, unsigned int flush) ++{ ++ u32 val; ++ ++ if ((flush & 2) && (readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & BIT(0))) { ++ h9p_rmw(pcie->base_port[0] + H9P_PORT_LTSSMCTL, 0, H9P_PORT_LTSSM_ENABLE); ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, val, ++ !(val & BIT(0)), 10, 100000); ++ return apple_s8003_port_stop(pcie, 0); ++ } ++ return 0; ++} ++ ++static int apple_s8003_powered_reset(struct apple_h9p_pcie *pcie, unsigned int flush) ++{ ++ int ret; ++ ++ if (!apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (!ret) ++ ret = apple_s8003_flush_link(pcie, flush); ++ if (!ret) ++ ret = apple_s8003_start_link(pcie, 0); ++ /* A failed powered reset stays under PERST; no full-cycle fallback. */ ++ if (ret) ++ gpiod_direction_output(pcie->perst[0], 1); ++ return ret; ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -2902,6 +3008,15 @@ static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + * validated on it, the other ports the restart sequence that is the + * only one validated on the Wi-Fi port. + */ ++ if (!port && apple_s8003_handoff_enabled() && ++ of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ apple_s8003_port_powered(pcie) && ++ gpiod_get_value_cansleep(pcie->perst[0]) == 1) { ++ ret = apple_s8003_powered_reset(pcie, READ_ONCE(ans_kexec_flush)); ++ if (!ret || (ret != -ETIMEDOUT && ret != -ENOLINK)) ++ return ret; ++ dev_warn(pcie->dev, "powered reset failed %d; full port fallback\n", ret); ++ } + if (pcie->hw->nvme_mode_ports & BIT(port)) + ret = apple_s8003_power_cycle_port(pcie, port); + else +@@ -2945,6 +3060,8 @@ static int apple_s8003_setup_storage_port(struct apple_h9p_pcie *pcie, + if (ret != -ETIMEDOUT && ret != -ENOLINK) + return ret; + ++ /* Probe/kexec may fall back to full reset; sleep resume never uses this. */ ++ + dev_warn(pcie->dev, "port %u: no link (%d), power cycling the port\n", + port, ret); + if ((readl(rc + S8003_RC_PORT_CTL(port)) & S8003_RC_PORT_CTL_ENABLE) && +@@ -3059,6 +3176,11 @@ static void apple_s8003_late_ports_work(struct work_struct *work) + unsigned int port; + int ret; + ++ /* Lab 0019: no secondary bus publication without storage bus 1. */ ++ if (of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") && ++ !(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return; ++ + for_each_set_bit(port, &pending, H9P_NUM_PORTS) { + ret = apple_s8003_setup_endpoint_port(pcie, port); + if (ret == -EPROBE_DEFER) { +@@ -3418,6 +3540,472 @@ static void apple_h9p_pcie_pwrctrl_cleanup(void *data) + pci_pwrctrl_destroy_devices(pcie->dev); + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static struct apple_h9p_pcie *apple_ans_endpoint_host(struct device *dev) ++{ ++ struct device *host = dev; ++ struct apple_h9p_pcie *pcie; ++ ++ while (host && host->bus == dev->bus) ++ host = host->parent; ++ if (!host || !host->parent) ++ return NULL; ++ pcie = apple_h9p_pcie_lookup(host->parent); ++ if (!pcie || !of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") || ++ apple_h9p_pcie_device_port(pcie, dev) != 0) ++ return NULL; ++ return pcie; ++} ++ ++int apple_ans_sleep_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie || h9p_suspend != 2 || !apple_s8003_handoff_enabled()) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_sleep_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_sleep_begin); ++ ++/* release6: caller froze and synchronized I/O, before SHN. */ ++/* Lab 0018: caller synchronized queues and stopped CC/RDY before this. */ ++int apple_ans_nvmmu_reset(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ struct apple_h9p_nvmmu *nvmmu; ++ int ret; ++ ++ if (!pcie || !(pcie->enabled_ports & BIT(0))) ++ return -ENODEV; ++ nvmmu = &pcie->nvmmu; ++ if (!nvmmu->ready) ++ return -ENODEV; ++ memset(nvmmu->tcb, 0, nvmmu->tcb_size); ++ memset(nvmmu->tcb_table, 0, nvmmu->tcb_table_size); ++ memset(nvmmu->tcb_sgl, 0, nvmmu->tcb_sgl_size); ++ ret = apple_h9p_nvmmu_program(pcie); ++ WRITE_ONCE(nvmmu->poisoned, !!ret); ++ if (ret) ++ return ret; ++ bitmap_zero(nvmmu->mapped, APPLE_ANS_NVMMU_MAX_REQS); ++ dev_info(dev, "NVMMU: stopped-controller cache/SART rebuilt\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_reset); ++ ++bool apple_ans_is_s8003(struct device *dev) ++{ ++ return apple_ans_endpoint_host(dev) != NULL; ++} ++EXPORT_SYMBOL_GPL(apple_ans_is_s8003); ++ ++int apple_ans_link_up(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -ENODEV; ++ if (!(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return -ENOLINK; ++ return !!(readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & ++ S8003_PORT_LINKSTS_UP); ++} ++EXPORT_SYMBOL_GPL(apple_ans_link_up); ++ ++/* Host-only evidence, no endpoint config/BAR reads or invalidate register. */ ++void apple_ans_host_dump(struct device *dev, const char *reason) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return; ++ dev_info(dev, "ANS host %s: ports %#x link %#x LTSSM %#x FIFO %#x poisoned %u mapped %*pb\n", ++ reason, pcie->enabled_ports, ++ readl(pcie->base_port[0] + H9P_PORT_LINKSTS), ++ readl(pcie->base_port[0] + H9P_PORT_LTSSMCTL), ++ readl(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO), ++ READ_ONCE(pcie->nvmmu.poisoned), APPLE_ANS_NVMMU_MAX_REQS, ++ pcie->nvmmu.mapped); ++} ++EXPORT_SYMBOL_GPL(apple_ans_host_dump); ++ ++int apple_ans_nvmmu_drain(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ u32 fifo; ++ int ret; ++ ++ if (!pcie || !pcie->nvmmu.ready) ++ return -ENODEV; ++ if (!bitmap_empty(pcie->nvmmu.mapped, APPLE_ANS_NVMMU_MAX_REQS)) ++ return -EBUSY; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ dev_info(dev, "Apple ANS: outgoing NVMMU FIFO empty check (20ms) result %d\n", ret); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_drain); ++ ++int apple_ans_sleep_stopped(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ unsigned int tag; ++ u32 fifo; ++ int ret; ++ ++ if (!pcie) ++ return -ENODEV; ++ /* All queues are drained and BME is clear before retiring translations. */ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } ++ /* The last invalidate also has to leave the FIFO before certification. */ ++ ret = readl_poll_timeout_atomic(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 1, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ ret = apple_ans_dart_set_active(dev, false); ++ if (!ret) ++ WRITE_ONCE(pcie->ans_sleep_stopped, true); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_sleep_stopped); ++ ++int apple_ans_handoff_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie || strcmp(ans_boot_mode, "handoff")) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_sleep_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); ++ ++static void apple_h9p_s8003_shutdown(struct platform_device *pdev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ unsigned int port; ++ ++ if (!kexec_in_progress || !pcie || ++ !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) ++ return; ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!strcmp(ans_boot_mode, "handoff") && ++ (!READ_ONCE(pcie->ans_sleep_stopped) || ++ (READ_ONCE(ans_keep_power) && !apple_s8003_port_powered(pcie)))) ++ panic("Apple ANS: refusing uncertified powered handoff"); ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; ++ ++ if (!(pcie->enabled_ports & BIT(port))) ++ continue; ++ /* release6: block root routing, retain refclk, then isolate. */ ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[port], 1)) ++ panic("Apple PCIe: PERST failed during handoff"); ++ if (!port && apple_s8003_handoff_enabled()) ++ apple_s8003_drain_link(pcie, READ_ONCE(ans_kexec_flush)); ++ if (apple_s8003_port_stop(pcie, port)) ++ panic("Apple PCIe: LTSSM stop failed during handoff"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (!port && pcie->nvmmu.ready) { ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ } ++ if (!port && !strcmp(ans_boot_mode, "handoff") && ++ !READ_ONCE(ans_keep_power)) ++ apple_s8003_port_off(pcie, port); ++ if (!port && apple_s8003_handoff_enabled() && ++ (!apple_s8003_port_powered(pcie) || ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) ++ panic("Apple ANS: outgoing powered port certificate failed"); ++ pcie->enabled_ports &= ~BIT(port); ++ } ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++} ++ ++static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *cfg = pcie->base_config; ++ ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!(pcie->enabled_ports & BIT(0))) ++ return; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[0], 1) || ++ apple_s8003_port_stop(pcie, 0)) ++ panic("ANS capture isolation failed"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ pcie->enabled_ports &= ~BIT(0); ++} ++ ++struct apple_h9p_crash_state { ++ struct apple_h9p_pcie *pcie; ++ int result; ++ bool busy; ++ bool found; ++}; ++ ++/* No NVMe driver, queues, SHN, or old DMA buffers are needed in phoenix. */ ++static int apple_h9p_crash_stop_ans(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_crash_state *state = data; ++ struct apple_h9p_pcie *pcie = state->pcie; ++ void __iomem *bar; ++ u32 csts = 0; ++ int i, ret; ++ ++ if (pdev->vendor != PCI_VENDOR_ID_APPLE || ++ ((pdev->class >> 8) != PCI_CLASS_STORAGE_EXPRESS && ++ (pdev->class >> 8) != PCI_CLASS_STORAGE_OTHER)) ++ return 0; ++ state->found = true; ++ if (pdev->driver) { ++ state->busy = true; ++ return 1; ++ } ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ state->result = -ENOLINK; ++ return 0; ++ } ++ ret = pci_enable_device_mem(pdev); ++ if (ret) { ++ state->result = ret; ++ return 0; ++ } ++ bar = pci_iomap_range(pdev, 0, 0, SZ_8K); ++ if (!bar) { ++ ret = -ENOMEM; ++ goto disable; ++ } ++ /* An unknown queue must never be rung or sent a shutdown notification. */ ++ dev_info(pcie->dev, "phoenix ANS: CC %#x CSTS %#x, clearing EN without SHN\n", ++ readl(bar + NVME_REG_CC), readl(bar + NVME_REG_CSTS)); ++ writel(0, bar + NVME_REG_CC); ++ ret = -ETIMEDOUT; ++ for (i = 0; i < 2000; i++) { ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ ret = -ENOLINK; ++ break; ++ } ++ csts = readl(bar + NVME_REG_CSTS); ++ if (csts == U32_MAX) { ++ ret = -EIO; ++ break; ++ } ++ if (!(csts & NVME_CSTS_RDY)) { ++ ret = 0; ++ break; ++ } ++ usleep_range(1000, 1100); ++ } ++ dev_info(pcie->dev, "phoenix ANS: disable result %d CSTS %#x; blocking bus master\n", ++ ret, csts); ++ pci_iounmap(pdev, bar); ++disable: ++ pci_clear_master(pdev); ++ pci_disable_device(pdev); ++ state->result = ret; ++ return 0; ++} ++ ++static ssize_t crash_quiesce_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ struct apple_h9p_crash_state state = { .pcie = pcie }; ++ ++ if (!pcie || !is_kdump_kernel() || ++ !of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !sysfs_streq(buf, "1")) ++ return -EPERM; ++ /* The endpoint gate must stay closed; don't park under a live driver. */ ++ if (pcie->late_ports || work_busy(&pcie->late_work.work)) ++ return -EBUSY; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_crash_stop_ans, &state); ++ if (state.busy) ++ return -EBUSY; ++ /* Isolation is still required after a bounded RDY timeout. */ ++ /* Wi-Fi stays linked until its own shutdown callback has run. */ ++ apple_h9p_capture_park(pcie); ++ dev_info(dev, "phoenix: ANS/PCIe quiesced (controller found %d result %d)\n", ++ state.found, state.result); ++ return count; ++} ++static DEVICE_ATTR_WO(crash_quiesce); ++ ++static struct attribute *apple_h9p_crash_attrs[] = { ++ &dev_attr_crash_quiesce.attr, ++ NULL, ++}; ++ ++static const struct attribute_group apple_h9p_crash_group = { ++ .attrs = apple_h9p_crash_attrs, ++}; ++ ++ ++/* Orchestrator-only fault injection; never enabled by a normal boot. */ ++static bool panic_resume_once; ++module_param(panic_resume_once, bool, 0600); ++MODULE_PARM_DESC(panic_resume_once, ++ "S8003: deliberately panic before one PCIe resume, only with kdump armed"); ++ ++static void apple_h9p_crash_resume_test(struct device *dev) ++{ ++ if (!of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !READ_ONCE(panic_resume_once)) ++ return; ++ WRITE_ONCE(panic_resume_once, false); ++ if (!kexec_crash_loaded()) { ++ dev_warn(dev, "resume panic test refused: no capture kernel armed\n"); ++ return; ++ } ++ panic("S8003: deliberate panic before PCIe resume"); ++} ++ ++ ++static bool apple_h9p_sleep_consumer_busy(void) ++{ ++ struct device_node *np; ++ struct platform_device *pdev; ++ struct usb_role_switch *sw; ++ struct path root; ++ bool busy; ++ ++ get_fs_root(current->fs, &root); ++ busy = root.dentry->d_sb->s_bdev && ++ MAJOR(root.dentry->d_sb->s_bdev->bd_dev) == NBD_MAJOR; ++ path_put(&root); ++ ++ np = of_find_compatible_node(NULL, NULL, "gpu,s8003"); ++ if (np) { ++ pdev = of_find_device_by_node(np); ++ if (pdev) { ++ busy |= pdev->dev.driver && pm_runtime_active(&pdev->dev); ++ put_device(&pdev->dev); ++ } ++ of_node_put(np); ++ } ++ np = of_find_compatible_node(NULL, NULL, "apple,s8000-usb-role-switch"); ++ if (np) { ++ sw = usb_role_switch_find_by_fwnode(of_fwnode_handle(np)); ++ if (!IS_ERR_OR_NULL(sw)) { ++ busy |= usb_role_switch_get_role(sw) == USB_ROLE_HOST; ++ usb_role_switch_put(sw); ++ } ++ of_node_put(np); ++ } ++ return busy; ++} ++ ++static int apple_h9p_sleep_precheck(struct notifier_block *nb, ++ unsigned long action, void *unused) ++{ ++ struct apple_h9p_pcie *pcie = container_of(nb, struct apple_h9p_pcie, ++ sleep_precheck); ++ ++ if (action != PM_SUSPEND_TO_IDLE) ++ return NOTIFY_DONE; ++ if (h9p_suspend != 2 || !apple_s8003_handoff_enabled() || ++ apple_h9p_sleep_consumer_busy()) { ++ dev_warn(pcie->dev, "sleep refused before sync: suspend=2 and ans_keep_power=1 ans_boot_mode=handoff required; active GPU/USB host or NBD root forbidden\n"); ++ return notifier_from_errno(-EBUSY); ++ } ++ return NOTIFY_DONE; ++} ++ ++static void apple_h9p_sleep_unregister(void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ unregister_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++} ++ ++static int apple_h9p_sleep_register(struct apple_h9p_pcie *pcie) ++{ ++ int ret; ++ ++ if (!of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) ++ return 0; ++ pcie->sleep_precheck.notifier_call = apple_h9p_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++ if (ret) ++ return ret; ++ return devm_add_action_or_reset(pcie->dev, apple_h9p_sleep_unregister, pcie); ++} ++ ++static int apple_h9p_s8003_suspend(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ if (!of_device_is_compatible(dev->of_node, "apple,s8003-pcie")) ++ return 0; ++ if (h9p_suspend != 2 || !apple_s8003_handoff_enabled() || ++ !READ_ONCE(pcie->ans_sleep_stopped) || !apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ cancel_delayed_work_sync(&pcie->late_work); ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (ret) ++ return ret; ++ apple_s8003_drain_link(pcie, READ_ONCE(ans_sleep_flush)); ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (ret) { ++ /* PM skips resume for a parent whose suspend failed. Restore now. */ ++ apple_s8003_powered_reset(pcie, READ_ONCE(ans_sleep_flush)); ++ return ret; ++ } ++ pcie->ans_sleep_powered = true; ++ dev_info(dev, "ANS sleep powered under PERST; no full-cycle fallback\n"); ++ return 0; ++} ++ ++static int apple_h9p_s8003_resume(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ apple_h9p_crash_resume_test(dev); ++ ++ if (!of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !pcie->ans_sleep_powered) ++ return 0; ++ ret = apple_s8003_powered_reset(pcie, READ_ONCE(ans_sleep_flush)); ++ pcie->ans_sleep_powered = false; ++ WRITE_ONCE(pcie->ans_sleep_stopped, false); ++ if (ret) ++ dev_err(dev, "ANS powered resume failed %d; endpoint isolated under PERST\n", ret); ++ else ++ dev_info(dev, "ANS powered resume trained, try=first\n"); ++ return ret; ++} ++ ++static DEFINE_SIMPLE_DEV_PM_OPS(apple_h9p_s8003_pm_ops, ++ apple_h9p_s8003_suspend, apple_h9p_s8003_resume); ++ ++static bool crash_quiesce_supported = true; ++module_param(crash_quiesce_supported, bool, 0444); ++ ++static bool h9p_probe = !IS_ENABLED(CONFIG_NVME_APPLE_ANS_EARLY_USB); ++module_param_named(probe, h9p_probe, bool, 0644); ++MODULE_PARM_DESC(probe, "Allow S8003 PCIe probe after USB rescue is ready"); ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3426,6 +4014,9 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + u32 doorbell; + int ret; + ++ if (!READ_ONCE(h9p_probe) && of_device_is_compatible(dev->of_node, "apple,s8003-pcie")) ++ return -EPROBE_DEFER; ++ + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; +@@ -3578,6 +4169,16 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (is_kdump_kernel() && of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) { ++ ret = devm_device_add_group(dev, &apple_h9p_crash_group); ++ if (ret) ++ return ret; ++ } ++ ++ ret = apple_h9p_sleep_register(pcie); ++ if (ret) ++ return ret; ++ + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +@@ -3660,8 +4261,10 @@ MODULE_DEVICE_TABLE(of, apple_h9p_pcie_of_match); + + static struct platform_driver apple_h9p_pcie_driver = { + .probe = apple_h9p_pcie_probe, ++ .shutdown = apple_h9p_s8003_shutdown, + .driver = { + .name = "pcie-apple-h9p", ++ .pm = pm_sleep_ptr(&apple_h9p_s8003_pm_ops), + .of_match_table = apple_h9p_pcie_of_match, + .suppress_bind_attrs = true, + }, +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index a59861010..788a343ee 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -17,11 +17,36 @@ struct device; + * Map @npages pages for FlatDMA request @tag, or retire the tag if @npages is + * zero. @to_memory: the controller writes the pages (an NVMe read). + */ ++#if IS_REACHABLE(CONFIG_APPLE_DART) ++int apple_ans_dart_set_active(struct device *dev, bool active); ++#else ++static inline int apple_ans_dart_set_active(struct device *dev, bool active) ++{ ++ return -EOPNOTSUPP; ++} ++#endif ++ + #if IS_REACHABLE(CONFIG_PCIE_APPLE_H9P) ++int apple_ans_sleep_begin(struct device *dev); ++int apple_ans_handoff_begin(struct device *dev); ++int apple_ans_sleep_stopped(struct device *dev); ++int apple_ans_nvmmu_drain(struct device *dev); ++int apple_ans_nvmmu_reset(struct device *dev); ++int apple_ans_link_up(struct device *dev); ++bool apple_ans_is_s8003(struct device *dev); ++void apple_ans_host_dump(struct device *dev, const char *reason); + int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova); + #else ++static inline int apple_ans_sleep_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_handoff_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_sleep_stopped(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_drain(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_reset(struct device *dev) { return -EOPNOTSUPP; } ++static inline int apple_ans_link_up(struct device *dev) { return 1; } ++static inline bool apple_ans_is_s8003(struct device *dev) { return false; } ++static inline void apple_ans_host_dump(struct device *dev, const char *reason) {} + static inline int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova) +diff --git a/include/linux/suspend.h b/include/linux/suspend.h +index b02876f1a..0a4719b20 100644 +--- a/include/linux/suspend.h ++++ b/include/linux/suspend.h +@@ -449,6 +449,15 @@ void restore_processor_state(void); + /* kernel/power/main.c */ + extern int register_pm_notifier(struct notifier_block *nb); + extern int unregister_pm_notifier(struct notifier_block *nb); ++/** ++ * register_pm_suspend_precheck_notifier - register an early suspend veto ++ * @nb: notifier whose action is the target suspend_state_t ++ * ++ * Prechecks run before filesystem sync and console preparation. They must ++ * only validate current policy/state and return; they have no post callback. ++ */ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb); ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb); + extern void ksys_sync_helper(void); + extern void pm_report_hw_sleep_time(u64 t); + extern void pm_report_max_hw_sleep(u64 t); +@@ -498,6 +507,16 @@ static inline int unregister_pm_notifier(struct notifier_block *nb) + return 0; + } + ++static inline int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return 0; ++} ++ ++static inline int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return 0; ++} ++ + static inline void pm_report_hw_sleep_time(u64 t) {}; + static inline void pm_report_max_hw_sleep(u64 t) {}; + +diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c +index dc770b9a6..819650bcd 100644 +--- a/kernel/kexec_core.c ++++ b/kernel/kexec_core.c +@@ -53,6 +53,7 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); + + /* Flag to indicate we are going to kexec a new kernel */ + bool kexec_in_progress = false; ++EXPORT_SYMBOL_GPL(kexec_in_progress); + + bool kexec_file_dbg_print; + +diff --git a/kernel/power/main.c b/kernel/power/main.c +index 5429e9f19..ae36afe43 100644 +--- a/kernel/power/main.c ++++ b/kernel/power/main.c +@@ -152,6 +152,32 @@ int pm_sleep_fs_sync(void) + /* Routines for PM-transition notifications */ + + static BLOCKING_NOTIFIER_HEAD(pm_chain_head); ++static BLOCKING_NOTIFIER_HEAD(pm_suspend_precheck_chain_head); ++ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_register(&pm_suspend_precheck_chain_head, ++ nb); ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_precheck_notifier); ++ ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_unregister(&pm_suspend_precheck_chain_head, ++ nb); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); ++ ++int pm_suspend_precheck_call_chain(suspend_state_t state) ++{ ++ int ret; ++ ++ /* Veto-only: callers have not changed system suspend state yet. */ ++ ret = blocking_notifier_call_chain(&pm_suspend_precheck_chain_head, ++ (unsigned long)state, NULL); ++ ++ return notifier_to_errno(ret); ++} + + int register_pm_notifier(struct notifier_block *nb) + { +diff --git a/kernel/power/power.h b/kernel/power/power.h +index 75b638438..1de8fe895 100644 +--- a/kernel/power/power.h ++++ b/kernel/power/power.h +@@ -240,6 +240,7 @@ static inline void suspend_test_finish(const char *label) {} + /* kernel/power/main.c */ + extern int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down); + extern int pm_notifier_call_chain(unsigned long val); ++int pm_suspend_precheck_call_chain(suspend_state_t state); + #endif + + #ifdef CONFIG_HIGHMEM +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 57c442686..4c5da415a 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -591,6 +591,13 @@ static int enter_state(suspend_state_t state) + if (!mutex_trylock(&system_transition_mutex)) + return -EBUSY; + ++ error = pm_suspend_precheck_call_chain(state); ++ if (error) { ++ dpm_save_failed_step(SUSPEND_PREPARE); ++ mutex_unlock(&system_transition_mutex); ++ return error; ++ } ++ + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); + From 419fe19d39cc8164125a4d3a62bfc7f9d63eb81f Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 05/33] Optional resident m1n1 EL3 PSCI Assisted-by: LLM --- boot/m1n1/README.md | 7 + ...resident-EL3-CPU-lifecycle-for-A7-A9.patch | 880 ++++++++++++++++++ boot/m1n1/psci-loader | 29 + chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + tests/test_m1n1_psci.py | 54 ++ 6 files changed, 976 insertions(+) create mode 100644 boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch create mode 100755 boot/m1n1/psci-loader create mode 100644 tests/test_m1n1_psci.py diff --git a/boot/m1n1/README.md b/boot/m1n1/README.md index bd78f93b..04150148 100644 --- a/boot/m1n1/README.md +++ b/boot/m1n1/README.md @@ -28,3 +28,10 @@ gives the same tree as Hoolock's tip before updating `sources.toml`. Scratch stays in ignored `boot/m1n1/.work//`. `--resume` reuses it only if the recorded patch set is unchanged. + +## Optional resident PSCI on A7-A9 + +Orchard patch 0014 adds the linux-a9 opt-in EL3 PSCI shim. It requires +`psci=1` as a payload variable and entry at EL3; EL1 keeps the spin table. +There is no per-board default. See [J71t loader instructions](../../devices/j71t/README.md) +for `psci-loader`, launch/RAM framing, Pongo evidence and the cold-boot gate. diff --git a/boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch b/boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch new file mode 100644 index 00000000..a677f9a7 --- /dev/null +++ b/boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch @@ -0,0 +1,880 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sat, 7 Nov 2026 15:00:00 +0100 +Subject: [PATCH] psci: Add opt-in resident EL3 CPU lifecycle for A7-A9 + +On A7-A9 m1n1 is entered at EL3 and stays resident: its image and the +per-CPU EL3 stacks are memreserved, and VBAR_EL3 points at its vectors +while Linux runs at EL1. Use that to give Linux PSCI 1.0 CPU_ON, CPU_OFF +and AFFINITY_INFO (plus VERSION, FEATURES and MIGRATE_INFO_TYPE). The +spin table has no cpu_die, so without this the kernel refuses kexec on +an SMP system, can't offline CPUs, and a crash kernel inherits CPUs stuck +in the old kernel. + +CPU_OFF does not power the core down. It cleans and invalidates its L1D, +stops its timers, and waits in WFE at EL3 (MMU off) on a one-line +mailbox. CPU_ON posts an entry point; the core leaves with a clean +I-cache and TLB, SCTLR_EL1 at its reset value, DAIF masked and SP_EL3 on +a static per-CPU stack. SYSTEM_OFF/SYSTEM_RESET, CPU_SUSPEND and +SMCCC_VERSION return NOT_SUPPORTED, so callers fall back to their own +methods and assume SMCCC 1.0. + +Opt-in with the payload variable psci=1, and only used when m1n1 itself +was entered at EL3. kboot then parks the secondaries in the pen (a core +that doesn't park is pruned), sets enable-method = "psci" on every CPU, +adds /psci (method = "smc") and records the re-entry address and the +original boot_args in /chosen. + +Re-entry: _reentry can be jumped to from a kexec'd trampoline (EL1, MMU +off, x0 = boot_args copy). It drops this image's lines from the caches, +flags the re-entry and restarts m1n1, which then skips the (stale) +payload area and runs the proxy. smp_start_secondaries() takes cores +back from the pen with CPU_ON, and the mailboxes and the psci=1 setting +live in .data so they survive the .bss clear. + +Rebase the linux-a9 shim onto Orchard's complete m1n1 series without +changing its runtime behavior. The payload variable is not a kernel +command-line option; a bare proxy loader also needs it appended. + +Assisted-by: LLM +--- + Makefile | 1 + + src/kboot.c | 56 +++++++++++++- + src/main.c | 8 +- + src/payload.c | 3 + + src/psci.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + src/psci.h | 78 ++++++++++++++++++++ + src/psci_asm.S | 170 ++++++++++++++++++++++++++++++++++++++++++ + src/smp.c | 73 ++++++++++++++++++ + src/smp.h | 1 + + src/start.S | 6 +- + src/startup.c | 6 +- + 11 files changed, 626 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index bb72266..4aefb37 100644 +--- a/Makefile ++++ b/Makefile +@@ -166,6 +166,7 @@ OBJECTS := \ + pcie.o \ + pmgr.o \ + proxy.o \ ++ psci.o psci_asm.o \ + ringbuffer.o \ + rtkit.o \ + sart.o \ +diff --git a/src/kboot.c b/src/kboot.c +index b75bb45..2316709 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -21,6 +21,7 @@ + #include "memory.h" + #include "pcie.h" + #include "pmgr.h" ++#include "psci.h" + #include "sep.h" + #include "sio.h" + #include "smc.h" +@@ -574,6 +575,53 @@ static int dt_set_smbios(void) + return 0; + } + ++/* ++ * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * (see psci.c) instead of the spin table, so the kernel can offline them ++ * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. ++ */ ++static int dt_set_psci(void) ++{ ++ static const char compat[] = "arm,psci-1.0\0arm,psci-0.2"; ++ int node; ++ ++ if (!psci_active()) { ++ printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ return 0; ++ } ++ ++ int cpus = fdt_path_offset(dt, "/cpus"); ++ if (cpus < 0) ++ bail("FDT: /cpus node not found in devtree\n"); ++ ++ fdt_for_each_subnode(node, dt, cpus) ++ { ++ if (strncmp(fdt_get_name(dt, node, NULL), "cpu@", 4)) ++ continue; ++ if (fdt_setprop_string(dt, node, "enable-method", "psci")) ++ bail("FDT: couldn't set enable-method\n"); ++ } ++ ++ node = fdt_path_offset(dt, "/psci"); ++ if (node < 0) ++ node = fdt_add_subnode(dt, 0, "psci"); ++ if (node < 0) ++ bail("FDT: couldn't add /psci\n"); ++ if (fdt_setprop(dt, node, "compatible", compat, sizeof(compat)) || ++ fdt_setprop_string(dt, node, "method", "smc")) ++ bail("FDT: couldn't fill /psci\n"); ++ ++ node = fdt_path_offset(dt, "/chosen"); ++ if (node < 0) ++ bail("FDT: /chosen node not found in devtree\n"); ++ if (fdt_setprop_u64(dt, node, "linux-a9,m1n1-reentry", (u64)_reentry) || ++ fdt_setprop(dt, node, "linux-a9,m1n1-boot-args", &cur_boot_args, sizeof(cur_boot_args))) ++ bail("FDT: couldn't set the m1n1 re-entry properties\n"); ++ ++ printf("PSCI: on, enable-method = \"psci\", m1n1 re-entry at %p\n", _reentry); ++ return 0; ++} ++ + static int dt_set_cpus(void) + { + int ret = 0; +@@ -587,6 +635,10 @@ static int dt_set_cpus(void) + if (!pruned_phandles) + bail("FDT: out of memory\n"); + ++ /* Secondaries wait in the PSCI pen; one that can't get there is pruned below */ ++ if (psci_active()) ++ printf("PSCI: parked %d secondary CPU(s)\n", smp_park_secondaries_psci()); ++ + /* Prune CPU nodes */ + int node, cpu = 0; + for (node = fdt_first_subnode(dt, cpus); node >= 0;) { +@@ -618,7 +670,7 @@ static int dt_set_cpus(void) + } else { + printf("FDT: Reserving stack for CPU %d 0x%lx\n", cpu, (uint64_t)secondary_stacks[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks[cpu], SECONDARY_STACK_SIZE); +- if (has_el3()) { ++ if (has_el3() && secondary_stacks_el3[cpu]) { + printf("FDT: Reserving EL3 stack for CPU %d 0x%lx\n", cpu, + (uint64_t)secondary_stacks_el3[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks_el3[cpu], SECONDARY_STACK_SIZE); +@@ -3206,6 +3258,8 @@ int kboot_prepare_dt(void *fdt) + return -1; + if (dt_set_cpus()) + return -1; ++ if (dt_set_psci()) ++ return -1; + if (dt_set_mac_addresses()) + return -1; + if (dt_set_wifi()) +diff --git a/src/main.c b/src/main.c +index d4e828c..3f32380 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -21,6 +21,7 @@ + #include "pcie.h" + #include "pmgr.h" + #include "sep.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "tps6598x.h" +@@ -117,9 +118,12 @@ void run_actions(void) + #endif + #endif + +- printf("Checking for payloads...\n"); ++ if (psci_reentered()) ++ printf("Re-entered from Linux: whatever follows m1n1 is stale, not loading payloads\n"); ++ else ++ printf("Checking for payloads...\n"); + +- if (payload_run() == 0) { ++ if (!psci_reentered() && payload_run() == 0) { + printf("Valid payload found\n"); + return; + } +diff --git a/src/payload.c b/src/payload.c +index 5b1547f..4ffc9e2 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -12,6 +12,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "utils.h" + +@@ -205,6 +206,8 @@ static bool check_var(u8 **p) + mitigations_configure(val); + } else if (IS_VAR("tso=")) { + enable_tso = val[0] == '1'; ++ } else if (IS_VAR("psci=")) { ++ psci_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +new file mode 100644 +index 0000000..2b8ca0d +--- /dev/null ++++ b/src/psci.c +@@ -0,0 +1,229 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++#include "smp.h" ++#include "types.h" ++#include "utils.h" ++ ++struct psci_cpu { ++ volatile u64 state; ++ volatile u64 entry; ++ volatile u64 context; ++ u64 pad[PSCI_CPU_SIZE / 8 - 3]; ++}; ++ ++_Static_assert(sizeof(struct psci_cpu) == PSCI_CPU_SIZE, "psci_cpu layout"); ++ ++/* ++ * Mailboxes, one cache line each, only ever accessed at EL3 with the MMU off. ++ * They live in .data rather than .bss so that a parked CPU's state survives ++ * m1n1 being re-entered from Linux (which clears .bss). ++ */ ++struct psci_cpu psci_cpus[PSCI_MAX_CPUS] ++ __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* Written with the MMU off only (_reentry sets reentry_flag, the first member) */ ++struct { ++ u64 reentry_flag; ++ u64 boot_el3; ++} psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* psci=1 from the payload, kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))); ++ ++/* EL3 stacks a CPU gets when it leaves the pen */ ++static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); ++ ++static bool psci_have_el3; ++static bool reentered; ++ ++/* EL3 side */ ++ ++static inline void psci_line_sync(volatile void *p) ++{ ++ __asm__ volatile("dc civac, %0\n\tdsb sy" ::"r"(p) : "memory"); ++} ++ ++static int psci_mpidr_index(u64 mpidr) ++{ ++ mpidr &= 0xff00ffffffUL; ++ ++ /* A7-A9: one cluster, Aff0 is the core */ ++ if (mpidr >= PSCI_MAX_CPUS) ++ return -1; ++ ++ return mpidr; ++} ++ ++static s64 psci_cpu_off(void) ++{ ++ int idx = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (idx < 0) ++ return PSCI_RET_DENIED; ++ ++ psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); ++} ++ ++static s64 psci_cpu_on(u64 mpidr, u64 entry, u64 context) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || !entry) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ struct psci_cpu *c = &psci_cpus[idx]; ++ ++ psci_line_sync(c); ++ switch (c->state) { ++ case PSCI_AFF_OFF: ++ break; ++ case PSCI_AFF_ON_PENDING: ++ return PSCI_RET_ON_PENDING; ++ default: ++ /* Running, or never seen by us (still in m1n1 or never started) */ ++ return PSCI_RET_ALREADY_ON; ++ } ++ ++ /* State before entry: the pen sets ON as soon as it sees the entry */ ++ c->context = context; ++ c->state = PSCI_AFF_ON_PENDING; ++ psci_line_sync(c); ++ c->entry = entry; ++ psci_line_sync(c); ++ sysop("sev"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++static s64 psci_affinity(u64 mpidr, u64 level) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || level) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ psci_line_sync(&psci_cpus[idx]); ++ return psci_cpus[idx].state; ++} ++ ++static s64 psci_features(u32 fid) ++{ ++ switch (fid) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ case PSCI_0_2_FN_CPU_OFF: ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return PSCI_RET_SUCCESS; ++ default: ++ /* ++ * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make ++ * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) ++{ ++ u32 fn = fid; ++ ++ /* SMC32 calls carry 32-bit arguments */ ++ if (!(fn & 0x40000000)) { ++ a1 = (u32)a1; ++ a2 = (u32)a2; ++ a3 = (u32)a3; ++ } ++ ++ switch (fn) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ return 0x10000; /* 1.0 */ ++ case PSCI_0_2_FN_CPU_OFF: ++ return psci_cpu_off(); ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ return psci_cpu_on(a1, a2, a3); ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ return psci_affinity(a1, a2); ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ return 2; /* no trusted OS, nothing to migrate */ ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return psci_features(a1); ++ default: ++ /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++/* EL1 side */ ++ ++void psci_note_boot_el(u64 el) ++{ ++ /* MMU still off: these .data accesses are uncached, like the writers' */ ++ if (psci_persist.reentry_flag) { ++ reentered = true; ++ psci_persist.reentry_flag = 0; ++ } ++ if (el == 3) ++ psci_persist.boot_el3 = 1; ++ ++ psci_have_el3 = psci_persist.boot_el3 && has_el3(); ++} ++ ++void psci_configure(const char *val) ++{ ++ psci_mode = val[0] == '1'; ++} ++ ++bool psci_active(void) ++{ ++ return psci_mode && psci_have_el3; ++} ++ ++bool psci_reentered(void) ++{ ++ return reentered; ++} ++ ++int psci_affinity_info(u64 mpidr) ++{ ++ return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); ++} ++ ++static void psci_self_off(void) ++{ ++ /* Only returns on failure; the core then stays in m1n1's loop */ ++ psci_smc(PSCI_0_2_FN_CPU_OFF, 0, 0, 0); ++} ++ ++int psci_park_cpu(int cpu, u64 mpidr) ++{ ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ ++ smp_call0(cpu, psci_self_off); ++ ++ for (int i = 0; i < 100; i++) { ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ udelay(1000); ++ } ++ ++ return -1; ++} ++ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top) ++{ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return -1; ++ ++ s64 ret = psci_smc(PSCI_0_2_FN64_CPU_ON, mpidr, (u64)psci_secondary_entry, ++ (u64)stack_top | (u64)index); ++ ++ return ret == PSCI_RET_SUCCESS ? 0 : -1; ++} +diff --git a/src/psci.h b/src/psci.h +new file mode 100644 +index 0000000..6769ebe +--- /dev/null ++++ b/src/psci.h +@@ -0,0 +1,78 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#ifndef PSCI_H ++#define PSCI_H ++ ++#ifndef __ASSEMBLER__ ++#include "types.h" ++#endif ++ ++/* ++ * Minimal PSCI 1.0 for SoCs where m1n1 owns EL3 (A7-A9 entered at EL3). ++ * Only CPU_ON/CPU_OFF/AFFINITY_INFO and the discovery calls are implemented. ++ * A CPU that calls CPU_OFF is not powered down: it waits at EL3, MMU and ++ * caches off, until CPU_ON hands it a new EL1 entry point. That is what the ++ * kernel needs to offline CPUs for kexec and CPU hotplug. ++ */ ++ ++#define PSCI_0_2_FN_PSCI_VERSION 0x84000000 ++#define PSCI_0_2_FN_CPU_SUSPEND 0x84000001 ++#define PSCI_0_2_FN_CPU_OFF 0x84000002 ++#define PSCI_0_2_FN_CPU_ON 0x84000003 ++#define PSCI_0_2_FN_AFFINITY_INFO 0x84000004 ++#define PSCI_0_2_FN_MIGRATE_INFO_TYPE 0x84000006 ++#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 ++#define PSCI_0_2_FN_SYSTEM_RESET 0x84000009 ++#define PSCI_1_0_FN_PSCI_FEATURES 0x8400000a ++#define PSCI_0_2_FN64_CPU_ON 0xc4000003 ++#define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 ++ ++#define PSCI_RET_SUCCESS 0 ++#define PSCI_RET_NOT_SUPPORTED -1 ++#define PSCI_RET_INVALID_PARAMS -2 ++#define PSCI_RET_DENIED -3 ++#define PSCI_RET_ALREADY_ON -4 ++#define PSCI_RET_ON_PENDING -5 ++#define PSCI_RET_INTERNAL_FAILURE -6 ++ ++#define PSCI_AFF_ON 0 ++#define PSCI_AFF_OFF 1 ++#define PSCI_AFF_ON_PENDING 2 ++ ++#define PSCI_MAX_CPUS 4 ++#define PSCI_STACK_SIZE 0x4000 ++ ++/* Layout shared with psci_asm.S */ ++#define PSCI_CPU_STATE 0x00 ++#define PSCI_CPU_ENTRY 0x08 ++#define PSCI_CPU_CONTEXT 0x10 ++#define PSCI_CPU_SIZE 0x80 ++ ++#ifndef __ASSEMBLER__ ++ ++/* Payload variable psci=1 */ ++void psci_configure(const char *val); ++/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++bool psci_active(void); ++/* Called once from _start_c with the EL m1n1 was entered at */ ++void psci_note_boot_el(u64 el); ++/* Set when m1n1 was re-entered from Linux (see psci_asm.S _reentry) */ ++bool psci_reentered(void); ++ ++/* EL1 side: SMC into our own EL3 */ ++s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3); ++int psci_affinity_info(u64 mpidr); ++/* Park a secondary that is spinning in smp_secondary_entry() at EL3 */ ++int psci_park_cpu(int cpu, u64 mpidr); ++/* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); ++ ++/* EL3 side */ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); ++void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_secondary_entry(void); ++void _reentry(void); ++ ++#endif ++ ++#endif +diff --git a/src/psci_asm.S b/src/psci_asm.S +new file mode 100644 +index 0000000..14f842f +--- /dev/null ++++ b/src/psci_asm.S +@@ -0,0 +1,170 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++ ++#define DCSW_OP_DCCISW 0x1 // memory.h ++ ++.extern _v_sp0_sync ++.extern _start ++.extern _cpu_reset_c ++.extern psci_handle ++.extern psci_persist ++ ++.text ++ ++/* ++ * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) ++ * goes to the PSCI dispatcher; everything else, and every exception taken ++ * at EL1/EL2, takes the usual path. ++ */ ++.globl _v_lower_sync ++.type _v_lower_sync, @function ++_v_lower_sync: ++ stp x16, x17, [sp, #-16]! ++ mrs x16, CurrentEL ++ cmp x16, #0xc ++ b.ne 1f ++ mrs x16, esr_el3 ++ lsr x17, x16, #26 ++ cmp x17, #0x17 // EC: SMC from AArch64 ++ b.ne 1f ++ tst x16, #0xffff // imm16 == 0: SMCCC ++ b.ne 1f ++ ldp x16, x17, [sp], #16 ++ b psci_smc_entry ++1: ++ ldp x16, x17, [sp], #16 ++ b _v_sp0_sync ++ ++/* ++ * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the ++ * caller. x0-x3 go straight to psci_handle(). ELR/SPSR are saved in case ++ * anything below takes an exception at EL3. ++ */ ++.type psci_smc_entry, @function ++psci_smc_entry: ++ stp x29, x30, [sp, #-16]! ++ mrs x29, elr_el3 ++ mrs x30, spsr_el3 ++ stp x29, x30, [sp, #-16]! ++ str x18, [sp, #-16]! ++ bl psci_handle ++ ldr x18, [sp], #16 ++ ldp x29, x30, [sp], #16 ++ msr elr_el3, x29 ++ msr spsr_el3, x30 ++ ldp x29, x30, [sp], #16 ++ eret ++ ++/* ++ * psci_pen(struct psci_cpu *me, u64 stack_top): CPU_OFF. Never returns. ++ * ++ * The core is not powered down. It writes its L1D back, marks itself OFF and ++ * waits in WFE at EL3 (MMU off, so every access here is non-cacheable) for ++ * CPU_ON to post an entry point. It then enters EL1 there with the MMU off, ++ * a clean I-cache and TLB, and x0 = context id, like a core coming out of ++ * reset. SP_EL3 is reset to this core's own static stack. ++ */ ++.globl psci_pen ++.type psci_pen, @function ++psci_pen: ++ mov x19, x0 ++ mov x20, x1 ++ ++ // A pending timer interrupt must not keep waking WFE. ++ msr cntp_ctl_el0, xzr ++ msr cntv_ctl_el0, xzr ++ isb ++ ++ // Clean and invalidate this core's L1D (L2 is shared and coherent). ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_level1 ++ dsb sy ++ ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_OFF ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ sev ++ ++1: ++ wfe ++ dc civac, x19 ++ dsb sy ++ ldr x2, [x19, #PSCI_CPU_ENTRY] ++ cbz x2, 1b ++ ++ ldr x0, [x19, #PSCI_CPU_CONTEXT] ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_ON ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ ++ // Nothing of the previous kernel's EL1 context may survive. ++ ic iallu ++ tlbi alle1 ++ dsb sy ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ msr elr_el3, x2 ++ mov sp, x20 ++ isb ++ eret ++ ++/* EL1 side: s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3) */ ++.globl psci_smc ++.type psci_smc, @function ++psci_smc: ++ smc #0 ++ ret ++ ++/* ++ * CPU_ON target when m1n1 takes a parked core back after a re-entry. ++ * x0 = EL1 stack top | m1n1 CPU index (stack tops are 16 KiB aligned). ++ */ ++.globl psci_secondary_entry ++.type psci_secondary_entry, @function ++psci_secondary_entry: ++ and x1, x0, #0xf ++ bic x0, x0, #0xf ++ msr tpidr_el1, x1 ++ mov sp, x0 ++ bl _cpu_reset_c ++ b . ++ ++/* ++ * Re-entry from Linux (a kexec'd trampoline, see docs/boot/warm-reboot.md): ++ * EL1, MMU and caches off, x0 = a copy of the original boot_args. ++ * Linux may hold clean lines of this image in its linear map, so drop them ++ * before m1n1 writes its .bss uncached and then reads it with caches on. ++ */ ++.globl _reentry ++.type _reentry, @function ++_reentry: ++ mov x19, x0 ++ adrp x1, _base ++ adrp x2, _end ++1: ++ dc civac, x1 ++ add x1, x1, #64 ++ cmp x1, x2 ++ b.lo 1b ++ dsb sy ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #1 ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ mov x0, x19 ++ b _start +diff --git a/src/smp.c b/src/smp.c +index 83eadff..7f5b336 100644 +--- a/src/smp.c ++++ b/src/smp.c +@@ -8,6 +8,7 @@ + #include "malloc.h" + #include "memory.h" + #include "pmgr.h" ++#include "psci.h" + #include "soc.h" + #include "string.h" + #include "types.h" +@@ -227,6 +228,74 @@ static void smp_stop_cpu(int index, int die, int cluster, int core, u64 impl, u6 + } + } + ++/* ++ * After a re-entry from Linux, a secondary that Linux offlined waits in the ++ * PSCI pen at EL3 rather than in smp_secondary_entry(). It can't be reset ++ * through the PMGR (it is running), so hand it an EL1 entry with CPU_ON. ++ */ ++static bool smp_reclaim_cpu(int index, u64 mpidr) ++{ ++ int i; ++ ++ if (index >= MAX_CPUS || spin_table[index].flag) ++ return false; ++ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return false; ++ ++ printf("Taking CPU %d back from the PSCI pen... ", index); ++ ++ memset(&spin_table[index], 0, sizeof(struct spin_table)); ++ target_cpu = index; ++ secondary_stacks[index] = memalign(0x4000, SECONDARY_STACK_SIZE); ++ ++ /* The core starts with its MMU off */ ++ dc_civac_range(&spin_table[index], sizeof(struct spin_table)); ++ dc_civac_range(&target_cpu, sizeof(target_cpu)); ++ dc_civac_range(&boot_cpu_idx, sizeof(boot_cpu_idx)); ++ dc_civac_range(&boot_cpu_mpidr, sizeof(boot_cpu_mpidr)); ++ ++ if (psci_unpark_cpu(index, mpidr, secondary_stacks[index] + SECONDARY_STACK_SIZE) < 0) { ++ printf("Failed (CPU_ON)\n"); ++ return true; ++ } ++ ++ for (i = 0; i < 100; i++) { ++ sysop("dmb ld"); ++ if (spin_table[index].flag) ++ break; ++ udelay(1000); ++ } ++ ++ printf("%s\n", i >= 100 ? "Failed!" : " Back."); ++ return true; ++} ++ ++/* ++ * Put every live secondary into the PSCI pen, for a kernel that boots them ++ * with enable-method = "psci". A CPU that doesn't make it is dropped (and ++ * then pruned from the DT by the caller). ++ */ ++int smp_park_secondaries_psci(void) ++{ ++ int parked = 0; ++ ++ for (int i = 0; i < MAX_CPUS; i++) { ++ if (i == boot_cpu_idx || !smp_is_alive(i)) ++ continue; ++ ++ u64 mpidr = spin_table[i].mpidr; ++ if (psci_park_cpu(i, mpidr) < 0) { ++ printf("PSCI: CPU %d (MPIDR 0x%lx) did not park, dropping it\n", i, mpidr); ++ memset(&spin_table[i], 0, sizeof(struct spin_table)); ++ continue; ++ } ++ parked++; ++ } ++ ++ return parked; ++} ++ + static u64 smp_get_iorvbar(int node, int cpu) + { + u64 cpu_impl_reg[2]; +@@ -405,6 +474,10 @@ void smp_start_secondaries(void) + u8 cluster = FIELD_GET(CPU_REG_CLUSTER, reg); + u8 die = FIELD_GET(CPU_REG_DIE, reg); + ++ if (psci_active() && psci_reentered() && ++ smp_reclaim_cpu(i, core | (cluster << 8) | (die << 16))) ++ continue; ++ + smp_start_cpu(i, die, cluster, core, iorvbar, pmgr_reg + cpu_start_off); + } + } +diff --git a/src/smp.h b/src/smp.h +index 0296978..6c90557 100644 +--- a/src/smp.h ++++ b/src/smp.h +@@ -18,6 +18,7 @@ void smp_secondary_prep_el3(void); + + void smp_start_secondaries(void); + void smp_stop_secondaries(bool deep_sleep); ++int smp_park_secondaries_psci(void); + + #define smp_call0(i, f) smp_call4(i, f, 0, 0, 0, 0) + #define smp_call1(i, f, a) smp_call4(i, f, a, 0, 0, 0) +diff --git a/src/start.S b/src/start.S +index aa11e18..a3f35ae 100644 +--- a/src/start.S ++++ b/src/start.S +@@ -11,6 +11,7 @@ + .extern _v_sp0_irq + .extern _v_sp0_fiq + .extern _v_sp0_serr ++.extern _v_lower_sync + .extern _reset_stack + .extern _cpu_reset_c + .extern wdt_reboot +@@ -41,7 +42,7 @@ _vectors_start: + .align 7 + b _v_sp0_serr + .align 7 +- b _v_sp0_sync ++ b _v_lower_sync + .align 7 + b _v_sp0_irq + .align 7 +@@ -103,6 +104,8 @@ _start: + + mov x0, x19 + mov x1, x20 ++ mrs x2, CurrentEL ++ lsr x2, x2, #2 + bl _start_c + b . + +@@ -115,6 +118,7 @@ start_el3: + msr tpidr_el3, xzr + adrp x6, _stack_bot + adr x7, _start_c ++ mov x2, #3 // _start_c: entered at EL3 + b el3_eret_to_el1 + + .globl exc_unk +diff --git a/src/startup.c b/src/startup.c +index 61afe16..167fae8 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -5,6 +5,7 @@ + #include "dockchannel_uart.h" + #include "exception.h" + #include "firmware.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "types.h" +@@ -159,12 +160,15 @@ void dump_boot_args(struct boot_args *ba) + } + + extern void get_device_info(void); +-void _start_c(void *boot_args, void *base) ++void _start_c(void *boot_args, void *base, u64 boot_el) + { + UNUSED(base); + u32 cpu_id = 0; + + memset64(_bss_start, 0, _bss_end - _bss_start); ++ psci_note_boot_el(boot_el); ++ if (psci_reentered()) ++ boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); + diff --git a/boot/m1n1/psci-loader b/boot/m1n1/psci-loader new file mode 100755 index 00000000..1f628bc9 --- /dev/null +++ b/boot/m1n1/psci-loader @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Make a J71t opt-in loader by appending m1n1's psci=1 payload variable. + +Use a bare m1n1.bin as input, before assembling any kernel/DTB/logo payload. +The output works with orchard run/build --loader or launch's loaders/m1n1.bin. +It still uses the spin table if m1n1 was entered at EL1. +""" +import argparse +from pathlib import Path +import shutil + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--device', required=True, choices=('j71t',)) + parser.add_argument('--loader', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if not args.loader.is_file() or not args.loader.stat().st_size: + parser.error('input must be a nonempty bare m1n1.bin') + # Exclusive creation: never replace the original or another boot artifact. + with args.loader.open('rb') as source, args.output.open('xb') as output: + shutil.copyfileobj(source, output) + output.write(b'psci=1\n') + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/chips/s8003.toml b/chips/s8003.toml index a8dc5c58..862cf6ac 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -24,3 +24,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: warm-storage. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-storage.md. + +# Local capability delta: psci. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/psci.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 594ff9a9..18157772 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -63,3 +63,6 @@ camera = "missing" # Local capability delta: warm-storage. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-storage.md. + +# Local capability delta: psci. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/psci.md. diff --git a/tests/test_m1n1_psci.py b/tests/test_m1n1_psci.py new file mode 100644 index 00000000..0f491dbf --- /dev/null +++ b/tests/test_m1n1_psci.py @@ -0,0 +1,54 @@ +"""Host-only framing checks for the explicit J71t PSCI loader.""" +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / 'boot/m1n1/psci-loader' + + +class PSCILoader(unittest.TestCase): + def test_payload_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as tmp: + loader, output = Path(tmp) / 'm1n1.bin', Path(tmp) / 'psci.bin' + original = b'bare loader\0' + loader.write_bytes(original) + cmd = [sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + self.assertEqual(loader.read_bytes(), original) + self.assertNotEqual(subprocess.run(cmd, capture_output=True).returncode, 0) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + + def test_manifest_psci_record_keeps_logo_first(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, data in {'loader': b'M1N1', 'dtb': b'DTB', 'kernel': b'KERNEL', + 'initrd': b'INITRD'}.items(): + (root / name).write_bytes(data) + logo = b'm1n1_logo_256128' + bytes(4 * (256 * 256 + 128 * 128)) + (root / 'logo').write_bytes(logo) + records = b'chosen.bootargs=console=tty0\npsci=1\n' + (root / 'profile.json').write_text(json.dumps({'bootargs': records.decode()})) + result = subprocess.run([sys.executable, '-I', str(ROOT / 'src/orchard/image/assemble.py'), + '--manifest', str(root / 'profile.json'), '--loader', str(root / 'loader'), + '--logo', str(root / 'logo'), '--dtb', str(root / 'dtb'), + '--kernel', str(root / 'kernel'), '--initramfs', str(root / 'initrd'), + '--output', str(root / 'payload')], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((root / 'payload').read_bytes().startswith(b'M1N1' + logo + records + b'DTB')) + + def test_board_requires_explicit_opt_in(self): + for device in ([], ['--device', 'j71b']): + result = subprocess.run([sys.executable, str(TOOL), *device, '--loader', 'unused', + '--output', 'unused'], capture_output=True) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() From a76d37602a13e224bc89ed4ad0ce41d41c3e7e78 Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 06/33] pinctrl/spi/input: quiesce S8000 peripherals for kexec Assisted-by: LLM --- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + ...0025-warm-peripherals-implementation.patch | 41 ++++++ ...0004-warm-peripherals-implementation.patch | 120 ++++++++++++++++++ ...0009-warm-peripherals-implementation.patch | 113 +++++++++++++++++ 5 files changed, 280 insertions(+) create mode 100644 kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch create mode 100644 kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch create mode 100644 kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch diff --git a/chips/s8003.toml b/chips/s8003.toml index 862cf6ac..49370529 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -27,3 +27,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: psci. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/psci.md. + +# Local capability delta: warm-peripherals. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-peripherals.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 18157772..fbcc3fa0 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -66,3 +66,6 @@ camera = "missing" # Local capability delta: psci. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/psci.md. + +# Local capability delta: warm-peripherals. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-peripherals.md. diff --git a/kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch b/kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch new file mode 100644 index 00000000..3ef9fca9 --- /dev/null +++ b/kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch @@ -0,0 +1,41 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] Input: apple-z2: pinctrl/spi/input: quiesce S8000 peripherals for kexec + +Retire inherited GPIO IRQ state, stop PIO SPI queues and chip selects, and quiesce Apple Z2 touch before its SPI parent shuts down. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/input/touchscreen/apple_z2.c b/drivers/input/touchscreen/apple_z2.c +index 161ebb4b8..145d6ffcf 100644 +--- a/drivers/input/touchscreen/apple_z2.c ++++ b/drivers/input/touchscreen/apple_z2.c +@@ -4584,6 +4584,16 @@ static struct spi_device_id apple_z2_of_id[] = { + }; + MODULE_DEVICE_TABLE(spi, apple_z2_of_id); + ++/* Children stop before the SPI controller drains its core queue. */ ++static void apple_z2_shutdown(struct spi_device *spi) ++{ ++ struct apple_z2 *z2 = spi_get_drvdata(spi); ++ ++ apple_z2_stop(z2); ++ /* Also wait when the IRQ thread already masked itself. */ ++ synchronize_irq(spi->irq); ++} ++ + static struct spi_driver apple_z2_driver = { + .driver = { + .name = "apple-z2", +@@ -4594,6 +4604,7 @@ static struct spi_driver apple_z2_driver = { + .id_table = apple_z2_of_id, + .probe = apple_z2_probe, + .remove = apple_z2_remove, ++ .shutdown = apple_z2_shutdown, + }; + + module_spi_driver(apple_z2_driver); diff --git a/kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch b/kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch new file mode 100644 index 00000000..5e4b4238 --- /dev/null +++ b/kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch @@ -0,0 +1,120 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pinctrl: pinctrl/spi/input: quiesce S8000 peripherals for kexec + +Retire inherited GPIO IRQ state, stop PIO SPI queues and chip selects, and quiesce Apple Z2 touch before its SPI parent shuts down. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c +index b97023c97..766fcf5bc 100644 +--- a/drivers/pinctrl/pinctrl-apple-gpio.c ++++ b/drivers/pinctrl/pinctrl-apple-gpio.c +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -46,6 +47,7 @@ struct apple_gpio_pinctrl { + + void __iomem *base; + struct regmap *map; ++ bool retire_irqs; + + struct pinctrl_desc pinctrl_desc; + struct gpio_chip gpio_chip; +@@ -379,8 +381,33 @@ static void apple_gpio_irq_handler(struct irq_desc *desc) + chained_irq_enter(chip, desc); + for (pinh = 0; pinh < gc->ngpio; pinh += 32) { + pending = readl_relaxed(pctl->base + REG_IRQ(*grpp, pinh)); +- for_each_set_bit(pinl, &pending, 32) +- generic_handle_domain_irq(gc->irq.domain, pinh + pinl); ++ for_each_set_bit(pinl, &pending, 32) { ++ unsigned int pin = pinh + pinl; ++ int ret; ++ ++ ret = pin < gc->ngpio ? ++ generic_handle_domain_irq(gc->irq.domain, pin) : -EINVAL; ++ if (ret && pctl->retire_irqs) { ++ /* An inherited, unmapped source has nobody to ack it. ++ * Mask only this GPIO, then clear this group's latch. ++ */ ++ if (pin < gc->ngpio) { ++ u32 reg = apple_gpio_get_reg(pctl, pin); ++ u32 mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ u32 off = FIELD_PREP(REG_GPIOx_MODE, ++ REG_GPIOx_IN_IRQ_OFF); ++ ++ if (!(reg & REG_GPIOx_PERIPH) && ++ mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, off); ++ } ++ writel(BIT(pinl), pctl->base + REG_IRQ(*grpp, pinh)); ++ dev_warn_ratelimited(pctl->dev, ++ "masked orphan inherited GPIO IRQ group=%u pin=%u\n", ++ *grpp, pin); ++ } ++ } + } + chained_irq_exit(chip, desc); + } +@@ -403,6 +430,35 @@ static const struct irq_chip apple_gpio_irqchip = { + + /* Probe & register */ + ++static void apple_gpio_quiesce_irqs(struct apple_gpio_pinctrl *pctl) ++{ ++ struct gpio_chip *gc = &pctl->gpio_chip; ++ unsigned int pin, group, mode, masked = 0; ++ u32 reg; ++ ++ if (!pctl->retire_irqs) ++ return; ++ /* Before chained parent IRQs are enabled, retire old GPIO IRQ modes. ++ * Preserve outputs, mux, pulls, data and grouping. Consumers rearm their ++ * own pins later; panic entry need not have run any shutdown callbacks. ++ */ ++ for (pin = 0; pin < gc->ngpio; pin++) { ++ reg = apple_gpio_get_reg(pctl, pin); ++ mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ if (!(reg & REG_GPIOx_PERIPH) && mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) { ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, ++ FIELD_PREP(REG_GPIOx_MODE, REG_GPIOx_IN_IRQ_OFF)); ++ masked++; ++ } ++ } ++ for (group = 0; group < gc->irq.num_parents; group++) ++ for (pin = 0; pin < gc->ngpio; pin += 32) ++ writel(GENMASK(min(32U, gc->ngpio - pin) - 1, 0), ++ pctl->base + REG_IRQ(group, pin)); ++ dev_info(pctl->dev, "inherited GPIO IRQ baseline: masked %u inherited inputs\n", masked); ++} ++ + static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + { + struct gpio_irq_chip *girq = &pctl->gpio_chip.irq; +@@ -452,6 +508,7 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + girq->handler = handle_level_irq; + } + ++ apple_gpio_quiesce_irqs(pctl); + ret = devm_gpiochip_add_data(pctl->dev, &pctl->gpio_chip, pctl); + + out_free_irq_data: +@@ -485,6 +542,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) + if (!pctl) + return -ENOMEM; + pctl->dev = &pdev->dev; ++ pctl->retire_irqs = of_device_is_compatible(pdev->dev.of_node, "apple,s8000-pinctrl"); + pctl->soc = of_device_get_match_data(&pdev->dev); + pctl->gpio_chip.irq.num_parents = nirqs; + dev_set_drvdata(&pdev->dev, pctl); diff --git a/kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch b/kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch new file mode 100644 index 00000000..738091e9 --- /dev/null +++ b/kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch @@ -0,0 +1,113 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] spi: pinctrl/spi/input: quiesce S8000 peripherals for kexec + +Retire inherited GPIO IRQ state, stop PIO SPI queues and chip selects, and quiesce Apple Z2 touch before its SPI parent shuts down. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c +index dc5154e1b..ed9f67a59 100644 +--- a/drivers/spi/spi-apple.c ++++ b/drivers/spi/spi-apple.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -141,6 +142,7 @@ struct apple_spi { + struct dmaengine_result dma_result; + const struct apple_spi_hw *hw; + u32 word_delay; ++ struct spi_controller *ctlr; + }; + + static inline void reg_write(struct apple_spi *spi, int offset, u32 value) +@@ -231,6 +233,19 @@ static void apple_spi_init(struct apple_spi *spi) + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + } + ++/* Lab 0015: normalize only the S8000 PIO register set, while powered. */ ++static int apple_spi_s8000_idle(struct apple_spi *spi) ++{ ++ reg_write(spi, APPLE_SPI_CTRL, 0); ++ reg_write(spi, APPLE_SPI_CFG, spi->hw->cfg_base); ++ reg_write(spi, APPLE_SPI_RXCNT, 0); ++ reg_write(spi, APPLE_SPI_TXCNT, 0); ++ reg_write(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS); ++ apple_spi_t8010_clear_status(spi); ++ /* Complete posted writes before publishing children or leaving Linux. */ ++ return readl(spi->regs + APPLE_SPI_CTRL) ? -EIO : 0; ++} ++ + static int apple_spi_prepare_message(struct spi_controller *ctlr, + struct spi_message *msg) + { +@@ -1120,6 +1135,7 @@ static int apple_spi_probe(struct platform_device *pdev) + + spi = spi_controller_get_devdata(ctlr); + platform_set_drvdata(pdev, spi); ++ spi->ctlr = ctlr; + init_completion(&spi->done); + init_completion(&spi->dma_done); + spi->hw = of_device_get_match_data(&pdev->dev); +@@ -1214,6 +1230,11 @@ static int apple_spi_probe(struct platform_device *pdev) + } + + apple_spi_init(spi); ++ if (spi->hw->pio_only) { ++ ret = apple_spi_s8000_idle(spi); ++ if (ret) ++ return ret; ++ } + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret < 0) +@@ -1223,6 +1244,32 @@ static int apple_spi_probe(struct platform_device *pdev) + return 0; + } + ++static void apple_spi_shutdown(struct platform_device *pdev) ++{ ++ struct apple_spi *spi = platform_get_drvdata(pdev); ++ struct spi_controller *ctlr = spi->ctlr; ++ unsigned int cs; ++ int ret; ++ ++ /* Other SoCs use different DMA/IRQ registers and keep their policy. */ ++ if (!spi->hw->pio_only) ++ return; ++ /* Children have shut down; drain the core queue before MMIO. */ ++ ret = spi_controller_suspend(ctlr); ++ if (!ret) ++ ret = pm_runtime_resume_and_get(&pdev->dev); ++ if (ret >= 0) { ++ if (ctlr->cs_gpiods) ++ for (cs = 0; cs < ctlr->num_chipselect; cs++) ++ gpiod_set_value_cansleep(ctlr->cs_gpiods[cs], 0); ++ ret = apple_spi_s8000_idle(spi); ++ } ++ if (ret < 0 && kexec_in_progress) ++ panic("S8000 SPI shutdown failed before kexec: %d", ret); ++ /* Keep runtime power pinned until the jump. */ ++ dev_info(&pdev->dev, "S8000 SPI shutdown result %d\n", ret); ++} ++ + static const struct apple_spi_hw apple_spi_hw_s8000 = { + .legacy_status = true, + .pio_only = true, +@@ -1260,6 +1307,7 @@ MODULE_DEVICE_TABLE(of, apple_spi_of_match); + + static struct platform_driver apple_spi_driver = { + .probe = apple_spi_probe, ++ .shutdown = apple_spi_shutdown, + .driver = { + .name = "apple-spi", + .of_match_table = apple_spi_of_match, From 1ca51ef055d0d8d1325ceb92346a84adde761e84 Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 07/33] J71t guarded Mac current profile Assisted-by: LLM --- boot/initramfs/root/init | 17 +- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + docs/bring-up/charging/j71t-mac-current.md | 119 +++++++++++++ .../0026-mac-current-implementation.patch | 74 ++++++++ .../usb/0011-mac-current-implementation.patch | 42 +++++ src/orchard/apple/linux_usb.py | 6 +- src/orchard/apple/usb.py | 6 +- tests/test_j71t_mac_current.py | 166 ++++++++++++++++++ 9 files changed, 433 insertions(+), 3 deletions(-) create mode 100644 docs/bring-up/charging/j71t-mac-current.md create mode 100644 kernel/patches/drivers/power/0026-mac-current-implementation.patch create mode 100644 kernel/patches/drivers/usb/0011-mac-current-implementation.patch create mode 100644 tests/test_j71t_mac_current.py diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index 9556f126..c852dd68 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -97,9 +97,24 @@ usb_gadget() { [ ! -e "$g" ] || return 0 name=$(sed -n 's/^NAME=//p' /newroot/etc/os-release /newroot/usr/lib/os-release 2>/dev/null | head -1 | tr -d "\"'") board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + # Opt-in compatibility identity in macOS's 2400 mA capability table. + # This requests a host grant; it does not set a charger limit. Keep the + # generic composite personality (no iOS/usbmux or preferred-config API). + pid=0x4145 + device_class=0x00 device_subclass=0x00 device_protocol=0x00 + if [ "$board" = apple,j71t ] && + [ "$(cat /newroot/etc/orchard/mac-usb-current 2>/dev/null)" = 1 ]; then + pid=0x1014 + # Standard IAD device: macOS defers its CDC composite match here, + # allowing the generic composite driver to deliver the grant. + device_class=0xef device_subclass=0x02 device_protocol=0x01 + fi mkdir "$g" && echo 0x05ac > "$g/idVendor" && - echo 0x4145 > "$g/idProduct" && + echo "$pid" > "$g/idProduct" && + echo "$device_class" > "$g/bDeviceClass" && + echo "$device_subclass" > "$g/bDeviceSubClass" && + echo "$device_protocol" > "$g/bDeviceProtocol" && mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" && echo 'Orchard Linux' > "$g/strings/0x409/manufacturer" && echo "${name:-Linux}" > "$g/strings/0x409/product" && diff --git a/chips/s8003.toml b/chips/s8003.toml index 6ae21ec1..4ea9cf07 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -18,3 +18,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: pmu-rtc. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/pmu-rtc.md. + +# Local capability delta: mac-current. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/mac-current.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 72069715..0c2a2e13 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -57,3 +57,6 @@ camera = "missing" # Local capability delta: pmu-rtc. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/pmu-rtc.md. + +# Local capability delta: mac-current. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/mac-current.md. diff --git a/docs/bring-up/charging/j71t-mac-current.md b/docs/bring-up/charging/j71t-mac-current.md new file mode 100644 index 00000000..6846b067 --- /dev/null +++ b/docs/bring-up/charging/j71t-mac-current.md @@ -0,0 +1,119 @@ +# J71t Mac extra-current candidate + +Host-tested on 2026-11-08; not yet qualified on an iPad. The matching root +initrd reads `/etc/orchard/mac-usb-current` from the mounted system. Exactly +`1` on `apple,j71t` selects the compatibility identity `05ac:1014`, explicit +IAD device class ef/02/01, and the existing ACM+NCM configuration at +500 mA MaxPower. Omarchy A carries that opt-in as a separate system change. +Removing the file selects the normal `05ac:4145` identity on the next boot. +Recovery, installer and RAM/NFS gadgets retain their existing identities. + +This Mac's AppleUSBHostMergeProperties personality lists `1014` in its +2400 mA UsbPowerSinkCapability table; `4145` is absent. The generic +AppleUSBHostCompositeDevice power-negotiation path can therefore allocate +extra power and send vendor device request `40/40`, zero length, signed +suspend mA in wValue and signed extra active mA in wIndex. No usbmux interface +or iOS preferred-configuration protocol is needed for this PID. Its IAD +class ef/02/01 matches Apple's generic composite personality. AppleUSBCDC's +matching IAD personality has IOMatchDefer=True; its non-deferred vendor +personality instead requires 0/0/0. Both ACM and NCM already emit IADs. The strings +continue to identify Orchard and Omarchy. This is a compatibility identity, +not an assigned Orchard PID or a claim to implement an Apple product. + +A new identity requires macOS accessory authorization, possibly on every boot +because the existing BOOT serial changes. Keep the Mac unlocked and reconnect +the cable if the device remains unregistered/unmatched. Do not silently change +the Mac's security settings. Orchard discovery recognizes this PID as a +system only when the existing serial contains a boot UUID and BOARD:j71t. + +The identity is never current evidence on the tablet. Only the host's real +request reaches composite_vbus_draw -> DWC2 -> lightning. The charger clamps +the allowance at its board maximum and retains pack guards, stepped ramp, +readback verification and source-epoch withdrawal. No request means the +existing 500 mA fallback. BC1.2 primary/secondary detection already exists; +SDP on this Mac must not be relabeled CDP just to obtain 1.5 A. + +A grant arriving while suspended cannot select its active allowance. The +existing policy discards the preceding active grant on suspend; resuming +needs a fresh grant. Reset and disconnect also clear it. The supplied suspend +field is logged but does not authorize an elevated suspend limit. This work +does not certify the pre-existing charger's USB suspend/preconfiguration +compliance: its unknown-source hardware fallback remains 500 mA. + +`current_max=0` on a configured link is a separate possible blocker. A report +is accepted only while present, with data enabled, in the current PMU source +epoch. Logs now show reported/accepted current, present/data and report epoch; +dynamic debug on the charger shows stale report/source epochs. Do not replay +an old allowance after a PMU event to make this number nonzero. A packet trace +and these logs must establish whether the live zero was a missing report, a +gated report, or epoch invalidation. That observation is still required. + +## Host verification + +Run `ORCHARD_KERNEL_SOURCE=/path/to/prepared/kernel PYTHONPATH=src python3 +-m unittest discover -s tests -p test_j71t_mac_current.py -v`. The C test +extracts the actual production functions and executes them with ASan/UBSan: +all request-type/request/short-length/vendor combinations, all signed extra +values, suspend, board caps, ramp/revoke, attachment and stale epochs. +The other tests run the initrd gadget function in a mock filesystem and test +both host discovery implementations. They do not contact USB hardware. + +## Device acceptance gate (orchestrator only) + +Boot the new Image, DTB, matching initrd and modules together, with the +Omarchy A opt-in present. Do not rebind a live gadget to edit its identity. + +On the unlocked Mac, approve the new accessory and capture `ioreg -p IOUSB +-l -w0` plus `ioreg -l -w0 -r -c IOUSBHostDevice`. The device should be +registered/matched/configured, have PID 0x1014, MaxPower 500, +UsbPowerSinkCapability 2400 and UsbPowerSinkAllocation above 500 (2400 if +available). Check AppleUSBHostCompositeDevice binding and +kUSBConfigurationCurrentOverride. The host's available 3000 mA alone is not +a grant. Preserve ACM serial and NCM network connectivity. + +On the tablet, enable dynamic debug for composite.c, dwc2/gadget.c and +apple_tristar_charger.c. Capture dmesg and lightning's online, usb_type, +current_max and input_current_limit alongside battery's status/current_now, +capacity, temperature, voltage and the UDC state. A usual full grant is +extra=1900, giving accepted/current_max=2400000, followed by a guarded ramp +to input_current_limit=2400000. Never exceed min(500+signed extra, 2400). +A smaller grant must produce a correspondingly smaller limit, rounded down. +SDP may remain correct; it does not imply that a software grant failed. + +At a moderate state of charge and normal pack temperature, hold brightness +and desktop workload fixed for at least 60 seconds: battery status must be +Charging and signed current_now positive. Full/thermal-limited packs are not +a valid charging-current acceptance test. If current_max stays zero, preserve +the report/epoch logs and EP0 trace; do not force a PMIC limit. + +Also test: a plain 500 mA host (no extra request), reset, unplug/replug to a +different source, a smaller/revoked grant, and Mac sleep/wake. Any loss of a +valid grant must stop elevated draw and return to the fallback. Suspend may +lose extra current until macOS issues a new grant; record that limitation. +Mac allocation without an actual request and accepted allowance is a failed +test, even if battery current happens to be positive with the panel off. + +## Sustained NCM transfers + +The runner reported two carrier-up stalls with ACM still working, recovered +by cycling only usb0. USB patch 0012 fixes a reproducible NCM timer bug: +NETDEV_TX_BUSY during a timer flush used to abandon the partial NTB. Retry +at the usual 300 us interval, advancing the timer expiry before restart. +This is a supported cause of silent NCM stalls; the runner's exact endpoint +state was not captured, so it is not proof that both incidents had this cause. + +A 16 KiB NTB does not need to fit in a dedicated FIFO at once: DWC2 splits +transfers and streams USB packets through the FIFO. The 2056-entry total is +in 32-bit words. No FIFO, DMA, ZLP or negotiated NTB-size change is justified +by those totals alone. ACM surviving also differs from the known shared-OUT +FIFO starvation fixed by USB patch 0008. Do not change those controls without +capturing endpoint state and request completions during a stall. + +Run test_ncm_flush.py with ORCHARD_KERNEL_SOURCE to execute the production +callback under repeated request exhaustion and subsequent recovery. On the +next device run count every `usb0-stall-bounce` marker, including the runner's +transient oi-usb0-watchdog. Use one watchdog at a time so bounces are attributable. +The Omarchy service's new carrier-up fallback cycles only usb0 after 15 seconds +without replies from the previously reachable 172.16.42.2 peer. It leaves +UDC, ACM and host charging-grant ownership intact. A clean sustained-transfer +acceptance run requires zero watchdog bounces, not merely successful retries. diff --git a/kernel/patches/drivers/power/0026-mac-current-implementation.patch b/kernel/patches/drivers/power/0026-mac-current-implementation.patch new file mode 100644 index 00000000..aaccda27 --- /dev/null +++ b/kernel/patches/drivers/power/0026-mac-current-implementation.patch @@ -0,0 +1,74 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] power: J71t guarded Mac current profile + +Use 05ac:1014 only for explicitly selected J71t Orchard gadgets. Preserve NCM and ACM, bound real host current grants and keep new grants inactive during suspend. Requires matching userspace opt-in; physical charging remains a device gate. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index b95142dad..9b3a0733e 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -669,7 +669,12 @@ static int tristar_identify(struct tristar_charger *chg, struct tristar_ident *i + */ + static unsigned int tristar_usb_ma(struct tristar_charger *chg) + { +- return chg->usb_epoch == READ_ONCE(chg->epoch) ? chg->usb_ma : 0; ++ if (chg->usb_epoch != READ_ONCE(chg->epoch)) { ++ dev_dbg(chg->dev, "USB budget stale: report epoch %u, source epoch %u\n", ++ chg->usb_epoch, READ_ONCE(chg->epoch)); ++ return 0; ++ } ++ return chg->usb_ma; + } + + /* +@@ -1231,7 +1236,8 @@ static int tristar_get_property(struct power_supply *psy, + break; + case POWER_SUPPLY_PROP_CURRENT_MAX: + /* What the identified source or USB host allows; 0 when unknown. */ +- val->intval = max(chg->source_ma, tristar_usb_ma(chg)) * 1000; ++ val->intval = min(max(chg->source_ma, tristar_usb_ma(chg)), ++ chg->max_ma) * 1000; + break; + default: + mutex_unlock(&chg->lock); +@@ -1254,23 +1260,29 @@ static int tristar_set_property(struct power_supply *psy, + const union power_supply_propval *val) + { + struct tristar_charger *chg = power_supply_get_drvdata(psy); +- unsigned int old_code; ++ unsigned int old_code, old_ma, report_ma; + + if (psp != POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT) + return -EINVAL; + + mutex_lock(&chg->lock); + old_code = chg->code; +- chg->usb_ma = chg->present && chg->usb_on ? max(val->intval, 0) / 1000 : 0; ++ old_ma = tristar_usb_ma(chg); ++ report_ma = max(val->intval, 0) / 1000; ++ chg->usb_ma = chg->present && chg->usb_on ? ++ min(report_ma, chg->max_ma) : 0; + chg->usb_epoch = READ_ONCE(chg->epoch); +- dev_dbg(chg->dev, "USB host allows %u mA\n", chg->usb_ma); ++ dev_info_ratelimited(chg->dev, ++ "USB budget: reported %u mA, accepted %u mA, present %u, data %u, epoch %u\n", ++ report_ma, chg->usb_ma, chg->present, chg->usb_on, ++ chg->usb_epoch); + if (!chg->stopping) { + tristar_update_target(chg); + mod_delayed_work(chg->wq, &chg->poll_work, 0); + } + mutex_unlock(&chg->lock); + +- if (READ_ONCE(chg->code) != old_code) ++ if (READ_ONCE(chg->code) != old_code || old_ma != READ_ONCE(chg->usb_ma)) + power_supply_changed(psy); + return 0; + } diff --git a/kernel/patches/drivers/usb/0011-mac-current-implementation.patch b/kernel/patches/drivers/usb/0011-mac-current-implementation.patch new file mode 100644 index 00000000..5cdb4766 --- /dev/null +++ b/kernel/patches/drivers/usb/0011-mac-current-implementation.patch @@ -0,0 +1,42 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] usb: J71t guarded Mac current profile + +Use 05ac:1014 only for explicitly selected J71t Orchard gadgets. Preserve NCM and ACM, bound real host current grants and keep new grants inactive during suspend. Requires matching userspace opt-in; physical charging remains a device gate. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 14e87fbc8..be66793e5 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -959,7 +959,7 @@ static void reset_config(struct usb_composite_dev *cdev) + static void composite_vbus_draw(struct usb_composite_dev *cdev, unsigned int mA) + { + cdev->vbus_draw_ma = mA; +- if (mA && cdev->apple_extra_ma) ++ if (mA && !cdev->suspended && cdev->apple_extra_ma) + mA = max(500 + cdev->apple_extra_ma, 0); + usb_gadget_vbus_draw(cdev->gadget, mA); + } +@@ -2665,6 +2665,7 @@ void composite_resume(struct usb_gadget *gadget) + DBG(cdev, "resume\n"); + if (cdev->driver->resume) + cdev->driver->resume(cdev); ++ cdev->suspended = 0; + if (cdev->config) { + list_for_each_entry(f, &cdev->config->functions, list) { + /* +@@ -2695,8 +2696,6 @@ void composite_resume(struct usb_gadget *gadget) + maxpower = min(maxpower, 100U); + composite_vbus_draw(cdev, maxpower); + } +- +- cdev->suspended = 0; + } + + /*-------------------------------------------------------------------------*/ diff --git a/src/orchard/apple/linux_usb.py b/src/orchard/apple/linux_usb.py index cdc4735c..b3a95803 100644 --- a/src/orchard/apple/linux_usb.py +++ b/src/orchard/apple/linux_usb.py @@ -36,11 +36,15 @@ def devices(root=Path('/sys/bus/usb/devices')): chip = re.search(r'\bCPID:([0-9a-fA-F]+)\b', serial) board = re.search(r'\bBDID:([0-9a-fA-F]+)\b', serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) + # A compatibility PID also belongs to real Apple hardware. Only our + # existing boot/profile identity makes it an Orchard system. + mac_current = (pid == 0x1014 and boot is not None and + profile is not None and profile.group(1) == 'j71t') ports = {str(Path('/dev') / p.name) for p in node.glob('**/tty/ttyACM*')} if len(ports) > 1: raise ValueError('Ambiguous USB serial ports on a device') product = (node / 'product').read_text().strip() if (node / 'product').exists() else '' - found.append(dict(location=node.name, mode=MODES.get(pid, 'other'), + found.append(dict(location=node.name, mode='system' if mac_current else MODES.get(pid, 'other'), ecid=int(ecid.group(1), 16) if ecid else None, boot_id=str(uuid.UUID(boot.group(1))) if boot else None, chip=int(chip.group(1), 16) if chip else None, diff --git a/src/orchard/apple/usb.py b/src/orchard/apple/usb.py index fa4090fd..7a63d431 100644 --- a/src/orchard/apple/usb.py +++ b/src/orchard/apple/usb.py @@ -38,6 +38,10 @@ def devices(): serial = node.get('USB Serial Number') or '' boot = re.search(r'\bBOOT:([0-9a-fA-F-]{36})\b', serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) + # A compatibility PID also belongs to real Apple hardware. Only our + # existing boot/profile identity makes it an Orchard system. + mac_current = (node.get('idProduct') == 0x1014 and boot is not None and + profile is not None and profile.group(1) == 'j71t') fields = [re.search(r'\b' + label + r':([0-9a-fA-F]+)\b', serial) for label in ('ECID', 'CPID', 'BDID')] descendants = list(node.get('IORegistryEntryChildren', [])) @@ -51,7 +55,7 @@ def devices(): ports.append(port) if len(ports) > 1: raise DeviceError('Ambiguous USB serial ports on the selected device') - found.append({'location': location, 'mode': MODES.get(node.get('idProduct'), 'other'), + found.append({'location': location, 'mode': 'system' if mac_current else MODES.get(node.get('idProduct'), 'other'), 'ecid': int(fields[0].group(1), 16) if fields[0] else None, 'boot_id': str(uuid.UUID(boot.group(1))) if boot else None, 'chip': int(fields[1].group(1), 16) if fields[1] else None, diff --git a/tests/test_j71t_mac_current.py b/tests/test_j71t_mac_current.py new file mode 100644 index 00000000..2a8a536c --- /dev/null +++ b/tests/test_j71t_mac_current.py @@ -0,0 +1,166 @@ +"""Host-only tests; ORCHARD_KERNEL_SOURCE selects the prepared patched kernel.""" +import os +from pathlib import Path +import plistlib +import re +import subprocess +import tempfile +import unittest +from unittest.mock import patch +from orchard.apple import usb, linux_usb + +HW = Path(__file__).resolve().parents[1] +BOOT = '12345678-1234-1234-1234-123456789abc' + + +def function(source, name): + start = re.search(r'^.*\b' + name + r'\([^;]*?\)\n\{', source, re.M).start() + brace = source.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + '\n' + + +class MacCurrent(unittest.TestCase): + def test_gadget_profile_preserves_network_serial_and_safe_default(self): + source = (HW / 'boot/initramfs/root/init').read_text() + start = source.index('usb_gadget() {') + script = source[start:source.index('\nusb_gadget ||', start)] + for board, flag, expected in [('j71t', '1', '0x1014'), ('j71t', '', '0x4145'), + ('j71t', '2400', '0x4145'), ('j71b', '1', '0x4145')]: + with self.subTest(board=board, flag=flag), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for n in ['sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', + 'newroot/etc/orchard', 'newroot/usr/lib', 'proc/device-tree', + 'proc/sys/kernel/random']: + (root / n).mkdir(parents=True) + (root / 'proc/device-tree/compatible').write_bytes(f'apple,{board}\0apple,s8003\0'.encode()) + (root / 'proc/sys/kernel/random/boot_id').write_text(BOOT) + (root / 'newroot/etc/os-release').write_text('NAME="Omarchy"\n') + if flag: + (root / 'newroot/etc/orchard/mac-usb-current').write_text(flag + '\n') + run = script + run = re.sub(r'/(sys|newroot|proc)/', lambda m: str(root) + m.group(), run) + run = run.replace('mkdir \"$g\" &&', 'mkdir -p \"$g/functions\" &&') + subprocess.run(['sh', '-ec', 'mount() { :; };\n' + run + '\nusb_gadget'], check=True, + capture_output=True) + g = root / 'sys/kernel/config/usb_gadget/orchard-system' + for key, value in [('idProduct', expected), ('idVendor', '0x05ac'), + ('bDeviceClass', '0xef' if expected == '0x1014' else '0x00'), + ('bDeviceSubClass', '0x02' if expected == '0x1014' else '0x00'), + ('bDeviceProtocol', '0x01' if expected == '0x1014' else '0x00'), ('configs/c.1/MaxPower', '500'), + ('UDC', 'usbdev')]: + self.assertEqual((g/key).read_text().strip(), value) + for name in ['ncm.usb0', 'acm.usb0']: + self.assertTrue((g/'configs/c.1'/name).is_symlink()) + self.assertEqual((g/'strings/0x409/serialnumber').read_text().strip(), f'BOOT:{BOOT} BOARD:{board}') + + def test_compatibility_pid_requires_orchard_identity_on_both_hosts(self): + for serial, mode in [(f'BOOT:{BOOT} BOARD:j71t', 'system'), ('AppleSerial', 'other'), + (f'BOOT:{BOOT} BOARD:j71b', 'other')]: + raw = plistlib.dumps([{'IOObjectClass': 'IOUSBHostDevice', 'idVendor': 0x05ac, + 'idProduct': 0x1014, 'locationID': 123, 'USB Serial Number': serial}]) + with patch.object(usb.sys, 'platform', 'darwin'), patch.object(usb.subprocess, 'run', + return_value=subprocess.CompletedProcess([], 0, raw, b'')): + self.assertEqual(usb.devices()[0]['mode'], mode) + with tempfile.TemporaryDirectory() as tmp: + node = Path(tmp)/'1-2'; node.mkdir() + for key, value in [('idVendor', '05ac'), ('idProduct', '1014'), ('serial', serial)]: + (node/key).write_text(value) + self.assertEqual(linux_usb.devices(Path(tmp))[0]['mode'], mode) + + @unittest.skipUnless(os.environ.get('ORCHARD_KERNEL_SOURCE'), 'set ORCHARD_KERNEL_SOURCE for production C tests') + def test_production_grant_and_charger_policy(self): + source = Path(os.environ['ORCHARD_KERNEL_SOURCE']) + composite = (source/'drivers/usb/gadget/composite.c').read_text() + charger = (source/'drivers/power/supply/apple_tristar_charger.c').read_text() + start = composite.index('if ((ctrl->bRequestType & ~USB_DIR_IN)') + match = composite[start+4:composite.index(' {', start)-1] + match = match.replace('ctrl->bRequestType', 'type').replace('ctrl->bRequest', 'request').replace('le16_to_cpu(cdev->desc.idVendor)', 'vendor') + c = r''' +#include +#include +#include +#include +typedef uint16_t u16; typedef int16_t s16; +#define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define READ_ONCE(a) (a) +#define USB_DIR_IN 0x80 +#define USB_TYPE_VENDOR 0x40 +#define USB_RECIP_DEVICE 0 +#define USB_APPLE_REQ_CURRENT_LIMIT 0x40 +#define USB_APPLE_VENDOR_ID 0x05ac +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_info_ratelimited(...) ((void)0) +#define EINVAL 22 +#define TRISTAR_CHG_FALLBACK_CODE 34 +#define TRISTAR_RAMPING 2 +#define TRISTAR_CHARGING 3 +struct usb_gadget {int dummy;}; +struct usb_composite_dev {struct usb_gadget *gadget; unsigned vbus_draw_ma; int apple_extra_ma; bool suspended;}; +static unsigned draw; +static void usb_gadget_vbus_draw(struct usb_gadget *g, unsigned ma){draw=ma;} +struct tristar_charger {unsigned usb_epoch, epoch, usb_ma, source_ma, max_ma, target_code, code, phase, step_ns; bool present,usb_on,stopping; void *dev,*wq; int lock,poll_work;}; +struct power_supply {struct tristar_charger *chg;}; +union power_supply_propval {int intval;}; +enum power_supply_property {POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT}; +static unsigned notifications; +static struct tristar_charger *power_supply_get_drvdata(struct power_supply *p){return p->chg;} +static void mutex_lock(int *p){assert(!*p);*p=1;} +static void mutex_unlock(int *p){assert(*p);*p=0;} +static void mod_delayed_work(void *wq,int *work,unsigned d){} +static void power_supply_changed(struct power_supply *p){notifications++;} +static void tristar_withdraw(struct tristar_charger *c,unsigned code){c->code=code;} +''' + for n in ['composite_vbus_draw', 'composite_apple_current_limit']: + c += function(composite, n) + for n in ['tristar_code_ma', 'tristar_ma_code', 'tristar_usb_ma', 'tristar_target_code', 'tristar_update_target', 'tristar_set_property']: + c += function(charger, n) + c += f'int match(unsigned type,unsigned request,unsigned w_length,unsigned vendor){{return {match};}}\n' + c += r''' +static void report(struct power_supply *p,int ma){union power_supply_propval v={.intval=ma*1000};assert(!tristar_set_property(p,POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,&v));} +int main(void){ + for(unsigned t=0;t<256;t++)for(unsigned r=0;r<256;r++)for(unsigned l=0;l<2;l++)for(unsigned v=0;v<2;v++) + assert(match(t,r,l,v?0x05ac:0x1234)==((t==0x40||t==0xc0)&&r==0x40&&!l&&v)); + struct usb_gadget g={0};struct usb_composite_dev d={.gadget=&g}; + composite_vbus_draw(&d,500);assert(draw==500); + composite_apple_current_limit(&d,500,1900);assert(draw==2400); + composite_vbus_draw(&d,500);assert(draw==2400); /* configuration preserves grant */ + for(unsigned i=0;i<65536;i++){ + composite_apple_current_limit(&d,0,i);assert(draw==(unsigned)max(500+(s16)i,0)); + } + d.suspended=true;composite_vbus_draw(&d,2); + composite_apple_current_limit(&d,500,1900);assert(draw==2); + d.suspended=false;composite_vbus_draw(&d,500);assert(draw==2400); + d.apple_extra_ma=0;composite_vbus_draw(&d,100);assert(draw==100); + composite_vbus_draw(&d,0);composite_apple_current_limit(&d,0,1900);assert(draw==0); + struct tristar_charger ch={.epoch=7,.usb_epoch=7,.max_ma=2400,.present=true,.usb_on=true,.phase=TRISTAR_CHARGING,.code=34}; + struct power_supply p={.chg=&ch}; + report(&p,2400);assert(ch.usb_ma==2400&&ch.target_code==186&&ch.phase==TRISTAR_RAMPING&¬ifications==1); + ch.code=186;ch.phase=TRISTAR_CHARGING;report(&p,500);assert(ch.code==34&&ch.target_code==34); + for(int ma=501;ma<=33267;ma++){ + report(&p,ma);assert(ch.usb_ma<=(unsigned)ma&&ch.usb_ma<=2400); + assert(tristar_code_ma(ch.target_code)<=min((unsigned)ma,2400)); + } + report(&p,-1);assert(!ch.usb_ma&&ch.target_code==34); + ch.usb_on=false;report(&p,2400);assert(!ch.usb_ma); + ch.usb_on=true;ch.present=false;report(&p,2400);assert(!ch.usb_ma); + ch.present=true;report(&p,2400);ch.epoch++;assert(!tristar_usb_ma(&ch)&&tristar_target_code(&ch)==34); + puts("PASS 262144 request tuples, all signed grants, suspend, board cap, ramp/revoke, attachment and epoch gates"); +} +''' + # Lifecycle invalidations remain in the actual callbacks. + for name in ['composite_reset', 'composite_disconnect', 'composite_suspend']: + self.assertIn('cdev->apple_extra_ma = 0;', function(composite, name)) + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp); (p/'test.c').write_text(c) + subprocess.run(['cc','-std=gnu11','-Wall','-Werror','-fsanitize=address,undefined', str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')], check=True) + +if __name__ == '__main__': + unittest.main() From ca3266cab0cdc75ed9424a2e674883ab937f518d Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 08/33] AOP sensors and certified warm reload kernel/configs/apple-16k/config: CONFIG_APPLE_AOP_GYRO=m CONFIG_APPLE_AOP_PRESSURE=m These options provide aop and its Kconfig dependencies on both page sizes. Assisted-by: LLM --- ...9-AOP-s-boot-arguments-and-sensor-ca.patch | 116 + chips/s8003.toml | 3 + devices/j71t/device.toml | 7 +- devices/j71t/firmware.json | 34 + docs/compatibility.md | 3 +- kernel/configs/apple-16k/config | 2 + .../s8000/0008-aop-implementation.patch | 121 + .../drivers/iio/0005-aop-implementation.patch | 760 ++++ .../pmdomain/0001-aop-implementation.patch | 24 + .../drivers/soc-apple/0014-aop-bindings.patch | 127 + .../soc-apple/0015-aop-implementation.patch | 3285 +++++++++++++++++ tests/test_j71t_data.py | 31 + 12 files changed, 4510 insertions(+), 3 deletions(-) create mode 100644 boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch create mode 100644 kernel/patches/devices/s8000/0008-aop-implementation.patch create mode 100644 kernel/patches/drivers/iio/0005-aop-implementation.patch create mode 100644 kernel/patches/drivers/pmdomain/0001-aop-implementation.patch create mode 100644 kernel/patches/drivers/soc-apple/0014-aop-bindings.patch create mode 100644 kernel/patches/drivers/soc-apple/0015-aop-implementation.patch create mode 100644 tests/test_j71t_data.py diff --git a/boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch b/boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch new file mode 100644 index 00000000..13245cd5 --- /dev/null +++ b/boot/m1n1/patches/orchard/0015-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 2 Nov 2026 15:07:08 +0100 +Subject: [PATCH] kboot: Pass the A9 AOP's boot arguments and sensor + calibration + +The A9's AOP firmware takes more than the ECID from its loader: the AOP +target number (iop-aop-nub aop-target), the chip's ID (/chosen chip-id) +and revision (/arm-io chip-revision). Its accelerometer, gyroscope and +pressure sensor also send no reports until the driver has given them +the unit's factory calibration, which iBoot resolves from syscfg into +the sensors' nodes below iop-aop-nub. + +Copy the three numbers into the AOP's node (alias aop) as +apple,aop-target, apple,chip-id and apple,chip-revision, and each +sensor's calibration properties into the AOP's child node of the same +name, prefixed with "apple,". Unprovisioned "syscfg/" references and +nodes the device tree doesn't have are skipped. The orientation +properties are not copied: the A9 firmware rejects them and the board's +mount-matrix describes the mounting. + +The values are the unit's own and never leave the device: the kernel +sends them to the AOP and nothing else. + +Assisted-by: LLM +--- + src/kboot.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 68 insertions(+), 1 deletion(-) + +diff --git a/src/kboot.c b/src/kboot.c +index ebd6282..7ce9d72 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -1005,6 +1005,73 @@ static int dt_set_als(void) + return 0; + } + ++/* ++ * The A9's AOP firmware also takes the AOP target number and the chip's ID and ++ * revision in its boot arguments, and each motion sensor's factory ++ * calibration, which iBoot resolves from syscfg into the sensor's node below ++ * iop-aop-nub. Copy them into the AOP's node and its sensor child nodes of the ++ * same name; the calibration keeps its name with an "apple," prefix. ++ */ ++static const char *const aop_sensor_calibrations[] = { ++ "low-temp-accel-offset", ++ "accel-sensitivity-calibration", ++ "accel-range-sensitivity-cal", ++ "accel-interrupt-calibration", ++ "gyro-sensitivity-calibration", ++ "gyro-range-sensitivity-cal", ++ "gyro-interrupt-calibration", ++ "gyro-trim-calibration", ++ "pressure-offset-calibration", ++ "pressure-accel-sensitivity", ++ "pressure-heater-resistance", ++}; ++ ++static int dt_set_aop_sensors(int node) ++{ ++ int nub = adt_path_offset(adt, "/arm-io/aop/iop-aop-nub"); ++ if (nub < 0) ++ return 0; ++ ++ u32 val; ++ if (ADT_GETPROP(adt, nub, "aop-target", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,aop-target", val)) ++ bail("FDT: failed to set apple,aop-target\n"); ++ ++ int chosen = adt_path_offset(adt, "/chosen"); ++ if (chosen >= 0 && ADT_GETPROP(adt, chosen, "chip-id", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-id", val)) ++ bail("FDT: failed to set apple,chip-id\n"); ++ ++ int arm_io = adt_path_offset(adt, "/arm-io"); ++ if (arm_io >= 0 && ADT_GETPROP(adt, arm_io, "chip-revision", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-revision", val)) ++ bail("FDT: failed to set apple,chip-revision\n"); ++ ++ int sensor = nub; ++ ADT_FOREACH_CHILD(adt, sensor) ++ { ++ const char *name = adt_get_name(adt, sensor); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(aop_sensor_calibrations); i++) { ++ u32 len; ++ const u8 *blob = adt_getprop(adt, sensor, aop_sensor_calibrations[i], &len); ++ if (!blob || !len || (len >= 7 && !memcmp(blob, "syscfg/", 7))) ++ continue; ++ ++ /* Earlier properties may have moved the sensor's node. */ ++ int child = fdt_subnode_offset(dt, node, name); ++ if (child < 0) ++ break; ++ ++ char prop[64]; ++ snprintf(prop, sizeof(prop), "apple,%s", aop_sensor_calibrations[i]); ++ if (fdt_setprop(dt, child, prop, blob, len)) ++ bail("FDT: failed to set %s/%s\n", name, prop); ++ } ++ } ++ return 0; ++} ++ + /* + * The AOP's firmware carries the chip's ECID in its boot arguments. Pass it + * only to the AOP's node; the driver removes it from the tree once read. +@@ -1028,7 +1095,7 @@ static int dt_set_aop(void) + + if (fdt_setprop_u64(dt, node, "apple,ecid", ecid)) + bail("FDT: failed to set apple,ecid\n"); +- return 0; ++ return dt_set_aop_sensors(node); + } + + static int dt_set_multitouch(void) diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..13220105 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -15,3 +15,6 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "J71t on hardware" stages = "remote_boot two-stage CPID list" + +# Local capability delta: aop. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/aop.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..d4fa8ae2 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -34,8 +34,8 @@ screen_off = "untested" backlight = "untested" touch = "untested" buttons = "untested" -accelerometer = "missing" -gyroscope = "missing" +accelerometer = "untested" +gyroscope = "untested" light_sensor = "untested" touch_id = "missing" storage_read = "untested" @@ -54,3 +54,6 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" + +# Local capability delta: aop. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/aop.md. diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index 3321e246..c03fb880 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -113,6 +113,12 @@ }, "device": "/usr/share/firmware/multitouch/J71t.mtprops", "sha256": "69f27fe207cc5810a748dac3dfe640b1c5a755709f49513b4b5125660784f0fc" + }, + "aop.im4p": { + "ipsw": { + "zip": "Firmware/AOP/aopfw-ipad6faop.im4p" + }, + "sha256": "5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa" } }, "outputs": [ @@ -279,6 +285,34 @@ "calibration_size": 1024 } ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f.bin", + "sha256": "af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP" + } + ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f-ext.bin", + "sha256": "2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP_ext" + } + ] } ] } diff --git a/docs/compatibility.md b/docs/compatibility.md index 52850565..1f8d2a8c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -161,7 +161,7 @@ Boards Orchard builds for. States: working, untested, missing, none (no such har | `n66` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n69u` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j71t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | +| `j71t` | untested | untested | untested | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | | `j72t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | | `n66m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n69` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | @@ -248,6 +248,7 @@ A yes means an enabled device tree node binds a driver in `7.3.0-rc1-orchard-app | Board | Feature | | | --- | --- | --- | +| `j71t` | Accelerometer | untested, but no driver binds | | `d10` | Bluetooth input devices | missing, but a driver binds | | `d10` | Bluetooth audio | missing, but a driver binds | | `d101` | Bluetooth input devices | missing, but a driver binds | diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index aae562f8..ddbbb543 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -5620,6 +5620,7 @@ CONFIG_APPLE_AOP_ACCEL=m # CONFIG_ADIS16260 is not set # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set +CONFIG_APPLE_AOP_GYRO=m # CONFIG_BMG160 is not set # CONFIG_FXAS21002C is not set # CONFIG_MPU3050_I2C is not set @@ -5855,6 +5856,7 @@ CONFIG_TSL2583=m # CONFIG_HP206C is not set # CONFIG_ZPA2326 is not set # CONFIG_ADP810 is not set +CONFIG_APPLE_AOP_PRESSURE=m # end of Pressure sensors # diff --git a/kernel/patches/devices/s8000/0008-aop-implementation.patch b/kernel/patches/devices/s8000/0008-aop-implementation.patch new file mode 100644 index 00000000..a2182740 --- /dev/null +++ b/kernel/patches/devices/s8000/0008-aop-implementation.patch @@ -0,0 +1,121 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +index 354533f26..338182cf5 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +@@ -11,6 +11,7 @@ + + / { + aliases { ++ aop = &aop; /* Used by loader */ + ausbphy0 = &ausbphy0; + nvme-hmb = &nvme_hmb; /* Used by loader */ + serial0 = &serial0; +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index bec208568..2a0a83383 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -446,6 +446,43 @@ wdt: watchdog@2102b0000 { + interrupt-parent = <&aic>; + interrupts = ; + }; ++ ++ aop: aop@210800000 { ++ compatible = "apple,s8000-aop"; ++ reg = <0x2 0x10800000 0x0 0x4000>, ++ <0x2 0x10000600 0x0 0x4>, ++ <0x2 0x10e00000 0x0 0x9b000>; ++ reg-names = "akf", "control", "sram"; ++ mboxes = <&aop_mbox>; ++ mbox-names = "mbox"; ++ status = "disabled"; ++ ++ aop_accel: accel { ++ compatible = "apple,s8000-aop-accel"; ++ }; ++ ++ aop_gyro: gyro { ++ compatible = "apple,s8000-aop-gyro"; ++ }; ++ ++ aop_pressure: pressure { ++ compatible = "apple,s8000-aop-pressure"; ++ }; ++ }; ++ ++ aop_mbox: mbox@210804000 { ++ compatible = "apple,s8000-a7iop-mailbox", "apple,t8010-a7iop-mailbox"; ++ reg = <0x2 0x10804000 0x0 0x1000>; ++ interrupt-parent = <&aic>; ++ interrupts = , ++ , ++ , ++ ; ++ interrupt-names = "send-empty", "send-not-empty", ++ "recv-empty", "recv-not-empty"; ++ #mbox-cells = <0>; ++ status = "disabled"; ++ }; + }; + + timer { +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 7e4d67d7f..347ec084c 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -16,6 +16,44 @@ / { + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; + }; + ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* Same tail page as release6, never a firmware DMA grant. */ ++ aop_shutdown: aop-shutdown@8583fc000 { ++ reg = <0x8 0x583fc000 0 0x4000>; ++ no-map; ++ }; ++ }; ++}; ++ ++&aop { ++ memory-region = <&aop_shutdown>; ++ apple,observe-domains = <&ps_aop_gpio &ps_aop &ps_aop_busif ++ &ps_aop_cpu &ps_aop_filter>; ++ apple,spi0-domain = <&ps_spi0>; ++ firmware-name = "apple/aop-ipad6f.bin", "apple/aop-ipad6f-ext.bin"; ++ status = "okay"; ++}; ++ ++/* ++ * The AOP reports the accelerometer in its sensor frame, which is the ++ * device frame (x right, y up, z out of the screen) negated: flat, screen up ++ * reads z = -1 g, upright y = -1 g. iio-sensor-proxy takes upright as ++ * negative y and needs z flipped. ++ */ ++&aop_accel { ++ mount-matrix = "1", "0", "0", ++ "0", "1", "0", ++ "0", "0", "-1"; ++}; ++ ++&aop_mbox { ++ status = "okay"; ++}; ++ + &touchscreen { + firmware-name = "apple/j71t-fwstart.z2fw"; + }; diff --git a/kernel/patches/drivers/iio/0005-aop-implementation.patch b/kernel/patches/drivers/iio/0005-aop-implementation.patch new file mode 100644 index 00000000..6d6ff334 --- /dev/null +++ b/kernel/patches/drivers/iio/0005-aop-implementation.patch @@ -0,0 +1,760 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] iio: apple-aop: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig +index b01598194..ed328a590 100644 +--- a/drivers/iio/accel/Kconfig ++++ b/drivers/iio/accel/Kconfig +@@ -213,7 +213,7 @@ config APPLE_AOP_ACCEL + select IIO_KFIFO_BUF + help + Say yes here to get support for the accelerometer that the Always-On +- Processor (AOP) of Apple A10 iPads and iPhones serves. ++ Processor (AOP) of Apple A9 and A10 iPads and iPhones serves. + + To compile this driver as a module, choose M here: the module will be + called apple_aop_accel. +diff --git a/drivers/iio/accel/apple_aop_accel.c b/drivers/iio/accel/apple_aop_accel.c +index 96c901361..e1b0e136d 100644 +--- a/drivers/iio/accel/apple_aop_accel.c ++++ b/drivers/iio/accel/apple_aop_accel.c +@@ -9,9 +9,13 @@ + * + * 0 u32 sequence + * 4 u16 reserved +- * 6 s32 x, y, z in 1/65536 g, in the device frame (x right, y up, +- * z out of the screen; +1 g opposite to gravity) ++ * 6 s32 x, y, z in 1/65536 g + * 18 u32 timestamp ++ * ++ * The T8010's firmware reports in the device frame (x right, y up, z out of ++ * the screen; +1 g opposite to gravity). The S8000's firmware first needs the ++ * unit's factory calibration and reports in the sensor's frame; the board's ++ * mount-matrix maps it. + */ + + #include +@@ -21,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -41,8 +46,16 @@ + /* 9.80665 m/s^2 / 65536 */ + #define AOP_ACCEL_SCALE_NANO 149637 + ++struct apple_aop_accel_hw { ++ const struct apple_aop_calibration *calibration; ++ unsigned int num_calibration; ++ /* No separate raw sample stream (raw interval 0) */ ++ bool no_raw_stream; ++}; ++ + struct apple_aop_accel { + struct device *dev; ++ const struct apple_aop_accel_hw *hw; + const struct apple_aop_service *service; + struct iio_mount_matrix orientation; + +@@ -75,7 +88,9 @@ static int apple_aop_accel_set_interval(struct apple_aop_accel *accel, u32 inter + int i, ret; + + for (i = 0; i < ARRAY_SIZE(tags); i++) { +- ret = apple_aop_accel_set_property(accel, tags[i], interval_us); ++ ret = apple_aop_accel_set_property(accel, tags[i], ++ tags[i] == AOP_PROP_RAW_INTERVAL && ++ accel->hw->no_raw_stream ? 0 : interval_us); + if (ret) + return ret; + } +@@ -230,6 +245,7 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return -ENOMEM; + accel = iio_priv(indio_dev); + accel->dev = dev; ++ accel->hw = device_get_match_data(dev); + accel->service = service; + mutex_init(&accel->lock); + spin_lock_init(&accel->sample_lock); +@@ -252,6 +268,15 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (accel->hw->num_calibration) { ++ ret = apple_aop_calibrate(service, dev, accel->hw->calibration, ++ accel->hw->num_calibration); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to calibrate\n"); ++ } ++ } ++ + ret = apple_aop_accel_set_interval(accel, AOP_ACCEL_DEFAULT_US); + if (ret) { + apple_aop_unlisten(service); +@@ -265,8 +290,25 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return devm_iio_device_register(dev, indio_dev); + } + ++/* Offset at low temperature, sensitivity (either), interrupt calibration */ ++static const struct apple_aop_calibration apple_aop_accel_s8000_calibration[] = { ++ { "apple,low-temp-accel-offset", 0x1a, 1 }, ++ { "apple,accel-sensitivity-calibration", 0x0b, 2 }, ++ { "apple,accel-range-sensitivity-cal", 0x27, 2 }, ++ { "apple,accel-interrupt-calibration", 0x19, 3 }, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_s8000 = { ++ .calibration = apple_aop_accel_s8000_calibration, ++ .num_calibration = ARRAY_SIZE(apple_aop_accel_s8000_calibration), ++ .no_raw_stream = true, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_t8010 = { }; ++ + static const struct of_device_id apple_aop_accel_of_match[] = { +- { .compatible = "apple,t8010-aop-accel" }, ++ { .compatible = "apple,s8000-aop-accel", .data = &apple_aop_accel_s8000 }, ++ { .compatible = "apple,t8010-aop-accel", .data = &apple_aop_accel_t8010 }, + { } + }; + MODULE_DEVICE_TABLE(of, apple_aop_accel_of_match); +diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig +index 3e193ee0f..8f506a65d 100644 +--- a/drivers/iio/gyro/Kconfig ++++ b/drivers/iio/gyro/Kconfig +@@ -63,6 +63,18 @@ config ADXRS450 + This driver can also be built as a module. If so, the module + will be called adxrs450. + ++config APPLE_AOP_GYRO ++ tristate "Apple AOP gyroscope" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the gyroscope that the Always-On ++ Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_gyro. ++ + config BMG160 + tristate "BOSCH BMG160 Gyro Sensor" + depends on (I2C || SPI_MASTER) +diff --git a/drivers/iio/gyro/Makefile b/drivers/iio/gyro/Makefile +index 0319b397d..212bbb814 100644 +--- a/drivers/iio/gyro/Makefile ++++ b/drivers/iio/gyro/Makefile +@@ -10,6 +10,7 @@ obj-$(CONFIG_ADIS16136) += adis16136.o + obj-$(CONFIG_ADIS16260) += adis16260.o + obj-$(CONFIG_ADXRS290) += adxrs290.o + obj-$(CONFIG_ADXRS450) += adxrs450.o ++obj-$(CONFIG_APPLE_AOP_GYRO) += apple_aop_gyro.o + obj-$(CONFIG_BMG160) += bmg160_core.o + obj-$(CONFIG_BMG160_I2C) += bmg160_i2c.o + obj-$(CONFIG_BMG160_SPI) += bmg160_spi.o +diff --git a/drivers/iio/gyro/apple_aop_gyro.c b/drivers/iio/gyro/apple_aop_gyro.c +new file mode 100644 +index 000000000..edbda1f43 +--- /dev/null ++++ b/drivers/iio/gyro/apple_aop_gyro.c +@@ -0,0 +1,291 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Gyroscope served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the gyroscope itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one motion report (EPIC ++ * subtype 0xc1) per report interval, laid out like the accelerometer's: ++ * ++ * 0 u16 sequence ++ * 2 u32 status ++ * 6 s32 x, y, z in 1/65536 degree per second ++ * 18 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_GYRO 0xc1 ++#define AOP_GYRO_LEN 22 ++#define AOP_GYRO_XYZ 6 ++ ++#define AOP_GYRO_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_GYRO_MIN_US 10000 /* 100 Hz */ ++#define AOP_GYRO_MAX_US 1000000 /* 1 Hz */ ++ ++/* pi / 180 rad/s / 65536 = 1171270 / 2^42 rad/s */ ++#define AOP_GYRO_SCALE 1171270 ++#define AOP_GYRO_SCALE_LOG2 42 ++ ++struct apple_aop_gyro { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ struct iio_mount_matrix orientation; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ s32 sample[3]; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(s32, scan, 3); ++}; ++ ++static int apple_aop_gyro_set_u32(struct apple_aop_gyro *gyro, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(gyro->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_gyro_set_interval(struct apple_aop_gyro *gyro, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ gyro->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_gyro_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int i; ++ ++ if (subtype != AOP_REPORT_GYRO || len < AOP_GYRO_LEN) ++ return; ++ ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ for (i = 0; i < 3; i++) ++ gyro->sample[i] = get_unaligned_le32(data + AOP_GYRO_XYZ + 4 * i); ++ gyro->have_sample = true; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ memcpy(gyro->scan, gyro->sample, sizeof(gyro->sample)); ++ iio_push_to_buffers_with_ts(indio_dev, gyro->scan, sizeof(gyro->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_gyro_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ *val = gyro->sample[chan->scan_index]; ++ ret = gyro->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ *val = AOP_GYRO_SCALE; ++ *val2 = AOP_GYRO_SCALE_LOG2; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&gyro->lock); ++ *val = USEC_PER_SEC / gyro->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % gyro->interval_us) * USEC_PER_SEC / ++ gyro->interval_us; ++ mutex_unlock(&gyro->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_gyro_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_GYRO_MIN_US, AOP_GYRO_MAX_US); ++ ++ mutex_lock(&gyro->lock); ++ ret = apple_aop_gyro_set_interval(gyro, interval); ++ mutex_unlock(&gyro->lock); ++ return ret; ++} ++ ++static const struct iio_mount_matrix * ++apple_aop_gyro_get_mount_matrix(const struct iio_dev *indio_dev, ++ const struct iio_chan_spec *chan) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ ++ return &gyro->orientation; ++} ++ ++static const struct iio_chan_spec_ext_info apple_aop_gyro_ext_info[] = { ++ IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, apple_aop_gyro_get_mount_matrix), ++ { } ++}; ++ ++#define APPLE_AOP_GYRO_CHANNEL(axis, index) { \ ++ .type = IIO_ANGL_VEL, \ ++ .modified = 1, \ ++ .channel2 = IIO_MOD_##axis, \ ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ ++ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \ ++ .scan_index = index, \ ++ .scan_type = { \ ++ .sign = 's', \ ++ .realbits = 32, \ ++ .storagebits = 32, \ ++ .endianness = IIO_CPU, \ ++ }, \ ++ .ext_info = apple_aop_gyro_ext_info, \ ++} ++ ++static const struct iio_chan_spec apple_aop_gyro_channels[] = { ++ APPLE_AOP_GYRO_CHANNEL(X, 0), ++ APPLE_AOP_GYRO_CHANNEL(Y, 1), ++ APPLE_AOP_GYRO_CHANNEL(Z, 2), ++ IIO_CHAN_SOFT_TIMESTAMP(3), ++}; ++ ++static const struct iio_info apple_aop_gyro_info = { ++ .read_raw = apple_aop_gyro_read_raw, ++ .write_raw = apple_aop_gyro_write_raw, ++}; ++ ++/* Sensitivity and interrupt calibration make it complete */ ++static const struct apple_aop_calibration apple_aop_gyro_calibration[] = { ++ { "apple,gyro-sensitivity-calibration", 0x0b, 1 }, ++ { "apple,gyro-interrupt-calibration", 0x19, 2 }, ++ { "apple,gyro-range-sensitivity-cal", 0x27, 0 }, ++ { "apple,gyro-trim-calibration", 0x20, 0 }, ++}; ++ ++static void apple_aop_gyro_stop(void *data) ++{ ++ struct apple_aop_gyro *gyro = data; ++ ++ apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(gyro->service); ++} ++ ++static int apple_aop_gyro_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_gyro *gyro; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*gyro)); ++ if (!indio_dev) ++ return -ENOMEM; ++ gyro = iio_priv(indio_dev); ++ gyro->dev = dev; ++ gyro->service = service; ++ mutex_init(&gyro->lock); ++ spin_lock_init(&gyro->sample_lock); ++ ++ ret = iio_read_mount_matrix(dev, &gyro->orientation); ++ if (ret) ++ return ret; ++ ++ indio_dev->name = "apple-aop-gyro"; ++ indio_dev->info = &apple_aop_gyro_info; ++ indio_dev->channels = apple_aop_gyro_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_gyro_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_gyro_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_gyro_calibration, ++ ARRAY_SIZE(apple_aop_gyro_calibration)); ++ if (!ret) ++ ret = apple_aop_gyro_set_interval(gyro, AOP_GYRO_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_gyro_stop, gyro); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_gyro_of_match[] = { ++ { .compatible = "apple,s8000-aop-gyro" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_gyro_of_match); ++ ++static struct platform_driver apple_aop_gyro_driver = { ++ .driver = { ++ .name = "apple-aop-gyro", ++ .of_match_table = apple_aop_gyro_of_match, ++ }, ++ .probe = apple_aop_gyro_probe, ++}; ++module_platform_driver(apple_aop_gyro_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP gyroscope driver"); ++MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig +index 838a8340c..18535ad99 100644 +--- a/drivers/iio/pressure/Kconfig ++++ b/drivers/iio/pressure/Kconfig +@@ -384,4 +384,16 @@ config ADP810 + To compile this driver as a module, choose M here: the module will + be called adp810 + ++config APPLE_AOP_PRESSURE ++ tristate "Apple AOP barometric pressure sensor" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the barometric pressure sensor that ++ the Always-On Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_pressure. ++ + endmenu +diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile +index bc0d11a20..bf3e27ba9 100644 +--- a/drivers/iio/pressure/Makefile ++++ b/drivers/iio/pressure/Makefile +@@ -9,6 +9,7 @@ obj-$(CONFIG_ABP2030PA) += abp2030pa.o + obj-$(CONFIG_ABP2030PA_I2C) += abp2030pa_i2c.o + obj-$(CONFIG_ABP2030PA_SPI) += abp2030pa_spi.o + obj-$(CONFIG_ADP810) += adp810.o ++obj-$(CONFIG_APPLE_AOP_PRESSURE) += apple_aop_pressure.o + obj-$(CONFIG_ROHM_BM1390) += rohm-bm1390.o + obj-$(CONFIG_BMP280) += bmp280.o + bmp280-objs := bmp280-core.o bmp280-regmap.o +diff --git a/drivers/iio/pressure/apple_aop_pressure.c b/drivers/iio/pressure/apple_aop_pressure.c +new file mode 100644 +index 000000000..8cd983c57 +--- /dev/null ++++ b/drivers/iio/pressure/apple_aop_pressure.c +@@ -0,0 +1,257 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Barometric pressure sensor served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the pressure sensor itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one report (EPIC subtype ++ * 0xc4) per report interval: ++ * ++ * 0 u16 sequence ++ * 2 u32 pressure in 1/65536 kPa ++ * 6 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_PRESSURE 0xc4 ++#define AOP_PRESSURE_LEN 10 ++#define AOP_PRESSURE_VALUE 2 ++ ++#define AOP_PRESSURE_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_PRESSURE_MIN_US 10000 /* 100 Hz */ ++#define AOP_PRESSURE_MAX_US 1000000 /* 1 Hz */ ++ ++struct apple_aop_pressure { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ u32 sample; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(u32, scan, 1); ++}; ++ ++static int apple_aop_pressure_set_u32(struct apple_aop_pressure *pressure, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(pressure->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_pressure_set_interval(struct apple_aop_pressure *pressure, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ pressure->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_pressure_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ ++ if (subtype != AOP_REPORT_PRESSURE || len < AOP_PRESSURE_LEN) ++ return; ++ ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ pressure->sample = get_unaligned_le32(data + AOP_PRESSURE_VALUE); ++ pressure->have_sample = true; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ pressure->scan[0] = pressure->sample; ++ iio_push_to_buffers_with_ts(indio_dev, pressure->scan, sizeof(pressure->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_pressure_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ *val = pressure->sample; ++ ret = pressure->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ /* kPa */ ++ *val = 1; ++ *val2 = 16; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&pressure->lock); ++ *val = USEC_PER_SEC / pressure->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % pressure->interval_us) * USEC_PER_SEC / ++ pressure->interval_us; ++ mutex_unlock(&pressure->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_pressure_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_PRESSURE_MIN_US, AOP_PRESSURE_MAX_US); ++ ++ mutex_lock(&pressure->lock); ++ ret = apple_aop_pressure_set_interval(pressure, interval); ++ mutex_unlock(&pressure->lock); ++ return ret; ++} ++ ++static const struct iio_chan_spec apple_aop_pressure_channels[] = { ++ { ++ .type = IIO_PRESSURE, ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), ++ .scan_index = 0, ++ .scan_type = { ++ .sign = 'u', ++ .realbits = 32, ++ .storagebits = 32, ++ .endianness = IIO_CPU, ++ }, ++ }, ++ IIO_CHAN_SOFT_TIMESTAMP(1), ++}; ++ ++static const struct iio_info apple_aop_pressure_info = { ++ .read_raw = apple_aop_pressure_read_raw, ++ .write_raw = apple_aop_pressure_write_raw, ++}; ++ ++/* The offset or the acceleration sensitivity makes it complete */ ++static const struct apple_aop_calibration apple_aop_pressure_calibration[] = { ++ { "apple,pressure-offset-calibration", 0x0b, 1 }, ++ { "apple,pressure-accel-sensitivity", 0xd3, 1 }, ++ { "apple,pressure-heater-resistance", 0xd4, 0 }, ++}; ++ ++static void apple_aop_pressure_stop(void *data) ++{ ++ struct apple_aop_pressure *pressure = data; ++ ++ apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(pressure->service); ++} ++ ++static int apple_aop_pressure_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_pressure *pressure; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*pressure)); ++ if (!indio_dev) ++ return -ENOMEM; ++ pressure = iio_priv(indio_dev); ++ pressure->dev = dev; ++ pressure->service = service; ++ mutex_init(&pressure->lock); ++ spin_lock_init(&pressure->sample_lock); ++ ++ indio_dev->name = "apple-aop-pressure"; ++ indio_dev->info = &apple_aop_pressure_info; ++ indio_dev->channels = apple_aop_pressure_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_pressure_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_pressure_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_pressure_calibration, ++ ARRAY_SIZE(apple_aop_pressure_calibration)); ++ if (!ret) ++ ret = apple_aop_pressure_set_interval(pressure, AOP_PRESSURE_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_pressure_stop, pressure); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_pressure_of_match[] = { ++ { .compatible = "apple,s8000-aop-pressure" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_pressure_of_match); ++ ++static struct platform_driver apple_aop_pressure_driver = { ++ .driver = { ++ .name = "apple-aop-pressure", ++ .of_match_table = apple_aop_pressure_of_match, ++ }, ++ .probe = apple_aop_pressure_probe, ++}; ++module_platform_driver(apple_aop_pressure_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP barometric pressure sensor driver"); ++MODULE_LICENSE("Dual MIT/GPL"); diff --git a/kernel/patches/drivers/pmdomain/0001-aop-implementation.patch b/kernel/patches/drivers/pmdomain/0001-aop-implementation.patch new file mode 100644 index 00000000..2fd722aa --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0001-aop-implementation.patch @@ -0,0 +1,24 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pmdomain: apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..a0559873c 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -417,6 +417,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + goto err_remove; + } + ++ platform_set_drvdata(pdev, ps); + return 0; + err_remove: + if (ps->has_genpd) { diff --git a/kernel/patches/drivers/soc-apple/0014-aop-bindings.patch b/kernel/patches/drivers/soc-apple/0014-aop-bindings.patch new file mode 100644 index 00000000..aaf9d2d5 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0014-aop-bindings.patch @@ -0,0 +1,127 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] dt-bindings: soc-apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +index ba035ff80..ea414a8f4 100644 +--- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml ++++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +@@ -59,6 +59,12 @@ properties: + items: + - const: apple,t8010-a7iop-mailbox + ++ - description: ++ The same A7 IOP mailbox on the A9. ++ items: ++ - const: apple,s8000-a7iop-mailbox ++ - const: apple,t8010-a7iop-mailbox ++ + reg: + maxItems: 1 + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +index 14726afa0..5d2485ac9 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +@@ -4,14 +4,14 @@ + $id: http://devicetree.org/schemas/soc/apple/apple,t8010-aop.yaml# + $schema: http://devicetree.org/meta-schemas/core.yaml# + +-title: Apple T8010 Always-On Processor ++title: Apple S8000 and T8010 Always-On Processor + + maintainers: + - Dalton Alexandre + + description: +- The Always-On Processor (AOP) of the A10 is an ARMv7 coprocessor that +- runs RTKit firmware and serves the motion sensors (accelerometer, ++ The Always-On Processor (AOP) of the A9 and A10 is an ARMv7 coprocessor ++ that runs RTKit firmware and serves the motion sensors (accelerometer, + gyroscope, compass, pressure) and the always-on microphones. It runs from + local SRAM, with a further firmware segment in system memory, and talks + to the application processor through an A7 IOP mailbox. Its firmware is +@@ -19,7 +19,9 @@ description: + + properties: + compatible: +- const: apple,t8010-aop ++ enum: ++ - apple,s8000-aop ++ - apple,t8010-aop + + reg: + items: +@@ -50,6 +52,38 @@ properties: + The chip's unique ID (ECID), filled in by the bootloader. The + firmware's boot arguments carry it. + ++ apple,aop-target: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The AOP target number from the Apple device tree, filled in by the ++ bootloader. The S8000 firmware's boot arguments carry it. ++ ++ apple,chip-id: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip ID (e.g. 0x8003), filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ apple,chip-revision: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip revision, filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ memory-region: ++ maxItems: 1 ++ description: Reserved 16 KiB shutdown transcript page for J71t warm reload. ++ ++ apple,observe-domains: ++ $ref: /schemas/types.yaml#/definitions/phandle-array ++ minItems: 5 ++ maxItems: 5 ++ description: AOP GPIO, AOP, bus interface, CPU and filter in lock order. ++ ++ apple,spi0-domain: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: Read-only SPI0 power-state provider observation. ++ + patternProperties: + "^(accel|gyro|compass|pressure)$": + type: object +@@ -58,6 +92,9 @@ patternProperties: + properties: + compatible: + enum: ++ - apple,s8000-aop-accel ++ - apple,s8000-aop-gyro ++ - apple,s8000-aop-pressure + - apple,t8010-aop-accel + - apple,t8010-aop-gyro + - apple,t8010-aop-compass +@@ -65,6 +102,15 @@ patternProperties: + + mount-matrix: + description: The sensor's mounting rotation in the device. ++ ++ patternProperties: ++ "^apple,(low-temp-accel-offset|(accel|gyro)-(sensitivity|interrupt)-calibration|(accel|gyro)-range-sensitivity-cal|gyro-trim-calibration|pressure-(offset-calibration|accel-sensitivity|heater-resistance))$": ++ $ref: /schemas/types.yaml#/definitions/uint8-array ++ maxItems: 64 ++ description: ++ The unit's factory calibration of the sensor, copied by the ++ bootloader from the property of the same name (without the ++ prefix) in the Apple device tree. The S8000 firmware takes it. + required: + - compatible + additionalProperties: false diff --git a/kernel/patches/drivers/soc-apple/0015-aop-implementation.patch b/kernel/patches/drivers/soc-apple/0015-aop-implementation.patch new file mode 100644 index 00000000..31a196d9 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0015-aop-implementation.patch @@ -0,0 +1,3285 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] soc: apple: AOP sensors and certified warm reload + +Expose S8000 acceleration, gyroscope and pressure through standard IIO, with mount matrices and original board calibration supplied by m1n1. Fold the cold PMGR provider-publication correction and final retained-grant RTKit/AFK shutdown ordering into the warm-reload implementation. Only a certified previous quiesce may reload; unknown and uncertified warm states refuse before register writes. This avoids replacing proven warm support with a blanket sensor blacklist. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/MAINTAINERS b/MAINTAINERS +index a4a69643c..12c292825 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -2068,6 +2068,8 @@ APPLE T8010 ALWAYS-ON PROCESSOR + M: Dalton Alexandre + S: Maintained + F: Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++F: drivers/iio/gyro/apple_aop_gyro.c ++F: drivers/iio/pressure/apple_aop_pressure.c + F: drivers/soc/apple/aop.c + F: include/linux/soc/apple/aop.h + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 17c017425..a4f6fbce4 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -479,3 +481,84 @@ MODULE_AUTHOR("Hector Martin "); + MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); + + module_platform_driver(apple_pmgr_ps_driver); ++ ++/* ++ * Read-only observer for bring-up. The caller's nodes follow the same ++ * child -> parent order as genpd power transitions. No domain is attached, ++ * resumed, pinned on, reset, or written by this function. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data) ++{ ++ struct platform_device *providers[5]; ++ struct apple_pmgr_ps *ps[5]; ++ unsigned long flags; ++ unsigned int i, j, found = 0; ++ int ret = 0; ++ bool accessible = true; ++ ++ if (!count || count > ARRAY_SIZE(providers) || !read) ++ return -EINVAL; ++ for (i = 0; i < count; i++) { ++ for (j = 0; j < i; j++) ++ if (nodes[i] == nodes[j]) { ++ ret = -EINVAL; ++ goto put; ++ } ++ providers[i] = of_find_device_by_node(nodes[i]); ++ if (!providers[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ device_lock(&providers[i]->dev); ++ found++; ++ if (!providers[i]->dev.driver || ++ strcmp(providers[i]->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ ps[i] = platform_get_drvdata(providers[i]); ++ if (!ps[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ } ++ ++ /* PMGR domains use GENPD_FLAG_IRQ_SAFE, i.e. genpd.slock. */ ++ local_irq_save(flags); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->genpd.slock, i); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->lock, i); ++ for (i = 0; i < count; i++) { ++ states[i].genpd_status = ps[i]->genpd.status; ++ ret = regmap_read(ps[i]->regmap, ps[i]->offset, &states[i].raw); ++ if (ret) ++ goto unlock; ++ if (states[i].genpd_status != GENPD_STATE_ON || ++ FIELD_GET(APPLE_PMGR_PS_ACTUAL, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ FIELD_GET(APPLE_PMGR_PS_TARGET, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ states[i].raw & (APPLE_PMGR_RESET | APPLE_PMGR_DEV_DISABLE | ++ APPLE_PMGR_PARENT_OFF)) ++ accessible = false; ++ } ++ if (accessible) ++ read(data); ++ else ++ ret = -EHOSTDOWN; ++unlock: ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->lock); ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->genpd.slock); ++ local_irq_restore(flags); ++put: ++ while (found) { ++ found--; ++ device_unlock(&providers[found]->dev); ++ put_device(&providers[found]->dev); ++ } ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_observe); +diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig +index e68a49f1b..c0af67b62 100644 +--- a/drivers/soc/apple/Kconfig ++++ b/drivers/soc/apple/Kconfig +@@ -39,16 +39,16 @@ config APPLE_RTKIT + Say 'y' here if you have an Apple SoC. + + config APPLE_AOP_T8010 +- tristate "Apple T8010 Always-On Processor" ++ tristate "Apple S8000 and T8010 Always-On Processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help +- The Always-On Processor (AOP) of the Apple A10 runs the firmware for +- the motion sensors and the always-on microphones. This driver loads +- and starts that firmware and provides its services to the sensor +- drivers. ++ The Always-On Processor (AOP) of the Apple A9 and A10 runs the ++ firmware for the motion sensors and the always-on microphones. This ++ driver loads and starts that firmware and provides its services to ++ the sensor drivers. + +- Say 'y' or 'm' here if you have an A10-based iPad or iPhone. ++ Say 'y' or 'm' here if you have an A9- or A10-based iPad or iPhone. + + config APPLE_PMP_T8015_V1 + tristate "Apple T8015 PMP" +diff --git a/drivers/soc/apple/aop-warm-record.h b/drivers/soc/apple/aop-warm-record.h +new file mode 100644 +index 000000000..a735809d5 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-record.h +@@ -0,0 +1,64 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#include ++#include ++#define A9_HO_PAGE 0x4000U ++#define A9_HO_LOG_OFFSET 0x800U ++#define A9_SL_MAGIC 0x4c533941U /* "A9SL" */ ++#define A9_SL_VERSION 1U ++ ++enum a9_ho_step { ++ A9_HO_STEP_BEGIN, ++ A9_HO_STEP_RATES, ++ A9_HO_STEP_AFK_SENT, ++ A9_HO_STEP_AFK_ACKED, ++ A9_HO_STEP_DISABLED, ++ A9_HO_STEP_AP_SENT, ++ A9_HO_STEP_AP_ACKED, ++ A9_HO_STEP_IOP_SENT, ++ A9_HO_STEP_IOP_ACKED, ++ A9_HO_STEP_RAM_STABLE, ++ A9_HO_STEP_MAILBOX_IDLE, ++ A9_HO_STEP_HALTED, ++ A9_HO_STEP_DRAINED, ++ A9_HO_STEPS, ++}; ++ ++static const char * const a9_ho_step_names[A9_HO_STEPS] = { ++ "begin", "rates-zero", "afk-sent", "afk-acked", "endpoint-disable", ++ "ap-sent", "ap-ack", "iop-sent", "iop-ack", "ram-stable", "mailbox-idle", ++ "halt", "drain", ++}; ++ ++struct a9_sl_header { ++ u32 magic, version, capacity, length; ++ u32 lines, truncated, checksum, mode; ++ u64 begin_ns; ++ u8 writer[16]; ++ char entry[64]; ++}; ++ ++_Static_assert(sizeof(struct a9_sl_header) == 120, "shutdown log ABI"); ++ ++#define A9_SL_CAPACITY (A9_HO_PAGE - A9_HO_LOG_OFFSET - sizeof(struct a9_sl_header)) ++ ++/* FNV-1a; incremental over appended text. */ ++static inline u32 a9_sl_hash(u32 hash, const char *text, size_t length) ++{ ++ size_t i; ++ ++ for (i = 0; i < length; i++) ++ hash = (hash ^ (u8)text[i]) * 16777619U; ++ return hash; ++} ++ ++#define A9_SL_HASH_SEED 2166136261U ++ ++/* A replayable previous transcript: complete header, bounded, intact. */ ++static inline bool a9_sl_valid(const struct a9_sl_header *h, const char *text) ++{ ++ return h->magic == A9_SL_MAGIC && h->version == A9_SL_VERSION && ++ h->capacity == A9_SL_CAPACITY && h->length <= h->capacity && ++ !h->entry[sizeof(h->entry) - 1] && ++ a9_sl_hash(A9_SL_HASH_SEED, text, h->length) == h->checksum; ++} ++ +diff --git a/drivers/soc/apple/aop-warm-spi0.c.h b/drivers/soc/apple/aop-warm-spi0.c.h +new file mode 100644 +index 000000000..bf883ffe8 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.c.h +@@ -0,0 +1,1340 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Ported from a9/aop-rc1: wake16, release6-d254a9f790. */ ++static void a9_aop_spi0_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_REGS; i++) ++ values[i] = readl(base + a9_spi0_offsets[i]); ++} ++ ++/* ++ * AOPSPIMaster_reset (0102f2f4) under the AOP-domain PMGR guard with IRQs ++ * off and the AOP CPU halted. Its provider power transition 3 is an ++ * AOP-local call with no AP register equivalent and is not replayed. A poll ++ * timeout still restores the saved control word. ++ */ ++static void a9_aop_spi0_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, s->before); ++ s->saved = s->before[A9_SPI0_I_CONTROL]; ++ s->asserted = a9_spi0_base_word(s->saved) | A9_SPI0_RESET; ++ writel(s->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ s->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(s->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(s->saved, control); ++ s->reset_done_ns = ktime_get_ns(); ++ a9_aop_spi0_snapshot(s->base, s->after); ++ if (!ret && (s->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ s->error = ret; ++} ++ ++/* ++ * Wake8 device evidence: SCM was 0 before/after the first controller reset, ++ * then became 3 at the channel-3 stop store. There is no firmware-derived ++ * SCM status write. Replay AOPSPIMaster_reset after the teardown so the ++ * controller reset, rather than an invented SCM write, is the one variable. ++ */ ++static void a9_aop_spi0_post_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, r->before); ++ r->saved = r->before[A9_SPI0_I_CONTROL]; ++ r->asserted = a9_spi0_base_word(r->saved) | A9_SPI0_RESET; ++ writel(r->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ r->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(r->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(r->saved, control); ++ a9_aop_spi0_snapshot(s->base, r->after); ++ if (!ret && (r->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ r->error = ret; ++} ++ ++static void a9_aop_spi0_ch3_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_CH3_REGS; i++) ++ values[i] = readl(base + a9_spi0_ch3_offsets[i]); ++} ++ ++static void a9_aop_spi0_trace_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int channel, i; ++ ++ s->trace.control = readl(s->base + A9_SPI0_CONTROL); ++ s->trace.clock = readl(s->base + A9_SPI0_CLOCK); ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->trace.scm[i] = readl(s->base + a9_spi0_scm_offsets[i]); ++ a9_aop_spi0_ch3_snapshot(s->base, s->trace.ch3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) ++ for (i = 0; i < A9_SPI0_CHANNEL_REGS; i++) ++ s->trace.channel[channel][i] = ++ readl(s->base + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel + ++ a9_spi0_channel_offsets[i]); ++} ++ ++/* Warm path only, CPU halted. Log outside the PMGR guard; no SCM writes. */ ++static int a9_aop_spi0_trace(struct apple_aop *aop, const char *stage) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *c = s->trace.ch3, *m = s->trace.scm; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_trace_read, s); ++ if (ret) { ++ s->intc.failed = "failed-spi0-trace-guard"; ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=%d: guard refused\n", stage, ret); ++ return ret; ++ } ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=0 ctl=%08x clk=%08x scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ stage, s->trace.control, s->trace.clock, m[0], m[1], m[2], m[3], ++ c[0], c[1], c[2], c[3], c[4], c[5], c[6]); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ v = s->trace.channel[channel]; ++ dev_info(aop->dev, "SPI0 SCM channel stage=%s ch=%02u ctl=%08x route=%08x rate=%08x cfg=%08x interval=%08x cursor=%08x desc=%08x pending0=%u pending1=%u\n", ++ stage, channel, v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ !!(c[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(c[A9_CH3_I_8D4] & BIT(2 * channel))); ++ } ++ return 0; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s); ++ ++/* ++ * Wake10 safety gate, strengthened in wake11 to retain every selected-channel ++ * teardown invariant. A fault observed by a copy sample remains terminal even ++ * if it is transient and the final read happens to clear. ++ */ ++static int a9_aop_spi0_gate(struct apple_aop *aop, const char *stage, ++ const char *check, bool include_copy) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *m, *c; ++ int ret; ++ ++ ret = a9_aop_spi0_trace(aop, stage); ++ if (ret) ++ return ret; ++ m = s->trace.scm; ++ c = s->trace.ch3; ++ if (m[0] || m[1]) ++ s->gate_failed = "failed-spi0-scm"; ++ else if (c[A9_CH3_I_8D0] || c[A9_CH3_I_8D4]) ++ s->gate_failed = "failed-spi0-fault"; ++ else if (include_copy && s->copy_faulted) ++ s->gate_failed = "failed-spi0-copy-sample"; ++ else { ++ s->gate_failed = a9_spi0_ch3_verify(c, s->teardown_stable & BIT(3)); ++ if (!s->gate_failed) ++ s->gate_failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = s->gate_failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 %s check errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x copy_faulted=%u result=%s\n", ++ check, ret, m[0], m[1], c[A9_CH3_I_8D0], c[A9_CH3_I_8D4], ++ s->copy_faulted, s->gate_failed ?: "pass"); ++ return ret; ++} ++ ++#define A9_SPI0_COPY_CHUNK 256U ++#define A9_SPI0_COPY_SAMPLE_US 5000U ++ ++struct a9_aop_spi0_copy_probe { ++ void __iomem *base; ++ u32 scm_lo, scm_hi, fault, fault_8d4; ++}; ++ ++static void a9_aop_spi0_copy_probe_read(void *data) ++{ ++ struct a9_aop_spi0_copy_probe *p = data; ++ ++ p->scm_lo = readl(p->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(p->base + A9_SPI0_SCM_HI); ++ p->fault = readl(p->base + A9_SPI0_FAULT); ++ p->fault_8d4 = readl(p->base + A9_SPI0_FAULT_8D4); ++} ++ ++/* ++ * Read-only samples spaced by at least 5 ms across both halves of the SRAM ++ * copy. Once a nonzero state is seen it remains a terminal fact, but the copy ++ * completes so later samples and the full after-copy channel snapshot expose ++ * whether the source is level, transient or moves between channels. ++ */ ++static int a9_aop_spi0_copy_sample(struct apple_aop *aop, const char *phase, ++ size_t offset, size_t size, u64 start_ns, ++ u64 *last_us, unsigned int *sample, ++ bool force) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_copy_probe p = { .base = s->base }; ++ u64 elapsed_us = div_u64(ktime_get_boottime_ns() - start_ns, ++ NSEC_PER_USEC); ++ bool first_fault, initial = !*sample; ++ int ret; ++ ++ if (!force && *sample && elapsed_us - *last_us < A9_SPI0_COPY_SAMPLE_US) ++ return 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_copy_probe_read, &p); ++ first_fault = !s->copy_faulted && ++ (p.scm_lo || p.scm_hi || p.fault || p.fault_8d4); ++ if (!ret && first_fault) ++ s->copy_faulted = true; ++ dev_info(aop->dev, "SPI0 COPY sample=%u phase=%s elapsed_us=%llu offset=0x%zx size=0x%zx scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ *sample, phase, elapsed_us, offset, size, p.scm_lo, p.scm_hi, ++ p.fault, p.fault_8d4, ret ? "guard" : s->copy_faulted ? "fault" : "pass"); ++ *last_us = elapsed_us; ++ (*sample)++; ++ if (ret) ++ return ret; ++ if (first_fault) { ++ ret = a9_aop_spi0_trace(aop, "copy-fault"); ++ if (ret) ++ return ret; ++ } ++ /* The offset-zero sample is the last no-write gate before the copy. */ ++ return !initial || !s->copy_faulted ? 0 : -EIO; ++} ++ ++static int a9_aop_copy_verify_wake10(struct apple_aop *aop, const u8 *src, ++ size_t size) ++{ ++ u32 chunk[A9_SPI0_COPY_CHUNK / 4]; ++ u64 start_ns = ktime_get_boottime_ns(), last_us = 0; ++ unsigned int sample = 0; ++ size_t done, n; ++ int ret; ++ ++ ret = a9_aop_spi0_copy_sample(aop, "write", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __iowrite32_copy(aop->sram + done, src + done, n / 4); ++ ret = a9_aop_spi0_copy_sample(aop, "write", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_copy_sample(aop, "verify", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __ioread32_copy(chunk, aop->sram + done, n / 4); ++ if (memcmp(chunk, src + done, n)) ++ return -EIO; ++ ret = a9_aop_spi0_copy_sample(aop, "verify", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void __iomem *a9_aop_spi0_channel_base(struct a9_aop_spi0 *s, ++ unsigned int channel) ++{ ++ return s->base + A9_SPI0_CHANNEL_BASE + A9_SPI0_CHANNEL_STRIDE * channel; ++} ++ ++/* ++ * Wake15: which other channels were RUN with a counting interval when this ++ * stop was issued, and which of them advanced across the stop. Read only; it ++ * separates a stop that had a ticking peer from the stop of the last live ++ * channel in the log. It never gates anything in-kernel. ++ */ ++static void a9_aop_spi0_peers_begin(struct a9_aop_spi0 *s, unsigned int channel, ++ u32 *interval, u32 *run) ++{ ++ unsigned int i; ++ u32 ctl; ++ ++ *run = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ interval[i] = 0; ++ if (i == channel) ++ continue; ++ ctl = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ interval[i] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ if ((ctl & A9_SPI0_CH3_RUN) && interval[i]) ++ *run |= BIT(i); ++ } ++} ++ ++static void a9_aop_spi0_stop_timing(struct a9_aop_spi0 *s, ++ struct a9_aop_spi0_ch3 *c, ++ const u32 *interval) ++{ ++ unsigned int i; ++ ++ c->peers_ticking = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ if ((c->peers_run & BIT(i)) && ++ readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]) != ++ interval[i]) ++ c->peers_ticking |= BIT(i); ++ } ++ c->reset_to_stop_us = c->stop_ns > s->reset_done_ns ? ++ div_u64(c->stop_ns - s->reset_done_ns, NSEC_PER_USEC) : 0; ++} ++ ++/* ++ * Wake12/13 captured channel 7, and wake13 channel 3, stopped but never idle: ++ * for 20 complete slices the control/progress word, config, interval, cursor, ++ * SCM and both fault banks were unchanged. This predicate is intentionally ++ * narrower than "RUN clear and no progress": the channel must be one of the ++ * two device-observed periodic clients, must carry that client's route, rate ++ * and config before the stop, and every sample must be byte-identical with SCM ++ * 3/0 (every earlier non-idle stop, which followed a channel-3 stop that had ++ * latched it) or 0/0 (a first stop that latches nothing), no fault, no cursor ++ * movement and a retained nonzero interval. ++ * It only permits the callback masks, descriptor clear, config acknowledge and ++ * the established controller reset; every later SCM, fault and copy gate stays. ++ */ ++static const char *a9_aop_spi0_stable_why(struct a9_aop_spi0_ch3 *c, ++ const struct a9_aop_channel_diag_slice *diag, ++ unsigned int channel) ++{ ++ const struct a9_aop_ch3_slice *first; ++ unsigned int i; ++ u32 route; ++ ++ if (channel == A9_SPI0_PHOSPHORUS_CHANNEL) ++ route = A9_SPI0_PHOSPHORUS_ROUTE; ++ else if (channel == A9_SPI0_MPU_ASYNC_CHANNEL) ++ route = A9_SPI0_MPU_ASYNC_ROUTE; ++ else ++ return "channel"; ++ if (c->poll != -ETIMEDOUT || c->slices != A9_SPI0_CH3_SLICES) ++ return "no-timeout"; ++ if (!(c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) || ++ c->before[A9_CH3_I_ROUTE] != route || ++ c->before[A9_CH3_I_RATE] != A9_SPI0_PHOSPHORUS_RATE || ++ c->before[A9_CH3_I_C3C] != A9_SPI0_PHOSPHORUS_CFG || ++ c->before[A9_CH3_I_DESC] || (c->stop & A9_SPI0_CH3_RUN)) ++ return "before-identity"; ++ if (c->before[A9_CH3_I_8D4] || (c->before[A9_CH3_I_8D0] & ~BIT(2 * channel))) ++ return "before-fault"; ++ ++ first = &c->slice[0]; ++ if ((first->c3 & A9_SPI0_STABLE_CTL_MASK) != A9_SPI0_STABLE_CTL_STOPPED || ++ (first->c3 & (A9_SPI0_CH3_RUN | A9_SPI0_CH3_IDLE | A9_SPI0_CH3_IRQ1))) ++ return "stopped-control"; ++ if ((first->scm_lo != 0 && first->scm_lo != 3) || first->scm_hi) ++ return "scm"; ++ if (first->fault || diag[0].fault_8d4) ++ return "sample-fault"; ++ if (diag[0].cfg != A9_SPI0_PHOSPHORUS_CFG || diag[0].cursor || !diag[0].interval) ++ return "cfg-cursor-interval"; ++ ++ for (i = 1; i < A9_SPI0_CH3_SLICES; i++) { ++ const struct a9_aop_ch3_slice *p = &c->slice[i]; ++ ++ if (p->c3 != first->c3 || p->scm_lo != first->scm_lo || ++ p->scm_hi != first->scm_hi || p->fault != first->fault || ++ diag[i].cfg != diag[0].cfg || ++ diag[i].interval != diag[0].interval || ++ diag[i].cursor != diag[0].cursor || ++ diag[i].fault_8d4 != diag[0].fault_8d4) ++ return "progress"; ++ } ++ if (c->slice[A9_SPI0_CH3_SLICES - 1].ns < ++ (u64)A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US * NSEC_PER_USEC) ++ return "short"; ++ return NULL; ++} ++ ++/* ++ * AOPSPIMaster_teardownChannel (0102e4f8) for channel 3, then the fault ++ * handler's acknowledge (0102f20a), under the same guard as the reset with ++ * the CPU halted. Every store is a read-modify-write except the descriptor, ++ * which the firmware zeroes. The idle wait runs in 1 ms slices, up to twice ++ * the firmware's own 10 ms SCMto bound; each slice ends with read-only ++ * samples of C3, the SCM state and +0x8d0 (wake15: and config, interval, ++ * cursor and +0x8d4). An idle timeout stops before the masks unless the ++ * stable-stop predicate holds. The acknowledge bit self-clears; +0x8d0 bit 6 ++ * shows whether it took effect. ++ */ ++static void a9_aop_spi0_ch3_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ void __iomem *c3 = s->base + A9_SPI0_CH3; ++ void __iomem *c3c = s->base + A9_SPI0_CH3_0C; ++ void __iomem *base = a9_aop_spi0_channel_base(s, 3); ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_ch3_snapshot(s->base, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(c3) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, c3); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, 3, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(c3, value, value & A9_SPI0_CH3_IDLE, ++ 1, A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->ch3_diag[c->slices]; ++ d->cfg = readl(c3c); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->ch3_diag, 3); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(3); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(c3) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, c3); ++ c->class1 = readl(c3) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, c3); ++ writel(0, s->base + A9_SPI0_CH3_DESC); ++ c->ack = readl(c3c) | A9_SPI0_CH3_ACK; ++ writel(c->ack, c3c); ++ } ++ a9_aop_spi0_ch3_snapshot(s->base, c->after); ++ c->failed = c->poll && !c->stable ? "failed-ch3-teardown-idle" : ++ a9_spi0_ch3_verify(c->after, c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++ ++static void a9_aop_spi0_channel_snapshot(struct a9_aop_spi0 *s, ++ unsigned int channel, u32 *values) ++{ ++ void __iomem *base = a9_aop_spi0_channel_base(s, channel); ++ ++ values[A9_CH3_I_C3] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ values[A9_CH3_I_ROUTE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_ROUTE]); ++ values[A9_CH3_I_RATE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_RATE]); ++ values[A9_CH3_I_C3C] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]); ++ values[A9_CH3_I_DESC] = readl(base + A9_SPI0_CHANNEL_DESC); ++ values[A9_CH3_I_8D0] = readl(s->base + A9_SPI0_FAULT); ++ values[A9_CH3_I_8D4] = readl(s->base + A9_SPI0_FAULT_8D4); ++} ++ ++static const char *a9_spi0_channel_state_verify(u32 control, u32 desc, ++ u32 fault, u32 fault_8d4, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ u32 pending = BIT(2 * channel); ++ ++ if (control & A9_SPI0_CH3_RUN) ++ return "failed-channel-teardown-run"; ++ if (control & A9_SPI0_CH3_IRQ0) ++ return "failed-channel-teardown-class0"; ++ if (control & A9_SPI0_CH3_IRQ1) ++ return "failed-channel-teardown-class1"; ++ if (!(control & A9_SPI0_CH3_IDLE) && !stable_stop) ++ return "failed-channel-teardown-idle"; ++ if (desc) ++ return "failed-channel-teardown-desc"; ++ if ((fault | fault_8d4) & pending) ++ return "failed-channel-teardown-pending"; ++ return NULL; ++} ++ ++static const char *a9_spi0_channel_verify(const u32 *after, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ return a9_spi0_channel_state_verify(after[A9_CH3_I_C3], ++ after[A9_CH3_I_DESC], ++ after[A9_CH3_I_8D0], ++ after[A9_CH3_I_8D4], channel, ++ stable_stop); ++} ++ ++/* ++ * Wake11 applies the channel-3 firmware sequence unchanged to another slot. ++ * Wake12 only appends read-only scheduler samples to that established poll. ++ */ ++static void a9_aop_spi0_channel_locked(void *data) ++{ ++ struct a9_aop_spi0_channel_op *op = data; ++ struct a9_aop_spi0 *s = op->spi0; ++ struct a9_aop_spi0_ch3 *c = op->state; ++ void __iomem *base = a9_aop_spi0_channel_base(s, op->channel); ++ void __iomem *control = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]; ++ void __iomem *config = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]; ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(control) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, control); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, op->channel, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(control, value, ++ value & A9_SPI0_CH3_IDLE, 1, ++ A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->channel_diag[c->slices]; ++ d->cfg = readl(config); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->channel_diag, op->channel); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(op->channel); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(control) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, control); ++ c->class1 = readl(control) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, control); ++ writel(0, base + A9_SPI0_CHANNEL_DESC); ++ c->ack = readl(config) | A9_SPI0_CH3_ACK; ++ writel(c->ack, config); ++ } ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->after); ++ c->failed = c->poll && !c->stable ? "failed-channel-teardown-idle" : ++ a9_spi0_channel_verify(c->after, op->channel, ++ c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++/* Called only if ps_spi0 is fully on; reads nothing. */ ++static void a9_aop_spi0_pmgr_on(void *data) ++{ ++ *(bool *)data = true; ++} ++ ++/* AP PMGR ps_spi0, read-only: never requested, resumed or written. ++ * 0 = on, -EHOSTDOWN = off (raw still filled), -ENODEV = no node. ++ */ ++static int a9_aop_spi0_pmgr(struct apple_aop *aop, u32 *raw) ++{ ++ struct apple_pmgr_observation state = {}; ++ bool on = false; ++ int ret; ++ ++ if (!aop->spi0_pmgr) ++ return -ENODEV; ++ ret = apple_pmgr_observe(&aop->spi0_pmgr, 1, &state, ++ a9_aop_spi0_pmgr_on, &on); ++ *raw = state.raw; ++ return ret ?: on ? 0 : -EIO; ++} ++ ++/* ++ * After a reset that cleared +0x8d0 bit 6, before the image copy: CPU halted, ++ * AP notifications masked. A refusal sets ch3.failed and nothing is loaded. ++ */ ++static int a9_aop_spi0_ch3_teardown(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *b = c->before, *a = c->after; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ unsigned int i; ++ bool source_progress = false; ++ int ret; ++ ++ aop->experiment_step = "spi0-ch3-teardown"; ++ dev_info(aop->dev, "SPI0 CH3TD begin pa=%llx cpu=halted: first channel-3 store follows\n", ++ A9_SPI0_PA + A9_SPI0_CH3); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, "spi0-ch3-teardown"); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_ch3_locked, s); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH3TD errno=%d ran=0: guard refused, channel 3 not touched\n", ++ ret); ++ return ret; ++ } ++ c->ran = true; ++ dev_info(aop->dev, "SPI0 CH3TD before c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH3TD idle-sample slice=%u at_us=%llu c3=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ i, div_u64(p->ns, NSEC_PER_USEC), p->c3, p->scm_lo, ++ p->scm_hi, p->fault, !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->ch3_diag[i]; ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ i, p->c3, d->cfg, d->interval, d->cursor, p->scm_lo, ++ p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->ch3_diag[0].interval || ++ d->cursor != s->ch3_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->ch3_diag[0].interval, ++ s->ch3_diag[c->slices - 1].interval, ++ s->ch3_diag[0].cursor, ++ s->ch3_diag[c->slices - 1].cursor, source_progress, ++ c->stable ? "stable-stop-reset-only" : "refused"); ++ /* beyond_fw_bound=1: idle, but later than the firmware would wait. */ ++ dev_info(aop->dev, "SPI0 CH3TD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ c->stop, c->idle, c->poll); ++ else if (c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->poll, c->class0, c->class1, c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH3TD after c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH3TD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ /* c3c_bit5 is the pulse's readback, logged only (0 at 19:50). */ ++ dev_info(aop->dev, "SPI0 CH3TD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x c3c_bit5=%u ch3_pending=%u result=%s\n", ++ c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), ++ a[A9_CH3_I_DESC], !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), c->failed ?: ++ c->stable ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static int a9_aop_spi0_channel_teardown(struct apple_aop *aop, ++ unsigned int channel) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->channel; ++ struct a9_aop_spi0_channel_op op = { ++ .spi0 = s, ++ .state = c, ++ .channel = channel, ++ }; ++ const u32 *b, *a; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ char checkpoint[32]; ++ unsigned int i; ++ bool source_progress = false; ++ bool stable_stop; ++ int ret; ++ ++ memset(c, 0, sizeof(*c)); ++ memset(s->channel_diag, 0, sizeof(s->channel_diag)); ++ s->teardown_failed_channel = channel; ++ aop->experiment_step = "spi0-channel-teardown"; ++ snprintf(checkpoint, sizeof(checkpoint), "spi0-ch%u-teardown", channel); ++ dev_info(aop->dev, "SPI0 CH%uTD begin pa=%llx cpu=halted: first channel-%u store follows\n", ++ channel, A9_SPI0_PA + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel, channel); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, checkpoint); ++ if (ret) { ++ c->failed = "failed-channel-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_channel_locked, &op); ++ if (ret) { ++ c->failed = "failed-channel-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH%uTD errno=%d ran=0: guard refused, channel %u not touched\n", ++ channel, ret, channel); ++ return ret; ++ } ++ c->ran = true; ++ stable_stop = s->teardown_stable & BIT(channel); ++ b = c->before; ++ a = c->after; ++ dev_info(aop->dev, "SPI0 CH%uTD before ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD idle-sample slice=%u at_us=%llu ctl=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ channel, i, div_u64(p->ns, NSEC_PER_USEC), p->c3, ++ p->scm_lo, p->scm_hi, p->fault, ++ !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->channel_diag[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ channel, i, p->c3, d->cfg, d->interval, d->cursor, ++ p->scm_lo, p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->channel_diag[0].interval || ++ d->cursor != s->channel_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ channel, c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->channel_diag[0].interval, ++ s->channel_diag[c->slices - 1].interval, ++ s->channel_diag[0].cursor, ++ s->channel_diag[c->slices - 1].cursor, ++ source_progress, ++ stable_stop ? "stable-stop-reset-only" : "refused"); ++ dev_info(aop->dev, "SPI0 CH%uTD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ channel, c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ channel, c->stop, c->idle, c->poll); ++ else if (stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->poll, c->class0, c->class1, ++ c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH%uTD after ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH%uTD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ channel, c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ dev_info(aop->dev, "SPI0 CH%uTD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x cfg_bit5=%u pending0=%u pending1=%u result=%s\n", ++ channel, c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), a[A9_CH3_I_DESC], ++ !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(a[A9_CH3_I_8D4] & BIT(2 * channel)), c->failed ?: ++ stable_stop ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s) ++{ ++ const u32 *v; ++ const char *failed = NULL; ++ unsigned int channel; ++ ++ if (s->teardown_selected != s->teardown_done) ++ return "failed-channel-teardown-incomplete"; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ v = s->trace.channel[channel]; ++ failed = a9_spi0_channel_state_verify(v[A9_CHANNEL_I_CTL], ++ v[A9_CHANNEL_I_DESC], ++ s->trace.ch3[A9_CH3_I_8D0], ++ s->trace.ch3[A9_CH3_I_8D4], channel, ++ s->teardown_stable & BIT(channel)); ++ if (failed) { ++ s->teardown_failed_channel = channel; ++ return failed; ++ } ++ } ++ return NULL; ++} ++ ++/* ++ * Channel 3 remains mandatory and executes first. Wake11 additionally selects ++ * every snapshot slot with run bit 0 (RUN) set and a nonzero interval, which ++ * excludes configured-but-idle slots while selecting any periodic client still ++ * owned by the old firmware. Wake16 returns to wake11's order after wake15's ++ * channel-7-first stop failed: all ten device stops fit "channel 3 idles when ++ * stopped within ~2 ms of the reset (5/5; 0/2 at 11 and 33 ms) and channel 7 ++ * never idles (0/3, at 0.3 ms, 1.9 ms and 15 ms)". Channel 3 goes first and ++ * early; channel 7 is then the stopped-without-idle channel the stable-stop ++ * fingerprint was captured from (wake11/12). ++ */ ++static int a9_aop_spi0_teardown_running(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ s->teardown_selected = BIT(3); ++ s->teardown_done = 0; ++ s->teardown_stable = 0; ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3) ++ continue; ++ v = s->trace.channel[channel]; ++ if ((v[A9_CHANNEL_I_CTL] & A9_SPI0_CH3_RUN) && ++ v[A9_CHANNEL_I_INTERVAL]) ++ s->teardown_selected |= BIT(channel); ++ } ++ dev_info(aop->dev, "SPI0 teardown select stage=after-controller-reset selected=%04x mandatory=0008 rule=ch3-or-run-and-interval order=ch3-then-others\n", ++ s->teardown_selected); ++ ret = a9_aop_spi0_ch3_teardown(aop); ++ if (ret) { ++ s->teardown_failed_channel = 3; ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=3 errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, ret, ++ s->ch3.failed ?: "failed-ch3-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ ret = a9_aop_spi0_channel_teardown(aop, channel); ++ if (ret) { ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=%u errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, channel, ret, ++ s->channel.failed ?: "failed-channel-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(channel); ++ } ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ dev_info(aop->dev, "SPI0 teardown complete selected=%04x done=%04x stable=%04x result=pass\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable); ++ return 0; ++} ++ ++/* ++ * Second copy of the firmware controller reset, after the stop has completed. ++ * The preceding after-ch3-teardown trace is the read-only before snapshot; ++ * after-post-teardown-reset is the matching read-only after snapshot. Refuse ++ * the image copy unless SCM is zero and every channel quiet-state gate holds. ++ */ ++static int a9_aop_spi0_post_reset(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ const u32 *b = r->before, *a = r->after; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.ran || ++ s->ch3.error || s->teardown_selected != s->teardown_done || r->ran) { ++ r->failed = "failed-post-reset-state"; ++ return -EPERM; ++ } ++ aop->experiment_step = "spi0-post-teardown-reset"; ++ dev_info(aop->dev, "SPI0 POSTRST begin pa=%llx reason=ch3-stop-latched-scm cpu=halted\n", ++ A9_SPI0_PA); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-post-teardown-reset"); ++ if (ret) { ++ r->failed = "failed-post-reset-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_post_reset_locked, s); ++ r->ran = !ret; ++ r->error = ret ?: r->error; ++ dev_info(aop->dev, "SPI0 POSTRST errno=%d ran=%d before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ r->error, r->ran, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 POSTRST after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], r->saved, r->asserted, ++ r->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ if (r->error) { ++ r->failed = "failed-post-reset-controller"; ++ return r->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-post-teardown-reset"); ++ if (ret) ++ return ret; ++ if (s->trace.scm[0] || s->trace.scm[1]) ++ r->failed = "failed-post-reset-scm"; ++ else { ++ r->failed = a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3)); ++ if (!r->failed) ++ r->failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = r->failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 POSTRST check errno=%d scm_lo=%08x scm_hi=%08x ch3_pending=%u result=%s\n", ++ ret, s->trace.scm[0], s->trace.scm[1], ++ !!(s->trace.ch3[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), ++ r->failed ?: "pass"); ++ dev_info(aop->dev, "SPI0 POSTRST channels selected=%04x done=%04x stable=%04x result=%s\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable, ++ r->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * The ten INTC words of a9_intc_offsets into state[@at], read only. The ++ * INTC is the AOP wrapper block (0xc1800000 in the firmware, space 2 as ++ * SPI0's verified 0xc1100000 = 0x210100000), which this driver reads ++ * unguarded for the mailbox and CPU control; no separate PMGR domain. ++ */ ++static void a9_aop_intc_snapshot(struct apple_aop *aop, unsigned int at) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ u32 *v = n->state[at]; ++ unsigned int i; ++ ++ for (i = 0; i < A9_INTC_REGS; i++) ++ v[i] = aop_rd(aop, a9_intc_offsets[i]); ++ n->states |= BIT(at); ++ dev_info(aop->dev, "AOP INTC state at=%s reset808=%08x current81c=%08x raise900=%08x ack980=%08x mask_a00=%08x mask_a04=%08x mask_a08=%08x mask_a80=%08x g7_c020=%08x g7_c024=%08x src_masked=%u view=%s\n", ++ a9_intc_at_name(at), v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ v[7], v[8], v[9], !!(v[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(v)); ++} ++ ++/* ++ * Round 4 (ds-aop-spi0-ch3-round4.md): the old session's SPI0 global IRQ ++ * 0x10014 stays latched in the AOP interrupt controller, and fresh firmware ++ * takes it as soon as SPI lifecycle event 1 enables it, before its own SPI0 ++ * reset (SCMErr:0x3,3). With the CPU halted and before the first SPI0 ++ * access, disable that source (group 1 disable bank, 0102c9ce). The bank ++ * reads back the mask, as the firmware's own reads of it show: bit 20 must ++ * read 1 after the store. ++ */ ++static int a9_aop_intc_disable(struct apple_aop *aop) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ const u32 *b = n->state[A9_INTC_AT_PRE_DISABLE]; ++ const u32 *a = n->state[A9_INTC_AT_POST_DISABLE]; ++ int ret; ++ ++ aop->experiment_step = "intc-disable"; ++ dev_info(aop->dev, "AOP INTC disable begin irq=0x10014 pa=%llx value=%08x cpu=halted: first INTC store follows\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "intc-disable"); ++ if (ret) { ++ n->failed = "failed-intc-disable-checkpoint"; ++ return ret; ++ } ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_DISABLE); ++ ret = aop_wr(aop, A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_DISABLE); ++ if (ret) { ++ n->failed = "failed-intc-disable"; ++ } else if (!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM)) { ++ n->failed = "failed-intc-disable-readback"; ++ ret = -EIO; ++ } ++ n->disabled = !ret; ++ dev_info(aop->dev, "AOP INTC disable errno=%d before=%08x value=%08x after=%08x mask_a80=%08x src_masked=%u view=%s result=%s\n", ++ ret, b[A9_INTC_I_MASK0], A9_INTC_SPI0_SCM, a[A9_INTC_I_MASK0], ++ a[A9_INTC_I_ENABLE0], !!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(a), n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Release policy for aop_spi0_reset=1. The reload is device-proven for exactly ++ * one retained state: the previous kernel quiesced and halted the firmware ++ * (wake16, 2026-10-29; decision prev-quiesce-halted). Auto-boot still set, a ++ * previous kernel that did not halt it, or an older kernel in the chain are ++ * different hardware states that no run has reloaded with the final teardown. ++ * Refuse them before the first halt or SPI0 access so the old firmware is ++ * left exactly as found; aop_spi0_untested=1 attempts them anyway. A skip ++ * (DFU cold, bad record, nothing ran) must also find the wrapper at its reset ++ * state: a record-less AOP that something already stopped is refused rather ++ * than cold-loaded. A stale record skips as before. ++ */ ++static int a9_aop_spi0_policy(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ enum a9_spi0_reason reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, ++ &aop->spi0.prev); ++ ++ if (a9_spi0_policy_allows(reason, outbox & AOP_OUTBOX_ENABLE, spi0_untested)) ++ return 0; ++ dev_info(aop->dev, "SPI0 policy decision=refuse reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d: only prev-quiesce-halted reloads and a skip needs the wrapper at its reset state block not touched\n", ++ a9_spi0_reason_name(reason), auto_boot, outbox, ++ a9_spi0_prev_name(aop->spi0.prev.state), aop->spi0.prev.hello, ++ aop->spi0.prev.halted); ++ return -EPERM; ++} ++ ++/* ++ * After the proven halt/drain, before any SRAM write. Only firmware that ran ++ * since the last SoC reset leaves SPI0 state behind: auto-boot 1, or the ++ * previous kernel's transcript bound by +0x4020 (a9_spi0_decide). Without ++ * that evidence the block is never touched. @auto_boot and @outbox are the ++ * boot snapshot from before the stop. Nonzero means: do not load or ++ * release the AOP. ++ */ ++static int a9_aop_spi0_prereset(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *b = s->before, *a = s->after; ++ const char *failed = NULL; ++ bool run; ++ int ret; ++ ++ s->auto_boot = auto_boot; ++ s->outbox = outbox; ++ s->reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, &s->prev); ++ s->decided = true; ++ run = a9_spi0_reason_runs(s->reason); ++ dev_info(aop->dev, "SPI0 reset decision=%s reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d prev_steps=0x%x prev_carry=%d%s\n", ++ run ? "run" : "skip", a9_spi0_reason_name(s->reason), auto_boot, ++ outbox, a9_spi0_prev_name(s->prev.state), s->prev.hello, ++ s->prev.halted, s->prev.steps, s->prev.carried, ++ run ? "" : ": block not touched"); ++ if (!run) { ++ s->skipped = a9_spi0_reason_name(s->reason); ++ return 0; ++ } ++ if (!s->base || !aop->force_halted || !aop->force_drained || ++ !a9_aop_force_stopped(aop)) ++ return -EPERM; ++ ret = a9_aop_intc_disable(aop); ++ if (ret) ++ return ret; ++ aop->experiment_step = "spi0-reset"; ++ s->pmgr_before_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_before); ++ /* Reaches ramoops before the first AP access to the block. */ ++ dev_info(aop->dev, "SPI0 reset begin pa=%llx cpu=halted ps_spi0=%08x errno=%d: first AP access follows\n", ++ A9_SPI0_PA, s->pmgr_before, s->pmgr_before_errno); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-reset"); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_trace(aop, "before-controller-reset"); ++ if (ret) ++ return ret; ++ /* ++ * Wake15: from the controller reset to the last channel stop nothing may ++ * drain a console in this task. Holding the console lock makes every ++ * dev_info() below only append to the log buffer; the flushed spi0-reset ++ * checkpoint above already put the begin line into ramoops, and the lines ++ * are emitted by console_unlock() once the stops are stored. ++ */ ++ console_lock(); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_reset_locked, s); ++ s->ran = !ret; ++ s->error = ret ?: s->error; ++ s->pmgr_after_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_after); ++ dev_info(aop->dev, "SPI0 reset errno=%d ran=%d pa=%llx before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ s->error, s->ran, A9_SPI0_PA, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 reset after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u provider3=not-replayed\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], s->saved, s->asserted, ++ s->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 pmgr ps_spi0 before=%08x errno=%d after=%08x errno=%d (read-only, never powered)\n", ++ s->pmgr_before, s->pmgr_before_errno, s->pmgr_after, ++ s->pmgr_after_errno); ++ if (s->error) { ++ console_unlock(); ++ return s->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-controller-reset"); ++ if (!ret) ++ ret = a9_aop_spi0_teardown_running(aop); ++ console_unlock(); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-ch3-teardown"); ++ if (!ret) ++ failed = a9_aop_spi0_selected_verify(s); ++ if (failed) { ++ s->channel.failed = failed; ++ ret = -EIO; ++ } ++ if (!ret) ++ ret = a9_aop_spi0_post_reset(aop); ++ return ret; ++} ++ ++/* Wake10 samples both SCM words and both fault classes, not only channel 3. */ ++static void a9_aop_spi0_fault_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ ++ s->prerun_scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ s->prerun_scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ s->prerun = readl(s->base + A9_SPI0_FAULT); ++ s->prerun_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ a9_aop_spi0_ch3_snapshot(s->base, s->ch3.reassert); ++} ++ ++/* ++ * Last AP look before RUN, CPU still halted, after the image copy: bit 6 ++ * must stay clear in every one of 31 guarded samples spaced >= 5 ms apart, ++ * spanning >= 150 ms. A reasserted channel-3 bit would crash the fresh ++ * image as before, so the first set sample refuses the release; nothing is ++ * retried. Any refusal sets ch3.failed. ++ */ ++static int a9_aop_spi0_recheck(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *r = c->reassert; ++ u64 start = ktime_get_boottime_ns(); ++ unsigned int i; ++ int ret = 0; ++ ++ for (i = 0; i < A9_SPI0_WINDOW_SAMPLES; i++) { ++ if (i) ++ usleep_range(A9_SPI0_WINDOW_US, A9_SPI0_WINDOW_US + 1000); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_fault_read, s); ++ c->samples = i + 1; ++ c->elapsed_ms = div_u64(ktime_get_boottime_ns() - start, NSEC_PER_MSEC); ++ if (ret) { ++ c->failed = "failed-ch3-window-guard"; ++ break; ++ } ++ if (s->prerun_scm_lo || s->prerun_scm_hi || s->prerun || ++ s->prerun_8d4) { ++ s->reasserted = c->reasserted = true; ++ c->reassert_sample = i; ++ c->reassert_ms = c->elapsed_ms; ++ c->failed = "failed-spi0-window-reasserted"; ++ ret = -EIO; ++ dev_info(aop->dev, "SPI0 WINDOW reasserted sample=%u at_ms=%llu scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ i, c->reassert_ms, s->prerun_scm_lo, s->prerun_scm_hi, ++ s->prerun, s->prerun_8d4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ dev_info(aop->dev, "SPI0 WINDOW channel3 c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ r[0], r[1], r[2], r[3], r[4], r[5], r[6]); ++ break; ++ } ++ } ++ /* usleep_range() cannot return early; this only trips on a bad clock. */ ++ if (!ret && c->elapsed_ms < A9_SPI0_WINDOW_MS) { ++ c->failed = "failed-ch3-window-short"; ++ ret = -EIO; ++ } ++ c->window_done = !ret; ++ dev_info(aop->dev, "SPI0 CH3TD window errno=%d samples=%u/%u elapsed_ms=%llu min_ms=%u result=%s\n", ++ ret, c->samples, A9_SPI0_WINDOW_SAMPLES, c->elapsed_ms, ++ A9_SPI0_WINDOW_MS, ret ? c->failed : "pass"); ++ dev_info(aop->dev, "SPI0 pre-run errno=%d 8d0=%08x ch3_pending=%u cpu=halted\n", ++ ret, s->prerun, !!(s->prerun & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 WINDOW state errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ ret, s->prerun_scm_lo, s->prerun_scm_hi, s->prerun, ++ s->prerun_8d4, ret ? c->failed : "pass"); ++ if (ret && c->reasserted) ++ a9_aop_spi0_trace(aop, "window-fault"); ++ return ret; ++} ++ ++/* SCM state and the channel-3 interval pair, read-only, into scm[scm_at]. */ ++static void a9_aop_spi0_scm_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->intc.scm[s->intc.scm_at][i] = readl(s->base + a9_spi0_scm_offsets[i]); ++} ++ ++/* ++ * After the clean pre-RUN window, CPU still halted: the firmware's group-1 ++ * pre-handler acknowledge for 0x10014 (0102c6de..0102c6f2) discards the ++ * old-session latch now that its SPI0 cause is quiet. Fresh firmware ++ * re-enables the source in event 1. The INTC words, the SCM state and the ++ * C3 interval pair are logged around it and never gated: with 0x10014 ++ * masked, no readable word shows whether it is still pending. ++ */ ++static int a9_aop_intc_ack(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ const u32 *p = n->scm[0], *q = n->scm[1]; ++ const u32 *b = n->state[A9_INTC_AT_PRE_ACK]; ++ const u32 *a = n->state[A9_INTC_AT_POST_ACK]; ++ int ret; ++ ++ aop->experiment_step = "intc-ack"; ++ n->scm_at = 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) { ++ n->failed = "failed-intc-ack-guard"; ++ dev_info(aop->dev, "AOP INTC pre-ack errno=%d: guard refused, no acknowledge\n", ++ ret); ++ return ret; ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_ACK); ++ dev_info(aop->dev, "AOP INTC pre-ack scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x cpu=halted\n", ++ p[0], p[1], p[2], p[3], b[A9_INTC_I_CURRENT]); ++ ret = aop_wr(aop, A9_INTC_ACK, A9_INTC_SPI0_SCM); ++ /* Also the posted-write barrier. */ ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_ACK); ++ n->acked = !ret; ++ dev_info(aop->dev, "AOP INTC ack errno=%d before=%08x value=%08x after=%08x\n", ++ ret, b[A9_INTC_I_ACK], A9_INTC_SPI0_SCM, a[A9_INTC_I_ACK]); ++ if (ret) { ++ n->failed = "failed-intc-ack"; ++ return ret; ++ } ++ n->scm_at = 1; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) ++ n->failed = "failed-intc-post-guard"; ++ dev_info(aop->dev, "AOP INTC post-ack errno=%d scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x result=%s\n", ++ ret, q[0], q[1], q[2], q[3], a[A9_INTC_I_CURRENT], n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Wake7 still took 0x10014 before SPI event 5 despite the targeted ack. ++ * Replay INTC lifecycle event 3 (0102c592/598/59c/5a2): 1/read/0/read at ++ * +0x808. aop_wr includes a posted-write read after each store. This resets ++ * all AOP-private interrupt state; it does not write the SCM status or MPU. ++ * No assumption that SCM must become zero. Only the final reset readback ++ * and the established channel-3 quiet conditions gate RUN. ++ */ ++static int a9_aop_intc_reset(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.window_done || ++ !n->acked || n->reset_done || ++ (aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) || readl(aop->control)) { ++ n->failed = "failed-intc-reset-state"; ++ return -EPERM; ++ } ++ ret = a9_aop_spi0_trace(aop, "pre-intc-reset"); ++ if (ret) ++ return ret; ++ aop->experiment_step = "intc-reset"; ++ dev_info(aop->dev, "AOP INTC reset begin pa=%llx sequence=1/read/0/read cpu=halted\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_RESET); ++ ret = aop_wr(aop, A9_INTC_RESET, 1); ++ n->reset_asserted = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-assert"; ++ return ret; ++ } ++ ret = aop_wr(aop, A9_INTC_RESET, 0); ++ n->reset_after = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-deassert"; ++ return ret; ++ } ++ if (n->reset_after) { ++ n->failed = "failed-intc-reset-readback"; ++ ret = -EIO; ++ } else { ++ ret = a9_aop_spi0_trace(aop, "post-intc-reset"); ++ if (!ret && a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3))) { ++ n->failed = "failed-intc-reset-ch3"; ++ ret = -EIO; ++ } else if (!ret && a9_aop_spi0_selected_verify(s)) { ++ n->failed = "failed-intc-reset-channel"; ++ ret = -EIO; ++ } ++ } ++ n->reset_done = !ret; ++ dev_info(aop->dev, "AOP INTC reset errno=%d asserted=%08x after=%08x result=%s\n", ++ ret, n->reset_asserted, n->reset_after, n->failed ?: "pass"); ++ return ret; ++} ++ +diff --git a/drivers/soc/apple/aop-warm-spi0.h b/drivers/soc/apple/aop-warm-spi0.h +new file mode 100644 +index 000000000..2470165ce +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.h +@@ -0,0 +1,543 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * J71t AOP reload/wake experiments, both opt-in and default off. Constants ++ * and evidence types only; the MMIO lives in apple-a9-aop-active.c. ++ * Evidence: docs/periph/ds-aop-reload-crash-and-wake.md. ++ * ++ * aop_spi0_reset=1: every halt + SRAM reload of a live AOP crashed in ++ * AOPSPIMaster_channelFaultIRQ (0102f252, DFAR 0x4c): the AOP-private SPI0 ++ * controller kept channel 3's +0x8d0 bit 6 while the fresh firmware's ++ * channel table was still empty. With the CPU halted, the AP replays the ++ * firmware's own controller reset AOPSPIMaster_reset (0102f2f4), then the ++ * firmware's own channel-3 teardown, and samples bit 6 for >= 150 ms after ++ * the image copy before it releases RUN once. Round 4: the old session's ++ * SPI0 global IRQ 0x10014 stays latched in the AOP interrupt controller ++ * (SCMErr:0x3,3 at the accel STARTEP), so the AP disables that source before ++ * the reset and acknowledges it after the clean window, just before RUN. ++ * Wake7: the channel-3 idle wait gets the firmware's own 10 ms bound twice ++ * over, sampled per 1 ms slice, and the disable is judged by the INTC mask ++ * readback, which the firmware itself reads. ++ * Wake8: after the targeted ack, replay INTC event 3 (+0x808: 1/read/0/read). ++ * This resets retained interrupt delivery, not the read-only SCM state. ++ * Wake9: wake8 device evidence showed the channel-3 stop itself changes SCM ++ * from 0 to 3. Replay the same SPI controller reset once more after teardown; ++ * require the post-reset SCM state to be zero and the teardown state to hold. ++ * Wake10: the second reset clears SCM, but a halted CPU then saw SCM 3 and ++ * +0x8d0 bit 14 (channel 7) during the firmware copy. Snapshot all 16 channel ++ * slots, sample global SCM/fault state across the copy, and gate the copy and ++ * final RUN on a completely quiet readback. ++ * Wake11: that snapshot proved channels 3 and 7 were both running. Keep the ++ * channel-3 sequence unchanged, then apply the same firmware teardown to every ++ * other run-bit + nonzero-interval channel before the post-teardown reset. ++ * Wake12: channel 7 is the phosphorus/BMP284 async client. Its firmware has no ++ * owner-specific stop path, so do not invent another store when the generic ++ * stop times out. Record the remaining scheduler state read-only and refuse. ++ * Wake13/14: every device stop of channel 3 that idled (wake8..wake12) came ++ * 1.5-2.2 ms after the controller reset; the two that did not (wake13/14) came ++ * 11/33 ms after it (pr_flush in the checkpoint), with class-0 state identical to ++ * wake12's passes. Wake15 closed that window (no pr_flush between the reset and ++ * the stops, console lock held), recorded the reset-to-stop latency and the peers ++ * per stop, and accepted one identity-pinned stopped-without-idle channel (3 or ++ * 7) as wake13 did for channel 7. Wake15's channel-7-first order then showed ++ * channel 7 never idles even at 0.3 ms (it is 0/3 over wake11/12/15), with ++ * nothing ticking. Wake16 therefore returns to channel 3 first (early, the only ++ * stop that ever idled) and then channel 7, which takes the stable-stop path ++ * (SCM 0/0 or 3/0, constant); every stop logs why the fingerprint did or did not ++ * hold (stable_why=). Wake14's class-0 prelude stays removed. ++ * ++ * aop_wake_wrapper=1: before 0x220, the AppleA7IOP status-3 resume wrapper ++ * writes (b4b854..b4b8b4). No CPU RUN, no auto-boot, no NMI. ++ */ ++#ifndef _A9_AOP_RELOAD_H ++#define _A9_AOP_RELOAD_H ++ ++#include ++#include ++#include "aop-warm-record.h" ++ ++/* Firmware device spi0: IOP 0x10100000 (space 2, 0x1000) via /arm-io ranges. */ ++#define A9_SPI0_PA 0x210100000ULL ++#define A9_SPI0_SIZE 0x1000U ++#define A9_SPI0_WRAPPER_PA 0x210800000ULL /* the only layout this applies to */ ++#define A9_SPI0_CONTROL 0x000U ++#define A9_SPI0_CLOCK 0x008U /* 0102f2aa programs 0x01b71b00 here */ ++#define A9_SPI0_FAULT 0x8d0U /* bit 2n = channel n, read by 0102f20a */ ++#define A9_SPI0_FAULT_8D4 0x8d4U ++#define A9_SPI0_CHANNEL_BASE 0x100U ++#define A9_SPI0_CHANNEL_STRIDE 0x080U ++#define A9_SPI0_CHANNELS 16U ++#define A9_SPI0_CHANNEL_REGS 7U ++#define A9_SPI0_CHANNEL_DESC 0x018U ++#define A9_SPI0_CH3 0x280U /* base + 0x100 + 0x80 * 3 */ ++#define A9_SPI0_CH3_0C 0x28cU ++#define A9_SPI0_RESET 0x100U /* control bit 8 */ ++#define A9_SPI0_ENABLE 0x001U /* control bit 0 */ ++#define A9_SPI0_CH3_PENDING 0x040U ++/* The firmware spins without bound; the AP gives each poll 1 ms. */ ++#define A9_SPI0_POLL_US 1000U ++#define A9_SPI0_DELAY_US 100U /* 0102b3f8(100, 0), twice */ ++#define A9_SPI0_REGS 6 ++ ++static const u32 a9_spi0_offsets[A9_SPI0_REGS] = { ++ A9_SPI0_CONTROL, A9_SPI0_CLOCK, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++ A9_SPI0_CH3, A9_SPI0_CH3_0C, ++}; ++ ++/* Snapshot indices into a9_spi0_offsets. */ ++enum { A9_SPI0_I_CONTROL, A9_SPI0_I_CLOCK, A9_SPI0_I_FAULT }; ++ ++/* ++ * Channel-3 teardown after the reset (ds-aop-spi0-ch3-round3.md section 5). ++ * The reset leaves C3 = 0x00010447: idle but still enabled, async-armed and ++ * with the class-0 fault IRQ on, so a level source re-latched +0x8d0 bit 6 ++ * within ~105 ms with the CPU halted. AOPSPIMaster_teardownChannel ++ * (0102e4f8) stops the channel and masks both IRQ classes; the fault ++ * handler (0102f20a) then acknowledges with C3C bit 5. ++ */ ++#define A9_SPI0_CH3_ROUTE 0x284U ++#define A9_SPI0_CH3_RATE 0x288U ++#define A9_SPI0_CH3_DESC 0x298U /* channel +0x18, zeroed on teardown */ ++#define A9_SPI0_CH3_RUN 0x001U /* C3 bit 0, channel run/enable */ ++#define A9_SPI0_CH3_IDLE 0x002U /* C3 bit 1, polled after the stop */ ++#define A9_SPI0_CH3_IRQ0 0x040U /* C3 bit 6, +0x8d0 class-0/fault enable */ ++#define A9_SPI0_CH3_IRQ1 0x100U /* C3 bit 8, +0x8d4 class-1 enable */ ++#define A9_SPI0_CHANNEL_PROGRESS_SHIFT 24U /* hardware-updated, not an IRQ class */ ++#define A9_SPI0_CH3_STOP 0x411U /* clear run bit 0 and bits 4, 10 */ ++#define A9_SPI0_CH3_CLASS0 0x450U /* mask class 0: bits 4, 6, 10 */ ++#define A9_SPI0_CH3_CLASS1 0x510U /* mask class 1: bits 4, 8, 10 */ ++#define A9_SPI0_CH3_ACK 0x020U /* C3C bit 5, ORed by 0102f20a; self-clearing */ ++#define A9_SPI0_PHOSPHORUS_CHANNEL 7U ++#define A9_SPI0_PHOSPHORUS_CTL_MASK 0x00ffffffU ++#define A9_SPI0_PHOSPHORUS_CTL_RUN 0x00010445U ++#define A9_SPI0_PHOSPHORUS_CTL_STOPPED 0x00010444U ++#define A9_SPI0_PHOSPHORUS_ROUTE 0x00000040U ++#define A9_SPI0_PHOSPHORUS_RATE 0x019000c8U ++#define A9_SPI0_PHOSPHORUS_CFG 0x00200009U ++/* Channel 3 is the MPU6900 async FIFO client (route 0x10, same rate/config). */ ++#define A9_SPI0_MPU_ASYNC_CHANNEL 3U ++#define A9_SPI0_MPU_ASYNC_ROUTE 0x00000010U ++/* Stopped control word, any top byte, class-0 enable optional (0x40). */ ++#define A9_SPI0_STABLE_CTL_STOPPED 0x00010404U ++#define A9_SPI0_STABLE_CTL_MASK 0x00ffffbfU ++/* Peer sampling and the "late stop" mark are reported, never gated, in-kernel. */ ++#define A9_SPI0_LATE_STOP_US 3000U ++/* ++ * AOPSPIMaster_setChannelEnabled(false) (0102ef7c) stores C3 & ~0x411 only ++ * if bit 0 was set, then spins on bit 1 until master +0x30, and panics ++ * "SCMto:%u" (0x010398da) past it. Event 1 sets +0x30 to 0x2710 us when it ++ * is 0 (0102e6fa, as in the live-cold object), through timer +0x18 = ++ * us * Hz / 1000000 (0102ff3c): a 10 ms bound. The AP polls in 1 ms slices, ++ * up to twice that, sampling C3, the SCM state and +0x8d0 after each slice. ++ */ ++#define A9_SPI0_CH3_IDLE_FW_US 10000U ++#define A9_SPI0_CH3_SLICE_US 1000U ++#define A9_SPI0_CH3_SLICES 20U /* 20 ms */ ++/* At least 150 ms (the 16:27 re-latch took ~105 ms) of samples before RUN. */ ++#define A9_SPI0_WINDOW_SAMPLES 31U ++#define A9_SPI0_WINDOW_US 5000U /* 30 sleeps >= 5 ms each */ ++#define A9_SPI0_WINDOW_MS 150U /* required first-to-last sample span */ ++#define A9_SPI0_CH3_REGS 7 ++ ++static const u32 a9_spi0_ch3_offsets[A9_SPI0_CH3_REGS] = { ++ A9_SPI0_CH3, A9_SPI0_CH3_ROUTE, A9_SPI0_CH3_RATE, A9_SPI0_CH3_0C, ++ A9_SPI0_CH3_DESC, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++}; ++ ++/* Every firmware-visible channel slot, read only in wake10 and selected in wake11. */ ++static const u32 a9_spi0_channel_offsets[A9_SPI0_CHANNEL_REGS] = { ++ 0x00U, 0x04U, 0x08U, 0x0cU, 0x10U, 0x14U, 0x18U, ++}; ++ ++enum { ++ A9_CHANNEL_I_CTL, A9_CHANNEL_I_ROUTE, A9_CHANNEL_I_RATE, A9_CHANNEL_I_CFG, ++ A9_CHANNEL_I_INTERVAL, A9_CHANNEL_I_CURSOR, A9_CHANNEL_I_DESC, ++}; ++ ++/* Snapshot indices into a9_spi0_ch3_offsets. */ ++enum { ++ A9_CH3_I_C3, A9_CH3_I_ROUTE, A9_CH3_I_RATE, A9_CH3_I_C3C, A9_CH3_I_DESC, ++ A9_CH3_I_8D0, A9_CH3_I_8D4, ++}; ++ ++/* One idle-poll slice: C3 as the slice ended, then SCM state and +0x8d0. */ ++struct a9_aop_ch3_slice { ++ u64 ns; /* since the stop store */ ++ u32 c3, scm_lo, scm_hi, fault; ++}; ++ ++/* Wake12 read-only scheduler evidence for an additional-channel stop poll. */ ++struct a9_aop_channel_diag_slice { ++ u32 cfg, interval, cursor, fault_8d4; ++}; ++ ++struct a9_aop_spi0_ch3 { ++ u32 before[A9_SPI0_CH3_REGS], after[A9_SPI0_CH3_REGS]; ++ u32 stop, idle, class0, class1, ack; /* words written / polled */ ++ int poll; /* idle-bit poll result */ ++ bool run_before; /* C3 bit 0 before the stop */ ++ unsigned int slices; /* idle-poll slices used */ ++ struct a9_aop_ch3_slice slice[A9_SPI0_CH3_SLICES]; ++ u64 idle_us; /* stop store to the idle read */ ++ int error; ++ bool ran; /* the guarded teardown executed */ ++ const char *failed; /* distinct refusal reason, or NULL */ ++ /* Wake15: stop latency after the controller reset and the peers' state. */ ++ u64 stop_ns, reset_to_stop_us; ++ u32 peers_run, peers_ticking; /* masks of the other 15 channels */ ++ bool stable; /* accepted stopped-without-idle identity */ ++ const char *stable_why; /* first failed fingerprint condition, NULL if stable */ ++ /* Pre-RUN window after the image copy; a refusal also sets failed. */ ++ unsigned int samples; ++ u64 elapsed_ms; ++ bool window_done, reasserted; ++ unsigned int reassert_sample; ++ u64 reassert_ms; ++ u32 reassert[A9_SPI0_CH3_REGS]; ++}; ++ ++struct a9_aop_spi0; ++ ++struct a9_aop_spi0_channel_op { ++ struct a9_aop_spi0 *spi0; ++ struct a9_aop_spi0_ch3 *state; ++ unsigned int channel; ++}; ++ ++/* ++ * AOP interrupt controller, wrapper offsets (ds-aop-spi0-ch3-round4.md ++ * section 3). IRQ 0x10014 is group 1, source 0x14: bank 0, bit 20. The ++ * group-1 disable/enable banks are write-one commands whose reads return ++ * the mask, 1 = disabled: the sleep command saves +0xa00..+0xa08 (0102c5ce) ++ * before masking all (0102c5da), the restore writes saved to +0xa00 and ++ * ~saved to +0xa80 (0102c9e6/0102c9f4), and aop_interrupt_write_enabled ++ * reads the target bank for the previous state (0102c9c0). So the disable ++ * is judged by bit 20 of the +0xa00 readback (wake6: ffffffff before the ++ * store, every source already masked by the quiesce). +0x81c only reports ++ * an unmasked pending source: the sleep path masks all, writes group 7 ++ * +0xc020/+0xc024 = 2 and spins until +0x81c reads 0 (0102c654), so ++ * all-ones there means the AP is not reading the INTC. +0x808 ends 0 after ++ * the event-3 pulse. +0x900 is the group-1 raise (0102c7c6) and +0x980 its ++ * pre-handler clear (0102c6f2); the firmware never reads them, so their ++ * readbacks are logged only. No register shows a masked source pending. ++ */ ++#define A9_INTC_RESET 0x808U /* event-3 pulse 1 then 0 (0102c592) */ ++#define A9_INTC_CURRENT 0x81cU /* current unmasked source */ ++#define A9_INTC_RAISE 0x900U /* group 1 raise, 0102c7c6 */ ++#define A9_INTC_ACK 0x980U /* group 1 pre-handler ack/clear, 0102c6f2 */ ++#define A9_INTC_DISABLE 0xa00U /* group 1 disable, store at 0102c9ce */ ++#define A9_INTC_ENABLE 0xa80U /* group 1 enable; reads the same mask */ ++#define A9_INTC_G7_DISABLE 0xc020U /* group 7 disable, 2 after sleep */ ++#define A9_INTC_SPI0_SCM 0x00100000U /* IRQ 0x10014 */ ++#define A9_INTC_ALL_ONES 0xffffffffU ++#define A9_INTC_REGS 10 ++ ++static const u32 a9_intc_offsets[A9_INTC_REGS] = { ++ A9_INTC_RESET, A9_INTC_CURRENT, A9_INTC_RAISE, A9_INTC_ACK, ++ A9_INTC_DISABLE, A9_INTC_DISABLE + 4, A9_INTC_DISABLE + 8, ++ A9_INTC_ENABLE, A9_INTC_G7_DISABLE, A9_INTC_G7_DISABLE + 4, ++}; ++ ++/* Snapshot indices into a9_intc_offsets. */ ++enum { ++ A9_INTC_I_RESET, A9_INTC_I_CURRENT, A9_INTC_I_RAISE, A9_INTC_I_ACK, ++ A9_INTC_I_MASK0, A9_INTC_I_MASK1, A9_INTC_I_MASK2, A9_INTC_I_ENABLE0, ++ A9_INTC_I_G7_0, A9_INTC_I_G7_1, ++}; ++ ++/* When the AP snapshots the INTC words. */ ++enum { ++ A9_INTC_AT_PRE_DISABLE, A9_INTC_AT_POST_DISABLE, A9_INTC_AT_PRE_ACK, ++ A9_INTC_AT_POST_ACK, A9_INTC_AT_N, ++}; ++ ++static inline const char *a9_intc_at_name(unsigned int at) ++{ ++ static const char * const names[] = { ++ "pre-disable", "post-disable", "pre-ack", "post-ack", ++ }; ++ ++ return at < A9_INTC_AT_N ? names[at] : "unknown"; ++} ++ ++/* "all-ones" if +0x81c reads ffffffff, which a live INTC cannot report. */ ++static inline const char *a9_intc_view(const u32 *state) ++{ ++ return state[A9_INTC_I_CURRENT] == A9_INTC_ALL_ONES ? "all-ones" : "intc"; ++} ++/* ++ * Read only, never written: the SCM state (also per idle-poll slice) and, ++ * around the acknowledge, the C3 interval pair. ++ */ ++#define A9_SPI0_SCM_LO 0x00cU ++#define A9_SPI0_SCM_HI 0x010U ++#define A9_SPI0_CH3_IVAL 0x290U ++#define A9_SPI0_CH3_CURSOR 0x294U ++#define A9_SPI0_SCM_REGS 4 ++ ++static const u32 a9_spi0_scm_offsets[A9_SPI0_SCM_REGS] = { ++ A9_SPI0_SCM_LO, A9_SPI0_SCM_HI, A9_SPI0_CH3_IVAL, A9_SPI0_CH3_CURSOR, ++}; ++ ++struct a9_aop_intc { ++ u32 state[A9_INTC_AT_N][A9_INTC_REGS]; /* around each store */ ++ unsigned int states; /* bit n: state[n] was read */ ++ u32 scm[2][A9_SPI0_SCM_REGS]; /* before / after the ack */ ++ unsigned int scm_at; /* scm[] row the guarded read fills */ ++ bool disabled, acked; ++ bool reset_done; ++ u32 reset_asserted, reset_after; ++ const char *failed; /* distinct refusal reason, or NULL */ ++}; ++ ++/* Read-only milestones. Wake10 gates the global SCM/fault words, not channels. */ ++struct a9_aop_spi0_trace { ++ u32 control, clock, scm[A9_SPI0_SCM_REGS], ch3[A9_SPI0_CH3_REGS]; ++ u32 channel[A9_SPI0_CHANNELS][A9_SPI0_CHANNEL_REGS]; ++}; ++ ++/* Wake9's second, post-channel-stop replay of AOPSPIMaster_reset. */ ++struct a9_aop_spi0_post_reset { ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved, asserted, polled; ++ int error; ++ bool ran; ++ const char *failed; ++}; ++ ++/* ++ * First failed readback after the teardown, or NULL if all hold. C3C bit 5 ++ * is a pulse: at 19:50 the write of 0x00200029 read back 0x00200009 while ++ * +0x8d0 bit 6 went 1 -> 0, so the acknowledge is judged by bit 6 alone and ++ * the C3C readback is only logged. ++ */ ++static inline const char *a9_spi0_ch3_verify(const u32 *after, bool stable) ++{ ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) ++ return "failed-ch3-teardown-run"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0) ++ return "failed-ch3-teardown-class0"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1) ++ return "failed-ch3-teardown-class1"; ++ if (!(after[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE) && !stable) ++ return "failed-ch3-teardown-idle"; ++ if (after[A9_CH3_I_DESC]) ++ return "failed-ch3-teardown-desc"; ++ if (after[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING) ++ return "failed-ch3-teardown-pending"; ++ return NULL; ++} ++ ++/* ++ * Has firmware run since the last SoC reset? auto-boot alone cannot say: ++ * the outgoing aop_quiesce=1 halt clears it, exactly like a DFU cold start. ++ * The previous kernel's reserved-RAM shutdown transcript says whether its ++ * firmware ran. That DRAM can outlive a DFU; wrapper +0x4020 outbox enable ++ * cannot: set after reset (12/12 DFU cold snapshots), cleared by the first ++ * store of every Linux stop. No SPI0 access is needed to decide. ++ */ ++enum a9_spi0_prev_state { ++ A9_SPI0_PREV_MISSING, /* no transcript magic, or none reserved */ ++ A9_SPI0_PREV_INVALID, /* magic present, header/checksum rejected */ ++ A9_SPI0_PREV_VALID, ++}; ++ ++struct a9_spi0_prev { ++ enum a9_spi0_prev_state state; ++ bool hello; /* "SHUTDOWN begin ... hello=1": its firmware ran */ ++ bool halted; /* "QUIESCE step=complete ... halted=1" + steps bit */ ++ bool carried; /* "SPI0-CARRY retained=1": ran earlier in the chain */ ++ u32 steps; /* "QUIESCE step=complete ... steps=0x..." */ ++}; ++ ++enum a9_spi0_reason { ++ /* Run the reset. */ ++ A9_SPI0_LIVE_AUTO_BOOT, /* auto-boot 1 before the stop */ ++ A9_SPI0_PREV_HALTED, /* previous kernel quiesced and halted it */ ++ A9_SPI0_PREV_LIVE, /* previous kernel's firmware ran, no halt */ ++ A9_SPI0_PREV_CARRIED, /* an earlier kernel in the chain ran it */ ++ /* Skip: the block is not touched. */ ++ A9_SPI0_FRESH, /* no transcript, auto-boot 0: DFU cold */ ++ A9_SPI0_PREV_BAD, /* transcript present but corrupt */ ++ A9_SPI0_PREV_NO_RUN, /* transcript: nothing ran in the chain */ ++ A9_SPI0_STALE, /* run evidence, wrapper at reset state */ ++}; ++ ++static inline const char *a9_spi0_reason_name(enum a9_spi0_reason reason) ++{ ++ static const char * const names[] = { ++ "live-auto-boot", "prev-quiesce-halted", "prev-live", "prev-carried", ++ "fresh-instance", "prev-transcript-invalid", "prev-no-run", ++ "stale-transcript", ++ }; ++ ++ return reason <= A9_SPI0_STALE ? names[reason] : "unknown"; ++} ++ ++static inline const char *a9_spi0_prev_name(enum a9_spi0_prev_state state) ++{ ++ return state == A9_SPI0_PREV_VALID ? "valid" : ++ state == A9_SPI0_PREV_INVALID ? "invalid" : "missing"; ++} ++ ++static inline bool a9_spi0_reason_runs(enum a9_spi0_reason reason) ++{ ++ return reason <= A9_SPI0_PREV_CARRIED; ++} ++ ++/* ++ * Release policy: a reason that would run the reload is allowed by default ++ * only for prev-quiesce-halted, the one state device-proven end to end. A ++ * skip (fresh, bad record, nothing ran) is allowed only while the wrapper is ++ * still at its reset state; stale records skip as before. Untested lifts both. ++ */ ++static inline bool a9_spi0_policy_allows(enum a9_spi0_reason reason, bool outbox_enabled, ++ bool untested) ++{ ++ if (untested || reason == A9_SPI0_PREV_HALTED || reason == A9_SPI0_STALE) ++ return true; ++ if (a9_spi0_reason_runs(reason)) ++ return false; ++ /* ++ * A skip means "nothing ran since the SoC reset", which only holds if the ++ * wrapper is still at its reset state (+0x4020 outbox enable set, 12/12 DFU ++ * cold starts). Without it some earlier kernel stopped the firmware and its ++ * record is lost: a cold load would land on a halted, never-torn-down AOP. ++ */ ++ return outbox_enabled; ++} ++ ++/* Token search inside one transcript line, which holds no NUL. */ ++static inline bool a9_spi0_line_has(const char *line, size_t length, ++ const char *token, size_t n, ++ const char **at) ++{ ++ size_t i; ++ ++ for (i = 0; n <= length && i <= length - n; i++) { ++ if (!memcmp(line + i, token, n)) { ++ *at = line + i + n; ++ return true; ++ } ++ } ++ return false; ++} ++ ++#define a9_spi0_has(line, length, token, at) \ ++ a9_spi0_line_has(line, length, token, sizeof(token) - 1, at) ++ ++static inline bool a9_spi0_line_starts(const char *line, size_t length, ++ const char *prefix, size_t n) ++{ ++ return length >= n && !memcmp(line, prefix, n); ++} ++ ++#define a9_spi0_starts(line, length, prefix) \ ++ a9_spi0_line_starts(line, length, prefix, sizeof(prefix) - 1) ++ ++static inline u32 a9_spi0_hex(const char *p, const char *end) ++{ ++ u32 value = 0; ++ ++ for (; p < end; p++) { ++ if (*p >= '0' && *p <= '9') ++ value = value << 4 | (u32)(*p - '0'); ++ else if (*p >= 'a' && *p <= 'f') ++ value = value << 4 | (u32)(*p - 'a' + 10); ++ else ++ break; ++ } ++ return value; ++} ++ ++/* Our own checksummed transcript; only whole-line prefixes count. */ ++static inline void a9_spi0_prev_parse(struct a9_spi0_prev *p, ++ const char *text, size_t length) ++{ ++ const char *end = text + length, *line, *next, *at; ++ size_t n; ++ ++ for (line = text; line < end; line = next < end ? next + 1 : end) { ++ next = memchr(line, '\n', end - line); ++ if (!next) ++ next = end; ++ n = next - line; ++ if (a9_spi0_starts(line, n, "SHUTDOWN begin ") && ++ a9_spi0_has(line, n, " hello=1 ", &at)) ++ p->hello = true; ++ if (a9_spi0_starts(line, n, "QUIESCE step=complete ")) { ++ if (a9_spi0_has(line, n, " steps=0x", &at)) ++ p->steps = a9_spi0_hex(at, next); ++ p->halted = a9_spi0_has(line, n, " halted=1 ", &at) && ++ (p->steps & (1U << A9_HO_STEP_HALTED)); ++ } ++ if (a9_spi0_starts(line, n, "SPI0-CARRY retained=1 ")) ++ p->carried = true; ++ } ++} ++ ++/* ++ * Skips unless there is positive evidence: running the reset on a fresh ++ * block would be the first AP access to SPI0 with nothing retained to ++ * clear (and ps_spi0 may gate it), while a skip on retained state only ++ * repeats the known, non-hanging accel STARTEP crash. ++ */ ++static inline enum a9_spi0_reason a9_spi0_decide(bool auto_boot, bool outbox_enabled, ++ const struct a9_spi0_prev *p) ++{ ++ if (auto_boot) ++ return A9_SPI0_LIVE_AUTO_BOOT; ++ if (p->state == A9_SPI0_PREV_MISSING) ++ return A9_SPI0_FRESH; ++ if (p->state != A9_SPI0_PREV_VALID) ++ return A9_SPI0_PREV_BAD; ++ if (!p->hello && !p->carried) ++ return A9_SPI0_PREV_NO_RUN; ++ if (outbox_enabled) ++ return A9_SPI0_STALE; ++ if (p->hello) ++ return p->halted ? A9_SPI0_PREV_HALTED : A9_SPI0_PREV_LIVE; ++ return A9_SPI0_PREV_CARRIED; ++} ++ ++struct a9_aop_spi0 { ++ void __iomem *base; /* mapped only with aop_spi0_reset=1 */ ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved; /* control word before the reset */ ++ u32 asserted; /* saved & ~0x101 | 0x100 */ ++ u32 polled; /* control word when the polls ended */ ++ u32 prerun; /* +0x8d0 just before RUN */ ++ u32 prerun_8d4, prerun_scm_lo, prerun_scm_hi; ++ u32 pmgr_before, pmgr_after; /* AP PMGR ps_spi0 raw, read-only */ ++ int pmgr_before_errno, pmgr_after_errno; ++ int error; ++ bool ran, reasserted, copy_faulted; ++ const char *gate_failed; ++ const char *skipped; ++ struct a9_spi0_prev prev; /* parsed at probe from the transcript */ ++ u32 auto_boot, outbox; /* boot snapshot, before the stop */ ++ enum a9_spi0_reason reason; ++ bool decided; ++ struct a9_aop_spi0_ch3 ch3; /* after a clean reset only */ ++ /* Wake11 reuses one bounded scratch record for each additional channel. */ ++ struct a9_aop_spi0_ch3 channel; ++ u32 teardown_selected, teardown_done; ++ unsigned int teardown_failed_channel; ++ struct a9_aop_spi0_post_reset post_reset; /* after the stop latches SCM */ ++ struct a9_aop_intc intc; /* IRQ 0x10014 around the reset */ ++ struct a9_aop_spi0_trace trace; /* latest read-only milestone */ ++ /* Appended so every pre-wake12 field, including ch3, keeps its layout. */ ++ struct a9_aop_channel_diag_slice channel_diag[A9_SPI0_CH3_SLICES]; ++ /* Wake13: exact phosphorus stopped-without-idle exception, by channel. */ ++ u32 teardown_stable; ++ /* Wake15: channel 3 shares the sampled diagnostics; reset completion time. */ ++ struct a9_aop_channel_diag_slice ch3_diag[A9_SPI0_CH3_SLICES]; ++ u64 reset_done_ns; ++}; ++ ++/* 0102f2f4: bits 8 and 0 clear; the reset write adds bit 8. */ ++static inline u32 a9_spi0_base_word(u32 saved) ++{ ++ return saved & ~(A9_SPI0_RESET | A9_SPI0_ENABLE); ++} ++ ++#endif +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index e8ae90098..8bcab7651 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -1,19 +1,28 @@ + // SPDX-License-Identifier: GPL-2.0-only OR MIT + /* +- * Apple T8010 Always-On Processor (AOP) ++ * Apple S8000 and T8010 Always-On Processor (AOP) + * + * The AOP runs RTKit firmware that the operating system loads: a main image + * into the AOP's SRAM and an external segment into system memory, with boot +- * arguments filled in. Its services talk AFK/EPIC over shared ring buffers on +- * the RTKit application endpoints, as on later SoCs (see the Asahi Linux AOP ++ * arguments filled in. On the S8000 (A9) the AOP sees its SRAM through a window ++ * that has to be set up first, and its firmware takes more boot arguments. ++ * Its services talk AFK/EPIC over shared ring buffers on the RTKit ++ * application endpoints, as on later SoCs (see the Asahi Linux AOP + * driver). Each service with a device tree child node of the same name gets a + * child platform device; its driver receives the service's reports through + * apple_aop_listen() and sends requests with apple_aop_call(). + * +- * The firmware cannot be restarted once stopped, so the AOP starts once per +- * boot and the driver cannot be unbound or unloaded after that. ++ * J71t can reload after a certified quiesce/halt/drain across kexec. Other ++ * retained states are refused. Unbind/unload after starting remains forbidden. + */ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include "aop-warm-spi0.h" + #include + #include + #include +@@ -41,6 +50,14 @@ + #define APPLE_AOP_IRQ_MASK_ALL 0x1111 + #define APPLE_AOP_IRQ_INBOX BIT(0) + #define APPLE_AOP_IRQ_OUTBOX BIT(12) ++/* S8000: the window through which the AOP sees its SRAM */ ++#define APPLE_AOP_REMAP_PA_LO 0x08 ++#define APPLE_AOP_REMAP_PA_HI 0x10 ++#define APPLE_AOP_REMAP_IOVA_LO 0x18 ++#define APPLE_AOP_REMAP_IOVA_HI 0x20 ++#define APPLE_AOP_REMAP_SIZE 0x28 ++#define APPLE_AOP_REMAP_RESERVED 0x30 ++#define APPLE_AOP_REMAP_ENABLE 0x38 + + #define APPLE_AOP_CONTROL_AUTO_BOOT 0x0 + +@@ -54,6 +71,13 @@ + #define APPLE_AOP_BOOTARG_ECID APPLE_AOP_FOURCC('E', 'C', 'I', 'D') + #define APPLE_AOP_BOOTARG_NONC APPLE_AOP_FOURCC('N', 'O', 'N', 'C') + #define APPLE_AOP_BOOTARG_EC0P APPLE_AOP_FOURCC('E', 'C', '0', 'p') ++#define APPLE_AOP_BOOTARG_AOPT APPLE_AOP_FOURCC('A', 'O', 'P', 't') ++#define APPLE_AOP_BOOTARG_NCAL APPLE_AOP_FOURCC('n', 'C', 'a', 'l') ++#define APPLE_AOP_BOOTARG_WRAD APPLE_AOP_FOURCC('W', 'r', 'A', 'd') ++#define APPLE_AOP_BOOTARG_SOC APPLE_AOP_FOURCC('S', 'O', 'C', '_') ++#define APPLE_AOP_BOOTARG_SOCR APPLE_AOP_FOURCC('S', 'O', 'C', 'R') ++#define APPLE_AOP_BOOTARG_STKG APPLE_AOP_FOURCC('S', 'T', 'K', 'G') ++#define APPLE_AOP_MAX_BOOTARGS 10 + + #define APPLE_AOP_HALT_TIMEOUT_US 10000 + +@@ -78,7 +102,9 @@ + #define AFK_OPC_GET_BUF_ACK 0xa1 + #define AFK_OPC_SEND 0xa2 + #define AFK_OPC_START 0xa3 +-#define AFK_OPC_SHUTDOWN_ACK 0xc1 ++#define AFK_OPC_SHUTDOWN 0xa4 ++#define AFK_OPC_SHUTDOWN_ACK 0x87 ++#define AFK_OPC_HIBERNATE_ACK 0xc3 + + #define AFK_BLOCK_STEP 0x40 + #define AFK_MAX_BUFFER SZ_1M +@@ -100,6 +126,12 @@ + #define APPLE_AOP_MAX_CALLS 8 + #define APPLE_AOP_CALL_TIMEOUT msecs_to_jiffies(1000) + ++/* Service properties */ ++#define APPLE_AOP_PROPERTY_MAX 64 ++#define APPLE_AOP_PROP_CALIBRATION 0x3d ++#define APPLE_AOP_CALIBRATION_PARTIAL 1 ++#define APPLE_AOP_CALIBRATION_COMPLETE 2 ++ + struct afk_qe_header { + __le32 magic; + __le32 size; +@@ -160,6 +192,8 @@ struct apple_aop_ep { + + struct mutex lock; /* tx ring, seq and calls */ + u16 seq; ++ bool initialized; ++ struct completion stopped; + struct apple_aop_call *calls[APPLE_AOP_MAX_CALLS]; + }; + +@@ -182,12 +216,25 @@ struct apple_aop_register_work { + struct apple_aop_service service; + }; + ++struct apple_aop_hw { ++ /* Set up the AOP's window onto its SRAM before starting it */ ++ bool sram_window; ++ /* A fresh NONC boot argument */ ++ bool nonce; ++ /* AOPt, SOC_, SOCR, WrAd, STKG and nCal boot arguments */ ++ bool soc_bootargs; ++}; ++ + struct apple_aop { + struct device *dev; ++ const struct apple_aop_hw *hw; + void __iomem *akf; + void __iomem *control; + void __iomem *sram; ++ resource_size_t akf_base; ++ resource_size_t sram_base; + resource_size_t sram_size; ++ u32 target, chip_id, chip_revision; + struct apple_rtkit *rtk; + + void *ext; +@@ -202,6 +249,16 @@ struct apple_aop { + struct mutex children_lock; /* children, removing */ + struct list_head children; + bool removing; ++ u64 accel_reports; ++ int shutdown_error; ++ /* J71t only; all other boards retain their original start/stop path. */ ++ bool warm, hello, iop_crashed, force_halted, force_drained; ++ void __iomem *warm_wrapper; ++ struct device_node *domains[5], *spi0_pmgr; ++ struct a9_aop_spi0 spi0; ++ const char *experiment_step; ++ struct a9_sl_header *shutdown_log; ++ char *shutdown_text; + }; + + static bool apple_aop_is_started(struct apple_aop *aop) +@@ -223,6 +280,409 @@ static bool apple_aop_halt(struct apple_aop *aop) + APPLE_AOP_HALT_TIMEOUT_US); + } + ++static bool a9_aop_ramoops_ready(void) ++{ ++ struct console *con; ++ bool ready = false; ++ int cookie; ++ ++ cookie = console_srcu_read_lock(); ++ for_each_console_srcu(con) { ++ if (!strcmp(con->name, "ramoops") && ++ (console_srcu_read_flags(con) & (CON_ENABLED | CON_SUSPENDED)) == ++ CON_ENABLED) { ++ ready = true; ++ break; ++ } ++ } ++ console_srcu_read_unlock(cookie); ++ return ready; ++} ++ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* J71t warm reload integration. SPI0 machinery is the release6 wake16 code. */ ++#define AOP_DOMAINS 5 ++#define AOP_CPU_CONTROL 0x44 ++#define AOP_CPU_START BIT(4) ++#define AOP_OUTBOX_CONTROL 0x4020 ++#define AOP_INBOX_CONTROL 0x4008 ++#define AOP_OUTBOX_ENABLE BIT(0) ++#define AOP_MBOX_FAULTS GENMASK(19, 18) ++#define cold_defer 1 ++#define spi0_untested false ++ ++static u32 aop_rd(struct apple_aop *aop, u32 offset) ++{ ++ return readl(aop->warm_wrapper + offset); ++} ++ ++static bool a9_aop_force_stopped(struct apple_aop *aop) ++{ ++ return !(aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) && ++ !readl(aop->control) && ++ !(aop_rd(aop, AOP_OUTBOX_CONTROL) & (AOP_OUTBOX_ENABLE | AOP_MBOX_FAULTS)) && ++ !(aop_rd(aop, AOP_INBOX_CONTROL) & AOP_MBOX_FAULTS); ++} ++ ++struct aop_warm_write { void __iomem *target; u32 value; }; ++static void aop_warm_store(void *data) ++{ ++ struct aop_warm_write *w = data; ++ ++ writel(w->value, w->target); ++ readl(w->target); ++} ++ ++static int aop_wr(struct apple_aop *aop, u32 offset, u32 value) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct aop_warm_write w = { aop->warm_wrapper + offset, value }; ++ ++ if ((aop_rd(aop, AOP_INBOX_CONTROL) | aop_rd(aop, AOP_OUTBOX_CONTROL)) & ++ AOP_MBOX_FAULTS) ++ return -EIO; ++ dev_info(aop->dev, "COLD-W step=%s bank=wrapper offset=%04x before=%08x value=%08x\n", ++ aop->experiment_step ?: "warm-stop", offset, readl(w.target), value); ++ { ++ int ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, aop_warm_store, &w); ++ ++ dev_info(aop->dev, "COLD-R offset=%04x after=%08x\n", offset, readl(w.target)); ++ return ret; ++ } ++} ++ ++static int a9_aop_cold_checkpoint_noflush(struct apple_aop *aop, const char *step) ++{ ++ if (!a9_aop_ramoops_ready()) ++ return -ENODEV; ++ dev_info(aop->dev, "COLD checkpoint=%s flush=skipped\n", step); ++ return 0; ++} ++ ++static int a9_aop_cold_checkpoint(struct apple_aop *aop, const char *step) ++{ ++ int ret = a9_aop_cold_checkpoint_noflush(aop, step); ++ ++ if (ret) ++ return ret; ++ if (!pr_flush(1000, false)) ++ return -ETIMEDOUT; ++ mb(); ++ return 0; ++} ++ ++#include "aop-warm-spi0.c.h" ++static int apple_aop_send(struct apple_aop_ep *ep, u8 opcode, u64 payload); ++ ++/* No cmdline override for untested states: default is the release6 policy. */ ++static atomic_t aop_warm_attempted = ATOMIC_INIT(0); ++ ++static int apple_aop_warm_stop(struct apple_aop *aop) ++{ ++ unsigned int fifo, i; ++ u32 offset, val; ++ int ret; ++ ++ ret = aop_wr(aop, 0x4020, aop_rd(aop, 0x4020) & ~BIT(0)); ++ if (!ret) ++ ret = aop_wr(aop, 0x4000, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0xc00, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0x44, aop_rd(aop, 0x44) & ~BIT(4)); ++ if (ret) ++ return ret; ++ writel(0, aop->control); ++ msleep(10); ++ if (!a9_aop_force_stopped(aop)) ++ return -EIO; ++ aop->force_halted = true; ++ for (fifo = 0; fifo < 2; fifo++) { ++ offset = fifo ? 0x4020 : 0x4008; ++ for (i = 0; i < 64; i++) { ++ val = aop_rd(aop, offset); ++ if (!a9_aop_force_stopped(aop) || (val & AOP_MBOX_FAULTS)) ++ return -EIO; ++ if (val & BIT(17)) ++ break; ++ readq(aop->warm_wrapper + (fifo ? 0x4038 : 0x4018)); ++ } ++ if (!(aop_rd(aop, offset) & BIT(17))) ++ return -ETIMEDOUT; ++ } ++ aop->force_drained = true; ++ return 0; ++} ++ ++static void apple_aop_warm_put_node(void *node) ++{ ++ of_node_put(node); ++} ++ ++static void apple_aop_warm_unmap(void *p) ++{ ++ memunmap(p); ++} ++ ++static int apple_aop_warm_prepare(struct apple_aop *aop) ++{ ++ struct device *dev = aop->dev; ++ struct device_node *np = dev->of_node, *mem; ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct a9_sl_header header; ++ struct resource res; ++ char *text; ++ u32 auto_boot, outbox; ++ unsigned int i; ++ int ret; ++ ++ if (!of_machine_is_compatible("apple,j71t") || !aop->hw->sram_window) ++ return 0; ++ /* Missing the new DT is not permission to run a retained instance. */ ++ mem = of_parse_phandle(np, "memory-region", 0); ++ if (!mem) ++ return dev_err_probe(dev, -EINVAL, "warm reload requires the matching J71t DT\n"); ++ ret = of_address_to_resource(mem, 0, &res); ++ of_node_put(mem); ++ if (ret || resource_size(&res) != A9_HO_PAGE || res.start != 0x8583fc000ULL || ++ aop->akf_base != A9_SPI0_WRAPPER_PA) ++ return -EINVAL; ++ aop->shutdown_log = memremap(res.start, A9_HO_PAGE, MEMREMAP_WB); ++ if (!aop->shutdown_log) ++ return -ENOMEM; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_unmap, aop->shutdown_log); ++ if (ret) ++ return ret; ++ /* Keep release6's transcript location and validation ABI. */ ++ aop->shutdown_log = (void *)aop->shutdown_log + A9_HO_LOG_OFFSET; ++ aop->shutdown_text = (char *)(aop->shutdown_log + 1); ++ memcpy(&header, aop->shutdown_log, sizeof(header)); ++ text = devm_kmemdup(dev, aop->shutdown_text, A9_SL_CAPACITY, GFP_KERNEL); ++ if (!text) ++ return -ENOMEM; ++ if (header.magic == A9_SL_MAGIC) { ++ aop->spi0.prev.state = A9_SPI0_PREV_INVALID; ++ if (a9_sl_valid(&header, text)) { ++ aop->spi0.prev.state = A9_SPI0_PREV_VALID; ++ a9_spi0_prev_parse(&aop->spi0.prev, text, header.length); ++ for (i = 0; i < header.length;) { ++ char *next = memchr(text + i, '\n', header.length - i); ++ unsigned int n = next ? next - text - i : header.length - i; ++ ++ dev_info(dev, "AOP-PREV %.*s\n", n, text + i); ++ i += n + 1; ++ } ++ } ++ } ++ aop->warm_wrapper = devm_ioremap(dev, aop->akf_base, 0x1c000); ++ if (!aop->warm_wrapper) ++ return -ENOMEM; ++ for (i = 0; i < AOP_DOMAINS; i++) { ++ aop->domains[i] = of_parse_phandle(np, "apple,observe-domains", i); ++ if (!aop->domains[i]) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->domains[i]); ++ if (ret) ++ return ret; ++ } ++ aop->spi0_pmgr = of_parse_phandle(np, "apple,spi0-domain", 0); ++ if (!aop->spi0_pmgr) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->spi0_pmgr); ++ if (ret) ++ return ret; ++ /* Check all domains before reading the wrapper; no power transitions. */ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, a9_aop_spi0_pmgr_on, &(bool){false}); ++ if (ret) ++ return dev_err_probe(dev, ret, "AOP domain guard refused\n"); ++ auto_boot = readl(aop->control); ++ outbox = aop_rd(aop, AOP_OUTBOX_CONTROL); ++ ret = a9_aop_spi0_policy(aop, auto_boot, outbox); ++ if (ret) ++ return ret; /* Before first halt, mask, SRAM or SPI0 write. */ ++ /* Orchard's old halt leaves +0x4020 enabled. On a known kexec, ++ * that alone cannot certify the cold wrapper baseline. Release6's ++ * wrapper discriminator was tied to its own outgoing stop routine. ++ */ ++ if (of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev) != A9_SPI0_PREV_HALTED) { ++ dev_info(dev, "SPI0 policy decision=refuse reason=uncertified-kexec auto_boot=%u out4020=%08x: block not touched\n", ++ auto_boot, outbox); ++ return -EPERM; ++ } ++ aop->warm = true; ++ aop->spi0.reason = a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev); ++ if (!a9_spi0_reason_runs(aop->spi0.reason)) { ++ /* Original cold path: no halt, drain, reset, or SPI0 access. */ ++ dev_info(dev, "SPI0 reset decision=skip reason=%s: block not touched\n", ++ a9_spi0_reason_name(aop->spi0.reason)); ++ return 0; ++ } ++ if (atomic_xchg(&aop_warm_attempted, 1)) ++ return dev_err_probe(dev, -EPERM, "refused-second-experiment-attempt\n"); ++ if (!a9_aop_ramoops_ready()) ++ return dev_err_probe(dev, -ENODEV, "warm reload requires ramoops console\n"); ++ aop->spi0.base = devm_ioremap(dev, A9_SPI0_PA, A9_SPI0_SIZE); ++ if (!aop->spi0.base) ++ return -ENOMEM; ++ /* Consume certification before the first control write. Reloading the ++ * module after a failure must not reuse it as a second attempt. ++ */ ++ aop->shutdown_log->magic = 0; ++ wmb(); ++ ret = apple_aop_warm_stop(aop); ++ if (!ret) ++ ret = a9_aop_spi0_prereset(aop, auto_boot, outbox); ++ if (ret) ++ dev_err(dev, "warm reload refused errno=%d; no copy/RUN, recover with DFU\n", ret); ++ return ret; ++} ++ ++static int apple_aop_warm_release(struct apple_aop *aop) ++{ ++ int ret; ++ ++ ret = a9_aop_spi0_recheck(aop); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-window"); ++ if (!ret) ++ ret = a9_aop_intc_ack(aop); ++ if (!ret) ++ ret = a9_aop_intc_reset(aop); ++ if (!ret) ++ ret = a9_aop_cold_checkpoint(aop, "before-run"); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "pre-run-final", "FINAL", true); ++ return ret; ++} ++ ++static void apple_aop_warm_hash(struct apple_aop *aop, u8 *digest) ++{ ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ if (aop->ext) ++ sha256_update(&ctx, aop->ext, aop->ext_size); ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) ++ if (aop->eps[i].buf) ++ sha256_update(&ctx, aop->eps[i].buf, aop->eps[i].buf_size); ++ sha256_final(&ctx, digest); ++} ++ ++static __printf(2, 3) void apple_aop_warm_log(struct apple_aop *aop, const char *fmt, ...) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ va_list args; ++ char line[512]; ++ size_t n; ++ ++ va_start(args, fmt); ++ n = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ dev_info(aop->dev, "%s", line); ++ if (n > h->capacity - h->length) { ++ h->truncated++; ++ return; ++ } ++ memcpy(aop->shutdown_text + h->length, line, n); ++ wmb(); ++ h->checksum = a9_sl_hash(h->checksum, line, n); ++ h->length += n; ++ h->lines++; ++ wmb(); ++} ++ ++static void apple_aop_warm_shutdown(struct apple_aop *aop) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ unsigned long deadline = jiffies + msecs_to_jiffies(10000), start; ++ unsigned int i, acked = 0; ++ int ret = -EIO; ++ ++ memset(h, 0, sizeof(*h)); ++ h->version = A9_SL_VERSION; ++ h->capacity = A9_SL_CAPACITY; ++ h->checksum = A9_SL_HASH_SEED; ++ h->mode = 1; ++ h->begin_ns = ktime_get_boottime_ns(); ++ wmb(); ++ h->magic = A9_SL_MAGIC; ++ apple_aop_warm_log(aop, "SHUTDOWN begin mode=quiesce hello=%d phase=stopping iop_crashed=%d\n", ++ aop->hello, aop->iop_crashed); ++ if (!aop->hello || aop->iop_crashed || aop->shutdown_error) ++ goto done; ++ /* Rates-zero has stopped reports. Stop one AFK at a time, max 2s ++ * per endpoint and 10s in total, including send time. Never force halt ++ * after a failed ACK. Keep every grant allocated for the entire hook. ++ */ ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ unsigned long limit; ++ ++ if (!ep->initialized) ++ continue; ++ if (time_after_eq(jiffies, deadline)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ reinit_completion(&ep->stopped); ++ start = jiffies; ++ ret = apple_aop_send(ep, AFK_OPC_SHUTDOWN, 0); ++ if (ret) ++ goto done; ++ limit = time_before(jiffies, deadline) ? min_t(unsigned long, ++ deadline - jiffies, msecs_to_jiffies(2000)) : 0; ++ if (!limit || !wait_for_completion_timeout(&ep->stopped, limit)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ acked++; ++ apple_aop_warm_log(aop, "QUIESCE step=afk-ack ep=0x%x state=stopped ms=%u\n", ++ ep->index, jiffies_to_msecs(jiffies - start)); ++ } ++ /* release6: complete every AFK stop before disabling endpoint routing. */ ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; ++ } ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ ++ if (!ep->initialized) ++ continue; ++ /* MGMT STARTEP bit 0 disables routing, no separate stop ACK. */ ++ ret = apple_rtkit_send_message(aop->rtk, 0, ++ (5ULL << 52) | ((u64)ep->index << 32) | 1, NULL, false); ++ if (ret) ++ goto done; ++ } ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; /* Never certify an incompletely initialized instance. */ ++ } ++ ret = apple_rtkit_shutdown_preserve(aop->rtk); ++ if (ret) ++ goto done; ++ msleep(20); ++ apple_aop_warm_hash(aop, before); ++ msleep(20); ++ apple_aop_warm_hash(aop, after); ++ ret = memcmp(before, after, sizeof(before)) ? -EBUSY : 0; ++ apple_aop_warm_log(aop, "QUIESCE step=ram-stable window_ms=20 status=%d\n", ret); ++ if (ret || aop->iop_crashed) ++ goto done; ++ ret = apple_aop_warm_stop(aop); ++done: ++ apple_aop_warm_log(aop, "QUIESCE step=complete status=%d quiesced=%d halted=%d drained=%d steps=0x%x afk_acks=%u iop_crashed=%d\n", ++ ret, !ret, aop->force_halted, aop->force_drained, ++ !ret ? 0x1bff : 0, acked, aop->iop_crashed); ++ if (ret) ++ dev_err(aop->dev, "quiesce failed; next AOP must refuse, recover with DFU\n"); ++} ++ + /* Firmware loading */ + + struct apple_aop_bootarg { +@@ -232,11 +692,17 @@ struct apple_aop_bootarg { + bool found; + }; + ++static void apple_aop_add_bootarg(struct apple_aop_bootarg *args, unsigned int *count, ++ u32 key, const void *value, u32 size) ++{ ++ args[(*count)++] = (struct apple_aop_bootarg){ .key = key, .value = value, .size = size }; ++} ++ + static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t size, +- struct apple_aop_bootarg *args, int count) ++ struct apple_aop_bootarg *args, unsigned int count) + { + size_t offset, end, pos; +- int i; ++ unsigned int i; + + offset = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_OFFSET); + end = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_SIZE); +@@ -269,13 +735,29 @@ static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t siz + return 0; + } + ++/* S8000: point the AOP's window at its SRAM; the firmware runs from there. */ ++static void apple_aop_map_sram(struct apple_aop *aop) ++{ ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_LO); ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_HI); ++ writel(aop->sram_size & ~0xfffULL, aop->akf + APPLE_AOP_REMAP_SIZE); ++ writel(0, aop->akf + APPLE_AOP_REMAP_RESERVED); ++ writel(lower_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_LO); ++ writel(upper_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_HI); ++ writel(1, aop->akf + APPLE_AOP_REMAP_ENABLE); ++} ++ + static int apple_aop_load(struct apple_aop *aop, u64 ecid) + { ++ struct apple_aop_bootarg args[APPLE_AOP_MAX_BOOTARGS]; + const struct firmware *main_fw, *ext_fw; + const char *main_name, *ext_name; + struct device_node *np = aop->dev->of_node; +- u8 prng[32], nonce[8]; +- __le64 ecid_le = cpu_to_le64(ecid), ext_le; ++ u8 prng[32], nonce[8], guard[4], ncal = 0; ++ __le64 ecid_le = cpu_to_le64(ecid), ext_le, wrapper_le; ++ __le32 target_le, chip_le, revision_le; ++ unsigned int count = 0; ++ size_t offset; + u8 *image; + int ret; + +@@ -316,28 +798,58 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + } + + get_random_bytes(prng, sizeof(prng)); +- get_random_bytes(nonce, sizeof(nonce)); +- { +- struct apple_aop_bootarg args[] = { +- { APPLE_AOP_BOOTARG_PRNG, prng, sizeof(prng) }, +- { APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le) }, +- { APPLE_AOP_BOOTARG_NONC, nonce, sizeof(nonce) }, +- { APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le) }, +- }; +- +- ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, +- ARRAY_SIZE(args)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_PRNG, &prng, sizeof(prng)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le)); ++ if (aop->hw->nonce) { ++ get_random_bytes(nonce, sizeof(nonce)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NONC, &nonce, sizeof(nonce)); ++ } ++ if (aop->hw->soc_bootargs) { ++ /* A random stack guard with one zero byte */ ++ get_random_bytes(guard, sizeof(guard)); ++ guard[get_random_u32_below(sizeof(guard))] = 0; ++ target_le = cpu_to_le32(aop->target); ++ chip_le = cpu_to_le32(aop->chip_id); ++ revision_le = cpu_to_le32(aop->chip_revision); ++ wrapper_le = cpu_to_le64(aop->akf_base); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_AOPT, ++ &target_le, sizeof(target_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NCAL, &ncal, sizeof(ncal)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_WRAD, ++ &wrapper_le, sizeof(wrapper_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOC, ++ &chip_le, sizeof(chip_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOCR, ++ &revision_le, sizeof(revision_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_STKG, &guard, sizeof(guard)); + } ++ ++ ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, count); + if (ret) { + dev_err(aop->dev, "unexpected boot arguments in %s: %d\n", main_name, ret); + } else { +- memcpy_toio(aop->sram, image, main_fw->size); +- memset_io(aop->sram + main_fw->size, 0, aop->sram_size - main_fw->size); ++ /* The SRAM takes 32-bit accesses, as Apple's loader makes them. */ ++ if (aop->spi0.ran) { ++ ret = a9_aop_spi0_gate(aop, "pre-copy-final", "PRECOPY", false); ++ if (!ret) ++ ret = a9_aop_copy_verify_wake10(aop, image, main_fw->size); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "after-copy", "AFTERCOPY", true); ++ if (ret) ++ goto out_image; ++ } else { ++ __iowrite32_copy(aop->sram, image, main_fw->size / 4); ++ } ++ for (offset = main_fw->size; offset + 4 <= aop->sram_size; offset += 4) ++ writel_relaxed(0, aop->sram + offset); + } + ++out_image: + kfree_sensitive(image); + memzero_explicit(prng, sizeof(prng)); + memzero_explicit(nonce, sizeof(nonce)); ++ memzero_explicit(guard, sizeof(guard)); + memzero_explicit(&ecid_le, sizeof(ecid_le)); + out_ext: + release_firmware(ext_fw); +@@ -448,7 +960,8 @@ static void apple_aop_announce(struct apple_aop_ep *ep, u32 channel, const void + const struct epic_service_announce *announce = data; + struct apple_aop_register_work *rw; + +- if (len < sizeof(*announce)) ++ /* Only the name and channel are used; the S8000's announcements are shorter. */ ++ if (len < offsetofend(struct epic_service_announce, channel)) + return; + + rw = kzalloc_obj(*rw); +@@ -475,6 +988,8 @@ static void apple_aop_report(struct apple_aop_ep *ep, u32 channel, u16 subtype, + struct apple_aop_listener *listener; + bool handled = false; + ++ if (ep->index == 0x21 && subtype == 0xc2) ++ aop->accel_reports++; + mutex_lock(&aop->listeners_lock); + list_for_each_entry(listener, &aop->listeners, list) { + if (listener->endpoint == ep->index && listener->channel == channel) { +@@ -631,6 +1146,7 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + + switch (FIELD_GET(AFK_MSG_OPCODE, msg)) { + case AFK_OPC_INIT: ++ ep->initialized = true; + apple_aop_send(ep, AFK_OPC_INIT_ACK, 0); + break; + case AFK_OPC_GET_BUF: +@@ -651,7 +1167,10 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + break; + case AFK_OPC_INIT_UNK: + case AFK_OPC_START_ACK: ++ break; + case AFK_OPC_SHUTDOWN_ACK: ++ case AFK_OPC_HIBERNATE_ACK: ++ complete(&ep->stopped); + break; + default: + dev_dbg(aop->dev, "endpoint 0x%02x: message 0x%llx\n", endpoint, msg); +@@ -662,7 +1181,8 @@ static void apple_aop_crashed(void *cookie, const void *crashlog, size_t crashlo + { + struct apple_aop *aop = cookie; + +- dev_err(aop->dev, "firmware crashed\n"); ++ aop->iop_crashed = true; ++ dev_err(aop->dev, "firmware crashed iop_crashed=1\n"); + } + + static const struct apple_rtkit_ops apple_aop_rtkit_ops = { +@@ -767,6 +1287,8 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + + if (!ep) + return -EINVAL; ++ if (service->aop->warm && READ_ONCE(service->aop->iop_crashed)) ++ return -EIO; + init_completion(&call.done); + + mutex_lock(&ep->lock); +@@ -797,6 +1319,58 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + } + EXPORT_SYMBOL_GPL(apple_aop_call); + ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len) ++{ ++ u8 buf[8 + APPLE_AOP_PROPERTY_MAX]; ++ int ret; ++ ++ if (len > APPLE_AOP_PROPERTY_MAX) ++ return -EINVAL; ++ put_unaligned_le32(0, buf); ++ put_unaligned_le32(key, buf + 4); ++ memcpy(buf + 8, value, len); ++ ++ ret = apple_aop_call(service, APPLE_AOP_SET_PROPERTY, buf, 8 + len, NULL, 0); ++ return ret > 0 ? -EIO : ret; ++} ++EXPORT_SYMBOL_GPL(apple_aop_set_property); ++ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count) ++{ ++ unsigned long needed = 0, applied = 0; ++ __le32 state; ++ unsigned int i; ++ int len, ret; ++ ++ for (i = 0; i < count; i++) { ++ const void *blob; ++ ++ if (cal[i].group) ++ needed |= BIT(cal[i].group); ++ blob = of_get_property(dev->of_node, cal[i].property, &len); ++ if (!blob) ++ continue; ++ ret = apple_aop_set_property(service, cal[i].key, blob, len); ++ if (ret) { ++ dev_warn(dev, "%s not applied: %d\n", cal[i].property, ret); ++ continue; ++ } ++ if (cal[i].group) ++ applied |= BIT(cal[i].group); ++ } ++ ++ if (needed && applied == needed) { ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_COMPLETE); ++ } else { ++ dev_warn(dev, "factory calibration incomplete\n"); ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_PARTIAL); ++ } ++ return apple_aop_set_property(service, APPLE_AOP_PROP_CALIBRATION, &state, sizeof(state)); ++} ++EXPORT_SYMBOL_GPL(apple_aop_calibrate); ++ + /* Driver */ + + static void apple_aop_free_buffers(struct apple_aop *aop) +@@ -825,27 +1399,35 @@ static int apple_aop_probe(struct platform_device *pdev) + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); + mutex_init(&aop->children_lock); + INIT_LIST_HEAD(&aop->children); + +- aop->akf = devm_platform_ioremap_resource_byname(pdev, "akf"); ++ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "akf"); ++ aop->akf = devm_ioremap_resource(dev, res); + if (IS_ERR(aop->akf)) + return PTR_ERR(aop->akf); ++ aop->akf_base = res->start; + aop->control = devm_platform_ioremap_resource_byname(pdev, "control"); + if (IS_ERR(aop->control)) + return PTR_ERR(aop->control); + aop->sram = devm_platform_get_and_ioremap_resource(pdev, 2, &res); + if (IS_ERR(aop->sram)) + return PTR_ERR(aop->sram); ++ aop->sram_base = res->start; + aop->sram_size = resource_size(res); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + +- if (apple_aop_is_started(aop)) ++ ret = apple_aop_warm_prepare(aop); ++ if (ret) ++ return ret; ++ ++ if (!aop->spi0.ran && apple_aop_is_started(aop)) + return dev_err_probe(dev, -EBUSY, + "already started; the AOP starts once per boot\n"); + +@@ -853,6 +1435,17 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + return dev_err_probe(dev, ret, "no apple,ecid from the bootloader\n"); + ++ if (aop->hw->soc_bootargs) { ++ ret = of_property_read_u32(np, "apple,aop-target", &aop->target); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-id", &aop->chip_id); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-revision", &aop->chip_revision); ++ if (ret) ++ return dev_err_probe(dev, ret, ++ "no AOP target or chip ID from the bootloader\n"); ++ } ++ + aop->eps = devm_kcalloc(dev, AFK_ENDPOINT_COUNT, sizeof(*aop->eps), GFP_KERNEL); + if (!aop->eps) + return -ENOMEM; +@@ -860,6 +1453,7 @@ static int apple_aop_probe(struct platform_device *pdev) + aop->eps[i].aop = aop; + aop->eps[i].index = AFK_ENDPOINT_START + i; + mutex_init(&aop->eps[i].lock); ++ init_completion(&aop->eps[i].stopped); + } + + ret = apple_aop_load(aop, ecid); +@@ -867,6 +1461,9 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + goto err_free; + ++ if (aop->hw->sram_window) ++ apple_aop_map_sram(aop); ++ + aop->rtk = apple_rtkit_init(dev, aop, "mbox", 0, &apple_aop_rtkit_ops); + if (IS_ERR(aop->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(aop->rtk), "failed to init RTKit\n"); +@@ -884,10 +1481,24 @@ static int apple_aop_probe(struct platform_device *pdev) + prop = of_find_property(np, "apple,ecid", NULL); + if (prop) + of_remove_property(np, prop); +- writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, +- aop->akf + APPLE_AOP_CPU_CONTROL); ++ if (aop->spi0.ran) { ++ ret = apple_aop_warm_release(aop); ++ if (ret) ++ goto err_halt; ++ } ++ if (aop->spi0.ran) { ++ aop->experiment_step = "warm-run"; ++ ret = aop_wr(aop, APPLE_AOP_CPU_CONTROL, ++ readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN); ++ if (ret) ++ goto err_halt; ++ } else { ++ writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, ++ aop->akf + APPLE_AOP_CPU_CONTROL); ++ } + writel(1, aop->control + APPLE_AOP_CONTROL_AUTO_BOOT); + ++ dev_info(dev, "AOP released warm=%d\n", aop->spi0.ran); + ret = apple_rtkit_wake(aop->rtk); + if (ret) { + dev_err(dev, "firmware did not boot: %d\n", ret); +@@ -900,6 +1511,8 @@ static int apple_aop_probe(struct platform_device *pdev) + dev_warn(dev, "failed to start endpoint 0x%02x: %d\n", i, ret); + } + ++ aop->hello = true; ++ dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); + return 0; + +@@ -929,27 +1542,81 @@ static void apple_aop_shutdown(struct platform_device *pdev) + mutex_lock(&aop->children_lock); + aop->removing = true; + list_for_each_entry_safe(child, tmp, &aop->children, list) { ++ ++ if (aop->warm) { ++ const struct apple_aop_service *service = dev_get_platdata(&child->pdev->dev); ++ static const u32 keys[] = { 0x16, 0x5c, 0 }; ++ ++ /* release6 sets the three accelerometer rates only. */ ++ if (strcmp(service->name, "accel")) ++ continue; ++ __le32 zero = 0; ++ unsigned int i; ++ int ret; ++ ++ for (i = 0; i < ARRAY_SIZE(keys); i++) { ++ ret = apple_aop_set_property(service, keys[i], &zero, sizeof(zero)); ++ if (ret) { ++ aop->shutdown_error = ret; ++ break; ++ } ++ } ++ } ++ /* Shutdown retains listeners, children and DMA grants, as release6 does. ++ * Unregister is remove-time work and can wait in IIO/driver core. ++ */ ++ if (aop->warm) ++ continue; + platform_device_unregister(child->pdev); + list_del(&child->list); + kfree(child); + } + mutex_unlock(&aop->children_lock); + ++ if (aop->warm) { ++ apple_aop_warm_shutdown(aop); ++ return; ++ } + if (!apple_aop_halt(aop)) + dev_err(aop->dev, "failed to halt\n"); + } + ++static const struct apple_aop_hw apple_aop_s8000_hw = { ++ .sram_window = true, ++ .soc_bootargs = true, ++}; ++ ++static const struct apple_aop_hw apple_aop_t8010_hw = { ++ .nonce = true, ++}; ++ + static const struct of_device_id apple_aop_of_match[] = { +- { .compatible = "apple,t8010-aop" }, ++ { .compatible = "apple,s8000-aop", .data = &apple_aop_s8000_hw }, ++ { .compatible = "apple,t8010-aop", .data = &apple_aop_t8010_hw }, + {} + }; + MODULE_DEVICE_TABLE(of, apple_aop_of_match); + ++static ssize_t warm_state_show(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ struct apple_aop *aop = dev_get_drvdata(dev); ++ ++ if (!aop) ++ return sysfs_emit(buf, "owned=0\n"); ++ return sysfs_emit(buf, "owned=%u reload=%u iop_crashed=%u accel_reports=%llu\n", ++ aop->hello, aop->spi0.ran, READ_ONCE(aop->iop_crashed), ++ READ_ONCE(aop->accel_reports)); ++} ++static DEVICE_ATTR_RO(warm_state); ++static struct attribute *apple_aop_attrs[] = { &dev_attr_warm_state.attr, NULL }; ++ATTRIBUTE_GROUPS(apple_aop); ++ + static struct platform_driver apple_aop_driver = { + .driver = { + .name = "apple-aop", + .of_match_table = apple_aop_of_match, + .suppress_bind_attrs = true, ++ .dev_groups = apple_aop_groups, + }, + .probe = apple_aop_probe, + .shutdown = apple_aop_shutdown, +@@ -957,5 +1624,5 @@ static struct platform_driver apple_aop_driver = { + module_platform_driver(apple_aop_driver); + + MODULE_AUTHOR("Dalton Alexandre "); +-MODULE_DESCRIPTION("Apple T8010 Always-On Processor driver"); ++MODULE_DESCRIPTION("Apple S8000 and T8010 Always-On Processor driver"); + MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c +index 725ab3fa7..d7928621a 100644 +--- a/drivers/soc/apple/rtkit.c ++++ b/drivers/soc/apple/rtkit.c +@@ -4,6 +4,8 @@ + * Copyright (C) The Asahi Linux Contributors + */ + ++#include ++#include + #include "rtkit-internal.h" + + enum { +@@ -867,6 +869,54 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) + } + EXPORT_SYMBOL_GPL(apple_rtkit_boot); + ++/* Unlike shutdown(), retain all grants until the caller halts the CPU. ++ * Reinit frees DMA buffers and is unsafe before that point on S8000. ++ */ ++static void apple_rtkit_grants_hash(struct apple_rtkit *rtk, u8 *digest) ++{ ++ struct apple_rtkit_shmem *buffers[] = { ++ &rtk->ioreport_buffer, &rtk->crashlog_buffer, ++ &rtk->oslog_buffer, &rtk->syslog_buffer, ++ }; ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ for (i = 0; i < ARRAY_SIZE(buffers); i++) ++ if (buffers[i]->buffer) ++ sha256_update(&ctx, buffers[i]->buffer, buffers[i]->size); ++ sha256_final(&ctx, digest); ++} ++ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk) ++{ ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ if (rtk->crashed) ++ return -EIO; ++ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_QUIESCED); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=ap-ack state=0x10\n"); ++ ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=iop-ack state=0x201\n"); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, before); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, after); ++ if (rtk->crashed || memcmp(before, after, sizeof(before))) ++ return -EBUSY; ++ /* No more callback can race the caller's halt/drain. Do not reinit. */ ++ apple_mbox_stop(rtk->mbox); ++ flush_workqueue(rtk->wq); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_rtkit_shutdown_preserve); ++ + int apple_rtkit_shutdown(struct apple_rtkit *rtk) + { + int ret; +diff --git a/include/linux/soc/apple/aop.h b/include/linux/soc/apple/aop.h +index 8c8bc71b6..b5188a0b1 100644 +--- a/include/linux/soc/apple/aop.h ++++ b/include/linux/soc/apple/aop.h +@@ -7,6 +7,8 @@ + + #include + ++struct device; ++ + struct apple_aop; + + /* +@@ -40,4 +42,30 @@ void apple_aop_unlisten(const struct apple_aop_service *service); + int apple_aop_call(const struct apple_aop_service *service, u16 subtype, + const void *data, size_t len, void *reply, size_t reply_len); + ++/* The EPIC notification subtype that sets a service property */ ++#define APPLE_AOP_SET_PROPERTY 0x4 ++ ++/* Set property @key of a service to @len (at most 64) bytes at @value. */ ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len); ++ ++/* ++ * A factory calibration blob of a sensor: the bootloader copies it from the ++ * Apple device tree into the sensor's node as @property, and the firmware ++ * takes it as property @key. The calibration is complete once a blob of each ++ * @group (1 to 31) was applied; group 0 blobs are not needed for that. ++ */ ++struct apple_aop_calibration { ++ const char *property; ++ u32 key; ++ u8 group; ++}; ++ ++/* ++ * Send the factory calibration found in @dev's node, then tell the firmware ++ * whether it is complete. Some firmware (S8000) sends no reports before. ++ */ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count); ++ + #endif /* _LINUX_SOC_APPLE_AOP_H_ */ +diff --git a/include/linux/soc/apple/pmgr-observe.h b/include/linux/soc/apple/pmgr-observe.h +new file mode 100644 +index 000000000..477b7fa22 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-observe.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++#define _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++ ++#include ++ ++struct device_node; ++ ++struct apple_pmgr_observation { ++ u32 raw; ++ int genpd_status; ++}; ++ ++/* ++ * Diagnostic only. Nodes must be in child-before-parent lock order, with no ++ * duplicates (maximum five). Takes no power/runtime-PM reference. read() must ++ * not sleep or acquire genpd locks; it runs with IRQs disabled and all domain ++ * locks held, only if every genpd and hardware actual/target state is ON and ++ * reset/disable/parent-off are clear. Hardware auto-PM remains untouched and is ++ * recorded in raw; locks exclude software transitions, not autonomous gating. ++ * -EHOSTDOWN means read() was skipped. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data); ++ ++#endif +diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h +index 1f9ebb618..c8e8df255 100644 +--- a/include/linux/soc/apple/rtkit.h ++++ b/include/linux/soc/apple/rtkit.h +@@ -124,6 +124,8 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); + * Shutdown the co-processor + */ + int apple_rtkit_shutdown(struct apple_rtkit *rtk); ++/* AP 0x10 / IOP 0x201, stable system RAM, stop RX; retain every grant. */ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk); + + /* + * Stop the mailbox of a shut-down co-processor, so that its power domain can +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index 3fcdf4b4e..fcc51b699 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4589,6 +4589,7 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) + { + return __pr_flush(NULL, timeout_ms, reset_on_progress); + } ++EXPORT_SYMBOL_GPL(pr_flush); + + /* + * Delayed printk version, for scheduler-internal messages: diff --git a/tests/test_j71t_data.py b/tests/test_j71t_data.py new file mode 100644 index 00000000..c6d83c08 --- /dev/null +++ b/tests/test_j71t_data.py @@ -0,0 +1,31 @@ +"""J71t firmware recipe: the AOP images.""" +import json +from pathlib import Path +import tomllib +import unittest + +HARDWARE = Path(__file__).resolve().parents[1] +DEVICE = HARDWARE / 'devices/j71t/device.toml' + + +class J71tData(unittest.TestCase): + def setUp(self): + device = tomllib.loads(DEVICE.read_text()) + self.recipe = json.loads((DEVICE.parent / device['firmware']['recipe']).read_text()) + + def test_aop_images_come_from_the_board_firmware(self): + aop = self.recipe['inputs']['aop.im4p'] + self.assertEqual(aop['ipsw'], {'zip': 'Firmware/AOP/aopfw-ipad6faop.im4p'}) + self.assertEqual(aop['sha256'], '5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa') + for name, role, digest in ( + ('aop-ipad6f.bin', 'AOP', 'af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8'), + ('aop-ipad6f-ext.bin', 'AOP_ext', '2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22')): + output, = [o for o in self.recipe['outputs'] if o['destination'] == 'lib/firmware/apple/' + name] + self.assertEqual(output['sha256'], digest) + self.assertEqual(output['steps'], [{'step': 'im4p-payload@1', 'input': 'aop.im4p'}, + {'step': 'aop-image@1', 'coprocessor': role}]) + self.assertFalse((HARDWARE / output['destination']).exists()) + + +if __name__ == '__main__': + unittest.main() From 6fd27377fce7ac055f4b978a38f9889fe965270a Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 09/33] usb: keep per-device system and RAM serials stable across boots Assisted-by: LLM --- boot/initramfs/root/init | 3 +- chips/s8003.toml | 3 ++ devices/j71t/device.toml | 3 ++ docs/device-protocol.md | 18 +++++++++ pyproject.toml | 2 +- src/orchard/apple/linux_usb.py | 16 ++++++-- src/orchard/apple/usb.py | 12 +++--- src/orchard/image/platform.py | 1 + src/orchard/ram_session.py | 43 ++++++++++++++------ src/orchard/usb-identity | 19 +++++++++ src/orchard/usb-session | 2 +- tests/test_device_state.py | 18 ++++++++- tests/test_identity.py | 73 ++++++++++++++++++++++++++++++++++ tests/test_j71t_mac_current.py | 42 ++++++++++++++++++- tests/test_platform_layer.py | 1 + tests/test_ram.py | 2 + tests/test_system_session.py | 57 ++++++++++++++++++++++++++ 17 files changed, 287 insertions(+), 28 deletions(-) create mode 100644 src/orchard/usb-identity diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index c852dd68..617e2607 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -109,6 +109,7 @@ usb_gadget() { # allowing the generic composite driver to deliver the grant. device_class=0xef device_subclass=0x02 device_protocol=0x01 fi + serial=$(sh /etc/orchard/usb-identity) || return 1 mkdir "$g" && echo 0x05ac > "$g/idVendor" && echo "$pid" > "$g/idProduct" && @@ -118,7 +119,7 @@ usb_gadget() { mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" && echo 'Orchard Linux' > "$g/strings/0x409/manufacturer" && echo "${name:-Linux}" > "$g/strings/0x409/product" && - echo "BOOT:$(cat /proc/sys/kernel/random/boot_id) BOARD:${board#apple,}" > "$g/strings/0x409/serialnumber" && + echo "$serial" > "$g/strings/0x409/serialnumber" && echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" && echo 500 > "$g/configs/c.1/MaxPower" && mkdir "$g/functions/ncm.usb0" "$g/functions/acm.usb0" && diff --git a/chips/s8003.toml b/chips/s8003.toml index 4ea9cf07..397b9902 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -21,3 +21,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: mac-current. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/mac-current.md. + +# Local capability delta: usb-stable-serial. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/usb-stable-serial.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 0c2a2e13..c0de3549 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -60,3 +60,6 @@ camera = "missing" # Local capability delta: mac-current. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/mac-current.md. + +# Local capability delta: usb-stable-serial. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/usb-stable-serial.md. diff --git a/docs/device-protocol.md b/docs/device-protocol.md index f15bfac0..518087d8 100644 --- a/docs/device-protocol.md +++ b/docs/device-protocol.md @@ -15,6 +15,24 @@ the host and device side of an install always come from the same CLI version. - Private bundle: exactly the device firmware data plus `mixin.json`; the install script writes it to the installed root's `/usr/lib/firmware`. +## USB identity + +Installed-system and standard RAM gadgets use `BOARD: ID:` as +their USB serial number. The ID is the first 12 hex characters of SHA-256 over +`orchard-` followed by the device-tree `serial-number` without NUL +padding. It identifies the unit across boots without publishing the hardware +serial. No descriptor changes per boot, so macOS can retain accessory approval. +Trees without a serial number warn and retain the legacy `BOOT:` identity to +keep rescue USB available; those still need approval each boot. + +For RAM sessions, ACM returns `/proc/sys/kernel/random/boot_id` and the SSH +host public key together on every connection. The host validates the boot ID +and pins that boot's key. Installed systems use their persistent SSH keys and +do not allocate per-boot host state. Recovery and install keep their existing +`ECID:`/`BOOT:` serials and bootstrap/status checks. The install-to-system +handoff also retains `BOOT:`: the CLI's install add-on can run on older initrds +without the stable-identity helper. Normal installed boots use the new format. + ## Operations `orchard-device OPERATION [BASE64_JSON]`; requests may include `protocol: 1`. diff --git a/pyproject.toml b/pyproject.toml index 0ec447e5..c8dd1a07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ package-dir = {"" = "src"} where = ["src"] [tool.setuptools.package-data] -orchard = ["py.typed", "usb-session"] +orchard = ["py.typed", "usb-session", "usb-identity"] [dependency-groups] dev = ["mypy==1.18.2", "jsonschema>=4.25,<5"] diff --git a/src/orchard/apple/linux_usb.py b/src/orchard/apple/linux_usb.py index b3a95803..2098ed9c 100644 --- a/src/orchard/apple/linux_usb.py +++ b/src/orchard/apple/linux_usb.py @@ -20,6 +20,12 @@ def serial_tags(serial): 'pwned': pwned.group(1) if pwned else ('checkra1n' if 'YOLO:checkra1n' in serial else None)} +def stable_id(serial): + """Hashed hardware identity of an Orchard gadget, absent on legacy boots.""" + match = re.search(r'\bID:([0-9a-fA-F]{12})\b', serial) + return match.group(1).lower() if match else None + + def devices(root=Path('/sys/bus/usb/devices')): found = [] for node in root.iterdir(): @@ -36,10 +42,11 @@ def devices(root=Path('/sys/bus/usb/devices')): chip = re.search(r'\bCPID:([0-9a-fA-F]+)\b', serial) board = re.search(r'\bBDID:([0-9a-fA-F]+)\b', serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) - # A compatibility PID also belongs to real Apple hardware. Only our - # existing boot/profile identity makes it an Orchard system. - mac_current = (pid == 0x1014 and boot is not None and - profile is not None and profile.group(1) == 'j71t') + usb_id = stable_id(serial) + # 1014 also belongs to real Apple devices. Require Orchard's board + # and stable ID (or the legacy BOOT tag), never the PID alone. + mac_current = (pid == 0x1014 and profile is not None and profile.group(1) == 'j71t' + and (boot is not None or usb_id is not None)) ports = {str(Path('/dev') / p.name) for p in node.glob('**/tty/ttyACM*')} if len(ports) > 1: raise ValueError('Ambiguous USB serial ports on a device') @@ -47,6 +54,7 @@ def devices(root=Path('/sys/bus/usb/devices')): found.append(dict(location=node.name, mode='system' if mac_current else MODES.get(pid, 'other'), ecid=int(ecid.group(1), 16) if ecid else None, boot_id=str(uuid.UUID(boot.group(1))) if boot else None, + usb_id=usb_id, chip=int(chip.group(1), 16) if chip else None, board=int(board.group(1), 16) if board else None, profile=profile.group(1) if profile else None, product=product, diff --git a/src/orchard/apple/usb.py b/src/orchard/apple/usb.py index 7a63d431..54804d7f 100644 --- a/src/orchard/apple/usb.py +++ b/src/orchard/apple/usb.py @@ -5,7 +5,7 @@ import sys import uuid -from .linux_usb import devices as linux_devices, m1n1_ports as linux_m1n1_ports, serial_tags +from .linux_usb import devices as linux_devices, m1n1_ports as linux_m1n1_ports, serial_tags, stable_id from orchard.state import EndpointUnavailable # 0x4144: a RAM payload's USB session (orchard/usb-session). @@ -38,10 +38,11 @@ def devices(): serial = node.get('USB Serial Number') or '' boot = re.search(r'\bBOOT:([0-9a-fA-F-]{36})\b', serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) - # A compatibility PID also belongs to real Apple hardware. Only our - # existing boot/profile identity makes it an Orchard system. - mac_current = (node.get('idProduct') == 0x1014 and boot is not None and - profile is not None and profile.group(1) == 'j71t') + usb_id = stable_id(serial) + # 1014 also belongs to real Apple devices. Require Orchard's board + # and stable ID (or the legacy BOOT tag), never the PID alone. + mac_current = (node.get('idProduct') == 0x1014 and profile is not None and profile.group(1) == 'j71t' + and (boot is not None or usb_id is not None)) fields = [re.search(r'\b' + label + r':([0-9a-fA-F]+)\b', serial) for label in ('ECID', 'CPID', 'BDID')] descendants = list(node.get('IORegistryEntryChildren', [])) @@ -58,6 +59,7 @@ def devices(): found.append({'location': location, 'mode': 'system' if mac_current else MODES.get(node.get('idProduct'), 'other'), 'ecid': int(fields[0].group(1), 16) if fields[0] else None, 'boot_id': str(uuid.UUID(boot.group(1))) if boot else None, + 'usb_id': usb_id, 'chip': int(fields[1].group(1), 16) if fields[1] else None, 'board': int(fields[2].group(1), 16) if fields[2] else None, 'profile': profile.group(1) if profile else None, diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index b39ae7bb..ee55ae55 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -111,6 +111,7 @@ def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_ entries = {name: (directory, b'') for name in ('bin', 'dev', 'proc', 'sys', 'etc', 'etc/orchard', 'newroot', 'lib', 'lib/modules', 'lib/modules/' + release)} entries['init'] = (stat.S_IFREG | 0o755, (hardware / 'boot/initramfs/root/init').read_bytes()) + entries['etc/orchard/usb-identity'] = (stat.S_IFREG | 0o644, (hardware / 'src/orchard/usb-identity').read_bytes()) entries['bin/busybox'] = (stat.S_IFREG | 0o755, Path(busybox).read_bytes()) entries['etc/orchard/modules'] = (stat.S_IFREG | 0o644, ('\n'.join(loaded) + '\n').encode()) if usb: diff --git a/src/orchard/ram_session.py b/src/orchard/ram_session.py index 1d1439ba..26c58889 100644 --- a/src/orchard/ram_session.py +++ b/src/orchard/ram_session.py @@ -18,6 +18,7 @@ import subprocess import sys import time +import uuid from .serial import SerialError, SerialPort from .state import cache_root, private_directory @@ -49,7 +50,9 @@ def identity(): def payload_files(): """RAM-root files that enable the USB session in a payload.""" helper = resources.files('orchard') / 'usb-session' - return {HELPER: Path(str(helper)), AUTHORIZED: identity().with_suffix('.pub')} + return {HELPER: Path(str(helper)), + 'usr/libexec/orchard/usb-identity': Path(str(resources.files('orchard') / 'usb-identity')), + AUTHORIZED: identity().with_suffix('.pub')} def target(): @@ -69,17 +72,20 @@ def target(): def _check(device): + # Stable descriptors identify the unit, not its boot. RAM SSH is pinned to + # the per-boot key read on the open ACM endpoint; a reboot invalidates it. + fields = ('location', 'mode', 'profile', 'usb_id', 'boot_id', 'serial_port') def check(): from .apple import usb - if not any(item['location'] == device['location'] and item.get('boot_id') == device.get('boot_id') + if not any(all(item.get(key) == device.get(key) for key in fields) for item in usb.devices()): raise ValueError('The RAM payload disconnected or rebooted') return check def connect(device=None, user=None, *, host=None, ram=False): - """SSH argv for the running payload (its host key pinned over serial once - per boot) or installed system (as its user, with this computer's key).""" + """SSH argv for the running payload (its boot and host key read over ACM) + or installed system (as its user, with this computer's key).""" if ram and (not host or user): raise ValueError('--ram requires --host and connects as root, not --user') if host: @@ -106,20 +112,33 @@ def connect(device=None, user=None, *, host=None, ram=False): argv = computer_key.ssh_argv(user) wait_authenticated(argv, _check(device), seconds=20) return argv - boot = device.get('boot_id') - if not boot: + if not (device.get('boot_id') or device.get('usb_id')): raise ValueError('The RAM payload has no boot identity; rebuild it with a current orchard') check = _check(device) + # Query every connection: the same stable USB serial can name a new boot. + # ACM supplies both boot ID and trusted host key without changing any USB + # descriptor, and also works with payloads built before stable identities. + with SerialPort(device['serial_port'], check) as serial: + code, text = serial.run('cat /proc/sys/kernel/random/boot_id && ' + 'cat /run/orchard/ssh_host_ed25519_key.pub', timeout=15) + if code: + raise ValueError('No SSH in this payload; its /init must run ' + '/' + HELPER + ' (or build without transport = false)') + lines = text.strip().splitlines() + try: + boot = str(uuid.UUID(lines[0].strip())) + except (ValueError, IndexError) as error: + raise ValueError('Invalid RAM boot identity over USB serial') from error + if device.get('boot_id') and boot != device['boot_id']: + raise ValueError('The RAM payload disconnected or rebooted') + pinned = HOST + ' ' + host_key('\n'.join(lines[1:])) + '\n' hosts = _directory() / f'known_hosts-{boot}' + if hosts.is_file() and hosts.read_text() != pinned: + raise ValueError('The RAM SSH host key changed within the same boot') if not hosts.is_file(): - with SerialPort(device['serial_port'], check) as serial: - code, text = serial.run('cat /run/orchard/ssh_host_ed25519_key.pub', timeout=15) - if code: - raise ValueError('No SSH in this payload; its /init must run ' - '/' + HELPER + ' (or build without transport = false)') for old in _directory().glob('known_hosts-*'): old.unlink() - hosts.write_text(HOST + ' ' + host_key(text) + '\n') + hosts.write_text(pinned) hosts.chmod(0o600) argv = ssh_argv(identity(), hosts, HOST, 22) wait_authenticated(argv, check, seconds=20) diff --git a/src/orchard/usb-identity b/src/orchard/usb-identity new file mode 100644 index 00000000..b6962ec8 --- /dev/null +++ b/src/orchard/usb-identity @@ -0,0 +1,19 @@ +#!/bin/sh +# Stable USB identity, shared by installed-system and RAM gadgets. The boot +# stage copies serial-number from Apple's tree; never publish it in the clear. +set -eu +tree=${ORCHARD_DEVICE_TREE:-/proc/device-tree} +board=$(tr '\000' '\n' < "$tree/compatible" | head -1) +serial= +if [ -r "$tree/serial-number" ]; then + serial=$(tr -d '\000' < "$tree/serial-number") +fi +if [ -n "$serial" ]; then + id=$(printf 'orchard-%s' "$serial" | sha256sum | cut -c1-12) + printf 'BOARD:%s ID:%s\n' "${board#apple,}" "$id" +else + # Keep rescue serial and SSH reachable on trees without a hardware ID. + # This legacy fallback cannot retain the host's accessory approval. + echo 'orchard: no device-tree serial-number; using per-boot USB identity' >&2 + printf 'BOOT:%s BOARD:%s\n' "$(cat /proc/sys/kernel/random/boot_id)" "${board#apple,}" +fi diff --git a/src/orchard/usb-session b/src/orchard/usb-session index aadde6a1..94c3e081 100755 --- a/src/orchard/usb-session +++ b/src/orchard/usb-session @@ -17,7 +17,7 @@ echo 0x05ac > "$g/idVendor" echo 0x4144 > "$g/idProduct" mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" echo "Orchard $name" > "$g/strings/0x409/product" -echo "BOOT:$(cat /proc/sys/kernel/random/boot_id)" > "$g/strings/0x409/serialnumber" +sh /usr/libexec/orchard/usb-identity > "$g/strings/0x409/serialnumber" echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" # Descriptor only: not charging capability or PMIC policy. echo 100 > "$g/configs/c.1/MaxPower" diff --git a/tests/test_device_state.py b/tests/test_device_state.py index 421dcd32..419a07e6 100644 --- a/tests/test_device_state.py +++ b/tests/test_device_state.py @@ -16,6 +16,7 @@ BOOT_A = '11111111-1111-4111-8111-111111111111' BOOT_B = '22222222-2222-4222-8222-222222222222' +USB_ID = 'f09dfe4c604a' class DeviceIsolation(unittest.TestCase): @@ -107,16 +108,29 @@ def test_linux_ram_payload_is_available_to_ram_session_tools(self): node = self.root/'1-2' node.mkdir() for name,value in {'idVendor':'05ac','idProduct':'4144','product':'Orchard charger-test', - 'serial':f'BOOT:{BOOT_B}'}.items(): + 'serial':f'BOARD:j71b ID:{USB_ID}'}.items(): (node/name).write_text(value) (node/'iface/tty/ttyACM0').mkdir(parents=True) rows = linux_usb.devices(self.root) self.assertEqual(len(rows),1) self.assertEqual(rows[0]['mode'],'ram') - self.assertEqual(rows[0]['boot_id'],BOOT_B) + self.assertIsNone(rows[0]['boot_id']) + self.assertEqual(rows[0]['usb_id'],USB_ID) self.assertEqual(rows[0]['serial_port'],'/dev/ttyACM0') self.assertIsNone(rows[0]['ecid']) + def test_macos_stable_identity_has_no_descriptor_boot_id(self): + for pid, mode in ((0x4144, 'ram'), (0x4145, 'system')): + with self.subTest(mode=mode): + raw = plistlib.dumps([{'IOObjectClass':'IOUSBHostDevice', 'idVendor':0x05ac, + 'idProduct':pid, 'locationID':123, + 'USB Serial Number':f'BOARD:j71b ID:{USB_ID}'}]) + with patch.object(usb.sys,'platform','darwin'), patch.object(usb.subprocess,'run',return_value= + subprocess.CompletedProcess([],0,raw,b'')): + row, = usb.devices() + self.assertEqual((row['mode'], row['profile'], row['usb_id']), (mode, 'j71b', USB_ID)) + self.assertIsNone(row['boot_id']) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_identity.py b/tests/test_identity.py index 514c2d9e..5a9524cc 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -1,9 +1,12 @@ """Boot identity publication from bounded ADT fixtures; no device activation.""" import importlib.util +import hashlib import json import os from pathlib import Path +import re import stat +import subprocess import struct import sys import tempfile @@ -107,5 +110,75 @@ def test_gadget_publishes_identity_before_binding_usb(self): self.assertLess(source.index('/serialnumber'),source.index('"$g/UDC"')) +class GadgetIdentity(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.tree = self.root / 'proc/device-tree' + self.tree.mkdir(parents=True) + (self.tree / 'compatible').write_bytes(b'apple,j71t\0apple,s8003\0') + boot = self.root / 'proc/sys/kernel/random/boot_id' + boot.parent.mkdir(parents=True) + boot.write_text(BOOT) + self.script = (ROOT / 'src/orchard/usb-identity').read_text().replace( + '/proc/sys/kernel/random/boot_id', str(boot)) + + def serial(self): + return subprocess.run(['sh', '-ec', self.script], check=True, capture_output=True, text=True, + env={**os.environ, 'ORCHARD_DEVICE_TREE': str(self.tree)}) + + def test_hardware_serial_is_hashed_stable_across_boots_and_distinct_per_unit(self): + number = self.tree / 'serial-number' + number.write_bytes(b'HardwareSerialA\0') + first = self.serial() + expected = hashlib.sha256(b'orchard-HardwareSerialA').hexdigest()[:12] + self.assertEqual(first.stdout.strip(), f'BOARD:j71t ID:{expected}') + self.assertNotIn('HardwareSerialA', first.stdout + first.stderr) + (self.root / 'proc/sys/kernel/random/boot_id').write_text('22222222-2222-4222-8222-222222222222') + self.assertEqual(self.serial().stdout, first.stdout) + number.write_bytes(b'HardwareSerialB\0') + self.assertNotEqual(self.serial().stdout, first.stdout) + + def test_missing_or_empty_serial_preserves_rescue_usb_with_a_warning(self): + for value in (None, b'', b'\0'): + if value is not None: + (self.tree / 'serial-number').write_bytes(value) + result = self.serial() + self.assertEqual(result.stdout.strip(), f'BOOT:{BOOT} BOARD:j71t') + self.assertIn('no device-tree serial-number', result.stderr) + + def test_installed_and_ram_gadgets_use_the_same_stable_serial_before_binding(self): + (self.tree / 'serial-number').write_bytes(b'HardwareSerialA\0') + expected = self.serial().stdout.strip() + for relative in ('boot/initramfs/root/init', 'src/orchard/usb-session'): + with self.subTest(path=relative): + source = (ROOT / relative).read_text() + if relative.endswith('/init'): + start = source.index('usb_gadget() {') + source = source[start:source.index('\nusb_gadget ||', start)] + '\nusb_gadget' + else: + source = source[source.index('name=$(cat'):source.index('\nifconfig ')] + for directory in ('sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', + 'newroot/etc', 'newroot/usr/lib', 'etc', 'usr/libexec/orchard'): + (self.root / directory).mkdir(parents=True, exist_ok=True) + (self.root / 'newroot/etc/os-release').write_text('NAME="Linux"\n') + (self.root / 'etc/hostname').write_text('ram-test\n') + for helper in ('etc/orchard/usb-identity', 'usr/libexec/orchard/usb-identity'): + path = self.root / helper + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.script) + source = re.sub(r'/(sys|newroot|etc|usr)/', lambda m: str(self.root) + m.group(), source) + source = source.replace('mkdir "$g"', 'mkdir -p "$g/functions"') + subprocess.run(['sh', '-ec', 'mount() { :; };\n' + source], check=True, capture_output=True, + env={**os.environ, 'ORCHARD_DEVICE_TREE': str(self.tree)}) + name = 'orchard-system' if relative.endswith('/init') else 'ram-test' + gadget = self.root / 'sys/kernel/config/usb_gadget' / name + self.assertEqual((gadget / 'strings/0x409/serialnumber').read_text().strip(), expected) + self.assertEqual((gadget / 'UDC').read_text().strip(), 'usbdev') + for function in ('ncm.usb0', 'acm.usb0'): + self.assertTrue((gadget / 'configs/c.1' / function).is_symlink()) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_j71t_mac_current.py b/tests/test_j71t_mac_current.py index 2a8a536c..17b875e2 100644 --- a/tests/test_j71t_mac_current.py +++ b/tests/test_j71t_mac_current.py @@ -35,15 +35,17 @@ def test_gadget_profile_preserves_network_serial_and_safe_default(self): root = Path(tmp) for n in ['sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', 'newroot/etc/orchard', 'newroot/usr/lib', 'proc/device-tree', - 'proc/sys/kernel/random']: + 'proc/sys/kernel/random', 'etc/orchard']: (root / n).mkdir(parents=True) (root / 'proc/device-tree/compatible').write_bytes(f'apple,{board}\0apple,s8003\0'.encode()) (root / 'proc/sys/kernel/random/boot_id').write_text(BOOT) (root / 'newroot/etc/os-release').write_text('NAME="Omarchy"\n') if flag: (root / 'newroot/etc/orchard/mac-usb-current').write_text(flag + '\n') + helper = (HW / "src/orchard/usb-identity").read_text().replace("/proc/", str(root) + "/proc/") + (root / "etc/orchard/usb-identity").write_text(helper) run = script - run = re.sub(r'/(sys|newroot|proc)/', lambda m: str(root) + m.group(), run) + run = re.sub(r'/(sys|newroot|proc|etc)/', lambda m: str(root) + m.group(), run) run = run.replace('mkdir \"$g\" &&', 'mkdir -p \"$g/functions\" &&') subprocess.run(['sh', '-ec', 'mount() { :; };\n' + run + '\nusb_gadget'], check=True, capture_output=True) @@ -164,3 +166,39 @@ def test_production_grant_and_charger_policy(self): if __name__ == '__main__': unittest.main() + + +"""A compatibility PID must never classify a real Apple device as Orchard.""" +from pathlib import Path +import plistlib +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from orchard.apple import usb, linux_usb + +BOOT = '12345678-1234-1234-1234-123456789abc' +STABLE = 'f09dfe4c604a' + + +class StableMacCurrent(unittest.TestCase): + def test_compatibility_pid_requires_orchard_identity_on_both_hosts(self): + for serial, mode in [(f'BOOT:{BOOT} BOARD:j71t', 'system'), + (f'BOARD:j71t ID:{STABLE}', 'system'), + ('AppleSerial', 'other'), ('BOARD:j71t', 'other'), + (f'ID:{STABLE}', 'other'), ('BOARD:j71t ID:bad', 'other'), + (f'BOARD:j71b ID:{STABLE}', 'other'), + (f'BOOT:{BOOT} BOARD:j71b', 'other')]: + with self.subTest(serial=serial): + raw = plistlib.dumps([{'IOObjectClass': 'IOUSBHostDevice', 'idVendor': 0x05ac, + 'idProduct': 0x1014, 'locationID': 123, 'USB Serial Number': serial}]) + with patch.object(usb.sys, 'platform', 'darwin'), patch.object(usb.subprocess, 'run', + return_value=subprocess.CompletedProcess([], 0, raw, b'')): + self.assertEqual(usb.devices()[0]['mode'], mode) + with tempfile.TemporaryDirectory() as tmp: + node = Path(tmp) / '1-2' + node.mkdir() + for key, value in [('idVendor', '05ac'), ('idProduct', '1014'), ('serial', serial)]: + (node / key).write_text(value) + self.assertEqual(linux_usb.devices(Path(tmp))[0]['mode'], mode) diff --git a/tests/test_platform_layer.py b/tests/test_platform_layer.py index 9acba731..013e405a 100644 --- a/tests/test_platform_layer.py +++ b/tests/test_platform_layer.py @@ -82,6 +82,7 @@ def test_the_initrd_carries_busybox_and_the_storage_modules_only(self): base = f'lib/modules/{RELEASE}/' self.assertEqual(names['bin/busybox'], b'static busybox') self.assertEqual(names['init'], (ROOT / 'boot/initramfs/root/init').read_bytes()) + self.assertEqual(names['etc/orchard/usb-identity'], (ROOT / 'src/orchard/usb-identity').read_bytes()) self.assertEqual(names['etc/orchard/modules'], b'nvme\n') self.assertEqual(sorted(name for name in names if name.endswith('.ko')), [base + 'kernel/drivers/nvme/host/nvme-core.ko', base + 'kernel/drivers/nvme/host/nvme.ko']) diff --git a/tests/test_ram.py b/tests/test_ram.py index 89c83aaa..6e430d29 100644 --- a/tests/test_ram.py +++ b/tests/test_ram.py @@ -475,6 +475,7 @@ def test_usb_session_is_added_unless_the_project_opts_out(self): ram.build_directory(example, hardware=self.hardware) archive = gzip.decompress((example / 'out/apple-16k/initramfs.cpio.gz').read_bytes()) self.assertIn(b'usr/libexec/orchard/usb-session\0', archive) + self.assertIn(b'usr/libexec/orchard/usb-identity\0', archive) self.assertIn(b'etc/orchard/authorized_keys\0', archive) public = (self.base / 'home/private/ram-ssh/id_ed25519.pub').read_bytes() self.assertIn(public, archive) @@ -484,6 +485,7 @@ def test_usb_session_is_added_unless_the_project_opts_out(self): ram.build_directory(example, hardware=self.hardware) archive = gzip.decompress((example / 'out/apple-16k/initramfs.cpio.gz').read_bytes()) self.assertNotIn(b'usr/libexec/orchard/usb-session\0', archive) + self.assertNotIn(b'usr/libexec/orchard/usb-identity\0', archive) self.assertNotIn(b'etc/orchard/authorized_keys\0', archive) def test_arbitrary_directory_layout_uses_active_hardware_package(self): diff --git a/tests/test_system_session.py b/tests/test_system_session.py index 197c0907..8f89b1d6 100644 --- a/tests/test_system_session.py +++ b/tests/test_system_session.py @@ -9,6 +9,9 @@ SYSTEM = {'location': 1, 'mode': 'system', 'product': 'Omarchy A', 'serial_port': '/dev/cu.x', 'boot_id': '00000000-0000-4000-8000-000000000001', 'profile': 'j71b'} PAYLOAD = {**SYSTEM, 'mode': 'ram', 'product': 'Orchard gpu_001'} +STABLE_PAYLOAD = {**PAYLOAD, 'boot_id': None, 'usb_id': 'f09dfe4c604a'} +BOOT_A = '11111111-1111-4111-8111-111111111111' +BOOT_B = '22222222-2222-4222-8222-222222222222' class SystemSession(unittest.TestCase): @@ -62,6 +65,60 @@ def test_without_a_user_it_says_how_to_get_one(self): with self.assertRaisesRegex(ValueError, '--user'): ram_session.connect(SYSTEM) + def ram_connect(self, reply, device=STABLE_PAYLOAD): + with mock.patch.object(ram_session, '_directory', return_value=Path(self.temp.name)), \ + mock.patch.object(ram_session, 'identity', return_value=Path('/ram-key')), \ + mock.patch.object(ram_session, 'SerialPort') as port, \ + mock.patch.object(ram_session, 'wait_authenticated') as wait: + port.return_value.__enter__.return_value.run.return_value = (0, reply) + argv = ram_session.connect(device) + command = port.return_value.__enter__.return_value.run.call_args.args[0] + self.assertIn('/proc/sys/kernel/random/boot_id', command) + self.assertIn('/run/orchard/ssh_host_ed25519_key.pub', command) + self.assertIn('StrictHostKeyChecking=yes', argv) + wait.assert_called_once() + return argv + + def test_stable_usb_identity_gets_fresh_boot_and_key_from_acm_on_each_connection(self): + # A second connect must not reuse a pin just because USB is unchanged. + first = self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n') + pin_a = Path(self.temp.name) / f'known_hosts-{BOOT_A}' + self.assertEqual(pin_a.read_text(), '172.16.42.1 ssh-ed25519 AAAA\n') + self.assertEqual(pin_a.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n'), first) + second = self.ram_connect(BOOT_B + '\nssh-ed25519 BBBB\n') + self.assertNotEqual(first, second) + self.assertFalse(pin_a.exists()) + self.assertEqual((Path(self.temp.name) / f'known_hosts-{BOOT_B}').read_text(), + '172.16.42.1 ssh-ed25519 BBBB\n') + + def test_legacy_ram_boot_tag_is_still_verified_against_acm(self): + self.ram_connect(PAYLOAD['boot_id'] + '\nssh-ed25519 AAAA\n', device=PAYLOAD) + with self.assertRaisesRegex(ValueError, 'disconnected or rebooted'): + self.ram_connect(BOOT_B + '\nssh-ed25519 AAAA\n', device=PAYLOAD) + + def test_invalid_acm_identity_or_changed_key_does_not_reuse_a_pin(self): + self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n') + for reply, error in [('garbage\nssh-ed25519 BBBB\n', 'Invalid RAM boot'), + ('../../escape\nssh-ed25519 BBBB\n', 'Invalid RAM boot'), + (BOOT_A + '\n', 'exactly one SSH host key'), + (BOOT_A + '\nssh-ed25519 BBBB\n', 'changed within the same boot')]: + with self.subTest(reply=reply), self.assertRaisesRegex(ValueError, error): + self.ram_connect(reply) + self.assertEqual((Path(self.temp.name) / f'known_hosts-{BOOT_A}').read_text(), + '172.16.42.1 ssh-ed25519 AAAA\n') + + def test_stable_usb_check_rejects_another_unit_or_mode_at_the_same_port(self): + check = ram_session._check(STABLE_PAYLOAD) + with mock.patch('orchard.apple.usb.devices', return_value=[STABLE_PAYLOAD]): + check() + for change in ({'usb_id': 'fedcba987654'}, {'mode': 'system'}, + {'profile': 'j71t'}, {'serial_port': '/dev/cu.other'}): + with self.subTest(change=change), \ + mock.patch('orchard.apple.usb.devices', return_value=[{**STABLE_PAYLOAD, **change}]), \ + self.assertRaisesRegex(ValueError, 'disconnected or rebooted'): + check() + if __name__ == '__main__': unittest.main() From 6c1601e6484a28dfa8c73f0e955fbc5e0036d1f4 Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 10/33] Optional resident m1n1 EL3 PSCI Assisted-by: LLM --- boot/m1n1/README.md | 7 + ...resident-EL3-CPU-lifecycle-for-A7-A9.patch | 880 ++++++++++++++++++ boot/m1n1/psci-loader | 29 + chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + tests/test_m1n1_psci.py | 54 ++ 6 files changed, 976 insertions(+) create mode 100644 boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch create mode 100755 boot/m1n1/psci-loader create mode 100644 tests/test_m1n1_psci.py diff --git a/boot/m1n1/README.md b/boot/m1n1/README.md index bd78f93b..04150148 100644 --- a/boot/m1n1/README.md +++ b/boot/m1n1/README.md @@ -28,3 +28,10 @@ gives the same tree as Hoolock's tip before updating `sources.toml`. Scratch stays in ignored `boot/m1n1/.work//`. `--resume` reuses it only if the recorded patch set is unchanged. + +## Optional resident PSCI on A7-A9 + +Orchard patch 0014 adds the linux-a9 opt-in EL3 PSCI shim. It requires +`psci=1` as a payload variable and entry at EL3; EL1 keeps the spin table. +There is no per-board default. See [J71t loader instructions](../../devices/j71t/README.md) +for `psci-loader`, launch/RAM framing, Pongo evidence and the cold-boot gate. diff --git a/boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch b/boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch new file mode 100644 index 00000000..a677f9a7 --- /dev/null +++ b/boot/m1n1/patches/orchard/0014-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch @@ -0,0 +1,880 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sat, 7 Nov 2026 15:00:00 +0100 +Subject: [PATCH] psci: Add opt-in resident EL3 CPU lifecycle for A7-A9 + +On A7-A9 m1n1 is entered at EL3 and stays resident: its image and the +per-CPU EL3 stacks are memreserved, and VBAR_EL3 points at its vectors +while Linux runs at EL1. Use that to give Linux PSCI 1.0 CPU_ON, CPU_OFF +and AFFINITY_INFO (plus VERSION, FEATURES and MIGRATE_INFO_TYPE). The +spin table has no cpu_die, so without this the kernel refuses kexec on +an SMP system, can't offline CPUs, and a crash kernel inherits CPUs stuck +in the old kernel. + +CPU_OFF does not power the core down. It cleans and invalidates its L1D, +stops its timers, and waits in WFE at EL3 (MMU off) on a one-line +mailbox. CPU_ON posts an entry point; the core leaves with a clean +I-cache and TLB, SCTLR_EL1 at its reset value, DAIF masked and SP_EL3 on +a static per-CPU stack. SYSTEM_OFF/SYSTEM_RESET, CPU_SUSPEND and +SMCCC_VERSION return NOT_SUPPORTED, so callers fall back to their own +methods and assume SMCCC 1.0. + +Opt-in with the payload variable psci=1, and only used when m1n1 itself +was entered at EL3. kboot then parks the secondaries in the pen (a core +that doesn't park is pruned), sets enable-method = "psci" on every CPU, +adds /psci (method = "smc") and records the re-entry address and the +original boot_args in /chosen. + +Re-entry: _reentry can be jumped to from a kexec'd trampoline (EL1, MMU +off, x0 = boot_args copy). It drops this image's lines from the caches, +flags the re-entry and restarts m1n1, which then skips the (stale) +payload area and runs the proxy. smp_start_secondaries() takes cores +back from the pen with CPU_ON, and the mailboxes and the psci=1 setting +live in .data so they survive the .bss clear. + +Rebase the linux-a9 shim onto Orchard's complete m1n1 series without +changing its runtime behavior. The payload variable is not a kernel +command-line option; a bare proxy loader also needs it appended. + +Assisted-by: LLM +--- + Makefile | 1 + + src/kboot.c | 56 +++++++++++++- + src/main.c | 8 +- + src/payload.c | 3 + + src/psci.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + src/psci.h | 78 ++++++++++++++++++++ + src/psci_asm.S | 170 ++++++++++++++++++++++++++++++++++++++++++ + src/smp.c | 73 ++++++++++++++++++ + src/smp.h | 1 + + src/start.S | 6 +- + src/startup.c | 6 +- + 11 files changed, 626 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index bb72266..4aefb37 100644 +--- a/Makefile ++++ b/Makefile +@@ -166,6 +166,7 @@ OBJECTS := \ + pcie.o \ + pmgr.o \ + proxy.o \ ++ psci.o psci_asm.o \ + ringbuffer.o \ + rtkit.o \ + sart.o \ +diff --git a/src/kboot.c b/src/kboot.c +index b75bb45..2316709 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -21,6 +21,7 @@ + #include "memory.h" + #include "pcie.h" + #include "pmgr.h" ++#include "psci.h" + #include "sep.h" + #include "sio.h" + #include "smc.h" +@@ -574,6 +575,53 @@ static int dt_set_smbios(void) + return 0; + } + ++/* ++ * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * (see psci.c) instead of the spin table, so the kernel can offline them ++ * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. ++ */ ++static int dt_set_psci(void) ++{ ++ static const char compat[] = "arm,psci-1.0\0arm,psci-0.2"; ++ int node; ++ ++ if (!psci_active()) { ++ printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ return 0; ++ } ++ ++ int cpus = fdt_path_offset(dt, "/cpus"); ++ if (cpus < 0) ++ bail("FDT: /cpus node not found in devtree\n"); ++ ++ fdt_for_each_subnode(node, dt, cpus) ++ { ++ if (strncmp(fdt_get_name(dt, node, NULL), "cpu@", 4)) ++ continue; ++ if (fdt_setprop_string(dt, node, "enable-method", "psci")) ++ bail("FDT: couldn't set enable-method\n"); ++ } ++ ++ node = fdt_path_offset(dt, "/psci"); ++ if (node < 0) ++ node = fdt_add_subnode(dt, 0, "psci"); ++ if (node < 0) ++ bail("FDT: couldn't add /psci\n"); ++ if (fdt_setprop(dt, node, "compatible", compat, sizeof(compat)) || ++ fdt_setprop_string(dt, node, "method", "smc")) ++ bail("FDT: couldn't fill /psci\n"); ++ ++ node = fdt_path_offset(dt, "/chosen"); ++ if (node < 0) ++ bail("FDT: /chosen node not found in devtree\n"); ++ if (fdt_setprop_u64(dt, node, "linux-a9,m1n1-reentry", (u64)_reentry) || ++ fdt_setprop(dt, node, "linux-a9,m1n1-boot-args", &cur_boot_args, sizeof(cur_boot_args))) ++ bail("FDT: couldn't set the m1n1 re-entry properties\n"); ++ ++ printf("PSCI: on, enable-method = \"psci\", m1n1 re-entry at %p\n", _reentry); ++ return 0; ++} ++ + static int dt_set_cpus(void) + { + int ret = 0; +@@ -587,6 +635,10 @@ static int dt_set_cpus(void) + if (!pruned_phandles) + bail("FDT: out of memory\n"); + ++ /* Secondaries wait in the PSCI pen; one that can't get there is pruned below */ ++ if (psci_active()) ++ printf("PSCI: parked %d secondary CPU(s)\n", smp_park_secondaries_psci()); ++ + /* Prune CPU nodes */ + int node, cpu = 0; + for (node = fdt_first_subnode(dt, cpus); node >= 0;) { +@@ -618,7 +670,7 @@ static int dt_set_cpus(void) + } else { + printf("FDT: Reserving stack for CPU %d 0x%lx\n", cpu, (uint64_t)secondary_stacks[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks[cpu], SECONDARY_STACK_SIZE); +- if (has_el3()) { ++ if (has_el3() && secondary_stacks_el3[cpu]) { + printf("FDT: Reserving EL3 stack for CPU %d 0x%lx\n", cpu, + (uint64_t)secondary_stacks_el3[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks_el3[cpu], SECONDARY_STACK_SIZE); +@@ -3206,6 +3258,8 @@ int kboot_prepare_dt(void *fdt) + return -1; + if (dt_set_cpus()) + return -1; ++ if (dt_set_psci()) ++ return -1; + if (dt_set_mac_addresses()) + return -1; + if (dt_set_wifi()) +diff --git a/src/main.c b/src/main.c +index d4e828c..3f32380 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -21,6 +21,7 @@ + #include "pcie.h" + #include "pmgr.h" + #include "sep.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "tps6598x.h" +@@ -117,9 +118,12 @@ void run_actions(void) + #endif + #endif + +- printf("Checking for payloads...\n"); ++ if (psci_reentered()) ++ printf("Re-entered from Linux: whatever follows m1n1 is stale, not loading payloads\n"); ++ else ++ printf("Checking for payloads...\n"); + +- if (payload_run() == 0) { ++ if (!psci_reentered() && payload_run() == 0) { + printf("Valid payload found\n"); + return; + } +diff --git a/src/payload.c b/src/payload.c +index 5b1547f..4ffc9e2 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -12,6 +12,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "utils.h" + +@@ -205,6 +206,8 @@ static bool check_var(u8 **p) + mitigations_configure(val); + } else if (IS_VAR("tso=")) { + enable_tso = val[0] == '1'; ++ } else if (IS_VAR("psci=")) { ++ psci_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +new file mode 100644 +index 0000000..2b8ca0d +--- /dev/null ++++ b/src/psci.c +@@ -0,0 +1,229 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++#include "smp.h" ++#include "types.h" ++#include "utils.h" ++ ++struct psci_cpu { ++ volatile u64 state; ++ volatile u64 entry; ++ volatile u64 context; ++ u64 pad[PSCI_CPU_SIZE / 8 - 3]; ++}; ++ ++_Static_assert(sizeof(struct psci_cpu) == PSCI_CPU_SIZE, "psci_cpu layout"); ++ ++/* ++ * Mailboxes, one cache line each, only ever accessed at EL3 with the MMU off. ++ * They live in .data rather than .bss so that a parked CPU's state survives ++ * m1n1 being re-entered from Linux (which clears .bss). ++ */ ++struct psci_cpu psci_cpus[PSCI_MAX_CPUS] ++ __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* Written with the MMU off only (_reentry sets reentry_flag, the first member) */ ++struct { ++ u64 reentry_flag; ++ u64 boot_el3; ++} psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* psci=1 from the payload, kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))); ++ ++/* EL3 stacks a CPU gets when it leaves the pen */ ++static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); ++ ++static bool psci_have_el3; ++static bool reentered; ++ ++/* EL3 side */ ++ ++static inline void psci_line_sync(volatile void *p) ++{ ++ __asm__ volatile("dc civac, %0\n\tdsb sy" ::"r"(p) : "memory"); ++} ++ ++static int psci_mpidr_index(u64 mpidr) ++{ ++ mpidr &= 0xff00ffffffUL; ++ ++ /* A7-A9: one cluster, Aff0 is the core */ ++ if (mpidr >= PSCI_MAX_CPUS) ++ return -1; ++ ++ return mpidr; ++} ++ ++static s64 psci_cpu_off(void) ++{ ++ int idx = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (idx < 0) ++ return PSCI_RET_DENIED; ++ ++ psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); ++} ++ ++static s64 psci_cpu_on(u64 mpidr, u64 entry, u64 context) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || !entry) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ struct psci_cpu *c = &psci_cpus[idx]; ++ ++ psci_line_sync(c); ++ switch (c->state) { ++ case PSCI_AFF_OFF: ++ break; ++ case PSCI_AFF_ON_PENDING: ++ return PSCI_RET_ON_PENDING; ++ default: ++ /* Running, or never seen by us (still in m1n1 or never started) */ ++ return PSCI_RET_ALREADY_ON; ++ } ++ ++ /* State before entry: the pen sets ON as soon as it sees the entry */ ++ c->context = context; ++ c->state = PSCI_AFF_ON_PENDING; ++ psci_line_sync(c); ++ c->entry = entry; ++ psci_line_sync(c); ++ sysop("sev"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++static s64 psci_affinity(u64 mpidr, u64 level) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || level) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ psci_line_sync(&psci_cpus[idx]); ++ return psci_cpus[idx].state; ++} ++ ++static s64 psci_features(u32 fid) ++{ ++ switch (fid) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ case PSCI_0_2_FN_CPU_OFF: ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return PSCI_RET_SUCCESS; ++ default: ++ /* ++ * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make ++ * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) ++{ ++ u32 fn = fid; ++ ++ /* SMC32 calls carry 32-bit arguments */ ++ if (!(fn & 0x40000000)) { ++ a1 = (u32)a1; ++ a2 = (u32)a2; ++ a3 = (u32)a3; ++ } ++ ++ switch (fn) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ return 0x10000; /* 1.0 */ ++ case PSCI_0_2_FN_CPU_OFF: ++ return psci_cpu_off(); ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ return psci_cpu_on(a1, a2, a3); ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ return psci_affinity(a1, a2); ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ return 2; /* no trusted OS, nothing to migrate */ ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return psci_features(a1); ++ default: ++ /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++/* EL1 side */ ++ ++void psci_note_boot_el(u64 el) ++{ ++ /* MMU still off: these .data accesses are uncached, like the writers' */ ++ if (psci_persist.reentry_flag) { ++ reentered = true; ++ psci_persist.reentry_flag = 0; ++ } ++ if (el == 3) ++ psci_persist.boot_el3 = 1; ++ ++ psci_have_el3 = psci_persist.boot_el3 && has_el3(); ++} ++ ++void psci_configure(const char *val) ++{ ++ psci_mode = val[0] == '1'; ++} ++ ++bool psci_active(void) ++{ ++ return psci_mode && psci_have_el3; ++} ++ ++bool psci_reentered(void) ++{ ++ return reentered; ++} ++ ++int psci_affinity_info(u64 mpidr) ++{ ++ return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); ++} ++ ++static void psci_self_off(void) ++{ ++ /* Only returns on failure; the core then stays in m1n1's loop */ ++ psci_smc(PSCI_0_2_FN_CPU_OFF, 0, 0, 0); ++} ++ ++int psci_park_cpu(int cpu, u64 mpidr) ++{ ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ ++ smp_call0(cpu, psci_self_off); ++ ++ for (int i = 0; i < 100; i++) { ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ udelay(1000); ++ } ++ ++ return -1; ++} ++ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top) ++{ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return -1; ++ ++ s64 ret = psci_smc(PSCI_0_2_FN64_CPU_ON, mpidr, (u64)psci_secondary_entry, ++ (u64)stack_top | (u64)index); ++ ++ return ret == PSCI_RET_SUCCESS ? 0 : -1; ++} +diff --git a/src/psci.h b/src/psci.h +new file mode 100644 +index 0000000..6769ebe +--- /dev/null ++++ b/src/psci.h +@@ -0,0 +1,78 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#ifndef PSCI_H ++#define PSCI_H ++ ++#ifndef __ASSEMBLER__ ++#include "types.h" ++#endif ++ ++/* ++ * Minimal PSCI 1.0 for SoCs where m1n1 owns EL3 (A7-A9 entered at EL3). ++ * Only CPU_ON/CPU_OFF/AFFINITY_INFO and the discovery calls are implemented. ++ * A CPU that calls CPU_OFF is not powered down: it waits at EL3, MMU and ++ * caches off, until CPU_ON hands it a new EL1 entry point. That is what the ++ * kernel needs to offline CPUs for kexec and CPU hotplug. ++ */ ++ ++#define PSCI_0_2_FN_PSCI_VERSION 0x84000000 ++#define PSCI_0_2_FN_CPU_SUSPEND 0x84000001 ++#define PSCI_0_2_FN_CPU_OFF 0x84000002 ++#define PSCI_0_2_FN_CPU_ON 0x84000003 ++#define PSCI_0_2_FN_AFFINITY_INFO 0x84000004 ++#define PSCI_0_2_FN_MIGRATE_INFO_TYPE 0x84000006 ++#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 ++#define PSCI_0_2_FN_SYSTEM_RESET 0x84000009 ++#define PSCI_1_0_FN_PSCI_FEATURES 0x8400000a ++#define PSCI_0_2_FN64_CPU_ON 0xc4000003 ++#define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 ++ ++#define PSCI_RET_SUCCESS 0 ++#define PSCI_RET_NOT_SUPPORTED -1 ++#define PSCI_RET_INVALID_PARAMS -2 ++#define PSCI_RET_DENIED -3 ++#define PSCI_RET_ALREADY_ON -4 ++#define PSCI_RET_ON_PENDING -5 ++#define PSCI_RET_INTERNAL_FAILURE -6 ++ ++#define PSCI_AFF_ON 0 ++#define PSCI_AFF_OFF 1 ++#define PSCI_AFF_ON_PENDING 2 ++ ++#define PSCI_MAX_CPUS 4 ++#define PSCI_STACK_SIZE 0x4000 ++ ++/* Layout shared with psci_asm.S */ ++#define PSCI_CPU_STATE 0x00 ++#define PSCI_CPU_ENTRY 0x08 ++#define PSCI_CPU_CONTEXT 0x10 ++#define PSCI_CPU_SIZE 0x80 ++ ++#ifndef __ASSEMBLER__ ++ ++/* Payload variable psci=1 */ ++void psci_configure(const char *val); ++/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++bool psci_active(void); ++/* Called once from _start_c with the EL m1n1 was entered at */ ++void psci_note_boot_el(u64 el); ++/* Set when m1n1 was re-entered from Linux (see psci_asm.S _reentry) */ ++bool psci_reentered(void); ++ ++/* EL1 side: SMC into our own EL3 */ ++s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3); ++int psci_affinity_info(u64 mpidr); ++/* Park a secondary that is spinning in smp_secondary_entry() at EL3 */ ++int psci_park_cpu(int cpu, u64 mpidr); ++/* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); ++ ++/* EL3 side */ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); ++void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_secondary_entry(void); ++void _reentry(void); ++ ++#endif ++ ++#endif +diff --git a/src/psci_asm.S b/src/psci_asm.S +new file mode 100644 +index 0000000..14f842f +--- /dev/null ++++ b/src/psci_asm.S +@@ -0,0 +1,170 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++ ++#define DCSW_OP_DCCISW 0x1 // memory.h ++ ++.extern _v_sp0_sync ++.extern _start ++.extern _cpu_reset_c ++.extern psci_handle ++.extern psci_persist ++ ++.text ++ ++/* ++ * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) ++ * goes to the PSCI dispatcher; everything else, and every exception taken ++ * at EL1/EL2, takes the usual path. ++ */ ++.globl _v_lower_sync ++.type _v_lower_sync, @function ++_v_lower_sync: ++ stp x16, x17, [sp, #-16]! ++ mrs x16, CurrentEL ++ cmp x16, #0xc ++ b.ne 1f ++ mrs x16, esr_el3 ++ lsr x17, x16, #26 ++ cmp x17, #0x17 // EC: SMC from AArch64 ++ b.ne 1f ++ tst x16, #0xffff // imm16 == 0: SMCCC ++ b.ne 1f ++ ldp x16, x17, [sp], #16 ++ b psci_smc_entry ++1: ++ ldp x16, x17, [sp], #16 ++ b _v_sp0_sync ++ ++/* ++ * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the ++ * caller. x0-x3 go straight to psci_handle(). ELR/SPSR are saved in case ++ * anything below takes an exception at EL3. ++ */ ++.type psci_smc_entry, @function ++psci_smc_entry: ++ stp x29, x30, [sp, #-16]! ++ mrs x29, elr_el3 ++ mrs x30, spsr_el3 ++ stp x29, x30, [sp, #-16]! ++ str x18, [sp, #-16]! ++ bl psci_handle ++ ldr x18, [sp], #16 ++ ldp x29, x30, [sp], #16 ++ msr elr_el3, x29 ++ msr spsr_el3, x30 ++ ldp x29, x30, [sp], #16 ++ eret ++ ++/* ++ * psci_pen(struct psci_cpu *me, u64 stack_top): CPU_OFF. Never returns. ++ * ++ * The core is not powered down. It writes its L1D back, marks itself OFF and ++ * waits in WFE at EL3 (MMU off, so every access here is non-cacheable) for ++ * CPU_ON to post an entry point. It then enters EL1 there with the MMU off, ++ * a clean I-cache and TLB, and x0 = context id, like a core coming out of ++ * reset. SP_EL3 is reset to this core's own static stack. ++ */ ++.globl psci_pen ++.type psci_pen, @function ++psci_pen: ++ mov x19, x0 ++ mov x20, x1 ++ ++ // A pending timer interrupt must not keep waking WFE. ++ msr cntp_ctl_el0, xzr ++ msr cntv_ctl_el0, xzr ++ isb ++ ++ // Clean and invalidate this core's L1D (L2 is shared and coherent). ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_level1 ++ dsb sy ++ ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_OFF ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ sev ++ ++1: ++ wfe ++ dc civac, x19 ++ dsb sy ++ ldr x2, [x19, #PSCI_CPU_ENTRY] ++ cbz x2, 1b ++ ++ ldr x0, [x19, #PSCI_CPU_CONTEXT] ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_ON ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ ++ // Nothing of the previous kernel's EL1 context may survive. ++ ic iallu ++ tlbi alle1 ++ dsb sy ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ msr elr_el3, x2 ++ mov sp, x20 ++ isb ++ eret ++ ++/* EL1 side: s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3) */ ++.globl psci_smc ++.type psci_smc, @function ++psci_smc: ++ smc #0 ++ ret ++ ++/* ++ * CPU_ON target when m1n1 takes a parked core back after a re-entry. ++ * x0 = EL1 stack top | m1n1 CPU index (stack tops are 16 KiB aligned). ++ */ ++.globl psci_secondary_entry ++.type psci_secondary_entry, @function ++psci_secondary_entry: ++ and x1, x0, #0xf ++ bic x0, x0, #0xf ++ msr tpidr_el1, x1 ++ mov sp, x0 ++ bl _cpu_reset_c ++ b . ++ ++/* ++ * Re-entry from Linux (a kexec'd trampoline, see docs/boot/warm-reboot.md): ++ * EL1, MMU and caches off, x0 = a copy of the original boot_args. ++ * Linux may hold clean lines of this image in its linear map, so drop them ++ * before m1n1 writes its .bss uncached and then reads it with caches on. ++ */ ++.globl _reentry ++.type _reentry, @function ++_reentry: ++ mov x19, x0 ++ adrp x1, _base ++ adrp x2, _end ++1: ++ dc civac, x1 ++ add x1, x1, #64 ++ cmp x1, x2 ++ b.lo 1b ++ dsb sy ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #1 ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ mov x0, x19 ++ b _start +diff --git a/src/smp.c b/src/smp.c +index 83eadff..7f5b336 100644 +--- a/src/smp.c ++++ b/src/smp.c +@@ -8,6 +8,7 @@ + #include "malloc.h" + #include "memory.h" + #include "pmgr.h" ++#include "psci.h" + #include "soc.h" + #include "string.h" + #include "types.h" +@@ -227,6 +228,74 @@ static void smp_stop_cpu(int index, int die, int cluster, int core, u64 impl, u6 + } + } + ++/* ++ * After a re-entry from Linux, a secondary that Linux offlined waits in the ++ * PSCI pen at EL3 rather than in smp_secondary_entry(). It can't be reset ++ * through the PMGR (it is running), so hand it an EL1 entry with CPU_ON. ++ */ ++static bool smp_reclaim_cpu(int index, u64 mpidr) ++{ ++ int i; ++ ++ if (index >= MAX_CPUS || spin_table[index].flag) ++ return false; ++ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return false; ++ ++ printf("Taking CPU %d back from the PSCI pen... ", index); ++ ++ memset(&spin_table[index], 0, sizeof(struct spin_table)); ++ target_cpu = index; ++ secondary_stacks[index] = memalign(0x4000, SECONDARY_STACK_SIZE); ++ ++ /* The core starts with its MMU off */ ++ dc_civac_range(&spin_table[index], sizeof(struct spin_table)); ++ dc_civac_range(&target_cpu, sizeof(target_cpu)); ++ dc_civac_range(&boot_cpu_idx, sizeof(boot_cpu_idx)); ++ dc_civac_range(&boot_cpu_mpidr, sizeof(boot_cpu_mpidr)); ++ ++ if (psci_unpark_cpu(index, mpidr, secondary_stacks[index] + SECONDARY_STACK_SIZE) < 0) { ++ printf("Failed (CPU_ON)\n"); ++ return true; ++ } ++ ++ for (i = 0; i < 100; i++) { ++ sysop("dmb ld"); ++ if (spin_table[index].flag) ++ break; ++ udelay(1000); ++ } ++ ++ printf("%s\n", i >= 100 ? "Failed!" : " Back."); ++ return true; ++} ++ ++/* ++ * Put every live secondary into the PSCI pen, for a kernel that boots them ++ * with enable-method = "psci". A CPU that doesn't make it is dropped (and ++ * then pruned from the DT by the caller). ++ */ ++int smp_park_secondaries_psci(void) ++{ ++ int parked = 0; ++ ++ for (int i = 0; i < MAX_CPUS; i++) { ++ if (i == boot_cpu_idx || !smp_is_alive(i)) ++ continue; ++ ++ u64 mpidr = spin_table[i].mpidr; ++ if (psci_park_cpu(i, mpidr) < 0) { ++ printf("PSCI: CPU %d (MPIDR 0x%lx) did not park, dropping it\n", i, mpidr); ++ memset(&spin_table[i], 0, sizeof(struct spin_table)); ++ continue; ++ } ++ parked++; ++ } ++ ++ return parked; ++} ++ + static u64 smp_get_iorvbar(int node, int cpu) + { + u64 cpu_impl_reg[2]; +@@ -405,6 +474,10 @@ void smp_start_secondaries(void) + u8 cluster = FIELD_GET(CPU_REG_CLUSTER, reg); + u8 die = FIELD_GET(CPU_REG_DIE, reg); + ++ if (psci_active() && psci_reentered() && ++ smp_reclaim_cpu(i, core | (cluster << 8) | (die << 16))) ++ continue; ++ + smp_start_cpu(i, die, cluster, core, iorvbar, pmgr_reg + cpu_start_off); + } + } +diff --git a/src/smp.h b/src/smp.h +index 0296978..6c90557 100644 +--- a/src/smp.h ++++ b/src/smp.h +@@ -18,6 +18,7 @@ void smp_secondary_prep_el3(void); + + void smp_start_secondaries(void); + void smp_stop_secondaries(bool deep_sleep); ++int smp_park_secondaries_psci(void); + + #define smp_call0(i, f) smp_call4(i, f, 0, 0, 0, 0) + #define smp_call1(i, f, a) smp_call4(i, f, a, 0, 0, 0) +diff --git a/src/start.S b/src/start.S +index aa11e18..a3f35ae 100644 +--- a/src/start.S ++++ b/src/start.S +@@ -11,6 +11,7 @@ + .extern _v_sp0_irq + .extern _v_sp0_fiq + .extern _v_sp0_serr ++.extern _v_lower_sync + .extern _reset_stack + .extern _cpu_reset_c + .extern wdt_reboot +@@ -41,7 +42,7 @@ _vectors_start: + .align 7 + b _v_sp0_serr + .align 7 +- b _v_sp0_sync ++ b _v_lower_sync + .align 7 + b _v_sp0_irq + .align 7 +@@ -103,6 +104,8 @@ _start: + + mov x0, x19 + mov x1, x20 ++ mrs x2, CurrentEL ++ lsr x2, x2, #2 + bl _start_c + b . + +@@ -115,6 +118,7 @@ start_el3: + msr tpidr_el3, xzr + adrp x6, _stack_bot + adr x7, _start_c ++ mov x2, #3 // _start_c: entered at EL3 + b el3_eret_to_el1 + + .globl exc_unk +diff --git a/src/startup.c b/src/startup.c +index 61afe16..167fae8 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -5,6 +5,7 @@ + #include "dockchannel_uart.h" + #include "exception.h" + #include "firmware.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "types.h" +@@ -159,12 +160,15 @@ void dump_boot_args(struct boot_args *ba) + } + + extern void get_device_info(void); +-void _start_c(void *boot_args, void *base) ++void _start_c(void *boot_args, void *base, u64 boot_el) + { + UNUSED(base); + u32 cpu_id = 0; + + memset64(_bss_start, 0, _bss_end - _bss_start); ++ psci_note_boot_el(boot_el); ++ if (psci_reentered()) ++ boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); + diff --git a/boot/m1n1/psci-loader b/boot/m1n1/psci-loader new file mode 100755 index 00000000..1f628bc9 --- /dev/null +++ b/boot/m1n1/psci-loader @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Make a J71t opt-in loader by appending m1n1's psci=1 payload variable. + +Use a bare m1n1.bin as input, before assembling any kernel/DTB/logo payload. +The output works with orchard run/build --loader or launch's loaders/m1n1.bin. +It still uses the spin table if m1n1 was entered at EL1. +""" +import argparse +from pathlib import Path +import shutil + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--device', required=True, choices=('j71t',)) + parser.add_argument('--loader', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if not args.loader.is_file() or not args.loader.stat().st_size: + parser.error('input must be a nonempty bare m1n1.bin') + # Exclusive creation: never replace the original or another boot artifact. + with args.loader.open('rb') as source, args.output.open('xb') as output: + shutil.copyfileobj(source, output) + output.write(b'psci=1\n') + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..541b54da 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -15,3 +15,6 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "J71t on hardware" stages = "remote_boot two-stage CPID list" + +# Local capability delta: psci. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/psci.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..40db2bbc 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -54,3 +54,6 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" + +# Local capability delta: psci. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/psci.md. diff --git a/tests/test_m1n1_psci.py b/tests/test_m1n1_psci.py new file mode 100644 index 00000000..0f491dbf --- /dev/null +++ b/tests/test_m1n1_psci.py @@ -0,0 +1,54 @@ +"""Host-only framing checks for the explicit J71t PSCI loader.""" +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / 'boot/m1n1/psci-loader' + + +class PSCILoader(unittest.TestCase): + def test_payload_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as tmp: + loader, output = Path(tmp) / 'm1n1.bin', Path(tmp) / 'psci.bin' + original = b'bare loader\0' + loader.write_bytes(original) + cmd = [sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + self.assertEqual(loader.read_bytes(), original) + self.assertNotEqual(subprocess.run(cmd, capture_output=True).returncode, 0) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + + def test_manifest_psci_record_keeps_logo_first(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, data in {'loader': b'M1N1', 'dtb': b'DTB', 'kernel': b'KERNEL', + 'initrd': b'INITRD'}.items(): + (root / name).write_bytes(data) + logo = b'm1n1_logo_256128' + bytes(4 * (256 * 256 + 128 * 128)) + (root / 'logo').write_bytes(logo) + records = b'chosen.bootargs=console=tty0\npsci=1\n' + (root / 'profile.json').write_text(json.dumps({'bootargs': records.decode()})) + result = subprocess.run([sys.executable, '-I', str(ROOT / 'src/orchard/image/assemble.py'), + '--manifest', str(root / 'profile.json'), '--loader', str(root / 'loader'), + '--logo', str(root / 'logo'), '--dtb', str(root / 'dtb'), + '--kernel', str(root / 'kernel'), '--initramfs', str(root / 'initrd'), + '--output', str(root / 'payload')], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((root / 'payload').read_bytes().startswith(b'M1N1' + logo + records + b'DTB')) + + def test_board_requires_explicit_opt_in(self): + for device in ([], ['--device', 'j71b']): + result = subprocess.run([sys.executable, str(TOOL), *device, '--loader', 'unused', + '--output', 'unused'], capture_output=True) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() From eb6a0b1a68f87a420fc80639816b7808e1ca7e89 Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 11/33] Quiet polled cover inputs and exclusive RTC event Assisted-by: LLM --- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + .../s800x/0002-pmu-rtc-implementation.patch | 232 ++++++++++++++++++ .../usb/0010-pmu-rtc-implementation.patch | 34 +++ 4 files changed, 272 insertions(+) create mode 100644 kernel/patches/devices/s800x/0002-pmu-rtc-implementation.patch create mode 100644 kernel/patches/drivers/usb/0010-pmu-rtc-implementation.patch diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..6ae21ec1 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -15,3 +15,6 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "J71t on hardware" stages = "remote_boot two-stage CPID list" + +# Local capability delta: pmu-rtc. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/pmu-rtc.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..72069715 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -54,3 +54,6 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" + +# Local capability delta: pmu-rtc. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/pmu-rtc.md. diff --git a/kernel/patches/devices/s800x/0002-pmu-rtc-implementation.patch b/kernel/patches/devices/s800x/0002-pmu-rtc-implementation.patch new file mode 100644 index 00000000..48b731e4 --- /dev/null +++ b/kernel/patches/devices/s800x/0002-pmu-rtc-implementation.patch @@ -0,0 +1,232 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: Quiet polled cover inputs and exclusive RTC event + +Collapse the intermediate IRQ-driven cover change and its storm fix into the final polled-cover implementation, keeping PMU events 48/58 masked. Reserve event 16 exclusively for the RTC while retaining Tristar event-72 wake and parent-line protection. Add the one-shot RTC alarm interface and accept both directions of the known Apple current-grant request; do not fabricate a host grant or claim Mac charging is fixed. Cover wake remains deferred. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +index eb6b6fe92..ca2ab85c2 100644 +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -191,11 +191,11 @@ tristar_charger: charger@180 { + interrupt-parent = <&pmu_gpio>; + /* + * The PMU line, ADC done, then the events the iPadOS +- * charger subscribes to. ++ * charger subscribes to, except RTC event 16: Linux gives that ++ * event exclusively to rtc@500, not to source identification. + */ + interrupts = <240 IRQ_TYPE_EDGE_RISING>, + <176 IRQ_TYPE_EDGE_RISING>, +- <16 IRQ_TYPE_EDGE_RISING>, + <40 IRQ_TYPE_EDGE_RISING>, + <41 IRQ_TYPE_EDGE_RISING>, + <42 IRQ_TYPE_EDGE_RISING>, +@@ -211,6 +211,9 @@ rtc@500 { + reg = <0x500 0x10>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; ++ interrupt-parent = <&pmu_gpio>; ++ interrupts = <16 IRQ_TYPE_EDGE_RISING>; ++ wakeup-source; + }; + + /* D2365 register 0x31c bit 0 */ +diff --git a/drivers/rtc/rtc-apple-pmic.c b/drivers/rtc/rtc-apple-pmic.c +index ee56618c8..d31d4c716 100644 +--- a/drivers/rtc/rtc-apple-pmic.c ++++ b/drivers/rtc/rtc-apple-pmic.c +@@ -12,6 +12,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -24,22 +25,30 @@ + #include + + #define RTC_COUNTER_SIZE 6 ++#define RTC_ALARM_SIZE 4 ++#define RTC_CONTROL_RUN BIT(0) ++#define RTC_CONTROL_ALARM BIT(6) + + /** + * struct apple_pmic_rtc_regs - RTC Hardware registers + * @control: RTC Control + * @counter: Time Counter ++ * @alarm: Alarm seconds counter ++ * @has_alarm: Alarm registers are present + */ + + struct apple_pmic_rtc_regs { + u32 control; + u32 counter; ++ u32 alarm; ++ bool has_alarm; + }; + + /** + * struct apple_pmic_rtc - RTC driver internal structure + * @rtc: RTC device + * @regmap: regmap used to access registers ++ * @dev: device used for logging and wake setup + * @nvmem_cell: nvmem cell for offset + * @regs: hardware registers + * @base: RTC base address +@@ -183,17 +192,107 @@ static int apple_pmic_rtc_read_time(struct device *dev, struct rtc_time *tm) + return 0; + } + ++static int apple_pmic_rtc_alarm_irq_enable(struct device *dev, ++ unsigned int enabled) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ ++ /* AppleD2365PMU 0xfffffff006bbbc1c writes these exact values. */ ++ return regmap_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_RUN | ++ (enabled ? RTC_CONTROL_ALARM : 0)); ++} ++ ++static int apple_pmic_rtc_read_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ unsigned int control; ++ time64_t secs; ++ int rc; ++ ++ rc = regmap_bulk_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ rc = regmap_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, &control); ++ if (rc) ++ return rc; ++ ++ secs = (time64_t)get_unaligned_le32(buf) + ap_rtc->offset; ++ rtc_time64_to_tm(secs, &alarm->time); ++ alarm->enabled = !!(control & RTC_CONTROL_ALARM); ++ alarm->pending = 0; ++ return 0; ++} ++ ++static int apple_pmic_rtc_set_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ time64_t secs; ++ int rc; ++ ++ if (!alarm->enabled) ++ return apple_pmic_rtc_alarm_irq_enable(dev, 0); ++ ++ secs = rtc_tm_to_time64(&alarm->time) - ap_rtc->offset; ++ if (secs <= 0 || secs > U32_MAX) ++ return -ERANGE; ++ put_unaligned_le32(secs, buf); ++ ++ /* iOS programs 0x508 first, then enables the one-shot at 0x500. */ ++ rc = regmap_bulk_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ return apple_pmic_rtc_alarm_irq_enable(dev, 1); ++} ++ ++static irqreturn_t apple_pmic_rtc_alarm_irq(int irq, void *data) ++{ ++ struct apple_pmic_rtc *ap_rtc = data; ++ int rc; ++ ++ /* The PMU event read is the acknowledge; iOS then clears alarm bit 6. */ ++ rc = regmap_update_bits(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_ALARM, 0); ++ if (rc) ++ dev_warn_ratelimited(ap_rtc->dev, ++ "failed to disable fired alarm: %d\n", rc); ++ rtc_update_irq(ap_rtc->rtc, 1, RTC_IRQF | RTC_AF); ++ return IRQ_HANDLED; ++} ++ + static const struct rtc_class_ops apple_pmic_rtc_ops = { + .read_time = apple_pmic_rtc_read_time, + .set_time = apple_pmic_rtc_set_time, + }; + ++static const struct rtc_class_ops apple_pmic_rtc_alarm_ops = { ++ .read_time = apple_pmic_rtc_read_time, ++ .set_time = apple_pmic_rtc_set_time, ++ .read_alarm = apple_pmic_rtc_read_alarm, ++ .set_alarm = apple_pmic_rtc_set_alarm, ++ .alarm_irq_enable = apple_pmic_rtc_alarm_irq_enable, ++}; ++ + static const struct apple_pmic_rtc_regs amber_regs = { + .counter = 0x6 + }; + + static const struct apple_pmic_rtc_regs antigua_regs = { +- .counter = 0x2 ++ .control = 0x0, ++ .counter = 0x2, ++ .alarm = 0x8, ++ .has_alarm = true, + }; + + static const struct apple_pmic_rtc_regs calpe_regs = { +@@ -230,6 +329,7 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + struct device_node *node = pdev->dev.of_node; + const struct of_device_id *match; + struct apple_pmic_rtc *ap_rtc; ++ int irq; + int rc; + + match = of_match_node(apple_pmic_id_table, node); +@@ -264,6 +364,31 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + ap_rtc->rtc->ops = &apple_pmic_rtc_ops; + ap_rtc->rtc->range_min = INT_MIN; + ap_rtc->rtc->range_max = INT_MAX; ++ if (ap_rtc->regs->has_alarm) { ++ irq = platform_get_irq_optional(pdev, 0); ++ if (irq == -EPROBE_DEFER) ++ return irq; ++ if (irq > 0) { ++ rc = devm_request_threaded_irq(&pdev->dev, irq, NULL, ++ apple_pmic_rtc_alarm_irq, ++ IRQF_ONESHOT, ++ "apple-pmic-rtc", ap_rtc); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot request alarm IRQ\n"); ++ device_init_wakeup(&pdev->dev, true); ++ rc = devm_pm_set_wake_irq(&pdev->dev, irq); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot set alarm wake IRQ\n"); ++ ap_rtc->rtc->ops = &apple_pmic_rtc_alarm_ops; ++ dev_info(&pdev->dev, ++ "D2365 alarm at 0x%03x on PMU event IRQ %d\n", ++ ap_rtc->base + ap_rtc->regs->alarm, irq); ++ } else if (irq != -ENXIO) { ++ return irq; ++ } ++ } + + return devm_rtc_register_device(ap_rtc->rtc); + } diff --git a/kernel/patches/drivers/usb/0010-pmu-rtc-implementation.patch b/kernel/patches/drivers/usb/0010-pmu-rtc-implementation.patch new file mode 100644 index 00000000..4713453e --- /dev/null +++ b/kernel/patches/drivers/usb/0010-pmu-rtc-implementation.patch @@ -0,0 +1,34 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] usb: Quiet polled cover inputs and exclusive RTC event + +Collapse the intermediate IRQ-driven cover change and its storm fix into the final polled-cover implementation, keeping PMU events 48/58 masked. Reserve event 16 exclusively for the RTC while retaining Tristar event-72 wake and parent-line protection. Add the one-shot RTC alarm interface and accept both directions of the known Apple current-grant request; do not fabricate a host grant or claim Mac charging is fixed. Cover wake remains deferred. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 1da3effb2..14e87fbc8 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -968,7 +968,7 @@ static int composite_apple_current_limit(struct usb_composite_dev *cdev, + u16 w_value, u16 w_index) + { + cdev->apple_extra_ma = (s16)w_index; +- DBG(cdev, "Apple host current grant: extra %d mA, suspend %d mA\n", ++ dev_info(&cdev->gadget->dev, "Apple host current grant: extra %d mA, suspend %d mA\n", + (s16)w_index, (s16)w_value); + composite_vbus_draw(cdev, cdev->vbus_draw_ma); + return 0; +@@ -2221,7 +2221,7 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) + goto check_value; + } + +- if (ctrl->bRequestType == (USB_DIR_OUT | USB_TYPE_VENDOR | ++ if ((ctrl->bRequestType & ~USB_DIR_IN) == (USB_TYPE_VENDOR | + USB_RECIP_DEVICE) && + ctrl->bRequest == USB_APPLE_REQ_CURRENT_LIMIT && !w_length && + le16_to_cpu(cdev->desc.idVendor) == USB_APPLE_VENDOR_ID) { From e4ea0b7c1c87f6318cda2db23e13ee8c83f3df1b Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 12/33] PCI/NVMe: retain the stopped ANS handoff across kexec kernel/configs/apple-16k/config: # CONFIG_NVME_APPLE_ANS_EARLY_USB is not set These options provide warm-storage and its Kconfig dependencies on both page sizes. kernel/configs/apple-4k/config: # CONFIG_NVME_APPLE_ANS_EARLY_USB is not set These options provide warm-storage and its Kconfig dependencies on both page sizes. Assisted-by: LLM --- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + kernel/configs/apple-16k/config | 1 + kernel/configs/apple-4k/config | 1 + .../0032-warm-storage-implementation.patch | 1579 +++++++++++++++++ 5 files changed, 1587 insertions(+) create mode 100644 kernel/patches/drivers/pci/0032-warm-storage-implementation.patch diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..ac45860e 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -15,3 +15,6 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "J71t on hardware" stages = "remote_boot two-stage CPID list" + +# Local capability delta: warm-storage. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-storage.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..1953c00a 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -54,3 +54,6 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" + +# Local capability delta: warm-storage. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-storage.md. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index aae562f8..33b0a61b 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -1815,6 +1815,7 @@ CONFIG_BLK_DEV_NBD=y # CONFIG_NVME_CORE=m CONFIG_BLK_DEV_NVME=m +# CONFIG_NVME_APPLE_ANS_EARLY_USB is not set # CONFIG_NVME_MULTIPATH is not set # CONFIG_NVME_VERBOSE_ERRORS is not set # CONFIG_NVME_HWMON is not set diff --git a/kernel/configs/apple-4k/config b/kernel/configs/apple-4k/config index bf332e9c..e10187a1 100644 --- a/kernel/configs/apple-4k/config +++ b/kernel/configs/apple-4k/config @@ -1816,6 +1816,7 @@ CONFIG_BLK_DEV_NBD=y # CONFIG_NVME_CORE=m CONFIG_BLK_DEV_NVME=m +# CONFIG_NVME_APPLE_ANS_EARLY_USB is not set # CONFIG_NVME_MULTIPATH is not set # CONFIG_NVME_VERBOSE_ERRORS is not set # CONFIG_NVME_HWMON is not set diff --git a/kernel/patches/drivers/pci/0032-warm-storage-implementation.patch b/kernel/patches/drivers/pci/0032-warm-storage-implementation.patch new file mode 100644 index 00000000..9f3131e9 --- /dev/null +++ b/kernel/patches/drivers/pci/0032-warm-storage-implementation.patch @@ -0,0 +1,1579 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] PCI: apple-h9p: PCI/NVMe: retain the stopped ANS handoff across kexec + +Pair the S8003 H9P powered handoff with an ANS stop certificate covering SHST, CC/RDY, BME, queue/tag retirement, final NVMMU FIFO acknowledgement and endpoint DART shutdown. Include the final fatal-kexec drain/isolation correction and the modular kexec-state export required by NVMe. Failed stops withhold the certificate; failed powered resume stays under PERST. Real suspend/resume and the Orchard warm ladder remain pending, with no automatic suspend enablement. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 1ee6add55..7906595cd 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -35,6 +35,7 @@ + #include + + #include ++#include + + #include "dma-iommu.h" + +@@ -1009,6 +1010,37 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++int apple_ans_dart_set_active(struct device *dev, bool active) ++{ ++ struct iommu_domain *domain = iommu_get_domain_for_dev(dev); ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ struct apple_dart_domain *d; ++ int i, ret; ++ ++ if (!of_machine_is_compatible("apple,j71t") || !domain || !cfg || ++ !cfg->stream_maps[0].dart || ++ (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) ++ return -ENODEV; ++ d = to_dart_domain(domain); ++ for_each_stream_map(i, cfg, map) { ++ if (map->dart->hw->type != DART_S5L8960X) ++ return -EOPNOTSUPP; ++ if (!active) { ++ apple_dart_hw_disable_dma(map); ++ continue; ++ } ++ ret = apple_dart_setup_translation(d, map); ++ if (ret) ++ return ret; ++ } ++ if (active) ++ WRITE_ONCE(d->flush_error, 0); ++ dev_info(dev, "Apple ANS: DART force_active(%u) complete\n", active); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_set_active); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { +diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig +index 31974c7dd..0504eb40d 100644 +--- a/drivers/nvme/host/Kconfig ++++ b/drivers/nvme/host/Kconfig +@@ -14,6 +14,15 @@ config BLK_DEV_NVME + To compile this driver as a module, choose M here: the + module will be called nvme. + ++config NVME_APPLE_ANS_EARLY_USB ++ bool "Defer S8000/S8003 PCIe/ANS until initramfs USB rescue is ready" ++ depends on BLK_DEV_NVME && PCIE_APPLE_H9P ++ help ++ Default pcie_apple_h9p.probe and nvme.apple_ans_probe to false ++ on S8000/S8003. Requires an initramfs which starts USB rescue, opens ++ both gates and reprobes the host and endpoints. Leave off with ++ the ordinary root initrd. Other boards are unchanged. ++ + config NVME_MULTIPATH + bool "NVMe multipath support" + depends on NVME_CORE +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index 5ba239e47..0db0cb71b 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -84,6 +84,7 @@ struct apple_ans_nvme { + u32 cc; + unsigned long cc_jiffies; + bool cc_written; ++ bool host_fault; + dma_addr_t hmb_dma; + u32 hmb_size; + struct apple_ans_nvme_req req[APPLE_ANS_NVMMU_MAX_REQS]; +@@ -218,13 +219,20 @@ static int nvme_pci_apple_ans_reset_dma(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long flags; +- unsigned int tag; + int ret; + +- for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { +- ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ans->host == &apple_ans_host_s8003) { ++ ret = apple_ans_nvmmu_reset(dev->dev); + if (ret) + return ret; ++ } else { ++ unsigned int tag; ++ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } + } + + spin_lock_irqsave(&ans->req_lock, flags); +@@ -258,6 +266,14 @@ static int nvme_pci_apple_ans_config(struct nvme_dev *dev) + return 0; + } + ++static bool apple_ans_probe = !IS_ENABLED(CONFIG_NVME_APPLE_ANS_EARLY_USB); ++module_param(apple_ans_probe, bool, 0644); ++ ++bool nvme_pci_apple_ans_probe_allowed(struct pci_dev *pdev) ++{ ++ return !nvme_pci_apple_ans_match(pdev) || READ_ONCE(apple_ans_probe); ++} ++ + static int nvme_pci_apple_ans_preinit(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); +@@ -326,10 +342,24 @@ static int nvme_pci_apple_ans_prepare_enable(struct nvme_dev *dev) + return 0; + } + ++static int nvme_pci_apple_ans_check_link(struct nvme_dev *dev) ++{ ++ struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); ++ ++ if (ans && ans->host == &apple_ans_host_s8003 && ++ (ans->host_fault || apple_ans_link_up(dev->dev) <= 0)) { ++ ans->host_fault = true; ++ return -EIO; ++ } ++ return 0; ++} ++ + static int nvme_pci_apple_ans_write_cc(struct nvme_dev *dev, u32 cc) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + writel(cc, dev->bar + NVME_REG_CC); + if (!ans) + return 0; +@@ -349,6 +379,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long deadline; + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + *csts = readl(dev->bar + NVME_REG_CSTS); + if (!ans || !ans->cc_written || !ans->host->rdy_timeout_ms || + (ans->cc & NVME_CC_SHN_MASK)) +@@ -360,6 +392,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + time_after(jiffies, deadline)) { + dev_err(dev->dev, "ANS RDY timeout (CC %#x, CSTS %#x)\n", + ans->cc, *csts); ++ ans->host_fault = true; ++ apple_ans_host_dump(dev->dev, "RDY deadline"); + return -ETIMEDOUT; + } + +@@ -419,6 +453,7 @@ static int nvme_pci_apple_ans_free_req(struct nvme_dev *dev, + dev_err_ratelimited(dev->dev, + "failed to release ANS DMA slot %u: %d\n", + tag, ret); ++ nvme_reset_ctrl(&dev->ctrl); + return ret; + } + +diff --git a/drivers/nvme/host/pci-internal.h b/drivers/nvme/host/pci-internal.h +index a62777abd..ef3ea7b99 100644 +--- a/drivers/nvme/host/pci-internal.h ++++ b/drivers/nvme/host/pci-internal.h +@@ -74,6 +74,8 @@ struct nvme_dev { + struct nvme_ctrl ctrl; + u32 last_ps; + bool hmb; ++ unsigned long apple_ans_flags; ++ atomic_t apple_ans_irq_count; + struct sg_table *hmb_sgt; + mempool_t *dmavec_mempool; + const struct nvme_pci_dma_ops *dma_ops; +@@ -195,6 +197,7 @@ struct nvme_iod { + + extern const struct nvme_pci_dma_ops nvme_pci_apple_ans_ops; + ++bool nvme_pci_apple_ans_probe_allowed(struct pci_dev *pdev); + bool nvme_pci_apple_ans_match(struct pci_dev *pdev); + + #endif /* _NVME_PCI_INTERNAL_H */ +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 9ffe2fb74..32cc6ba89 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -5,6 +5,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -17,11 +18,17 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + #include + #include + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -1559,6 +1566,9 @@ static irqreturn_t nvme_irq(int irq, void *data) + DEFINE_IO_COMP_BATCH(iob); + bool found; + ++ if (!nvmeq->qid && dev->dma_ops == &nvme_pci_apple_ans_ops) ++ atomic_inc(&dev->apple_ans_irq_count); ++ + if (nvme_apple_ans_shared_vector(dev)) + found = nvme_apple_ans_irq_vector(dev, nvmeq->cq_vector, &iob); + else +@@ -1836,6 +1846,40 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) + "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n"); + } + ++/* No endpoint MMIO/config or doorbell reads: safe first evidence for ANS. */ ++static void nvme_apple_ans_dump(struct nvme_dev *dev, const char *reason) ++{ ++ unsigned int i; ++ ++ dev_info(dev->dev, "Apple ANS: snapshot %s state %d online %u allocated %u vectors %u recovery_used %u\n", ++ reason, nvme_ctrl_state(&dev->ctrl), dev->online_queues, ++ dev->ctrl.queue_count, dev->num_vecs, ++ test_bit(0, &dev->apple_ans_flags)); ++ apple_ans_host_dump(dev->dev, reason); ++ for (i = 0; i < dev->ctrl.queue_count; i++) { ++ struct nvme_queue *q = &dev->queues[i]; ++ u16 head = READ_ONCE(q->cq_head); ++ struct nvme_completion *cqe; ++ int irq = -1; ++ ++ if (!q->cqes || head >= q->q_depth) ++ continue; ++ cqe = &q->cqes[head]; ++ if (test_bit(NVMEQ_ENABLED, &q->flags) && ++ !test_bit(NVMEQ_POLLED, &q->flags)) ++ irq = pci_irq_vector(to_pci_dev(dev->dev), q->cq_vector); ++ dev_info(dev->dev, "Apple ANS: q%u flags %#lx depth %u SQ tail %u/%u CQ head %u phase %u next status %#x cid %#x sqhead %u vector %u irq %d ANS admin IRQ actions %u SQ %pad CQ %pad\n", ++ i, READ_ONCE(q->flags), q->q_depth, ++ READ_ONCE(q->sq_tail), READ_ONCE(q->last_sq_tail), ++ head, READ_ONCE(q->cq_phase), ++ le16_to_cpu(READ_ONCE(cqe->status)), ++ le16_to_cpu(READ_ONCE(cqe->command_id)), ++ le16_to_cpu(READ_ONCE(cqe->sq_head)), q->cq_vector, ++ irq, atomic_read(&dev->apple_ans_irq_count), ++ &q->sq_dma_addr, &q->cq_dma_addr); ++ } ++} ++ + static enum blk_eh_timer_return nvme_timeout(struct request *req) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); +@@ -1844,8 +1888,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + struct request *abort_req; + struct nvme_command cmd = { }; + struct pci_dev *pdev = to_pci_dev(dev->dev); +- u32 csts = readl(dev->bar + NVME_REG_CSTS); ++ u32 csts = ~0U; + u8 opcode; ++ bool ans = dev->dma_ops == &nvme_pci_apple_ans_ops && ++ apple_ans_is_s8003(dev->dev); ++ ++ if (ans) { ++ nvme_apple_ans_dump(dev, "timeout before CSTS"); ++ if (dev->dma_ops->read_csts(dev, &csts)) ++ goto disable; ++ } else { ++ csts = readl(dev->bar + NVME_REG_CSTS); ++ } + + /* + * Shutdown the device immediately if we see it is disconnected. This +@@ -1921,6 +1975,11 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + * returned to the driver, or if this is the admin queue. + */ + opcode = nvme_req(req)->cmd->common.opcode; ++ if (ans) { ++ if (test_and_set_bit(0, &dev->apple_ans_flags)) ++ nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); ++ goto disable; ++ } + if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) { + dev_warn(dev->ctrl.device, + "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n", +@@ -2804,7 +2863,32 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, + return a->mode; + } + ++static ssize_t apple_ans_diag_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); ++ u32 cc, csts; ++ ++ if (ndev->dma_ops != &nvme_pci_apple_ans_ops) ++ return -EOPNOTSUPP; ++ if (!sysfs_streq(buf, "host") && !sysfs_streq(buf, "endpoint")) ++ return -EINVAL; ++ mutex_lock(&ndev->shutdown_lock); ++ nvme_apple_ans_dump(ndev, "sysfs"); ++ if (sysfs_streq(buf, "endpoint") && apple_ans_link_up(ndev->dev) > 0) { ++ if (!ndev->dma_ops->read_csts(ndev, &csts)) { ++ cc = readl(ndev->bar + NVME_REG_CC); ++ dev_info(dev, "ANS endpoint CC %#x CSTS %#x\n", cc, csts); ++ } ++ } ++ mutex_unlock(&ndev->shutdown_lock); ++ return count; ++} ++static DEVICE_ATTR_WO(apple_ans_diag); ++ + static struct attribute *nvme_pci_attrs[] = { ++ &dev_attr_apple_ans_diag.attr, + &dev_attr_cmb.attr, + &dev_attr_cmbloc.attr, + &dev_attr_cmbsz.attr, +@@ -3283,11 +3367,69 @@ static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev) + return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY); + } + +-static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static bool apple_ans_sleep_pair = true; ++module_param(apple_ans_sleep_pair, bool, 0644); ++static bool apple_ans_sleep_cc0_ok; ++module_param(apple_ans_sleep_cc0_ok, bool, 0644); ++ ++static int nvme_apple_ans_sleep_precheck(struct notifier_block *nb, ++ unsigned long action, void *unused) ++{ ++ if (action == PM_SUSPEND_TO_IDLE && ++ of_machine_is_compatible("apple,s8003") && ++ !READ_ONCE(apple_ans_sleep_pair)) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_DONE; ++} ++ ++static struct notifier_block nvme_apple_ans_sleep_nb = { ++ .notifier_call = nvme_apple_ans_sleep_precheck, ++}; ++ ++/* SHST alone does not certify a stopped controller. Keep endpoint MMIO before ++ * clearing BME and retiring translations, as in suspend-r1c/r1d. */ ++static int nvme_apple_ans_sleep_disable(struct nvme_dev *dev, int shutdown_ret) ++{ ++ struct pci_dev *pdev = to_pci_dev(dev->dev); ++ u32 csts = ~0U, cc, expected; ++ int ret, err; ++ ++ ret = dev->dma_ops->read_csts(dev, &csts); ++ if (shutdown_ret || ret || (csts & NVME_CSTS_CFS) || ++ (csts & NVME_CSTS_SHST_MASK) != NVME_CSTS_SHST_CMPLT) { ++ pci_clear_master(pdev); ++ return -EBUSY; ++ } ++ dev->ctrl.ctrl_config &= ~(NVME_CC_ENABLE | NVME_CC_SHN_MASK); ++ expected = dev->ctrl.ctrl_config; ++ ret = dev->dma_ops->write_cc(dev, expected); ++ if (!ret) { ++ ret = read_poll_timeout(dev->dma_ops->read_csts, err, ++ err || !(csts & NVME_CSTS_RDY), ++ 1000, 2000000, false, dev, &csts); ++ if (!ret) ++ ret = err; ++ } ++ if (ret) ++ goto block_dma; ++ cc = readl(dev->bar + NVME_REG_CC); ++ if ((!kexec_in_progress && csts) || (csts & NVME_CSTS_CFS) || ++ (cc & (NVME_CC_ENABLE | NVME_CC_SHN_MASK)) || ++ (!kexec_in_progress && cc != expected && ++ !(READ_ONCE(apple_ans_sleep_cc0_ok) && !cc))) ++ ret = -EBUSY; ++block_dma: ++ pci_clear_master(pdev); ++ return ret; ++} ++ ++static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool paired) + { + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); + dead = nvme_pci_ctrl_is_dead(dev); +@@ -3298,15 +3440,26 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown) +- nvme_wait_freeze_timeout(&dev->ctrl); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) ++ panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); + } + + nvme_quiesce_io_queues(&dev->ctrl); + ++ if (paired && (dead || !dev->ctrl.queue_count)) ++ panic("Apple ANS: dead outgoing controller; refusing kexec"); + if (!dead && dev->ctrl.queue_count > 0) { ++ if (paired) { ++ nvme_sync_io_queues(&dev->ctrl); ++ ret = apple_ans_nvmmu_drain(dev->dev); ++ if (ret) ++ panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ } + nvme_delete_io_queues(dev); +- nvme_disable_ctrl(&dev->ctrl, shutdown); ++ dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); ++ ret = nvme_disable_ctrl(&dev->ctrl, shutdown); ++ if (paired) ++ ret = nvme_apple_ans_sleep_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } + nvme_suspend_io_queues(dev); +@@ -3329,7 +3482,15 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) + nvme_unquiesce_admin_queue(&dev->ctrl); + } ++ if (paired && !ret) ++ ret = apple_ans_sleep_stopped(dev->dev); + mutex_unlock(&dev->shutdown_lock); ++ return ret; ++} ++ ++static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++{ ++ nvme_dev_disable_paired(dev, shutdown, false); + } + + static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) +@@ -3775,7 +3936,10 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) + struct nvme_dev *dev; + int result = -ENOMEM; + ++ if (!nvme_pci_apple_ans_probe_allowed(pdev)) ++ return -EPROBE_DEFER; + dev = nvme_pci_alloc_dev(pdev, id); ++ + if (IS_ERR(dev)) + return PTR_ERR(dev); + +@@ -3920,6 +4084,12 @@ static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); + ++ if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) ++ panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + nvme_disable_prepare_reset(dev, true); + } + +@@ -3970,6 +4140,14 @@ static int nvme_resume(struct device *dev) + struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); + struct nvme_ctrl *ctrl = &ndev->ctrl; + ++ if (apple_ans_is_s8003(ndev->dev) && ndev->dma_ops == &nvme_pci_apple_ans_ops && ++ ndev->last_ps == U32_MAX) { ++ int ret = apple_ans_dart_set_active(ndev->dev, true); ++ ++ if (ret) ++ return ret; ++ } ++ + if (ndev->last_ps == U32_MAX || + nvme_set_power_state(ctrl, ndev->last_ps) != 0) + goto reset; +@@ -3989,6 +4167,19 @@ static int nvme_suspend(struct device *dev) + int ret = -EBUSY; + + ndev->last_ps = U32_MAX; ++ if (apple_ans_is_s8003(ndev->dev) && ndev->dma_ops == &nvme_pci_apple_ans_ops) { ++ if (!READ_ONCE(apple_ans_sleep_pair) || apple_ans_sleep_begin(ndev->dev)) ++ return -EBUSY; ++ if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || ++ !nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) ++ return -EBUSY; ++ ret = nvme_dev_disable_paired(ndev, true, true); ++ if (ret) { ++ apple_ans_dart_set_active(ndev->dev, true); ++ nvme_try_sched_reset(ctrl); ++ } ++ return ret; ++ } + + /* + * The platform does not remove power for a kernel managed suspend so +@@ -4356,12 +4547,20 @@ static struct pci_driver nvme_driver = { + + static int __init nvme_init(void) + { ++ int ret; ++ + BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); + BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); + BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); + BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2); + +- return pci_register_driver(&nvme_driver); ++ ret = register_pm_suspend_precheck_notifier(&nvme_apple_ans_sleep_nb); ++ if (ret) ++ return ret; ++ ret = pci_register_driver(&nvme_driver); ++ if (ret) ++ unregister_pm_suspend_precheck_notifier(&nvme_apple_ans_sleep_nb); ++ return ret; + } + + static void __exit nvme_exit(void) +@@ -4369,6 +4568,7 @@ static void __exit nvme_exit(void) + kfree(nvme_pci_quirk_list); + pci_unregister_driver(&nvme_driver); + flush_workqueue(nvme_wq); ++ unregister_pm_suspend_precheck_notifier(&nvme_apple_ans_sleep_nb); + } + + MODULE_AUTHOR("Matthew Wilcox "); +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 15f7a0cbe..52a79fba6 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -15,6 +15,8 @@ + + #include + #include ++#include ++#include + #include + #include + #include +@@ -38,12 +40,20 @@ + #include + #include + #include ++#include + #include + #include + #include ++#include ++#include ++#include ++#include ++#include ++#include + + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -248,7 +258,7 @@ + #define S8003_PORT_LINKCDMSTS_LTSSM GENMASK(14, 9) + #define S8003_LTSSM_L0 0x11 + +-/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (j71t) */ ++/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (S8003) */ + #define S8003_PERST_TO_REFCLK_US 60 + #define S8003_REFCLK_TO_PERST_US 100 + #define S8003_PERST_HOLD_MS 100 +@@ -269,7 +279,7 @@ + + /* + * NVMMU invalidate FIFO status, in the port 0 block (iBoot sets the live ADT +- * nvme-mmu0 invalidate-fifo-workaround = 1 on j71t). iOS (AppleNVMeMMU, ++ * nvme-mmu0 invalidate-fifo-workaround = 1 on S8003). iOS (AppleNVMeMMU, + * 0xfffffff005eebc9c) waits up to 20 ms for it to drain before every + * invalidate and never reads the invalidate register back. + */ +@@ -429,7 +439,7 @@ static const struct apple_h9p_tunable t7000_port_tunables[] = { + }; + + /* +- * S8003 tunables, which iBoot puts in the live ADT; copied from the j71t ++ * S8003 tunables, which iBoot puts in the live ADT; copied from the S8003 + * live ADT (iOS 16.7.10 iBoot), where both bridges carry the same values. + * apcie-phy-tunables and apcie-common-tunables apply after the common init, + * in that order. +@@ -582,6 +592,7 @@ struct apple_h9p_tunables { + struct apple_h9p_nvmmu { + struct apple_h9p_pcie *pcie; + void __iomem *base; ++ bool poisoned; + /* Programmed: the ANS glue may map through it */ + bool ready; + /* Serializes the invalidate FIFO wait and the invalidate write */ +@@ -711,6 +722,8 @@ struct apple_h9p_pcie { + u32 late_ports; + unsigned int late_tries; + struct delayed_work late_work; ++ bool ans_sleep_stopped, ans_sleep_powered; ++ struct notifier_block sleep_precheck; + + struct apple_h9p_nvmmu nvmmu; + +@@ -1554,6 +1567,64 @@ static irqreturn_t apple_h9p_nvmmu_irq(int irq, void *data) + return IRQ_HANDLED; + } + ++static int apple_h9p_nvmmu_program(struct apple_h9p_pcie *pcie) ++{ ++ struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; ++ u32 val; ++ int ret; ++ ++ if (pcie->hw->nvmmu_inv_fifo) { ++ /* ++ * iOS's S8003 order: the invalidate FIFO drained, the old SART ++ * aperture off before its address changes, the zeroed scratch ++ * page and TCBs visible, then scratch page, TCB base, init, SART. ++ */ ++ ret = readl_poll_timeout(pcie->base_port[0] + ++ S8003_PORT_NVMMU_INV_FIFO, val, ++ val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ dma_wmb(); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ } else { ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ } ++ h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, ++ nvmmu->base + H9P_NVMMU_TCB_CTRL); ++ ++ ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, ++ pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ if (ret) ++ return ret; ++ ++ h9p_writel_flush(nvmmu->va_base - 0x80000000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_BASE); ++ h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, ++ H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_END); ++ h9p_writel_flush(nvmmu->pa_base >> 20, ++ nvmmu->base + H9P_NVMMU_SART_PA_BASE); ++ h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ ++ return 0; ++} ++ + static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + { + struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; +@@ -1561,7 +1632,7 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + struct device_node *mem_np; + struct resource res; + resource_size_t size; +- u32 iova, val; ++ u32 iova; + int irq; + int ret; + +@@ -1630,55 +1701,10 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + } + + spin_lock_init(&nvmmu->inv_lock); +- if (pcie->hw->nvmmu_inv_fifo) { +- /* +- * iOS's S8003 order: the invalidate FIFO drained, the old SART +- * aperture off before its address changes, the zeroed scratch +- * page and TCBs visible, then scratch page, TCB base, init, SART. +- */ +- ret = readl_poll_timeout(pcie->base_port[0] + +- S8003_PORT_NVMMU_INV_FIFO, val, +- val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, +- 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); +- if (ret) +- goto out_put_node; +- h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); +- dma_wmb(); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- } else { +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- } +- h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, +- nvmmu->base + H9P_NVMMU_TCB_CTRL); +- +- ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, +- pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ ret = apple_h9p_nvmmu_program(pcie); + if (ret) + goto out_put_node; + +- h9p_writel_flush(nvmmu->va_base - 0x80000000U, +- nvmmu->base + H9P_NVMMU_SART_VA_BASE); +- h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, +- H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, +- nvmmu->base + H9P_NVMMU_SART_VA_END); +- h9p_writel_flush(nvmmu->pa_base >> 20, +- nvmmu->base + H9P_NVMMU_SART_PA_BASE); +- h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); +- + irq = platform_get_irq_byname_optional(pcie->pdev, "nvmmu"); + if (irq > 0) { + ret = devm_request_irq(dev, irq, apple_h9p_nvmmu_irq, 0, +@@ -1726,6 +1752,7 @@ static int apple_h9p_nvmmu_invalidate(struct apple_h9p_pcie *pcie, + spin_unlock_irqrestore(&nvmmu->inv_lock, flags); + + if (ret) { ++ WRITE_ONCE(nvmmu->poisoned, true); + dev_err_ratelimited(pcie->dev, + "NVMMU invalidate FIFO not empty (%#x), tag %u\n", + val, tag); +@@ -1781,6 +1808,8 @@ int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + nvmmu = &pcie->nvmmu; + if (!nvmmu->ready) + return -EOPNOTSUPP; ++ if (READ_ONCE(nvmmu->poisoned)) ++ return -EIO; + + tcb = (u32 *)nvmmu->tcb + tag * H9P_NVMMU_TCB_DWORDS; + sgl = (u32 *)nvmmu->tcb_sgl + tag * H9P_NVMMU_SGL_WORDS; +@@ -2705,6 +2734,8 @@ static int apple_s8003_power_cycle_port(struct apple_h9p_pcie *pcie, + return ret; + } + ++ if (!port && pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); + apple_s8003_port_off(pcie, port); + usleep_range(S8003_POWER_OFF_SETTLE_US, S8003_POWER_OFF_SETTLE_US + 1000); + ret = apple_h9p_wait(rc + S8003_RC_PORT_CTL(port), +@@ -2887,6 +2918,81 @@ static int apple_s8003_start_link(struct apple_h9p_pcie *pcie, + return ret; + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* S8003 adaptation of suspend-r1d/r1e/r1f to Orchard's host driver. */ ++static unsigned int h9p_suspend; ++module_param_named(suspend, h9p_suspend, uint, 0644); ++static bool ans_keep_power; ++module_param(ans_keep_power, bool, 0644); ++static char *ans_boot_mode = "handoff"; ++module_param(ans_boot_mode, charp, 0444); ++static unsigned int ans_sleep_flush = 3; ++module_param(ans_sleep_flush, uint, 0644); ++static unsigned int ans_kexec_flush; ++module_param(ans_kexec_flush, uint, 0644); ++ ++static bool apple_s8003_handoff_enabled(void) ++{ ++ return READ_ONCE(ans_keep_power) && !strcmp(ans_boot_mode, "handoff"); ++} ++ ++static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *rc = pcie->base_rc; ++ ++ return (readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_ENABLE) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_ON) && ++ (readl(rc + S8003_RC_PORT_REFCLK(0)) & S8003_RC_PORT_REFCLK_ON) && ++ !(readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_RESET) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_REFCLK_NOGATE) && ++ (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); ++} ++ ++/* Best effort, bounded, while PERST is asserted. Never cycle port power. */ ++static void apple_s8003_drain_link(struct apple_h9p_pcie *pcie, unsigned int flush) ++{ ++ u32 val; ++ int ret; ++ ++ if (!(flush & 1)) ++ return; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, val, ++ !(val & BIT(0)), 10, 100000); ++ dev_info(pcie->dev, "ANS link drain: status=%#x drained=%u\n", val, !ret); ++} ++ ++static int apple_s8003_flush_link(struct apple_h9p_pcie *pcie, unsigned int flush) ++{ ++ u32 val; ++ ++ if ((flush & 2) && (readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & BIT(0))) { ++ h9p_rmw(pcie->base_port[0] + H9P_PORT_LTSSMCTL, 0, H9P_PORT_LTSSM_ENABLE); ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, val, ++ !(val & BIT(0)), 10, 100000); ++ return apple_s8003_port_stop(pcie, 0); ++ } ++ return 0; ++} ++ ++static int apple_s8003_powered_reset(struct apple_h9p_pcie *pcie, unsigned int flush) ++{ ++ int ret; ++ ++ if (!apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (!ret) ++ ret = apple_s8003_flush_link(pcie, flush); ++ if (!ret) ++ ret = apple_s8003_start_link(pcie, 0); ++ /* A failed powered reset stays under PERST; no full-cycle fallback. */ ++ if (ret) ++ gpiod_direction_output(pcie->perst[0], 1); ++ return ret; ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -2902,6 +3008,15 @@ static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + * validated on it, the other ports the restart sequence that is the + * only one validated on the Wi-Fi port. + */ ++ if (!port && apple_s8003_handoff_enabled() && ++ of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ apple_s8003_port_powered(pcie) && ++ gpiod_get_value_cansleep(pcie->perst[0]) == 1) { ++ ret = apple_s8003_powered_reset(pcie, READ_ONCE(ans_kexec_flush)); ++ if (!ret || (ret != -ETIMEDOUT && ret != -ENOLINK)) ++ return ret; ++ dev_warn(pcie->dev, "powered reset failed %d; full port fallback\n", ret); ++ } + if (pcie->hw->nvme_mode_ports & BIT(port)) + ret = apple_s8003_power_cycle_port(pcie, port); + else +@@ -2945,6 +3060,8 @@ static int apple_s8003_setup_storage_port(struct apple_h9p_pcie *pcie, + if (ret != -ETIMEDOUT && ret != -ENOLINK) + return ret; + ++ /* Probe/kexec may fall back to full reset; sleep resume never uses this. */ ++ + dev_warn(pcie->dev, "port %u: no link (%d), power cycling the port\n", + port, ret); + if ((readl(rc + S8003_RC_PORT_CTL(port)) & S8003_RC_PORT_CTL_ENABLE) && +@@ -3059,6 +3176,11 @@ static void apple_s8003_late_ports_work(struct work_struct *work) + unsigned int port; + int ret; + ++ /* Lab 0019: no secondary bus publication without storage bus 1. */ ++ if (of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") && ++ !(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return; ++ + for_each_set_bit(port, &pending, H9P_NUM_PORTS) { + ret = apple_s8003_setup_endpoint_port(pcie, port); + if (ret == -EPROBE_DEFER) { +@@ -3418,6 +3540,472 @@ static void apple_h9p_pcie_pwrctrl_cleanup(void *data) + pci_pwrctrl_destroy_devices(pcie->dev); + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static struct apple_h9p_pcie *apple_ans_endpoint_host(struct device *dev) ++{ ++ struct device *host = dev; ++ struct apple_h9p_pcie *pcie; ++ ++ while (host && host->bus == dev->bus) ++ host = host->parent; ++ if (!host || !host->parent) ++ return NULL; ++ pcie = apple_h9p_pcie_lookup(host->parent); ++ if (!pcie || !of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") || ++ apple_h9p_pcie_device_port(pcie, dev) != 0) ++ return NULL; ++ return pcie; ++} ++ ++int apple_ans_sleep_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie || h9p_suspend != 2 || !apple_s8003_handoff_enabled()) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_sleep_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_sleep_begin); ++ ++/* release6: caller froze and synchronized I/O, before SHN. */ ++/* Lab 0018: caller synchronized queues and stopped CC/RDY before this. */ ++int apple_ans_nvmmu_reset(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ struct apple_h9p_nvmmu *nvmmu; ++ int ret; ++ ++ if (!pcie || !(pcie->enabled_ports & BIT(0))) ++ return -ENODEV; ++ nvmmu = &pcie->nvmmu; ++ if (!nvmmu->ready) ++ return -ENODEV; ++ memset(nvmmu->tcb, 0, nvmmu->tcb_size); ++ memset(nvmmu->tcb_table, 0, nvmmu->tcb_table_size); ++ memset(nvmmu->tcb_sgl, 0, nvmmu->tcb_sgl_size); ++ ret = apple_h9p_nvmmu_program(pcie); ++ WRITE_ONCE(nvmmu->poisoned, !!ret); ++ if (ret) ++ return ret; ++ bitmap_zero(nvmmu->mapped, APPLE_ANS_NVMMU_MAX_REQS); ++ dev_info(dev, "NVMMU: stopped-controller cache/SART rebuilt\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_reset); ++ ++bool apple_ans_is_s8003(struct device *dev) ++{ ++ return apple_ans_endpoint_host(dev) != NULL; ++} ++EXPORT_SYMBOL_GPL(apple_ans_is_s8003); ++ ++int apple_ans_link_up(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -ENODEV; ++ if (!(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return -ENOLINK; ++ return !!(readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & ++ S8003_PORT_LINKSTS_UP); ++} ++EXPORT_SYMBOL_GPL(apple_ans_link_up); ++ ++/* Host-only evidence, no endpoint config/BAR reads or invalidate register. */ ++void apple_ans_host_dump(struct device *dev, const char *reason) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return; ++ dev_info(dev, "ANS host %s: ports %#x link %#x LTSSM %#x FIFO %#x poisoned %u mapped %*pb\n", ++ reason, pcie->enabled_ports, ++ readl(pcie->base_port[0] + H9P_PORT_LINKSTS), ++ readl(pcie->base_port[0] + H9P_PORT_LTSSMCTL), ++ readl(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO), ++ READ_ONCE(pcie->nvmmu.poisoned), APPLE_ANS_NVMMU_MAX_REQS, ++ pcie->nvmmu.mapped); ++} ++EXPORT_SYMBOL_GPL(apple_ans_host_dump); ++ ++int apple_ans_nvmmu_drain(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ u32 fifo; ++ int ret; ++ ++ if (!pcie || !pcie->nvmmu.ready) ++ return -ENODEV; ++ if (!bitmap_empty(pcie->nvmmu.mapped, APPLE_ANS_NVMMU_MAX_REQS)) ++ return -EBUSY; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ dev_info(dev, "Apple ANS: outgoing NVMMU FIFO empty check (20ms) result %d\n", ret); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_drain); ++ ++int apple_ans_sleep_stopped(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ unsigned int tag; ++ u32 fifo; ++ int ret; ++ ++ if (!pcie) ++ return -ENODEV; ++ /* All queues are drained and BME is clear before retiring translations. */ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } ++ /* The last invalidate also has to leave the FIFO before certification. */ ++ ret = readl_poll_timeout_atomic(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 1, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ ret = apple_ans_dart_set_active(dev, false); ++ if (!ret) ++ WRITE_ONCE(pcie->ans_sleep_stopped, true); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_sleep_stopped); ++ ++int apple_ans_handoff_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie || strcmp(ans_boot_mode, "handoff")) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_sleep_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); ++ ++static void apple_h9p_s8003_shutdown(struct platform_device *pdev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ unsigned int port; ++ ++ if (!kexec_in_progress || !pcie || ++ !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) ++ return; ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!strcmp(ans_boot_mode, "handoff") && ++ (!READ_ONCE(pcie->ans_sleep_stopped) || ++ (READ_ONCE(ans_keep_power) && !apple_s8003_port_powered(pcie)))) ++ panic("Apple ANS: refusing uncertified powered handoff"); ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; ++ ++ if (!(pcie->enabled_ports & BIT(port))) ++ continue; ++ /* release6: block root routing, retain refclk, then isolate. */ ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[port], 1)) ++ panic("Apple PCIe: PERST failed during handoff"); ++ if (!port && apple_s8003_handoff_enabled()) ++ apple_s8003_drain_link(pcie, READ_ONCE(ans_kexec_flush)); ++ if (apple_s8003_port_stop(pcie, port)) ++ panic("Apple PCIe: LTSSM stop failed during handoff"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (!port && pcie->nvmmu.ready) { ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ } ++ if (!port && !strcmp(ans_boot_mode, "handoff") && ++ !READ_ONCE(ans_keep_power)) ++ apple_s8003_port_off(pcie, port); ++ if (!port && apple_s8003_handoff_enabled() && ++ (!apple_s8003_port_powered(pcie) || ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) ++ panic("Apple ANS: outgoing powered port certificate failed"); ++ pcie->enabled_ports &= ~BIT(port); ++ } ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++} ++ ++static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *cfg = pcie->base_config; ++ ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!(pcie->enabled_ports & BIT(0))) ++ return; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[0], 1) || ++ apple_s8003_port_stop(pcie, 0)) ++ panic("ANS capture isolation failed"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ pcie->enabled_ports &= ~BIT(0); ++} ++ ++struct apple_h9p_crash_state { ++ struct apple_h9p_pcie *pcie; ++ int result; ++ bool busy; ++ bool found; ++}; ++ ++/* No NVMe driver, queues, SHN, or old DMA buffers are needed in phoenix. */ ++static int apple_h9p_crash_stop_ans(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_crash_state *state = data; ++ struct apple_h9p_pcie *pcie = state->pcie; ++ void __iomem *bar; ++ u32 csts = 0; ++ int i, ret; ++ ++ if (pdev->vendor != PCI_VENDOR_ID_APPLE || ++ ((pdev->class >> 8) != PCI_CLASS_STORAGE_EXPRESS && ++ (pdev->class >> 8) != PCI_CLASS_STORAGE_OTHER)) ++ return 0; ++ state->found = true; ++ if (pdev->driver) { ++ state->busy = true; ++ return 1; ++ } ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ state->result = -ENOLINK; ++ return 0; ++ } ++ ret = pci_enable_device_mem(pdev); ++ if (ret) { ++ state->result = ret; ++ return 0; ++ } ++ bar = pci_iomap_range(pdev, 0, 0, SZ_8K); ++ if (!bar) { ++ ret = -ENOMEM; ++ goto disable; ++ } ++ /* An unknown queue must never be rung or sent a shutdown notification. */ ++ dev_info(pcie->dev, "phoenix ANS: CC %#x CSTS %#x, clearing EN without SHN\n", ++ readl(bar + NVME_REG_CC), readl(bar + NVME_REG_CSTS)); ++ writel(0, bar + NVME_REG_CC); ++ ret = -ETIMEDOUT; ++ for (i = 0; i < 2000; i++) { ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ ret = -ENOLINK; ++ break; ++ } ++ csts = readl(bar + NVME_REG_CSTS); ++ if (csts == U32_MAX) { ++ ret = -EIO; ++ break; ++ } ++ if (!(csts & NVME_CSTS_RDY)) { ++ ret = 0; ++ break; ++ } ++ usleep_range(1000, 1100); ++ } ++ dev_info(pcie->dev, "phoenix ANS: disable result %d CSTS %#x; blocking bus master\n", ++ ret, csts); ++ pci_iounmap(pdev, bar); ++disable: ++ pci_clear_master(pdev); ++ pci_disable_device(pdev); ++ state->result = ret; ++ return 0; ++} ++ ++static ssize_t crash_quiesce_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ struct apple_h9p_crash_state state = { .pcie = pcie }; ++ ++ if (!pcie || !is_kdump_kernel() || ++ !of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !sysfs_streq(buf, "1")) ++ return -EPERM; ++ /* The endpoint gate must stay closed; don't park under a live driver. */ ++ if (pcie->late_ports || work_busy(&pcie->late_work.work)) ++ return -EBUSY; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_crash_stop_ans, &state); ++ if (state.busy) ++ return -EBUSY; ++ /* Isolation is still required after a bounded RDY timeout. */ ++ /* Wi-Fi stays linked until its own shutdown callback has run. */ ++ apple_h9p_capture_park(pcie); ++ dev_info(dev, "phoenix: ANS/PCIe quiesced (controller found %d result %d)\n", ++ state.found, state.result); ++ return count; ++} ++static DEVICE_ATTR_WO(crash_quiesce); ++ ++static struct attribute *apple_h9p_crash_attrs[] = { ++ &dev_attr_crash_quiesce.attr, ++ NULL, ++}; ++ ++static const struct attribute_group apple_h9p_crash_group = { ++ .attrs = apple_h9p_crash_attrs, ++}; ++ ++ ++/* Orchestrator-only fault injection; never enabled by a normal boot. */ ++static bool panic_resume_once; ++module_param(panic_resume_once, bool, 0600); ++MODULE_PARM_DESC(panic_resume_once, ++ "S8003: deliberately panic before one PCIe resume, only with kdump armed"); ++ ++static void apple_h9p_crash_resume_test(struct device *dev) ++{ ++ if (!of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !READ_ONCE(panic_resume_once)) ++ return; ++ WRITE_ONCE(panic_resume_once, false); ++ if (!kexec_crash_loaded()) { ++ dev_warn(dev, "resume panic test refused: no capture kernel armed\n"); ++ return; ++ } ++ panic("S8003: deliberate panic before PCIe resume"); ++} ++ ++ ++static bool apple_h9p_sleep_consumer_busy(void) ++{ ++ struct device_node *np; ++ struct platform_device *pdev; ++ struct usb_role_switch *sw; ++ struct path root; ++ bool busy; ++ ++ get_fs_root(current->fs, &root); ++ busy = root.dentry->d_sb->s_bdev && ++ MAJOR(root.dentry->d_sb->s_bdev->bd_dev) == NBD_MAJOR; ++ path_put(&root); ++ ++ np = of_find_compatible_node(NULL, NULL, "gpu,s8003"); ++ if (np) { ++ pdev = of_find_device_by_node(np); ++ if (pdev) { ++ busy |= pdev->dev.driver && pm_runtime_active(&pdev->dev); ++ put_device(&pdev->dev); ++ } ++ of_node_put(np); ++ } ++ np = of_find_compatible_node(NULL, NULL, "apple,s8000-usb-role-switch"); ++ if (np) { ++ sw = usb_role_switch_find_by_fwnode(of_fwnode_handle(np)); ++ if (!IS_ERR_OR_NULL(sw)) { ++ busy |= usb_role_switch_get_role(sw) == USB_ROLE_HOST; ++ usb_role_switch_put(sw); ++ } ++ of_node_put(np); ++ } ++ return busy; ++} ++ ++static int apple_h9p_sleep_precheck(struct notifier_block *nb, ++ unsigned long action, void *unused) ++{ ++ struct apple_h9p_pcie *pcie = container_of(nb, struct apple_h9p_pcie, ++ sleep_precheck); ++ ++ if (action != PM_SUSPEND_TO_IDLE) ++ return NOTIFY_DONE; ++ if (h9p_suspend != 2 || !apple_s8003_handoff_enabled() || ++ apple_h9p_sleep_consumer_busy()) { ++ dev_warn(pcie->dev, "sleep refused before sync: suspend=2 and ans_keep_power=1 ans_boot_mode=handoff required; active GPU/USB host or NBD root forbidden\n"); ++ return notifier_from_errno(-EBUSY); ++ } ++ return NOTIFY_DONE; ++} ++ ++static void apple_h9p_sleep_unregister(void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ unregister_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++} ++ ++static int apple_h9p_sleep_register(struct apple_h9p_pcie *pcie) ++{ ++ int ret; ++ ++ if (!of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) ++ return 0; ++ pcie->sleep_precheck.notifier_call = apple_h9p_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++ if (ret) ++ return ret; ++ return devm_add_action_or_reset(pcie->dev, apple_h9p_sleep_unregister, pcie); ++} ++ ++static int apple_h9p_s8003_suspend(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ if (!of_device_is_compatible(dev->of_node, "apple,s8003-pcie")) ++ return 0; ++ if (h9p_suspend != 2 || !apple_s8003_handoff_enabled() || ++ !READ_ONCE(pcie->ans_sleep_stopped) || !apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ cancel_delayed_work_sync(&pcie->late_work); ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (ret) ++ return ret; ++ apple_s8003_drain_link(pcie, READ_ONCE(ans_sleep_flush)); ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (ret) { ++ /* PM skips resume for a parent whose suspend failed. Restore now. */ ++ apple_s8003_powered_reset(pcie, READ_ONCE(ans_sleep_flush)); ++ return ret; ++ } ++ pcie->ans_sleep_powered = true; ++ dev_info(dev, "ANS sleep powered under PERST; no full-cycle fallback\n"); ++ return 0; ++} ++ ++static int apple_h9p_s8003_resume(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ apple_h9p_crash_resume_test(dev); ++ ++ if (!of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !pcie->ans_sleep_powered) ++ return 0; ++ ret = apple_s8003_powered_reset(pcie, READ_ONCE(ans_sleep_flush)); ++ pcie->ans_sleep_powered = false; ++ WRITE_ONCE(pcie->ans_sleep_stopped, false); ++ if (ret) ++ dev_err(dev, "ANS powered resume failed %d; endpoint isolated under PERST\n", ret); ++ else ++ dev_info(dev, "ANS powered resume trained, try=first\n"); ++ return ret; ++} ++ ++static DEFINE_SIMPLE_DEV_PM_OPS(apple_h9p_s8003_pm_ops, ++ apple_h9p_s8003_suspend, apple_h9p_s8003_resume); ++ ++static bool crash_quiesce_supported = true; ++module_param(crash_quiesce_supported, bool, 0444); ++ ++static bool h9p_probe = !IS_ENABLED(CONFIG_NVME_APPLE_ANS_EARLY_USB); ++module_param_named(probe, h9p_probe, bool, 0644); ++MODULE_PARM_DESC(probe, "Allow S8003 PCIe probe after USB rescue is ready"); ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3426,6 +4014,9 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + u32 doorbell; + int ret; + ++ if (!READ_ONCE(h9p_probe) && of_device_is_compatible(dev->of_node, "apple,s8003-pcie")) ++ return -EPROBE_DEFER; ++ + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; +@@ -3578,6 +4169,16 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (is_kdump_kernel() && of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) { ++ ret = devm_device_add_group(dev, &apple_h9p_crash_group); ++ if (ret) ++ return ret; ++ } ++ ++ ret = apple_h9p_sleep_register(pcie); ++ if (ret) ++ return ret; ++ + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +@@ -3660,8 +4261,10 @@ MODULE_DEVICE_TABLE(of, apple_h9p_pcie_of_match); + + static struct platform_driver apple_h9p_pcie_driver = { + .probe = apple_h9p_pcie_probe, ++ .shutdown = apple_h9p_s8003_shutdown, + .driver = { + .name = "pcie-apple-h9p", ++ .pm = pm_sleep_ptr(&apple_h9p_s8003_pm_ops), + .of_match_table = apple_h9p_pcie_of_match, + .suppress_bind_attrs = true, + }, +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index a59861010..788a343ee 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -17,11 +17,36 @@ struct device; + * Map @npages pages for FlatDMA request @tag, or retire the tag if @npages is + * zero. @to_memory: the controller writes the pages (an NVMe read). + */ ++#if IS_REACHABLE(CONFIG_APPLE_DART) ++int apple_ans_dart_set_active(struct device *dev, bool active); ++#else ++static inline int apple_ans_dart_set_active(struct device *dev, bool active) ++{ ++ return -EOPNOTSUPP; ++} ++#endif ++ + #if IS_REACHABLE(CONFIG_PCIE_APPLE_H9P) ++int apple_ans_sleep_begin(struct device *dev); ++int apple_ans_handoff_begin(struct device *dev); ++int apple_ans_sleep_stopped(struct device *dev); ++int apple_ans_nvmmu_drain(struct device *dev); ++int apple_ans_nvmmu_reset(struct device *dev); ++int apple_ans_link_up(struct device *dev); ++bool apple_ans_is_s8003(struct device *dev); ++void apple_ans_host_dump(struct device *dev, const char *reason); + int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova); + #else ++static inline int apple_ans_sleep_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_handoff_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_sleep_stopped(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_drain(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_reset(struct device *dev) { return -EOPNOTSUPP; } ++static inline int apple_ans_link_up(struct device *dev) { return 1; } ++static inline bool apple_ans_is_s8003(struct device *dev) { return false; } ++static inline void apple_ans_host_dump(struct device *dev, const char *reason) {} + static inline int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova) +diff --git a/include/linux/suspend.h b/include/linux/suspend.h +index b02876f1a..0a4719b20 100644 +--- a/include/linux/suspend.h ++++ b/include/linux/suspend.h +@@ -449,6 +449,15 @@ void restore_processor_state(void); + /* kernel/power/main.c */ + extern int register_pm_notifier(struct notifier_block *nb); + extern int unregister_pm_notifier(struct notifier_block *nb); ++/** ++ * register_pm_suspend_precheck_notifier - register an early suspend veto ++ * @nb: notifier whose action is the target suspend_state_t ++ * ++ * Prechecks run before filesystem sync and console preparation. They must ++ * only validate current policy/state and return; they have no post callback. ++ */ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb); ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb); + extern void ksys_sync_helper(void); + extern void pm_report_hw_sleep_time(u64 t); + extern void pm_report_max_hw_sleep(u64 t); +@@ -498,6 +507,16 @@ static inline int unregister_pm_notifier(struct notifier_block *nb) + return 0; + } + ++static inline int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return 0; ++} ++ ++static inline int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return 0; ++} ++ + static inline void pm_report_hw_sleep_time(u64 t) {}; + static inline void pm_report_max_hw_sleep(u64 t) {}; + +diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c +index dc770b9a6..819650bcd 100644 +--- a/kernel/kexec_core.c ++++ b/kernel/kexec_core.c +@@ -53,6 +53,7 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); + + /* Flag to indicate we are going to kexec a new kernel */ + bool kexec_in_progress = false; ++EXPORT_SYMBOL_GPL(kexec_in_progress); + + bool kexec_file_dbg_print; + +diff --git a/kernel/power/main.c b/kernel/power/main.c +index 5429e9f19..ae36afe43 100644 +--- a/kernel/power/main.c ++++ b/kernel/power/main.c +@@ -152,6 +152,32 @@ int pm_sleep_fs_sync(void) + /* Routines for PM-transition notifications */ + + static BLOCKING_NOTIFIER_HEAD(pm_chain_head); ++static BLOCKING_NOTIFIER_HEAD(pm_suspend_precheck_chain_head); ++ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_register(&pm_suspend_precheck_chain_head, ++ nb); ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_precheck_notifier); ++ ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_unregister(&pm_suspend_precheck_chain_head, ++ nb); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); ++ ++int pm_suspend_precheck_call_chain(suspend_state_t state) ++{ ++ int ret; ++ ++ /* Veto-only: callers have not changed system suspend state yet. */ ++ ret = blocking_notifier_call_chain(&pm_suspend_precheck_chain_head, ++ (unsigned long)state, NULL); ++ ++ return notifier_to_errno(ret); ++} + + int register_pm_notifier(struct notifier_block *nb) + { +diff --git a/kernel/power/power.h b/kernel/power/power.h +index 75b638438..1de8fe895 100644 +--- a/kernel/power/power.h ++++ b/kernel/power/power.h +@@ -240,6 +240,7 @@ static inline void suspend_test_finish(const char *label) {} + /* kernel/power/main.c */ + extern int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down); + extern int pm_notifier_call_chain(unsigned long val); ++int pm_suspend_precheck_call_chain(suspend_state_t state); + #endif + + #ifdef CONFIG_HIGHMEM +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 57c442686..4c5da415a 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -591,6 +591,13 @@ static int enter_state(suspend_state_t state) + if (!mutex_trylock(&system_transition_mutex)) + return -EBUSY; + ++ error = pm_suspend_precheck_call_chain(state); ++ if (error) { ++ dpm_save_failed_step(SUSPEND_PREPARE); ++ mutex_unlock(&system_transition_mutex); ++ return error; ++ } ++ + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); + From 33047d223df787c7a01fb8d278a9d6be74f1b4dd Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 13/33] pinctrl/spi/input: quiesce S8000 peripherals for kexec Assisted-by: LLM --- chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + ...0025-warm-peripherals-implementation.patch | 41 ++++++ ...0004-warm-peripherals-implementation.patch | 120 ++++++++++++++++++ ...0009-warm-peripherals-implementation.patch | 113 +++++++++++++++++ 5 files changed, 280 insertions(+) create mode 100644 kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch create mode 100644 kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch create mode 100644 kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..a83f0ba7 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -15,3 +15,6 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "J71t on hardware" stages = "remote_boot two-stage CPID list" + +# Local capability delta: warm-peripherals. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-peripherals.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..9a5ec064 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -54,3 +54,6 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" + +# Local capability delta: warm-peripherals. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-peripherals.md. diff --git a/kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch b/kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch new file mode 100644 index 00000000..3ef9fca9 --- /dev/null +++ b/kernel/patches/drivers/input/0025-warm-peripherals-implementation.patch @@ -0,0 +1,41 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] Input: apple-z2: pinctrl/spi/input: quiesce S8000 peripherals for kexec + +Retire inherited GPIO IRQ state, stop PIO SPI queues and chip selects, and quiesce Apple Z2 touch before its SPI parent shuts down. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/input/touchscreen/apple_z2.c b/drivers/input/touchscreen/apple_z2.c +index 161ebb4b8..145d6ffcf 100644 +--- a/drivers/input/touchscreen/apple_z2.c ++++ b/drivers/input/touchscreen/apple_z2.c +@@ -4584,6 +4584,16 @@ static struct spi_device_id apple_z2_of_id[] = { + }; + MODULE_DEVICE_TABLE(spi, apple_z2_of_id); + ++/* Children stop before the SPI controller drains its core queue. */ ++static void apple_z2_shutdown(struct spi_device *spi) ++{ ++ struct apple_z2 *z2 = spi_get_drvdata(spi); ++ ++ apple_z2_stop(z2); ++ /* Also wait when the IRQ thread already masked itself. */ ++ synchronize_irq(spi->irq); ++} ++ + static struct spi_driver apple_z2_driver = { + .driver = { + .name = "apple-z2", +@@ -4594,6 +4604,7 @@ static struct spi_driver apple_z2_driver = { + .id_table = apple_z2_of_id, + .probe = apple_z2_probe, + .remove = apple_z2_remove, ++ .shutdown = apple_z2_shutdown, + }; + + module_spi_driver(apple_z2_driver); diff --git a/kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch b/kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch new file mode 100644 index 00000000..5e4b4238 --- /dev/null +++ b/kernel/patches/drivers/pinctrl/0004-warm-peripherals-implementation.patch @@ -0,0 +1,120 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pinctrl: pinctrl/spi/input: quiesce S8000 peripherals for kexec + +Retire inherited GPIO IRQ state, stop PIO SPI queues and chip selects, and quiesce Apple Z2 touch before its SPI parent shuts down. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c +index b97023c97..766fcf5bc 100644 +--- a/drivers/pinctrl/pinctrl-apple-gpio.c ++++ b/drivers/pinctrl/pinctrl-apple-gpio.c +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -46,6 +47,7 @@ struct apple_gpio_pinctrl { + + void __iomem *base; + struct regmap *map; ++ bool retire_irqs; + + struct pinctrl_desc pinctrl_desc; + struct gpio_chip gpio_chip; +@@ -379,8 +381,33 @@ static void apple_gpio_irq_handler(struct irq_desc *desc) + chained_irq_enter(chip, desc); + for (pinh = 0; pinh < gc->ngpio; pinh += 32) { + pending = readl_relaxed(pctl->base + REG_IRQ(*grpp, pinh)); +- for_each_set_bit(pinl, &pending, 32) +- generic_handle_domain_irq(gc->irq.domain, pinh + pinl); ++ for_each_set_bit(pinl, &pending, 32) { ++ unsigned int pin = pinh + pinl; ++ int ret; ++ ++ ret = pin < gc->ngpio ? ++ generic_handle_domain_irq(gc->irq.domain, pin) : -EINVAL; ++ if (ret && pctl->retire_irqs) { ++ /* An inherited, unmapped source has nobody to ack it. ++ * Mask only this GPIO, then clear this group's latch. ++ */ ++ if (pin < gc->ngpio) { ++ u32 reg = apple_gpio_get_reg(pctl, pin); ++ u32 mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ u32 off = FIELD_PREP(REG_GPIOx_MODE, ++ REG_GPIOx_IN_IRQ_OFF); ++ ++ if (!(reg & REG_GPIOx_PERIPH) && ++ mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, off); ++ } ++ writel(BIT(pinl), pctl->base + REG_IRQ(*grpp, pinh)); ++ dev_warn_ratelimited(pctl->dev, ++ "masked orphan inherited GPIO IRQ group=%u pin=%u\n", ++ *grpp, pin); ++ } ++ } + } + chained_irq_exit(chip, desc); + } +@@ -403,6 +430,35 @@ static const struct irq_chip apple_gpio_irqchip = { + + /* Probe & register */ + ++static void apple_gpio_quiesce_irqs(struct apple_gpio_pinctrl *pctl) ++{ ++ struct gpio_chip *gc = &pctl->gpio_chip; ++ unsigned int pin, group, mode, masked = 0; ++ u32 reg; ++ ++ if (!pctl->retire_irqs) ++ return; ++ /* Before chained parent IRQs are enabled, retire old GPIO IRQ modes. ++ * Preserve outputs, mux, pulls, data and grouping. Consumers rearm their ++ * own pins later; panic entry need not have run any shutdown callbacks. ++ */ ++ for (pin = 0; pin < gc->ngpio; pin++) { ++ reg = apple_gpio_get_reg(pctl, pin); ++ mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ if (!(reg & REG_GPIOx_PERIPH) && mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) { ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, ++ FIELD_PREP(REG_GPIOx_MODE, REG_GPIOx_IN_IRQ_OFF)); ++ masked++; ++ } ++ } ++ for (group = 0; group < gc->irq.num_parents; group++) ++ for (pin = 0; pin < gc->ngpio; pin += 32) ++ writel(GENMASK(min(32U, gc->ngpio - pin) - 1, 0), ++ pctl->base + REG_IRQ(group, pin)); ++ dev_info(pctl->dev, "inherited GPIO IRQ baseline: masked %u inherited inputs\n", masked); ++} ++ + static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + { + struct gpio_irq_chip *girq = &pctl->gpio_chip.irq; +@@ -452,6 +508,7 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + girq->handler = handle_level_irq; + } + ++ apple_gpio_quiesce_irqs(pctl); + ret = devm_gpiochip_add_data(pctl->dev, &pctl->gpio_chip, pctl); + + out_free_irq_data: +@@ -485,6 +542,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) + if (!pctl) + return -ENOMEM; + pctl->dev = &pdev->dev; ++ pctl->retire_irqs = of_device_is_compatible(pdev->dev.of_node, "apple,s8000-pinctrl"); + pctl->soc = of_device_get_match_data(&pdev->dev); + pctl->gpio_chip.irq.num_parents = nirqs; + dev_set_drvdata(&pdev->dev, pctl); diff --git a/kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch b/kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch new file mode 100644 index 00000000..738091e9 --- /dev/null +++ b/kernel/patches/drivers/spi/0009-warm-peripherals-implementation.patch @@ -0,0 +1,113 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] spi: pinctrl/spi/input: quiesce S8000 peripherals for kexec + +Retire inherited GPIO IRQ state, stop PIO SPI queues and chip selects, and quiesce Apple Z2 touch before its SPI parent shuts down. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c +index dc5154e1b..ed9f67a59 100644 +--- a/drivers/spi/spi-apple.c ++++ b/drivers/spi/spi-apple.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -141,6 +142,7 @@ struct apple_spi { + struct dmaengine_result dma_result; + const struct apple_spi_hw *hw; + u32 word_delay; ++ struct spi_controller *ctlr; + }; + + static inline void reg_write(struct apple_spi *spi, int offset, u32 value) +@@ -231,6 +233,19 @@ static void apple_spi_init(struct apple_spi *spi) + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + } + ++/* Lab 0015: normalize only the S8000 PIO register set, while powered. */ ++static int apple_spi_s8000_idle(struct apple_spi *spi) ++{ ++ reg_write(spi, APPLE_SPI_CTRL, 0); ++ reg_write(spi, APPLE_SPI_CFG, spi->hw->cfg_base); ++ reg_write(spi, APPLE_SPI_RXCNT, 0); ++ reg_write(spi, APPLE_SPI_TXCNT, 0); ++ reg_write(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS); ++ apple_spi_t8010_clear_status(spi); ++ /* Complete posted writes before publishing children or leaving Linux. */ ++ return readl(spi->regs + APPLE_SPI_CTRL) ? -EIO : 0; ++} ++ + static int apple_spi_prepare_message(struct spi_controller *ctlr, + struct spi_message *msg) + { +@@ -1120,6 +1135,7 @@ static int apple_spi_probe(struct platform_device *pdev) + + spi = spi_controller_get_devdata(ctlr); + platform_set_drvdata(pdev, spi); ++ spi->ctlr = ctlr; + init_completion(&spi->done); + init_completion(&spi->dma_done); + spi->hw = of_device_get_match_data(&pdev->dev); +@@ -1214,6 +1230,11 @@ static int apple_spi_probe(struct platform_device *pdev) + } + + apple_spi_init(spi); ++ if (spi->hw->pio_only) { ++ ret = apple_spi_s8000_idle(spi); ++ if (ret) ++ return ret; ++ } + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret < 0) +@@ -1223,6 +1244,32 @@ static int apple_spi_probe(struct platform_device *pdev) + return 0; + } + ++static void apple_spi_shutdown(struct platform_device *pdev) ++{ ++ struct apple_spi *spi = platform_get_drvdata(pdev); ++ struct spi_controller *ctlr = spi->ctlr; ++ unsigned int cs; ++ int ret; ++ ++ /* Other SoCs use different DMA/IRQ registers and keep their policy. */ ++ if (!spi->hw->pio_only) ++ return; ++ /* Children have shut down; drain the core queue before MMIO. */ ++ ret = spi_controller_suspend(ctlr); ++ if (!ret) ++ ret = pm_runtime_resume_and_get(&pdev->dev); ++ if (ret >= 0) { ++ if (ctlr->cs_gpiods) ++ for (cs = 0; cs < ctlr->num_chipselect; cs++) ++ gpiod_set_value_cansleep(ctlr->cs_gpiods[cs], 0); ++ ret = apple_spi_s8000_idle(spi); ++ } ++ if (ret < 0 && kexec_in_progress) ++ panic("S8000 SPI shutdown failed before kexec: %d", ret); ++ /* Keep runtime power pinned until the jump. */ ++ dev_info(&pdev->dev, "S8000 SPI shutdown result %d\n", ret); ++} ++ + static const struct apple_spi_hw apple_spi_hw_s8000 = { + .legacy_status = true, + .pio_only = true, +@@ -1260,6 +1307,7 @@ MODULE_DEVICE_TABLE(of, apple_spi_of_match); + + static struct platform_driver apple_spi_driver = { + .probe = apple_spi_probe, ++ .shutdown = apple_spi_shutdown, + .driver = { + .name = "apple-spi", + .of_match_table = apple_spi_of_match, From 0886d8cc7c12d5aa013b2fd5a128847136c6ce8e Mon Sep 17 00:00:00 2001 From: brkosebay Date: Sun, 8 Nov 2026 16:00:00 +0100 Subject: [PATCH 14/33] Warm Restart evidence, reset guard and pristine DT Assisted-by: LLM --- boot/initramfs/root/init | 32 ++++ chips/s8003.toml | 3 + devices/j71t/device.toml | 3 + .../0009-warm-kernel-implementation.patch | 175 ++++++++++++++++++ .../drm/0011-warm-kernel-implementation.patch | 26 +++ .../of/0001-warm-kernel-implementation.patch | 33 ++++ .../0002-warm-kernel-implementation.patch | 126 +++++++++++++ .../0001-warm-kernel-implementation.patch | 47 +++++ src/orchard/image/platform.py | 15 +- src/orchard/service.py | 4 +- tests/test_platform_layer.py | 2 + 11 files changed, 462 insertions(+), 4 deletions(-) create mode 100644 kernel/patches/devices/s8000/0009-warm-kernel-implementation.patch create mode 100644 kernel/patches/drivers/drm/0011-warm-kernel-implementation.patch create mode 100644 kernel/patches/drivers/of/0001-warm-kernel-implementation.patch create mode 100644 kernel/patches/drivers/pmdomain/0002-warm-kernel-implementation.patch create mode 100644 kernel/patches/drivers/watchdog/0001-warm-kernel-implementation.patch diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index 9556f126..34a27921 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -10,6 +10,7 @@ mount -t devtmpfs -o mode=0755,nosuid dev /dev exec /dev/console 2>&1 stop() { + echo "<3>orchard initrd: $1; rescue shell" > /dev/kmsg echo "orchard initrd: $1" echo 'Starting a shell on the console; the system did not start.' exec sh @@ -31,8 +32,11 @@ done [ ! -e /etc/orchard/usb-modules ] || for module in $(cat /etc/orchard/usb-modules); do modprobe "$module" || echo "orchard initrd: could not load $module; no USB" done +echo '<6>orchard initrd: storage load begin' > /dev/kmsg for module in $(cat /etc/orchard/modules); do + echo "<6>orchard initrd: modprobe $module begin" > /dev/kmsg modprobe "$module" || stop "could not load $module" + echo "<6>orchard initrd: modprobe $module complete" > /dev/kmsg done # An install boot: orchard's add-on, appended to this initrd by the CLI, writes the @@ -87,8 +91,35 @@ done mkdir -p /newroot mount ${fstype:+-t "$fstype"} -o "$mode${flags:+,$flags}" "$device" /newroot || stop "could not mount $device" +echo '<6>orchard initrd: root mounted' > /dev/kmsg [ -x "/newroot$init" ] || [ -L "/newroot$init" ] || stop "the root has no $init" +# Preserve the running boot's *pristine* tree before any package update can +# replace /boot or same-release module assets. The matching initrd embeds cold +# DTBs; warm boots name their retained stage instead. Never infer it by uname. +mkdir -p /run /newroot/run +mount -t tmpfs -o mode=0755,nosuid,nodev tmpfs /run +mkdir -p /run/orchard-kexec +warm_pristine= +for argument in $(cat /proc/cmdline); do + case "$argument" in orchard_warm.dtb=*) warm_pristine=${argument#orchard_warm.dtb=} ;; esac +done +if [ -n "$warm_pristine" ]; then + case "$warm_pristine" in + /var/tmp/orchard-warm.*/pristine.dtb) + case "$warm_pristine" in *..*) stop 'invalid warm DT provenance' ;; esac + cp "/newroot$warm_pristine" /run/orchard-kexec/pristine.dtb || stop 'missing warm pristine DT' ;; + *) stop 'invalid warm DT provenance' ;; + esac +elif [ -f /etc/orchard/pristine-map ]; then + board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + while read compatible file; do + [ "$compatible" != "$board" ] || cp "/etc/orchard/dtbs/$file" /run/orchard-kexec/pristine.dtb +done < /etc/orchard/pristine-map +fi +cat /proc/sys/kernel/random/boot_id > /run/orchard-kexec/boot-id +mount --move /run /newroot/run || stop 'cannot retain boot provenance' + # USB for the host: network (usb0) and a serial console (ttyGS0), named after # the system, so `orchard devices` sees it. What runs on them is the system's. usb_gadget() { @@ -112,6 +143,7 @@ usb_gadget() { basename "$(ls -d /sys/class/udc/* | head -1)" > "$g/UDC" } usb_gadget || echo 'orchard initrd: no USB gadget; continuing without USB' +echo '<6>orchard initrd: USB setup complete; switching root' > /dev/kmsg for name in dev proc sys; do mount --move "/$name" "/newroot/$name" diff --git a/chips/s8003.toml b/chips/s8003.toml index 49370529..b3596a18 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -30,3 +30,6 @@ stages = "remote_boot two-stage CPID list" # Local capability delta: warm-peripherals. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-peripherals.md. + +# Local capability delta: warm-kernel. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-kernel.md. diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index fbcc3fa0..45384a30 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -69,3 +69,6 @@ camera = "missing" # Local capability delta: warm-peripherals. # Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-peripherals.md. + +# Local capability delta: warm-kernel. +# Integration evidence / qualification limits: linux-a9 docs/release/orchard-prs-20261108-final/warm-kernel.md. diff --git a/kernel/patches/devices/s8000/0009-warm-kernel-implementation.patch b/kernel/patches/devices/s8000/0009-warm-kernel-implementation.patch new file mode 100644 index 00000000..2dff6460 --- /dev/null +++ b/kernel/patches/devices/s8000/0009-warm-kernel-implementation.patch @@ -0,0 +1,175 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: Warm Restart evidence, reset guard and pristine DT + +Preserve warm power-domain holds and the ADBE boot framebuffer, add bounded shutdown checkpoints and the cold-reset guard, and retain a correctly mapped ramoops reservation across kexec. Embed per-board pristine DTBs in the matching initrd and archive the actual running boot's pristine tree before updates replace its files. Kernel/module/BLS pairing must remain explicit; DFU is known to erase the retained logs. The new Orchard cold-baseline and three-hop ladder is pending. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h +index 5e1211c54..bd22a913d 100644 +--- a/arch/arm64/include/asm/mmu.h ++++ b/arch/arm64/include/asm/mmu.h +@@ -62,6 +62,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void) + + extern void arm64_memblock_init(void); + extern void paging_init(void); ++extern void apple_soc_publish_pstore(void); + extern void bootmem_init(void); + extern void create_mapping_noalloc(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot); +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index 29c6100f0..45c23e3d9 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -332,6 +332,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) + arm64_memblock_init(); + + paging_init(); ++ apple_soc_publish_pstore(); + + acpi_table_upgrade(); + +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index fbf215ecc..affb37089 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -22,6 +22,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -193,6 +195,91 @@ static int __init early_mem(char *p) + } + early_param("mem", early_mem); + ++static phys_addr_t apple_soc_pstore_fdt __initdata; ++ ++/* ++ * Reserve before paging/CMA/crashkernel, without touching the linear map: ++ * paging_init() has not built it yet. Prepare the replacement FDT through a ++ * temporary early mapping and publish its linear alias only after paging. ++ * Never choose a fresh address in kdump; reuse the first kernel's region. ++ */ ++static void __init apple_soc_reserve_pstore(void) ++{ ++ void *fdt = initial_boot_params, *copy; ++ phys_addr_t base, copy_pa; ++ fdt64_t reg[2]; ++ char name[40]; ++ int parent, node, size, ret; ++ ++ if (!IS_BUILTIN(CONFIG_PSTORE_RAM) || !fdt || ++ (fdt_node_check_compatible(fdt, 0, "apple,s8000") && ++ fdt_node_check_compatible(fdt, 0, "apple,s8003"))) ++ return; ++ if (fdt_node_offset_by_compatible(fdt, -1, "ramoops") >= 0) ++ return; ++ if (is_kdump_kernel()) { ++ pr_warn("Apple pstore: no inherited region in capture kernel\n"); ++ return; ++ } ++ parent = fdt_path_offset(fdt, "/reserved-memory"); ++ if (parent < 0 || fdt_address_cells(fdt, parent) != 2 || ++ fdt_size_cells(fdt, parent) != 2) ++ return; ++ ++ /* Existing firmware, kernel and initrd reservations are already known. */ ++ base = memblock_phys_alloc(SZ_2M, SZ_2M); ++ if (!base) ++ return; ++ size = fdt_totalsize(fdt) + SZ_4K; ++ copy_pa = memblock_phys_alloc(size, SMP_CACHE_BYTES); ++ if (!copy_pa) ++ goto free_region; ++ copy = early_memremap(copy_pa, size); ++ if (!copy) ++ goto free_copy; ++ ret = fdt_open_into(fdt, copy, size); ++ if (ret) ++ goto unmap_copy; ++ parent = fdt_path_offset(copy, "/reserved-memory"); ++ snprintf(name, sizeof(name), "ramoops@%llx", (u64)base); ++ node = fdt_add_subnode(copy, parent, name); ++ if (node < 0) ++ goto unmap_copy; ++ reg[0] = cpu_to_fdt64(base); ++ reg[1] = cpu_to_fdt64(SZ_2M); ++ if (fdt_setprop(copy, node, "compatible", "ramoops", sizeof("ramoops")) || ++ fdt_setprop(copy, node, "reg", reg, sizeof(reg)) || ++ fdt_setprop(copy, node, "no-map", NULL, 0) || ++ fdt_setprop_u32(copy, node, "record-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "console-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "max-reason", 2) || ++ fdt_add_mem_rsv(copy, base, SZ_2M)) ++ goto unmap_copy; ++ /* No cached linear alias of ramoops' write-combined persistent mapping. */ ++ if (memblock_mark_nomap(base, SZ_2M)) ++ goto unmap_copy; ++ early_memunmap(copy, size); ++ apple_soc_pstore_fdt = copy_pa; ++ pr_info("Apple pstore: reserved 2 MiB at %pa, carried in kexec FDT\n", &base); ++ return; ++ ++unmap_copy: ++ early_memunmap(copy, size); ++free_copy: ++ memblock_phys_free(copy_pa, size); ++free_region: ++ memblock_phys_free(base, SZ_2M); ++ pr_warn("Apple pstore: reservation failed\n"); ++} ++ ++/* The linear map now exists; unflattening must see the prepared FDT. */ ++void __init apple_soc_publish_pstore(void) ++{ ++ if (apple_soc_pstore_fdt) ++ early_init_dt_verify(phys_to_virt(apple_soc_pstore_fdt), ++ apple_soc_pstore_fdt); ++} ++ + void __init arm64_memblock_init(void) + { + s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); +@@ -298,6 +385,7 @@ void __init arm64_memblock_init(void) + } + + early_init_fdt_scan_reserved_mem(); ++ apple_soc_reserve_pstore(); + } + + void __init bootmem_init(void) +diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c +index 2eb0d4fa2..45b9b51df 100644 +--- a/fs/pstore/ram.c ++++ b/fs/pstore/ram.c +@@ -9,6 +9,7 @@ + #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + + #include ++#include + #include + #include + #include +@@ -729,6 +730,13 @@ static int ramoops_probe(struct platform_device *pdev) + phys_addr_t paddr; + int err = -EINVAL; + ++ /* Leave the original panic/console records intact across phoenix. */ ++ if (is_kdump_kernel() && (of_machine_is_compatible("apple,s8000") || ++ of_machine_is_compatible("apple,s8003"))) { ++ pr_info("capture kernel: preserving Apple records for the full boot\n"); ++ return -ENODEV; ++ } ++ + /* + * Only a single ramoops area allowed at a time, so fail extra + * probes. diff --git a/kernel/patches/drivers/drm/0011-warm-kernel-implementation.patch b/kernel/patches/drivers/drm/0011-warm-kernel-implementation.patch new file mode 100644 index 00000000..a71533ea --- /dev/null +++ b/kernel/patches/drivers/drm/0011-warm-kernel-implementation.patch @@ -0,0 +1,26 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] drm/apple-g5p: Warm Restart evidence, reset guard and pristine DT + +Preserve warm power-domain holds and the ADBE boot framebuffer, add bounded shutdown checkpoints and the cold-reset guard, and retain a correctly mapped ramoops reservation across kexec. Embed per-board pristine DTBs in the matching initrd and archive the actual running boot's pristine tree before updates replace its files. Kernel/module/BLS pairing must remain explicit; DFU is known to erase the retained logs. The new Orchard cold-baseline and three-hop ladder is pending. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 24c76fef9..363086cd7 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -1090,6 +1090,9 @@ static void adbe_shutdown(struct platform_device *pdev) + struct adbe *adbe = platform_get_drvdata(pdev); + + drm_atomic_helper_shutdown(&adbe->drm); ++ timer_delete_sync(&adbe->flip_timer); ++ /* release9: restore the boot buffer even if the CRTC was never enabled. */ ++ adbe_park(adbe); + writel(adbe->saved_inten, adbe->regs + ADP_INTEN); + } + diff --git a/kernel/patches/drivers/of/0001-warm-kernel-implementation.patch b/kernel/patches/drivers/of/0001-warm-kernel-implementation.patch new file mode 100644 index 00000000..8fa188d9 --- /dev/null +++ b/kernel/patches/drivers/of/0001-warm-kernel-implementation.patch @@ -0,0 +1,33 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] of: Warm Restart evidence, reset guard and pristine DT + +Preserve warm power-domain holds and the ADBE boot framebuffer, add bounded shutdown checkpoints and the cold-reset guard, and retain a correctly mapped ramoops reservation across kexec. Embed per-board pristine DTBs in the matching initrd and archive the actual running boot's pristine tree before updates replace its files. Kernel/module/BLS pairing must remain explicit; DFU is known to erase the retained logs. The new Orchard cold-baseline and three-hop ladder is pending. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c +index 8c9d6395d..76cf3e5a4 100644 +--- a/drivers/of/of_reserved_mem.c ++++ b/drivers/of/of_reserved_mem.c +@@ -275,6 +275,16 @@ void __init fdt_scan_reserved_mem_late(void) + return; + } + ++ /* Early architecture fixups may have added static reservations. */ ++ total_reserved_mem_cnt = reserved_mem_count; ++ fdt_for_each_subnode(child, fdt, node) { ++ int len; ++ ++ if (of_fdt_device_is_available(fdt, child) && ++ fdt_getprop(fdt, child, "reg", &len) && len) ++ total_reserved_mem_cnt++; ++ } ++ + /* Attempt dynamic allocation of a new reserved_mem array */ + if (alloc_reserved_mem_array()) + return; diff --git a/kernel/patches/drivers/pmdomain/0002-warm-kernel-implementation.patch b/kernel/patches/drivers/pmdomain/0002-warm-kernel-implementation.patch new file mode 100644 index 00000000..c58ac9fb --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0002-warm-kernel-implementation.patch @@ -0,0 +1,126 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pmdomain: apple: Warm Restart evidence, reset guard and pristine DT + +Preserve warm power-domain holds and the ADBE boot framebuffer, add bounded shutdown checkpoints and the cold-reset guard, and retain a correctly mapped ramoops reservation across kexec. Embed per-board pristine DTBs in the matching initrd and archive the actual running boot's pristine tree before updates replace its files. Kernel/module/BLS pairing must remain explicit; DFU is known to erase the retained logs. The new Orchard cold-baseline and three-hop ladder is pending. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 0ae0eda2d..38f705973 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -60,12 +60,23 @@ struct apple_pmgr_ps { + u32 min_state; + bool has_genpd; + bool manual_pm; ++ bool warm; ++ bool keep_on; + }; + + #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) + #define rcdev_to_apple_pmgr_ps(_rcdev) container_of(_rcdev, struct apple_pmgr_ps, rcdev) + #define clk_hw_to_apple_pmgr_gate(_hw) container_of(_hw, struct apple_pmgr_gate, hw) + ++/* release6/release9: inherited J71t firmware is not a cold reset baseline. */ ++static bool kexec_keep_on = true; ++ ++static int __init apple_pmgr_keep_on_setup(char *arg) ++{ ++ return kstrtobool(arg, &kexec_keep_on); ++} ++early_param("apple_pmgr.kexec_keep_on", apple_pmgr_keep_on_setup); ++ + static int apple_pmgr_ps_set(struct apple_pmgr_ps *ps, u32 pstate, bool auto_enable) + { + int ret; +@@ -85,17 +96,21 @@ static int apple_pmgr_ps_set(struct apple_pmgr_ps *ps, u32 pstate, bool auto_ena + + dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", ps->name, pstate, reg); + +- regmap_write(ps->regmap, ps->offset, reg); ++ ret = regmap_write(ps->regmap, ps->offset, reg); ++ if (ret) ++ return ret; + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, reg, + (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, + APPLE_PMGR_PS_SET_TIMEOUT); +- if (ret < 0) ++ if (ret < 0) { + dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", + ps->name, pstate, reg); ++ return ret; ++ } + +- if (auto_enable) { ++ if (auto_enable && !ps->keep_on) { + /* Not all devices implement this; this is a no-op where not implemented. */ + reg &= ~APPLE_PMGR_FLAGS; + reg |= APPLE_PMGR_AUTO_ENABLE; +@@ -138,6 +153,9 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) + unsigned long flags; + int ret; + ++ if (ps->keep_on) ++ return -EBUSY; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_PWRGATE, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -170,6 +188,10 @@ static void apple_pmgr_clk_disable(struct clk_hw *hw) + unsigned long flags; + int ret; + ++ /* The lab has no separate clk gate sharing this power-state word. */ ++ if (ps->keep_on) ++ return; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, gate->off_state, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -308,6 +330,10 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + return ret; + } + ++ ps->warm = of_device_is_compatible(node, "apple,s8000-pmgr-pwrstate") && ++ of_property_read_bool(of_chosen, "linux,booted-from-kexec"); ++ ps->keep_on = ps->warm && kexec_keep_on; ++ ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); + clock_only = of_property_read_bool(node, "apple,clock-gate-only"); + if (!clock_only) { + ps->genpd.flags |= GENPD_FLAG_IRQ_SAFE; +@@ -317,12 +343,17 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); +- if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) ++ if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && ++ !(ps->warm && ps->manual_pm)) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, + FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + + active = apple_pmgr_ps_is_active(ps); +- ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); ++ if (ps->warm && active && !ps->manual_pm) { ++ ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_ACTIVE, false); ++ if (ret) ++ return ret; ++ } + if (!clock_only && of_property_read_bool(node, "apple,always-on")) { + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + if (!active) { +@@ -333,7 +364,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + /* Turn on auto-PM if the domain is already on */ +- if (active) ++ if (active && !ps->warm) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, + ps->manual_pm ? 0 : APPLE_PMGR_AUTO_ENABLE); + diff --git a/kernel/patches/drivers/watchdog/0001-warm-kernel-implementation.patch b/kernel/patches/drivers/watchdog/0001-warm-kernel-implementation.patch new file mode 100644 index 00000000..39bab6de --- /dev/null +++ b/kernel/patches/drivers/watchdog/0001-warm-kernel-implementation.patch @@ -0,0 +1,47 @@ +From: brkosebay +Date: Sun, 08 Nov 2026 16:00:00 +0100 +Subject: [PATCH] watchdog: Warm Restart evidence, reset guard and pristine DT + +Preserve warm power-domain holds and the ADBE boot framebuffer, add bounded shutdown checkpoints and the cold-reset guard, and retain a correctly mapped ramoops reservation across kexec. Embed per-board pristine DTBs in the matching initrd and archive the actual running boot's pristine tree before updates replace its files. Kernel/module/BLS pairing must remain explicit; DFU is known to erase the retained logs. The new Orchard cold-baseline and three-hop ladder is pending. + +Integration fixups are folded into the final delta. Device observations apply +to the integration build; the isolated topic requires qualification. + +Assisted-by: LLM +--- + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index f53395329..c1b4bb64b 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -53,6 +53,18 @@ + + #define APPLE_WDT_TIMEOUT_DEFAULT 30 + ++/* ++ * On iDevices booted through a bootrom exploit (checkm8 -> pongoOS -> m1n1), ++ * a SoC reset boots the stock OS, and getting back to Linux needs a host ++ * computer and a manual DFU entry. With cold_restart=0 the restart handler ++ * refuses, so a reboot that wasn't meant to leave Linux (a failed kexec, a ++ * panic timeout, sysrq-b) halts instead. Writable at runtime, so a ++ * deliberate "reboot to iOS" can turn it back on first. ++ */ ++static bool cold_restart = true; ++module_param(cold_restart, bool, 0644); ++MODULE_PARM_DESC(cold_restart, "Allow restarts through a SoC reset (default: true)"); ++ + struct apple_wdt { + struct watchdog_device wdd; + void __iomem *regs; +@@ -123,6 +135,11 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + ++ if (!READ_ONCE(cold_restart)) { ++ pr_emerg("apple-wdt: SoC reset refused (apple_wdt.cold_restart=0), halting\n"); ++ return -EPERM; ++ } ++ + writel_relaxed(APPLE_WDT_CTRL_RESET_EN, wdt->regs + APPLE_WDT_WD1_CTRL); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_BITE_TIME); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_CUR_TIME); diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index b39ae7bb..52302497 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -85,7 +85,7 @@ def tools(hardware=None, cache=None, platform='apple-16k'): return prepare_tools(hardware, cache, device=sorted(boards(platform, hardware))[0]) / 'busybox' -def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES): +def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES, pristine_dtbs=None): """orchard's initrd for installed systems: BusyBox, the storage modules and their dependencies. MODULES is {path: (data, mode)} as module_files gives it. @@ -115,6 +115,15 @@ def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_ entries['etc/orchard/modules'] = (stat.S_IFREG | 0o644, ('\n'.join(loaded) + '\n').encode()) if usb: entries['etc/orchard/usb-modules'] = (stat.S_IFREG | 0o644, ('\n'.join(usb) + '\n').encode()) + if pristine_dtbs: + entries['etc/orchard/dtbs'] = (directory, b'') + mapping = [] + for board, data in sorted(pristine_dtbs.items()): + if not board.replace('-', '').isalnum(): + raise ValueError('Invalid pristine DT board') + entries[f'etc/orchard/dtbs/{board}.dtb'] = (stat.S_IFREG | 0o644, data) + mapping.append(f'apple,{board} {board}.dtb\n') + entries['etc/orchard/pristine-map'] = (stat.S_IFREG | 0o644, ''.join(mapping).encode()) dependency_lines = [] for name in sorted(wanted): target = f'lib/modules/{release}/{name}' @@ -153,13 +162,13 @@ def files(bundle, platform, board_dtbs, *, busybox, hardware=None): base = f'usr/lib/modules/{release}/' result[base + 'vmlinuz'] = ((bundle / described['image']).read_bytes(), 0o644) modules_to_load = storage_modules(hardware, platform) - result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, - storage_modules=modules_to_load), 0o644) for dtb in sorted(set(board_dtbs.values())): path = bundle / 'dtbs' / (dtb + '.dtb') if path.is_symlink() or not path.is_file(): raise ValueError(f'Kernel bundle lacks {path.name}') result[base + 'dtb/' + path.name] = (path.read_bytes(), 0o644) + pristine = {board: result[base + 'dtb/' + dtb + '.dtb'][0] for board, dtb in board_dtbs.items()} + result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, pristine_dtbs=pristine, storage_modules=modules_to_load), 0o644) result.update(regulatory_files()) defaults = hardware / 'system/defaults' for path in sorted(defaults.rglob('*')): diff --git a/src/orchard/service.py b/src/orchard/service.py index a1dd700c..953f6066 100644 --- a/src/orchard/service.py +++ b/src/orchard/service.py @@ -150,7 +150,9 @@ def entry_for(record, devicetree): name = record.get('name', 'Linux') return {'id': re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') or 'linux', 'title': name, # Quiet as the recovery: no console text over the splash; the system maps its console. - 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0', + 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0' + ( + ' pcie_apple_h9p.ans_keep_power=1 pcie_apple_h9p.ans_boot_mode=handoff' + ' panic=0 apple_wdt.cold_restart=0' if devicetree == 's8003-j71t.dtb' else ''), 'devicetree': devicetree} diff --git a/tests/test_platform_layer.py b/tests/test_platform_layer.py index 9acba731..53914cef 100644 --- a/tests/test_platform_layer.py +++ b/tests/test_platform_layer.py @@ -83,6 +83,8 @@ def test_the_initrd_carries_busybox_and_the_storage_modules_only(self): self.assertEqual(names['bin/busybox'], b'static busybox') self.assertEqual(names['init'], (ROOT / 'boot/initramfs/root/init').read_bytes()) self.assertEqual(names['etc/orchard/modules'], b'nvme\n') + self.assertEqual(names['etc/orchard/dtbs/j71t.dtb'], b's8003-j71t') + self.assertIn(b'apple,j71t j71t.dtb\n', names['etc/orchard/pristine-map']) self.assertEqual(sorted(name for name in names if name.endswith('.ko')), [base + 'kernel/drivers/nvme/host/nvme-core.ko', base + 'kernel/drivers/nvme/host/nvme.ko']) self.assertEqual(names[base + 'modules.dep'], From 0674e1dbde8dc80a1fc7e7ce4566d9189469cdff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 15/33] j71t: Guarded Mac current profile A Mac grants an iPad more than the standard USB current, but the charger took the gadget's reported budget unchecked and the composite gadget could draw the extra current while the bus was suspended. Cap and log the USB host budget in the TriStar charger, draw no extra Apple current while suspended, and let the initramfs and orchard USB helpers opt J71t into the Mac current profile. Documented in docs/bring-up/charging/j71t-mac-current.md, with tests. Assisted-by: LLM --- boot/initramfs/root/init | 52 +++++++- docs/bring-up/charging/j71t-mac-current.md | 119 ++++++++++++++++++ ...le-tristar-charger-cap-and-log-the-U.patch | 82 ++++++++++++ ...site-draw-no-extra-Apple-current-whi.patch | 47 +++++++ src/orchard/apple/linux_usb.py | 14 ++- src/orchard/apple/usb.py | 10 +- 6 files changed, 319 insertions(+), 5 deletions(-) create mode 100644 docs/bring-up/charging/j71t-mac-current.md create mode 100644 kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch create mode 100644 kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index 9556f126..86f05636 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -10,6 +10,7 @@ mount -t devtmpfs -o mode=0755,nosuid dev /dev exec /dev/console 2>&1 stop() { + echo "<3>orchard initrd: $1; rescue shell" > /dev/kmsg echo "orchard initrd: $1" echo 'Starting a shell on the console; the system did not start.' exec sh @@ -31,8 +32,11 @@ done [ ! -e /etc/orchard/usb-modules ] || for module in $(cat /etc/orchard/usb-modules); do modprobe "$module" || echo "orchard initrd: could not load $module; no USB" done +echo '<6>orchard initrd: storage load begin' > /dev/kmsg for module in $(cat /etc/orchard/modules); do + echo "<6>orchard initrd: modprobe $module begin" > /dev/kmsg modprobe "$module" || stop "could not load $module" + echo "<6>orchard initrd: modprobe $module complete" > /dev/kmsg done # An install boot: orchard's add-on, appended to this initrd by the CLI, writes the @@ -87,8 +91,35 @@ done mkdir -p /newroot mount ${fstype:+-t "$fstype"} -o "$mode${flags:+,$flags}" "$device" /newroot || stop "could not mount $device" +echo '<6>orchard initrd: root mounted' > /dev/kmsg [ -x "/newroot$init" ] || [ -L "/newroot$init" ] || stop "the root has no $init" +# Preserve the running boot's *pristine* tree before any package update can +# replace /boot or same-release module assets. The matching initrd embeds cold +# DTBs; warm boots name their retained stage instead. Never infer it by uname. +mkdir -p /run /newroot/run +mount -t tmpfs -o mode=0755,nosuid,nodev tmpfs /run +mkdir -p /run/orchard-kexec +warm_pristine= +for argument in $(cat /proc/cmdline); do + case "$argument" in orchard_warm.dtb=*) warm_pristine=${argument#orchard_warm.dtb=} ;; esac +done +if [ -n "$warm_pristine" ]; then + case "$warm_pristine" in + /var/tmp/orchard-warm.*/pristine.dtb) + case "$warm_pristine" in *..*) stop 'invalid warm DT provenance' ;; esac + cp "/newroot$warm_pristine" /run/orchard-kexec/pristine.dtb || stop 'missing warm pristine DT' ;; + *) stop 'invalid warm DT provenance' ;; + esac +elif [ -f /etc/orchard/pristine-map ]; then + board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + while read compatible file; do + [ "$compatible" != "$board" ] || cp "/etc/orchard/dtbs/$file" /run/orchard-kexec/pristine.dtb +done < /etc/orchard/pristine-map +fi +cat /proc/sys/kernel/random/boot_id > /run/orchard-kexec/boot-id +mount --move /run /newroot/run || stop 'cannot retain boot provenance' + # USB for the host: network (usb0) and a serial console (ttyGS0), named after # the system, so `orchard devices` sees it. What runs on them is the system's. usb_gadget() { @@ -97,13 +128,29 @@ usb_gadget() { [ ! -e "$g" ] || return 0 name=$(sed -n 's/^NAME=//p' /newroot/etc/os-release /newroot/usr/lib/os-release 2>/dev/null | head -1 | tr -d "\"'") board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + # Opt-in compatibility identity in macOS's 2400 mA capability table. + # This requests a host grant; it does not set a charger limit. Keep the + # generic composite personality (no iOS/usbmux or preferred-config API). + pid=0x4145 + device_class=0x00 device_subclass=0x00 device_protocol=0x00 + if [ "$board" = apple,j71t ] && + [ "$(cat /newroot/etc/orchard/mac-usb-current 2>/dev/null)" = 1 ]; then + pid=0x1014 + # Standard IAD device: macOS defers its CDC composite match here, + # allowing the generic composite driver to deliver the grant. + device_class=0xef device_subclass=0x02 device_protocol=0x01 + fi + serial=$(sh /etc/orchard/usb-identity) || return 1 mkdir "$g" && echo 0x05ac > "$g/idVendor" && - echo 0x4145 > "$g/idProduct" && + echo "$pid" > "$g/idProduct" && + echo "$device_class" > "$g/bDeviceClass" && + echo "$device_subclass" > "$g/bDeviceSubClass" && + echo "$device_protocol" > "$g/bDeviceProtocol" && mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" && echo 'Orchard Linux' > "$g/strings/0x409/manufacturer" && echo "${name:-Linux}" > "$g/strings/0x409/product" && - echo "BOOT:$(cat /proc/sys/kernel/random/boot_id) BOARD:${board#apple,}" > "$g/strings/0x409/serialnumber" && + echo "$serial" > "$g/strings/0x409/serialnumber" && echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" && echo 500 > "$g/configs/c.1/MaxPower" && mkdir "$g/functions/ncm.usb0" "$g/functions/acm.usb0" && @@ -112,6 +159,7 @@ usb_gadget() { basename "$(ls -d /sys/class/udc/* | head -1)" > "$g/UDC" } usb_gadget || echo 'orchard initrd: no USB gadget; continuing without USB' +echo '<6>orchard initrd: USB setup complete; switching root' > /dev/kmsg for name in dev proc sys; do mount --move "/$name" "/newroot/$name" diff --git a/docs/bring-up/charging/j71t-mac-current.md b/docs/bring-up/charging/j71t-mac-current.md new file mode 100644 index 00000000..af31232a --- /dev/null +++ b/docs/bring-up/charging/j71t-mac-current.md @@ -0,0 +1,119 @@ +# J71t Mac extra-current candidate + +Host-tested on 2026-11-08; not yet qualified on an iPad. The matching root +initrd reads `/etc/orchard/mac-usb-current` from the mounted system. Exactly +`1` on `apple,j71t` selects the compatibility identity `05ac:1014`, explicit +IAD device class ef/02/01, and the existing ACM+NCM configuration at +500 mA MaxPower. Omarchy A carries that opt-in as a separate system change. +Removing the file selects the normal `05ac:4145` identity on the next boot. +Recovery, installer and RAM/NFS gadgets retain their existing identities. + +This Mac's AppleUSBHostMergeProperties personality lists `1014` in its +2400 mA UsbPowerSinkCapability table; `4145` is absent. The generic +AppleUSBHostCompositeDevice power-negotiation path can therefore allocate +extra power and send vendor device request `40/40`, zero length, signed +suspend mA in wValue and signed extra active mA in wIndex. No usbmux interface +or iOS preferred-configuration protocol is needed for this PID. Its IAD +class ef/02/01 matches Apple's generic composite personality. AppleUSBCDC's +matching IAD personality has IOMatchDefer=True; its non-deferred vendor +personality instead requires 0/0/0. Both ACM and NCM already emit IADs. The strings +continue to identify Orchard and Omarchy. This is a compatibility identity, +not an assigned Orchard PID or a claim to implement an Apple product. + +A new identity requires macOS accessory authorization, possibly on every boot +because the existing BOOT serial changes. Keep the Mac unlocked and reconnect +the cable if the device remains unregistered/unmatched. Do not silently change +the Mac's security settings. Orchard discovery recognizes this PID as a +system only when the existing serial contains a boot UUID and BOARD:j71t. + +The identity is never current evidence on the tablet. Only the host's real +request reaches composite_vbus_draw -> DWC2 -> lightning. The charger clamps +the allowance at its board maximum and retains pack guards, stepped ramp, +readback verification and source-epoch withdrawal. No request means the +existing 500 mA fallback. BC1.2 primary/secondary detection already exists; +SDP on this Mac must not be relabeled CDP just to obtain 1.5 A. + +A grant arriving while suspended cannot select its active allowance. The +existing policy discards the preceding active grant on suspend; resuming +needs a fresh grant. Reset and disconnect also clear it. The supplied suspend +field is logged but does not authorize an elevated suspend limit. This work +does not certify the pre-existing charger's USB suspend/preconfiguration +compliance: its unknown-source hardware fallback remains 500 mA. + +`current_max=0` on a configured link is a separate possible blocker. A report +is accepted only while present, with data enabled, in the current PMU source +epoch. Logs now show reported/accepted current, present/data and report epoch; +dynamic debug on the charger shows stale report/source epochs. Do not replay +an old allowance after a PMU event to make this number nonzero. A packet trace +and these logs must establish whether the live zero was a missing report, a +gated report, or epoch invalidation. That observation is still required. + +## Host verification + +Run `ORCHARD_KERNEL_SOURCE=/path/to/prepared/kernel PYTHONPATH=src python3 +-m unittest discover -s tests -p test_j71t_mac_current.py -v`. The C test +extracts the actual production functions and executes them with ASan/UBSan: +all request-type/request/short-length/vendor combinations, all signed extra +values, suspend, board caps, ramp/revoke, attachment and stale epochs. +The other tests run the initrd gadget function in a mock filesystem and test +both host discovery implementations. They do not contact USB hardware. + +## Device acceptance check + +Boot the new Image, DTB, matching initrd and modules together, with the +Omarchy A opt-in present. Do not rebind a live gadget to edit its identity. + +On the unlocked Mac, approve the new accessory and capture `ioreg -p IOUSB +-l -w0` plus `ioreg -l -w0 -r -c IOUSBHostDevice`. The device should be +registered/matched/configured, have PID 0x1014, MaxPower 500, +UsbPowerSinkCapability 2400 and UsbPowerSinkAllocation above 500 (2400 if +available). Check AppleUSBHostCompositeDevice binding and +kUSBConfigurationCurrentOverride. The host's available 3000 mA alone is not +a grant. Preserve ACM serial and NCM network connectivity. + +On the tablet, enable dynamic debug for composite.c, dwc2/gadget.c and +apple_tristar_charger.c. Capture dmesg and lightning's online, usb_type, +current_max and input_current_limit alongside battery's status/current_now, +capacity, temperature, voltage and the UDC state. A usual full grant is +extra=1900, giving accepted/current_max=2400000, followed by a guarded ramp +to input_current_limit=2400000. Never exceed min(500+signed extra, 2400). +A smaller grant must produce a correspondingly smaller limit, rounded down. +SDP may remain correct; it does not imply that a software grant failed. + +At a moderate state of charge and normal pack temperature, hold brightness +and desktop workload fixed for at least 60 seconds: battery status must be +Charging and signed current_now positive. Full/thermal-limited packs are not +a valid charging-current acceptance test. If current_max stays zero, preserve +the report/epoch logs and EP0 trace; do not force a PMIC limit. + +Also test: a plain 500 mA host (no extra request), reset, unplug/replug to a +different source, a smaller/revoked grant, and Mac sleep/wake. Any loss of a +valid grant must stop elevated draw and return to the fallback. Suspend may +lose extra current until macOS issues a new grant; record that limitation. +Mac allocation without an actual request and accepted allowance is a failed +test, even if battery current happens to be positive with the panel off. + +## Sustained NCM transfers + +The runner reported two carrier-up stalls with ACM still working, recovered +by cycling only usb0. USB patch 0012 fixes a reproducible NCM timer bug: +NETDEV_TX_BUSY during a timer flush used to abandon the partial NTB. Retry +at the usual 300 us interval, advancing the timer expiry before restart. +This is a supported cause of silent NCM stalls; the runner's exact endpoint +state was not captured, so it is not proof that both incidents had this cause. + +A 16 KiB NTB does not need to fit in a dedicated FIFO at once: DWC2 splits +transfers and streams USB packets through the FIFO. The 2056-entry total is +in 32-bit words. No FIFO, DMA, ZLP or negotiated NTB-size change is justified +by those totals alone. ACM surviving also differs from the known shared-OUT +FIFO starvation fixed by USB patch 0008. Do not change those controls without +capturing endpoint state and request completions during a stall. + +Run test_ncm_flush.py with ORCHARD_KERNEL_SOURCE to execute the production +callback under repeated request exhaustion and subsequent recovery. On the +next device run count every `usb0-stall-bounce` marker, including the runner's +transient oi-usb0-watchdog. Use one watchdog at a time so bounces are attributable. +The Omarchy service's new carrier-up fallback cycles only usb0 after 15 seconds +without replies from the previously reachable 172.16.42.2 peer. It leaves +UDC, ACM and host charging-grant ownership intact. A clean sustained-transfer +acceptance run requires zero watchdog bounces, not merely successful retries. diff --git a/kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch b/kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch new file mode 100644 index 00000000..08928664 --- /dev/null +++ b/kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch @@ -0,0 +1,82 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] power: supply: apple_tristar_charger: cap and log the USB + host budget + +An Apple host can grant more current than the standard USB budget, and +the gadget reports the granted budget to the charger. A stale report +from an earlier connection was already dropped, but a report above the +charger's maximum was passed on as is. + +Cap the reported budget and the input current limit at the charger's +maximum, re-evaluate the source when only the budget changes, and log +the reported and accepted values with the connection epoch. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 24 +++++++++++++++----- + 1 file changed, 18 insertions(+), 6 deletions(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index a953cedcb..c50bfc861 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -649,7 +649,12 @@ static int tristar_identify(struct tristar_charger *chg, struct tristar_ident *i + */ + static unsigned int tristar_usb_ma(struct tristar_charger *chg) + { +- return chg->usb_epoch == READ_ONCE(chg->epoch) ? chg->usb_ma : 0; ++ if (chg->usb_epoch != READ_ONCE(chg->epoch)) { ++ dev_dbg(chg->dev, "USB budget stale: report epoch %u, source epoch %u\n", ++ chg->usb_epoch, READ_ONCE(chg->epoch)); ++ return 0; ++ } ++ return chg->usb_ma; + } + + /* +@@ -1140,7 +1145,8 @@ static int tristar_get_property(struct power_supply *psy, + break; + case POWER_SUPPLY_PROP_CURRENT_MAX: + /* What the identified source or USB host allows; 0 when unknown. */ +- val->intval = max(chg->source_ma, tristar_usb_ma(chg)) * 1000; ++ val->intval = min(max(chg->source_ma, tristar_usb_ma(chg)), ++ chg->max_ma) * 1000; + break; + default: + mutex_unlock(&chg->lock); +@@ -1163,23 +1169,29 @@ static int tristar_set_property(struct power_supply *psy, + const union power_supply_propval *val) + { + struct tristar_charger *chg = power_supply_get_drvdata(psy); +- unsigned int old_code; ++ unsigned int old_code, old_ma, report_ma; + + if (psp != POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT) + return -EINVAL; + + mutex_lock(&chg->lock); + old_code = chg->code; +- chg->usb_ma = chg->present && chg->usb_on ? max(val->intval, 0) / 1000 : 0; ++ old_ma = tristar_usb_ma(chg); ++ report_ma = max(val->intval, 0) / 1000; ++ chg->usb_ma = chg->present && chg->usb_on ? ++ min(report_ma, chg->max_ma) : 0; + chg->usb_epoch = READ_ONCE(chg->epoch); +- dev_dbg(chg->dev, "USB host allows %u mA\n", chg->usb_ma); ++ dev_info_ratelimited(chg->dev, ++ "USB budget: reported %u mA, accepted %u mA, present %u, data %u, epoch %u\n", ++ report_ma, chg->usb_ma, chg->present, chg->usb_on, ++ chg->usb_epoch); + if (!chg->stopping) { + tristar_update_target(chg); + mod_delayed_work(chg->wq, &chg->poll_work, 0); + } + mutex_unlock(&chg->lock); + +- if (READ_ONCE(chg->code) != old_code) ++ if (READ_ONCE(chg->code) != old_code || old_ma != READ_ONCE(chg->usb_ma)) + power_supply_changed(psy); + return 0; + } diff --git a/kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch b/kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch new file mode 100644 index 00000000..74f69a05 --- /dev/null +++ b/kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch @@ -0,0 +1,47 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] usb: gadget: composite: draw no extra Apple current while + suspended + +The extra current an Apple host grants must not be drawn while the bus +is suspended. reset_config() could apply it during suspend, because the +suspended flag was cleared only at the end of composite_resume(). Skip +the extra current while suspended and clear the flag before the +configuration is applied again on resume. + +Assisted-by: LLM +--- + drivers/usb/gadget/composite.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 1da3effb2..ea76d3918 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -959,7 +959,7 @@ static void reset_config(struct usb_composite_dev *cdev) + static void composite_vbus_draw(struct usb_composite_dev *cdev, unsigned int mA) + { + cdev->vbus_draw_ma = mA; +- if (mA && cdev->apple_extra_ma) ++ if (mA && !cdev->suspended && cdev->apple_extra_ma) + mA = max(500 + cdev->apple_extra_ma, 0); + usb_gadget_vbus_draw(cdev->gadget, mA); + } +@@ -2665,6 +2665,7 @@ void composite_resume(struct usb_gadget *gadget) + DBG(cdev, "resume\n"); + if (cdev->driver->resume) + cdev->driver->resume(cdev); ++ cdev->suspended = 0; + if (cdev->config) { + list_for_each_entry(f, &cdev->config->functions, list) { + /* +@@ -2695,8 +2696,6 @@ void composite_resume(struct usb_gadget *gadget) + maxpower = min(maxpower, 100U); + composite_vbus_draw(cdev, maxpower); + } +- +- cdev->suspended = 0; + } + + /*-------------------------------------------------------------------------*/ diff --git a/src/orchard/apple/linux_usb.py b/src/orchard/apple/linux_usb.py index 6095440c..ca8aae49 100644 --- a/src/orchard/apple/linux_usb.py +++ b/src/orchard/apple/linux_usb.py @@ -30,6 +30,12 @@ def serial_tags(serial): 'pwned': pwned.group(1) if pwned else ('checkra1n' if 'YOLO:checkra1n' in serial else None)} +def stable_id(serial): + """Hashed per-device identity, absent on legacy gadgets.""" + match = re.search(r'\bID:([0-9a-fA-F]{12})\b', serial) + return match.group(1).lower() if match else None + + def devices(root=Path('/sys/bus/usb/devices')): found = [] for node in root.iterdir(): @@ -45,14 +51,20 @@ def devices(root=Path('/sys/bus/usb/devices')): ecid = re.search(r'\bECID:([0-9a-fA-F]+)\b', serial) chip = re.search(r'\bCPID:([0-9a-fA-F]+)\b', serial) board = re.search(r'\bBDID:([0-9a-fA-F]+)\b', serial) + usb_id = stable_id(serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) + # A compatibility PID also belongs to real Apple hardware. Only our + # existing boot/profile identity makes it an Orchard system. + mac_current = (pid == 0x1014 and (boot is not None or usb_id is not None) and + profile is not None and profile.group(1) == 'j71t') ports = {str(Path('/dev') / p.name) for p in node.glob('**/tty/ttyACM*')} if len(ports) > 1: raise ValueError('Ambiguous USB serial ports on a device') product = (node / 'product').read_text().strip() if (node / 'product').exists() else '' - found.append(dict(location=node.name, mode=MODES.get(pid, 'other'), + found.append(dict(location=node.name, mode='system' if mac_current else MODES.get(pid, 'other'), ecid=int(ecid.group(1), 16) if ecid else None, boot_id=str(uuid.UUID(boot.group(1))) if boot else None, + usb_id=usb_id, chip=int(chip.group(1), 16) if chip else None, board=int(board.group(1), 16) if board else None, profile=profile.group(1) if profile else None, product=product, diff --git a/src/orchard/apple/usb.py b/src/orchard/apple/usb.py index 1b1a4bf3..263ef546 100644 --- a/src/orchard/apple/usb.py +++ b/src/orchard/apple/usb.py @@ -5,7 +5,7 @@ import sys import uuid -from .linux_usb import MODES, M1N1, devices as linux_devices, m1n1_ports as linux_m1n1_ports, serial_tags +from .linux_usb import MODES, M1N1, devices as linux_devices, m1n1_ports as linux_m1n1_ports, serial_tags, stable_id from orchard.state import EndpointUnavailable @@ -33,7 +33,12 @@ def devices(): continue serial = node.get('USB Serial Number') or '' boot = re.search(r'\bBOOT:([0-9a-fA-F-]{36})\b', serial) + usb_id = stable_id(serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) + # A compatibility PID also belongs to real Apple hardware. Only our + # existing boot/profile identity makes it an Orchard system. + mac_current = (node.get('idProduct') == 0x1014 and (boot is not None or usb_id is not None) and + profile is not None and profile.group(1) == 'j71t') fields = [re.search(r'\b' + label + r':([0-9a-fA-F]+)\b', serial) for label in ('ECID', 'CPID', 'BDID')] descendants = list(node.get('IORegistryEntryChildren', [])) @@ -47,9 +52,10 @@ def devices(): ports.append(port) if len(ports) > 1: raise DeviceError('Ambiguous USB serial ports on the selected device') - found.append({'location': location, 'mode': MODES.get(node.get('idProduct'), 'other'), + found.append({'location': location, 'mode': 'system' if mac_current else MODES.get(node.get('idProduct'), 'other'), 'ecid': int(fields[0].group(1), 16) if fields[0] else None, 'boot_id': str(uuid.UUID(boot.group(1))) if boot else None, + 'usb_id': usb_id, 'chip': int(fields[1].group(1), 16) if fields[1] else None, 'board': int(fields[2].group(1), 16) if fields[2] else None, 'profile': profile.group(1) if profile else None, From a54e204901823de88b1fdbf9b2199078fc5a155e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 16/33] j71t: AOP sensors with a retained firmware state for reload The A9 AOP hosts the accelerometer, gyroscope, pressure and ambient-light sensors. Its firmware needs boot arguments and per-device calibration that m1n1 did not pass on S8003, and a reloaded AOP (after kexec or sleep) lost its sensor firmware state, so the sensors did not come back. m1n1 passes the A9 AOP boot arguments and sensor calibration. The kernel selects a certified AOP reload path through device-tree match data (bindings, DT, mailbox, pmdomain, IIO and AOP driver patches), keeps the AOP power domain and firmware state for that reload, and refuses a suspend that would interrupt the firmware's boot. The J71t firmware recipe gains the AOP pieces, the gyroscope and pressure drivers are enabled, and J71t's accelerometer and gyroscope move from missing to working, and its light sensor from untested to working. Assisted-by: LLM --- ...9-AOP-s-boot-arguments-and-sensor-ca.patch | 116 + devices/j71t/device.toml | 6 +- devices/j71t/firmware.json | 44 + docs/compatibility.md | 3 +- kernel/configs/apple-16k/config | 2 + kernel/configs/apple-4k/config | 5 +- ...select-certified-AOP-reload-through-.patch | 126 + ...lect-certified-AOP-reload-through-DT.patch | 26 + ...lect-certified-AOP-reload-through-DT.patch | 771 ++++ ...box-select-certified-AOP-reload-thro.patch | 30 + ...elect-certified-AOP-reload-through-D.patch | 127 + ...apple-select-certified-AOP-reload-th.patch | 116 + ...lect-certified-AOP-reload-through-DT.patch | 3181 +++++++++++++++++ ...fuse-pre-sync-sleep-during-firmware-.patch | 80 + tests/test_j71t_data.py | 31 + 15 files changed, 4659 insertions(+), 5 deletions(-) create mode 100644 boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch create mode 100644 kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch create mode 100644 kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch create mode 100644 kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch create mode 100644 kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch create mode 100644 kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch create mode 100644 kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch create mode 100644 kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch create mode 100644 kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch create mode 100644 tests/test_j71t_data.py diff --git a/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch new file mode 100644 index 00000000..6a30fc37 --- /dev/null +++ b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 2 Nov 2026 15:07:08 +0100 +Subject: [PATCH] kboot: Pass the A9 AOP's boot arguments and sensor + calibration + +The A9's AOP firmware takes more than the ECID from its loader: the AOP +target number (iop-aop-nub aop-target), the chip's ID (/chosen chip-id) +and revision (/arm-io chip-revision). Its accelerometer, gyroscope and +pressure sensor also send no reports until the driver has given them the +unit's factory calibration, which iBoot resolves from syscfg into the +sensors' nodes below iop-aop-nub. + +Copy the three numbers into the AOP's node (alias aop) as +apple,aop-target, apple,chip-id and apple,chip-revision, and each +sensor's calibration properties into the AOP's child node of the same +name, prefixed with "apple,". Unprovisioned "syscfg/" references and +nodes the device tree doesn't have are skipped. The orientation +properties are not copied: the A9 firmware rejects them and the board's +mount-matrix describes the mounting. + +The values are the unit's own and never leave the device: the kernel +sends them to the AOP and nothing else. + +Assisted-by: LLM +--- + src/kboot.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 68 insertions(+), 1 deletion(-) + +diff --git a/src/kboot.c b/src/kboot.c +index ebd6282..7ce9d72 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -1005,6 +1005,73 @@ static int dt_set_als(void) + return 0; + } + ++/* ++ * The A9's AOP firmware also takes the AOP target number and the chip's ID and ++ * revision in its boot arguments, and each motion sensor's factory ++ * calibration, which iBoot resolves from syscfg into the sensor's node below ++ * iop-aop-nub. Copy them into the AOP's node and its sensor child nodes of the ++ * same name; the calibration keeps its name with an "apple," prefix. ++ */ ++static const char *const aop_sensor_calibrations[] = { ++ "low-temp-accel-offset", ++ "accel-sensitivity-calibration", ++ "accel-range-sensitivity-cal", ++ "accel-interrupt-calibration", ++ "gyro-sensitivity-calibration", ++ "gyro-range-sensitivity-cal", ++ "gyro-interrupt-calibration", ++ "gyro-trim-calibration", ++ "pressure-offset-calibration", ++ "pressure-accel-sensitivity", ++ "pressure-heater-resistance", ++}; ++ ++static int dt_set_aop_sensors(int node) ++{ ++ int nub = adt_path_offset(adt, "/arm-io/aop/iop-aop-nub"); ++ if (nub < 0) ++ return 0; ++ ++ u32 val; ++ if (ADT_GETPROP(adt, nub, "aop-target", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,aop-target", val)) ++ bail("FDT: failed to set apple,aop-target\n"); ++ ++ int chosen = adt_path_offset(adt, "/chosen"); ++ if (chosen >= 0 && ADT_GETPROP(adt, chosen, "chip-id", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-id", val)) ++ bail("FDT: failed to set apple,chip-id\n"); ++ ++ int arm_io = adt_path_offset(adt, "/arm-io"); ++ if (arm_io >= 0 && ADT_GETPROP(adt, arm_io, "chip-revision", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-revision", val)) ++ bail("FDT: failed to set apple,chip-revision\n"); ++ ++ int sensor = nub; ++ ADT_FOREACH_CHILD(adt, sensor) ++ { ++ const char *name = adt_get_name(adt, sensor); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(aop_sensor_calibrations); i++) { ++ u32 len; ++ const u8 *blob = adt_getprop(adt, sensor, aop_sensor_calibrations[i], &len); ++ if (!blob || !len || (len >= 7 && !memcmp(blob, "syscfg/", 7))) ++ continue; ++ ++ /* Earlier properties may have moved the sensor's node. */ ++ int child = fdt_subnode_offset(dt, node, name); ++ if (child < 0) ++ break; ++ ++ char prop[64]; ++ snprintf(prop, sizeof(prop), "apple,%s", aop_sensor_calibrations[i]); ++ if (fdt_setprop(dt, child, prop, blob, len)) ++ bail("FDT: failed to set %s/%s\n", name, prop); ++ } ++ } ++ return 0; ++} ++ + /* + * The AOP's firmware carries the chip's ECID in its boot arguments. Pass it + * only to the AOP's node; the driver removes it from the tree once read. +@@ -1028,7 +1095,7 @@ static int dt_set_aop(void) + + if (fdt_setprop_u64(dt, node, "apple,ecid", ecid)) + bail("FDT: failed to set apple,ecid\n"); +- return 0; ++ return dt_set_aop_sensors(node); + } + + static int dt_set_multitouch(void) diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..1e16858f 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -34,9 +34,9 @@ screen_off = "untested" backlight = "untested" touch = "untested" buttons = "untested" -accelerometer = "missing" -gyroscope = "missing" -light_sensor = "untested" +accelerometer = "working" +gyroscope = "working" +light_sensor = "working" touch_id = "missing" storage_read = "untested" storage_write = "untested" diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index 3321e246..39e7e406 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -113,6 +113,12 @@ }, "device": "/usr/share/firmware/multitouch/J71t.mtprops", "sha256": "69f27fe207cc5810a748dac3dfe640b1c5a755709f49513b4b5125660784f0fc" + }, + "aop.im4p": { + "ipsw": { + "zip": "Firmware/AOP/aopfw-ipad6faop.im4p" + }, + "sha256": "5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa" } }, "outputs": [ @@ -279,6 +285,44 @@ "calibration_size": 1024 } ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f.bin", + "sha256": "af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP" + } + ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f-ext.bin", + "sha256": "2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP_ext" + } + ] + }, + { + "destination": "lib/firmware/apple/agx-g5p-c0.ldr", + "sha256": "33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca", + "steps": [ + { + "step": "agx-g5p-ldr-from-kernelcache@1", + "input": "kernelcache" + } + ] } ] } diff --git a/docs/compatibility.md b/docs/compatibility.md index 25ce8915..54c462e4 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -161,7 +161,7 @@ Boards Orchard builds for. States: working, untested, missing, none (no such har | `n66` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n69u` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j71t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | +| `j71t` | untested | untested | untested | untested | untested | working | working | working | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | | `j72t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | | `n66m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | | `n69` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | @@ -248,6 +248,7 @@ A yes means an enabled device tree node binds a driver in `7.3.0-rc1-orchard-app | Board | Feature | | | --- | --- | --- | +| `j71t` | Accelerometer | working, but no driver binds | | `d10` | Bluetooth input devices | missing, but a driver binds | | `d10` | Bluetooth audio | missing, but a driver binds | | `d101` | Bluetooth input devices | missing, but a driver binds | diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index d3236c57..4949d63c 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -5624,6 +5624,7 @@ CONFIG_APPLE_AOP_ACCEL=m # CONFIG_ADIS16260 is not set # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set +CONFIG_APPLE_AOP_GYRO=m # CONFIG_BMG160 is not set # CONFIG_FXAS21002C is not set # CONFIG_MPU3050_I2C is not set @@ -5859,6 +5860,7 @@ CONFIG_TSL2583=m # CONFIG_HP206C is not set # CONFIG_ZPA2326 is not set # CONFIG_ADP810 is not set +CONFIG_APPLE_AOP_PRESSURE=m # end of Pressure sensors # diff --git a/kernel/configs/apple-4k/config b/kernel/configs/apple-4k/config index 106c382a..ce7fad3f 100644 --- a/kernel/configs/apple-4k/config +++ b/kernel/configs/apple-4k/config @@ -5192,7 +5192,7 @@ CONFIG_APPLE_DART=y CONFIG_APPLE_MAILBOX=y # CONFIG_APPLE_PMGR_MISC is not set CONFIG_APPLE_RTKIT=y -# CONFIG_APPLE_AOP_T8010 is not set +CONFIG_APPLE_AOP_T8010=m CONFIG_APPLE_PMP_T8015_V1=y CONFIG_APPLE_SIO=m CONFIG_APPLE_T8010_WLAN=y @@ -5312,6 +5312,7 @@ CONFIG_IIO_CONSUMERS_PER_TRIGGER=2 # CONFIG_ADXL372_I2C is not set # CONFIG_ADXL380_SPI is not set # CONFIG_ADXL380_I2C is not set +CONFIG_APPLE_AOP_ACCEL=m # CONFIG_BMA180 is not set # CONFIG_BMA220 is not set # CONFIG_BMA400 is not set @@ -5625,6 +5626,7 @@ CONFIG_IIO_CONSUMERS_PER_TRIGGER=2 # CONFIG_ADIS16260 is not set # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set +CONFIG_APPLE_AOP_GYRO=m # CONFIG_BMG160 is not set # CONFIG_FXAS21002C is not set # CONFIG_MPU3050_I2C is not set @@ -5860,6 +5862,7 @@ CONFIG_TSL2583=m # CONFIG_HP206C is not set # CONFIG_ZPA2326 is not set # CONFIG_ADP810 is not set +CONFIG_APPLE_AOP_PRESSURE=m # end of Pressure sensors # diff --git a/kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch b/kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch new file mode 100644 index 00000000..02ad3882 --- /dev/null +++ b/kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch @@ -0,0 +1,126 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + .../arm64/boot/dts/apple/s800-0-3-common.dtsi | 1 + + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 37 ++++++++++++++++++ + arch/arm64/boot/dts/apple/s8003-j71t.dts | 39 +++++++++++++++++++ + 3 files changed, 77 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +index 354533f26..338182cf5 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +@@ -11,6 +11,7 @@ + + / { + aliases { ++ aop = &aop; /* Used by loader */ + ausbphy0 = &ausbphy0; + nvme-hmb = &nvme_hmb; /* Used by loader */ + serial0 = &serial0; +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 3277a73bb..75ec4a403 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -415,6 +415,43 @@ + interrupt-parent = <&aic>; + interrupts = ; + }; ++ ++ aop: aop@210800000 { ++ compatible = "apple,s8000-aop"; ++ reg = <0x2 0x10800000 0x0 0x4000>, ++ <0x2 0x10000600 0x0 0x4>, ++ <0x2 0x10e00000 0x0 0x9b000>; ++ reg-names = "akf", "control", "sram"; ++ mboxes = <&aop_mbox>; ++ mbox-names = "mbox"; ++ status = "disabled"; ++ ++ aop_accel: accel { ++ compatible = "apple,s8000-aop-accel"; ++ }; ++ ++ aop_gyro: gyro { ++ compatible = "apple,s8000-aop-gyro"; ++ }; ++ ++ aop_pressure: pressure { ++ compatible = "apple,s8000-aop-pressure"; ++ }; ++ }; ++ ++ aop_mbox: mbox@210804000 { ++ compatible = "apple,s8000-a7iop-mailbox", "apple,t8010-a7iop-mailbox"; ++ reg = <0x2 0x10804000 0x0 0x1000>; ++ interrupt-parent = <&aic>; ++ interrupts = , ++ , ++ , ++ ; ++ interrupt-names = "send-empty", "send-not-empty", ++ "recv-empty", "recv-not-empty"; ++ #mbox-cells = <0>; ++ status = "disabled"; ++ }; + }; + + timer { +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..2728cbd5f 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -16,6 +16,45 @@ + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; + }; + ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* Same tail page as release6, never a firmware DMA grant. */ ++ aop_shutdown: aop-shutdown@8583fc000 { ++ reg = <0x8 0x583fc000 0 0x4000>; ++ no-map; ++ }; ++ }; ++}; ++ ++&aop { ++ apple,certified-reload; ++ memory-region = <&aop_shutdown>; ++ apple,observe-domains = <&ps_aop_gpio &ps_aop &ps_aop_busif ++ &ps_aop_cpu &ps_aop_filter>; ++ apple,spi0-domain = <&ps_spi0>; ++ firmware-name = "apple/aop-ipad6f.bin", "apple/aop-ipad6f-ext.bin"; ++ status = "okay"; ++}; ++ ++/* ++ * The AOP reports the accelerometer in its sensor frame, which is the ++ * device frame (x right, y up, z out of the screen) negated: flat, screen up ++ * reads z = -1 g, upright y = -1 g. iio-sensor-proxy takes upright as ++ * negative y and needs z flipped. ++ */ ++&aop_accel { ++ mount-matrix = "1", "0", "0", ++ "0", "1", "0", ++ "0", "0", "-1"; ++}; ++ ++&aop_mbox { ++ status = "okay"; ++}; ++ + &touchscreen { + firmware-name = "apple/j71t-fwstart.z2fw"; + }; diff --git a/kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..6fd6514b --- /dev/null +++ b/kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index 3fcdf4b4e..fcc51b699 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4589,6 +4589,7 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) + { + return __pr_flush(NULL, timeout_ms, reset_on_progress); + } ++EXPORT_SYMBOL_GPL(pr_flush); + + /* + * Delayed printk version, for scheduler-internal messages: diff --git a/kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..0bf54215 --- /dev/null +++ b/kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,771 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] iio: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/iio/accel/Kconfig | 2 +- + drivers/iio/accel/apple_aop_accel.c | 50 +++- + drivers/iio/gyro/Kconfig | 12 + + drivers/iio/gyro/Makefile | 1 + + drivers/iio/gyro/apple_aop_gyro.c | 291 ++++++++++++++++++++++ + drivers/iio/pressure/Kconfig | 12 + + drivers/iio/pressure/Makefile | 1 + + drivers/iio/pressure/apple_aop_pressure.c | 257 +++++++++++++++++++ + 8 files changed, 621 insertions(+), 5 deletions(-) + create mode 100644 drivers/iio/gyro/apple_aop_gyro.c + create mode 100644 drivers/iio/pressure/apple_aop_pressure.c + +diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig +index b01598194..ed328a590 100644 +--- a/drivers/iio/accel/Kconfig ++++ b/drivers/iio/accel/Kconfig +@@ -213,7 +213,7 @@ config APPLE_AOP_ACCEL + select IIO_KFIFO_BUF + help + Say yes here to get support for the accelerometer that the Always-On +- Processor (AOP) of Apple A10 iPads and iPhones serves. ++ Processor (AOP) of Apple A9 and A10 iPads and iPhones serves. + + To compile this driver as a module, choose M here: the module will be + called apple_aop_accel. +diff --git a/drivers/iio/accel/apple_aop_accel.c b/drivers/iio/accel/apple_aop_accel.c +index 96c901361..e1b0e136d 100644 +--- a/drivers/iio/accel/apple_aop_accel.c ++++ b/drivers/iio/accel/apple_aop_accel.c +@@ -9,9 +9,13 @@ + * + * 0 u32 sequence + * 4 u16 reserved +- * 6 s32 x, y, z in 1/65536 g, in the device frame (x right, y up, +- * z out of the screen; +1 g opposite to gravity) ++ * 6 s32 x, y, z in 1/65536 g + * 18 u32 timestamp ++ * ++ * The T8010's firmware reports in the device frame (x right, y up, z out of ++ * the screen; +1 g opposite to gravity). The S8000's firmware first needs the ++ * unit's factory calibration and reports in the sensor's frame; the board's ++ * mount-matrix maps it. + */ + + #include +@@ -21,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -41,8 +46,16 @@ + /* 9.80665 m/s^2 / 65536 */ + #define AOP_ACCEL_SCALE_NANO 149637 + ++struct apple_aop_accel_hw { ++ const struct apple_aop_calibration *calibration; ++ unsigned int num_calibration; ++ /* No separate raw sample stream (raw interval 0) */ ++ bool no_raw_stream; ++}; ++ + struct apple_aop_accel { + struct device *dev; ++ const struct apple_aop_accel_hw *hw; + const struct apple_aop_service *service; + struct iio_mount_matrix orientation; + +@@ -75,7 +88,9 @@ static int apple_aop_accel_set_interval(struct apple_aop_accel *accel, u32 inter + int i, ret; + + for (i = 0; i < ARRAY_SIZE(tags); i++) { +- ret = apple_aop_accel_set_property(accel, tags[i], interval_us); ++ ret = apple_aop_accel_set_property(accel, tags[i], ++ tags[i] == AOP_PROP_RAW_INTERVAL && ++ accel->hw->no_raw_stream ? 0 : interval_us); + if (ret) + return ret; + } +@@ -230,6 +245,7 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return -ENOMEM; + accel = iio_priv(indio_dev); + accel->dev = dev; ++ accel->hw = device_get_match_data(dev); + accel->service = service; + mutex_init(&accel->lock); + spin_lock_init(&accel->sample_lock); +@@ -252,6 +268,15 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (accel->hw->num_calibration) { ++ ret = apple_aop_calibrate(service, dev, accel->hw->calibration, ++ accel->hw->num_calibration); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to calibrate\n"); ++ } ++ } ++ + ret = apple_aop_accel_set_interval(accel, AOP_ACCEL_DEFAULT_US); + if (ret) { + apple_aop_unlisten(service); +@@ -265,8 +290,25 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return devm_iio_device_register(dev, indio_dev); + } + ++/* Offset at low temperature, sensitivity (either), interrupt calibration */ ++static const struct apple_aop_calibration apple_aop_accel_s8000_calibration[] = { ++ { "apple,low-temp-accel-offset", 0x1a, 1 }, ++ { "apple,accel-sensitivity-calibration", 0x0b, 2 }, ++ { "apple,accel-range-sensitivity-cal", 0x27, 2 }, ++ { "apple,accel-interrupt-calibration", 0x19, 3 }, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_s8000 = { ++ .calibration = apple_aop_accel_s8000_calibration, ++ .num_calibration = ARRAY_SIZE(apple_aop_accel_s8000_calibration), ++ .no_raw_stream = true, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_t8010 = { }; ++ + static const struct of_device_id apple_aop_accel_of_match[] = { +- { .compatible = "apple,t8010-aop-accel" }, ++ { .compatible = "apple,s8000-aop-accel", .data = &apple_aop_accel_s8000 }, ++ { .compatible = "apple,t8010-aop-accel", .data = &apple_aop_accel_t8010 }, + { } + }; + MODULE_DEVICE_TABLE(of, apple_aop_accel_of_match); +diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig +index 3e193ee0f..8f506a65d 100644 +--- a/drivers/iio/gyro/Kconfig ++++ b/drivers/iio/gyro/Kconfig +@@ -63,6 +63,18 @@ config ADXRS450 + This driver can also be built as a module. If so, the module + will be called adxrs450. + ++config APPLE_AOP_GYRO ++ tristate "Apple AOP gyroscope" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the gyroscope that the Always-On ++ Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_gyro. ++ + config BMG160 + tristate "BOSCH BMG160 Gyro Sensor" + depends on (I2C || SPI_MASTER) +diff --git a/drivers/iio/gyro/Makefile b/drivers/iio/gyro/Makefile +index 0319b397d..212bbb814 100644 +--- a/drivers/iio/gyro/Makefile ++++ b/drivers/iio/gyro/Makefile +@@ -10,6 +10,7 @@ obj-$(CONFIG_ADIS16136) += adis16136.o + obj-$(CONFIG_ADIS16260) += adis16260.o + obj-$(CONFIG_ADXRS290) += adxrs290.o + obj-$(CONFIG_ADXRS450) += adxrs450.o ++obj-$(CONFIG_APPLE_AOP_GYRO) += apple_aop_gyro.o + obj-$(CONFIG_BMG160) += bmg160_core.o + obj-$(CONFIG_BMG160_I2C) += bmg160_i2c.o + obj-$(CONFIG_BMG160_SPI) += bmg160_spi.o +diff --git a/drivers/iio/gyro/apple_aop_gyro.c b/drivers/iio/gyro/apple_aop_gyro.c +new file mode 100644 +index 000000000..edbda1f43 +--- /dev/null ++++ b/drivers/iio/gyro/apple_aop_gyro.c +@@ -0,0 +1,291 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Gyroscope served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the gyroscope itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one motion report (EPIC ++ * subtype 0xc1) per report interval, laid out like the accelerometer's: ++ * ++ * 0 u16 sequence ++ * 2 u32 status ++ * 6 s32 x, y, z in 1/65536 degree per second ++ * 18 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_GYRO 0xc1 ++#define AOP_GYRO_LEN 22 ++#define AOP_GYRO_XYZ 6 ++ ++#define AOP_GYRO_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_GYRO_MIN_US 10000 /* 100 Hz */ ++#define AOP_GYRO_MAX_US 1000000 /* 1 Hz */ ++ ++/* pi / 180 rad/s / 65536 = 1171270 / 2^42 rad/s */ ++#define AOP_GYRO_SCALE 1171270 ++#define AOP_GYRO_SCALE_LOG2 42 ++ ++struct apple_aop_gyro { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ struct iio_mount_matrix orientation; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ s32 sample[3]; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(s32, scan, 3); ++}; ++ ++static int apple_aop_gyro_set_u32(struct apple_aop_gyro *gyro, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(gyro->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_gyro_set_interval(struct apple_aop_gyro *gyro, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ gyro->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_gyro_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int i; ++ ++ if (subtype != AOP_REPORT_GYRO || len < AOP_GYRO_LEN) ++ return; ++ ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ for (i = 0; i < 3; i++) ++ gyro->sample[i] = get_unaligned_le32(data + AOP_GYRO_XYZ + 4 * i); ++ gyro->have_sample = true; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ memcpy(gyro->scan, gyro->sample, sizeof(gyro->sample)); ++ iio_push_to_buffers_with_ts(indio_dev, gyro->scan, sizeof(gyro->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_gyro_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ *val = gyro->sample[chan->scan_index]; ++ ret = gyro->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ *val = AOP_GYRO_SCALE; ++ *val2 = AOP_GYRO_SCALE_LOG2; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&gyro->lock); ++ *val = USEC_PER_SEC / gyro->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % gyro->interval_us) * USEC_PER_SEC / ++ gyro->interval_us; ++ mutex_unlock(&gyro->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_gyro_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_GYRO_MIN_US, AOP_GYRO_MAX_US); ++ ++ mutex_lock(&gyro->lock); ++ ret = apple_aop_gyro_set_interval(gyro, interval); ++ mutex_unlock(&gyro->lock); ++ return ret; ++} ++ ++static const struct iio_mount_matrix * ++apple_aop_gyro_get_mount_matrix(const struct iio_dev *indio_dev, ++ const struct iio_chan_spec *chan) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ ++ return &gyro->orientation; ++} ++ ++static const struct iio_chan_spec_ext_info apple_aop_gyro_ext_info[] = { ++ IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, apple_aop_gyro_get_mount_matrix), ++ { } ++}; ++ ++#define APPLE_AOP_GYRO_CHANNEL(axis, index) { \ ++ .type = IIO_ANGL_VEL, \ ++ .modified = 1, \ ++ .channel2 = IIO_MOD_##axis, \ ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ ++ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \ ++ .scan_index = index, \ ++ .scan_type = { \ ++ .sign = 's', \ ++ .realbits = 32, \ ++ .storagebits = 32, \ ++ .endianness = IIO_CPU, \ ++ }, \ ++ .ext_info = apple_aop_gyro_ext_info, \ ++} ++ ++static const struct iio_chan_spec apple_aop_gyro_channels[] = { ++ APPLE_AOP_GYRO_CHANNEL(X, 0), ++ APPLE_AOP_GYRO_CHANNEL(Y, 1), ++ APPLE_AOP_GYRO_CHANNEL(Z, 2), ++ IIO_CHAN_SOFT_TIMESTAMP(3), ++}; ++ ++static const struct iio_info apple_aop_gyro_info = { ++ .read_raw = apple_aop_gyro_read_raw, ++ .write_raw = apple_aop_gyro_write_raw, ++}; ++ ++/* Sensitivity and interrupt calibration make it complete */ ++static const struct apple_aop_calibration apple_aop_gyro_calibration[] = { ++ { "apple,gyro-sensitivity-calibration", 0x0b, 1 }, ++ { "apple,gyro-interrupt-calibration", 0x19, 2 }, ++ { "apple,gyro-range-sensitivity-cal", 0x27, 0 }, ++ { "apple,gyro-trim-calibration", 0x20, 0 }, ++}; ++ ++static void apple_aop_gyro_stop(void *data) ++{ ++ struct apple_aop_gyro *gyro = data; ++ ++ apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(gyro->service); ++} ++ ++static int apple_aop_gyro_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_gyro *gyro; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*gyro)); ++ if (!indio_dev) ++ return -ENOMEM; ++ gyro = iio_priv(indio_dev); ++ gyro->dev = dev; ++ gyro->service = service; ++ mutex_init(&gyro->lock); ++ spin_lock_init(&gyro->sample_lock); ++ ++ ret = iio_read_mount_matrix(dev, &gyro->orientation); ++ if (ret) ++ return ret; ++ ++ indio_dev->name = "apple-aop-gyro"; ++ indio_dev->info = &apple_aop_gyro_info; ++ indio_dev->channels = apple_aop_gyro_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_gyro_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_gyro_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_gyro_calibration, ++ ARRAY_SIZE(apple_aop_gyro_calibration)); ++ if (!ret) ++ ret = apple_aop_gyro_set_interval(gyro, AOP_GYRO_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_gyro_stop, gyro); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_gyro_of_match[] = { ++ { .compatible = "apple,s8000-aop-gyro" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_gyro_of_match); ++ ++static struct platform_driver apple_aop_gyro_driver = { ++ .driver = { ++ .name = "apple-aop-gyro", ++ .of_match_table = apple_aop_gyro_of_match, ++ }, ++ .probe = apple_aop_gyro_probe, ++}; ++module_platform_driver(apple_aop_gyro_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP gyroscope driver"); ++MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig +index 838a8340c..18535ad99 100644 +--- a/drivers/iio/pressure/Kconfig ++++ b/drivers/iio/pressure/Kconfig +@@ -384,4 +384,16 @@ config ADP810 + To compile this driver as a module, choose M here: the module will + be called adp810 + ++config APPLE_AOP_PRESSURE ++ tristate "Apple AOP barometric pressure sensor" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the barometric pressure sensor that ++ the Always-On Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_pressure. ++ + endmenu +diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile +index bc0d11a20..bf3e27ba9 100644 +--- a/drivers/iio/pressure/Makefile ++++ b/drivers/iio/pressure/Makefile +@@ -9,6 +9,7 @@ obj-$(CONFIG_ABP2030PA) += abp2030pa.o + obj-$(CONFIG_ABP2030PA_I2C) += abp2030pa_i2c.o + obj-$(CONFIG_ABP2030PA_SPI) += abp2030pa_spi.o + obj-$(CONFIG_ADP810) += adp810.o ++obj-$(CONFIG_APPLE_AOP_PRESSURE) += apple_aop_pressure.o + obj-$(CONFIG_ROHM_BM1390) += rohm-bm1390.o + obj-$(CONFIG_BMP280) += bmp280.o + bmp280-objs := bmp280-core.o bmp280-regmap.o +diff --git a/drivers/iio/pressure/apple_aop_pressure.c b/drivers/iio/pressure/apple_aop_pressure.c +new file mode 100644 +index 000000000..8cd983c57 +--- /dev/null ++++ b/drivers/iio/pressure/apple_aop_pressure.c +@@ -0,0 +1,257 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Barometric pressure sensor served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the pressure sensor itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one report (EPIC subtype ++ * 0xc4) per report interval: ++ * ++ * 0 u16 sequence ++ * 2 u32 pressure in 1/65536 kPa ++ * 6 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_PRESSURE 0xc4 ++#define AOP_PRESSURE_LEN 10 ++#define AOP_PRESSURE_VALUE 2 ++ ++#define AOP_PRESSURE_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_PRESSURE_MIN_US 10000 /* 100 Hz */ ++#define AOP_PRESSURE_MAX_US 1000000 /* 1 Hz */ ++ ++struct apple_aop_pressure { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ u32 sample; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(u32, scan, 1); ++}; ++ ++static int apple_aop_pressure_set_u32(struct apple_aop_pressure *pressure, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(pressure->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_pressure_set_interval(struct apple_aop_pressure *pressure, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ pressure->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_pressure_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ ++ if (subtype != AOP_REPORT_PRESSURE || len < AOP_PRESSURE_LEN) ++ return; ++ ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ pressure->sample = get_unaligned_le32(data + AOP_PRESSURE_VALUE); ++ pressure->have_sample = true; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ pressure->scan[0] = pressure->sample; ++ iio_push_to_buffers_with_ts(indio_dev, pressure->scan, sizeof(pressure->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_pressure_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ *val = pressure->sample; ++ ret = pressure->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ /* kPa */ ++ *val = 1; ++ *val2 = 16; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&pressure->lock); ++ *val = USEC_PER_SEC / pressure->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % pressure->interval_us) * USEC_PER_SEC / ++ pressure->interval_us; ++ mutex_unlock(&pressure->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_pressure_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_PRESSURE_MIN_US, AOP_PRESSURE_MAX_US); ++ ++ mutex_lock(&pressure->lock); ++ ret = apple_aop_pressure_set_interval(pressure, interval); ++ mutex_unlock(&pressure->lock); ++ return ret; ++} ++ ++static const struct iio_chan_spec apple_aop_pressure_channels[] = { ++ { ++ .type = IIO_PRESSURE, ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), ++ .scan_index = 0, ++ .scan_type = { ++ .sign = 'u', ++ .realbits = 32, ++ .storagebits = 32, ++ .endianness = IIO_CPU, ++ }, ++ }, ++ IIO_CHAN_SOFT_TIMESTAMP(1), ++}; ++ ++static const struct iio_info apple_aop_pressure_info = { ++ .read_raw = apple_aop_pressure_read_raw, ++ .write_raw = apple_aop_pressure_write_raw, ++}; ++ ++/* The offset or the acceleration sensitivity makes it complete */ ++static const struct apple_aop_calibration apple_aop_pressure_calibration[] = { ++ { "apple,pressure-offset-calibration", 0x0b, 1 }, ++ { "apple,pressure-accel-sensitivity", 0xd3, 1 }, ++ { "apple,pressure-heater-resistance", 0xd4, 0 }, ++}; ++ ++static void apple_aop_pressure_stop(void *data) ++{ ++ struct apple_aop_pressure *pressure = data; ++ ++ apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(pressure->service); ++} ++ ++static int apple_aop_pressure_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_pressure *pressure; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*pressure)); ++ if (!indio_dev) ++ return -ENOMEM; ++ pressure = iio_priv(indio_dev); ++ pressure->dev = dev; ++ pressure->service = service; ++ mutex_init(&pressure->lock); ++ spin_lock_init(&pressure->sample_lock); ++ ++ indio_dev->name = "apple-aop-pressure"; ++ indio_dev->info = &apple_aop_pressure_info; ++ indio_dev->channels = apple_aop_pressure_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_pressure_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_pressure_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_pressure_calibration, ++ ARRAY_SIZE(apple_aop_pressure_calibration)); ++ if (!ret) ++ ret = apple_aop_pressure_set_interval(pressure, AOP_PRESSURE_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_pressure_stop, pressure); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_pressure_of_match[] = { ++ { .compatible = "apple,s8000-aop-pressure" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_pressure_of_match); ++ ++static struct platform_driver apple_aop_pressure_driver = { ++ .driver = { ++ .name = "apple-aop-pressure", ++ .of_match_table = apple_aop_pressure_of_match, ++ }, ++ .probe = apple_aop_pressure_probe, ++}; ++module_platform_driver(apple_aop_pressure_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP barometric pressure sensor driver"); ++MODULE_LICENSE("Dual MIT/GPL"); diff --git a/kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch b/kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch new file mode 100644 index 00000000..f98a0dbf --- /dev/null +++ b/kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: mailbox: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. +Describe the compatible and retained-reload property. + +Assisted-by: LLM +--- + .../devicetree/bindings/mailbox/apple,mailbox.yaml | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +index ba035ff80..8821623d3 100644 +--- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml ++++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +@@ -22,6 +22,12 @@ description: + properties: + compatible: + oneOf: ++ - description: ++ The same A7 IOP mailbox on the A9. ++ items: ++ - const: apple,s8000-a7iop-mailbox ++ - const: apple,t8010-a7iop-mailbox ++ + - description: + ASC mailboxes are the most common variant found on the M1 used + for example for the display controller, the system management diff --git a/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch b/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch new file mode 100644 index 00000000..d9a8480b --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch @@ -0,0 +1,127 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 84 ++++++++++++++++++++++++++ + 1 file changed, 84 insertions(+) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..0ae0eda2d 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -417,6 +419,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + goto err_remove; + } + ++ platform_set_drvdata(pdev, ps); + return 0; + err_remove: + if (ps->has_genpd) { +@@ -446,3 +449,84 @@ MODULE_AUTHOR("Hector Martin "); + MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); + + module_platform_driver(apple_pmgr_ps_driver); ++ ++/* ++ * Read-only observer for bring-up. The caller's nodes follow the same ++ * child -> parent order as genpd power transitions. No domain is attached, ++ * resumed, pinned on, reset, or written by this function. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data) ++{ ++ struct platform_device *providers[5]; ++ struct apple_pmgr_ps *ps[5]; ++ unsigned long flags; ++ unsigned int i, j, found = 0; ++ int ret = 0; ++ bool accessible = true; ++ ++ if (!count || count > ARRAY_SIZE(providers) || !read) ++ return -EINVAL; ++ for (i = 0; i < count; i++) { ++ for (j = 0; j < i; j++) ++ if (nodes[i] == nodes[j]) { ++ ret = -EINVAL; ++ goto put; ++ } ++ providers[i] = of_find_device_by_node(nodes[i]); ++ if (!providers[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ device_lock(&providers[i]->dev); ++ found++; ++ if (!providers[i]->dev.driver || ++ strcmp(providers[i]->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ ps[i] = platform_get_drvdata(providers[i]); ++ if (!ps[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ } ++ ++ /* PMGR domains use GENPD_FLAG_IRQ_SAFE, i.e. genpd.slock. */ ++ local_irq_save(flags); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->genpd.slock, i); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->lock, i); ++ for (i = 0; i < count; i++) { ++ states[i].genpd_status = ps[i]->genpd.status; ++ ret = regmap_read(ps[i]->regmap, ps[i]->offset, &states[i].raw); ++ if (ret) ++ goto unlock; ++ if (states[i].genpd_status != GENPD_STATE_ON || ++ FIELD_GET(APPLE_PMGR_PS_ACTUAL, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ FIELD_GET(APPLE_PMGR_PS_TARGET, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ states[i].raw & (APPLE_PMGR_RESET | APPLE_PMGR_DEV_DISABLE | ++ APPLE_PMGR_PARENT_OFF)) ++ accessible = false; ++ } ++ if (accessible) ++ read(data); ++ else ++ ret = -EHOSTDOWN; ++unlock: ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->lock); ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->genpd.slock); ++ local_irq_restore(flags); ++put: ++ while (found) { ++ found--; ++ device_unlock(&providers[found]->dev); ++ put_device(&providers[found]->dev); ++ } ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_observe); diff --git a/kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch b/kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch new file mode 100644 index 00000000..ad082c85 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: soc-apple: select certified AOP reload through + DT + +A retained AOP must not be reset as though it were a fresh boot. +Describe the compatible and retained-reload property. + +Assisted-by: LLM +--- + .../bindings/soc/apple/apple,t8010-aop.yaml | 58 +++++++++++++++++-- + 1 file changed, 54 insertions(+), 4 deletions(-) + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +index 14726afa0..df6621d74 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +@@ -4,22 +4,28 @@ + $id: http://devicetree.org/schemas/soc/apple/apple,t8010-aop.yaml# + $schema: http://devicetree.org/meta-schemas/core.yaml# + +-title: Apple T8010 Always-On Processor ++title: Apple S8000 and T8010 Always-On Processor + + maintainers: + - Dalton Alexandre + + description: +- The Always-On Processor (AOP) of the A10 is an ARMv7 coprocessor that +- runs RTKit firmware and serves the motion sensors (accelerometer, ++ The Always-On Processor (AOP) of the A9 and A10 is an ARMv7 coprocessor ++ that runs RTKit firmware and serves the motion sensors (accelerometer, + gyroscope, compass, pressure) and the always-on microphones. It runs from + local SRAM, with a further firmware segment in system memory, and talks + to the application processor through an A7 IOP mailbox. Its firmware is + board-specific and is loaded by the operating system. + + properties: ++ apple,certified-reload: ++ type: boolean ++ description: Firmware supports the retained shutdown certificate for warm reload. ++ + compatible: +- const: apple,t8010-aop ++ enum: ++ - apple,s8000-aop ++ - apple,t8010-aop + + reg: + items: +@@ -50,6 +56,38 @@ properties: + The chip's unique ID (ECID), filled in by the bootloader. The + firmware's boot arguments carry it. + ++ apple,aop-target: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The AOP target number from the Apple device tree, filled in by the ++ bootloader. The S8000 firmware's boot arguments carry it. ++ ++ apple,chip-id: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip ID (e.g. 0x8003), filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ apple,chip-revision: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip revision, filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ memory-region: ++ maxItems: 1 ++ description: Reserved 16 KiB shutdown transcript page for J71t warm reload. ++ ++ apple,observe-domains: ++ $ref: /schemas/types.yaml#/definitions/phandle-array ++ minItems: 5 ++ maxItems: 5 ++ description: AOP GPIO, AOP, bus interface, CPU and filter in lock order. ++ ++ apple,spi0-domain: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: Read-only SPI0 power-state provider observation. ++ + patternProperties: + "^(accel|gyro|compass|pressure)$": + type: object +@@ -58,6 +96,9 @@ patternProperties: + properties: + compatible: + enum: ++ - apple,s8000-aop-accel ++ - apple,s8000-aop-gyro ++ - apple,s8000-aop-pressure + - apple,t8010-aop-accel + - apple,t8010-aop-gyro + - apple,t8010-aop-compass +@@ -65,6 +106,15 @@ patternProperties: + + mount-matrix: + description: The sensor's mounting rotation in the device. ++ ++ patternProperties: ++ "^apple,(low-temp-accel-offset|(accel|gyro)-(sensitivity|interrupt)-calibration|(accel|gyro)-range-sensitivity-cal|gyro-trim-calibration|pressure-(offset-calibration|accel-sensitivity|heater-resistance))$": ++ $ref: /schemas/types.yaml#/definitions/uint8-array ++ maxItems: 64 ++ description: ++ The unit's factory calibration of the sensor, copied by the ++ bootloader from the property of the same name (without the ++ prefix) in the Apple device tree. The S8000 firmware takes it. + required: + - compatible + additionalProperties: false diff --git a/kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..279d4de7 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,3181 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] soc: apple: aop: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + MAINTAINERS | 2 + + drivers/soc/apple/Kconfig | 12 +- + drivers/soc/apple/aop-warm-record.h | 63 ++ + drivers/soc/apple/aop-warm-spi0.c.h | 1339 ++++++++++++++++++++++++ + drivers/soc/apple/aop-warm-spi0.h | 543 ++++++++++ + drivers/soc/apple/aop.c | 729 ++++++++++++- + drivers/soc/apple/rtkit.c | 50 + + include/linux/soc/apple/aop.h | 28 + + include/linux/soc/apple/pmgr-observe.h | 27 + + include/linux/soc/apple/rtkit.h | 2 + + 10 files changed, 2758 insertions(+), 37 deletions(-) + create mode 100644 drivers/soc/apple/aop-warm-record.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.c.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.h + create mode 100644 include/linux/soc/apple/pmgr-observe.h + +diff --git a/MAINTAINERS b/MAINTAINERS +index a4a69643c..12c292825 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -2068,6 +2068,8 @@ APPLE T8010 ALWAYS-ON PROCESSOR + M: Dalton Alexandre + S: Maintained + F: Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++F: drivers/iio/gyro/apple_aop_gyro.c ++F: drivers/iio/pressure/apple_aop_pressure.c + F: drivers/soc/apple/aop.c + F: include/linux/soc/apple/aop.h + +diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig +index e68a49f1b..c0af67b62 100644 +--- a/drivers/soc/apple/Kconfig ++++ b/drivers/soc/apple/Kconfig +@@ -39,16 +39,16 @@ config APPLE_RTKIT + Say 'y' here if you have an Apple SoC. + + config APPLE_AOP_T8010 +- tristate "Apple T8010 Always-On Processor" ++ tristate "Apple S8000 and T8010 Always-On Processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help +- The Always-On Processor (AOP) of the Apple A10 runs the firmware for +- the motion sensors and the always-on microphones. This driver loads +- and starts that firmware and provides its services to the sensor +- drivers. ++ The Always-On Processor (AOP) of the Apple A9 and A10 runs the ++ firmware for the motion sensors and the always-on microphones. This ++ driver loads and starts that firmware and provides its services to ++ the sensor drivers. + +- Say 'y' or 'm' here if you have an A10-based iPad or iPhone. ++ Say 'y' or 'm' here if you have an A9- or A10-based iPad or iPhone. + + config APPLE_PMP_T8015_V1 + tristate "Apple T8015 PMP" +diff --git a/drivers/soc/apple/aop-warm-record.h b/drivers/soc/apple/aop-warm-record.h +new file mode 100644 +index 000000000..237d78d17 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-record.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#include ++#include ++#define A9_HO_PAGE 0x4000U ++#define A9_HO_LOG_OFFSET 0x800U ++#define A9_SL_MAGIC 0x4c533941U /* "A9SL" */ ++#define A9_SL_VERSION 1U ++ ++enum a9_ho_step { ++ A9_HO_STEP_BEGIN, ++ A9_HO_STEP_RATES, ++ A9_HO_STEP_AFK_SENT, ++ A9_HO_STEP_AFK_ACKED, ++ A9_HO_STEP_DISABLED, ++ A9_HO_STEP_AP_SENT, ++ A9_HO_STEP_AP_ACKED, ++ A9_HO_STEP_IOP_SENT, ++ A9_HO_STEP_IOP_ACKED, ++ A9_HO_STEP_RAM_STABLE, ++ A9_HO_STEP_MAILBOX_IDLE, ++ A9_HO_STEP_HALTED, ++ A9_HO_STEP_DRAINED, ++ A9_HO_STEPS, ++}; ++ ++static const char * const a9_ho_step_names[A9_HO_STEPS] = { ++ "begin", "rates-zero", "afk-sent", "afk-acked", "endpoint-disable", ++ "ap-sent", "ap-ack", "iop-sent", "iop-ack", "ram-stable", "mailbox-idle", ++ "halt", "drain", ++}; ++ ++struct a9_sl_header { ++ u32 magic, version, capacity, length; ++ u32 lines, truncated, checksum, mode; ++ u64 begin_ns; ++ u8 writer[16]; ++ char entry[64]; ++}; ++ ++_Static_assert(sizeof(struct a9_sl_header) == 120, "shutdown log ABI"); ++ ++#define A9_SL_CAPACITY (A9_HO_PAGE - A9_HO_LOG_OFFSET - sizeof(struct a9_sl_header)) ++ ++/* FNV-1a; incremental over appended text. */ ++static inline u32 a9_sl_hash(u32 hash, const char *text, size_t length) ++{ ++ size_t i; ++ ++ for (i = 0; i < length; i++) ++ hash = (hash ^ (u8)text[i]) * 16777619U; ++ return hash; ++} ++ ++#define A9_SL_HASH_SEED 2166136261U ++ ++/* A replayable previous transcript: complete header, bounded, intact. */ ++static inline bool a9_sl_valid(const struct a9_sl_header *h, const char *text) ++{ ++ return h->magic == A9_SL_MAGIC && h->version == A9_SL_VERSION && ++ h->capacity == A9_SL_CAPACITY && h->length <= h->capacity && ++ !h->entry[sizeof(h->entry) - 1] && ++ a9_sl_hash(A9_SL_HASH_SEED, text, h->length) == h->checksum; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.c.h b/drivers/soc/apple/aop-warm-spi0.c.h +new file mode 100644 +index 000000000..3c421d207 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.c.h +@@ -0,0 +1,1339 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Ported from a9/aop-release: wake16, release6-d254a9f790. */ ++static void a9_aop_spi0_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_REGS; i++) ++ values[i] = readl(base + a9_spi0_offsets[i]); ++} ++ ++/* ++ * AOPSPIMaster_reset (0102f2f4) under the AOP-domain PMGR guard with IRQs ++ * off and the AOP CPU halted. Its provider power transition 3 is an ++ * AOP-local call with no AP register equivalent and is not replayed. A poll ++ * timeout still restores the saved control word. ++ */ ++static void a9_aop_spi0_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, s->before); ++ s->saved = s->before[A9_SPI0_I_CONTROL]; ++ s->asserted = a9_spi0_base_word(s->saved) | A9_SPI0_RESET; ++ writel(s->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ s->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(s->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(s->saved, control); ++ s->reset_done_ns = ktime_get_ns(); ++ a9_aop_spi0_snapshot(s->base, s->after); ++ if (!ret && (s->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ s->error = ret; ++} ++ ++/* ++ * Wake8 device evidence: SCM was 0 before/after the first controller reset, ++ * then became 3 at the channel-3 stop store. There is no firmware-derived ++ * SCM status write. Replay AOPSPIMaster_reset after the teardown so the ++ * controller reset, rather than an invented SCM write, is the one variable. ++ */ ++static void a9_aop_spi0_post_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, r->before); ++ r->saved = r->before[A9_SPI0_I_CONTROL]; ++ r->asserted = a9_spi0_base_word(r->saved) | A9_SPI0_RESET; ++ writel(r->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ r->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(r->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(r->saved, control); ++ a9_aop_spi0_snapshot(s->base, r->after); ++ if (!ret && (r->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ r->error = ret; ++} ++ ++static void a9_aop_spi0_ch3_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_CH3_REGS; i++) ++ values[i] = readl(base + a9_spi0_ch3_offsets[i]); ++} ++ ++static void a9_aop_spi0_trace_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int channel, i; ++ ++ s->trace.control = readl(s->base + A9_SPI0_CONTROL); ++ s->trace.clock = readl(s->base + A9_SPI0_CLOCK); ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->trace.scm[i] = readl(s->base + a9_spi0_scm_offsets[i]); ++ a9_aop_spi0_ch3_snapshot(s->base, s->trace.ch3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) ++ for (i = 0; i < A9_SPI0_CHANNEL_REGS; i++) ++ s->trace.channel[channel][i] = ++ readl(s->base + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel + ++ a9_spi0_channel_offsets[i]); ++} ++ ++/* Warm path only, CPU halted. Log outside the PMGR guard; no SCM writes. */ ++static int a9_aop_spi0_trace(struct apple_aop *aop, const char *stage) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *c = s->trace.ch3, *m = s->trace.scm; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_trace_read, s); ++ if (ret) { ++ s->intc.failed = "failed-spi0-trace-guard"; ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=%d: guard refused\n", stage, ret); ++ return ret; ++ } ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=0 ctl=%08x clk=%08x scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ stage, s->trace.control, s->trace.clock, m[0], m[1], m[2], m[3], ++ c[0], c[1], c[2], c[3], c[4], c[5], c[6]); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ v = s->trace.channel[channel]; ++ dev_info(aop->dev, "SPI0 SCM channel stage=%s ch=%02u ctl=%08x route=%08x rate=%08x cfg=%08x interval=%08x cursor=%08x desc=%08x pending0=%u pending1=%u\n", ++ stage, channel, v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ !!(c[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(c[A9_CH3_I_8D4] & BIT(2 * channel))); ++ } ++ return 0; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s); ++ ++/* ++ * Wake10 safety gate, strengthened in wake11 to retain every selected-channel ++ * teardown invariant. A fault observed by a copy sample remains terminal even ++ * if it is transient and the final read happens to clear. ++ */ ++static int a9_aop_spi0_gate(struct apple_aop *aop, const char *stage, ++ const char *check, bool include_copy) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *m, *c; ++ int ret; ++ ++ ret = a9_aop_spi0_trace(aop, stage); ++ if (ret) ++ return ret; ++ m = s->trace.scm; ++ c = s->trace.ch3; ++ if (m[0] || m[1]) ++ s->gate_failed = "failed-spi0-scm"; ++ else if (c[A9_CH3_I_8D0] || c[A9_CH3_I_8D4]) ++ s->gate_failed = "failed-spi0-fault"; ++ else if (include_copy && s->copy_faulted) ++ s->gate_failed = "failed-spi0-copy-sample"; ++ else { ++ s->gate_failed = a9_spi0_ch3_verify(c, s->teardown_stable & BIT(3)); ++ if (!s->gate_failed) ++ s->gate_failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = s->gate_failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 %s check errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x copy_faulted=%u result=%s\n", ++ check, ret, m[0], m[1], c[A9_CH3_I_8D0], c[A9_CH3_I_8D4], ++ s->copy_faulted, s->gate_failed ?: "pass"); ++ return ret; ++} ++ ++#define A9_SPI0_COPY_CHUNK 256U ++#define A9_SPI0_COPY_SAMPLE_US 5000U ++ ++struct a9_aop_spi0_copy_probe { ++ void __iomem *base; ++ u32 scm_lo, scm_hi, fault, fault_8d4; ++}; ++ ++static void a9_aop_spi0_copy_probe_read(void *data) ++{ ++ struct a9_aop_spi0_copy_probe *p = data; ++ ++ p->scm_lo = readl(p->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(p->base + A9_SPI0_SCM_HI); ++ p->fault = readl(p->base + A9_SPI0_FAULT); ++ p->fault_8d4 = readl(p->base + A9_SPI0_FAULT_8D4); ++} ++ ++/* ++ * Read-only samples spaced by at least 5 ms across both halves of the SRAM ++ * copy. Once a nonzero state is seen it remains a terminal fact, but the copy ++ * completes so later samples and the full after-copy channel snapshot expose ++ * whether the source is level, transient or moves between channels. ++ */ ++static int a9_aop_spi0_copy_sample(struct apple_aop *aop, const char *phase, ++ size_t offset, size_t size, u64 start_ns, ++ u64 *last_us, unsigned int *sample, ++ bool force) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_copy_probe p = { .base = s->base }; ++ u64 elapsed_us = div_u64(ktime_get_boottime_ns() - start_ns, ++ NSEC_PER_USEC); ++ bool first_fault, initial = !*sample; ++ int ret; ++ ++ if (!force && *sample && elapsed_us - *last_us < A9_SPI0_COPY_SAMPLE_US) ++ return 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_copy_probe_read, &p); ++ first_fault = !s->copy_faulted && ++ (p.scm_lo || p.scm_hi || p.fault || p.fault_8d4); ++ if (!ret && first_fault) ++ s->copy_faulted = true; ++ dev_info(aop->dev, "SPI0 COPY sample=%u phase=%s elapsed_us=%llu offset=0x%zx size=0x%zx scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ *sample, phase, elapsed_us, offset, size, p.scm_lo, p.scm_hi, ++ p.fault, p.fault_8d4, ret ? "guard" : s->copy_faulted ? "fault" : "pass"); ++ *last_us = elapsed_us; ++ (*sample)++; ++ if (ret) ++ return ret; ++ if (first_fault) { ++ ret = a9_aop_spi0_trace(aop, "copy-fault"); ++ if (ret) ++ return ret; ++ } ++ /* The offset-zero sample is the last no-write gate before the copy. */ ++ return !initial || !s->copy_faulted ? 0 : -EIO; ++} ++ ++static int a9_aop_copy_verify_wake10(struct apple_aop *aop, const u8 *src, ++ size_t size) ++{ ++ u32 chunk[A9_SPI0_COPY_CHUNK / 4]; ++ u64 start_ns = ktime_get_boottime_ns(), last_us = 0; ++ unsigned int sample = 0; ++ size_t done, n; ++ int ret; ++ ++ ret = a9_aop_spi0_copy_sample(aop, "write", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __iowrite32_copy(aop->sram + done, src + done, n / 4); ++ ret = a9_aop_spi0_copy_sample(aop, "write", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_copy_sample(aop, "verify", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __ioread32_copy(chunk, aop->sram + done, n / 4); ++ if (memcmp(chunk, src + done, n)) ++ return -EIO; ++ ret = a9_aop_spi0_copy_sample(aop, "verify", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void __iomem *a9_aop_spi0_channel_base(struct a9_aop_spi0 *s, ++ unsigned int channel) ++{ ++ return s->base + A9_SPI0_CHANNEL_BASE + A9_SPI0_CHANNEL_STRIDE * channel; ++} ++ ++/* ++ * Wake15: which other channels were RUN with a counting interval when this ++ * stop was issued, and which of them advanced across the stop. Read only; it ++ * separates a stop that had a ticking peer from the stop of the last live ++ * channel in the log. It never gates anything in-kernel. ++ */ ++static void a9_aop_spi0_peers_begin(struct a9_aop_spi0 *s, unsigned int channel, ++ u32 *interval, u32 *run) ++{ ++ unsigned int i; ++ u32 ctl; ++ ++ *run = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ interval[i] = 0; ++ if (i == channel) ++ continue; ++ ctl = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ interval[i] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ if ((ctl & A9_SPI0_CH3_RUN) && interval[i]) ++ *run |= BIT(i); ++ } ++} ++ ++static void a9_aop_spi0_stop_timing(struct a9_aop_spi0 *s, ++ struct a9_aop_spi0_ch3 *c, ++ const u32 *interval) ++{ ++ unsigned int i; ++ ++ c->peers_ticking = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ if ((c->peers_run & BIT(i)) && ++ readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]) != ++ interval[i]) ++ c->peers_ticking |= BIT(i); ++ } ++ c->reset_to_stop_us = c->stop_ns > s->reset_done_ns ? ++ div_u64(c->stop_ns - s->reset_done_ns, NSEC_PER_USEC) : 0; ++} ++ ++/* ++ * Wake12/13 captured channel 7, and wake13 channel 3, stopped but never idle: ++ * for 20 complete slices the control/progress word, config, interval, cursor, ++ * SCM and both fault banks were unchanged. This predicate is intentionally ++ * narrower than "RUN clear and no progress": the channel must be one of the ++ * two device-observed periodic clients, must carry that client's route, rate ++ * and config before the stop, and every sample must be byte-identical with SCM ++ * 3/0 (every earlier non-idle stop, which followed a channel-3 stop that had ++ * latched it) or 0/0 (a first stop that latches nothing), no fault, no cursor ++ * movement and a retained nonzero interval. ++ * It only permits the callback masks, descriptor clear, config acknowledge and ++ * the established controller reset; every later SCM, fault and copy gate stays. ++ */ ++static const char *a9_aop_spi0_stable_why(struct a9_aop_spi0_ch3 *c, ++ const struct a9_aop_channel_diag_slice *diag, ++ unsigned int channel) ++{ ++ const struct a9_aop_ch3_slice *first; ++ unsigned int i; ++ u32 route; ++ ++ if (channel == A9_SPI0_PHOSPHORUS_CHANNEL) ++ route = A9_SPI0_PHOSPHORUS_ROUTE; ++ else if (channel == A9_SPI0_MPU_ASYNC_CHANNEL) ++ route = A9_SPI0_MPU_ASYNC_ROUTE; ++ else ++ return "channel"; ++ if (c->poll != -ETIMEDOUT || c->slices != A9_SPI0_CH3_SLICES) ++ return "no-timeout"; ++ if (!(c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) || ++ c->before[A9_CH3_I_ROUTE] != route || ++ c->before[A9_CH3_I_RATE] != A9_SPI0_PHOSPHORUS_RATE || ++ c->before[A9_CH3_I_C3C] != A9_SPI0_PHOSPHORUS_CFG || ++ c->before[A9_CH3_I_DESC] || (c->stop & A9_SPI0_CH3_RUN)) ++ return "before-identity"; ++ if (c->before[A9_CH3_I_8D4] || (c->before[A9_CH3_I_8D0] & ~BIT(2 * channel))) ++ return "before-fault"; ++ ++ first = &c->slice[0]; ++ if ((first->c3 & A9_SPI0_STABLE_CTL_MASK) != A9_SPI0_STABLE_CTL_STOPPED || ++ (first->c3 & (A9_SPI0_CH3_RUN | A9_SPI0_CH3_IDLE | A9_SPI0_CH3_IRQ1))) ++ return "stopped-control"; ++ if ((first->scm_lo != 0 && first->scm_lo != 3) || first->scm_hi) ++ return "scm"; ++ if (first->fault || diag[0].fault_8d4) ++ return "sample-fault"; ++ if (diag[0].cfg != A9_SPI0_PHOSPHORUS_CFG || diag[0].cursor || !diag[0].interval) ++ return "cfg-cursor-interval"; ++ ++ for (i = 1; i < A9_SPI0_CH3_SLICES; i++) { ++ const struct a9_aop_ch3_slice *p = &c->slice[i]; ++ ++ if (p->c3 != first->c3 || p->scm_lo != first->scm_lo || ++ p->scm_hi != first->scm_hi || p->fault != first->fault || ++ diag[i].cfg != diag[0].cfg || ++ diag[i].interval != diag[0].interval || ++ diag[i].cursor != diag[0].cursor || ++ diag[i].fault_8d4 != diag[0].fault_8d4) ++ return "progress"; ++ } ++ if (c->slice[A9_SPI0_CH3_SLICES - 1].ns < ++ (u64)A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US * NSEC_PER_USEC) ++ return "short"; ++ return NULL; ++} ++ ++/* ++ * AOPSPIMaster_teardownChannel (0102e4f8) for channel 3, then the fault ++ * handler's acknowledge (0102f20a), under the same guard as the reset with ++ * the CPU halted. Every store is a read-modify-write except the descriptor, ++ * which the firmware zeroes. The idle wait runs in 1 ms slices, up to twice ++ * the firmware's own 10 ms SCMto bound; each slice ends with read-only ++ * samples of C3, the SCM state and +0x8d0 (wake15: and config, interval, ++ * cursor and +0x8d4). An idle timeout stops before the masks unless the ++ * stable-stop predicate holds. The acknowledge bit self-clears; +0x8d0 bit 6 ++ * shows whether it took effect. ++ */ ++static void a9_aop_spi0_ch3_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ void __iomem *c3 = s->base + A9_SPI0_CH3; ++ void __iomem *c3c = s->base + A9_SPI0_CH3_0C; ++ void __iomem *base = a9_aop_spi0_channel_base(s, 3); ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_ch3_snapshot(s->base, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(c3) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, c3); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, 3, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(c3, value, value & A9_SPI0_CH3_IDLE, ++ 1, A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->ch3_diag[c->slices]; ++ d->cfg = readl(c3c); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->ch3_diag, 3); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(3); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(c3) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, c3); ++ c->class1 = readl(c3) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, c3); ++ writel(0, s->base + A9_SPI0_CH3_DESC); ++ c->ack = readl(c3c) | A9_SPI0_CH3_ACK; ++ writel(c->ack, c3c); ++ } ++ a9_aop_spi0_ch3_snapshot(s->base, c->after); ++ c->failed = c->poll && !c->stable ? "failed-ch3-teardown-idle" : ++ a9_spi0_ch3_verify(c->after, c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++ ++static void a9_aop_spi0_channel_snapshot(struct a9_aop_spi0 *s, ++ unsigned int channel, u32 *values) ++{ ++ void __iomem *base = a9_aop_spi0_channel_base(s, channel); ++ ++ values[A9_CH3_I_C3] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ values[A9_CH3_I_ROUTE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_ROUTE]); ++ values[A9_CH3_I_RATE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_RATE]); ++ values[A9_CH3_I_C3C] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]); ++ values[A9_CH3_I_DESC] = readl(base + A9_SPI0_CHANNEL_DESC); ++ values[A9_CH3_I_8D0] = readl(s->base + A9_SPI0_FAULT); ++ values[A9_CH3_I_8D4] = readl(s->base + A9_SPI0_FAULT_8D4); ++} ++ ++static const char *a9_spi0_channel_state_verify(u32 control, u32 desc, ++ u32 fault, u32 fault_8d4, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ u32 pending = BIT(2 * channel); ++ ++ if (control & A9_SPI0_CH3_RUN) ++ return "failed-channel-teardown-run"; ++ if (control & A9_SPI0_CH3_IRQ0) ++ return "failed-channel-teardown-class0"; ++ if (control & A9_SPI0_CH3_IRQ1) ++ return "failed-channel-teardown-class1"; ++ if (!(control & A9_SPI0_CH3_IDLE) && !stable_stop) ++ return "failed-channel-teardown-idle"; ++ if (desc) ++ return "failed-channel-teardown-desc"; ++ if ((fault | fault_8d4) & pending) ++ return "failed-channel-teardown-pending"; ++ return NULL; ++} ++ ++static const char *a9_spi0_channel_verify(const u32 *after, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ return a9_spi0_channel_state_verify(after[A9_CH3_I_C3], ++ after[A9_CH3_I_DESC], ++ after[A9_CH3_I_8D0], ++ after[A9_CH3_I_8D4], channel, ++ stable_stop); ++} ++ ++/* ++ * Wake11 applies the channel-3 firmware sequence unchanged to another slot. ++ * Wake12 only appends read-only scheduler samples to that established poll. ++ */ ++static void a9_aop_spi0_channel_locked(void *data) ++{ ++ struct a9_aop_spi0_channel_op *op = data; ++ struct a9_aop_spi0 *s = op->spi0; ++ struct a9_aop_spi0_ch3 *c = op->state; ++ void __iomem *base = a9_aop_spi0_channel_base(s, op->channel); ++ void __iomem *control = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]; ++ void __iomem *config = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]; ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(control) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, control); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, op->channel, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(control, value, ++ value & A9_SPI0_CH3_IDLE, 1, ++ A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->channel_diag[c->slices]; ++ d->cfg = readl(config); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->channel_diag, op->channel); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(op->channel); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(control) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, control); ++ c->class1 = readl(control) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, control); ++ writel(0, base + A9_SPI0_CHANNEL_DESC); ++ c->ack = readl(config) | A9_SPI0_CH3_ACK; ++ writel(c->ack, config); ++ } ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->after); ++ c->failed = c->poll && !c->stable ? "failed-channel-teardown-idle" : ++ a9_spi0_channel_verify(c->after, op->channel, ++ c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++/* Called only if ps_spi0 is fully on; reads nothing. */ ++static void a9_aop_spi0_pmgr_on(void *data) ++{ ++ *(bool *)data = true; ++} ++ ++/* AP PMGR ps_spi0, read-only: never requested, resumed or written. ++ * 0 = on, -EHOSTDOWN = off (raw still filled), -ENODEV = no node. ++ */ ++static int a9_aop_spi0_pmgr(struct apple_aop *aop, u32 *raw) ++{ ++ struct apple_pmgr_observation state = {}; ++ bool on = false; ++ int ret; ++ ++ if (!aop->spi0_pmgr) ++ return -ENODEV; ++ ret = apple_pmgr_observe(&aop->spi0_pmgr, 1, &state, ++ a9_aop_spi0_pmgr_on, &on); ++ *raw = state.raw; ++ return ret ?: on ? 0 : -EIO; ++} ++ ++/* ++ * After a reset that cleared +0x8d0 bit 6, before the image copy: CPU halted, ++ * AP notifications masked. A refusal sets ch3.failed and nothing is loaded. ++ */ ++static int a9_aop_spi0_ch3_teardown(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *b = c->before, *a = c->after; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ unsigned int i; ++ bool source_progress = false; ++ int ret; ++ ++ aop->experiment_step = "spi0-ch3-teardown"; ++ dev_info(aop->dev, "SPI0 CH3TD begin pa=%llx cpu=halted: first channel-3 store follows\n", ++ A9_SPI0_PA + A9_SPI0_CH3); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, "spi0-ch3-teardown"); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_ch3_locked, s); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH3TD errno=%d ran=0: guard refused, channel 3 not touched\n", ++ ret); ++ return ret; ++ } ++ c->ran = true; ++ dev_info(aop->dev, "SPI0 CH3TD before c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH3TD idle-sample slice=%u at_us=%llu c3=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ i, div_u64(p->ns, NSEC_PER_USEC), p->c3, p->scm_lo, ++ p->scm_hi, p->fault, !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->ch3_diag[i]; ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ i, p->c3, d->cfg, d->interval, d->cursor, p->scm_lo, ++ p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->ch3_diag[0].interval || ++ d->cursor != s->ch3_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->ch3_diag[0].interval, ++ s->ch3_diag[c->slices - 1].interval, ++ s->ch3_diag[0].cursor, ++ s->ch3_diag[c->slices - 1].cursor, source_progress, ++ c->stable ? "stable-stop-reset-only" : "refused"); ++ /* beyond_fw_bound=1: idle, but later than the firmware would wait. */ ++ dev_info(aop->dev, "SPI0 CH3TD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ c->stop, c->idle, c->poll); ++ else if (c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->poll, c->class0, c->class1, c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH3TD after c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH3TD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ /* c3c_bit5 is the pulse's readback, logged only (0 at 19:50). */ ++ dev_info(aop->dev, "SPI0 CH3TD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x c3c_bit5=%u ch3_pending=%u result=%s\n", ++ c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), ++ a[A9_CH3_I_DESC], !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), c->failed ?: ++ c->stable ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static int a9_aop_spi0_channel_teardown(struct apple_aop *aop, ++ unsigned int channel) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->channel; ++ struct a9_aop_spi0_channel_op op = { ++ .spi0 = s, ++ .state = c, ++ .channel = channel, ++ }; ++ const u32 *b, *a; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ char checkpoint[32]; ++ unsigned int i; ++ bool source_progress = false; ++ bool stable_stop; ++ int ret; ++ ++ memset(c, 0, sizeof(*c)); ++ memset(s->channel_diag, 0, sizeof(s->channel_diag)); ++ s->teardown_failed_channel = channel; ++ aop->experiment_step = "spi0-channel-teardown"; ++ snprintf(checkpoint, sizeof(checkpoint), "spi0-ch%u-teardown", channel); ++ dev_info(aop->dev, "SPI0 CH%uTD begin pa=%llx cpu=halted: first channel-%u store follows\n", ++ channel, A9_SPI0_PA + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel, channel); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, checkpoint); ++ if (ret) { ++ c->failed = "failed-channel-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_channel_locked, &op); ++ if (ret) { ++ c->failed = "failed-channel-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH%uTD errno=%d ran=0: guard refused, channel %u not touched\n", ++ channel, ret, channel); ++ return ret; ++ } ++ c->ran = true; ++ stable_stop = s->teardown_stable & BIT(channel); ++ b = c->before; ++ a = c->after; ++ dev_info(aop->dev, "SPI0 CH%uTD before ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD idle-sample slice=%u at_us=%llu ctl=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ channel, i, div_u64(p->ns, NSEC_PER_USEC), p->c3, ++ p->scm_lo, p->scm_hi, p->fault, ++ !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->channel_diag[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ channel, i, p->c3, d->cfg, d->interval, d->cursor, ++ p->scm_lo, p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->channel_diag[0].interval || ++ d->cursor != s->channel_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ channel, c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->channel_diag[0].interval, ++ s->channel_diag[c->slices - 1].interval, ++ s->channel_diag[0].cursor, ++ s->channel_diag[c->slices - 1].cursor, ++ source_progress, ++ stable_stop ? "stable-stop-reset-only" : "refused"); ++ dev_info(aop->dev, "SPI0 CH%uTD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ channel, c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ channel, c->stop, c->idle, c->poll); ++ else if (stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->poll, c->class0, c->class1, ++ c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH%uTD after ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH%uTD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ channel, c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ dev_info(aop->dev, "SPI0 CH%uTD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x cfg_bit5=%u pending0=%u pending1=%u result=%s\n", ++ channel, c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), a[A9_CH3_I_DESC], ++ !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(a[A9_CH3_I_8D4] & BIT(2 * channel)), c->failed ?: ++ stable_stop ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s) ++{ ++ const u32 *v; ++ const char *failed = NULL; ++ unsigned int channel; ++ ++ if (s->teardown_selected != s->teardown_done) ++ return "failed-channel-teardown-incomplete"; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ v = s->trace.channel[channel]; ++ failed = a9_spi0_channel_state_verify(v[A9_CHANNEL_I_CTL], ++ v[A9_CHANNEL_I_DESC], ++ s->trace.ch3[A9_CH3_I_8D0], ++ s->trace.ch3[A9_CH3_I_8D4], channel, ++ s->teardown_stable & BIT(channel)); ++ if (failed) { ++ s->teardown_failed_channel = channel; ++ return failed; ++ } ++ } ++ return NULL; ++} ++ ++/* ++ * Channel 3 remains mandatory and executes first. Wake11 additionally selects ++ * every snapshot slot with run bit 0 (RUN) set and a nonzero interval, which ++ * excludes configured-but-idle slots while selecting any periodic client still ++ * owned by the old firmware. Wake16 returns to wake11's order after wake15's ++ * channel-7-first stop failed: all ten device stops fit "channel 3 idles when ++ * stopped within ~2 ms of the reset (5/5; 0/2 at 11 and 33 ms) and channel 7 ++ * never idles (0/3, at 0.3 ms, 1.9 ms and 15 ms)". Channel 3 goes first and ++ * early; channel 7 is then the stopped-without-idle channel the stable-stop ++ * fingerprint was captured from (wake11/12). ++ */ ++static int a9_aop_spi0_teardown_running(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ s->teardown_selected = BIT(3); ++ s->teardown_done = 0; ++ s->teardown_stable = 0; ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3) ++ continue; ++ v = s->trace.channel[channel]; ++ if ((v[A9_CHANNEL_I_CTL] & A9_SPI0_CH3_RUN) && ++ v[A9_CHANNEL_I_INTERVAL]) ++ s->teardown_selected |= BIT(channel); ++ } ++ dev_info(aop->dev, "SPI0 teardown select stage=after-controller-reset selected=%04x mandatory=0008 rule=ch3-or-run-and-interval order=ch3-then-others\n", ++ s->teardown_selected); ++ ret = a9_aop_spi0_ch3_teardown(aop); ++ if (ret) { ++ s->teardown_failed_channel = 3; ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=3 errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, ret, ++ s->ch3.failed ?: "failed-ch3-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ ret = a9_aop_spi0_channel_teardown(aop, channel); ++ if (ret) { ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=%u errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, channel, ret, ++ s->channel.failed ?: "failed-channel-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(channel); ++ } ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ dev_info(aop->dev, "SPI0 teardown complete selected=%04x done=%04x stable=%04x result=pass\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable); ++ return 0; ++} ++ ++/* ++ * Second copy of the firmware controller reset, after the stop has completed. ++ * The preceding after-ch3-teardown trace is the read-only before snapshot; ++ * after-post-teardown-reset is the matching read-only after snapshot. Refuse ++ * the image copy unless SCM is zero and every channel quiet-state gate holds. ++ */ ++static int a9_aop_spi0_post_reset(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ const u32 *b = r->before, *a = r->after; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.ran || ++ s->ch3.error || s->teardown_selected != s->teardown_done || r->ran) { ++ r->failed = "failed-post-reset-state"; ++ return -EPERM; ++ } ++ aop->experiment_step = "spi0-post-teardown-reset"; ++ dev_info(aop->dev, "SPI0 POSTRST begin pa=%llx reason=ch3-stop-latched-scm cpu=halted\n", ++ A9_SPI0_PA); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-post-teardown-reset"); ++ if (ret) { ++ r->failed = "failed-post-reset-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_post_reset_locked, s); ++ r->ran = !ret; ++ r->error = ret ?: r->error; ++ dev_info(aop->dev, "SPI0 POSTRST errno=%d ran=%d before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ r->error, r->ran, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 POSTRST after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], r->saved, r->asserted, ++ r->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ if (r->error) { ++ r->failed = "failed-post-reset-controller"; ++ return r->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-post-teardown-reset"); ++ if (ret) ++ return ret; ++ if (s->trace.scm[0] || s->trace.scm[1]) ++ r->failed = "failed-post-reset-scm"; ++ else { ++ r->failed = a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3)); ++ if (!r->failed) ++ r->failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = r->failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 POSTRST check errno=%d scm_lo=%08x scm_hi=%08x ch3_pending=%u result=%s\n", ++ ret, s->trace.scm[0], s->trace.scm[1], ++ !!(s->trace.ch3[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), ++ r->failed ?: "pass"); ++ dev_info(aop->dev, "SPI0 POSTRST channels selected=%04x done=%04x stable=%04x result=%s\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable, ++ r->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * The ten INTC words of a9_intc_offsets into state[@at], read only. The ++ * INTC is the AOP wrapper block (0xc1800000 in the firmware, space 2 as ++ * SPI0's verified 0xc1100000 = 0x210100000), which this driver reads ++ * unguarded for the mailbox and CPU control; no separate PMGR domain. ++ */ ++static void a9_aop_intc_snapshot(struct apple_aop *aop, unsigned int at) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ u32 *v = n->state[at]; ++ unsigned int i; ++ ++ for (i = 0; i < A9_INTC_REGS; i++) ++ v[i] = aop_rd(aop, a9_intc_offsets[i]); ++ n->states |= BIT(at); ++ dev_info(aop->dev, "AOP INTC state at=%s reset808=%08x current81c=%08x raise900=%08x ack980=%08x mask_a00=%08x mask_a04=%08x mask_a08=%08x mask_a80=%08x g7_c020=%08x g7_c024=%08x src_masked=%u view=%s\n", ++ a9_intc_at_name(at), v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ v[7], v[8], v[9], !!(v[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(v)); ++} ++ ++/* ++ * Round 4 (ds-aop-spi0-ch3-round4.md): the old session's SPI0 global IRQ ++ * 0x10014 stays latched in the AOP interrupt controller, and fresh firmware ++ * takes it as soon as SPI lifecycle event 1 enables it, before its own SPI0 ++ * reset (SCMErr:0x3,3). With the CPU halted and before the first SPI0 ++ * access, disable that source (group 1 disable bank, 0102c9ce). The bank ++ * reads back the mask, as the firmware's own reads of it show: bit 20 must ++ * read 1 after the store. ++ */ ++static int a9_aop_intc_disable(struct apple_aop *aop) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ const u32 *b = n->state[A9_INTC_AT_PRE_DISABLE]; ++ const u32 *a = n->state[A9_INTC_AT_POST_DISABLE]; ++ int ret; ++ ++ aop->experiment_step = "intc-disable"; ++ dev_info(aop->dev, "AOP INTC disable begin irq=0x10014 pa=%llx value=%08x cpu=halted: first INTC store follows\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "intc-disable"); ++ if (ret) { ++ n->failed = "failed-intc-disable-checkpoint"; ++ return ret; ++ } ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_DISABLE); ++ ret = aop_wr(aop, A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_DISABLE); ++ if (ret) { ++ n->failed = "failed-intc-disable"; ++ } else if (!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM)) { ++ n->failed = "failed-intc-disable-readback"; ++ ret = -EIO; ++ } ++ n->disabled = !ret; ++ dev_info(aop->dev, "AOP INTC disable errno=%d before=%08x value=%08x after=%08x mask_a80=%08x src_masked=%u view=%s result=%s\n", ++ ret, b[A9_INTC_I_MASK0], A9_INTC_SPI0_SCM, a[A9_INTC_I_MASK0], ++ a[A9_INTC_I_ENABLE0], !!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(a), n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Release policy for aop_spi0_reset=1. The reload is device-proven for exactly ++ * one retained state: the previous kernel quiesced and halted the firmware ++ * (wake16, 2026-10-29; decision prev-quiesce-halted). Auto-boot still set, a ++ * previous kernel that did not halt it, or an older kernel in the chain are ++ * different hardware states that no run has reloaded with the final teardown. ++ * Refuse them before the first halt or SPI0 access so the old firmware is ++ * left exactly as found; aop_spi0_untested=1 attempts them anyway. A skip ++ * (DFU cold, bad record, nothing ran) must also find the wrapper at its reset ++ * state: a record-less AOP that something already stopped is refused rather ++ * than cold-loaded. A stale record skips as before. ++ */ ++static int a9_aop_spi0_policy(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ enum a9_spi0_reason reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, ++ &aop->spi0.prev); ++ ++ if (a9_spi0_policy_allows(reason, outbox & AOP_OUTBOX_ENABLE, spi0_untested)) ++ return 0; ++ dev_info(aop->dev, "SPI0 policy decision=refuse reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d: only prev-quiesce-halted reloads and a skip needs the wrapper at its reset state block not touched\n", ++ a9_spi0_reason_name(reason), auto_boot, outbox, ++ a9_spi0_prev_name(aop->spi0.prev.state), aop->spi0.prev.hello, ++ aop->spi0.prev.halted); ++ return -EPERM; ++} ++ ++/* ++ * After the proven halt/drain, before any SRAM write. Only firmware that ran ++ * since the last SoC reset leaves SPI0 state behind: auto-boot 1, or the ++ * previous kernel's transcript bound by +0x4020 (a9_spi0_decide). Without ++ * that evidence the block is never touched. @auto_boot and @outbox are the ++ * boot snapshot from before the stop. Nonzero means: do not load or ++ * release the AOP. ++ */ ++static int a9_aop_spi0_prereset(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *b = s->before, *a = s->after; ++ const char *failed = NULL; ++ bool run; ++ int ret; ++ ++ s->auto_boot = auto_boot; ++ s->outbox = outbox; ++ s->reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, &s->prev); ++ s->decided = true; ++ run = a9_spi0_reason_runs(s->reason); ++ dev_info(aop->dev, "SPI0 reset decision=%s reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d prev_steps=0x%x prev_carry=%d%s\n", ++ run ? "run" : "skip", a9_spi0_reason_name(s->reason), auto_boot, ++ outbox, a9_spi0_prev_name(s->prev.state), s->prev.hello, ++ s->prev.halted, s->prev.steps, s->prev.carried, ++ run ? "" : ": block not touched"); ++ if (!run) { ++ s->skipped = a9_spi0_reason_name(s->reason); ++ return 0; ++ } ++ if (!s->base || !aop->force_halted || !aop->force_drained || ++ !a9_aop_force_stopped(aop)) ++ return -EPERM; ++ ret = a9_aop_intc_disable(aop); ++ if (ret) ++ return ret; ++ aop->experiment_step = "spi0-reset"; ++ s->pmgr_before_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_before); ++ /* Reaches ramoops before the first AP access to the block. */ ++ dev_info(aop->dev, "SPI0 reset begin pa=%llx cpu=halted ps_spi0=%08x errno=%d: first AP access follows\n", ++ A9_SPI0_PA, s->pmgr_before, s->pmgr_before_errno); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-reset"); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_trace(aop, "before-controller-reset"); ++ if (ret) ++ return ret; ++ /* ++ * Wake15: from the controller reset to the last channel stop nothing may ++ * drain a console in this task. Holding the console lock makes every ++ * dev_info() below only append to the log buffer; the flushed spi0-reset ++ * checkpoint above already put the begin line into ramoops, and the lines ++ * are emitted by console_unlock() once the stops are stored. ++ */ ++ console_lock(); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_reset_locked, s); ++ s->ran = !ret; ++ s->error = ret ?: s->error; ++ s->pmgr_after_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_after); ++ dev_info(aop->dev, "SPI0 reset errno=%d ran=%d pa=%llx before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ s->error, s->ran, A9_SPI0_PA, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 reset after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u provider3=not-replayed\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], s->saved, s->asserted, ++ s->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 pmgr ps_spi0 before=%08x errno=%d after=%08x errno=%d (read-only, never powered)\n", ++ s->pmgr_before, s->pmgr_before_errno, s->pmgr_after, ++ s->pmgr_after_errno); ++ if (s->error) { ++ console_unlock(); ++ return s->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-controller-reset"); ++ if (!ret) ++ ret = a9_aop_spi0_teardown_running(aop); ++ console_unlock(); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-ch3-teardown"); ++ if (!ret) ++ failed = a9_aop_spi0_selected_verify(s); ++ if (failed) { ++ s->channel.failed = failed; ++ ret = -EIO; ++ } ++ if (!ret) ++ ret = a9_aop_spi0_post_reset(aop); ++ return ret; ++} ++ ++/* Wake10 samples both SCM words and both fault classes, not only channel 3. */ ++static void a9_aop_spi0_fault_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ ++ s->prerun_scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ s->prerun_scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ s->prerun = readl(s->base + A9_SPI0_FAULT); ++ s->prerun_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ a9_aop_spi0_ch3_snapshot(s->base, s->ch3.reassert); ++} ++ ++/* ++ * Last AP look before RUN, CPU still halted, after the image copy: bit 6 ++ * must stay clear in every one of 31 guarded samples spaced >= 5 ms apart, ++ * spanning >= 150 ms. A reasserted channel-3 bit would crash the fresh ++ * image as before, so the first set sample refuses the release; nothing is ++ * retried. Any refusal sets ch3.failed. ++ */ ++static int a9_aop_spi0_recheck(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *r = c->reassert; ++ u64 start = ktime_get_boottime_ns(); ++ unsigned int i; ++ int ret = 0; ++ ++ for (i = 0; i < A9_SPI0_WINDOW_SAMPLES; i++) { ++ if (i) ++ usleep_range(A9_SPI0_WINDOW_US, A9_SPI0_WINDOW_US + 1000); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_fault_read, s); ++ c->samples = i + 1; ++ c->elapsed_ms = div_u64(ktime_get_boottime_ns() - start, NSEC_PER_MSEC); ++ if (ret) { ++ c->failed = "failed-ch3-window-guard"; ++ break; ++ } ++ if (s->prerun_scm_lo || s->prerun_scm_hi || s->prerun || ++ s->prerun_8d4) { ++ s->reasserted = c->reasserted = true; ++ c->reassert_sample = i; ++ c->reassert_ms = c->elapsed_ms; ++ c->failed = "failed-spi0-window-reasserted"; ++ ret = -EIO; ++ dev_info(aop->dev, "SPI0 WINDOW reasserted sample=%u at_ms=%llu scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ i, c->reassert_ms, s->prerun_scm_lo, s->prerun_scm_hi, ++ s->prerun, s->prerun_8d4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ dev_info(aop->dev, "SPI0 WINDOW channel3 c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ r[0], r[1], r[2], r[3], r[4], r[5], r[6]); ++ break; ++ } ++ } ++ /* usleep_range() cannot return early; this only trips on a bad clock. */ ++ if (!ret && c->elapsed_ms < A9_SPI0_WINDOW_MS) { ++ c->failed = "failed-ch3-window-short"; ++ ret = -EIO; ++ } ++ c->window_done = !ret; ++ dev_info(aop->dev, "SPI0 CH3TD window errno=%d samples=%u/%u elapsed_ms=%llu min_ms=%u result=%s\n", ++ ret, c->samples, A9_SPI0_WINDOW_SAMPLES, c->elapsed_ms, ++ A9_SPI0_WINDOW_MS, ret ? c->failed : "pass"); ++ dev_info(aop->dev, "SPI0 pre-run errno=%d 8d0=%08x ch3_pending=%u cpu=halted\n", ++ ret, s->prerun, !!(s->prerun & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 WINDOW state errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ ret, s->prerun_scm_lo, s->prerun_scm_hi, s->prerun, ++ s->prerun_8d4, ret ? c->failed : "pass"); ++ if (ret && c->reasserted) ++ a9_aop_spi0_trace(aop, "window-fault"); ++ return ret; ++} ++ ++/* SCM state and the channel-3 interval pair, read-only, into scm[scm_at]. */ ++static void a9_aop_spi0_scm_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->intc.scm[s->intc.scm_at][i] = readl(s->base + a9_spi0_scm_offsets[i]); ++} ++ ++/* ++ * After the clean pre-RUN window, CPU still halted: the firmware's group-1 ++ * pre-handler acknowledge for 0x10014 (0102c6de..0102c6f2) discards the ++ * old-session latch now that its SPI0 cause is quiet. Fresh firmware ++ * re-enables the source in event 1. The INTC words, the SCM state and the ++ * C3 interval pair are logged around it and never gated: with 0x10014 ++ * masked, no readable word shows whether it is still pending. ++ */ ++static int a9_aop_intc_ack(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ const u32 *p = n->scm[0], *q = n->scm[1]; ++ const u32 *b = n->state[A9_INTC_AT_PRE_ACK]; ++ const u32 *a = n->state[A9_INTC_AT_POST_ACK]; ++ int ret; ++ ++ aop->experiment_step = "intc-ack"; ++ n->scm_at = 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) { ++ n->failed = "failed-intc-ack-guard"; ++ dev_info(aop->dev, "AOP INTC pre-ack errno=%d: guard refused, no acknowledge\n", ++ ret); ++ return ret; ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_ACK); ++ dev_info(aop->dev, "AOP INTC pre-ack scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x cpu=halted\n", ++ p[0], p[1], p[2], p[3], b[A9_INTC_I_CURRENT]); ++ ret = aop_wr(aop, A9_INTC_ACK, A9_INTC_SPI0_SCM); ++ /* Also the posted-write barrier. */ ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_ACK); ++ n->acked = !ret; ++ dev_info(aop->dev, "AOP INTC ack errno=%d before=%08x value=%08x after=%08x\n", ++ ret, b[A9_INTC_I_ACK], A9_INTC_SPI0_SCM, a[A9_INTC_I_ACK]); ++ if (ret) { ++ n->failed = "failed-intc-ack"; ++ return ret; ++ } ++ n->scm_at = 1; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) ++ n->failed = "failed-intc-post-guard"; ++ dev_info(aop->dev, "AOP INTC post-ack errno=%d scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x result=%s\n", ++ ret, q[0], q[1], q[2], q[3], a[A9_INTC_I_CURRENT], n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Wake7 still took 0x10014 before SPI event 5 despite the targeted ack. ++ * Replay INTC lifecycle event 3 (0102c592/598/59c/5a2): 1/read/0/read at ++ * +0x808. aop_wr includes a posted-write read after each store. This resets ++ * all AOP-private interrupt state; it does not write the SCM status or MPU. ++ * No assumption that SCM must become zero. Only the final reset readback ++ * and the established channel-3 quiet conditions gate RUN. ++ */ ++static int a9_aop_intc_reset(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.window_done || ++ !n->acked || n->reset_done || ++ (aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) || readl(aop->control)) { ++ n->failed = "failed-intc-reset-state"; ++ return -EPERM; ++ } ++ ret = a9_aop_spi0_trace(aop, "pre-intc-reset"); ++ if (ret) ++ return ret; ++ aop->experiment_step = "intc-reset"; ++ dev_info(aop->dev, "AOP INTC reset begin pa=%llx sequence=1/read/0/read cpu=halted\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_RESET); ++ ret = aop_wr(aop, A9_INTC_RESET, 1); ++ n->reset_asserted = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-assert"; ++ return ret; ++ } ++ ret = aop_wr(aop, A9_INTC_RESET, 0); ++ n->reset_after = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-deassert"; ++ return ret; ++ } ++ if (n->reset_after) { ++ n->failed = "failed-intc-reset-readback"; ++ ret = -EIO; ++ } else { ++ ret = a9_aop_spi0_trace(aop, "post-intc-reset"); ++ if (!ret && a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3))) { ++ n->failed = "failed-intc-reset-ch3"; ++ ret = -EIO; ++ } else if (!ret && a9_aop_spi0_selected_verify(s)) { ++ n->failed = "failed-intc-reset-channel"; ++ ret = -EIO; ++ } ++ } ++ n->reset_done = !ret; ++ dev_info(aop->dev, "AOP INTC reset errno=%d asserted=%08x after=%08x result=%s\n", ++ ret, n->reset_asserted, n->reset_after, n->failed ?: "pass"); ++ return ret; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.h b/drivers/soc/apple/aop-warm-spi0.h +new file mode 100644 +index 000000000..2470165ce +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.h +@@ -0,0 +1,543 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * J71t AOP reload/wake experiments, both opt-in and default off. Constants ++ * and evidence types only; the MMIO lives in apple-a9-aop-active.c. ++ * Evidence: docs/periph/ds-aop-reload-crash-and-wake.md. ++ * ++ * aop_spi0_reset=1: every halt + SRAM reload of a live AOP crashed in ++ * AOPSPIMaster_channelFaultIRQ (0102f252, DFAR 0x4c): the AOP-private SPI0 ++ * controller kept channel 3's +0x8d0 bit 6 while the fresh firmware's ++ * channel table was still empty. With the CPU halted, the AP replays the ++ * firmware's own controller reset AOPSPIMaster_reset (0102f2f4), then the ++ * firmware's own channel-3 teardown, and samples bit 6 for >= 150 ms after ++ * the image copy before it releases RUN once. Round 4: the old session's ++ * SPI0 global IRQ 0x10014 stays latched in the AOP interrupt controller ++ * (SCMErr:0x3,3 at the accel STARTEP), so the AP disables that source before ++ * the reset and acknowledges it after the clean window, just before RUN. ++ * Wake7: the channel-3 idle wait gets the firmware's own 10 ms bound twice ++ * over, sampled per 1 ms slice, and the disable is judged by the INTC mask ++ * readback, which the firmware itself reads. ++ * Wake8: after the targeted ack, replay INTC event 3 (+0x808: 1/read/0/read). ++ * This resets retained interrupt delivery, not the read-only SCM state. ++ * Wake9: wake8 device evidence showed the channel-3 stop itself changes SCM ++ * from 0 to 3. Replay the same SPI controller reset once more after teardown; ++ * require the post-reset SCM state to be zero and the teardown state to hold. ++ * Wake10: the second reset clears SCM, but a halted CPU then saw SCM 3 and ++ * +0x8d0 bit 14 (channel 7) during the firmware copy. Snapshot all 16 channel ++ * slots, sample global SCM/fault state across the copy, and gate the copy and ++ * final RUN on a completely quiet readback. ++ * Wake11: that snapshot proved channels 3 and 7 were both running. Keep the ++ * channel-3 sequence unchanged, then apply the same firmware teardown to every ++ * other run-bit + nonzero-interval channel before the post-teardown reset. ++ * Wake12: channel 7 is the phosphorus/BMP284 async client. Its firmware has no ++ * owner-specific stop path, so do not invent another store when the generic ++ * stop times out. Record the remaining scheduler state read-only and refuse. ++ * Wake13/14: every device stop of channel 3 that idled (wake8..wake12) came ++ * 1.5-2.2 ms after the controller reset; the two that did not (wake13/14) came ++ * 11/33 ms after it (pr_flush in the checkpoint), with class-0 state identical to ++ * wake12's passes. Wake15 closed that window (no pr_flush between the reset and ++ * the stops, console lock held), recorded the reset-to-stop latency and the peers ++ * per stop, and accepted one identity-pinned stopped-without-idle channel (3 or ++ * 7) as wake13 did for channel 7. Wake15's channel-7-first order then showed ++ * channel 7 never idles even at 0.3 ms (it is 0/3 over wake11/12/15), with ++ * nothing ticking. Wake16 therefore returns to channel 3 first (early, the only ++ * stop that ever idled) and then channel 7, which takes the stable-stop path ++ * (SCM 0/0 or 3/0, constant); every stop logs why the fingerprint did or did not ++ * hold (stable_why=). Wake14's class-0 prelude stays removed. ++ * ++ * aop_wake_wrapper=1: before 0x220, the AppleA7IOP status-3 resume wrapper ++ * writes (b4b854..b4b8b4). No CPU RUN, no auto-boot, no NMI. ++ */ ++#ifndef _A9_AOP_RELOAD_H ++#define _A9_AOP_RELOAD_H ++ ++#include ++#include ++#include "aop-warm-record.h" ++ ++/* Firmware device spi0: IOP 0x10100000 (space 2, 0x1000) via /arm-io ranges. */ ++#define A9_SPI0_PA 0x210100000ULL ++#define A9_SPI0_SIZE 0x1000U ++#define A9_SPI0_WRAPPER_PA 0x210800000ULL /* the only layout this applies to */ ++#define A9_SPI0_CONTROL 0x000U ++#define A9_SPI0_CLOCK 0x008U /* 0102f2aa programs 0x01b71b00 here */ ++#define A9_SPI0_FAULT 0x8d0U /* bit 2n = channel n, read by 0102f20a */ ++#define A9_SPI0_FAULT_8D4 0x8d4U ++#define A9_SPI0_CHANNEL_BASE 0x100U ++#define A9_SPI0_CHANNEL_STRIDE 0x080U ++#define A9_SPI0_CHANNELS 16U ++#define A9_SPI0_CHANNEL_REGS 7U ++#define A9_SPI0_CHANNEL_DESC 0x018U ++#define A9_SPI0_CH3 0x280U /* base + 0x100 + 0x80 * 3 */ ++#define A9_SPI0_CH3_0C 0x28cU ++#define A9_SPI0_RESET 0x100U /* control bit 8 */ ++#define A9_SPI0_ENABLE 0x001U /* control bit 0 */ ++#define A9_SPI0_CH3_PENDING 0x040U ++/* The firmware spins without bound; the AP gives each poll 1 ms. */ ++#define A9_SPI0_POLL_US 1000U ++#define A9_SPI0_DELAY_US 100U /* 0102b3f8(100, 0), twice */ ++#define A9_SPI0_REGS 6 ++ ++static const u32 a9_spi0_offsets[A9_SPI0_REGS] = { ++ A9_SPI0_CONTROL, A9_SPI0_CLOCK, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++ A9_SPI0_CH3, A9_SPI0_CH3_0C, ++}; ++ ++/* Snapshot indices into a9_spi0_offsets. */ ++enum { A9_SPI0_I_CONTROL, A9_SPI0_I_CLOCK, A9_SPI0_I_FAULT }; ++ ++/* ++ * Channel-3 teardown after the reset (ds-aop-spi0-ch3-round3.md section 5). ++ * The reset leaves C3 = 0x00010447: idle but still enabled, async-armed and ++ * with the class-0 fault IRQ on, so a level source re-latched +0x8d0 bit 6 ++ * within ~105 ms with the CPU halted. AOPSPIMaster_teardownChannel ++ * (0102e4f8) stops the channel and masks both IRQ classes; the fault ++ * handler (0102f20a) then acknowledges with C3C bit 5. ++ */ ++#define A9_SPI0_CH3_ROUTE 0x284U ++#define A9_SPI0_CH3_RATE 0x288U ++#define A9_SPI0_CH3_DESC 0x298U /* channel +0x18, zeroed on teardown */ ++#define A9_SPI0_CH3_RUN 0x001U /* C3 bit 0, channel run/enable */ ++#define A9_SPI0_CH3_IDLE 0x002U /* C3 bit 1, polled after the stop */ ++#define A9_SPI0_CH3_IRQ0 0x040U /* C3 bit 6, +0x8d0 class-0/fault enable */ ++#define A9_SPI0_CH3_IRQ1 0x100U /* C3 bit 8, +0x8d4 class-1 enable */ ++#define A9_SPI0_CHANNEL_PROGRESS_SHIFT 24U /* hardware-updated, not an IRQ class */ ++#define A9_SPI0_CH3_STOP 0x411U /* clear run bit 0 and bits 4, 10 */ ++#define A9_SPI0_CH3_CLASS0 0x450U /* mask class 0: bits 4, 6, 10 */ ++#define A9_SPI0_CH3_CLASS1 0x510U /* mask class 1: bits 4, 8, 10 */ ++#define A9_SPI0_CH3_ACK 0x020U /* C3C bit 5, ORed by 0102f20a; self-clearing */ ++#define A9_SPI0_PHOSPHORUS_CHANNEL 7U ++#define A9_SPI0_PHOSPHORUS_CTL_MASK 0x00ffffffU ++#define A9_SPI0_PHOSPHORUS_CTL_RUN 0x00010445U ++#define A9_SPI0_PHOSPHORUS_CTL_STOPPED 0x00010444U ++#define A9_SPI0_PHOSPHORUS_ROUTE 0x00000040U ++#define A9_SPI0_PHOSPHORUS_RATE 0x019000c8U ++#define A9_SPI0_PHOSPHORUS_CFG 0x00200009U ++/* Channel 3 is the MPU6900 async FIFO client (route 0x10, same rate/config). */ ++#define A9_SPI0_MPU_ASYNC_CHANNEL 3U ++#define A9_SPI0_MPU_ASYNC_ROUTE 0x00000010U ++/* Stopped control word, any top byte, class-0 enable optional (0x40). */ ++#define A9_SPI0_STABLE_CTL_STOPPED 0x00010404U ++#define A9_SPI0_STABLE_CTL_MASK 0x00ffffbfU ++/* Peer sampling and the "late stop" mark are reported, never gated, in-kernel. */ ++#define A9_SPI0_LATE_STOP_US 3000U ++/* ++ * AOPSPIMaster_setChannelEnabled(false) (0102ef7c) stores C3 & ~0x411 only ++ * if bit 0 was set, then spins on bit 1 until master +0x30, and panics ++ * "SCMto:%u" (0x010398da) past it. Event 1 sets +0x30 to 0x2710 us when it ++ * is 0 (0102e6fa, as in the live-cold object), through timer +0x18 = ++ * us * Hz / 1000000 (0102ff3c): a 10 ms bound. The AP polls in 1 ms slices, ++ * up to twice that, sampling C3, the SCM state and +0x8d0 after each slice. ++ */ ++#define A9_SPI0_CH3_IDLE_FW_US 10000U ++#define A9_SPI0_CH3_SLICE_US 1000U ++#define A9_SPI0_CH3_SLICES 20U /* 20 ms */ ++/* At least 150 ms (the 16:27 re-latch took ~105 ms) of samples before RUN. */ ++#define A9_SPI0_WINDOW_SAMPLES 31U ++#define A9_SPI0_WINDOW_US 5000U /* 30 sleeps >= 5 ms each */ ++#define A9_SPI0_WINDOW_MS 150U /* required first-to-last sample span */ ++#define A9_SPI0_CH3_REGS 7 ++ ++static const u32 a9_spi0_ch3_offsets[A9_SPI0_CH3_REGS] = { ++ A9_SPI0_CH3, A9_SPI0_CH3_ROUTE, A9_SPI0_CH3_RATE, A9_SPI0_CH3_0C, ++ A9_SPI0_CH3_DESC, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++}; ++ ++/* Every firmware-visible channel slot, read only in wake10 and selected in wake11. */ ++static const u32 a9_spi0_channel_offsets[A9_SPI0_CHANNEL_REGS] = { ++ 0x00U, 0x04U, 0x08U, 0x0cU, 0x10U, 0x14U, 0x18U, ++}; ++ ++enum { ++ A9_CHANNEL_I_CTL, A9_CHANNEL_I_ROUTE, A9_CHANNEL_I_RATE, A9_CHANNEL_I_CFG, ++ A9_CHANNEL_I_INTERVAL, A9_CHANNEL_I_CURSOR, A9_CHANNEL_I_DESC, ++}; ++ ++/* Snapshot indices into a9_spi0_ch3_offsets. */ ++enum { ++ A9_CH3_I_C3, A9_CH3_I_ROUTE, A9_CH3_I_RATE, A9_CH3_I_C3C, A9_CH3_I_DESC, ++ A9_CH3_I_8D0, A9_CH3_I_8D4, ++}; ++ ++/* One idle-poll slice: C3 as the slice ended, then SCM state and +0x8d0. */ ++struct a9_aop_ch3_slice { ++ u64 ns; /* since the stop store */ ++ u32 c3, scm_lo, scm_hi, fault; ++}; ++ ++/* Wake12 read-only scheduler evidence for an additional-channel stop poll. */ ++struct a9_aop_channel_diag_slice { ++ u32 cfg, interval, cursor, fault_8d4; ++}; ++ ++struct a9_aop_spi0_ch3 { ++ u32 before[A9_SPI0_CH3_REGS], after[A9_SPI0_CH3_REGS]; ++ u32 stop, idle, class0, class1, ack; /* words written / polled */ ++ int poll; /* idle-bit poll result */ ++ bool run_before; /* C3 bit 0 before the stop */ ++ unsigned int slices; /* idle-poll slices used */ ++ struct a9_aop_ch3_slice slice[A9_SPI0_CH3_SLICES]; ++ u64 idle_us; /* stop store to the idle read */ ++ int error; ++ bool ran; /* the guarded teardown executed */ ++ const char *failed; /* distinct refusal reason, or NULL */ ++ /* Wake15: stop latency after the controller reset and the peers' state. */ ++ u64 stop_ns, reset_to_stop_us; ++ u32 peers_run, peers_ticking; /* masks of the other 15 channels */ ++ bool stable; /* accepted stopped-without-idle identity */ ++ const char *stable_why; /* first failed fingerprint condition, NULL if stable */ ++ /* Pre-RUN window after the image copy; a refusal also sets failed. */ ++ unsigned int samples; ++ u64 elapsed_ms; ++ bool window_done, reasserted; ++ unsigned int reassert_sample; ++ u64 reassert_ms; ++ u32 reassert[A9_SPI0_CH3_REGS]; ++}; ++ ++struct a9_aop_spi0; ++ ++struct a9_aop_spi0_channel_op { ++ struct a9_aop_spi0 *spi0; ++ struct a9_aop_spi0_ch3 *state; ++ unsigned int channel; ++}; ++ ++/* ++ * AOP interrupt controller, wrapper offsets (ds-aop-spi0-ch3-round4.md ++ * section 3). IRQ 0x10014 is group 1, source 0x14: bank 0, bit 20. The ++ * group-1 disable/enable banks are write-one commands whose reads return ++ * the mask, 1 = disabled: the sleep command saves +0xa00..+0xa08 (0102c5ce) ++ * before masking all (0102c5da), the restore writes saved to +0xa00 and ++ * ~saved to +0xa80 (0102c9e6/0102c9f4), and aop_interrupt_write_enabled ++ * reads the target bank for the previous state (0102c9c0). So the disable ++ * is judged by bit 20 of the +0xa00 readback (wake6: ffffffff before the ++ * store, every source already masked by the quiesce). +0x81c only reports ++ * an unmasked pending source: the sleep path masks all, writes group 7 ++ * +0xc020/+0xc024 = 2 and spins until +0x81c reads 0 (0102c654), so ++ * all-ones there means the AP is not reading the INTC. +0x808 ends 0 after ++ * the event-3 pulse. +0x900 is the group-1 raise (0102c7c6) and +0x980 its ++ * pre-handler clear (0102c6f2); the firmware never reads them, so their ++ * readbacks are logged only. No register shows a masked source pending. ++ */ ++#define A9_INTC_RESET 0x808U /* event-3 pulse 1 then 0 (0102c592) */ ++#define A9_INTC_CURRENT 0x81cU /* current unmasked source */ ++#define A9_INTC_RAISE 0x900U /* group 1 raise, 0102c7c6 */ ++#define A9_INTC_ACK 0x980U /* group 1 pre-handler ack/clear, 0102c6f2 */ ++#define A9_INTC_DISABLE 0xa00U /* group 1 disable, store at 0102c9ce */ ++#define A9_INTC_ENABLE 0xa80U /* group 1 enable; reads the same mask */ ++#define A9_INTC_G7_DISABLE 0xc020U /* group 7 disable, 2 after sleep */ ++#define A9_INTC_SPI0_SCM 0x00100000U /* IRQ 0x10014 */ ++#define A9_INTC_ALL_ONES 0xffffffffU ++#define A9_INTC_REGS 10 ++ ++static const u32 a9_intc_offsets[A9_INTC_REGS] = { ++ A9_INTC_RESET, A9_INTC_CURRENT, A9_INTC_RAISE, A9_INTC_ACK, ++ A9_INTC_DISABLE, A9_INTC_DISABLE + 4, A9_INTC_DISABLE + 8, ++ A9_INTC_ENABLE, A9_INTC_G7_DISABLE, A9_INTC_G7_DISABLE + 4, ++}; ++ ++/* Snapshot indices into a9_intc_offsets. */ ++enum { ++ A9_INTC_I_RESET, A9_INTC_I_CURRENT, A9_INTC_I_RAISE, A9_INTC_I_ACK, ++ A9_INTC_I_MASK0, A9_INTC_I_MASK1, A9_INTC_I_MASK2, A9_INTC_I_ENABLE0, ++ A9_INTC_I_G7_0, A9_INTC_I_G7_1, ++}; ++ ++/* When the AP snapshots the INTC words. */ ++enum { ++ A9_INTC_AT_PRE_DISABLE, A9_INTC_AT_POST_DISABLE, A9_INTC_AT_PRE_ACK, ++ A9_INTC_AT_POST_ACK, A9_INTC_AT_N, ++}; ++ ++static inline const char *a9_intc_at_name(unsigned int at) ++{ ++ static const char * const names[] = { ++ "pre-disable", "post-disable", "pre-ack", "post-ack", ++ }; ++ ++ return at < A9_INTC_AT_N ? names[at] : "unknown"; ++} ++ ++/* "all-ones" if +0x81c reads ffffffff, which a live INTC cannot report. */ ++static inline const char *a9_intc_view(const u32 *state) ++{ ++ return state[A9_INTC_I_CURRENT] == A9_INTC_ALL_ONES ? "all-ones" : "intc"; ++} ++/* ++ * Read only, never written: the SCM state (also per idle-poll slice) and, ++ * around the acknowledge, the C3 interval pair. ++ */ ++#define A9_SPI0_SCM_LO 0x00cU ++#define A9_SPI0_SCM_HI 0x010U ++#define A9_SPI0_CH3_IVAL 0x290U ++#define A9_SPI0_CH3_CURSOR 0x294U ++#define A9_SPI0_SCM_REGS 4 ++ ++static const u32 a9_spi0_scm_offsets[A9_SPI0_SCM_REGS] = { ++ A9_SPI0_SCM_LO, A9_SPI0_SCM_HI, A9_SPI0_CH3_IVAL, A9_SPI0_CH3_CURSOR, ++}; ++ ++struct a9_aop_intc { ++ u32 state[A9_INTC_AT_N][A9_INTC_REGS]; /* around each store */ ++ unsigned int states; /* bit n: state[n] was read */ ++ u32 scm[2][A9_SPI0_SCM_REGS]; /* before / after the ack */ ++ unsigned int scm_at; /* scm[] row the guarded read fills */ ++ bool disabled, acked; ++ bool reset_done; ++ u32 reset_asserted, reset_after; ++ const char *failed; /* distinct refusal reason, or NULL */ ++}; ++ ++/* Read-only milestones. Wake10 gates the global SCM/fault words, not channels. */ ++struct a9_aop_spi0_trace { ++ u32 control, clock, scm[A9_SPI0_SCM_REGS], ch3[A9_SPI0_CH3_REGS]; ++ u32 channel[A9_SPI0_CHANNELS][A9_SPI0_CHANNEL_REGS]; ++}; ++ ++/* Wake9's second, post-channel-stop replay of AOPSPIMaster_reset. */ ++struct a9_aop_spi0_post_reset { ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved, asserted, polled; ++ int error; ++ bool ran; ++ const char *failed; ++}; ++ ++/* ++ * First failed readback after the teardown, or NULL if all hold. C3C bit 5 ++ * is a pulse: at 19:50 the write of 0x00200029 read back 0x00200009 while ++ * +0x8d0 bit 6 went 1 -> 0, so the acknowledge is judged by bit 6 alone and ++ * the C3C readback is only logged. ++ */ ++static inline const char *a9_spi0_ch3_verify(const u32 *after, bool stable) ++{ ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) ++ return "failed-ch3-teardown-run"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0) ++ return "failed-ch3-teardown-class0"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1) ++ return "failed-ch3-teardown-class1"; ++ if (!(after[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE) && !stable) ++ return "failed-ch3-teardown-idle"; ++ if (after[A9_CH3_I_DESC]) ++ return "failed-ch3-teardown-desc"; ++ if (after[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING) ++ return "failed-ch3-teardown-pending"; ++ return NULL; ++} ++ ++/* ++ * Has firmware run since the last SoC reset? auto-boot alone cannot say: ++ * the outgoing aop_quiesce=1 halt clears it, exactly like a DFU cold start. ++ * The previous kernel's reserved-RAM shutdown transcript says whether its ++ * firmware ran. That DRAM can outlive a DFU; wrapper +0x4020 outbox enable ++ * cannot: set after reset (12/12 DFU cold snapshots), cleared by the first ++ * store of every Linux stop. No SPI0 access is needed to decide. ++ */ ++enum a9_spi0_prev_state { ++ A9_SPI0_PREV_MISSING, /* no transcript magic, or none reserved */ ++ A9_SPI0_PREV_INVALID, /* magic present, header/checksum rejected */ ++ A9_SPI0_PREV_VALID, ++}; ++ ++struct a9_spi0_prev { ++ enum a9_spi0_prev_state state; ++ bool hello; /* "SHUTDOWN begin ... hello=1": its firmware ran */ ++ bool halted; /* "QUIESCE step=complete ... halted=1" + steps bit */ ++ bool carried; /* "SPI0-CARRY retained=1": ran earlier in the chain */ ++ u32 steps; /* "QUIESCE step=complete ... steps=0x..." */ ++}; ++ ++enum a9_spi0_reason { ++ /* Run the reset. */ ++ A9_SPI0_LIVE_AUTO_BOOT, /* auto-boot 1 before the stop */ ++ A9_SPI0_PREV_HALTED, /* previous kernel quiesced and halted it */ ++ A9_SPI0_PREV_LIVE, /* previous kernel's firmware ran, no halt */ ++ A9_SPI0_PREV_CARRIED, /* an earlier kernel in the chain ran it */ ++ /* Skip: the block is not touched. */ ++ A9_SPI0_FRESH, /* no transcript, auto-boot 0: DFU cold */ ++ A9_SPI0_PREV_BAD, /* transcript present but corrupt */ ++ A9_SPI0_PREV_NO_RUN, /* transcript: nothing ran in the chain */ ++ A9_SPI0_STALE, /* run evidence, wrapper at reset state */ ++}; ++ ++static inline const char *a9_spi0_reason_name(enum a9_spi0_reason reason) ++{ ++ static const char * const names[] = { ++ "live-auto-boot", "prev-quiesce-halted", "prev-live", "prev-carried", ++ "fresh-instance", "prev-transcript-invalid", "prev-no-run", ++ "stale-transcript", ++ }; ++ ++ return reason <= A9_SPI0_STALE ? names[reason] : "unknown"; ++} ++ ++static inline const char *a9_spi0_prev_name(enum a9_spi0_prev_state state) ++{ ++ return state == A9_SPI0_PREV_VALID ? "valid" : ++ state == A9_SPI0_PREV_INVALID ? "invalid" : "missing"; ++} ++ ++static inline bool a9_spi0_reason_runs(enum a9_spi0_reason reason) ++{ ++ return reason <= A9_SPI0_PREV_CARRIED; ++} ++ ++/* ++ * Release policy: a reason that would run the reload is allowed by default ++ * only for prev-quiesce-halted, the one state device-proven end to end. A ++ * skip (fresh, bad record, nothing ran) is allowed only while the wrapper is ++ * still at its reset state; stale records skip as before. Untested lifts both. ++ */ ++static inline bool a9_spi0_policy_allows(enum a9_spi0_reason reason, bool outbox_enabled, ++ bool untested) ++{ ++ if (untested || reason == A9_SPI0_PREV_HALTED || reason == A9_SPI0_STALE) ++ return true; ++ if (a9_spi0_reason_runs(reason)) ++ return false; ++ /* ++ * A skip means "nothing ran since the SoC reset", which only holds if the ++ * wrapper is still at its reset state (+0x4020 outbox enable set, 12/12 DFU ++ * cold starts). Without it some earlier kernel stopped the firmware and its ++ * record is lost: a cold load would land on a halted, never-torn-down AOP. ++ */ ++ return outbox_enabled; ++} ++ ++/* Token search inside one transcript line, which holds no NUL. */ ++static inline bool a9_spi0_line_has(const char *line, size_t length, ++ const char *token, size_t n, ++ const char **at) ++{ ++ size_t i; ++ ++ for (i = 0; n <= length && i <= length - n; i++) { ++ if (!memcmp(line + i, token, n)) { ++ *at = line + i + n; ++ return true; ++ } ++ } ++ return false; ++} ++ ++#define a9_spi0_has(line, length, token, at) \ ++ a9_spi0_line_has(line, length, token, sizeof(token) - 1, at) ++ ++static inline bool a9_spi0_line_starts(const char *line, size_t length, ++ const char *prefix, size_t n) ++{ ++ return length >= n && !memcmp(line, prefix, n); ++} ++ ++#define a9_spi0_starts(line, length, prefix) \ ++ a9_spi0_line_starts(line, length, prefix, sizeof(prefix) - 1) ++ ++static inline u32 a9_spi0_hex(const char *p, const char *end) ++{ ++ u32 value = 0; ++ ++ for (; p < end; p++) { ++ if (*p >= '0' && *p <= '9') ++ value = value << 4 | (u32)(*p - '0'); ++ else if (*p >= 'a' && *p <= 'f') ++ value = value << 4 | (u32)(*p - 'a' + 10); ++ else ++ break; ++ } ++ return value; ++} ++ ++/* Our own checksummed transcript; only whole-line prefixes count. */ ++static inline void a9_spi0_prev_parse(struct a9_spi0_prev *p, ++ const char *text, size_t length) ++{ ++ const char *end = text + length, *line, *next, *at; ++ size_t n; ++ ++ for (line = text; line < end; line = next < end ? next + 1 : end) { ++ next = memchr(line, '\n', end - line); ++ if (!next) ++ next = end; ++ n = next - line; ++ if (a9_spi0_starts(line, n, "SHUTDOWN begin ") && ++ a9_spi0_has(line, n, " hello=1 ", &at)) ++ p->hello = true; ++ if (a9_spi0_starts(line, n, "QUIESCE step=complete ")) { ++ if (a9_spi0_has(line, n, " steps=0x", &at)) ++ p->steps = a9_spi0_hex(at, next); ++ p->halted = a9_spi0_has(line, n, " halted=1 ", &at) && ++ (p->steps & (1U << A9_HO_STEP_HALTED)); ++ } ++ if (a9_spi0_starts(line, n, "SPI0-CARRY retained=1 ")) ++ p->carried = true; ++ } ++} ++ ++/* ++ * Skips unless there is positive evidence: running the reset on a fresh ++ * block would be the first AP access to SPI0 with nothing retained to ++ * clear (and ps_spi0 may gate it), while a skip on retained state only ++ * repeats the known, non-hanging accel STARTEP crash. ++ */ ++static inline enum a9_spi0_reason a9_spi0_decide(bool auto_boot, bool outbox_enabled, ++ const struct a9_spi0_prev *p) ++{ ++ if (auto_boot) ++ return A9_SPI0_LIVE_AUTO_BOOT; ++ if (p->state == A9_SPI0_PREV_MISSING) ++ return A9_SPI0_FRESH; ++ if (p->state != A9_SPI0_PREV_VALID) ++ return A9_SPI0_PREV_BAD; ++ if (!p->hello && !p->carried) ++ return A9_SPI0_PREV_NO_RUN; ++ if (outbox_enabled) ++ return A9_SPI0_STALE; ++ if (p->hello) ++ return p->halted ? A9_SPI0_PREV_HALTED : A9_SPI0_PREV_LIVE; ++ return A9_SPI0_PREV_CARRIED; ++} ++ ++struct a9_aop_spi0 { ++ void __iomem *base; /* mapped only with aop_spi0_reset=1 */ ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved; /* control word before the reset */ ++ u32 asserted; /* saved & ~0x101 | 0x100 */ ++ u32 polled; /* control word when the polls ended */ ++ u32 prerun; /* +0x8d0 just before RUN */ ++ u32 prerun_8d4, prerun_scm_lo, prerun_scm_hi; ++ u32 pmgr_before, pmgr_after; /* AP PMGR ps_spi0 raw, read-only */ ++ int pmgr_before_errno, pmgr_after_errno; ++ int error; ++ bool ran, reasserted, copy_faulted; ++ const char *gate_failed; ++ const char *skipped; ++ struct a9_spi0_prev prev; /* parsed at probe from the transcript */ ++ u32 auto_boot, outbox; /* boot snapshot, before the stop */ ++ enum a9_spi0_reason reason; ++ bool decided; ++ struct a9_aop_spi0_ch3 ch3; /* after a clean reset only */ ++ /* Wake11 reuses one bounded scratch record for each additional channel. */ ++ struct a9_aop_spi0_ch3 channel; ++ u32 teardown_selected, teardown_done; ++ unsigned int teardown_failed_channel; ++ struct a9_aop_spi0_post_reset post_reset; /* after the stop latches SCM */ ++ struct a9_aop_intc intc; /* IRQ 0x10014 around the reset */ ++ struct a9_aop_spi0_trace trace; /* latest read-only milestone */ ++ /* Appended so every pre-wake12 field, including ch3, keeps its layout. */ ++ struct a9_aop_channel_diag_slice channel_diag[A9_SPI0_CH3_SLICES]; ++ /* Wake13: exact phosphorus stopped-without-idle exception, by channel. */ ++ u32 teardown_stable; ++ /* Wake15: channel 3 shares the sampled diagnostics; reset completion time. */ ++ struct a9_aop_channel_diag_slice ch3_diag[A9_SPI0_CH3_SLICES]; ++ u64 reset_done_ns; ++}; ++ ++/* 0102f2f4: bits 8 and 0 clear; the reset write adds bit 8. */ ++static inline u32 a9_spi0_base_word(u32 saved) ++{ ++ return saved & ~(A9_SPI0_RESET | A9_SPI0_ENABLE); ++} ++ ++#endif +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index e8ae90098..d208eacb6 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -1,19 +1,28 @@ + // SPDX-License-Identifier: GPL-2.0-only OR MIT + /* +- * Apple T8010 Always-On Processor (AOP) ++ * Apple S8000 and T8010 Always-On Processor (AOP) + * + * The AOP runs RTKit firmware that the operating system loads: a main image + * into the AOP's SRAM and an external segment into system memory, with boot +- * arguments filled in. Its services talk AFK/EPIC over shared ring buffers on +- * the RTKit application endpoints, as on later SoCs (see the Asahi Linux AOP ++ * arguments filled in. On the S8000 (A9) the AOP sees its SRAM through a window ++ * that has to be set up first, and its firmware takes more boot arguments. ++ * Its services talk AFK/EPIC over shared ring buffers on the RTKit ++ * application endpoints, as on later SoCs (see the Asahi Linux AOP + * driver). Each service with a device tree child node of the same name gets a + * child platform device; its driver receives the service's reports through + * apple_aop_listen() and sends requests with apple_aop_call(). + * +- * The firmware cannot be restarted once stopped, so the AOP starts once per +- * boot and the driver cannot be unbound or unloaded after that. ++ * J71t can reload after a certified quiesce/halt/drain across kexec. Other ++ * retained states are refused. Unbind/unload after starting remains forbidden. + */ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include "aop-warm-spi0.h" + #include + #include + #include +@@ -41,6 +50,14 @@ + #define APPLE_AOP_IRQ_MASK_ALL 0x1111 + #define APPLE_AOP_IRQ_INBOX BIT(0) + #define APPLE_AOP_IRQ_OUTBOX BIT(12) ++/* S8000: the window through which the AOP sees its SRAM */ ++#define APPLE_AOP_REMAP_PA_LO 0x08 ++#define APPLE_AOP_REMAP_PA_HI 0x10 ++#define APPLE_AOP_REMAP_IOVA_LO 0x18 ++#define APPLE_AOP_REMAP_IOVA_HI 0x20 ++#define APPLE_AOP_REMAP_SIZE 0x28 ++#define APPLE_AOP_REMAP_RESERVED 0x30 ++#define APPLE_AOP_REMAP_ENABLE 0x38 + + #define APPLE_AOP_CONTROL_AUTO_BOOT 0x0 + +@@ -54,6 +71,13 @@ + #define APPLE_AOP_BOOTARG_ECID APPLE_AOP_FOURCC('E', 'C', 'I', 'D') + #define APPLE_AOP_BOOTARG_NONC APPLE_AOP_FOURCC('N', 'O', 'N', 'C') + #define APPLE_AOP_BOOTARG_EC0P APPLE_AOP_FOURCC('E', 'C', '0', 'p') ++#define APPLE_AOP_BOOTARG_AOPT APPLE_AOP_FOURCC('A', 'O', 'P', 't') ++#define APPLE_AOP_BOOTARG_NCAL APPLE_AOP_FOURCC('n', 'C', 'a', 'l') ++#define APPLE_AOP_BOOTARG_WRAD APPLE_AOP_FOURCC('W', 'r', 'A', 'd') ++#define APPLE_AOP_BOOTARG_SOC APPLE_AOP_FOURCC('S', 'O', 'C', '_') ++#define APPLE_AOP_BOOTARG_SOCR APPLE_AOP_FOURCC('S', 'O', 'C', 'R') ++#define APPLE_AOP_BOOTARG_STKG APPLE_AOP_FOURCC('S', 'T', 'K', 'G') ++#define APPLE_AOP_MAX_BOOTARGS 10 + + #define APPLE_AOP_HALT_TIMEOUT_US 10000 + +@@ -78,7 +102,9 @@ + #define AFK_OPC_GET_BUF_ACK 0xa1 + #define AFK_OPC_SEND 0xa2 + #define AFK_OPC_START 0xa3 +-#define AFK_OPC_SHUTDOWN_ACK 0xc1 ++#define AFK_OPC_SHUTDOWN 0xa4 ++#define AFK_OPC_SHUTDOWN_ACK 0x87 ++#define AFK_OPC_HIBERNATE_ACK 0xc3 + + #define AFK_BLOCK_STEP 0x40 + #define AFK_MAX_BUFFER SZ_1M +@@ -100,6 +126,12 @@ + #define APPLE_AOP_MAX_CALLS 8 + #define APPLE_AOP_CALL_TIMEOUT msecs_to_jiffies(1000) + ++/* Service properties */ ++#define APPLE_AOP_PROPERTY_MAX 64 ++#define APPLE_AOP_PROP_CALIBRATION 0x3d ++#define APPLE_AOP_CALIBRATION_PARTIAL 1 ++#define APPLE_AOP_CALIBRATION_COMPLETE 2 ++ + struct afk_qe_header { + __le32 magic; + __le32 size; +@@ -160,6 +192,8 @@ struct apple_aop_ep { + + struct mutex lock; /* tx ring, seq and calls */ + u16 seq; ++ bool initialized; ++ struct completion stopped; + struct apple_aop_call *calls[APPLE_AOP_MAX_CALLS]; + }; + +@@ -182,12 +216,25 @@ struct apple_aop_register_work { + struct apple_aop_service service; + }; + ++struct apple_aop_hw { ++ /* Set up the AOP's window onto its SRAM before starting it */ ++ bool sram_window; ++ /* A fresh NONC boot argument */ ++ bool nonce; ++ /* AOPt, SOC_, SOCR, WrAd, STKG and nCal boot arguments */ ++ bool soc_bootargs; ++}; ++ + struct apple_aop { + struct device *dev; ++ const struct apple_aop_hw *hw; + void __iomem *akf; + void __iomem *control; + void __iomem *sram; ++ resource_size_t akf_base; ++ resource_size_t sram_base; + resource_size_t sram_size; ++ u32 target, chip_id, chip_revision; + struct apple_rtkit *rtk; + + void *ext; +@@ -202,6 +249,16 @@ struct apple_aop { + struct mutex children_lock; /* children, removing */ + struct list_head children; + bool removing; ++ u64 accel_reports; ++ int shutdown_error; ++ /* J71t only; all other boards retain their original start/stop path. */ ++ bool warm, hello, iop_crashed, force_halted, force_drained; ++ void __iomem *warm_wrapper; ++ struct device_node *domains[5], *spi0_pmgr; ++ struct a9_aop_spi0 spi0; ++ const char *experiment_step; ++ struct a9_sl_header *shutdown_log; ++ char *shutdown_text; + }; + + static bool apple_aop_is_started(struct apple_aop *aop) +@@ -223,6 +280,409 @@ static bool apple_aop_halt(struct apple_aop *aop) + APPLE_AOP_HALT_TIMEOUT_US); + } + ++static bool a9_aop_ramoops_ready(void) ++{ ++ struct console *con; ++ bool ready = false; ++ int cookie; ++ ++ cookie = console_srcu_read_lock(); ++ for_each_console_srcu(con) { ++ if (!strcmp(con->name, "ramoops") && ++ (console_srcu_read_flags(con) & (CON_ENABLED | CON_SUSPENDED)) == ++ CON_ENABLED) { ++ ready = true; ++ break; ++ } ++ } ++ console_srcu_read_unlock(cookie); ++ return ready; ++} ++ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* J71t warm reload integration. SPI0 machinery is the release6 wake16 code. */ ++#define AOP_DOMAINS 5 ++#define AOP_CPU_CONTROL 0x44 ++#define AOP_CPU_START BIT(4) ++#define AOP_OUTBOX_CONTROL 0x4020 ++#define AOP_INBOX_CONTROL 0x4008 ++#define AOP_OUTBOX_ENABLE BIT(0) ++#define AOP_MBOX_FAULTS GENMASK(19, 18) ++#define cold_defer 1 ++#define spi0_untested false ++ ++static u32 aop_rd(struct apple_aop *aop, u32 offset) ++{ ++ return readl(aop->warm_wrapper + offset); ++} ++ ++static bool a9_aop_force_stopped(struct apple_aop *aop) ++{ ++ return !(aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) && ++ !readl(aop->control) && ++ !(aop_rd(aop, AOP_OUTBOX_CONTROL) & (AOP_OUTBOX_ENABLE | AOP_MBOX_FAULTS)) && ++ !(aop_rd(aop, AOP_INBOX_CONTROL) & AOP_MBOX_FAULTS); ++} ++ ++struct aop_warm_write { void __iomem *target; u32 value; }; ++static void aop_warm_store(void *data) ++{ ++ struct aop_warm_write *w = data; ++ ++ writel(w->value, w->target); ++ readl(w->target); ++} ++ ++static int aop_wr(struct apple_aop *aop, u32 offset, u32 value) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct aop_warm_write w = { aop->warm_wrapper + offset, value }; ++ ++ if ((aop_rd(aop, AOP_INBOX_CONTROL) | aop_rd(aop, AOP_OUTBOX_CONTROL)) & ++ AOP_MBOX_FAULTS) ++ return -EIO; ++ dev_info(aop->dev, "COLD-W step=%s bank=wrapper offset=%04x before=%08x value=%08x\n", ++ aop->experiment_step ?: "warm-stop", offset, readl(w.target), value); ++ { ++ int ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, aop_warm_store, &w); ++ ++ dev_info(aop->dev, "COLD-R offset=%04x after=%08x\n", offset, readl(w.target)); ++ return ret; ++ } ++} ++ ++static int a9_aop_cold_checkpoint_noflush(struct apple_aop *aop, const char *step) ++{ ++ if (!a9_aop_ramoops_ready()) ++ return -ENODEV; ++ dev_info(aop->dev, "COLD checkpoint=%s flush=skipped\n", step); ++ return 0; ++} ++ ++static int a9_aop_cold_checkpoint(struct apple_aop *aop, const char *step) ++{ ++ int ret = a9_aop_cold_checkpoint_noflush(aop, step); ++ ++ if (ret) ++ return ret; ++ if (!pr_flush(1000, false)) ++ return -ETIMEDOUT; ++ mb(); ++ return 0; ++} ++ ++#include "aop-warm-spi0.c.h" ++static int apple_aop_send(struct apple_aop_ep *ep, u8 opcode, u64 payload); ++ ++/* No cmdline override for untested states: default is the release6 policy. */ ++static atomic_t aop_warm_attempted = ATOMIC_INIT(0); ++ ++static int apple_aop_warm_stop(struct apple_aop *aop) ++{ ++ unsigned int fifo, i; ++ u32 offset, val; ++ int ret; ++ ++ ret = aop_wr(aop, 0x4020, aop_rd(aop, 0x4020) & ~BIT(0)); ++ if (!ret) ++ ret = aop_wr(aop, 0x4000, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0xc00, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0x44, aop_rd(aop, 0x44) & ~BIT(4)); ++ if (ret) ++ return ret; ++ writel(0, aop->control); ++ msleep(10); ++ if (!a9_aop_force_stopped(aop)) ++ return -EIO; ++ aop->force_halted = true; ++ for (fifo = 0; fifo < 2; fifo++) { ++ offset = fifo ? 0x4020 : 0x4008; ++ for (i = 0; i < 64; i++) { ++ val = aop_rd(aop, offset); ++ if (!a9_aop_force_stopped(aop) || (val & AOP_MBOX_FAULTS)) ++ return -EIO; ++ if (val & BIT(17)) ++ break; ++ readq(aop->warm_wrapper + (fifo ? 0x4038 : 0x4018)); ++ } ++ if (!(aop_rd(aop, offset) & BIT(17))) ++ return -ETIMEDOUT; ++ } ++ aop->force_drained = true; ++ return 0; ++} ++ ++static void apple_aop_warm_put_node(void *node) ++{ ++ of_node_put(node); ++} ++ ++static void apple_aop_warm_unmap(void *p) ++{ ++ memunmap(p); ++} ++ ++static int apple_aop_warm_prepare(struct apple_aop *aop) ++{ ++ struct device *dev = aop->dev; ++ struct device_node *np = dev->of_node, *mem; ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct a9_sl_header header; ++ struct resource res; ++ char *text; ++ u32 auto_boot, outbox; ++ unsigned int i; ++ int ret; ++ ++ if (!of_property_read_bool(aop->dev->of_node, "apple,certified-reload") || !aop->hw->sram_window) ++ return 0; ++ /* Missing the new DT is not permission to run a retained instance. */ ++ mem = of_parse_phandle(np, "memory-region", 0); ++ if (!mem) ++ return dev_err_probe(dev, -EINVAL, "warm reload requires the matching J71t DT\n"); ++ ret = of_address_to_resource(mem, 0, &res); ++ of_node_put(mem); ++ if (ret || resource_size(&res) != A9_HO_PAGE || res.start != 0x8583fc000ULL || ++ aop->akf_base != A9_SPI0_WRAPPER_PA) ++ return -EINVAL; ++ aop->shutdown_log = memremap(res.start, A9_HO_PAGE, MEMREMAP_WB); ++ if (!aop->shutdown_log) ++ return -ENOMEM; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_unmap, aop->shutdown_log); ++ if (ret) ++ return ret; ++ /* Keep release6's transcript location and validation ABI. */ ++ aop->shutdown_log = (void *)aop->shutdown_log + A9_HO_LOG_OFFSET; ++ aop->shutdown_text = (char *)(aop->shutdown_log + 1); ++ memcpy(&header, aop->shutdown_log, sizeof(header)); ++ text = devm_kmemdup(dev, aop->shutdown_text, A9_SL_CAPACITY, GFP_KERNEL); ++ if (!text) ++ return -ENOMEM; ++ if (header.magic == A9_SL_MAGIC) { ++ aop->spi0.prev.state = A9_SPI0_PREV_INVALID; ++ if (a9_sl_valid(&header, text)) { ++ aop->spi0.prev.state = A9_SPI0_PREV_VALID; ++ a9_spi0_prev_parse(&aop->spi0.prev, text, header.length); ++ for (i = 0; i < header.length;) { ++ char *next = memchr(text + i, '\n', header.length - i); ++ unsigned int n = next ? next - text - i : header.length - i; ++ ++ dev_info(dev, "AOP-PREV %.*s\n", n, text + i); ++ i += n + 1; ++ } ++ } ++ } ++ aop->warm_wrapper = devm_ioremap(dev, aop->akf_base, 0x1c000); ++ if (!aop->warm_wrapper) ++ return -ENOMEM; ++ for (i = 0; i < AOP_DOMAINS; i++) { ++ aop->domains[i] = of_parse_phandle(np, "apple,observe-domains", i); ++ if (!aop->domains[i]) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->domains[i]); ++ if (ret) ++ return ret; ++ } ++ aop->spi0_pmgr = of_parse_phandle(np, "apple,spi0-domain", 0); ++ if (!aop->spi0_pmgr) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->spi0_pmgr); ++ if (ret) ++ return ret; ++ /* Check all domains before reading the wrapper; no power transitions. */ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, a9_aop_spi0_pmgr_on, &(bool){false}); ++ if (ret) ++ return dev_err_probe(dev, ret, "AOP domain guard refused\n"); ++ auto_boot = readl(aop->control); ++ outbox = aop_rd(aop, AOP_OUTBOX_CONTROL); ++ ret = a9_aop_spi0_policy(aop, auto_boot, outbox); ++ if (ret) ++ return ret; /* Before first halt, mask, SRAM or SPI0 write. */ ++ /* Orchard's old halt leaves +0x4020 enabled. On a known kexec, ++ * that alone cannot certify the cold wrapper baseline. Release6's ++ * wrapper discriminator was tied to its own outgoing stop routine. ++ */ ++ if (of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev) != A9_SPI0_PREV_HALTED) { ++ dev_info(dev, "SPI0 policy decision=refuse reason=uncertified-kexec auto_boot=%u out4020=%08x: block not touched\n", ++ auto_boot, outbox); ++ return -EPERM; ++ } ++ aop->warm = true; ++ aop->spi0.reason = a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev); ++ if (!a9_spi0_reason_runs(aop->spi0.reason)) { ++ /* Original cold path: no halt, drain, reset, or SPI0 access. */ ++ dev_info(dev, "SPI0 reset decision=skip reason=%s: block not touched\n", ++ a9_spi0_reason_name(aop->spi0.reason)); ++ return 0; ++ } ++ if (atomic_xchg(&aop_warm_attempted, 1)) ++ return dev_err_probe(dev, -EPERM, "refused-second-experiment-attempt\n"); ++ if (!a9_aop_ramoops_ready()) ++ return dev_err_probe(dev, -ENODEV, "warm reload requires ramoops console\n"); ++ aop->spi0.base = devm_ioremap(dev, A9_SPI0_PA, A9_SPI0_SIZE); ++ if (!aop->spi0.base) ++ return -ENOMEM; ++ /* Consume certification before the first control write. Reloading the ++ * module after a failure must not reuse it as a second attempt. ++ */ ++ aop->shutdown_log->magic = 0; ++ wmb(); ++ ret = apple_aop_warm_stop(aop); ++ if (!ret) ++ ret = a9_aop_spi0_prereset(aop, auto_boot, outbox); ++ if (ret) ++ dev_err(dev, "warm reload refused errno=%d; no copy/RUN, recover with DFU\n", ret); ++ return ret; ++} ++ ++static int apple_aop_warm_release(struct apple_aop *aop) ++{ ++ int ret; ++ ++ ret = a9_aop_spi0_recheck(aop); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-window"); ++ if (!ret) ++ ret = a9_aop_intc_ack(aop); ++ if (!ret) ++ ret = a9_aop_intc_reset(aop); ++ if (!ret) ++ ret = a9_aop_cold_checkpoint(aop, "before-run"); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "pre-run-final", "FINAL", true); ++ return ret; ++} ++ ++static void apple_aop_warm_hash(struct apple_aop *aop, u8 *digest) ++{ ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ if (aop->ext) ++ sha256_update(&ctx, aop->ext, aop->ext_size); ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) ++ if (aop->eps[i].buf) ++ sha256_update(&ctx, aop->eps[i].buf, aop->eps[i].buf_size); ++ sha256_final(&ctx, digest); ++} ++ ++static __printf(2, 3) void apple_aop_warm_log(struct apple_aop *aop, const char *fmt, ...) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ va_list args; ++ char line[512]; ++ size_t n; ++ ++ va_start(args, fmt); ++ n = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ dev_info(aop->dev, "%s", line); ++ if (n > h->capacity - h->length) { ++ h->truncated++; ++ return; ++ } ++ memcpy(aop->shutdown_text + h->length, line, n); ++ wmb(); ++ h->checksum = a9_sl_hash(h->checksum, line, n); ++ h->length += n; ++ h->lines++; ++ wmb(); ++} ++ ++static void apple_aop_warm_shutdown(struct apple_aop *aop) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ unsigned long deadline = jiffies + msecs_to_jiffies(10000), start; ++ unsigned int i, acked = 0; ++ int ret = -EIO; ++ ++ memset(h, 0, sizeof(*h)); ++ h->version = A9_SL_VERSION; ++ h->capacity = A9_SL_CAPACITY; ++ h->checksum = A9_SL_HASH_SEED; ++ h->mode = 1; ++ h->begin_ns = ktime_get_boottime_ns(); ++ wmb(); ++ h->magic = A9_SL_MAGIC; ++ apple_aop_warm_log(aop, "SHUTDOWN begin mode=quiesce hello=%d phase=stopping iop_crashed=%d\n", ++ aop->hello, aop->iop_crashed); ++ if (!aop->hello || aop->iop_crashed || aop->shutdown_error) ++ goto done; ++ /* Rates-zero has stopped reports. Stop one AFK at a time, max 2s ++ * per endpoint and 10s in total, including send time. Never force halt ++ * after a failed ACK. Keep every grant allocated for the entire hook. ++ */ ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ unsigned long limit; ++ ++ if (!ep->initialized) ++ continue; ++ if (time_after_eq(jiffies, deadline)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ reinit_completion(&ep->stopped); ++ start = jiffies; ++ ret = apple_aop_send(ep, AFK_OPC_SHUTDOWN, 0); ++ if (ret) ++ goto done; ++ limit = time_before(jiffies, deadline) ? min_t(unsigned long, ++ deadline - jiffies, msecs_to_jiffies(2000)) : 0; ++ if (!limit || !wait_for_completion_timeout(&ep->stopped, limit)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ acked++; ++ apple_aop_warm_log(aop, "QUIESCE step=afk-ack ep=0x%x state=stopped ms=%u\n", ++ ep->index, jiffies_to_msecs(jiffies - start)); ++ } ++ /* release6: complete every AFK stop before disabling endpoint routing. */ ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; ++ } ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ ++ if (!ep->initialized) ++ continue; ++ /* MGMT STARTEP bit 0 disables routing, no separate stop ACK. */ ++ ret = apple_rtkit_send_message(aop->rtk, 0, ++ (5ULL << 52) | ((u64)ep->index << 32) | 1, NULL, false); ++ if (ret) ++ goto done; ++ } ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; /* Never certify an incompletely initialized instance. */ ++ } ++ ret = apple_rtkit_shutdown_preserve(aop->rtk); ++ if (ret) ++ goto done; ++ msleep(20); ++ apple_aop_warm_hash(aop, before); ++ msleep(20); ++ apple_aop_warm_hash(aop, after); ++ ret = memcmp(before, after, sizeof(before)) ? -EBUSY : 0; ++ apple_aop_warm_log(aop, "QUIESCE step=ram-stable window_ms=20 status=%d\n", ret); ++ if (ret || aop->iop_crashed) ++ goto done; ++ ret = apple_aop_warm_stop(aop); ++done: ++ apple_aop_warm_log(aop, "QUIESCE step=complete status=%d quiesced=%d halted=%d drained=%d steps=0x%x afk_acks=%u iop_crashed=%d\n", ++ ret, !ret, aop->force_halted, aop->force_drained, ++ !ret ? 0x1bff : 0, acked, aop->iop_crashed); ++ if (ret) ++ dev_err(aop->dev, "quiesce failed; next AOP must refuse, recover with DFU\n"); ++} ++ + /* Firmware loading */ + + struct apple_aop_bootarg { +@@ -232,11 +692,17 @@ struct apple_aop_bootarg { + bool found; + }; + ++static void apple_aop_add_bootarg(struct apple_aop_bootarg *args, unsigned int *count, ++ u32 key, const void *value, u32 size) ++{ ++ args[(*count)++] = (struct apple_aop_bootarg){ .key = key, .value = value, .size = size }; ++} ++ + static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t size, +- struct apple_aop_bootarg *args, int count) ++ struct apple_aop_bootarg *args, unsigned int count) + { + size_t offset, end, pos; +- int i; ++ unsigned int i; + + offset = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_OFFSET); + end = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_SIZE); +@@ -269,13 +735,29 @@ static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t siz + return 0; + } + ++/* S8000: point the AOP's window at its SRAM; the firmware runs from there. */ ++static void apple_aop_map_sram(struct apple_aop *aop) ++{ ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_LO); ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_HI); ++ writel(aop->sram_size & ~0xfffULL, aop->akf + APPLE_AOP_REMAP_SIZE); ++ writel(0, aop->akf + APPLE_AOP_REMAP_RESERVED); ++ writel(lower_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_LO); ++ writel(upper_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_HI); ++ writel(1, aop->akf + APPLE_AOP_REMAP_ENABLE); ++} ++ + static int apple_aop_load(struct apple_aop *aop, u64 ecid) + { ++ struct apple_aop_bootarg args[APPLE_AOP_MAX_BOOTARGS]; + const struct firmware *main_fw, *ext_fw; + const char *main_name, *ext_name; + struct device_node *np = aop->dev->of_node; +- u8 prng[32], nonce[8]; +- __le64 ecid_le = cpu_to_le64(ecid), ext_le; ++ u8 prng[32], nonce[8], guard[4], ncal = 0; ++ __le64 ecid_le = cpu_to_le64(ecid), ext_le, wrapper_le; ++ __le32 target_le, chip_le, revision_le; ++ unsigned int count = 0; ++ size_t offset; + u8 *image; + int ret; + +@@ -316,28 +798,58 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + } + + get_random_bytes(prng, sizeof(prng)); +- get_random_bytes(nonce, sizeof(nonce)); +- { +- struct apple_aop_bootarg args[] = { +- { APPLE_AOP_BOOTARG_PRNG, prng, sizeof(prng) }, +- { APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le) }, +- { APPLE_AOP_BOOTARG_NONC, nonce, sizeof(nonce) }, +- { APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le) }, +- }; +- +- ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, +- ARRAY_SIZE(args)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_PRNG, &prng, sizeof(prng)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le)); ++ if (aop->hw->nonce) { ++ get_random_bytes(nonce, sizeof(nonce)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NONC, &nonce, sizeof(nonce)); ++ } ++ if (aop->hw->soc_bootargs) { ++ /* A random stack guard with one zero byte */ ++ get_random_bytes(guard, sizeof(guard)); ++ guard[get_random_u32_below(sizeof(guard))] = 0; ++ target_le = cpu_to_le32(aop->target); ++ chip_le = cpu_to_le32(aop->chip_id); ++ revision_le = cpu_to_le32(aop->chip_revision); ++ wrapper_le = cpu_to_le64(aop->akf_base); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_AOPT, ++ &target_le, sizeof(target_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NCAL, &ncal, sizeof(ncal)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_WRAD, ++ &wrapper_le, sizeof(wrapper_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOC, ++ &chip_le, sizeof(chip_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOCR, ++ &revision_le, sizeof(revision_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_STKG, &guard, sizeof(guard)); + } ++ ++ ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, count); + if (ret) { + dev_err(aop->dev, "unexpected boot arguments in %s: %d\n", main_name, ret); + } else { +- memcpy_toio(aop->sram, image, main_fw->size); +- memset_io(aop->sram + main_fw->size, 0, aop->sram_size - main_fw->size); ++ /* The SRAM takes 32-bit accesses, as Apple's loader makes them. */ ++ if (aop->spi0.ran) { ++ ret = a9_aop_spi0_gate(aop, "pre-copy-final", "PRECOPY", false); ++ if (!ret) ++ ret = a9_aop_copy_verify_wake10(aop, image, main_fw->size); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "after-copy", "AFTERCOPY", true); ++ if (ret) ++ goto out_image; ++ } else { ++ __iowrite32_copy(aop->sram, image, main_fw->size / 4); ++ } ++ for (offset = main_fw->size; offset + 4 <= aop->sram_size; offset += 4) ++ writel_relaxed(0, aop->sram + offset); + } + ++out_image: + kfree_sensitive(image); + memzero_explicit(prng, sizeof(prng)); + memzero_explicit(nonce, sizeof(nonce)); ++ memzero_explicit(guard, sizeof(guard)); + memzero_explicit(&ecid_le, sizeof(ecid_le)); + out_ext: + release_firmware(ext_fw); +@@ -448,7 +960,8 @@ static void apple_aop_announce(struct apple_aop_ep *ep, u32 channel, const void + const struct epic_service_announce *announce = data; + struct apple_aop_register_work *rw; + +- if (len < sizeof(*announce)) ++ /* Only the name and channel are used; the S8000's announcements are shorter. */ ++ if (len < offsetofend(struct epic_service_announce, channel)) + return; + + rw = kzalloc_obj(*rw); +@@ -475,6 +988,8 @@ static void apple_aop_report(struct apple_aop_ep *ep, u32 channel, u16 subtype, + struct apple_aop_listener *listener; + bool handled = false; + ++ if (ep->index == 0x21 && subtype == 0xc2) ++ aop->accel_reports++; + mutex_lock(&aop->listeners_lock); + list_for_each_entry(listener, &aop->listeners, list) { + if (listener->endpoint == ep->index && listener->channel == channel) { +@@ -631,6 +1146,7 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + + switch (FIELD_GET(AFK_MSG_OPCODE, msg)) { + case AFK_OPC_INIT: ++ ep->initialized = true; + apple_aop_send(ep, AFK_OPC_INIT_ACK, 0); + break; + case AFK_OPC_GET_BUF: +@@ -651,7 +1167,10 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + break; + case AFK_OPC_INIT_UNK: + case AFK_OPC_START_ACK: ++ break; + case AFK_OPC_SHUTDOWN_ACK: ++ case AFK_OPC_HIBERNATE_ACK: ++ complete(&ep->stopped); + break; + default: + dev_dbg(aop->dev, "endpoint 0x%02x: message 0x%llx\n", endpoint, msg); +@@ -662,7 +1181,8 @@ static void apple_aop_crashed(void *cookie, const void *crashlog, size_t crashlo + { + struct apple_aop *aop = cookie; + +- dev_err(aop->dev, "firmware crashed\n"); ++ aop->iop_crashed = true; ++ dev_err(aop->dev, "firmware crashed iop_crashed=1\n"); + } + + static const struct apple_rtkit_ops apple_aop_rtkit_ops = { +@@ -767,6 +1287,8 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + + if (!ep) + return -EINVAL; ++ if (service->aop->warm && READ_ONCE(service->aop->iop_crashed)) ++ return -EIO; + init_completion(&call.done); + + mutex_lock(&ep->lock); +@@ -797,6 +1319,58 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + } + EXPORT_SYMBOL_GPL(apple_aop_call); + ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len) ++{ ++ u8 buf[8 + APPLE_AOP_PROPERTY_MAX]; ++ int ret; ++ ++ if (len > APPLE_AOP_PROPERTY_MAX) ++ return -EINVAL; ++ put_unaligned_le32(0, buf); ++ put_unaligned_le32(key, buf + 4); ++ memcpy(buf + 8, value, len); ++ ++ ret = apple_aop_call(service, APPLE_AOP_SET_PROPERTY, buf, 8 + len, NULL, 0); ++ return ret > 0 ? -EIO : ret; ++} ++EXPORT_SYMBOL_GPL(apple_aop_set_property); ++ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count) ++{ ++ unsigned long needed = 0, applied = 0; ++ __le32 state; ++ unsigned int i; ++ int len, ret; ++ ++ for (i = 0; i < count; i++) { ++ const void *blob; ++ ++ if (cal[i].group) ++ needed |= BIT(cal[i].group); ++ blob = of_get_property(dev->of_node, cal[i].property, &len); ++ if (!blob) ++ continue; ++ ret = apple_aop_set_property(service, cal[i].key, blob, len); ++ if (ret) { ++ dev_warn(dev, "%s not applied: %d\n", cal[i].property, ret); ++ continue; ++ } ++ if (cal[i].group) ++ applied |= BIT(cal[i].group); ++ } ++ ++ if (needed && applied == needed) { ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_COMPLETE); ++ } else { ++ dev_warn(dev, "factory calibration incomplete\n"); ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_PARTIAL); ++ } ++ return apple_aop_set_property(service, APPLE_AOP_PROP_CALIBRATION, &state, sizeof(state)); ++} ++EXPORT_SYMBOL_GPL(apple_aop_calibrate); ++ + /* Driver */ + + static void apple_aop_free_buffers(struct apple_aop *aop) +@@ -825,27 +1399,35 @@ static int apple_aop_probe(struct platform_device *pdev) + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); + mutex_init(&aop->children_lock); + INIT_LIST_HEAD(&aop->children); + +- aop->akf = devm_platform_ioremap_resource_byname(pdev, "akf"); ++ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "akf"); ++ aop->akf = devm_ioremap_resource(dev, res); + if (IS_ERR(aop->akf)) + return PTR_ERR(aop->akf); ++ aop->akf_base = res->start; + aop->control = devm_platform_ioremap_resource_byname(pdev, "control"); + if (IS_ERR(aop->control)) + return PTR_ERR(aop->control); + aop->sram = devm_platform_get_and_ioremap_resource(pdev, 2, &res); + if (IS_ERR(aop->sram)) + return PTR_ERR(aop->sram); ++ aop->sram_base = res->start; + aop->sram_size = resource_size(res); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + +- if (apple_aop_is_started(aop)) ++ ret = apple_aop_warm_prepare(aop); ++ if (ret) ++ return ret; ++ ++ if (!aop->spi0.ran && apple_aop_is_started(aop)) + return dev_err_probe(dev, -EBUSY, + "already started; the AOP starts once per boot\n"); + +@@ -853,6 +1435,17 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + return dev_err_probe(dev, ret, "no apple,ecid from the bootloader\n"); + ++ if (aop->hw->soc_bootargs) { ++ ret = of_property_read_u32(np, "apple,aop-target", &aop->target); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-id", &aop->chip_id); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-revision", &aop->chip_revision); ++ if (ret) ++ return dev_err_probe(dev, ret, ++ "no AOP target or chip ID from the bootloader\n"); ++ } ++ + aop->eps = devm_kcalloc(dev, AFK_ENDPOINT_COUNT, sizeof(*aop->eps), GFP_KERNEL); + if (!aop->eps) + return -ENOMEM; +@@ -860,6 +1453,7 @@ static int apple_aop_probe(struct platform_device *pdev) + aop->eps[i].aop = aop; + aop->eps[i].index = AFK_ENDPOINT_START + i; + mutex_init(&aop->eps[i].lock); ++ init_completion(&aop->eps[i].stopped); + } + + ret = apple_aop_load(aop, ecid); +@@ -867,6 +1461,9 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + goto err_free; + ++ if (aop->hw->sram_window) ++ apple_aop_map_sram(aop); ++ + aop->rtk = apple_rtkit_init(dev, aop, "mbox", 0, &apple_aop_rtkit_ops); + if (IS_ERR(aop->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(aop->rtk), "failed to init RTKit\n"); +@@ -884,10 +1481,24 @@ static int apple_aop_probe(struct platform_device *pdev) + prop = of_find_property(np, "apple,ecid", NULL); + if (prop) + of_remove_property(np, prop); +- writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, +- aop->akf + APPLE_AOP_CPU_CONTROL); ++ if (aop->spi0.ran) { ++ ret = apple_aop_warm_release(aop); ++ if (ret) ++ goto err_halt; ++ } ++ if (aop->spi0.ran) { ++ aop->experiment_step = "warm-run"; ++ ret = aop_wr(aop, APPLE_AOP_CPU_CONTROL, ++ readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN); ++ if (ret) ++ goto err_halt; ++ } else { ++ writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, ++ aop->akf + APPLE_AOP_CPU_CONTROL); ++ } + writel(1, aop->control + APPLE_AOP_CONTROL_AUTO_BOOT); + ++ dev_info(dev, "AOP released warm=%d\n", aop->spi0.ran); + ret = apple_rtkit_wake(aop->rtk); + if (ret) { + dev_err(dev, "firmware did not boot: %d\n", ret); +@@ -900,6 +1511,8 @@ static int apple_aop_probe(struct platform_device *pdev) + dev_warn(dev, "failed to start endpoint 0x%02x: %d\n", i, ret); + } + ++ aop->hello = true; ++ dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); + return 0; + +@@ -929,27 +1542,81 @@ static void apple_aop_shutdown(struct platform_device *pdev) + mutex_lock(&aop->children_lock); + aop->removing = true; + list_for_each_entry_safe(child, tmp, &aop->children, list) { ++ ++ if (aop->warm) { ++ const struct apple_aop_service *service = dev_get_platdata(&child->pdev->dev); ++ static const u32 keys[] = { 0x16, 0x5c, 0 }; ++ ++ /* release6 sets the three accelerometer rates only. */ ++ if (strcmp(service->name, "accel")) ++ continue; ++ __le32 zero = 0; ++ unsigned int i; ++ int ret; ++ ++ for (i = 0; i < ARRAY_SIZE(keys); i++) { ++ ret = apple_aop_set_property(service, keys[i], &zero, sizeof(zero)); ++ if (ret) { ++ aop->shutdown_error = ret; ++ break; ++ } ++ } ++ } ++ /* Shutdown retains listeners, children and DMA grants, as release6 does. ++ * Unregister is remove-time work and can wait in IIO/driver core. ++ */ ++ if (aop->warm) ++ continue; + platform_device_unregister(child->pdev); + list_del(&child->list); + kfree(child); + } + mutex_unlock(&aop->children_lock); + ++ if (aop->warm) { ++ apple_aop_warm_shutdown(aop); ++ return; ++ } + if (!apple_aop_halt(aop)) + dev_err(aop->dev, "failed to halt\n"); + } + ++static const struct apple_aop_hw apple_aop_s8000_hw = { ++ .sram_window = true, ++ .soc_bootargs = true, ++}; ++ ++static const struct apple_aop_hw apple_aop_t8010_hw = { ++ .nonce = true, ++}; ++ + static const struct of_device_id apple_aop_of_match[] = { +- { .compatible = "apple,t8010-aop" }, ++ { .compatible = "apple,s8000-aop", .data = &apple_aop_s8000_hw }, ++ { .compatible = "apple,t8010-aop", .data = &apple_aop_t8010_hw }, + {} + }; + MODULE_DEVICE_TABLE(of, apple_aop_of_match); + ++static ssize_t warm_state_show(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ struct apple_aop *aop = dev_get_drvdata(dev); ++ ++ if (!aop) ++ return sysfs_emit(buf, "owned=0\n"); ++ return sysfs_emit(buf, "owned=%u reload=%u iop_crashed=%u accel_reports=%llu\n", ++ aop->hello, aop->spi0.ran, READ_ONCE(aop->iop_crashed), ++ READ_ONCE(aop->accel_reports)); ++} ++static DEVICE_ATTR_RO(warm_state); ++static struct attribute *apple_aop_attrs[] = { &dev_attr_warm_state.attr, NULL }; ++ATTRIBUTE_GROUPS(apple_aop); ++ + static struct platform_driver apple_aop_driver = { + .driver = { + .name = "apple-aop", + .of_match_table = apple_aop_of_match, + .suppress_bind_attrs = true, ++ .dev_groups = apple_aop_groups, + }, + .probe = apple_aop_probe, + .shutdown = apple_aop_shutdown, +@@ -957,5 +1624,5 @@ static struct platform_driver apple_aop_driver = { + module_platform_driver(apple_aop_driver); + + MODULE_AUTHOR("Dalton Alexandre "); +-MODULE_DESCRIPTION("Apple T8010 Always-On Processor driver"); ++MODULE_DESCRIPTION("Apple S8000 and T8010 Always-On Processor driver"); + MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c +index 725ab3fa7..d7928621a 100644 +--- a/drivers/soc/apple/rtkit.c ++++ b/drivers/soc/apple/rtkit.c +@@ -4,6 +4,8 @@ + * Copyright (C) The Asahi Linux Contributors + */ + ++#include ++#include + #include "rtkit-internal.h" + + enum { +@@ -867,6 +869,54 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) + } + EXPORT_SYMBOL_GPL(apple_rtkit_boot); + ++/* Unlike shutdown(), retain all grants until the caller halts the CPU. ++ * Reinit frees DMA buffers and is unsafe before that point on S8000. ++ */ ++static void apple_rtkit_grants_hash(struct apple_rtkit *rtk, u8 *digest) ++{ ++ struct apple_rtkit_shmem *buffers[] = { ++ &rtk->ioreport_buffer, &rtk->crashlog_buffer, ++ &rtk->oslog_buffer, &rtk->syslog_buffer, ++ }; ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ for (i = 0; i < ARRAY_SIZE(buffers); i++) ++ if (buffers[i]->buffer) ++ sha256_update(&ctx, buffers[i]->buffer, buffers[i]->size); ++ sha256_final(&ctx, digest); ++} ++ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk) ++{ ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ if (rtk->crashed) ++ return -EIO; ++ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_QUIESCED); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=ap-ack state=0x10\n"); ++ ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=iop-ack state=0x201\n"); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, before); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, after); ++ if (rtk->crashed || memcmp(before, after, sizeof(before))) ++ return -EBUSY; ++ /* No more callback can race the caller's halt/drain. Do not reinit. */ ++ apple_mbox_stop(rtk->mbox); ++ flush_workqueue(rtk->wq); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_rtkit_shutdown_preserve); ++ + int apple_rtkit_shutdown(struct apple_rtkit *rtk) + { + int ret; +diff --git a/include/linux/soc/apple/aop.h b/include/linux/soc/apple/aop.h +index 8c8bc71b6..b5188a0b1 100644 +--- a/include/linux/soc/apple/aop.h ++++ b/include/linux/soc/apple/aop.h +@@ -7,6 +7,8 @@ + + #include + ++struct device; ++ + struct apple_aop; + + /* +@@ -40,4 +42,30 @@ void apple_aop_unlisten(const struct apple_aop_service *service); + int apple_aop_call(const struct apple_aop_service *service, u16 subtype, + const void *data, size_t len, void *reply, size_t reply_len); + ++/* The EPIC notification subtype that sets a service property */ ++#define APPLE_AOP_SET_PROPERTY 0x4 ++ ++/* Set property @key of a service to @len (at most 64) bytes at @value. */ ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len); ++ ++/* ++ * A factory calibration blob of a sensor: the bootloader copies it from the ++ * Apple device tree into the sensor's node as @property, and the firmware ++ * takes it as property @key. The calibration is complete once a blob of each ++ * @group (1 to 31) was applied; group 0 blobs are not needed for that. ++ */ ++struct apple_aop_calibration { ++ const char *property; ++ u32 key; ++ u8 group; ++}; ++ ++/* ++ * Send the factory calibration found in @dev's node, then tell the firmware ++ * whether it is complete. Some firmware (S8000) sends no reports before. ++ */ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count); ++ + #endif /* _LINUX_SOC_APPLE_AOP_H_ */ +diff --git a/include/linux/soc/apple/pmgr-observe.h b/include/linux/soc/apple/pmgr-observe.h +new file mode 100644 +index 000000000..477b7fa22 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-observe.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++#define _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++ ++#include ++ ++struct device_node; ++ ++struct apple_pmgr_observation { ++ u32 raw; ++ int genpd_status; ++}; ++ ++/* ++ * Diagnostic only. Nodes must be in child-before-parent lock order, with no ++ * duplicates (maximum five). Takes no power/runtime-PM reference. read() must ++ * not sleep or acquire genpd locks; it runs with IRQs disabled and all domain ++ * locks held, only if every genpd and hardware actual/target state is ON and ++ * reset/disable/parent-off are clear. Hardware auto-PM remains untouched and is ++ * recorded in raw; locks exclude software transitions, not autonomous gating. ++ * -EHOSTDOWN means read() was skipped. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data); ++ ++#endif +diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h +index 1f9ebb618..c8e8df255 100644 +--- a/include/linux/soc/apple/rtkit.h ++++ b/include/linux/soc/apple/rtkit.h +@@ -124,6 +124,8 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); + * Shutdown the co-processor + */ + int apple_rtkit_shutdown(struct apple_rtkit *rtk); ++/* AP 0x10 / IOP 0x201, stable system RAM, stop RX; retain every grant. */ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk); + + /* + * Stop the mailbox of a shut-down co-processor, so that its power domain can diff --git a/kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch b/kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch new file mode 100644 index 00000000..8a4e6613 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch @@ -0,0 +1,80 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] soc: apple: aop: refuse pre-sync sleep during firmware boot + +A suspend while the AOP firmware is still starting stops it part way +through its boot, and the sensors behind it do not come back. Register a +suspend precheck notifier that refuses sleep with -EBUSY until the +firmware has reported ready. + +Assisted-by: LLM +--- + drivers/soc/apple/aop.c | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -226,6 +227,8 @@ + }; + + struct apple_aop { ++ struct notifier_block sleep_precheck; ++ bool sleep_ready; + struct device *dev; + const struct apple_aop_hw *hw; + void __iomem *akf; +@@ -1385,6 +1388,22 @@ + dma_free_coherent(aop->dev, aop->ext_size, aop->ext, aop->ext_dma); + } + ++static int apple_aop_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_aop *aop = container_of(nb, struct apple_aop, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ return READ_ONCE(aop->sleep_ready) ? NOTIFY_OK : notifier_from_errno(-EBUSY); ++} ++ ++static void apple_aop_sleep_unregister(void *data) ++{ ++ struct apple_aop *aop = data; ++ ++ unregister_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++} ++ + static int apple_aop_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -1399,6 +1418,13 @@ + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->sleep_precheck.notifier_call = apple_aop_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++ if (ret) ++ return ret; ++ ret = devm_add_action_or_reset(dev, apple_aop_sleep_unregister, aop); ++ if (ret) ++ return ret; + aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); +@@ -1514,6 +1540,7 @@ + aop->hello = true; + dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); ++ WRITE_ONCE(aop->sleep_ready, true); + return 0; + + err_halt: diff --git a/tests/test_j71t_data.py b/tests/test_j71t_data.py new file mode 100644 index 00000000..c6d83c08 --- /dev/null +++ b/tests/test_j71t_data.py @@ -0,0 +1,31 @@ +"""J71t firmware recipe: the AOP images.""" +import json +from pathlib import Path +import tomllib +import unittest + +HARDWARE = Path(__file__).resolve().parents[1] +DEVICE = HARDWARE / 'devices/j71t/device.toml' + + +class J71tData(unittest.TestCase): + def setUp(self): + device = tomllib.loads(DEVICE.read_text()) + self.recipe = json.loads((DEVICE.parent / device['firmware']['recipe']).read_text()) + + def test_aop_images_come_from_the_board_firmware(self): + aop = self.recipe['inputs']['aop.im4p'] + self.assertEqual(aop['ipsw'], {'zip': 'Firmware/AOP/aopfw-ipad6faop.im4p'}) + self.assertEqual(aop['sha256'], '5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa') + for name, role, digest in ( + ('aop-ipad6f.bin', 'AOP', 'af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8'), + ('aop-ipad6f-ext.bin', 'AOP_ext', '2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22')): + output, = [o for o in self.recipe['outputs'] if o['destination'] == 'lib/firmware/apple/' + name] + self.assertEqual(output['sha256'], digest) + self.assertEqual(output['steps'], [{'step': 'im4p-payload@1', 'input': 'aop.im4p'}, + {'step': 'aop-image@1', 'coprocessor': role}]) + self.assertFalse((HARDWARE / output['destination']).exists()) + + +if __name__ == '__main__': + unittest.main() From bbe62ba5b96388d62a920dd474d9ed3ff29dacf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 17/33] usb: Keep per-device system and RAM serials stable across boots The USB serial number changed between installed boots and RAM sessions, so the host saw a new device every boot and could not keep its SSH known_hosts entry or serial console name. Share one BOARD/hashed-ID serial format between installed boots and RAM sessions (usb-identity), and get the per-boot SSH identity through the existing ACM console. Documented in docs/device-protocol.md, with tests. Assisted-by: LLM --- docs/device-protocol.md | 18 +++ pyproject.toml | 2 +- src/orchard/image/platform.py | 30 ++++- src/orchard/ram_session.py | 43 +++++-- src/orchard/usb-identity | 19 +++ src/orchard/usb-session | 2 +- tests/test_identity.py | 73 ++++++++++++ tests/test_j71t_mac_current.py | 204 +++++++++++++++++++++++++++++++++ tests/test_platform_layer.py | 3 + tests/test_ram.py | 2 + tests/test_system_session.py | 57 +++++++++ 11 files changed, 436 insertions(+), 17 deletions(-) create mode 100644 src/orchard/usb-identity create mode 100644 tests/test_j71t_mac_current.py diff --git a/docs/device-protocol.md b/docs/device-protocol.md index f15bfac0..518087d8 100644 --- a/docs/device-protocol.md +++ b/docs/device-protocol.md @@ -15,6 +15,24 @@ the host and device side of an install always come from the same CLI version. - Private bundle: exactly the device firmware data plus `mixin.json`; the install script writes it to the installed root's `/usr/lib/firmware`. +## USB identity + +Installed-system and standard RAM gadgets use `BOARD: ID:` as +their USB serial number. The ID is the first 12 hex characters of SHA-256 over +`orchard-` followed by the device-tree `serial-number` without NUL +padding. It identifies the unit across boots without publishing the hardware +serial. No descriptor changes per boot, so macOS can retain accessory approval. +Trees without a serial number warn and retain the legacy `BOOT:` identity to +keep rescue USB available; those still need approval each boot. + +For RAM sessions, ACM returns `/proc/sys/kernel/random/boot_id` and the SSH +host public key together on every connection. The host validates the boot ID +and pins that boot's key. Installed systems use their persistent SSH keys and +do not allocate per-boot host state. Recovery and install keep their existing +`ECID:`/`BOOT:` serials and bootstrap/status checks. The install-to-system +handoff also retains `BOOT:`: the CLI's install add-on can run on older initrds +without the stable-identity helper. Normal installed boots use the new format. + ## Operations `orchard-device OPERATION [BASE64_JSON]`; requests may include `protocol: 1`. diff --git a/pyproject.toml b/pyproject.toml index f48aa158..ad17df07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ package-dir = {"" = "src"} where = ["src"] [tool.setuptools.package-data] -orchard = ["py.typed", "usb-session"] +orchard = ["py.typed", "usb-session", "usb-identity"] [dependency-groups] dev = ["mypy==1.18.2", "jsonschema>=4.25,<5"] diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index b39ae7bb..9985c837 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -85,7 +85,7 @@ def tools(hardware=None, cache=None, platform='apple-16k'): return prepare_tools(hardware, cache, device=sorted(boards(platform, hardware))[0]) / 'busybox' -def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES): +def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES, pristine_dtbs=None): """orchard's initrd for installed systems: BusyBox, the storage modules and their dependencies. MODULES is {path: (data, mode)} as module_files gives it. @@ -111,10 +111,20 @@ def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_ entries = {name: (directory, b'') for name in ('bin', 'dev', 'proc', 'sys', 'etc', 'etc/orchard', 'newroot', 'lib', 'lib/modules', 'lib/modules/' + release)} entries['init'] = (stat.S_IFREG | 0o755, (hardware / 'boot/initramfs/root/init').read_bytes()) + entries['etc/orchard/usb-identity'] = (stat.S_IFREG | 0o644, (hardware / 'src/orchard/usb-identity').read_bytes()) entries['bin/busybox'] = (stat.S_IFREG | 0o755, Path(busybox).read_bytes()) entries['etc/orchard/modules'] = (stat.S_IFREG | 0o644, ('\n'.join(loaded) + '\n').encode()) if usb: entries['etc/orchard/usb-modules'] = (stat.S_IFREG | 0o644, ('\n'.join(usb) + '\n').encode()) + if pristine_dtbs: + entries['etc/orchard/dtbs'] = (directory, b'') + mapping = [] + for board, data in sorted(pristine_dtbs.items()): + if not board.replace('-', '').isalnum(): + raise ValueError('Invalid pristine DT board') + entries[f'etc/orchard/dtbs/{board}.dtb'] = (stat.S_IFREG | 0o644, data) + mapping.append(f'apple,{board} {board}.dtb\n') + entries['etc/orchard/pristine-map'] = (stat.S_IFREG | 0o644, ''.join(mapping).encode()) dependency_lines = [] for name in sorted(wanted): target = f'lib/modules/{release}/{name}' @@ -143,6 +153,19 @@ def storage_modules(hardware, platform): return tuple(modules) + +def device_features(hardware, boards): + """Public board data for runtime policy, matched against the live DT compatible.""" + from .. import hardware as catalog + records = [] + for board in sorted(boards): + described = catalog.device(board, root=hardware) + records.append({'compatible': described['device'].get('dt_compatible', []), + 'features': described.get('features', {})}) + return {'usr/lib/orchard/device-features.json': + ((json.dumps(records, sort_keys=True) + '\n').encode(), 0o644)} + + def files(bundle, platform, board_dtbs, *, busybox, hardware=None): """(kernel release, {path: (data, mode)}) of PLATFORM's layer from a staged kernel BUNDLE.""" hardware = Path(hardware or ROOT) @@ -153,14 +176,15 @@ def files(bundle, platform, board_dtbs, *, busybox, hardware=None): base = f'usr/lib/modules/{release}/' result[base + 'vmlinuz'] = ((bundle / described['image']).read_bytes(), 0o644) modules_to_load = storage_modules(hardware, platform) - result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, - storage_modules=modules_to_load), 0o644) for dtb in sorted(set(board_dtbs.values())): path = bundle / 'dtbs' / (dtb + '.dtb') if path.is_symlink() or not path.is_file(): raise ValueError(f'Kernel bundle lacks {path.name}') result[base + 'dtb/' + path.name] = (path.read_bytes(), 0o644) + pristine = {board: result[base + 'dtb/' + dtb + '.dtb'][0] for board, dtb in board_dtbs.items()} + result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, pristine_dtbs=pristine, storage_modules=modules_to_load), 0o644) result.update(regulatory_files()) + result.update(device_features(hardware, board_dtbs)) defaults = hardware / 'system/defaults' for path in sorted(defaults.rglob('*')): if path.is_symlink(): diff --git a/src/orchard/ram_session.py b/src/orchard/ram_session.py index 1d1439ba..26c58889 100644 --- a/src/orchard/ram_session.py +++ b/src/orchard/ram_session.py @@ -18,6 +18,7 @@ import subprocess import sys import time +import uuid from .serial import SerialError, SerialPort from .state import cache_root, private_directory @@ -49,7 +50,9 @@ def identity(): def payload_files(): """RAM-root files that enable the USB session in a payload.""" helper = resources.files('orchard') / 'usb-session' - return {HELPER: Path(str(helper)), AUTHORIZED: identity().with_suffix('.pub')} + return {HELPER: Path(str(helper)), + 'usr/libexec/orchard/usb-identity': Path(str(resources.files('orchard') / 'usb-identity')), + AUTHORIZED: identity().with_suffix('.pub')} def target(): @@ -69,17 +72,20 @@ def target(): def _check(device): + # Stable descriptors identify the unit, not its boot. RAM SSH is pinned to + # the per-boot key read on the open ACM endpoint; a reboot invalidates it. + fields = ('location', 'mode', 'profile', 'usb_id', 'boot_id', 'serial_port') def check(): from .apple import usb - if not any(item['location'] == device['location'] and item.get('boot_id') == device.get('boot_id') + if not any(all(item.get(key) == device.get(key) for key in fields) for item in usb.devices()): raise ValueError('The RAM payload disconnected or rebooted') return check def connect(device=None, user=None, *, host=None, ram=False): - """SSH argv for the running payload (its host key pinned over serial once - per boot) or installed system (as its user, with this computer's key).""" + """SSH argv for the running payload (its boot and host key read over ACM) + or installed system (as its user, with this computer's key).""" if ram and (not host or user): raise ValueError('--ram requires --host and connects as root, not --user') if host: @@ -106,20 +112,33 @@ def connect(device=None, user=None, *, host=None, ram=False): argv = computer_key.ssh_argv(user) wait_authenticated(argv, _check(device), seconds=20) return argv - boot = device.get('boot_id') - if not boot: + if not (device.get('boot_id') or device.get('usb_id')): raise ValueError('The RAM payload has no boot identity; rebuild it with a current orchard') check = _check(device) + # Query every connection: the same stable USB serial can name a new boot. + # ACM supplies both boot ID and trusted host key without changing any USB + # descriptor, and also works with payloads built before stable identities. + with SerialPort(device['serial_port'], check) as serial: + code, text = serial.run('cat /proc/sys/kernel/random/boot_id && ' + 'cat /run/orchard/ssh_host_ed25519_key.pub', timeout=15) + if code: + raise ValueError('No SSH in this payload; its /init must run ' + '/' + HELPER + ' (or build without transport = false)') + lines = text.strip().splitlines() + try: + boot = str(uuid.UUID(lines[0].strip())) + except (ValueError, IndexError) as error: + raise ValueError('Invalid RAM boot identity over USB serial') from error + if device.get('boot_id') and boot != device['boot_id']: + raise ValueError('The RAM payload disconnected or rebooted') + pinned = HOST + ' ' + host_key('\n'.join(lines[1:])) + '\n' hosts = _directory() / f'known_hosts-{boot}' + if hosts.is_file() and hosts.read_text() != pinned: + raise ValueError('The RAM SSH host key changed within the same boot') if not hosts.is_file(): - with SerialPort(device['serial_port'], check) as serial: - code, text = serial.run('cat /run/orchard/ssh_host_ed25519_key.pub', timeout=15) - if code: - raise ValueError('No SSH in this payload; its /init must run ' - '/' + HELPER + ' (or build without transport = false)') for old in _directory().glob('known_hosts-*'): old.unlink() - hosts.write_text(HOST + ' ' + host_key(text) + '\n') + hosts.write_text(pinned) hosts.chmod(0o600) argv = ssh_argv(identity(), hosts, HOST, 22) wait_authenticated(argv, check, seconds=20) diff --git a/src/orchard/usb-identity b/src/orchard/usb-identity new file mode 100644 index 00000000..b6962ec8 --- /dev/null +++ b/src/orchard/usb-identity @@ -0,0 +1,19 @@ +#!/bin/sh +# Stable USB identity, shared by installed-system and RAM gadgets. The boot +# stage copies serial-number from Apple's tree; never publish it in the clear. +set -eu +tree=${ORCHARD_DEVICE_TREE:-/proc/device-tree} +board=$(tr '\000' '\n' < "$tree/compatible" | head -1) +serial= +if [ -r "$tree/serial-number" ]; then + serial=$(tr -d '\000' < "$tree/serial-number") +fi +if [ -n "$serial" ]; then + id=$(printf 'orchard-%s' "$serial" | sha256sum | cut -c1-12) + printf 'BOARD:%s ID:%s\n' "${board#apple,}" "$id" +else + # Keep rescue serial and SSH reachable on trees without a hardware ID. + # This legacy fallback cannot retain the host's accessory approval. + echo 'orchard: no device-tree serial-number; using per-boot USB identity' >&2 + printf 'BOOT:%s BOARD:%s\n' "$(cat /proc/sys/kernel/random/boot_id)" "${board#apple,}" +fi diff --git a/src/orchard/usb-session b/src/orchard/usb-session index aadde6a1..94c3e081 100755 --- a/src/orchard/usb-session +++ b/src/orchard/usb-session @@ -17,7 +17,7 @@ echo 0x05ac > "$g/idVendor" echo 0x4144 > "$g/idProduct" mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" echo "Orchard $name" > "$g/strings/0x409/product" -echo "BOOT:$(cat /proc/sys/kernel/random/boot_id)" > "$g/strings/0x409/serialnumber" +sh /usr/libexec/orchard/usb-identity > "$g/strings/0x409/serialnumber" echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" # Descriptor only: not charging capability or PMIC policy. echo 100 > "$g/configs/c.1/MaxPower" diff --git a/tests/test_identity.py b/tests/test_identity.py index 514c2d9e..5a9524cc 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -1,9 +1,12 @@ """Boot identity publication from bounded ADT fixtures; no device activation.""" import importlib.util +import hashlib import json import os from pathlib import Path +import re import stat +import subprocess import struct import sys import tempfile @@ -107,5 +110,75 @@ def test_gadget_publishes_identity_before_binding_usb(self): self.assertLess(source.index('/serialnumber'),source.index('"$g/UDC"')) +class GadgetIdentity(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.tree = self.root / 'proc/device-tree' + self.tree.mkdir(parents=True) + (self.tree / 'compatible').write_bytes(b'apple,j71t\0apple,s8003\0') + boot = self.root / 'proc/sys/kernel/random/boot_id' + boot.parent.mkdir(parents=True) + boot.write_text(BOOT) + self.script = (ROOT / 'src/orchard/usb-identity').read_text().replace( + '/proc/sys/kernel/random/boot_id', str(boot)) + + def serial(self): + return subprocess.run(['sh', '-ec', self.script], check=True, capture_output=True, text=True, + env={**os.environ, 'ORCHARD_DEVICE_TREE': str(self.tree)}) + + def test_hardware_serial_is_hashed_stable_across_boots_and_distinct_per_unit(self): + number = self.tree / 'serial-number' + number.write_bytes(b'HardwareSerialA\0') + first = self.serial() + expected = hashlib.sha256(b'orchard-HardwareSerialA').hexdigest()[:12] + self.assertEqual(first.stdout.strip(), f'BOARD:j71t ID:{expected}') + self.assertNotIn('HardwareSerialA', first.stdout + first.stderr) + (self.root / 'proc/sys/kernel/random/boot_id').write_text('22222222-2222-4222-8222-222222222222') + self.assertEqual(self.serial().stdout, first.stdout) + number.write_bytes(b'HardwareSerialB\0') + self.assertNotEqual(self.serial().stdout, first.stdout) + + def test_missing_or_empty_serial_preserves_rescue_usb_with_a_warning(self): + for value in (None, b'', b'\0'): + if value is not None: + (self.tree / 'serial-number').write_bytes(value) + result = self.serial() + self.assertEqual(result.stdout.strip(), f'BOOT:{BOOT} BOARD:j71t') + self.assertIn('no device-tree serial-number', result.stderr) + + def test_installed_and_ram_gadgets_use_the_same_stable_serial_before_binding(self): + (self.tree / 'serial-number').write_bytes(b'HardwareSerialA\0') + expected = self.serial().stdout.strip() + for relative in ('boot/initramfs/root/init', 'src/orchard/usb-session'): + with self.subTest(path=relative): + source = (ROOT / relative).read_text() + if relative.endswith('/init'): + start = source.index('usb_gadget() {') + source = source[start:source.index('\nusb_gadget ||', start)] + '\nusb_gadget' + else: + source = source[source.index('name=$(cat'):source.index('\nifconfig ')] + for directory in ('sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', + 'newroot/etc', 'newroot/usr/lib', 'etc', 'usr/libexec/orchard'): + (self.root / directory).mkdir(parents=True, exist_ok=True) + (self.root / 'newroot/etc/os-release').write_text('NAME="Linux"\n') + (self.root / 'etc/hostname').write_text('ram-test\n') + for helper in ('etc/orchard/usb-identity', 'usr/libexec/orchard/usb-identity'): + path = self.root / helper + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.script) + source = re.sub(r'/(sys|newroot|etc|usr)/', lambda m: str(self.root) + m.group(), source) + source = source.replace('mkdir "$g"', 'mkdir -p "$g/functions"') + subprocess.run(['sh', '-ec', 'mount() { :; };\n' + source], check=True, capture_output=True, + env={**os.environ, 'ORCHARD_DEVICE_TREE': str(self.tree)}) + name = 'orchard-system' if relative.endswith('/init') else 'ram-test' + gadget = self.root / 'sys/kernel/config/usb_gadget' / name + self.assertEqual((gadget / 'strings/0x409/serialnumber').read_text().strip(), expected) + self.assertEqual((gadget / 'UDC').read_text().strip(), 'usbdev') + for function in ('ncm.usb0', 'acm.usb0'): + self.assertTrue((gadget / 'configs/c.1' / function).is_symlink()) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_j71t_mac_current.py b/tests/test_j71t_mac_current.py new file mode 100644 index 00000000..17b875e2 --- /dev/null +++ b/tests/test_j71t_mac_current.py @@ -0,0 +1,204 @@ +"""Host-only tests; ORCHARD_KERNEL_SOURCE selects the prepared patched kernel.""" +import os +from pathlib import Path +import plistlib +import re +import subprocess +import tempfile +import unittest +from unittest.mock import patch +from orchard.apple import usb, linux_usb + +HW = Path(__file__).resolve().parents[1] +BOOT = '12345678-1234-1234-1234-123456789abc' + + +def function(source, name): + start = re.search(r'^.*\b' + name + r'\([^;]*?\)\n\{', source, re.M).start() + brace = source.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + '\n' + + +class MacCurrent(unittest.TestCase): + def test_gadget_profile_preserves_network_serial_and_safe_default(self): + source = (HW / 'boot/initramfs/root/init').read_text() + start = source.index('usb_gadget() {') + script = source[start:source.index('\nusb_gadget ||', start)] + for board, flag, expected in [('j71t', '1', '0x1014'), ('j71t', '', '0x4145'), + ('j71t', '2400', '0x4145'), ('j71b', '1', '0x4145')]: + with self.subTest(board=board, flag=flag), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for n in ['sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', + 'newroot/etc/orchard', 'newroot/usr/lib', 'proc/device-tree', + 'proc/sys/kernel/random', 'etc/orchard']: + (root / n).mkdir(parents=True) + (root / 'proc/device-tree/compatible').write_bytes(f'apple,{board}\0apple,s8003\0'.encode()) + (root / 'proc/sys/kernel/random/boot_id').write_text(BOOT) + (root / 'newroot/etc/os-release').write_text('NAME="Omarchy"\n') + if flag: + (root / 'newroot/etc/orchard/mac-usb-current').write_text(flag + '\n') + helper = (HW / "src/orchard/usb-identity").read_text().replace("/proc/", str(root) + "/proc/") + (root / "etc/orchard/usb-identity").write_text(helper) + run = script + run = re.sub(r'/(sys|newroot|proc|etc)/', lambda m: str(root) + m.group(), run) + run = run.replace('mkdir \"$g\" &&', 'mkdir -p \"$g/functions\" &&') + subprocess.run(['sh', '-ec', 'mount() { :; };\n' + run + '\nusb_gadget'], check=True, + capture_output=True) + g = root / 'sys/kernel/config/usb_gadget/orchard-system' + for key, value in [('idProduct', expected), ('idVendor', '0x05ac'), + ('bDeviceClass', '0xef' if expected == '0x1014' else '0x00'), + ('bDeviceSubClass', '0x02' if expected == '0x1014' else '0x00'), + ('bDeviceProtocol', '0x01' if expected == '0x1014' else '0x00'), ('configs/c.1/MaxPower', '500'), + ('UDC', 'usbdev')]: + self.assertEqual((g/key).read_text().strip(), value) + for name in ['ncm.usb0', 'acm.usb0']: + self.assertTrue((g/'configs/c.1'/name).is_symlink()) + self.assertEqual((g/'strings/0x409/serialnumber').read_text().strip(), f'BOOT:{BOOT} BOARD:{board}') + + def test_compatibility_pid_requires_orchard_identity_on_both_hosts(self): + for serial, mode in [(f'BOOT:{BOOT} BOARD:j71t', 'system'), ('AppleSerial', 'other'), + (f'BOOT:{BOOT} BOARD:j71b', 'other')]: + raw = plistlib.dumps([{'IOObjectClass': 'IOUSBHostDevice', 'idVendor': 0x05ac, + 'idProduct': 0x1014, 'locationID': 123, 'USB Serial Number': serial}]) + with patch.object(usb.sys, 'platform', 'darwin'), patch.object(usb.subprocess, 'run', + return_value=subprocess.CompletedProcess([], 0, raw, b'')): + self.assertEqual(usb.devices()[0]['mode'], mode) + with tempfile.TemporaryDirectory() as tmp: + node = Path(tmp)/'1-2'; node.mkdir() + for key, value in [('idVendor', '05ac'), ('idProduct', '1014'), ('serial', serial)]: + (node/key).write_text(value) + self.assertEqual(linux_usb.devices(Path(tmp))[0]['mode'], mode) + + @unittest.skipUnless(os.environ.get('ORCHARD_KERNEL_SOURCE'), 'set ORCHARD_KERNEL_SOURCE for production C tests') + def test_production_grant_and_charger_policy(self): + source = Path(os.environ['ORCHARD_KERNEL_SOURCE']) + composite = (source/'drivers/usb/gadget/composite.c').read_text() + charger = (source/'drivers/power/supply/apple_tristar_charger.c').read_text() + start = composite.index('if ((ctrl->bRequestType & ~USB_DIR_IN)') + match = composite[start+4:composite.index(' {', start)-1] + match = match.replace('ctrl->bRequestType', 'type').replace('ctrl->bRequest', 'request').replace('le16_to_cpu(cdev->desc.idVendor)', 'vendor') + c = r''' +#include +#include +#include +#include +typedef uint16_t u16; typedef int16_t s16; +#define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define READ_ONCE(a) (a) +#define USB_DIR_IN 0x80 +#define USB_TYPE_VENDOR 0x40 +#define USB_RECIP_DEVICE 0 +#define USB_APPLE_REQ_CURRENT_LIMIT 0x40 +#define USB_APPLE_VENDOR_ID 0x05ac +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_info_ratelimited(...) ((void)0) +#define EINVAL 22 +#define TRISTAR_CHG_FALLBACK_CODE 34 +#define TRISTAR_RAMPING 2 +#define TRISTAR_CHARGING 3 +struct usb_gadget {int dummy;}; +struct usb_composite_dev {struct usb_gadget *gadget; unsigned vbus_draw_ma; int apple_extra_ma; bool suspended;}; +static unsigned draw; +static void usb_gadget_vbus_draw(struct usb_gadget *g, unsigned ma){draw=ma;} +struct tristar_charger {unsigned usb_epoch, epoch, usb_ma, source_ma, max_ma, target_code, code, phase, step_ns; bool present,usb_on,stopping; void *dev,*wq; int lock,poll_work;}; +struct power_supply {struct tristar_charger *chg;}; +union power_supply_propval {int intval;}; +enum power_supply_property {POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT}; +static unsigned notifications; +static struct tristar_charger *power_supply_get_drvdata(struct power_supply *p){return p->chg;} +static void mutex_lock(int *p){assert(!*p);*p=1;} +static void mutex_unlock(int *p){assert(*p);*p=0;} +static void mod_delayed_work(void *wq,int *work,unsigned d){} +static void power_supply_changed(struct power_supply *p){notifications++;} +static void tristar_withdraw(struct tristar_charger *c,unsigned code){c->code=code;} +''' + for n in ['composite_vbus_draw', 'composite_apple_current_limit']: + c += function(composite, n) + for n in ['tristar_code_ma', 'tristar_ma_code', 'tristar_usb_ma', 'tristar_target_code', 'tristar_update_target', 'tristar_set_property']: + c += function(charger, n) + c += f'int match(unsigned type,unsigned request,unsigned w_length,unsigned vendor){{return {match};}}\n' + c += r''' +static void report(struct power_supply *p,int ma){union power_supply_propval v={.intval=ma*1000};assert(!tristar_set_property(p,POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,&v));} +int main(void){ + for(unsigned t=0;t<256;t++)for(unsigned r=0;r<256;r++)for(unsigned l=0;l<2;l++)for(unsigned v=0;v<2;v++) + assert(match(t,r,l,v?0x05ac:0x1234)==((t==0x40||t==0xc0)&&r==0x40&&!l&&v)); + struct usb_gadget g={0};struct usb_composite_dev d={.gadget=&g}; + composite_vbus_draw(&d,500);assert(draw==500); + composite_apple_current_limit(&d,500,1900);assert(draw==2400); + composite_vbus_draw(&d,500);assert(draw==2400); /* configuration preserves grant */ + for(unsigned i=0;i<65536;i++){ + composite_apple_current_limit(&d,0,i);assert(draw==(unsigned)max(500+(s16)i,0)); + } + d.suspended=true;composite_vbus_draw(&d,2); + composite_apple_current_limit(&d,500,1900);assert(draw==2); + d.suspended=false;composite_vbus_draw(&d,500);assert(draw==2400); + d.apple_extra_ma=0;composite_vbus_draw(&d,100);assert(draw==100); + composite_vbus_draw(&d,0);composite_apple_current_limit(&d,0,1900);assert(draw==0); + struct tristar_charger ch={.epoch=7,.usb_epoch=7,.max_ma=2400,.present=true,.usb_on=true,.phase=TRISTAR_CHARGING,.code=34}; + struct power_supply p={.chg=&ch}; + report(&p,2400);assert(ch.usb_ma==2400&&ch.target_code==186&&ch.phase==TRISTAR_RAMPING&¬ifications==1); + ch.code=186;ch.phase=TRISTAR_CHARGING;report(&p,500);assert(ch.code==34&&ch.target_code==34); + for(int ma=501;ma<=33267;ma++){ + report(&p,ma);assert(ch.usb_ma<=(unsigned)ma&&ch.usb_ma<=2400); + assert(tristar_code_ma(ch.target_code)<=min((unsigned)ma,2400)); + } + report(&p,-1);assert(!ch.usb_ma&&ch.target_code==34); + ch.usb_on=false;report(&p,2400);assert(!ch.usb_ma); + ch.usb_on=true;ch.present=false;report(&p,2400);assert(!ch.usb_ma); + ch.present=true;report(&p,2400);ch.epoch++;assert(!tristar_usb_ma(&ch)&&tristar_target_code(&ch)==34); + puts("PASS 262144 request tuples, all signed grants, suspend, board cap, ramp/revoke, attachment and epoch gates"); +} +''' + # Lifecycle invalidations remain in the actual callbacks. + for name in ['composite_reset', 'composite_disconnect', 'composite_suspend']: + self.assertIn('cdev->apple_extra_ma = 0;', function(composite, name)) + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp); (p/'test.c').write_text(c) + subprocess.run(['cc','-std=gnu11','-Wall','-Werror','-fsanitize=address,undefined', str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')], check=True) + +if __name__ == '__main__': + unittest.main() + + +"""A compatibility PID must never classify a real Apple device as Orchard.""" +from pathlib import Path +import plistlib +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from orchard.apple import usb, linux_usb + +BOOT = '12345678-1234-1234-1234-123456789abc' +STABLE = 'f09dfe4c604a' + + +class StableMacCurrent(unittest.TestCase): + def test_compatibility_pid_requires_orchard_identity_on_both_hosts(self): + for serial, mode in [(f'BOOT:{BOOT} BOARD:j71t', 'system'), + (f'BOARD:j71t ID:{STABLE}', 'system'), + ('AppleSerial', 'other'), ('BOARD:j71t', 'other'), + (f'ID:{STABLE}', 'other'), ('BOARD:j71t ID:bad', 'other'), + (f'BOARD:j71b ID:{STABLE}', 'other'), + (f'BOOT:{BOOT} BOARD:j71b', 'other')]: + with self.subTest(serial=serial): + raw = plistlib.dumps([{'IOObjectClass': 'IOUSBHostDevice', 'idVendor': 0x05ac, + 'idProduct': 0x1014, 'locationID': 123, 'USB Serial Number': serial}]) + with patch.object(usb.sys, 'platform', 'darwin'), patch.object(usb.subprocess, 'run', + return_value=subprocess.CompletedProcess([], 0, raw, b'')): + self.assertEqual(usb.devices()[0]['mode'], mode) + with tempfile.TemporaryDirectory() as tmp: + node = Path(tmp) / '1-2' + node.mkdir() + for key, value in [('idVendor', '05ac'), ('idProduct', '1014'), ('serial', serial)]: + (node / key).write_text(value) + self.assertEqual(linux_usb.devices(Path(tmp))[0]['mode'], mode) diff --git a/tests/test_platform_layer.py b/tests/test_platform_layer.py index 9acba731..b7029826 100644 --- a/tests/test_platform_layer.py +++ b/tests/test_platform_layer.py @@ -82,7 +82,10 @@ def test_the_initrd_carries_busybox_and_the_storage_modules_only(self): base = f'lib/modules/{RELEASE}/' self.assertEqual(names['bin/busybox'], b'static busybox') self.assertEqual(names['init'], (ROOT / 'boot/initramfs/root/init').read_bytes()) + self.assertEqual(names['etc/orchard/usb-identity'], (ROOT / 'src/orchard/usb-identity').read_bytes()) self.assertEqual(names['etc/orchard/modules'], b'nvme\n') + self.assertEqual(names['etc/orchard/dtbs/j71t.dtb'], b's8003-j71t') + self.assertIn(b'apple,j71t j71t.dtb\n', names['etc/orchard/pristine-map']) self.assertEqual(sorted(name for name in names if name.endswith('.ko')), [base + 'kernel/drivers/nvme/host/nvme-core.ko', base + 'kernel/drivers/nvme/host/nvme.ko']) self.assertEqual(names[base + 'modules.dep'], diff --git a/tests/test_ram.py b/tests/test_ram.py index 89c83aaa..6e430d29 100644 --- a/tests/test_ram.py +++ b/tests/test_ram.py @@ -475,6 +475,7 @@ def test_usb_session_is_added_unless_the_project_opts_out(self): ram.build_directory(example, hardware=self.hardware) archive = gzip.decompress((example / 'out/apple-16k/initramfs.cpio.gz').read_bytes()) self.assertIn(b'usr/libexec/orchard/usb-session\0', archive) + self.assertIn(b'usr/libexec/orchard/usb-identity\0', archive) self.assertIn(b'etc/orchard/authorized_keys\0', archive) public = (self.base / 'home/private/ram-ssh/id_ed25519.pub').read_bytes() self.assertIn(public, archive) @@ -484,6 +485,7 @@ def test_usb_session_is_added_unless_the_project_opts_out(self): ram.build_directory(example, hardware=self.hardware) archive = gzip.decompress((example / 'out/apple-16k/initramfs.cpio.gz').read_bytes()) self.assertNotIn(b'usr/libexec/orchard/usb-session\0', archive) + self.assertNotIn(b'usr/libexec/orchard/usb-identity\0', archive) self.assertNotIn(b'etc/orchard/authorized_keys\0', archive) def test_arbitrary_directory_layout_uses_active_hardware_package(self): diff --git a/tests/test_system_session.py b/tests/test_system_session.py index 197c0907..8f89b1d6 100644 --- a/tests/test_system_session.py +++ b/tests/test_system_session.py @@ -9,6 +9,9 @@ SYSTEM = {'location': 1, 'mode': 'system', 'product': 'Omarchy A', 'serial_port': '/dev/cu.x', 'boot_id': '00000000-0000-4000-8000-000000000001', 'profile': 'j71b'} PAYLOAD = {**SYSTEM, 'mode': 'ram', 'product': 'Orchard gpu_001'} +STABLE_PAYLOAD = {**PAYLOAD, 'boot_id': None, 'usb_id': 'f09dfe4c604a'} +BOOT_A = '11111111-1111-4111-8111-111111111111' +BOOT_B = '22222222-2222-4222-8222-222222222222' class SystemSession(unittest.TestCase): @@ -62,6 +65,60 @@ def test_without_a_user_it_says_how_to_get_one(self): with self.assertRaisesRegex(ValueError, '--user'): ram_session.connect(SYSTEM) + def ram_connect(self, reply, device=STABLE_PAYLOAD): + with mock.patch.object(ram_session, '_directory', return_value=Path(self.temp.name)), \ + mock.patch.object(ram_session, 'identity', return_value=Path('/ram-key')), \ + mock.patch.object(ram_session, 'SerialPort') as port, \ + mock.patch.object(ram_session, 'wait_authenticated') as wait: + port.return_value.__enter__.return_value.run.return_value = (0, reply) + argv = ram_session.connect(device) + command = port.return_value.__enter__.return_value.run.call_args.args[0] + self.assertIn('/proc/sys/kernel/random/boot_id', command) + self.assertIn('/run/orchard/ssh_host_ed25519_key.pub', command) + self.assertIn('StrictHostKeyChecking=yes', argv) + wait.assert_called_once() + return argv + + def test_stable_usb_identity_gets_fresh_boot_and_key_from_acm_on_each_connection(self): + # A second connect must not reuse a pin just because USB is unchanged. + first = self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n') + pin_a = Path(self.temp.name) / f'known_hosts-{BOOT_A}' + self.assertEqual(pin_a.read_text(), '172.16.42.1 ssh-ed25519 AAAA\n') + self.assertEqual(pin_a.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n'), first) + second = self.ram_connect(BOOT_B + '\nssh-ed25519 BBBB\n') + self.assertNotEqual(first, second) + self.assertFalse(pin_a.exists()) + self.assertEqual((Path(self.temp.name) / f'known_hosts-{BOOT_B}').read_text(), + '172.16.42.1 ssh-ed25519 BBBB\n') + + def test_legacy_ram_boot_tag_is_still_verified_against_acm(self): + self.ram_connect(PAYLOAD['boot_id'] + '\nssh-ed25519 AAAA\n', device=PAYLOAD) + with self.assertRaisesRegex(ValueError, 'disconnected or rebooted'): + self.ram_connect(BOOT_B + '\nssh-ed25519 AAAA\n', device=PAYLOAD) + + def test_invalid_acm_identity_or_changed_key_does_not_reuse_a_pin(self): + self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n') + for reply, error in [('garbage\nssh-ed25519 BBBB\n', 'Invalid RAM boot'), + ('../../escape\nssh-ed25519 BBBB\n', 'Invalid RAM boot'), + (BOOT_A + '\n', 'exactly one SSH host key'), + (BOOT_A + '\nssh-ed25519 BBBB\n', 'changed within the same boot')]: + with self.subTest(reply=reply), self.assertRaisesRegex(ValueError, error): + self.ram_connect(reply) + self.assertEqual((Path(self.temp.name) / f'known_hosts-{BOOT_A}').read_text(), + '172.16.42.1 ssh-ed25519 AAAA\n') + + def test_stable_usb_check_rejects_another_unit_or_mode_at_the_same_port(self): + check = ram_session._check(STABLE_PAYLOAD) + with mock.patch('orchard.apple.usb.devices', return_value=[STABLE_PAYLOAD]): + check() + for change in ({'usb_id': 'fedcba987654'}, {'mode': 'system'}, + {'profile': 'j71t'}, {'serial_port': '/dev/cu.other'}): + with self.subTest(change=change), \ + mock.patch('orchard.apple.usb.devices', return_value=[{**STABLE_PAYLOAD, **change}]), \ + self.assertRaisesRegex(ValueError, 'disconnected or rebooted'): + check() + if __name__ == '__main__': unittest.main() From e8b5e4b7ea065fc76b13c4f6f5296c3d2e6a8439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 18/33] j71t: Wake from sleep on the Smart Cover Opening the Smart Cover should wake the iPad, but the PMIC GPIO driver had no notion of wake-only events, and arming the cover events while the cover was absent aborted every suspend. Describe wake-only events in the apple,d2365-pmic-gpio binding, let the driver wake from sleep on the listed GPIO events, and list the Smart Cover events in the J71t DT. Assisted-by: LLM --- ...j71t-wake-from-sleep-on-the-Smart-Co.patch | 30 ++ ...-apple-d2365-pmic-gpio-describe-wake.patch | 69 +++ ...-wake-from-sleep-on-listed-GPIO-even.patch | 446 ++++++++++++++++++ 3 files changed, 545 insertions(+) create mode 100644 kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch create mode 100644 kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch create mode 100644 kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch diff --git a/kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch b/kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch new file mode 100644 index 00000000..4f9d26c1 --- /dev/null +++ b/kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:27:23 +0000 +Subject: [PATCH] arm64: dts: apple: j71t: wake from sleep on the Smart Cover + events + +PMIC GPIO lines 0 and 10 are the Smart Cover hall sensors. Their events, +0x30 and 0x3a, latch while the iPad sleeps; list them as wake-only so +that opening the cover wakes the iPad. The polled switches and their +reserved range are unchanged: lines 0 and 10 are already outside +gpio-reserved-ranges. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index acb62901e..2a62f0d95 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -40,3 +40,8 @@ &pcie0 { + &usbdev { + apple,retain-gadget-in-s2idle; + }; ++ ++/* Smart Cover hall sensors (PMIC GPIO 0 and 10): wake-only. */ ++&pmu_gpio { ++ apple,wake-events = <0x30 0x3a>; ++}; diff --git a/kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch b/kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch new file mode 100644 index 00000000..24f22212 --- /dev/null +++ b/kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch @@ -0,0 +1,69 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:27:23 +0000 +Subject: [PATCH] dt-bindings: gpio: apple,d2365-pmic-gpio: describe wake-only + events + +Add apple,wake-events: GPIO events (0x30 plus the line) that end system +sleep and have no interrupt consumer, such as the iPad 5 Smart Cover +hall sensors on lines 0 and 10. A hall sensor that is unmasked while the +system is awake drives the PMU line on every change of the magnet field, +so these events are unmasked only for the duration of a sleep and cannot +be mapped as interrupts. Only the D2365 has the event controller they +need. + +Assisted-by: LLM +--- + .../bindings/gpio/apple,d2365-pmic-gpio.yaml | 18 ++++++++++++++++++ + 1 file changed, 18 insertions(+) + +diff --git a/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml b/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml +index c1cae13ab..6241bfc50 100644 +--- a/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml ++++ b/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml +@@ -54,6 +54,21 @@ properties: + maxItems: 1 + description: PMU event output of the PMIC, active low. + ++ apple,wake-events: ++ $ref: /schemas/types.yaml#/definitions/uint32-array ++ minItems: 1 ++ maxItems: 24 ++ items: ++ minimum: 0x30 ++ maximum: 0x47 ++ description: | ++ GPIO events (0x30 plus the line number) that end system sleep and have ++ no interrupt consumer, such as the iPad 5 Smart Cover hall sensors on ++ lines 0 and 10 (events 0x30 and 0x3a). They stay masked while the ++ system is awake, where the line is read through the GPIO instead, and ++ are unmasked only while it sleeps: a hall sensor drives the PMU line on ++ every change of the magnet field. They cannot be mapped as interrupts. ++ + required: + - compatible + - gpio-controller +@@ -61,6 +76,7 @@ required: + + dependencies: + interrupt-controller: ["#interrupt-cells"] ++ apple,wake-events: [interrupt-controller] + + allOf: + - if: +@@ -74,6 +90,7 @@ allOf: + interrupt-controller: false + "#interrupt-cells": false + interrupts: false ++ apple,wake-events: false + + additionalProperties: false + +@@ -92,6 +109,7 @@ examples: + #interrupt-cells = <2>; + interrupt-controller; + interrupts = <38 IRQ_TYPE_LEVEL_LOW>; ++ apple,wake-events = <0x30 0x3a>; + }; + }; + ... diff --git a/kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch b/kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch new file mode 100644 index 00000000..67b97c85 --- /dev/null +++ b/kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch @@ -0,0 +1,446 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:27:23 +0000 +Subject: [PATCH] gpio: apple-d2365: wake from sleep on listed GPIO events + +The Smart Cover halls cannot be interrupt consumers: unmasked while the +system is awake, they keep the PMU line asserted (the 2026-11-07 storm +with IRQ gpio-keys). The switches stay polled; make the halls wake-only +instead. + +Events in apple,wake-events get no interrupt mapping and stay masked +while awake. Every event latches whether masked or not, so ->suspend +stops the event thread and reads the block before it unmasks them, or +the cover close that started the suspend would end it at once. Consumer +events in that read are dispatched as the thread would have, and a wake +source among them ends the transition. ->suspend then unmasks the wake +and wake-only events and takes a wake reference on the PMU line, which +no child interrupt holds for these events. Any failure there masks the +halls again before returning, because ->resume will not run. ->resume +masks them and, while the thread is still gated, reads the latched +events and the line levels as the wake reason, then dispatches the +consumers. + +The read of the event block is factored out of the thread and shared; +the line interrupt's consumers and the storm accounting stay in the +thread. At probe, each wake line's boot configuration is logged next to +the rule the iOS sleep path applies (config byte & 0xc1 == 1). + +Assisted-by: LLM +--- + drivers/gpio/gpio-apple-d2365.c | 258 +++++++++++++++++++++++++++++--- + 1 file changed, 235 insertions(+), 23 deletions(-) + +diff --git a/drivers/gpio/gpio-apple-d2365.c b/drivers/gpio/gpio-apple-d2365.c +index c516dfe30..604d4cba3 100644 +--- a/drivers/gpio/gpio-apple-d2365.c ++++ b/drivers/gpio/gpio-apple-d2365.c +@@ -11,6 +11,12 @@ + * One more interrupt, numbered after the last event, is the line itself: its + * consumers run before the block is read, for a consumer that must act + * within the shortest time after an assertion (charger withdrawal). ++ * ++ * GPIO events listed in apple,wake-events have no consumer: they stay ++ * masked while the system is awake and are unmasked only while it sleeps. ++ * An event latches whether it is masked or not, so the block is read ++ * before they are unmasked, or a move from before the suspend (the cover ++ * close that started it) would end the sleep at once. + */ + + #include +@@ -72,6 +78,10 @@ + #define D2365_EVENT_MASK 0x1c0 + #define D2365_EVENT_BYTES 30 + #define D2365_GPIO_EVENT 0x30 ++/* iOS keeps a GPIO event unmasked in sleep if its config byte & 0xc1 == 1. */ ++#define D2365_GPIO_SLEEP_RULE_MASK 0xc1 ++#define D2365_GPIO_SLEEP_RULE_VALUE 0x01 ++#define APPLE_PMIC_INPUT_BYTES 3 + + #define APPLE_PMIC_EVENT_BYTES_MAX 30 + #define APPLE_PMIC_EVENTS_MAX (APPLE_PMIC_EVENT_BYTES_MAX * 8) +@@ -116,6 +126,8 @@ struct apple_pmic_gpio { + struct delayed_work irq_retry; + DECLARE_BITMAP(enabled, APPLE_PMIC_IRQS_MAX); /* unmasked while awake */ + DECLARE_BITMAP(wake, APPLE_PMIC_EVENTS_MAX); /* unmasked while asleep */ ++ DECLARE_BITMAP(wake_only, APPLE_PMIC_EVENTS_MAX); /* apple,wake-events */ ++ u8 armed_level[APPLE_PMIC_INPUT_BYTES]; + u8 mask[APPLE_PMIC_EVENT_BYTES_MAX]; /* as last written */ + u8 boot_mask[APPLE_PMIC_EVENT_BYTES_MAX]; + int wake_delta; /* parent wake references to apply */ +@@ -126,6 +138,8 @@ struct apple_pmic_gpio { + bool irq_held; + bool removing; + bool shutting_down; ++ bool wake_only_armed; /* wake_only unmasked for this sleep */ ++ bool parent_wake_armed; /* our own wake reference on the line */ + }; + + static unsigned int d2207_gpio_config_reg(unsigned int offset) +@@ -285,6 +299,9 @@ static int apple_pmic_irq_map(struct irq_domain *domain, unsigned int irq, + + if (hwirq > gpio->nevents) + return -EINVAL; ++ /* Unmasked while awake, a hall sensor holds the PMU line. */ ++ if (hwirq < gpio->nevents && test_bit(hwirq, gpio->wake_only)) ++ return -EINVAL; + + irq_set_chip_data(irq, gpio); + irq_set_chip_and_handler(irq, &apple_pmic_irq_chip, handle_simple_irq); +@@ -346,7 +363,8 @@ static bool apple_pmic_event_gate(struct apple_pmic_gpio *gpio) + * suspend_device_irqs() must abort the transition, as the armed + * interrupt does afterwards. + */ +- if (!bitmap_empty(gpio->wake, gpio->nevents)) ++ if (!bitmap_empty(gpio->wake, gpio->nevents) || ++ gpio->wake_only_armed) + pm_system_wakeup(); + apple_pmic_irq_hold(gpio, false); + return false; +@@ -363,6 +381,41 @@ static void apple_pmic_dispatch(struct apple_pmic_gpio *gpio, + handle_nested_irq(child); + } + ++/* ++ * Read (and so clear) the event block: @latched gets every set event, ++ * @pending the ones with a consumer. Caller holds irq_lock. ++ */ ++static int apple_pmic_event_read(struct apple_pmic_gpio *gpio, ++ unsigned long *latched, unsigned long *pending) ++{ ++ const struct apple_pmic_event_info *ev = gpio->info->events; ++ u8 events[APPLE_PMIC_EVENT_BYTES_MAX]; ++ unsigned int hwirq; ++ int ret; ++ ++ bitmap_zero(latched, APPLE_PMIC_EVENTS_MAX); ++ bitmap_zero(pending, APPLE_PMIC_EVENTS_MAX); ++ ret = regmap_bulk_read(gpio->regmap, ev->status_base, events, ev->bytes); ++ if (ret) ++ return ret; ++ ++ for (hwirq = 0; hwirq < gpio->nevents; hwirq++) ++ if (events[hwirq / 8] & BIT(hwirq % 8)) ++ __set_bit(hwirq, latched); ++ bitmap_and(pending, latched, gpio->enabled, gpio->nevents); ++ return 0; ++} ++ ++/* Consumers may mask their interrupt, which takes the bus lock. */ ++static void apple_pmic_dispatch_events(struct apple_pmic_gpio *gpio, ++ const unsigned long *pending) ++{ ++ unsigned int hwirq; ++ ++ for_each_set_bit(hwirq, pending, gpio->nevents) ++ apple_pmic_dispatch(gpio, hwirq); ++} ++ + /* + * Per assertion of the PMU line: first the consumers of the line itself, + * then one read of the event block; every set event with a consumer is then +@@ -377,10 +430,8 @@ static void apple_pmic_dispatch(struct apple_pmic_gpio *gpio, + static irqreturn_t apple_pmic_event_thread(int irq, void *data) + { + struct apple_pmic_gpio *gpio = data; +- const struct apple_pmic_event_info *ev = gpio->info->events; ++ DECLARE_BITMAP(latched, APPLE_PMIC_EVENTS_MAX); + DECLARE_BITMAP(pending, APPLE_PMIC_EVENTS_MAX); +- u8 events[APPLE_PMIC_EVENT_BYTES_MAX]; +- unsigned int hwirq; + int ret; + + bitmap_zero(pending, APPLE_PMIC_EVENTS_MAX); +@@ -408,15 +459,10 @@ static irqreturn_t apple_pmic_event_thread(int irq, void *data) + } + + /* Reading the events clears them and releases the line. */ +- ret = regmap_bulk_read(gpio->regmap, ev->status_base, events, ev->bytes); ++ ret = apple_pmic_event_read(gpio, latched, pending); + if (ret) + goto hold; + gpio->retries = 0; +- +- for (hwirq = 0; hwirq < gpio->nevents; hwirq++) +- if (events[hwirq / 8] & BIT(hwirq % 8) && +- test_bit(hwirq, gpio->enabled)) +- __set_bit(hwirq, pending); + mutex_unlock(&gpio->irq_lock); + goto dispatch; + +@@ -429,9 +475,7 @@ static irqreturn_t apple_pmic_event_thread(int irq, void *data) + mutex_unlock(&gpio->irq_lock); + + dispatch: +- /* Consumers may mask their interrupt, which takes the bus lock. */ +- for_each_set_bit(hwirq, pending, gpio->nevents) +- apple_pmic_dispatch(gpio, hwirq); ++ apple_pmic_dispatch_events(gpio, pending); + return IRQ_HANDLED; + + unlock: +@@ -481,6 +525,59 @@ static void apple_pmic_event_stop(void *data) + free_irq(gpio->parent_irq, gpio); + } + ++/* What the boot firmware left on a wake-only line, next to iOS's sleep rule. */ ++static void apple_pmic_wake_report(struct apple_pmic_gpio *gpio, ++ unsigned int event) ++{ ++ const struct apple_pmic_gpio_info *info = gpio->info; ++ struct device *dev = gpio->chip.parent; ++ unsigned int line = event - info->events->gpio_event; ++ unsigned int reg = info->config_reg(line); ++ unsigned int cfg, cfg_hi, level; ++ ++ if (regmap_read(gpio->regmap, reg, &cfg) || ++ regmap_read(gpio->regmap, reg + 1, &cfg_hi) || ++ regmap_read(gpio->regmap, info->input_base + line / 8, &level)) { ++ dev_warn(dev, "wake event 0x%02x: cannot read line %u\n", ++ event, line); ++ return; ++ } ++ dev_info(dev, ++ "wake event 0x%02x: line %u config 0x%02x/0x%02x level %u, mask at boot %s, iOS keeps it %s in sleep\n", ++ event, line, cfg, cfg_hi, !!(level & BIT(line % 8)), ++ gpio->boot_mask[event / 8] & BIT(event % 8) ? "masked" : "open", ++ (cfg & D2365_GPIO_SLEEP_RULE_MASK) == D2365_GPIO_SLEEP_RULE_VALUE ? ++ "unmasked" : "masked"); ++} ++ ++static void apple_pmic_wake_only_parse(struct apple_pmic_gpio *gpio) ++{ ++ const struct apple_pmic_event_info *ev = gpio->info->events; ++ struct device *dev = gpio->chip.parent; ++ u32 events[APPLE_PMIC_EVENTS_MAX / 8]; ++ int count, i; ++ ++ count = device_property_count_u32(dev, "apple,wake-events"); ++ if (count <= 0) ++ return; ++ if (count > gpio->info->ngpio || ++ device_property_read_u32_array(dev, "apple,wake-events", events, ++ count)) { ++ dev_warn(dev, "ignoring unreadable apple,wake-events\n"); ++ return; ++ } ++ for (i = 0; i < count; i++) { ++ if (events[i] < ev->gpio_event || ++ events[i] >= ev->gpio_event + gpio->info->ngpio || ++ events[i] >= gpio->nevents) { ++ dev_warn(dev, "ignoring wake event 0x%x: not a GPIO event\n", ++ events[i]); ++ continue; ++ } ++ __set_bit(events[i], gpio->wake_only); ++ } ++} ++ + static int apple_pmic_event_probe(struct platform_device *pdev, + struct apple_pmic_gpio *gpio) + { +@@ -496,6 +593,7 @@ static int apple_pmic_event_probe(struct platform_device *pdev, + if (gpio->parent_irq < 0) + return gpio->parent_irq; + gpio->nevents = ev->bytes * 8; ++ apple_pmic_wake_only_parse(gpio); + + ret = devm_mutex_init(dev, &gpio->irq_lock); + if (ret) +@@ -540,7 +638,19 @@ static int apple_pmic_event_probe(struct platform_device *pdev, + if (ret) + return dev_err_probe(dev, ret, + "failed to request the PMU interrupt\n"); +- return devm_add_action_or_reset(dev, apple_pmic_event_stop, gpio); ++ ret = devm_add_action_or_reset(dev, apple_pmic_event_stop, gpio); ++ if (ret) ++ return ret; ++ ++ if (!bitmap_empty(gpio->wake_only, gpio->nevents)) { ++ unsigned int event; ++ ++ for_each_set_bit(event, gpio->wake_only, gpio->nevents) ++ apple_pmic_wake_report(gpio, event); ++ dev_info(dev, "wake-only events %*pbl on IRQ %d\n", ++ gpio->nevents, gpio->wake_only, gpio->parent_irq); ++ } ++ return 0; + } + + static struct platform_driver d2365_gpio_driver; +@@ -576,32 +686,108 @@ void apple_d2365_gpio_quiesce(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_d2365_gpio_quiesce); + ++/* ++ * Drop what latched while awake before the wake-only events are unmasked, ++ * with the event thread stopped. Consumer events in that read are run as ++ * the thread would have run them; a wake source among them ends this ++ * transition, as it would once the line is armed. ++ */ ++static void apple_pmic_wake_only_drop(struct apple_pmic_gpio *gpio) ++{ ++ struct device *dev = gpio->chip.parent; ++ const struct apple_pmic_gpio_info *info = gpio->info; ++ DECLARE_BITMAP(latched, APPLE_PMIC_EVENTS_MAX); ++ DECLARE_BITMAP(pending, APPLE_PMIC_EVENTS_MAX); ++ int ret; ++ ++ mutex_lock(&gpio->irq_lock); ++ ret = apple_pmic_event_read(gpio, latched, pending); ++ if (regmap_bulk_read(gpio->regmap, info->input_base, gpio->armed_level, ++ APPLE_PMIC_INPUT_BYTES)) ++ memset(gpio->armed_level, 0xff, sizeof(gpio->armed_level)); ++ mutex_unlock(&gpio->irq_lock); ++ if (ret) { ++ /* Unknown: as the thread does, every consumer may have an event. */ ++ dev_warn(dev, "cannot drop the latched events: %d\n", ret); ++ bitmap_copy(pending, gpio->enabled, gpio->nevents); ++ } ++ if (bitmap_intersects(pending, gpio->wake, gpio->nevents)) ++ pm_system_wakeup(); ++ apple_pmic_dispatch_events(gpio, pending); ++} ++ + static int apple_pmic_gpio_suspend(struct device *dev) + { + struct apple_pmic_gpio *gpio = dev_get_drvdata(dev); ++ DECLARE_BITMAP(armed, APPLE_PMIC_EVENTS_MAX); ++ bool wake_only = false; + int ret; + + if (!gpio->domain) + return 0; + + cancel_delayed_work_sync(&gpio->irq_retry); ++ if (!bitmap_empty(gpio->wake_only, gpio->nevents)) { ++ wake_only = true; ++ disable_irq(gpio->parent_irq); ++ apple_pmic_wake_only_drop(gpio); ++ } ++ + mutex_lock(&gpio->irq_lock); + /* Only wake-enabled events may pull the line while asleep. */ +- ret = apple_pmic_event_sync(gpio, gpio->wake); +- if (ret) { +- mutex_unlock(&gpio->irq_lock); +- return ret; +- } ++ bitmap_or(armed, gpio->wake, gpio->wake_only, gpio->nevents); ++ ret = apple_pmic_event_sync(gpio, armed); ++ if (ret) ++ goto unwind; + gpio->suspended = true; ++ gpio->wake_only_armed = wake_only; + mutex_unlock(&gpio->irq_lock); ++ if (wake_only) ++ enable_irq(gpio->parent_irq); + /* A thread that passed the gate before it closed has finished now. */ + synchronize_irq(gpio->parent_irq); ++ ++ if (wake_only) { ++ /* No child interrupt takes a wake reference for these. */ ++ ret = irq_set_irq_wake(gpio->parent_irq, 1); ++ if (ret) { ++ dev_err(dev, "cannot arm the PMU interrupt for wake: %d\n", ++ ret); ++ mutex_lock(&gpio->irq_lock); ++ gpio->suspended = false; ++ gpio->wake_only_armed = false; ++ goto unwind_enabled; ++ } ++ gpio->parent_wake_armed = true; ++ dev_info(dev, "wake events %*pbl armed, GPIO levels %*phN\n", ++ gpio->nevents, gpio->wake_only, APPLE_PMIC_INPUT_BYTES, ++ gpio->armed_level); ++ } + return 0; ++ ++unwind: ++ /* resume() will not run for this device: mask the halls again here. */ ++ if (wake_only) ++ enable_irq(gpio->parent_irq); ++unwind_enabled: ++ if (apple_pmic_event_sync(gpio, gpio->enabled)) ++ dev_err(dev, "event mask restore failed\n"); ++ /* A line held while gated is not re-enabled by a resume of ours. */ ++ if (gpio->irq_held) { ++ gpio->irq_held = false; ++ enable_irq(gpio->parent_irq); ++ } ++ mutex_unlock(&gpio->irq_lock); ++ return ret; + } + + static int apple_pmic_gpio_resume(struct device *dev) + { + struct apple_pmic_gpio *gpio = dev_get_drvdata(dev); ++ DECLARE_BITMAP(latched, APPLE_PMIC_EVENTS_MAX); ++ DECLARE_BITMAP(pending, APPLE_PMIC_EVENTS_MAX); ++ u8 level[APPLE_PMIC_INPUT_BYTES] = {}; ++ bool armed, read = false; + int ret; + + if (!gpio->domain) +@@ -609,17 +795,43 @@ static int apple_pmic_gpio_resume(struct device *dev) + + /* Our I2C ancestors have completed their resume callbacks. */ + mutex_lock(&gpio->irq_lock); +- gpio->suspended = false; +- gpio->retries = 0; + ret = apple_pmic_event_sync(gpio, gpio->enabled); + if (ret) + dev_warn(dev, "event mask write failed: %d\n", ret); +- /* An event that arrived while gated is still latched. */ ++ armed = gpio->wake_only_armed; ++ gpio->wake_only_armed = false; ++ bitmap_zero(pending, APPLE_PMIC_EVENTS_MAX); ++ if (armed) { ++ /* Still gated: the thread cannot take the wake reason first. */ ++ regmap_bulk_read(gpio->regmap, gpio->info->input_base, level, ++ sizeof(level)); ++ read = !apple_pmic_event_read(gpio, latched, pending); ++ } ++ gpio->suspended = false; ++ gpio->retries = 0; ++ /* An event that arrived while gated is still latched, unless read here. */ + if (gpio->irq_held) { + gpio->irq_held = false; + enable_irq(gpio->parent_irq); + } + mutex_unlock(&gpio->irq_lock); ++ ++ if (armed) { ++ if (read) { ++ bitmap_and(latched, latched, gpio->wake_only, gpio->nevents); ++ dev_info(dev, "wake events disarmed, latched %*pbl, GPIO levels %*phN (armed at %*phN)\n", ++ gpio->nevents, latched, APPLE_PMIC_INPUT_BYTES, ++ level, APPLE_PMIC_INPUT_BYTES, gpio->armed_level); ++ } else { ++ /* Unknown: every consumer may have an event. */ ++ bitmap_copy(pending, gpio->enabled, gpio->nevents); ++ } ++ if (gpio->parent_wake_armed) { ++ irq_set_irq_wake(gpio->parent_irq, 0); ++ gpio->parent_wake_armed = false; ++ } ++ apple_pmic_dispatch_events(gpio, pending); ++ } + return 0; + } + +@@ -728,13 +940,13 @@ static int apple_pmic_gpio_probe(struct platform_device *pdev) + gpio->regmap = dev_get_regmap(dev->parent, NULL); + if (!gpio->regmap) + return dev_err_probe(dev, -ENODEV, "parent regmap unavailable\n"); ++ gpio->chip.parent = dev; + + ret = apple_pmic_event_probe(pdev, gpio); + if (ret) + return ret; + + gpio->chip.label = gpio->info->label; +- gpio->chip.parent = dev; + gpio->chip.owner = THIS_MODULE; + gpio->chip.base = -1; + gpio->chip.ngpio = gpio->info->ngpio; From 7d41c329d7057dbba8237899b5d34a57a0651617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 19/33] m1n1: Resident EL3 PSCI for A7-A9, on by default A7-A9 have no EL2 and boot without any PSCI firmware, so Linux on an iPad 5 could not bring secondary CPUs down and up again, and a kexec had nowhere to park them. CPU hotplug, kexec and a clean poweroff all need a PSCI implementation that stays resident after m1n1 hands over. Carry m1n1 patches that keep a small EL3 PSCI implementation resident: CPU_ON/CPU_OFF with a firmware rendezvous (EL1 WFI/WFE trap to EL3), SYSTEM_OFF, and a guarded SYSTEM_RESET that boots the default entry again. The loader marks retained hardware in the device tree after a PSCI re-entry. The resident PSCI is on by default on these SoCs; the shipped boot framing keeps psci_reset=0, so SYSTEM_RESET stays off unless a loader enables it. Adds boot/m1n1/psci-loader and its tests. Assisted-by: LLM --- boot/m1n1/README.md | 13 + ...resident-EL3-CPU-lifecycle-for-A7-A9.patch | 880 ++++++++++++++++++ ...-the-resident-EL3-PSCI-on-by-default.patch | 84 ++ ...I-WFE-to-EL3-for-a-firmware-CPU-rend.patch | 286 ++++++ ...YSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch | 497 ++++++++++ ...-default-entry-after-a-PSCI-SYSTEM_R.patch | 298 ++++++ ...etained-hardware-after-PSCI-re-entry.patch | 38 + boot/m1n1/psci-loader | 39 + tests/test_m1n1_psci.py | 81 ++ 9 files changed, 2216 insertions(+) create mode 100644 boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch create mode 100644 boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch create mode 100644 boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch create mode 100644 boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch create mode 100644 boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch create mode 100644 boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch create mode 100755 boot/m1n1/psci-loader create mode 100644 tests/test_m1n1_psci.py diff --git a/boot/m1n1/README.md b/boot/m1n1/README.md index bd78f93b..0a7c6e1c 100644 --- a/boot/m1n1/README.md +++ b/boot/m1n1/README.md @@ -28,3 +28,16 @@ gives the same tree as Hoolock's tip before updating `sources.toml`. Scratch stays in ignored `boot/m1n1/.work//`. `--resume` reuses it only if the recorded patch set is unchanged. + +## Resident PSCI on A7-A9 + +Orchard patch 0015 adds the resident EL3 PSCI monitor (CPU_ON, CPU_OFF, +AFFINITY_INFO); 0016-0019 make it the default whenever m1n1 is entered at EL3 +and add SYSTEM_RESET (warm reboot back into m1n1, which boots the default BLS +entry with the last launch's DTB) and SYSTEM_OFF (halt). Any PSCI-aware +kernel reboots, powers off and kexecs through its ordinary paths; m1n1 takes +IPI-stopped CPUs back by trapping EL1 WFI/WFE to EL3. EL1/EL2 entries keep +the spin table. Payload variables (append to `m1n1.bin`, e.g. with +`psci-loader`): `psci=0` turns PSCI off, `psci_reset=0` keeps only the CPU +lifecycle (no WFI/WFE traps), `psci_reset=proxy` stops in the proxy after a +reset. 0020 marks retained hardware in the device tree after a re-entry. diff --git a/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch b/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch new file mode 100644 index 00000000..4eb87533 --- /dev/null +++ b/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch @@ -0,0 +1,880 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sat, 7 Nov 2026 15:00:00 +0100 +Subject: [PATCH] psci: Add opt-in resident EL3 CPU lifecycle for A7-A9 + +On A7-A9 m1n1 is entered at EL3 and stays resident: its image and the +per-CPU EL3 stacks are memreserved, and VBAR_EL3 points at its vectors +while Linux runs at EL1. Use that to give Linux PSCI 1.0 CPU_ON, CPU_OFF +and AFFINITY_INFO (plus VERSION, FEATURES and MIGRATE_INFO_TYPE). The +spin table has no cpu_die, so without this the kernel refuses kexec on +an SMP system, can't offline CPUs, and a crash kernel inherits CPUs +stuck in the old kernel. + +CPU_OFF does not power the core down. It cleans and invalidates its L1D, +stops its timers, and waits in WFE at EL3 (MMU off) on a one-line +mailbox. CPU_ON posts an entry point; the core leaves with a clean +I-cache and TLB, SCTLR_EL1 at its reset value, DAIF masked and SP_EL3 on +a static per-CPU stack. SYSTEM_OFF/SYSTEM_RESET, CPU_SUSPEND and +SMCCC_VERSION return NOT_SUPPORTED, so callers fall back to their own +methods and assume SMCCC 1.0. + +Opt-in with the payload variable psci=1, and only used when m1n1 itself +was entered at EL3. kboot then parks the secondaries in the pen (a core +that doesn't park is pruned), sets enable-method = "psci" on every CPU, +adds /psci (method = "smc") and records the re-entry address and the +original boot_args in /chosen. + +Re-entry: _reentry can be jumped to from a kexec'd trampoline (EL1, MMU +off, x0 = boot_args copy). It drops this image's lines from the caches, +flags the re-entry and restarts m1n1, which then skips the (stale) +payload area and runs the proxy. smp_start_secondaries() takes cores +back from the pen with CPU_ON, and the mailboxes and the psci=1 setting +live in .data so they survive the .bss clear. + +Rebase the resident boot shim onto Orchard's complete m1n1 series +without changing its runtime behavior. The payload variable is not a +kernel command-line option; a bare proxy loader also needs it appended. + +Assisted-by: LLM +--- + Makefile | 1 + + src/kboot.c | 56 +++++++++++++- + src/main.c | 8 +- + src/payload.c | 3 + + src/psci.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + src/psci.h | 78 ++++++++++++++++++++ + src/psci_asm.S | 170 ++++++++++++++++++++++++++++++++++++++++++ + src/smp.c | 73 ++++++++++++++++++ + src/smp.h | 1 + + src/start.S | 6 +- + src/startup.c | 6 +- + 11 files changed, 626 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index bb72266..4aefb37 100644 +--- a/Makefile ++++ b/Makefile +@@ -166,6 +166,7 @@ OBJECTS := \ + pcie.o \ + pmgr.o \ + proxy.o \ ++ psci.o psci_asm.o \ + ringbuffer.o \ + rtkit.o \ + sart.o \ +diff --git a/src/kboot.c b/src/kboot.c +index b75bb45..2316709 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -21,6 +21,7 @@ + #include "memory.h" + #include "pcie.h" + #include "pmgr.h" ++#include "psci.h" + #include "sep.h" + #include "sio.h" + #include "smc.h" +@@ -574,6 +575,53 @@ static int dt_set_smbios(void) + return 0; + } + ++/* ++ * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * (see psci.c) instead of the spin table, so the kernel can offline them ++ * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. ++ */ ++static int dt_set_psci(void) ++{ ++ static const char compat[] = "arm,psci-1.0\0arm,psci-0.2"; ++ int node; ++ ++ if (!psci_active()) { ++ printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ return 0; ++ } ++ ++ int cpus = fdt_path_offset(dt, "/cpus"); ++ if (cpus < 0) ++ bail("FDT: /cpus node not found in devtree\n"); ++ ++ fdt_for_each_subnode(node, dt, cpus) ++ { ++ if (strncmp(fdt_get_name(dt, node, NULL), "cpu@", 4)) ++ continue; ++ if (fdt_setprop_string(dt, node, "enable-method", "psci")) ++ bail("FDT: couldn't set enable-method\n"); ++ } ++ ++ node = fdt_path_offset(dt, "/psci"); ++ if (node < 0) ++ node = fdt_add_subnode(dt, 0, "psci"); ++ if (node < 0) ++ bail("FDT: couldn't add /psci\n"); ++ if (fdt_setprop(dt, node, "compatible", compat, sizeof(compat)) || ++ fdt_setprop_string(dt, node, "method", "smc")) ++ bail("FDT: couldn't fill /psci\n"); ++ ++ node = fdt_path_offset(dt, "/chosen"); ++ if (node < 0) ++ bail("FDT: /chosen node not found in devtree\n"); ++ if (fdt_setprop_u64(dt, node, "linux-a9,m1n1-reentry", (u64)_reentry) || ++ fdt_setprop(dt, node, "linux-a9,m1n1-boot-args", &cur_boot_args, sizeof(cur_boot_args))) ++ bail("FDT: couldn't set the m1n1 re-entry properties\n"); ++ ++ printf("PSCI: on, enable-method = \"psci\", m1n1 re-entry at %p\n", _reentry); ++ return 0; ++} ++ + static int dt_set_cpus(void) + { + int ret = 0; +@@ -587,6 +635,10 @@ static int dt_set_cpus(void) + if (!pruned_phandles) + bail("FDT: out of memory\n"); + ++ /* Secondaries wait in the PSCI pen; one that can't get there is pruned below */ ++ if (psci_active()) ++ printf("PSCI: parked %d secondary CPU(s)\n", smp_park_secondaries_psci()); ++ + /* Prune CPU nodes */ + int node, cpu = 0; + for (node = fdt_first_subnode(dt, cpus); node >= 0;) { +@@ -618,7 +670,7 @@ static int dt_set_cpus(void) + } else { + printf("FDT: Reserving stack for CPU %d 0x%lx\n", cpu, (uint64_t)secondary_stacks[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks[cpu], SECONDARY_STACK_SIZE); +- if (has_el3()) { ++ if (has_el3() && secondary_stacks_el3[cpu]) { + printf("FDT: Reserving EL3 stack for CPU %d 0x%lx\n", cpu, + (uint64_t)secondary_stacks_el3[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks_el3[cpu], SECONDARY_STACK_SIZE); +@@ -3206,6 +3258,8 @@ int kboot_prepare_dt(void *fdt) + return -1; + if (dt_set_cpus()) + return -1; ++ if (dt_set_psci()) ++ return -1; + if (dt_set_mac_addresses()) + return -1; + if (dt_set_wifi()) +diff --git a/src/main.c b/src/main.c +index d4e828c..3f32380 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -21,6 +21,7 @@ + #include "pcie.h" + #include "pmgr.h" + #include "sep.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "tps6598x.h" +@@ -117,9 +118,12 @@ void run_actions(void) + #endif + #endif + +- printf("Checking for payloads...\n"); ++ if (psci_reentered()) ++ printf("Re-entered from Linux: whatever follows m1n1 is stale, not loading payloads\n"); ++ else ++ printf("Checking for payloads...\n"); + +- if (payload_run() == 0) { ++ if (!psci_reentered() && payload_run() == 0) { + printf("Valid payload found\n"); + return; + } +diff --git a/src/payload.c b/src/payload.c +index 5b1547f..4ffc9e2 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -12,6 +12,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "utils.h" + +@@ -205,6 +206,8 @@ static bool check_var(u8 **p) + mitigations_configure(val); + } else if (IS_VAR("tso=")) { + enable_tso = val[0] == '1'; ++ } else if (IS_VAR("psci=")) { ++ psci_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +new file mode 100644 +index 0000000..2b8ca0d +--- /dev/null ++++ b/src/psci.c +@@ -0,0 +1,229 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++#include "smp.h" ++#include "types.h" ++#include "utils.h" ++ ++struct psci_cpu { ++ volatile u64 state; ++ volatile u64 entry; ++ volatile u64 context; ++ u64 pad[PSCI_CPU_SIZE / 8 - 3]; ++}; ++ ++_Static_assert(sizeof(struct psci_cpu) == PSCI_CPU_SIZE, "psci_cpu layout"); ++ ++/* ++ * Mailboxes, one cache line each, only ever accessed at EL3 with the MMU off. ++ * They live in .data rather than .bss so that a parked CPU's state survives ++ * m1n1 being re-entered from Linux (which clears .bss). ++ */ ++struct psci_cpu psci_cpus[PSCI_MAX_CPUS] ++ __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* Written with the MMU off only (_reentry sets reentry_flag, the first member) */ ++struct { ++ u64 reentry_flag; ++ u64 boot_el3; ++} psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* psci=1 from the payload, kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))); ++ ++/* EL3 stacks a CPU gets when it leaves the pen */ ++static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); ++ ++static bool psci_have_el3; ++static bool reentered; ++ ++/* EL3 side */ ++ ++static inline void psci_line_sync(volatile void *p) ++{ ++ __asm__ volatile("dc civac, %0\n\tdsb sy" ::"r"(p) : "memory"); ++} ++ ++static int psci_mpidr_index(u64 mpidr) ++{ ++ mpidr &= 0xff00ffffffUL; ++ ++ /* A7-A9: one cluster, Aff0 is the core */ ++ if (mpidr >= PSCI_MAX_CPUS) ++ return -1; ++ ++ return mpidr; ++} ++ ++static s64 psci_cpu_off(void) ++{ ++ int idx = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (idx < 0) ++ return PSCI_RET_DENIED; ++ ++ psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); ++} ++ ++static s64 psci_cpu_on(u64 mpidr, u64 entry, u64 context) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || !entry) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ struct psci_cpu *c = &psci_cpus[idx]; ++ ++ psci_line_sync(c); ++ switch (c->state) { ++ case PSCI_AFF_OFF: ++ break; ++ case PSCI_AFF_ON_PENDING: ++ return PSCI_RET_ON_PENDING; ++ default: ++ /* Running, or never seen by us (still in m1n1 or never started) */ ++ return PSCI_RET_ALREADY_ON; ++ } ++ ++ /* State before entry: the pen sets ON as soon as it sees the entry */ ++ c->context = context; ++ c->state = PSCI_AFF_ON_PENDING; ++ psci_line_sync(c); ++ c->entry = entry; ++ psci_line_sync(c); ++ sysop("sev"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++static s64 psci_affinity(u64 mpidr, u64 level) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || level) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ psci_line_sync(&psci_cpus[idx]); ++ return psci_cpus[idx].state; ++} ++ ++static s64 psci_features(u32 fid) ++{ ++ switch (fid) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ case PSCI_0_2_FN_CPU_OFF: ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return PSCI_RET_SUCCESS; ++ default: ++ /* ++ * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make ++ * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) ++{ ++ u32 fn = fid; ++ ++ /* SMC32 calls carry 32-bit arguments */ ++ if (!(fn & 0x40000000)) { ++ a1 = (u32)a1; ++ a2 = (u32)a2; ++ a3 = (u32)a3; ++ } ++ ++ switch (fn) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ return 0x10000; /* 1.0 */ ++ case PSCI_0_2_FN_CPU_OFF: ++ return psci_cpu_off(); ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ return psci_cpu_on(a1, a2, a3); ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ return psci_affinity(a1, a2); ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ return 2; /* no trusted OS, nothing to migrate */ ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return psci_features(a1); ++ default: ++ /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++/* EL1 side */ ++ ++void psci_note_boot_el(u64 el) ++{ ++ /* MMU still off: these .data accesses are uncached, like the writers' */ ++ if (psci_persist.reentry_flag) { ++ reentered = true; ++ psci_persist.reentry_flag = 0; ++ } ++ if (el == 3) ++ psci_persist.boot_el3 = 1; ++ ++ psci_have_el3 = psci_persist.boot_el3 && has_el3(); ++} ++ ++void psci_configure(const char *val) ++{ ++ psci_mode = val[0] == '1'; ++} ++ ++bool psci_active(void) ++{ ++ return psci_mode && psci_have_el3; ++} ++ ++bool psci_reentered(void) ++{ ++ return reentered; ++} ++ ++int psci_affinity_info(u64 mpidr) ++{ ++ return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); ++} ++ ++static void psci_self_off(void) ++{ ++ /* Only returns on failure; the core then stays in m1n1's loop */ ++ psci_smc(PSCI_0_2_FN_CPU_OFF, 0, 0, 0); ++} ++ ++int psci_park_cpu(int cpu, u64 mpidr) ++{ ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ ++ smp_call0(cpu, psci_self_off); ++ ++ for (int i = 0; i < 100; i++) { ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ udelay(1000); ++ } ++ ++ return -1; ++} ++ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top) ++{ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return -1; ++ ++ s64 ret = psci_smc(PSCI_0_2_FN64_CPU_ON, mpidr, (u64)psci_secondary_entry, ++ (u64)stack_top | (u64)index); ++ ++ return ret == PSCI_RET_SUCCESS ? 0 : -1; ++} +diff --git a/src/psci.h b/src/psci.h +new file mode 100644 +index 0000000..6769ebe +--- /dev/null ++++ b/src/psci.h +@@ -0,0 +1,78 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#ifndef PSCI_H ++#define PSCI_H ++ ++#ifndef __ASSEMBLER__ ++#include "types.h" ++#endif ++ ++/* ++ * Minimal PSCI 1.0 for SoCs where m1n1 owns EL3 (A7-A9 entered at EL3). ++ * Only CPU_ON/CPU_OFF/AFFINITY_INFO and the discovery calls are implemented. ++ * A CPU that calls CPU_OFF is not powered down: it waits at EL3, MMU and ++ * caches off, until CPU_ON hands it a new EL1 entry point. That is what the ++ * kernel needs to offline CPUs for kexec and CPU hotplug. ++ */ ++ ++#define PSCI_0_2_FN_PSCI_VERSION 0x84000000 ++#define PSCI_0_2_FN_CPU_SUSPEND 0x84000001 ++#define PSCI_0_2_FN_CPU_OFF 0x84000002 ++#define PSCI_0_2_FN_CPU_ON 0x84000003 ++#define PSCI_0_2_FN_AFFINITY_INFO 0x84000004 ++#define PSCI_0_2_FN_MIGRATE_INFO_TYPE 0x84000006 ++#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 ++#define PSCI_0_2_FN_SYSTEM_RESET 0x84000009 ++#define PSCI_1_0_FN_PSCI_FEATURES 0x8400000a ++#define PSCI_0_2_FN64_CPU_ON 0xc4000003 ++#define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 ++ ++#define PSCI_RET_SUCCESS 0 ++#define PSCI_RET_NOT_SUPPORTED -1 ++#define PSCI_RET_INVALID_PARAMS -2 ++#define PSCI_RET_DENIED -3 ++#define PSCI_RET_ALREADY_ON -4 ++#define PSCI_RET_ON_PENDING -5 ++#define PSCI_RET_INTERNAL_FAILURE -6 ++ ++#define PSCI_AFF_ON 0 ++#define PSCI_AFF_OFF 1 ++#define PSCI_AFF_ON_PENDING 2 ++ ++#define PSCI_MAX_CPUS 4 ++#define PSCI_STACK_SIZE 0x4000 ++ ++/* Layout shared with psci_asm.S */ ++#define PSCI_CPU_STATE 0x00 ++#define PSCI_CPU_ENTRY 0x08 ++#define PSCI_CPU_CONTEXT 0x10 ++#define PSCI_CPU_SIZE 0x80 ++ ++#ifndef __ASSEMBLER__ ++ ++/* Payload variable psci=1 */ ++void psci_configure(const char *val); ++/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++bool psci_active(void); ++/* Called once from _start_c with the EL m1n1 was entered at */ ++void psci_note_boot_el(u64 el); ++/* Set when m1n1 was re-entered from Linux (see psci_asm.S _reentry) */ ++bool psci_reentered(void); ++ ++/* EL1 side: SMC into our own EL3 */ ++s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3); ++int psci_affinity_info(u64 mpidr); ++/* Park a secondary that is spinning in smp_secondary_entry() at EL3 */ ++int psci_park_cpu(int cpu, u64 mpidr); ++/* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); ++ ++/* EL3 side */ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); ++void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_secondary_entry(void); ++void _reentry(void); ++ ++#endif ++ ++#endif +diff --git a/src/psci_asm.S b/src/psci_asm.S +new file mode 100644 +index 0000000..14f842f +--- /dev/null ++++ b/src/psci_asm.S +@@ -0,0 +1,170 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++ ++#define DCSW_OP_DCCISW 0x1 // memory.h ++ ++.extern _v_sp0_sync ++.extern _start ++.extern _cpu_reset_c ++.extern psci_handle ++.extern psci_persist ++ ++.text ++ ++/* ++ * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) ++ * goes to the PSCI dispatcher; everything else, and every exception taken ++ * at EL1/EL2, takes the usual path. ++ */ ++.globl _v_lower_sync ++.type _v_lower_sync, @function ++_v_lower_sync: ++ stp x16, x17, [sp, #-16]! ++ mrs x16, CurrentEL ++ cmp x16, #0xc ++ b.ne 1f ++ mrs x16, esr_el3 ++ lsr x17, x16, #26 ++ cmp x17, #0x17 // EC: SMC from AArch64 ++ b.ne 1f ++ tst x16, #0xffff // imm16 == 0: SMCCC ++ b.ne 1f ++ ldp x16, x17, [sp], #16 ++ b psci_smc_entry ++1: ++ ldp x16, x17, [sp], #16 ++ b _v_sp0_sync ++ ++/* ++ * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the ++ * caller. x0-x3 go straight to psci_handle(). ELR/SPSR are saved in case ++ * anything below takes an exception at EL3. ++ */ ++.type psci_smc_entry, @function ++psci_smc_entry: ++ stp x29, x30, [sp, #-16]! ++ mrs x29, elr_el3 ++ mrs x30, spsr_el3 ++ stp x29, x30, [sp, #-16]! ++ str x18, [sp, #-16]! ++ bl psci_handle ++ ldr x18, [sp], #16 ++ ldp x29, x30, [sp], #16 ++ msr elr_el3, x29 ++ msr spsr_el3, x30 ++ ldp x29, x30, [sp], #16 ++ eret ++ ++/* ++ * psci_pen(struct psci_cpu *me, u64 stack_top): CPU_OFF. Never returns. ++ * ++ * The core is not powered down. It writes its L1D back, marks itself OFF and ++ * waits in WFE at EL3 (MMU off, so every access here is non-cacheable) for ++ * CPU_ON to post an entry point. It then enters EL1 there with the MMU off, ++ * a clean I-cache and TLB, and x0 = context id, like a core coming out of ++ * reset. SP_EL3 is reset to this core's own static stack. ++ */ ++.globl psci_pen ++.type psci_pen, @function ++psci_pen: ++ mov x19, x0 ++ mov x20, x1 ++ ++ // A pending timer interrupt must not keep waking WFE. ++ msr cntp_ctl_el0, xzr ++ msr cntv_ctl_el0, xzr ++ isb ++ ++ // Clean and invalidate this core's L1D (L2 is shared and coherent). ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_level1 ++ dsb sy ++ ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_OFF ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ sev ++ ++1: ++ wfe ++ dc civac, x19 ++ dsb sy ++ ldr x2, [x19, #PSCI_CPU_ENTRY] ++ cbz x2, 1b ++ ++ ldr x0, [x19, #PSCI_CPU_CONTEXT] ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_ON ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ ++ // Nothing of the previous kernel's EL1 context may survive. ++ ic iallu ++ tlbi alle1 ++ dsb sy ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ msr elr_el3, x2 ++ mov sp, x20 ++ isb ++ eret ++ ++/* EL1 side: s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3) */ ++.globl psci_smc ++.type psci_smc, @function ++psci_smc: ++ smc #0 ++ ret ++ ++/* ++ * CPU_ON target when m1n1 takes a parked core back after a re-entry. ++ * x0 = EL1 stack top | m1n1 CPU index (stack tops are 16 KiB aligned). ++ */ ++.globl psci_secondary_entry ++.type psci_secondary_entry, @function ++psci_secondary_entry: ++ and x1, x0, #0xf ++ bic x0, x0, #0xf ++ msr tpidr_el1, x1 ++ mov sp, x0 ++ bl _cpu_reset_c ++ b . ++ ++/* ++ * Re-entry from Linux (a kexec'd trampoline, see docs/boot/warm-reboot.md): ++ * EL1, MMU and caches off, x0 = a copy of the original boot_args. ++ * Linux may hold clean lines of this image in its linear map, so drop them ++ * before m1n1 writes its .bss uncached and then reads it with caches on. ++ */ ++.globl _reentry ++.type _reentry, @function ++_reentry: ++ mov x19, x0 ++ adrp x1, _base ++ adrp x2, _end ++1: ++ dc civac, x1 ++ add x1, x1, #64 ++ cmp x1, x2 ++ b.lo 1b ++ dsb sy ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #1 ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ mov x0, x19 ++ b _start +diff --git a/src/smp.c b/src/smp.c +index 83eadff..7f5b336 100644 +--- a/src/smp.c ++++ b/src/smp.c +@@ -8,6 +8,7 @@ + #include "malloc.h" + #include "memory.h" + #include "pmgr.h" ++#include "psci.h" + #include "soc.h" + #include "string.h" + #include "types.h" +@@ -227,6 +228,74 @@ static void smp_stop_cpu(int index, int die, int cluster, int core, u64 impl, u6 + } + } + ++/* ++ * After a re-entry from Linux, a secondary that Linux offlined waits in the ++ * PSCI pen at EL3 rather than in smp_secondary_entry(). It can't be reset ++ * through the PMGR (it is running), so hand it an EL1 entry with CPU_ON. ++ */ ++static bool smp_reclaim_cpu(int index, u64 mpidr) ++{ ++ int i; ++ ++ if (index >= MAX_CPUS || spin_table[index].flag) ++ return false; ++ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return false; ++ ++ printf("Taking CPU %d back from the PSCI pen... ", index); ++ ++ memset(&spin_table[index], 0, sizeof(struct spin_table)); ++ target_cpu = index; ++ secondary_stacks[index] = memalign(0x4000, SECONDARY_STACK_SIZE); ++ ++ /* The core starts with its MMU off */ ++ dc_civac_range(&spin_table[index], sizeof(struct spin_table)); ++ dc_civac_range(&target_cpu, sizeof(target_cpu)); ++ dc_civac_range(&boot_cpu_idx, sizeof(boot_cpu_idx)); ++ dc_civac_range(&boot_cpu_mpidr, sizeof(boot_cpu_mpidr)); ++ ++ if (psci_unpark_cpu(index, mpidr, secondary_stacks[index] + SECONDARY_STACK_SIZE) < 0) { ++ printf("Failed (CPU_ON)\n"); ++ return true; ++ } ++ ++ for (i = 0; i < 100; i++) { ++ sysop("dmb ld"); ++ if (spin_table[index].flag) ++ break; ++ udelay(1000); ++ } ++ ++ printf("%s\n", i >= 100 ? "Failed!" : " Back."); ++ return true; ++} ++ ++/* ++ * Put every live secondary into the PSCI pen, for a kernel that boots them ++ * with enable-method = "psci". A CPU that doesn't make it is dropped (and ++ * then pruned from the DT by the caller). ++ */ ++int smp_park_secondaries_psci(void) ++{ ++ int parked = 0; ++ ++ for (int i = 0; i < MAX_CPUS; i++) { ++ if (i == boot_cpu_idx || !smp_is_alive(i)) ++ continue; ++ ++ u64 mpidr = spin_table[i].mpidr; ++ if (psci_park_cpu(i, mpidr) < 0) { ++ printf("PSCI: CPU %d (MPIDR 0x%lx) did not park, dropping it\n", i, mpidr); ++ memset(&spin_table[i], 0, sizeof(struct spin_table)); ++ continue; ++ } ++ parked++; ++ } ++ ++ return parked; ++} ++ + static u64 smp_get_iorvbar(int node, int cpu) + { + u64 cpu_impl_reg[2]; +@@ -405,6 +474,10 @@ void smp_start_secondaries(void) + u8 cluster = FIELD_GET(CPU_REG_CLUSTER, reg); + u8 die = FIELD_GET(CPU_REG_DIE, reg); + ++ if (psci_active() && psci_reentered() && ++ smp_reclaim_cpu(i, core | (cluster << 8) | (die << 16))) ++ continue; ++ + smp_start_cpu(i, die, cluster, core, iorvbar, pmgr_reg + cpu_start_off); + } + } +diff --git a/src/smp.h b/src/smp.h +index 0296978..6c90557 100644 +--- a/src/smp.h ++++ b/src/smp.h +@@ -18,6 +18,7 @@ void smp_secondary_prep_el3(void); + + void smp_start_secondaries(void); + void smp_stop_secondaries(bool deep_sleep); ++int smp_park_secondaries_psci(void); + + #define smp_call0(i, f) smp_call4(i, f, 0, 0, 0, 0) + #define smp_call1(i, f, a) smp_call4(i, f, a, 0, 0, 0) +diff --git a/src/start.S b/src/start.S +index aa11e18..a3f35ae 100644 +--- a/src/start.S ++++ b/src/start.S +@@ -11,6 +11,7 @@ + .extern _v_sp0_irq + .extern _v_sp0_fiq + .extern _v_sp0_serr ++.extern _v_lower_sync + .extern _reset_stack + .extern _cpu_reset_c + .extern wdt_reboot +@@ -41,7 +42,7 @@ _vectors_start: + .align 7 + b _v_sp0_serr + .align 7 +- b _v_sp0_sync ++ b _v_lower_sync + .align 7 + b _v_sp0_irq + .align 7 +@@ -103,6 +104,8 @@ _start: + + mov x0, x19 + mov x1, x20 ++ mrs x2, CurrentEL ++ lsr x2, x2, #2 + bl _start_c + b . + +@@ -115,6 +118,7 @@ start_el3: + msr tpidr_el3, xzr + adrp x6, _stack_bot + adr x7, _start_c ++ mov x2, #3 // _start_c: entered at EL3 + b el3_eret_to_el1 + + .globl exc_unk +diff --git a/src/startup.c b/src/startup.c +index 61afe16..167fae8 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -5,6 +5,7 @@ + #include "dockchannel_uart.h" + #include "exception.h" + #include "firmware.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "types.h" +@@ -159,12 +160,15 @@ void dump_boot_args(struct boot_args *ba) + } + + extern void get_device_info(void); +-void _start_c(void *boot_args, void *base) ++void _start_c(void *boot_args, void *base, u64 boot_el) + { + UNUSED(base); + u32 cpu_id = 0; + + memset64(_bss_start, 0, _bss_end - _bss_start); ++ psci_note_boot_el(boot_el); ++ if (psci_reentered()) ++ boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); + diff --git a/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch b/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch new file mode 100644 index 00000000..edfc7592 --- /dev/null +++ b/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch @@ -0,0 +1,84 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 15:56:40 +0100 +Subject: [PATCH] psci: Turn the resident EL3 PSCI on by default + +When m1n1 was entered at EL3 (A7-A9 booted by Pongo) it stays resident +and already implements CPU_ON/CPU_OFF/AFFINITY_INFO for the kernel, but +only when the payload asked for it with psci=1. Without it the kernel +gets the spin table and can neither offline CPUs nor kexec on SMP, so +every loader had to be built with the variable appended. + +Make it the default. psci=0 turns it off again (spin table, no /psci +node), psci=1 is still accepted. Nothing changes where m1n1 is entered +at EL1 or EL2: psci_active() still requires the EL3 entry. + +Assisted-by: LLM +--- + src/kboot.c | 4 ++-- + src/psci.c | 6 +++--- + src/psci.h | 4 ++-- + 3 files changed, 7 insertions(+), 7 deletions(-) + +diff --git a/src/kboot.c b/src/kboot.c +index 7123d60..a5f4238 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -576,7 +576,7 @@ static int dt_set_smbios(void) + } + + /* +- * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * When m1n1 owns EL3 (A7-A9), CPUs boot through m1n1's PSCI + * (see psci.c) instead of the spin table, so the kernel can offline them + * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. + */ +@@ -586,7 +586,7 @@ static int dt_set_psci(void) + int node; + + if (!psci_active()) { +- printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ printf("PSCI: off (m1n1 not entered at EL3, or psci=0)\n"); + return 0; + } + +diff --git a/src/psci.c b/src/psci.c +index 2b8ca0d..25e2a4f 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -28,8 +28,8 @@ struct { + u64 boot_el3; + } psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + +-/* psci=1 from the payload, kept across a re-entry */ +-static u32 psci_mode __attribute__((section(".data"))); ++/* On unless the payload says psci=0; kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))) = 1; + + /* EL3 stacks a CPU gets when it leaves the pen */ + static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); +@@ -177,7 +177,7 @@ void psci_note_boot_el(u64 el) + + void psci_configure(const char *val) + { +- psci_mode = val[0] == '1'; ++ psci_mode = val[0] != '0'; + } + + bool psci_active(void) +diff --git a/src/psci.h b/src/psci.h +index 6769ebe..9a213af 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -50,9 +50,9 @@ + + #ifndef __ASSEMBLER__ + +-/* Payload variable psci=1 */ ++/* Payload variable psci=0 turns PSCI off; anything else (psci=1) leaves it on */ + void psci_configure(const char *val); +-/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++/* m1n1 entered at EL3 and not psci=0: the kernel uses PSCI */ + bool psci_active(void); + /* Called once from _start_c with the EL m1n1 was entered at */ + void psci_note_boot_el(u64 el); diff --git a/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch b/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch new file mode 100644 index 00000000..a1c26412 --- /dev/null +++ b/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch @@ -0,0 +1,286 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 15:58:57 +0100 +Subject: [PATCH] psci: Trap EL1 WFI/WFE to EL3 for a firmware CPU rendezvous + +An ordinary arm64 reboot or power-off does not offline the secondaries +with CPU_OFF: smp_send_stop() leaves them spinning in "wfe; wfi" at EL1 +with interrupts masked, still ON as far as the monitor knows. A9 has no +way to reset a core from another one (Apple and m1n1 power cores down +from the core itself), so SYSTEM_RESET and SYSTEM_OFF need those CPUs to +come to EL3 on their own. + +Give the monitor that hook: a private SiP call, made by m1n1 on the boot +CPU right before it starts a kernel, sets SCR_EL3.TWI and TWE there and +on every CPU that later leaves the pen, and records the AIC and watchdog +bases for later EL3 use. A trapped WFE returns at once (a spurious +wake-up is architectural, and the exception return may have cleared the +exclusive monitor whose event the caller waits for); a trapped WFI waits +at EL3 with IRQ/FIQ routed there only around the wfi, so an interrupt +meant for EL1 still wakes it. While a rendezvous is pending, a trapped +CPU instead parks itself in the pen exactly as CPU_OFF would, cleaning +its L1D on the way. + +The trap enable has no user yet. The CPUs that have been in the pen are +recorded, so a rendezvous knows which ones the kernel was given. + +Assisted-by: LLM +--- + src/psci.c | 56 ++++++++++++++++++++++++++++++++++++++ + src/psci.h | 20 ++++++++++++++ + src/psci_asm.S | 73 ++++++++++++++++++++++++++++++++++++++++++++++++-- + 3 files changed, 147 insertions(+), 2 deletions(-) + +diff --git a/src/psci.c b/src/psci.c +index 25e2a4f..af4f325 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -28,6 +28,24 @@ struct { + u64 boot_el3; + } psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + ++/* ++ * Rendezvous state, one line, only accessed at EL3 with the MMU off. A CPU ++ * that wants all others in the pen (SYSTEM_RESET/SYSTEM_OFF) sets pending; ++ * every other CPU then parks itself at its next trapped WFI/WFE. ++ */ ++struct psci_rdv { ++ volatile u64 pending; ++ volatile u64 traps; /* SCR_EL3.TWI|TWE for CPUs leaving the pen */ ++ volatile u64 cpus; /* MPIDR indices that have been in the pen: handed to the kernel */ ++ u64 aic; ++ u64 wdt; ++ u64 pad[PSCI_CPU_SIZE / 8 - 5]; ++} psci_rdv __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++_Static_assert(sizeof(struct psci_rdv) == PSCI_CPU_SIZE, "psci_rdv layout"); ++_Static_assert(offsetof(struct psci_rdv, pending) == PSCI_RDV_PENDING, "psci_rdv layout"); ++_Static_assert(offsetof(struct psci_rdv, traps) == PSCI_RDV_TRAPS, "psci_rdv layout"); ++ + /* On unless the payload says psci=0; kept across a re-entry */ + static u32 psci_mode __attribute__((section(".data"))) = 1; + +@@ -62,6 +80,10 @@ static s64 psci_cpu_off(void) + if (idx < 0) + return PSCI_RET_DENIED; + ++ psci_line_sync(&psci_rdv); ++ psci_rdv.cpus |= BIT(idx); ++ psci_line_sync(&psci_rdv); ++ + psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); + } + +@@ -128,6 +150,32 @@ static s64 psci_features(u32 fid) + } + } + ++/* Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the pen */ ++static s64 psci_m1n1_handoff(u64 traps, u64 aic, u64 wdt) ++{ ++ u64 bits = traps ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ ++ psci_rdv.pending = 0; ++ psci_rdv.traps = bits; ++ psci_rdv.aic = aic; ++ psci_rdv.wdt = wdt; ++ psci_line_sync(&psci_rdv); ++ ++ msr(SCR_EL3, (mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)) | bits); ++ sysop("isb"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++/* A trapped WFI/WFE saw the rendezvous flag: become CPU_OFF right here */ ++void psci_rendezvous_off(void) ++{ ++ psci_cpu_off(); ++ /* Not a CPU we know (can't happen after the pen saw it): stay out of the way */ ++ for (;;) ++ sysop("wfe"); ++} ++ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + { + u32 fn = fid; +@@ -154,6 +202,8 @@ u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + return 2; /* no trusted OS, nothing to migrate */ + case PSCI_1_0_FN_PSCI_FEATURES: + return psci_features(a1); ++ case PSCI_M1N1_SIP_HANDOFF: ++ return psci_m1n1_handoff(a1, a2, a3); + default: + /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ + return PSCI_RET_NOT_SUPPORTED; +@@ -190,6 +240,12 @@ bool psci_reentered(void) + return reentered; + } + ++void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base) ++{ ++ if (psci_active()) ++ psci_smc(PSCI_M1N1_SIP_HANDOFF, wfx_traps, aic_base, wdt_base); ++} ++ + int psci_affinity_info(u64 mpidr) + { + return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); +diff --git a/src/psci.h b/src/psci.h +index 9a213af..1d3a88e 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -27,6 +27,13 @@ + #define PSCI_0_2_FN64_CPU_ON 0xc4000003 + #define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 + ++/* ++ * m1n1-private SiP call, made by m1n1 itself on the boot CPU right before it ++ * starts a kernel: x1 = 1 to trap EL1 WFI/WFE to EL3 on this and every CPU ++ * that later leaves the pen, x2 = AIC base, x3 = primary watchdog base. ++ */ ++#define PSCI_M1N1_SIP_HANDOFF 0xc200ff00 ++ + #define PSCI_RET_SUCCESS 0 + #define PSCI_RET_NOT_SUPPORTED -1 + #define PSCI_RET_INVALID_PARAMS -2 +@@ -48,6 +55,15 @@ + #define PSCI_CPU_CONTEXT 0x10 + #define PSCI_CPU_SIZE 0x80 + ++/* struct psci_rdv, one line, EL3 only (MMU off) */ ++#define PSCI_RDV_PENDING 0x00 ++#define PSCI_RDV_TRAPS 0x08 ++ ++#define SCR_EL3_IRQ (1 << 1) ++#define SCR_EL3_FIQ (1 << 2) ++#define SCR_EL3_TWI (1 << 12) ++#define SCR_EL3_TWE (1 << 13) ++ + #ifndef __ASSEMBLER__ + + /* Payload variable psci=0 turns PSCI off; anything else (psci=1) leaves it on */ +@@ -67,9 +83,13 @@ int psci_park_cpu(int cpu, u64 mpidr); + /* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ + int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); + ++/* EL1, boot CPU, last thing before the kernel: see PSCI_M1N1_SIP_HANDOFF */ ++void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base); ++ + /* EL3 side */ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); + void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_rendezvous_off(void) __attribute__((noreturn)); + void psci_secondary_entry(void); + void _reentry(void); + +diff --git a/src/psci_asm.S b/src/psci_asm.S +index 14f842f..ff31f3a 100644 +--- a/src/psci_asm.S ++++ b/src/psci_asm.S +@@ -9,13 +9,16 @@ + .extern _cpu_reset_c + .extern psci_handle + .extern psci_persist ++.extern psci_rdv ++.extern psci_rendezvous_off + + .text + + /* + * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) +- * goes to the PSCI dispatcher; everything else, and every exception taken +- * at EL1/EL2, takes the usual path. ++ * goes to the PSCI dispatcher and a trapped WFI/WFE to psci_wfx_entry; ++ * everything else, and every exception taken at EL1/EL2, takes the usual ++ * path. + */ + .globl _v_lower_sync + .type _v_lower_sync, @function +@@ -26,6 +29,8 @@ _v_lower_sync: + b.ne 1f + mrs x16, esr_el3 + lsr x17, x16, #26 ++ cmp x17, #0x01 // EC: trapped WFI/WFE ++ b.eq 2f + cmp x17, #0x17 // EC: SMC from AArch64 + b.ne 1f + tst x16, #0xffff // imm16 == 0: SMCCC +@@ -35,6 +40,59 @@ _v_lower_sync: + 1: + ldp x16, x17, [sp], #16 + b _v_sp0_sync ++2: ++ ldp x16, x17, [sp], #16 ++ b psci_wfx_entry ++ ++/* ++ * WFI/WFE from EL1/EL0, trapped by SCR_EL3.TWI/TWE (see PSCI_M1N1_SIP_HANDOFF). ++ * This is how firmware gets hold of a CPU that the kernel stopped with an ++ * IPI and left spinning in "wfe; wfi" with interrupts masked (an ordinary ++ * reboot does that; only kexec and hotplug use CPU_OFF): while a ++ * SYSTEM_RESET/SYSTEM_OFF rendezvous is pending, the CPU parks itself as if ++ * it had called CPU_OFF. Otherwise: ++ * - WFE completes at once. A spurious WFE wake-up is architectural, and ++ * waiting here could miss the event the caller waits for (the exception ++ * return may clear its exclusive monitor). ++ * - WFI waits here, with IRQ/FIQ routed to EL3 only around the wfi so that ++ * an interrupt for EL1 wakes it (DAIF is masked at EL3, nothing is taken). ++ * Both then return past the instruction. Only x0-x3 are used. ++ */ ++.type psci_wfx_entry, @function ++psci_wfx_entry: ++ stp x0, x1, [sp, #-16]! ++ stp x2, x3, [sp, #-16]! ++ adrp x0, psci_rdv ++ add x0, x0, :lo12:psci_rdv ++ dc civac, x0 ++ dsb sy ++ ldr x1, [x0, #PSCI_RDV_PENDING] ++ cbnz x1, 3f ++ mrs x1, esr_el3 ++ tbnz x1, #0, 2f // ISS.TI bit 0 set: WFE (or WFET) ++ mrs x2, scr_el3 ++ orr x3, x2, #(SCR_EL3_IRQ | SCR_EL3_FIQ) ++ msr scr_el3, x3 ++ isb ++ dsb sy ++ wfi ++ msr scr_el3, x2 ++ isb ++ dc civac, x0 ++ dsb sy ++ ldr x1, [x0, #PSCI_RDV_PENDING] ++ cbnz x1, 3f ++2: ++ mrs x1, elr_el3 ++ add x1, x1, #4 ++ msr elr_el3, x1 ++ ldp x2, x3, [sp], #16 ++ ldp x0, x1, [sp], #16 ++ eret ++3: ++ add sp, sp, #32 ++ bl psci_rendezvous_off ++ b . + + /* + * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the +@@ -106,6 +164,17 @@ psci_pen: + ic iallu + tlbi alle1 + dsb sy ++ ++ // WFI/WFE traps as the boot CPU set them for the kernel (psci_rdv.traps) ++ adrp x3, psci_rdv ++ add x3, x3, :lo12:psci_rdv ++ dc civac, x3 ++ dsb sy ++ ldr x4, [x3, #PSCI_RDV_TRAPS] ++ mrs x1, scr_el3 ++ bic x1, x1, #(SCR_EL3_TWI | SCR_EL3_TWE) ++ orr x1, x1, x4 ++ msr scr_el3, x1 + mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off + movk x1, #0x30d0, lsl #16 + msr sctlr_el1, x1 diff --git a/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch b/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch new file mode 100644 index 00000000..e5dd449e --- /dev/null +++ b/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch @@ -0,0 +1,497 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 16:01:53 +0100 +Subject: [PATCH] psci: Implement SYSTEM_RESET and SYSTEM_OFF with a CPU + rendezvous + +Give the kernel's ordinary reboot and power-off paths somewhere to go. +On SYSTEM_RESET or SYSTEM_OFF the calling CPU sets the rendezvous flag, +sends an event and an AIC IPI to every other CPU the kernel was given, +and waits up to a second for all of them to park in the pen through the +WFI/WFE traps. If one does not arrive, it clears the flag and returns +without having changed anything, so the kernel goes on to its next +restart or power-off method as it does today. + +Once they are parked it disarms the primary watchdog (the kernel, or +systemd's reboot watchdog, may have armed it), stops its timers and +drops the traps. SYSTEM_RESET then turns the EL1 MMU and caches off, +writes back every data cache level by set/way, invalidates the EL1 TLB +and I-cache and restarts m1n1 at EL1h with the original iBoot boot_args, +saved in .data on the first cold start before top_of_memory_alloc() +changes them. psci_reentered() and the new psci_reset_reentered() tell +the restarted m1n1 where it came from. SYSTEM_OFF parks the caller too: +there is no PMU driver here, so this halts rather than powers off. + +The payload variable psci_reset selects what SYSTEM_RESET leads to: boot +(the default, used by the next change), proxy, or 0 for NOT_SUPPORTED +with no WFI/WFE traps. It is reported by PSCI_FEATURES and armed by +kboot_boot() through the private handoff call, which also records the +boot CPU and needs an AIC v1 and a watchdog in the ADT. + +Assisted-by: LLM +--- + src/kboot.c | 1 + + src/payload.c | 2 + + src/psci.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++-- + src/psci.h | 25 +++++- + src/psci_asm.S | 44 +++++++++- + src/startup.c | 1 + + 6 files changed, 280 insertions(+), 13 deletions(-) + +diff --git a/src/kboot.c b/src/kboot.c +index a5f4238..299e388 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -3301,6 +3301,7 @@ int kboot_boot(void *kernel) + + printf("Setting SMP mode to WFE...\n"); + smp_set_wfe_mode(true); ++ psci_handoff(); + printf("Preparing to boot kernel at %p with fdt at %p\n", kernel, dt); + + next_stage.entry = kernel; +diff --git a/src/payload.c b/src/payload.c +index 4ffc9e2..0b7dc14 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -208,6 +208,8 @@ static bool check_var(u8 **p) + enable_tso = val[0] == '1'; + } else if (IS_VAR("psci=")) { + psci_configure(val); ++ } else if (IS_VAR("psci_reset=")) { ++ psci_reset_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +index af4f325..1821ebc 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -1,9 +1,19 @@ + /* SPDX-License-Identifier: MIT */ + + #include "psci.h" ++#include "adt.h" ++#include "aic_regs.h" ++#include "memory.h" + #include "smp.h" ++#include "string.h" + #include "types.h" + #include "utils.h" ++#include "xnuboot.h" ++ ++#define WDT_CTL 0x1c ++ ++/* SYSTEM_RESET/SYSTEM_OFF wait this long for the other CPUs to park */ ++#define RENDEZVOUS_TIMEOUT_US 1000000 + + struct psci_cpu { + volatile u64 state; +@@ -39,7 +49,8 @@ struct psci_rdv { + volatile u64 cpus; /* MPIDR indices that have been in the pen: handed to the kernel */ + u64 aic; + u64 wdt; +- u64 pad[PSCI_CPU_SIZE / 8 - 5]; ++ u64 reset; /* enum psci_reset_mode the kernel was started with */ ++ u64 pad[PSCI_CPU_SIZE / 8 - 6]; + } psci_rdv __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + + _Static_assert(sizeof(struct psci_rdv) == PSCI_CPU_SIZE, "psci_rdv layout"); +@@ -49,11 +60,18 @@ _Static_assert(offsetof(struct psci_rdv, traps) == PSCI_RDV_TRAPS, "psci_rdv lay + /* On unless the payload says psci=0; kept across a re-entry */ + static u32 psci_mode __attribute__((section(".data"))) = 1; + ++/* psci_reset=0|proxy|boot, kept across a re-entry */ ++static u32 psci_reset_mode __attribute__((section(".data"))) = PSCI_RESET_BOOT; ++ ++/* The iBoot boot_args m1n1 was first started with; SYSTEM_RESET starts m1n1 with them again */ ++static struct boot_args reset_boot_args __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ + /* EL3 stacks a CPU gets when it leaves the pen */ + static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); + + static bool psci_have_el3; + static bool reentered; ++static bool reset_reentered; + + /* EL3 side */ + +@@ -129,9 +147,113 @@ static s64 psci_affinity(u64 mpidr, u64 level) + return psci_cpus[idx].state; + } + ++/* ++ * Gets every other CPU the kernel was given into the pen: they park at their ++ * next trapped WFI/WFE (see psci_wfx_entry). A CPU waiting in WFI is woken ++ * with an IPI, one in WFE with an event. If one doesn't arrive in time, ++ * nothing has been changed that the kernel cares about. ++ */ ++static bool psci_rendezvous(int self) ++{ ++ psci_line_sync(&psci_rdv); ++ u64 others = psci_rdv.cpus & ~BIT(self); ++ ++ psci_rdv.pending = 1; ++ psci_line_sync(&psci_rdv); ++ sysop("sev"); ++ ++ for (int i = 0; i < PSCI_MAX_CPUS; i++) { ++ if (!(others & BIT(i))) ++ continue; ++ psci_line_sync(&psci_cpus[i]); ++ if (psci_cpus[i].state != PSCI_AFF_OFF) ++ write32(psci_rdv.aic + AIC_IPI_SEND, AIC_IPI_SEND_CPU(i)); ++ } ++ ++ u64 timeout = timeout_calculate(RENDEZVOUS_TIMEOUT_US); ++ do { ++ bool all = true; ++ ++ for (int i = 0; i < PSCI_MAX_CPUS; i++) { ++ if (!(others & BIT(i))) ++ continue; ++ psci_line_sync(&psci_cpus[i]); ++ if (psci_cpus[i].state != PSCI_AFF_OFF) ++ all = false; ++ } ++ if (all) { ++ psci_rdv.pending = 0; ++ psci_line_sync(&psci_rdv); ++ return true; ++ } ++ sysop("sev"); ++ udelay(100); ++ } while (!timeout_expired(timeout)); ++ ++ psci_rdv.pending = 0; ++ psci_line_sync(&psci_rdv); ++ return false; ++} ++ ++/* Everything that must happen before this CPU leaves the kernel for good */ ++static s64 psci_system_stop(void) ++{ ++ int self = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ psci_line_sync(&psci_rdv); ++ if (psci_rdv.reset == PSCI_RESET_OFF) ++ return PSCI_RET_NOT_SUPPORTED; ++ if (self < 0 || !psci_rendezvous(self)) ++ return PSCI_RET_INTERNAL_FAILURE; ++ ++ msr(CNTP_CTL_EL0, 0); ++ msr(CNTV_CTL_EL0, 0); ++ /* The kernel (or systemd's reboot watchdog) may have armed it */ ++ write32(psci_rdv.wdt + WDT_CTL, 0); ++ read32(psci_rdv.wdt + WDT_CTL); ++ ++ /* ++ * The WFx traps go away with the kernel; the next handoff sets them ++ * again, and the next kboot records which CPUs the next kernel gets. ++ */ ++ psci_rdv.traps = 0; ++ psci_rdv.cpus = 0; ++ psci_line_sync(&psci_rdv); ++ msr(SCR_EL3, mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)); ++ sysop("isb"); ++ return PSCI_RET_SUCCESS; ++} ++ ++/* Returns only if the other CPUs could not be stopped: the kernel tries its next method */ ++static s64 psci_system_reset(void) ++{ ++ s64 ret = psci_system_stop(); ++ ++ if (ret != PSCI_RET_SUCCESS) ++ return ret; ++ ++ psci_system_reset_entry(&reset_boot_args); ++} ++ ++/* No PMU driver here: this halts the SoC with every CPU parked, it does not power off */ ++static s64 psci_system_off(void) ++{ ++ s64 ret = psci_system_stop(); ++ ++ if (ret != PSCI_RET_SUCCESS) ++ return ret; ++ ++ for (;;) ++ sysop("wfi"); ++} ++ + static s64 psci_features(u32 fid) + { + switch (fid) { ++ case PSCI_0_2_FN_SYSTEM_OFF: ++ case PSCI_0_2_FN_SYSTEM_RESET: ++ psci_line_sync(&psci_rdv); ++ return psci_rdv.reset != PSCI_RESET_OFF ? PSCI_RET_SUCCESS : PSCI_RET_NOT_SUPPORTED; + case PSCI_0_2_FN_PSCI_VERSION: + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_CPU_ON: +@@ -144,21 +266,35 @@ static s64 psci_features(u32 fid) + default: + /* + * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make +- * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ * no other SMCs), CPU_SUSPEND and SYSTEM_RESET2. + */ + return PSCI_RET_NOT_SUPPORTED; + } + } + +-/* Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the pen */ +-static s64 psci_m1n1_handoff(u64 traps, u64 aic, u64 wdt) ++/* ++ * Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the ++ * pen, and with them SYSTEM_RESET/SYSTEM_OFF (reset = enum psci_reset_mode). ++ */ ++static s64 psci_m1n1_handoff(u64 reset, u64 aic, u64 wdt) + { +- u64 bits = traps ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ if (reset > PSCI_RESET_PROXY || (reset && (!aic || !wdt))) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ u64 bits = reset ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ int self = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (self < 0) ++ return PSCI_RET_DENIED; + ++ psci_line_sync(&psci_rdv); ++ /* The secondaries were added when kboot parked them */ ++ psci_rdv.cpus |= BIT(self); + psci_rdv.pending = 0; + psci_rdv.traps = bits; + psci_rdv.aic = aic; + psci_rdv.wdt = wdt; ++ psci_rdv.reset = reset; + psci_line_sync(&psci_rdv); + + msr(SCR_EL3, (mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)) | bits); +@@ -202,10 +338,13 @@ u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + return 2; /* no trusted OS, nothing to migrate */ + case PSCI_1_0_FN_PSCI_FEATURES: + return psci_features(a1); ++ case PSCI_0_2_FN_SYSTEM_RESET: ++ return psci_system_reset(); ++ case PSCI_0_2_FN_SYSTEM_OFF: ++ return psci_system_off(); + case PSCI_M1N1_SIP_HANDOFF: + return psci_m1n1_handoff(a1, a2, a3); + default: +- /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ + return PSCI_RET_NOT_SUPPORTED; + } + } +@@ -216,6 +355,7 @@ void psci_note_boot_el(u64 el) + { + /* MMU still off: these .data accesses are uncached, like the writers' */ + if (psci_persist.reentry_flag) { ++ reset_reentered = psci_persist.reentry_flag == PSCI_REENTRY_RESET; + reentered = true; + psci_persist.reentry_flag = 0; + } +@@ -240,10 +380,72 @@ bool psci_reentered(void) + return reentered; + } + +-void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base) ++void psci_reset_configure(const char *val) ++{ ++ if (val[0] == '0') ++ psci_reset_mode = PSCI_RESET_OFF; ++ else if (!strncmp(val, "proxy", 5)) ++ psci_reset_mode = PSCI_RESET_PROXY; ++ else ++ psci_reset_mode = PSCI_RESET_BOOT; ++} ++ ++enum psci_reset_mode psci_reset_mode_get(void) + { +- if (psci_active()) +- psci_smc(PSCI_M1N1_SIP_HANDOFF, wfx_traps, aic_base, wdt_base); ++ return psci_reset_mode; ++} ++ ++bool psci_reset_reentered(void) ++{ ++ return reset_reentered; ++} ++ ++void psci_reset_save_boot_args(void) ++{ ++ if (reentered) ++ return; ++ memcpy(&reset_boot_args, &cur_boot_args, sizeof(reset_boot_args)); ++ dc_civac_range(&reset_boot_args, sizeof(reset_boot_args)); ++} ++ ++/* ++ * The CPU rendezvous needs the AIC v1 IPI (A7-A9) and the primary watchdog ++ * (wdt-version 1 there, nothing else to stop). Without them SYSTEM_RESET and ++ * SYSTEM_OFF stay NOT_SUPPORTED and the kernel uses its own methods. ++ */ ++void psci_handoff(void) ++{ ++ enum psci_reset_mode reset = psci_reset_mode; ++ u64 aic = 0, wdt = 0; ++ int path[8]; ++ ++ if (!psci_active()) ++ return; ++ ++ if (reset != PSCI_RESET_OFF) { ++ int node = adt_path_offset_trace(adt, "/arm-io/aic", path); ++ if (node < 0 || !adt_is_compatible(adt, node, "aic,1") || ++ adt_get_reg(adt, path, "reg", 0, &aic, NULL) < 0) { ++ printf("PSCI: SYSTEM_RESET/SYSTEM_OFF unavailable: no AIC v1\n"); ++ reset = PSCI_RESET_OFF; ++ } ++ node = adt_path_offset_trace(adt, "/arm-io/wdt", path); ++ if (node < 0 || adt_get_reg(adt, path, "reg", 0, &wdt, NULL) < 0) { ++ printf("PSCI: SYSTEM_RESET/SYSTEM_OFF unavailable: no watchdog\n"); ++ reset = PSCI_RESET_OFF; ++ } ++ } ++ ++ if (psci_smc(PSCI_M1N1_SIP_HANDOFF, reset, aic, wdt) != PSCI_RET_SUCCESS) { ++ printf("PSCI: handoff refused, SYSTEM_RESET/SYSTEM_OFF unavailable\n"); ++ psci_smc(PSCI_M1N1_SIP_HANDOFF, PSCI_RESET_OFF, 0, 0); ++ return; ++ } ++ ++ static const char *const names[] = {"off", "warm reboot into the default entry", ++ "warm reboot into the proxy"}; ++ printf("PSCI: SYSTEM_RESET: %s%s\n", names[reset], ++ reset ? "; EL1 WFI/WFE trap to EL3" : ""); + } + + int psci_affinity_info(u64 mpidr) +diff --git a/src/psci.h b/src/psci.h +index 1d3a88e..7c81af7 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -29,8 +29,9 @@ + + /* + * m1n1-private SiP call, made by m1n1 itself on the boot CPU right before it +- * starts a kernel: x1 = 1 to trap EL1 WFI/WFE to EL3 on this and every CPU +- * that later leaves the pen, x2 = AIC base, x3 = primary watchdog base. ++ * starts a kernel: x1 = enum psci_reset_mode (nonzero also traps EL1 WFI/WFE ++ * to EL3 on this and every CPU that later leaves the pen), x2 = AIC base, ++ * x3 = primary watchdog base. + */ + #define PSCI_M1N1_SIP_HANDOFF 0xc200ff00 + +@@ -59,6 +60,10 @@ + #define PSCI_RDV_PENDING 0x00 + #define PSCI_RDV_TRAPS 0x08 + ++/* psci_persist.reentry_flag */ ++#define PSCI_REENTRY_KEXEC 1 ++#define PSCI_REENTRY_RESET 2 ++ + #define SCR_EL3_IRQ (1 << 1) + #define SCR_EL3_FIQ (1 << 2) + #define SCR_EL3_TWI (1 << 12) +@@ -83,13 +88,27 @@ int psci_park_cpu(int cpu, u64 mpidr); + /* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ + int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); + ++/* What SYSTEM_RESET does: payload variable psci_reset=0|proxy|boot (default boot) */ ++enum psci_reset_mode { ++ PSCI_RESET_OFF = 0, /* NOT_SUPPORTED: the kernel resets the SoC its own way */ ++ PSCI_RESET_BOOT = 1, /* back into m1n1, which boots the default boot entry */ ++ PSCI_RESET_PROXY = 2, /* back into m1n1, which only runs the proxy */ ++}; ++void psci_reset_configure(const char *val); ++enum psci_reset_mode psci_reset_mode_get(void); ++/* Set when m1n1 was re-entered by SYSTEM_RESET (psci_reentered() is too) */ ++bool psci_reset_reentered(void); ++/* _start_c: keep the first boot_args, SYSTEM_RESET starts m1n1 with them */ ++void psci_reset_save_boot_args(void); ++ + /* EL1, boot CPU, last thing before the kernel: see PSCI_M1N1_SIP_HANDOFF */ +-void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base); ++void psci_handoff(void); + + /* EL3 side */ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); + void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); + void psci_rendezvous_off(void) __attribute__((noreturn)); ++void psci_system_reset_entry(void *boot_args) __attribute__((noreturn)); + void psci_secondary_entry(void); + void _reentry(void); + +diff --git a/src/psci_asm.S b/src/psci_asm.S +index ff31f3a..61a6a02 100644 +--- a/src/psci_asm.S ++++ b/src/psci_asm.S +@@ -230,10 +230,52 @@ _reentry: + + adrp x1, psci_persist // .reentry_flag + add x1, x1, :lo12:psci_persist +- mov x2, #1 ++ mov x2, #PSCI_REENTRY_KEXEC + str x2, [x1] + dc civac, x1 + dsb sy + + mov x0, x19 + b _start ++ ++/* ++ * psci_system_reset_entry(boot_args): the end of SYSTEM_RESET, at EL3 on the ++ * calling CPU once every other CPU is in the pen (L1D clean) and nothing ++ * else runs. The kernel's EL1 MMU and caches may still be on; its stack and ++ * memory are never used again. Turn EL1 translation and caching off, write ++ * all data caches back, flag the reset and restart m1n1 at EL1h with the ++ * original boot_args, like a cold boot. _start resets the stack and reapplies ++ * the relocations; SP_EL3 goes back to the boot stack. ++ */ ++.globl psci_system_reset_entry ++.type psci_system_reset_entry, @function ++psci_system_reset_entry: ++ msr daifset, #0xf ++ mov x19, x0 ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ isb ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_all ++ tlbi alle1 ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #PSCI_REENTRY_RESET ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ adrp x1, _start ++ add x1, x1, :lo12:_start ++ msr elr_el3, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ adrp x1, _stack_bot_el3 ++ mov sp, x1 ++ mov x0, x19 ++ eret +diff --git a/src/startup.c b/src/startup.c +index 167fae8..4d9ac39 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -171,6 +171,7 @@ void _start_c(void *boot_args, void *base, u64 boot_el) + boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); ++ psci_reset_save_boot_args(); + + adt = + (void *)(((u64)cur_boot_args.devtree) - cur_boot_args.virt_base + cur_boot_args.phys_base); diff --git a/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch b/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch new file mode 100644 index 00000000..a8039549 --- /dev/null +++ b/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch @@ -0,0 +1,298 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 16:03:26 +0100 +Subject: [PATCH] bootmgr: Boot the default entry after a PSCI SYSTEM_RESET + +SYSTEM_RESET brings m1n1 back with every other CPU parked and the +original boot_args, so m1n1 can finish the reboot itself the way the +host's orchard launch starts a system: bring main storage up, take the +first bootable entry in BLS order (the same default the host suggests; +there is still no remembered default) and start it with bootmgr_start(). +The board DTB file name comes from the last launch, which m1n1 now keeps +in .data; if no entry was ever launched (a kernel from the payload, say) +m1n1 doesn't guess the board and stays in the proxy. + +A re-entered m1n1 also skips the stale payload area for the logo, keeps +the picture on screen instead of clearing it and drawing the logo (the +entry's splash replaces it), and leaves SEP alone, which was never +reset. A 96x96 square at the top left shows how far it got: cyan back in +m1n1, green loading the entry, magenta no previous launch, red storage +or entry failure. Any failure ends in the USB proxy, as a cold boot with +nothing to do does; m1n1 never turns it into a cold reset. + +With psci_reset=proxy it stops in the proxy right after the re-entry. + +Assisted-by: LLM +--- + rust/src/bls.rs | 48 +++++++++++++++++++++++++++++ + src/bootmgr.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ + src/bootmgr.h | 2 ++ + src/fb.c | 4 ++- + src/main.c | 16 +++++++--- + 5 files changed, 146 insertions(+), 5 deletions(-) + +diff --git a/rust/src/bls.rs b/rust/src/bls.rs +index 35ddc1b..4570d63 100644 +--- a/rust/src/bls.rs ++++ b/rust/src/bls.rs +@@ -629,6 +629,54 @@ pub unsafe extern "C" fn rust_boot_entries( + text.leak().as_mut_ptr() as *mut c_void + } + ++/// The entry a boot without a host starts: the first bootable one in BLS ++/// order, as the host's `orchard launch` suggests (there is no remembered ++/// default). ++pub fn default_entry(listing: &Listing) -> Option<&Entry> { ++ listing.entries.iter().find(|entry| entry.bootable()) ++} ++ ++/// Writes the id of main storage's default entry (see `default_entry`), ++/// NUL-terminated, to `id` (`size` bytes). Main storage must be up; ++/// `devicetree` may be NULL. Returns 0, or -1 with the reason on the console. ++#[no_mangle] ++pub unsafe extern "C" fn rust_boot_default_entry( ++ devicetree: *const c_char, ++ id: *mut c_char, ++ size: c_size_t, ++) -> c_int { ++ let devicetree = match devicetree.is_null() { ++ true => None, ++ false => unsafe { CStr::from_ptr(devicetree) }.to_str().ok(), ++ }; ++ let listing = read_entries(storage::MainStorage::new, devicetree); ++ log(&listing); ++ let entry = match &listing { ++ Err(error) => { ++ println!("bls: {}", error); ++ return -1; ++ } ++ Ok(listing) => match default_entry(listing) { ++ Some(entry) => entry, ++ None => { ++ println!("bls: no bootable entry"); ++ return -1; ++ } ++ }, ++ }; ++ let name = entry.id.as_bytes(); ++ if name.len() >= size || name.contains(&0) { ++ println!("bls: entry id too long"); ++ return -1; ++ } ++ unsafe { ++ core::ptr::copy_nonoverlapping(name.as_ptr(), id as *mut u8, name.len()); ++ *id.add(name.len()) = 0; ++ } ++ println!("bls: default entry {}", entry.id); ++ 0 ++} ++ + #[repr(C)] + pub struct BootFiles { + kernel: *mut c_void, +diff --git a/src/bootmgr.c b/src/bootmgr.c +index b91b6a6..a746673 100644 +--- a/src/bootmgr.c ++++ b/src/bootmgr.c +@@ -8,6 +8,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "utils.h" +@@ -32,6 +33,32 @@ struct boot_files { + + int rust_boot_entry_load(const char *id, const char *devicetree, bool splash, + struct boot_files *files); ++int rust_boot_default_entry(const char *devicetree, char *id, size_t size); ++ ++/* ++ * The board DTB file name of the last launch (the host's orchard launch ++ * passes it), kept across a SYSTEM_RESET so m1n1 can boot an entry without ++ * the host and without guessing the board. ++ */ ++static struct { ++ bool launched; ++ bool have_devicetree; ++ char devicetree[MAX_NAME]; ++} last_launch __attribute__((section(".data"))); ++ ++/* Warm reset progress, top left */ ++#define MARK_REENTERED 0x00b7ff ++#define MARK_LOADING 0x00c853 ++#define MARK_NO_LAUNCH 0xff00ff ++#define MARK_FAILED 0xff1744 ++ ++static void bootmgr_mark(u32 color) ++{ ++ if (!fb.ptr) ++ return; ++ fb_fill(0, 0, min(fb.width, 96U), min(fb.height, 96U), int2rgb(color)); ++ fb_update(); ++} + + /* Called from Rust: the entry's splash, pixels as b, g, r, 0, centred on black. */ + void bootmgr_paint_splash(const void *pixels, u32 width, u32 height) +@@ -104,6 +131,11 @@ int bootmgr_start(const char *id_arg, const char *devicetree_arg, u64 flags) + return -1; + } + ++ last_launch.launched = true; ++ last_launch.have_devicetree = devicetree_arg; ++ if (devicetree_arg) ++ memcpy(last_launch.devicetree, devicetree, sizeof(devicetree)); ++ + /* The kernel, initrd and DT are m1n1's now, past any limit the host set. */ + heapblock_set_limit(NULL); + +@@ -136,8 +168,57 @@ int bootmgr_start(const char *id_arg, const char *devicetree_arg, u64 flags) + return kboot_boot(kernel); + } + ++/* ++ * m1n1 was restarted by a PSCI SYSTEM_RESET from the kernel: every other CPU ++ * is parked, the watchdog is off and memory is as after a cold start. Do ++ * what the host's orchard launch would: bring up main storage and start the ++ * default entry with the last launch's board DTB. Returns 0 when the kernel ++ * is set up to run, -1 (proxy) otherwise. ++ */ ++int bootmgr_warm_reset(void) ++{ ++ char id[MAX_NAME]; ++ const char *devicetree = last_launch.have_devicetree ? last_launch.devicetree : NULL; ++ ++ bootmgr_mark(MARK_REENTERED); ++ if (psci_reset_mode_get() == PSCI_RESET_PROXY) { ++ printf("bootmgr: warm reset: psci_reset=proxy, not booting\n"); ++ return -1; ++ } ++ if (!last_launch.launched) { ++ printf("bootmgr: warm reset: no boot entry was launched before, not guessing the board\n"); ++ bootmgr_mark(MARK_NO_LAUNCH); ++ return -1; ++ } ++ ++ printf("bootmgr: warm reset: starting the default entry (devicetree %s)\n", ++ devicetree ? devicetree : "from the entry"); ++ if (!main_storage_init()) { ++ printf("bootmgr: warm reset: main storage did not come up\n"); ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ if (rust_boot_default_entry(devicetree, id, sizeof(id)) < 0) { ++ main_storage_shutdown(); ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ bootmgr_mark(MARK_LOADING); ++ if (bootmgr_start(id, devicetree, 0) < 0) { ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ return 0; ++} ++ + #else + ++int bootmgr_warm_reset(void) ++{ ++ printf("bootmgr: not supported in this build\n"); ++ return -1; ++} ++ + int bootmgr_start(const char *id, const char *devicetree, u64 flags) + { + UNUSED(id); +diff --git a/src/bootmgr.h b/src/bootmgr.h +index 8a03baa..f3b86ee 100644 +--- a/src/bootmgr.h ++++ b/src/bootmgr.h +@@ -8,5 +8,7 @@ + #define BOOTMGR_NO_SPLASH BIT(0) + + int bootmgr_start(const char *id, const char *devicetree, u64 flags); ++/* After a PSCI SYSTEM_RESET: boot the default entry the way the last launch did */ ++int bootmgr_warm_reset(void); + + #endif +diff --git a/src/fb.c b/src/fb.c +index ba606b2..0aa88b8 100644 +--- a/src/fb.c ++++ b/src/fb.c +@@ -1,6 +1,7 @@ + /* SPDX-License-Identifier: MIT */ + + #include "fb.h" ++#include "psci.h" + #include "assert.h" + #include "iodev.h" + #include "malloc.h" +@@ -464,14 +465,16 @@ void fb_init(bool clear) + &orig_logo); + } + ++ // After a re-entry from Linux the payload area behind m1n1 is stale ++ bool payload_ok = !psci_reentered(); + u32 *splash, splash_w, splash_h; +- if (payload_splash(&splash, &splash_w, &splash_h) && splash_w <= fb.width && ++ if (payload_ok && payload_splash(&splash, &splash_w, &splash_h) && splash_w <= fb.width && + splash_h <= fb.height) { + custom_logo.ptr = splash; + custom_logo.width = splash_w; + custom_logo.height = splash_h; + logo = &custom_logo; +- } else if (payload_logo(&custom_128, &custom_256)) { ++ } else if (payload_ok && payload_logo(&custom_128, &custom_256)) { + custom_logo = *logo; + if (custom_logo.width == 256) { + custom_logo.ptr = custom_256; +diff --git a/src/main.c b/src/main.c +index 3f32380..8966e41 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -7,6 +7,7 @@ + + #include "adt.h" + #include "aic.h" ++#include "bootmgr.h" + #include "cpufreq.h" + #include "display.h" + #include "exception.h" +@@ -127,6 +128,9 @@ void run_actions(void) + printf("Valid payload found\n"); + return; + } ++ ++ if (psci_reset_reentered() && bootmgr_warm_reset() == 0) ++ return; + fb_set_active(true); + + printf("No valid payload found\n"); +@@ -173,9 +177,11 @@ void m1n1_main(void) + // Kick DCP to sleep, so dodgy monitors which cause reconnect cycles don't cause us to lose the + // framebuffer. + display_shutdown(DCP_SLEEP_IF_EXTERNAL); +- // On idevice we need to always clear, because otherwise it looks scuffed on white devices +- fb_init(!is_mac); +- fb_display_logo(); ++ // On idevice we need to always clear, because otherwise it looks scuffed on white devices. ++ // After a warm reset, keep the picture until the next system's splash. ++ fb_init(!is_mac && !psci_reset_reentered()); ++ if (!psci_reset_reentered()) ++ fb_display_logo(); + #ifdef FB_SILENT_MODE + fb_set_active(!cur_boot_args.video.display); + #else +@@ -184,7 +190,9 @@ void m1n1_main(void) + #endif + + cpufreq_fixup(); +- sep_init(); ++ // SEP was never reset; it is still running what it ran at the first boot ++ if (!psci_reset_reentered()) ++ sep_init(); + #endif + + printf("Initialization complete.\n"); diff --git a/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch b/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch new file mode 100644 index 00000000..08935588 --- /dev/null +++ b/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch @@ -0,0 +1,38 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 12:00:00 +0100 +Subject: [PATCH] kboot: Mark retained hardware after PSCI re-entry + +SYSTEM_RESET re-enters m1n1 without resetting all peripherals. Mark the +next kernel's /chosen with the existing linux,booted-from-kexec boolean +so retained hardware is not mistaken for the cold iBoot baseline. + +Use psci_reentered(), which covers both SYSTEM_RESET and the kexec +trampoline. This also preserves the contract when kboot loads a fresh +BLS devicetree instead of inheriting the outgoing kernel's tree. + +The Orchard kernel's G5P, PMGR, PCIe/ANS and AOP consumers all use this +property to select retained-state handling, with their existing hardware +and certification guards. The only other reader is PowerPC-only. No +kernel consumer needs a separate PSCI-specific property. + +Assisted-by: LLM +--- +diff --git a/src/kboot.c b/src/kboot.c +index 1d4b26f..7dc9874 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -278,6 +278,13 @@ static int dt_set_chosen(void) + printf("FDT: %s = '%s'\n", name, value); + } + ++ /* Re-entry retains peripherals, just like a kernel-to-kernel handoff. */ ++ if (psci_reentered()) { ++ if (fdt_setprop(dt, node, "linux,booted-from-kexec", NULL, 0)) ++ bail("FDT: couldn't mark retained hardware after m1n1 re-entry\n"); ++ printf("FDT: linux,booted-from-kexec (m1n1 re-entry)\n"); ++ } ++ + if (initrd_start && initrd_size) { + if (fdt_setprop_u64(dt, node, "linux,initrd-start", (u64)initrd_start)) + bail("FDT: couldn't set chosen.linux,initrd-start property\n"); diff --git a/boot/m1n1/psci-loader b/boot/m1n1/psci-loader new file mode 100755 index 00000000..c21f1801 --- /dev/null +++ b/boot/m1n1/psci-loader @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Append m1n1 PSCI payload variables to a bare m1n1.bin. + +m1n1 entered at EL3 (A7-A9 from Pongo) provides PSCI by default, including +SYSTEM_RESET as a warm reboot into the default boot entry; a bare m1n1.bin +needs no framing for that. Use this to get the other settings, before +assembling any kernel/DTB/logo payload. The output works with +orchard run/build --loader or launch's loaders/m1n1.bin. m1n1 entered at EL1 +always uses the spin table. +""" +import argparse +from pathlib import Path +import shutil + +RESET = {'boot': b'', 'proxy': b'psci_reset=proxy\n', 'off': b'psci_reset=0\n'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--device', required=True, choices=('j71t',)) + parser.add_argument('--loader', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + group = parser.add_mutually_exclusive_group() + group.add_argument('--system-reset', choices=tuple(RESET), default='boot', + help='boot: warm reboot into the default entry (default); proxy: stop in ' + 'the m1n1 proxy; off: NOT_SUPPORTED, kernel resets the SoC itself') + group.add_argument('--no-psci', action='store_true', help='psci=0: spin table, no PSCI') + args = parser.parse_args() + if not args.loader.is_file() or not args.loader.stat().st_size: + parser.error('input must be a nonempty bare m1n1.bin') + # Exclusive creation: never replace the original or another boot artifact. + with args.loader.open('rb') as source, args.output.open('xb') as output: + shutil.copyfileobj(source, output) + output.write(b'psci=0\n' if args.no_psci else b'psci=1\n' + RESET[args.system_reset]) + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/tests/test_m1n1_psci.py b/tests/test_m1n1_psci.py new file mode 100644 index 00000000..85091696 --- /dev/null +++ b/tests/test_m1n1_psci.py @@ -0,0 +1,81 @@ +"""Host-only framing checks for the J71t PSCI loader variables.""" +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / 'boot/m1n1/psci-loader' + + +class PSCILoader(unittest.TestCase): + def test_payload_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as tmp: + loader, output = Path(tmp) / 'm1n1.bin', Path(tmp) / 'psci.bin' + original = b'bare loader\0' + loader.write_bytes(original) + cmd = [sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + self.assertEqual(loader.read_bytes(), original) + self.assertNotEqual(subprocess.run(cmd, capture_output=True).returncode, 0) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + + def test_reset_and_off_variables(self): + cases = {('--system-reset', 'proxy'): b'psci=1\npsci_reset=proxy\n', + ('--system-reset', 'off'): b'psci=1\npsci_reset=0\n', + ('--system-reset', 'boot'): b'psci=1\n', + ('--no-psci',): b'psci=0\n'} + with tempfile.TemporaryDirectory() as tmp: + loader = Path(tmp) / 'm1n1.bin' + loader.write_bytes(b'bare') + for n, (extra, records) in enumerate(cases.items()): + output = Path(tmp) / f'out{n}.bin' + result = subprocess.run([sys.executable, str(TOOL), '--device', 'j71t', '--loader', + str(loader), '--output', str(output), *extra], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), b'bare' + records) + both = subprocess.run([sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(Path(tmp) / 'x.bin'), '--no-psci', '--system-reset', + 'proxy'], capture_output=True) + self.assertNotEqual(both.returncode, 0) + + def test_series_has_reset_patches(self): + names = sorted(p.name for p in (ROOT / 'boot/m1n1/patches/orchard').glob('*.patch')) + for prefix in ('0015-psci-', '0016-psci-Turn-the-resident-EL3-PSCI-on-by-default', + '0017-psci-Trap-EL1-WFI-WFE', '0018-psci-Implement-SYSTEM_RESET', + '0019-bootmgr-Boot-the-default-entry'): + self.assertTrue(any(n.startswith(prefix) for n in names), prefix) + + def test_manifest_psci_record_keeps_logo_first(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, data in {'loader': b'M1N1', 'dtb': b'DTB', 'kernel': b'KERNEL', + 'initrd': b'INITRD'}.items(): + (root / name).write_bytes(data) + logo = b'm1n1_logo_256128' + bytes(4 * (256 * 256 + 128 * 128)) + (root / 'logo').write_bytes(logo) + records = b'chosen.bootargs=console=tty0\npsci=1\n' + (root / 'profile.json').write_text(json.dumps({'bootargs': records.decode()})) + result = subprocess.run([sys.executable, '-I', str(ROOT / 'src/orchard/image/assemble.py'), + '--manifest', str(root / 'profile.json'), '--loader', str(root / 'loader'), + '--logo', str(root / 'logo'), '--dtb', str(root / 'dtb'), + '--kernel', str(root / 'kernel'), '--initramfs', str(root / 'initrd'), + '--output', str(root / 'payload')], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((root / 'payload').read_bytes().startswith(b'M1N1' + logo + records + b'DTB')) + + def test_board_requires_explicit_opt_in(self): + for device in ([], ['--device', 'j71b']): + result = subprocess.run([sys.executable, str(TOOL), *device, '--loader', 'unused', + '--output', 'unused'], capture_output=True) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() From 190bb139b29337d3dfdb1eed690332d81427bab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 20/33] j71t: One owner for the PMU and RTC wake interrupts The TriStar charger subscribed to the PMU's RTC event 16 for source identification. An acknowledgement there could take the RTC's wake event, and the polled cover inputs had no driver enabled. Give RTC event 16 to the RTC alone and keep the PMU and RTC event split (DT, RTC and USB patches), so one consumer's acknowledgement cannot take the other's wake event. Enable KEYBOARD_GPIO_POLLED for the polled cover inputs. Assisted-by: LLM --- kernel/configs/apple-16k/config | 2 +- kernel/configs/apple-4k/config | 2 +- ...preserve-PMU-and-RTC-interrupt-owner.patch | 42 ++++ ...erve-PMU-and-RTC-interrupt-ownership.patch | 206 ++++++++++++++++++ ...erve-PMU-and-RTC-interrupt-ownership.patch | 36 +++ 5 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch create mode 100644 kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch create mode 100644 kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index 4949d63c..f11529b5 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -2118,7 +2118,7 @@ CONFIG_INPUT_KEYBOARD=y # CONFIG_KEYBOARD_DLINK_DIR685 is not set # CONFIG_KEYBOARD_LKKBD is not set CONFIG_KEYBOARD_GPIO=y -# CONFIG_KEYBOARD_GPIO_POLLED is not set +CONFIG_KEYBOARD_GPIO_POLLED=y # CONFIG_KEYBOARD_TCA8418 is not set # CONFIG_KEYBOARD_MATRIX is not set # CONFIG_KEYBOARD_CHARLIEPLEX is not set diff --git a/kernel/configs/apple-4k/config b/kernel/configs/apple-4k/config index ce7fad3f..f135bd7e 100644 --- a/kernel/configs/apple-4k/config +++ b/kernel/configs/apple-4k/config @@ -2119,7 +2119,7 @@ CONFIG_INPUT_KEYBOARD=y # CONFIG_KEYBOARD_DLINK_DIR685 is not set # CONFIG_KEYBOARD_LKKBD is not set CONFIG_KEYBOARD_GPIO=y -# CONFIG_KEYBOARD_GPIO_POLLED is not set +CONFIG_KEYBOARD_GPIO_POLLED=y # CONFIG_KEYBOARD_TCA8418 is not set # CONFIG_KEYBOARD_MATRIX is not set # CONFIG_KEYBOARD_CHARLIEPLEX is not set diff --git a/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch b/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch new file mode 100644 index 00000000..ff7c75cc --- /dev/null +++ b/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch @@ -0,0 +1,42 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +index 2ce40b469..20de6953f 100644 +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -191,11 +191,11 @@ + interrupt-parent = <&pmu_gpio>; + /* + * The PMU line, ADC done, then the events the iPadOS +- * charger subscribes to. ++ * charger subscribes to, except RTC event 16: Linux gives that ++ * event exclusively to rtc@500, not to source identification. + */ + interrupts = <240 IRQ_TYPE_EDGE_RISING>, + <176 IRQ_TYPE_EDGE_RISING>, +- <16 IRQ_TYPE_EDGE_RISING>, + <40 IRQ_TYPE_EDGE_RISING>, + <41 IRQ_TYPE_EDGE_RISING>, + <42 IRQ_TYPE_EDGE_RISING>, +@@ -218,6 +218,9 @@ + reg = <0x500 0x10>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; ++ interrupt-parent = <&pmu_gpio>; ++ interrupts = <16 IRQ_TYPE_EDGE_RISING>; ++ wakeup-source; + }; + + /* D2365 register 0x31c bit 0 */ diff --git a/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch b/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch new file mode 100644 index 00000000..e1ea9505 --- /dev/null +++ b/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch @@ -0,0 +1,206 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] rtc: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + drivers/rtc/rtc-apple-pmic.c | 127 ++++++++++++++++++++++++++++++++++- + 1 file changed, 126 insertions(+), 1 deletion(-) + +diff --git a/drivers/rtc/rtc-apple-pmic.c b/drivers/rtc/rtc-apple-pmic.c +index ee56618c8..d31d4c716 100644 +--- a/drivers/rtc/rtc-apple-pmic.c ++++ b/drivers/rtc/rtc-apple-pmic.c +@@ -12,6 +12,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -24,22 +25,30 @@ + #include + + #define RTC_COUNTER_SIZE 6 ++#define RTC_ALARM_SIZE 4 ++#define RTC_CONTROL_RUN BIT(0) ++#define RTC_CONTROL_ALARM BIT(6) + + /** + * struct apple_pmic_rtc_regs - RTC Hardware registers + * @control: RTC Control + * @counter: Time Counter ++ * @alarm: Alarm seconds counter ++ * @has_alarm: Alarm registers are present + */ + + struct apple_pmic_rtc_regs { + u32 control; + u32 counter; ++ u32 alarm; ++ bool has_alarm; + }; + + /** + * struct apple_pmic_rtc - RTC driver internal structure + * @rtc: RTC device + * @regmap: regmap used to access registers ++ * @dev: device used for logging and wake setup + * @nvmem_cell: nvmem cell for offset + * @regs: hardware registers + * @base: RTC base address +@@ -183,17 +192,107 @@ static int apple_pmic_rtc_read_time(struct device *dev, struct rtc_time *tm) + return 0; + } + ++static int apple_pmic_rtc_alarm_irq_enable(struct device *dev, ++ unsigned int enabled) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ ++ /* AppleD2365PMU 0xfffffff006bbbc1c writes these exact values. */ ++ return regmap_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_RUN | ++ (enabled ? RTC_CONTROL_ALARM : 0)); ++} ++ ++static int apple_pmic_rtc_read_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ unsigned int control; ++ time64_t secs; ++ int rc; ++ ++ rc = regmap_bulk_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ rc = regmap_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, &control); ++ if (rc) ++ return rc; ++ ++ secs = (time64_t)get_unaligned_le32(buf) + ap_rtc->offset; ++ rtc_time64_to_tm(secs, &alarm->time); ++ alarm->enabled = !!(control & RTC_CONTROL_ALARM); ++ alarm->pending = 0; ++ return 0; ++} ++ ++static int apple_pmic_rtc_set_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ time64_t secs; ++ int rc; ++ ++ if (!alarm->enabled) ++ return apple_pmic_rtc_alarm_irq_enable(dev, 0); ++ ++ secs = rtc_tm_to_time64(&alarm->time) - ap_rtc->offset; ++ if (secs <= 0 || secs > U32_MAX) ++ return -ERANGE; ++ put_unaligned_le32(secs, buf); ++ ++ /* iOS programs 0x508 first, then enables the one-shot at 0x500. */ ++ rc = regmap_bulk_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ return apple_pmic_rtc_alarm_irq_enable(dev, 1); ++} ++ ++static irqreturn_t apple_pmic_rtc_alarm_irq(int irq, void *data) ++{ ++ struct apple_pmic_rtc *ap_rtc = data; ++ int rc; ++ ++ /* The PMU event read is the acknowledge; iOS then clears alarm bit 6. */ ++ rc = regmap_update_bits(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_ALARM, 0); ++ if (rc) ++ dev_warn_ratelimited(ap_rtc->dev, ++ "failed to disable fired alarm: %d\n", rc); ++ rtc_update_irq(ap_rtc->rtc, 1, RTC_IRQF | RTC_AF); ++ return IRQ_HANDLED; ++} ++ + static const struct rtc_class_ops apple_pmic_rtc_ops = { + .read_time = apple_pmic_rtc_read_time, + .set_time = apple_pmic_rtc_set_time, + }; + ++static const struct rtc_class_ops apple_pmic_rtc_alarm_ops = { ++ .read_time = apple_pmic_rtc_read_time, ++ .set_time = apple_pmic_rtc_set_time, ++ .read_alarm = apple_pmic_rtc_read_alarm, ++ .set_alarm = apple_pmic_rtc_set_alarm, ++ .alarm_irq_enable = apple_pmic_rtc_alarm_irq_enable, ++}; ++ + static const struct apple_pmic_rtc_regs amber_regs = { + .counter = 0x6 + }; + + static const struct apple_pmic_rtc_regs antigua_regs = { +- .counter = 0x2 ++ .control = 0x0, ++ .counter = 0x2, ++ .alarm = 0x8, ++ .has_alarm = true, + }; + + static const struct apple_pmic_rtc_regs calpe_regs = { +@@ -230,6 +329,7 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + struct device_node *node = pdev->dev.of_node; + const struct of_device_id *match; + struct apple_pmic_rtc *ap_rtc; ++ int irq; + int rc; + + match = of_match_node(apple_pmic_id_table, node); +@@ -264,6 +364,31 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + ap_rtc->rtc->ops = &apple_pmic_rtc_ops; + ap_rtc->rtc->range_min = INT_MIN; + ap_rtc->rtc->range_max = INT_MAX; ++ if (ap_rtc->regs->has_alarm) { ++ irq = platform_get_irq_optional(pdev, 0); ++ if (irq == -EPROBE_DEFER) ++ return irq; ++ if (irq > 0) { ++ rc = devm_request_threaded_irq(&pdev->dev, irq, NULL, ++ apple_pmic_rtc_alarm_irq, ++ IRQF_ONESHOT, ++ "apple-pmic-rtc", ap_rtc); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot request alarm IRQ\n"); ++ device_init_wakeup(&pdev->dev, true); ++ rc = devm_pm_set_wake_irq(&pdev->dev, irq); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot set alarm wake IRQ\n"); ++ ap_rtc->rtc->ops = &apple_pmic_rtc_alarm_ops; ++ dev_info(&pdev->dev, ++ "D2365 alarm at 0x%03x on PMU event IRQ %d\n", ++ ap_rtc->base + ap_rtc->regs->alarm, irq); ++ } else if (irq != -ENXIO) { ++ return irq; ++ } ++ } + + return devm_rtc_register_device(ap_rtc->rtc); + } diff --git a/kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch b/kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch new file mode 100644 index 00000000..58875bd3 --- /dev/null +++ b/kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch @@ -0,0 +1,36 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] usb: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + drivers/usb/gadget/composite.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 1da3effb2..14e87fbc8 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -968,7 +968,7 @@ static int composite_apple_current_limit(struct usb_composite_dev *cdev, + u16 w_value, u16 w_index) + { + cdev->apple_extra_ma = (s16)w_index; +- DBG(cdev, "Apple host current grant: extra %d mA, suspend %d mA\n", ++ dev_info(&cdev->gadget->dev, "Apple host current grant: extra %d mA, suspend %d mA\n", + (s16)w_index, (s16)w_value); + composite_vbus_draw(cdev, cdev->vbus_draw_ma); + return 0; +@@ -2221,7 +2221,7 @@ unknown: + goto check_value; + } + +- if (ctrl->bRequestType == (USB_DIR_OUT | USB_TYPE_VENDOR | ++ if ((ctrl->bRequestType & ~USB_DIR_IN) == (USB_TYPE_VENDOR | + USB_RECIP_DEVICE) && + ctrl->bRequest == USB_APPLE_REQ_CURRENT_LIMIT && !w_length && + le16_to_cpu(cdev->desc.idVendor) == USB_APPLE_VENDOR_ID) { From d2247b1c61e0af5811b670e2e22c4e370cb7784f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 21/33] PCI/NVMe: Keep the ANS handoff across kexec, cold-reset when it fails On J71t the NVMe controller (ANS) sits behind the H9P PCIe port, and resetting that port loses the controller until a cold boot. A kexec therefore has to hand over a stopped but powered ANS, with its DART translations retained, or the next kernel loses its root filesystem. The first version of this handoff panicked when a step failed, which with panic=0 is an indefinite hang. Keep the certified ANS kexec handoff: DT and bindings describe it, the DART, H9P and NVMe drivers stop queues and translations in order and the next kernel picks them up, and core/printk export what the modules need. Every failure on the outgoing side now ends in a native cold reset through apple_cold_reset() instead of a panic, with explicit returns after it. The port is handed over powered, with PERST# asserted and the link stopped, and the incoming kernel brings it up with the cold-boot sequence (port power cycle, NVMe mode, port enable, then link training). Retraining the powered port by PERST# alone, as iOS never does, left the root port in Detect.Quiet and cost a timeout on every warm boot. Assisted-by: LLM --- ...keep-the-certified-ANS-kexec-handoff.patch | 32 + ...keep-the-certified-ANS-kexec-handoff.patch | 26 + ...sg_dump_desc-for-module-cold-reset-d.patch | 27 + ...u-keep-the-certified-ANS-kexec-hando.patch | 28 + ...-keep-the-certified-ANS-kexec-handof.patch | 56 ++ ...keep-the-certified-ANS-kexec-handoff.patch | 28 + ...keep-the-certified-ANS-kexec-handoff.patch | 668 ++++++++++++++++++ ...keep-the-certified-ANS-kexec-handoff.patch | 337 +++++++++ ...eset-after-an-uncertified-kexec-hand.patch | 85 +++ ...eset-after-unsafe-ANS-kexec-shutdown.patch | 76 ++ ...se-explicit-returns-after-cold-reset.patch | 62 ++ 11 files changed, 1425 insertions(+) create mode 100644 kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch create mode 100644 kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch create mode 100644 kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch create mode 100644 kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch create mode 100644 kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch create mode 100644 kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch create mode 100644 kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch create mode 100644 kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch create mode 100644 kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch create mode 100644 kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch create mode 100644 kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch diff --git a/kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..dab3a029 --- /dev/null +++ b/kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,32 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..0bc19727d 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -11,6 +11,14 @@ + #include "s8003.dtsi" + #include "s800x-ipad5.dtsi" + ++&pcie0 { ++ apple,ans-kexec-handoff; ++}; ++ ++&pcie0_dart0 { ++ apple,ans-dart-handoff; ++}; ++ + / { + compatible = "apple,j71t", "apple,s8003", "apple,arm-platform"; + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; diff --git a/kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..c3550de8 --- /dev/null +++ b/kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + kernel/kexec_core.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c +index dc770b9a6..819650bcd 100644 +--- a/kernel/kexec_core.c ++++ b/kernel/kexec_core.c +@@ -53,6 +53,7 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); + + /* Flag to indicate we are going to kexec a new kernel */ + bool kexec_in_progress = false; ++EXPORT_SYMBOL_GPL(kexec_in_progress); + + bool kexec_file_dbg_print; + diff --git a/kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch b/kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch new file mode 100644 index 00000000..054cba10 --- /dev/null +++ b/kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch @@ -0,0 +1,27 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:54:33 +0100 +Subject: [PATCH] printk: export kmsg_dump_desc for module cold-reset + diagnostics + +The VXD shutdown fail-safe is a GPL module. Allow it to dump emergency +evidence before a cold reset, matching the built-in GPU and PCIe paths. +No printk behavior changes. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index fcc51b699..a812ec1e5 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4882,6 +4882,7 @@ void kmsg_dump_desc(enum kmsg_dump_reason reason, const char *desc) + } + rcu_read_unlock(); + } ++EXPORT_SYMBOL_GPL(kmsg_dump_desc); + + /** + * kmsg_dump_get_line - retrieve one kmsg log line diff --git a/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch b/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch new file mode 100644 index 00000000..e442e50e --- /dev/null +++ b/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: iommu: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/iommu/apple,dart.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml +index df72722fc..adcbd21fc 100644 +--- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml ++++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml +@@ -21,6 +21,10 @@ description: |+ + fault conditions. + + properties: ++ apple,ans-dart-handoff: ++ type: boolean ++ description: Allow the storage endpoint to retire and rebuild this DART stream during kexec. ++ + compatible: + oneOf: + - enum: diff --git a/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch b/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch new file mode 100644 index 00000000..e5204e73 --- /dev/null +++ b/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch @@ -0,0 +1,56 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] iommu/apple-dart: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + drivers/iommu/apple-dart.c | 24 ++++++++++++++++++++++++ + 1 file changed, 24 insertions(+) + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 1ee6add55..5aff4bc3e 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -35,6 +35,7 @@ + #include + + #include ++#include + + #include "dma-iommu.h" + +@@ -1009,6 +1010,29 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++int apple_ans_dart_stop(struct device *dev) ++{ ++ struct iommu_domain *domain = iommu_get_domain_for_dev(dev); ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ int i; ++ ++ if (!domain || !cfg || ++ !cfg->stream_maps[0].dart || ++ !of_property_read_bool(cfg->stream_maps[0].dart->dev->of_node, ++ "apple,ans-dart-handoff") || ++ (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) ++ return -ENODEV; ++ for_each_stream_map(i, cfg, map) { ++ if (map->dart->hw->type != DART_S5L8960X) ++ return -EOPNOTSUPP; ++ apple_dart_hw_disable_dma(map); ++ } ++ dev_info(dev, "Apple ANS: DART stopped for handoff\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_stop); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { diff --git a/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..c4b7b0ee --- /dev/null +++ b/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: pci: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +index f6497a4e1..f9b72ce26 100644 +--- a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml ++++ b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +@@ -140,6 +140,10 @@ allOf: + - apple,nvme-hmb + + properties: ++ apple,ans-kexec-handoff: ++ type: boolean ++ description: Retain storage port power across a certified stopped-DMA kexec handoff. ++ + compatible: + oneOf: + - enum: diff --git a/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..e030657f --- /dev/null +++ b/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,668 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] PCI: apple-h9p: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +The storage port is handed over powered, with its reference clock running, +PERST# asserted and the LTSSM stopped: the state the boot loader leaves it +in, except that the link is down. The incoming kernel brings it up as on a +cold boot, with the full port power cycle, NVMe mode and the port enable +sequence before the link is trained. iOS trains this link only after that +sequence and never re-trains a powered port by toggling PERST# alone; +doing so after a kexec leaves the root port in Detect.Quiet with the LTSSM +enable not latched. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 459 +++++++++++++++++++++--- + include/linux/apple-ans.h | 23 ++ + 2 files changed, 432 insertions(+), 50 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 15f7a0cbe..f0095394a 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -15,6 +15,8 @@ + + #include + #include ++#include ++#include + #include + #include + #include +@@ -38,12 +40,15 @@ + #include + #include + #include ++#include + #include + #include + #include ++#include + + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -248,7 +253,7 @@ + #define S8003_PORT_LINKCDMSTS_LTSSM GENMASK(14, 9) + #define S8003_LTSSM_L0 0x11 + +-/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (j71t) */ ++/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (S8003) */ + #define S8003_PERST_TO_REFCLK_US 60 + #define S8003_REFCLK_TO_PERST_US 100 + #define S8003_PERST_HOLD_MS 100 +@@ -269,7 +274,7 @@ + + /* + * NVMMU invalidate FIFO status, in the port 0 block (iBoot sets the live ADT +- * nvme-mmu0 invalidate-fifo-workaround = 1 on j71t). iOS (AppleNVMeMMU, ++ * nvme-mmu0 invalidate-fifo-workaround = 1 on S8003). iOS (AppleNVMeMMU, + * 0xfffffff005eebc9c) waits up to 20 ms for it to drain before every + * invalidate and never reads the invalidate register back. + */ +@@ -429,7 +434,7 @@ static const struct apple_h9p_tunable t7000_port_tunables[] = { + }; + + /* +- * S8003 tunables, which iBoot puts in the live ADT; copied from the j71t ++ * S8003 tunables, which iBoot puts in the live ADT; copied from the S8003 + * live ADT (iOS 16.7.10 iBoot), where both bridges carry the same values. + * apcie-phy-tunables and apcie-common-tunables apply after the common init, + * in that order. +@@ -582,6 +587,7 @@ struct apple_h9p_tunables { + struct apple_h9p_nvmmu { + struct apple_h9p_pcie *pcie; + void __iomem *base; ++ bool poisoned; + /* Programmed: the ANS glue may map through it */ + bool ready; + /* Serializes the invalidate FIFO wait and the invalidate write */ +@@ -711,6 +717,7 @@ struct apple_h9p_pcie { + u32 late_ports; + unsigned int late_tries; + struct delayed_work late_work; ++ bool ans_handoff_stopped; + + struct apple_h9p_nvmmu nvmmu; + +@@ -1554,6 +1561,64 @@ static irqreturn_t apple_h9p_nvmmu_irq(int irq, void *data) + return IRQ_HANDLED; + } + ++static int apple_h9p_nvmmu_program(struct apple_h9p_pcie *pcie) ++{ ++ struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; ++ u32 val; ++ int ret; ++ ++ if (pcie->hw->nvmmu_inv_fifo) { ++ /* ++ * iOS's S8003 order: the invalidate FIFO drained, the old SART ++ * aperture off before its address changes, the zeroed scratch ++ * page and TCBs visible, then scratch page, TCB base, init, SART. ++ */ ++ ret = readl_poll_timeout(pcie->base_port[0] + ++ S8003_PORT_NVMMU_INV_FIFO, val, ++ val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ dma_wmb(); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ } else { ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ } ++ h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, ++ nvmmu->base + H9P_NVMMU_TCB_CTRL); ++ ++ ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, ++ pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ if (ret) ++ return ret; ++ ++ h9p_writel_flush(nvmmu->va_base - 0x80000000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_BASE); ++ h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, ++ H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_END); ++ h9p_writel_flush(nvmmu->pa_base >> 20, ++ nvmmu->base + H9P_NVMMU_SART_PA_BASE); ++ h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ ++ return 0; ++} ++ + static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + { + struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; +@@ -1561,7 +1626,7 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + struct device_node *mem_np; + struct resource res; + resource_size_t size; +- u32 iova, val; ++ u32 iova; + int irq; + int ret; + +@@ -1630,55 +1695,10 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + } + + spin_lock_init(&nvmmu->inv_lock); +- if (pcie->hw->nvmmu_inv_fifo) { +- /* +- * iOS's S8003 order: the invalidate FIFO drained, the old SART +- * aperture off before its address changes, the zeroed scratch +- * page and TCBs visible, then scratch page, TCB base, init, SART. +- */ +- ret = readl_poll_timeout(pcie->base_port[0] + +- S8003_PORT_NVMMU_INV_FIFO, val, +- val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, +- 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); +- if (ret) +- goto out_put_node; +- h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); +- dma_wmb(); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- } else { +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- } +- h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, +- nvmmu->base + H9P_NVMMU_TCB_CTRL); +- +- ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, +- pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ ret = apple_h9p_nvmmu_program(pcie); + if (ret) + goto out_put_node; + +- h9p_writel_flush(nvmmu->va_base - 0x80000000U, +- nvmmu->base + H9P_NVMMU_SART_VA_BASE); +- h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, +- H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, +- nvmmu->base + H9P_NVMMU_SART_VA_END); +- h9p_writel_flush(nvmmu->pa_base >> 20, +- nvmmu->base + H9P_NVMMU_SART_PA_BASE); +- h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); +- + irq = platform_get_irq_byname_optional(pcie->pdev, "nvmmu"); + if (irq > 0) { + ret = devm_request_irq(dev, irq, apple_h9p_nvmmu_irq, 0, +@@ -1726,6 +1746,7 @@ static int apple_h9p_nvmmu_invalidate(struct apple_h9p_pcie *pcie, + spin_unlock_irqrestore(&nvmmu->inv_lock, flags); + + if (ret) { ++ WRITE_ONCE(nvmmu->poisoned, true); + dev_err_ratelimited(pcie->dev, + "NVMMU invalidate FIFO not empty (%#x), tag %u\n", + val, tag); +@@ -1781,6 +1802,8 @@ int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + nvmmu = &pcie->nvmmu; + if (!nvmmu->ready) + return -EOPNOTSUPP; ++ if (READ_ONCE(nvmmu->poisoned)) ++ return -EIO; + + tcb = (u32 *)nvmmu->tcb + tag * H9P_NVMMU_TCB_DWORDS; + sgl = (u32 *)nvmmu->tcb_sgl + tag * H9P_NVMMU_SGL_WORDS; +@@ -2705,6 +2728,8 @@ static int apple_s8003_power_cycle_port(struct apple_h9p_pcie *pcie, + return ret; + } + ++ if (!port && pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); + apple_s8003_port_off(pcie, port); + usleep_range(S8003_POWER_OFF_SETTLE_US, S8003_POWER_OFF_SETTLE_US + 1000); + ret = apple_h9p_wait(rc + S8003_RC_PORT_CTL(port), +@@ -2887,6 +2912,18 @@ static int apple_s8003_start_link(struct apple_h9p_pcie *pcie, + return ret; + } + ++static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *rc = pcie->base_rc; ++ ++ return (readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_ENABLE) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_ON) && ++ (readl(rc + S8003_RC_PORT_REFCLK(0)) & S8003_RC_PORT_REFCLK_ON) && ++ !(readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_RESET) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_REFCLK_NOGATE) && ++ (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -3059,6 +3096,11 @@ static void apple_s8003_late_ports_work(struct work_struct *work) + unsigned int port; + int ret; + ++ /* Lab 0019: no secondary bus publication without storage bus 1. */ ++ if (of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") && ++ !(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return; ++ + for_each_set_bit(port, &pending, H9P_NUM_PORTS) { + ret = apple_s8003_setup_endpoint_port(pcie, port); + if (ret == -EPROBE_DEFER) { +@@ -3418,6 +3460,314 @@ static void apple_h9p_pcie_pwrctrl_cleanup(void *data) + pci_pwrctrl_destroy_devices(pcie->dev); + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static struct apple_h9p_pcie *apple_ans_endpoint_host(struct device *dev) ++{ ++ struct device *host = dev; ++ struct apple_h9p_pcie *pcie; ++ ++ while (host && host->bus == dev->bus) ++ host = host->parent; ++ if (!host || !host->parent) ++ return NULL; ++ pcie = apple_h9p_pcie_lookup(host->parent); ++ if (!pcie || !of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") || ++ apple_h9p_pcie_device_port(pcie, dev) != 0) ++ return NULL; ++ return pcie; ++} ++ ++ ++/* release6: caller froze and synchronized I/O, before SHN. */ ++/* Lab 0018: caller synchronized queues and stopped CC/RDY before this. */ ++int apple_ans_nvmmu_reset(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ struct apple_h9p_nvmmu *nvmmu; ++ int ret; ++ ++ if (!pcie || !(pcie->enabled_ports & BIT(0))) ++ return -ENODEV; ++ nvmmu = &pcie->nvmmu; ++ if (!nvmmu->ready) ++ return -ENODEV; ++ memset(nvmmu->tcb, 0, nvmmu->tcb_size); ++ memset(nvmmu->tcb_table, 0, nvmmu->tcb_table_size); ++ memset(nvmmu->tcb_sgl, 0, nvmmu->tcb_sgl_size); ++ ret = apple_h9p_nvmmu_program(pcie); ++ WRITE_ONCE(nvmmu->poisoned, !!ret); ++ if (ret) ++ return ret; ++ bitmap_zero(nvmmu->mapped, APPLE_ANS_NVMMU_MAX_REQS); ++ dev_info(dev, "NVMMU: stopped-controller cache/SART rebuilt\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_reset); ++ ++bool apple_ans_is_s8003(struct device *dev) ++{ ++ return apple_ans_endpoint_host(dev) != NULL; ++} ++EXPORT_SYMBOL_GPL(apple_ans_is_s8003); ++ ++int apple_ans_link_up(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -ENODEV; ++ if (!(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return -ENOLINK; ++ return !!(readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & ++ S8003_PORT_LINKSTS_UP); ++} ++EXPORT_SYMBOL_GPL(apple_ans_link_up); ++ ++/* Host-only evidence, no endpoint config/BAR reads or invalidate register. */ ++void apple_ans_host_dump(struct device *dev, const char *reason) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return; ++ dev_info(dev, "ANS host %s: ports %#x link %#x LTSSM %#x FIFO %#x poisoned %u mapped %*pb\n", ++ reason, pcie->enabled_ports, ++ readl(pcie->base_port[0] + H9P_PORT_LINKSTS), ++ readl(pcie->base_port[0] + H9P_PORT_LTSSMCTL), ++ readl(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO), ++ READ_ONCE(pcie->nvmmu.poisoned), APPLE_ANS_NVMMU_MAX_REQS, ++ pcie->nvmmu.mapped); ++} ++EXPORT_SYMBOL_GPL(apple_ans_host_dump); ++ ++int apple_ans_nvmmu_drain(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ u32 fifo; ++ int ret; ++ ++ if (!pcie || !pcie->nvmmu.ready) ++ return -ENODEV; ++ if (!bitmap_empty(pcie->nvmmu.mapped, APPLE_ANS_NVMMU_MAX_REQS)) ++ return -EBUSY; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ dev_info(dev, "Apple ANS: outgoing NVMMU FIFO empty check (20ms) result %d\n", ret); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_drain); ++ ++int apple_ans_handoff_stopped(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ unsigned int tag; ++ u32 fifo; ++ int ret; ++ ++ if (!pcie) ++ return -ENODEV; ++ /* All queues are drained and BME is clear before retiring translations. */ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } ++ /* The last invalidate also has to leave the FIFO before certification. */ ++ ret = readl_poll_timeout_atomic(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 1, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ ret = apple_ans_dart_stop(dev); ++ if (!ret) ++ WRITE_ONCE(pcie->ans_handoff_stopped, true); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_stopped); ++ ++int apple_ans_handoff_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_handoff_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); ++ ++static void apple_h9p_s8003_shutdown(struct platform_device *pdev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ unsigned int port; ++ ++ if (!kexec_in_progress || !pcie || ++ !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) ++ return; ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!READ_ONCE(pcie->ans_handoff_stopped) || ++ (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && ++ !apple_s8003_port_powered(pcie))) ++ panic("Apple ANS: refusing uncertified powered handoff"); ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; ++ ++ if (!(pcie->enabled_ports & BIT(port))) ++ continue; ++ /* release6: block root routing, retain refclk, then isolate. */ ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[port], 1)) ++ panic("Apple PCIe: PERST failed during handoff"); ++ if (apple_s8003_port_stop(pcie, port)) ++ panic("Apple PCIe: LTSSM stop failed during handoff"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (!port && pcie->nvmmu.ready) { ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ } ++ if (!port && !of_property_read_bool(pcie->dev->of_node, ++ "apple,ans-kexec-handoff")) ++ apple_s8003_port_off(pcie, port); ++ if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && ++ (!apple_s8003_port_powered(pcie) || ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) ++ panic("Apple ANS: outgoing powered port certificate failed"); ++ pcie->enabled_ports &= ~BIT(port); ++ } ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++} ++ ++static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *cfg = pcie->base_config; ++ ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!(pcie->enabled_ports & BIT(0))) ++ return; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[0], 1) || ++ apple_s8003_port_stop(pcie, 0)) ++ panic("ANS capture isolation failed"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ pcie->enabled_ports &= ~BIT(0); ++} ++ ++struct apple_h9p_crash_state { ++ struct apple_h9p_pcie *pcie; ++ int result; ++ bool busy; ++ bool found; ++}; ++ ++/* No NVMe driver, queues, SHN, or old DMA buffers are needed in phoenix. */ ++static int apple_h9p_crash_stop_ans(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_crash_state *state = data; ++ struct apple_h9p_pcie *pcie = state->pcie; ++ void __iomem *bar; ++ u32 csts = 0; ++ int i, ret; ++ ++ if (pdev->vendor != PCI_VENDOR_ID_APPLE || ++ ((pdev->class >> 8) != PCI_CLASS_STORAGE_EXPRESS && ++ (pdev->class >> 8) != PCI_CLASS_STORAGE_OTHER)) ++ return 0; ++ state->found = true; ++ if (pdev->driver) { ++ state->busy = true; ++ return 1; ++ } ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ state->result = -ENOLINK; ++ return 0; ++ } ++ ret = pci_enable_device_mem(pdev); ++ if (ret) { ++ state->result = ret; ++ return 0; ++ } ++ bar = pci_iomap_range(pdev, 0, 0, SZ_8K); ++ if (!bar) { ++ ret = -ENOMEM; ++ goto disable; ++ } ++ /* An unknown queue must never be rung or sent a shutdown notification. */ ++ dev_info(pcie->dev, "phoenix ANS: CC %#x CSTS %#x, clearing EN without SHN\n", ++ readl(bar + NVME_REG_CC), readl(bar + NVME_REG_CSTS)); ++ writel(0, bar + NVME_REG_CC); ++ ret = -ETIMEDOUT; ++ for (i = 0; i < 2000; i++) { ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ ret = -ENOLINK; ++ break; ++ } ++ csts = readl(bar + NVME_REG_CSTS); ++ if (csts == U32_MAX) { ++ ret = -EIO; ++ break; ++ } ++ if (!(csts & NVME_CSTS_RDY)) { ++ ret = 0; ++ break; ++ } ++ usleep_range(1000, 1100); ++ } ++ dev_info(pcie->dev, "phoenix ANS: disable result %d CSTS %#x; blocking bus master\n", ++ ret, csts); ++ pci_iounmap(pdev, bar); ++disable: ++ pci_clear_master(pdev); ++ pci_disable_device(pdev); ++ state->result = ret; ++ return 0; ++} ++ ++static ssize_t crash_quiesce_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ struct apple_h9p_crash_state state = { .pcie = pcie }; ++ ++ if (!pcie || !is_kdump_kernel() || ++ !of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !sysfs_streq(buf, "1")) ++ return -EPERM; ++ /* The endpoint gate must stay closed; don't park under a live driver. */ ++ if (pcie->late_ports || work_busy(&pcie->late_work.work)) ++ return -EBUSY; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_crash_stop_ans, &state); ++ if (state.busy) ++ return -EBUSY; ++ /* Isolation is still required after a bounded RDY timeout. */ ++ /* Wi-Fi stays linked until its own shutdown callback has run. */ ++ apple_h9p_capture_park(pcie); ++ dev_info(dev, "phoenix: ANS/PCIe quiesced (controller found %d result %d)\n", ++ state.found, state.result); ++ return count; ++} ++static DEVICE_ATTR_WO(crash_quiesce); ++ ++static struct attribute *apple_h9p_crash_attrs[] = { ++ &dev_attr_crash_quiesce.attr, ++ NULL, ++}; ++ ++static const struct attribute_group apple_h9p_crash_group = { ++ .attrs = apple_h9p_crash_attrs, ++}; ++ ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3426,6 +3776,7 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + u32 doorbell; + int ret; + ++ + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; +@@ -3578,6 +3929,13 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (is_kdump_kernel() && of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) { ++ ret = devm_device_add_group(dev, &apple_h9p_crash_group); ++ if (ret) ++ return ret; ++ } ++ ++ + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +@@ -3660,6 +4018,7 @@ MODULE_DEVICE_TABLE(of, apple_h9p_pcie_of_match); + + static struct platform_driver apple_h9p_pcie_driver = { + .probe = apple_h9p_pcie_probe, ++ .shutdown = apple_h9p_s8003_shutdown, + .driver = { + .name = "pcie-apple-h9p", + .of_match_table = apple_h9p_pcie_of_match, +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index a59861010..10be3ddef 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -17,11 +17,34 @@ struct device; + * Map @npages pages for FlatDMA request @tag, or retire the tag if @npages is + * zero. @to_memory: the controller writes the pages (an NVMe read). + */ ++#if IS_REACHABLE(CONFIG_APPLE_DART) ++int apple_ans_dart_stop(struct device *dev); ++#else ++static inline int apple_ans_dart_stop(struct device *dev) ++{ ++ return -EOPNOTSUPP; ++} ++#endif ++ + #if IS_REACHABLE(CONFIG_PCIE_APPLE_H9P) ++int apple_ans_handoff_begin(struct device *dev); ++int apple_ans_handoff_stopped(struct device *dev); ++int apple_ans_nvmmu_drain(struct device *dev); ++int apple_ans_nvmmu_reset(struct device *dev); ++int apple_ans_link_up(struct device *dev); ++bool apple_ans_is_s8003(struct device *dev); ++void apple_ans_host_dump(struct device *dev, const char *reason); + int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova); + #else ++static inline int apple_ans_handoff_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_handoff_stopped(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_drain(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_reset(struct device *dev) { return -EOPNOTSUPP; } ++static inline int apple_ans_link_up(struct device *dev) { return 1; } ++static inline bool apple_ans_is_s8003(struct device *dev) { return false; } ++static inline void apple_ans_host_dump(struct device *dev, const char *reason) {} + static inline int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova) diff --git a/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..62d5bda6 --- /dev/null +++ b/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,337 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] nvme-pci: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + drivers/nvme/host/pci-apple-ans.c | 33 +++++++- + drivers/nvme/host/pci-internal.h | 2 + + drivers/nvme/host/pci.c | 128 ++++++++++++++++++++++++++++-- + 3 files changed, 155 insertions(+), 8 deletions(-) + +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index 5ba239e47..fdd9fa091 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -84,6 +84,7 @@ struct apple_ans_nvme { + u32 cc; + unsigned long cc_jiffies; + bool cc_written; ++ bool host_fault; + dma_addr_t hmb_dma; + u32 hmb_size; + struct apple_ans_nvme_req req[APPLE_ANS_NVMMU_MAX_REQS]; +@@ -218,13 +219,20 @@ static int nvme_pci_apple_ans_reset_dma(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long flags; +- unsigned int tag; + int ret; + +- for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { +- ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ans->host == &apple_ans_host_s8003) { ++ ret = apple_ans_nvmmu_reset(dev->dev); + if (ret) + return ret; ++ } else { ++ unsigned int tag; ++ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } + } + + spin_lock_irqsave(&ans->req_lock, flags); +@@ -326,10 +334,24 @@ static int nvme_pci_apple_ans_prepare_enable(struct nvme_dev *dev) + return 0; + } + ++static int nvme_pci_apple_ans_check_link(struct nvme_dev *dev) ++{ ++ struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); ++ ++ if (ans && ans->host == &apple_ans_host_s8003 && ++ (ans->host_fault || apple_ans_link_up(dev->dev) <= 0)) { ++ ans->host_fault = true; ++ return -EIO; ++ } ++ return 0; ++} ++ + static int nvme_pci_apple_ans_write_cc(struct nvme_dev *dev, u32 cc) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + writel(cc, dev->bar + NVME_REG_CC); + if (!ans) + return 0; +@@ -349,6 +371,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long deadline; + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + *csts = readl(dev->bar + NVME_REG_CSTS); + if (!ans || !ans->cc_written || !ans->host->rdy_timeout_ms || + (ans->cc & NVME_CC_SHN_MASK)) +@@ -360,6 +384,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + time_after(jiffies, deadline)) { + dev_err(dev->dev, "ANS RDY timeout (CC %#x, CSTS %#x)\n", + ans->cc, *csts); ++ ans->host_fault = true; ++ apple_ans_host_dump(dev->dev, "RDY deadline"); + return -ETIMEDOUT; + } + +@@ -419,6 +445,7 @@ static int nvme_pci_apple_ans_free_req(struct nvme_dev *dev, + dev_err_ratelimited(dev->dev, + "failed to release ANS DMA slot %u: %d\n", + tag, ret); ++ nvme_reset_ctrl(&dev->ctrl); + return ret; + } + +diff --git a/drivers/nvme/host/pci-internal.h b/drivers/nvme/host/pci-internal.h +index a62777abd..d3fe17867 100644 +--- a/drivers/nvme/host/pci-internal.h ++++ b/drivers/nvme/host/pci-internal.h +@@ -74,6 +74,8 @@ struct nvme_dev { + struct nvme_ctrl ctrl; + u32 last_ps; + bool hmb; ++ unsigned long apple_ans_flags; ++ atomic_t apple_ans_irq_count; + struct sg_table *hmb_sgt; + mempool_t *dmavec_mempool; + const struct nvme_pci_dma_ops *dma_ops; +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 9ffe2fb74..40b2448d1 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -5,6 +5,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -17,6 +18,10 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + #include + #include + #include +@@ -1559,6 +1564,9 @@ static irqreturn_t nvme_irq(int irq, void *data) + DEFINE_IO_COMP_BATCH(iob); + bool found; + ++ if (!nvmeq->qid && dev->dma_ops == &nvme_pci_apple_ans_ops) ++ atomic_inc(&dev->apple_ans_irq_count); ++ + if (nvme_apple_ans_shared_vector(dev)) + found = nvme_apple_ans_irq_vector(dev, nvmeq->cq_vector, &iob); + else +@@ -1836,6 +1844,40 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) + "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n"); + } + ++/* No endpoint MMIO/config or doorbell reads: safe first evidence for ANS. */ ++static void nvme_apple_ans_dump(struct nvme_dev *dev, const char *reason) ++{ ++ unsigned int i; ++ ++ dev_info(dev->dev, "Apple ANS: snapshot %s state %d online %u allocated %u vectors %u recovery_used %u\n", ++ reason, nvme_ctrl_state(&dev->ctrl), dev->online_queues, ++ dev->ctrl.queue_count, dev->num_vecs, ++ test_bit(0, &dev->apple_ans_flags)); ++ apple_ans_host_dump(dev->dev, reason); ++ for (i = 0; i < dev->ctrl.queue_count; i++) { ++ struct nvme_queue *q = &dev->queues[i]; ++ u16 head = READ_ONCE(q->cq_head); ++ struct nvme_completion *cqe; ++ int irq = -1; ++ ++ if (!q->cqes || head >= q->q_depth) ++ continue; ++ cqe = &q->cqes[head]; ++ if (test_bit(NVMEQ_ENABLED, &q->flags) && ++ !test_bit(NVMEQ_POLLED, &q->flags)) ++ irq = pci_irq_vector(to_pci_dev(dev->dev), q->cq_vector); ++ dev_info(dev->dev, "Apple ANS: q%u flags %#lx depth %u SQ tail %u/%u CQ head %u phase %u next status %#x cid %#x sqhead %u vector %u irq %d ANS admin IRQ actions %u SQ %pad CQ %pad\n", ++ i, READ_ONCE(q->flags), q->q_depth, ++ READ_ONCE(q->sq_tail), READ_ONCE(q->last_sq_tail), ++ head, READ_ONCE(q->cq_phase), ++ le16_to_cpu(READ_ONCE(cqe->status)), ++ le16_to_cpu(READ_ONCE(cqe->command_id)), ++ le16_to_cpu(READ_ONCE(cqe->sq_head)), q->cq_vector, ++ irq, atomic_read(&dev->apple_ans_irq_count), ++ &q->sq_dma_addr, &q->cq_dma_addr); ++ } ++} ++ + static enum blk_eh_timer_return nvme_timeout(struct request *req) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); +@@ -1844,8 +1886,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + struct request *abort_req; + struct nvme_command cmd = { }; + struct pci_dev *pdev = to_pci_dev(dev->dev); +- u32 csts = readl(dev->bar + NVME_REG_CSTS); ++ u32 csts = ~0U; + u8 opcode; ++ bool ans = dev->dma_ops == &nvme_pci_apple_ans_ops && ++ apple_ans_is_s8003(dev->dev); ++ ++ if (ans) { ++ nvme_apple_ans_dump(dev, "timeout before CSTS"); ++ if (dev->dma_ops->read_csts(dev, &csts)) ++ goto disable; ++ } else { ++ csts = readl(dev->bar + NVME_REG_CSTS); ++ } + + /* + * Shutdown the device immediately if we see it is disconnected. This +@@ -1921,6 +1973,11 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + * returned to the driver, or if this is the admin queue. + */ + opcode = nvme_req(req)->cmd->common.opcode; ++ if (ans) { ++ if (test_and_set_bit(0, &dev->apple_ans_flags)) ++ nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); ++ goto disable; ++ } + if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) { + dev_warn(dev->ctrl.device, + "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n", +@@ -2804,6 +2861,7 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, + return a->mode; + } + ++ + static struct attribute *nvme_pci_attrs[] = { + &dev_attr_cmb.attr, + &dev_attr_cmbloc.attr, +@@ -3283,11 +3341,46 @@ static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev) + return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY); + } + +-static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++/* Certify stopped DMA before the kexec PCIe handoff. */ ++static int nvme_apple_ans_handoff_disable(struct nvme_dev *dev, int shutdown_ret) ++{ ++ struct pci_dev *pdev = to_pci_dev(dev->dev); ++ u32 csts = ~0U, cc, expected; ++ int ret, err; ++ ++ ret = dev->dma_ops->read_csts(dev, &csts); ++ if (shutdown_ret || ret || (csts & NVME_CSTS_CFS) || ++ (csts & NVME_CSTS_SHST_MASK) != NVME_CSTS_SHST_CMPLT) { ++ pci_clear_master(pdev); ++ return -EBUSY; ++ } ++ dev->ctrl.ctrl_config &= ~(NVME_CC_ENABLE | NVME_CC_SHN_MASK); ++ expected = dev->ctrl.ctrl_config; ++ ret = dev->dma_ops->write_cc(dev, expected); ++ if (!ret) { ++ ret = read_poll_timeout(dev->dma_ops->read_csts, err, ++ err || !(csts & NVME_CSTS_RDY), ++ 1000, 2000000, false, dev, &csts); ++ if (!ret) ++ ret = err; ++ } ++ if (ret) ++ goto block_dma; ++ cc = readl(dev->bar + NVME_REG_CC); ++ if ((csts & NVME_CSTS_CFS) || ++ (cc & (NVME_CC_ENABLE | NVME_CC_SHN_MASK))) ++ ret = -EBUSY; ++block_dma: ++ pci_clear_master(pdev); ++ return ret; ++} ++ ++static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool paired) + { + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); + dead = nvme_pci_ctrl_is_dead(dev); +@@ -3298,15 +3391,26 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown) +- nvme_wait_freeze_timeout(&dev->ctrl); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) ++ panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); + } + + nvme_quiesce_io_queues(&dev->ctrl); + ++ if (paired && (dead || !dev->ctrl.queue_count)) ++ panic("Apple ANS: dead outgoing controller; refusing kexec"); + if (!dead && dev->ctrl.queue_count > 0) { ++ if (paired) { ++ nvme_sync_io_queues(&dev->ctrl); ++ ret = apple_ans_nvmmu_drain(dev->dev); ++ if (ret) ++ panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ } + nvme_delete_io_queues(dev); +- nvme_disable_ctrl(&dev->ctrl, shutdown); ++ dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); ++ ret = nvme_disable_ctrl(&dev->ctrl, shutdown); ++ if (paired) ++ ret = nvme_apple_ans_handoff_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } + nvme_suspend_io_queues(dev); +@@ -3329,7 +3433,15 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) + nvme_unquiesce_admin_queue(&dev->ctrl); + } ++ if (paired && !ret) ++ ret = apple_ans_handoff_stopped(dev->dev); + mutex_unlock(&dev->shutdown_lock); ++ return ret; ++} ++ ++static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++{ ++ nvme_dev_disable_paired(dev, shutdown, false); + } + + static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) +@@ -3920,6 +4032,12 @@ static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); + ++ if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) ++ panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + nvme_disable_prepare_reset(dev, true); + } + diff --git a/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch b/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch new file mode 100644 index 00000000..f82f0350 --- /dev/null +++ b/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch @@ -0,0 +1,85 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:07 +0100 +Subject: [PATCH] PCI: apple: cold-reset after an uncertified kexec handoff + +The warm guard sets panic_timeout to zero. Neither an uncertified ANS +stop nor failed PERST, LTSSM stop or powered-port certificate can safely +continue into the incoming kernel. Flush the reason and request a cold +watchdog reset instead of hanging in panic. Return without further port +writes if restart unexpectedly returns. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 9 +++++---- + include/linux/soc/apple/cold-reset.h | 19 +++++++++++++++++++ + 2 files changed, 24 insertions(+), 4 deletions(-) + create mode 100644 include/linux/soc/apple/cold-reset.h + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index f0095394a..28305b0ee 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -49,6 +49,7 @@ + #include + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -3609,7 +3610,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!READ_ONCE(pcie->ans_handoff_stopped) || + (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + !apple_s8003_port_powered(pcie))) +- panic("Apple ANS: refusing uncertified powered handoff"); ++ return apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); + for (port = 0; port < H9P_NUM_PORTS; port++) { + void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; + +@@ -3621,9 +3622,9 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, + S8003_RC_PORT_PWR_REFCLK_NOGATE); + if (gpiod_direction_output(pcie->perst[port], 1)) +- panic("Apple PCIe: PERST failed during handoff"); ++ return apple_cold_reset("Apple PCIe: PERST failed during handoff"); + if (apple_s8003_port_stop(pcie, port)) +- panic("Apple PCIe: LTSSM stop failed during handoff"); ++ return apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + udelay(S8003_PERST_TO_REFCLK_US); + if (!port && pcie->nvmmu.ready) { + h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); +@@ -3635,7 +3636,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || + gpiod_get_value_cansleep(pcie->perst[0]) != 1)) +- panic("Apple ANS: outgoing powered port certificate failed"); ++ return apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); + pcie->enabled_ports &= ~BIT(port); + } + dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); +diff --git a/include/linux/soc/apple/cold-reset.h b/include/linux/soc/apple/cold-reset.h +new file mode 100644 +index 000000000..34467be50 +--- /dev/null ++++ b/include/linux/soc/apple/cold-reset.h +@@ -0,0 +1,19 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_COLD_RESET_H ++#define _LINUX_SOC_APPLE_COLD_RESET_H ++ ++#include ++#include ++#include ++ ++/* Shutdown has no safe return to a warm kernel after an unproven DMA stop. */ ++static inline void apple_cold_reset(const char *reason) ++{ ++ pr_emerg("Apple unsafe handover: %s; forcing cold watchdog reset\n", reason); ++ pr_flush(1000, true); ++ kmsg_dump_desc(KMSG_DUMP_PANIC, reason); ++ WRITE_ONCE(reboot_mode, REBOOT_COLD); ++ emergency_restart(); ++} ++ ++#endif diff --git a/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch b/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch new file mode 100644 index 00000000..5075181b --- /dev/null +++ b/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch @@ -0,0 +1,76 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:08:51 +0100 +Subject: [PATCH] nvme-pci: cold-reset after unsafe ANS kexec shutdown + +An unsafe ANS shutdown cannot panic under a zero panic timeout. Use the +shared logged and flushed cold-reset path for each failed certificate. +If reset unexpectedly returns, leave without further device operations. + +Assisted-by: LLM +--- + drivers/nvme/host/pci.c | 29 +++++++++++++++++++++-------- + 1 file changed, 21 insertions(+), 8 deletions(-) + +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 40b2448d1..2f356a8ce 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -3391,20 +3392,30 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) +- panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) { ++ apple_cold_reset("Apple ANS: outgoing I/O did not drain"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + + nvme_quiesce_io_queues(&dev->ctrl); + +- if (paired && (dead || !dev->ctrl.queue_count)) +- panic("Apple ANS: dead outgoing controller; refusing kexec"); ++ if (paired && (dead || !dev->ctrl.queue_count)) { ++ apple_cold_reset("Apple ANS: dead outgoing controller"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + if (!dead && dev->ctrl.queue_count > 0) { + if (paired) { + nvme_sync_io_queues(&dev->ctrl); + ret = apple_ans_nvmmu_drain(dev->dev); +- if (ret) +- panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ if (ret) { ++ dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); ++ apple_cold_reset("Apple ANS: outgoing drain failed"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + nvme_delete_io_queues(dev); + dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); +@@ -4034,8 +4045,10 @@ static void nvme_shutdown(struct pci_dev *pdev) + + if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && + dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { +- if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) +- panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) { ++ apple_cold_reset("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + return; + } + nvme_disable_prepare_reset(dev, true); diff --git a/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch b/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch new file mode 100644 index 00000000..5b49b55d --- /dev/null +++ b/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch @@ -0,0 +1,62 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:08:51 +0100 +Subject: [PATCH] PCI: apple: use explicit returns after cold reset + +Use standard C void returns while retaining the early exit if the reset +helper unexpectedly returns. Successful handoff ordering is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 24 ++++++++++++++++-------- + 1 file changed, 16 insertions(+), 8 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 28305b0ee..5890d632a 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3609,8 +3609,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + cancel_delayed_work_sync(&pcie->late_work); + if (!READ_ONCE(pcie->ans_handoff_stopped) || + (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && +- !apple_s8003_port_powered(pcie))) +- return apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); ++ !apple_s8003_port_powered(pcie))) { ++ apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); ++ return; ++ } + for (port = 0; port < H9P_NUM_PORTS; port++) { + void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; + +@@ -3621,10 +3623,14 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); + h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, + S8003_RC_PORT_PWR_REFCLK_NOGATE); +- if (gpiod_direction_output(pcie->perst[port], 1)) +- return apple_cold_reset("Apple PCIe: PERST failed during handoff"); +- if (apple_s8003_port_stop(pcie, port)) +- return apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); ++ if (gpiod_direction_output(pcie->perst[port], 1)) { ++ apple_cold_reset("Apple PCIe: PERST failed during handoff"); ++ return; ++ } ++ if (apple_s8003_port_stop(pcie, port)) { ++ apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); ++ return; ++ } + udelay(S8003_PERST_TO_REFCLK_US); + if (!port && pcie->nvmmu.ready) { + h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); +@@ -3635,8 +3641,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_s8003_port_off(pcie, port); + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || +- gpiod_get_value_cansleep(pcie->perst[0]) != 1)) +- return apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) { ++ apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); ++ return; ++ } + pcie->enabled_ports &= ~BIT(port); + } + dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); From 8286fdfd11cc5bb8c2cd41fc273c2e6c91f4e5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 22/33] pinctrl/spi/input: Quiesce S8000 peripherals for kexec After a kexec the next kernel found the S8000 GPIO controller with interrupt modes armed by the previous kernel, the PIO SPI controller with queued transfers and an asserted chip select, and the touch controller still running. An orphaned GPIO interrupt fires forever and touch did not come back. Retire inherited GPIO interrupt modes before the chained parent interrupts are enabled, drain and idle the S8000 PIO SPI controller on shutdown and at probe, and stop the Apple Z2 touch controller before its SPI parent shuts down. Assisted-by: LLM --- ...top-the-touch-controller-on-shutdown.patch | 48 +++++++ ...io-retire-inherited-GPIO-interrupt-m.patch | 130 ++++++++++++++++++ ...and-idle-the-S8000-PIO-controller-on.patch | 122 ++++++++++++++++ 3 files changed, 300 insertions(+) create mode 100644 kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch create mode 100644 kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch create mode 100644 kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch diff --git a/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch b/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch new file mode 100644 index 00000000..88cd9ce1 --- /dev/null +++ b/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch @@ -0,0 +1,48 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] Input: apple-z2: stop the touch controller on shutdown + +A kexec hands the next kernel an S8000 SPI bus that the touch controller +may still be using: its interrupt thread can run and the SPI core can +still hold its messages. + +Add a shutdown callback that stops the controller and waits for its +interrupt, also when the threaded handler has already masked itself. SPI +children shut down before their controller, so the SPI driver can then +drain its queue. + +Assisted-by: LLM +--- + drivers/input/touchscreen/apple_z2.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/drivers/input/touchscreen/apple_z2.c b/drivers/input/touchscreen/apple_z2.c +index 92a04edcd..5dcf3579d 100644 +--- a/drivers/input/touchscreen/apple_z2.c ++++ b/drivers/input/touchscreen/apple_z2.c +@@ -4599,6 +4599,16 @@ static struct spi_device_id apple_z2_of_id[] = { + }; + MODULE_DEVICE_TABLE(spi, apple_z2_of_id); + ++/* Children stop before the SPI controller drains its core queue. */ ++static void apple_z2_shutdown(struct spi_device *spi) ++{ ++ struct apple_z2 *z2 = spi_get_drvdata(spi); ++ ++ apple_z2_stop(z2); ++ /* Also wait when the IRQ thread already masked itself. */ ++ synchronize_irq(spi->irq); ++} ++ + static struct spi_driver apple_z2_driver = { + .driver = { + .name = "apple-z2", +@@ -4609,6 +4619,7 @@ static struct spi_driver apple_z2_driver = { + .id_table = apple_z2_of_id, + .probe = apple_z2_probe, + .remove = apple_z2_remove, ++ .shutdown = apple_z2_shutdown, + }; + + module_spi_driver(apple_z2_driver); diff --git a/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch b/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch new file mode 100644 index 00000000..dc5b66a6 --- /dev/null +++ b/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch @@ -0,0 +1,130 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pinctrl: apple-gpio: retire inherited GPIO interrupt modes + after kexec + +After a kexec, or after a panic that ran no shutdown callbacks, GPIO +pins can still be in an interrupt mode set up by the previous kernel. +Once the chained parent interrupts are enabled, such a pin raises an +interrupt that nobody maps or acknowledges, and it keeps firing. + +On apple,s8000-pinctrl, switch every GPIO interrupt mode off at probe, +before the parent interrupts are enabled, and keep outputs, mux, pulls, +data and grouping. Consumers arm their own pins again later. In the +handler, a pending pin without a mapping is masked and its group latch +cleared, with a rate-limited warning. + +Assisted-by: LLM +--- + drivers/pinctrl/pinctrl-apple-gpio.c | 62 +++++++++++++++++++++++++++- + 1 file changed, 60 insertions(+), 2 deletions(-) + +diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c +index b97023c97..766fcf5bc 100644 +--- a/drivers/pinctrl/pinctrl-apple-gpio.c ++++ b/drivers/pinctrl/pinctrl-apple-gpio.c +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -46,6 +47,7 @@ struct apple_gpio_pinctrl { + + void __iomem *base; + struct regmap *map; ++ bool retire_irqs; + + struct pinctrl_desc pinctrl_desc; + struct gpio_chip gpio_chip; +@@ -379,8 +381,33 @@ static void apple_gpio_irq_handler(struct irq_desc *desc) + chained_irq_enter(chip, desc); + for (pinh = 0; pinh < gc->ngpio; pinh += 32) { + pending = readl_relaxed(pctl->base + REG_IRQ(*grpp, pinh)); +- for_each_set_bit(pinl, &pending, 32) +- generic_handle_domain_irq(gc->irq.domain, pinh + pinl); ++ for_each_set_bit(pinl, &pending, 32) { ++ unsigned int pin = pinh + pinl; ++ int ret; ++ ++ ret = pin < gc->ngpio ? ++ generic_handle_domain_irq(gc->irq.domain, pin) : -EINVAL; ++ if (ret && pctl->retire_irqs) { ++ /* An inherited, unmapped source has nobody to ack it. ++ * Mask only this GPIO, then clear this group's latch. ++ */ ++ if (pin < gc->ngpio) { ++ u32 reg = apple_gpio_get_reg(pctl, pin); ++ u32 mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ u32 off = FIELD_PREP(REG_GPIOx_MODE, ++ REG_GPIOx_IN_IRQ_OFF); ++ ++ if (!(reg & REG_GPIOx_PERIPH) && ++ mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, off); ++ } ++ writel(BIT(pinl), pctl->base + REG_IRQ(*grpp, pinh)); ++ dev_warn_ratelimited(pctl->dev, ++ "masked orphan inherited GPIO IRQ group=%u pin=%u\n", ++ *grpp, pin); ++ } ++ } + } + chained_irq_exit(chip, desc); + } +@@ -403,6 +430,35 @@ static const struct irq_chip apple_gpio_irqchip = { + + /* Probe & register */ + ++static void apple_gpio_quiesce_irqs(struct apple_gpio_pinctrl *pctl) ++{ ++ struct gpio_chip *gc = &pctl->gpio_chip; ++ unsigned int pin, group, mode, masked = 0; ++ u32 reg; ++ ++ if (!pctl->retire_irqs) ++ return; ++ /* Before chained parent IRQs are enabled, retire old GPIO IRQ modes. ++ * Preserve outputs, mux, pulls, data and grouping. Consumers rearm their ++ * own pins later; panic entry need not have run any shutdown callbacks. ++ */ ++ for (pin = 0; pin < gc->ngpio; pin++) { ++ reg = apple_gpio_get_reg(pctl, pin); ++ mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ if (!(reg & REG_GPIOx_PERIPH) && mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) { ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, ++ FIELD_PREP(REG_GPIOx_MODE, REG_GPIOx_IN_IRQ_OFF)); ++ masked++; ++ } ++ } ++ for (group = 0; group < gc->irq.num_parents; group++) ++ for (pin = 0; pin < gc->ngpio; pin += 32) ++ writel(GENMASK(min(32U, gc->ngpio - pin) - 1, 0), ++ pctl->base + REG_IRQ(group, pin)); ++ dev_info(pctl->dev, "inherited GPIO IRQ baseline: masked %u inherited inputs\n", masked); ++} ++ + static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + { + struct gpio_irq_chip *girq = &pctl->gpio_chip.irq; +@@ -452,6 +508,7 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + girq->handler = handle_level_irq; + } + ++ apple_gpio_quiesce_irqs(pctl); + ret = devm_gpiochip_add_data(pctl->dev, &pctl->gpio_chip, pctl); + + out_free_irq_data: +@@ -485,6 +542,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) + if (!pctl) + return -ENOMEM; + pctl->dev = &pdev->dev; ++ pctl->retire_irqs = of_device_is_compatible(pdev->dev.of_node, "apple,s8000-pinctrl"); + pctl->soc = of_device_get_match_data(&pdev->dev); + pctl->gpio_chip.irq.num_parents = nirqs; + dev_set_drvdata(&pdev->dev, pctl); diff --git a/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch b/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch new file mode 100644 index 00000000..daa284cf --- /dev/null +++ b/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch @@ -0,0 +1,122 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] spi: apple: drain and idle the S8000 PIO controller on + shutdown + +On S8000 the SPI controller runs in PIO mode. A kexec left it with +queued transfers, counters and an asserted chip select, and the next +kernel probed it in that state. + +On shutdown, suspend the SPI core queue, deassert the GPIO chip selects +and write the PIO registers back to idle while the controller is +powered. Do the same at probe, so that a controller inherited in any +state starts idle. Other SoCs use different DMA and interrupt registers +and keep their current behaviour. + +Assisted-by: LLM +--- + drivers/spi/spi-apple.c | 48 +++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 48 insertions(+) + +diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c +index dc5154e1b..ed9f67a59 100644 +--- a/drivers/spi/spi-apple.c ++++ b/drivers/spi/spi-apple.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -141,6 +142,7 @@ struct apple_spi { + struct dmaengine_result dma_result; + const struct apple_spi_hw *hw; + u32 word_delay; ++ struct spi_controller *ctlr; + }; + + static inline void reg_write(struct apple_spi *spi, int offset, u32 value) +@@ -231,6 +233,19 @@ static void apple_spi_init(struct apple_spi *spi) + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + } + ++/* Lab 0015: normalize only the S8000 PIO register set, while powered. */ ++static int apple_spi_s8000_idle(struct apple_spi *spi) ++{ ++ reg_write(spi, APPLE_SPI_CTRL, 0); ++ reg_write(spi, APPLE_SPI_CFG, spi->hw->cfg_base); ++ reg_write(spi, APPLE_SPI_RXCNT, 0); ++ reg_write(spi, APPLE_SPI_TXCNT, 0); ++ reg_write(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS); ++ apple_spi_t8010_clear_status(spi); ++ /* Complete posted writes before publishing children or leaving Linux. */ ++ return readl(spi->regs + APPLE_SPI_CTRL) ? -EIO : 0; ++} ++ + static int apple_spi_prepare_message(struct spi_controller *ctlr, + struct spi_message *msg) + { +@@ -1120,6 +1135,7 @@ static int apple_spi_probe(struct platform_device *pdev) + + spi = spi_controller_get_devdata(ctlr); + platform_set_drvdata(pdev, spi); ++ spi->ctlr = ctlr; + init_completion(&spi->done); + init_completion(&spi->dma_done); + spi->hw = of_device_get_match_data(&pdev->dev); +@@ -1214,6 +1230,11 @@ static int apple_spi_probe(struct platform_device *pdev) + } + + apple_spi_init(spi); ++ if (spi->hw->pio_only) { ++ ret = apple_spi_s8000_idle(spi); ++ if (ret) ++ return ret; ++ } + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret < 0) +@@ -1223,6 +1244,32 @@ static int apple_spi_probe(struct platform_device *pdev) + return 0; + } + ++static void apple_spi_shutdown(struct platform_device *pdev) ++{ ++ struct apple_spi *spi = platform_get_drvdata(pdev); ++ struct spi_controller *ctlr = spi->ctlr; ++ unsigned int cs; ++ int ret; ++ ++ /* Other SoCs use different DMA/IRQ registers and keep their policy. */ ++ if (!spi->hw->pio_only) ++ return; ++ /* Children have shut down; drain the core queue before MMIO. */ ++ ret = spi_controller_suspend(ctlr); ++ if (!ret) ++ ret = pm_runtime_resume_and_get(&pdev->dev); ++ if (ret >= 0) { ++ if (ctlr->cs_gpiods) ++ for (cs = 0; cs < ctlr->num_chipselect; cs++) ++ gpiod_set_value_cansleep(ctlr->cs_gpiods[cs], 0); ++ ret = apple_spi_s8000_idle(spi); ++ } ++ if (ret < 0 && kexec_in_progress) ++ panic("S8000 SPI shutdown failed before kexec: %d", ret); ++ /* Keep runtime power pinned until the jump. */ ++ dev_info(&pdev->dev, "S8000 SPI shutdown result %d\n", ret); ++} ++ + static const struct apple_spi_hw apple_spi_hw_s8000 = { + .legacy_status = true, + .pio_only = true, +@@ -1260,6 +1307,7 @@ MODULE_DEVICE_TABLE(of, apple_spi_of_match); + + static struct platform_driver apple_spi_driver = { + .probe = apple_spi_probe, ++ .shutdown = apple_spi_shutdown, + .driver = { + .name = "apple-spi", + .of_match_table = apple_spi_of_match, From 57e91ab3b84a2977877ea843372f3428fab51234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 23/33] kernel: Retain the tethered warm boot state across kexec A warm restart through kexec needs the next kernel to see the pristine device tree from boot, the pstore region, the display state and the power domains that must not be cut while the next kernel starts. DT, OF, DRM, pmdomain and pstore patches retain the board-selected warm handoff state, so probe and shutdown keep what the next kernel needs: the pristine boot device tree, the pstore region, the display state and the Apple pmgr warm hold. The Orchard service file learns the retained state. Assisted-by: LLM --- ...-retain-the-tethered-warm-boot-state.patch | 26 +++ ...-retain-the-tethered-warm-boot-state.patch | 28 +++ ...-retain-the-tethered-warm-boot-state.patch | 35 ++++ ...-retain-the-tethered-warm-boot-state.patch | 113 +++++++++++ ...re-retain-the-tethered-warm-boot-sta.patch | 68 +++++++ ...-retain-the-tethered-warm-boot-state.patch | 180 ++++++++++++++++++ src/orchard/service.py | 3 +- 7 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch create mode 100644 kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch create mode 100644 kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch create mode 100644 kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch create mode 100644 kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch create mode 100644 kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch diff --git a/kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..36c9879f --- /dev/null +++ b/kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 3277a73bb..f6449a7e6 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -443,3 +443,7 @@ pmu { + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/ { ++ apple,reserve-pstore; ++}; diff --git a/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..2865725b --- /dev/null +++ b/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] drm: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/gpu/drm/adbe/adbe_drv.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..7ed2de3d7 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -1065,6 +1065,9 @@ static void adbe_shutdown(struct platform_device *pdev) + struct adbe *adbe = platform_get_drvdata(pdev); + + drm_atomic_helper_shutdown(&adbe->drm); ++ timer_delete_sync(&adbe->flip_timer); ++ /* release9: restore the boot buffer even if the CRTC was never enabled. */ ++ adbe_park(adbe); + writel(adbe->saved_inten, adbe->regs + ADP_INTEN); + } + diff --git a/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..64a11814 --- /dev/null +++ b/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,35 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] of: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/of/of_reserved_mem.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c +index 8c9d6395d..76cf3e5a4 100644 +--- a/drivers/of/of_reserved_mem.c ++++ b/drivers/of/of_reserved_mem.c +@@ -275,6 +275,16 @@ void __init fdt_scan_reserved_mem_late(void) + return; + } + ++ /* Early architecture fixups may have added static reservations. */ ++ total_reserved_mem_cnt = reserved_mem_count; ++ fdt_for_each_subnode(child, fdt, node) { ++ int len; ++ ++ if (of_fdt_device_is_available(fdt, child) && ++ fdt_getprop(fdt, child, "reg", &len) && len) ++ total_reserved_mem_cnt++; ++ } ++ + /* Attempt dynamic allocation of a new reserved_mem array */ + if (alloc_reserved_mem_array()) + return; diff --git a/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..0ae20f3c --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,113 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 34 +++++++++++++++++++++----- + 1 file changed, 28 insertions(+), 6 deletions(-) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..7995d3b71 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -58,6 +58,8 @@ struct apple_pmgr_ps { + u32 min_state; + bool has_genpd; + bool manual_pm; ++ bool warm; ++ bool keep_on; + }; + + #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) +@@ -83,17 +85,21 @@ static int apple_pmgr_ps_set(struct apple_pmgr_ps *ps, u32 pstate, bool auto_ena + + dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", ps->name, pstate, reg); + +- regmap_write(ps->regmap, ps->offset, reg); ++ ret = regmap_write(ps->regmap, ps->offset, reg); ++ if (ret) ++ return ret; + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, reg, + (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, + APPLE_PMGR_PS_SET_TIMEOUT); +- if (ret < 0) ++ if (ret < 0) { + dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", + ps->name, pstate, reg); ++ return ret; ++ } + +- if (auto_enable) { ++ if (auto_enable && !ps->keep_on) { + /* Not all devices implement this; this is a no-op where not implemented. */ + reg &= ~APPLE_PMGR_FLAGS; + reg |= APPLE_PMGR_AUTO_ENABLE; +@@ -136,6 +142,9 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) + unsigned long flags; + int ret; + ++ if (ps->keep_on) ++ return -EBUSY; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_PWRGATE, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -168,6 +177,10 @@ static void apple_pmgr_clk_disable(struct clk_hw *hw) + unsigned long flags; + int ret; + ++ /* The lab has no separate clk gate sharing this power-state word. */ ++ if (ps->keep_on) ++ return; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, gate->off_state, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -306,6 +319,10 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + return ret; + } + ++ ps->warm = of_device_is_compatible(node, "apple,s8000-pmgr-pwrstate") && ++ of_property_read_bool(of_chosen, "linux,booted-from-kexec"); ++ ps->keep_on = ps->warm; ++ ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); + clock_only = of_property_read_bool(node, "apple,clock-gate-only"); + if (!clock_only) { + ps->genpd.flags |= GENPD_FLAG_IRQ_SAFE; +@@ -315,12 +332,17 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); +- if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) ++ if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && ++ !(ps->warm && ps->manual_pm)) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, + FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + + active = apple_pmgr_ps_is_active(ps); +- ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); ++ if (ps->warm && active && !ps->manual_pm) { ++ ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_ACTIVE, false); ++ if (ret) ++ return ret; ++ } + if (!clock_only && of_property_read_bool(node, "apple,always-on")) { + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + if (!active) { +@@ -331,7 +353,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + /* Turn on auto-PM if the domain is already on */ +- if (active) ++ if (active && !ps->warm) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, + ps->manual_pm ? 0 : APPLE_PMGR_AUTO_ENABLE); + diff --git a/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch b/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch new file mode 100644 index 00000000..af7dc5ff --- /dev/null +++ b/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch @@ -0,0 +1,68 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: pstore: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + .../devicetree/bindings/arm/apple,pstore.yaml | 31 +++++++++++++++++++ + .../bindings/reserved-memory/ramoops.yaml | 4 +++ + 2 files changed, 35 insertions(+) + create mode 100644 Documentation/devicetree/bindings/arm/apple,pstore.yaml + +diff --git a/Documentation/devicetree/bindings/arm/apple,pstore.yaml b/Documentation/devicetree/bindings/arm/apple,pstore.yaml +new file mode 100644 +index 000000000..241dee447 +--- /dev/null ++++ b/Documentation/devicetree/bindings/arm/apple,pstore.yaml +@@ -0,0 +1,31 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/arm/apple,pstore.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: Apple retained pstore allocation ++ ++maintainers: ++ - brkosebay ++ ++description: | ++ Select early allocation on the root node. The generated ramoops node ++ carries apple,preserve-in-capture so a capture kernel leaves evidence intact. ++ Its address is carried in the kexec FDT, never reallocated on capture boots. ++ ++select: ++ properties: ++ compatible: ++ contains: ++ enum: ++ - apple,s8000 ++ - apple,s8003 ++ required: ++ - compatible ++properties: ++ apple,reserve-pstore: ++ type: boolean ++ description: Reserve an aligned 2 MiB no-map ramoops area before paging. ++ ++additionalProperties: true +diff --git a/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml b/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml +index 45cc39ecc..0b349c0ba 100644 +--- a/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml ++++ b/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml +@@ -30,6 +30,10 @@ allOf: + - $ref: reserved-memory.yaml + + properties: ++ apple,preserve-in-capture: ++ type: boolean ++ description: Preserve the previous kernel records by refusing capture-kernel probe. ++ + compatible: + const: ramoops + diff --git a/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..1bac3a8f --- /dev/null +++ b/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,180 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pstore: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + arch/arm64/include/asm/mmu.h | 1 + + arch/arm64/kernel/setup.c | 1 + + arch/arm64/mm/init.c | 88 ++++++++++++++++++++++++++++++++++++ + fs/pstore/ram.c | 8 ++++ + 4 files changed, 98 insertions(+) + +diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h +index 5e1211c54..bd22a913d 100644 +--- a/arch/arm64/include/asm/mmu.h ++++ b/arch/arm64/include/asm/mmu.h +@@ -62,6 +62,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void) + + extern void arm64_memblock_init(void); + extern void paging_init(void); ++extern void apple_soc_publish_pstore(void); + extern void bootmem_init(void); + extern void create_mapping_noalloc(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot); +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index 29c6100f0..45c23e3d9 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -332,6 +332,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) + arm64_memblock_init(); + + paging_init(); ++ apple_soc_publish_pstore(); + + acpi_table_upgrade(); + +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index fbf215ecc..3e458f58e 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -22,6 +22,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -193,6 +195,91 @@ static int __init early_mem(char *p) + } + early_param("mem", early_mem); + ++static phys_addr_t apple_soc_pstore_fdt __initdata; ++ ++/* ++ * Reserve before paging/CMA/crashkernel, without touching the linear map: ++ * paging_init() has not built it yet. Prepare the replacement FDT through a ++ * temporary early mapping and publish its linear alias only after paging. ++ * Never choose a fresh address in kdump; reuse the first kernel's region. ++ */ ++static void __init apple_soc_reserve_pstore(void) ++{ ++ void *fdt = initial_boot_params, *copy; ++ phys_addr_t base, copy_pa; ++ fdt64_t reg[2]; ++ char name[40]; ++ int parent, node, size, ret; ++ ++ if (!IS_BUILTIN(CONFIG_PSTORE_RAM) || !fdt || ++ !fdt_getprop(fdt, 0, "apple,reserve-pstore", NULL)) ++ return; ++ if (fdt_node_offset_by_compatible(fdt, -1, "ramoops") >= 0) ++ return; ++ if (is_kdump_kernel()) { ++ pr_warn("Apple pstore: no inherited region in capture kernel\n"); ++ return; ++ } ++ parent = fdt_path_offset(fdt, "/reserved-memory"); ++ if (parent < 0 || fdt_address_cells(fdt, parent) != 2 || ++ fdt_size_cells(fdt, parent) != 2) ++ return; ++ ++ /* Existing firmware, kernel and initrd reservations are already known. */ ++ base = memblock_phys_alloc(SZ_2M, SZ_2M); ++ if (!base) ++ return; ++ size = fdt_totalsize(fdt) + SZ_4K; ++ copy_pa = memblock_phys_alloc(size, SMP_CACHE_BYTES); ++ if (!copy_pa) ++ goto free_region; ++ copy = early_memremap(copy_pa, size); ++ if (!copy) ++ goto free_copy; ++ ret = fdt_open_into(fdt, copy, size); ++ if (ret) ++ goto unmap_copy; ++ parent = fdt_path_offset(copy, "/reserved-memory"); ++ snprintf(name, sizeof(name), "ramoops@%llx", (u64)base); ++ node = fdt_add_subnode(copy, parent, name); ++ if (node < 0) ++ goto unmap_copy; ++ reg[0] = cpu_to_fdt64(base); ++ reg[1] = cpu_to_fdt64(SZ_2M); ++ if (fdt_setprop(copy, node, "compatible", "ramoops", sizeof("ramoops")) || ++ fdt_setprop(copy, node, "reg", reg, sizeof(reg)) || ++ fdt_setprop(copy, node, "no-map", NULL, 0) || ++ fdt_setprop_u32(copy, node, "record-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "console-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "max-reason", 2) || ++ fdt_setprop(copy, node, "apple,preserve-in-capture", NULL, 0) || ++ fdt_add_mem_rsv(copy, base, SZ_2M)) ++ goto unmap_copy; ++ /* No cached linear alias of ramoops' write-combined persistent mapping. */ ++ if (memblock_mark_nomap(base, SZ_2M)) ++ goto unmap_copy; ++ early_memunmap(copy, size); ++ apple_soc_pstore_fdt = copy_pa; ++ pr_info("Apple pstore: reserved 2 MiB at %pa, carried in kexec FDT\n", &base); ++ return; ++ ++unmap_copy: ++ early_memunmap(copy, size); ++free_copy: ++ memblock_phys_free(copy_pa, size); ++free_region: ++ memblock_phys_free(base, SZ_2M); ++ pr_warn("Apple pstore: reservation failed\n"); ++} ++ ++/* The linear map now exists; unflattening must see the prepared FDT. */ ++void __init apple_soc_publish_pstore(void) ++{ ++ if (apple_soc_pstore_fdt) ++ early_init_dt_verify(phys_to_virt(apple_soc_pstore_fdt), ++ apple_soc_pstore_fdt); ++} ++ + void __init arm64_memblock_init(void) + { + s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); +@@ -298,6 +385,7 @@ void __init arm64_memblock_init(void) + } + + early_init_fdt_scan_reserved_mem(); ++ apple_soc_reserve_pstore(); + } + + void __init bootmem_init(void) +diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c +index 2eb0d4fa2..b26a3a0d5 100644 +--- a/fs/pstore/ram.c ++++ b/fs/pstore/ram.c +@@ -9,6 +9,7 @@ + #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + + #include ++#include + #include + #include + #include +@@ -729,6 +730,13 @@ static int ramoops_probe(struct platform_device *pdev) + phys_addr_t paddr; + int err = -EINVAL; + ++ /* Leave the original panic/console records intact across phoenix. */ ++ if (is_kdump_kernel() && of_property_read_bool(pdev->dev.of_node, ++ "apple,preserve-in-capture")) { ++ pr_info("capture kernel: preserving Apple records for the full boot\n"); ++ return -ENODEV; ++ } ++ + /* + * Only a single ramoops area allowed at a time, so fail extra + * probes. diff --git a/src/orchard/service.py b/src/orchard/service.py index 0d5d76c7..0340f16c 100644 --- a/src/orchard/service.py +++ b/src/orchard/service.py @@ -150,7 +150,8 @@ def entry_for(record, devicetree): name = record.get('name', 'Linux') return {'id': re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') or 'linux', 'title': name, # Quiet as the recovery: no console text over the splash; the system maps its console. - 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0', + 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0' + ( + ' panic=0' if devicetree == 's8003-j71t.dtb' else ''), 'devicetree': devicetree} From 92d22c85424ce2626fcb1e017b076e8fa99fd844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 24/33] watchdog: apple: Prefer a cold reset and guard the warm reset path On a tethered J71t boot a restart must be a native cold reset, but the watchdog's restart handler ran below PSCI's, and its restart guard dereferenced a parent that probe never set. Describe J71t's tethered boot in DT. On such a board the watchdog restart handler registers above PSCI and performs the native cold reset; in the warm and soft modes it leaves the watchdog alone and returns success, so PSCI's restart runs. Probe sets the parent before registering, and the restart callback tolerates a missing one. Other devices keep the current priority. Assisted-by: LLM --- ...ts-apple-describe-J71t-tethered-boot.patch | 24 +++++++++ ...hdog-retain-the-tethered-warm-boot-s.patch | 31 +++++++++++ ...-retain-the-tethered-warm-boot-state.patch | 54 +++++++++++++++++++ ...nitialize-the-parent-before-register.patch | 37 +++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch create mode 100644 kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch create mode 100644 kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch create mode 100644 kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch diff --git a/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch b/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch new file mode 100644 index 00000000..70a0484d --- /dev/null +++ b/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:20:28 +0100 +Subject: [PATCH] arm64: dts: apple: describe J71t tethered boot + +Record the board fact for the droppable watchdog reset guard. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..0df58a56f 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -24,3 +24,7 @@ &touchscreen { + &wifi { + brcm,board-type = "apple,feara"; + }; ++ ++&wdt { ++ apple,tethered-boot; ++}; diff --git a/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch b/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch new file mode 100644 index 00000000..e166e49d --- /dev/null +++ b/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: watchdog: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/watchdog/apple,wdt.yaml | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +index ca4cf01db..a8179c208 100644 +--- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml ++++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +@@ -13,6 +13,12 @@ allOf: + - $ref: watchdog.yaml# + + properties: ++ apple,tethered-boot: ++ type: boolean ++ description: | ++ A SoC reset leaves the tethered OS. The restart handler refuses warm ++ and soft reboot modes; cold reboot retains the normal SoC reset. ++ + compatible: + oneOf: + - items: diff --git a/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..a235291f --- /dev/null +++ b/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,54 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 23:30:21 +0100 +Subject: [PATCH] watchdog: apple: prioritize cold reset and delegate guarded + warm reset + +Tethered cold reset must beat PSCI priority 129: register at priority +255. On warm/soft modes do not program WD1, but return success so +watchdog_restart_notifier returns NOTIFY_DONE and PSCI can run; an error +would stop the chain. Other devices retain priority 128. + +Assisted-by: LLM +--- + drivers/watchdog/apple_wdt.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index f53395329..04e287fe8 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + + /* +@@ -123,6 +124,12 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + ++ if (of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && ++ (mode == REBOOT_WARM || mode == REBOOT_SOFT)) { ++ pr_emerg("apple-wdt: warm/soft reset delegated to PSCI on tethered boot\n"); ++ return 0; /* Continue the restart chain; NOTIFY_BAD would stop PSCI. */ ++ } ++ + writel_relaxed(APPLE_WDT_CTRL_RESET_EN, wdt->regs + APPLE_WDT_WD1_CTRL); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_BITE_TIME); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_CUR_TIME); +@@ -190,7 +197,11 @@ static int apple_wdt_probe(struct platform_device *pdev) + watchdog_init_timeout(&wdt->wdd, 0, dev); + apple_wdt_set_timeout(&wdt->wdd, wdt->wdd.timeout); + watchdog_stop_on_unregister(&wdt->wdd); +- watchdog_set_restart_priority(&wdt->wdd, 128); ++ /* The tethered cold escape must beat PSCI (priority 129). Warm/soft ++ * still refuse above, allowing the PSCI handler to run next. ++ */ ++ watchdog_set_restart_priority(&wdt->wdd, ++ of_property_read_bool(dev->of_node, "apple,tethered-boot") ? 255 : 128); + + return devm_watchdog_register_device(dev, &wdt->wdd); + } diff --git a/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch b/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch new file mode 100644 index 00000000..67e6cb21 --- /dev/null +++ b/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:06 +0100 +Subject: [PATCH] watchdog: apple: initialize the parent before registering + restart + +The tethered restart guard dereferences the parent, but probe never set +it. Initialize it before registration and tolerate a missing parent in +the restart callback so the cold watchdog writes remain reachable. + +Assisted-by: LLM +--- + drivers/watchdog/apple_wdt.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index 04e287fe8..6de9f863c 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -124,7 +124,8 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + +- if (of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && ++ if (wdd->parent && ++ of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && + (mode == REBOOT_WARM || mode == REBOOT_SOFT)) { + pr_emerg("apple-wdt: warm/soft reset delegated to PSCI on tethered boot\n"); + return 0; /* Continue the restart chain; NOTIFY_BAD would stop PSCI. */ +@@ -185,6 +186,7 @@ static int apple_wdt_probe(struct platform_device *pdev) + + platform_set_drvdata(pdev, wdt); + ++ wdt->wdd.parent = dev; + wdt->wdd.ops = &apple_wdt_ops; + wdt->wdd.info = &apple_wdt_info; + wdt->wdd.max_hw_heartbeat_ms = U32_MAX / wdt->clk_rate * 1000; From 40a8b18d6c9fb646ef2e5a44f559451e5b203419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 25/33] j71t: Suspend to idle with retained ANS power and early driver admission Suspend on J71t lost the NVMe controller (its port must keep power and refclk), the DART translations, the USB gadget link and Wi-Fi after resume, and drivers could only refuse sleep after the filesystems had been synced. Add a driver admission step and a precheck notifier before sync to the PM core, with a suspend_ready report. Pair suspend to idle with retained ANS power in the H9P driver, stop and rebuild the ANS DART and NVMe controller around sleep, keep the dwc2 gadget connected, stop the VXD firmware, and keep Wi-Fi down when its link is lost. The PMIC GPIO driver refuses sleep when it cannot program the wake mask, the charger uses J71t's plug event as a wake source, and the J71t DT enables paired sleep, the s2idle gadget and charger wake. On restart and poweroff, NVMe stops the Apple ANS as it does before a kexec, and the H9P driver hands the S8003 ports off in the same powered state; these two patches sit here because they build on this topic's H9P and NVMe sleep code. Assisted-by: LLM --- ...j71t-enable-paired-sleep-and-the-s2i.patch | 30 ++ ...4-dts-apple-retain-J71t-charger-wake.patch | 24 ++ ...-driver-admission-before-sync-throug.patch | 205 +++++++++ ...suspend-admission-readiness-in-sys-p.patch | 176 ++++++++ ...-refuse-sleep-when-the-wake-mask-wri.patch | 33 ++ ...-save-and-rebuild-translations-aroun.patch | 97 +++++ ...stop-the-A9-firmware-for-system-slee.patch | 330 ++++++++++++++ ...d-answer-the-suspend-readiness-query.patch | 45 ++ ...ple-ans-declare-retained-DART-resume.patch | 27 ++ ...-ANS-sleep-stop-and-rebuild-through-.patch | 163 +++++++ ...describe-paired-sleep-and-optional-h.patch | 39 ++ ...ir-s2idle-with-retained-ANS-power-an.patch | 402 ++++++++++++++++++ ...ke-the-handoff-link-flush-a-separate.patch | 37 ++ ...ple-ANS-before-a-restart-as-before-a.patch | 61 +++ ...nd-off-S8003-ports-on-restart-and-po.patch | 70 +++ ...le-tristar-select-J71t-USB-wake-even.patch | 56 +++ ...describe-board-selected-s2idle-gadge.patch | 31 ++ ...retained-s2idle-gadget-from-board-da.patch | 160 +++++++ ...stay-down-after-host-sleep-link-loss.patch | 37 ++ tests/test_d2365_poweroff.py | 2 +- 20 files changed, 2024 insertions(+), 1 deletion(-) create mode 100644 kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch create mode 100644 kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch create mode 100644 kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch create mode 100644 kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch create mode 100644 kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch create mode 100644 kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch create mode 100644 kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch create mode 100644 kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch create mode 100644 kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch create mode 100644 kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch create mode 100644 kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch create mode 100644 kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch create mode 100644 kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch create mode 100644 kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch create mode 100644 kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch create mode 100644 kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch create mode 100644 kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch create mode 100644 kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch create mode 100644 kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch diff --git a/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch b/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch new file mode 100644 index 00000000..43ae9e86 --- /dev/null +++ b/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] arm64: dts: apple: j71t: enable paired sleep and the s2idle + gadget + +Mark the J71t ANS port for paired system sleep and keep the USB gadget +connected over suspend to idle. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 3ee634e79..d03bc4ff6 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -75,3 +75,11 @@ + &wdt { + apple,tethered-boot; + }; ++ ++&pcie0 { ++ apple,paired-system-sleep; ++}; ++ ++&usbdev { ++ apple,retain-gadget-in-s2idle; ++}; diff --git a/kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch b/kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch new file mode 100644 index 00000000..9de5ab14 --- /dev/null +++ b/kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: retain charger wake after shared PMU data + +Keep the charger marked as a wake source after the shared PMU data has +been applied. Preserve the separate RTC event ownership. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -185,6 +185,7 @@ + }; + + tristar_charger: charger@180 { ++ wakeup-source; + compatible = "apple,j71t-tristar-charger", "apple,tristar-charger"; + reg = <0x180 0x10>, <0xa00 0x04>, <0x4000 0x03>; + reg-names = "status", "control", "adc"; diff --git a/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch b/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch new file mode 100644 index 00000000..e510724f --- /dev/null +++ b/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch @@ -0,0 +1,205 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:37 +0000 +Subject: [PATCH] PM: sleep: reserve driver admission before sync through + resume or abort + +Some drivers can only enter system sleep while no new work is admitted, +and they have to decide that before the filesystems are synced. Once +sync has run, a refusal from a suspend callback aborts the transition +late, after work that cannot be undone. There is no hook between the +start of a transition and sync for this. + +Add a small admission list to kernel/power/suspend.c. A driver registers +a prepare/complete pair. enter_state() calls every prepare under +system_transition_mutex before sync, then a blocking precheck notifier +chain on which a driver can refuse the transition with -EBUSY. complete +runs in reverse order after resume and after every abort, including a +failed prepare, and the chain is then called with +PM_SUSPEND_PRECHECK_COMPLETE so that notifier users can drop their +state. pm_suspend_admission_reserved() lets a precheck ask whether a +device's admission is held. + +Assisted-by: LLM +--- + include/linux/pm-suspend-admission.h | 29 ++++++++++ + include/linux/suspend.h | 10 ++++ + kernel/power/suspend.c | 82 ++++++++++++++++++++++++++++ + 3 files changed, 121 insertions(+) + create mode 100644 include/linux/pm-suspend-admission.h + +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +new file mode 100644 +index 000000000..f1c92e058 +--- /dev/null ++++ b/include/linux/pm-suspend-admission.h +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_PM_SUSPEND_ADMISSION_H ++#define _LINUX_PM_SUSPEND_ADMISSION_H ++#include ++#include ++ ++/* Driver-owned reversible admission gate. Registration must precede exposing ++ * hardware work. Unregister before removing the hardware or freeing this object. ++ * prepare must reserve admission without destructive MMIO; complete releases it ++ * after resume or ANY abort, including a failed prepare. No PM-op inference. ++ */ ++struct pm_suspend_admission { ++ struct list_head node; ++ struct device *dev; ++ int (*prepare)(struct device *dev); ++ void (*complete)(struct device *dev); ++ bool reserved; ++}; ++#ifdef CONFIG_SUSPEND ++int register_pm_suspend_admission(struct pm_suspend_admission *admission); ++void unregister_pm_suspend_admission(struct pm_suspend_admission *admission); ++/* Only callable from a precheck notifier, under system_transition_mutex. */ ++bool pm_suspend_admission_reserved(struct device *dev); ++#else ++static inline int register_pm_suspend_admission(struct pm_suspend_admission *a) { return 0; } ++static inline void unregister_pm_suspend_admission(struct pm_suspend_admission *a) {} ++static inline bool pm_suspend_admission_reserved(struct device *dev) { return false; } ++#endif ++#endif +diff --git a/include/linux/suspend.h b/include/linux/suspend.h +index b02876f1a..93082b990 100644 +--- a/include/linux/suspend.h ++++ b/include/linux/suspend.h +@@ -142,9 +142,19 @@ struct platform_s2idle_ops { + void (*end)(void); + }; + ++#ifdef CONFIG_SUSPEND ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb); ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb); ++#else ++static inline int register_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } ++static inline int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } ++#endif ++#define PM_SUSPEND_PRECHECK_COMPLETE 0 ++ + #ifdef CONFIG_SUSPEND + extern suspend_state_t pm_suspend_target_state; + extern suspend_state_t mem_sleep_current; ++ + extern suspend_state_t mem_sleep_default; + + /** +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 57c442686..307e6d5f3 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -9,6 +9,7 @@ + + #define pr_fmt(fmt) "PM: " fmt + ++#include + #include + #include + #include +@@ -573,8 +574,68 @@ static void suspend_finish(void) + * Fail if that's not the case. Otherwise, prepare for system suspend, make the + * system enter the given sleep state and clean up after wakeup. + */ ++static LIST_HEAD(suspend_admissions); ++ ++int register_pm_suspend_admission(struct pm_suspend_admission *a) ++{ ++ struct pm_suspend_admission *other; ++ ++ if (!a->dev || !a->prepare || !a->complete) ++ return -EINVAL; ++ mutex_lock(&system_transition_mutex); ++ list_for_each_entry(other, &suspend_admissions, node) { ++ if (other->dev == a->dev) { ++ mutex_unlock(&system_transition_mutex); ++ return -EEXIST; ++ } ++ } ++ a->reserved = false; ++ get_device(a->dev); ++ list_add_tail(&a->node, &suspend_admissions); ++ mutex_unlock(&system_transition_mutex); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_admission); ++ ++void unregister_pm_suspend_admission(struct pm_suspend_admission *a) ++{ ++ /* Serializes removal/module unload with reservation through complete. */ ++ mutex_lock(&system_transition_mutex); ++ list_del(&a->node); ++ put_device(a->dev); ++ mutex_unlock(&system_transition_mutex); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_admission); ++ ++bool pm_suspend_admission_reserved(struct device *dev) ++{ ++ struct pm_suspend_admission *a; ++ ++ lockdep_assert_held(&system_transition_mutex); ++ list_for_each_entry(a, &suspend_admissions, node) ++ if (a->dev == dev) ++ return a->reserved; ++ return false; ++} ++EXPORT_SYMBOL_GPL(pm_suspend_admission_reserved); ++ ++static BLOCKING_NOTIFIER_HEAD(suspend_precheck_chain); ++ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_register(&suspend_precheck_chain, nb); ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_precheck_notifier); ++ ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_unregister(&suspend_precheck_chain, nb); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); ++ + static int enter_state(suspend_state_t state) + { ++ struct pm_suspend_admission *admission; + int error; + + trace_suspend_resume(TPS("suspend_enter"), state, true); +@@ -594,6 +655,18 @@ static int enter_state(suspend_state_t state) + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); + ++ list_for_each_entry(admission, &suspend_admissions, node) { ++ /* Also release a provider whose prepare returned an error. */ ++ admission->reserved = true; ++ error = admission->prepare(admission->dev); ++ if (error) ++ goto Unlock; ++ } ++ error = notifier_to_errno(blocking_notifier_call_chain( ++ &suspend_precheck_chain, state, NULL)); ++ if (error) ++ goto Unlock; ++ + if (sync_on_suspend_enabled) { + trace_suspend_resume(TPS("sync_filesystems"), 0, true); + +@@ -622,6 +695,15 @@ static int enter_state(suspend_state_t state) + pm_pr_dbg("Finishing wakeup.\n"); + suspend_finish(); + Unlock: ++ list_for_each_entry_reverse(admission, &suspend_admissions, node) { ++ if (!admission->reserved) ++ continue; ++ admission->complete(admission->dev); ++ admission->reserved = false; ++ } ++ /* Release driver admission reservations on success and every abort. */ ++ blocking_notifier_call_chain(&suspend_precheck_chain, ++ PM_SUSPEND_PRECHECK_COMPLETE, NULL); + mutex_unlock(&system_transition_mutex); + return error; + } diff --git a/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch b/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch new file mode 100644 index 00000000..88392fe4 --- /dev/null +++ b/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch @@ -0,0 +1,176 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:11:36 +0000 +Subject: [PATCH] PM: sleep: report suspend admission readiness in + /sys/power/suspend_ready + +Driver admission providers and the early suspend prechecks decide inside +enter_state() whether a suspend is admitted, but userspace has no way to +ask before requesting one. A desktop that offers Suspend, or a power +daemon that would suspend on idle, can only try it and see the +transition fail. + +Add an optional non-reserving check callback to struct +pm_suspend_admission and a read-only /sys/power/suspend_ready. Reading +it takes system_transition_mutex without waiting, runs every provider's +check and then every precheck notifier for the current mem_sleep state, +and reports "ready" or "not-ready " for the first refusal. +While the query runs, pm_suspend_admission_reserved() also reports a +provider whose check passed, so prechecks that require a reserved +provider answer as they would during a suspend. A provider without a +check reports not ready. No reservation is taken, nothing is logged by +the query itself, and a later suspend reserves and checks again. + +Assisted-by: LLM +--- + Documentation/ABI/testing/sysfs-power | 18 ++++++++ + include/linux/pm-suspend-admission.h | 10 ++++- + kernel/power/suspend.c | 63 ++++++++++++++++++++++++++- + 3 files changed, 89 insertions(+), 2 deletions(-) + +diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power +index d38da0779..98cc5c2b5 100644 +--- a/Documentation/ABI/testing/sysfs-power ++++ b/Documentation/ABI/testing/sysfs-power +@@ -470,3 +470,21 @@ Description: + + Minimum value: 1 + Default value: 3 ++ ++What: /sys/power/suspend_ready ++Date: November 2026 ++Contact: Linux PM list ++Description: ++ Read-only report of whether a suspend to the current ++ mem_sleep state would be admitted now. Reading runs the ++ admission a suspend runs before syncing filesystems: each ++ registered driver admission provider's non-reserving check, ++ then each early suspend precheck. It takes no reservation ++ and changes no state; a later suspend checks again. ++ ++ The file reads "ready", or one line ++ "not-ready " naming the first refusal: a ++ device name for a provider, the precheck callback's symbol, ++ "transition" while a system transition is in progress, or ++ "state" when the mem_sleep state is not available. ++ is a negative decimal errno value. +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +index f1c92e058..0b1a1c57b 100644 +--- a/include/linux/pm-suspend-admission.h ++++ b/include/linux/pm-suspend-admission.h +@@ -8,18 +8,26 @@ + * hardware work. Unregister before removing the hardware or freeing this object. + * prepare must reserve admission without destructive MMIO; complete releases it + * after resume or ANY abort, including a failed prepare. No PM-op inference. ++ * check, optional, answers /sys/power/suspend_ready: whether prepare would ++ * admit now, without reserving or any side effect. Work prepare would drain ++ * is not a refusal. A provider without check reports not ready. + */ + struct pm_suspend_admission { + struct list_head node; + struct device *dev; + int (*prepare)(struct device *dev); + void (*complete)(struct device *dev); ++ int (*check)(struct device *dev); + bool reserved; ++ bool checked; /* check passed during a readiness query */ + }; + #ifdef CONFIG_SUSPEND + int register_pm_suspend_admission(struct pm_suspend_admission *admission); + void unregister_pm_suspend_admission(struct pm_suspend_admission *admission); +-/* Only callable from a precheck notifier, under system_transition_mutex. */ ++/* ++ * Only callable from a precheck notifier, under system_transition_mutex. ++ * During a readiness query it reports a provider whose check passed. ++ */ + bool pm_suspend_admission_reserved(struct device *dev); + #else + static inline int register_pm_suspend_admission(struct pm_suspend_admission *a) { return 0; } +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 307e6d5f3..670f50ad8 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -590,6 +590,7 @@ int register_pm_suspend_admission(struct pm_suspend_admission *a) + } + } + a->reserved = false; ++ a->checked = false; + get_device(a->dev); + list_add_tail(&a->node, &suspend_admissions); + mutex_unlock(&system_transition_mutex); +@@ -614,7 +615,7 @@ bool pm_suspend_admission_reserved(struct device *dev) + lockdep_assert_held(&system_transition_mutex); + list_for_each_entry(a, &suspend_admissions, node) + if (a->dev == dev) +- return a->reserved; ++ return a->reserved || a->checked; + return false; + } + EXPORT_SYMBOL_GPL(pm_suspend_admission_reserved); +@@ -633,6 +634,66 @@ int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) + } + EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); + ++/* ++ * The admission enter_state() would run, read only: each provider's check ++ * instead of its reservation, then each precheck notifier for the current ++ * mem_sleep state. No reservation is taken and nothing is logged here; a ++ * later suspend still reserves and checks again. ++ */ ++static ssize_t suspend_ready_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ suspend_state_t state = mem_sleep_current; ++ struct pm_suspend_admission *a; ++ struct notifier_block *nb; ++ ssize_t len = 0; ++ int error; ++ ++ if (!mutex_trylock(&system_transition_mutex)) ++ return sysfs_emit(buf, "not-ready transition %d\n", -EBUSY); ++ if (state != PM_SUSPEND_TO_IDLE && !valid_state(state)) { ++ len = sysfs_emit(buf, "not-ready state %d\n", -EINVAL); ++ goto unlock; ++ } ++ list_for_each_entry(a, &suspend_admissions, node) { ++ error = a->check ? a->check(a->dev) : -EOPNOTSUPP; ++ if (error) { ++ len = sysfs_emit(buf, "not-ready %s %d\n", ++ dev_name(a->dev), error); ++ goto clear; ++ } ++ a->checked = true; ++ } ++ /* Walk the chain to name the refusing notifier, not only its errno. */ ++ down_read(&suspend_precheck_chain.rwsem); ++ for (nb = rcu_dereference_raw(suspend_precheck_chain.head); nb; ++ nb = rcu_dereference_raw(nb->next)) { ++ error = notifier_to_errno(nb->notifier_call(nb, state, NULL)); ++ if (error) { ++ len = sysfs_emit(buf, "not-ready %ps %d\n", ++ nb->notifier_call, error); ++ break; ++ } ++ } ++ up_read(&suspend_precheck_chain.rwsem); ++ if (!len) ++ len = sysfs_emit(buf, "ready\n"); ++clear: ++ list_for_each_entry(a, &suspend_admissions, node) ++ a->checked = false; ++unlock: ++ mutex_unlock(&system_transition_mutex); ++ return len; ++} ++ ++power_attr_ro(suspend_ready); ++ ++static int __init suspend_ready_init(void) ++{ ++ return sysfs_create_file(power_kobj, &suspend_ready_attr.attr); ++} ++late_initcall(suspend_ready_init); ++ + static int enter_state(suspend_state_t state) + { + struct pm_suspend_admission *admission; diff --git a/kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch b/kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch new file mode 100644 index 00000000..47656153 --- /dev/null +++ b/kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch @@ -0,0 +1,33 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] gpio: apple-d2365: refuse sleep when the wake mask write + fails + +apple_pmic_gpio_suspend() warned and went on when writing the event wake +mask failed. The system then slept with wake sources the PMIC would not +report, and could not be woken by them. Return the error so that the +suspend is aborted, and drop the IRQ lock on that path. + +Assisted-by: LLM +--- + drivers/gpio/gpio-apple-d2365.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpio/gpio-apple-d2365.c b/drivers/gpio/gpio-apple-d2365.c +index ba66e8eab..c516dfe30 100644 +--- a/drivers/gpio/gpio-apple-d2365.c ++++ b/drivers/gpio/gpio-apple-d2365.c +@@ -588,8 +588,10 @@ static int apple_pmic_gpio_suspend(struct device *dev) + mutex_lock(&gpio->irq_lock); + /* Only wake-enabled events may pull the line while asleep. */ + ret = apple_pmic_event_sync(gpio, gpio->wake); +- if (ret) +- dev_warn(dev, "event wake mask write failed: %d\n", ret); ++ if (ret) { ++ mutex_unlock(&gpio->irq_lock); ++ return ret; ++ } + gpio->suspended = true; + mutex_unlock(&gpio->irq_lock); + /* A thread that passed the gate before it closed has finished now. */ diff --git a/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch b/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch new file mode 100644 index 00000000..711841f3 --- /dev/null +++ b/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch @@ -0,0 +1,97 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] iommu/apple-dart: save and rebuild translations around + certified ANS stop + +apple_ans_dart_stop() stops the ANS DART before a kexec handoff. System +sleep needs the same stop and its reverse. + +Save the DART's translation state when apple_ans_dart_stop() stops it, +and add apple_ans_dart_start() to restore it. The regular suspend +callback skips a DART stopped this way, which would otherwise save its +state a second time, and resume clears the flag. The NVMe sleep path +uses the pair to stop the ANS translations before sleep and rebuild them +on resume. + +Assisted-by: LLM +--- + drivers/iommu/apple-dart.c | 29 +++++++++++++++++++++++++++++ + include/linux/apple-ans.h | 2 ++ + 2 files changed, 31 insertions(+) + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 5aff4bc3e..5c28d4e4b 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -217,6 +217,7 @@ enum dart_type { + * @iommu: iommu core device + */ + struct apple_dart { ++ bool ans_stopped; + struct device *dev; + const struct apple_dart_hw *hw; + +@@ -1010,6 +1011,9 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++static __maybe_unused int apple_dart_suspend(struct device *dev); ++static __maybe_unused int apple_dart_resume(struct device *dev); ++ + int apple_ans_dart_stop(struct device *dev) + { + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); +@@ -1026,6 +1030,8 @@ int apple_ans_dart_stop(struct device *dev) + for_each_stream_map(i, cfg, map) { + if (map->dart->hw->type != DART_S5L8960X) + return -EOPNOTSUPP; ++ apple_dart_suspend(map->dart->dev); ++ map->dart->ans_stopped = true; + apple_dart_hw_disable_dma(map); + } + dev_info(dev, "Apple ANS: DART stopped for handoff\n"); +@@ -1033,6 +1039,25 @@ int apple_ans_dart_stop(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_ans_dart_stop); + ++int apple_ans_dart_start(struct device *dev) ++{ ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ int i, ret; ++ ++ if (!cfg) ++ return -ENODEV; ++ for_each_stream_map(i, cfg, map) { ++ if (!map->dart->ans_stopped) ++ continue; ++ ret = apple_dart_resume(map->dart->dev); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_start); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { +@@ -2017,6 +2042,9 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + ++ if (dart->ans_stopped) ++ return 0; ++ + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = dart->hw->read_tcr(dart, sid); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) +@@ -2046,6 +2074,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) + dart->hw->write_tcr(dart, sid, dart->save_tcr[sid]); + } + ++ dart->ans_stopped = false; + return 0; + } + diff --git a/kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch b/kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch new file mode 100644 index 00000000..15940583 --- /dev/null +++ b/kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch @@ -0,0 +1,330 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:23:32 +0000 +Subject: [PATCH] media: apple-vxd: stop the A9 firmware for system sleep + +The DEVA firmware boots on the first open of the video node and stays +resident until the driver is unbound. udev and media daemons open the +node at boot, so the firmware is resident in practice. Over s2idle the +decoder power domain goes off and the resident firmware does not survive +it, and the J71t sleep precheck refuses any bound decoder without a +reserved sleep admission. + +Follow AppleD5500 instead, which stops the core and frees the firmware +when its last client closes (terminateDeviceInternal, fffffff00628a604: +clear the ready magic, write 1 to 0x200, known state v1, free, power +down). Register a sleep admission provider for the A9 variant. Its +prepare does no MMIO: it refuses while a file is open and otherwise +closes the open gate until complete, after resume or any abort. The +system suspend callback then stops the firmware in that order and frees +what it used; the next open boots it again, as the first open after +probe does. If the core does not reach its known state, every buffer it +may still read is kept and the sleep is refused. + +The T8010 variant registers no provider and keeps its behaviour. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 63 ++++++++++++++- + .../media/platform/apple/vxd/apple-vxd-fw.c | 77 +++++++++++++++++++ + .../media/platform/apple/vxd/apple-vxd-regs.h | 6 ++ + .../media/platform/apple/vxd/apple-vxd-v4l2.c | 9 ++- + drivers/media/platform/apple/vxd/apple-vxd.h | 12 +++ + 5 files changed, 164 insertions(+), 3 deletions(-) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index ab7515fee..31a024233 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -245,7 +245,51 @@ static int apple_vxd_runtime_suspend(struct device *dev) + return 0; + } + ++/* ++ * Sleep admission: no MMIO. Refuse while a file is open (its contexts and ++ * any queued pictures belong to the running firmware); otherwise close the ++ * open gate until complete, after resume or any abort. ++ */ ++static int apple_vxd_sleep_prepare(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ int ret = 0; ++ ++ mutex_lock(&vxd->power_lock); ++ if (vxd->users) ++ ret = -EBUSY; ++ else ++ vxd->sleep_gate = true; ++ mutex_unlock(&vxd->power_lock); ++ if (ret) ++ dev_info(dev, "system sleep refused: decoder in use\n"); ++ return ret; ++} ++ ++static void apple_vxd_sleep_complete(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ ++ mutex_lock(&vxd->power_lock); ++ vxd->sleep_gate = false; ++ mutex_unlock(&vxd->power_lock); ++} ++ ++static int apple_vxd_suspend(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ ++ if (!vxd->variant->sleep_stop) ++ return 0; ++ ++ /* No open file: nothing can queue a picture, let the last one finish. */ ++ flush_work(&vxd->job_done_work); ++ cancel_delayed_work_sync(&vxd->watchdog_work); ++ return apple_vxd_fw_sleep_stop(vxd); ++} ++ + static const struct dev_pm_ops apple_vxd_pm_ops = { ++ .suspend = pm_sleep_ptr(apple_vxd_suspend), + RUNTIME_PM_OPS(apple_vxd_runtime_suspend, apple_vxd_runtime_resume, NULL) + }; + +@@ -254,6 +298,7 @@ static const struct apple_vxd_variant s8000_vxd_variant = { + .num_gart_pts = 512, /* "setting 2GB mapping" [E] */ + /* HEVCSupported = ((rev - 7) < 4) && ((0xD >> (rev - 7)) & 1) [E] */ + .hevc_supported = true, ++ .sleep_stop = true, + }; + + /* The T8010 core has two pixel pipes; it reports them in CORE_ID. */ +@@ -353,14 +398,27 @@ static int apple_vxd_probe(struct platform_device *pdev) + if (ret) + goto err_disable_runtime_pm; + ++ /* Before the video node exists: nothing can open it unadmitted. */ ++ if (variant->sleep_stop) { ++ vxd->sleep_admission.dev = &pdev->dev; ++ vxd->sleep_admission.prepare = apple_vxd_sleep_prepare; ++ vxd->sleep_admission.complete = apple_vxd_sleep_complete; ++ ret = register_pm_suspend_admission(&vxd->sleep_admission); ++ if (ret) ++ goto err_gart_fini; ++ } ++ + ret = apple_vxd_h264_register(vxd); + if (!ret) + ret = apple_vxd_v4l2_init(vxd); + if (ret) +- goto err_gart_fini; ++ goto err_admission; + + return 0; + ++err_admission: ++ if (variant->sleep_stop) ++ unregister_pm_suspend_admission(&vxd->sleep_admission); + err_gart_fini: + apple_vxd_gart_fini(vxd); + err_disable_runtime_pm: +@@ -373,6 +431,9 @@ static void apple_vxd_remove(struct platform_device *pdev) + { + struct apple_vxd_dev *vxd = platform_get_drvdata(pdev); + ++ /* Before any teardown; also waits out a transition in progress. */ ++ if (vxd->variant->sleep_stop) ++ unregister_pm_suspend_admission(&vxd->sleep_admission); + cancel_delayed_work_sync(&vxd->watchdog_work); + apple_vxd_v4l2_cleanup(vxd); + apple_vxd_fw_cleanup(vxd); +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-fw.c b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +index 753ca4944..ede737efd 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-fw.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +@@ -435,6 +435,83 @@ int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd) + return ret; + } + ++/* ++ * One reference per open file. A sleep that has been admitted closes the ++ * gate first, so an open between admission and the device's suspend ++ * callback fails instead of booting the firmware behind it. ++ */ ++int apple_vxd_fw_open(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ if (vxd->sleep_gate) { ++ mutex_unlock(&vxd->power_lock); ++ return -EBUSY; ++ } ++ vxd->users++; ++ mutex_unlock(&vxd->power_lock); ++ ++ ret = apple_vxd_boot_firmware(vxd); ++ if (ret) ++ apple_vxd_fw_close(vxd); ++ return ret; ++} ++ ++void apple_vxd_fw_close(struct apple_vxd_dev *vxd) ++{ ++ mutex_lock(&vxd->power_lock); ++ if (!WARN_ON(!vxd->users)) ++ vxd->users--; ++ mutex_unlock(&vxd->power_lock); ++} ++ ++/* ++ * System sleep, after admission and with no open file: stop the MTX in ++ * AppleD5500's terminate order (clear the ready magic, the 0x200 write, ++ * known state v1), then free what the running firmware used. The next ++ * open boots it again, as the first open after probe does. If the core ++ * does not reach its known state, keep every buffer it may still read ++ * and refuse the sleep; the decoder then stays failed (-EIO) until rebind. ++ */ ++int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ if (!vxd->sleep_gate || vxd->users) { ++ mutex_unlock(&vxd->power_lock); ++ return -EBUSY; ++ } ++ if (!vxd->fw_loaded) { ++ mutex_unlock(&vxd->power_lock); ++ return 0; ++ } ++ ++ apple_vxd_mbox_stop(vxd); ++ vxd->hw_ready = false; ++ vxd_write(vxd, VXD_REG_FW_READY_MAGIC, 0); ++ vxd_write(vxd, VXD_REG_VECT_CFG, VXD_VECT_CFG_TERMINATE); ++ ret = vxd_known_state_v1(vxd, "Terminate"); ++ if (ret) { ++ dev_err(vxd->dev, "firmware stop for sleep failed: %d\n", ret); ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++ } ++ ++ apple_vxd_cmdbufs_fini(vxd); ++ apple_vxd_gart_unmap(vxd, &vxd->fw_map); ++ dma_free_coherent(vxd->dev, vxd->fw_size, vxd->fw_vaddr, vxd->fw_dma); ++ vxd->fw_vaddr = NULL; ++ vxd->fw_loaded = false; ++ mutex_unlock(&vxd->power_lock); ++ ++ /* The reference boot took; the PM core holds its own until complete. */ ++ pm_runtime_put_noidle(vxd->dev); ++ dev_info(vxd->dev, "firmware stopped for system sleep\n"); ++ return 0; ++} ++ + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd) + { + bool was_loaded; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-regs.h b/drivers/media/platform/apple/vxd/apple-vxd-regs.h +index edeccb8e0..fdbe48903 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-regs.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd-regs.h +@@ -89,6 +89,12 @@ + + /* VEC block and entropy decoder pipes */ + #define VXD_REG_VECT_CFG 0x0200 ++/* ++ * AppleD5500's terminate path (fffffff00628c474 -> fffffff00629972c [E]) ++ * writes 1 here after clearing the ready magic and before known state v1. ++ * Nothing beyond that order is known about the write. ++ */ ++#define VXD_VECT_CFG_TERMINATE 0x00000001 + #define VXD_REG_VECT_CFG2 0x0208 + #define VXD_VECT_CFG2_MSGS_MINUS1 0x000000c7 /* core[0xb4] - 1 */ + #define VXD_REG_ENTDEC_0_SIZE 0x0340 +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +index 23c051f1d..e9467232d 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +@@ -885,13 +885,15 @@ static int apple_vxd_open(struct file *filp) + * can be read, and keeps running (and the block powered) until the + * driver is unbound. Later opens find it running. + */ +- ret = apple_vxd_boot_firmware(vxd); ++ ret = apple_vxd_fw_open(vxd); + if (ret) + return ret; + + ctx = kzalloc_obj(*ctx); +- if (!ctx) ++ if (!ctx) { ++ apple_vxd_fw_close(vxd); + return -ENOMEM; ++ } + + ctx->vxd = vxd; + v4l2_fh_init(&ctx->fh, vdev); +@@ -900,6 +902,7 @@ static int apple_vxd_open(struct file *filp) + if (ret) { + v4l2_fh_exit(&ctx->fh); + kfree(ctx); ++ apple_vxd_fw_close(vxd); + return ret; + } + +@@ -910,11 +913,13 @@ static int apple_vxd_open(struct file *filp) + static int apple_vxd_release(struct file *filp) + { + struct apple_vxd_ctx *ctx = file_to_vxd_ctx(filp); ++ struct apple_vxd_dev *vxd = ctx->vxd; + + v4l2_fh_del(&ctx->fh, filp); + apple_vxd_ctx_fini(ctx); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); ++ apple_vxd_fw_close(vxd); + + return 0; + } +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index bba04f75a..81fef0f1a 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -66,6 +67,11 @@ struct apple_vxd_variant { + u32 num_gart_pts; /* mapped page tables (512 = 2 GiB) */ + bool hevc_supported; + bool has_integration; /* second reg range, see VXD_INTEG_CTRL */ ++ /* ++ * Stop the firmware for system sleep the way AppleD5500 does when its ++ * last client closes, and boot it again on the next open. ++ */ ++ bool sleep_stop; + }; + + /* +@@ -284,6 +290,9 @@ struct apple_vxd_dev { + /* Core state. */ + struct mutex power_lock; /* boot vs. runtime suspend */ + bool hw_ready; /* firmware took its init message */ ++ unsigned int users; /* open files, under power_lock */ ++ bool sleep_gate; /* sleep admitted: no new open */ ++ struct pm_suspend_admission sleep_admission; + u32 core_id; + u32 core_rev; + u32 num_entropy_pipes; +@@ -448,6 +457,9 @@ void apple_vxd_dev_buf_free(struct apple_vxd_dev *vxd, + + /* fw.c */ + int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_open(struct apple_vxd_dev *vxd); ++void apple_vxd_fw_close(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd); + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd); + + /* mbox.c */ diff --git a/kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch b/kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch new file mode 100644 index 00000000..fdae4cdb --- /dev/null +++ b/kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:11:36 +0000 +Subject: [PATCH] media: apple-vxd: answer the suspend readiness query + +The A9 decoder's sleep admission refuses while a file is open and +otherwise closes its open gate until complete. Provide the matching +check: report -EBUSY while a file is open, without touching the gate. + +Assisted-by: LLM +--- + drivers/media/platform/apple/vxd/apple-vxd-core.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index 31a024233..dff75927d 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -266,6 +266,18 @@ static int apple_vxd_sleep_prepare(struct device *dev) + return ret; + } + ++/* Readiness query: what prepare would decide, without closing the gate. */ ++static int apple_vxd_sleep_check(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ ret = vxd->users ? -EBUSY : 0; ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++} ++ + static void apple_vxd_sleep_complete(struct device *dev) + { + struct apple_vxd_dev *vxd = dev_get_drvdata(dev); +@@ -403,6 +415,7 @@ static int apple_vxd_probe(struct platform_device *pdev) + vxd->sleep_admission.dev = &pdev->dev; + vxd->sleep_admission.prepare = apple_vxd_sleep_prepare; + vxd->sleep_admission.complete = apple_vxd_sleep_complete; ++ vxd->sleep_admission.check = apple_vxd_sleep_check; + ret = register_pm_suspend_admission(&vxd->sleep_admission); + if (ret) + goto err_gart_fini; diff --git a/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch b/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch new file mode 100644 index 00000000..462bcde9 --- /dev/null +++ b/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch @@ -0,0 +1,27 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:00:00 +0100 +Subject: [PATCH] apple-ans: declare retained DART resume after handoff API + +Declare the retained DART resume operation after the ANS interface has +been introduced, so the paired sleep consumer can call it. + +Assisted-by: LLM +--- + include/linux/apple-ans.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index 10be3ddef..a312a683c 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -19,7 +19,9 @@ struct device; + */ + #if IS_REACHABLE(CONFIG_APPLE_DART) + int apple_ans_dart_stop(struct device *dev); ++int apple_ans_dart_start(struct device *dev); + #else ++static inline int apple_ans_dart_start(struct device *dev) { return -EOPNOTSUPP; } + static inline int apple_ans_dart_stop(struct device *dev) + { + return -EOPNOTSUPP; diff --git a/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch b/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch new file mode 100644 index 00000000..210631ae --- /dev/null +++ b/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch @@ -0,0 +1,163 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] nvme-pci: certify ANS sleep stop and rebuild through + warm-storage APIs + +On S8003 the NVMe controller (ANS) sits behind a PCIe port that keeps +power and reference clock over system sleep, while its DART loses its +translations. The generic nvme_suspend() path neither drains the paired +controller nor rebuilds the DART, so storage did not come back after +suspend to idle. + +For the paired ANS controller, nvme_suspend() now begins the handoff, +moves the controller to RESETTING and shuts it down through +nvme_dev_disable_paired(). preinit restores the DART with +apple_ans_dart_start(). In nvme_dev_disable_paired() a failure while +preparing for sleep unfreezes the queues and returns -EBUSY, so the +suspend is aborted instead of taking the cold-reset path used for kexec. +Resume returns -ENODEV when the PCIe link was lost over sleep. + +Assisted-by: LLM +--- + drivers/nvme/host/pci-apple-ans.c | 7 ++++ + drivers/nvme/host/pci.c | 54 +++++++++++++++++++++++++------ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index fdd9fa091..bdd9f991f 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -6,6 +6,7 @@ + * Copyright (C) 2026 Paul Praschl + */ + ++#include + #include + #include + #include +@@ -276,6 +277,12 @@ static int nvme_pci_apple_ans_preinit(struct nvme_dev *dev) + if (!ans) + return 0; + ++ if (apple_ans_is_s8003(dev->dev)) { ++ ret = apple_ans_dart_start(dev->dev); ++ if (ret) ++ return ret; ++ } ++ + core_mask = readl(dev->bar + APPLE_ANS_REG_CORE_MASK); + log_size = readl(dev->bar + APPLE_ANS_REG_LOG_SIZE); + hmb_size = readl(dev->bar + APPLE_ANS_REG_HMB_SIZE_REQ); +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 2f356a8ce..6868811d5 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -3381,6 +3381,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ bool for_sleep = paired && !kexec_in_progress; + int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); +@@ -3393,28 +3394,40 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + * if doing a safe shutdown. + */ + if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) { +- apple_cold_reset("Apple ANS: outgoing I/O did not drain"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (for_sleep) { ++ nvme_unfreeze(&dev->ctrl); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EBUSY; ++ } else { ++ apple_cold_reset("Apple ANS: outgoing I/O did not drain"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + } + + nvme_quiesce_io_queues(&dev->ctrl); + + if (paired && (dead || !dev->ctrl.queue_count)) { +- apple_cold_reset("Apple ANS: dead outgoing controller"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (!for_sleep) { ++ apple_cold_reset("Apple ANS: dead outgoing controller"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } ++ goto sleep_failed; + } + if (!dead && dev->ctrl.queue_count > 0) { + if (paired) { + nvme_sync_io_queues(&dev->ctrl); + ret = apple_ans_nvmmu_drain(dev->dev); + if (ret) { +- dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); +- apple_cold_reset("Apple ANS: outgoing drain failed"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (!for_sleep) { ++ dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); ++ apple_cold_reset("Apple ANS: outgoing drain failed"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } ++ goto sleep_failed; + } + } + nvme_delete_io_queues(dev); +@@ -3424,6 +3437,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + ret = nvme_apple_ans_handoff_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } ++sleep_failed: + nvme_suspend_io_queues(dev); + nvme_suspend_queue(dev, 0); + pci_free_irq_vectors(pdev); +@@ -4101,6 +4115,9 @@ static int nvme_resume(struct device *dev) + struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); + struct nvme_ctrl *ctrl = &ndev->ctrl; + ++ if (pci_dev_is_disconnected(to_pci_dev(dev))) ++ return -ENODEV; ++ + if (ndev->last_ps == U32_MAX || + nvme_set_power_state(ctrl, ndev->last_ps) != 0) + goto reset; +@@ -4120,6 +4137,20 @@ static int nvme_suspend(struct device *dev) + int ret = -EBUSY; + + ndev->last_ps = U32_MAX; ++ if (ndev->dma_ops == &nvme_pci_apple_ans_ops && apple_ans_is_s8003(dev)) { ++ ret = apple_ans_handoff_begin(dev); ++ if (ret) ++ return ret; ++ if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || ++ !nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) ++ return -EBUSY; ++ ret = nvme_dev_disable_paired(ndev, true, true); ++ if (ret) { ++ nvme_try_sched_reset(ctrl); ++ return -EBUSY; ++ } ++ return 0; ++ } + + /* + * The platform does not remove power for a kernel managed suspend so +@@ -4201,6 +4232,9 @@ static int nvme_simple_resume(struct device *dev) + struct pci_dev *pdev = to_pci_dev(dev); + struct nvme_dev *ndev = pci_get_drvdata(pdev); + ++ if (pci_dev_is_disconnected(to_pci_dev(dev))) ++ return -ENODEV; ++ + return nvme_try_sched_reset(&ndev->ctrl); + } + diff --git a/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch b/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch new file mode 100644 index 00000000..5d551f21 --- /dev/null +++ b/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch @@ -0,0 +1,39 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] dt-bindings: pci: describe paired sleep and optional + handoff link flush + +Document two boolean properties of the S8003 H9P controller. +apple,paired-system-sleep marks an ANS controller whose port keeps power +across system sleep and is stopped and restarted together with it. +apple,ans-kexec-link-flush makes a warm handoff consume a stale link-up +status under PERST before the link is stopped. + +Assisted-by: LLM +--- + .../devicetree/bindings/pci/apple,t8010-pcie.yaml | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +index f9b72ce26..cebf60d8a 100644 +--- a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml ++++ b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +@@ -231,6 +231,17 @@ properties: + without exposing a root bridge. This is used when an enabled port shares + PHY state with a firmware-enabled sibling port. + ++ apple,ans-kexec-link-flush: ++ type: boolean ++ description: ++ Consume stale ANS link status under PERST during warm handoff. ++ ++ apple,paired-system-sleep: ++ type: boolean ++ description: ++ The ANS controller is paired with retained-power port reset over system ++ sleep. ++ + interrupt-controller: true + + '#interrupt-cells': diff --git a/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch b/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch new file mode 100644 index 00000000..8ad218f5 --- /dev/null +++ b/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch @@ -0,0 +1,402 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] PCI: apple-h9p: pair s2idle with retained ANS power and + early admission + +System sleep on J71t has to keep the ANS port powered: removing its +power or reference clock loses the NVMe controller until a cold boot. + +Add sleep support to the S8003 path of the H9P driver, enabled by +apple,paired-system-sleep. prepare pins the endpoints and holds off +config accesses. suspend asserts PERST and stops the link but keeps +power and refclk, and a failure before the first destructive write rolls +back with power kept. resume brings the ports back through the powered +reset and marks endpoints that did not return as disconnected. The +powered reset (PERST, LTSSM stop and a fresh link training, without a +power cycle) is added here with its only caller; the common block is not +reinitialized across s2idle. + +A suspend precheck refuses with -EBUSY, before sync, anything but +suspend to idle, and also refuses while a filesystem is mounted from an +NBD device, port bring-up work is pending, the port is not powered, or a +bound GPU, VXD or SEP driver holds no reserved sleep admission. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 278 +++++++++++++++++++++++- + 1 file changed, 277 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 5890d632a..7a0a832e3 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -51,6 +51,11 @@ + #include + #include + ++#include ++#include ++#include ++#include ++#include "../pci.h" + #include "pci-host-common.h" + + #define H9P_NUM_PORTS 4 +@@ -719,6 +724,13 @@ struct apple_h9p_pcie { + unsigned int late_tries; + struct delayed_work late_work; + bool ans_handoff_stopped; ++ bool ans_sleep_powered; ++ u32 resume_ports; ++ struct pci_dev *sleep_pinned[32]; ++ unsigned int sleep_npinned; ++ bool sleep_pin_overflow; ++ struct notifier_block sleep_precheck; ++ + + struct apple_h9p_nvmmu nvmmu; + +@@ -1094,6 +1106,9 @@ static int apple_h9p_pcie_attach_genpd(struct apple_h9p_pcie *pcie) + return PTR_ERR(pcie->pd_dev[i]); + } + ++ if (of_property_read_bool(dev->of_node, "apple,paired-system-sleep")) ++ dev_pm_syscore_device(pcie->pd_dev[i], true); ++ + pcie->pd_link[i] = device_link_add(dev, pcie->pd_dev[i], + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | +@@ -1239,6 +1254,20 @@ static void apple_h9p_limit_config_field(u32 *val, int where, int size, + *val = (*val & ~mask) | (value & mask); + } + ++static bool apple_h9p_sleep_config_blocked(struct pci_bus *bus) ++{ ++ struct pci_config_window *cfg = bus->sysdata; ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(cfg->parent); ++ struct pci_bus *root = bus; ++ ++ if (!pcie || bus->number == cfg->busr.start) ++ return false; ++ while (root->parent && root->parent->parent) ++ root = root->parent; ++ return root->self && !(READ_ONCE(pcie->enabled_ports) & ++ BIT(PCI_SLOT(root->self->devfn))); ++} ++ + static int apple_h9p_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) + { +@@ -1258,6 +1287,10 @@ static int apple_h9p_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + pcie = NULL; + } + ++ if (apple_h9p_sleep_config_blocked(bus)) { ++ *val = ~0U; ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ } + ret = pci_generic_config_read(bus, devfn, where, size, val); + if (ret != PCIBIOS_SUCCESSFUL || !pcie) + return ret; +@@ -1295,6 +1328,9 @@ static int apple_h9p_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + return PCIBIOS_DEVICE_NOT_FOUND; + } + ++ if (apple_h9p_sleep_config_blocked(bus)) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ + if (where <= PCI_INTERRUPT_LINE && where + size > PCI_INTERRUPT_LINE) + val |= 0xffu << ((PCI_INTERRUPT_LINE - where) << 3); + +@@ -2925,6 +2961,33 @@ static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) + (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); + } + ++static int apple_s8003_powered_reset(struct apple_h9p_pcie *pcie) ++{ ++ int ret; ++ ++ if (!apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (!ret && READ_ONCE(pcie->ans_sleep_powered) && ++ (readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & S8003_PORT_LINKSTS_UP)) { ++ u32 val; ++ ++ h9p_rmw(pcie->base_port[0] + H9P_PORT_LTSSMCTL, 0, ++ H9P_PORT_LTSSM_ENABLE); ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ ret = apple_s8003_port_stop(pcie, 0); ++ } ++ if (!ret) ++ ret = apple_s8003_start_link(pcie, 0); ++ /* A failed powered reset stays under PERST; no full-cycle fallback. */ ++ if (ret) ++ gpiod_direction_output(pcie->perst[0], 1); ++ return ret; ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -2940,6 +3003,8 @@ static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + * validated on it, the other ports the restart sequence that is the + * only one validated on the Wi-Fi port. + */ ++ if (!port && READ_ONCE(pcie->ans_sleep_powered)) ++ return apple_s8003_powered_reset(pcie); + if (pcie->hw->nvme_mode_ports & BIT(port)) + ret = apple_s8003_power_cycle_port(pcie, port); + else +@@ -3598,6 +3663,208 @@ int apple_ans_handoff_begin(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); + ++ ++static bool apple_h9p_sleep_enabled(struct apple_h9p_pcie *pcie) ++{ ++ return of_property_read_bool(pcie->dev->of_node, "apple,paired-system-sleep"); ++} ++ ++static int apple_h9p_sleep_pin(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ if (pdev->dev_flags & PCI_DEV_FLAGS_NO_D3) ++ return 0; ++ if (pcie->sleep_npinned == ARRAY_SIZE(pcie->sleep_pinned)) { ++ pcie->sleep_pin_overflow = true; ++ return 1; ++ } ++ pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; ++ pcie->sleep_pinned[pcie->sleep_npinned++] = pci_dev_get(pdev); ++ return 0; ++} ++ ++static void apple_h9p_sleep_complete(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ ++ while (pcie && pcie->sleep_npinned) { ++ struct pci_dev *pdev = pcie->sleep_pinned[--pcie->sleep_npinned]; ++ ++ pdev->dev_flags &= ~PCI_DEV_FLAGS_NO_D3; ++ pci_dev_put(pdev); ++ } ++ if (pcie) ++ pcie->ans_sleep_powered = false; ++} ++ ++static int apple_h9p_sleep_prepare(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ pcie->ans_handoff_stopped = false; ++ pcie->ans_sleep_powered = false; ++ pcie->sleep_pin_overflow = false; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_sleep_pin, pcie); ++ ret = pcie->sleep_pin_overflow ? -EBUSY : 0; ++ if (ret) ++ apple_h9p_sleep_complete(dev); ++ return ret; ++} ++ ++static void apple_h9p_sleep_cut_off(struct apple_h9p_pcie *pcie, unsigned int port) ++{ ++ struct pci_dev *rp; ++ ++ gpiod_direction_output(pcie->perst[port], 1); ++ apple_s8003_port_stop(pcie, port); ++ /* Never remove ANS power/refclk, even after failed resume. */ ++ pcie->enabled_ports &= ~BIT(port); ++ rp = pci_get_slot(pcie->bridge->bus, PCI_DEVFN(port, 0)); ++ if (!rp) ++ return; ++ if (rp->subordinate) ++ pci_walk_bus(rp->subordinate, pci_dev_set_disconnected, NULL); ++ pci_dev_set_disconnected(rp, NULL); ++ pci_dev_put(rp); ++} ++ ++static int apple_h9p_sleep_resume(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ unsigned int port; ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ struct pci_dev *rp; ++ ++ if (!(pcie->resume_ports & BIT(port))) ++ continue; ++ if (port && !gpiod_get_value_cansleep(pcie->perst[port]) && ++ apple_h9p_link_up(pcie, port)) ++ continue; ++ ret = port ? apple_s8003_setup_port_tries(pcie, port) : ++ apple_s8003_powered_reset(pcie); ++ if (ret) { ++ apple_h9p_sleep_cut_off(pcie, port); ++ dev_err(dev, "port %u lost after sleep: %d; endpoints disconnected\n", port, ret); ++ continue; ++ } ++ pcie->enabled_ports |= BIT(port); ++ msleep(PCIE_RESET_CONFIG_WAIT_MS); ++ rp = pci_get_slot(pcie->bridge->bus, PCI_DEVFN(port, 0)); ++ if (rp) { ++ pci_restore_state(rp); ++ pci_dev_put(rp); ++ } ++ } ++ if (pcie->enabled_ports & BIT(0)) { ++ ret = apple_h9p_nvmmu_program(pcie); ++ if (ret) ++ apple_h9p_sleep_cut_off(pcie, 0); ++ } ++ pcie->resume_ports = 0; ++ return 0; ++} ++ ++static int apple_h9p_sleep_suspend(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ void __iomem *cfg; ++ u32 val; ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ cfg = pcie->base_config; ++ if (!READ_ONCE(pcie->ans_handoff_stopped) || !apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ pcie->resume_ports = pcie->enabled_ports; ++ /* Latch before the first destructive write; rollback must retain power. */ ++ pcie->ans_sleep_powered = true; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) { ++ /* r1e: consume the falling link before stopping LTSSM. */ ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ ret = apple_s8003_port_stop(pcie, 0); ++ } ++ if (ret) { ++ apple_h9p_sleep_resume(dev); ++ return ret; ++ } ++ udelay(S8003_PERST_TO_REFCLK_US); ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->enabled_ports &= ~BIT(0); ++ return 0; ++} ++ ++static void apple_h9p_sleep_nbd(struct super_block *sb, void *data) ++{ ++ if (sb->s_bdev && !strncmp(sb->s_bdev->bd_disk->disk_name, "nbd", 3)) ++ *(bool *)data = true; ++} ++ ++/* Until GS provides a reserved admission capability, bound accelerators fail closed. */ ++static int apple_h9p_sleep_stack(struct device *dev, void *data) ++{ ++ const char *name; ++ ++ if (!dev->driver) ++ return 0; ++ name = dev->driver->name; ++ if (strstr(name, "g5p") || strstr(name, "powervr") || ++ strstr(name, "vxd") || strstr(name, "sep")) { ++ if (pm_suspend_admission_reserved(dev)) ++ return 0; ++ dev_warn(dev, "system sleep refused: no reserved safe sleep capability\n"); ++ return -EBUSY; ++ } ++ return 0; ++} ++ ++static int apple_h9p_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_h9p_pcie *pcie = container_of(nb, struct apple_h9p_pcie, sleep_precheck); ++ bool nbd = false; ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ if (!apple_h9p_sleep_enabled(pcie)) ++ return NOTIFY_DONE; ++ if (action != PM_SUSPEND_TO_IDLE) ++ return notifier_from_errno(-EBUSY); ++ iterate_supers(apple_h9p_sleep_nbd, &nbd); ++ if (nbd || pcie->late_ports || work_busy(&pcie->late_work.work) || ++ !pcie->pd_dev || !apple_s8003_port_powered(pcie) || ++ bus_for_each_dev(&platform_bus_type, NULL, NULL, apple_h9p_sleep_stack)) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_OK; ++} ++ ++static void apple_h9p_sleep_unregister(void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ unregister_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++} ++ ++static const struct dev_pm_ops apple_h9p_sleep_ops = { ++ .prepare = apple_h9p_sleep_prepare, ++ .complete = apple_h9p_sleep_complete, ++ .suspend_noirq = apple_h9p_sleep_suspend, ++ .resume_noirq = apple_h9p_sleep_resume, ++}; ++ + static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + { + struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); +@@ -3627,6 +3894,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_cold_reset("Apple PCIe: PERST failed during handoff"); + return; + } ++ + if (apple_s8003_port_stop(pcie, port)) { + apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + return; +@@ -3777,6 +4045,9 @@ static const struct attribute_group apple_h9p_crash_group = { + }; + + ++static int apple_h9p_sleep_precheck(struct notifier_block *, unsigned long, void *); ++static void apple_h9p_sleep_unregister(void *); ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3948,7 +4219,11 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +- return 0; ++ pcie->sleep_precheck.notifier_call = apple_h9p_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++ if (ret) ++ return ret; ++ return devm_add_action_or_reset(dev, apple_h9p_sleep_unregister, pcie); + } + + static const struct apple_h9p_hw apple_h9p_hw_t7000 = { +@@ -4032,6 +4307,7 @@ static struct platform_driver apple_h9p_pcie_driver = { + .name = "pcie-apple-h9p", + .of_match_table = apple_h9p_pcie_of_match, + .suppress_bind_attrs = true, ++ .pm = pm_sleep_ptr(&apple_h9p_sleep_ops), + }, + }; + module_platform_driver(apple_h9p_pcie_driver); diff --git a/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch b/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch new file mode 100644 index 00000000..bd2908f3 --- /dev/null +++ b/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] PCI: apple-h9p: make the handoff link flush a separate + opt-in + +Waiting for the link-up status to fall under PERST before LTSSM is +stopped was tied to the sleep path. A warm handoff can see the same +stale status, but not every board needs the wait. + +Do the flush in the outgoing kexec shutdown path only, and only when +apple,ans-kexec-link-flush is set, so it stays off unless a board asks +for it. The incoming kernel restarts the storage port with the cold-boot +power cycle, which needs no flush. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 7a0a832e3..46c24f2b5 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3894,7 +3894,12 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_cold_reset("Apple PCIe: PERST failed during handoff"); + return; + } ++ if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-link-flush")) { ++ u32 val; + ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ } + if (apple_s8003_port_stop(pcie, port)) { + apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + return; diff --git a/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch b/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch new file mode 100644 index 00000000..9e84e7ac --- /dev/null +++ b/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch @@ -0,0 +1,61 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:15:38 +0100 +Subject: [PATCH] nvme-pci: stop Apple ANS before a restart as before a kexec + +A restart that re-enters the boot loader (PSCI SYSTEM_RESET to m1n1) +leaves the S8003 ANS controller to a loader that brings its port up again +and reads from it. On a plain shutdown nvme_disable_prepare_reset() sets +SHN but keeps CC.EN, and power-cycling the port in that state hangs the +SoC. + +Take the paired, certified stop that a kexec handoff already uses on every +restart as well: drain, SHN, CC.EN and RDY clear, bus mastering off and the +NVMMU/DART retired. Such a failure is not a sleep refusal, so it no longer +takes the sleep branch; it ends in a cold watchdog reset, as for kexec. +Power-off and halt keep the existing shutdown. + +Assisted-by: LLM +--- + drivers/nvme/host/pci.c | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 6868811d5..9fe60de0e 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -3381,7 +3381,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; +- bool for_sleep = paired && !kexec_in_progress; ++ bool for_sleep = paired && !kexec_in_progress && system_state != SYSTEM_RESTART; + int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); +@@ -4053,14 +4053,23 @@ static void nvme_reset_done(struct pci_dev *pdev) + flush_work(&dev->ctrl.reset_work); + } + ++/* ++ * A restart re-enters the boot loader, which starts the storage port again ++ * and reads from the controller. It needs the controller stopped as iBoot ++ * leaves it (SHN complete, CC.EN and RDY clear, DMA certified stopped): the ++ * stop a kexec handoff takes. Anything less ends in a cold reset. ++ */ + static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); ++ bool restart = !kexec_in_progress && system_state == SYSTEM_RESTART; + +- if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ if ((kexec_in_progress || restart) && apple_ans_is_s8003(dev->dev) && + dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { + if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) { +- apple_cold_reset("Apple ANS: refusing unsafe powered kexec handoff"); ++ apple_cold_reset(restart ? ++ "Apple ANS: unsafe controller stop before restart" : ++ "Apple ANS: refusing unsafe powered kexec handoff"); + return; + } + return; diff --git a/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch b/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch new file mode 100644 index 00000000..179866ce --- /dev/null +++ b/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch @@ -0,0 +1,70 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:15:39 +0100 +Subject: [PATCH] PCI: apple-h9p: hand off S8003 ports on restart and power off + Wi-Fi + +On a restart that re-enters the boot loader the S8003 host's shutdown did +nothing, so the Wi-Fi function kept its link and bus mastering. Its +firmware can keep writing into memory that the boot loader and the next +kernel reuse, and the boot loader does not touch ports other than storage. + +Run the kexec handoff on restart as well: block root routing, assert PERST, +stop the LTSSM and disable the NVMMU SART, with the same certificate and +cold-reset fallback. The storage port then ends powered with PERST held, +as for kexec. Ports other than storage are additionally powered off with +the iOS port power-down, the state the boot loader leaves them in and from +which probe restarts them. The kexec handoff itself is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 5d6e717ee..90491965d 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3891,12 +3891,20 @@ static const struct dev_pm_ops apple_h9p_sleep_ops = { + .resume_noirq = apple_h9p_sleep_resume, + }; + ++/* ++ * A kexec and a restart take the same handoff. After a restart the boot ++ * loader starts the storage port again from this state, as it does from ++ * iBoot's, but it does not stop an endpoint left on any other port: Wi-Fi ++ * would keep DMAing into memory the next kernel reuses. Those ports are ++ * therefore powered off after the isolation, as the boot loader leaves them. ++ */ + static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + { + struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ bool restart = !kexec_in_progress && system_state == SYSTEM_RESTART; + unsigned int port; + +- if (!kexec_in_progress || !pcie || ++ if ((!kexec_in_progress && !restart) || !pcie || + !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) + return; + cancel_delayed_work_sync(&pcie->late_work); +@@ -3938,6 +3946,8 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!port && !of_property_read_bool(pcie->dev->of_node, + "apple,ans-kexec-handoff")) + apple_s8003_port_off(pcie, port); ++ if (port && restart) ++ apple_s8003_port_off(pcie, port); + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || + gpiod_get_value_cansleep(pcie->perst[0]) != 1)) { +@@ -3946,7 +3956,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + } + pcie->enabled_ports &= ~BIT(port); + } +- dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++ if (restart) ++ dev_info(pcie->dev, "restart: PCIe PERST asserted, LTSSM stopped, ports 1+ powered off; NVMMU SART disabled\n"); ++ else ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); + } + + static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) diff --git a/kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch b/kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch new file mode 100644 index 00000000..7a81d907 --- /dev/null +++ b/kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch @@ -0,0 +1,56 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] power: supply: apple-tristar: select J71t USB wake event + from match data + +The charger treats one TriStar event as the plug event that may wake the +system. J71t reports plug changes on event 0x49 instead of the default +one. Take the plug event from the match data and give +apple,j71t-tristar-charger 0x49. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index a953cedcb..8a869725d 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -124,6 +124,7 @@ struct tristar_charger { + struct completion adc_done; + int irqs[TRISTAR_IRQS_MAX]; + unsigned int nirqs; ++ unsigned int plug_event; + int plug_irq; /* wakes the system, 0 without one */ + u32 status_base; + u32 control_base; +@@ -1321,13 +1322,15 @@ static int tristar_probe(struct platform_device *pdev) + if (ret <= TRISTAR_IRQ_SOURCE || ret > TRISTAR_IRQS_MAX) + return dev_err_probe(dev, -EINVAL, + "expected PMU line, ADC and source event interrupts\n"); ++ chg->plug_event = device_get_match_data(dev) ? ++ *(const unsigned int *)device_get_match_data(dev) : TRISTAR_EVENT_PLUG; + chg->nirqs = ret; + for (i = 0; i < chg->nirqs; i++) { + chg->irqs[i] = platform_get_irq(pdev, i); + if (chg->irqs[i] < 0) + return chg->irqs[i]; + if (i >= TRISTAR_IRQ_SOURCE && +- irqd_to_hwirq(irq_get_irq_data(chg->irqs[i])) == TRISTAR_EVENT_PLUG) ++ irqd_to_hwirq(irq_get_irq_data(chg->irqs[i])) == chg->plug_event) + chg->plug_irq = chg->irqs[i]; + } + +@@ -1454,7 +1457,10 @@ static int tristar_resume(struct device *dev) + + static DEFINE_SIMPLE_DEV_PM_OPS(tristar_pm_ops, tristar_suspend, tristar_resume); + ++static const unsigned int j71t_plug_event = 0x49; ++ + static const struct of_device_id tristar_of_match[] = { ++ { .compatible = "apple,j71t-tristar-charger", .data = &j71t_plug_event }, + { .compatible = "apple,tristar-charger" }, + { } + }; diff --git a/kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch b/kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch new file mode 100644 index 00000000..b7291559 --- /dev/null +++ b/kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] dt-bindings: usb: describe board-selected s2idle gadget + retention + +Document apple,retain-gadget-in-s2idle: the board keeps the peripheral +controller and its PHY powered during suspend to idle, so the gadget +stays connected. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/usb/dwc2.yaml | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/Documentation/devicetree/bindings/usb/dwc2.yaml b/Documentation/devicetree/bindings/usb/dwc2.yaml +index b1f66a663..81df7017a 100644 +--- a/Documentation/devicetree/bindings/usb/dwc2.yaml ++++ b/Documentation/devicetree/bindings/usb/dwc2.yaml +@@ -172,6 +172,11 @@ properties: + description: If present indicates that we need to reset the PHY when we + detect a wakeup. This is due to a hardware errata. + ++ apple,retain-gadget-in-s2idle: ++ type: boolean ++ description: ++ Keep the peripheral controller and PHY powered during suspend to idle. ++ + usb-psy-name: + $ref: /schemas/types.yaml#/definitions/string + description: diff --git a/kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch b/kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch new file mode 100644 index 00000000..8e03537a --- /dev/null +++ b/kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch @@ -0,0 +1,160 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] usb: dwc2: select retained s2idle gadget from board data + +A USB gadget network link, and anything on it such as a root over NBD, +survives a short suspend to idle when the controller, its PHY and the +gadget are left connected. The host sees NAKs while the system sleeps. + +Keep the gadget connected over suspend to idle when the board sets +apple,retain-gadget-in-s2idle and the port is in device mode and active. +The controller and the PHY are put on the wakeup path so that their +power domains stay on. A suspend precheck refuses any other sleep state, +or a port that is not a connected peripheral, with -EBUSY before sync. + +Assisted-by: LLM +--- + drivers/usb/dwc2/core.h | 5 +++ + drivers/usb/dwc2/platform.c | 63 +++++++++++++++++++++++++++++++++++++ + 2 files changed, 68 insertions(+) + +diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h +index ae563cadc..9dbddbb0d 100644 +--- a/drivers/usb/dwc2/core.h ++++ b/drivers/usb/dwc2/core.h +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include "hw.h" + +@@ -882,6 +883,8 @@ struct dwc2_hregs_backup { + * @reset_phy_on_wake: Quirk saying that we should assert PHY reset on a + * remote wakeup. + * @phy_off_for_suspend: Status of whether we turned the PHY off at suspend. ++ * @kept_for_s2idle: The gadget was left connected and running over ++ * suspend-to-idle (dwc2.s2idle_keep_gadget). + * @need_phy_for_wake: Quirk saying that we should keep the PHY on at + * suspend if we need USB to wake us up. + * @frame_number: Frame number read from the core. For both device +@@ -1088,6 +1091,8 @@ struct dwc2_hsotg { + unsigned int reset_phy_on_wake:1; + unsigned int need_phy_for_wake:1; + unsigned int phy_off_for_suspend:1; ++ unsigned int kept_for_s2idle:1; ++ struct notifier_block sleep_precheck; + u16 frame_number; + + struct phy *phy; +diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c +index ef0d73077..fabee8363 100644 +--- a/drivers/usb/dwc2/platform.c ++++ b/drivers/usb/dwc2/platform.c +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + + #include + +@@ -436,6 +437,9 @@ int dwc2_check_core_version(struct dwc2_hsotg *hsotg) + * in the device private data. This allows the driver to access the dwc2_hsotg + * structure on subsequent calls to driver methods for this device. + */ ++static int dwc2_sleep_precheck(struct notifier_block *, unsigned long, void *); ++static void dwc2_sleep_unregister(void *); ++ + static int dwc2_driver_probe(struct platform_device *dev) + { + struct dwc2_hsotg *hsotg; +@@ -625,6 +629,14 @@ static int dwc2_driver_probe(struct platform_device *dev) + } + } + #endif /* CONFIG_USB_DWC2_PERIPHERAL || CONFIG_USB_DWC2_DUAL_ROLE */ ++ hsotg->sleep_precheck.notifier_call = dwc2_sleep_precheck; ++ retval = register_pm_suspend_precheck_notifier(&hsotg->sleep_precheck); ++ if (retval) ++ goto error; ++ retval = devm_add_action_or_reset(&dev->dev, dwc2_sleep_unregister, hsotg); ++ if (retval) ++ goto error; ++ + return 0; + + #if IS_ENABLED(CONFIG_USB_DWC2_PERIPHERAL) || \ +@@ -646,6 +658,49 @@ static int dwc2_driver_probe(struct platform_device *dev) + return retval; + } + ++/* ++ * Suspend-to-idle with dwc2.s2idle_keep_gadget: leave the controller and ++ * its PHY powered and the gadget connected. Its interrupt is off while the ++ * system sleeps, so the host sees NAKs and nothing else; a gadget network ++ * link (and anything on it, e.g. an NBD root) survives a short sleep. ++ * The controller and PHY are put on the wakeup path so that their power ++ * domains stay on (GENPD_FLAG_ACTIVE_WAKEUP). ++ */ ++static bool dwc2_keep_gadget_for_s2idle(struct dwc2_hsotg *dwc2) ++{ ++ if (!device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") || !pm_suspend_no_platform() || ++ !dwc2_is_device_mode(dwc2) || dwc2->lx_state != DWC2_L0) ++ return false; ++ ++ device_set_wakeup_path(dwc2->dev); ++ if (dwc2->phy && dwc2->phy->dev.parent) ++ device_set_wakeup_path(dwc2->phy->dev.parent); ++ dwc2->kept_for_s2idle = 1; ++ dev_info(dwc2->dev, "s2idle: gadget stays connected\n"); ++ return true; ++} ++ ++static int dwc2_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct dwc2_hsotg *dwc2 = container_of(nb, struct dwc2_hsotg, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ if (device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") && ++ (action != PM_SUSPEND_TO_IDLE || ++ (dwc2->op_state != OTG_STATE_B_PERIPHERAL && ++ dwc2->op_state != OTG_STATE_B_IDLE))) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_OK; ++} ++ ++static void dwc2_sleep_unregister(void *data) ++{ ++ struct dwc2_hsotg *dwc2 = data; ++ ++ unregister_pm_suspend_precheck_notifier(&dwc2->sleep_precheck); ++} ++ + static int __maybe_unused dwc2_suspend(struct device *dev) + { + struct dwc2_hsotg *dwc2 = dev_get_drvdata(dev); +@@ -655,6 +710,9 @@ static int __maybe_unused dwc2_suspend(struct device *dev) + if (!dwc2->ll_hw_enabled) + return 0; + ++ if (dwc2_keep_gadget_for_s2idle(dwc2)) ++ return 0; ++ + is_device_mode = dwc2_is_device_mode(dwc2); + if (is_device_mode) + dwc2_hsotg_suspend(dwc2); +@@ -735,6 +793,11 @@ static int __maybe_unused dwc2_resume(struct device *dev) + if (!dwc2->ll_hw_enabled) + return 0; + ++ if (dwc2->kept_for_s2idle) { ++ dwc2->kept_for_s2idle = 0; ++ return 0; ++ } ++ + if (dwc2->phy_off_for_suspend && dwc2->ll_hw_enabled) { + ret = __dwc2_lowlevel_hw_enable(dwc2); + if (ret) diff --git a/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch b/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch new file mode 100644 index 00000000..91be9b20 --- /dev/null +++ b/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] wifi: brcmfmac: stay down after host sleep link loss + +When the PCIe link does not come back after system sleep, the host +bridge marks the device disconnected. brcmf_pcie_pm_leave_D3() then went +on to access the BARs, which can stall the CPU. Return -ENODEV and keep +the bus down instead; firmware commands and transmits are then refused +early. + +Assisted-by: LLM +--- + .../net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +index d7c6a402f..a07f7c8ad 100644 +--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c ++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +@@ -2696,6 +2696,16 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) + devinfo = bus->bus_priv.pcie->devinfo; + brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); + ++ /* ++ * The host bridge cut the device off because its link did not come ++ * back. Any BAR access now could stall the CPU, so stay down; with the ++ * bus down, firmware commands and transmits are refused early. ++ */ ++ if (pci_dev_is_disconnected(devinfo->pdev)) { ++ brcmf_err(bus, "device lost over suspend, staying down\n"); ++ return -ENODEV; ++ } ++ + /* Check if device is still up and running, if so we are ready */ + if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { + brcmf_dbg(PCIE, "Try to wakeup device....\n"); diff --git a/tests/test_d2365_poweroff.py b/tests/test_d2365_poweroff.py index 6e262908..52a3fadd 100644 --- a/tests/test_d2365_poweroff.py +++ b/tests/test_d2365_poweroff.py @@ -192,7 +192,7 @@ def test_both_prepare_and_atomic_final_handlers_are_registered(self): class D2365EventQuiesce(unittest.TestCase): def test_shutdown_drains_irq_and_retry_after_closing_gate(self): - post = patched_source('drivers/gpio/gpio-apple-d2365.c') + post = patched_source('drivers/gpio/gpio-apple-d2365.c', every=True) start = post.index('void apple_d2365_gpio_quiesce(') shutdown = post[start:post.index('\n}', start) + 2] self.assertNotIn('.shutdown = apple_pmic_gpio_shutdown', post) From 557ef01b656a85decbc30388764a49b754de9701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Thu, 12 Nov 2026 09:00:00 +0100 Subject: [PATCH 26/33] apple: G5P render and compute for J71t with the device's own firmware The A9 GPU (PowerVR-derived G5P) had no kernel driver, so the desktop rendered in software. Add the apple-g5p DRM driver: bindings and J71t DT, the render/compute UAPI, the firmware transport and recovery core, buffer VMs with cache visibility, typed render and compute submission, and the GEM exporter's CPU cache contract. The firmware recipe extracts the G5P firmware from the device's own IPSW (src/orchard/apple/agx_g5p.py), and DRM_APPLE_G5P is enabled for apple-16k. Assisted-by: LLM --- docs/firmware-recipes.md | 13 + kernel/configs/apple-16k/config | 1 + ...4-arm64-dts-apple-enable-G5P-on-J71t.patch | 53 + ...u-describe-Apple-S8003-G5P-resources.patch | 46 + ...fine-the-render-and-compute-userspac.patch | 651 + ...d-firmware-transport-and-recovery-co.patch | 14822 ++++++++++++++++ ...d-buffer-VM-and-cache-visibility-sup.patch | 1027 ++ ...d-typed-render-and-compute-submissio.patch | 8973 ++++++++++ ...ative-GEM-exporter-CPU-cache-contrac.patch | 147 + src/orchard/apple/agx_g5p.py | 19 + src/orchard/recipe.py | 7 +- tests/test_agx_g5p.py | 29 + 12 files changed, 25787 insertions(+), 1 deletion(-) create mode 100644 kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch create mode 100644 kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch create mode 100644 kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch create mode 100644 kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch create mode 100644 kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch create mode 100644 kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch create mode 100644 kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch create mode 100644 src/orchard/apple/agx_g5p.py create mode 100644 tests/test_agx_g5p.py diff --git a/docs/firmware-recipes.md b/docs/firmware-recipes.md index 6eb10f97..2d0ca8db 100644 --- a/docs/firmware-recipes.md +++ b/docs/firmware-recipes.md @@ -238,3 +238,16 @@ The decompressed N61 kernel's SHA-256 is These fields are explicit recipe parameters, including zero-valued overrides. Physical qualification of firmware loading and input events remains separate from converting this shared firmware file. + + +## S8003 G5P META firmware + +`agx-g5p-ldr-from-kernelcache@1` reads the unique `__TEXT,__const` section +of `com.apple.AGXFirmwareKextG5P` in the user's own legacy prelinked +kernelcache. It rejects any size other than 119112 bytes or any digest other +than `33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca`. +The existing J71t kernelcache input names the active device kernelcache and +the pinned restore-image fallback. The normal recipe installer writes the +verified output to `/lib/firmware/apple/agx-g5p-c0.ldr`. No payload patch, +calibration, firmware bytes or captured job is stored in the source tree. +Other C0 revisions and other boards require separate digest qualification. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index f11529b5..28029ed8 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -3940,6 +3940,7 @@ CONFIG_DRM_SIMPLEDRM=y # CONFIG_DRM_VKMS is not set # CONFIG_DRM_VMWGFX is not set CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DRM_APPLE_G5P=y # # Frame buffer Devices diff --git a/kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch b/kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch new file mode 100644 index 00000000..4b13d168 --- /dev/null +++ b/kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch @@ -0,0 +1,53 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] arm64: dts: apple: enable G5P on J71t + +Use chip compatible matching; only the qualified board enables the node. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + arch/arm64/boot/dts/apple/s8003.dtsi | 17 +++++++++++++++++ + 2 files changed, 21 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 5b129d695..323c27e27 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -76,6 +76,10 @@ + apple,tethered-boot; + }; + ++&gpu_diag { ++ status = "okay"; ++}; ++ + &pcie0 { + apple,paired-system-sleep; + }; +diff --git a/arch/arm64/boot/dts/apple/s8003.dtsi b/arch/arm64/boot/dts/apple/s8003.dtsi +index b51588941..b7b7a2188 100644 +--- a/arch/arm64/boot/dts/apple/s8003.dtsi ++++ b/arch/arm64/boot/dts/apple/s8003.dtsi +@@ -336,3 +336,20 @@ + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/* G5P resources; enable only on a board qualified with this driver. */ ++&{/soc} { ++ gpu_diag: gpu@201000000 { ++ status = "disabled"; ++ compatible = "apple,s8003-gpu"; ++ reg = <0x2 0x01000000 0x0 0x100000>, ++ <0x2 0x01e00000 0x0 0x100000>; ++ reg-names = "rgx", "fender"; ++ power-domains = <&ps_gfx>; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ }; ++}; diff --git a/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch b/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch new file mode 100644 index 00000000..d0acb8a6 --- /dev/null +++ b/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch @@ -0,0 +1,46 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] dt-bindings: gpu: describe Apple S8003 G5P resources + +Use chip compatible matching; only the qualified board enables the node. + +Assisted-by: LLM +--- + .../bindings/gpu/apple,s8003-g5p.yaml | 27 +++++++++++++++++++ + 1 file changed, 27 insertions(+) + create mode 100644 Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml + +diff --git a/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +new file mode 100644 +index 000000000..76aebc183 +--- /dev/null ++++ b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +@@ -0,0 +1,27 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/gpu/apple,s8003-g5p.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++title: Apple S8003 G5P GPU ++maintainers: ++ - Berkay Kösebay ++description: | ++ G5P GPU with META firmware on S8003. The compatible is Apple's inherited ++ ADT spelling used by the qualified driver; no new vendor prefix is implied. ++properties: ++ compatible: ++ const: apple,s8003-gpu ++ reg: ++ maxItems: 2 ++ reg-names: ++ items: ++ - const: rgx ++ - const: fender ++ interrupts: ++ minItems: 5 ++ maxItems: 5 ++ power-domains: ++ maxItems: 1 ++required: [compatible, reg, reg-names, interrupts, power-domains] ++additionalProperties: false diff --git a/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch b/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch new file mode 100644 index 00000000..4da6e1b6 --- /dev/null +++ b/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch @@ -0,0 +1,651 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: define the render and compute userspace + interface + +Import the sealed warmport ABI, before pipeline stage wait extensions. + +Assisted-by: LLM +--- + include/uapi/drm/apple_g5p_drm.h | 631 +++++++++++++++++++++++++++++++ + 1 file changed, 631 insertions(+) + create mode 100644 include/uapi/drm/apple_g5p_drm.h + +diff --git a/include/uapi/drm/apple_g5p_drm.h b/include/uapi/drm/apple_g5p_drm.h +new file mode 100644 +index 000000000..5e41ab533 +--- /dev/null ++++ b/include/uapi/drm/apple_g5p_drm.h +@@ -0,0 +1,631 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++#ifndef _UAPI_APPLE_G5P_DRM_H_ ++#define _UAPI_APPLE_G5P_DRM_H_ ++ ++#include "drm.h" ++ ++/* G5P merged winsys ABI. Retired ioctl numbers remain reserved. */ ++#define DRM_G5P_UAPI_VERSION 1 ++#define DRM_G5P_CAP_BO_CPU (1ULL << 0) ++#define DRM_G5P_CAP_VM_PREPARE (1ULL << 1) ++#define DRM_G5P_CAP_SYNCOBJ (1ULL << 2) ++#define DRM_G5P_CAP_RENDER (1ULL << 5) ++#define DRM_G5P_CAP_PRIME (1ULL << 6) ++#define DRM_G5P_CAP_CODE_BASE_VERIFIED (1ULL << 7) ++#define DRM_G5P_CAP_VM_LIVE (1ULL << 8) ++#define DRM_G5P_CAP_CONTEXT_LIVE (1ULL << 9) ++#define DRM_G5P_CAP_JOB_STATUS (1ULL << 10) ++#define DRM_G5P_CAP_CONTEXT_STATUS (1ULL << 11) ++#define DRM_G5P_CAP_FAULT_INFO (1ULL << 13) ++ ++/* LIVE means the owned user catalogue has completed firmware attachment. */ ++#define DRM_G5P_CONTEXT_LIVE 3 ++#define DRM_G5P_CONTEXT_HOST_ONLY 0 ++#define DRM_G5P_CONTEXT_LOST 1 ++ ++struct drm_g5p_get_context_status { ++ __u32 context; ++ __u32 flags; ++ __u32 state; ++ __u32 map_count; ++ __u64 mapped_bytes; ++ __u64 reserved; ++}; ++ ++/* First terminal firmware event, retained independently of last_event. */ ++#define DRM_G5P_FAULT_NONE 0 ++#define DRM_G5P_FAULT_RECOVERY 1 ++#define DRM_G5P_FAULT_PAGE_REQUEST 2 ++#define DRM_G5P_FAULT_CHANNEL 3 ++#define DRM_G5P_FAULT_UNKNOWN 4 ++#define DRM_G5P_FAULT_VALID_CHANNEL (1U << 0) ++struct drm_g5p_get_fault_info { ++ __u32 flags; ++ __u32 kind; ++ __u64 event_serial; /* One-based arrival order; zero if no fault. */ ++ __u32 valid; ++ __u32 channel_class; /* Event8: 0 memory failure, 1 killed, others opaque. */ ++ __u32 channel_token; /* Firmware token, not a DRM context handle. */ ++ __u32 channel_stamp; ++ __u32 raw[11]; /* Includes unknown fields; no decoded fault VA is claimed. */ ++ __u32 reserved; ++}; ++ ++struct drm_g5p_get_info { ++ __u32 size; ++ __u32 flags; ++ __u32 version; ++ __u32 gpu_page_shift; ++ __u64 capabilities; ++ __u64 va_start; ++ __u64 va_end; ++ __u64 reserved_start; ++ __u64 reserved_end; ++ __u64 code_base; /* Zero until CODE_BASE_VERIFIED. */ ++ __u64 max_bo_size; ++ __u32 bvnc[4]; /* Static target, not a new MMIO measurement. */ ++ __u64 reserved[2]; ++}; ++ ++/* GPU access roles; CPU initialization/mmap is permitted for every role. */ ++#define DRM_G5P_BO_READ (1U << 0) ++#define DRM_G5P_BO_WRITE (1U << 1) ++#define DRM_G5P_BO_EXEC (1U << 2) ++struct drm_g5p_create_bo { ++ __u64 size; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_bo_mmap_offset { ++ __u32 handle; ++ __u32 flags; ++ __u64 offset; ++}; ++ ++#define DRM_G5P_BO_SYNC_TO_DEVICE 0 ++#define DRM_G5P_BO_SYNC_TO_CPU 1 ++struct drm_g5p_bo_sync { ++ __u32 handle; ++ __u32 op; ++ __u64 offset; ++ __u64 size; ++}; ++ ++struct drm_g5p_create_vm { ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_vm { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++#define DRM_G5P_VM_MAP 0 ++#define DRM_G5P_VM_UNMAP 1 ++struct drm_g5p_vm_bind { ++ __u32 vm; ++ __u32 bo; ++ __u64 va; ++ __u64 size; ++ __u64 offset; ++ __u32 op; ++ __u32 flags; ++}; ++ ++struct drm_g5p_create_context { ++ __u32 vm; ++ __u32 type; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_context { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* A point of zero uses a binary syncobj; nonzero uses a timeline point. */ ++#define DRM_G5P_SYNC_SIGNAL (1U << 0) ++struct drm_g5p_sync { ++ __u32 handle; ++ __u32 flags; ++ __u64 point; ++}; ++ ++/* Privileged one-shot kernel fixture; all payload/BO fields must be zero. ++ * The empty user context authorizes ownership only; its VM is not attached. ++ * Does not advertise general COMPUTE or attach user-provided GPU addresses. ++ */ ++ ++/* Geometry is all zero/NONE while CODE_BASE_VERIFIED is clear. */ ++#define DRM_G5P_BASE_NONE 0 ++#define DRM_G5P_BASE_DEVICE 1 ++#define DRM_G5P_BASE_VM 2 ++#define DRM_G5P_BASE_CONTEXT 3 ++struct drm_g5p_get_exec_info { ++ __u32 size; ++ __u32 flags; ++ __u32 base_scope; ++ __u32 pad; ++ __u64 usc_base; ++ __u64 pds_base; ++ __u64 window_size; ++ __u64 allocation_alignment; ++ __u32 usc_offset_bits; ++ __u32 pds_offset_bits; ++ __u64 reserved[3]; ++}; ++ ++#define DRM_G5P_JOB_PENDING 0 ++#define DRM_G5P_JOB_COMPLETE 1 ++struct drm_g5p_get_job_status { ++ __u64 job_id; ++ __u32 flags; ++ __u32 state; ++ __s32 error; /* -EINPROGRESS while pending; immutable after completion. */ ++ __u32 pad; ++ __u64 reset_generation; /* Zero: no reset/recovery support. */ ++}; ++ ++#define DRM_G5P_GET_INFO 0x00 ++#define DRM_G5P_CREATE_BO 0x01 ++#define DRM_G5P_BO_MMAP_OFFSET 0x02 ++#define DRM_G5P_BO_SYNC 0x03 ++#define DRM_G5P_CREATE_VM 0x04 ++#define DRM_G5P_DESTROY_VM 0x05 ++#define DRM_G5P_VM_BIND 0x06 ++#define DRM_G5P_CREATE_CONTEXT 0x07 ++#define DRM_G5P_DESTROY_CONTEXT 0x08 ++#define DRM_G5P_GET_EXEC_INFO 0x0b ++#define DRM_G5P_GET_JOB_STATUS 0x0c ++#define DRM_G5P_GET_CONTEXT_STATUS 0x0d ++#define DRM_G5P_GET_FAULT_INFO 0x0e ++ ++#define DRM_IOCTL_G5P_GET_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_INFO, struct drm_g5p_get_info) ++#define DRM_IOCTL_G5P_CREATE_BO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_BO, struct drm_g5p_create_bo) ++#define DRM_IOCTL_G5P_BO_MMAP_OFFSET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_BO_MMAP_OFFSET, \ ++ struct drm_g5p_bo_mmap_offset) ++#define DRM_IOCTL_G5P_BO_SYNC \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_BO_SYNC, struct drm_g5p_bo_sync) ++#define DRM_IOCTL_G5P_CREATE_VM \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_VM, struct drm_g5p_create_vm) ++#define DRM_IOCTL_G5P_DESTROY_VM \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_VM, \ ++ struct drm_g5p_destroy_vm) ++#define DRM_IOCTL_G5P_VM_BIND \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_VM_BIND, struct drm_g5p_vm_bind) ++#define DRM_IOCTL_G5P_CREATE_CONTEXT \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_CONTEXT, \ ++ struct drm_g5p_create_context) ++#define DRM_IOCTL_G5P_DESTROY_CONTEXT \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_CONTEXT, \ ++ struct drm_g5p_destroy_context) ++ ++#define DRM_IOCTL_G5P_GET_EXEC_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_EXEC_INFO, \ ++ struct drm_g5p_get_exec_info) ++#define DRM_IOCTL_G5P_GET_JOB_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_JOB_STATUS, \ ++ struct drm_g5p_get_job_status) ++ ++#define DRM_IOCTL_G5P_GET_CONTEXT_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_CONTEXT_STATUS, \ ++ struct drm_g5p_get_context_status) ++ ++#define DRM_IOCTL_G5P_GET_FAULT_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_FAULT_INFO, \ ++ struct drm_g5p_get_fault_info) ++ ++#define DRM_G5P_CAP_IDLE_VM_BIND (1ULL << 18) ++ ++#define DRM_G5P_CAP_BOUNDED_RENDER (1ULL << 19) ++#define DRM_G5P_CAP_TYPED_RENDER (1ULL << 20) ++#define DRM_G5P_CAP_RENDER_USER_GEOMETRY (1ULL << 21) ++#define DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY (1ULL << 22) ++#define DRM_G5P_CAP_TYPED_RENDER_V2 (1ULL << 23) ++/* Bit 24 is reserved for COMPUTE_BLIT_V1 and bit 25 is FIXED_DATA_B1 on ++ * a9/release, so the next bit free in every branch is 26. ++ */ ++#define DRM_G5P_CAP_TYPED_RENDER_V3 (1ULL << 26) ++ ++/** ++ * DRM_G5P_CAP_RENDER_MULTI_CONTEXT - several render contexts share the GPU ++ * ++ * Reported together with TYPED_RENDER_V3. Any number of render contexts, in any ++ * number of files, may submit TYPED_V3 jobs. Each context has its own VM and its ++ * own firmware slot pair, and the jobs of all contexts run one at a time in ++ * submission order. Without this bit, the first context to submit owns the GPU ++ * until reboot and every other context is refused. The other render profiles ++ * stay single-context. ++ */ ++#define DRM_G5P_CAP_RENDER_MULTI_CONTEXT (1ULL << 27) ++ ++/** ++ * DRM_G5P_CAP_RENDER_UNPRIVILEGED - TYPED_V3 render needs no CAP_SYS_RAWIO ++ * ++ * Reported together with RENDER_MULTI_CONTEXT. Any opener of the render node may ++ * create render contexts and submit TYPED_V3 jobs. A render VM then maps none of ++ * the firmware's memory beyond the kernel-built window the GPU's own requesters ++ * need, and the contexts close cleanly: a closed context's firmware binding is ++ * removed and its memory freed. Pinned memory is limited device-wide. The other ++ * render profiles, compute and the diagnostic ioctls still need CAP_SYS_RAWIO. ++ */ ++#define DRM_G5P_CAP_RENDER_UNPRIVILEGED (1ULL << 28) ++ ++/** ++ * DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH - TYPED_V3 carries pixel_output_width ++ * ++ * Reported together with TYPED_RENDER_V3. struct drm_g5p_render_pass_v3 has ++ * pixel_output_width, so a TYPED_V3 pass in the V1 dialect (all-zero ext) can ++ * write several colour targets. Without this bit the field is reserved and ++ * must be zero. ++ */ ++#define DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH (1ULL << 29) ++ ++/* drm_g5p_create_context.type; flags must be zero. */ ++#define DRM_G5P_CONTEXT_RENDER 1 ++ ++#define DRM_G5P_RENDER_PROFILE_TYPED_V3 4 ++ ++/* LINEAR64_V1: kind is also the resource's position in the canonical array. */ ++ ++#define DRM_G5P_RENDER_ACCESS_READ (1U << 0) ++#define DRM_G5P_RENDER_ACCESS_WRITE (1U << 1) ++#define DRM_G5P_RENDER_ACCESS_IMPLICIT (1U << 2) ++#define DRM_G5P_RENDER_ACCESS_CACHE (1U << 3) ++ ++#define DRM_G5P_RENDER_MAX_RESOURCES 256 ++#define DRM_G5P_RENDER_MAX_SYNCS 17 ++#define DRM_G5P_RENDER_MAX_VDM_BYTES 0x10000 ++ ++/* A GPU VA range [va, va + size) inside one existing mapping of handle's BO ++ * in the context VM, at BO byte offset `offset`. Never a CPU pointer, PA or ++ * META address. LINEAR64_V1 requires offset 0 and the exact whole BO. ++ */ ++struct drm_g5p_render_resource { ++ __u32 handle; ++ __u32 kind; ++ __u64 offset; ++ __u64 size; ++ __u64 va; ++}; ++ ++/** ++ * DRM_G5P_CAP_RENDER_MSAA - 4x multisampled render targets and passes ++ * ++ * Reported with TYPED_RENDER_V3 and RENDER_KERNEL_GEOMETRY. ++ * CREATE_RENDER_TARGET accepts samples = 4 (kernel-derived geometry; with ++ * DRM_G5P_RT_USER_GEOMETRY userspace may only restate it), and a TYPED_V3 ++ * pass carries samples, msaa_flags and sample_positions in struct ++ * drm_g5p_render_pass. Without this bit those fields are reserved. ++ */ ++#define DRM_G5P_CAP_RENDER_MSAA (1ULL << 31) ++ ++/** ++ * DRM_G5P_CAP_RENDER_OCCLUSION_QUERY - TYPED_V3 carries ISP query storage ++ * ++ * The V2 extension names the query-result allocation used by ISP visibility ++ * tests. The kernel validates the complete range and writes its base into the ++ * two per-partition ISP_OCLQRY_BASE work-record slots. A zero VA and size ++ * disable visibility-result storage. ++ */ ++#define DRM_G5P_CAP_RENDER_OCCLUSION_QUERY (1ULL << 33) ++ ++/* drm_g5p_render_pass.msaa_flags: the end of tile averages each pixel's ++ * samples into the (one-sample) surface the EOT's PBE words name. ++ */ ++#define DRM_G5P_RENDER_MSAA_RESOLVE (1U << 0) ++ ++struct drm_g5p_render_pass { ++ __u32 size; /* sizeof(struct drm_g5p_render_pass) */ ++ __u32 flags; /* zero */ ++ __u64 vdm_va; /* VDM control stream, READ, 4-byte aligned */ ++ __u32 vdm_size; /* bytes, multiple of 4, <= MAX_VDM_BYTES */ ++ __u32 vdm_control; /* V1/V2: TA W+0x88 value, multiple of 16; V3: zero */ ++ __u64 scratch_va; /* TA scratch block, WRITE, 16-byte aligned */ ++ __u64 scratch_size; ++ __u64 scratch_ptr[6]; /* TA scratch/marker pointers, in a WRITE range */ ++ __u64 pool7_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool7_size; ++ __u64 pool8_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool8_size; ++ __u32 context_pds[2][6]; /* TA context store/load PDS descriptors */ ++ __u32 resume_pds[2][4]; /* TA resume PDS descriptors */ ++ __u32 ta_pds[4]; /* TA state PDS descriptor; word 3 zero */ ++ __u32 background[8]; /* 3D background (load/clear) object */ ++ __u32 eot[4]; /* 3D end-of-tile object; word 3 zero */ ++ __u32 eot_resume[4]; /* 3D partial-render EOT; word 3 zero */ ++ __u32 restore[4]; /* 3D partial-render restore words */ ++ /* DRM_G5P_CAP_RENDER_MSAA, TYPED_V3 only; zero in every other profile. ++ * samples: 0 or 1 with a one-sample target, 4 with a four-sample one. ++ * sample_positions: one byte per sample, x in [3:0] and y in [7:4] in ++ * 1/16 px (PPP_MULTISAMPLECTL); 0 takes the target's standard pattern. ++ * The kernel writes the AA mode, ISP_CTL and output-partition words ++ * for the sample count. The draws' ISPCTL.upass (0x7 at four samples, ++ * 0xf at one), the PBE/EOT words and the fragment programs are ++ * userspace's, as for one sample; see docs/gpu/experiments/msaa.md. ++ */ ++ __u32 samples; ++ __u32 msaa_flags; /* DRM_G5P_RENDER_MSAA_* */ ++ __u64 sample_positions; ++ __u64 reserved[6]; ++}; ++ ++/* ++ * TYPED_V2 additions. Each field is a 3D register image that the firmware ++ * uploads from the normal-3D work object. The CR offsets are the firmware ++ * store targets (its 3D upload puts W3c0 on ROGUE_CR_ISP_CTL 0xf38); the ++ * Rogue names are inferred, graded in docs/gpu/render-uapi.md. ++ */ ++#define DRM_G5P_RENDER_MAX_TABLE_ENTRIES 65536 /* ISPDBSC 16-bit indices */ ++#define DRM_G5P_RENDER_ZLS_REGS 6 ++ ++struct drm_g5p_render_pass_ext { ++ /* ISP_SCISSOR_BASE (CR fb0): IPF_SCISSOR_WORD_0/1 pairs, 8 bytes each, ++ * indexed by TA_STATE_ISPDBSC.scindex. READ, 4-byte aligned. ++ */ ++ __u64 scissor_va; ++ __u32 scissor_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ __u32 dbias_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ /* ISP_DBIAS_BASE (CR fb8): {constant, slope, clamp} f32 triples, ++ * 12 bytes each, indexed by ISPDBSC.dbindex. READ, 4-byte aligned. ++ */ ++ __u64 dbias_va; ++ /* PDS_BGRND1_BASE (CR 6a8): background object texture/varying data ++ * (B-relative, like background[]); LOAD_OP_LOAD's texture state. ++ */ ++ __u64 background_texture; ++ __u32 bgobj_depth; /* ISP_BGOBJDEPTH (CR f80): depth clear, ZLS format */ ++ __u32 bgobj_vals; /* ISP_BGOBJVALS (CR f88): stencil [7:0], mask [8], bgtag [9] */ ++ /* Pixel output width (2, 4 or 8): the 6ca7ae0 inputs to ISP_CTL's ++ * partition fields and the six per-core output words. 2 for one ++ * 32-bpp colour attachment (the captured value). ++ */ ++ __u32 pixel_output_width; ++ /* Bit i: zls[i] is an address ([39:4]; [63:40] zero; [3:0] flags) ++ * inside a listed READ|WRITE range. The kernel does not decode ZLS. ++ */ ++ __u32 zls_address_mask; ++ /* Opaque ZLS register images, CR f48, f50, f58, f60, f68 and fd0: ++ * Rogue ISP_ZLSCTL, ZLOAD/ZSTORE and STENCIL_LOAD/STORE bases plus ++ * one more; positions proven, order unqualified. Zero: no ZLS. ++ */ ++ __u64 zls[DRM_G5P_RENDER_ZLS_REGS]; ++ __u64 occlusion_query_va; /* ISP_OCLQRY_BASE, 16-byte aligned, READ|WRITE */ ++ __u64 occlusion_query_size; /* bytes in the listed allocation */ ++ __u64 reserved[2]; ++}; ++ ++struct drm_g5p_render_pass_v2 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++}; ++ ++#define DRM_G5P_RENDER_MAX_PROGRAM_BYTES 0x1000 ++#define DRM_G5P_RENDER_STREAM_TERMINATE 0xc0000000U ++ ++struct drm_g5p_render_pass_v3 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++ __u64 ta_program_va; /* -> W+0x88 / R0+0x470 as an offset from pds_base */ ++ __u32 ta_program_size; /* bytes, 16..MAX_PROGRAM_BYTES, multiple of 16 */ ++ __u32 vdm_end; /* bytes from vdm_va past STREAM_TERMINATE, <= vdm_size */ ++ __u32 pixel_output_width; /* V1 dialect: 0 (= 2), 2, 4 or 8 */ ++ __u32 pad; /* zero */ ++ __u64 reserved[3]; ++}; ++ ++struct drm_g5p_submit_render_typed { ++ __u32 context; ++ __u32 flags; /* zero */ ++ __u32 profile; ++ __u32 render_target; /* handle from CREATE_RENDER_TARGET */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[] */ ++ __u64 pass; /* struct drm_g5p_render_pass */ ++ __u32 pass_size; ++ __u32 pad; ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; ++}; ++ ++/* ++ * Render-target memory geometry: the AGXRenderTargetMemoryG5 arithmetic ++ * outputs (firmware round16 section 3, R-offsets in comments). The kernel ++ * computes them for its verified table (64x64x1), or with ++ * RENDER_KERNEL_GEOMETRY for any size; with RENDER_USER_GEOMETRY userspace ++ * supplies them and the kernel bounds the memory they imply (and, with ++ * RENDER_KERNEL_GEOMETRY, requires its own result). ++ */ ++struct drm_g5p_rt_geometry { ++ __u32 header_bytes; /* R10 */ ++ __u32 region_bytes; /* R20 */ ++ __u64 packed_state; /* Rb0 */ ++ __u32 state_word; /* Rb8 */ ++ __u32 padded_dims; /* Rbc */ ++ __u64 scale[3]; /* Rc0, Rc8, Rd0 packed float pairs */ ++ __u64 ta_state[2]; /* Rf0, Rf8 */ ++ __u64 ta_tiling[2]; /* R100, R108 */ ++ __u32 ta_tiling_word; /* R110 */ ++ __u32 tile_count; /* R118 */ ++ __u64 secondary_bytes; /* R138 */ ++ __u64 secondary_align; /* R178 */ ++ __u64 reserved[2]; ++}; ++ ++#define DRM_G5P_RT_USER_GEOMETRY (1U << 0) ++struct drm_g5p_create_render_target { ++ __u32 context; /* render context; RT memory lives in its VM */ ++ __u32 flags; ++ __u32 width; ++ __u32 height; ++ __u32 samples; /* 1, or 4 with DRM_G5P_CAP_RENDER_MSAA */ ++ __u32 layers; /* 1 */ ++ __u32 handle; /* output */ ++ __u32 pad; ++ __u64 main_va; /* output: kernel-private GPU VA */ ++ __u64 main_bytes; /* output */ ++ __u64 secondary_va; /* output */ ++ __u64 secondary_bytes; /* output: mapped bytes */ ++ struct drm_g5p_rt_geometry geometry; /* input if USER, else output */ ++ __u32 pm_blocks; /* output with RENDER_KERNEL_GEOMETRY: context PM pool */ ++ __u32 pad2; ++ __u64 reserved; ++}; ++ ++struct drm_g5p_destroy_render_target { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* GET_RENDER_INFO.flags, output only (input must be zero). ++ * VARIANT_ACTIVE: a render_user_* hypothesis knob (stage, cache/mmu/timestamp ++ * variant, 3d_after_ta, pm_policy, cache_repeat, wait_ms, pm/memory/rt/slot ++ * variant, profile1_vdm_control) differs from its default, so a context ++ * created now latches a non-default job behaviour. GET_RENDER_INFO has no ++ * context: it reports what CREATE_CONTEXT would latch at this moment. ++ */ ++#define DRM_G5P_RENDER_INFO_VARIANT_ACTIVE (1U << 0) ++ ++struct drm_g5p_get_render_info { ++ __u32 size; /* sizeof, exact */ ++ __u32 flags; /* in: zero; out: RENDER_INFO_* */ ++ __u32 profiles; /* bit N: profile N accepted */ ++ __u32 max_render_targets; ++ __u64 private_va_start; /* kernel window in render VMs */ ++ __u64 private_va_end; ++ __u64 user_va_min; /* lowest mappable VA in a render VM */ ++ __u64 usc_base; /* common USC/PDS base, 32-bit offsets */ ++ __u64 pds_base; ++ __u64 code_window; /* bytes above usc_base usable for code */ ++ __u32 pm_blocks; /* parameter buffer blocks per context */ ++ __u32 pm_block_bytes; ++ __u32 max_queued_jobs; ++ __u32 max_vdm_bytes; ++ __u32 job_timeout_ms; ++ __u32 retained_results; ++ /* RENDER_KERNEL_GEOMETRY only, else zero (64x64 only). */ ++ __u32 max_render_width; ++ __u32 max_render_height; ++ __u32 max_pm_blocks; /* PM pool growth bound per context */ ++ __u32 pad; ++ union { ++ __u64 reserved[2]; /* in: zero */ ++ struct { ++ /* TYPED_V3: every GPU VA the TA/3D read ends at or below ++ * this; 0 makes no statement. Out only. ++ */ ++ __u64 hw_va_limit; ++ __u64 reserved1; ++ }; ++ }; ++}; ++ ++#define DRM_G5P_CREATE_RENDER_TARGET 0x11 ++#define DRM_G5P_DESTROY_RENDER_TARGET 0x12 ++#define DRM_G5P_GET_RENDER_INFO 0x13 ++#define DRM_G5P_SUBMIT_RENDER_TYPED 0x14 ++ ++#define DRM_IOCTL_G5P_SUBMIT_RENDER_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_RENDER_TYPED, \ ++ struct drm_g5p_submit_render_typed) ++#define DRM_IOCTL_G5P_CREATE_RENDER_TARGET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_RENDER_TARGET, \ ++ struct drm_g5p_create_render_target) ++#define DRM_IOCTL_G5P_DESTROY_RENDER_TARGET \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_RENDER_TARGET, \ ++ struct drm_g5p_destroy_render_target) ++#define DRM_IOCTL_G5P_GET_RENDER_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_RENDER_INFO, \ ++ struct drm_g5p_get_render_info) ++ ++/** ++ * DRM_G5P_CAP_TYPED_COMPUTE - compute jobs on render contexts (SUBMIT_COMPUTE_TYPED) ++ * ++ * Reported together with TYPED_RENDER_V3. A ++ * render context (DRM_G5P_CONTEXT_RENDER) runs compute jobs beside its TYPED_V3 render jobs, ++ * in the same VM, in submission order, serialized with every other context's jobs. The ++ * kernel builds the firmware's compute work (W, microsequence, stamps, channel, slot); the ++ * job carries the CDM control stream and the register images iOS's userspace hands its ++ * kernel, which the kernel bounds but does not decode. Compute needs an epoch a render job ++ * brought up: on a device no render job has started, SUBMIT_COMPUTE_TYPED fails with EAGAIN. ++ */ ++#define DRM_G5P_CAP_TYPED_COMPUTE (1ULL << 30) ++ ++#define DRM_G5P_COMPUTE_MAX_STREAM_BYTES 0x10000 ++#define DRM_G5P_COMPUTE_STREAM_TERMINATE 0x80000000U ++ ++/* A PDS program reference: code and data offsets from GET_RENDER_INFO.pds_base (16-byte ++ * aligned; code in a READ resource whose BO carries DRM_G5P_BO_EXEC, data in a READ resource) ++ * and its opaque control word (Rogue's PDS1 position: sizes and temps). ++ */ ++struct drm_g5p_compute_pds { ++ __u32 code; ++ __u32 data; ++ __u32 control; ++ __u32 pad; /* zero */ ++}; ++ ++/* ++ * One compute job. Every *_va is a GPU VA inside a listed resource with the stated access. ++ * cdm_va/cdm_size the CDM control stream (W+0x70, R0+0x480): READ, 4-byte aligned, ++ * cdm_size <= MAX_STREAM_BYTES, in a native (not PRIME-imported) BO ++ * cdm_end bytes from cdm_va past the stream's terminate word (the dword at ++ * cdm_va + cdm_end - 4 is STREAM_TERMINATE), <= cdm_size ++ * state_va[4] CDM context-state words (W+0x1ec..0x204), READ|WRITE, 4-byte aligned ++ * state_tail_va CDM context-state tail (W+0x60, R0+0x498; W+0x20c), READ|WRITE, 16 bytes, ++ * 16-byte aligned ++ * config[2] CDM configuration words (W+0x10/0x14, R0+0x4c0/0x4c8), opaque ++ * store context-store program (W+0x21c/0x224, R0+0x4a8/0x4b0) ++ * resume context-resume program (W+0x22c/0x234) ++ * terminate terminate program (W+0x68, R0+0x4b8); control must be zero ++ */ ++struct drm_g5p_compute_pass { ++ __u32 size; /* sizeof(struct drm_g5p_compute_pass) */ ++ __u32 flags; /* zero */ ++ __u64 cdm_va; ++ __u32 cdm_size; ++ __u32 cdm_end; ++ __u64 state_va[4]; ++ __u64 state_tail_va; ++ __u32 config[2]; ++ struct drm_g5p_compute_pds store; ++ struct drm_g5p_compute_pds resume; ++ struct drm_g5p_compute_pds terminate; ++ __u64 reserved[4]; /* zero */ ++}; ++ ++/* Command 0x15. resources lists every BO the job references (1..RENDER_MAX_RESOURCES, ++ * TYPED_V1 access-mask kinds, IMPLICIT and CACHE included); pass_size is exactly ++ * sizeof(struct drm_g5p_compute_pass). Syncs, job ids, results and the error contract are ++ * SUBMIT_RENDER_TYPED's: every error other than EINTR/EFAULT on copy-out precedes fence ++ * replacement and hardware publication. ++ */ ++struct drm_g5p_submit_compute_typed { ++ __u32 context; /* a DRM_G5P_CONTEXT_RENDER context */ ++ __u32 flags; /* zero */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[]: exactly one SIGNAL, at most 16 inputs */ ++ __u64 pass; /* struct drm_g5p_compute_pass */ ++ __u32 pass_size; ++ __u32 pad; /* zero */ ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; /* zero */ ++}; ++ ++#define DRM_G5P_SUBMIT_COMPUTE_TYPED 0x15 ++ ++#define DRM_IOCTL_G5P_SUBMIT_COMPUTE_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_COMPUTE_TYPED, \ ++ struct drm_g5p_submit_compute_typed) ++ ++#endif diff --git a/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch b/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch new file mode 100644 index 00000000..5e33a414 --- /dev/null +++ b/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch @@ -0,0 +1,14822 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add firmware transport and recovery core + +[Orchard: from brkosebay/linux d67f133590a; fixed default policy and +removed idle experiments.] Load the original user-supplied META firmware +and retain bounded recovery. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/bits-compat.h | 20 + + drivers/gpu/apple-g5p/boot-layout.h | 306 ++++ + drivers/gpu/apple-g5p/boot-r73b.inc | 1179 +++++++++++++ + drivers/gpu/apple-g5p/boot-r74f.inc | 532 ++++++ + drivers/gpu/apple-g5p/boot-r74g.inc | 288 +++ + drivers/gpu/apple-g5p/boot.inc | 1665 ++++++++++++++++++ + drivers/gpu/apple-g5p/compute-user-layout.h | 207 +++ + drivers/gpu/apple-g5p/core.inc | 567 ++++++ + drivers/gpu/apple-g5p/diagnostics.inc | 181 ++ + drivers/gpu/apple-g5p/dirtystart108.h | 454 +++++ + drivers/gpu/apple-g5p/dirtystart108.inc | 280 +++ + drivers/gpu/apple-g5p/drm-validate.h | 40 + + drivers/gpu/apple-g5p/fender.h | 87 + + drivers/gpu/apple-g5p/gpu-oc.h | 149 ++ + drivers/gpu/apple-g5p/heartbeat116.h | 175 ++ + drivers/gpu/apple-g5p/idle-power.inc | 38 + + drivers/gpu/apple-g5p/kick.inc | 27 + + drivers/gpu/apple-g5p/latency123.h | 63 + + drivers/gpu/apple-g5p/ldr.h | 154 ++ + drivers/gpu/apple-g5p/lifecycle.inc | 1032 +++++++++++ + drivers/gpu/apple-g5p/meta71c114.h | 65 + + drivers/gpu/apple-g5p/mmio-licence.h | 148 ++ + drivers/gpu/apple-g5p/mmio.inc | 295 ++++ + drivers/gpu/apple-g5p/mmu-attrs.inc | 54 + + drivers/gpu/apple-g5p/mmu-tables.inc | 57 + + drivers/gpu/apple-g5p/pack.h | 89 + + drivers/gpu/apple-g5p/perf-envelope.h | 107 ++ + drivers/gpu/apple-g5p/power-r33.inc | 70 + + drivers/gpu/apple-g5p/power-r41.inc | 135 ++ + drivers/gpu/apple-g5p/r74e112.h | 137 ++ + drivers/gpu/apple-g5p/r87b-attach.h | 34 + + drivers/gpu/apple-g5p/readback.h | 40 + + drivers/gpu/apple-g5p/recover.inc | 101 ++ + drivers/gpu/apple-g5p/render-layout-r22.h | 54 + + drivers/gpu/apple-g5p/render-layout.h | 59 + + drivers/gpu/apple-g5p/render-user-bind.h | 285 +++ + drivers/gpu/apple-g5p/render-user-close.h | 91 + + drivers/gpu/apple-g5p/render-user-evwait.h | 52 + + drivers/gpu/apple-g5p/render-user-fault.h | 60 + + drivers/gpu/apple-g5p/render-user-faultreg.h | 124 ++ + drivers/gpu/apple-g5p/render-user-geometry.h | 144 ++ + drivers/gpu/apple-g5p/render-user-layout.h | 651 +++++++ + drivers/gpu/apple-g5p/render-user-loss.h | 120 ++ + drivers/gpu/apple-g5p/render-user-mmu.h | 60 + + drivers/gpu/apple-g5p/render-user-msaa.h | 88 + + drivers/gpu/apple-g5p/render-user-pmset.h | 77 + + drivers/gpu/apple-g5p/render-user-quiesce.h | 145 ++ + drivers/gpu/apple-g5p/render-user-recover.h | 107 ++ + drivers/gpu/apple-g5p/render-user-retire.h | 248 +++ + drivers/gpu/apple-g5p/render-user-sched.h | 155 ++ + drivers/gpu/apple-g5p/render-user-serial.h | 122 ++ + drivers/gpu/apple-g5p/render-user-sync.h | 50 + + drivers/gpu/apple-g5p/render-user-template.h | 24 + + drivers/gpu/apple-g5p/render-user-terminal.h | 103 ++ + drivers/gpu/apple-g5p/render-user-touch.h | 102 ++ + drivers/gpu/apple-g5p/render-user-wait.h | 266 +++ + drivers/gpu/apple-g5p/render-user-wake.h | 273 +++ + drivers/gpu/apple-g5p/render-user-window.h | 74 + + drivers/gpu/apple-g5p/render-variants-r25.h | 48 + + drivers/gpu/apple-g5p/retire108.h | 434 +++++ + drivers/gpu/apple-g5p/teardown.h | 740 ++++++++ + drivers/gpu/apple-g5p/teardown.inc | 694 ++++++++ + drivers/gpu/apple-g5p/transport-budget.inc | 6 + + drivers/gpu/apple-g5p/vm-mutation.h | 36 + + drivers/gpu/apple-g5p/work-layout.h | 21 + + 65 files changed, 14289 insertions(+) + create mode 100644 drivers/gpu/apple-g5p/bits-compat.h + create mode 100644 drivers/gpu/apple-g5p/boot-layout.h + create mode 100644 drivers/gpu/apple-g5p/boot-r73b.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74f.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74g.inc + create mode 100644 drivers/gpu/apple-g5p/boot.inc + create mode 100644 drivers/gpu/apple-g5p/compute-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/core.inc + create mode 100644 drivers/gpu/apple-g5p/diagnostics.inc + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.h + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.inc + create mode 100644 drivers/gpu/apple-g5p/drm-validate.h + create mode 100644 drivers/gpu/apple-g5p/fender.h + create mode 100644 drivers/gpu/apple-g5p/gpu-oc.h + create mode 100644 drivers/gpu/apple-g5p/heartbeat116.h + create mode 100644 drivers/gpu/apple-g5p/idle-power.inc + create mode 100644 drivers/gpu/apple-g5p/kick.inc + create mode 100644 drivers/gpu/apple-g5p/latency123.h + create mode 100644 drivers/gpu/apple-g5p/ldr.h + create mode 100644 drivers/gpu/apple-g5p/lifecycle.inc + create mode 100644 drivers/gpu/apple-g5p/meta71c114.h + create mode 100644 drivers/gpu/apple-g5p/mmio-licence.h + create mode 100644 drivers/gpu/apple-g5p/mmio.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-attrs.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-tables.inc + create mode 100644 drivers/gpu/apple-g5p/pack.h + create mode 100644 drivers/gpu/apple-g5p/perf-envelope.h + create mode 100644 drivers/gpu/apple-g5p/power-r33.inc + create mode 100644 drivers/gpu/apple-g5p/power-r41.inc + create mode 100644 drivers/gpu/apple-g5p/r74e112.h + create mode 100644 drivers/gpu/apple-g5p/r87b-attach.h + create mode 100644 drivers/gpu/apple-g5p/readback.h + create mode 100644 drivers/gpu/apple-g5p/recover.inc + create mode 100644 drivers/gpu/apple-g5p/render-layout-r22.h + create mode 100644 drivers/gpu/apple-g5p/render-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-bind.h + create mode 100644 drivers/gpu/apple-g5p/render-user-close.h + create mode 100644 drivers/gpu/apple-g5p/render-user-evwait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-fault.h + create mode 100644 drivers/gpu/apple-g5p/render-user-faultreg.h + create mode 100644 drivers/gpu/apple-g5p/render-user-geometry.h + create mode 100644 drivers/gpu/apple-g5p/render-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-loss.h + create mode 100644 drivers/gpu/apple-g5p/render-user-mmu.h + create mode 100644 drivers/gpu/apple-g5p/render-user-msaa.h + create mode 100644 drivers/gpu/apple-g5p/render-user-pmset.h + create mode 100644 drivers/gpu/apple-g5p/render-user-quiesce.h + create mode 100644 drivers/gpu/apple-g5p/render-user-recover.h + create mode 100644 drivers/gpu/apple-g5p/render-user-retire.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sched.h + create mode 100644 drivers/gpu/apple-g5p/render-user-serial.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sync.h + create mode 100644 drivers/gpu/apple-g5p/render-user-template.h + create mode 100644 drivers/gpu/apple-g5p/render-user-terminal.h + create mode 100644 drivers/gpu/apple-g5p/render-user-touch.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wake.h + create mode 100644 drivers/gpu/apple-g5p/render-user-window.h + create mode 100644 drivers/gpu/apple-g5p/render-variants-r25.h + create mode 100644 drivers/gpu/apple-g5p/retire108.h + create mode 100644 drivers/gpu/apple-g5p/teardown.h + create mode 100644 drivers/gpu/apple-g5p/teardown.inc + create mode 100644 drivers/gpu/apple-g5p/transport-budget.inc + create mode 100644 drivers/gpu/apple-g5p/vm-mutation.h + create mode 100644 drivers/gpu/apple-g5p/work-layout.h + +diff --git a/drivers/gpu/apple-g5p/bits-compat.h b/drivers/gpu/apple-g5p/bits-compat.h +new file mode 100644 +index 000000000..608e9b8ec +--- /dev/null ++++ b/drivers/gpu/apple-g5p/bits-compat.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_BITS_COMPAT_H ++#define G5P_BITS_COMPAT_H ++ ++#ifdef __KERNEL__ ++#include ++#include ++#else ++#ifndef BIT ++#define BIT(nr) (1UL << (nr)) ++#endif ++#ifndef BIT_ULL ++#define BIT_ULL(nr) (1ULL << (nr)) ++#endif ++#ifndef ARRAY_SIZE ++#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) ++#endif ++#endif ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-layout.h b/drivers/gpu/apple-g5p/boot-layout.h +new file mode 100644 +index 000000000..752e6873f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-layout.h +@@ -0,0 +1,306 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_BOOT_LAYOUT_H ++#define APPLE_G5P_BOOT_LAYOUT_H ++ ++#include "bits-compat.h" ++#include "pack.h" ++ ++#define G5P_GPU_PAGE 0x4000U ++#define G5P_ROOT_SIZE 0x8000U ++#define G5P_SHARED_SIZE 0x60000U ++#define G5P_IMAGE_VA 0xe1c0000000ULL ++#define G5P_SHARED_VA 0xe1c0200000ULL ++#define G5P_ROOT_VA 0xe1c3200000ULL ++#define G5P_PIO_VA 0xe1c4200000ULL ++#define G5P_PIO_SIZE 0x10000U ++ ++/* c95da0 descriptors, sizes at 5940b90; c97b6c packs on 64-byte boundaries. */ ++struct g5p_boot_layout { ++ u32 offset[0x90]; ++ u8 group[0x90]; ++ u32 used[2]; ++}; ++ ++static inline void g5p_put32(u8 *p, u32 value) ++{ ++ p[0] = value; ++ p[1] = value >> 8; ++ p[2] = value >> 16; ++ p[3] = value >> 24; ++} ++ ++static inline void g5p_put64(u8 *p, u64 value) ++{ ++ g5p_put32(p, value); ++ g5p_put32(p + 4, value >> 32); ++} ++ ++static inline u32 g5p_meta_address(u64 va) ++{ ++ /* ca0744: root range 8 bypasses META caches; other shared RAM is global. */ ++ u32 prefix = va >= G5P_ROOT_VA && va < G5P_ROOT_VA + 0x1000000 ? ++ 0x40000000 : ++ 0xc0000000; ++ ++ return ((u32)va + 0x40000000) | prefix; ++} ++ ++static inline int g5p_plan_layout(struct g5p_boot_layout *p) ++{ ++ static const struct { ++ u8 cpu, gpu, group; ++ u32 size; ++ } objects[] = { ++ { 0x44, 0x50, 0, 0x1490 }, ++ { 0x47, 0x51, 0, 0x990 }, ++ { 0x48, 0x52, 0, 0x8e0 }, ++ { 0x49, 0x53, 0, 0x8c0 }, ++ { 0x4a, 0x54, 0, 0x938 }, ++ { 0x4b, 0x55, 0, 0x28 }, ++ { 0x4c, 0x56, 0, 8 }, ++ { 0x4d, 0x57, 0, 0x78c }, ++ { 0x4e, 0x58, 0, 0x84 }, ++ { 0x5e, 0x5f, 1, 0x2058 }, ++ { 0x70, 0x78, 1, 0x20 }, ++ { 0x71, 0x79, 1, 0x80 }, ++ { 0x72, 0x7a, 1, 0x20 }, ++ { 0x73, 0x7b, 1, 0x20 }, ++ { 0x74, 0x7c, 1, 0x2c00 }, ++ { 0x75, 0x7d, 1, 0xb000 }, ++ { 0x4f, 0x59, 1, 0x36000 }, ++ { 0x76, 0x7e, 1, 0x2c00 }, ++ { 0x77, 0x7f, 1, 0x2400 }, ++ { 0x6c, 0x6e, 1, 0x20 }, ++ { 0x6d, 0x6f, 1, 0x1400 }, ++ { 0x60, 0x62, 1, 0x20 }, ++ { 0x64, 0x66, 1, 0x20 }, ++ { 0x68, 0x6a, 1, 0x20 }, ++ { 0x61, 0x63, 1, 0x1400 }, ++ { 0x65, 0x67, 1, 0x1400 }, ++ { 0x69, 0x6b, 1, 0x1400 }, ++ /* Separate c95da0 allocations, placed after the recovered group. */ ++ { 0x5c, 0x5d, 1, 8 }, ++ { 0x85, 0x88, 1, 0x400 }, ++ { 0x84, 0x87, 1, 0x400 }, ++ { 0x86, 0x89, 1, 0x800 }, ++ }; ++ unsigned int i; ++ ++ memset(p, 0, sizeof(*p)); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) { ++ u32 group = objects[i].group; ++ u32 off = (p->used[group] + 63) & ~63U; ++ ++ p->offset[objects[i].cpu] = off; ++ p->offset[objects[i].gpu] = off; ++ p->group[objects[i].cpu] = group; ++ p->group[objects[i].gpu] = group; ++ p->used[group] = off + objects[i].size; ++ } ++ return p->used[0] <= G5P_ROOT_SIZE && p->used[1] <= G5P_SHARED_SIZE ? ++ 0 : ++ -EINVAL; ++} ++ ++static inline u64 g5p_object_va(const struct g5p_boot_layout *p, u32 slot) ++{ ++ return (p->group[slot] ? G5P_SHARED_VA : G5P_ROOT_VA) + p->offset[slot]; ++} ++ ++static inline u8 *g5p_object(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 slot) ++{ ++ return (p->group[slot] ? shared : root) + p->offset[slot]; ++} ++ ++/* ca9b34 + cab44c + cac314: 40-bit PA, 16 KiB GPU pages. */ ++static inline u32 g5p_catalogue_entry(u64 pa) ++{ ++ return (u32)(pa >> 8) | 1; ++} ++ ++static inline u64 g5p_directory_entry(u64 pa) ++{ ++ /* Geometry bytes 3c..3f are 08 05 05 15: size, entry/PA/VA shifts. */ ++ return pa | 3; ++} ++ ++static inline u64 g5p_leaf_entry(u64 pa, u32 attributes) ++{ ++ /* caa6e4 range attributes -> ca9a60: META protect, CC / SLC bypass. */ ++ return (pa & 0xfffffff000ULL) | (attributes & 0xc) | ++ ((u64)(attributes & 0x10) << 58) | 1; ++} ++ ++static inline void g5p_make_boot_config(struct g5p_ldr_image *im, ++ const struct g5p_boot_layout *p) ++{ ++ static const u32 bases[] = { 0x10000000, 0x20000000, 0x40000000, ++ 0x60001000 }; ++ static const u32 sizes[] = { 0x18000, 0x4000, 0x4000, 0x4000 }; ++ static const u32 offsets[] = { 0x4000, 0x1c000, 0, 0x20000 }; ++ u8 *out = im->data + 0x200; ++ unsigned int seg, i; ++ ++ for (seg = 0; seg < 4; seg++) { ++ u64 va = G5P_IMAGE_VA + offsets[seg]; ++ u32 reg = 0x04850000 + seg * 16; ++ u32 values[] = { bases[seg] | 0xf02, sizes[seg] - 1, (u32)va, ++ (u32)(va >> 32) | 0x300000 }; ++ ++ if (seg == 2) { ++ g5p_put32(out, reg + 12); ++ g5p_put32(out + 4, (u32)(va >> 32) | 0x100); ++ out += 8; ++ continue; ++ } ++ for (i = 0; i < 4; i++, out += 8) { ++ g5p_put32(out, reg + i * 4); ++ g5p_put32(out + 4, values[i]); ++ } ++ } ++ for (i = 0; i < im->config_count; i++, out += 8) { ++ g5p_put32(out, im->config[i][0]); ++ g5p_put32(out + 4, im->config[i][1]); ++ } ++ g5p_put64(out, 0); ++ g5p_put32(out + 8, g5p_meta_address(G5P_IMAGE_VA + 0x3ff8)); ++ g5p_put32(im->data + 0x3ff8, g5p_meta_address(g5p_object_va(p, 0x50))); ++ g5p_put32(im->data + 0x3ffc, g5p_meta_address(g5p_object_va(p, 0x5f))); ++} ++ ++/* c96be8/c9edcc/ca3ccc. Unknown platform policy fields remain zero. */ ++static inline void g5p_make_init(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, const struct g5p_ldr_image *im, ++ u64 catalogue, u64 timestamp) ++{ ++ static const u8 roots[] = { ++ 0x62, 0x63, 0x66, 0x67, 0x6a, 0x6b, 0x6e, 0x6f, ++ 0x78, 0x7c, 0x79, 0x7d, 0x7a, 0x7e, 0x7b, 0x7f, ++ }; ++ static const u8 objects[] = { 0x51, 0x52, 0x53, 0x54, 0x55 }; ++ static const u8 tail[] = { 0x58, 0x56, 0x57, 0x88, 0x59 }; ++ static const u64 pio_pa[] = { 0x201e00000ULL, 0x20e100000ULL, ++ 0x20e104000ULL, 0x201000000ULL }; ++ static const u32 pio_size[] = { 0x5000, 0x4000, 0x4000, 0x20000 }; ++ static const u32 pio_offset[] = { 0, 0x8000, 0xc000, 0 }; ++ u8 *init = g5p_object(p, root, shared, 0x44); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *ctrl = g5p_object(p, root, shared, 0x5e); ++ unsigned int i; ++ ++ memset(root, 0, G5P_ROOT_SIZE); ++ memset(shared, 0, G5P_SHARED_SIZE); ++ for (i = 0; i < ARRAY_SIZE(roots); i++) ++ g5p_put32(init + i * 4, ++ g5p_meta_address(g5p_object_va(p, roots[i]))); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) ++ g5p_put32(init + 0x64 + i * 4, ++ g5p_meta_address(g5p_object_va(p, objects[i]))); ++ for (i = 0; i < ARRAY_SIZE(tail); i++) ++ g5p_put32(init + 0x44c + i * 4, ++ g5p_meta_address(g5p_object_va(p, tail[i]))); ++ for (i = 0; i < 3; i++) { ++ u8 *channel = g5p_object(p, root, shared, 0x47 + i); ++ u32 ptr = g5p_meta_address(g5p_object_va(p, 0x51 + i)); ++ ++ g5p_put32(channel + 4, ptr); ++ g5p_put32(channel + 12, ptr + 8); ++ } ++ g5p_put32(g5p_object(p, root, shared, 0x48) + 0x778, 0xffffffff); ++ g5p_put32(init + 0x480, ++ ((u32)(G5P_IMAGE_VA + 0x24000) + 0x40000000) | 0x40000000); ++ g5p_put64(init + 0x484, G5P_IMAGE_VA + 0x24000); ++ g5p_put32(init + 0x48c, 0x55dc); ++ g5p_put64(meta, 0x600000000ULL); ++ g5p_put64(meta + 8, 0xffc00000); ++ g5p_put64(meta + 0x10, 0x300000000ULL); ++ g5p_put64(meta + 0x18, 0x300000000ULL); ++ for (i = 0; i < ARRAY_SIZE(pio_pa); i++) { ++ u8 *entry = meta + 0x630 + i * 16; ++ ++ g5p_put64(entry, pio_pa[i]); ++ g5p_put32(entry + 8, i == 3 ? 0 : ++ g5p_meta_address(G5P_PIO_VA + ++ pio_offset[i])); ++ g5p_put32(entry + 12, pio_size[i]); ++ } ++ g5p_put32(meta + 0x6c0, 1); ++ g5p_put32(meta + 0x6c8, 1); ++ g5p_put64(meta + 0x6d8, 0x100000000ULL); ++ g5p_put32(meta + 0x6e0, 1); ++ g5p_put32(meta + 0x6e4, 1); ++ /* readChipInfo version 15 -> tables 5942650/5942674 -> 1,5; N=6. */ ++ g5p_put32(meta + 0x704, 1); ++ g5p_put32(meta + 0x708, 5); ++ g5p_put32(meta + 0x70c, 6); ++ g5p_put32(meta + 0x758, 0x14); ++ g5p_put32(meta + 0x75c, 0x4000); ++ g5p_put32(meta + 0x760, 0x1020); ++ g5p_put32(meta + 0x764, im->t1_pc); ++ g5p_put32(meta + 0x768, im->t1_sp); ++ g5p_put64(meta + 0x774, G5P_IMAGE_VA); ++ g5p_put64(meta + 0x77c, 0x300000); ++ g5p_put64(meta + 0x784, timestamp); ++ /* Fixed portion of c974f0; dynamic power-policy fields remain zero. */ ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 8, 0x400000000ULL); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x10, 0x3f800000); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x28, 0x100000001ULL); ++ g5p_put32(g5p_object(p, root, shared, 0x4a) + 0x38, 100); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x8f8, timestamp); ++ g5p_put32(ctrl + 0x24, 0x78); ++ g5p_put32(ctrl + 0x30, 1); ++ g5p_put32(ctrl + 0x1044, g5p_meta_address(g5p_object_va(p, 0x87))); ++ g5p_put32(ctrl + 0x1048, g5p_meta_address(g5p_object_va(p, 0x89))); ++ g5p_put32(ctrl + 0x104c, 1); ++ g5p_put64(g5p_object(p, root, shared, 0x86), catalogue); ++} ++ ++/* Explicit boot-time hypotheses; never modify a live root table. */ ++static inline void g5p_dc_init_variant(const struct g5p_boot_layout *p, ++ u8 *root, u8 *shared, u32 variant) ++{ ++ /* ca0744 uses 40 aliases for cache bypass. Applying that alias to DC ++ * is an unverified hypothesis, not recovered normal host behavior. ++ */ ++ if (variant == 1) ++ g5p_put32(root + 0x18, g5p_le32(root + 0x18) & ~0x80000000U); ++ if (variant == 1 || variant == 6) ++ g5p_put32(root + 0x1c, g5p_le32(root + 0x1c) & ~0x80000000U); ++ /* c96be8 chooses this policy from host+1af8 bits; both values exist. ++ * Whether zero is appropriate for our incomplete policy is unknown. ++ */ ++ if (variant == 5) ++ g5p_put32(shared + p->offset[0x5e] + 0x30, 0); ++ /* MINIM 10001ac8 enters policy1000c0d8 only when meta6c0/6c8 ++ * are both nonzero. Experimental pre-MTS bypass; no live length writes. ++ */ ++ if (variant == 7) ++ g5p_put32(g5p_object(p, root, shared, 0x4d) + 0x6c8, 0); ++ ++ /* Round13: recovered iOS inputs; variants 8/9 retain meta6c8=1. ++ * Neither is a complete DVFS policy (table+3c remains absent). ++ */ ++ if (variant == 8 || variant == 9) { ++ u8 *policy = g5p_object(p, root, shared, 0x4a); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *control = g5p_object(p, root, shared, 0x5e); ++ static const u32 mhz[] = { 0, 340, 474, 550, 723 }; ++ ++ g5p_put32(policy + 4, 16); ++ if (variant == 9) { ++ g5p_put32(meta + 0x710, 4); ++ g5p_put32(meta + 0x714, 5); ++ for (u32 i = 0; i < 16; i++) ++ g5p_put32(meta + 0x718 + 4 * i, ++ i < ARRAY_SIZE(mhz) ? mhz[i] : 0); ++ g5p_put32(control + 0x1028, 3348); ++ g5p_put32(control + 0x102c, 400); ++ g5p_put32(control + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++ } ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +new file mode 100644 +index 000000000..df2560f5e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -0,0 +1,1179 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R73b crash-first IRQ174 diagnostics. Included by boot.inc after struct ++ * g5p_boot. Case zero allocates nothing and keeps the r73 IRQ order. ++ * Rows live in CPU RAM only; they are formatted after the epoch, never in the ++ * IRQ service loop. The first 128 rows are retained, the last 128 roll. ++ */ ++#include ++#include ++#include ++#include ++#include "r87b-attach.h" ++#include "r74e112.h" ++#include "heartbeat116.h" ++ ++#define G5P_R73B_ROWS 256 ++#define G5P_R73B_HEAD 128 ++#define G5P_R73B_IRQ_CAP 8192 ++#define G5P_R73B_PANIC_ROWS 32 ++#define G5P_R73B_TRACE_ROWS 128 ++#define G5P_R73B_TRACE_WORDS 9 ++#define G5P_R74_IRQ_CAP 262144 ++#define G5P_R74_IRQ_CAP_MAX 1048576 ++#define G5P_R74_EVENT_FLOOR (256 + 4 * 65536) ++#define G5P_R74_STAGES 8 ++#define G5P_R74_MILESTONES 64 ++#define G5P_R74_MAX_TICKS 16 ++#define G5P_R74_RGX 9 ++#define G5P_R74_WORD1 8 ++#define G5P_R74F_KINDS 3 ++#define G5P_R74F_OFF_US 1000 ++#define G5P_R74F_ON_US 1800 ++#define G5P_R74F_BELL_MS 250 ++#define G5P_R74F_MMU_SLOTS 2 ++#define G5P_R74F_MMU_US 1000 ++#define G5P_R74G_GRACE_ALIVES 4 ++#define G5P_R74G_GRACE_US 20000 ++#define G5P_R74G_COMPANION 0x3980 ++#define G5P_R74G_RECOVERIES 0x3994 ++#define G5P_R74G_CLASS 0x3998 ++#define G5P_R74G_BAD_RING BIT(0) ++#define G5P_R74G_BAD_RECORD BIT(1) ++#define G5P_R74G_BAD_PA BIT(2) ++#define G5P_R74G_BAD_HOST BIT(3) ++#define G5P_R74G_BAD_CACHED BIT(4) ++#define G5P_R74G_BAD_FAILED BIT(5) ++#define G5P_R74G_BAD_COMPANION BIT(6) ++#define G5P_R74G_BAD_MUTATION BIT(7) ++#define G5P_R74H_TAIL 128 ++#define G5P_R74H_PHYS_MAX 1024 ++#define G5P_R74H_PTES 10 ++ ++/* First-ALIVE submit: wait (reads only) for a GPC request in flight to finish. */ ++static const unsigned int boot_alive_settle_us = 2000; ++MODULE_PARM_DESC( ++ boot_alive_settle_us, ++ "default0 (single read, as before); 1..5000: when the first-ALIVE submit reads " ++ "R1+0x1000 with bit 4 set (a GPC performance-state request in flight), re-read " ++ "it every 10 us for up to this long before the unchanged state==1 check, as " ++ "Apple's setGPCPerformanceState waits for bit 4; R1 reads only; latched at map"); ++ ++enum g5p_r73b_phase { ++ G5P_R73B_RELEASE_ENTRY = 1, ++ G5P_R73B_RELEASE_FIFO, ++ G5P_R73B_RELEASE_GPC, ++ G5P_R73B_RELEASE_EXIT, ++ G5P_R73B_IRQ, ++ G5P_R73B_DRAIN_BEFORE, ++ G5P_R73B_DRAIN_AFTER, ++ G5P_R73B_DRAIN_BUSY, ++ G5P_R73B_CAP, ++ G5P_R73B_EVENTS_EXIT, ++ G5P_R73B_DC_PUBLISHED, ++ G5P_R73B_DC_KICKED, ++ G5P_R73B_WAIT_EXIT, ++ G5P_R73B_EXPORT, ++ G5P_R73B_PHASES, ++}; ++ ++static const char *const g5p_r73b_phase_names[G5P_R73B_PHASES] = { ++ "unknown", "release-entry", "release-fifo", "release-gpc", ++ "release-exit", "irq", "drain-before", "drain-after", ++ "drain-busy", "cap", "events-exit", "dc-published", ++ "dc-kicked", "wait-exit", "export", ++}; ++ ++enum g5p_r73b_owner { ++ G5P_R73B_NONE, ++ G5P_R73B_OWNER_IRQ, ++ G5P_R73B_OWNER_EVENTS, ++ G5P_R73B_OWNER_PREPARE, ++ G5P_R73B_OWNER_PUBLISH, ++ G5P_R73B_OWNER_KICK, ++ G5P_R73B_OWNER_WAIT, ++}; ++ ++/* Row columns: event, log lane 0, statistics, trace, device control. */ ++static const u8 g5p_r73b_row_slots[5] = { 0x70, 0x71, 0x72, 0x73, 0x6c }; ++ ++struct g5p_r73b_row { ++ u32 seq; /* sequence + 1, published last; zero means empty/torn */ ++ u32 phase; ++ u64 t_ns; ++ u32 irq, ack, released, owner, cap_seen; ++ s32 result; ++ u32 cp[5][2]; ++}; ++ ++enum g5p_r74_step { ++ G5P_R74_IRQ_TEST_REFUSED, ++ G5P_R74_PREPARE_BEGIN, ++ G5P_R74_PREPARE_END, ++ G5P_R74_ATTACH_BEGIN, ++ G5P_R74_ATTACH_PUBLISHED, ++ G5P_R74_ATTACH_DOORBELL_BEGIN, ++ G5P_R74_ATTACH_DOORBELL_END, ++ G5P_R74_ATTACH_CONSUMED, ++ G5P_R74_ATTACH_END, ++ G5P_R74_TA_BEGIN, ++ G5P_R74_TA_PUBLISH, ++ G5P_R74_TA_DOORBELL_RETURNED, ++ G5P_R74_OBSERVE, ++ G5P_R74_WAIT_TICK, ++ G5P_R74_WAIT_EXIT, ++ G5P_R74_REPORT_END, ++ G5P_R74_CMD8_CONSUMED, ++ G5P_R74_TA_CONSUMED, ++ G5P_R74_TA_RAM_COMPLETE, ++ G5P_R74_TA_RAM_TIMEOUT, ++ G5P_R74F_ATTACH_ARMED, ++ G5P_R74F_ATTACH_REACTIVATED, ++ G5P_R74F_ATTACH_BELL, ++ G5P_R74F_MMU_CLOSED, ++ G5P_R74F_TA_ARMED, ++ G5P_R74F_TA_REACTIVATED, ++ G5P_R74F_TA_BELL, ++ G5P_R74F_STOP, ++ G5P_R74G_FW_MMU, ++ G5P_R74G_ACK_SKIPPED, ++ G5P_R74G_TA_GRACE, ++ G5P_R74G_TA_FALLBACK, ++ G5P_R74_STEPS, ++}; ++ ++static const char *const g5p_r74_steps[G5P_R74_STEPS] = { ++ "irq-test-refused", ++ "prepare-begin", ++ "prepare-end", ++ "attach-begin", ++ "attach-dc3-published", ++ "attach-doorbell-begin", ++ "attach-doorbell-end", ++ "attach-consumed", ++ "attach-end", ++ "ta-begin", ++ "ta-publish", ++ "ta-doorbell-returned", ++ "observe", ++ "wait-tick", ++ "wait-exit", ++ "report-end", ++ "cmd8-consumed", ++ "ta-consumed", ++ "ta-ram-complete", ++ "ta-ram-timeout", ++ "attach-bell-armed", ++ "attach-reactivated", ++ "attach-bell", ++ "first-mmu-window-closed", ++ "ta-bell-armed", ++ "ta-reactivated", ++ "ta-bell", ++ "r74f-stop", ++ "firmware-mmu-complete", ++ "r31-ack-skipped", ++ "ta-grace-end", ++ "ta-fallback", ++}; ++ ++/* R74 read-only witnesses; every address is already read elsewhere in-tree. */ ++ ++struct g5p_r74_milestone { ++ u64 t_ns; ++ u32 step, irqs, events; ++ s32 ret; ++ u32 dc[2], ta[2]; ++}; ++ ++enum g5p_r74f_kind { ++ G5P_R74F_DC3, ++ G5P_R74F_TA, ++ G5P_R74F_3D, /* r96: ring 0x64, doorbell 0x11, owned by the ALIVE decoder like the TA's */ ++}; ++ ++/* idle -> published (gate closed) -> open (reactivated) -> firing -> ++ * rang; published/open -> cancelled. FIRING lives under event_lock. ++ */ ++enum g5p_r74f_state { ++ G5P_R74F_IDLE, ++ G5P_R74F_PUBLISHED, ++ G5P_R74F_OPEN, ++ G5P_R74F_FIRING, ++ G5P_R74F_RANG, ++ G5P_R74F_CANCELLED, ++ G5P_R74F_STATES, ++}; ++ ++enum g5p_r74f_cancel { ++ G5P_R74F_CANCEL_NONE, ++ G5P_R74F_CANCEL_REACTIVATION, ++ G5P_R74F_CANCEL_NO_ALIVE, ++ G5P_R74F_CANCEL_CONSUMED, ++ G5P_R74F_CANCEL_PROTOCOL, ++ G5P_R74F_CANCEL_ALIVE_ADVANCED, ++ G5P_R74F_CANCEL_SELF_DRAIN, ++ G5P_R74F_CANCELS, ++}; ++ ++enum g5p_r74f_stop { ++ G5P_R74F_STOP_NONE, ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ G5P_R74F_STOP_MMU_CLOSED, ++ G5P_R74F_STOP_TA_REACTIVATION, ++ G5P_R74F_STOP_TA_NOT_KICKED, ++ G5P_R74G_STOP_DC3_TAGS, ++ G5P_R74G_STOP_DC3_RAM, ++ G5P_R74G_STOP_COMPANION, ++ G5P_R74G_STOP_TA_NOT_CONSUMED, ++ G5P_R74G_STOP_TA_WINDOW_MISSED, ++ G5P_R74F_STOPS, ++}; ++ ++enum g5p_r74f_path { ++ G5P_R74F_PATH_NONE, ++ G5P_R74F_PATH_IMMEDIATE, ++ G5P_R74F_PATH_DEFERRED, ++ G5P_R74F_PATHS, ++}; ++ ++/* One generation. Its owner writes the receipt, then releases state. */ ++struct g5p_r74f_bell { ++ atomic_t state; ++ u32 kind, generation, ring, start, target, path, cancel; ++ u32 arm_alive, open_alive, bell_alive; ++ u32 first_fence, first_state, fence, bell_state; ++ s32 first_ret, bell_ret, power_ret; ++ u32 reactivated, core_fence, gpc_fence, off_raw, on_raw; ++ u32 attempted, kicked, complete; ++ u32 consumer_before, consumer_after, producer_after; ++ u32 alive_done; ++ u64 arm_ns, power_start_ns, off_ns, on_ns, open_ns; ++ u64 bell_start_ns, bell_ns, done_ns; ++}; ++ ++enum g5p_r74g_grace { ++ G5P_R74G_GRACE_NONE, ++ G5P_R74G_GRACE_CONSUMED, ++ G5P_R74G_GRACE_ALIVES_END, ++ G5P_R74G_GRACE_WALL, ++ G5P_R74G_GRACE_FAULT, ++ G5P_R74G_GRACES, ++}; ++ ++/* The RAM-visible results of the firmware DC3 handler: phase 0 at the DC3 ++ * consume, phase 1 again just before the TA producer moves. ++ */ ++struct g5p_r74g_dc3 { ++ u64 pa[2], expected_pa, t_ns; ++ u32 slot[2], host[2], cached[2], rec[2][5]; ++ u32 start, target, consumer, producer, tag; ++ u32 failed, event_failed, companion, recoveries, classification, bad; ++}; ++ ++struct g5p_r73b { ++ struct device *dev; ++ struct g5p_boot *boot; ++ u32 case_id, watchdog_ms, dc_variant, meta6c8, r73e_case; ++ bool ack_first, aux, fast_enabled, fast_window; ++ bool early_release_ready, early_power_abort; ++ atomic_t early_once, early_busy; ++ u64 early_irq_ns, early_power_start_ns; ++ u64 early_off_ns, early_on_ns; ++ u32 early_state, early_after_state, early_power_raw; ++ s32 early_ret, early_power_ret; ++ /* wake112 (r74e112.h): the R74E proof deferred to the first-ALIVE owner. */ ++ atomic_t early_proof; ++ u32 early_proof_gen, early_proof_gen_now, early_proof_why; ++ s32 early_defer_ret; ++ /* boot_alive_settle_us: limit, first in-flight value seen, re-reads, time spent. */ ++ u32 settle_us, settle_first, settle_samples; ++ u64 settle_ns; ++ u64 fast_release_ns, fast_alive_ns, fast_submit_ns; ++ u32 fast_state; ++ s32 fast_ret; ++ /* nopatch: Apple's unmodified image; policy record +0/+4/+8/+c/+34 taken before the first bell. */ ++ bool nopatch; ++ u32 np_pre[5]; ++ atomic_t seq, acks, cap_seen, busy, drains, drain_errors, exported, ++ trace_saved; ++ atomic_t watchdog_state; /* 0 off, 1 armed, 2 disarmed, 3 fired, 4 terminal */ ++ int watchdog_cpu; ++ u32 owner; ++ struct mutex drain_lock; /* single auxiliary-ring owner */ ++ struct hrtimer watchdog; ++ struct g5p_r73b_row rows[G5P_R73B_ROWS]; ++ u32 trace_rows[G5P_R73B_TRACE_ROWS][G5P_R73B_TRACE_WORDS]; ++ u32 irq_cap, r74_case, r74_mask, r74_ticks, r74_word1_count; ++ u32 r74_word1[G5P_R74_WORD1]; ++ atomic_t r74_seq, r74_irq_refused; ++ struct g5p_r74_milestone r74_milestones[G5P_R74_MILESTONES]; ++ ++ /* r74f_alive and FIRING are written under event_lock only. */ ++ u32 r74f_case, r74f_alive, r74f_active, r74f_generation, r74f_stop; ++ ++ u32 r87b_attach; ++ s32 r74f_stop_value; ++ atomic_t r74f_bells; ++ struct g5p_r74f_bell r74f_bell[G5P_R74F_KINDS]; ++ ++ /* R74g: no host MMU pass, publish-only TA grace, one fallback bell. */ ++ u32 r74g_case, r74g_ack_skipped, r74g_grace_end, r74g_opened; ++ u32 r74g_grace_alive; ++ u64 r74g_grace_ns; ++ struct g5p_r74g_dc3 r74g_dc3[2]; ++ /* R74h: RAM-only trace tail and checkpoints; NULL when off. */ ++ u32 r74h_case; ++ ++ ++ ++ /* Heartbeat116 (heartbeat116.h): the R74 window detector; IRQ thread only. */ ++ bool hb_on; ++ u32 hb_seed, hb_empty, hb_cross; ++ atomic_t hb_late; ++ struct g5p_hb_cfg hb_cfg; ++ struct g5p_hb hb; ++}; ++ ++/* The ctrl+0 word installed before release. */ ++static u32 g5p_boot_ctrl0(struct g5p_boot *b) ++{ ++ return b->r78_ctrl0; ++} ++ ++ ++ ++/* render-r96.inc: the receipts at export (RAM only) */ ++ ++/* render-r94.inc: the RAM state of the record and the code BO at export, read only */ ++ ++/* boot-r80.inc: the two IRQ-owned plateau banks, read-only */ ++ ++/* render-r84b.inc: the late raw dump and the CPU visibility step, RAM only */ ++ ++static int g5p_boot_drain(struct g5p_diag *g); ++static u32 g5p_transport_event_limit(void); ++static int g5p_r73c_fast_control(struct g5p_diag *g); ++static int g5p_r73e_prepare(struct g5p_diag *g); ++static int g5p_r73e_irq_submit(struct g5p_diag *g); ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner); ++ ++static const char *g5p_r73b_name(u32 id) ++{ ++ return id == 1 ? "meta1-bounded-all-rings" : ++ id == 2 ? "meta0-bounded-all-rings-control" : ++ id == 3 ? "meta1-event-only-reference" : ++ "off"; ++} ++ ++static bool g5p_r73b_sampled(u32 n) ++{ ++ return n <= 16 || is_power_of_2(n); ++} ++ ++static u32 *g5p_r73b_ptr(struct g5p_boot *b, u32 off) ++{ ++ return (u32 *)(b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r73b_word(struct g5p_boot *b, u32 off) ++{ ++ return READ_ONCE(*g5p_r73b_ptr(b, off)); ++} ++ ++static u32 g5p_r73b_slot(u32 seq) ++{ ++ if (seq < G5P_R73B_HEAD) ++ return seq; ++ return G5P_R73B_HEAD + ++ (seq - G5P_R73B_HEAD) % (G5P_R73B_ROWS - G5P_R73B_HEAD); ++} ++ ++/* Lockless: a writer owns its slot by sequence; seq is published last. */ ++static void g5p_r73b_crumb(struct g5p_boot *b, u32 phase, u32 owner, int result) ++{ ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r73b_row *row; ++ u32 seq; ++ ++ if (!r) ++ return; ++ seq = atomic_inc_return(&r->seq) - 1; ++ row = &r->rows[g5p_r73b_slot(seq)]; ++ WRITE_ONCE(row->seq, 0); ++ /* Invalidate before rewriting so a reader never pairs old seq/new data. */ ++ smp_wmb(); ++ row->phase = phase; ++ row->t_ns = ktime_get_mono_fast_ns(); ++ row->irq = atomic_read(&b->irqs); ++ row->ack = atomic_read(&r->acks); ++ row->released = READ_ONCE(b->released); ++ row->owner = owner; ++ row->result = result; ++ row->cap_seen = atomic_read(&r->cap_seen); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r73b_row_slots); i++) { ++ u32 ctl = b->layout.offset[g5p_r73b_row_slots[i]]; ++ ++ row->cp[i][0] = g5p_r73b_word(b, ctl); ++ row->cp[i][1] = g5p_r73b_word(b, ctl + 0x10); ++ } ++ /* Publish the row contents before its sequence. */ ++ smp_wmb(); ++ WRITE_ONCE(row->seq, seq + 1); ++} ++ ++static const char *g5p_r74_name(u32 id) ++{ ++ return id == 1 ? "forced-ready-meta1-natural-ta" : ++ id == 2 ? "meta0-unpatched-natural-ta-control" : ++ "off"; ++} ++ ++static const char *g5p_r74_step_name(u32 step) ++{ ++ return step < G5P_R74_STEPS ? g5p_r74_steps[step] : "unknown"; ++} ++ ++static u32 g5p_r74_case(struct g5p_boot *b) ++{ ++ return b && b->r73b ? 1 : 0; ++} ++ ++/* R74b case 1: after cmd8 retirement, optional host MMIO must not race the ++ * firmware policy loop power-gating RGX. Functional setup/kick sites opt in ++ * explicitly; this predicate gates observations only. ++ */ ++static bool g5p_r74b_ram_only(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ ++ return g5p_r74_case(b) == 1 && b && READ_ONCE(b->released) && ++ READ_ONCE(b->r72_meta6c8); ++} ++ ++/* Process context only, never per IRQ: one console line per r74 step, so a ++ * whole-SoC freeze is localized by the last line netconsole delivered. ++ */ ++static void g5p_r74_mark(struct g5p_diag *g, u32 step, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_r74_milestone m; ++ u32 seq; ++ ++ if (!r || !1) ++ return; ++ m.t_ns = ktime_get_ns(); ++ m.step = step; ++ m.ret = ret; ++ m.irqs = atomic_read(&b->irqs); ++ m.events = READ_ONCE(b->events); ++ m.dc[0] = g5p_r73b_word(b, b->layout.offset[0x6c]); ++ m.dc[1] = g5p_r73b_word(b, b->layout.offset[0x6c] + 0x10); ++ m.ta[0] = g5p_r73b_word(b, b->layout.offset[0x60]); ++ m.ta[1] = g5p_r73b_word(b, b->layout.offset[0x60] + 0x10); ++ seq = atomic_inc_return(&r->r74_seq) - 1; ++ if (seq < G5P_R74_MILESTONES) ++ r->r74_milestones[seq] = m; ++ dev_emerg( ++ g->dev, ++ "R74 milestone seq=%u step=%s ret=%d irqs=%u events=%u dc=%u/%u ta=%u/%u " ++ "t_ns=%llu end=r74\n", ++ seq, g5p_r74_step_name(step), ret, m.irqs, m.events, m.dc[0], ++ m.dc[1], m.ta[0], m.ta[1], m.t_ns); ++} ++ ++/* R74 keeps repeated ALIVEs quiet whatever their word1: each new value is ++ * printed once (eight at most), so a moved loop cannot flood the console. ++ */ ++static bool g5p_r74_word1(struct g5p_boot *b, u32 word1) ++{ ++ struct g5p_r73b *r = b->r73b; ++ ++ ; ++ ; ++ for (u32 i = 0; i < r->r74_word1_count; i++) ++ if (r->r74_word1[i] == word1) ++ return false; ++ if (r->r74_word1_count == G5P_R74_WORD1) ++ return false; ++ r->r74_word1[r->r74_word1_count++] = word1; ++ return true; ++} ++ ++static bool g5p_r73b_copy_row(struct g5p_r73b *r, u32 seq, ++ struct g5p_r73b_row *out) ++{ ++ struct g5p_r73b_row *row = &r->rows[g5p_r73b_slot(seq)]; ++ ++ if (READ_ONCE(row->seq) != seq + 1) ++ return false; ++ /* Pairs with the writer's publication barrier. */ ++ smp_rmb(); ++ *out = *row; ++ /* Re-check that the slot was not recycled while copying. */ ++ smp_rmb(); ++ return READ_ONCE(row->seq) == seq + 1; ++} ++ ++static int g5p_r73b_format(char *buf, size_t len, ++ const struct g5p_r73b_row *row) ++{ ++ u32 phase = row->phase < G5P_R73B_PHASES ? row->phase : 0; ++ ++ return scnprintf( ++ buf, len, ++ "R73B crumb seq=%u t_ns=%llu phase=%s irq=%u ack=%u released=%u event=%u/%u " ++ "log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d cap=%u end=r73b\n", ++ row->seq - 1, row->t_ns, g5p_r73b_phase_names[phase], row->irq, ++ row->ack, row->released, row->cp[0][0], row->cp[0][1], ++ row->cp[1][0], row->cp[1][1], row->cp[2][0], row->cp[2][1], ++ row->cp[3][0], row->cp[3][1], row->cp[4][0], row->cp[4][1], ++ row->owner, row->result, row->cap_seen); ++} ++ ++/* Snapshot the producer once and consume at most that modular distance ++ * (<= 255). One consumer store returns the whole batch; a producer that ++ * advances meanwhile is left for the next pass, never chased. ++ */ ++static int g5p_r73b_drain_ring(struct g5p_boot *b, u32 ctl_slot, u32 lane, ++ u32 data_slot, u32 stride, u32 ring, ++ u32 *consumed) ++{ ++ u32 ctl = b->layout.offset[ctl_slot] + lane * 0x20; ++ u32 data = b->layout.offset[data_slot] + ++ (ctl_slot == 0x71 ? lane * 0x2c00 : 0); ++ u32 consumer = g5p_r73b_word(b, ctl); ++ u32 producer = g5p_r73b_word(b, ctl + 0x10); ++ u32 count, hash = b->r73_aux_hash[ring] ?: 2166136261U; ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ count = (producer - consumer) & 255; ++ if (!count) ++ return 0; ++ /* Producer index before the slot payloads it covers. */ ++ dma_rmb(); ++ for (u32 n = 0; n < count; n++) { ++ u32 words[G5P_R73B_TRACE_WORDS] = { 0 }; ++ u32 off = data + consumer * stride; ++ ++ for (u32 i = 0; i < stride / 4; i++) { ++ u32 value = g5p_r73b_word(b, off + i * 4); ++ ++ if (i < ARRAY_SIZE(words)) ++ words[i] = value; ++ hash = (hash ^ value) * 16777619U; ++ } ++ if (ring == 5 && ++ atomic_read(&b->r73b->trace_saved) < G5P_R73B_TRACE_ROWS) { ++ u32 saved = atomic_read(&b->r73b->trace_saved); ++ ++ memcpy(b->r73b->trace_rows[saved], words, ++ sizeof(words)); ++ /* Publish this complete row before increasing the visible count. */ ++ smp_wmb(); ++ atomic_inc(&b->r73b->trace_saved); ++ }; ++ ; ++ if (!b->r73_aux_entries[ring]) ++ memcpy(b->r73_aux_first[ring], words, ++ sizeof(b->r73_aux_first[ring])); ++ memcpy(b->r73_aux_last[ring], words, ++ sizeof(b->r73_aux_last[ring])); ++ b->r73_aux_entries[ring]++; ++ consumer = (consumer + 1) & 255; ++ } ++ b->r73_aux_hash[ring] = hash; ++ /* dma_wmb() ordering plus the slot loads above: Apple returns the ++ * consumer only after its reads, with a full barrier. ++ */ ++ dma_mb(); ++ WRITE_ONCE(*g5p_r73b_ptr(b, ctl), consumer); ++ *consumed += count; ++ return 0; ++} ++ ++/* Apple order after the ACK: four log lanes, events, statistics, trace. */ ++static int g5p_r73b_aux_pass(struct g5p_diag *g, u32 *consumed) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret = 0; ++ ++ for (u32 lane = 0; lane < 4 && !ret; lane++) ++ ret = g5p_r73b_drain_ring(b, 0x71, lane, 0x75, 44, lane, ++ consumed); ++ if (!ret) ++ ret = g5p_boot_drain(g); ++ /* R80: ring 4 has one consumer, the drain thread (retains every record). */ ++ if (!ret && !false) ++ ret = g5p_r73b_drain_ring(b, 0x72, 0, 0x76, 44, 4, consumed); ++ if (!ret) ++ ret = g5p_r73b_drain_ring(b, 0x73, 0, 0x77, 36, 5, consumed); ++ return ret; ++} ++ ++static bool g5p_r73b_pending(struct g5p_boot *b, bool aux) ++{ ++ static const u8 slots[] = { 0x70, 0x71, 0x72, 0x73 }; ++ ++ for (u32 i = 0; i < (aux ? ARRAY_SIZE(slots) : 1); i++) { ++ for (u32 lane = 0; lane < (slots[i] == 0x71 ? 4 : 1); lane++) { ++ u32 ctl = b->layout.offset[slots[i]] + lane * 0x20; ++ ++ if (g5p_r73b_word(b, ctl) != ++ g5p_r73b_word(b, ctl + 0x10)) ++ return true; ++ } ++ } ++ return false; ++} ++ ++/* The IRQ thread only trylocks the auxiliary rings; a miss still runs the ++ * spinlock-protected event decoder. Process-context paths lock normally. ++ * Drains with work, errors or sampled entries record before/after rows. ++ */ ++static int g5p_r73b_service(struct g5p_diag *g, u32 owner, bool sampled) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 consumed = 0, before; ++ bool rows, aux; ++ int ret; ++ ++ if (!r) ++ return g5p_boot_drain(g); ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* The atomic r73c window services only the event ring. Auxiliary ++ * traffic is retained until cmd8 is either published or refused. */ ++ aux = 1 && !READ_ONCE(r->fast_window); ++ if (aux && owner == G5P_R73B_OWNER_IRQ && ++ !mutex_trylock(&r->drain_lock)) { ++ atomic_inc(&r->busy); ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BUSY, READ_ONCE(r->owner), ++ -EBUSY); ++ ret = g5p_boot_drain(g); ++ goto out; ++ } ++ if (aux && owner != G5P_R73B_OWNER_IRQ) ++ mutex_lock(&r->drain_lock); ++ if (aux) ++ WRITE_ONCE(r->owner, owner); ++ atomic_inc(&r->drains); ++ rows = sampled || g5p_r73b_pending(b, aux); ++ if (rows) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BEFORE, owner, 0); ++ before = READ_ONCE(b->events); ++ ret = aux ? g5p_r73b_aux_pass(g, &consumed) : g5p_boot_drain(g); ++ consumed += READ_ONCE(b->events) - before; ++ if (rows || ret) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_AFTER, owner, ret ?: consumed); ++ if (aux) { ++ WRITE_ONCE(r->owner, G5P_R73B_NONE); ++ mutex_unlock(&r->drain_lock); ++ } ++out: ++ if (ret) { ++ atomic_inc(&r->drain_errors); ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return ret; ++} ++ ++/* Heartbeat116: an R74 epoch bounds IRQ174 and its ring entries per window. */ ++static bool g5p_hb116_on(struct g5p_boot *b) ++{ ++ return b->r73b && b->r73b->hb_on; ++} ++ ++/* Auxiliary-ring entries consumed so far (RAM, monotonic, may wrap). */ ++static u32 g5p_hb116_aux(struct g5p_boot *b) ++{ ++ u32 sum = 0; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(b->r73_aux_entries); i++) ++ sum += READ_ONCE(b->r73_aux_entries[i]); ++ return sum; ++} ++ ++static u32 g5p_hb116_progress(struct g5p_boot *b) ++{ ++ return READ_ONCE(b->events) + g5p_hb116_aux(b); ++} ++ ++/* The lifetime cap's terminal stop, once: final row, cap_seen, failed, one disable. */ ++static void g5p_hb116_trip(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb *h = &r->hb; ++ u32 prior = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (atomic_cmpxchg(&r->cap_seen, 0, 1)) ++ return; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "HB116 trip why=%s prior_failed=%u windows=%u stalled=%u ms=%llu irqs=%u " ++ "events=%u acks=%u empty=%u aux=%u total_irqs=%u total_events=%u t_ns=%llu " ++ "stop=masked late=ack-only end=hb116\n", ++ g5p_hb_why_name[h->why < G5P_HBW_N ? h->why : 0], prior, ++ h->windows, h->stalled, h->trip.ns / NSEC_PER_MSEC, ++ h->trip.irqs, h->trip.events, h->trip.acks, h->trip.empty, ++ h->trip.aux, n, READ_ONCE(b->events), h->trip_ns); ++} ++ ++/* Receipts only: the first window past each Wake112 lifetime bound; nothing trips here. */ ++static void g5p_hb116_cross(struct g5p_diag *g, u32 irqs, u32 events) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ u32 limit = g5p_transport_event_limit(); ++ ++ if (!(r->hb_cross & 1) && irqs > r->irq_cap) { ++ r->hb_cross |= 1; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=irq-cap value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ r->irq_cap, irqs, events, r->hb.windows); ++ } ++ if (!(r->hb_cross & 2) && events >= limit) { ++ r->hb_cross |= 2; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=event-limit value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ limit, irqs, events, r->hb.windows); ++ } ++} ++ ++/* After the entry's ACK and service: account it, close a window, or trip. */ ++static void g5p_hb116_irq(struct g5p_diag *g, int irq, u32 n, u32 before) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb_win *w = &r->hb.last; ++ u32 seed = r->hb_seed, events = READ_ONCE(b->events); ++ struct g5p_hb_in in; ++ ++ if (g5p_hb116_progress(b) == before) ++ r->hb_empty++; ++ in.now_ns = ktime_get_mono_fast_ns(); ++ in.irqs = n + seed; ++ in.events = events + seed; ++ in.acks = atomic_read(&r->acks) + seed; ++ in.empty = r->hb_empty + seed; ++ in.aux = g5p_hb116_aux(b) + seed; ++ switch (g5p_hb_step(&r->hb, &r->hb_cfg, &in)) { ++ case G5P_HBR_OPEN: ++ case G5P_HBR_CLOSED: ++ break; ++ case G5P_HBR_TRIP: ++ g5p_hb116_trip(g, irq, n); ++ return; ++ } ++ g5p_hb116_cross(g, n, events); ++ if (g5p_hb_report(&r->hb)) ++ g5p_job_printk( ++ g->dev, ++ "HB116 window n=%u t_ns=%llu ms=%llu irqs=%u events=%u acks=%u empty=%u aux=%u " ++ "stalled=%u peak_irqs=%u peak_events=%u total_irqs=%u total_events=%u " ++ "total_acks=%d end=hb116\n", ++ r->hb.windows, in.now_ns, w->ns / NSEC_PER_MSEC, ++ w->irqs, w->events, w->acks, w->empty, w->aux, ++ r->hb.stalled, r->hb.peak_irqs, r->hb.peak_events, n, ++ events, atomic_read(&r->acks)); ++} ++ ++/* Every entry ACKs. Over the cap: ACK, final row, cap_seen, one disable; ++ * the kernel stays alive so the runner can export the record. ++ * Heartbeat116: an R74 epoch has no lifetime cap. The window detector runs ++ * after the ACK and the service; after its trip an entry only ACKs. ++ */ ++static irqreturn_t g5p_r73b_irq(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ bool sampled = g5p_r73b_sampled(n); ++ u32 before = 0; ++ ++ if (r->hb_on && atomic_read(&r->cap_seen)) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ atomic_inc(&r->hb_late); ++ return IRQ_HANDLED; ++ } ++ if (!r->hb_on && n > r->irq_cap) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ if (!atomic_read(&r->cap_seen)) { ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (!atomic_cmpxchg(&r->cap_seen, 0, 1)) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ } ++ } ++ return IRQ_HANDLED; ++ } ++ { ++ /* KPF level6G: Apple ACKs the AIC meta interrupt before ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ } ++ if (sampled) ++ g5p_r73b_crumb(b, G5P_R73B_IRQ, G5P_R73B_OWNER_IRQ, n); ++ if (r->hb_on) ++ before = g5p_hb116_progress(b); ++ if (READ_ONCE(b->released)) ++ g5p_r73b_service(g, G5P_R73B_OWNER_IRQ, sampled); ++ if (1 && READ_ONCE(b->alive)) ++ g5p_r73e_alive_submit(g, G5P_R73B_OWNER_IRQ); ++ ; ++ ; ++ ; ++ ; ++ ; ++ if (r->hb_on) ++ g5p_hb116_irq(g, irq, n, before); ++ return IRQ_HANDLED; ++} ++ ++static void g5p_r73b_disarm(struct g5p_r73b *r) ++{ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 2) == 1) ++ hrtimer_cancel(&r->watchdog); ++} ++ ++static void g5p_r73b_stop(void *data) ++{ ++ g5p_r73b_disarm(data); ++} ++ ++/* Cached CPU RAM only: no MMIO, no locks, then a deliberate panic so a ++ * silent hang becomes a pstore/ramoops record. ++ */ ++static enum hrtimer_restart g5p_r73b_watchdog_fire(struct hrtimer *timer) ++{ ++ struct g5p_r73b *r = container_of(timer, struct g5p_r73b, watchdog); ++ struct g5p_boot *b = r->boot; ++ struct g5p_r73b_row row; ++ u32 total = atomic_read(&r->seq), first, last; ++ char line[320]; ++ bool found = false; ++ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 3) != 1) ++ return HRTIMER_NORESTART; ++ /* A contained cap or a latched failure is terminal: the IRQ only ACKs, ++ * service returns -EIO and the runner sees failed=1. Keep that host ++ * alive for the export instead of turning it into a deliberate panic. ++ */ ++ if (atomic_read(&r->cap_seen) || READ_ONCE(b->failed)) { ++ atomic_set(&r->watchdog_state, 4); ++ dev_emerg( ++ r->dev, ++ "R73B watchdog terminal case=%u irqs=%d acks=%d cap_seen=%d failed=%u rows=%u " ++ "no-panic end=r73b\n", ++ r->case_id, atomic_read(&b->irqs), ++ atomic_read(&r->acks), atomic_read(&r->cap_seen), ++ READ_ONCE(b->failed), total); ++ return HRTIMER_NORESTART; ++ } ++ dev_emerg( ++ r->dev, ++ "R73B receipt case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u release=%s end=r73b\n", ++ r->case_id, g5p_r73b_name(r->case_id), r->ack_first, r->aux, ++ r->irq_cap, r->watchdog_ms, r->dc_variant, r->meta6c8, ++ init_utsname()->release); ++ first = total > G5P_R73B_PANIC_ROWS ? total - G5P_R73B_PANIC_ROWS : 0; ++ for (u32 seq = first; seq < total; seq++) { ++ if (!g5p_r73b_copy_row(r, seq, &row)) ++ continue; ++ g5p_r73b_format(line, sizeof(line), &row); ++ dev_emerg(r->dev, "%s", line); ++ } ++ for (last = total; last > first && !found; last--) ++ found = g5p_r73b_copy_row(r, last - 1, &row); ++ if (!found) ++ memset(&row, 0, sizeof(row)); ++ panic("G5P R73B watchdog: release epoch incomplete after %ums case=%u irqs=%d acks=%d " ++ "cap_seen=%d busy=%d rows=%u last seq=%u phase=%s irq=%u ack=%u released=%u " ++ "event=%u/%u log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d " ++ "cap=%u end=r73b\n", ++ 0, 1, atomic_read(&b->irqs), atomic_read(&r->acks), ++ atomic_read(&r->cap_seen), atomic_read(&r->busy), total, ++ row.seq ? row.seq - 1 : 0, ++ g5p_r73b_phase_names[row.phase < G5P_R73B_PHASES ? row.phase : 0], ++ row.irq, row.ack, row.released, row.cp[0][0], row.cp[0][1], ++ row.cp[1][0], row.cp[1][1], row.cp[2][0], row.cp[2][1], ++ row.cp[3][0], row.cp[3][1], row.cp[4][0], row.cp[4][1], row.owner, ++ row.result, row.cap_seen); ++ return HRTIMER_NORESTART; ++} ++ ++static void g5p_r73b_release_entry(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ ++ if (!r) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_ENTRY, G5P_R73B_NONE, 0); ++ { ++ return; ++ } ++} ++ ++static void g5p_r73b_release_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_EXIT, G5P_R73B_NONE, ret); ++ if (ret) ++ g5p_r73b_disarm(b->r73b); ++} ++ ++/* The release epoch ends when the host has returned from the ALIVE wait. */ ++static void g5p_r73b_events_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_EVENTS_EXIT, G5P_R73B_OWNER_EVENTS, ret); ++ g5p_r73b_disarm(b->r73b); ++} ++ ++static int g5p_r73b_latch(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const u32 id = (1), ms = (0); ++ const u32 fast = (1); ++ const u32 early = (1); ++ const u32 r74 = (1), cap = (1048576); ++ const u32 r74f = (1); ++ const u32 r74g = (1); ++ const u32 r74h = (0); ++ ++ struct g5p_r73b *r; ++ int ret; ++ ++ ; ++ ; ++ ; ++ ; ++ ; ++ ; ++ /* nopatch replaces r73f case 1 under the whole r74/r74f/r74g path, and nothing else. */ ++ /* nopatch's receipts and its r74 gate are the TYPED_V3 path's; any other path (r74 0) runs ++ * Apple's image anyway (only r73f patches it) and gets neither. ++ */ ++ if (1 != g5p_fw_unpatched_want((1), 0, 0, 0) || ++ (1 && r74 == 1 && (0 || r74f != 1 || r74g != 1))) { ++ dev_err(g->dev, ++ "NOPATCH reject phase=map unpatched=%u/%u r73f_case=%u path=%u/%u/%u end=nopatch\n", ++ 1, (1), 0, r74, r74f, r74g); ++ return -EINVAL; ++ } ++ /* R74 rides only on an exact r73f case 1 (or nopatch) or 4 with the raised limits. */ ++ if (r74 && (!id || r74 > 2 || 0 != (r74 == 1 ? (1 ? 0 : 1) : 4) || ++ 1 != (r74 == 1) || !(1) || cap < G5P_R74_IRQ_CAP || ++ cap > G5P_R74_IRQ_CAP_MAX || ++ g5p_transport_event_limit() < G5P_R74_EVENT_FLOOR)) { ++ dev_err(g->dev, ++ "R74 reject phase=map case=%u r73b_case=%u r73f_case=%u meta6c8=%u " ++ "render_engine=%u irq_cap=%u event_limit=%u end=r74\n", ++ r74, id, 0, b->r72_meta6c8, (1), cap, ++ g5p_transport_event_limit()); ++ return -EINVAL; ++ } ++ ++ /* R74f rides only on the exact r74 case 1 forced-ready epoch. */ ++ ; ++ ++ /* R74g rides only on the exact r74f case 1 epoch. */ ++ ; ++ ++ /* R74h observes only the exact r74g case 1 epoch. */ ++ ; ++ ++ /* The bounded serialized drain supersedes r73's IRQ-context drain. */ ++ ; ++ ; ++ ; ++ ; ++ if (id > 3 || fast > 1 || (fast && id == 3) || early > 2 || ++ (early && (!fast || id != early)) || b->r73_aux_drain || 7 != 7 || ++ 1 != (id == 2 ? 0 : 1) || (ms && (ms < 1000 || ms > 30000))) ++ return -EINVAL; ++ r = devm_kzalloc(g->dev, sizeof(*r), GFP_KERNEL); ++ if (!r) ++ return -ENOMEM; ++ r->dev = g->dev; ++ r->boot = b; ++ r->case_id = id; ++ r->watchdog_ms = ms; ++ r->dc_variant = 7; ++ r->meta6c8 = 1; ++ r->aux = id != 3; ++ r->ack_first = true; ++ r->fast_enabled = fast; ++ r->r73e_case = early; ++ r->r74_case = r74; ++ r->r74f_case = r74f; ++ r->r74g_case = r74g; ++ r->r74h_case = r74h; ++ r->settle_us = min_t(u32, READ_ONCE(boot_alive_settle_us), 5000); ++ r->nopatch = 1 && r74 == 1; ++ r->irq_cap = r74 ? cap : G5P_R73B_IRQ_CAP; ++ /* Heartbeat116: in an R74 epoch the window detector replaces both lifetime bounds. */ ++ r->hb_on = r74 != 0; ++ r->hb_seed = r74 ? (0) : 0; ++ g5p_hb_cfg_default(&r->hb_cfg); ++ atomic_set(&r->early_once, 0); ++ atomic_set(&r->early_busy, 0); ++ atomic_set(&r->early_proof, G5P_PX_NONE); ++ r->watchdog_cpu = -1; ++ mutex_init(&r->drain_lock); ++ hrtimer_setup(&r->watchdog, g5p_r73b_watchdog_fire, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL_PINNED_HARD); ++ ret = devm_add_action_or_reset(g->dev, g5p_r73b_stop, r); ++ if (ret) ++ return ret; ++ b->r73_aux_drain = 1; ++ b->r73b = r; ++ dev_info( ++ g->dev, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), r->ack_first, r->aux, r->irq_cap, ms, ++ r->dc_variant, r->meta6c8, G5P_R73B_ROWS, ++ init_utsname()->release); ++ g5p_blob_printf( ++ g, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), 1, 1, r->irq_cap, ms, 7, 1, ++ G5P_R73B_ROWS, init_utsname()->release); ++ g5p_blob_printf(g, "R73C latched fast=%u case=%u meta6c8=%u end=r73c\n", ++ 1, id, 1); ++ dev_info(g->dev, "R73E latched case=%u meta6c8=%u fast=%u end=r73e\n", ++ early, r->meta6c8, r->fast_enabled); ++ ; ++ ; ++ if (r->nopatch) ++ dev_info( ++ g->dev, ++ "NOPATCH latched fw=apple-original r73f_case=0 meta6c8=%u r73e_case=%u r74=%u " ++ "r74f=%u r74g=%u force_policy_ready=0 end=nopatch\n", ++ r->meta6c8, early, r74, r74f, r74g); ++ { ++ dev_info( ++ g->dev, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, r->meta6c8, ++ r->irq_cap, g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, 1, r->irq_cap, ++ g5p_transport_event_limit()); ++ dev_info( ++ g->dev, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ g5p_blob_printf( ++ g, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ g5p_blob_printf( ++ g, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ }; ++ /* TYPED_V3 production path: the self-drain DC3 attach policy of r87b rides on r74f + r74g ++ * alone, with none of the r74h / r80 instrumentation. r74g has the firmware own the first ++ * MMU pass, which is what makes self-drain safe (r87b-attach.h); r74h only observes. Every ++ * other r8x / r9x knob is still refused below without boot_r80_case. ++ */ ++ { ++ r->r87b_attach = G5P_R87B_ATTACH_SELF_DRAIN; ++ dev_info( ++ g->dev, ++ "R87B latched policy=1 name=self-drain r74f=%u r74g=%u r74h=%u r80=0 " ++ "production_path=1 end=r87b\n", ++ r74f, r74g, r74h); ++ } ++ { ++ { ++ { ++ { ++ { ++ if ((1) && !r->r87b_attach) { ++ dev_err(g->dev, ++ "R87B reject phase=map r87b knob without boot_r80_case policy=%u end=r87b\n", ++ (1)); ++ return -EBADMSG; ++ } else { ++ { ++ { ++ ; ++ ; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ } ++ return 0; ++} ++ ++/* Process context, cached CPU RAM only; never waits for the command lock. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +new file mode 100644 +index 000000000..71cf79dbe +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -0,0 +1,532 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74f deferred doorbell. Included by boot.inc before the event decoder. ++ * Diagnostic bypass on the r74 forced-ready epoch only; not power policy. ++ * ++ * A DC3 pair or a natural TA is published once, append-only, before this ++ * code runs; its producer is never rewound, republished or torn down. If ++ * the immediate functional gate finds GPC inactive, that generation stays ++ * pending (never b->failed), the host repeats the r74e release transition ++ * once, and the first type-0 ALIVE decoded afterwards owns one doorbell ++ * attempt. The owner runs under event_lock: producer confirm, R1+14 fence, ++ * active check, dma_wmb(), doorbell 0x10. Receipts stay in RAM; console ++ * lines are printed afterwards from process context. ++ */ ++ ++static const char *const g5p_r74f_kinds[G5P_R74F_KINDS] = { "dc3", "ta", "3d" }; ++ ++static const char *const g5p_r74f_states[G5P_R74F_STATES] = { ++ "idle", "published", "open", "firing", "rang", "cancelled", ++}; ++ ++static const char *const g5p_r74f_cancels[G5P_R74F_CANCELS] = { ++ "none", "reactivation-failed", ++ "no-alive", "consumed-before-bell", ++ "protocol", "alive-during-reactivation", ++ "self-drain", ++}; ++ ++static const char *const g5p_r74f_stops[G5P_R74F_STOPS] = { ++ "none", ++ "attach-reactivation-failed", ++ "attach-published-not-kicked", ++ "attach-kicked-not-consumed", ++ "first-mmu-window-closed", ++ "ta-reactivation-failed", ++ "ta-published-not-kicked", ++ "firmware-dc3-tag-mismatch", ++ "dc3-ram-validation-refused", ++ "firmware-companion-recovery", ++ "ta-published-not-consumed", ++ "ta-window-missed", ++}; ++ ++static const char *const g5p_r74f_paths[G5P_R74F_PATHS] = { ++ "none", ++ "immediate", ++ "deferred", ++}; ++ ++static bool g5p_r74f_on(struct g5p_boot *b) ++{ ++ return b && b->r73b && 1; ++} ++ ++static u32 *g5p_r74f_ctl(struct g5p_boot *b, u32 kind) ++{ ++ return (u32 *)g5p_boot_shared(b, kind == G5P_R74F_TA ? 0x60 : ++ kind == G5P_R74F_3D ? 0x64 : ++ 0x6c); ++} ++ ++/* Owner only, event_lock held with interrupts off. Between the active check ++ * and the doorbell there is only dma_wmb(): no console, no other MMIO. ++ */ ++static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ u32 *fence, u32 *state) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 consumer = READ_ONCE(ctl[0]); ++ u32 producer = smp_load_acquire(&ctl[4]); ++ u32 queued = (f->target - f->start) & 255; ++ u32 progressed = (consumer - f->start) & 255; ++ u32 expected = f->kind == G5P_R74F_DC3 ? 2 : 1; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* Do not turn corrupted modulo-256 ring state into a doorbell. */ ++ if (consumer > 255 || producer > 255 || queued != expected || ++ progressed > queued) ++ return -EPROTO; ++ /* Acquire: pairs with the producer store published before the state. */ ++ if (producer != f->target) ++ return -EPROTO; ++ if (consumer == f->target) ++ return -EALREADY; ++ writel(7, g->window[1] + 0x14); ++ *fence = readl(g->window[1] + 0x14); ++ if (*fence & ~7U) ++ return -EPROTO; ++ *state = readl(g->window[1] + 0x1000); ++ /* perf60 (gpc_any_pstate): a GPC request in flight or a not-yet-awake GPC after the wake ++ * above; re-read for up to 2 ms, as Apple's setGPCPerformanceState waits for bit 4. Reads only. ++ */ ++ { ++ u64 t0 = ktime_get_mono_fast_ns(); ++ ++ while (!g5p_gpc_ok(*state) && ktime_get_mono_fast_ns() - t0 < ++ 2000ULL * NSEC_PER_USEC) { ++ udelay(10); ++ *state = readl(g->window[1] + 0x1000); ++ } ++ } ++ if (!g5p_gpc_ok(*state)) ++ return -EHOSTDOWN; ++ dma_wmb(); ++ writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); ++ return 0; ++} ++ ++/* Owner only: record the attempt, then publish it with the state release. */ ++static void g5p_r74f_rang(struct g5p_r73b *r, struct g5p_r74f_bell *f, int ret, ++ u32 fence, u32 state, u32 path) ++{ ++ f->bell_ns = ktime_get_mono_fast_ns(); ++ f->path = path; ++ f->fence = fence; ++ f->bell_state = state; ++ f->bell_ret = ret; ++ if (!ret) { ++ f->kicked = 1; ++ atomic_inc(&r->r74f_bells); ++ } else if (ret == -EALREADY) { ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ } else if (ret != -EHOSTDOWN) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ } ++ atomic_set_release(&f->state, G5P_R74F_RANG); ++} ++ ++/* g5p_boot_drain(), event_lock held: one type-0 ALIVE was just decoded. */ ++static void g5p_r74f_alive(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ struct g5p_r74f_bell *f; ++ u32 kind, fence = U32_MAX, state = U32_MAX; ++ int ret; ++ ++ WRITE_ONCE(r->r74f_alive, r->r74f_alive + 1); ++ kind = READ_ONCE(r->r74f_active); ++ if (!kind) ++ return; ++ f = &r->r74f_bell[kind - 1]; ++ if (atomic_read_acquire(&f->state) != G5P_R74F_OPEN || ++ r->r74f_alive <= f->open_alive) ++ return; ++ /* Sole owner: exactly one decoder moves OPEN to FIRING. */ ++ if (atomic_cmpxchg(&f->state, G5P_R74F_OPEN, G5P_R74F_FIRING) != ++ G5P_R74F_OPEN) ++ return; ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_DEFERRED); ++} ++ ++static int g5p_r74f_poll(struct g5p_diag *g, u32 alive, u32 expected, ++ u32 timeout_us, u64 *elapsed_ns, u32 *raw) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u64 start = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ *raw = U32_MAX; ++ for (;;) { ++ if (READ_ONCE(g->boot->failed)) { ++ ret = -EIO; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ /* Serialize the ordinal check and exactly one R1 read with the ++ * decoder. Once ALIVE advances, no further R1 MMIO is licensed. ++ */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ break; ++ } ++ *raw = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if ((*raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ return ret; ++} ++ ++/* Absolute freshness budget from the DC3 doorbell. The first MMU pass may ++ * touch R0 only while the same active window is still demonstrably fresh. ++ */ ++ ++/* The r74e release transition, once per generation, on r74f receipts and ++ * without console output: core request clear, GPC 0 within 1.0 ms, GPC 1 ++ * within 1.8 ms. Every R1 access is serialized against ALIVE decoding; ++ * ordinal advance cancels the generation before any further R1 MMIO. ++ */ ++static int g5p_r74f_reactivate(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u32 alive = f->arm_alive; ++ int ret; ++ ++ f->reactivated = 1; ++ f->power_start_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ f->power_ret = -EHOSTDOWN; ++ return -EHOSTDOWN; ++ } ++ writel(0, g->window[1] + 0x20000); ++ f->core_fence = readl(g->window[1] + 0x20000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, 0, G5P_R74F_OFF_US, &f->off_ns, ++ &f->off_raw); ++ if (!ret) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ goto out; ++ } ++ /* boot_gpu_perf_seed: the firmware believes pstate q; never drop the GPC under it */ ++ writel(0x10 | (b->perf_poweron ?: ++ b->perf_seed ?: ++ 1), ++ g->window[1] + 0x1000); ++ f->gpc_fence = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, ++ b->perf_poweron ?: ++ b->perf_seed ?: ++ 1, ++ G5P_R74F_ON_US, &f->on_ns, &f->on_raw); ++ } ++out: ++ f->power_ret = ret; ++ return ret; ++} ++ ++/* Process context, before any publication: no overlap with a pending ring. */ ++static int g5p_r74f_admit(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *dc = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ ++ if (!g5p_r74b_ram_only(g) || READ_ONCE(b->failed) || ++ READ_ONCE(r->r74f_stop) || ++ atomic_read(&r->r74f_bell[kind].state) != G5P_R74F_IDLE) ++ return -EPERM; ++ if (kind == G5P_R74F_DC3) ++ return READ_ONCE(r->r74f_active) ? -EBUSY : 0; ++ /* M7: the TA wakeup never shares a doorbell with a pending DC3 pair. */ ++ dma_rmb(); ++ if (READ_ONCE(r->r74f_active) != G5P_R74F_DC3 + 1 || ++ !r->r74f_bell[G5P_R74F_DC3].complete || ++ READ_ONCE(dc[0]) != READ_ONCE(dc[4])) ++ return -EBUSY; ++ return 0; ++} ++ ++static void g5p_r74f_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 state = atomic_read_acquire(&f->state); ++ char line[3][512]; ++ ++ dma_rmb(); ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74F bell kind=%s generation=%u ring=%02x start=%u target=%u state=%s path=%s " ++ "first_ret=%d first_fence=%08x first_state=%08x attempted=%u fence=%08x " ++ "bell_state=%08x bell_ret=%d kicked=%u bells=%d cancel=%s complete=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->ring, f->start, ++ f->target, g5p_r74f_states[state < G5P_R74F_STATES ? state : 0], ++ g5p_r74f_paths[f->path], f->first_ret, f->first_fence, ++ f->first_state, f->attempted, f->fence, f->bell_state, ++ f->bell_ret, f->kicked, atomic_read(&b->r73b->r74f_bells), ++ g5p_r74f_cancels[f->cancel], f->complete); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74F ring kind=%s generation=%u arm_alive=%u open_alive=%u bell_alive=%u " ++ "alive_now=%u consumer_before=%u consumer_after=%u producer_after=%u " ++ "consumer_now=%u producer_now=%u failed=%u event_failed=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->arm_alive, ++ f->open_alive, f->bell_alive, READ_ONCE(b->r73b->r74f_alive), ++ f->consumer_before, f->consumer_after, f->producer_after, ++ READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), READ_ONCE(b->failed), ++ READ_ONCE(b->event_failed)); ++ scnprintf( ++ line[2], sizeof(line[2]), ++ "R74F power kind=%s generation=%u reactivated=%u core_fence=%08x gpc_fence=%08x " ++ "off_ns=%llu on_ns=%llu off_raw=%08x on_raw=%08x power_ret=%d arm_ns=%llu " ++ "power_start_ns=%llu open_ns=%llu bell_start_ns=%llu bell_ns=%llu done_ns=%llu " ++ "end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->reactivated, ++ f->core_fence, f->gpc_fence, f->off_ns, f->on_ns, f->off_raw, ++ f->on_raw, f->power_ret, f->arm_ns, f->power_start_ns, ++ f->open_ns, f->bell_start_ns, f->bell_ns, f->done_ns); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++/* r87b receipt: the DC3 attach policy and what the host did after the publish. ++ * RAM reads only; one line per DC3 generation (console at the consume/timeout ++ * point, blob at export). ++ */ ++static void g5p_r87b_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 policy = READ_ONCE(b->r73b->r87b_attach); ++ char line[640]; ++ ++ dma_rmb(); ++ scnprintf( ++ line, sizeof(line), ++ "R87B attach policy=%u name=%s generation=%u gate_first_ret=%d " ++ "gate_first_state=%08x gate_closed=%u path=%s cancel=%s bell_rung=%u bells=%d " ++ "reactivated=%u cons_before=%u prod_before=%u prod_pub=%u cons_after=%u " ++ "prod_after=%u consumed=%u arm_alive=%u alive_done=%u alive_delta=%u " ++ "pub_to_done_ns=%llu irqs=%d events=%u end=r87b\n", ++ policy, policy ? "self-drain" : "original", f->generation, ++ f->first_ret, f->first_state, f->first_ret == -EHOSTDOWN, ++ g5p_r74f_paths[f->path], g5p_r74f_cancels[f->cancel], f->kicked, ++ atomic_read(&b->r73b->r74f_bells), f->reactivated, ++ f->consumer_before, f->start, f->target, f->consumer_after, ++ f->producer_after, f->complete, f->arm_alive, f->alive_done, ++ f->alive_done - f->arm_alive, ++ f->done_ns >= f->arm_ns ? f->done_ns - f->arm_ns : 0, ++ atomic_read(&b->irqs), READ_ONCE(b->events)); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* A retained boundary: the publication stays, nothing is failed. */ ++static int g5p_r74f_stop(struct g5p_diag *g, u32 stop, int value) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ ++ r->r74f_stop_value = value; ++ WRITE_ONCE(r->r74f_stop, stop); ++ g5p_r74_mark(g, G5P_R74F_STOP, stop); ++ dev_emerg(g->dev, "R74F stop reason=%s value=%d failed=%u end=r74f\n", ++ g5p_r74f_stops[stop], value, READ_ONCE(g->boot->failed)); ++ return -EINPROGRESS; ++} ++ ++/* After the reactivation: open the generation, wait (bounded) for the ++ * ALIVE owner while the event drain stays alive, then cancel if unrung. ++ */ ++static int g5p_r74f_open(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags, deadline; ++ int ret; ++ ++ ret = g5p_r74f_reactivate(g, f); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (ret) { ++ f->cancel = ret == -EHOSTDOWN ? G5P_R74F_CANCEL_ALIVE_ADVANCED : ++ G5P_R74F_CANCEL_REACTIVATION; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ f->open_alive = r->r74f_alive; ++ f->open_ns = ktime_get_mono_fast_ns(); ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (ret) ++ return 0; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ /* The IRQ path is the sole event consumer and bell owner. */ ++ if (READ_ONCE(b->failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* Process context, command mutex held, producer already published. Returns ++ * 0 after one doorbell, 1 if consumed without one, -EINPROGRESS when the ++ * publication is retained unrung, else a fatal protocol error. ++ */ ++static int g5p_r74f_bell(struct g5p_diag *g, u32 kind, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(b, kind); ++ u32 fence = U32_MAX, state = U32_MAX, now; ++ unsigned long flags; ++ int ret, drained = 0; ++ bool ta = kind == G5P_R74F_TA; ++ bool self_drain; ++ ++ if (kind == G5P_R74F_3D) ++ return -EINVAL; /* the 3D bell is armed by g5p_r96_3d_arm(): never the DC3 / self-drain path */ ++ f->kind = kind; ++ f->ring = ta ? 0x60 : 0x6c; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ /* The immediate functional gate r74e used, without console output. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, kind + 1); ++ atomic_set(&f->state, G5P_R74F_FIRING); ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ f->first_ret = ret; ++ f->first_fence = fence; ++ f->first_state = state; ++ /* r87b self-drain: a closed DC3 gate is only a phase sample of the ++ * firmware's GPC window. The host never repairs it: no R1+20000 clear, no ++ * GPC off/on, no bell. The caller waits RAM-only for the firmware to ++ * consume the pair. Only the DC3 kind; an active gate rang above. ++ */ ++ self_drain = ret == -EHOSTDOWN && kind == G5P_R74F_DC3 && ++ READ_ONCE(r->r87b_attach) == G5P_R87B_ATTACH_SELF_DRAIN; ++ if (self_drain) { ++ f->cancel = G5P_R74F_CANCEL_SELF_DRAIN; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (ret == -EHOSTDOWN) { ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ } else { ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_IMMEDIATE); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (self_drain) { ++ drained = 1; ++ g5p_r74_mark(g, G5P_R74F_ATTACH_ARMED, state); ++ } else if (ret == -EHOSTDOWN) { ++ drained = g5p_r74f_open(g, f); ++ /* Functional reactivation precedes all console/milestone output. */ ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_ARMED : G5P_R74F_ATTACH_ARMED, ++ state); ++ g5p_r74_mark(g, ++ ta ? G5P_R74F_TA_REACTIVATED : ++ G5P_R74F_ATTACH_REACTIVATED, ++ f->power_ret); ++ } ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_BELL : G5P_R74F_ATTACH_BELL, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ if (drained) ++ return drained; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_REACTIVATION || ++ f->cancel == G5P_R74F_CANCEL_ALIVE_ADVANCED) ++ return g5p_r74f_stop( ++ g, ++ ta ? G5P_R74F_STOP_TA_REACTIVATION : ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ f->power_ret); ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* RAM only: the consumer this generation reached, at consume or timeout. */ ++static void g5p_r74f_consumed(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_r74f_bell *f = &g->boot->r73b->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(g->boot, kind); ++ ++ dma_rmb(); ++ f->consumer_after = READ_ONCE(ctl[0]); ++ f->producer_after = READ_ONCE(ctl[4]); ++ f->complete = f->consumer_after == f->target; ++ f->done_ns = ktime_get_mono_fast_ns(); ++ f->alive_done = READ_ONCE(g->boot->r73b->r74f_alive); ++ if (kind == G5P_R74F_DC3) ++ g5p_r87b_line(g, f, false); ++} ++ ++/* g5p_r22_mmu(): attach's first pass only; records the gate it already read. */ ++ ++/* g5p_r22_command(): an r74f boundary keeps the epoch, drain and backing. */ ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74g.inc b/drivers/gpu/apple-g5p/boot-r74g.inc +new file mode 100644 +index 000000000..6ff269317 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74g.inc +@@ -0,0 +1,288 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74g. Included by boot.inc after boot-r74f.inc; r74f case 1 only. It is a ++ * diagnostic bypass on the forced-ready epoch, not power policy. ++ * ++ * The firmware DC3 handler (0x1000218c..0x10002218) writes e140/e148 for ++ * slot N, copies host_generation[N] to cached_generation[N], then issues ++ * the e138 (N << 3) | 4 catalogue invalidation and waits for it. The DC ++ * dispatcher advances the consumer only after the handler returns, so a ++ * consumed DC3 pair is the completion receipt. r74g checks the RAM-visible ++ * results and issues no host MMU MMIO. The natural TA is then published ++ * without a doorbell. A RAM-only grace (four type-0 ALIVEs, 20 ms cap) ++ * lets the firmware drain it on its own. If it stays pending, the r74f ++ * ALIVE owner gets one fenced fallback doorbell. Receipts print afterwards. ++ */ ++ ++/* perf60: with the policy held in state 0 the firmware is quiet (no ALIVE stream, no self-drain), so ++ * job 1's TA bell, owned by "the first ALIVE after the grace", would never ring. ++ */ ++/* release8 default */ ++ ++static const char *const g5p_r74g_graces[G5P_R74G_GRACES] = { ++ "none", "consumed", "alives", "wall", "fault", ++}; ++ ++static bool g5p_r74g_on(struct g5p_boot *b) ++{ ++ return g5p_r74f_on(b) && 1; ++} ++ ++static u32 g5p_r74g_root(struct g5p_boot *b, u32 off) ++{ ++ return g5p_le32(b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r74g_shared(struct g5p_boot *b, u32 at) ++{ ++ return g5p_le32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE); ++} ++ ++/* RAM only: the DC3 ring, both records in attach order (3D, then TA), the ++ * PA[N] entries, host and cached tags, both failure latches and the ++ * root+3980 companion word, whose value 2 is firmware recovery/escape. ++ */ ++static u32 g5p_r74g_check(struct g5p_diag *g, u32 phase, const u32 *slot, ++ u64 pa, u32 tag, u32 start) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r74g_dc3 *d = &b->r73b->r74g_dc3[phase]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ u32 bad = 0; ++ ++ dma_rmb(); ++ d->start = start; ++ d->target = (start + 2) & 255; ++ d->tag = tag; ++ d->expected_pa = pa; ++ d->consumer = READ_ONCE(ctl[0]); ++ d->producer = READ_ONCE(ctl[4]); ++ if (d->consumer != d->target || d->producer != d->target) ++ bad |= G5P_R74G_BAD_RING; ++ for (u32 i = 0; i < 2; i++) { ++ u32 at = b->layout.offset[0x6d] + ((start + i) & 255) * 20; ++ ++ for (u32 j = 0; j < 5; j++) { ++ d->rec[i][j] = g5p_r74g_shared(b, at + j * 4); ++ if (d->rec[i][j] != (j == 0 ? 3 : ++ j == 1 ? slot[1 - i] : ++ 0)) ++ bad |= G5P_R74G_BAD_RECORD; ++ } ++ } ++ for (u32 e = 0; e < 2; e++) { ++ u8 *p = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ ++ d->slot[e] = slot[e]; ++ d->pa[e] = (u64)g5p_le32(p + 4) << 32 | g5p_le32(p); ++ d->host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + slot[e] * 4); ++ d->cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + slot[e] * 4); ++ if (d->pa[e] != pa) ++ bad |= G5P_R74G_BAD_PA; ++ if (d->host[e] != tag) ++ bad |= G5P_R74G_BAD_HOST; ++ if (d->cached[e] != tag) ++ bad |= G5P_R74G_BAD_CACHED; ++ } ++ d->failed = READ_ONCE(b->failed); ++ d->event_failed = READ_ONCE(b->event_failed); ++ if (d->failed || d->event_failed) ++ bad |= G5P_R74G_BAD_FAILED; ++ d->companion = g5p_r74g_root(b, G5P_R74G_COMPANION); ++ d->recoveries = g5p_r74g_root(b, G5P_R74G_RECOVERIES); ++ d->classification = g5p_r74g_root(b, G5P_R74G_CLASS); ++ if (d->companion == 2) ++ bad |= G5P_R74G_BAD_COMPANION; ++ d->bad = bad; ++ d->t_ns = ktime_get_mono_fast_ns(); ++ return bad; ++} ++ ++/* A refused check is retained, never b->failed: companion 2 is firmware ++ * recovery, a cached-tag-only mismatch is the firmware tag copy, and ++ * anything else (a host tag included) is a refused RAM validation. ++ */ ++static int g5p_r74g_refuse(struct g5p_diag *g, u32 bad) ++{ ++ if (bad & G5P_R74G_BAD_COMPANION) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_COMPANION, bad); ++ if (bad == G5P_R74G_BAD_CACHED) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_TAGS, bad); ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_RAM, bad); ++} ++ ++/* Two bounded lines: every field at full width still ends in end=r74g. */ ++static void g5p_r74g_dc3_line(struct g5p_diag *g, u32 phase, bool blob) ++{ ++ const struct g5p_r74g_dc3 *d = &g->boot->r73b->r74g_dc3[phase]; ++ const char *name = phase ? "ta-preflight" : "dc3-consume"; ++ char line[2][512]; ++ ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74G dc3 phase=%s bad=%02x start=%u target=%u consumer=%u producer=%u tag=%08x " ++ "slot=%u/%u host=%08x/%08x cached=%08x/%08x failed=%u event_failed=%u " ++ "companion=%u recoveries=%u class=%u host_mmu=skipped t_ns=%llu end=r74g\n", ++ name, d->bad, d->start, d->target, d->consumer, d->producer, ++ d->tag, d->slot[0], d->slot[1], d->host[0], d->host[1], ++ d->cached[0], d->cached[1], d->failed, d->event_failed, ++ d->companion, d->recoveries, d->classification, d->t_ns); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74G dc3-ram phase=%s pa=%llx/%llx expected_pa=%llx rec0=%x,%x,%x,%x,%x " ++ "rec1=%x,%x,%x,%x,%x end=r74g\n", ++ name, d->pa[0], d->pa[1], d->expected_pa, d->rec[0][0], ++ d->rec[0][1], d->rec[0][2], d->rec[0][3], d->rec[0][4], ++ d->rec[1][0], d->rec[1][1], d->rec[1][2], d->rec[1][3], ++ d->rec[1][4]); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++static void g5p_r74g_ta_line(struct g5p_diag *g, bool blob) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ const struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 end = r->r74g_grace_end; ++ char line[256]; ++ ++ scnprintf( ++ line, sizeof(line), ++ "R74G ta grace_end=%s arm_alive=%u grace_alive=%u grace_ns=%llu opened=%u " ++ "open_alive=%u ack_skipped=%u end=r74g\n", ++ g5p_r74g_graces[end < G5P_R74G_GRACES ? end : 0], f->arm_alive, ++ r->r74g_grace_alive, r->r74g_grace_ns - f->arm_ns, ++ r->r74g_opened, f->open_alive, r->r74g_ack_skipped); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* Process context, command mutex held, TA producer already published once. ++ * Grace: RAM reads and sleeps only, no MMIO and no doorbell, while the IRQ ++ * thread keeps decoding. It ends at the consume, four type-0 ALIVEs after ++ * the arm, or the 20 ms wall cap. If the TA is still pending, the ++ * generation opens, and the first ALIVE decoded after that owns the single ++ * r74f fallback attempt. Returns 0 after one doorbell, 1 if consumed without ++ * one, -EINPROGRESS for a retained boundary, else a fatal error. ++ */ ++static int g5p_r74g_ta(struct g5p_diag *g, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_TA); ++ unsigned long flags, deadline; ++ u32 end = G5P_R74G_GRACE_NONE, now; ++ int ret = 0; ++ ++ f->kind = G5P_R74F_TA; ++ f->ring = 0x60; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, G5P_R74F_TA + 1); ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ while (end == G5P_R74G_GRACE_NONE) { ++ ; ++ dma_rmb(); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) ++ end = G5P_R74G_GRACE_FAULT; ++ else if (READ_ONCE(ctl[0]) == target) ++ end = G5P_R74G_GRACE_CONSUMED; ++ else if (READ_ONCE(r->r74f_alive) - f->arm_alive >= ++ G5P_R74G_GRACE_ALIVES) ++ end = G5P_R74G_GRACE_ALIVES_END; ++ else if (ktime_get_mono_fast_ns() - f->arm_ns >= ++ (u64)G5P_R74G_GRACE_US * 1000) ++ end = G5P_R74G_GRACE_WALL; ++ else ++ usleep_range(100, 200); ++ } ++ spin_lock_irqsave(&b->event_lock, flags); ++ r->r74g_grace_end = end; ++ r->r74g_grace_alive = r->r74f_alive; ++ r->r74g_grace_ns = ktime_get_mono_fast_ns(); ++ dma_rmb(); ++ if (end == G5P_R74G_GRACE_FAULT) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (READ_ONCE(ctl[0]) == target) { ++ /* Drained by the firmware policy loop: never ring. */ ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ r->r74g_opened = 1; ++ f->open_alive = r->r74f_alive; ++ f->open_ns = r->r74g_grace_ns; ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ /* ta1_host_bell: the host owns it instead, in the same lock section a decoder would. */ ++ if ((1) && atomic_cmpxchg(&f->state, G5P_R74F_OPEN, ++ G5P_R74F_FIRING) == G5P_R74F_OPEN) { ++ u32 fence = U32_MAX, state = U32_MAX; ++ int rung; ++ ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ rung = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, rung, fence, state, ++ G5P_R74F_PATH_IMMEDIATE); ++ } ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (end == G5P_R74G_GRACE_FAULT) ++ ret = -EIO; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, G5P_R74G_TA_GRACE, end); ++ if (r->r74g_opened) ++ g5p_r74_mark(g, G5P_R74G_TA_FALLBACK, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ g5p_r74g_ta_line(g, false); ++ if (ret) ++ return ret; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_CONSUMED) ++ return 1; ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_NOT_CONSUMED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_WINDOW_MISSED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +new file mode 100644 +index 000000000..6a62f2eb0 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -0,0 +1,1665 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Experimental G2 stages. Included after the bounded META slave primitives. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "boot-layout.h" ++#include "render-user-pmset.h" ++#include "render-user-recover.h" ++#include "fender.h" ++#include "readback.h" ++#include "meta71c114.h" ++#include "latency123.h" ++#include "perf-envelope.h" ++ ++/* Boot deltas are selected before map; no live render-time reconfiguration. */ ++ ++ ++ ++/* R46 (diag-fable51 T7): the first SOFT_RESET write of the host reset FIFO ++ * also asserts bits 56-59, VBS/VCE/TE3/TILING_CORE in the PBE2_IN_XE ++ * layout. Apple's mask 00e7fffffffffc1d is exactly the other layout's ++ * MASKFULL, in which those bits do not exist and TE/GPP/UVS/TLA/PM/VDM are ++ * already asserted; expected to be a no-op. The unchanged second write ++ * releases them. The FIFO forbids a readback here. ++ */ ++#define G5P_BOOT_RESET1 0x00e7fffffffffc1dULL ++#define G5P_BOOT_R46_RESET_BITS (0xfULL << 56) ++ ++/* R72b: isolate Apple meta+0x6c8 while retaining dc7 transport. */ ++ ++/* The event ring's stamp events (type 1) dump once per epoch, then a count at each power of two. */ ++ ++/* R73: mirror Apple's auxiliary firmware-ring returns on IRQ174. */ ++ ++/* Latency114: the pstate-1 frequency word meta+0x71c (meta71c114.h). */ ++ ++/* Latency123 (latency123.h): arm A boot words, and the live ctl+0x04 tick-service mask. */ ++ ++/* perf60 clock (perf-envelope.h): the firmware's own pstate request, pinned before release. */ ++/* release8 default */ ++/* GPU overclock (gpu-oc.h, docs/power/ds-gpu-dvfs.md §9 R5): PSINFO record 5 = stock record 4 ++ * with only the PLL multiplier moved (no voltage change), written before release while the GPC is at ++ * state 1, the firmware's table extended to state 5, the release powering on at 5. Needs ++ * boot_gpu_perf 500 and boot_gpu_perf_poweron 1. A thermal stop (drm-render-v3.inc) reboots the ++ * firmware at state 4 when a SoC zone reaches boot_gpu_oc_temp_mc. Record 5 is volatile PMGR ++ * state (LLB rewrites the table only at SoC reset; iBoot reads it once to build the ADT before us) ++ * and is cleared on every boot that does not overclock. ++ */ ++ ++static bool ++ g5p_oc_tripped; /* the thermal stop fired: every later epoch of this boot runs state 4 */ ++ ++static const char *const g5p_oc_zone_names[] = { "apple-a9-soc0", ++ "apple-a9-soc1", ++ "apple-a9-tcc2" }; ++ ++/* The hottest of the three SoC zones (millidegrees); the number of zones read. */ ++static u32 g5p_oc_temp(int *max_mc, const char **zone) ++{ ++ u32 n = 0; ++ ++ *max_mc = INT_MIN; ++ *zone = "none"; ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_zone_names); i++) { ++ struct thermal_zone_device *tz = ++ thermal_zone_get_zone_by_name(g5p_oc_zone_names[i]); ++ int t; ++ ++ if (IS_ERR_OR_NULL(tz) || thermal_zone_get_temp(tz, &t)) ++ continue; ++ n++; ++ if (t > *max_mc) { ++ *max_mc = t; ++ *zone = g5p_oc_zone_names[i]; ++ } ++ } ++ return n; ++} ++ ++static int g5p_oc_read(struct g5p_diag *g, u32 n, u32 w[4]) ++{ ++ for (u32 i = 0; i < 4; i++) ++ if (regmap_read(g->pmgr, G5P_PSINFO_REC(n) + 4 * i, &w[i])) ++ return -EIO; ++ return 0; ++} ++ ++/* Write record 5 from the live record 4 and read it back; the GPC is at state 1 (map time). ++ * Slot 5 is read first: nonzero and not a record this driver writes -> refused, left untouched. ++ */ ++ ++/* A boot without the overclock clears a record 5 this driver wrote; anything else stays. */ ++static void g5p_oc_clear(struct g5p_diag *g) ++{ ++ u32 r4[4], r5[4]; ++ ++ if (!g->pmgr || g5p_oc_read(g, 4, r4) || ++ g5p_oc_read(g, G5P_OC_STATE, r5) || ++ !(r5[0] || r5[1] || r5[2] || r5[3])) ++ return; ++ if (!g5p_oc_is_ours(r4, r5)) { ++ dev_info( ++ g->dev, ++ "GPUPERF oc slot 5 not ours, left: %08x %08x %08x %08x end=gpuperf\n", ++ r5[0], r5[1], r5[2], r5[3]); ++ return; ++ } ++ for (u32 i = 0; i < 4; i++) ++ regmap_write(g->pmgr, G5P_PSINFO_REC(G5P_OC_STATE) + 4 * i, 0); ++ g5p_oc_read(g, G5P_OC_STATE, r5); ++ dev_info( ++ g->dev, ++ "GPUPERF oc record 5 cleared (no overclock this epoch) word0=%08x end=gpuperf\n", ++ r5[0]); ++} ++ ++/* perf60 pin >100, the host's own route: power the GPC on at q (R1+0x1000 = 0x10 | q) in the release, ++ * as the GPC1 request the kext and we already make, with q in place of 1; the policy record carries ++ * the seed-1 words (+0x28 = +0x2c = q) and stays "never sampled" (+0 = 0), so the boot sampler only ++ * initialises; it derives +0x34 and its clock entry priv+0x2c4 from Fender (0x1001c838). ++ */ ++/* release8 default */ ++/* perf60 pin >100 (perf-envelope.h g5p_perf_seed): the boot sampler applies the pin before ALIVE. */ ++ ++/* perf60 stage 4 (render-user-pmset.h): firmware-GiB pages for per-context PM records. */ ++/* release8 default */ ++/* perf60 A/B (docs/gpu/experiments/perf60-clock.md): the firmware's per-tick full flush alone. */ ++ ++/* Latched by map, before firmware sees any pointers. Zero preserves round 9. */ ++ ++/* Explicit decoded VA input only: zero is a valid address when valid=true. ++ * Never infer an address or context from revision-dependent e150 bits. ++ */ ++static unsigned long long mmu_fault_va; ++ ++static const bool boot_experiment = 1; ++ ++struct g5p_dma_page { ++ u8 *cpu; ++ dma_addr_t dma; ++}; ++ ++struct g5p_boot { ++ u32 r27_variant, r27_mode_writes, r46_reset, r50_mode, r50_writes; ++ ++ bool table_clean, preinstall_context, compute_boot_vm; ++ u32 compute_slot; ++ ++ bool compute_numerical, user_compute_owned; ++ ++ struct g5p_boot_layout layout; ++ struct g5p_dma_page image[G5P_IMAGE_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page root[G5P_ROOT_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page shared[G5P_SHARED_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page render_meta[2]; ++ /* perf60 stage 4 (render-user-pmset.h): boot_pm_sets per-context PM sets, 3 pages each */ ++ u32 pm_sets; ++ struct g5p_dma_page pm_set[G5P_PMSET_MAX * G5P_PMSET_PAGES]; ++ struct g5p_dma_page catalogue, directory, zero, leaf[4]; ++ struct g5p_dma_page sink_directory, sink_leaf, sink_page; ++ u32 setup_variant; ++ bool sink_ready, sink_dumped; ++ u32 leaf_index[4]; ++ u32 leaves, pages, mapped, fifo, events, alive_word; ++ u32 fender_raw[ARRAY_SIZE(g5p_fender_regs)]; ++ u32 rgx_raw[ARRAY_SIZE(g5p_rgx_regs)]; ++ const char *snapshot_stage; ++ bool snapshot_valid, rgx_valid; ++ u32 rgx_count; ++ int fifo_error; ++ int irq; ++ atomic_t irqs; ++ spinlock_t event_lock; ++ void __iomem *swirq; ++ bool maps_ready, armed, copied, released, alive, failed, irq_ready; ++ bool release_ack, release_ack_attempted; ++ bool fault_va_valid; ++ u64 fault_va; ++ bool work_ready, work_submitted, work_complete; ++ u32 work_target, control_target; ++ u32 dc_variant, work_variant; ++ u32 r72_meta6c8; ++ u32 meta71c; ++ u32 fw_power_words; ++ u32 gpu_perf; ++ u32 perf_seed; /* boot_gpu_perf_seed: the seeded pstate q, 0 = not seeded */ ++ u32 perf_seed_mode; /* boot_gpu_perf_seed as latched (1..5) */ ++ u32 perf_poweron; /* boot_gpu_perf_poweron: the pstate q the release requests, 0 = 1 as before */ ++ u32 gpu_oc_mhz; /* boot_gpu_oc_mhz as applied (state 5), 0 = no overclock */ ++ /* record 5's voltage codes */ ++ u64 seed_stamp, ++ seed_gpc1; /* seed 3/4: the stamp and the GPC1 write (arch timer) */ ++ u32 meta6d4; ++ u32 r78_ctrl0; ++ u32 r73_aux_drain; ++ u32 r73_aux_entries[6]; ++ u32 r73_aux_hash[6]; ++ u32 r73_aux_first[6][3]; ++ u32 r73_aux_last[6][3]; ++ struct g5p_r73b *r73b; ++ bool irq_capped; ++ bool dc_observe; ++ ++ bool control_ready, control_submitted, control_complete; ++ bool irq_tested, stopped, parked; ++ bool control_timed_out, work_timed_out, event_failed; ++ bool context_submitted, context_complete, context_timed_out; ++ u32 context_target; ++ /* The epoch's last recovery packet (render-user-recover.h), under event_lock. */ ++ struct g5p_fw_recovery fw_recovery; ++ u32 fw_recoveries; ++ u32 event_n[10]; /* events drained this epoch by type (9: 9 and above) */ ++}; ++ ++#include "boot-r73b.inc" ++ ++/* The fault-status subset is read-only, including while META reset is held. ++ * Never access R0 with GPC off/busy; no slave-port transaction is involved. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage); ++ ++static int g5p_boot_alloc_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ p->cpu = dmam_alloc_coherent(g->dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ /* There is no GPU bus translation in the recovered GART path. */ ++ if ((p->dma & (G5P_GPU_PAGE - 1)) || p->dma >= BIT_ULL(40) || ++ dma_to_phys(g->dev, p->dma) != p->dma) { ++ dev_err(g->dev, ++ "unsupported DMA translation/alignment at %pad\n", ++ &p->dma); ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ g->boot->pages++; ++ return 0; ++} ++ ++/* Deliberately map one16KiB page only, not the iOS all-address fallback. ++ * Own distinct directory/leaf/backing; do not alias firmware tables or e160. ++ * No live table surgery: this helper runs only during first map construction. ++ */ ++static int g5p_boot_sink_init(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ; ++ ; ++ if (!b->catalogue.cpu || b->maps_ready || b->armed || b->released || ++ b->sink_ready || b->sink_directory.cpu || b->sink_leaf.cpu || ++ b->sink_page.cpu) ++ return -EPERM; ++ if (g5p_le32(b->catalogue.cpu)) ++ return -EEXIST; ++ ret = g5p_boot_alloc_page(g, &b->sink_directory); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_leaf); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_page); ++ if (ret) ++ return ret; ++ /* Round12 iOS optional dummy-tree leaf flags0: valid writable PA|1. ++ * Only cat[0]/dir[0]/leaf[0] are installed; every other hole stays absent. ++ */ ++ g5p_put64(b->sink_leaf.cpu, g5p_leaf_entry(b->sink_page.dma, 0)); ++ g5p_put64(b->sink_directory.cpu, g5p_directory_entry(b->sink_leaf.dma)); ++ dma_wmb(); ++ g5p_put32(b->catalogue.cpu, g5p_catalogue_entry(b->sink_directory.dma)); ++ dma_wmb(); ++ b->mapped++; ++ b->sink_ready = true; ++ g5p_mmu_record( ++ g, ++ "MMU sink-installed VA=0 bytes=4000 context=0 leaf_flags=0; pre-release " ++ "containment experiment\n"); ++ g5p_blob_printf( ++ g, ++ "MMU sink PC=%pad cat0=%08x dir=%pad dir0=%016llx leaf=%pad pte0=%016llx " ++ "backing=%pad; initially zero\n", ++ &b->catalogue.dma, g5p_catalogue_entry(b->sink_directory.dma), ++ &b->sink_directory.dma, g5p_directory_entry(b->sink_leaf.dma), ++ &b->sink_leaf.dma, g5p_leaf_entry(b->sink_page.dma, 0), ++ &b->sink_page.dma); ++ return 0; ++} ++ ++static int g5p_boot_map_page(struct g5p_diag *g, u64 va, u64 pa, u32 attributes) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ u32 ti = (va >> 14) & 127, i; ++ int ret; ++ ++ if ((va | pa) & (G5P_GPU_PAGE - 1)) ++ return -EINVAL; ++ /* This experiment only owns the recovered firmware heap's catalogue slot. */ ++ if (ci != ((G5P_IMAGE_VA >> 30) & 1023)) ++ return -ERANGE; ++ for (i = 0; i < b->leaves; i++) ++ if (b->leaf_index[i] == di) ++ break; ++ if (i == b->leaves) { ++ if (i == ARRAY_SIZE(b->leaf)) ++ return -ENOSPC; ++ ret = g5p_boot_alloc_page(g, &b->leaf[i]); ++ if (ret) ++ return ret; ++ b->leaf_index[i] = di; ++ g5p_put64(b->directory.cpu + di * 8, ++ g5p_directory_entry(b->leaf[i].dma)); ++ b->leaves++; ++ } ++ if (g5p_le32(b->leaf[i].cpu + ti * 8)) ++ return -EEXIST; ++ g5p_put64(b->leaf[i].cpu + ti * 8, g5p_leaf_entry(pa, attributes)); ++ g5p_put32(b->catalogue.cpu + ci * 4, ++ g5p_catalogue_entry(b->directory.dma)); ++ b->mapped++; ++ return 0; ++} ++ ++#include "mmu-attrs.inc" ++#include "mmu-tables.inc" ++ ++static int g5p_boot_region(struct g5p_diag *g, struct g5p_dma_page *p, u32 size, ++ u64 va) ++{ ++ u32 offset; ++ u32 attributes = va == G5P_IMAGE_VA ? 0x10 : 0x14; ++ int ret; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) { ++ ret = g5p_boot_alloc_page(g, p); ++ if (ret) ++ return ret; ++ ret = g5p_boot_map_page(g, va + offset, p->dma, attributes); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void g5p_boot_copy(struct g5p_dma_page *p, const u8 *data, u32 size) ++{ ++ u32 offset; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) ++ memcpy(p->cpu, data + offset, G5P_GPU_PAGE); ++} ++ ++static u8 *g5p_boot_shared(struct g5p_boot *b, u32 slot) ++{ ++ u32 off = b->layout.offset[slot]; ++ ++ return b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static int g5p_boot_maps(struct g5p_diag *g) ++{ ++ struct g5p_boot *b; ++ u8 *root = NULL, *shared = NULL; ++ bool m71c_wrote; ++ u32 i, m71c, seed; ++ int ret; ++ ++ if (!g->loaded || !g->id_checked || PAGE_SIZE != G5P_GPU_PAGE) ++ return -EPERM; ++ if (g->core_id != 0x011b0602 || g->core_revision != 0x00090f00) ++ return -ENODEV; ++ if (g->boot) ++ return g->boot->maps_ready ? 0 : -EIO; ++ b = devm_kzalloc(g->dev, sizeof(*b), GFP_KERNEL); ++ if (!b) ++ return -ENOMEM; ++ g->boot = b; ++ b->r50_mode = (0); ++ ; ++ b->r27_variant = (0); ++ ; ++ b->r46_reset = (0); ++ ; ++ b->r72_meta6c8 = (1); ++ b->r78_ctrl0 = (0); ++ if (b->r78_ctrl0 && b->r78_ctrl0 != 0x5f) { ++ b->failed = true; ++ return -EINVAL; ++ }; ++ b->r73_aux_drain = (0); ++ if (b->r73_aux_drain > 1 || (b->r73_aux_drain && !1)) { ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->release_ack = (0); ++ b->table_clean = (1); ++ b->compute_boot_vm = (0); ++ b->compute_slot = 0 ? 0 : (7); ++ b->preinstall_context = (0) || 0; ++ ; ++ b->setup_variant = (1); ++ ; ++ b->fault_va = READ_ONCE(mmu_fault_va); ++ b->fault_va_valid = (0); ++ ; ++ b->dc_variant = (7); ++ b->work_variant = (0); ++ b->dc_observe = (0); ++ ; ++ /* iOS: meta+0x71c = 340, the pstate-1 frequency in MHz. */ ++ b->meta71c = (340); ++ m71c = g5p_m71c_check(b->meta71c, 7); ++ if (m71c != G5P_M71C_OK) { ++ dev_err(g->dev, ++ "META71C refused requested=%u variant=%u why=%s before_release=1 end=meta71c\n", ++ b->meta71c, b->dc_variant, g5p_m71c_whys[m71c]); ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->fw_power_words = (0); ++ b->gpu_perf = 400; ++ b->meta6d4 = false; ++ seed = (0); ++ b->gpu_oc_mhz = 0; ++ ; ++ b->perf_poweron = ++ (1) ? (0 ? G5P_OC_STATE : g5p_perf_seed_q(b->gpu_perf)) : 0; ++ b->perf_seed = seed && !b->perf_poweron ? g5p_perf_seed_q(b->gpu_perf) : ++ 0; ++ b->perf_seed_mode = b->perf_seed ? seed : 0; ++ if (!(g5p_perf_valid(b->gpu_perf) || ++ (0 && b->gpu_perf == G5P_OC_PIN)) || ++ (b->gpu_perf && b->fw_power_words) || seed > 5 || (1) > 1 || ++ (b->gpu_perf > 100 && !(1)) || b->meta6d4 > 1 || ++ (b->meta6d4 && !b->gpu_perf)) { ++ dev_err(g->dev, ++ "GPUPERF refused pin=%u power_words=%u gpc_any_pstate=%u meta6d4=%u seed=%u " ++ "before_release=1 end=gpuperf\n", ++ b->gpu_perf, b->fw_power_words, (1), b->meta6d4, seed); ++ b->failed = true; ++ return -EINVAL; ++ } ++ if (b->fw_power_words > G5P_L123_FULL || (0) > G5P_L123_TICK_MASK) { ++ dev_err(g->dev, ++ "L123 refused power_words=%u tick_mask=%u before_release=1 end=l123\n", ++ b->fw_power_words, (0)); ++ b->failed = true; ++ return -EINVAL; ++ } ++ ret = g5p_r73b_latch(g); ++ if (ret) { ++ b->failed = true; ++ return ret; ++ }; ++ dev_info( ++ g->dev, ++ "DC matrix latched variant=%u observe=%u (changes require reboot)\n", ++ b->dc_variant, b->dc_observe); ++ dev_info( ++ g->dev, ++ "COMPUTE VM latched slot=%u boot_vm=%u preinstall=%u (changes require reboot)\n", ++ b->compute_slot, b->compute_boot_vm, b->preinstall_context); ++ spin_lock_init(&b->event_lock); ++ atomic_set(&b->irqs, 0); ++ ret = g5p_plan_layout(&b->layout); ++ if (ret) ++ goto out; ++ root = kvzalloc(G5P_ROOT_SIZE, GFP_KERNEL); ++ shared = kvzalloc(G5P_SHARED_SIZE, GFP_KERNEL); ++ if (!root || !shared) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_boot_alloc_page(g, &b->catalogue); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->directory); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->zero); ++ if (ret) ++ goto out; ++ ret = g5p_boot_sink_init(g); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->image, G5P_IMAGE_SIZE, G5P_IMAGE_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->root, G5P_ROOT_SIZE, G5P_ROOT_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->shared, G5P_SHARED_SIZE, G5P_SHARED_VA); ++ if (ret) ++ goto out; ++ /* R22 metadata mappings are installed before release, only behind its ++ * default-off gate. Compute/boot defaults allocate no additional pages. ++ */ ++ { ++ ret = g5p_boot_region(g, b->render_meta, 2 * G5P_GPU_PAGE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE); ++ if (ret) ++ goto out; ++ b->pm_sets = min((8), G5P_PMSET_MAX); ++ if (b->pm_sets) { ++ ret = g5p_boot_region(g, b->pm_set, ++ b->pm_sets * G5P_PMSET_PAGES * ++ G5P_GPU_PAGE, ++ G5P_PMSET_VA); ++ if (ret) ++ goto out; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-sets boot sets=%u va=%#llx..%#llx before_release=1 end=v3\n", ++ b->pm_sets, G5P_PMSET_VA, ++ g5p_pmset_va(b->pm_sets, 0)); ++ } ++ } ++ ret = 0; ++ if (ret) ++ goto out; ++ /* c80758: Fender (2 pages), AIC timer, AIC software interrupt. */ ++ for (i = 0; i < 4; i++) { ++ u64 pa = i < 2 ? 0x201e00000ULL + i * G5P_GPU_PAGE : ++ i == 2 ? 0x20e100000ULL : ++ 0x20e104000ULL; ++ ++ ret = g5p_boot_map_page(g, G5P_PIO_VA + i * G5P_GPU_PAGE, pa, ++ 0x18); ++ if (ret) ++ goto out; ++ } ++ g5p_make_boot_config(&g->image, &b->layout); ++ g5p_make_init(&b->layout, root, shared, &g->image, b->catalogue.dma, ++ arch_timer_read_counter()); ++ g5p_dc_init_variant(&b->layout, root, shared, 7); ++ /* meta+0x784 is the abstime OFFSET the firmware adds to its timebase (0x100221e8 = PIO-1 ++ * counter + priv+688, loaded from meta+0x784 at 0x1001ff58); iOS writes ++ * ml_get_abstime_offset(). Linux's timebase is the counter itself: 0. (make_init's absolute ++ * stamp put the firmware's clock ~uptime ahead of every host stamp in policy+0x8f8.) ++ */ ++ g5p_put64(g5p_object(&b->layout, root, shared, 0x4d) + 0x784, 0); ++ if (g5p_boot_ctrl0(b)) ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x5e), ++ g5p_boot_ctrl0(b)); ++ dev_info( ++ g->dev, ++ "R78 ctrl0 requested=%08x installed=%08x before_release=1 end=r78\n", ++ g5p_boot_ctrl0(b), ++ g5p_le32(g5p_object(&b->layout, root, shared, 0x5e))); ++ ; ++ ; ++ { ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x4d) + 0x6c8, ++ 1); ++ }; ++ m71c = g5p_m71c_install(&b->layout, root, shared, b->meta71c, ++ &m71c_wrote); ++ dev_info( ++ g->dev, ++ "META71C requested=%u installed=%08x variant=%u wrote=%u before_release=1 " ++ "end=meta71c\n", ++ b->meta71c, m71c, b->dc_variant, m71c_wrote); ++ if (m71c != b->meta71c) { ++ ret = -EIO; ++ goto out; ++ } ++ if (b->fw_power_words) ++ g5p_l123_arm_a(&b->layout, root, shared, b->fw_power_words); ++ if (b->gpu_perf) { ++ u8 *pol = g5p_object(&b->layout, root, shared, G5P_PERF_POLICY); ++ u8 *mt = g5p_object(&b->layout, root, shared, G5P_PERF_META); ++ u8 *ct = g5p_object(&b->layout, root, shared, G5P_PERF_CTL); ++ ++ g5p_perf_install(&b->layout, root, shared, b->gpu_perf); ++ { ++ g5p_oc_clear(g); ++ } ++ if (b->meta6d4) ++ g5p_put32(mt + 0x6d4, 1); ++ if (b->perf_poweron) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_poweron, ++ false); ++ dev_info( ++ g->dev, ++ "GPUPERF poweron q=%u policy28=%u/%u policy0=%u policy4=%u seed=%u ignored " ++ "before_release=1 end=gpuperf\n", ++ b->perf_poweron, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), seed); ++ } else { ++ dev_info( ++ g->dev, ++ "GPUPERF poweron ignored pin=%u (pins 200..400 only) before_release=1 end=gpuperf\n", ++ b->gpu_perf); ++ } ++ if (seed && !b->perf_seed && !b->perf_poweron) ++ dev_info( ++ g->dev, ++ "GPUPERF seed ignored pin=%u (seeds pins 200..400 only) before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf); ++ if (b->perf_seed) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_seed, ++ seed >= 2); ++ /* seed 5: meta+0x784 is the abstime OFFSET the firmware adds to its timebase ++ * (0x100221e8 = PIO-1 counter + priv+688, loaded from meta+0x784 at 0x1001ff58); ++ * iOS writes ml_get_abstime_offset(), make_init the absolute counter, so every ++ * policy+0x8f8 stamp lay ~counter in the firmware's past. Linux has no offset. ++ */ ++ if (seed == 5) ++ g5p_put64(mt + 0x784, 0); ++ dev_info( ++ g->dev, ++ "GPUPERF seed mode=%u q=%u policy28=%u/%u policy30=%u policy8=%u policy0=%u " ++ "policy4=%u meta784=%llx before_release=1 end=gpuperf\n", ++ seed, b->perf_seed, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0x30), ++ g5p_le32(pol + 8), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), ++ (u64)g5p_le32(mt + 0x784) | ++ (u64)g5p_le32(mt + 0x788) << 32); ++ } ++ dev_info( ++ g->dev, ++ "GPUPERF boot pin=%u table=%u/%u/%u,%u,%u,%u,%u limits=%u/%u/%u ovr=%u/%u " ++ "policy688=%u/%u policy6f0=%u/%u policy4=%u policy8=%u/%u q=%u mhz=%u " ++ "gpc_any_pstate=%u seed=%u/%u policy0=%u policy28=%u/%u before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf, g5p_le32(mt + 0x710), g5p_le32(mt + 0x714), ++ g5p_le32(mt + 0x718), g5p_le32(mt + 0x71c), ++ g5p_le32(mt + 0x720), g5p_le32(mt + 0x724), ++ g5p_le32(mt + 0x728), g5p_le32(ct + 0x1028), ++ g5p_le32(ct + 0x102c), g5p_le32(ct + 0x1030), ++ g5p_le32(ct + 0x1038), g5p_le32(ct + 0x1040), ++ g5p_le32(pol + 0x688), g5p_le32(pol + 0x68c), ++ g5p_le32(pol + 0x6f0), g5p_le32(pol + 0x6f4), ++ g5p_le32(pol + 4), g5p_le32(pol + 8), ++ g5p_le32(pol + 0xc), b->gpu_perf / 100, ++ b->gpu_perf / 100 < ARRAY_SIZE(g5p_perf_mhz) ? ++ g5p_perf_mhz[b->gpu_perf / 100] : ++ b->gpu_oc_mhz, ++ (1), seed, b->perf_seed, g5p_le32(pol + 0), ++ g5p_le32(pol + 0x28), g5p_le32(pol + 0x2c)); ++ } ++ g5p_put32(g5p_object(&b->layout, root, shared, G5P_L123_CTL) + 4, (0)); ++ { ++ u8 *meta = g5p_object(&b->layout, root, shared, G5P_L123_META); ++ ++ dev_info( ++ g->dev, ++ "L123 boot power_words=%u meta6d4=%u table=%u/%u/%u,%u,%u,%u,%u policy4=%u " ++ "ctl04=%u limits=%u/%u/%u before_release=1 end=l123\n", ++ b->fw_power_words, g5p_le32(meta + 0x6d4), ++ g5p_le32(meta + 0x710), g5p_le32(meta + 0x714), ++ g5p_le32(meta + 0x718), g5p_le32(meta + 0x71c), ++ g5p_le32(meta + 0x720), g5p_le32(meta + 0x724), ++ g5p_le32(meta + 0x728), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_POLICY) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1028), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x102c), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1030)); ++ /* the accessor's licence bound (mmio-access.md §4.2 step 4): meta+0x710 as written */ ++ WRITE_ONCE(g->mmio_max_pstate, ++ g5p_mmio_max_pstate(g5p_le32(meta + 0x710))); ++ }; ++ ; ++ g5p_boot_copy(b->image, g->image.data, G5P_IMAGE_SIZE); ++ g5p_boot_copy(b->root, root, G5P_ROOT_SIZE); ++ g5p_boot_copy(b->shared, shared, G5P_SHARED_SIZE); ++ dma_wmb(); ++ ret = 0; ++ if (ret) ++ goto out; ++ g5p_tables_clean(g, "boot-before-release", NULL, 0); ++ b->maps_ready = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 maps: %u x 16KiB allocations; %u PTEs, %u leaves, PC=%pad\n", ++ b->pages, b->mapped, b->leaves, &b->catalogue.dma); ++ dev_info( ++ g->dev, ++ "G2 init: root=%llx control=%llx event-ctl=%llx event-data=%llx\n", ++ g5p_object_va(&b->layout, 0x50), ++ g5p_object_va(&b->layout, 0x5f), ++ g5p_object_va(&b->layout, 0x78), ++ g5p_object_va(&b->layout, 0x7c)); ++ dev_info( ++ g->dev, ++ "G2 experimental zero policy fields; fault observation only in map\n"); ++out: ++ kvfree(root); ++ kvfree(shared); ++ if (ret) ++ b->failed = true; ++ g5p_ds_hook_maps(g, ret); ++ return ret; ++} ++ ++#include "boot-r74f.inc" ++#include "boot-r74g.inc" ++ ++static u32 g5p_transport_event_limit(void); ++ ++static int g5p_boot_drain(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ dev_err(g->dev, "G2 invalid event indices %u/%u\n", consumer, ++ producer); ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ for (n = 0; n < 16 && consumer != producer; n++) { ++ u32 words[11], off = b->layout.offset[0x74] + consumer * 44, t, ++ n; ++ u8 *src; ++ unsigned int i; ++ bool want; ++ ++ /* Heartbeat116: an R74 epoch bounds ring entries per window ++ * in the IRQ thread instead (heartbeat116.h EVENTS). ++ */ ++ if (!g5p_hb116_on(b) && ++ b->events >= g5p_transport_event_limit()) { ++ ret = -EOVERFLOW; ++ goto out; ++ } ++ /* Entries may straddle DMA pages. */ ++ for (i = 0; i < ARRAY_SIZE(words); i++) { ++ u32 at = off + i * 4; ++ ++ src = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ words[i] = g5p_le32(src); ++ } ++ /* A healthy meta6c8 setup loop can emit thousands of identical ++ * ALIVEs. Keep its first four packets and the exact aggregate, ++ * without making printk throughput part of the experiment. ++ * R74 also quiets a moved loop: each new word1 prints once. ++ */ ++ want = !b->r73b || words[0] || b->events < 4 || ++ (words[1] != 0x8000a120 && g5p_r74_word1(b, words[1])); ++ /* Stamp events (type 1, two per job) dump once per epoch, then only count: ++ * at a few jobs per frame they were most of the log and evicted the faults ++ * (release8 bench, 306 k lines). Every other type dumps as before. ++ */ ++ t = min_t(u32, words[0], ARRAY_SIZE(b->event_n) - 1); ++ n = ++b->event_n[t]; ++ if (words[0] == 1 && n > 1 && !(0)) { ++ if (is_power_of_2(n)) ++ g5p_job_printk( ++ g->dev, ++ "G2 events type=1 n=%u: stamp events counted, dumped once (boot_event_dump 1: each)\n", ++ n); ++ want = false; ++ } ++ /* Alive (0) and stamp (1) events are routine, one or more per job: job_log. Every other ++ * type (recovery, page request, channel error, fault) prints as before. ++ */ ++ if (want && words[0] > 1) { ++ dev_info(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ print_hex_dump(KERN_INFO, ++ "G2 event raw: ", DUMP_PREFIX_NONE, 16, ++ 4, words, sizeof(words), false); ++ } else if (want) { ++ g5p_job_printk(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ g5p_job_hex("G2 event raw: ", words, sizeof(words)); ++ }; ++ if (g->event_notify) ++ g->event_notify(g->event_cookie, words); ++ /* c99788 routes these to recovery/page-request/channel-error paths. */ ++ if (words[0] != 0 && words[0] != 1) { ++ WRITE_ONCE(b->failed, true); ++ switch (words[0]) { ++ case G5P_FW_EVENT_RECOVERY: { ++ /* The kext's fields (render-user-recover.h). */ ++ struct g5p_fw_recovery rec = ++ g5p_fw_recovery_decode(words); ++ ++ b->fw_recovery = rec; ++ b->fw_recoveries++; ++ dev_err(g->dev, ++ "G2 recovery event slot=%u flag=%u code=%08x w18=%08x w20=%08x w24=%08x n=%u: " ++ "the firmware stopped; iOS restarts the GPU\n", ++ rec.slot, rec.flag, rec.code, rec.w18, ++ rec.w20, rec.w24, b->fw_recoveries); ++ break; ++ } ++ case 7: ++ dev_err(g->dev, ++ "G2 page-request event payload=%08x/%08x/%08x; address encoding unresolved\n", ++ words[1], words[2], words[3]); ++ break; ++ case 8: ++ dev_err(g->dev, ++ "G2 channel-error event kind=%u engine=%u token=%08x stamp=%08x; no " ++ "acknowledgment command guessed\n", ++ words[1], words[2], words[3], words[4]); ++ break; ++ } ++ dev_err(g->dev, ++ "G2 unsupported/fault event=%u; epoch failed, entry retained (the render device " ++ "recovers it if it may)\n", ++ words[0]); ++ ret = -EIO; ++ break; ++ } ++ if (words[0] == 0 && READ_ONCE(b->released) && ++ !READ_ONCE(b->stopped)) { ++ b->alive_word = words[1]; ++ WRITE_ONCE(b->alive, true); ++ } ++ b->events++; ++ consumer = (consumer + 1) & 255; ++ /* c99788 uses a full barrier: finish slot reads before returning it. */ ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ /* Return the ALIVE slot before ringing work. No logging or ++ * other MMIO may be inserted between these two operations. ++ */ ++ if (!words[0] && g5p_r74f_on(b) && ++ READ_ONCE(b->r73b->owner) == G5P_R73B_OWNER_IRQ && ++ READ_ONCE(b->released) && !READ_ONCE(b->stopped)) ++ g5p_r74f_alive(g); ++ } ++out: ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++static irqreturn_t g5p_boot_irq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ struct g5p_boot *b = g->boot; ++ u32 n = atomic_inc_return(&b->irqs); ++ int ret = 0; ++ ++ if (b->r73b) ++ return g5p_r73b_irq(g, irq, n); ++ if (n > g5p_transport_event_limit()) { ++ /* Never leave the software source asserted, even when capped. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (!b->irq_capped) { ++ b->irq_capped = true; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "G2 IRQ174 storm capped; reboot required\n"); ++ } ++ return IRQ_HANDLED; ++ } ++ /* KPF level6G: Apple uses this AIC ACK before all ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (READ_ONCE(b->released)) ++ ret = g5p_boot_drain(g); ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return IRQ_HANDLED; ++} ++ ++static int g5p_boot_irq_setup(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b->irq_ready) ++ return 0; ++ b->swirq = devm_ioremap(g->dev, 0x20e104000ULL, 0x100); ++ if (!b->swirq) ++ return -ENOMEM; ++ b->irq = platform_get_irq(to_platform_device(g->dev), 4); ++ if (b->irq < 0) ++ return b->irq; ++ ret = devm_request_threaded_irq(g->dev, b->irq, NULL, g5p_boot_irq, ++ IRQF_ONESHOT, "g5p-meta", g); ++ if (ret) ++ return ret; ++ b->irq_ready = true; ++ dev_info(g->dev, "G2 IRQ174 registered as Linux IRQ %d\n", b->irq); ++ return 0; ++} ++ ++/* Cache first, then report: sysfs status never issues GPU MMIO. */ ++static void g5p_boot_snapshot(struct g5p_diag *g, const char *stage, ++ enum g5p_readback_state mode) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 i, state; ++ ++ b->snapshot_stage = stage; ++ b->snapshot_valid = false; ++ b->rgx_valid = false; ++ b->rgx_count = 0; ++ g5p_power_snapshot(g, stage); ++ dev_info(g->dev, "G2 snapshot %s: R1 first\n", stage); ++ for (i = 0; i < ARRAY_SIZE(g5p_fender_regs); i++) { ++ b->fender_raw[i] = readl(g->window[1] + g5p_fender_regs[i]); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x\n", g5p_fender_regs[i], ++ b->fender_raw[i]); ++ } ++ b->snapshot_valid = true; ++ state = b->fender_raw[4]; ++ /* GPC=1 did not make META a18 safe while bit32 was held. */ ++ b->rgx_count = g5p_readback_rgx_count(state, mode); ++ if (!b->rgx_count) { ++ dev_info(g->dev, "G2 R0 snapshot skipped: R1+1000=%08x\n", ++ state); ++ return; ++ } ++ for (i = 0; i < b->rgx_count; i++) { ++ dev_info(g->dev, "G2 R R0+%05x\n", g5p_rgx_regs[i]); ++ b->rgx_raw[i] = readl(g->rgx + g5p_rgx_regs[i]); ++ dev_info(g->dev, "G2 raw R0+%05x=%08x\n", g5p_rgx_regs[i], ++ b->rgx_raw[i]); ++ } ++ b->rgx_valid = true; ++} ++ ++static int g5p_fender_write(void *ctx, u32 reg, u32 value) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ ++ /* recovery108: never an R1 access after a FINAL teardown retired the device. */ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 W R1+%05x=%08x\n", reg, value); ++ writel(value, g->window[1] + reg); ++ raw = readl(g->window[1] + reg); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x (write fence)\n", reg, raw); ++ return 0; ++} ++ ++static int g5p_fender_poll(void *ctx, u32 reg, u32 mask, u32 expected) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 poll R1+%05x mask=%08x expected=%08x (100ms)\n", ++ reg, mask, expected); ++ ret = readl_poll_timeout(g->window[1] + reg, raw, ++ (raw & mask) == expected, 10, 100000); ++ dev_info(g->dev, "G2 poll raw R1+%05x=%08x ret=%d\n", reg, raw, ret); ++ return ret; ++} ++ ++static int g5p_r74e_power_poll(struct g5p_diag *g, u32 expected, u32 timeout_us, ++ u64 *elapsed_ns, u32 *samples) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u64 start = ktime_get_mono_fast_ns(); ++ u32 raw = U32_MAX, n = 0; ++ int ret; ++ ++ for (;;) { ++ if (READ_ONCE(b->alive)) { ++ ret = -EHOSTDOWN; ++ break; ++ } ++ raw = readl(g->window[1] + 0x1000); ++ n++; ++ if ((raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ /* wake112: the reads taken; 0 means raw is still the U32_MAX initializer, not a read. */ ++ *samples = n; ++ r->early_power_raw = raw; ++ return ret; ++} ++ ++/* wake112 (r74e112.h): a failed deferred proof is the cold power abort, absorbing. */ ++static void g5p_r74e_fail(struct g5p_r73b *r, u32 why) ++{ ++ r->early_proof_why = why; ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ atomic_set_release(&r->early_proof, G5P_PX_FAILED); ++} ++ ++/* The once-owner takes the deferred proof right before its one R1+0x1000 read. ++ * false: that read must not happen, and the proof has failed. ++ */ ++static bool g5p_r74e_take(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_px_pre p; ++ u32 why; ++ ++ if (atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, G5P_PX_TAKEN) != ++ G5P_PX_DEFERRED) ++ return false; ++ p = (struct g5p_px_pre){ ++ .gen = r->early_proof_gen, ++ .gen_now = g5p_ds_gen(), ++ .alive = READ_ONCE(b->alive), ++ .released = READ_ONCE(b->released), ++ .stopped = READ_ONCE(b->stopped), ++ .failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ .r1_allowed = g5p_r8_r1_allowed(), ++ }; ++ r->early_proof_gen_now = p.gen_now; ++ why = g5p_px_check(&p); ++ if (why) ++ g5p_r74e_fail(r, why); ++ return !why; ++} ++ ++/* The owner's read decides: only the state field 1 licenses the release. */ ++static void g5p_r74e_settle(struct g5p_r73b *r, u32 raw) ++{ ++ if (!g5p_px_on(raw)) { ++ g5p_r74e_fail(r, G5P_PXW_STATE); ++ return; ++ } ++ WRITE_ONCE(r->early_release_ready, true); ++ atomic_set_release(&r->early_proof, G5P_PX_PROVEN); ++} ++ ++/* No ALIVE wait ends with the proof still owed: close it, no register access. ++ * true: it was still owed. ++ */ ++static bool g5p_r74e_close(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ ++ if (!r || atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, ++ G5P_PX_TAKEN) != G5P_PX_DEFERRED) ++ return false; ++ g5p_r74e_fail(r, G5P_PXW_UNTAKEN); ++ return true; ++} ++ ++/* boot_gpu_perf_seed 3/4: the sampler's last-sample time policy+0x8f8 = now. Seed 3 stamps at the ++ * start of the release (flown 2026-11-03: 38 ms before the stream's end, the window was missed); ++ * seed 4 stamps as the last host write before the one that lets META run, the GPC1 request ++ * R1+0x1000 = 0x11 (writel orders the RAM store before it). ++ */ ++static void g5p_perf_seed_stamp(struct g5p_boot *b, u32 mode) ++{ ++ u32 o; ++ ++ if (mode == 4 ? b->perf_seed_mode < 4 : b->perf_seed_mode != mode) ++ return; /* the GPC1 stamp serves seeds 4 and 5 */ ++ o = b->layout.offset[G5P_PERF_POLICY] + 0x8f8; ++ b->seed_stamp = arch_timer_read_counter(); ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)b->seed_stamp)); ++ o += 4; ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)(b->seed_stamp >> 32))); ++ wmb(); ++} ++ ++static int g5p_r74e_fender_start(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxp latch; ++ bool on_ran = false; ++ u32 samples = ++ 0; /* wake112: the reads of the poll that ran last (the printed raw) */ ++ int ret; ++ ++ r->early_power_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_r74e_power_poll(g, 0, 1000, &r->early_off_ns, ++ &samples); ++ if (!ret) { ++ g5p_perf_seed_stamp(b, 4); ++ ret = g5p_fender_write(g, 0x1000, ++ 0x10 | (b->perf_poweron ?: 1)); ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ if (!ret) { ++ on_ran = true; ++ /* poweron: the first request above pstate 1, a voltage step; a longer bound */ ++ ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, ++ b->perf_poweron ? 10000 : 1800, ++ &r->early_on_ns, &samples); ++ if (b->perf_poweron) ++ dev_emerg( ++ g->dev, ++ "GPUPERF poweron q=%u r1_1000=%08x on_ns=%llu ret=%d end=gpuperf\n", ++ b->perf_poweron, r->early_power_raw, ++ r->early_on_ns, ret); ++ } ++ /* wake112: only ALIVE preempting the on-poll before its first read defers; the ++ * first-ALIVE owner owes the proof. A sampled on-poll (0x11, ...) aborts as before. ++ */ ++ latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples); ++ if (latch == G5P_PXP_DEFER) { ++ r->early_defer_ret = ret; ++ r->early_proof_gen = g5p_ds_gen(); ++ ret = 0; ++ atomic_set_release(&r->early_proof, G5P_PX_DEFERRED); ++ } ++ r->early_power_ret = ret; ++ if (latch == G5P_PXP_ABORT) { ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ } else if (latch == G5P_PXP_READY) { ++ WRITE_ONCE(r->early_release_ready, true); ++ } ++ dev_emerg( ++ g->dev, ++ "R74E power off_ns=%llu on_ns=%llu raw=%08x ret=%d release_ready=%u alive=%u " ++ "sampled=%u end=r74e\n", ++ r->early_off_ns, r->early_on_ns, r->early_power_raw, ret, ++ r->early_release_ready, READ_ONCE(b->alive), samples); ++ if (latch == G5P_PXP_DEFER) ++ dev_emerg( ++ g->dev, ++ "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u end=r74e112\n", ++ r->early_proof_gen, r->early_defer_ret, ++ r->early_power_raw, READ_ONCE(b->alive)); ++ return ret; ++} ++ ++static void g5p_fifo_word(struct g5p_diag *g, u32 word) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 raw; ++ ++ if (b->fifo_error) ++ return; ++ if (b->fifo < 0x10000 || b->fifo >= 0x10200) { ++ b->fifo_error = -EOVERFLOW; ++ return; ++ } ++ writel(word, g->window[1] + b->fifo); ++ raw = readl(g->window[1] + b->fifo); ++ if (raw != word) { ++ dev_err(g->dev, ++ "G2 FIFO verify R1+%05x actual=%08x expected=%08x\n", ++ b->fifo, raw, word); ++ b->fifo_error = -EIO; ++ } ++ b->fifo += 4; ++} ++ ++static void g5p_fifo32(struct g5p_diag *g, u32 reg, u32 value, u32 flags) ++{ ++ dev_info(g->dev, "G2 FIFO R0+%05x=%08x flags=%02x\n", reg, value, ++ flags); ++ g5p_fifo_word(g, (reg & 0xff00ffff) | (flags & 0xff) << 16); ++ g5p_fifo_word(g, value); ++} ++ ++static void g5p_fifo64(struct g5p_diag *g, u32 reg, u64 value) ++{ ++ g5p_fifo32(g, reg + 4, value >> 32, 0); ++ g5p_fifo32(g, reg, value, 0xff); ++} ++ ++static int g5p_boot_fifo_dump(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off, raw, end = min(b->fifo, 0x10200U); ++ ++ if (end <= 0x10000) ++ return -ENODATA; ++ dev_info(g->dev, "G2 FIFO dump R1+10000..%05x (<=128 words)\n", ++ end - 4); ++ for (off = 0x10000; off < end; off += 4) { ++ raw = readl(g->window[1] + off); ++ dev_info(g->dev, "G2 FIFO raw R1+%05x=%08x\n", off, raw); ++ } ++ return 0; ++} ++ ++static u64 g5p_snapshot64(struct g5p_boot *b, u32 reg) ++{ ++ u32 i; ++ ++ for (i = 0; i + 1 < b->rgx_count; i++) ++ if (g5p_rgx_regs[i] == reg && g5p_rgx_regs[i + 1] == reg + 4) ++ return (u64)b->rgx_raw[i + 1] << 32 | b->rgx_raw[i]; ++ return 0; ++} ++ ++static void g5p_release_observe(void *ctx, const char *stage) ++{ ++ ; ++ ; ++} ++ ++/* A diagnostic notification ACK only, never engine repair or a retry loop. ++ * Snapshot first, require the active wrapper, ACK only200 once, fence and ++ * re-sample. A cleared event alone is not evidence the underlying fault healed. ++ */ ++static int g5p_release_ack(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 state, event, after; ++ ++ if (!b->release_ack) ++ return 0; ++ if (!b->released || b->stopped || b->release_ack_attempted) ++ return -EPERM; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_fault_snapshot(g, "release-ack-before"); ++ g5p_mmu_progress(g, "release-ack-event", 0x130); ++ event = readl(g->rgx + 0x130); ++ b->release_ack_attempted = true; ++ if (event & 0x200) { ++ g5p_mmu_progress(g, "release-ack-write", 0x138); ++ writel(0x200, g->rgx + 0x138); ++ /* Ordered same-device read fences the single write. */ ++ g5p_mmu_progress(g, "release-ack-fence", 0x130); ++ after = readl(g->rgx + 0x130); ++ g5p_mmu_record( ++ g, ++ "MMU release-ack mask=00000200 before=%08x after=%08x; notification only\n", ++ event, after); ++ } else { ++ g5p_mmu_record(g, ++ "MMU release-ack skipped=no-event before=%08x\n", ++ event); ++ } ++ g5p_fault_snapshot(g, "release-ack-after"); ++ return 0; ++} ++ ++static int g5p_boot_program_stream(struct g5p_diag *g, bool release) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ u64 reset; ++ int ret; ++ ++ if (b->armed && !release) ++ return -EALREADY; ++ ret = g5p_boot_irq_setup(g); ++ if (ret) ++ return ret; ++ g5p_boot_snapshot(g, release ? "before-release" : "before-arm", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ ; ++ ret = 0; ++ if (ret) ++ goto out; ++ /* Slave-port access remains forbidden for the rest of this boot. */ ++ g->reset_transition = true; ++ /* c9fa10: core request false, GPC 0. Establish an actual state edge; ++ * wake left GPC at 1, so writing 1 again did not prove FIFO execution. ++ * No R0 accesses between this point and the successful active-state poll. ++ */ ++ dev_info(g->dev, "G2 Fender park: core=0, GPC=0\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ goto out; ++ /* c9fc10: core request true before prepareBoot c9feb4. */ ++ dev_info(g->dev, "G2 Fender prepare: core=20, require GPC=0\n"); ++ ret = g5p_fender_hold(&io); ++ if (ret) ++ goto out; ++ dev_info(g->dev, ++ "G2 init stream: master boot, release=%u; FIFO reset\n", ++ release); ++ ret = g5p_fender_write(g, 0x10, 0); ++ if (ret) ++ goto out; ++ /* setMode(3), after Fender prepare cleared +10, before firmware runs. ++ * Both arm and release execute prepare; never reapply after ALIVE. ++ */ ++ ; ++ ; ++ ; ++ g5p_fender_write(g, 0, 0x40000000); ++ ; ++ ; ++ b->fifo = 0x10000; ++ b->fifo_error = 0; ++ ; ++ ; ++ g5p_fifo64(g, 0x100, ++ G5P_BOOT_RESET1 | (0 ? G5P_BOOT_R46_RESET_BITS : 0)); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x00e7fe071ffffc1dULL); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x100000000ULL); ++ ; ++ ; ++ g5p_fifo32(g, 0xbf8, 1, 0xff); ++ ; ++ ; ++ g5p_fifo32(g, 0xe140, 0, 0xff); ++ ; ++ ; ++ g5p_fifo64(g, 0xe148, b->catalogue.dma >> 12); ++ ; ++ ; ++ g5p_fifo64(g, 0xe200, 3); ++ ; ++ ; ++ g5p_fifo64(g, 0xe208, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fifo64(g, 0xe210, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fender_write(g, 0x2010, 0xbf); ++ ; ++ ; ++ g5p_fender_write(g, 0x2014, 0x233); ++ ; ++ ; ++ g5p_fifo64(g, 0xe160, b->zero.dma); ++ ; ++ ; ++ g5p_fifo64(g, 0xbb8, 0); ++ ; ++ ; ++ if (release) { ++ g5p_fifo64(g, 0x100, 0); ++ ; ++ } ++ g5p_fifo_word(g, 0xffff0000); ++ ; ++ ; ++ ret = b->fifo_error; ++ if (ret) ++ goto out; ++ dma_wmb(); ++ dev_info( ++ g->dev, ++ "G2 FIFO stored and verified: %u words (not execution proof)\n", ++ (b->fifo - 0x10000) / 4); ++ if (release) { ++ WRITE_ONCE(b->released, true); ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_FIFO, G5P_R73B_NONE, 0); ++ ret = 0; ++ if (ret) ++ goto out; ++ } ++ /* c9f954: core request false, then GPC 1. */ ++ dev_info(g->dev, "G2 Fender execute: core=0, GPC=1 (from 0)\n"); ++ if (release && !(b->r73b && 1)) { ++ g5p_perf_seed_stamp( ++ b, ++ 4); /* the non-r74e start: as close as this path gets */ ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ ret = release && b->r73b && 1 ? g5p_r74e_fender_start(g) : ++ release ? g5p_fender_start_observed_q(&io, g5p_release_observe, ++ b->perf_poweron ?: 1) : ++ g5p_fender_start(&io); ++ if (release) ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_GPC, G5P_R73B_NONE, ret); ++ if (ret) ++ goto out; ++ /* r74e: after GPC1 is active, wait for ALIVE without optional MMIO. */ ++ if (release && b->r73b && 1) ++ return 0; ++ usleep_range(10, 20); ++ g5p_boot_snapshot(g, release ? "after-release" : "after-arm", ++ release ? G5P_READBACK_WRAPPER_ONLY : ++ G5P_READBACK_RESET_HELD); ++ ; ++ /* Only the fault-status subset above is sampled after release. */ ++ if (release) ++ return g5p_release_ack(g); ++ if (!b->rgx_valid) ++ return -EIO; ++ reset = g5p_snapshot64(b, 0x100); ++ dev_info(g->dev, "G2 verify reset actual=%016llx expected=%016llx\n", ++ reset, 0x100000000ULL); ++ dev_info( ++ g->dev, ++ "G2 expected catalogue=%llx scratch=%llx SLC=3 (not read: state safety)\n", ++ (u64)b->catalogue.dma >> 12, (u64)b->zero.dma); ++ if (reset != 0x100000000ULL) ++ return -EIO; ++ dev_info( ++ g->dev, ++ "G2 reset-held gate verified; remaining FIFO effects await firmware event\n"); ++ b->armed = true; ++ return 0; ++out: ++ /* A slow r74e epoch is scoreable without another unsafe MMIO read. */ ++ if (release && b->r73b && 1 && READ_ONCE(b->r73b->early_power_abort)) ++ return ret; ++ /* Wrapper-only snapshot if GPC remains off/busy. */ ++ g5p_boot_snapshot(g, "transition-or-FIFO-failed", ++ G5P_READBACK_WRAPPER_ONLY); ++ return ret; ++} ++ ++static int g5p_boot_arm(struct g5p_diag *g) ++{ ++ return g5p_boot_program_stream(g, false); ++} ++ ++static int g5p_boot_load(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off; ++ ++ /* c9f1bc/ca43b0: segment and LDR configuration pairs are supplied in ++ * mapped RAM. Polling MSLVCTRL1 in the reset-held state locks this SoC. ++ */ ++ for (off = 0; off < G5P_IMAGE_SIZE; off += G5P_GPU_PAGE) ++ if (memcmp(b->image[off / G5P_GPU_PAGE].cpu, ++ g->image.data + off, G5P_GPU_PAGE)) ++ return -EBADMSG; ++ dma_wmb(); ++ b->copied = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 RAM load verified bytes=%x segment_pairs=13 LDR_pairs=27; no slave-port MMIO\n", ++ G5P_IMAGE_SIZE); ++ return 0; ++} ++ ++static int g5p_boot_release(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u64 seed_ts = 0; ++ int ret; ++ ++ if (b->released) ++ return -EALREADY; ++ /* recovery108: fail closed before the release touches anything. */ ++ ret = g5p_ds_hook_release_gate(g); ++ if (ret) ++ return ret; ++ g5p_r73b_release_entry(g); ++ if (b->r73b && 1) { ++ ret = g5p_r73e_prepare(g); ++ if (ret) ++ return ret; ++ } ++ dev_info(g->dev, ++ "G2 release: META master boot; no GPU job submitted\n"); ++ /* Reproduce the complete original stream, including its final reset release. ++ * Do not assume the experimental arm stage preserves a FIFO read cursor. ++ */ ++ /* boot_gpu_perf_seed 3: the sampler's last-sample time = now, as close to the release as the ++ * host gets (perf-envelope.h); the firmware divides (now - policy+0x8f8) by 24000 * policy+4. ++ */ ++ if (b->perf_seed_mode >= 3) { ++ seed_ts = arch_timer_read_counter(); ++ g5p_perf_seed_stamp(b, 3); ++ } ++ ret = g5p_boot_program_stream(g, true); ++ if (seed_ts) { ++ u64 end = arch_timer_read_counter(), ++ rate = arch_timer_get_rate(); ++ ++ /* always printed on a seed 3/4 boot, before ALIVE is awaited */ ++ dev_emerg( ++ g->dev, ++ "GPUPERF seed mode=%u stamp=%llx release_to_gpc1_us=%llu stamp_to_gpc1_us=%llu " ++ "gpc1_to_stream_end_us=%llu stamp_to_stream_end_us=%llu period_us=16000 " ++ "before_alive=1 end=gpuperf\n", ++ b->perf_seed_mode, b->seed_stamp, ++ b->seed_gpc1 ? ++ div_u64((b->seed_gpc1 - seed_ts) * 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? div_u64((b->seed_gpc1 - b->seed_stamp) * ++ 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? ++ div_u64((end - b->seed_gpc1) * 1000000ULL, ++ rate) : ++ 0, ++ div_u64((end - b->seed_stamp) * 1000000ULL, rate)); ++ } ++ g5p_r73b_release_exit(g, ret); ++ g5p_ds_hook_release(g, ret); ++ if (ret) ++ return ret; ++ if (b->r73b && 1) ++ return g5p_r73c_fast_control(g); ++ return 0; ++} ++ ++static int g5p_boot_events_once(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 3 * HZ; ++ u64 submit_wait_start = 0; ++ u32 pass = 0, submit_wait_passes = 0; ++ int ret; ++ ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_EVENTS, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->alive)) { ++ if (b->r73b && 1) { ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ if (ret == -EAGAIN) ++ submit_wait_start = ++ ktime_get_mono_fast_ns(); ++ while (ret == -EAGAIN && ++ ktime_get_mono_fast_ns() - ++ submit_wait_start < ++ 100ULL * NSEC_PER_MSEC) { ++ usleep_range(50, 100); ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ submit_wait_passes++; ++ } ++ if (submit_wait_passes) ++ dev_info( ++ g->dev, ++ "R73C alive-submit synchronize wait_ns=%llu passes=%u ret=%d submitted=%u end=r73c\n", ++ ktime_get_mono_fast_ns() - ++ submit_wait_start, ++ submit_wait_passes, ret, ++ READ_ONCE( ++ b->control_submitted)); ++ if (ret) ++ return ret; ++ } ++ dev_info( ++ g->dev, ++ "G2 ALIVE type=0 word1=%08x events=%u IRQ174=%d\n", ++ b->alive_word, b->events, ++ atomic_read(&b->irqs)); ++ g5p_r41_bridge_firmware(g); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, ++ "G2 no alive event: events=%u IRQ174=%d; preserve baseline\n", ++ b->events, atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++/* recovery108: every ALIVE wait (release fast path and the events command) is an ALIVE event. */ ++static int g5p_boot_events(struct g5p_diag *g) ++{ ++ int ret = g5p_boot_events_once(g); ++ ++ /* wake112: no ALIVE wait ends with an R74E proof still owed (r74e112.h). */ ++ if (g5p_r74e_close(g)) { ++ dev_emerg( ++ g->dev, ++ "R74E112 close gen=%u why=untaken ret=%d end=r74e112\n", ++ g->boot->r73b->early_proof_gen, ret); ++ if (!ret) ++ ret = -EHOSTDOWN; ++ } ++ g5p_ds_hook_alive(g, ret); ++ return ret; ++} ++ ++static int g5p_boot_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b && b->stopped) ++ return -ESHUTDOWN; ++ if (sysfs_streq(command, "map")) ++ return g5p_boot_maps(g); ++ /* Read-only diagnostics remain available after a latched boot failure. */ ++ if (b && g->powered && g->fender_ready && g->id_checked) { ++ if (sysfs_streq(command, "snapshot")) { ++ g5p_boot_snapshot(g, "explicit", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ return 0; ++ } ++ if (sysfs_streq(command, "fifo")) ++ return g5p_boot_fifo_dump(g); ++ } ++ if (!boot_experiment || !b || !b->maps_ready || b->failed) ++ return -EPERM; ++ if (sysfs_streq(command, "arm")) { ++ ret = g5p_boot_arm(g); ++ } else if (sysfs_streq(command, "load") && b->armed && !b->released && ++ !b->copied) { ++ ret = g5p_boot_load(g); ++ g5p_ds_hook_load(g, ret); ++ } else if (sysfs_streq(command, "release") && b->copied) { ++ ret = g5p_boot_release(g); ++ } else if (sysfs_streq(command, "events") && b->released) { ++ ret = g5p_boot_events(g); ++ g5p_r73b_events_exit(g, ret); ++ } else { ++ return -EPERM; ++ } ++ if (ret && ret != -EALREADY) ++ b->failed = true; ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/compute-user-layout.h b/drivers/gpu/apple-g5p/compute-user-layout.h +new file mode 100644 +index 000000000..78b459832 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/compute-user-layout.h +@@ -0,0 +1,207 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CDM channel and per-job firmware objects. Layout helpers perform no I/O. ++ * The kernel owns the channel, control watermarks, pointer ring, work record, ++ * completion microsequence, timestamp queue, stamps and dependency object. ++ * GPU program instructions are provided by userspace, never embedded here. ++ * ++ * Compute uses the context's TA MMU slot. The scheduler serializes TA and CDM ++ * work across contexts, and a lost CDM job blocks the channel until retirement ++ * or recovery. Stamp identity zero is safe because the TA event latch is armed ++ * only while executing a TA phase; completion also requires coherent stamps. ++ */ ++#ifndef G5P_COMPUTE_USER_LAYOUT_H ++#define G5P_COMPUTE_USER_LAYOUT_H ++ ++#define G5P_CU_ENGINE 2U ++#define G5P_CU_OFFSET (G5P_R22_3D + 0x3000U) /* in the shared region */ ++#define G5P_CU_BYTES 0x1000U ++#define G5P_CU_W 0x400U ++#define G5P_CU_W_BYTES 0x290U /* through the timestamps at W+0x280..0x288 */ ++#define G5P_CU_SKU 0x800U ++#define G5P_CU_SKU_BYTES 0xc0U ++#define G5P_CU_TQ 0x900U ++#define G5P_CU_START 0x940U /* W258: the retirement mirror */ ++#define G5P_CU_END 0x944U /* W25c: finish33's stamp */ ++#define G5P_CU_DEP 0x980U ++#define G5P_CU_DEP_BYTES 0x280U ++#define G5P_CU_IDENTITY 0U ++#define G5P_CU_SEQ_MAX 0x7fffffffU ++ ++/* Firmware configuration W+0x244 and compute record type. */ ++#define G5P_CU_W244 0x00a00004U ++#define G5P_CU_TAG 3U ++ ++/* The validated register images of one job (compute-user-validate.h fills it). */ ++struct g5p_cu_regs { ++ u64 stream; /* W+0x70 -> R0+0x480 */ ++ u64 terminate; /* the stream's terminate word: W+0x214, W+0x23c */ ++ u64 state[4]; /* W+0x1ec..0x204 */ ++ u64 tail; /* W+0x60 -> R0+0x498, W+0x20c */ ++ u32 config[2]; /* W+0x10, W+0x14 -> R0+0x4c0, 0x4c8 */ ++ /* {code | data << 32}: W+0x21c, W+0x22c, W+0x68 -> R0+0x4b8 */ ++ u64 store, resume, term; ++ u32 store1, resume1; /* W+0x224 -> R0+0x4b0, W+0x234 */ ++}; ++ ++/* Job `seq` (1-based) occupies pointer-ring entry (seq - 1) % 16 and publishes seq % 16. */ ++static inline u32 g5p_cu_ring_index(u32 seq) ++{ ++ return (seq - 1) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_cu_endpoint(u32 seq) ++{ ++ return seq % G5P_WORK_SLOTS; ++} ++ ++/* The channel at its epoch start: header, ring capacity, timestamp queue, dependency object ++ * (g5p_compute_dependency()); watermarks, ring and stamps zero. `p` is the channel's ++ * G5P_CU_BYTES, `meta` its META address. ++ */ ++static inline void g5p_cu_channel_init(u8 *p, u32 meta) ++{ ++ u8 *d = p + G5P_CU_DEP; ++ ++ memset(p, 0, G5P_CU_BYTES); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_CU_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_CU_TQ + 4, meta + G5P_CU_TQ); ++ g5p_put64(d + 0x30, 1); ++ g5p_put64(d + 0x48, ~0ULL); ++ g5p_put32(d + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(d + off + 8, 0xf); ++} ++ ++/* The SKU microsequence of job `seq`: start32, timestamp, wait COMPUTE_FINISHED (event 4), ++ * wait (R0+0x4a0 & 1) == 1, timestamp, finish33 (W25c = W260), the backlink, end. ++ * Each sequence reference uses the current job number. ++ */ ++static inline u32 g5p_cu_sku(u8 *sku, u32 meta, u32 global, u32 slot, u32 seq) ++{ ++ u32 w = meta + G5P_CU_W, end = meta + G5P_CU_END; ++ const u32 words[] = { ++ 0x32, ++ w + 0x10, ++ w + 0x60, ++ global + 0x720, ++ meta, ++ slot, ++ 0, ++ 0, ++ 0, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ 0x80000018, ++ w + 0x278, ++ w + 0x280, ++ w + 0x280, ++ seq, ++ 0x101, ++ 0x004a000a, ++ 1, ++ 1, ++ 0x18, ++ w + 0x278, ++ w + 0x280, ++ w + 0x284, ++ seq, ++ 0x33, ++ global + 0x720, ++ meta, ++ slot, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ end, ++ seq, ++ (u32)-0x6c, ++ 0x40000017, ++ }; ++ ++ memset(sku, 0, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(words); i++) ++ g5p_put32(sku + i * 4, words[i]); ++ return ARRAY_SIZE(words) * 4; ++} ++ ++/* Every per-job object of job `seq` on slot `slot`: W, SKU and its pointer-ring entry. The ++ * channel header, the dependency object and the timestamp queue stay as initialised. ++ */ ++static inline void g5p_cu_job(u8 *p, u32 meta, u32 global, u32 slot, u32 seq, ++ const struct g5p_cu_regs *k) ++{ ++ u8 *w = p + G5P_CU_W; ++ ++ memset(w, 0, G5P_CU_W_BYTES); ++ g5p_put32(w, G5P_CU_TAG); ++ g5p_put32(w + 8, slot); ++ g5p_put32(w + 0xc, meta + G5P_CU_DEP); ++ g5p_put32(w + 0x10, k->config[0]); ++ g5p_put32(w + 0x14, k->config[1]); ++ g5p_put64(w + 0x60, k->tail); ++ g5p_put64(w + 0x68, k->term); ++ g5p_put64(w + 0x70, k->stream); ++ /* The six selected-resource entries at W+0x78..0xa7 stay zero (disabled). */ ++ g5p_put32(w + 0x1e0, meta + G5P_CU_SKU); ++ g5p_put32(w + 0x1e4, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x1ec + i * 8, k->state[i]); ++ g5p_put64(w + 0x20c, k->tail); ++ g5p_put64(w + 0x214, k->terminate); ++ g5p_put64(w + 0x21c, k->store); ++ g5p_put32(w + 0x224, k->store1); ++ g5p_put64(w + 0x22c, k->resume); ++ g5p_put32(w + 0x234, k->resume1); ++ g5p_put64(w + 0x23c, k->terminate); ++ g5p_put32(w + 0x244, G5P_CU_W244); ++ g5p_put32(w + 0x258, meta + G5P_CU_START); ++ g5p_put32(w + 0x25c, meta + G5P_CU_END); ++ g5p_put32(w + 0x260, seq); ++ g5p_put32(w + 0x264, G5P_CU_IDENTITY); ++ g5p_put32(w + 0x270, seq); ++ g5p_cu_sku(p + G5P_CU_SKU, meta, global, slot, seq); ++ g5p_put32(p + G5P_WORK_RING + g5p_cu_ring_index(seq) * 4, ++ meta + G5P_CU_W); ++} ++ ++/* Retirement of job `seq` (the r19d predicate, RAM only): W still names our stamps and ++ * sequence, finish33's stamp and the retirement mirror both at seq, all five channel ++ * watermarks at its endpoint. ++ */ ++static inline bool g5p_cu_retired(const u8 *p, u32 meta, u32 seq) ++{ ++ const u8 *w = p + G5P_CU_W; ++ u32 endpoint = g5p_cu_endpoint(seq); ++ ++ if (g5p_le32(w + 0x258) != meta + G5P_CU_START || ++ g5p_le32(w + 0x25c) != meta + G5P_CU_END || ++ g5p_le32(w + 0x260) != seq || g5p_le32(p + G5P_CU_START) != seq || ++ g5p_le32(p + G5P_CU_END) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* The stamps are the previous job's or ours, and no watermark runs past our endpoint's ++ * predecessor: anything else is a protocol error (another writer, or the wrong channel). ++ */ ++static inline bool g5p_cu_stamp_sane(const u8 *p, u32 seq) ++{ ++ u32 start = g5p_le32(p + G5P_CU_START), end = g5p_le32(p + G5P_CU_END); ++ ++ return (start == seq - 1 || start == seq) && ++ (end == seq - 1 || end == seq); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +new file mode 100644 +index 000000000..709efed16 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -0,0 +1,567 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared private bring-up implementation for mutually exclusive frontends. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++/* perf60 clock: which GPC states license an R0 access (a bell, a TLB pass, a register sample). */ ++/* release8 default */ ++ ++/* The GPC is powered and not mid-transition: state 1 only, or (gpc_any_pstate) any nonzero ++ * pstate nibble with no request in flight. A parked GPC (nibble 0) never qualifies. ++ */ ++static inline bool g5p_gpc_ok(u32 raw) ++{ ++ ; ++ ; ++ return !(raw & 0x10) && (raw & 0xf); ++} ++ ++/* The pure headers (readback.h, r74e112.h, render-user-wake.h, dirtystart108.h) judge the GPC state ++ * through G5P_GPC_ON(); their default is the old == 1, for the host tests. The kernel takes the rule. ++ */ ++#define G5P_GPC_ON(raw) g5p_gpc_ok(raw) ++ ++#include "pack.h" ++#include "work-layout.h" ++#include "mmio-licence.h" ++#include "gpu-oc.h" ++#include ++#include "readback.h" ++ ++/* nopatch: the TYPED_V3 boot path (r74/r74f/r74g, meta6c8=1) on Apple's original image. The release8 ++ * default since its device pass (2026-11-03, release8-nopatch a10a8556b9): an explicit firmware patch ++ * request (boot_r73f_case 1..4, or the r75/r79 cases that ride on it) takes the patched path instead, ++ * so the patched fallback is boot_r73f_case 1 alone (g5p_fw_unpatched_want()). ++ */ ++/* release8 default */ ++ ++/* The effective nopatch latch: the parameter, unless a patched image is asked for. */ ++static u32 g5p_fw_unpatched_want(u32 param, u32 r73f, u32 r75, u32 r79) ++{ ++ return param == 1 && !r73f && !r75 && !r79; ++} ++ ++#define G5P_MSLVDATAX 0x0a00 ++#define G5P_MSLVDATAT 0x0a08 ++#define G5P_MSLVCTRL0 0x0a10 ++#define G5P_MSLVCTRL1 0x0a18 ++#define G5P_MSLVIRQSTATUS 0x0ac8 ++#define G5P_SP_READY (BIT(24) | BIT(26)) ++#define G5P_FW_NAME "apple/agx-g5p-c0.ldr" ++ ++struct g5p_boot; ++ ++struct g5p_diag { ++ struct g5p_diagnostics *diagnostics; ++ struct g5p_boot *boot; ++ ++ spinlock_t slot_lock; ++ DECLARE_BITMAP(user_slots, 256); ++ struct device *dev; ++ void __iomem *window[2]; ++ void __iomem *rgx; ++ struct regmap *pmgr; ++ /* Serializes commands and all hardware access. */ ++ struct mutex lock; ++ bool powered; ++ bool loaded; ++ ++ struct g5p_ldr_image image; ++ bool fender_ready; ++ bool id_checked; ++ int last_result; ++ u32 words_written; ++ u32 core_id, core_revision; ++ u32 last_sp_addr; ++ bool reset_transition; ++ void (*event_notify)(void *cookie, const u32 words[11]); ++ void *event_cookie; ++ /* The GPU register licence (mmio.inc, docs/gpu/experiments/mmio-access.md §4). mmio_lock is ++ * a leaf: taken inside event_lock where both are held, never the reverse, nothing inside. ++ * mmio_off forbids R1/R0 through the accessor from before any PMGR gfx gate until Fender is ++ * up again (g5p_mmio_gate(), cleared by g5p_fender_wake_once()). ++ */ ++ spinlock_t mmio_lock; ++ bool mmio_off; ++ u32 mmio_max_pstate; ++ struct g5p_mmio_self *mmio_self; ++}; ++ ++/* Before every PMGR gfx gate (boot R33 off, dirty-start, kexec quiesce and in-boot recovery ++ * cycles): no accessor MMIO from here until g5p_fender_wake_once() brings Fender up again. ++ * Apple clears fender+0xc inside setPlatformPower(0), before its gate (G5). ++ */ ++static void g5p_mmio_gate(struct g5p_diag *g) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = true; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++} ++ ++/* gpu-recovery108: retire predicates (teardown.inc) and dirty-start hooks (dirtystart108.inc). */ ++static bool g5p_r8_r1_allowed(void); ++static bool g5p_r8_is_retired(void); ++static void g5p_ds_hook_probe(struct g5p_diag *g); ++static void g5p_ds_hook_power(struct g5p_diag *g); ++static int g5p_ds_hook_reset(struct g5p_diag *g); ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret); ++static int g5p_ds_hook_release_gate(struct g5p_diag *g); ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret); ++/* wake112: the reset generation of a deferred R74E proof (dirtystart108.inc). */ ++static u32 g5p_ds_gen(void); ++ ++#include "diagnostics.inc" ++ ++/* The GPU ADT clock/power gate124 resolves to ps-regs[4]+index8*8. ++ * Read via its existing syscon, sharing the provider mapping without writes. ++ */ ++static void g5p_power_observer_init(struct g5p_diag *g) ++{ ++ struct device_node *domain, *parent; ++ u32 offset; ++ ++ domain = of_parse_phandle(g->dev->of_node, "power-domains", 0); ++ if (!domain) ++ return; ++ parent = of_get_parent(domain); ++ if (of_device_is_compatible(domain, "apple,s8000-pmgr-pwrstate") && ++ !of_property_read_u32(domain, "reg", &offset) && ++ offset == 0x80340 && parent && ++ of_device_is_compatible(parent, "apple,s8000-pmgr")) { ++ g->pmgr = syscon_node_to_regmap(parent); ++ if (IS_ERR(g->pmgr)) ++ g->pmgr = NULL; ++ } ++ of_node_put(parent); ++ of_node_put(domain); ++} ++ ++/* pmgr-pwrstate.c masks; history flags are sticky, not current clock state. */ ++static void g5p_power_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) { ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx unavailable: provider not resolved\n", ++ stage); ++ return; ++ } ++ dev_info(g->dev, "POWER %s reading PMGR gfx20e080340 gate124\n", stage); ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (ret) { ++ dev_info(g->dev, "POWER %s PMGR gfx read error=%d\n", stage, ++ ret); ++ return; ++ } ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx20e080340=%08x target=%x actual=%x auto=%u auto_state=%x " ++ "min=%x reset=%u parent_off=%u disabled=%u; current outer-domain fields only\n", ++ stage, raw, raw & 0xf, (raw >> 4) & 0xf, (raw >> 28) & 1, ++ (raw >> 24) & 0xf, (raw >> 16) & 0xf, raw >> 31, ++ (raw >> 11) & 1, (raw >> 10) & 1); ++ dev_info( ++ g->dev, ++ "POWER %s PMGR history was_clkgated=%u was_pwrgated=%u; sticky history, not " ++ "current state\n", ++ stage, (raw >> 9) & 1, (raw >> 8) & 1); ++} ++ ++static int g5p_power(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->powered) ++ return 0; ++ dev_info(g->dev, "request gfx power domain\n"); ++ ret = pm_runtime_resume_and_get(g->dev); ++ if (ret < 0) ++ return ret; ++ g->powered = true; ++ dev_info(g->dev, "gfx runtime PM active\n"); ++ g5p_ds_hook_power(g); ++ return 0; ++} ++ ++#include "power-r33.inc" ++#include "power-r41.inc" ++ ++static bool boot_gfx_off_done; ++ ++static int g5p_fender_wake_once(struct g5p_diag *g) ++{ ++ void __iomem *fender = g->window[1]; ++ u32 value; ++ int ret; ++ ++ ret = g5p_power(g); ++ if (ret) ++ return ret; ++ /* recovery108 mode 3: the one bounded incoming cycle, before any GPU MMIO. */ ++ ret = g5p_ds_hook_reset(g); ++ if (ret) ++ return ret; ++ /* R33: once per boot, before the first Fender write (no GPU MMIO yet). */ ++ ; ++ ret = g5p_r41_bridge_prepare(g); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "W R1+10000=ffff0000 (Apple ...6c93fe8)\n"); ++ writel(0xffff0000, fender + 0x10000); ++ dev_info(g->dev, "W R1+00000=40000000 (Apple ...6c93ff0)\n"); ++ writel(0x40000000, fender); ++ dev_info(g->dev, "W R1+01000=00000011 (Apple ...6c93ff8)\n"); ++ writel(0x11, fender + 0x1000); ++ dev_info(g->dev, "poll R1+01000 bit4 clear\n"); ++ ret = readl_poll_timeout(fender + 0x1000, value, !(value & BIT(4)), 10, ++ 100000); ++ dev_info(g->dev, "Fender wake status=%08x result=%d\n", value, ret); ++ if (ret) ++ return ret; ++ usleep_range(10, 20); ++ g->fender_ready = true; ++ { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = false; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++ } ++ return 0; ++} ++ ++static int g5p_fender_wake(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->fender_ready) ++ return 0; ++ ret = g5p_fender_wake_once(g); ++ g5p_ds_hook_wake(g, ret); ++ return ret; ++} ++ ++static int g5p_sp_wait(struct g5p_diag *g) ++{ ++ u32 value; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = readl_poll_timeout(g->rgx + G5P_MSLVCTRL1, value, ++ (value & G5P_SP_READY) == G5P_SP_READY, 10, ++ 100000); ++ if (ret) ++ dev_err(g->dev, "slave port timeout CTRL1=%08x words=%u\n", ++ value, g->words_written); ++ if (ret) ++ dev_err(g->dev, "slave target=%08x\n", g->last_sp_addr); ++ return ret; ++} ++ ++static int g5p_meta_read(struct g5p_diag *g, u32 addr, u32 *out) ++{ ++ int ret; ++ ++ g->last_sp_addr = addr; ++ ret = g5p_sp_wait(g); ++ ++ if (ret) ++ return ret; ++ writel(addr | 1, g->rgx + G5P_MSLVCTRL0); ++ readl(g->rgx + G5P_MSLVCTRL0); ++ ret = g5p_sp_wait(g); ++ if (ret) ++ return ret; ++ *out = readl(g->rgx + G5P_MSLVDATAX); ++ return 0; ++} ++ ++static int g5p_firmware(struct g5p_diag *g, bool prepare) ++{ ++ static const u8 expected[SHA256_DIGEST_SIZE] = { ++ 0x33, 0xae, 0x20, 0xcb, 0xbe, 0x24, 0x98, 0xe4, ++ 0xf1, 0x19, 0xba, 0x4e, 0x9a, 0x55, 0x46, 0x7e, ++ 0x4e, 0x01, 0x92, 0xd4, 0xc8, 0x6d, 0x5b, 0xf1, ++ 0x90, 0x3a, 0xdc, 0xe6, 0x3f, 0xfa, 0x5d, 0xca, ++ }; ++ /* SHA-256 of the packed image of that LDR, unmodified (nopatch's receipt) */ ++ static const u8 g5p_fw_packed_sha[SHA256_DIGEST_SIZE] = { ++ 0x48, 0xea, 0x03, 0x14, 0x08, 0x23, 0x96, 0x23, ++ 0xd8, 0x6f, 0x79, 0xb8, 0xa5, 0xda, 0x55, 0x7e, ++ 0x25, 0x3d, 0xb1, 0x6f, 0xb6, 0x23, 0x47, 0x95, ++ 0xe8, 0xed, 0xba, 0xa5, 0xcd, 0x9b, 0xb9, 0x1a, ++ }; ++ const struct firmware *fw; ++ u8 digest[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ ret = request_firmware_direct(&fw, G5P_FW_NAME, g->dev); ++ if (ret) ++ return ret; ++ sha256(fw->data, fw->size, digest); ++ if (fw->size != 119112 || memcmp(digest, expected, sizeof(expected))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, NULL, NULL); ++ if (ret) ++ goto out; ++ if (prepare && !g->loaded) { ++ g->image.data = kvzalloc(G5P_IMAGE_SIZE, GFP_KERNEL); ++ if (!g->image.data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, g5p_pack_record, ++ &g->image); ++ if (!ret && g->image.config_count != G5P_CONFIG_WRITES) ++ ret = -EINVAL; ++ if (!ret) { ++ sha256(g->image.data, G5P_IMAGE_SIZE, digest); ++ if (memcmp(digest, g5p_fw_packed_sha, sizeof(digest))) ++ ret = -EBADMSG; ++ } ++ if (ret) { ++ kvfree(g->image.data); ++ memset(&g->image, 0, sizeof(g->image)); ++ goto out; ++ } ++ g->loaded = true; ++ dev_info( ++ g->dev, ++ "NOPATCH packed apple_original=1 packed_sha256=%32phN end=nopatch\n", ++ digest); ++ } ++out: ++ release_firmware(fw); ++ return ret; ++} ++ ++static int g5p_sample(struct g5p_diag *g) ++{ ++ u32 value; ++ unsigned int thread, reg; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ if (!g->rgx) ++ return -EINVAL; ++ dev_info(g->dev, "RGX CTRL1=%08x IRQSTATUS=%08x\n", ++ readl(g->rgx + G5P_MSLVCTRL1), ++ readl(g->rgx + G5P_MSLVIRQSTATUS)); ++ for (thread = 0; thread < 2; thread++) { ++ for (reg = 0; reg <= 0x10; reg += 0x10) { ++ ret = g5p_meta_read( ++ g, 0x04800000 + thread * 0x1000 + reg, &value); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "META T%u %s=%08x\n", thread, ++ reg ? "STATUS" : "ENABLE", value); ++ } ++ } ++ return 0; ++} ++ ++static int g5p_read_id(struct g5p_diag *g) ++{ ++ u32 words[2], version; ++ static const u32 offsets[] = { 0x20, 0x18 }; ++ unsigned int i; ++ ++ if (!g->fender_ready || READ_ONCE(g->reset_transition)) ++ return -EPERM; ++ g->id_checked = false; ++ for (i = 0; i < ARRAY_SIZE(offsets); i++) { ++ dev_info(g->dev, "R R0+%05x\n", offsets[i]); ++ words[i] = readl(g->rgx + offsets[i]); ++ dev_info(g->dev, "R0+%05x=%08x\n", offsets[i], words[i]); ++ } ++ g->core_revision = words[0]; ++ g->core_id = words[1]; ++ if ((!words[0] && !words[1]) || words[0] == ~0U || words[1] == ~0U) ++ return -ENODEV; ++ /* Match Apple readChipInfo at ...6ca644c before deeper META accesses. */ ++ version = (words[0] >> 8) & 0xff; ++ if ((words[0] & 0xff0000) != 0x90000 || ++ (version != 7 && version != 10 && version != 13 && version != 15)) ++ return -ENODEV; ++ g->id_checked = true; ++ return 0; ++} ++ ++#include "boot.inc" ++#include "mmio.inc" ++#include "kick.inc" ++#include "lifecycle.inc" ++#include "teardown.inc" ++#include "dirtystart108.inc" ++#include "recover.inc" ++#include "idle-power.inc" ++#include "transport-budget.inc" ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++ ++static ssize_t result_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ return sysfs_emit( ++ buf, ++ "last=%d loaded=%u maps=%u armed=%u copied=%u released=%u alive=%u " ++ "control_complete=%u failed=%u\n", ++ READ_ONCE(g->last_result), READ_ONCE(g->loaded), ++ b && b->maps_ready, b && b->armed, b && b->copied, ++ b && b->released, b && b->alive, b && b->control_complete, ++ b && b->failed); ++} ++static DEVICE_ATTR_RO(result); ++ ++/* Cached raw words from the last explicit/automatic snapshot: no GPU MMIO. */ ++ ++/* One bring-up phase, as the command file and the probe-time bring-up run it. *ran is set ++ * when the phase itself ran (and its result was recorded). ++ */ ++static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) ++{ ++ int ret; ++ ++ *ran = false; ++ if (g5p_r8_is_retired()) ++ return -ESHUTDOWN; ++ guard(mutex) ++ (&g->lock); ++ if (g->boot && (g->boot->stopped || g->boot->user_compute_owned)) ++ return -EBUSY; ++ if (sysfs_streq(buf, "validate") || sysfs_streq(buf, "prepare")) ++ ret = g5p_firmware(g, sysfs_streq(buf, "prepare")); ++ else if (sysfs_streq(buf, "wake")) ++ ret = g5p_fender_wake(g); ++ else if (sysfs_streq(buf, "id")) ++ ret = g5p_read_id(g); ++ else if (sysfs_streq(buf, "sample")) ++ ret = g->id_checked ? g5p_sample(g) : -EPERM; ++ else if (sysfs_streq(buf, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(buf, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(buf, "control-wait")) ++ ret = g5p_control_wait(g); ++ else if (sysfs_streq(buf, "map") || sysfs_streq(buf, "arm") || ++ sysfs_streq(buf, "load") || sysfs_streq(buf, "release") || ++ sysfs_streq(buf, "events")) ++ ret = g5p_boot_command(g, buf); ++ else ++ return -EOPNOTSUPP; ++ g->last_result = ret; ++ *ran = true; ++ return ret; ++} ++ ++static ssize_t command_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ bool ran; ++ int ret; ++ ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (sysfs_streq(buf, "td-kexec")) ++ return g5p_td_command(g, buf) ?: count; ++ ret = g5p_run_phase(g, buf, &ran); ++ if (ran) ++ dev_info(dev, "command %.*s result=%d\n", (int)count, buf, ret); ++ return ret ? ret : count; ++} ++static DEVICE_ATTR_WO(command); ++ ++static struct attribute *g5p_attrs[] = { ++ &dev_attr_command.attr, ++ &dev_attr_result.attr, ++ &dev_attr_recovery_status.attr, ++ &dev_attr_idle_power_status.attr, ++ NULL, ++}; ++static const struct attribute_group g5p_group = { .attrs = g5p_attrs }; ++__ATTRIBUTE_GROUPS(g5p); ++ ++static int g5p_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct g5p_diag *g; ++ unsigned int i; ++ ++ g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL); ++ if (!g) ++ return -ENOMEM; ++ g->dev = dev; ++ if (g5p_diagnostics_init(g)) ++ return -ENOMEM; ++ mutex_init(&g->lock); ++ spin_lock_init(&g->slot_lock); ++ spin_lock_init(&g->mmio_lock); ++ g->mmio_max_pstate = 1; ++ bitmap_set(g->user_slots, 0, 7); ++ for (i = 0; i < 2; i++) { ++ g->window[i] = devm_platform_ioremap_resource(pdev, i); ++ if (IS_ERR(g->window[i])) ++ return PTR_ERR(g->window[i]); ++ } ++ g->rgx = g->window[0]; ++ g5p_power_observer_init(g); ++ platform_set_drvdata(pdev, g); ++ if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40))) ++ return -ENODEV; ++ pm_runtime_enable(dev); ++ g5p_td_attach(g); ++ g5p_ds_hook_probe(g); ++ ; ++ dev_info(dev, "mapped both ADT windows; awaiting explicit command\n"); ++ return 0; ++} ++ ++static void g5p_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ /* With teardown on, keep the PM reference: an outgoing PMGR off is unvalidated. */ ++ bool keep_power; ++ ++ ; ++ keep_power = g5p_td_remove(g); ++ if (g->powered && !keep_power) { ++ g5p_mmio_gate(g); ++ pm_runtime_put_sync(g->dev); ++ } ++ pm_runtime_disable(g->dev); ++ kvfree(g->image.data); ++} ++ +diff --git a/drivers/gpu/apple-g5p/diagnostics.inc b/drivers/gpu/apple-g5p/diagnostics.inc +new file mode 100644 +index 000000000..184ce954b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/diagnostics.inc +@@ -0,0 +1,181 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Cached append-only text. Readers never take the command lock or touch MMIO. ++ * Preserve earliest records on overflow, expose dropped bytes explicitly. ++ */ ++#define G5P_BLOB_SIZE SZ_4M ++#define G5P_MMU_HISTORY_SIZE SZ_32K ++ ++struct g5p_diagnostics { ++ spinlock_t lock; ++ u32 used, history_used; ++ u64 dropped, history_dropped; ++ char stage[40]; ++ u32 next_register; ++ bool reading; ++ char history[G5P_MMU_HISTORY_SIZE]; ++ char blob[G5P_BLOB_SIZE]; ++}; ++ ++static void g5p_diagnostics_free(void *data) ++{ ++ kvfree(data); ++} ++ ++static int g5p_diagnostics_init(struct g5p_diag *g) ++{ ++ g->diagnostics = kvzalloc(sizeof(*g->diagnostics), GFP_KERNEL); ++ if (!g->diagnostics) ++ return -ENOMEM; ++ spin_lock_init(&g->diagnostics->lock); ++ return devm_add_action_or_reset(g->dev, g5p_diagnostics_free, ++ g->diagnostics); ++} ++ ++static void g5p_diagnostic_append(struct g5p_diag *g, bool history, ++ const char *text, size_t len) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ u32 *used = history ? &d->history_used : &d->used; ++ u64 *dropped = history ? &d->history_dropped : &d->dropped; ++ u32 capacity = history ? G5P_MMU_HISTORY_SIZE : G5P_BLOB_SIZE; ++ char *dst = history ? d->history : d->blob; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ /* Keep complete lines; once full, never replace earlier evidence. */ ++ if (*dropped || len > capacity - *used) { ++ *dropped += len; ++ } else { ++ memcpy(dst + *used, text, len); ++ *used += len; ++ } ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_blob_printf(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ char line[512]; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, false, line, len); ++} ++ ++static void g5p_blob_hex(struct g5p_diag *g, const char *name, const u8 *data, ++ u32 len) ++{ ++ for (u32 off = 0; off < len; off += 16) { ++ char text[80]; ++ ++ hex_dump_to_buffer(data + off, min_t(u32, len - off, 16), 16, 4, ++ text, sizeof(text), false); ++ g5p_blob_printf(g, "%s: %04x: %s\n", name, off, text); ++ } ++} ++ ++/* R1+1000 uses synthetic address11000; other offsets denote R0. */ ++static void g5p_mmu_progress(struct g5p_diag *g, const char *stage, u32 next) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ strscpy(d->stage, stage, sizeof(d->stage)); ++ d->next_register = next; ++ d->reading = true; ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_mmu_record(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ dev_info(g->dev, "%s", line); ++} ++ ++/* The routine per-job and per-event lines (a job's publish, wake and success receipt, the firmware's ++ * alive and stamp events). At desktop job rates they are hundreds of lines a second, more than netconsole ++ * can send: its queue grew to 700 MB of unreclaimable slab (release8, 55 min). Faults, losses, ++ * recoveries, terminal latches and every failed job keep their own levels and always print. ++ * job_log 0 dev_dbg (dynamic debug: off unless enabled in dynamic_debug/control) ++ * 1 KERN_DEBUG: in dmesg for the parsers (RENDER-USER job=, RENDER publish), below the ++ * default console loglevel, so not on a console or netconsole ++ * 2 KERN_INFO as before (short netconsole traces only: floods at desktop job rates) ++ */ ++static unsigned int job_log = 0; ++module_param(job_log, uint, 0600); ++MODULE_PARM_DESC( ++ job_log, ++ "default0 routine per-job/per-event lines are dev_dbg; 1 KERN_DEBUG (dmesg, no " ++ "console); 2 KERN_INFO as before (floods netconsole)"); ++ ++#define g5p_job_printk(dev, fmt, ...) \ ++ do { \ ++ switch (READ_ONCE(job_log)) { \ ++ case 0: \ ++ dev_dbg(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ case 1: \ ++ dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ default: \ ++ dev_info(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ } \ ++ } while (0) ++ ++/* g5p_mmu_record for a routine per-job line: the blob keeps it, the console sees it only at job_log 2. */ ++static __printf(2, 3) __maybe_unused ++ void g5p_mmu_record_job(struct g5p_diag *g, const char *fmt, ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ g5p_job_printk(g->dev, "%s", line); ++} ++ ++/* A routine event's raw words at the job_log level. */ ++static __maybe_unused void g5p_job_hex(const char *prefix, const void *buf, ++ size_t len) ++{ ++ switch (READ_ONCE(job_log)) { ++ case 0: ++ print_hex_dump_debug(prefix, DUMP_PREFIX_NONE, 16, 4, buf, len, ++ false); ++ break; ++ case 1: ++ print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ default: ++ print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ } ++} +diff --git a/drivers/gpu/apple-g5p/dirtystart108.h b/drivers/gpu/apple-g5p/dirtystart108.h +new file mode 100644 +index 000000000..fcf7432fa +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.h +@@ -0,0 +1,454 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DIRTYSTART108_H ++#define G5P_DIRTYSTART108_H ++ ++#include "bits-compat.h" ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * G5P incoming dirty-start state machine (series gpu-recovery108, patch 0007). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 4. ++ * ++ * The bootstrap is stepped by userspace (map, load, release, events, ++ * control), so this is an event-driven machine, not one run: the glue feeds ++ * one event per driver step with the step's result and RAM-only inputs, and ++ * asks the machine at two gates (before release, at the DRM claim). It ++ * assumes nothing pristine: the origin of the power state is classified from ++ * the kexec flag, the requested incoming PMGR cycle and the PMGR word, and ++ * every step's own result is evidence. ++ * ++ * PROBE origin inputs: booted from kexec, kexec_gpu_cycle requested ++ * POWER the PMGR gfx word, read once after the PM reference (PMGR read) ++ * RESET the bounded incoming power cycle (g5p_r33_gfx_off(), one attempt ++ * per boot, only in mode 3, only before the first wrapper wake) ++ * WAKE the Fender wake result ++ * MAPS fresh shared state: every ring control word in range (after the wake: ++ * map needs the core id, read through the woken wrapper) ++ * LOAD the firmware image memcmp of the RAM copy (b->copied) ++ * RELEASE the reset-stream/release result, the R74E power result ++ * ALIVE the first ALIVE wait result and, when read, the first-ALIVE ++ * GPC word (r73e); 0x10/0x11 is the C3 closed gate ++ * CLAIM the DRM claim asks; answers from the ALIVE proof ++ * ++ * Modes (parameter dirty_start): ++ * 0 observe: receipts only, both gates open (combined107 behaviour) ++ * 1 gate (default): fail closed on evidence the driver itself produced (a ++ * failed or out-of-order step, a power abort, a GPC word other than ++ * 0x01 at the first ALIVE, a ring out of range); a claim needs ALIVE ++ * proven in the current reset generation. Equal to combined107 on every ++ * bootstrap that succeeds. ++ * 2 strict: also refuse release and claim for an origin that is not cold ++ * and not a witnessed cycle (warm-unproven, warm-uncycled, unknown), ++ * unless a reset succeeded in this generation ++ * 3 strict + reset: as 2, and the glue may spend the one RESET attempt ++ * on such an origin before the wrapper wake (destructive: PMGR write) ++ * ++ * Generations: gen starts at 1 at PROBE and steps once per RESET attempt. A ++ * reset voids every proof of the previous generation; ALIVE proven in an ++ * earlier generation never exposes. gen never decreases. ++ * ++ * Pure policy: no register access, no kernel API except scnprintf(). Each ++ * event emits exactly one DS108 line through the emit callback. ++ */ ++ ++enum g5p_ds_ev { ++ G5P_DSE_PROBE, ++ G5P_DSE_POWER, ++ G5P_DSE_RESET, ++ G5P_DSE_WAKE, ++ G5P_DSE_MAPS, ++ G5P_DSE_LOAD, ++ G5P_DSE_RELEASE, ++ G5P_DSE_ALIVE, ++ G5P_DSE_CLAIM, ++ G5P_DSE_RECOVER, /* an in-boot recovery cycled the gate after a proven epoch */ ++ G5P_DSE_N, ++}; ++ ++static const char *const g5p_ds_ev_name[G5P_DSE_N] = { ++ "probe", "power", "reset", "wake", "maps", ++ "load", "release", "alive", "claim", "recover", ++}; ++ ++enum g5p_ds_origin { ++ G5P_DSO_UNKNOWN, /* not probed, or the PMGR word could not be read */ ++ G5P_DSO_COLD, /* not booted from kexec */ ++ G5P_DSO_WARM_CYCLED, /* kexec, incoming cycle requested and witnessed (was_pwrgated) */ ++ G5P_DSO_WARM_UNPROVEN, /* kexec, cycle requested, no witness in the PMGR word */ ++ G5P_DSO_WARM_UNCYCLED, /* kexec, no incoming cycle requested */ ++ G5P_DSO_N, ++}; ++ ++static const char *const g5p_ds_origin_name[G5P_DSO_N] = { ++ "unknown", "cold", "warm-cycled", "warm-unproven", "warm-uncycled", ++}; ++ ++enum g5p_ds_why { ++ G5P_DSW_NONE, ++ G5P_DSW_PROTOCOL, /* a step out of order (its predecessor not ok in this gen) */ ++ G5P_DSW_WAKE, /* the wrapper wake failed */ ++ G5P_DSW_MAPS, /* map failed or a ring control word out of range */ ++ G5P_DSW_LOAD, /* the RAM image is not the firmware (memcmp) */ ++ G5P_DSW_RELEASE, /* release / reset stream failed */ ++ G5P_DSW_POWER_ABORT, /* the R74E power poll or the first-ALIVE gate aborted */ ++ G5P_DSW_GPC, /* first-ALIVE GPC word not 0x01 (0x10/0x11: C3) */ ++ G5P_DSW_ALIVE, /* no ALIVE, or the ALIVE wait failed */ ++ G5P_DSW_RESET, /* the incoming power cycle failed: power state unknown */ ++ G5P_DSW_N, ++}; ++ ++static const char *const g5p_ds_why_name[G5P_DSW_N] = { ++ "none", "protocol", "wake", "maps", "load", ++ "release", "power-abort", "gpc", "alive", "reset", ++}; ++ ++/* Gate answers (not latched): why a gate is closed right now. */ ++enum g5p_ds_gate { ++ G5P_DSG_OPEN, ++ G5P_DSG_FAILED, /* the fail-closed latch */ ++ G5P_DSG_NOT_PROVEN, /* no ALIVE proof in the current generation */ ++ G5P_DSG_ORIGIN, /* strict: origin neither cold nor a witnessed cycle */ ++ G5P_DSG_STATE, /* the claim snapshot contradicts the proof */ ++ G5P_DSG_N, ++}; ++ ++static const char *const g5p_ds_gate_name[G5P_DSG_N] = { ++ "open", "failed", "not-proven", "origin", "state", ++}; ++ ++#define G5P_DS_MODE_MAX 3 ++#define G5P_DS_GPC_ON 0x01u ++#define G5P_DS_PMGR_PWRGATED \ ++ BIT(8) /* sticky was_pwrgated history bit of 0x80340 */ ++ ++#define G5P_DSF_RINGS_BUSY \ ++ BIT(0) /* a ring not empty at map (data, not a failure) */ ++#define G5P_DSF_ALIVE_LATE \ ++ BIT(1) /* ALIVE first seen at the claim (no ALIVE event) */ ++#define G5P_DSF_RESET_RAN BIT(2) /* a RESET attempt ran this boot */ ++#define G5P_DSF_RESET_OK \ ++ BIT(3) /* ... and succeeded (gen of the success = gen) */ ++/* a second attempt or one after the wake was refused */ ++#define G5P_DSF_RESET_REFUSED BIT(4) ++#define G5P_DSF_DUP BIT(5) /* an event repeated (recorded, not a failure) */ ++#define G5P_DSF_PMGR_UNREAD BIT(6) /* the PMGR word could not be read */ ++ ++/* One flat input record; each event reads only its own fields (RAM or step results). */ ++struct g5p_ds_in { ++ bool kexec; /* PROBE: chosen linux,booted-from-kexec */ ++ bool cycle_requested; /* PROBE: apple_pmgr.kexec_gpu_cycle=1 on the cmdline */ ++ bool pmgr_ok; /* POWER: the PMGR read succeeded */ ++ u32 pmgr_raw; /* POWER: the gfx word */ ++ int ret; /* RESET/WAKE/MAPS/LOAD/RELEASE/ALIVE: the step's return */ ++ u32 rings_bad; /* MAPS: control words > 255 */ ++ u32 rings_busy; /* MAPS: rings not empty */ ++ int power_ret; /* RELEASE: R74E power result */ ++ u32 power_raw; /* RELEASE: its last R1+0x1000 sample */ ++ bool power_abort; /* RELEASE/ALIVE/CLAIM: r73b early_power_abort */ ++ bool gpc_valid; /* ALIVE: the r73e first-ALIVE word was read */ ++ u32 gpc; /* ALIVE: that word */ ++ bool alive, released, failed, ++ control_complete; /* ALIVE/CLAIM snapshot */ ++}; ++ ++struct g5p_ds { ++ u32 gen; /* reset generation, monotonic, 0 before PROBE */ ++ u32 seen, ok; /* event bitmasks, current generation */ ++ u32 flags; ++ enum g5p_ds_origin origin; ++ bool kexec, cycle; /* PROBE inputs, kept for POWER */ ++ u32 pmgr_raw; ++ u32 resets; /* RESET attempts this boot (at most one runs) */ ++ u32 reset_ok_gen; /* gen in which a RESET succeeded, 0 never */ ++ u32 alive_gen; /* gen in which ALIVE was proven, 0 never */ ++ bool failed; /* fail-closed latch (absorbing) */ ++ enum g5p_ds_why why; ++ u32 events, lines, claims, refused; ++ enum g5p_ds_ev last; ++ enum g5p_ds_gate last_gate; ++}; ++ ++static inline bool g5p_ds_ok(const struct g5p_ds *d, enum g5p_ds_ev e) ++{ ++ return d->ok & (1u << e); ++} ++ ++static inline void g5p_ds_fail(struct g5p_ds *d, enum g5p_ds_why why) ++{ ++ if (!d->failed) ++ d->why = why; ++ d->failed = true; ++} ++ ++static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool cycle, ++ bool pmgr_ok, u32 raw) ++{ ++ if (!pmgr_ok) ++ return G5P_DSO_UNKNOWN; ++ if (!kexec) ++ return G5P_DSO_COLD; ++ if (!cycle) ++ return G5P_DSO_WARM_UNCYCLED; ++ return (raw & G5P_DS_PMGR_PWRGATED) ? G5P_DSO_WARM_CYCLED : ++ G5P_DSO_WARM_UNPROVEN; ++} ++ ++/* The origin is trusted (cold, witnessed cycle) or a reset succeeded in this generation. */ ++static inline bool g5p_ds_origin_trusted(const struct g5p_ds *d) ++{ ++ return d->origin == G5P_DSO_COLD || d->origin == G5P_DSO_WARM_CYCLED || ++ (d->reset_ok_gen && d->reset_ok_gen == d->gen); ++} ++ ++/* Should the glue spend the one RESET attempt now (mode 3, before the wake)? */ ++static inline bool g5p_ds_reset_wanted(const struct g5p_ds *d, u32 mode) ++{ ++ return mode >= 3 && !d->failed && !d->resets && ++ !(d->seen & (1u << G5P_DSE_WAKE)) && ++ (d->seen & (1u << G5P_DSE_POWER)) && !g5p_ds_origin_trusted(d); ++} ++ ++/* The release gate (asked by the glue right before the reset stream runs). */ ++static inline enum g5p_ds_gate g5p_ds_release_gate(const struct g5p_ds *d, ++ u32 mode) ++{ ++ if (!mode) ++ return G5P_DSG_OPEN; ++ if (d->failed) ++ return G5P_DSG_FAILED; ++ if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ return G5P_DSG_ORIGIN; ++ return G5P_DSG_OPEN; ++} ++ ++/* The ALIVE proof the claim needs: proven in this generation, nothing failed since. */ ++static inline bool g5p_ds_proven(const struct g5p_ds *d) ++{ ++ return !d->failed && d->alive_gen && d->alive_gen == d->gen && ++ g5p_ds_ok(d, G5P_DSE_RELEASE) && g5p_ds_ok(d, G5P_DSE_LOAD); ++} ++ ++static inline int g5p_ds_line(struct g5p_ds *d, char *buf, size_t n, ++ enum g5p_ds_ev e, int ret, u32 a, u32 b) ++{ ++ d->lines++; ++ return scnprintf(buf, n, ++ "DS108 gen=%u ev=%s ret=%d origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x pmgr=%08x a=%x b=%x gate=%s", ++ d->gen, g5p_ds_ev_name[e], ret, g5p_ds_origin_name[d->origin], ++ d->failed, g5p_ds_why_name[d->why], d->seen, d->ok, d->flags, ++ d->pmgr_raw, a, b, g5p_ds_gate_name[d->last_gate]); ++} ++ ++/* ++ * Feed one event. Returns, for CLAIM, 0 (expose) or -EAGAIN/-EIO (refuse); ++ * for every other event 0. mode is the current dirty_start parameter; at mode 0 ++ * the claim answers 0 whatever the state (the legacy driver decides alone). ++ * line/n receive the event's receipt (always written). ++ */ ++static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in, u32 mode, char *line, ++ size_t n) ++{ ++ u32 bit = 1u << e, a = 0, b = 0; ++ int ret = 0; ++ ++ if (mode > G5P_DS_MODE_MAX) ++ mode = G5P_DS_MODE_MAX; ++ d->events++; ++ d->last = e; ++ if ((d->seen & bit) && e != G5P_DSE_CLAIM) ++ d->flags |= G5P_DSF_DUP; ++ switch (e) { ++ case G5P_DSE_PROBE: ++ /* The only event that may start a generation: probe runs once per boot. */ ++ if (!d->gen) ++ d->gen = 1; ++ d->origin = G5P_DSO_UNKNOWN; ++ a = in->kexec; ++ b = in->cycle_requested; ++ d->kexec = in->kexec; ++ d->cycle = in->cycle_requested; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_POWER: ++ if (!(d->seen & (1u << G5P_DSE_PROBE))) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->origin = g5p_ds_origin_of(d->kexec, d->cycle, in->pmgr_ok, ++ in->pmgr_raw); ++ if (!in->pmgr_ok) ++ d->flags |= G5P_DSF_PMGR_UNREAD; ++ d->pmgr_raw = in->pmgr_ok ? in->pmgr_raw : 0; ++ a = in->pmgr_ok; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RESET: ++ a = in->ret; ++ if (d->resets || (d->seen & (1u << G5P_DSE_WAKE))) { ++ /* The glue must not have run it; record a refused attempt and fail ++ * closed. ++ */ ++ d->flags |= G5P_DSF_RESET_REFUSED; ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->resets++; ++ d->flags |= G5P_DSF_RESET_RAN; ++ /* A reset voids the previous generation's proofs (power epoch boundary). */ ++ d->gen++; ++ /* Unreachable today (a reset runs before the wake, so before every later ++ * step); kept so that a reordered glue cannot carry a proof across it. ++ */ ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->flags |= G5P_DSF_RESET_OK; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_WAKE: ++ d->seen |= bit; ++ if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_WAKE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_MAPS: ++ d->seen |= bit; ++ a = in->rings_bad; ++ b = in->rings_busy; ++ if (in->rings_busy) ++ d->flags |= G5P_DSF_RINGS_BUSY; ++ /* map needs the core id, read through the woken wrapper */ ++ if (!g5p_ds_ok(d, G5P_DSE_WAKE)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret || in->rings_bad) ++ g5p_ds_fail(d, G5P_DSW_MAPS); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_LOAD: ++ d->seen |= bit; ++ if (!g5p_ds_ok(d, G5P_DSE_MAPS)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_LOAD); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RELEASE: ++ d->seen |= bit; ++ a = (u32)in->power_ret; ++ b = in->power_raw; ++ if (!g5p_ds_ok(d, G5P_DSE_LOAD)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->power_abort || in->power_ret) ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_RELEASE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_ALIVE: ++ d->seen |= bit; ++ a = in->gpc_valid; ++ b = in->gpc; ++ if (!g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ } else if (in->power_abort) { ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ } else if (in->gpc_valid && !G5P_GPC_ON(in->gpc)) { ++ g5p_ds_fail(d, G5P_DSW_GPC); ++ } else if (in->ret || !in->alive || in->failed) { ++ g5p_ds_fail(d, G5P_DSW_ALIVE); ++ } else { ++ d->ok |= bit; ++ d->alive_gen = d->gen; ++ } ++ break; ++ case G5P_DSE_RECOVER: ++ /* The render frontend's in-boot recovery (recover.inc): the gate was cycled ++ * after this generation's release, with the firmware frozen. Like a RESET it ++ * starts a generation that voids every proof of the last one, and it is a ++ * witnessed cycle for the origin gate; the bootstrap's steps then run again ++ * and must each succeed in the new generation. Not after a failure, and not ++ * before a release: there is no epoch to recover then. ++ */ ++ a = in->ret; ++ if (d->failed || !g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->gen++; ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ /* The old ALIVE proof is void; the claim's late path may prove the new one. */ ++ d->alive_gen = 0; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_CLAIM: ++ d->claims++; ++ d->seen |= bit; ++ a = in->alive | in->released << 1 | in->failed << 2 | ++ in->control_complete << 3 | in->power_abort << 4; ++ /* ALIVE reached through a path without the ALIVE event: accept the RAM ++ * latch as the proof only after a successful release of this generation. ++ */ ++ if (!d->failed && !d->alive_gen && in->alive && !in->failed && ++ !in->power_abort && g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ d->alive_gen = d->gen; ++ d->ok |= 1u << G5P_DSE_ALIVE; ++ d->flags |= G5P_DSF_ALIVE_LATE; ++ } ++ if (!mode) ++ d->last_gate = G5P_DSG_OPEN; ++ else if (d->failed) ++ d->last_gate = G5P_DSG_FAILED; ++ else if (!g5p_ds_proven(d)) ++ d->last_gate = G5P_DSG_NOT_PROVEN; ++ else if (!in->alive || !in->released || in->failed || ++ in->power_abort || !in->control_complete) ++ d->last_gate = G5P_DSG_STATE; ++ else if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ d->last_gate = G5P_DSG_ORIGIN; ++ else ++ d->last_gate = G5P_DSG_OPEN; ++ if (d->last_gate == G5P_DSG_OPEN) { ++ d->ok |= bit; ++ } else { ++ d->refused++; ++ ret = d->last_gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++ } ++ break; ++ default: ++ break; ++ } ++ g5p_ds_line(d, line, n, e, e == G5P_DSE_CLAIM ? ret : in->ret, a, b); ++ return ret; ++} ++ ++ ++ ++static inline void g5p_ds_init(struct g5p_ds *d) ++{ ++ *d = (struct g5p_ds){ .last = G5P_DSE_N }; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +new file mode 100644 +index 000000000..be023e7e7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -0,0 +1,280 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the incoming dirty-start machine (dirtystart108.h), series ++ * gpu-recovery108 (0007). Design: docs/gpu/experiments/recovery108-driver.md. ++ * ++ * Included from core.inc after teardown.inc, so it may use the teardown ring ++ * read (RAM), g5p_r33_gfx_off() and the retire layer's state. Patch 0008 calls ++ * the hooks from the driver steps; until then nothing here runs. ++ * ++ * Register access added by this file: one PMGR read of the gfx word after the ++ * PM reference (POWER, the same read g5p_power_snapshot() makes), and in mode 3 ++ * only one bounded PMGR cycle through g5p_r33_gfx_off() before the first ++ * wrapper wake. Every other input is RAM or a step's own return value. ++ */ ++#include ++#include "dirtystart108.h" ++ ++static struct g5p_ds g5p_ds_state = { .last = G5P_DSE_N }; ++static DEFINE_SPINLOCK(g5p_ds_lock); ++ ++/* Latched from /chosen at probe, before POWER or the first wrapper wake. */ ++static u32 g5p_ds_boot_mode = 1; ++ ++static u32 g5p_ds_mode(void) ++{ ++ return READ_ONCE(g5p_ds_boot_mode); ++} ++ ++/* One event, one DS108 line (dev_emerg: netconsole at any loglevel). */ ++static int g5p_ds_feed(struct g5p_diag *g, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in) ++{ ++ unsigned long flags; ++ char line[224]; ++ int ret; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ ret = g5p_ds_event(&g5p_ds_state, e, in, g5p_ds_mode(), line, ++ sizeof(line)); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "%s\n", line); ++ return ret; ++} ++ ++/* wake112: the current reset generation, recorded with a deferred R74E proof (RAM, leaf lock). */ ++static u32 g5p_ds_gen(void) ++{ ++ unsigned long flags; ++ u32 gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return gen; ++} ++ ++/* The last apple_pmgr.kexec_gpu_cycle= token of the chosen bootargs (the one the kernel obeys). */ ++static bool g5p_ds_cycle_requested(struct device_node *chosen) ++{ ++ static const char key[] = "apple_pmgr.kexec_gpu_cycle="; ++ const char *args, *p, *last = NULL; ++ char tok[8]; ++ bool v = false; ++ size_t n; ++ ++ if (!chosen || of_property_read_string(chosen, "bootargs", &args)) ++ return false; ++ for (p = strstr(args, key); p; p = strstr(p + 1, key)) ++ if (p == args || p[-1] == ' ') ++ last = p; ++ if (!last) ++ return false; ++ last += sizeof(key) - 1; ++ n = strcspn(last, " "); ++ if (!n || n >= sizeof(tok)) ++ return false; ++ memcpy(tok, last, n); ++ tok[n] = 0; ++ return !kstrtobool(tok, &v) && v; ++} ++ ++static void g5p_ds_hook_probe(struct g5p_diag *g) ++{ ++ struct device_node *chosen = of_find_node_by_path("/chosen"); ++ struct g5p_ds_in in = {}; ++ ++ in.kexec = chosen && ++ of_property_read_bool(chosen, "linux,booted-from-kexec"); ++ in.cycle_requested = g5p_ds_cycle_requested(chosen); ++ of_node_put(chosen); ++ /* Retained warm state needs strict origin checks and the bounded reset. */ ++ WRITE_ONCE(g5p_ds_boot_mode, in.kexec ? 3 : 1); ++ dev_info(g->dev, "DS108 boot mode=%u kexec=%u\n", ++ g5p_ds_mode(), in.kexec); ++ g5p_ds_feed(g, G5P_DSE_PROBE, &in); ++} ++ ++/* After the PM reference is taken: one PMGR read (the gfx word), no write. */ ++static void g5p_ds_hook_power(struct g5p_diag *g) ++{ ++ struct g5p_ds_in in = {}; ++ u32 raw = 0; ++ ++ in.pmgr_ok = g->pmgr && !regmap_read(g->pmgr, 0x80340, &raw); ++ in.pmgr_raw = raw; ++ g5p_ds_feed(g, G5P_DSE_POWER, &in); ++} ++ ++/* Before the first wrapper wake. Mode 3 only, one attempt per boot, through the ++ * r33 helper (its own refusal rules: min/reset/auto or not on; every exit leaves ++ * the gate requested on). Returns the cycle's error; the wake then fails closed. ++ */ ++static int g5p_ds_hook_reset(struct g5p_diag *g) ++{ ++ u32 ms = clamp_t(u32, (10), 1, G5P_R33_GFX_OFF_MAX_MS); ++ struct g5p_ds_in in = {}; ++ unsigned long flags; ++ bool want; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ want = g5p_ds_reset_wanted(&g5p_ds_state, g5p_ds_mode()); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ if (!want) ++ return 0; ++ g5p_mmio_gate(g); ++ in.ret = g5p_r33_gfx_off(g->dev, g->pmgr, ms); ++ g5p_ds_feed(g, G5P_DSE_RESET, &in); ++ boot_gfx_off_done = true; /* one PMGR cycle per boot, whoever runs it */ ++ return in.ret; ++} ++ ++/* The legacy boot_gfx_off_ms cycle (R33) is the same mechanism: record it as the reset. */ ++ ++/* An in-boot recovery's gate cycle (recover.inc): a new generation, or a failed machine. */ ++static int g5p_ds_hook_recover(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ unsigned long flags; ++ bool failed; ++ ++ g5p_ds_feed(g, G5P_DSE_RECOVER, &in); ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ failed = g5p_ds_state.failed; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return failed ? -EIO : 0; ++} ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_WAKE, &in); ++} ++ ++/* Fresh shared state: every ring control word in range (RAM, the teardown ring read). */ ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ u32 i, cons, prod; ++ ++ for (i = 0; !ret && i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ in.rings_bad++; ++ continue; ++ } ++ if (cons > 255 || prod > 255) ++ in.rings_bad++; ++ else if (cons != prod) ++ in.rings_busy++; ++ } ++ g5p_ds_feed(g, G5P_DSE_MAPS, &in); ++} ++ ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_LOAD, &in); ++} ++ ++/* Asked before the release touches anything. -EIO: the fail-closed latch; -EAGAIN: strict origin. */ ++static int g5p_ds_hook_release_gate(struct g5p_diag *g) ++{ ++ enum g5p_ds_gate gate; ++ unsigned long flags; ++ u32 mode = g5p_ds_mode(), gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gate = g5p_ds_release_gate(&g5p_ds_state, mode); ++ g5p_ds_state.last_gate = gate; ++ if (gate != G5P_DSG_OPEN) ++ g5p_ds_state.refused++; ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "DS108 gen=%u gate=release mode=%u answer=%s\n", gen, ++ mode, g5p_ds_gate_name[gate]); ++ if (gate == G5P_DSG_OPEN) ++ return 0; ++ return gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++} ++ ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (r) { ++ in.power_ret = r->early_power_ret; ++ in.power_raw = r->early_power_raw; ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ } ++ g5p_ds_feed(g, G5P_DSE_RELEASE, &in); ++} ++ ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed); ++ } ++ if (r) { ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ /* r73e reads R1+0x1000 at the first ALIVE; 0 means not read. */ ++ in.gpc = READ_ONCE(r->early_state); ++ in.gpc_valid = 1 && in.gpc; ++ } ++ g5p_ds_feed(g, G5P_DSE_ALIVE, &in); ++} ++ ++/* The DRM claim: 0 expose, -EAGAIN not proven / strict origin, -EIO the fail-closed latch. */ ++static int __maybe_unused g5p_ds_hook_claim(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_ds_in in = {}; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ in.control_complete = READ_ONCE(b->control_complete); ++ in.power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ return g5p_ds_feed(g, G5P_DSE_CLAIM, &in); ++} ++ ++/* RAM only: the retire layer and the dirty-start machine, for capture without a log. */ ++static ssize_t recovery_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct g5p_r8 *s = &g5p_r8_state; ++ unsigned long flags; ++ struct g5p_ds d; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ d = g5p_ds_state; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return sysfs_emit( ++ buf, ++ "r8 gen=%u retired=%d mode=%s class=%s verdict=%s flags=%02x admit_closed=%u " ++ "inside=%u detach_queued=%u lines=%u\n" ++ "ds gen=%u mode=%u origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x " ++ "pmgr=%08x alive_gen=%u resets=%u reset_ok_gen=%u events=%u claims=%u refused=%u " ++ "gate=%s\n", ++ READ_ONCE(s->gen), READ_ONCE(s->retired), ++ g5p_td_mode_name[s->mode], g5p_r8_class_name[s->cls], ++ g5p_td_verdict_name[s->verdict], s->flags, ++ g5p_r8_admit_closed(&s->admit), g5p_r8_admit_inside(&s->admit), ++ READ_ONCE(s->detach_queued), s->lines, d.gen, g5p_ds_mode(), ++ g5p_ds_origin_name[d.origin], d.failed, g5p_ds_why_name[d.why], ++ d.seen, d.ok, d.flags, d.pmgr_raw, d.alive_gen, d.resets, ++ d.reset_ok_gen, d.events, d.claims, d.refused, ++ g5p_ds_gate_name[d.last_gate]); ++} ++static DEVICE_ATTR_RO(recovery_status); +diff --git a/drivers/gpu/apple-g5p/drm-validate.h b/drivers/gpu/apple-g5p/drm-validate.h +new file mode 100644 +index 000000000..671e13ab8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/drm-validate.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DRM_VALIDATE_H ++#define G5P_DRM_VALIDATE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_USER_VA_START 0x10000ULL ++#define G5P_USER_VA_END 0x10000000000ULL ++#define G5P_USER_RESERVED_START G5P_IMAGE_VA ++#define G5P_USER_RESERVED_END (G5P_PIO_VA + G5P_PIO_SIZE) ++#define G5P_MAX_BO_SIZE (64ULL << 20) ++/* VA bookkeeping: the sum of mapping sizes in one VM. A GEM bound twice ++ * (GBM alloc, then the EGL import of its own dma-buf) counts twice here. ++ */ ++#define G5P_MAX_VM_BYTES BIT_ULL(30) ++/* Memory behind one VM's mappings, counted once per GEM object. */ ++#define G5P_MAX_PINNED_BYTES (768ULL << 20) ++ ++static inline bool g5p_user_pa_valid(u64 pa) ++{ ++ return pa < 0x10000000000ULL && !(pa & (G5P_GPU_PAGE - 1)); ++} ++ ++static inline bool g5p_user_range_valid(u64 va, u64 size) ++{ ++ if (!size || ((va | size) & (G5P_GPU_PAGE - 1)) || ++ va < G5P_USER_VA_START || va >= G5P_USER_VA_END || ++ size > G5P_USER_VA_END - va) ++ return false; ++ return va >= G5P_USER_RESERVED_END || ++ va + size <= G5P_USER_RESERVED_START; ++} ++ ++static inline bool g5p_bo_range_valid(u64 offset, u64 size, u64 bo_size) ++{ ++ return size && !((offset | size) & (G5P_GPU_PAGE - 1)) && ++ offset < bo_size && size <= bo_size - offset; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/fender.h b/drivers/gpu/apple-g5p/fender.h +new file mode 100644 +index 000000000..cbdf203c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/fender.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_FENDER_H ++#define G5P_FENDER_H ++ ++/* Every poll callback must enforce a deadline and retain its final raw word. ++ * Apple 20H350: stop c9fa10, prepare c9fc10, boot c9f954; ++ * Fender helpers ca4ccc (GPC state) and ca6544 (core-power request). ++ * Hardware interpretation of the 0 -> 1 transition remains experimental. ++ */ ++struct g5p_fender_io { ++ void *ctx; ++ int (*write)(void *ctx, u32 reg, u32 value); ++ int (*poll)(void *ctx, u32 reg, u32 mask, u32 expected); ++}; ++ ++static inline int g5p_fender_idle(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x1000, 0x10, 0); ++ if (ret) ++ return ret; ++ ret = io->write(io->ctx, 0x1000, 0x10); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++static inline int g5p_fender_hold(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0x20); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x20000, 0x20, 0x20); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++/* q: the pstate the GPC is requested at (R1+0x1000 = 0x10 | q, poll for q); 1 as before, ++ * or boot_gpu_perf_poweron's pin (boot.inc). ++ */ ++static inline int ++g5p_fender_start_observed_q(const struct g5p_fender_io *io, ++ void (*observe)(void *, const char *), u32 q) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-core-clear"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc0-verified"); ++ ret = io->write(io->ctx, 0x1000, 0x10 | q); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc1-requested"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, q); ++ if (!ret && observe) ++ observe(io->ctx, "release-gpc1-active"); ++ return ret; ++} ++ ++static inline int g5p_fender_start_observed(const struct g5p_fender_io *io, ++ void (*observe)(void *, ++ const char *)) ++{ ++ return g5p_fender_start_observed_q(io, observe, 1); ++} ++ ++static inline int g5p_fender_start(const struct g5p_fender_io *io) ++{ ++ return g5p_fender_start_observed(io, NULL); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/gpu-oc.h b/drivers/gpu/apple-g5p/gpu-oc.h +new file mode 100644 +index 000000000..733aba9d4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/gpu-oc.h +@@ -0,0 +1,149 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * GPU overclock record (boot_gpu_oc_mhz): docs/power/ds-gpu-dvfs.md §3-4, §9 R5; ++ * docs/gpu/experiments/clock-mechanisms.md §2. Host test: scripts/gpu/perf60-host/test_oc.c. ++ * ++ * The hardware applies PSINFO record n (PMGR+0x70000 + 0x10*n, written by LLB 180395fd8 for ++ * slots 0-4 once per SoC reset, volatile) when Fender is asked for pstate n. Word 0 = core-rail ++ * DAC code [31:24] | PLL valid [21] | multiplier [20:12] | predivider [8:4] | postdivider-1 [3:0]; ++ * word 1 [7:0] = buck8 code; word 2 = timing selectors. ++ * ++ * PLL: VCO = 24 MHz * mult / prediv, f = VCO / postdiv. Apple's GPU-domain (0x17) descriptors ++ * (LLB 0x1803bc718, all three chip sections) run VCOs from 800 to 2160 MHz (2160 = id 0x20 of ++ * section 0x2f0001ff, 90/1/3); the live states use 948..1650. The overclock keeps postdiv 2 (every ++ * live state at or above 474 MHz uses 2 or 3) and the VCO inside Apple's range: f <= 1080 MHz. ++ * Apple's own descriptors at 804 (67/1/2), 850 (425/6/2), 900 (75/1/2); elsewhere record 4's ++ * dividers (prediv 4, postdiv 2: f = 3 MHz * mult) on a 3 MHz grid. ++ * ++ * Voltage: record 5 keeps record 4's timing word; its core (rail 1, PMIC 0x11, 450 mV + 3.125 mV * ++ * code, GPU only) and buck8 (rail 8, PMIC 0x18, 600 mV + 3.125 mV * code) codes default to record ++ * 4's (138 = 881.25 mV, 124 = 987.5 mV), never lower, raised only on request in whole 3.125 mV ++ * codes, ++ * and never above the ceilings: the boot parameters (default state 4 + 50 mV, the CPU overclock's ++ * rail-offset cap) bounded by the compiled maxima below (state 4 + 150 mV; raising those needs a ++ * rebuild). The encoding itself reaches 1296 mV on both rails; that is not an operating point. ++ */ ++#ifndef G5P_GPU_OC_H ++#define G5P_GPU_OC_H ++ ++#include "bits-compat.h" ++ ++#define G5P_PSINFO_BASE 0x70000U /* PMGR syscon offset */ ++#define G5P_PSINFO_REC(n) (G5P_PSINFO_BASE + 0x10U * (n)) ++#define G5P_OC_STATE 5U ++#define G5P_OC_PIN 500U /* boot_gpu_perf for state 5 */ ++#define G5P_OC_STOCK4 \ ++ 0x8a2f1041U /* record 4 word 0 as LLB writes it on this chip */ ++#define G5P_OC_STOCK4_BUCK8 124U /* record 4 word 1 [7:0] */ ++#define G5P_OC_MIN_MHZ 726U /* the first grid step above 723 */ ++#define G5P_OC_VCO_MAX 2160U /* Apple's highest GPU-domain VCO */ ++#define G5P_OC_MAX_MHZ (G5P_OC_VCO_MAX / 2) /* postdiv 2: 1080 */ ++#define G5P_OC_VCO_STOCK_MAX \ ++ 1650U /* the highest VCO of a live state (550 MHz) */ ++#define G5P_OC_VALID BIT(21) ++#define G5P_OC_PLL_MASK 0x001ff1ffU /* mult, prediv, postdiv-1 */ ++#define G5P_OC_UV_STEP 3125U ++#define G5P_OC_CORE_BASE_UV 450000U ++#define G5P_OC_BUCK8_BASE_UV 600000U ++/* 881250 */ ++#define G5P_OC_CORE_STOCK_UV \ ++ (G5P_OC_CORE_BASE_UV + G5P_OC_UV_STEP * (G5P_OC_STOCK4 >> 24)) ++/* 987500 */ ++#define G5P_OC_BUCK8_STOCK_UV \ ++ (G5P_OC_BUCK8_BASE_UV + G5P_OC_UV_STEP * G5P_OC_STOCK4_BUCK8) ++/* the boot parameters' default ceiling: P4 + 50 mV */ ++#define G5P_OC_DEFAULT_HEADROOM_UV 50000U ++#define G5P_OC_ABS_HEADROOM_UV \ ++ 150000U /* compiled: P4 + 150 mV, no parameter goes above */ ++ ++struct g5p_oc_pll { ++ u32 mhz, mult, prediv, postdiv; ++ bool llb; /* Apple's own descriptor */ ++}; ++ ++static const struct g5p_oc_pll g5p_oc_llb[] = { ++ { 804, 67, 1, 2, true }, /* LLB id 0x22, all three GPU sections */ ++ { 850, 425, 6, 2, true }, /* LLB id 0x23, the live section */ ++ { 900, 75, 1, 2, true }, /* LLB id 0x24, the live section */ ++}; ++ ++/* ++ * The PLL for a requested MHz: Apple's descriptor at 804/850/900 (the request or its 3 MHz ++ * rounding), ++ * else the 3 MHz grid (rounded down) with record 4's dividers; false outside 726..1080. ++ */ ++static inline bool g5p_oc_pll(u32 want, struct g5p_oc_pll *p) ++{ ++ u32 mhz = want / 3 * 3; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_llb); i++) ++ if (want == g5p_oc_llb[i].mhz || mhz == g5p_oc_llb[i].mhz) { ++ *p = g5p_oc_llb[i]; ++ return true; ++ } ++ if (mhz < G5P_OC_MIN_MHZ || mhz > G5P_OC_MAX_MHZ) ++ return false; ++ *p = (struct g5p_oc_pll){ mhz, mhz / 3, 4, 2, false }; ++ return true; ++} ++ ++/* A rail voltage as a code: exact multiples of 3.125 mV above the base only. */ ++static inline bool g5p_oc_code(u32 uv, u32 base, u32 *code) ++{ ++ if (uv < base || (uv - base) % G5P_OC_UV_STEP || ++ (uv - base) / G5P_OC_UV_STEP > 0xff) ++ return false; ++ *code = (uv - base) / G5P_OC_UV_STEP; ++ return true; ++} ++ ++/* The voltage codes for a request (0 = state 4's), bounded below by state 4 and above by the ++ * ceilings (each already clamped to the compiled maximum by g5p_oc_ceiling()). ++ */ ++ ++/* Record 5's words 0 and 1 from the live record 4 (stock only), a PLL and the voltage codes. */ ++static inline bool g5p_oc_words(const u32 r4[4], u32 want, u32 core, u32 buck8, ++ u32 w[4], struct g5p_oc_pll *p) ++{ ++ if (r4[0] != G5P_OC_STOCK4 || (r4[1] & 0xff) != G5P_OC_STOCK4_BUCK8 || ++ !g5p_oc_pll(want, p) || core < (G5P_OC_STOCK4 >> 24) || ++ core > 0xff || buck8 < G5P_OC_STOCK4_BUCK8 || buck8 > 0xff) ++ return false; ++ w[0] = core << 24 | G5P_OC_VALID | p->mult << 12 | p->prediv << 4 | ++ (p->postdiv - 1); ++ w[1] = (r4[1] & ~0xffU) | buck8; ++ w[2] = r4[2]; ++ w[3] = r4[3]; ++ return true; ++} ++ ++/* Hz of a record word 0, as iBoot 87001c3ac decodes it (0 if the predivider is 0). */ ++static inline u64 g5p_oc_hz(u32 w0) ++{ ++ u32 mult = (w0 >> 12) & 0x1ff, prediv = (w0 >> 4) & 0x1f, ++ postdiv = (w0 & 0xf) + 1; ++ ++ return prediv ? 24000000ULL * mult / prediv / postdiv : 0; ++} ++ ++/* A nonzero record 5 that this driver writes: record 4's timing and word-3 words, valid, a PLL ++ * g5p_oc_pll() produces, voltage codes between state 4's and the compiled maxima. Anything else in ++ * slot 5 is not ours: never overwritten or cleared. ++ */ ++static inline bool g5p_oc_is_ours(const u32 r4[4], const u32 r5[4]) ++{ ++ u32 core = r5[0] >> 24, buck8 = r5[1] & 0xff, w[4], cmax, bmax; ++ struct g5p_oc_pll p; ++ u64 hz = g5p_oc_hz(r5[0]); ++ ++ if (!g5p_oc_code(G5P_OC_CORE_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_CORE_BASE_UV, &cmax) || ++ !g5p_oc_code(G5P_OC_BUCK8_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_BUCK8_BASE_UV, &bmax) || ++ core > cmax || buck8 > bmax || !hz || hz % 1000000ULL || ++ !g5p_oc_words(r4, (u32)(hz / 1000000ULL), core, buck8, w, &p)) ++ return false; ++ return w[0] == r5[0] && w[1] == r5[1] && w[2] == r5[2] && w[3] == r5[3]; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/heartbeat116.h b/drivers/gpu/apple-g5p/heartbeat116.h +new file mode 100644 +index 000000000..428146764 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/heartbeat116.h +@@ -0,0 +1,175 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_HEARTBEAT116_H ++#define G5P_HEARTBEAT116_H ++ ++/* ++ * Heartbeat116: the R74 IRQ174 storm bound as a window rate, not a lifetime count. ++ * ++ * A healthy R74 epoch takes IRQ174 at about 575/s and consumes about 290 event-ring ++ * entries/s, idle or not: one type-0 ALIVE per firmware scheduler iteration, and two IRQ ++ * entries per ring entry (irqs = 2 * events - 18 over the Warm113 C4-prep R74 milestones; ++ * r81 measured 577/289 per s). Wake112 bounded both counts over the epoch's lifetime: ++ * IRQ entries against boot_r74_irq_cap (1048576) and ring entries against the compute event ++ * limit (524544). Either bound latched failed/event_failed, so an idle, healthy GPU faulted ++ * after about 30 minutes. ++ * ++ * Here both bounds are rates. A window opens at the first IRQ entry and closes at the first ++ * entry at least window_ns later, then re-bases on the totals of that entry. Totals are u32 ++ * and every in-window count is a u32 difference, so a wrap of a total is harmless. ++ * ++ * RATE more than irq_ceiling IRQ entries in the open window (checked on every entry) ++ * EVENTS more than event_ceiling ring entries in the open window (checked on every entry) ++ * STALL stall_windows consecutive closed windows in which ++ * irqs > ratio * events + slack. Healthy traffic is 2:1. A latch that re-fires ++ * without ring progress, or a heartbeat after a fault stopped the drain, is not. ++ * ++ * A trip is absorbing. Nothing here clears a failure; the glue adds the same terminal stop ++ * as the lifetime cap had (cap_seen, failed, event_failed, one disable_irq_nosync). ++ * ++ * Pure policy: no register access, no lock, no kernel API. The caller serializes (the R73B ++ * IRQ thread is the only caller). ++ */ ++ ++#define G5P_HB_WINDOW_NS 1000000000ULL ++#define G5P_HB_IRQ_CEILING 16384U ++#define G5P_HB_EVENT_CEILING 8192U ++#define G5P_HB_RATIO 8U ++#define G5P_HB_SLACK 512U ++#define G5P_HB_STALL_WINDOWS 3U ++ ++enum g5p_hb_why { ++ G5P_HBW_NONE, ++ G5P_HBW_RATE, ++ G5P_HBW_EVENTS, ++ G5P_HBW_STALL, ++ G5P_HBW_N, ++}; ++ ++static const char *const g5p_hb_why_name[G5P_HBW_N] = { ++ "none", ++ "rate", ++ "events", ++ "stall", ++}; ++ ++enum g5p_hb_res { ++ G5P_HBR_OPEN, /* counted in the open window */ ++ G5P_HBR_CLOSED, /* this entry closed a window (h->last) */ ++ G5P_HBR_TRIP, /* tripped now or earlier */ ++}; ++ ++struct g5p_hb_cfg { ++ u64 window_ns; ++ u32 irq_ceiling, event_ceiling, ratio, slack, stall_windows; ++}; ++ ++/* The totals the IRQ thread samples after servicing one entry. */ ++struct g5p_hb_in { ++ u64 now_ns; ++ u32 irqs; /* IRQ174 entries (b->irqs) */ ++ u32 events; /* event-ring entries consumed (b->events) */ ++ u32 acks; /* AIC ACKs written by the R73B handler */ ++ u32 empty; /* entries whose service consumed no ring entry */ ++ u32 aux; /* auxiliary-ring entries consumed */ ++}; ++ ++/* One window: its start and length and its in-window counts. */ ++struct g5p_hb_win { ++ u64 start_ns, ns; ++ u32 irqs, events, acks, empty, aux; ++}; ++ ++struct g5p_hb { ++ u32 open; /* zero until the first entry (kzalloc) */ ++ u32 why; /* enum g5p_hb_why, written once */ ++ u32 windows; /* windows closed */ ++ u32 stalled; /* consecutive unproductive closed windows */ ++ u32 peak_irqs, peak_events; ++ u64 start_ns; ++ struct g5p_hb_in base; /* totals at the window's first entry */ ++ struct g5p_hb_win last; /* the last closed window */ ++ struct g5p_hb_win trip; /* the open window at the trip */ ++ u64 trip_ns; ++}; ++ ++static inline void g5p_hb_cfg_default(struct g5p_hb_cfg *c) ++{ ++ c->window_ns = G5P_HB_WINDOW_NS; ++ c->irq_ceiling = G5P_HB_IRQ_CEILING; ++ c->event_ceiling = G5P_HB_EVENT_CEILING; ++ c->ratio = G5P_HB_RATIO; ++ c->slack = G5P_HB_SLACK; ++ c->stall_windows = G5P_HB_STALL_WINDOWS; ++} ++ ++static inline void g5p_hb_count(const struct g5p_hb *h, ++ const struct g5p_hb_in *in, ++ struct g5p_hb_win *w) ++{ ++ w->start_ns = h->start_ns; ++ w->ns = in->now_ns >= h->start_ns ? in->now_ns - h->start_ns : 0; ++ w->irqs = in->irqs - h->base.irqs; ++ w->events = in->events - h->base.events; ++ w->acks = in->acks - h->base.acks; ++ w->empty = in->empty - h->base.empty; ++ w->aux = in->aux - h->base.aux; ++} ++ ++static inline bool g5p_hb_unproductive(const struct g5p_hb_cfg *c, ++ const struct g5p_hb_win *w) ++{ ++ return (u64)w->irqs > (u64)c->ratio * w->events + c->slack; ++} ++ ++static inline enum g5p_hb_res ++g5p_hb_trip(struct g5p_hb *h, u32 why, const struct g5p_hb_win *w, u64 now_ns) ++{ ++ h->why = why; ++ h->trip = *w; ++ h->trip_ns = now_ns; ++ return G5P_HBR_TRIP; ++} ++ ++/* One serviced IRQ entry. */ ++static inline enum g5p_hb_res g5p_hb_step(struct g5p_hb *h, ++ const struct g5p_hb_cfg *c, ++ const struct g5p_hb_in *in) ++{ ++ struct g5p_hb_win w; ++ ++ if (h->why) ++ return G5P_HBR_TRIP; ++ if (!h->open) { ++ h->open = 1; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ return G5P_HBR_OPEN; ++ } ++ g5p_hb_count(h, in, &w); ++ if (w.irqs > c->irq_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_RATE, &w, in->now_ns); ++ if (w.events > c->event_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_EVENTS, &w, in->now_ns); ++ if (w.ns < c->window_ns) ++ return G5P_HBR_OPEN; ++ h->last = w; ++ h->windows++; ++ if (w.irqs > h->peak_irqs) ++ h->peak_irqs = w.irqs; ++ if (w.events > h->peak_events) ++ h->peak_events = w.events; ++ h->stalled = g5p_hb_unproductive(c, &w) ? h->stalled + 1 : 0; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ if (h->stalled >= c->stall_windows) ++ return g5p_hb_trip(h, G5P_HBW_STALL, &w, in->now_ns); ++ return G5P_HBR_CLOSED; ++} ++ ++/* Receipts: the first four closed windows, then every 64th. */ ++static inline bool g5p_hb_report(const struct g5p_hb *h) ++{ ++ return h->windows <= 4 || !(h->windows % 64); ++} ++ ++#endif /* G5P_HEARTBEAT116_H */ +diff --git a/drivers/gpu/apple-g5p/idle-power.inc b/drivers/gpu/apple-g5p/idle-power.inc +new file mode 100644 +index 000000000..d2ce71879 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/idle-power.inc +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Software/shared-RAM observation only. Never wake or probe GPU registers. */ ++static ssize_t idle_power_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b; ++ u32 p, state, target, mirror, period; ++ int ret; ++ ++ if (!mutex_trylock(&g->lock)) ++ return sysfs_emit(buf, "available=0 why=busy mmio=0\n"); ++ b = g->boot; ++ if (!b || !b->maps_ready) { ++ ret = sysfs_emit(buf, "available=0 why=no-maps mmio=0\n"); ++ goto out; ++ } ++ p = b->layout.offset[0x4a]; ++ dma_rmb(); ++ period = g5p_dc_root_word(b, p + 4); ++ state = g5p_dc_root_word(b, p + 8); ++ target = g5p_dc_root_word(b, p + 0x28); ++ mirror = g5p_dc_root_word(b, p + 0x34); ++ /* +34 is both a sampled Fender nibble and the setter's requested index. ++ * It is not a completed transition or stable-off certificate. Stale RAM ++ * remains readable across a stopped/failed epoch; label it explicitly. ++ */ ++ ret = sysfs_emit(buf, ++ "available=1 gen=%u alive=%u failed=%u stopped=%u powered=%u " ++ "policy_state=%u target=%u policy_pstate=%u sample_ms=%u " ++ "meta71c=%u source=shared-ram atomic=0 hw_ack=0 mmio=0\n", ++ g5p_ds_gen(), b->alive, b->failed, b->stopped, ++ g->powered, state, target, mirror, period, b->meta71c); ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++static DEVICE_ATTR_RO(idle_power_status); +diff --git a/drivers/gpu/apple-g5p/kick.inc b/drivers/gpu/apple-g5p/kick.inc +new file mode 100644 +index 000000000..6ef7395c6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/kick.inc +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared pre-kick step recovered from ca0358/ca05d4 and ca6644. */ ++static int g5p_prepare_kick(struct g5p_diag *g, const char *name) ++{ ++ u32 raw, state; ++ ++ if (gfx_bridge_replay && !strcmp(name, "RENDER-TA")) ++ g5p_r41_bridge_ta(g); ++ /* ca05d4 always invokes Fender +80 before inspecting GPC state. ++ * ca6644: W R1+14=7, then read fence. Hardware bit names unknown. ++ * This is distinct from acknowledging the host AIC interrupt. ++ */ ++ dev_info(g->dev, "%s Fender W R1+00014=00000007 before MTS kick\n", ++ name); ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ dev_info(g->dev, "%s Fender raw R1+00014=%08x (write fence)\n", name, ++ raw); ++ if (raw & ~7U) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ if (READ_ONCE(g->boot->failed)) ++ return -EIO; ++ return 0; ++} +diff --git a/drivers/gpu/apple-g5p/latency123.h b/drivers/gpu/apple-g5p/latency123.h +new file mode 100644 +index 000000000..164dc8e68 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/latency123.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: firmware words a powered iOS GPU has and our bootstrap leaves zero ++ * (docs/gpu/firmware-re/wake-latency-re.md sections 3.4 and 4.2). ++ * ++ * Arm A. The firmware copies meta+0x6c0..0x758 into its private block once, at init ++ * (MINIM 0x100009ac -> 0x10003ca4), so these words only act from the boot image: ++ * meta+0x6d4 = 1 per-tick cache maintenance 0x1001b638 (688/1720/e230/e138) ++ * meta+0x710/0x714 pstate table max index / count ++ * meta+0x718.. pstate MHz; the tick period is table[Fender+0x1000 & 0xf] ++ * policy+0x4 = 16 governor sample period, ms (the governor reads it live) ++ * 1 = capped: table { 0, 340 }, max index 1. The firmware's own pstate setter ++ * (0x1001c88c, Fender+0x1000 = index | 0x10) can then only ask for pstate 1, the state ++ * the host already requests, so the clock does not move. ++ * 2 = full: table { 0, 340, 474, 550, 723 }, max index 4, plus the limits README rank 2 ++ * pairs with it (control+0x1028/0x102c/0x1030 = 3348/400/400, policy+0x7cc/0x6fc = ++ * 400). The firmware may then raise the GPU clock itself; nothing on Linux programs ++ * a GPU voltage table, so this arm runs after the capped one. ++ * ++ * Arm B. ctl+0x04 is the per-engine tick-service mask (iOS 7). Every tick re-copies ++ * ctl+0x00..0x20 (0x10001328), so the word is live; the boot image gets the value the ++ * parameter holds at map and the render path re-applies it before each job. ++ * ++ * RAM only. Temporary A/B code: the winning words become unconditional init writes. ++ */ ++#ifndef APPLE_G5P_LATENCY123_H ++#define APPLE_G5P_LATENCY123_H ++ ++#include "bits-compat.h" ++#define G5P_L123_POLICY 0x4aU ++#define G5P_L123_META 0x4dU ++#define G5P_L123_CTL 0x5eU ++#define G5P_L123_TICK_MASK 7U ++#define G5P_L123_CAPPED 1U ++#define G5P_L123_FULL 2U ++ ++static const u32 g5p_l123_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline void g5p_l123_arm_a(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 arm) ++{ ++ u8 *meta = g5p_object(p, root, shared, G5P_L123_META); ++ u8 *policy = g5p_object(p, root, shared, G5P_L123_POLICY); ++ u8 *ctl = g5p_object(p, root, shared, G5P_L123_CTL); ++ u32 count = arm == G5P_L123_FULL ? ARRAY_SIZE(g5p_l123_mhz) : 2; ++ u32 i; ++ ++ g5p_put32(meta + 0x6d4, 1); ++ g5p_put32(meta + 0x710, count - 1); ++ g5p_put32(meta + 0x714, count); ++ for (i = 0; i < count; i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_l123_mhz[i]); ++ g5p_put32(policy + 4, 16); ++ if (arm != G5P_L123_FULL) ++ return; ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, 400); ++ g5p_put32(ctl + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/ldr.h b/drivers/gpu/apple-g5p/ldr.h +new file mode 100644 +index 000000000..165aacf85 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/ldr.h +@@ -0,0 +1,154 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Deliberately small Apple G5P META LDR reader. No allocation or MMIO here. */ ++#ifndef APPLE_G5P_LDR_H ++#define APPLE_G5P_LDR_H ++ ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_LDR_MAX_RECORDS 128 ++#define G5P_LDR_MAX_BYTES (256 * 1024) ++ ++enum g5p_ldr_op { G5P_LOAD, G5P_ZERO, G5P_CONFIG, G5P_START }; ++ ++typedef int (*g5p_ldr_emit)(void *ctx, enum g5p_ldr_op op, u32 address, ++ u32 size, const u8 *data); ++ ++static inline u32 g5p_le32(const u8 *p) ++{ ++ return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; ++} ++ ++static inline u16 g5p_le16(const u8 *p) ++{ ++ return (u16)p[0] | (u16)p[1] << 8; ++} ++ ++static inline bool g5p_ldr_bounds(size_t len, u32 off, u32 size) ++{ ++ return off <= len && size <= len - off; ++} ++ ++static inline int g5p_ldr_l2(const u8 *fw, size_t len, u32 off, u16 tag, ++ const u8 **data, u32 *size) ++{ ++ u16 bytes; ++ ++ if (!g5p_ldr_bounds(len, off, 4)) ++ return -EINVAL; ++ bytes = g5p_le16(fw + off + 2); ++ if (bytes < 6 || !g5p_ldr_bounds(len, off, bytes) || ++ g5p_le16(fw + off) != tag) ++ return -EINVAL; ++ *data = fw + off + 4; ++ *size = bytes - 6; ++ return 0; ++} ++ ++/* Validate in a first pass (emit=NULL); only then use a hardware callback. */ ++static inline int g5p_ldr_walk(const u8 *fw, size_t len, g5p_ldr_emit emit, ++ void *ctx) ++{ ++ u32 pos, seen[G5P_LDR_MAX_RECORDS], total = 0; ++ unsigned int nr = 0, starts = 0; ++ ++ if (len != 119112 || g5p_le32(fw) != 0x01aa5500 || ++ !g5p_ldr_bounds(len, g5p_le32(fw + 4), 1)) ++ return -EINVAL; ++ pos = g5p_le32(fw + 8); ++ while (pos != 0xffffffff) { ++ const u8 *p, *data; ++ u32 command, bytes, addr = 0, size = 0, l2, next; ++ enum g5p_ldr_op op; ++ unsigned int i; ++ int ret; ++ ++ if (nr == G5P_LDR_MAX_RECORDS || !g5p_ldr_bounds(len, pos, 12)) ++ return -EINVAL; ++ for (i = 0; i < nr; i++) ++ if (seen[i] == pos) ++ return -EINVAL; ++ seen[nr++] = pos; ++ p = fw + pos; ++ command = g5p_le16(p); ++ bytes = g5p_le16(p + 2); ++ next = g5p_le32(p + 4); ++ if (bytes < 12 || (bytes & 3) || ++ !g5p_ldr_bounds(len, pos, bytes)) ++ return -EINVAL; ++ if (command & 0x10) { ++ if (command != 0x10 || bytes != 16) ++ return -EINVAL; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), 0x8010, ++ &data, &size); ++ if (ret) ++ return ret; ++ goto next_record; ++ } ++ switch (command) { ++ case 0: ++ if (bytes != 20) ++ return -EINVAL; ++ op = G5P_LOAD; ++ addr = g5p_le32(p + 8); ++ l2 = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, l2, 0x8000, &data, &size); ++ if (ret) ++ return ret; ++ break; ++ case 4: ++ if (bytes != 24) ++ return -EINVAL; ++ op = G5P_ZERO; ++ addr = g5p_le32(p + 8); ++ size = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 16), 0x8004, ++ &data, &l2); ++ if (ret || l2) ++ return -EINVAL; ++ break; ++ case 5: ++ case 3: ++ if (bytes != 16) ++ return -EINVAL; ++ op = command == 5 ? G5P_CONFIG : G5P_START; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), ++ 0x8000 | command, &data, &size); ++ if (ret || !size || size % (command == 5 ? 12 : 16)) ++ return -EINVAL; ++ if (command == 5) { ++ for (i = 0; i < size; i += 12) ++ if (g5p_le32(data + i) != 2 || ++ (g5p_le32(data + i + 4) & 3)) ++ return -EINVAL; ++ } else { ++ if (++starts != 1 || size != 32 || ++ next != 0xffffffff || g5p_le32(data) != 1 || ++ g5p_le32(data + 16) != 0) ++ return -EINVAL; ++ } ++ break; ++ default: ++ return -EINVAL; ++ } ++ if (op == G5P_LOAD || op == G5P_ZERO) { ++ if ((addr & 3) || (size & 3) || !size || ++ size > G5P_LDR_MAX_BYTES || ++ addr > (u32)~0U - size || ++ total > G5P_LDR_MAX_BYTES - size) ++ return -EINVAL; ++ total += size; ++ } ++ if (emit) { ++ ret = emit(ctx, op, addr, size, data); ++ if (ret) ++ return ret; ++ } ++next_record: ++ pos = next; ++ } ++ return nr == 94 && starts == 1 ? 0 : -EINVAL; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +new file mode 100644 +index 000000000..9a896569c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -0,0 +1,1032 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Post-alive transport and park experiments; never implicit recovery. */ ++static const bool control_experiment = 1; ++static const bool irq_experiment = 0; ++static const bool teardown_experiment = 1; ++ ++/* A complete sparse image: first sample omits only zero words, later samples ++ * emit every changed word (including transitions to zero). All allocated ++ * root/shared memory is covered, including unknown status/trace buffers. ++ * These are CPU coherent RAM observations, never proof of META cache contents. ++ * IRQs can update memory concurrently; this is deliberately not an atomic dump. ++ */ ++static u32 g5p_dc_root_word(struct g5p_boot *b, u32 offset) ++{ ++ return READ_ONCE(*(u32 *)(b->root[offset / G5P_GPU_PAGE].cpu + ++ offset % G5P_GPU_PAGE)); ++} ++ ++/* nopatch: the policy record (slot 0x4a) words 0x1000d044 decides readiness from: +8 state, ++ * +0xc substate, +0x34 pstate mirror; +0 and +4 (sample period) for context. iOS keeps +8 at 0. ++ */ ++static const u32 g5p_np_words[5] = { 0, 4, 8, 0xc, 0x34 }; ++ ++static void g5p_np_read(struct g5p_boot *b, u32 *w) ++{ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_np_words); i++) ++ w[i] = g5p_dc_root_word(b, b->layout.offset[0x4a] + ++ g5p_np_words[i]); ++} ++ ++static void g5p_np_print(struct g5p_diag *g, const char *stage, u32 seq, u32 e, ++ const u32 *w) ++{ ++ dev_info( ++ g->dev, ++ "NOPATCH policy stage=%s seq=%u engine=%u +0=%08x +4=%08x +8=%08x +c=%08x " ++ "+34=%08x events=%u end=nopatch\n", ++ stage, seq, e, w[0], w[1], w[2], w[3], w[4], ++ READ_ONCE(g->boot->events)); ++} ++ ++/* RAM only; nothing unless the unpatched image is latched. */ ++static void g5p_np_policy(struct g5p_diag *g, const char *stage, u32 seq, u32 e) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 w[ARRAY_SIZE(g5p_np_words)]; ++ ++ if (!b || !b->r73b || !b->r73b->nopatch) ++ return; ++ g5p_np_read(b, w); ++ g5p_np_print(g, stage, seq, e, w); ++} ++ ++/* Read fixed allocated policy objects, never follow firmware-owned pointers. ++ * 1000d044/1000d504 test these states;1001c900 stores requested pstate. ++ * These are sampled boundary predicates, NOT entry/return breadcrumbs or PC. ++ */ ++ ++/* Firmware MMU recovery uses event mask200 and reads e150/e158. ++ * Read raw qword halves without ACK/clear; concurrent updates can tear them. ++ * An absent sampled event does not exclude an earlier firmware-handled fault. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 event, lo, hi, lo2, hi2; ++ ++ g5p_mmu_progress(g, stage, 0x130); ++ event = readl(g->rgx + 0x130); ++ g5p_mmu_progress(g, stage, 0xe150); ++ lo = readl(g->rgx + 0xe150); ++ g5p_mmu_progress(g, stage, 0xe154); ++ hi = readl(g->rgx + 0xe154); ++ g5p_mmu_progress(g, stage, 0xe158); ++ lo2 = readl(g->rgx + 0xe158); ++ g5p_mmu_progress(g, stage, 0xe15c); ++ hi2 = readl(g->rgx + 0xe15c); ++ g5p_mmu_record( ++ g, ++ "MMU %s event130=%08x pending=%u e150=%08x:%08x e158=%08x:%08x raw-split read-only\n", ++ stage, event, !!(event & 0x200), hi, lo, hi2, lo2); ++} ++ ++/* Same active-wrapper gate as CDM observations. Never write/ACK a register. ++ * 1001c9b8 samples R0+164/160/164; bound host sampling even on rollover. ++ * A split sample with different high words is explicitly unstable. ++ */ ++ ++static void g5p_dc_snapshot(struct g5p_diag *g, const char *stage, bool first) ++{ ++ { ++ return; ++ } ++} ++ ++static int g5p_control_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 *work = (u32 *)g5p_boot_shared(b, 0x68); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ u32 ctl_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6e)); ++ u32 data_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6f)); ++ u8 *init = b->root[0].cpu; ++ u8 *handoff = b->image[0].cpu + 0x3ff8; ++ u32 *events = (u32 *)g5p_boot_shared(b, 0x70); ++ ++ /* c9f868/c9f884: validate the loaded boot handoff, not just its builder. */ ++ if (g5p_le32(handoff) != g5p_meta_address(G5P_ROOT_VA) || ++ g5p_le32(handoff + 4) != ++ g5p_meta_address(g5p_object_va(&b->layout, 0x5f))) ++ return -EPROTO; ++ /* Drain under event_lock before checking that ALIVE has been returned. */ ++ if (g5p_r73b_service(g, G5P_R73B_OWNER_PREPARE, true)) ++ return -EIO; ++ if (READ_ONCE(events[0]) > 255 || READ_ONCE(events[4]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(events[0]) != READ_ONCE(events[4])) ++ return -EBUSY; ++ dev_info(g->dev, ++ "DC boot handoff verified; event consumer=%u producer=%u\n", ++ READ_ONCE(events[0]), READ_ONCE(events[4])); ++ ++ ; ++ ; ++ ; ++ ; ++ /* c96be8: init+18/+1c are META addresses, never CPU/DMA pointers. */ ++ if (g5p_le32(init + 0x18) != ctl_meta || ++ g5p_le32(init + 0x1c) != data_meta) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC mapping ctl GPU=%llx META=%08x data GPU=%llx META=%08x; init verified\n", ++ g5p_object_va(&b->layout, 0x6e), ctl_meta, ++ g5p_object_va(&b->layout, 0x6f), data_meta); ++ ++ if (b->work_ready) ++ return -EBUSY; ++ if (b->control_ready) ++ return b->r73b && 1 ? 0 : -EALREADY; ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (consumer != producer || READ_ONCE(work[0]) != READ_ONCE(work[4]) || ++ (b->work_submitted && !b->work_complete)) ++ return -EBUSY; ++ /* c993f8, size=0: no payload copy, ctrl+1020=0, command={8,0,0,0,0}. ++ * No claim that this is an opcode-independent firmware NOP. ++ */ ++ g5p_put32(g5p_boot_shared(b, 0x5e) + 0x1020, 0); ++ b->control_target = (producer + 1) & 255; ++ b->control_ready = true; ++ dev_info( ++ g->dev, ++ "DC prepare cmd=8 payload_words=0 consumer=%u producer=%u target=%u; RAM only\n", ++ consumer, producer, b->control_target); ++ return 0; ++} ++ ++static int g5p_r73e_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ret = g5p_control_prepare(g); ++ if (!ret) ++ WRITE_ONCE(b->r73b->fast_window, true); ++ dev_info( ++ g->dev, ++ "R73E cmd8 prepared-before-release ret=%d target=%u end=r73e\n", ++ ret, b->control_target); ++ return ret; ++} ++ ++/* Apple's GPC precondition (AGXMetaFender_setGPCPerformanceState, ...6ca4ccc): a request in ++ * device test shows R1+0x1000 bit 4; wait for it to clear before judging the state. R1 reads only, ++ * bounded by boot_alive_settle_us; with 0 the first read stands, as before. ++ */ ++static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw) ++{ ++ u64 start; ++ ++ if (!r->settle_us || !(raw & 0x10)) ++ return raw; ++ if (!r->settle_first) ++ r->settle_first = raw; ++ start = ktime_get_mono_fast_ns(); ++ while ((raw & 0x10) && ktime_get_mono_fast_ns() - start < ++ (u64)r->settle_us * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(g->window[1] + 0x1000); ++ r->settle_samples++; ++ } ++ r->settle_ns += ktime_get_mono_fast_ns() - start; ++ return raw; ++} ++ ++static int g5p_r73e_irq_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 off, raw; ++ enum g5p_pxg gate; ++ ++ if (!b->control_ready || b->control_submitted) ++ return -EPERM; ++ /* wake112: release_ready, or a deferred R74E proof that the read below settles. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate != G5P_PXG_READY && gate != G5P_PXG_PROVE) ++ return -EHOSTDOWN; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ if (gate == G5P_PXG_PROVE && !g5p_r74e_take(g)) ++ return -EHOSTDOWN; ++ r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (gate == G5P_PXG_PROVE) ++ g5p_r74e_settle(r, r->early_state); ++ if (!g5p_gpc_ok(r->early_state)) ++ return -EHOSTDOWN; ++ /* nopatch: five RAM loads, the state the readiness gate will see before this bell. */ ++ if (r->nopatch) ++ g5p_np_read(b, r->np_pre); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_IRQ, 0); ++ /* Functional pre-kick sequence only; no optional diagnostics here. */ ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ if (raw & ~7U) ++ return -EPROTO; ++ r->early_after_state = ++ g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (!g5p_gpc_ok(r->early_after_state)) ++ return -EHOSTDOWN; ++ dev_emerg( ++ g->dev, ++ "R73E doorbell begin first-ALIVE W0+00b00 value=00000010 end=r73e\n"); ++ writel(0x10, g->rgx + 0xb00); ++ dev_emerg(g->dev, "R73E doorbell end first-ALIVE W0+00b00 end=r73e\n"); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_IRQ, 0); ++ return 0; ++} ++ ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxg gate; ++ u32 px; ++ int ret; ++ ++ /* wake112: a deferred R74E proof admits the once-owner as release_ready does. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate == G5P_PXG_REFUSE) ++ return -EHOSTDOWN; ++ if (gate == G5P_PXG_WAIT) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_busy, 0, 1)) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_once, 0, 1)) { ++ ret = READ_ONCE(r->early_ret); ++ goto out; ++ } ++ r->early_irq_ns = ktime_get_mono_fast_ns(); ++ r->early_ret = g5p_r73e_irq_submit(g); ++ ret = r->early_ret; ++ WRITE_ONCE(r->fast_window, false); ++ if (ret == -EHOSTDOWN) ++ WRITE_ONCE(r->early_power_abort, true); ++ dev_emerg( ++ g->dev, ++ "R74E ALIVE submit case=%u owner=%u state=%08x after=%08x ret=%d " ++ "release_ready=%u power_abort=%u submitted=%u end=r74e\n", ++ r->r73e_case, owner, r->early_state, r->early_after_state, ret, ++ r->early_release_ready, r->early_power_abort, ++ READ_ONCE(b->control_submitted)); ++ if (r->settle_first) ++ dev_emerg( ++ g->dev, ++ "R74E settle first=%08x state=%08x after=%08x samples=%u ns=%llu limit_us=%u " ++ "ret=%d end=r74e\n", ++ r->settle_first, r->early_state, r->early_after_state, ++ r->settle_samples, r->settle_ns, r->settle_us, ret); ++ if (r->nopatch && READ_ONCE(b->control_submitted)) ++ g5p_np_print(g, "pre-bell", 0, 0, r->np_pre); ++ px = atomic_read(&r->early_proof); ++ /* raw: the proof read, or U32_MAX when it did not happen (a refusal or no take). */ ++ if (px != G5P_PX_NONE) ++ dev_emerg( ++ g->dev, ++ "R74E112 proof owner=%u state=%s why=%s gen=%u/%u raw=%08x release_ready=%u " ++ "power_abort=%u end=r74e112\n", ++ owner, g5p_px_name[min_t(u32, px, G5P_PX_N - 1)], ++ g5p_px_why_name[min_t(u32, r->early_proof_why, ++ G5P_PXW_N - 1)], ++ r->early_proof_gen, r->early_proof_gen_now, ++ px == G5P_PX_PROVEN || ++ r->early_proof_why == G5P_PXW_STATE ? ++ r->early_state : ++ U32_MAX, ++ r->early_release_ready, r->early_power_abort); ++out: ++ atomic_set(&r->early_busy, 0); ++ return ret; ++} ++ ++static int g5p_control_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 state, off; ++ int ret; ++ ++ if (b->control_submitted) ++ return b->r73b && 1 ? 0 : -EPERM; ++ if (!b->control_ready) ++ return -EPERM; ++ /* R73 synchronous pre-publish pass closes the race where trace records ++ * do not raise a final IRQ. A negative result is scoreable only when the ++ * factor reaches cmd8 with every outbound auxiliary consumer caught up. ++ */ ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_PUBLISH, true); ++ if (ret) ++ return ret; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ dev_info(g->dev, "DC raw R1+01000=%08x (require active before kick)\n", ++ state); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_dc_snapshot(g, "before-publish", true); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ /* ca05d4: full dmb ish after producer, before Fender and MTS. */ ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_PUBLISH, 0); ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *src = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ if (g5p_le32(src) != (i ? 0 : 8)) ++ return -EBADMSG; ++ } ++ if (READ_ONCE(ctl[4]) != b->control_target) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC slot verified words=00000008/0/0/0/0 producer=%u; full barrier\n", ++ b->control_target); ++ ret = g5p_prepare_kick(g, "DC"); ++ if (ret) ++ return ret; ++ ; ++ if (b->r73_aux_drain) { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_KICK, true); ++ if (ret) ++ return ret; ++ } ++ g5p_dc_snapshot(g, "before-kick", false); ++ dev_info( ++ g->dev, ++ "DC publish cmd=8 slot=%u target=%u; W R0+00b00=%08x variant=%u\n", ++ producer, b->control_target, b->dc_variant == 4 ? 0x12 : 0x10, ++ b->dc_variant); ++ /* 0x12 is verified for CL, NOT for DC: variant 4 tests dispatch routing. */ ++ writel(7 == 4 ? 0x12 : 0x10, g->rgx + 0xb00); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_KICK, 0); ++ g5p_dc_snapshot(g, "after-kick", false); ++ return 0; ++} ++ ++/* perf60 pin >100 (seed 1-5, poweron): the policy record as the firmware left it, from RAM, plus the ++ * real Fender state through an R1-only ticket of the register licence (any pstate, bit 29 allowed). ++ * The sampler 0x1000aa80 stores its own "now" at policy+0x918 on every call (0x1000ab44) and the ++ * current state at +0x34 (0x1000ab18), so fw_now against the host's arch timer names the ++ * firmware's clock, and +0x34 / +0 / +0x8f8 name the branch it took. ++ */ ++static void g5p_perf_readback(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 pol = b->layout.offset[0x4a], meta = b->layout.offset[0x4d]; ++ u64 host = arch_timer_read_counter(), rate = arch_timer_get_rate(); ++ u64 p8f8, p918, p920, m784; ++ struct g5p_mmio_ticket t; ++ ++ if (!b->gpu_perf || (!b->perf_seed && !b->perf_poweron)) ++ return; ++ dma_rmb(); ++ p8f8 = g5p_dc_root_word(b, pol + 0x8f8) | ++ (u64)g5p_dc_root_word(b, pol + 0x8fc) << 32; ++ p918 = g5p_dc_root_word(b, pol + 0x918) | ++ (u64)g5p_dc_root_word(b, pol + 0x91c) << 32; ++ p920 = g5p_dc_root_word(b, pol + 0x920) | ++ (u64)g5p_dc_root_word(b, pol + 0x924) << 32; ++ m784 = g5p_dc_root_word(b, meta + 0x784) | ++ (u64)g5p_dc_root_word(b, meta + 0x788) << 32; ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ dev_emerg( ++ g->dev, ++ "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " ++ "pol28=%u pol2c=%u pol30=%u pol34=%u p8f8=%llx fw_now918=%llx p920=%llx " ++ "meta784=%llx host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " ++ "host_minus_fw_counter_us=%lld r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", ++ why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, ++ g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30), ++ g5p_dc_root_word(b, pol + 0x34), p8f8, p918, p920, m784, host, ++ b->seed_stamp, ++ b->seed_stamp ? ++ div_s64((s64)(p918 - m784 - b->seed_stamp) * 1000000LL, ++ rate) : ++ 0, ++ div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate), ++ g5p_mmio_did(&t), t.gpc, t.r1_0); ++} ++ ++static int g5p_r73c_fast_control(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ int ret; ++ ++ r->fast_release_ns = ktime_get_mono_fast_ns(); ++ WRITE_ONCE(r->fast_window, true); ++ ret = g5p_boot_events(g); ++ r->fast_alive_ns = ktime_get_mono_fast_ns(); ++ if (!ret) ++ ret = g5p_control_prepare(g); ++ if (!ret) { ++ r->fast_state = readl(g->window[1] + 0x1000); ++ ret = g5p_control_submit(g); ++ } ++ r->fast_submit_ns = ktime_get_mono_fast_ns(); ++ r->fast_ret = ret; ++ WRITE_ONCE(r->fast_window, false); ++ g5p_r73b_events_exit(g, ret); ++ dev_info( ++ g->dev, ++ "R73C fast release_to_alive_ns=%llu release_to_submit_ns=%llu state=%08x ret=%d " ++ "control_submitted=%u end=r73c\n", ++ r->fast_alive_ns - r->fast_release_ns, ++ r->fast_submit_ns - r->fast_release_ns, r->fast_state, ret, ++ READ_ONCE(b->control_submitted)); ++ g5p_perf_readback(g, ret ? "alive-timeout" : "alive"); ++ return ret; ++} ++ ++/* perf60 pin 200 (docs/gpu/experiments/perf60-clock.md "Pin 200: the guarded drain read" and ++ * "Clock-change drain RE" §4): at a boot control timeout with boot_gpu_perf set (before any job, ++ * the firmware alive), every read goes through the GPU register licence (mmio.inc) and nothing ++ * wakes the GPU (no R1+0x14 write, mmio-access.md G6): ++ * r1: R1 +0, +8, +0x14, +0x1000, +0x1004, +0x20000 (one R1-only ticket, any pstate) ++ * drain: 11f00 688 1720 1728 e230 e238 e138, then the clock words R0+0x0/0x4/0x8/0xc ++ * timer: R0+0x160/0x164, twice, 1 ms apart ++ * tx: T0/T1 TXENABLE/TXSTATUS (0x04800000/10, 0x04801000/10) through the META slave port ++ * pc: (bit 1 only) T0 PC: RQ 0x0480fff8 = 0x10005, poll bit 31, read 0x0480fff0; x8, 100 us ++ * apart. This is the driver's first slave-port write (CTRL0 = addr, DATAT = value). ++ * One ticket per group or slave-port op (<= G5P_MMIO_MAX_OPS), a refusal retried <= 2 ms. ++ */ ++ ++#define G5P_GPP_BUDGET_US 2000U ++ ++/* One META slave-port read in one ticket: ready, CTRL0 = addr | 1, read back, ready (<= 4 ++ * polls), DATAX: 8 ops. -EBUSY when the port is not ready within the ticket. ++ */ ++static int g5p_mmio_sp_read(struct g5p_diag *g, u32 addr, u32 *out, u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1 = 0; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr | 1)) { ++ g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL0); ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ *out = g5p_mmio_r0_read(g, &t, G5P_MSLVDATAX); ++ ret = 0; ++ break; ++ } ++ } ++ } ++ if (!g5p_mmio_release(g, &t) && !ret) ++ ret = -ESTALE; ++ return ret; ++} ++ ++/* One META slave-port write in one ticket: ready, CTRL0 = addr, DATAT = value, ready (<= 4). */ ++static int g5p_mmio_sp_write(struct g5p_diag *g, u32 addr, u32 value, ++ u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr) && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVDATAT, value)) { ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ ret = 0; ++ break; ++ } ++ } ++ } ++ g5p_mmio_release(g, &t); ++ return ret; ++} ++ ++/* A group of plain R0 reads (<= G5P_MMIO_MAX_OPS) under one ticket; returns the ticket's did. */ ++static const char *g5p_gpp_r0(struct g5p_diag *g, const u32 *off, u32 *v, u32 n, ++ u32 *tries, u32 *gpc) ++{ ++ struct g5p_mmio_ticket t; ++ ++ if (g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries)) { ++ *gpc = t.gpc; ++ return g5p_mmio_did(&t); ++ } ++ for (u32 i = 0; i < n; i++) ++ v[i] = g5p_mmio_r0_read(g, &t, off[i]); ++ g5p_mmio_release(g, &t); ++ *gpc = t.gpc; ++ g5p_mmio_sample_line(g, G5P_MMS_PROBE, &t, v, n, true); ++ return g5p_mmio_did(&t); ++} ++ ++/* What a slave-port op's return means, for the dc-probe lines. */ ++static const char *g5p_sp_why(int ret) ++{ ++ switch (ret) { ++ case 0: ++ return "ok"; ++ case -ENODATA: ++ return "not-run"; ++ case -EPERM: ++ return "forbidden-after-release"; /* g->reset_transition: boot.inc's post-release ban */ ++ case -ENODEV: ++ return "licence-off"; ++ case -EAGAIN: ++ return "licence-refused"; ++ case -EBUSY: ++ return "port-not-ready"; ++ case -ESTALE: ++ return "stale"; ++ case -ETIMEDOUT: ++ return "rq-no-ready"; ++ default: ++ return "error"; ++ } ++} ++ ++static void g5p_gpu_perf_probe(struct g5p_diag *g) ++{ ++ static const u32 drain[] = { 0x11f00, 0x688, 0x1720, 0x1728, ++ 0xe230, 0xe238, 0xe138 }; ++ static const u32 clk[] = { 0x0, 0x4, 0x8, 0xc }; ++ static const u32 timer[] = { 0x160, 0x164 }; ++ static const u32 r1off[] = { 0x0, 0x8, 0x14, 0x1000, 0x1004, 0x20000 }; ++ static const u32 tx[] = { 0x04800000, 0x04800010, 0x04801000, ++ 0x04801010 }; ++ u32 mode = (0); ++ struct g5p_boot *b = g->boot; ++ u32 vd[ARRAY_SIZE(drain)], vc[ARRAY_SIZE(clk)], vt[2][2], ++ r1[ARRAY_SIZE(r1off)]; ++ u32 vx[ARRAY_SIZE(tx)], pc[8], pcrq[8], gate = U32_MAX, tries = 0, ++ gpc = U32_MAX, gpc_x; ++ int txr[ARRAY_SIZE(tx)], pcr[8]; ++ const char *d_r1 = "skip-guard", *d_drain = "skip-guard", ++ *d_clk = "skip-guard"; ++ const char *d_t0 = "skip-guard", *d_t1 = "skip-guard"; ++ struct g5p_mmio_ticket t; ++ ++ if (!mode || !b->gpu_perf) ++ return; ++ memset(vd, 0xff, sizeof(vd)); ++ memset(vc, 0xff, sizeof(vc)); ++ memset(vt, 0xff, sizeof(vt)); ++ memset(r1, 0xff, sizeof(r1)); ++ memset(vx, 0xff, sizeof(vx)); ++ memset(pc, 0xff, sizeof(pc)); ++ memset(pcrq, 0xff, sizeof(pcrq)); ++ for (u32 i = 0; i < ARRAY_SIZE(txr); i++) ++ txr[i] = -ENODATA; ++ for (u32 i = 0; i < ARRAY_SIZE(pcr); i++) ++ pcr[i] = -ENODATA; ++ if (g->pmgr && regmap_read(g->pmgr, 0x80340, &gate)) ++ gate = U32_MAX; ++ if (READ_ONCE(b->failed)) ++ goto print; ++ /* r1: before anything else, R1-only (valid at any pstate and with bit 29 set) */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) { ++ for (u32 i = 0; i < ARRAY_SIZE(r1off); i++) ++ r1[i] = g5p_mmio_r1_read(g, &t, r1off[i]); ++ g5p_mmio_release(g, &t); ++ } ++ d_r1 = g5p_mmio_did(&t); ++ /* gpc: the drain group's licence read; every group's ticket also has its own sample line */ ++ d_drain = g5p_gpp_r0(g, drain, vd, ARRAY_SIZE(drain), &tries, &gpc); ++ d_clk = g5p_gpp_r0(g, clk, vc, ARRAY_SIZE(clk), &tries, &gpc_x); ++ d_t0 = g5p_gpp_r0(g, timer, vt[0], 2, &tries, &gpc_x); ++ usleep_range(1000, 1100); ++ d_t1 = g5p_gpp_r0(g, timer, vt[1], 2, &tries, &gpc_x); ++ for (u32 i = 0; i < ARRAY_SIZE(tx); i++) ++ txr[i] = g5p_mmio_sp_read(g, tx[i], &vx[i], &tries); ++ if (mode & 2) { ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) { ++ pcr[i] = g5p_mmio_sp_write(g, 0x0480fff8, 0x10005, ++ &tries); ++ for (u32 k = 0; !pcr[i] && k < 4; k++) { ++ pcr[i] = g5p_mmio_sp_read(g, 0x0480fff8, ++ &pcrq[i], &tries); ++ if (pcr[i] || (pcrq[i] & BIT(31))) ++ break; ++ udelay(10); ++ } ++ if (!pcr[i]) ++ pcr[i] = pcrq[i] & BIT(31) ? ++ g5p_mmio_sp_read(g, 0x0480fff0, ++ &pc[i], ++ &tries) : ++ -ETIMEDOUT; ++ udelay(100); ++ } ++ } ++print: ++ dev_err(g->dev, ++ "GPUPERF dc-probe did=%s gate=%08x r1=%s r1_0=%08x r1_8=%08x r1_14=%08x " ++ "r1_1000=%08x r1_1004=%08x r1_20000=%08x gpc=%08x tries=%u end=gpuperf\n", ++ d_drain, gate, d_r1, r1[0], r1[1], r1[2], r1[3], r1[4], r1[5], ++ gpc, tries); ++ dev_err(g->dev, ++ "GPUPERF dc-probe drain=%s 11f00=%08x 688=%08x 1720=%08x 1728=%08x e230=%08x " ++ "e238=%08x e138=%08x clk=%s clk=%08x:%08x clkstat=%08x:%08x end=gpuperf\n", ++ d_drain, vd[0], vd[1], vd[2], vd[3], vd[4], vd[5], vd[6], d_clk, ++ vc[1], vc[0], vc[3], vc[2]); ++ dev_err(g->dev, ++ "GPUPERF dc-probe timer=%s/%s 160=%08x/%08x 164=%08x/%08x gap_us=1000 " ++ "txen0=%08x/%s txst0=%08x/%s txen1=%08x/%s txst1=%08x/%s end=gpuperf\n", ++ d_t0, d_t1, vt[0][0], vt[1][0], vt[0][1], vt[1][1], vx[0], ++ g5p_sp_why(txr[0]), vx[1], g5p_sp_why(txr[1]), vx[2], ++ g5p_sp_why(txr[2]), vx[3], g5p_sp_why(txr[3])); ++ if (mode & 2) ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) ++ dev_err(g->dev, ++ "GPUPERF dc-probe pc i=%u ret=%s rq=%08x t0pc=%08x end=gpuperf\n", ++ i, g5p_sp_why(pcr[i]), pcrq[i], pc[i]); ++} ++ ++static int g5p_control_wait(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long start = jiffies, deadline = start + 3 * HZ; ++ bool sampled = false, rekicked = false; ++ u32 consumer, producer, pass = 0; ++ int ret; ++ ++ if (!b->control_submitted || b->control_complete) ++ return -EPERM; ++ do { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_WAIT, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ consumer = READ_ONCE(ctl[0]); ++ producer = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ if (consumer > 255 || producer != b->control_target || ++ (consumer != b->control_target && ++ consumer != ((b->control_target - 1) & 255))) ++ return -EPROTO; ++ if (consumer == b->control_target) { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (g5p_r74_case(b) == 1) { ++ WRITE_ONCE(b->control_complete, true); ++ g5p_r74_mark(g, G5P_R74_CMD8_CONSUMED, ++ consumer); ++ dev_emerg( ++ g->dev, ++ "R74B cmd8-consumed consumer=%u producer=%u IRQ174=%d ram_only=1 end=r74b\n", ++ consumer, producer, ++ atomic_read(&b->irqs)); ++ g5p_np_policy(g, "cmd8-consumed", consumer, 0); ++ return 0; ++ } ++ g5p_dc_snapshot(g, "consumed", false); ++ b->control_complete = true; ++ ; ++ dev_info( ++ g->dev, ++ "DC consumed cmd=8 consumer=%u producer=%u IRQ174=%d; transport only\n", ++ consumer, producer, atomic_read(&b->irqs)); ++ return 0; ++ } ++ if (!sampled && !time_before(jiffies, start + HZ / 10)) { ++ sampled = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "100ms", false); ++ g5p_np_policy(g, "100ms", consumer, 0); ++ } ++ if (7 == 3 && !rekicked && !time_before(jiffies, start + HZ)) { ++ /* Re-notify the SAME slot once; never publish a second command. */ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_prepare_kick(g, "DC re-kick"); ++ if (ret) ++ return ret; ++ mb(); ++ writel(0x10, g->rgx + 0xb00); ++ rekicked = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "re-kick", false); ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "timeout", false); ++ g5p_np_policy(g, "timeout", consumer, 0); ++ dev_err(g->dev, ++ "DC timeout consumer=%u producer=%u target=%u; retain RAM, no retry\n", ++ consumer, producer, b->control_target); ++ /* perf60 (boot_gpu_perf): the policy-state words at the stall, RAM only. pol34 = the pstate the ++ * firmware last requested (its setter stores it before the Fender write): q means it reached the ++ * setter, 1 that it stopped before (the clock-change drain 0x1001cec8 -> 0x1001b638). ++ */ ++ if (b->gpu_perf) { ++ u32 pol = b->layout.offset[0x4a]; ++ ++ dma_rmb(); ++ dev_err(g->dev, ++ "GPUPERF dc-timeout pin=%u pol8=%u/%u q=%u x=%u pol34=%u req=%u/%u ceil=%u/%u " ++ "seed=%u pol0=%u pol4=%u pol28=%u pol2c=%u pol30=%u end=gpuperf\n", ++ b->gpu_perf, g5p_dc_root_word(b, pol + 8), ++ g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x38), ++ g5p_dc_root_word(b, pol + 0x34), ++ g5p_dc_root_word(b, pol + 0x6f0), ++ g5p_dc_root_word(b, pol + 0x6f4), ++ g5p_dc_root_word(b, pol + 0x688), ++ g5p_dc_root_word(b, pol + 0x68c), b->perf_seed, ++ g5p_dc_root_word(b, pol + 0), ++ g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30)); ++ g5p_gpu_perf_probe(g); ++ g5p_perf_readback(g, "dc-timeout"); ++ } ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ b->control_timed_out = true; ++ return -ETIMEDOUT; ++} ++ ++static int g5p_irq_test(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + HZ / 10; ++ int before = atomic_read(&b->irqs), ret; ++ ++ if (b->irq_tested || !b->irq_ready || g5p_r74_case(b)) ++ return -EPERM; ++ /* ADT set/ack addresses, not a GPU slave-port transaction. Firmware may ++ * also interrupt concurrently: this tests delivery, not event provenance. ++ */ ++ dev_info(g->dev, "IRQ test baseline=%d W AIC+014=00004000\n", before); ++ writel(0x4000, b->swirq + 0x14); ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (atomic_read(&b->irqs) > before) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ b->irq_tested = true; ++ ; ++ dev_info( ++ g->dev, ++ "IRQ delivery observed before=%d after=%d; no shader completion implied\n", ++ before, atomic_read(&b->irqs)); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, "IRQ delivery timeout before=%d after=%d\n", before, ++ atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++static int g5p_boot_park(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ static const u32 slots[] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ bool failed_control = READ_ONCE(b->failed) && b->control_timed_out; ++ bool failed_work = READ_ONCE(b->failed) && b->work_timed_out; ++ bool failed_context = READ_ONCE(b->failed) && b->context_timed_out; ++ bool failed_timeout = failed_control || failed_work || failed_context; ++ int ret; ++ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ if (READ_ONCE(b->event_failed) || ++ (b->control_submitted && !b->control_complete && !failed_control) || ++ (b->work_submitted && !b->work_complete && !failed_work) || ++ (b->context_submitted && !b->context_complete && !failed_context)) ++ return -EBUSY; ++ /* A failed drain must never be bypassed. On the sole permitted failure ++ * path require the event ring empty below without consuming anything. ++ */ ++ if (!failed_timeout) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(slots); i++) { ++ u32 *ctl = (u32 *)g5p_boot_shared(b, slots[i]); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (((failed_control || failed_context) && slots[i] == 0x6c) || ++ (failed_work && slots[i] == 0x68)) { ++ u32 target = failed_context && slots[i] == 0x6c ? ++ b->context_target : ++ slots[i] == 0x6c ? b->control_target : ++ b->work_target; ++ ++ if (producer != target || ++ (consumer != producer && ++ consumer != ((producer - 1) & 255))) ++ return -EPROTO; ++ } else if (consumer != producer) { ++ return -EBUSY; ++ } ++ } ++ if (READ_ONCE(b->event_failed)) ++ return -EIO; ++ if (failed_timeout) ++ dev_info( ++ g->dev, ++ "PARK failed %s timeout; abandon command, preserve failure and RAM\n", ++ failed_context ? "CONTEXT" : ++ failed_work ? "CL" : ++ "DC"); ++ /* Freeze commands before the first transition; no retry even if it fails. ++ * c9fa10 enabled-Fender branch: core=0, GPC=0, no a18 access. ++ */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ dev_info( ++ g->dev, ++ "PARK begin core=0 GPC=0; wrapper only, DMA memory retained\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ return ret; ++ /* Keep the IRQ/ack handler and every allocation alive until reboot. ++ * GPC=0 alone is not proof that outstanding bus traffic has drained. ++ */ ++ b->parked = true; ++ dev_info( ++ g->dev, ++ "PARK GPC=0 verified; quarantine until reboot, no PMGR off or DMA free\n"); ++ return 0; ++} ++ ++static int g5p_lifecycle_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (!boot_experiment || !b || !b->alive || b->stopped) ++ return -EPERM; ++ if (b->failed && ++ !(sysfs_streq(command, "park") && teardown_experiment && ++ (b->control_timed_out || b->work_timed_out || ++ b->context_timed_out))) ++ return -EPERM; ++ if (!strncmp(command, "control-", 8) && control_experiment) { ++ if (sysfs_streq(command, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(command, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(command, "control-wait")) ++ ret = g5p_control_wait(g); ++ else ++ return -EINVAL; ++ } else if (sysfs_streq(command, "irq-test") && ++ (g5p_r74_case(b) || (1))) { ++ /* R74 never runs the AIC self-test r73f case 01 stopped in. */ ++ if (b->r73b) ++ atomic_inc(&b->r73b->r74_irq_refused); ++ g5p_r74_mark(g, G5P_R74_IRQ_TEST_REFUSED, -EPERM); ++ dev_err(g->dev, ++ "R74 refuse irq-test case=%u irq_tested=%u end=r74\n", ++ g5p_r74_case(b), b->irq_tested); ++ return -EPERM; ++ } else if (sysfs_streq(command, "irq-test") && irq_experiment) { ++ ret = g5p_irq_test(g); ++ } else if (sysfs_streq(command, "park") && teardown_experiment) { ++ ret = g5p_boot_park(g); ++ } else { ++ return -EPERM; ++ } ++ if (sysfs_streq(command, "control-wait")) ++ g5p_r73b_crumb(b, G5P_R73B_WAIT_EXIT, G5P_R73B_OWNER_WAIT, ret); ++ if (ret && ret != -EALREADY && ret != -EPERM) ++ WRITE_ONCE(b->failed, true); ++ return ret; ++} ++ ++/* Outgoing-side kexec/reboot quiesce; see docs/gpu/experiments/gpu-reset-recovery-production.md ++ * section 6.10. NOT a reset: Fender core=0/GPC=0 only, no PMGR off, no DMA free and ++ * no R0 access. It acts only when the command lock is free (no job or sysfs command is ++ * inside a hardware wait), the firmware was released and the pstate policy is ++ * host-owned (init meta+0x6c8 == 0); every other state is left alone. Unlike ++ * g5p_boot_park() it does not require drained rings: round 10 flew park after an ++ * unconsumed DC command, and at shutdown there is nothing left to drain for. ++ * GPC=0 is not a proof that bus traffic drained (see g5p_boot_park). ++ */ ++ ++static void g5p_td_shutdown(struct g5p_diag *g); ++ ++static void g5p_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ struct g5p_boot *b; ++ int ret; ++ ++ if (!g) ++ return; ++ /* Idempotent second chance after the reboot notifier; a no-op unless teardown != 0. */ ++ g5p_td_shutdown(g); ++ { ++ return; ++ }; ++ /* A worker or sysfs writer stuck in MMIO holds g->lock: never queue behind it. */ ++ if (!mutex_trylock(&g->lock)) { ++ dev_warn(g->dev, "SHUTDOWN park skipped: command lock busy\n"); ++ return; ++ } ++ b = g->boot; ++ /* R1 is touched only with ps_gfx held on (the retained runtime-PM reference), ++ * the wrapper woken and the firmware released. ++ */ ++ if (!b || !g->powered || !g->fender_ready || !b->maps_ready || ++ !b->released || b->stopped || b->parked) { ++ dev_info(g->dev, "SHUTDOWN park not needed\n"); ++ goto out; ++ } ++ /* With firmware-owned policy (init meta+0x6c8 != 0) the firmware also writes ++ * R1+0x1000, so a host write would race it (dsx-r74b). RAM-only read of the ++ * host-written init word, the same one g5p_dc_snapshot() reports. ++ */ ++ if (g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8)) { ++ dev_info( ++ g->dev, ++ "SHUTDOWN park refused: firmware-owned policy (meta+0x6c8 != 0)\n"); ++ goto out; ++ } ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ /* Freeze commands before the first transition; no retry even if it fails. */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ ret = g5p_fender_idle(&io); ++ if (!ret) ++ WRITE_ONCE(b->parked, true); ++ dev_info( ++ g->dev, ++ "SHUTDOWN park core=0 GPC=0 result=%d; DMA memory retained, no PMGR off\n", ++ ret); ++out: ++ mutex_unlock(&g->lock); ++} +diff --git a/drivers/gpu/apple-g5p/meta71c114.h b/drivers/gpu/apple-g5p/meta71c114.h +new file mode 100644 +index 000000000..cc84dbd82 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/meta71c114.h +@@ -0,0 +1,65 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency114: one RAM dword, meta+0x71c, in the bootstrap image. ++ * ++ * meta (object 0x4d) +0x718 is the firmware's pstate frequency table in MHz. ++ * Its init copies the table to private+0x26c at boot (MINIM 0x1000122c..127c), ++ * and the scheduler loop reloads its timer from entry 1 on both branches ++ * (0x8000a1a4 via private+0x2c4 at pstate 1, 0x8000a338 via private+0x270). ++ * Linux leaves the table zero; Apple has { 0, 340, 474, 550, 723 }. The A/B ++ * installs entry 1 only: 0 is the control (nothing is stored, variant 7 leaves ++ * the word zero), 340 the factor. Everything else is refused at map, before ++ * any store. The store goes into the staging buffer, before g5p_boot_copy() ++ * publishes the image and long before the firmware is released. There is no ++ * register access here. ++ */ ++#ifndef APPLE_G5P_META71C114_H ++#define APPLE_G5P_META71C114_H ++ ++#define G5P_M71C_SLOT 0x4dU ++#define G5P_M71C_OFFSET 0x71cU ++#define G5P_M71C_CONTROL 0U ++#define G5P_M71C_FACTOR 340U ++#define G5P_M71C_VARIANT 7U ++ ++enum { ++ G5P_M71C_OK, ++ G5P_M71C_BAD_VALUE, ++ G5P_M71C_BAD_VARIANT, ++ G5P_M71C_WHYS, ++}; ++ ++static const char *const g5p_m71c_whys[G5P_M71C_WHYS] = { ++ "none", ++ "value", ++ "variant", ++}; ++ ++/* Latched at map: only the two pre-registered arms, the factor on dc7 only. */ ++static inline u32 g5p_m71c_check(u32 requested, u32 variant) ++{ ++ if (requested != G5P_M71C_CONTROL && requested != G5P_M71C_FACTOR) ++ return G5P_M71C_BAD_VALUE; ++ if (requested == G5P_M71C_FACTOR && variant != G5P_M71C_VARIANT) ++ return G5P_M71C_BAD_VARIANT; ++ return G5P_M71C_OK; ++} ++ ++/* ++ * One store into the staging image (the factor only) and its read-back. The ++ * caller fails the map unless the read-back equals the request. ++ */ ++static inline u32 g5p_m71c_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 requested, bool *wrote) ++{ ++ u8 *word = g5p_object(p, root, shared, G5P_M71C_SLOT) + G5P_M71C_OFFSET; ++ ++ *wrote = false; ++ if (requested == G5P_M71C_FACTOR) { ++ g5p_put32(word, requested); ++ *wrote = true; ++ } ++ return g5p_le32(word); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio-licence.h b/drivers/gpu/apple-g5p/mmio-licence.h +new file mode 100644 +index 000000000..d9d23ca2a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio-licence.h +@@ -0,0 +1,148 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU register licence (docs/gpu/experiments/mmio-access.md §4), pure so the host test ++ * (scripts/gpu/perf60-host/test_mmio.c) drives the same code. iOS's ++ * AGXMetaFirmware_isGPURegisterAccessible (fffffff006ca10ec) licenses an R0 access when PMGR gfx ++ * is on and the wrapper is up (fender+0xc), R1+0 bit 29 is clear (the firmware's power-down / ++ * clock-change routine 0x1001cec8 raises it before the GPC goes to 0) and the GPC pstate nibble is ++ * 1..meta+0x710. We are stricter: R1+0x1000 bit 4 (a transition in flight) must be clear and ++ * R1+0 == 0 (a dead wrapper, Apple's ca05d4 vt+0x70 branch) refuses too. ++ * ++ * Order, and nothing else touches the bus: the RAM terms (no MMIO when any fails), then R1+0, then ++ * R1+0x1000, then at most G5P_MMIO_MAX_OPS R0 accesses, then R1+0x1000 and R1+0 again at release ++ * to account a sample whose licence changed under it (stale: data loss, not exclusion). ++ * A refusal never waits and never wakes (no R1+0x14 write): the caller decides whether to retry. ++ */ ++#ifndef G5P_MMIO_LICENCE_H ++#define G5P_MMIO_LICENCE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_MMIO_MAX_OPS 8U /* R0 accesses per ticket */ ++#define G5P_MMIO_R1_CTRL \ ++ 0x0U /* R1+0: bit 29 = firmware power-down in progress */ ++#define G5P_MMIO_R1_GPC \ ++ 0x1000U /* R1+0x1000: bit 4 transition, nibble = pstate */ ++#define G5P_MMIO_BIT29 BIT(29) ++ ++enum g5p_mmio_why { ++ G5P_MMIO_OK, ++ G5P_MMIO_OFF, /* RAM: power epoch not readable (no MMIO done) -> -ENODEV */ ++ G5P_MMIO_BUSY, /* the lock was held (try mode, IRQ census) -> -EAGAIN, no MMIO */ ++ G5P_MMIO_DEAD, /* R1+0 == 0 */ ++ G5P_MMIO_FWOFF, /* R1+0 bit 29 */ ++ G5P_MMIO_TRANS, /* R1+0x1000 bit 4 */ ++ G5P_MMIO_PARKED, /* nibble 0 */ ++ G5P_MMIO_OVERMAX, /* nibble > max pstate */ ++ G5P_MMIO_PARKEPOCH, /* R0 in a park epoch (g5p_mmio_r0_regime_ok) */ ++ G5P_MMIO_NR, ++}; ++ ++static const char *const g5p_mmio_why_names[G5P_MMIO_NR] = { ++ "read", "skip-off", "skip-busy", ++ "skip-dead", "skip-bit29", "skip-transition", ++ "skip-parked", "skip-overmax", "skip-park-epoch", ++}; ++ ++/* The RAM preconditions (§4.2 step 2). b->failed is deliberately not one: the fault read runs ++ * after it latches, as Apple's does. ++ */ ++struct g5p_mmio_ram { ++ bool powered, fender_ready, mmio_off, have_boot, released, stopped; ++}; ++ ++static inline bool g5p_mmio_ram_ok(const struct g5p_mmio_ram *r) ++{ ++ return r->powered && r->fender_ready && !r->mmio_off && r->have_boot && ++ r->released && !r->stopped; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_r1_0(u32 r1_0) ++{ ++ if (!r1_0) ++ return G5P_MMIO_DEAD; ++ if (r1_0 & G5P_MMIO_BIT29) ++ return G5P_MMIO_FWOFF; ++ return G5P_MMIO_OK; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_gpc(u32 gpc, u32 max) ++{ ++ if (gpc & 0x10) ++ return G5P_MMIO_TRANS; ++ if (!(gpc & 0xf)) ++ return G5P_MMIO_PARKED; ++ if ((gpc & 0xf) > max) ++ return G5P_MMIO_OVERMAX; ++ return G5P_MMIO_OK; ++} ++ ++/* ++ * R0 only where the firmware cannot park under it. The governor's park does not raise R1+0 ++ * bit 29 (mmio-access.md §2.3: only the power-down routine 0x1001cec8 does, a few calls before it ++ * drops the GPC), so against a park the licence's R1 checks are a race: a check that passes can be ++ * followed by the GPC going to 0 before the R0 load. The never-park envelope (boot_gpu_perf >= ++ * 100, latched at map: the policy writer's x >= 100, q >= 1, the governor never selects pstate 0) ++ * has no such window; at pin 400/500 the 3D-completion and clock samples saw bit 29 0 times, while ++ * the park tuple's rung 5 (2026-11-03) saw it on 749 of 983 3D completions and then froze the SoC. ++ * iOS reads R0 only at a fault event, never at idle. `override` = mmio_r0_park (DFU-class rungs ++ * only). ++ */ ++static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool override) ++{ ++ return override || gpu_perf >= 100; ++} ++ ++/* ++ * meta+0x710 as written into the boot image; 0 (no table) means the firmware stays at pstate 1. ++ */ ++static inline u32 g5p_mmio_max_pstate(u32 meta710) ++{ ++ return meta710 < 1 ? 1 : meta710 > 15 ? 15 : meta710; ++} ++ ++typedef u32 (*g5p_mmio_rd_t)(void *ctx, u32 off); ++ ++/* §4.2 steps 2-4 under the caller's lock: no bus access unless the RAM terms pass, R1+0 before ++ * R1+0x1000, nothing after a refusal. *r1_0 / *gpc stay U32_MAX when not read. ++ */ ++static inline enum g5p_mmio_why g5p_mmio_licence(const struct g5p_mmio_ram *ram, ++ u32 max, g5p_mmio_rd_t rd, ++ void *ctx, u32 *r1_0, u32 *gpc) ++{ ++ enum g5p_mmio_why why; ++ ++ *r1_0 = 0xffffffffU; ++ *gpc = 0xffffffffU; ++ if (!g5p_mmio_ram_ok(ram)) ++ return G5P_MMIO_OFF; ++ *r1_0 = rd(ctx, G5P_MMIO_R1_CTRL); ++ why = g5p_mmio_check_r1_0(*r1_0); ++ if (why) ++ return why; ++ *gpc = rd(ctx, G5P_MMIO_R1_GPC); ++ return g5p_mmio_check_gpc(*gpc, max); ++} ++ ++/* An R0 op is allowed while ops < G5P_MMIO_MAX_OPS; the counter counts refusals too. */ ++static inline bool g5p_mmio_op_take(u32 *ops) ++{ ++ return (*ops)++ < G5P_MMIO_MAX_OPS; ++} ++ ++/* §4.3 release: the licence fields (nibble, bit 4, bit 29) unchanged and still licensing. */ ++static inline bool g5p_mmio_stale(u32 r1_0, u32 gpc, u32 r1_0_end, u32 gpc_end, ++ u32 max) ++{ ++ if ((gpc & 0x1f) != (gpc_end & 0x1f) || ++ (r1_0 & G5P_MMIO_BIT29) != (r1_0_end & G5P_MMIO_BIT29)) ++ return true; ++ return g5p_mmio_check_r1_0(r1_0_end) || ++ g5p_mmio_check_gpc(gpc_end, max); ++} ++ ++/* mmio_selftest rung 3/5 expectations: CORE_REVISION and CORE_ID as read at bring-up. */ ++#define G5P_MMIO_EXPECT_20 0x00090f00U ++#define G5P_MMIO_EXPECT_18 0x011b0602U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio.inc b/drivers/gpu/apple-g5p/mmio.inc +new file mode 100644 +index 000000000..b8c25bd8d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio.inc +@@ -0,0 +1,295 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The one GPU register accessor (docs/gpu/experiments/mmio-access.md §4), Apple's licence from ++ * AGXMetaFirmware_isGPURegisterAccessible (mmio-licence.h) under g->mmio_lock: ++ * ++ * if (!g5p_mmio_hold(g, &t, 0)) { ++ * v = g5p_mmio_r0_read(g, &t, off); // <= G5P_MMIO_MAX_OPS, nothing else in between ++ * g5p_mmio_release(g, &t); // false: the licence changed under the sample ++ * } ++ * ++ * hold takes mmio_lock with IRQs off and returns with it held on success. It refuses instead of ++ * waiting and never wakes the GPU (no R1+0x14 write): -ENODEV when the power epoch is not ++ * readable (RAM only, no MMIO done), -EAGAIN when parked, bit 29, in transition, above the ++ * pstate table, or (G5P_MMIO_TRY) when the lock is taken; -EPERM for an R0 ticket in an epoch ++ * the firmware may park (boot_gpu_perf < 100, unless mmio_r0_park). Callable from hard IRQ, ++ * threaded IRQ and process context, inside event_lock and g->lock. Between hold and release: no ++ * printk, no other MMIO, no sleep, no allocation. ++ * ++ * mmio_selftest (0600, default 0, live) drives the proof rungs of §6: ++ * bit 0 rung 1/1b: 1 kHz hrtimer census, R1+0 and R1+0x1000 only (no R0) ++ * bit 1 rung 3/4: every 100 ms hold -> R0+0x20, R0+0x18 -> release (DFU-risk with parks: rung 4) ++ * bit 2 rung 5: the same at every 3D completion decision (drm side; DFU-risk, park tuple) ++ * bit 3 idle check: after 5, 30 and 120 s with no job, the PMGR gfx word, then (gfx on only) ++ * R1+0x1000 and R1+0; never R0 ++ * Lines: GPU-MMIO sample (per sample, at most one per source per second unless the outcome ++ * changes or the sample is stale/bad), GPU-MMIO idle, GPU-MMIO stats (every 10 s while on and ++ * once when it goes back to 0). ++ */ ++#define G5P_MMIO_TRY \ ++ BIT(0) /* trylock: refuse busy instead of spinning (the IRQ census) */ ++/* An R1-only ticket, counted in the census class: held whenever the RAM terms pass (R1 is ++ * readable while PMGR gfx is on, mmio-access.md §1.3), R1+0 and R1+0x1000 both read even when the ++ * licence refuses; R0 only if the licence passed (t->r0_ok). ++ */ ++#define G5P_MMIO_CENSUS BIT(1) ++ ++struct g5p_mmio_ticket { ++ u32 r1_0, gpc, r1_0_end, gpc_end, flags, ops; ++ u64 t_ns, t_end_ns; ++ unsigned long irq; ++ enum g5p_mmio_why why; ++ bool held, stale, r0_ok; ++}; ++ ++enum { ++ G5P_MMS_CENSUS, ++ G5P_MMS_IDLE, ++ G5P_MMS_3D, ++ G5P_MMS_FAULT, ++ G5P_MMS_PROBE, ++ G5P_MMS_NR ++}; ++static const char *const g5p_mms_names[G5P_MMS_NR] = { ++ "census", "idle-100ms", "3d-done", "fault-read", "dc-probe", ++}; ++ ++#define G5P_MMIO_RING 32 ++struct g5p_mmio_rec { ++ u64 t_ns; ++ u32 r1_0, gpc, us; ++ u8 why, stale; ++}; ++ ++struct g5p_mmio_self { ++ struct g5p_diag *g; ++ struct hrtimer timer; ++ struct delayed_work work; ++ bool census_on, was_on; ++ u64 stats_ns; ++ /* counters: [0] R1-only census tickets, [1] R0 tickets */ ++ atomic_t n[2][G5P_MMIO_NR]; ++ atomic_t stale[2], r0_ops, bad; ++ /* census transitions, written by the hrtimer, drained by the work */ ++ spinlock_t ring_lock; ++ struct g5p_mmio_rec ring[G5P_MMIO_RING]; ++ u32 ring_head, ring_tail, ring_drop; ++ struct g5p_mmio_rec census_last; ++ u8 census_prev_why, census_prev_stale; ++ /* per-source print limiter */ ++ u64 printed_ns[G5P_MMS_NR]; ++ u8 printed_why[G5P_MMS_NR]; ++ /* idle check */ ++ u64 job_ns; ++ u32 idle_done; ++ /* bit 4, the GPU clock: per Fender pstate nibble, the highest rate seen and the sample counts */ ++ u32 clk_max_khz[16], clk_cr_khz[16], clk_n[16], clk_busy[16], clk_reset; ++ u64 clk_printed_ns; ++}; ++ ++/* mmio_selftest bit 4: the gap between the two tickets of one clock sample */ ++ ++/* R0 tickets in a park epoch (boot_gpu_perf < 100): refused unless 1 (g5p_mmio_r0_regime_ok). */ ++ ++static u32 g5p_mmio_rd_r1(void *ctx, u32 off) ++{ ++ return readl((void __iomem *)ctx + off); ++} ++ ++static void g5p_mmio_count(struct g5p_diag *g, const struct g5p_mmio_ticket *t) ++{ ++ struct g5p_mmio_self *s = READ_ONCE(g->mmio_self); ++ u32 cls = t->flags & G5P_MMIO_CENSUS ? 0 : 1; ++ ++ if (!s) ++ return; ++ atomic_inc(&s->n[cls][t->why]); ++ if (t->stale) ++ atomic_inc(&s->stale[cls]); ++ if (t->ops && cls) ++ atomic_add(min_t(u32, t->ops, G5P_MMIO_MAX_OPS), &s->r0_ops); ++} ++ ++static int g5p_mmio_hold(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags) ++{ ++ struct g5p_mmio_ram ram = { 0 }; ++ struct g5p_boot *b; ++ unsigned long irq; ++ ++ memset(t, 0, sizeof(*t)); ++ t->flags = flags; ++ t->r1_0 = t->gpc = t->r1_0_end = t->gpc_end = U32_MAX; ++ if (flags & G5P_MMIO_TRY) { ++ if (!spin_trylock_irqsave(&g->mmio_lock, irq)) { ++ t->why = G5P_MMIO_BUSY; ++ goto refused; ++ } ++ } else { ++ spin_lock_irqsave(&g->mmio_lock, irq); ++ } ++ t->t_ns = ktime_get_mono_fast_ns(); ++ ram.mmio_off = g->mmio_off; ++ ram.powered = READ_ONCE(g->powered); ++ ram.fender_ready = READ_ONCE(g->fender_ready); ++ /* mmio_off first: a boot struct behind a gate is never dereferenced */ ++ b = ram.mmio_off ? NULL : READ_ONCE(g->boot); ++ if (b) { ++ ram.have_boot = true; ++ ram.released = READ_ONCE(b->released); ++ ram.stopped = READ_ONCE(b->stopped); ++ } ++ /* before any bus access: an R0 ticket in a park epoch is refused outright */ ++ if (!(flags & G5P_MMIO_CENSUS) && b && ++ !g5p_mmio_r0_regime_ok(b->gpu_perf, (0))) { ++ t->why = G5P_MMIO_PARKEPOCH; ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ t->why = g5p_mmio_licence(&ram, READ_ONCE(g->mmio_max_pstate), ++ g5p_mmio_rd_r1, g->window[1], &t->r1_0, ++ &t->gpc); ++ if (t->why && (!(flags & G5P_MMIO_CENSUS) || t->why == G5P_MMIO_OFF)) { ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ if (t->gpc == U32_MAX) ++ t->gpc = readl( ++ g->window[1] + ++ G5P_MMIO_R1_GPC); /* census: after an R1+0 refusal */ ++ t->r0_ok = !t->why; ++ t->irq = irq; ++ t->held = true; ++ return 0; ++refused: ++ g5p_mmio_count(g, t); ++ /* a park epoch does not change by waiting: -EPERM, never retried (g5p_mmio_hold_wait) */ ++ return t->why == G5P_MMIO_OFF ? -ENODEV : ++ t->why == G5P_MMIO_PARKEPOCH ? -EPERM : ++ -EAGAIN; ++} ++ ++/* Only between a successful hold and its release; past G5P_MMIO_MAX_OPS no bus access. */ ++static u32 g5p_mmio_r0_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->rgx + off); ++} ++ ++/* R0 writes count as ops too (the slave-port request words of the pin-200 probe). */ ++static bool g5p_mmio_r0_write(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off, u32 v) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return false; ++ writel(v, g->rgx + off); ++ return true; ++} ++ ++/* Any R1 word under a held ticket (census or licensed); counts as an op. */ ++static u32 g5p_mmio_r1_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->window[1] + off); ++} ++ ++/* Process context: refusals (-EAGAIN) retried outside the lock every 20-40 us, <= budget_us. */ ++static int g5p_mmio_hold_wait(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags, u32 budget_us, u32 *tries) ++{ ++ u64 t0 = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ might_sleep(); ++ for (;;) { ++ (*tries)++; ++ ret = g5p_mmio_hold(g, t, flags); ++ if (ret != -EAGAIN || ktime_get_mono_fast_ns() - t0 >= ++ (u64)budget_us * NSEC_PER_USEC) ++ return ret; ++ usleep_range(20, 40); ++ } ++} ++ ++static bool g5p_mmio_release(struct g5p_diag *g, struct g5p_mmio_ticket *t) ++{ ++ if (!t->held) ++ return false; ++ t->gpc_end = readl(g->window[1] + G5P_MMIO_R1_GPC); ++ t->r1_0_end = readl(g->window[1] + G5P_MMIO_R1_CTRL); ++ t->stale = t->r0_ok && ++ g5p_mmio_stale(t->r1_0, t->gpc, t->r1_0_end, t->gpc_end, ++ READ_ONCE(g->mmio_max_pstate)); ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ t->held = false; ++ spin_unlock_irqrestore(&g->mmio_lock, t->irq); ++ g5p_mmio_count(g, t); ++ return !t->stale; ++} ++ ++static const char *g5p_mmio_did(const struct g5p_mmio_ticket *t) ++{ ++ return t->stale ? "stale" : g5p_mmio_why_names[t->why]; ++} ++ ++static u32 g5p_mmio_us(const struct g5p_mmio_ticket *t) ++{ ++ return t->t_end_ns > t->t_ns ? ++ (u32)div_u64(t->t_end_ns - t->t_ns, NSEC_PER_USEC) : ++ 0; ++} ++ ++/* The per-sample line, after release. force: print regardless of the per-source limiter. */ ++static void g5p_mmio_sample_line(struct g5p_diag *g, u32 src, ++ const struct g5p_mmio_ticket *t, u32 *v, ++ u32 nv, bool force) ++{ ++ struct g5p_mmio_self *s = g->mmio_self; ++ u64 now = ktime_get_mono_fast_ns(); ++ char vs[9 * G5P_MMIO_MAX_OPS + 1] = "-"; ++ u32 len = 0; ++ ++ if (s && !force && !t->stale && s->printed_why[src] == t->why && ++ now - s->printed_ns[src] < NSEC_PER_SEC) ++ return; ++ if (s) { ++ s->printed_ns[src] = now; ++ s->printed_why[src] = t->why; ++ } ++ for (u32 i = 0; i < nv && i < G5P_MMIO_MAX_OPS; i++) ++ len += scnprintf(vs + len, sizeof(vs) - len, "%s%08x", ++ i ? "," : "", v[i]); ++ dev_info( ++ g->dev, ++ "GPU-MMIO sample why=%s did=%s r1_0=%08x gpc=%08x r1_0_end=%08x gpc_end=%08x " ++ "max=%u v=%s us=%u end=mmio\n", ++ g5p_mms_names[src], g5p_mmio_did(t), t->r1_0, t->gpc, ++ t->r1_0_end, t->gpc_end, READ_ONCE(g->mmio_max_pstate), vs, ++ g5p_mmio_us(t)); ++} ++ ++/* Rung 3/5: one licensed ticket, CORE_REVISION and CORE_ID. */ ++ ++/* drm side: rung 5 at the 3D completion decision, and the idle clock for bit 3. */ ++ ++/* Bit 4: the GPU core clock. CR_TIMER (R0+0x160) ticks once per 256 core clocks whether or not a ++ * phase is open (r3clk: 2823 kHz at pstate 4 = 722.7 MHz, 3703 kHz at 948 MHz), so CR_TIMER over ++ * host time x 256 is the clock at any moment. The Rogue PERF_TA_CYCLE / PERF_3D_CYCLE / ++ * PERF_TA_OR_3D_CYCLE counters (R0+0x6020/0x6028/0x6038, 32 bit) advance at the core clock while a ++ * TA or 3D phase is open (340.0 MHz through open TA phases, docs/gpu/experiments/46-device-tests.md): ++ * over host time, the highest rate per pstate (a window busy all through) is the clock again. One ++ * sample = two licensed tickets, mmio_clock_gap_us apart, each reading the three counters and ++ * CR_TIMER's low word back to back. A cycle rate above 2 GHz is a firmware counter clear ++ * (R0+0x6000 = 1, 0x1001ab00) inside the window and is dropped. ++ */ ++ ++/* Bit 3: does the GPC power down at full idle in state 0? The PMGR word is always readable; ++ * R1 only while the gate reads on (actual == target == 0xf), through a census ticket so the RAM ++ * terms and the lock still apply. A refused licence still returns the R1 words it read. ++ */ +diff --git a/drivers/gpu/apple-g5p/mmu-attrs.inc b/drivers/gpu/apple-g5p/mmu-attrs.inc +new file mode 100644 +index 000000000..694552016 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-attrs.inc +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include ++#include ++ ++/* Read the actual stable kernel mapping of a retained DMA allocation. ++ * DMA memory may be a vmalloc PTE or a cached linear-map block. ++ */ ++static u64 g5p_table_cpu_descriptor(unsigned long va) ++{ ++ pgd_t *pgd = pgd_offset_k(va); ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ ++ if (pgd_none(READ_ONCE(*pgd)) || pgd_bad(READ_ONCE(*pgd))) ++ return 0; ++ p4d = p4d_offset(pgd, va); ++ if (p4d_none(READ_ONCE(*p4d)) || p4d_bad(READ_ONCE(*p4d))) ++ return 0; ++ pud = pud_offset(p4d, va); ++ if (pud_leaf(READ_ONCE(*pud))) ++ return pud_val(READ_ONCE(*pud)); ++ if (pud_none(READ_ONCE(*pud)) || pud_bad(READ_ONCE(*pud))) ++ return 0; ++ pmd = pmd_offset(pud, va); ++ if (pmd_leaf(READ_ONCE(*pmd))) ++ return pmd_val(READ_ONCE(*pmd)); ++ if (pmd_none(READ_ONCE(*pmd)) || pmd_bad(READ_ONCE(*pmd))) ++ return 0; ++ return pte_val(READ_ONCE(*pte_offset_kernel(pmd, va))); ++} ++ ++static void g5p_table_attrs(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *p) ++{ ++ u64 desc, mair = read_sysreg(mair_el1); ++ u32 index, attr; ++ ++ if (!p->cpu) ++ return; ++ desc = g5p_table_cpu_descriptor((unsigned long)p->cpu); ++ index = (desc & PTE_ATTRINDX_MASK) >> 2; ++ attr = (mair >> (index * 8)) & 0xff; ++ g5p_mmu_record( ++ g, ++ "MMU allocation %s DMA=%pad coherent=%u cpu_desc=%016llx AttrIndx=%u " ++ "MAIR=%016llx attr=%02x table_memtype=%s\n", ++ stage, &p->dma, dev_is_dma_coherent(g->dev), desc, index, mair, ++ attr, ++ !desc ? "unknown" : ++ attr == 0xff ? "Normal-WB" : ++ attr == 0x44 ? "Normal-NC" : ++ "other"); ++} +diff --git a/drivers/gpu/apple-g5p/mmu-tables.inc b/drivers/gpu/apple-g5p/mmu-tables.inc +new file mode 100644 +index 000000000..94ae6ed8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-tables.inc +@@ -0,0 +1,57 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Fixed, pre-release aliases only. No firmware patch or live table surgery. */ ++#define G5P_TABLE_ALIAS_VA (G5P_ROOT_VA + 0x100000ULL) ++#define G5P_TABLE_COOKIE 0x3ffcU ++ ++static void g5p_table_clean_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ unsigned long start; ++ ++ if (!p->cpu) ++ return; ++ start = (unsigned long)p->cpu; ++ /* Explicit architecture PoC operation, even if DT calls this coherent. ++ * dma_sync_single_for_device is not an API for coherent allocations. ++ */ ++ dcache_clean_poc(start, start + G5P_GPU_PAGE); ++} ++ ++static void g5p_tables_clean(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *user, u32 count) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ g5p_mmu_record( ++ g, ++ "MMU tables %s clean_poc=%u dma_coherent=%u dma_pgprot=%016llx; coherent DMA " ++ "API, extra clean is experimental\n", ++ stage, 1, dev_is_dma_coherent(g->dev), ++ (u64)pgprot_val(dma_pgprot(g->dev, PAGE_KERNEL, 0))); ++ for (u32 i = 0; i < count; i++) ++ g5p_table_attrs(g, stage, &user[i]); ++ g5p_table_attrs(g, stage, &b->catalogue); ++ g5p_table_attrs(g, stage, &b->directory); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_attrs(g, stage, &b->leaf[i]); ++ ; ++ ; ++ for (u32 i = 0; i < count; i++) ++ g5p_table_clean_page(g, &user[i]); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_clean_page(g, &b->leaf[i]); ++ g5p_table_clean_page(g, &b->sink_leaf); ++ g5p_table_clean_page(g, &b->sink_directory); ++ g5p_table_clean_page(g, &b->directory); ++ g5p_table_clean_page(g, &b->catalogue); ++ dsb(sy); ++ dma_wmb(); ++ g5p_mmu_record( ++ g, ++ "MMU tables %s PoC complete user_pages=%u boot_leaves=%u; dsb sy before publication\n", ++ stage, count, b->leaves); ++} ++ ++/* Deliberately separate from g5p_meta_read(): the generic slave guard stays ++ * latched after reset. This opt-in reader can address only the preinstalled ++ * table aliases and one immutable root control word, never private META RAM. ++ */ +diff --git a/drivers/gpu/apple-g5p/pack.h b/drivers/gpu/apple-g5p/pack.h +new file mode 100644 +index 000000000..942b44ba3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/pack.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_PACK_H ++#define APPLE_G5P_PACK_H ++ ++#include "bits-compat.h" ++#include "ldr.h" ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_IMAGE_SIZE 0x30000 ++#define G5P_CONFIG_WRITES 27 ++ ++struct g5p_ldr_image { ++ u8 *data; ++ u32 config[G5P_CONFIG_WRITES][2]; ++ u32 config_count; ++ u32 t1_pc; ++ u32 t1_sp; ++}; ++ ++/* Exact C0 ranges derived from AGXMetaFirmware_layoutAndLoadLDR ...6c9f1bc. ++ * GPU virtual mappings and the boot config at +0x200 are intentionally separate. ++ */ ++static inline int g5p_image_offset(u32 addr, u32 size, u32 *offset) ++{ ++ static const struct { ++ u32 base, size, packed; ++ } segments[] = { ++ { 0x10000000, 0x18000, 0x04000 }, ++ { 0x20000000, 0x04000, 0x1c000 }, ++ { 0xc0000000, 0x04000, 0x00000 }, ++ { 0xe0001000, 0x04000, 0x20000 }, ++ { 0x80000080, 0x055dc, 0x24000 }, ++ }; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(segments); i++) { ++ if (addr >= segments[i].base && size <= segments[i].size && ++ addr - segments[i].base <= segments[i].size - size) { ++ *offset = segments[i].packed + addr - segments[i].base; ++ return 0; ++ } ++ } ++ return -EINVAL; ++} ++ ++static inline int g5p_pack_record(void *ctx, enum g5p_ldr_op op, u32 addr, ++ u32 size, const u8 *data) ++{ ++ struct g5p_ldr_image *image = ctx; ++ u32 i, offset; ++ int ret; ++ ++ switch (op) { ++ case G5P_LOAD: ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memcpy(image->data + offset, data, size); ++ break; ++ case G5P_ZERO: ++ /* Apple does not apply the local/core-memory ZeroMem records here. */ ++ if (addr >= 0x80000000 && addr < 0x88000000) ++ break; ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memset(image->data + offset, 0, size); ++ break; ++ case G5P_CONFIG: ++ for (i = 0; i < size; i += 12) { ++ if (image->config_count == G5P_CONFIG_WRITES) ++ return -EINVAL; ++ image->config[image->config_count][0] = ++ g5p_le32(data + i + 4); ++ image->config[image->config_count++][1] = ++ g5p_le32(data + i + 8); ++ } ++ break; ++ case G5P_START: ++ image->t1_sp = g5p_le32(data + 4); ++ image->t1_pc = g5p_le32(data + 8); ++ break; ++ } ++ return 0; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/perf-envelope.h b/drivers/gpu/apple-g5p/perf-envelope.h +new file mode 100644 +index 000000000..57387bcba +--- /dev/null ++++ b/drivers/gpu/apple-g5p/perf-envelope.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 clock: the inputs of the firmware's policy-state writer (0x10008218) and of its governor, ++ * so the firmware itself moves the GPC from pstate 1 to q = pin / 100. Both compute ++ * x = min(policy+0x6f0, policy+0x688, ctrl+0x102c) ++ * (META CS is unsigned lower; iOS's live policy+0x38 equals this min in every sample), store ++ * policy+8 = 0 when x % 100 == 0 (2 when x < 100), and request pstate q = x / 100 (policy+0x28). ++ * +0x6f0 and +0x688 are recomputed by the firmware from its coefficient block unless their ++ * overrides +0x6f4 / +0x68c are nonzero, and the firmware itself reloads those overrides from ++ * the control object on every pass, whatever the record held: +0x6f4 = ctrl+0x1040 (0x1000a1e4, ++ * 0x1000b388, 0x1000c568) and +0x68c = ctrl+0x1038 (0x1000a74c, 0x1000b10c, 0x1000c2fc). iOS ++ * leaves both control words 0 and arms the sampler (policy+4 = 16); this boot pins them ++ * (ctrl+0x1040 = pin, ctrl+0x1038 = 400) and leaves the sampler off (policy+4 = 0), so there is ++ * no idle parking at pstate 0. The pstate table and limits are iOS's (live j71t capture, ++ * 2026-11-01): meta+0x710/0x714 = 4/5, MHz {0, 340, 474, 550, 723}, ctrl+0x1028/0x1030 = ++ * 3348/400; the floor ctrl+0x102c is the pin. RAM only, before release. ++ */ ++#ifndef APPLE_G5P_PERF_ENVELOPE_H ++#define APPLE_G5P_PERF_ENVELOPE_H ++ ++#include "bits-compat.h" ++#define G5P_PERF_POLICY 0x4aU ++#define G5P_PERF_META 0x4dU ++#define G5P_PERF_CTL 0x5eU ++#define G5P_PERF_TOP 400U ++ ++static const u32 g5p_perf_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline bool g5p_perf_valid(u32 pin) ++{ ++ return pin <= G5P_PERF_TOP && !(pin % 100); ++} ++ ++static inline void g5p_perf_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 pin) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ u8 *meta = g5p_object(p, root, shared, G5P_PERF_META); ++ u8 *ctl = g5p_object(p, root, shared, G5P_PERF_CTL); ++ u32 i; ++ ++ g5p_put32(meta + 0x710, ARRAY_SIZE(g5p_perf_mhz) - 1); ++ g5p_put32(meta + 0x714, ARRAY_SIZE(g5p_perf_mhz)); ++ for (i = 0; i < ARRAY_SIZE(g5p_perf_mhz); i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_perf_mhz[i]); ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, pin); ++ g5p_put32(ctl + 0x1030, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1038, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1040, pin); ++ g5p_put32(policy + 0x688, G5P_PERF_TOP); ++ g5p_put32(policy + 0x68c, G5P_PERF_TOP); ++ g5p_put32(policy + 0x6f0, pin); ++ g5p_put32(policy + 0x6f4, pin); ++} ++ ++/* boot_gpu_oc_mhz (gpu-oc.h): state 5 = the overclock record; the firmware's table, ceilings and ++ * policy limits extended to index 5 so the pin 500 and the power-on request of state 5 are in range ++ * (the firmware derives its clock entry priv+0x2c4 = meta+0x718[state] from Fender, 0x1001c838). ++ */ ++ ++/* ++ * boot_gpu_perf_seed (perf60-clock.md "Clock-change drain RE" §3): in policy state 0 the ++ * firmware ++ * never changes pstate while powered; only its boot sampler (0x1000aa80, before ALIVE) moves the ++ * GPC, when policy+0x28 != Fender & 0xf. Seeding the policy record with the firmware's own ++ * steady-state output for the pin, +0x28 = +0x2c = q, +0x30 = 0, +8 = 0, makes that sampler ++ * request q through its setter (0x1001c9b8 -> 0x1001c88c: pol34 = q, R1+0x1000 = q | 0x10), ++ * which is ++ * iOS's park -> reboot -> sampler path. q = pin / 100 for pins 200..400, else 0 (no seed). ++ */ ++static inline u32 g5p_perf_seed_q(u32 pin) ++{ ++ return pin >= 200 && pin <= G5P_PERF_TOP && !(pin % 100) ? pin / 100 : ++ 0; ++} ++ ++/* Seed 2 adds what the sampler needs to get past its first-call init: 0x1000aa80 at 0x1000abb0 ++ * takes policy+0 == 0 as "never sampled" and only initialises (0x1000ae88: +0 = 1, its time words) ++ * without reaching the target compare at 0x1000ada4 or the setter; its sampling path divides by ++ * policy+4 (0x10022f88 at 0x1000ac28). The boot call at 0x10000b54 is its only caller, so a fresh ++ * record never moves Fender. iOS's record persists across firmware reboots: +0 = 1 and +4 = 16 in ++ * every sample of the 2026-11-01 capture (idle and load). Seed 2 writes those two as iOS has them. ++ * Seed 2 on hardware (2026-11-03): no ALIVE, Fender 0. With +0 = 1 the sampler takes its sampling ++ * path when (now - policy+0x8f8) / 24000 / policy+4 >= 1 (0x1000ac10..ac5c): +0x8f8 is the host's ++ * map-time stamp, ~3 s old at release, so it runs the load governor (0x1000afa4 -> 0x1000b050, busy ++ * accumulators policy+0x62c..0x68c, all 0 on a fresh record) and lands on pstate 0. Seed 3 ++ * (boot.inc ++ * g5p_boot_release) restamps +0x8f8 right before the release stream: < 1 period elapsed, the load ++ * update is skipped (0x1000ac64 -> 0x1000acb0 -> 0x1000ada4) and the compare against +0x28 = q ++ * reaches the setter (0x1000ae68 -> 0x1001c9b8). ++ */ ++static inline void g5p_perf_seed(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 q, bool ios_record) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ ++ g5p_put64(policy + 0x28, (u64)q << 32 | q); ++ g5p_put32(policy + 0x30, 0); ++ g5p_put32(policy + 8, 0); ++ if (ios_record) { ++ g5p_put32(policy + 0, 1); ++ g5p_put32(policy + 4, 16); ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/power-r33.inc b/drivers/gpu/apple-g5p/power-r33.inc +new file mode 100644 +index 000000000..ce8a18f3c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r33.inc +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R33 (ds-r33-b knob A): the kexec GFX cycle holds ps_gfx off for ~10 us, and ++ * R0+0x418/0x41c (no reset) keeps the previous boot's value through it. At the ++ * first Fender wake, before any GPU MMIO and before any GPU IRQ is requested, ++ * hold the same PMGR gate (0x80340, gate124) off for boot_gfx_off_ms, then ++ * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset, ++ * auto). Every exit leaves the gate requested on. ++ */ ++#define G5P_R33_GFX_PS 0x80340U ++#define G5P_R33_GFX_OFF_MAX_MS 2000U ++#define G5P_R33_GFX_POLL_US 10000U ++ ++static inline bool g5p_r33_gfx_on(u32 raw) ++{ ++ return (raw & 0xf) == 0xf && ((raw >> 4) & 0xf) == 0xf; ++} ++ ++/* Only a plain, fully-on, software-controlled gate may be cycled. */ ++static inline bool g5p_r33_gfx_cyclable(u32 raw) ++{ ++ return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31) && ++ !((raw >> 28) & 1); ++} ++ ++static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) ++{ ++ u32 before, held, v = 0, after = 0; ++ u64 t0; ++ int ret, on; ++ ++ if (!ms) ++ return 0; ++ if (ms > G5P_R33_GFX_OFF_MAX_MS || !pmgr) ++ return -EINVAL; ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &before); ++ if (ret) ++ return ret; ++ if (!g5p_r33_gfx_cyclable(before)) { ++ dev_info( ++ dev, ++ "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset/auto or not on\n", ++ ms, before); ++ return -EBUSY; ++ } ++ ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0); ++ if (!ret) ++ ret = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, v, ++ !((v >> 4) & 0xf), 10, ++ G5P_R33_GFX_POLL_US); ++ t0 = ktime_get_ns(); ++ if (!ret) ++ msleep(ms); ++ held = (u32)((ktime_get_ns() - t0) / 1000); ++ if (!ret) ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &v); ++ /* Never leave the GPU domain off, whatever happened above. */ ++ on = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0xf); ++ if (!on) ++ on = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, after, ++ g5p_r33_gfx_on(after), 10, ++ G5P_R33_GFX_POLL_US); ++ dev_info( ++ dev, ++ "POWER gfx-off ms=%u held_us=%u before=%08x end=%08x after=%08x off_ret=%d " ++ "on_ret=%d\n", ++ ms, held, before, v, after, ret, on); ++ if (!ret && (v >> 4) & 0xf) ++ ret = -EIO; ++ return ret ?: on; ++} +diff --git a/drivers/gpu/apple-g5p/power-r41.inc b/drivers/gpu/apple-g5p/power-r41.inc +new file mode 100644 +index 000000000..9e5c1e668 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r41.inc +@@ -0,0 +1,135 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* J71t live ADT + ApplePMGR_restoreDeviceBridgeSettings (5a39918). ++ * Volatile fabric MMIO only. GFX selects bridge 8, then bridge 9; the two ++ * version-0 counter controls follow bridge 8's table, before bridge 9. ++ * Keep devm_ioremap/readl/writel identical to boot.inc's swirq mapping. ++ */ ++static const unsigned int gfx_bridge_replay = 0; ++ ++struct g5p_r41_write { ++ u32 bridge, offset, value; ++}; ++ ++static const struct g5p_r41_write g5p_r41_writes[] = { ++ { 8, 0x0000, 0x00000120 }, { 8, 0x0018, 0x80010005 }, ++ { 8, 0x001c, 0x00010001 }, { 8, 0x410c, 0x00000003 }, ++ { 8, 0x4100, 0x00000003 }, { 9, 0x0000, 0x00697101 }, ++ { 9, 0x0004, 0x00697101 }, { 9, 0x0008, 0x00697101 }, ++ { 9, 0x000c, 0x00697101 }, { 9, 0x0010, 0x00697101 }, ++ { 9, 0x0014, 0x00697101 }, { 9, 0x0018, 0x00697101 }, ++ { 9, 0x001c, 0x00697101 }, { 9, 0x0020, 0x00697101 }, ++ { 9, 0x0024, 0x00697101 }, { 9, 0x0028, 0x00697101 }, ++ { 9, 0x002c, 0x00697101 }, { 9, 0x0030, 0x00697101 }, ++ { 9, 0x0034, 0x00697101 }, { 9, 0x0038, 0x00697101 }, ++ { 9, 0x003c, 0x00697101 }, { 9, 0x0040, 0x00697101 }, ++ { 9, 0x0044, 0x00697101 }, { 9, 0x0048, 0x00697101 }, ++ { 9, 0x004c, 0x00697101 }, { 9, 0x0050, 0x00697101 }, ++ { 9, 0x0054, 0x00697101 }, { 9, 0x0058, 0x00697101 }, ++ { 9, 0x005c, 0x00697101 }, { 9, 0x0060, 0x00697101 }, ++ { 9, 0x0064, 0x00697101 }, { 9, 0x0068, 0x00697101 }, ++ { 9, 0x006c, 0x00697101 }, { 9, 0x0070, 0x00697101 }, ++ { 9, 0x0074, 0x00697101 }, { 9, 0x0078, 0x00697101 }, ++ { 9, 0x007c, 0x00697101 }, ++}; ++ ++static struct { ++ void __iomem *window[2]; ++ unsigned int mode; ++ int result; ++ bool attempted, ready, firmware_logged, ta_logged; ++} g5p_r41_bridge; ++ ++/* Reject anything outside the exact write allowlist, including unaligned ++ * offsets and other registers inside the two mapped ADT ranges. ++ */ ++static bool g5p_r41_valid(const struct g5p_r41_write *w) ++{ ++ if (w->offset & 3) ++ return false; ++ if (w->bridge == 8) ++ return w->offset < 0x5000 && ++ (w->offset == 0 || w->offset == 0x18 || ++ w->offset == 0x1c || w->offset == 0x410c || ++ w->offset == 0x4100); ++ return w->bridge == 9 && w->offset <= 0x7c; ++} ++ ++static void g5p_r41_bridge_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ u32 value = ++ readl(g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ ++ dev_info( ++ g->dev, ++ "BRIDGE r41 stage=%s mode=%u index=%u bridge=%u off=%04x value=%08x want=%08x\n", ++ stage, g5p_r41_bridge.mode, i, w->bridge, w->offset, ++ value, w->value); ++ } ++} ++ ++/* Called under g->lock after ACTIVE (and any r33 off/on), before Fender. ++ * Latch errors too: never retry a partial experiment in the same boot. ++ */ ++static int g5p_r41_bridge_prepare(struct g5p_diag *g) ++{ ++ unsigned int i, mode = gfx_bridge_replay; ++ ++ if (!mode) ++ return 0; ++ if (g5p_r41_bridge.attempted) ++ return g5p_r41_bridge.result; ++ g5p_r41_bridge.attempted = true; ++ g5p_r41_bridge.mode = mode; ++ g5p_r41_bridge.result = -EINVAL; ++ if (mode > 4) ++ return -EINVAL; ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) ++ if (!g5p_r41_valid(&g5p_r41_writes[i])) ++ return -EINVAL; ++ g5p_r41_bridge.result = -ENOMEM; ++ g5p_r41_bridge.window[0] = devm_ioremap(g->dev, 0x201f00000ULL, 0x5000); ++ if (!g5p_r41_bridge.window[0]) ++ return -ENOMEM; ++ g5p_r41_bridge.window[1] = devm_ioremap(g->dev, 0x201f80000ULL, 0x1000); ++ if (!g5p_r41_bridge.window[1]) ++ return -ENOMEM; ++ /* Read all 37 even in split modes; never read unmapped/partial state. */ ++ g5p_r41_bridge_snapshot(g, "before"); ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ ++ if (mode == 1 || (mode == 3 && w->bridge != 8) || ++ (mode == 4 && w->bridge != 9)) ++ continue; ++ writel(w->value, ++ g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ dev_info( ++ g->dev, ++ "BRIDGE r41 write mode=%u index=%u bridge=%u off=%04x value=%08x\n", ++ mode, i, w->bridge, w->offset, w->value); ++ } ++ g5p_r41_bridge_snapshot(g, "after"); ++ g5p_r41_bridge.ready = true; ++ g5p_r41_bridge.result = 0; ++ return 0; ++} ++ ++static void g5p_r41_bridge_firmware(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.firmware_logged) { ++ g5p_r41_bridge.firmware_logged = true; ++ g5p_r41_bridge_snapshot(g, "firmware-alive"); ++ } ++} ++ ++static void g5p_r41_bridge_ta(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.ta_logged) { ++ g5p_r41_bridge.ta_logged = true; ++ g5p_r41_bridge_snapshot(g, "first-TA-kick"); ++ } ++} +diff --git a/drivers/gpu/apple-g5p/r74e112.h b/drivers/gpu/apple-g5p/r74e112.h +new file mode 100644 +index 000000000..cacac73ac +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r74e112.h +@@ -0,0 +1,137 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R74E112_H ++#define G5P_R74E112_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * Wake112: the R74E GPC1 power proof when ALIVE preempts the on-poll. ++ * ++ * R74E (boot.inc) requests GPC=1 and polls R1+0x1000 until the state field reads 1. ++ * The poll stops as soon as ALIVE is decoded: after ALIVE only the first-ALIVE owner ++ * may read R1. On a warm, kexec-cycled boot the firmware can reach ALIVE before the ++ * poll's first read (Wake111 20261031T201022Z: off_ns=1500 on_ns=375 raw=ffffffff, ++ * the U32_MAX "not read" sentinel). Wake111 then latched a power abort although the ++ * state was never sampled. ++ * ++ * Wake112 defers that one proof to the owner instead, but only when the on-poll took ++ * no sample: the poll counts its reads, and the U32_MAX raw is never taken as "not read". ++ * The owner's existing first-ALIVE read of R1+0x1000 (g5p_r73e_irq_submit, before any ++ * register write) is the proof. State 1 sets release_ready and the unchanged cold ++ * sequence continues. Any other value (0, 0x10, 0x11, ffffffff, ...) is an absorbing ++ * power abort. An on-poll that reads state 1 never defers. An on-poll that sampled ++ * before ALIVE (0x11 in rounds 74e..79b, or any other word) and every other R74E failure ++ * (off-poll, timeout, GPC1 write refused) abort exactly as before. ++ * ++ * States (r73b->early_proof): ++ * NONE no deferral: the poll decided, or the release has not run ++ * DEFERRED ALIVE preempted the on-poll; one owner read is owed (generation recorded) ++ * TAKEN the once-owner claimed it; it settles with no exit in between ++ * PROVEN that read had state 1: release_ready ++ * FAILED any other value, another generation, a lost precondition, or an ALIVE ++ * wait that ended with the proof still owed: power abort, absorbing ++ * ++ * Pure policy: no register access, no lock, no kernel API. ++ */ ++ ++#define G5P_PX_STATE_MASK 0x1fU ++#define G5P_PX_STATE_ON 0x01U ++ ++enum g5p_px { ++ G5P_PX_NONE, ++ G5P_PX_DEFERRED, ++ G5P_PX_TAKEN, ++ G5P_PX_PROVEN, ++ G5P_PX_FAILED, ++ G5P_PX_N, ++}; ++ ++static const char *const g5p_px_name[G5P_PX_N] = { ++ "none", "deferred", "taken", "proven", "failed", ++}; ++ ++/* Why a deferred proof failed (NONE: it did not). */ ++enum g5p_px_why { ++ G5P_PXW_NONE, ++ G5P_PXW_STATE, /* the read's state field is not 1 */ ++ G5P_PXW_STALE, /* deferred in another reset generation */ ++ G5P_PXW_LOST, /* ALIVE or the release lost, stopped or failed since */ ++ G5P_PXW_RETIRED, /* a FINAL teardown forbids R1 */ ++ G5P_PXW_UNTAKEN, /* the ALIVE wait ended with the proof still owed */ ++ G5P_PXW_N, ++}; ++ ++static const char *const g5p_px_why_name[G5P_PXW_N] = { ++ "none", "state", "stale", "lost", "retired", "untaken", ++}; ++ ++/* What the R74E release path latches. */ ++enum g5p_pxp { ++ G5P_PXP_READY, /* the on-poll read state 1: release_ready (cold) */ ++ G5P_PXP_DEFER, /* ALIVE preempted the on-poll before its first read */ ++ G5P_PXP_ABORT, /* anything else: power abort (cold) */ ++}; ++ ++/* ++ * on_ran: the core request, the off-poll and the GPC1 write succeeded and the on-poll ++ * ran. ret: the last step's result. preempted: the on-poll stopped on ALIVE. ++ * samples: the on-poll's R1+0x1000 reads. Only a preemption with no read defers (no ++ * abort, no release_ready); a sampled one saw the state and aborts. ++ */ ++static inline enum g5p_pxp g5p_px_poll(bool on_ran, int ret, bool preempted, ++ u32 samples) ++{ ++ if (!on_ran) ++ return G5P_PXP_ABORT; ++ if (!ret) ++ return G5P_PXP_READY; ++ return preempted && !samples ? G5P_PXP_DEFER : G5P_PXP_ABORT; ++} ++ ++/* The first-ALIVE owner's admission. */ ++enum g5p_pxg { ++ G5P_PXG_READY, /* release_ready: the cold owner path */ ++ G5P_PXG_PROVE, /* a deferred proof is owed: the once-owner's read decides */ ++ G5P_PXG_WAIT, /* nothing proven yet, or another owner holds the proof */ ++ G5P_PXG_REFUSE, /* power abort or a failed proof: no register access */ ++}; ++ ++static inline enum g5p_pxg g5p_px_gate(bool power_abort, bool release_ready, ++ u32 px) ++{ ++ if (power_abort || px == G5P_PX_FAILED) ++ return G5P_PXG_REFUSE; ++ if (release_ready) ++ return G5P_PXG_READY; ++ if (px == G5P_PX_DEFERRED) ++ return G5P_PXG_PROVE; ++ return G5P_PXG_WAIT; ++} ++ ++/* RAM facts the once-owner samples after taking the proof, before its one read. */ ++struct g5p_px_pre { ++ u32 gen, gen_now; /* DS108 generation at the deferral and now */ ++ bool alive, released, stopped, failed; ++ bool r1_allowed; /* g5p_r8_r1_allowed() */ ++}; ++ ++static inline enum g5p_px_why g5p_px_check(const struct g5p_px_pre *p) ++{ ++ if (!p->gen || p->gen != p->gen_now) ++ return G5P_PXW_STALE; ++ if (!p->alive || !p->released || p->stopped || p->failed) ++ return G5P_PXW_LOST; ++ if (!p->r1_allowed) ++ return G5P_PXW_RETIRED; ++ return G5P_PXW_NONE; ++} ++ ++/* The proof: only the state field 1 is power-on. */ ++static inline bool g5p_px_on(u32 raw) ++{ ++ return G5P_GPC_ON(raw); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/r87b-attach.h b/drivers/gpu/apple-g5p/r87b-attach.h +new file mode 100644 +index 000000000..81600ac06 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r87b-attach.h +@@ -0,0 +1,34 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R87B_ATTACH_H ++#define G5P_R87B_ATTACH_H ++/* R87b DIAGNOSTIC ONLY, never production-safe: the DC3 attach policy selector. ++ * (docs/gpu/experiments/87b-build-report.md) ++ * ++ * Two campaigns (r85 stage 2, r87 stage 1) never reached GPU work: the DC3 attach pair was ++ * published, the immediate gate (R1+0x1000 & 0x1f == 1) read 0x10/0x11, the host GPC off/on ++ * repair timed out (-110) and attach stopped with -115 before the RAM-only consume wait ran, ++ * after which the machine froze. Every retained instance of that failure shows the firmware ++ * consuming the pair on its own (dc=3/3 at export, no bell rung). boot_r87b_attach selects: ++ * 0 the original attach policy, exactly r87: a closed gate is repaired by the host (R1+0x20000 ++ * clear, GPC off/on, ALIVE-owned doorbell). The matched control. ++ * 1 self-drain (the r82b design, docs/gpu/experiments/82-attach-reactivation-triage.md): a ++ * closed DC3 gate is only a phase sample of the firmware's GPC window. The host does no ++ * repair and rings no bell; it keeps the RAM-only 3 s consume wait of render-r74f.inc and ++ * continues through the firmware-MMU attach path (r74g) when the firmware consumed the pair. ++ * An active gate still rings the bell at once (the unchanged 80 % path). The TA kind is ++ * never affected. ++ * The selector is latched at map and read only through the latched copy. A non-zero value needs ++ * the r80 epoch (which implies r74h, r74g and r74f case 1): self-drain is only safe because r74g ++ * has the firmware own the first MMU pass, so no host R0 access is licensed by a rung bell. ++ * The TYPED_V3 production path latches policy 1 with boot_r74f_case = boot_r74g_case = 1 and NO ++ * r80 / r74h (boot-r73b.inc, "production path"); g5p_r87b_attach_valid() describes the r80 ++ * epoch only. ++ */ ++#include ++#include ++ ++#define G5P_R87B_ATTACH_ORIGINAL 0U ++#define G5P_R87B_ATTACH_SELF_DRAIN 1U ++#define G5P_R87B_ATTACH_MAX 1U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/readback.h b/drivers/gpu/apple-g5p/readback.h +new file mode 100644 +index 000000000..fdd71ab30 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/readback.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_READBACK_H ++#define G5P_READBACK_H ++ ++#include "bits-compat.h" ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++enum g5p_readback_state { ++ G5P_READBACK_WRAPPER_ONLY, ++ G5P_READBACK_BASELINE, ++ G5P_READBACK_RESET_HELD, ++}; ++ ++/* Returning wrapper registers in exp03; never a META slave-port register. */ ++static const u32 g5p_fender_regs[] = { ++ 0, 8, 0x10, 0x14, 0x1000, 0x1004, 0x2010, 0x2014, 0x20000, ++}; ++ ++/* Reset halves first: the only post-arm R0 readback needed by the gate. */ ++static const u32 g5p_rgx_regs[] = { ++ 0x100, 0x104, 0, 4, 8, 0xc, 0x18, 0x20, 0x80, 0x84, 0x88, 0x8c, ++}; ++ ++static inline u32 g5p_readback_rgx_count(u32 gpc, enum g5p_readback_state state) ++{ ++ if (!G5P_GPC_ON(gpc)) ++ return 0; ++ if (state == G5P_READBACK_RESET_HELD) ++ return 2; ++ return state == G5P_READBACK_BASELINE ? ARRAY_SIZE(g5p_rgx_regs) : 0; ++} ++ ++static inline bool g5p_slave_access_allowed(bool reset_transition) ++{ ++ return !reset_transition; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/recover.inc b/drivers/gpu/apple-g5p/recover.inc +new file mode 100644 +index 000000000..f550d1010 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/recover.inc +@@ -0,0 +1,101 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * In-boot GPU recovery (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault isolation). ++ * ++ * A render job that stopped the firmware (its event stream went silent, or a fault) blocks the ++ * one TA/3D channel for every client, and the firmware has no abort. iOS ends that state with a ++ * firmware stop, a platform power cycle and a new firmware boot (kext 6c9e070, firmware-re README ++ * "Fault recovery"). This is the same, built from two proven pieces: ++ * - the reboot quiesce's freeze, IRQ mask and PMGR gfx gate cycle (teardown.inc g5p_kq_*), which ++ * on hardware cycled a wedged, failed device safely (deskfold reboot notifier: plan=-+cycle, ++ * then the next kernel came up warm-cycled); ++ * - the bootstrap's own stage functions (wake, map, arm, load, release, events, control), the ++ * same code the sysfs commands run after the next kernel's cycle on every warm hop. ++ * The old struct g5p_boot is frozen and left allocated (devm; a few hundred KiB per recovery), ++ * so a stale pointer anywhere still reads valid memory; a new one is built by the stages. id and ++ * sample are not rerun (they refuse once the boot saw a reset transition; map needs only the ++ * cached core id), nor is irq-test (refused in an r74 epoch). The firmware image is reused. ++ * ++ * Core lock held, no job in flight (the render scheduler's worker). Returns 0 with a live new ++ * epoch, or an error with the device left frozen and failed. ++ */ ++ ++/* The bootstrap's stages after the wake, as the sysfs runner orders them. */ ++static const char *const g5p_recover_stages[] = { ++ "map", ++ "arm", ++ "load", ++ "release", ++ "events", ++ "control-prepare", ++ "control-submit", ++ "control-wait", ++}; ++ ++static int __maybe_unused g5p_recover(struct g5p_diag *g, const char *why, ++ u32 *stage_out) ++{ ++ struct g5p_boot *old = g->boot; ++ struct g5p_kq_ctx kq = { .g = g }; ++ u64 t0 = ktime_get_ns(); ++ u32 stage = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ *stage_out = 0; ++ if (!old || !READ_ONCE(old->released) || g5p_r8_is_retired() || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || !g->pmgr || ++ !g->id_checked) ++ return -EPERM; ++ /* The quiesce's freeze and IRQ mask, then the line is given back for the new epoch. */ ++ g5p_kq_freeze(&kq); ++ g5p_kq_irq_off(&kq); ++ if (old->irq_ready && old->irq > 0) ++ devm_free_irq(g->dev, old->irq, g); ++ /* mmio-access.md G5: the GPU is marked unusable before its power is cut (Apple clears ++ * fender+0xc inside setPlatformPower(0), before the gate); g5p_kq_cycle() also sets mmio_off. ++ */ ++ WRITE_ONCE(g->fender_ready, false); ++ ret = g5p_kq_cycle(&kq); ++ ret = g5p_ds_hook_recover(g, ret) ?: ret; ++ if (ret) ++ goto out; ++ /* A new epoch: the stages build a new struct g5p_boot from here. */ ++ WRITE_ONCE(g->boot, NULL); ++ stage = 1; ++ ret = g5p_fender_wake(g); ++ for (u32 i = 0; !ret && i < ARRAY_SIZE(g5p_recover_stages); i++) { ++ const char *s = g5p_recover_stages[i]; ++ ++ stage = i + 2; ++ ret = strncmp(s, "control-", 8) ? g5p_boot_command(g, s) : ++ g5p_lifecycle_command(g, s); ++ if (ret == -EALREADY) ++ ret = 0; ++ } ++ if (!ret && (!g->boot || !READ_ONCE(g->boot->alive) || ++ !READ_ONCE(g->boot->control_complete))) ++ ret = -EIO; ++ /* The claim gate again: the held jobs need ALIVE proven in the new generation. */ ++ if (!ret) ++ ret = g5p_ds_hook_claim(g); ++ if (!ret) { ++ /* The render claim is kept: no diagnostic command may reset this epoch. */ ++ WRITE_ONCE(g->boot->user_compute_owned, true); ++ stage = 0; ++ } ++out: ++ if (ret && g->boot) ++ WRITE_ONCE(g->boot->failed, true); ++ g->last_result = ret; ++ *stage_out = stage; ++ dev_emerg( ++ g->dev, ++ "GPU-RECOVER why=%s ret=%d stage=%s ms=%llu gen=%u end=recover\n", ++ why, ret, ++ !stage ? (ret ? "cycle" : "done") : ++ stage == 1 ? "wake" : ++ g5p_recover_stages[stage - 2], ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC, g5p_ds_gen()); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/render-layout-r22.h b/drivers/gpu/apple-g5p/render-layout-r22.h +new file mode 100644 +index 000000000..8391a8320 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout-r22.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_R22_H ++#define G5P_RENDER_LAYOUT_R22_H ++ ++#include "render-layout.h" ++ ++#define G5P_R22_TA 0x54000U ++#define G5P_R22_3D 0x58000U ++#define G5P_R22_PM 0x5c000U ++#define G5P_R22_W 0x400U ++#define G5P_R22_TA_SKU 0xa00U ++#define G5P_R22_3D_SKU 0xd00U ++#define G5P_R22_TQ 0x1800U ++#define G5P_R22_STAMP 0x1840U ++#define G5P_R22_DEP 0x1900U ++#define G5P_R22_BLOCKS 32U ++#define G5P_R22_PHYS_PAGES (G5P_R22_BLOCKS * 8) ++#define G5P_R22_AUX 0x101000000ULL ++#define G5P_R22_RT 0x101020000ULL ++#define G5P_R22_RT_AUX 0x101030000ULL ++ ++/* 6cb124c/13f4/14dc: PA PPNs, GPU F and META links are distinct. ++ * Caller provides complete 16KiB control/list/F allocations. Check the whole ++ * physical input before writing anything. No synthesized PM catalogue encoder. ++ */ ++ ++/* 6ca6d5c arithmetic oracle + 6cba480/6c7b64c. Native legacy mapping ++ * dialect has bit36 clear: L=H, Laux=Haux. No PA substitution or float math. ++ */ ++ ++static const u32 g5p_r22_wait_words[] = { ++ 0x201, 0x138204, 0x10, 0x8388204, 0, 0x104000a, 0x1f, ++ 0x1f, 0x8388204, 1, 0x104000a, 0x1f, 0x1f, 0xf48008, ++ 0x10000015, 4, 0, 0x14, 0x109f, 0x13000a, 0x4000, ++ 0x4000, 0x138204, 0x4000, 0x8388204, 0, 0x104000a, 8, ++ 8, 0x8388204, 1, 0x104000a, 8, 8, ++}; ++ ++/* Full normal timestamped recipes, independent stamp identities TA0/3D1. ++ * k is dependency+4 (zero on the proven fresh initializer), not MMU tag. ++ */ ++ ++/* PM bank0 save writes runtime qwords before the a0/a8/b0 backing anchors. */ ++static inline bool g5p_r22_pm_saved(const u8 *state) ++{ ++ for (u32 off = 8; off < 0xa0; off += 4) ++ if (g5p_le32(state + off)) ++ return true; ++ return false; ++} ++ ++/* Never trust a firmware pointer until it equals our retained stamp word. */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-layout.h b/drivers/gpu/apple-g5p/render-layout.h +new file mode 100644 +index 000000000..f2a0373da +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_H ++#define G5P_RENDER_LAYOUT_H ++ ++#include "boot-layout.h" ++ ++/* Offline construction primitives only. No TA/3D publication path exists. ++ * 6caf9f8/6cb2378: caller must recover N/S1/S2/S3, not infer them from USC ++ * count or target dimensions. Sizes are bounded to u32 allocation extents. ++ */ ++struct g5p_pm_sizes { ++ u32 control, page_lists, backing; ++}; ++ ++static inline int g5p_pm_sizes(u32 n, u32 blocks, u64 s1, u64 s2, u64 s3, ++ struct g5p_pm_sizes *out) ++{ ++ u64 limit = 0xffffffffULL; ++ u64 a, b; ++ ++ if (!n || !blocks || !s1 || !s2 || !s3) ++ return -EINVAL; ++ if (n > (limit - 0x80) / 0xc0 || blocks > (limit - 0x44) / 0x24 || ++ s1 > limit / n || s2 > limit / n || s3 > limit) ++ return -EOVERFLOW; ++ a = s1 * n; ++ b = s2 * n; ++ if (a > limit - b || a + b > limit - s3) ++ return -EOVERFLOW; ++ out->control = 0x80 + n * 0xc0; ++ out->page_lists = 0x44 + blocks * 0x24; ++ out->backing = a + b + s3; ++ return 0; ++} ++ ++/* Only four recovered backing-pointer writes, not a complete PM record. ++ * Caller owns a fully initialized0xc0 record and validates GPU mappings. ++ */ ++static inline int g5p_pm_record_backing(u8 *record, u32 index, u32 n, u64 base, ++ u64 s1, u64 s2, u64 s3, u32 meta) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (index >= n) ++ return -EINVAL; ++ ret = g5p_pm_sizes(n, 1, s1, s2, s3, &sizes); ++ if (ret) ++ return ret; ++ if (base > ~0ULL - sizes.backing) ++ return -EOVERFLOW; ++ g5p_put64(record + 0xa8, base + index * s1); ++ g5p_put64(record + 0xa0, base + n * s1 + index * s2); ++ g5p_put64(record + 0xb0, base + n * s1 + n * s2); ++ g5p_put32(record + 0xb8, meta); ++ return 0; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-bind.h b/drivers/gpu/apple-g5p/render-user-bind.h +new file mode 100644 +index 000000000..3bbbb1e0f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-bind.h +@@ -0,0 +1,285 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Firmware slot pairs of render contexts (docs/gpu/multi-client-design.md, stage 1). ++ * ++ * Each render context owns a TA and a 3D firmware slot in 7..255, both pointing at its own ++ * page catalogue. The pair is bound once by two DC3 records {3, slot, 0, 0, 0}, 3D first, ++ * then TA, with the catalogue PA in 0x86[slot] and the host tag in 0x84[slot]. The firmware's ++ * DC3 handler installs CBASE and copies the host tag to 0x85[slot]. ++ * ++ * These helpers only read and write those RAM words, so they cannot reach a register and are ++ * safe after a job has retired. Pure functions, host-tested ++ * (scripts/gpu/multiclient1-host/test_render_bind.c). The caller provides g5p_le32(), ++ * g5p_put32() and g5p_put64(). ++ */ ++#ifndef G5P_RENDER_USER_BIND_H ++#define G5P_RENDER_USER_BIND_H ++ ++#define G5P_RU_SLOT_MIN 7U ++#define G5P_RU_SLOT_MAX 255U ++/* RAM-only wait for the firmware to consume a published DC3 batch. */ ++#define G5P_RU_BIND_MS 3000U ++/* Host tag of a pre-bound spare pair (render_user_spare_pairs): never zero. */ ++#define G5P_RU_SPARE_TAG 0x00400000U ++/* Host tag of a parked pair (render_user_close_park): never zero, never a spare's. */ ++#define G5P_RU_PARK_TAG 0x00500000U ++#define G5P_RU_MAX_SPARES 32U ++ ++/* The words one pair's binding touches. Pointers, because the DC records may straddle the ++ * shared pages. Index [e]: 0 TA slot, 1 3D slot. ++ */ ++struct g5p_ru_bind_mem { ++ u8 *rec[2][5]; /* the records at start and start + 1 */ ++ u8 *pa[2]; /* 0x86[slot], 64 bit */ ++ u8 *host[2]; /* 0x84[slot] */ ++ u8 *cached[2]; /* 0x85[slot] */ ++}; ++ ++/* One RAM sample of those words and of the DC ring control. */ ++struct g5p_ru_bind_sample { ++ u32 cons, prod; ++ u32 rec[2][5]; ++ u64 pa[2]; ++ u32 host[2], cached[2]; ++}; ++ ++/* g5p_ru_bind_busy() */ ++#define G5P_RU_BIND_BUSY_RING 0x1U /* the DC ring is not idle */ ++#define G5P_RU_BIND_BUSY_SLOT 0x2U /* a slot has a PA or a host tag */ ++ ++/* g5p_ru_bind_check(), the r74g criteria */ ++#define G5P_RU_BIND_BAD_RING 0x1U ++#define G5P_RU_BIND_BAD_RECORD 0x2U ++#define G5P_RU_BIND_BAD_PA 0x4U ++#define G5P_RU_BIND_BAD_HOST 0x8U ++#define G5P_RU_BIND_BAD_CACHED 0x10U ++ ++enum g5p_ru_bind_wait { ++ G5P_RU_BIND_PENDING, ++ G5P_RU_BIND_CONSUMED, ++ G5P_RU_BIND_PROTO, ++ G5P_RU_BIND_TIMEOUT, ++}; ++ ++static inline bool g5p_ru_slot_used(const unsigned long *map, u32 slot) ++{ ++ return (map[slot / BITS_PER_LONG] >> (slot % BITS_PER_LONG)) & 1; ++} ++ ++/* The slot pair of a new render context: the two lowest free slots in 7..255. The lower one ++ * is the 3D slot, or the TA slot with ta_first (render_user_slot_variant 1). False when fewer ++ * than two are free. slot[0] is TA, slot[1] is 3D. ++ */ ++static inline bool g5p_ru_slot_pair(const unsigned long *map, bool ta_first, ++ u32 *slot) ++{ ++ u32 pick[2], n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < 2; s++) ++ if (!g5p_ru_slot_used(map, s)) ++ pick[n++] = s; ++ if (n < 2) ++ return false; ++ slot[0] = pick[ta_first ? 0 : 1]; ++ slot[1] = pick[ta_first ? 1 : 0]; ++ return true; ++} ++ ++/* Word j of record i of a pair: {3, slot, 0, 0, 0}, the 3D slot's record first. */ ++static inline u32 g5p_ru_bind_record(const u32 *slot, u32 i, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot[1 - i] : 0; ++} ++ ++static inline void g5p_ru_bind_read(const struct g5p_ru_bind_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_bind_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ s->rec[i][j] = g5p_le32(m->rec[i][j]); ++ for (u32 e = 0; e < 2; e++) { ++ s->pa[e] = (u64)g5p_le32(m->pa[e] + 4) << 32 | ++ g5p_le32(m->pa[e]); ++ s->host[e] = g5p_le32(m->host[e]); ++ s->cached[e] = g5p_le32(m->cached[e]); ++ } ++} ++ ++/* Before a publish: the ring must be idle and both slots unused (zero PA and host tag). */ ++static inline u32 g5p_ru_bind_busy(const struct g5p_ru_bind_sample *s) ++{ ++ u32 busy = 0; ++ ++ if (s->prod > 255 || s->cons != s->prod) ++ busy |= G5P_RU_BIND_BUSY_RING; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] || s->host[e]) ++ busy |= G5P_RU_BIND_BUSY_SLOT; ++ return busy; ++} ++ ++/* ++ * A parked pair (render_user_close_park, release8 quiet2): a closed context's slots, bound to the ++ * kernel's park catalogue (it maps only the firmware window) with the park tag, so that the PM's ++ * late write-back of that context's free list resolves instead of faulting on an empty PC level. ++ * Both slots on `park`, both host tags the park tag. park 0: nothing is parked. ++ */ ++static inline bool g5p_ru_bind_parked(const struct g5p_ru_bind_sample *s, ++ u64 park) ++{ ++ if (!park) ++ return false; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] != park || s->host[e] != G5P_RU_PARK_TAG) ++ return false; ++ return true; ++} ++ ++/* PA and tag of both slots, then the two records. The caller cleans the catalogue first and ++ * stores the producer afterwards, each behind a barrier. ++ */ ++static inline void g5p_ru_bind_publish(const struct g5p_ru_bind_mem *m, ++ const u32 *slot, u64 root, u32 tag) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ u32 e = 1 - i; ++ ++ g5p_put64(m->pa[e], root); ++ g5p_put32(m->host[e], tag); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[i][j], g5p_ru_bind_record(slot, i, j)); ++ } ++} ++ ++/* After the ring reached `target`: the records intact, both slots on `root`, and the firmware ++ * tag equal to the host tag (the DC3 handler ran for both). ++ */ ++static inline u32 g5p_ru_bind_check(const struct g5p_ru_bind_sample *s, ++ const u32 *slot, u64 root, u32 tag, ++ u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[i][j] != g5p_ru_bind_record(slot, i, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ for (u32 e = 0; e < 2; e++) { ++ if (s->pa[e] != root) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host[e] != tag) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached[e] != tag) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ } ++ return bad; ++} ++ ++/* One sample of a published batch. The caller takes `expired` before reading the ring, so ++ * the sample taken at the deadline still counts. ++ */ ++static inline int g5p_ru_bind_step(u32 cons, u32 prod, u32 target, bool expired) ++{ ++ if (prod != target || cons > 255) ++ return G5P_RU_BIND_PROTO; ++ if (cons == target) ++ return G5P_RU_BIND_CONSUMED; ++ return expired ? G5P_RU_BIND_TIMEOUT : G5P_RU_BIND_PENDING; ++} ++ ++/* ++ * Unbind (iOS AGXGart_detachSlotsAndDestroyAddressSpace, firmware-re README round 15 §1): PA ++ * and host tag of a slot go to zero and one DC3 record {3, slot, 0, 0, 0} follows. The DC3 ++ * handler then installs CBASE 0, copies the zero tag to 0x85[slot] and invalidates the slot's ++ * PC level. The slots of one batch take consecutive ring entries, one record each, behind a ++ * single producer store. The caller must have retired every use of the slots first. ++ */ ++#define G5P_RU_USER_SLOTS (G5P_RU_SLOT_MAX - G5P_RU_SLOT_MIN + 1) ++ ++struct g5p_ru_slot_mem { ++ u8 *rec[5]; /* the slot's record in the batch */ ++ u8 *pa, *host, ++ *cached; /* 0x86[slot] (64 bit), 0x84[slot], 0x85[slot] */ ++}; ++ ++struct g5p_ru_slot_sample { ++ u32 cons, prod; ++ u32 rec[5]; ++ u64 pa; ++ u32 host, cached; ++}; ++ ++/* The user slots the firmware may hold: those with a PA or a host tag. At most `max`. */ ++static inline u32 g5p_ru_held_slots(const u8 *pa, const u8 *host, u32 *slots, ++ u32 max) ++{ ++ u32 n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < max; s++) ++ if (g5p_le32(pa + s * 8) || g5p_le32(pa + s * 8 + 4) || ++ g5p_le32(host + s * 4)) ++ slots[n++] = s; ++ return n; ++} ++ ++/* A batch of n records fits the 256-entry ring behind an idle consumer. */ ++static inline bool g5p_ru_unbind_fits(u32 n) ++{ ++ return n && n <= G5P_RU_USER_SLOTS; ++} ++ ++static inline u32 g5p_ru_unbind_record(u32 slot, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot : 0; ++} ++ ++static inline void g5p_ru_slot_read(const struct g5p_ru_slot_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_slot_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 j = 0; j < 5; j++) ++ s->rec[j] = g5p_le32(m->rec[j]); ++ s->pa = (u64)g5p_le32(m->pa + 4) << 32 | g5p_le32(m->pa); ++ s->host = g5p_le32(m->host); ++ s->cached = g5p_le32(m->cached); ++} ++ ++/* PA and host tag to zero, then the record. The caller stores the producer behind a barrier. */ ++static inline void g5p_ru_unbind_publish(const struct g5p_ru_slot_mem *m, ++ u32 slot) ++{ ++ g5p_put64(m->pa, 0); ++ g5p_put32(m->host, 0); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[j], g5p_ru_unbind_record(slot, j)); ++} ++ ++/* After the ring reached `target`: the record intact, PA and host tag still zero, and the ++ * firmware's tag zero too, which only its DC3 handler writes (the slot was bound with a ++ * non-zero tag). The G5P_RU_BIND_BAD_* bits. ++ */ ++static inline u32 g5p_ru_unbind_check(const struct g5p_ru_slot_sample *s, ++ u32 slot, u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[j] != g5p_ru_unbind_record(slot, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ if (s->pa) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ return bad; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-close.h b/drivers/gpu/apple-g5p/render-user-close.h +new file mode 100644 +index 000000000..2aa461bce +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-close.h +@@ -0,0 +1,91 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Release of a closed render context (docs/gpu/multi-client-design.md, stage 2). ++ * ++ * A TYPED_V3 context whose handle is gone (DESTROY_CONTEXT or file close) gets one close item. ++ * It runs after the context's last job, in the job order, and decides from plain values what ++ * the firmware may still hold: ++ * ++ * RELEASE the firmware never held the pair: drop the references, free everything; ++ * UNBIND the pair is bound and every use retired: RAM-only DC3 unbind ++ * (g5p_rv3_unbind_slots()), then free on success, retain on failure; ++ * RETAIN the firmware may still use the pair or its memory: keep the context, its ++ * slots and its memory until reboot (the kexec quiesce detaches them then). ++ * ++ * The rule follows iOS AGXGart_detachSlotsAndDestroyAddressSpace (firmware-re README round 15 ++ * §1, "Linux sequence" step 8): stop new jobs, wait for every engine use, publish PA and tag ++ * zero, consume the DC3, and only then recycle the slots and free the translations. A slot ++ * returns to the allocator only after its unbind was consumed and verified, so an old unbind ++ * can never clear a new owner's PA, and the lowest free pair may be reused at once (iOS pops ++ * a freed slot off its free stack first). ++ * ++ * Pure function, host-tested (scripts/gpu/multiclient2-host/test_render_close.c). ++ */ ++#ifndef G5P_RENDER_USER_CLOSE_H ++#define G5P_RENDER_USER_CLOSE_H ++ ++enum g5p_ru_close_plan { ++ G5P_RU_CLOSE_RELEASE, ++ G5P_RU_CLOSE_UNBIND, ++ G5P_RU_CLOSE_RETAIN, ++}; ++ ++/* Why a bound or published pair is retained (bits). */ ++#define G5P_RU_CLOSE_UNVERIFIED \ ++ 0x01U /* a bind was published but never checked out */ ++#define G5P_RU_CLOSE_BAD \ ++ 0x02U /* a job of this context failed on the hardware */ ++#define G5P_RU_CLOSE_TERMINAL 0x04U /* the epoch noted a timeout or fault */ ++#define G5P_RU_CLOSE_FAILED 0x08U /* the device failed */ ++#define G5P_RU_CLOSE_STOPPED 0x10U /* the device stopped (teardown, quiesce) */ ++#define G5P_RU_CLOSE_SHUTDOWN \ ++ 0x20U /* admission closed for good (reboot quiesce) */ ++#define G5P_RU_CLOSE_QUEUED 0x40U /* a job of this context is still queued */ ++#define G5P_RU_CLOSE_PENDING 0x80U /* an earlier DC batch was never consumed */ ++#define G5P_RU_CLOSE_LOCK 0x100U /* the command lock could not be taken */ ++ ++struct g5p_ru_close_in { ++ bool retained; /* the firmware may hold the pair: a DC3 bind was published */ ++ bool bound; /* that bind was consumed and checked */ ++ bool bad, term_failed, dev_failed, stopped, shutdown, dc_pending; ++ u32 queued; ++}; ++ ++static inline const char *g5p_ru_close_name(int plan) ++{ ++ return plan == G5P_RU_CLOSE_RELEASE ? "release" : ++ plan == G5P_RU_CLOSE_UNBIND ? "unbind" : ++ "retain"; ++} ++ ++/* The plan, and in *why the G5P_RU_CLOSE_* bits that made it RETAIN (0 otherwise). */ ++static inline int g5p_ru_close_plan(const struct g5p_ru_close_in *in, u32 *why) ++{ ++ u32 w = 0; ++ ++ *why = 0; ++ if (!in->retained) ++ return in->queued ? (*why = G5P_RU_CLOSE_QUEUED, ++ G5P_RU_CLOSE_RETAIN) : ++ G5P_RU_CLOSE_RELEASE; ++ if (!in->bound) ++ w |= G5P_RU_CLOSE_UNVERIFIED; ++ if (in->bad) ++ w |= G5P_RU_CLOSE_BAD; ++ if (in->term_failed) ++ w |= G5P_RU_CLOSE_TERMINAL; ++ if (in->dev_failed) ++ w |= G5P_RU_CLOSE_FAILED; ++ if (in->stopped) ++ w |= G5P_RU_CLOSE_STOPPED; ++ if (in->shutdown) ++ w |= G5P_RU_CLOSE_SHUTDOWN; ++ if (in->queued) ++ w |= G5P_RU_CLOSE_QUEUED; ++ if (in->dc_pending) ++ w |= G5P_RU_CLOSE_PENDING; ++ *why = w; ++ return w ? G5P_RU_CLOSE_RETAIN : G5P_RU_CLOSE_UNBIND; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +new file mode 100644 +index 000000000..ccb7d2c8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The phase wait's pause between two RAM samples (perf60-fillrate.md §11.3 cut 1): woken by the ++ * firmware event, not a fixed 1-2 ms sleep. Host test: scripts/gpu/perf60-host/test_evwait.c. ++ * ++ * Protocol (g5p_rv3_wait): drain -> seen = event_serial -> sample RAM -> decide -> pause. ++ * The IRQ thread's drain decodes each firmware event, bumps event_serial under status_lock and ++ * wakes the queue. The pause sleeps until event_serial != seen or the safety timeout, so an ++ * event decoded at any point after `seen` was read (before, during or after the sample) ends the ++ * pause at once: no lost wakeup. The timeout is the safety net for a RAM state that changes ++ * without an event; the predicate stays level-triggered on every sample, as before. ++ * Once the phase's own completion event is latched (s->event) but the RAM rules still wait ++ * (idle words, retire stamp, PM receipt land a little after the event), the pause is a short ++ * G5P_EW_POST_US poll: there is no further event to wait for. ++ */ ++#ifndef G5P_RENDER_USER_EVWAIT_H ++#define G5P_RENDER_USER_EVWAIT_H ++ ++#define G5P_EW_POST_US \ ++ 100U /* after the phase's own event: completion is imminent */ ++#define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ ++ ++enum g5p_ew_act { ++ G5P_EW_LEGACY, /* render_user_event_wake 0: usleep_range(1000, 2000), release8 */ ++ G5P_EW_SHORT, /* the phase's event seen: sleep G5P_EW_POST_US */ ++ G5P_EW_EVENT, /* sleep until any event since `seen`, at most *wait_us */ ++}; ++ ++static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, ++ u32 *wait_us) ++{ ++ if (!knob_us) { ++ *wait_us = 0; ++ return G5P_EW_LEGACY; ++ } ++ if (phase_event) { ++ *wait_us = G5P_EW_POST_US; ++ return G5P_EW_SHORT; ++ } ++ *wait_us = knob_us < G5P_EW_POST_US ? G5P_EW_POST_US : ++ knob_us > G5P_EW_MAX_US ? G5P_EW_MAX_US : ++ knob_us; ++ return G5P_EW_EVENT; ++} ++ ++/* The wait condition: an event was decoded since `seen` was read. */ ++static inline bool g5p_ew_woken(u64 seen, u64 serial) ++{ ++ return serial != seen; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-fault.h b/drivers/gpu/apple-g5p/render-user-fault.h +new file mode 100644 +index 000000000..2af9f7513 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-fault.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * A client job that faults (docs/gpu/experiments/faultiso.md). Pure decisions, host-tested ++ * (scripts/gpu/faultiso-host/test_fault_iso.c). ++ * ++ * On this firmware a GPU page fault is not survivable in place. The firmware acknowledges the ++ * MMU fault itself (R0+0x138), reports it as event 4 and parks (firmware-re README "Fault ++ * ownership"); iOS answers event 4 only with its restart tail (kext 6c81dac -> 6c834a0: firmware ++ * stop, context and dependency reset, shared state rebuilt, firmware boot), and has no ++ * per-context ACK, channel reset or slot unbind for it. The no-reset path iOS does have, the ++ * firmware's own job kill (event 8, then device-control command 0xa), is not a page fault. So a ++ * job that stops the firmware, by a fault event or by a silent event stream, loses its context ++ * and the GPU is recovered in the boot (render_user_recover), or the epoch ends. ++ */ ++#ifndef G5P_RENDER_USER_FAULT_H ++#define G5P_RENDER_USER_FAULT_H ++ ++/* The errno a lost job reports: a timeout's own, else the fault's (-EIO for a fault event or a ++ * silent stream, -EPROTO, or the drain's error). `result` is the phase's G5P_RU_WAIT_*. ++ */ ++static inline int g5p_ru_lost_errno(u32 result, int err) ++{ ++ return result == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++} ++ ++/* ++ * Does a job that failed after touching the hardware fail the device too? The legacy profiles ++ * always do. A TYPED_V3 job does not when it timed out (nothing is wrong with the protocol and ++ * the firmware may still finish), when its bind failed (the DC ring is not on the path of the ++ * contexts already bound), or when it was lost (the executor ruled: the channel drains, or a ++ * recovery replaces the epoch). Its context is quarantined in every case. ++ */ ++static inline bool g5p_ru_job_fails_device(bool v3, int ret, bool bind_failed, ++ bool lost) ++{ ++ return !v3 || (ret != -ETIMEDOUT && !bind_failed && !lost); ++} ++ ++/* ++ * A device failure noted while a job or a drain runs (a fault event decoded mid-phase), or while ++ * a recovery is pending, is not yet ruled on: the executor either loses that job's context and ++ * recovers, or ends the epoch. Until then the other contexts wait (submit accepted and held, ++ * VM_BIND busy, context status not lost) instead of failing. The executor sets `recovering` ++ * before it clears `executing`, so a reader that loads `executing` with acquire and then ++ * `recovering` always sees one of them through a recovery. ++ */ ++static inline bool g5p_ru_ruling(bool executing, bool recovering) ++{ ++ return executing || recovering; ++} ++ ++/* render_user_test_fault_job: the context's job about to start (it has run `jobs` before) gets ++ * the unmapped control stream. Never the epoch's first job: its attach and bell are special. ++ */ ++static inline bool g5p_ru_fault_due(u32 fault_job, u32 jobs, bool first) ++{ ++ return fault_job && jobs + 1 == fault_job && !first; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-faultreg.h b/drivers/gpu/apple-g5p/render-user-faultreg.h +new file mode 100644 +index 000000000..83cac51f7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-faultreg.h +@@ -0,0 +1,124 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 fault address (render_user_fault_read, drm-render-v3.inc): decode of a GPU MMU fault ++ * status bank, R0+0xe150/e154 (bank 0) or R0+0xe158/e15c (bank 1), as high32:low32. ++ * ++ * The firmware's fault path (source 4 -> 1001a494) reads both banks with a 64-bit load, keeps ++ * only a class in its companion record and posts event 4 without the status, so the address ++ * exists only in these registers. iOS reads them itself in its fault handler (kext 6ca5c14, ++ * accelerator virtual+740) with exactly these fields (docs/gpu/firmware-re/README.md "Complete ++ * e150 decode"): bit 0 valid, 2:1 type (0 page invalid, 2 read-only, 3 PM/META protected), ++ * 3 read (0 write), 5:4 level (3 = catalogue), 11:6 requester (iOS table 6ff5ed8), 19:12 ++ * sideband, 27:20 MMU context (the slot), VA = (status >> 24) & 0xfffffffff0 (16-byte precision). ++ * Pure, host-tested (scripts/gpu/perf60-host/test_fault_decode.c). ++ */ ++#ifndef G5P_RENDER_USER_FAULTREG_H ++#define G5P_RENDER_USER_FAULTREG_H ++ ++#include "bits-compat.h" ++struct g5p_ru_mmu_fault { ++ bool valid, read; ++ u8 type, level, requester, sideband, context; ++ u64 va; ++}; ++ ++static inline struct g5p_ru_mmu_fault g5p_ru_mmu_fault_decode(u32 lo, u32 hi) ++{ ++ u64 s = (u64)hi << 32 | lo; ++ struct g5p_ru_mmu_fault f = { ++ .valid = s & 1, ++ .type = (s >> 1) & 3, ++ .read = (s >> 3) & 1, ++ .level = (s >> 4) & 3, ++ .requester = (s >> 6) & 0x3f, ++ .sideband = (s >> 12) & 0xff, ++ .context = (s >> 20) & 0xff, ++ .va = (s >> 24) & 0xfffffffff0ULL, ++ }; ++ ++ return f; ++} ++ ++static const char *const g5p_ru_mmu_fault_types[4] = { ++ "page-invalid", ++ "type1", ++ "read-only", ++ "pm-meta-protected", ++}; ++ ++/* Requester names, iOS kext table 6ff5ed8 (16-byte {name, class} entries; read from the j71t ++ * 16.7.10 kernelcache, 2026-11-03). 56 entries; a larger index prints its number only. ++ */ ++static const char *const g5p_ru_mmu_requesters[] = { ++ "META", ++ "TLA", ++ "HOST", ++ "VDMM", ++ "CDM", ++ "IPP", ++ "PM", ++ "TILING", ++ "MCU_TEXAS0", ++ "PDS_TEXAS0", ++ "PBE0_TEXAS0", ++ "PBE1_TEXAS0", ++ "VDMS_BLACKPEARL0", ++ "IPF_BLACKPEARL0", ++ "ISP_BLACKPEARL0", ++ "TPF_BLACKPEARL0", ++ "USCS_BLACKPEARL0", ++ "PPP_BLACKPEARL0", ++ "VCE_BLACKPEARL0", ++ "FBCDC_BLACKPEARL0", ++ "MCU_TEXAS1", ++ "PDS_TEXAS1", ++ "PBE0_TEXAS1", ++ "PBE1_TEXAS1", ++ "MCU_TEXAS2", ++ "PDS_TEXAS2", ++ "PBE0_TEXAS2", ++ "PBE1_TEXAS2", ++ "VDMS_BLACKPEARL1", ++ "IPF_BLACKPEARL1", ++ "ISP_BLACKPEARL1", ++ "TPF_BLACKPEARL1", ++ "USCS_BLACKPEARL1", ++ "PPP_BLACKPEARL1", ++ "VCE_BLACKPEARL1", ++ "FBCDC_BLACKPEARL1", ++ "MCU_TEXAS3", ++ "PDS_TEXAS3", ++ "PBE0_TEXAS3", ++ "PBE1_TEXAS3", ++ "MCU_TEXAS4", ++ "PDS_TEXAS4", ++ "PBE0_TEXAS4", ++ "PBE1_TEXAS4", ++ "VDMS_BLACKPEARL2", ++ "IPF_BLACKPEARL2", ++ "ISP_BLACKPEARL2", ++ "TPF_BLACKPEARL2", ++ "USCS_BLACKPEARL2", ++ "PPP_BLACKPEARL2", ++ "VCE_BLACKPEARL2", ++ "FBCDC_BLACKPEARL2", ++ "MCU_TEXAS5", ++ "PDS_TEXAS5", ++ "PBE0_TEXAS5", ++ "PBE1_TEXAS5", ++}; ++ ++static inline const char *g5p_ru_mmu_requester(u32 requester) ++{ ++ if (requester >= ARRAY_SIZE(g5p_ru_mmu_requesters)) ++ return "?"; ++ return g5p_ru_mmu_requesters[requester]; ++} ++ ++/* The range test of the VM edit log: does [va, va + size) hold the fault address? */ ++static inline bool g5p_ru_mmu_fault_in(u64 fault_va, u64 va, u64 size) ++{ ++ return fault_va >= va && fault_va - va < size; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-geometry.h b/drivers/gpu/apple-g5p/render-user-geometry.h +new file mode 100644 +index 000000000..2117d567b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-geometry.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_GEOMETRY_H ++#define G5P_RENDER_USER_GEOMETRY_H ++ ++/* Kernel-derived render-target geometry: AGXRenderTargetMemoryG5's arithmetic ++ * leaf 6ca6d5c (firmware round16 section 3) for one sample at the pixel ++ * centre (8,8) or four samples (docs/gpu/experiments/msaa.md), in integer ++ * arithmetic only. It is Rogue's non-simple ++ * parameter format: 32x32 tiles, 4x4 macrotiles of ++ * ALIGN(DIV_ROUND_UP(tiles, 4), 4) tiles per axis, five-byte region headers ++ * per padded tile, a tail pointer cache of next_pow2(max(padded tiles))^2 ++ * eight-byte entries. The scale pairs are the leaf's IEEE singles ++ * 0x3e89309c / n, 0x3fddb3d9 / n (fdiv, round to nearest even) and n * 16. ++ * tools/gpu/test_rt_oracle.c compares every output with the unmodified leaf ++ * for every width and height in 1..G5P_RU_MAX_DIM. ++ * ++ * G5P_RU_MAX_DIM is rogue_get_render_size_max() for this tiling class; it ++ * is a memory bound (16 MiB private window, 8 MiB main ring), not a field ++ * limit: the 9-bit TE macrotile and 10-bit padded-size fields would hold ++ * twice as much. ++ */ ++#define G5P_RU_MAX_DIM 8192U ++#define G5P_RU_SCALE_A 0x3e89309cU ++#define G5P_RU_SCALE_B 0x3fddb3d9U ++ ++static inline u32 g5p_ru_fls(u32 v) ++{ ++ return v ? 32 - __builtin_clz(v) : 0; ++} ++ ++/* ucvtf of 1 <= n < 2^24: exact. */ ++static inline u32 g5p_ru_f32(u32 n) ++{ ++ u32 bits = g5p_ru_fls(n); ++ ++ return ((126 + bits) << 23) | ((n << (24 - bits)) & 0x7fffff); ++} ++ ++/* fdiv of a normal positive single by 1 <= n < 2^24, round to nearest even. ++ * The quotient q of the 24-bit significand shifted by fls(n) + 3 has 27 or ++ * 28 bits; s of them are rounding bits, the remainder is the sticky bit. ++ * Results stay normal for the two leaf constants and n <= G5P_RU_MAX_DIM. ++ */ ++static inline u32 g5p_ru_fdiv(u32 a, u32 n) ++{ ++ u32 shift = g5p_ru_fls(n) + 3; ++ u64 num = (u64)((a & 0x7fffff) | 0x800000) << shift; ++ u32 q = num / n, sticky = num % n != 0; ++ u32 s = g5p_ru_fls(q) - 24, m = q >> s, low = q & ((1U << s) - 1); ++ ++ if (low > 1U << (s - 1) || ++ (low == 1U << (s - 1) && (sticky || (m & 1)))) ++ m++; ++ if (m >> 24) { ++ m >>= 1; ++ s++; ++ } ++ return ((((a >> 23) & 0xff) - shift + s) << 23) | (m & 0x7fffff); ++} ++ ++/* Sample positions, one byte per sample: x in [3:0], y in [7:4], 1/16 px. ++ * One sample sits at the pixel centre; four use the standard Vulkan/D3D ++ * pattern, which is also Metal's (record+0x69c..0x6b8 of the iOS ++ * msaa4-resolve/-store captures: (6,2) (14,6) (2,10) (10,14)). ++ */ ++#define G5P_RU_POSITIONS_1X 0x88ULL ++#define G5P_RU_POSITIONS_4X 0xeaa26e26ULL ++ ++/* The leaf for one or four samples (6ca6d5c's 1, 2, 4, 8 table, of which ++ * Linux uses 1 and 4). Four samples are 2x2 per pixel: the region headers ++ * and the tail pointer cache are sized for twice the tiles in x and y, the ++ * ISP macrotile size (Rbc) doubles in y, Rb8 is log2(samples) (5940f40) ++ * and the TE word Rf0 is 5. Rb0 and Rf8 are the positions word, packed by ++ * the leaf as x0 | y0 << 4 | x1 << 8 | ... | y3 << 28. The scales, the TE ++ * tiling words and the tile count do not depend on the samples. ++ */ ++static inline int g5p_ru_rt_derive_samples(u32 width, u32 height, u32 samples, ++ u64 positions, struct g5p_ru_rt *rt) ++{ ++ u32 tx = (width + 31) / 32, ty = (height + 31) / 32; ++ u32 mx = (((tx + 3) >> 2) + 3) & ~3U, my = (((ty + 3) >> 2) + 3) & ~3U; ++ u32 s = samples == 4 ? 2 : 1; ++ u32 px = mx * s * 4, py = my * s * 4, tpc; ++ ++ if (!width || !height || width > G5P_RU_MAX_DIM || ++ height > G5P_RU_MAX_DIM || (samples != 1 && samples != 4) || ++ positions >> (samples * 8)) ++ return -EINVAL; ++ tpc = 1U << g5p_ru_fls((px > py ? px : py) - 1); ++ *rt = (struct g5p_ru_rt) { ++ .header = px * py * 5, ++ .region = mx * my * s * s * 5, ++ .packed = positions, ++ .state = s - 1 ? 2 : 0, ++ .padded = (mx & 0x3ff) << 16 | ((my * s) & 0x3ff), ++ .scale = { ++ g5p_ru_fdiv(G5P_RU_SCALE_A, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_A, height) << 32, ++ g5p_ru_fdiv(G5P_RU_SCALE_B, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_B, height) << 32, ++ (g5p_ru_f32(width) + (4U << 23)) | ++ (u64)(g5p_ru_f32(height) + (4U << 23)) << 32, ++ }, ++ .ta_state = { s - 1 ? 5 : 0, positions }, ++ .ta_tiling = { ++ (u64)((ty - 1) << 12 | (tx - 1)) << 32 | ++ ((height - 1) << 16 | (width - 1)), ++ (u64)(my << 18 | my << 10 | 3 * my) << 32 | ++ (mx << 18 | mx << 10 | 3 * mx), ++ }, ++ .ta_tiling_word = mx * my, ++ .tiles = tx * ty, ++ .secondary = (u64)tpc * tpc * 8, ++ .secondary_align = 0x40, ++ }; ++ return 0; ++} ++ ++/* g5p_ru_rt64 is this function's result for 64x64. */ ++ ++/* PM pool for a kernel-derived target. Linux policy, not hardware: Rogue's ++ * minimum free list (50 4KiB pages, non-XE cores) plus one 4KiB PM page per ++ * tile, in 128KiB blocks, never below the latched pool and at most one F ++ * page. 64x64 (4 tiles) needs 2, so the latched 32 stays. There is no PM ++ * growth during a job: a frame that exhausts the pool fails like any OOM. ++ */ ++static inline u32 g5p_ru_pm_blocks_for(u32 tiles, u32 latched) ++{ ++ u64 need = ((50ULL + tiles) * 0x1000 + G5P_RU_BLOCK_BYTES - 1) / ++ G5P_RU_BLOCK_BYTES; ++ ++ if (need < latched) ++ need = latched; ++ return need > G5P_RU_MAX_BLOCKS ? G5P_RU_MAX_BLOCKS : need; ++} ++ ++ ++ ++static inline int g5p_ru_rt_derive(u32 width, u32 height, struct g5p_ru_rt *rt) ++{ ++ return g5p_ru_rt_derive_samples(width, height, 1, G5P_RU_POSITIONS_1X, rt); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +new file mode 100644 +index 000000000..822709ed8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -0,0 +1,651 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_LAYOUT_H ++#define G5P_RENDER_USER_LAYOUT_H ++ ++#include "bits-compat.h" ++/* Parameterized forms of the r22 fixed constructors for userspace render. ++ * Pure byte builders: no allocation, MMIO, locking or firmware publication. ++ * With r22's inputs every builder reproduces render-layout-r22.h exactly; ++ * tools/gpu/test_render_user.c checks that equivalence byte for byte. ++ */ ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++#include "render-user-template.h" ++ ++/* Kernel-private window in every render VM. r22's aux/RT addresses are the ++ * first allocations, so a 64x64 target keeps the fixed path's W anchors. ++ * Root entry 0 carries the boot VA0 sink, so user maps start at 1 GiB. ++ */ ++#define G5P_RU_PRIVATE_VA G5P_R22_AUX ++#define G5P_RU_PRIVATE_SIZE 0x1000000ULL ++#define G5P_RU_USER_VA_MIN 0x40000000ULL ++/* Catalogue entry of the firmware GiB, borrowed from the boot tables. */ ++#define G5P_RU_FIRMWARE_CI ((G5P_IMAGE_VA >> 30) & 1023) ++#define G5P_RU_AUX_BYTES 0x10680U ++#define G5P_RU_AUX_PAGES 5U ++#define G5P_RU_RT_ALIGN 0x10000ULL ++#define G5P_RU_MAX_BLOCKS 128U ++#define G5P_RU_BLOCK_BYTES 0x20000U ++#define G5P_RU_MAX_HEADER 0x80000U ++#define G5P_RU_MAX_SECONDARY 0x100000ULL ++#define G5P_RU_TA_W_BYTES 0x560U ++#define G5P_RU_3D_W_BYTES 0x8c0U ++#define G5P_RU_CMD_BYTES 0x40U ++ ++/* Kernel mirror of drm_g5p_rt_geometry (firmware round16 R-offsets). */ ++struct g5p_ru_rt { ++ u32 header, region; /* R10, R20 */ ++ u64 packed; /* Rb0 */ ++ u32 state, padded; /* Rb8, Rbc */ ++ u64 scale[3]; /* Rc0, Rc8, Rd0 */ ++ u64 ta_state[2]; /* Rf0, Rf8 */ ++ u64 ta_tiling[2]; /* R100, R108 */ ++ u32 ta_tiling_word, tiles; /* R110, R118 */ ++ u64 secondary, secondary_align; /* R138, R178 */ ++}; ++ ++/* 6ca6d5c executed offline for 64,64,1,8,8 (round16 section 3). */ ++static const struct g5p_ru_rt g5p_ru_rt64 = { ++ .header = 0x500, ++ .region = 0x50, ++ .packed = 0x88, ++ .state = 0, ++ .padded = 0x40004, ++ .scale = { 0x3b89309c3b89309cULL, 0x3cddb3d93cddb3d9ULL, ++ 0x4480000044800000ULL }, ++ .ta_state = { 0, 0x88 }, ++ .ta_tiling = { 0x00001001003f003fULL, 0x0010100c0010100cULL }, ++ .ta_tiling_word = 0x10, ++ .tiles = 4, ++ .secondary = 0x800, ++ .secondary_align = 0x40, ++}; ++ ++#include "render-user-geometry.h" ++ ++struct g5p_ru_rt_sizes { ++ u64 slice, main, secondary_map; ++}; ++ ++/* 6cb3d68: slice=R10 rounded to 4KiB; main ring is the power-of-two capacity ++ * for sixteen slices. 6cb40e0 places the secondary extent at offset 0 of ++ * its own mapping. Bounds are Linux allocation policy, not hardware limits. ++ */ ++static inline int g5p_ru_rt_sizes(const struct g5p_ru_rt *rt, ++ struct g5p_ru_rt_sizes *out) ++{ ++ u64 main = 0x1000; ++ ++ if (!rt->header || rt->header > G5P_RU_MAX_HEADER || !rt->secondary || ++ rt->secondary > G5P_RU_MAX_SECONDARY || !rt->secondary_align || ++ (rt->secondary_align & (rt->secondary_align - 1)) || ++ rt->secondary_align > G5P_GPU_PAGE || !rt->tiles) ++ return -EINVAL; ++ out->slice = ((u64)rt->header + 0xfff) & ~0xfffULL; ++ while (main < out->slice * 16) ++ main <<= 1; ++ out->main = main < G5P_GPU_PAGE ? G5P_GPU_PAGE : main; ++ out->secondary_map = (rt->secondary + G5P_GPU_PAGE - 1) & ++ ~(u64)(G5P_GPU_PAGE - 1); ++ return 0; ++} ++ ++/* 6cba480/6c7b64c anchors. Native mapping dialect: L=H, Laux=Haux. ++ * Equal to g5p_r22_rt() for g5p_ru_rt64 at the r22 addresses. 3D W58 and ++ * Wa8 are the importer's literal Rb0 and R20>>2 for the verified geometry. ++ */ ++static inline void g5p_ru_rt_fill(u8 *ta, u8 *td, const struct g5p_ru_rt *rt, ++ u64 main_va, u64 secondary_va) ++{ ++ g5p_put64(ta + 0x28, main_va); ++ g5p_put64(ta + 0x30, secondary_va); ++ g5p_put64(ta + 0x38, rt->ta_state[0]); ++ g5p_put64(ta + 0x40, rt->ta_state[1]); ++ g5p_put64(ta + 0x444, secondary_va); ++ g5p_put32(ta + 0x44c, rt->secondary); ++ g5p_put64(ta + 0x3ac, rt->ta_tiling[0]); ++ g5p_put64(ta + 0x3b4, rt->ta_tiling[1]); ++ g5p_put32(ta + 0x3bc, rt->ta_tiling_word); ++ g5p_put64(td + 0x20, main_va); ++ g5p_put64(td + 0x28, rt->packed); ++ g5p_put32(td + 0x30, rt->state); ++ g5p_put32(td + 0x34, rt->padded); ++ g5p_put32(td + 0x38, rt->region); ++ g5p_put64(td + 0x3c, rt->scale[0]); ++ g5p_put64(td + 0x44, rt->scale[1]); ++ g5p_put64(td + 0x4c, rt->scale[2]); ++ g5p_put64(td + 0x58, rt->packed); ++ g5p_put32(td + 0xa8, rt->region >> 2); ++ g5p_put32(td + 0x3a0, rt->padded); ++ g5p_put64(td + 0x3d0, rt->scale[0]); ++ g5p_put64(td + 0x3d8, rt->scale[1]); ++ g5p_put32(td + 0x3e0, rt->scale[2]); ++ g5p_put32(td + 0x3e4, rt->scale[0]); ++} ++ ++/* Validated typed render inputs, already range-checked by the caller. */ ++struct g5p_ru_pass { ++ u64 vdm_va; ++ u32 vdm_size, vdm_control; ++ u64 scratch_va, scratch_size, scratch_ptr[6]; ++ u64 pool7_va, pool7_size, pool8_va, pool8_size; ++ u32 context_pds[2][6], resume_pds[2][4], ta_pds[4]; ++ u32 background[8], eot[4], eot_resume[4], restore[4]; ++}; ++ ++/* Importer chains 6cba8a0/6c7b100/6c7b34c -> 6ca9710/6ca7ae0 (offline ++ * differential map in make-render-user-template.py). Every user VA field ++ * and code-relative descriptor lands here; the rest of W is the frozen ++ * dialect-1 scalar table or a kernel RT/channel field. ++ */ ++static inline void g5p_ru_fill_ta(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_TA_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_ta_fixed); i++) ++ g5p_put32(w + g5p_ru_ta_fixed[i][0], g5p_ru_ta_fixed[i][1]); ++ g5p_put64(w + 0x48, p->scratch_ptr[4]); ++ g5p_put64(w + 0x50, p->scratch_ptr[5]); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put32(w + 0x58 + i * 4, p->context_pds[0][i]); ++ g5p_put32(w + 0x70 + i * 4, p->context_pds[1][i]); ++ } ++ g5p_put64(w + 0x88, p->vdm_control & ~0xfULL); ++ g5p_put64(w + 0x90, p->scratch_va & ~7ULL); ++ g5p_put64(w + 0x98, p->vdm_va & ~3ULL); ++ for (u32 i = 0; i < 4; i++) { ++ g5p_put32(w + 0xa0 + i * 4, p->resume_pds[0][i]); ++ g5p_put32(w + 0xb0 + i * 4, p->resume_pds[1][i]); ++ } ++ for (u32 i = 0; i < 3; i++) ++ g5p_put32(w + 0xc0 + i * 4, p->ta_pds[i]); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x468 + i * 8, p->scratch_ptr[i]); ++ g5p_put64(w + 0x488, p->scratch_ptr[4]); ++} ++ ++static inline void g5p_ru_fill_3d(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_3D_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_3d_fixed); i++) ++ g5p_put32(w + g5p_ru_3d_fixed[i][0], g5p_ru_3d_fixed[i][1]); ++ g5p_put64(w + 0x60, p->pool8_va & ~3ULL); ++ g5p_put64(w + 0x68, p->pool7_va & ~3ULL); ++ g5p_put64(w + 0x490, p->pool7_va); ++ g5p_put64(w + 0x4a0, p->pool8_va); ++ for (u32 i = 0; i < 8; i++) ++ g5p_put32(w + 0x5b8 + i * 4, p->background[i]); ++ g5p_put32(w + 0xb0, p->background[0]); ++ g5p_put32(w + 0xb4, p->background[1]); ++ for (u32 i = 4; i < 8; i++) ++ g5p_put32(w + 0xc0 + (i - 4) * 4, p->background[i]); ++ /* 6ca7ae0 swaps the first EOT qword's halves in its B upload. */ ++ g5p_put32(w + 0x3b4, p->eot[1]); ++ g5p_put32(w + 0x3b8, p->eot[0]); ++ g5p_put32(w + 0x3bc, p->eot[2]); ++ for (u32 i = 0; i < 3; i++) { ++ g5p_put32(w + 0x680 + i * 4, p->eot[i]); ++ g5p_put32(w + 0x694 + i * 4, p->eot_resume[i]); ++ } ++ g5p_put32(w + 0x600, p->restore[0]); ++ g5p_put32(w + 0x604, p->restore[1]); ++ g5p_put32(w + 0x610, p->restore[2]); ++ g5p_put32(w + 0x614, p->restore[3]); ++} ++ ++/* TYPED_V2 additions, already validated by g5p_ru_pass2_check(). */ ++struct g5p_ru_pass2 { ++ u64 scissor_va, dbias_va, background_texture, zls[6]; ++ u64 occlusion_query_va, occlusion_query_size; ++ u32 scissor_count, dbias_count, bgobj_depth, bgobj_vals; ++ u32 pixel_width, zls_mask; ++}; ++ ++/* Dialect-2 words over g5p_ru_fill_3d() (TYPED_V2 has zero pools). Each ++ * register image goes to its B-upload word and to the normal builder's ++ * copy (6c7a06c); make-render-user-dialect2.py proves every position by ++ * perturbing record 2d8/2e0 (pools), 2f0/310/338/360/388/308 (ZLS), 508 ++ * (background texture) and 4d0 (clears) through the r22 importer. ++ * Pixel output: 6ca7ae0 for D2f8 = width, one sample (D6e0 = 0), D630 = ++ * D6b2 = 0, six cores. Widths 2/4/8 keep two or more partitions, so W3c0 ++ * and W3c4 keep their dialect-1 values; width 2 is dialect 1 exactly. ++ */ ++static inline void g5p_ru_fill_3d_width(u8 *w, u32 width) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by D2f8 - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u32 count = 0x4000 / (width << 10); ++ u32 code = width_code[width - 2]; ++ ++ g5p_put32(w + 0x3c8, count & 0x1f); ++ g5p_put32(w + 0x3cc, code >> 4 | 0xf8); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(w + 0x108 + i * 8, code | (count - 1) << 7 | 0x43); ++ g5p_put32(w + 0x6b0, width); ++} ++ ++static inline void g5p_ru_fill_3d_v2(u8 *w, const struct g5p_ru_pass2 *e) ++{ ++ static const u16 zls_w[6][2] = { ++ { 0x80, 0x618 }, { 0x88, 0x628 }, { 0x90, 0x630 }, ++ { 0x98, 0x648 }, { 0xa0, 0x650 }, { 0x138, 0x6d0 }, ++ }; ++ ++ g5p_put64(w + 0x60, e->scissor_va & ~3ULL); ++ g5p_put64(w + 0x4a0, e->scissor_va); ++ g5p_put64(w + 0x68, e->dbias_va & ~3ULL); ++ g5p_put64(w + 0x490, e->dbias_va); ++ /* ++ * D318 is copied to both partition records. The normal 3D upload masks ++ * the low nibble before programming ISP_OCLQRY_BASE. ++ */ ++ g5p_put64(w + 0x70, e->occlusion_query_va); ++ g5p_put64(w + 0x78, e->occlusion_query_va); ++ g5p_put64(w + 0xb8, e->background_texture); ++ g5p_put64(w + 0x5f0, e->background_texture); ++ g5p_put32(w + 0x3a4, e->bgobj_depth); ++ g5p_put32(w + 0x6a8, e->bgobj_depth); ++ g5p_put32(w + 0x3a8, e->bgobj_vals); ++ g5p_put32(w + 0x6ac, e->bgobj_vals); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put64(w + zls_w[i][0], e->zls[i]); ++ g5p_put64(w + zls_w[i][1], e->zls[i]); ++ } ++ g5p_ru_fill_3d_width(w, e->pixel_width); ++} ++ ++/* TYPED_V3 additions, already validated by g5p_ru_pass3_check(). */ ++struct g5p_ru_pass3 { ++ u64 ta_program_va; ++ u32 ta_program_size, vdm_end; ++ u32 pixel_width, pad; /* V1 dialect output width, 0 = 2 */ ++}; ++ ++/* Dialect-3 word over g5p_ru_fill_ta(): W+0x88 (-> R0+0x470) is the exact ++ * byte offset of the TA PDS program from pds_base, a plain 64-bit store with ++ * no mask. g5p_ru_fill_ta() leaves it zero for a V3 pass (vdm_control is ++ * zero), so V1/V2 bytes are untouched; the stream length never reaches W. ++ */ ++static inline void g5p_ru_fill_ta_v3(u8 *w, u64 program_off) ++{ ++ g5p_put64(w + 0x88, program_off); ++} ++ ++/* 6cb2378 selected state S at control+80: explicit zeros plus the four ++ * recovered backing pointers. N=1, PM ID0/record0. aux_va covers 10680. ++ */ ++static inline int g5p_ru_pm_state(u8 *control, u64 aux_va, u64 list_va) ++{ ++ memset(control + 0x80, 0, 0xc0); ++ return g5p_pm_record_backing(control + 0x80, 0, 1, aux_va, 0x80, ++ 0x10000, 0x600, ++ g5p_meta_address(list_va + 0x40)); ++} ++ ++/* 6cb124c/13f4/14dc pool for `blocks` resident = maximum blocks. Rewrites ++ * control+0..7f, the whole list and F; S at control+80 is not touched. ++ * physical[] holds blocks*8 16KiB PAs. Validate everything before writing. ++ */ ++static inline int g5p_ru_pm_check(u64 list_va, u64 free_va, const u64 *physical, ++ u32 blocks) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (!blocks || blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ ret = g5p_pm_sizes(1, blocks, 0x80, 0x10000, 0x600, &sizes); ++ if (ret) ++ return ret; ++ if ((list_va | free_va) & (G5P_GPU_PAGE - 1) || ++ list_va >= (1ULL << 40) || free_va >= (1ULL << 40) || ++ 0x80 + blocks * 0x24 > G5P_GPU_PAGE || ++ blocks * 32 * 4 > G5P_GPU_PAGE) ++ return -EINVAL; ++ for (u32 i = 0; i < blocks * 8; i++) ++ if ((physical[i] & (G5P_GPU_PAGE - 1)) || ++ physical[i] >= (1ULL << 40)) ++ return -ERANGE; ++ return 0; ++} ++ ++static inline int g5p_ru_pm_pool(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks) ++{ ++ u32 previous = 0; ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ memset(control, 0, 0x80); ++ memset(list, 0, G5P_GPU_PAGE); ++ memset(free, 0, G5P_GPU_PAGE); ++ for (u32 j = 0; j < blocks; j++) { ++ u32 at = 0x80 + j * 0x24; ++ ++ g5p_put32(list + at, previous); ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[j * 8 + k] >> 12; ++ ++ g5p_put32(list + at + 4 + k * 4, ppn); ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * 4, ++ ppn + p); ++ } ++ previous = g5p_meta_address(list_va + at); ++ } ++ g5p_put32(list + 0x60, 1); ++ g5p_put32(control + 0x10, previous); ++ g5p_put64(control + 0x14, free_va); ++ g5p_put32(control + 0x1c, blocks * 32 * 4); ++ g5p_put32(control + 0x20, blocks * 32); ++ g5p_put32(control + 0x24, blocks * 32 - 1); ++ g5p_put32(control + 0x34, G5P_GPU_PAGE); ++ g5p_put32(control + 0x40, g5p_meta_address(list_va)); ++ g5p_put32(control + 0x54, blocks * 32); ++ g5p_put32(control + 0x58, blocks * 32); ++ return 0; ++} ++ ++/* Complete fresh PM object: g5p_r22_pm() when blocks=32, aux=G5P_R22_AUX. */ ++static inline int g5p_ru_pm(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks, ++ u64 aux_va) ++{ ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ if ((aux_va & (G5P_GPU_PAGE - 1)) || ++ aux_va > (1ULL << 40) - G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ return -EINVAL; ++ memset(control, 0, G5P_GPU_PAGE); ++ ret = g5p_ru_pm_pool(control, list, free, list_va, free_va, physical, ++ blocks); ++ return ret ?: g5p_ru_pm_state(control, aux_va, list_va); ++} ++ ++/* The r25 latched selections that can still yield a frame. Context/host/SKU ++ * probes and the heap-relative RT dialect are diagnostic-only in the fixed ++ * path (pair rejected) and are not offered here; blocks is pm_blocks. ++ */ ++static inline bool g5p_ru_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return g5p_r25_valid(v) && !g5p_r25_diagnostic(v) && !v->blocks; ++} ++ ++/* r25 render_pm_variant 2/3 on a pool from g5p_ru_pm()/g5p_ru_pm_pool(): ++ * reverse only F's block order, or report F's whole mapped page as its ++ * extent. Equal to g5p_r25_pool() for 32, 64 and 128 blocks. ++ */ ++static inline void g5p_ru_pm_variant(u8 *control, u8 *free, const u64 *physical, ++ u32 blocks, u32 variant) ++{ ++ if (variant == 2) ++ for (u32 j = 0; j < blocks; j++) ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[(blocks - 1 - j) * 8 + k] >> ++ 12; ++ ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * ++ 4, ++ ppn + p); ++ } ++ if (variant == 3) ++ g5p_put32(control + 0x1c, G5P_GPU_PAGE); ++} ++ ++/* r25 render_pm_variant 1, after g5p_ru_job() built the TA page: tag6+10 ++ * names an owned holder with the linked head (outside CH/ring/W/SKU/TQ/ ++ * dependency); F/count/head start clear and last=ffffffff, so 10005368 ++ * rebuilds them. Equal to g5p_r25_inputs() for the same pool. ++ */ ++ ++/* Persistent per-channel objects (6c89420 layout): written once at attach. ++ * Control indices, TQ and the dependency object stay firmware-owned after. ++ */ ++static inline void g5p_ru_channel_init(u8 *p, u32 meta) ++{ ++ memset(p, 0, G5P_GPU_PAGE); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_R22_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_R22_TQ + 4, meta + G5P_R22_TQ); ++ g5p_put64(p + G5P_R22_DEP + 0x30, 1); ++ g5p_put64(p + G5P_R22_DEP + 0x48, ~0ULL); ++ g5p_put32(p + G5P_R22_DEP + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(p + G5P_R22_DEP + off + 8, 0xf); ++} ++ ++/* Pointer-ring positions of job `seq` (1-based): two entries per job. */ ++static inline u32 g5p_ru_ring_index(u32 seq) ++{ ++ return (2 * (seq - 1)) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_ru_endpoint(u32 seq) ++{ ++ return (2 * seq) % G5P_WORK_SLOTS; ++} ++ ++/* perf60 job slots (render_user_job_slots, drm-render-v3.inc). A job's command, W record and ++ * microsequence ("SKU") live in slot 0 (the r22 offsets) or slot 1, in the channel page's free ++ * tail; the firmware reaches both through the job's two pointer-ring entries. The channel header, ++ * control, pointer ring, TQ, stamps and dependency object stay shared. Slot 1 clears TQ/STAMP/DEP ++ * (0x1800..0x1b70) and the r25 holder (G5P_R25_HEAD, 0x3000). ++ */ ++#define G5P_RU_SLOT1 0x1c00U ++#define G5P_RU_SLOT1_END \ ++ (G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES + 0x180U) ++ ++static inline u32 g5p_ru_cmd_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 : G5P_WORK_CMD; ++} ++ ++static inline u32 g5p_ru_w_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 + G5P_RU_CMD_BYTES : G5P_R22_W; ++} ++ ++static inline u32 g5p_ru_sku_at(u32 js, bool td) ++{ ++ if (js) ++ return G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES; ++ return td ? G5P_R22_3D_SKU : G5P_R22_TA_SKU; ++} ++ ++/* Per-job objects for sequence `seq`: W image (fixed+typed+RT), channel W ++ * fields, the full timestamped microsequence, the tag6 PM bootstrap (TA) or ++ * tag4 TA-finish dependency (3D) and both pointer-ring entries. With seq=1 ++ * this equals g5p_r22_channel(). Stamp target and sequence are both seq. ++ * ta_finish is M(TA stamp word addressed by TA.W530), used by 3D only. ++ */ ++static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, ++ bool td, u32 seq, const u8 *image, ++ u32 ta_finish, bool zero_timestamps, u32 js, ++ u32 pmid) ++{ ++ u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; ++ u32 finish = meta + G5P_R22_STAMP + 4; ++ u32 sku = g5p_ru_sku_at(js, td), cmd = g5p_ru_cmd_at(js); ++ u32 ring = g5p_ru_ring_index(seq); ++ u8 *work = p + g5p_ru_w_at(js); ++ u32 cursor = sku, starts, finishes; ++ ++ memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); ++ memset(p + sku, 0, td ? 0x180 : 0xc0); ++ g5p_put32(work, td ? 1 : 0); ++ g5p_put32(work + 4, slot); ++ g5p_put32(work + (td ? 0x14 : 0xc), meta + G5P_R22_DEP); ++ g5p_put32(work + (td ? 0x18 : 0x14), pm); ++ if (!td && pmid) ++ g5p_put32(work + 0x10, ++ pmid); /* TA W+10: the PM identity (iOS Dcb0) */ ++ g5p_put32(work + (td ? 0x1c : 0x18), state); ++ g5p_put32(work + (td ? 0xc : 0x454), meta + sku); ++ g5p_put32(work + (td ? 0x10 : 0x458), td ? 0x180 : 0xc0); ++ g5p_put32(work + (td ? 0x888 : 0x52c), finish - 4); ++ g5p_put32(work + (td ? 0x88c : 0x530), finish); ++ g5p_put32(work + (td ? 0x890 : 0x534), seq); ++ g5p_put32(work + (td ? 0x894 : 0x538), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x898 : 0x53c), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x8a0 : 0x544), seq); ++ { ++ u32 ta_start[] = { ++ 0x23, ++ w + 0x3a4, ++ w + 0x20, ++ state, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ pmid, ++ 0, ++ 0, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ 0, ++ seq, ++ }; ++ u32 td_start[] = { ++ 0x25, ++ w + 0x3a0, ++ w + 0x58, ++ state, ++ global + 0x724, ++ w + 0x7f0, ++ w + 0x7f4, ++ w + 0x878, ++ meta, ++ w, ++ slot, ++ 0, ++ pmid, ++ g5p_le32(work + 0x800), ++ 0, ++ 1, ++ w + 0x6d8, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ 0, ++ seq, ++ }; ++ u32 ta_finish[] = { ++ 0x24, ++ state, ++ pm, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ seq, ++ finish, ++ seq, ++ (u32)-0x70, ++ }; ++ u32 td_finish[] = { ++ 0x26, ++ seq, ++ finish, ++ seq, ++ state, ++ pm, ++ 1, ++ global + 0x724, ++ w + 0x878, ++ w + 0x7f0, ++ meta, ++ w, ++ slot, ++ w + 0x6d8, ++ 0, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ (u32)-0x104, ++ }; ++ u32 result = w + (td ? 0x8ac : 0x550); ++ u32 timestamp = w + (td ? 0x8b4 : 0x558); ++ u32 ts0[] = { 0x80000018, result, timestamp, timestamp, seq }; ++ u32 ts1[] = { 0x18, result, timestamp, timestamp + 4, seq }; ++ ++ starts = td ? ARRAY_SIZE(td_start) : ARRAY_SIZE(ta_start); ++ finishes = td ? ARRAY_SIZE(td_finish) : ARRAY_SIZE(ta_finish); ++ for (u32 i = 0; i < starts; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_start[i] : ta_start[i]); ++ for (u32 i = 0; i < ARRAY_SIZE(ts0); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts0[i]); ++ if (td) { ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r22_wait_words); ++ i++, cursor += 4) ++ g5p_put32(p + cursor, g5p_r22_wait_words[i]); ++ } else { ++ g5p_put32(p + cursor, 0x801); ++ cursor += 4; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(ts1); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts1[i]); ++ for (u32 i = 0; i < finishes; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_finish[i] : ta_finish[i]); ++ g5p_put32(p + cursor, 0x40000017); ++ } ++ memset(p + cmd, 0, G5P_RU_CMD_BYTES); ++ g5p_put32(p + cmd, td ? 4 : 6); ++ g5p_put32(p + cmd + 4, td ? ta_finish : slot); ++ g5p_put32(p + cmd + 8, ++ td ? seq : pmid); /* tag6 {6, STA, PMid, M(control)} */ ++ g5p_put32(p + cmd + 12, td ? seq & 0x7fffffff : pm); ++ g5p_put32(p + G5P_WORK_RING + ring * 4, meta + cmd); ++ g5p_put32(p + G5P_WORK_RING + ((ring + 1) % G5P_WORK_SLOTS) * 4, w); ++} ++ ++/* Retirement of job `seq`: both W pointers still ours, finish word and ++ * retirement mirror at seq, all five channel watermarks at its endpoint. ++ * g5p_r22_retired() is the seq=1 case. ++ */ ++static inline bool g5p_ru_retired_at(const u8 *p, u32 meta, bool td, u32 seq, ++ u32 js) ++{ ++ const u8 *w = p + g5p_ru_w_at(js); ++ u32 endpoint = g5p_ru_endpoint(seq); ++ ++ if (g5p_le32(w + (td ? 0x888 : 0x52c)) != meta + G5P_R22_STAMP || ++ g5p_le32(w + (td ? 0x88c : 0x530)) != meta + G5P_R22_STAMP + 4 || ++ g5p_le32(w + (td ? 0x890 : 0x534)) != seq || ++ g5p_le32(p + G5P_R22_STAMP) != seq || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* A stamp beyond the published target, or a watermark not on the even ++ * two-entry grid, is a protocol error rather than "not yet retired". ++ */ ++static inline bool g5p_ru_stamp_sane(const u8 *p, u32 seq) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ s32 ahead = (s32)(g5p_le32(p + G5P_R22_STAMP + i * 4) - seq); ++ ++ if (ahead > 0) ++ return false; ++ } ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) >= G5P_WORK_SLOTS) ++ return false; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-loss.h b/drivers/gpu/apple-g5p/render-user-loss.h +new file mode 100644 +index 000000000..2aad49d88 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-loss.h +@@ -0,0 +1,120 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context fault isolation (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault ++ * isolation). Pure decisions, host-tested (scripts/gpu/multiclient2-host/test_render_loss.c). ++ * ++ * The hardware gives no way to abandon a job: the firmware has no abort (iOS ends a ++ * non-progressing engine with a firmware stop, a platform power cycle and a new firmware boot, ++ * kext 6c9e070, firmware-re README "Fault recovery"), the TA and 3D channels are FIFO, and the ++ * one TA and one 3D engine serve every context. So one context's job can be *lost* (its client ++ * gets an error, its context is dead) without ending the device only while the firmware is ++ * alive, and only if the stuck phase still finishes: until then no other job may be published ++ * behind it (it would wait behind the same engine and be lost in turn). ++ * ++ * loss at the end of a phase that did not complete: ++ * NONE it completed; ++ * CONTEXT a phase timeout on a job other than the epoch's first, with the firmware ++ * alive (its event stream never went silent, no fault event, no protocol ++ * error) and isolation on: the job and its context are lost, the device is ++ * not; the channel is blocked behind the stuck phase; ++ * DEVICE anything else: the epoch is terminal, as before. ++ * drain while blocked, the scheduler runs no job and waits, RAM only, for the stuck phase: ++ * a late TA gets its 3D published (the channel and PM state then stay consistent), ++ * a late 3D ends it; the channel is then clear and every held job runs. A silent ++ * event stream, a fault, a refused 3D or the drain bound make the epoch terminal. ++ */ ++#ifndef G5P_RENDER_USER_LOSS_H ++#define G5P_RENDER_USER_LOSS_H ++ ++enum g5p_ru_loss { ++ G5P_RU_LOSS_NONE, ++ G5P_RU_LOSS_CONTEXT, ++ G5P_RU_LOSS_DEVICE, ++}; ++ ++struct g5p_ru_loss_in { ++ u32 result; /* G5P_RU_WAIT_* of the phase */ ++ bool first; /* the epoch's first job (its attach and bell) */ ++ bool isolate; /* render_user_drain_ms != 0 */ ++ bool silent; /* the event stream stopped: the firmware is dead */ ++ bool fault; /* a fault event, a failed boot, a protocol or drain error */ ++}; ++ ++static inline int g5p_ru_loss(const struct g5p_ru_loss_in *in) ++{ ++ if (in->result == G5P_RU_WAIT_COMPLETE) ++ return G5P_RU_LOSS_NONE; ++ if (in->result != G5P_RU_WAIT_TIMEOUT || in->first || !in->isolate || ++ in->silent || in->fault) ++ return G5P_RU_LOSS_DEVICE; ++ return G5P_RU_LOSS_CONTEXT; ++} ++ ++/* The drain of a blocked channel. */ ++enum g5p_ru_drain_state { ++ G5P_RU_DRAIN_TA, /* waiting for the stuck TA */ ++ G5P_RU_DRAIN_TD, /* waiting for the stuck job's 3D */ ++ G5P_RU_DRAIN_CLEAR, /* done: the channel is clear */ ++ G5P_RU_DRAIN_DEAD, /* the epoch is terminal */ ++}; ++ ++/* What the drain does next. */ ++enum g5p_ru_drain_action { ++ G5P_RU_DRAIN_WAIT, /* sample again */ ++ G5P_RU_DRAIN_PUBLISH_TD, /* publish the stuck job's 3D, wait for it */ ++ G5P_RU_DRAIN_UNBLOCK, /* release the held jobs */ ++ G5P_RU_DRAIN_TERMINAL, /* note the epoch terminal, release the held jobs (they fail) */ ++}; ++ ++/* One step from the result of the phase wait (G5P_RU_WAIT_*) in state *st. A refused 3D ++ * publish is reported by the caller as G5P_RU_WAIT_FAULT on the TD state. ++ */ ++static inline int g5p_ru_drain_step(u32 *st, u32 result) ++{ ++ if (*st >= G5P_RU_DRAIN_CLEAR) ++ return *st == G5P_RU_DRAIN_CLEAR ? G5P_RU_DRAIN_UNBLOCK : ++ G5P_RU_DRAIN_TERMINAL; ++ if (result == G5P_RU_WAIT_PENDING) ++ return G5P_RU_DRAIN_WAIT; ++ if (result != G5P_RU_WAIT_COMPLETE) { ++ *st = G5P_RU_DRAIN_DEAD; ++ return G5P_RU_DRAIN_TERMINAL; ++ } ++ if (*st == G5P_RU_DRAIN_TA) { ++ *st = G5P_RU_DRAIN_TD; ++ return G5P_RU_DRAIN_PUBLISH_TD; ++ } ++ *st = G5P_RU_DRAIN_CLEAR; ++ return G5P_RU_DRAIN_UNBLOCK; ++} ++ ++/* ++ * The drain bound must leave the held clients' own wait intact. A client waits for its job up to ++ * the job_timeout_ms GET_RENDER_INFO reports plus a margin (the ICD: + 3 s, so 2 * hard + obs + 3 s ++ * = 125 s with latency101 on, 33 s with it off), and a held job has already waited out the stuck ++ * phase (up to the hard bound, or render_user_wait_ms) before the drain starts, then runs after ++ * it. Valid: 0 (isolation off), or from 1000 up to hard + obs - 10000 with latency101 on (hard ++ * 60000, obs 2000: 52000), and up to 8000 with it off. An invalid value at the time of a loss ++ * turns isolation off for it. ++ */ ++static inline bool g5p_ru_drain_valid(u32 drain_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!drain_ms) ++ return true; ++ if (drain_ms < 1000) ++ return false; ++ if (!hard_ms) ++ return drain_ms <= 8000; ++ return hard_ms + obs_ms >= 11000 && ++ drain_ms <= hard_ms + obs_ms - 10000; ++} ++ ++ ++ ++static inline const char *g5p_ru_loss_name(int loss) ++{ ++ return loss == G5P_RU_LOSS_NONE ? "none" : ++ loss == G5P_RU_LOSS_CONTEXT ? "context" : "device"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-mmu.h b/drivers/gpu/apple-g5p/render-user-mmu.h +new file mode 100644 +index 000000000..5a05dccbf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-mmu.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 MMU invalidation arm (render_user_mmu_inval, drm-render-v3.inc), pure helpers. ++ * ++ * The only invalidation of a user MMU context besides DC3 is the firmware's per-kick tag compare ++ * (10017ac4): diff = host_tag[S] ^ cached_tag[S]; bits 31:22 set -> e138 = (S << 3) | 4 (PC), ++ * else bits 21:12 -> | 2 (PD), else bits 11:0 -> | 1 (PT); then cached = host. One invalidate per ++ * kick, the highest changed field (docs/gpu/firmware-re/README.md "generation producer"). A PC ++ * field change therefore asks for the firmware's PC-level invalidate of that context, the level ++ * Apple's own producer (6cabe70) and DC3 (1000218c) use as the strongest one. ++ * ++ * g5p_ru_mmu_pc_tag(): a host tag whose PC field differs from both slots' cached PC fields, so ++ * the next kick of each slot takes the PC branch. The PD and PT fields are the given tag's. ++ * Never 0 (0 is the unbound tag) and never equal to either cached tag. Host-tested ++ * (scripts/gpu/perf60-host/test_mmu_inval.c). ++ */ ++#ifndef G5P_RENDER_USER_MMU_H ++#define G5P_RENDER_USER_MMU_H ++ ++#define G5P_RU_MMU_PC_MASK 0xffc00000U ++#define G5P_RU_MMU_PC_ONE 0x00400000U ++ ++/* render_user_mmu_inval bits */ ++#define G5P_RU_MMU_EDIT \ ++ 0x1U /* every VM edit publication (and spare adoption) asks for PC */ ++#define G5P_RU_MMU_SWITCH \ ++ 0x2U /* a bound context's job after another context's job: PC retag */ ++#define G5P_RU_MMU_TOUCH \ ++ 0x4U /* NC read of every table line before the tag store */ ++#define G5P_RU_MMU_EVERY \ ++ 0x8U /* every job of a bound context: PC retag (superset of SWITCH) */ ++ ++static inline u32 g5p_ru_mmu_pc_tag(u32 tag, u32 cached0, u32 cached1) ++{ ++ u32 pc = (tag & G5P_RU_MMU_PC_MASK) + G5P_RU_MMU_PC_ONE; ++ ++ /* At most two values are excluded, so three candidates always contain a free one. */ ++ for (u32 i = 0; i < 3; i++, pc += G5P_RU_MMU_PC_ONE) { ++ u32 t = (pc & G5P_RU_MMU_PC_MASK) | (tag & ~G5P_RU_MMU_PC_MASK); ++ ++ if (((t ^ cached0) & G5P_RU_MMU_PC_MASK) && ++ ((t ^ cached1) & G5P_RU_MMU_PC_MASK) && t) ++ return t; ++ } ++ return 0; /* unreachable: kept for the host test */ ++} ++ ++/* ++ * Retag on a job: the published host tag, or the cached one if none, moved to a fresh PC field. ++ */ ++static inline bool g5p_ru_mmu_retag_due(u32 mode, bool bound_before, ++ bool switched) ++{ ++ if (!bound_before) ++ return false; /* this job's DC3 or adoption publishes its own tag */ ++ return (mode & G5P_RU_MMU_EVERY) || ++ ((mode & G5P_RU_MMU_SWITCH) && switched); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-msaa.h b/drivers/gpu/apple-g5p/render-user-msaa.h +new file mode 100644 +index 000000000..f27f197a6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-msaa.h +@@ -0,0 +1,88 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_MSAA_H ++#define G5P_RENDER_USER_MSAA_H ++ ++#include "bits-compat.h" ++/* TYPED_V3 4x multisampling: the per-job words a pass with a 4-sample render ++ * target needs on top of the V1/V2 3D words and the target's geometry ++ * (g5p_ru_rt_derive_samples()). Pure byte builders over the kernel's own ++ * work records: no allocation, MMIO, locking or firmware publication. ++ * ++ * Evidence (docs/gpu/experiments/msaa.md): the iOS gles3/gles3b captures ++ * 20261101T175626Z, msaa4-resolve, msaa4-store, g3b-a2c and g3b-samplemask, ++ * against g3-base, and the importer's 3D/TA fillers 6ca7ae0/6ca9710 executed ++ * with the 4-sample descriptor (build/gpu/msaa/filler-msaa.c): ++ * record+0x4d8 0xf -> 0x7 D6a8 -> 3D W3c0[3:0] (ISP_CTL; the draws' ++ * ISPCTL.upass drops 0xf -> 0x7 with it) ++ * record+0x640 0 -> 1 D6ad -> 3D W3c0 bit 17 and W86c (resolve only; ++ * bit 17 is ISP_CTL.PROCESS_EMPTY_TILES, which ++ * Linux's template sets for every pass since ++ * 0e05f993cf, so only W86c is new) ++ * RT Rb8 0 -> 2 D6e0 -> 3D Wd0 (AA mode, CR f30), and the ++ * pixel-output shift 10 -> 11 in W3c0 [21], ++ * W3c4, W3c8 and the six W108 words ++ * RT Rb0 / Rf8 3D W58 (CR fa8) / TA W40 (PPP_MULTISAMPLECTL) ++ * The rest of the target's geometry reaches W through g5p_ru_rt_fill(). ++ */ ++#define G5P_RU_MSAA_RESOLVE 1U /* DRM_G5P_RENDER_MSAA_RESOLVE */ ++ ++struct g5p_ru_msaa { ++ u32 samples, flags; /* 0 = 1 */ ++ u64 positions; /* 0 = the target's */ ++}; ++ ++/* samples 0 or 1 is the one-sample pass and must use a one-sample target; ++ * 4 must use a four-sample target. Positions are one byte per sample, so a ++ * word with bits above the sample count is malformed; zero takes the ++ * target's (the standard pattern). Resolve exists only with four samples. ++ * Nothing here is a hardware hazard: positions are a 4-bit grid and the ++ * resolve bit changes what the PBE writes, not where. ++ */ ++static inline int g5p_ru_msaa_check(const struct g5p_ru_msaa *m, u32 rt_samples) ++{ ++ u32 s = m->samples ? m->samples : 1; ++ ++ if ((s != 1 && s != 4) || s != rt_samples || ++ m->flags & ~G5P_RU_MSAA_RESOLVE || (m->flags && s == 1) || ++ m->positions >> (s * 8)) ++ return -EINVAL; ++ return 0; ++} ++ ++/* After g5p_ru_fill_3d(), the dialect's width words and g5p_ru_rt_fill(). ++ * A one-sample pass is left as it is, unless it names its position. ++ */ ++static inline void g5p_ru_fill_msaa(u8 *ta, u8 *td, const struct g5p_ru_msaa *m, ++ const struct g5p_ru_rt *rt) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by width - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u64 positions = m->positions ? m->positions : rt->packed; ++ bool resolve = m->flags & G5P_RU_MSAA_RESOLVE; ++ u32 width, count, mode, w3c0; ++ ++ if (m->samples != 4 && !m->positions) ++ return; ++ g5p_put64(ta + 0x40, positions); ++ g5p_put64(td + 0x58, positions); ++ if (m->samples != 4) ++ return; ++ /* The width the dialect wrote (2, 4 or 8); 6ca7ae0 shifts by 11 with an ++ * AA mode, so each width has half the output partitions. ++ */ ++ width = g5p_le32(td + 0x6b0); ++ count = 0x4000 / (width << 11); ++ mode = count > 1 ? 2 : count; ++ w3c0 = g5p_le32(td + 0x3c0) & ~(0xfU | BIT(21)); ++ g5p_put32(td + 0x3c0, ++ w3c0 | 0x7 | (resolve ? BIT(17) : 0) | (mode - 1) << 21); ++ g5p_put32(td + 0x3c4, mode * 0x204); ++ g5p_put32(td + 0x3c8, count & 0x1f); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(td + 0x108 + i * 8, ++ width_code[width - 2] | (count - 1) << 7 | 0x43); ++ g5p_put64(td + 0xd0, rt->state); ++ g5p_put32(td + 0x86c, resolve); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-pmset.h b/drivers/gpu/apple-g5p/render-user-pmset.h +new file mode 100644 +index 000000000..170738f1f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-pmset.h +@@ -0,0 +1,77 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context PM records (perf60 stage 4, docs/gpu/multi-client-design.md risk R3). ++ * ++ * iOS acquires a ParamBuffer ID per prepare from a 127-entry pool (6cb9afc) and builds a PM ++ * control/state/aux object and its block list per PM (6cb2378); one PM is never shared by two ++ * work queues. Linux built one device PM (control+S at shared+G5P_R22_PM, list and F in the ++ * two render_meta pages, aux at G5P_RU_PRIVATE_VA) for every render context; with the GPU ++ * never parked, one context's job then started from the PM state another context's job left. ++ * ++ * boot_pm_sets reserves N sets in the firmware GiB at boot, right after render_meta: set s is ++ * three pages {control + S, list, F} at G5P_PMSET_VA + s * 3 pages, mapped like the shared ++ * pages (they are firmware objects the firmware and the PM requesters address by meta VA). ++ * A render context takes one set, its own pool pages and its own aux pages. PM ID: 0 stays ++ * the device PM's; set s may carry s + 1. ++ * ++ * Pure helpers, host-tested (scripts/gpu/perf60-host/test_pmset.c). ++ */ ++#ifndef G5P_RENDER_USER_PMSET_H ++#define G5P_RENDER_USER_PMSET_H ++ ++#define G5P_PMSET_MAX 8U ++#define G5P_PMSET_PAGES 3U /* control + S, list, F */ ++#define G5P_PMSET_VA (G5P_SHARED_VA + G5P_SHARED_SIZE + 2 * G5P_GPU_PAGE) ++#define G5P_PMSET_END \ ++ (G5P_PMSET_VA + G5P_PMSET_MAX * G5P_PMSET_PAGES * G5P_GPU_PAGE) ++ ++enum { G5P_PMSET_CONTROL, G5P_PMSET_LIST, G5P_PMSET_FREE }; ++ ++static inline u64 g5p_pmset_va(u32 set, u32 part) ++{ ++ return G5P_PMSET_VA + ++ ((u64)set * G5P_PMSET_PAGES + part) * G5P_GPU_PAGE; ++} ++ ++static inline u32 g5p_pmset_id(u32 set) ++{ ++ return set + 1; ++} ++ ++/* ++ * The PM's write-back target (release8 bench-gaps3, 2026-11-03). After the GPU switches from ++ * context A's jobs to context B's, the firmware's PM keeps writing A's free list F back, through ++ * A's TA slot (its MMU context), on B's later jobs, until a job of a third context runs: two ++ * faults, one on the 2nd and one on the 3rd job of B after A's last, both the first job after A ++ * was unbound (page-invalid level 2 in A's MMU context, A's set F page; with A still bound the ++ * same writes landed). So a context's slots and set stay bound while it is the context of the ++ * last job or the previous *distinct* one: g5p_pmset_track() at every job end, g5p_pmset_holds() ++ * at close. That bounds when the slots may go, not when the PM stops: two later faults (release8 ++ * quiet2, seq 858 and 893) wrote the free list of a context two switches back, after this rule had ++ * released it. So a released pair is parked rather than left empty (render_user_close_park, ++ * drm-render-v3.inc), and a released set is reused last (g5p_pmset_pick()). ++ */ ++static inline bool g5p_pmset_switched(const void *last, const void *job) ++{ ++ return last != job; ++} ++ ++static inline bool g5p_pmset_holds(const void *last, const void *prev, ++ const void *ctx) ++{ ++ return ctx && (ctx == last || ctx == prev); ++} ++ ++/* A free set, round-robin from `next` (a released set is reused last). -1: none free. */ ++static inline int g5p_pmset_pick(const bool *used, u32 n, u32 next) ++{ ++ for (u32 i = 0; i < n; i++) { ++ u32 s = (next + i) % n; ++ ++ if (!used[s]) ++ return s; ++ } ++ return -1; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-quiesce.h b/drivers/gpu/apple-g5p/render-user-quiesce.h +new file mode 100644 +index 000000000..3671e415e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-quiesce.h +@@ -0,0 +1,145 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU quiesce before a kexec, reboot or power-off (docs/gpu/experiments/mcfold2-kexec.md). ++ * ++ * Without it the firmware keeps running from the outgoing kernel's RAM across the relocation: ++ * it still writes its event rings, and the MMU still holds a CBASE for every bound render slot ++ * pointing at page tables the next kernel reuses, until the next kernel's PMGR GFX cycle about ++ * 0.4 s into its boot. The quiesce ends that window on the outgoing side, in this order: ++ * ++ * 1 close job admission and wait (bounded) for submitters inside; ++ * 2 take the command lock (bounded trylock), so no job or command is in flight; ++ * 3 DETACH: every user slot the firmware may hold is unbound by a RAM-only DC3 batch ++ * (render-user-bind.h), only when the lock is ours, the firmware is healthy, no timeout ++ * or fault was noted and the DC ring is idle; ++ * 4 freeze the boot state and mask the GPU IRQ (no host register access after this); ++ * 5 CYCLE: one bounded PMGR GFX off/on, the same gate cycle the next kernel performs with ++ * apple_pmgr.kexec_gpu_cycle=1, only when no host thread can be inside a GPU register ++ * access: the lock is ours, or its holder is the TYPED_V3 executor in its RAM-only regime. ++ * ++ * Pure functions over the ops below, host-tested (scripts/gpu/mcfold2-host/test_kexec_quiesce.c). ++ */ ++#ifndef G5P_RENDER_USER_QUIESCE_H ++#define G5P_RENDER_USER_QUIESCE_H ++ ++/* kexec_quiesce module parameter */ ++#define G5P_RU_KQ_OFF 0U ++#define G5P_RU_KQ_RAM 1U /* detach only, no PMGR access */ ++#define G5P_RU_KQ_FULL 2U /* detach and cycle (default) */ ++ ++/* Plan */ ++#define G5P_RU_KQ_DETACH 0x1U ++#define G5P_RU_KQ_CYCLE 0x2U ++ ++/* Why a step was skipped */ ++#define G5P_RU_KQ_WHY_OFF 0x001U /* kexec_quiesce=0 */ ++#define G5P_RU_KQ_WHY_DOWN 0x002U /* the firmware never ran on this boot */ ++#define G5P_RU_KQ_WHY_BUSY 0x004U /* the command lock is not ours: no detach */ ++#define G5P_RU_KQ_WHY_FAILED \ ++ 0x008U /* fault or protocol failure: firmware state unknown */ ++#define G5P_RU_KQ_WHY_WEDGED \ ++ 0x010U /* a TYPED_V3 timeout or fault: a job may hold its slots */ ++#define G5P_RU_KQ_WHY_RING 0x020U /* the DC ring is not idle */ ++#define G5P_RU_KQ_WHY_NONE 0x040U /* no user slot is held */ ++#define G5P_RU_KQ_WHY_HOLDER \ ++ 0x080U /* unknown lock holder, maybe inside MMIO: no cycle */ ++#define G5P_RU_KQ_WHY_NO_PMGR 0x100U ++#define G5P_RU_KQ_WHY_RAM 0x200U /* kexec_quiesce=1: no cycle */ ++#define G5P_RU_KQ_WHY_NO_DRM 0x400U /* no render frontend: nobody can detach */ ++ ++struct g5p_ru_kq_in { ++ u32 mode; /* kexec_quiesce */ ++ bool booted; /* the firmware was released on this boot */ ++ bool locked; /* the command lock is ours */ ++ bool failed; /* b->failed or b->event_failed */ ++ bool frontend; /* a render frontend that can detach is registered */ ++ bool term_failed; /* the TYPED_V3 latch noted a timeout or fault */ ++ bool ram_only; /* the TYPED_V3 latch is quiet: its executor touches no register */ ++ bool executing; /* the TYPED_V3 executor is inside a job */ ++ bool ring_idle; /* DC consumer == producer <= 255 */ ++ bool pmgr; /* the PMGR regmap is present */ ++ u32 held; /* user slots with a PA or a host tag */ ++ u32 cons, prod; /* the DC ring when sampled */ ++}; ++ ++static inline u32 g5p_ru_kq_plan(const struct g5p_ru_kq_in *in, u32 *why) ++{ ++ u32 plan = 0, w = 0; ++ ++ if (in->mode == G5P_RU_KQ_OFF || !in->booted) { ++ *why = in->mode == G5P_RU_KQ_OFF ? G5P_RU_KQ_WHY_OFF : ++ G5P_RU_KQ_WHY_DOWN; ++ return 0; ++ } ++ if (!in->locked) ++ w |= G5P_RU_KQ_WHY_BUSY; ++ if (in->failed) ++ w |= G5P_RU_KQ_WHY_FAILED; ++ if (in->term_failed) ++ w |= G5P_RU_KQ_WHY_WEDGED; ++ if (!in->ring_idle) ++ w |= G5P_RU_KQ_WHY_RING; ++ if (!in->held) ++ w |= G5P_RU_KQ_WHY_NONE; ++ if (!in->frontend) ++ w |= G5P_RU_KQ_WHY_NO_DRM; ++ if (!w) ++ plan |= G5P_RU_KQ_DETACH; ++ if (in->mode < G5P_RU_KQ_FULL) ++ w |= G5P_RU_KQ_WHY_RAM; ++ else if (!in->pmgr) ++ w |= G5P_RU_KQ_WHY_NO_PMGR; ++ else if (!in->locked && !(in->executing && in->ram_only)) ++ w |= G5P_RU_KQ_WHY_HOLDER; ++ else ++ plan |= G5P_RU_KQ_CYCLE; ++ *why = w; ++ return plan; ++} ++ ++struct g5p_ru_kq_ops { ++ void *ctx; ++ u32 (*admit_close)(void *ctx); /* close admission; submitters still inside */ ++ bool (*lock)(void *ctx); /* bounded trylock of the command lock */ ++ void (*unlock)(void *ctx); ++ void (*inputs)(void *ctx, struct g5p_ru_kq_in *in); /* RAM only */ ++ int (*detach)(void *ctx); /* RAM only */ ++ void (*freeze)(void *ctx); /* RAM only */ ++ void (*irq_off)(void *ctx); /* AIC ack, mask, wait for the handler */ ++ int (*cycle)(void *ctx); /* PMGR only */ ++}; ++ ++struct g5p_ru_kq_out { ++ struct g5p_ru_kq_in in; ++ u32 inside, plan, why; ++ int detach_ret, cycle_ret; ++ bool froze; ++}; ++ ++/* One quiesce. detach_ret/cycle_ret stay 0 for a step that did not run (see plan). */ ++static inline void g5p_ru_kq_run(const struct g5p_ru_kq_ops *o, u32 mode, ++ struct g5p_ru_kq_out *out) ++{ ++ *out = (struct g5p_ru_kq_out){ .in.mode = mode }; ++ if (mode == G5P_RU_KQ_OFF) { ++ out->why = G5P_RU_KQ_WHY_OFF; ++ return; ++ } ++ out->inside = o->admit_close(o->ctx); ++ out->in.locked = o->lock(o->ctx); ++ o->inputs(o->ctx, &out->in); ++ out->plan = g5p_ru_kq_plan(&out->in, &out->why); ++ if (out->plan & G5P_RU_KQ_DETACH) ++ out->detach_ret = o->detach(o->ctx); ++ if (out->in.booted) { ++ o->freeze(o->ctx); ++ o->irq_off(o->ctx); ++ out->froze = true; ++ } ++ if (out->plan & G5P_RU_KQ_CYCLE) ++ out->cycle_ret = o->cycle(o->ctx); ++ if (out->in.locked) ++ o->unlock(o->ctx); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-recover.h b/drivers/gpu/apple-g5p/render-user-recover.h +new file mode 100644 +index 000000000..51982c5c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-recover.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * In-boot GPU recovery: the firmware's recovery packet and the recovery decision. Pure, ++ * host-tested (scripts/gpu/perf60-host/test_recover.c). ++ * ++ * The recovery packet is firmware event type 4 (44 bytes). The AGX kext's event drain ++ * (AGXFirmware_drainFirmwareEventRing 6c99788, case 4) reads +0x0c (the faulting slot, 0x80 for ++ * -1), +0x10 (a flag, accel+0x1af8 bit 3), +0x18, +0x20 and +0x24 (accel+0x1a1c/0x1af4/0x1af0), ++ * marks a recovery pending (accel+0x1a24 = 2) and calls IOGPUScheduler_signalHardwareError(sched, ++ * 1) (6c67b4c): "GPURestartSignaled", the scheduler enqueues one GPU restart (a second signal ++ * before it runs is "Redundant hardware error"). The restart is ++ * AGXMetaFirmware_stopForRecoveryOptionalPowerCycle (6ca1e40) and ++ * AGXMetaFirmware_rebootAfterRecovery (6ca1efc), which panics only if the firmware does not boot ++ * again. There is no per-boot limit: every recovery packet restarts the GPU. The ++ * kext does not read +0x04. ++ * ++ * On A9/Linux the slot matches the phase the host waited in: 0 on every TA fault, 1 on the 3D fault ++ * of cf7 (2026-11-02); +0x20 = 8 and +0x24 = 0x81818105 (TA) / 0x81810581 (3D) so far. ++ */ ++#ifndef G5P_RENDER_USER_RECOVER_H ++#define G5P_RENDER_USER_RECOVER_H ++ ++#define G5P_FW_EVENT_RECOVERY 4 ++#define G5P_FW_RECOVERY_NOSLOT 0x80 ++ ++struct g5p_fw_recovery { ++ u32 code; /* +0x04, not read by the kext */ ++ u32 slot; /* +0x0c, G5P_FW_RECOVERY_NOSLOT for -1 */ ++ u32 flag; /* +0x10 != 0 */ ++ u32 w18, w20, w24; ++}; ++ ++/* `w` is the 11-word event entry, type 4. */ ++static inline struct g5p_fw_recovery g5p_fw_recovery_decode(const u32 *w) ++{ ++ struct g5p_fw_recovery r = { ++ .code = w[1], ++ .slot = w[3] == 0xffffffffU ? G5P_FW_RECOVERY_NOSLOT : w[3], ++ .flag = w[4] != 0, ++ .w18 = w[6], ++ .w20 = w[8], ++ .w24 = w[9], ++ }; ++ ++ return r; ++} ++ ++/* ++ * May a job that stopped the firmware (a recovery packet, a silent event stream, a hard timeout) ++ * be recovered from in this boot, the job's context lost and every other kept? ++ * ++ * budget == 0 never (render_user_recover 0: the epoch ends, as before); ++ * retired/closed never (the boot is shutting the GPU down); ++ * streak_max == 0 the old rule: `budget` recoveries per boot, never on the epoch's first job; ++ * streak_max != 0 Apple's rule, every time, with two guards: at most streak_max recoveries in a ++ * row with no job completing between them (a GPU that faults again before it ++ * finishes anything is not recovering), and at most G5P_RU_RECOVER_BOOT_MAX per ++ * boot (recover.inc leaves each old epoch allocated, ~0.8 MiB: 50 x 16 KiB pages ++ * and its struct, so a stale pointer still reads valid memory; this bounds that ++ * to ~26 MiB). The boot's first epoch's first job stays terminal (a failed ++ * bring-up, not a client); the first job of a later epoch (the attach after a ++ * recovery) is recoverable like any other. ++ */ ++#define G5P_RU_RECOVER_BOOT_MAX 32 ++ ++struct g5p_ru_recover_in { ++ bool first; /* the job runs its epoch's attach */ ++ bool retired; /* the GPU was retired (kexec / reboot quiesce) */ ++ bool closed; /* admission closed */ ++ u32 recoveries; /* recoveries this boot */ ++ u32 streak; /* recoveries since the last job completed */ ++ u32 budget; /* render_user_recover */ ++ u32 streak_max; /* render_user_recover_streak */ ++}; ++ ++static inline bool g5p_ru_recoverable(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget || in->retired || in->closed) ++ return false; ++ if (!in->streak_max) ++ return !in->first && in->recoveries < in->budget; ++ if (in->first && !in->recoveries) ++ return false; ++ return in->streak < in->streak_max && ++ in->recoveries < G5P_RU_RECOVER_BOOT_MAX; ++} ++ ++/* Why g5p_ru_recoverable() said no (for the terminal line), or "-" when it said yes. */ ++static inline const char * ++g5p_ru_recover_refusal(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget) ++ return "recover-off"; ++ if (in->retired || in->closed) ++ return "shutdown"; ++ if (!in->streak_max) ++ return in->first ? "first-job" : ++ in->recoveries >= in->budget ? "per-boot-budget" : ++ "-"; ++ if (in->first && !in->recoveries) ++ return "first-job"; ++ if (in->streak >= in->streak_max) ++ return "streak"; ++ return in->recoveries >= G5P_RU_RECOVER_BOOT_MAX ? "boot-max" : "-"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-retire.h b/drivers/gpu/apple-g5p/render-user-retire.h +new file mode 100644 +index 000000000..6e5729003 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-retire.h +@@ -0,0 +1,248 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 3D completion: target coherence and engine idle, decided separately. Pure ++ * functions, no kernel services beyond the integer types, so the host suite compiles this ++ * file as it is. Include render-user-wait.h first. ++ * ++ * combined107 P4R seq 4 (build/gpu/combined107/raw/20261031T145710Z): the 3D target state ++ * was coherent (ring consumer = producer = target 4, both stamps 4, W+0x888/0x88c ours, ++ * W+0x890 = 4, all five channel watermarks at endpoint 8, and the firmware's type-1 3D ++ * completion event with the endpoint pair 7/8 decoded at publish + 19999 ms), yet the ++ * wait timed out at 60002 ms. The wait is a 1-2 ms RAM poll (no waitqueue, no wake), so ++ * this was not a lost edge: the done predicate also required the 3D global's idle words ++ * (G+0x724/0x728/0x72c/0x734/0x738) to be zero, and they never cleared. ++ * ++ * The 3D phase is complete when either ++ * (a) the target is coherent and the idle words are zero (the flown r96 predicate), or ++ * (b) the target is coherent and the matching 3D completion event was decoded after this ++ * job's 3D publish (idle_stuck). The event is the firmware's completion report, the ++ * signal a fence follows on Asahi too; the idle words lag it by an arbitrary time. ++ * settle_ms (default 0) only adds a minimum length of the coherent run, or ++ * (c) the target has been coherent, unbroken, for G5P_RU_RT_HOLD_MS (at least settle_ms): ++ * tahang1, as the TA's rule (c), so a lost or late event does not hold a job whose ++ * stamps landed. The PM state needs no clause here: the TA phase, which the 3D ++ * publish waits for, required it saved. ++ * Target coherent = ring consumer == target && g5p_ru_retired(): both stamps == seq, the W ++ * stamp pointers ours, the W sequence word == seq, every channel watermark == endpoint. ++ * The stamps carry the identity (31-bit sequence, one per job); the watermarks and the ++ * event's endpoint pair repeat every eight jobs and only corroborate. The TA phase has its ++ * own rules (tahang1, below). ++ */ ++#ifndef G5P_RENDER_USER_RETIRE_H ++#define G5P_RENDER_USER_RETIRE_H ++ ++#define G5P_RU_RT_DEF_SETTLE_MS 0U /* default of render_user_idle_settle_ms */ ++#define G5P_RU_RT_MAX_SETTLE_MS \ ++ 5000U /* far below the smallest per-phase bound (20 s) */ ++#define G5P_RU_RT_GRID 16U /* the channel watermark grid, G5P_WORK_SLOTS */ ++#define G5P_RU_RT_HOLD_MS \ ++ 500U /* rule (c), raised to settle_ms when that is longer */ ++ ++/* Clauses of the 3D predicate, as the receipt's `unsat` bits (set = not satisfied). */ ++#define G5P_RU_RT_CONSUMED 0x1U /* ring consumer != target */ ++#define G5P_RU_RT_RETIRED 0x2U /* g5p_ru_retired() false */ ++#define G5P_RU_RT_IDLE 0x4U /* an idle word is non-zero */ ++#define G5P_RU_RT_EVENT 0x8U /* no matching completion event latched */ ++#define G5P_RU_RT_SETTLE \ ++ 0x10U /* coherent, run shorter than the settle or hold window */ ++#define G5P_RU_RT_PM 0x20U /* TA receipt only: the PM state is not saved */ ++ ++enum g5p_ru_rt_how { ++ G5P_RU_RT_PENDING, ++ G5P_RU_RT_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_RT_IDLE_STUCK, /* rule (b) */ ++ G5P_RU_RT_HOLD, /* rule (c) */ ++}; ++ ++struct g5p_ru_retire { ++ u64 settle_ns; /* minimum coherent run before rule (b); 0 = none */ ++ u64 hold_ns; /* rule (c): max(settle_ns, G5P_RU_RT_HOLD_MS) */ ++ u64 run_ns; /* first sample of the current coherent run; 0 = none */ ++ u64 coherent_ns; /* first coherent sample of the phase */ ++ u64 event_ns; /* first sample that saw the matching event latched */ ++ u64 busy_ns; /* first sample with an idle word set */ ++ u64 clear_ns; /* first all-clear sample after busy_ns */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample */ ++ u32 how; /* enum g5p_ru_rt_how of the last sample */ ++ u32 decided; /* enum g5p_ru_rt_how of the sample that decided COMPLETE */ ++}; ++ ++static inline bool g5p_ru_rt_settle_valid(u32 settle_ms) ++{ ++ return settle_ms <= G5P_RU_RT_MAX_SETTLE_MS; ++} ++ ++static inline void g5p_ru_rt_start(struct g5p_ru_retire *t, u32 settle_ms) ++{ ++ u32 hold_ms = settle_ms > G5P_RU_RT_HOLD_MS ? settle_ms : ++ G5P_RU_RT_HOLD_MS; ++ ++ *t = (struct g5p_ru_retire){ ++ .settle_ns = (u64)settle_ms * 1000000ULL, ++ .hold_ns = (u64)hold_ms * 1000000ULL, ++ }; ++} ++ ++/* The firmware's 3D completion is a type-1 event with notification bit 1 (engine 1) whose ++ * words 8/9 are the channel endpoint pair (2 seq - 1, 2 seq): 1/2, 3/4, 5/6, 7/8 for ++ * combined107 seq 1..4. The grid wraps at G5P_RU_RT_GRID, so the pair is compared modulo ++ * the grid whether the firmware reports it wrapped or as a running count: endpoint 0 ++ * (seq 8) matches 15/0 and 15/16 alike. ++ */ ++static inline bool g5p_ru_rt_event_match(const u32 words[11], u32 endpoint) ++{ ++ return words[0] == 1 && (words[1] & 2) && ++ words[9] % G5P_RU_RT_GRID == endpoint % G5P_RU_RT_GRID && ++ (words[9] - words[8]) % G5P_RU_RT_GRID == 1; ++} ++ ++/* One RAM sample of the 3D phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_rt_how); the caller passes `!= G5P_RU_RT_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_rt_step(struct g5p_ru_retire *t, u64 now_ns, ++ bool consumed, bool retired, bool idle, ++ bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ if (!idle && !t->busy_ns) ++ t->busy_ns = now_ns; ++ if (idle && t->busy_ns && !t->clear_ns) ++ t->clear_ns = now_ns; ++ t->how = G5P_RU_RT_PENDING; ++ if (!consumed || !retired) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ t->run_ns = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_RT_IDLE_CLEAR; ++ else if (event && now_ns - t->run_ns >= t->settle_ns) ++ t->how = G5P_RU_RT_IDLE_STUCK; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_RT_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_rt_note(struct g5p_ru_retire *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++/* ms from `from_ns` to `to_ns` for the receipt; -1 when either is missing or out of order. */ ++static inline s64 g5p_ru_rt_ms(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000000ULL); ++} ++ ++/* ++ * tahang1: the TA phase. mcfold1 desktop seq 251 (after T1) and seq 339 (one client): ring ++ * consumed at 2 ms, the firmware's type-1 TA completion event decoded at 7.7-8.4 ms, both ++ * stamps, the W words and the watermarks at the job and the PM state saved, yet a TA idle word ++ * (G+0x724/0x728/0x72c) set from 7-8 ms on never cleared, and the r74b predicate (which needs ++ * the idle words zero) timed out at 60 s (`unsat=0x4`). On 49 of 241 good TAs of the same ++ * boot that word was set at 7-12 ms and cleared 2 ms later. The 3D engine shows the same lag ++ * (rule (b) above). ++ * ++ * The TA phase is complete when the target is coherent (ring consumer == target and ++ * g5p_ru_retired()) with the PM state saved, and ++ * (a) the idle words are zero (the flown r74b predicate), or ++ * (b) a type-1 TA completion event was decoded after this job's TA publish, or ++ * (c) the target has stayed coherent, unbroken, for G5P_RU_TA_HOLD_MS. ++ * The stamps carry the job's identity; the TA event does not (its words are the same every ++ * job), it only reports that the firmware finished a TA after the publish. Rule (c) keeps a ++ * lost or late event from holding a job whose stamps already landed. ++ */ ++#define G5P_RU_TA_HOLD_MS 500U ++ ++enum g5p_ru_ta_how { ++ G5P_RU_TA_PENDING, ++ G5P_RU_TA_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_TA_EVENT, /* rule (b) */ ++ G5P_RU_TA_HOLD, /* rule (c) */ ++ G5P_RU_TA_HOWS, ++}; ++ ++struct g5p_ru_ta { ++ u64 hold_ns; /* rule (c) */ ++ u64 run_ns; /* first sample of the current coherent run */ ++ u64 coherent_ns; /* first coherent sample of the phase; 0 = none */ ++ u64 event_ns; /* first sample that saw the event latched; 0 = none */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample (SETTLE: run below the hold) */ ++ u32 how; /* enum g5p_ru_ta_how of the last sample */ ++ u32 decided; /* enum g5p_ru_ta_how of the sample that decided COMPLETE */ ++}; ++ ++static inline void g5p_ru_ta_start(struct g5p_ru_ta *t) ++{ ++ *t = (struct g5p_ru_ta){ .hold_ns = ++ (u64)G5P_RU_TA_HOLD_MS * 1000000ULL }; ++} ++ ++/* The firmware's TA completion: a type-1 event with notification bit 0 (engine 0). */ ++static inline bool g5p_ru_ta_event_match(const u32 words[11]) ++{ ++ return words[0] == 1 && (words[1] & 1); ++} ++ ++/* One RAM sample of the TA phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_ta_how); the caller passes `!= G5P_RU_TA_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_ta_step(struct g5p_ru_ta *t, u64 now_ns, bool consumed, ++ bool retired, bool pm, bool idle, bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT) | ++ (pm ? 0 : G5P_RU_RT_PM); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ t->how = G5P_RU_TA_PENDING; ++ if (!consumed || !retired || !pm) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_TA_IDLE_CLEAR; ++ else if (event) ++ t->how = G5P_RU_TA_EVENT; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_TA_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_ta_note(struct g5p_ru_ta *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sched.h b/drivers/gpu/apple-g5p/render-user-sched.h +new file mode 100644 +index 000000000..174a29e23 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sched.h +@@ -0,0 +1,155 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Render job scheduling across contexts (docs/gpu/multi-client-design.md, stage 2), shaped ++ * like drm_sched so that stage 4 can swap drm_sched in without a UAPI or structural change: ++ * ++ * entity = one render context (drm_sched_entity): a FIFO of its accepted jobs, and at ++ * most one close item, which runs after its last job (drm_sched_entity_destroy()); ++ * run list = the entities with work, served round-robin: each pick takes the first entity ++ * whose head job is ready and moves it to the tail (drm_sched's RR policy); ++ * ready = every dependency of the head job has signalled, it failed before the hardware, ++ * its input deadline passed, or admission closed for good (the job then fails ++ * without the hardware). Readiness is the caller's predicate; the kernel's adds ++ * dma-fence callbacks (drm_sched_job_add_dependency()) so that an unsignalled ++ * input of one client never blocks another client's jobs. ++ * ++ * One credit: the single shared TA/3D channel and PM record run one job at a time, so the ++ * caller runs each picked item to completion before the next pick. Only the head of an ++ * entity is looked at, so each context's fences signal in submission order. ++ * ++ * The lists are plain list_heads; the caller holds its scheduler lock around every call. ++ * Host-tested (scripts/gpu/multiclient2-host/test_render_sched.c). ++ */ ++#ifndef G5P_RENDER_USER_SCHED_H ++#define G5P_RENDER_USER_SCHED_H ++ ++struct g5p_rs_job { ++ struct list_head link; ++}; ++ ++struct g5p_rs_entity { ++ struct list_head link; /* in g5p_rs.run while it has work */ ++ struct list_head jobs; /* FIFO of g5p_rs_job */ ++ bool in_run, close_pending; ++}; ++ ++struct g5p_rs { ++ struct list_head run; ++}; ++ ++/* g5p_rs_pick() result */ ++enum g5p_rs_pick { ++ G5P_RS_IDLE, /* nothing ready; *wake holds the earliest deadline if any job waits */ ++ G5P_RS_JOB, /* *job was popped from *entity */ ++ G5P_RS_CLOSE, /* *entity has no job left and its close item is due */ ++}; ++ ++/* Head-job predicate: true when it may run now. *deadline is the time at which it becomes ++ * ready on its own (its input deadline); the predicate leaves it alone if there is none. ++ */ ++typedef bool (*g5p_rs_ready_fn)(struct g5p_rs_job *job, void *cookie, ++ unsigned long *deadline); ++ ++static inline void g5p_rs_init(struct g5p_rs *s) ++{ ++ INIT_LIST_HEAD(&s->run); ++} ++ ++static inline void g5p_rs_entity_init(struct g5p_rs_entity *e) ++{ ++ INIT_LIST_HEAD(&e->link); ++ INIT_LIST_HEAD(&e->jobs); ++ e->in_run = false; ++ e->close_pending = false; ++} ++ ++static inline void g5p_rs_enlist(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (!e->in_run) { ++ list_add_tail(&e->link, &s->run); ++ e->in_run = true; ++ } ++} ++ ++/* An accepted job joins the end of its entity's FIFO. */ ++static inline void g5p_rs_push(struct g5p_rs *s, struct g5p_rs_entity *e, ++ struct g5p_rs_job *job) ++{ ++ list_add_tail(&job->link, &e->jobs); ++ g5p_rs_enlist(s, e); ++} ++ ++/* The entity's close item: due once its FIFO is empty. False if one is already queued. */ ++static inline bool g5p_rs_close(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (e->close_pending) ++ return false; ++ e->close_pending = true; ++ g5p_rs_enlist(s, e); ++ return true; ++} ++ ++/* After the close item ran: the entity leaves the scheduler for good. */ ++static inline void g5p_rs_closed(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ (void)s; ++ e->close_pending = false; ++ if (e->in_run) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } ++} ++ ++/* ++ * The next item to run, round-robin over the entities with work. A picked entity moves to ++ * the tail; one left without a job and without a close item leaves the run list. Entities ++ * whose head is not ready keep their place. `has_wake` reports whether *wake is set. ++ */ ++static inline int g5p_rs_pick(struct g5p_rs *s, g5p_rs_ready_fn ready, ++ void *cookie, struct g5p_rs_entity **entity, ++ struct g5p_rs_job **job, unsigned long *wake, ++ bool *has_wake) ++{ ++ struct g5p_rs_entity *e; ++ ++ *has_wake = false; ++ list_for_each_entry(e, &s->run, link) { ++ struct g5p_rs_job *head; ++ unsigned long deadline; ++ bool timed = false; ++ ++ if (list_empty(&e->jobs)) { ++ /* Without a close item it left the run list at its last pick. */ ++ if (!e->close_pending) ++ continue; ++ *entity = e; ++ *job = NULL; ++ list_move_tail(&e->link, &s->run); ++ return G5P_RS_CLOSE; ++ } ++ head = list_first_entry(&e->jobs, struct g5p_rs_job, link); ++ deadline = 0; ++ if (!ready(head, cookie, &deadline)) { ++ timed = deadline != 0; ++ if (timed && ++ (!*has_wake || (long)(deadline - *wake) < 0)) { ++ *wake = deadline; ++ *has_wake = true; ++ } ++ continue; ++ } ++ list_del_init(&head->link); ++ *entity = e; ++ *job = head; ++ if (list_empty(&e->jobs) && !e->close_pending) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } else { ++ list_move_tail(&e->link, &s->run); ++ } ++ return G5P_RS_JOB; ++ } ++ return G5P_RS_IDLE; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-serial.h b/drivers/gpu/apple-g5p/render-user-serial.h +new file mode 100644 +index 000000000..e5cffa4e3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-serial.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial jobs of every render context (a compositor and its clients: one TA+3D job ++ * per frame each), on the device's one TA/3D channel pair. ++ * Pure functions, host-tested (scripts/gpu/multiclient1-host/test_render_admit.c). ++ * ++ * Semantics: ++ * - jobs of all contexts run strictly one after another: the queue is FIFO, each job is ++ * TA publish -> TA complete -> 3D publish -> 3D complete before the next job starts, and ++ * there is never a TA(N+1) in flight with a 3D(N); ++ * - the epoch's first job brings the device up: PM, channels and its context's DC3 attach ++ * with the r74f/r74g bells (one-shot). Every later context binds its own slot pair at its ++ * first job with a DC3 pair published in RAM only (render-user-bind.h), no doorbell; ++ * - every job after the first only rebuilds the PM pool, builds its pages for the device's ++ * sequence number, publishes with NO doorbell and waits RAM-only; ++ * - the first job of a context that does not complete (timeout, fault, protocol error) ++ * marks that context bad: every job still queued behind it fails with -EIO without ++ * touching the hardware, and every later submit is refused (-EIO); ++ * - TYPED_V3 never mixes with the other profiles, on a context or on the device; ++ * - after any timeout or fault (render-user-terminal.h `failed`) no job starts at all, and ++ * once the epoch is quiet the device cannot be brought up (that rings a doorbell); binding ++ * another context is RAM only and stays allowed; ++ * - the sequence number is the device's, 31 bit; the channel watermark grid wraps ++ * (g5p_ru_endpoint). ++ */ ++#ifndef G5P_RENDER_USER_SERIAL_H ++#define G5P_RENDER_USER_SERIAL_H ++ ++enum g5p_ru_v3_refusal { ++ G5P_RU_V3_OK, ++ G5P_RU_V3_NO_TUPLE, /* boot tuple (r74g + r87b self-drain, no r74h/r80) absent */ ++ G5P_RU_V3_MIXED, /* another profile ran on this context or brought the device up */ ++ G5P_RU_V3_OVERFLOW, /* sequence space exhausted */ ++ G5P_RU_V3_SLOTS, /* not two distinct slots in 7..255 */ ++ G5P_RU_V3_VARIANT, /* r22/r25 variant, 3d_after_ta, timestamps or pm_policy */ ++ G5P_RU_V3_BOUND, /* per-phase bound outside 20000..30000 ms */ ++ G5P_RU_V3_STATE, /* executor only: bind state and job counts disagree */ ++ G5P_RU_V3_TERMINAL, /* terminal epoch: timeout/fault, or quiet and the device down */ ++ G5P_RU_V3_REFUSALS, ++}; ++ ++static const char *const g5p_ru_v3_why[G5P_RU_V3_REFUSALS] = { ++ "", ++ "transport is not ready for TYPED_V3 rendering", ++ "another profile ran on this context or brought the device up", ++ "sequence space exhausted", ++ "slots are not two distinct slots in 7..255", ++ "an r22/r25 variant, 3d_after_ta, timestamps or pm_policy is latched", ++ "per-phase bound outside 20000..30000 ms", ++ "bind state and job counts disagree", ++ "terminal epoch (timeout or fault, or quiet and the device not up): no hardware access", ++}; ++ ++/* Everything the admission needs, as plain values. */ ++struct g5p_ru_v3_state { ++ bool tuple; /* boot tuple present */ ++ bool v3_accepted; /* a V3 job was accepted on this context */ ++ bool bound; /* this context's slot pair was published */ ++ u32 jobs; /* jobs that started on this context */ ++ bool device_up; /* the epoch's first job built PM and channels */ ++ bool device_v3; /* ... through the TYPED_V3 attach */ ++ u32 sequence; /* the device's last job that started (0 = none) */ ++ u32 slot_ta, slot_3d; ++ bool variant; ++ u32 wait_ms; ++ bool quiet; /* a terminal note was made: mmio_ok is false */ ++ bool terminal_failed; /* ... and it was, or a later one is, a timeout or fault */ ++}; ++ ++/* Any two distinct user slots: unbinds leave holes, so a pair need not be adjacent. */ ++static inline bool g5p_ru_v3_slots_valid(u32 slot_ta, u32 slot_3d) ++{ ++ return slot_ta >= 7 && slot_3d >= 7 && slot_ta <= 255 && ++ slot_3d <= 255 && slot_ta != slot_3d; ++} ++ ++#define G5P_RU_V3_SEQ_MAX 0x7fffffffU ++ ++/* At submit time and again in the executor (`executing`): a refusal here touched nothing. */ ++static inline int g5p_ru_v3_admit(const struct g5p_ru_v3_state *s, ++ bool executing) ++{ ++ if (!s->tuple) ++ return G5P_RU_V3_NO_TUPLE; ++ /* After any timeout or fault nothing new starts (closes the window between a failing job's ++ * unlock and the context being marked bad). Once quiet, the device cannot be brought up, ++ * because that rings a doorbell; a job on an up device publishes with no doorbell and ++ * touches no register, and so does the bind of a new context. ++ */ ++ if (s->terminal_failed || (s->quiet && !s->device_up)) ++ return G5P_RU_V3_TERMINAL; ++ /* A job of another profile ran on this context, or another profile brought the device ++ * up: only V3 may follow V3. ++ */ ++ if ((s->jobs && !s->v3_accepted) || (s->device_up && !s->device_v3)) ++ return G5P_RU_V3_MIXED; ++ if (s->sequence >= G5P_RU_V3_SEQ_MAX) ++ return G5P_RU_V3_OVERFLOW; ++ if (!g5p_ru_v3_slots_valid(s->slot_ta, s->slot_3d)) ++ return G5P_RU_V3_SLOTS; ++ if (s->variant) ++ return G5P_RU_V3_VARIANT; ++ if (!g5p_ru_v3_bound_valid(s->wait_ms)) ++ return G5P_RU_V3_BOUND; ++ /* The executor: the device is up exactly when a job has started on it, and a context is ++ * bound exactly when it has run a job, always on an up device. ++ */ ++ if (executing && (s->device_up != !!s->sequence || ++ s->bound != !!s->jobs || (s->bound && !s->device_up))) ++ return G5P_RU_V3_STATE; ++ return G5P_RU_V3_OK; ++} ++ ++/* Verbose receipts for each context's first jobs and every failure; a compositor prints no ++ * line per frame. ++ */ ++static inline bool g5p_ru_v3_verbose(u32 jobs, int ret) ++{ ++ return jobs <= 8 || ret; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sync.h b/drivers/gpu/apple-g5p/render-user-sync.h +new file mode 100644 +index 000000000..d212d3199 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sync.h +@@ -0,0 +1,50 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CPU cache maintenance by range (perf60-fillrate.md §11.3 cut 5, kernel half). Host test: ++ * scripts/gpu/perf60-host/test_sync_range.c. ++ * ++ * render_user_sync_range bit 0: BO_SYNC honours offset/size (already validated page aligned by ++ * g5p_bo_range_valid) instead of syncing the whole BO. Bit 1: a job's ACCESS_CACHE maintenance ++ * (clean before, invalidate after retirement) covers the resource's BO range [offset, offset + ++ * size), the bytes the job can reach through that resource, instead of the whole BO. The window ++ * is widened to whole GPU pages and clamped to the BO. Off: the whole BO, as release8. ++ * The window is walked over the sg_table's DMA segments in order (their lengths concatenate to ++ * the BO's bytes); each segment gets the intersection, relative to the segment. ++ */ ++#ifndef G5P_RENDER_USER_SYNC_H ++#define G5P_RENDER_USER_SYNC_H ++ ++#define G5P_SYNC_ALIGN 0x4000ULL ++ ++/* ++ * [lo, hi) = [off, off + size) widened to pages, clamped to the BO; false if empty or outside. ++ */ ++static inline bool g5p_sync_window(u64 off, u64 size, u64 bo_size, u64 *lo, ++ u64 *hi) ++{ ++ u64 end; ++ ++ if (!size || off >= bo_size) ++ return false; ++ end = size > bo_size - off ? bo_size : off + size; ++ *lo = off & ~(G5P_SYNC_ALIGN - 1); ++ *hi = (end + G5P_SYNC_ALIGN - 1) & ~(G5P_SYNC_ALIGN - 1); ++ if (*hi > bo_size) ++ *hi = bo_size; ++ return *hi > *lo; ++} ++ ++/* The part of segment [pos, pos + len) inside [lo, hi): *start relative to the segment. */ ++static inline bool g5p_sync_clip(u64 pos, u64 len, u64 lo, u64 hi, u64 *start, ++ u64 *n) ++{ ++ u64 a = pos > lo ? pos : lo, b = pos + len < hi ? pos + len : hi; ++ ++ if (b <= a) ++ return false; ++ *start = a - pos; ++ *n = b - a; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-template.h b/drivers/gpu/apple-g5p/render-user-template.h +new file mode 100644 +index 000000000..92bd1604a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-template.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Generated by make-render-user-template.py from the hash-pinned r22 importer. */ ++#ifndef G5P_RENDER_USER_TEMPLATE_H ++#define G5P_RENDER_USER_TEMPLATE_H ++ ++/* Frozen dialect-1 W scalars {offset, value}: color-only, one sample, ++ * normal 3D, no depth/resource arrays. Typed and kernel fields excluded. ++ */ ++static const u32 g5p_ru_ta_fixed[][2] = { ++ { 0x020, 0x1e3ce508 }, { 0x024, 0x1e3ce508 }, { 0x3a4, 0x00000088 }, ++ { 0x3a8, 0x00000002 }, { 0x3c0, 0x00020000 }, { 0x514, 0x08002358 }, ++ { 0x528, 0x00000001 }, ++}; ++ ++static const u32 g5p_ru_3d_fixed[][2] = { ++ { 0x108, 0x000003c3 }, { 0x110, 0x000003c3 }, { 0x118, 0x000003c3 }, ++ { 0x120, 0x000003c3 }, { 0x128, 0x000003c3 }, { 0x130, 0x000003c3 }, ++ { 0x3a4, 0x3f800000 }, { 0x3a8, 0x00000300 }, { 0x3b0, 0x0000001f }, ++ { 0x3c0, 0x0023000f }, { 0x3c4, 0x00000408 }, { 0x3c8, 0x00000008 }, ++ { 0x3cc, 0x000000f8 }, { 0x6a8, 0x3f800000 }, { 0x6ac, 0x00000300 }, ++ { 0x6b0, 0x00000002 }, { 0x868, 0x08002358 }, ++}; ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-terminal.h b/drivers/gpu/apple-g5p/render-user-terminal.h +new file mode 100644 +index 000000000..4ced26eaf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-terminal.h +@@ -0,0 +1,103 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 terminal state of the epoch. Pure functions over one small struct; the ++ * kernel supplies READ_ONCE/WRITE_ONCE/cmpxchg and the host suite shims them, so the ++ * host compiles this file as it is. ++ * ++ * One latch, first writer wins. The first terminal event of the epoch (the natural TA ++ * complete, a timeout, a fault or a protocol error) fixes `kind`; from then on `quiet` ++ * is set and never cleared, and g5p_ru_mmio_ok() is false: the host touches no register ++ * (R0, R1, AIC, doorbell) and no diagnostic bank. RAM stays accessible. A later ++ * failure (the 3D after a completed TA, job N+1 after job N) cannot change `kind` but ++ * sets the sticky `failed`, which is what the admission uses: after any timeout or fault ++ * no new job starts, from any queue, even one that passed the pre-lock context checks ++ * before the failing job marked the context bad. ++ * ++ * The recovery series (patches/gpu-recovery) is independent of this file. Its ++ * g5p_td_note_terminal() takes {1 complete, 2 timeout}; g5p_ru_td_kind() maps ours. ++ */ ++#ifndef G5P_RENDER_USER_TERMINAL_H ++#define G5P_RENDER_USER_TERMINAL_H ++ ++enum g5p_ru_term_kind { ++ G5P_RU_TERM_NONE, ++ G5P_RU_TERM_COMPLETE, /* the natural TA completed (RAM) */ ++ G5P_RU_TERM_TIMEOUT, /* a phase ran out its deadline */ ++ G5P_RU_TERM_FAULT, /* protocol error, drain error, attach or publish failure */ ++ G5P_RU_TERM_KINDS, ++}; ++ ++static const char *const g5p_ru_term_names[G5P_RU_TERM_KINDS] = { ++ "none", ++ "complete", ++ "timeout", ++ "fault", ++}; ++ ++struct g5p_ru_terminal { ++ int kind; /* first writer wins */ ++ int quiet; /* set with the first note, never cleared */ ++ int failed; /* sticky: any timeout or fault was noted */ ++ int captured; /* the CAPTURE entry (recovery hook) was taken, once */ ++}; ++ ++/* A new firmware epoch (the in-boot recovery, recover.inc) starts a new latch; nothing else ++ * clears it. ++ */ ++static inline void g5p_ru_term_reset(struct g5p_ru_terminal *t) ++{ ++ WRITE_ONCE(t->kind, 0); ++ WRITE_ONCE(t->quiet, 0); ++ WRITE_ONCE(t->failed, 0); ++ WRITE_ONCE(t->captured, 0); ++} ++ ++/* The kind a job's return value stands for (only called for ret < 0, ret != -ECANCELED). */ ++static inline int g5p_ru_term_from_ret(int ret) ++{ ++ return ret == -ETIMEDOUT ? G5P_RU_TERM_TIMEOUT : G5P_RU_TERM_FAULT; ++} ++ ++/* Returns true for the first note of the epoch. */ ++static inline bool g5p_ru_term_note(struct g5p_ru_terminal *t, int kind) ++{ ++ bool first; ++ ++ if (kind <= G5P_RU_TERM_NONE || kind >= G5P_RU_TERM_KINDS) ++ kind = G5P_RU_TERM_FAULT; ++ first = cmpxchg(&t->kind, G5P_RU_TERM_NONE, kind) == G5P_RU_TERM_NONE; ++ if (kind >= G5P_RU_TERM_TIMEOUT) ++ WRITE_ONCE(t->failed, 1); ++ WRITE_ONCE(t->quiet, 1); ++ return first; ++} ++ ++static inline int g5p_ru_term_kind(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->kind); ++} ++ ++static inline bool g5p_ru_term_failed(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->failed); ++} ++ ++/* The single predicate for register access: false from the first terminal note on. */ ++static inline bool g5p_ru_mmio_ok(const struct g5p_ru_terminal *t) ++{ ++ return !READ_ONCE(t->quiet); ++} ++ ++/* True once per epoch for a timeout or fault: the caller may take the recovery CAPTURE entry. */ ++static inline bool g5p_ru_term_capture_due(struct g5p_ru_terminal *t, int kind) ++{ ++ return kind >= G5P_RU_TERM_TIMEOUT && cmpxchg(&t->captured, 0, 1) == 0; ++} ++ ++/* Recovery's terminal encoding: 1 complete, 2 timeout (a fault is a no-completion exit: 2). */ ++static inline int g5p_ru_td_kind(int kind) ++{ ++ return kind == G5P_RU_TERM_COMPLETE ? 1 : 2; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-touch.h b/drivers/gpu/apple-g5p/render-user-touch.h +new file mode 100644 +index 000000000..145bdbf2d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-touch.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 RAM-touch latency A/B (release). Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. Include render-user-wait.h first. ++ * ++ * Question (docs/gpu/experiments/latency101-device-analysis.md section 9): is the host's bulk read ++ * of the receipt pages what releases a stalled phase (the +6 ms atom after the job-1 receipt)? ++ * ++ * render_user_touch_ms = T, default 0 = off. On, every phase of every job records the first wait ++ * sample at or after publish + T that left the phase undecided (still pending, deadline not ++ * reached). Odd sequences are the touch arm: right after that sample the host reads the ++ * receipt's read set once (RAM only, folded into a sum). Even sequences are the control arm: the ++ * same instant is recorded and nothing is read. Neither arm changes a sample, the sleep, the ++ * deadline, a latch or the result, and neither prints before the job ends. ++ * ++ * Off: g5p_ru_touch_start() leaves the record zero and g5p_ru_touch_due() is never true, so the ++ * wait loop runs the latency101 instructions unchanged. ++ */ ++#ifndef G5P_RENDER_USER_TOUCH_H ++#define G5P_RENDER_USER_TOUCH_H ++ ++#define G5P_RU_TOUCH_MAX_MS \ ++ 20000U /* never at or past the smallest soft bound */ ++ ++struct g5p_ru_touch { ++ u64 at_ns; /* publish + T; 0 = off for this phase */ ++ u64 seen_ns; /* the undecided sample at or after at_ns that took it; 0 = not reached */ ++ u64 done_ns; /* end of the read (touch arm), seen_ns (control arm) */ ++ u32 bytes; /* bytes read: 0 on the control arm */ ++ u32 sum; /* fold of the words read: the loads cannot be elided */ ++ bool sweep; /* this phase is in the touch arm */ ++}; ++ ++/* 0 (off), or 1..G5P_RU_TOUCH_MAX_MS with latency101 on (hard_ms != 0) and strictly before the ++ * effective soft bound (g5p_ru_lat_soft()), so the instant always falls in the plain wait. ++ */ ++static inline bool g5p_ru_touch_valid(u32 touch_ms, u32 soft_ms, u32 hard_ms) ++{ ++ if (!touch_ms) ++ return true; ++ return hard_ms && touch_ms <= G5P_RU_TOUCH_MAX_MS && touch_ms < soft_ms; ++} ++ ++/* Odd sequences are the touch arm, even ones the control. */ ++static inline bool g5p_ru_touch_arm(u32 seq) ++{ ++ return seq & 1; ++} ++ ++static inline void g5p_ru_touch_start(struct g5p_ru_touch *t, u64 start_ns, ++ u32 touch_ms, u32 seq) ++{ ++ *t = (struct g5p_ru_touch){ 0 }; ++ if (!touch_ms) ++ return; ++ t->at_ns = start_ns + (u64)touch_ms * 1000000ULL; ++ t->sweep = g5p_ru_touch_arm(seq); ++} ++ ++/* ++ * Asked after g5p_ru_wait_step() of the sample taken at now_ns: true once, for the first sample ++ * at ++ * or after the instant whose step left the phase undecided and not expired. A completion, a fault ++ * or the deadline in the same sample wins, and then the instant is never taken. ++ */ ++static inline bool g5p_ru_touch_due(const struct g5p_ru_touch *t, u64 now_ns, ++ u32 result, bool expired) ++{ ++ return t->at_ns && !t->seen_ns && !result && !expired && ++ now_ns >= t->at_ns; ++} ++ ++/* The instant was taken at seen_ns (a sample time, never 0); done_ns is when the read ended. */ ++static inline void g5p_ru_touch_mark(struct g5p_ru_touch *t, u64 seen_ns, ++ u64 done_ns, u32 bytes, u32 sum) ++{ ++ t->seen_ns = seen_ns; ++ t->done_ns = done_ns < seen_ns ? seen_ns : done_ns; ++ t->bytes = bytes; ++ t->sum = sum; ++} ++ ++/* Receipt helper: to - from in microseconds, -1 when an end is missing or the order is wrong. */ ++static inline s64 g5p_ru_touch_us(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000); ++} ++ ++/* Receipt: how long after the instant (the end of the read) the phase's deciding sample saw it ++ * complete; -1 when the instant was not taken or the phase did not complete. ++ */ ++static inline s64 g5p_ru_touch_after_us(const struct g5p_ru_touch *t, ++ u32 result, u64 end_ns) ++{ ++ if (!t->seen_ns || result != G5P_RU_WAIT_COMPLETE) ++ return -1; ++ return g5p_ru_touch_us(t->done_ns, end_ns); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wait.h b/drivers/gpu/apple-g5p/render-user-wait.h +new file mode 100644 +index 000000000..b9b5f1c49 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wait.h +@@ -0,0 +1,266 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial wait policy. Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. ++ * ++ * Evidence behind the numbers (r96 raw timelines, 2026-10-30): ++ * - the natural TA completion event arrived +12.0060 s and +12.0062 s after the ++ * publish in the two boots that a 12000 ms cap turned into timeouts; the whole ++ * delay is inside the firmware's start23, and once it returns the hardware and ++ * the retirement take less than ~1 ms; ++ * - nothing the host does shortens the delay, so the wait is RAM-only: after the ++ * DC3 attach the host touches no register until the job has ended. ++ * ++ * Policy: ++ * - one phase (TA, then 3D) waits at most `bound` from its own publish: 20 s by ++ * default, never below 20 s, never above the absolute cap; ++ * - the whole job (TA publish to the end of the 3D phase) never exceeds the ++ * absolute cap of 30 s; ++ * - the deadline is decided on RAM samples. A completion seen in the sample that ++ * reaches the deadline wins; a deadline is only final after one closing sample ++ * taken at or after it, which closes the gap between the last poll and the ++ * expiry; a completion after that is evidence only (`late_after`), never the ++ * result; ++ * - a fault wins over a completion in the same sample and over the deadline. ++ */ ++#ifndef G5P_RENDER_USER_WAIT_H ++#define G5P_RENDER_USER_WAIT_H ++ ++#define G5P_RU_V3_WAIT_MS 20000U /* default and minimum per-phase bound */ ++#define G5P_RU_V3_CAP_MS \ ++ 30000U /* absolute job bound, and the largest per-phase bound */ ++ ++/* ++ * latency101 (default off: render_user_soft_ms = render_user_hard_ms = 0 leaves every number above, ++ * and the behaviour of every function below, exactly as it was). When on, the per-phase deadline ++ * is the HARD bound and the job bound is two hard bounds; the SOFT bound marks a phase "slow" ++ * (RAM-only record, no print, no latch). A silent event stream means a dead firmware from the ++ * phase's publish on (faultiso; before it, only from the soft bound). See ++ * docs/gpu/experiments/prod98c-bimodal-latency.md and docs/gpu/experiments/faultiso.md. ++ */ ++#define G5P_RU_LAT_MIN_SOFT_MS \ ++ 20000U /* never below the proven per-phase bound */ ++#define G5P_RU_LAT_DEF_SOFT_MS 45000U ++#define G5P_RU_LAT_MAX_HARD_MS \ ++ 60000U /* largest hard bound; the job bound is twice this */ ++#define G5P_RU_LAT_DEF_OBS_MS \ ++ 2000U /* RAM-only look after a timeout, pre-receipt */ ++#define G5P_RU_LAT_MAX_OBS_MS 5000U ++#define G5P_RU_LAT_SILENT_MS \ ++ 1000U /* no event decoded for this long in a phase = silent */ ++/* The silence clock runs from the publish (the host suites key on this). */ ++#define G5P_RU_LAT_SILENT_FROM_PUBLISH 1 ++ ++enum g5p_ru_wait_result { ++ G5P_RU_WAIT_PENDING, ++ G5P_RU_WAIT_COMPLETE, ++ G5P_RU_WAIT_TIMEOUT, ++ G5P_RU_WAIT_FAULT, ++}; ++ ++struct g5p_ru_wait { ++ u64 start_ns; /* the publish (producer store) of this phase */ ++ u64 deadline_ns; /* start + bound, clamped to the job's absolute deadline */ ++ u64 end_ns; /* the sample that decided */ ++ u32 result; ++ bool expired; /* a sample at or after the deadline has been seen */ ++ bool late_gap; /* completion first seen by the closing sample */ ++ bool late_after; /* completion seen after a timeout was decided: evidence only */ ++ /* latency101; all zero (and never read) unless soft_ns != 0 */ ++ u64 soft_ns; /* start + soft bound; 0 = the grace is off */ ++ u64 soft_seen_ns; /* the first sample at or after soft_ns */ ++ u64 late_ns; /* the first retired sample after the timeout was decided */ ++ u64 event_ns; /* sample time of the last change of the decoded-event counter */ ++ u64 busy_ns; /* the first sample whose idle words showed the engine busy */ ++ u32 events; /* the decoded-event counter at that sample */ ++ bool slow; /* the phase passed its soft bound and is still pending */ ++ bool silent; /* the event stream stopped during the phase: firmware dead */ ++ bool late_soft; /* completed after the soft bound: a success, never a failure */ ++}; ++ ++/* latency101 parameters. Off is hard = soft = 0. On: hard in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000. The job bound the waiters must outlast is two ++ * hard bounds plus one observation window. ++ */ ++static inline u32 g5p_ru_lat_soft(u32 soft_ms, u32 hard_ms) ++{ ++ if (soft_ms) ++ return soft_ms; ++ return hard_ms > G5P_RU_LAT_DEF_SOFT_MS ? G5P_RU_LAT_DEF_SOFT_MS : ++ hard_ms; ++} ++ ++static inline bool g5p_ru_lat_valid(u32 soft_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!hard_ms) ++ return !soft_ms; ++ soft_ms = g5p_ru_lat_soft(soft_ms, hard_ms); ++ return hard_ms <= G5P_RU_LAT_MAX_HARD_MS && ++ soft_ms >= G5P_RU_LAT_MIN_SOFT_MS && soft_ms <= hard_ms && ++ obs_ms <= G5P_RU_LAT_MAX_OBS_MS; ++} ++ ++static inline u32 g5p_ru_lat_job_ms(u32 hard_ms, u32 obs_ms) ++{ ++ return 2 * hard_ms + obs_ms; ++} ++ ++/* A per-phase bound is valid when it is in [G5P_RU_V3_WAIT_MS, G5P_RU_V3_CAP_MS]. */ ++static inline bool g5p_ru_v3_bound_valid(u32 ms) ++{ ++ return ms >= G5P_RU_V3_WAIT_MS && ms <= G5P_RU_V3_CAP_MS; ++} ++ ++/* The deadline of a phase: its own bound, clamped to the job's absolute deadline. ++ * A job deadline of 0 means "no job deadline yet" (the first phase). ++ */ ++static inline u64 g5p_ru_wait_deadline(u64 start_ns, u32 bound_ms, ++ u64 job_deadline_ns) ++{ ++ u64 d = start_ns + (u64)bound_ms * 1000000ULL; ++ ++ return job_deadline_ns && job_deadline_ns < d ? job_deadline_ns : d; ++} ++ ++static inline void g5p_ru_wait_start(struct g5p_ru_wait *w, u64 start_ns, ++ u32 bound_ms, u64 job_deadline_ns) ++{ ++ *w = (struct g5p_ru_wait){ ++ .start_ns = start_ns, ++ .deadline_ns = g5p_ru_wait_deadline(start_ns, bound_ms, ++ job_deadline_ns), ++ }; ++} ++ ++/* ++ * latency101: hard_ms is the per-phase deadline (clamped to the job's deadline as above), ++ * soft_ms ++ * (never past the deadline) the point where a still-pending phase is called slow. `events` seeds ++ * the silence clock. ++ */ ++static inline void g5p_ru_wait_start_lat(struct g5p_ru_wait *w, u64 start_ns, ++ u32 soft_ms, u32 hard_ms, ++ u64 job_deadline_ns, u32 events) ++{ ++ g5p_ru_wait_start(w, start_ns, hard_ms, job_deadline_ns); ++ w->soft_ns = start_ns + (u64)soft_ms * 1000000ULL; ++ if (w->soft_ns > w->deadline_ns) ++ w->soft_ns = w->deadline_ns; ++ w->events = events; ++ w->event_ns = start_ns; ++} ++ ++/* The first sample whose idle words (G+0x724..) show the engine busy, i.e. the firmware's ++ * start path has run. Recorded once; it splits a late start (busy_ns close to the ++ * completion) from slow hardware (busy_ns early). A busy window shorter than the poll ++ * period can be missed: 0 = not seen. ++ */ ++static inline void g5p_ru_wait_busy(struct g5p_ru_wait *w, u64 now_ns, ++ bool busy) ++{ ++ if (busy && !w->busy_ns) ++ w->busy_ns = now_ns; ++} ++ ++/* One look at the decoded-event counter, taken with every RAM sample, before g5p_ru_wait_step(). ++ * The ALIVE stream runs at about 290 events/s whatever the work does (r96c: 29442 polls, no gap ++ * over 3.4 ms; latency101: 287 events/s through every stall of 0.55-33 s), so a stream that has ++ * been silent for G5P_RU_LAT_SILENT_MS is a dead firmware, not a slow one, at any point of the ++ * phase. Returns true then; the caller reports it as a fault. The silence clock starts at the ++ * publish (vpds rung vk, 2026-11-01: the counter froze 1 s after the TA's publish, and a clock ++ * that only ran from the soft bound called it at 45 s, with every other client held behind it). ++ * The caller drains the ring before each sample, so a host that slept longer than the window ++ * sees the events it missed and is not silent. Never true when latency101 is off. ++ */ ++static inline bool g5p_ru_wait_grace(struct g5p_ru_wait *w, u64 now_ns, ++ u32 events) ++{ ++ if (!w->soft_ns) ++ return false; ++ if (events != w->events) { ++ w->events = events; ++ w->event_ns = now_ns; ++ } ++ if (now_ns >= w->soft_ns && !w->soft_seen_ns) ++ w->soft_seen_ns = now_ns; ++ w->silent = now_ns - w->event_ns >= ++ (u64)G5P_RU_LAT_SILENT_MS * 1000000ULL; ++ return w->silent; ++} ++ ++/* One RAM sample taken at `now_ns`. Returns the result so far (PENDING until ++ * decided). A sample at or after the deadline only marks the wait expired unless ++ * it shows completion or a fault; the caller must then take one closing sample and ++ * call g5p_ru_wait_close(). ++ */ ++static inline u32 g5p_ru_wait_step(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (w->soft_ns && now_ns >= w->soft_ns) ++ w->slow = true; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ w->end_ns = now_ns; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->end_ns = now_ns; ++ w->late_soft = w->slow; ++ } else if (now_ns >= w->deadline_ns) { ++ w->expired = true; ++ } ++ return w->result; ++} ++ ++/* The closing sample, taken after the loop left because of the deadline. This is ++ * the only place a timeout is decided. ++ */ ++static inline u32 g5p_ru_wait_close(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->late_gap = true; ++ w->late_soft = w->slow; ++ } else { ++ w->result = G5P_RU_WAIT_TIMEOUT; ++ } ++ w->end_ns = now_ns; ++ return w->result; ++} ++ ++/* A sample after the result was decided: a completion after a timeout is recorded ++ * and nothing else happens (no 3D, no recovery, no register access). ++ */ ++static inline void g5p_ru_wait_after(struct g5p_ru_wait *w, bool complete) ++{ ++ if (w->result == G5P_RU_WAIT_TIMEOUT && complete) ++ w->late_after = true; ++} ++ ++/* latency101: the RAM-only observation window after a timeout was decided, before the receipt is ++ * printed and the terminal latch is taken. g5p_ru_wait_after() looks once, microseconds after the ++ * decision; a completion that arrives milliseconds later (prod98c seq 3: 5.49 ms) was never ++ * recorded, so `late_after=0/0` was a false negative. Returns true while the window is open. A ++ * retired sample ends it and is recorded in late_after/late_ns; it never changes the result (a ++ * deadline stays final). obs_ms = 0 closes it at once (the legacy behaviour). ++ */ ++static inline bool g5p_ru_wait_observe(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, u32 obs_ms) ++{ ++ if (w->result != G5P_RU_WAIT_TIMEOUT || w->late_after) ++ return false; ++ if (complete) { ++ w->late_after = true; ++ w->late_ns = now_ns; ++ return false; ++ } ++ return now_ns < w->end_ns + (u64)obs_ms * 1000000ULL; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +new file mode 100644 +index 000000000..64a51eaf4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -0,0 +1,273 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 publish-time MTS wake (wake110). Pure functions over plain values; the one ++ * register sequence goes through a callback table (the fender.h pattern), so the host ++ * suite compiles this file as it is and records every access in order. ++ * ++ * Question (docs/gpu/experiments/wake110-apple-mts-dispatch.md): is the 0.5-33 s TA/3D ++ * start latency caused by the missing Apple submit kick? Apple's TA and 3D submit ++ * wrappers (iPad6,11 kernelcache 0xfffffff006ca00a0 / 0xfffffff006ca01f8) do, every time: ++ * ring entry, dmb ish, Fender vt+0x80 (R1+0x14 = 7, read back), Fender vt+0x18 ++ * (R1+0x1000 & 0xf), then R0+0xb00 = 0x10 (TA, DM0|COUNTED) or 0x11 (3D, DM1|COUNTED). ++ * The MTS store is reached from every arm; when the GPC nibble is 0 Apple first runs a ++ * power arm (up to forcePowerCycleAndReboot). Linux V3 publishes job N >= 2's TA and ++ * every 3D with no kick at all. ++ * ++ * render_user_wake (latched per context): 0 off (default: no call has any effect), ++ * 1 the A/B, 2 every eligible phase is a factor (the candidate production fix, for use ++ * only after the A/B proved it). The TA of job 1 is never a phase of this file (the r74g ++ * grace and its fenced fallback bell own it). In the A/B each engine's phases are paired ++ * in publish order, (0,1), (2,3), ...; exactly one of each pair is the factor, which one ++ * is fixed by render_user_wake_seed before the context exists, and bit 31 of the seed ++ * inverts the whole schedule (the mirror stage). The verdict recomputes the arm of every ++ * receipt from the seed. ++ * ++ * At the publish, right after the producer store and its barrier, both arms take the same ++ * lock and read the same RAM state and run the same guards; the control stops there (no ++ * register). The factor then runs the Apple order: R1+0x14 = 7, read back (any bit ++ * outside 7: stop), R1+0x1000 (state & 0x1f != 1, the GPC is not active: stop, never ++ * repaired, never a power arm), dma_wmb(), R0+0xb00 = 0x10 | engine. At most one attempt ++ * per phase. Never after a timeout or fault (the terminal latch's sticky `failed`), never ++ * with the boot failed or stopped, never unpowered, never with an r74f bell in flight, ++ * never once the phase shows any sign of retirement, never on a ring that is not exactly ++ * this publish. A consumer already at the target does not cancel (Apple never checks ++ * it); it is recorded. The natural TA completion (terminal kind COMPLETE, the epoch's ++ * first note) is not a failure: the factor's sequence is the one licensed exception to the ++ * r96 "no register after the TA complete" envelope, default off. ++ */ ++#ifndef G5P_RENDER_USER_WAKE_H ++#define G5P_RENDER_USER_WAKE_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++#define G5P_WK_OFF 0U ++#define G5P_WK_AB 1U ++#define G5P_WK_ALWAYS 2U ++#define G5P_WK_MIRROR 0x80000000U /* seed bit 31: every arm inverted */ ++ ++#define G5P_WK_R1_FENDER 0x14U /* Fender vt+0x80: write 7, read back */ ++#define G5P_WK_R1_GPC 0x1000U /* GPC state: & 0x1f == 1 is active */ ++#define G5P_WK_R0_MTS 0xb00U /* MTS_SCHEDULE, write only */ ++#define G5P_WK_MTS_TA 0x10U /* DM0 | COUNTED */ ++#define G5P_WK_MTS_3D 0x11U /* DM1 | COUNTED */ ++ ++/* wake111: the publish-time power evidence, one bit per RAM fact, sampled under the event lock ++ * and printed per phase (power=TT/DD). Every required bit must be set. reset_transition is ++ * the permanent post-release slave-port lockout (set right before the release, never ++ * cleared), not a live reset state: Wake110 vetoed on it, so every phase skipped. It is ++ * recorded (SLAVE_LOCK) and is neither required nor a veto; the slave-port ban is unchanged. ++ */ ++#define G5P_WK_PWR_CORE 0x01U /* g->powered */ ++#define G5P_WK_PWR_FENDER 0x02U /* g->fender_ready */ ++#define G5P_WK_PWR_RELEASED 0x04U /* b->released */ ++#define G5P_WK_PWR_ALIVE 0x08U /* b->alive */ ++#define G5P_WK_PWR_CONTROL 0x10U /* b->control_complete */ ++#define G5P_WK_PWR_DS108 \ ++ 0x20U /* DS108 claim proof, current generation, no power abort */ ++#define G5P_WK_PWR_SLAVE_LOCK 0x40U /* g->reset_transition: evidence only */ ++#define G5P_WK_PWR_REQUIRED \ ++ (G5P_WK_PWR_CORE | G5P_WK_PWR_FENDER | G5P_WK_PWR_RELEASED | \ ++ G5P_WK_PWR_ALIVE | G5P_WK_PWR_CONTROL | G5P_WK_PWR_DS108) ++ ++enum g5p_wk_arm { ++ G5P_WK_ARM_NONE, /* wake off */ ++ G5P_WK_ARM_EXCLUDED, /* the TA of job 1 (r74g) */ ++ G5P_WK_ARM_CONTROL, ++ G5P_WK_ARM_FACTOR, ++ G5P_WK_ARMS, ++}; ++ ++enum g5p_wk_did { ++ G5P_WK_DID_NONE, /* off, or never reached */ ++ G5P_WK_DID_RANG, /* factor: fence, GPC active, 0xb00 written */ ++ G5P_WK_DID_SHAM, /* control: every guard passed, no register */ ++ G5P_WK_DID_EXCLUDED, ++ G5P_WK_SKIP_TWICE, /* this phase was already decided */ ++ G5P_WK_SKIP_LICENSE, /* a timeout or fault was noted (terminal failed) */ ++ G5P_WK_SKIP_FAULT, /* boot failed, event failed or stopped */ ++ G5P_WK_SKIP_POWER, /* one of G5P_WK_PWR_REQUIRED is absent */ ++ G5P_WK_SKIP_R74F, /* an r74f bell is published, open or firing */ ++ G5P_WK_SKIP_RING, /* the ring is not exactly this publish */ ++ G5P_WK_SKIP_RETIRED, /* a stamp == seq or a completion event since the publish */ ++ G5P_WK_SKIP_FENCE, /* R1+0x14 read back outside 7: no further register */ ++ G5P_WK_SKIP_GPC, /* GPC not active: no 0xb00 */ ++ G5P_WK_DIDS, ++}; ++ ++static const char *const g5p_wk_arm_names[G5P_WK_ARMS] = { ++ "none", ++ "excluded", ++ "control", ++ "factor", ++}; ++ ++static const char *const g5p_wk_did_names[G5P_WK_DIDS] = { ++ "none", "rang", "sham", "excluded", "skip-twice", ++ "skip-license", "skip-fault", "skip-power", "skip-r74f", "skip-ring", ++ "skip-retired", "skip-fence", "skip-gpc", ++}; ++ ++/* Everything the decision reads, sampled under the event lock right after the producer store. */ ++struct g5p_wk_in { ++ u32 mode, seed, seq, e; ++ bool attempted; /* this phase already went through g5p_wk_decide() */ ++ bool term_failed; /* g5p_ru_term_failed() */ ++ bool boot_failed; /* b->failed || b->event_failed || b->stopped */ ++ u32 power; /* g5p_wk_power_mask(); the slave lock is evidence, not a veto */ ++ bool r74f_pending; /* an r74f bell in PUBLISHED, OPEN or FIRING */ ++ u32 start, target, prod, ++ cons; /* ring: producer before this publish, our target, now */ ++ u32 stamp[2]; /* this engine's stamp words */ ++ u32 type1; /* this engine's completion events decoded since the phase armed */ ++}; ++ ++/* The register access of the factor, in order. The kernel binds them to writel/readl on the ++ * Fender window and the RGX window and to dma_wmb(); the host suite records them. ++ */ ++struct g5p_wk_io { ++ void *ctx; ++ void (*w1)(void *ctx, u32 reg, u32 value); ++ u32 (*r1)(void *ctx, u32 reg); ++ void (*wbarrier)(void *ctx); ++ void (*w0)(void *ctx, u32 reg, u32 value); ++}; ++ ++static inline bool g5p_wk_mode_valid(u32 mode) ++{ ++ return mode <= G5P_WK_ALWAYS; ++} ++ ++static inline u32 g5p_wk_mix(u32 x) ++{ ++ x ^= x >> 16; ++ x *= 0x7feb352dU; ++ x ^= x >> 15; ++ x *= 0x846ca68bU; ++ x ^= x >> 16; ++ return x; ++} ++ ++/* The phase's index among its engine's eligible phases: TA of job seq >= 2 -> seq - 2, ++ * 3D of job seq >= 1 -> seq - 1. ++ */ ++static inline u32 g5p_wk_index(u32 seq, u32 e) ++{ ++ return e ? seq - 1 : seq - 2; ++} ++ ++static inline u32 g5p_wk_arm(u32 mode, u32 seed, u32 seq, u32 e) ++{ ++ u32 i, flip; ++ ++ if (!mode || !g5p_wk_mode_valid(mode) || !seq || e > 1) ++ return G5P_WK_ARM_NONE; ++ if (!e && seq == 1) ++ return G5P_WK_ARM_EXCLUDED; ++ if (mode == G5P_WK_ALWAYS) ++ return G5P_WK_ARM_FACTOR; ++ i = g5p_wk_index(seq, e); ++ flip = g5p_wk_mix(g5p_wk_mix(seed & ~G5P_WK_MIRROR) + (e << 24) + ++ (i >> 1)) & ++ 1; ++ flip ^= !!(seed & G5P_WK_MIRROR); ++ return ((i & 1) ^ flip) ? G5P_WK_ARM_FACTOR : G5P_WK_ARM_CONTROL; ++} ++ ++static inline u32 g5p_wk_mts(u32 e) ++{ ++ return G5P_WK_MTS_TA | e; ++} ++ ++/* wake111: the evidence word, one bit per fact (the caller samples each fact once). */ ++static inline u32 g5p_wk_power_mask(bool powered, bool fender_ready, ++ bool released, bool alive, ++ bool control_complete, bool ds108, ++ bool slave_lock) ++{ ++ return (powered ? G5P_WK_PWR_CORE : 0) | ++ (fender_ready ? G5P_WK_PWR_FENDER : 0) | ++ (released ? G5P_WK_PWR_RELEASED : 0) | ++ (alive ? G5P_WK_PWR_ALIVE : 0) | ++ (control_complete ? G5P_WK_PWR_CONTROL : 0) | ++ (ds108 ? G5P_WK_PWR_DS108 : 0) | ++ (slave_lock ? G5P_WK_PWR_SLAVE_LOCK : 0); ++} ++ ++static inline bool g5p_wk_power_ready(u32 power) ++{ ++ return (power & G5P_WK_PWR_REQUIRED) == G5P_WK_PWR_REQUIRED; ++} ++ ++/* The RAM guards, common to both arms, in priority order. 0 = pass. */ ++static inline u32 g5p_wk_guard(const struct g5p_wk_in *in) ++{ ++ if (in->attempted) ++ return G5P_WK_SKIP_TWICE; ++ if (in->term_failed) ++ return G5P_WK_SKIP_LICENSE; ++ if (in->boot_failed) ++ return G5P_WK_SKIP_FAULT; ++ if (!g5p_wk_power_ready(in->power)) ++ return G5P_WK_SKIP_POWER; ++ if (in->r74f_pending) ++ return G5P_WK_SKIP_R74F; ++ if (in->start > 255 || in->target > 255 || in->prod != in->target || ++ in->cons > 255 || ((in->target - in->start) & 255) != 1 || ++ ((in->cons - in->start) & 255) > 1) ++ return G5P_WK_SKIP_RING; ++ if (in->stamp[0] == in->seq || in->stamp[1] == in->seq || in->type1) ++ return G5P_WK_SKIP_RETIRED; ++ return 0; ++} ++ ++/* The factor's register sequence (Apple's order). The caller set *fence and *gpc to U32_MAX ++ * (not read). ++ */ ++static inline u32 g5p_wk_ring(const struct g5p_wk_io *io, u32 e, u32 *fence, ++ u32 *gpc) ++{ ++ io->w1(io->ctx, G5P_WK_R1_FENDER, 7); ++ *fence = io->r1(io->ctx, G5P_WK_R1_FENDER); ++ if (*fence & ~7U) ++ return G5P_WK_SKIP_FENCE; ++ *gpc = io->r1(io->ctx, G5P_WK_R1_GPC); ++ if (!G5P_GPC_ON(*gpc)) ++ return G5P_WK_SKIP_GPC; ++ /* the publish's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ io->wbarrier(io->ctx); ++ io->w0(io->ctx, G5P_WK_R0_MTS, g5p_wk_mts(e)); ++ return G5P_WK_DID_RANG; ++} ++ ++/* One phase's decision and, for a factor that passed every guard, its register sequence ++ * (the control arm never touches io). ++ */ ++static inline u32 g5p_wk_decide(const struct g5p_wk_in *in, ++ const struct g5p_wk_io *io, u32 *fence, ++ u32 *gpc) ++{ ++ u32 arm = g5p_wk_arm(in->mode, in->seed, in->seq, in->e), skip; ++ ++ *fence = U32_MAX; ++ *gpc = U32_MAX; ++ if (arm == G5P_WK_ARM_NONE) ++ return G5P_WK_DID_NONE; ++ if (arm == G5P_WK_ARM_EXCLUDED) ++ return G5P_WK_DID_EXCLUDED; ++ skip = g5p_wk_guard(in); ++ if (skip) ++ return skip; ++ if (arm == G5P_WK_ARM_CONTROL) ++ return G5P_WK_DID_SHAM; ++ return g5p_wk_ring(io, in->e, fence, gpc); ++} ++ ++/* A decided phase never decides again (the caller stores this before it unlocks). */ ++static inline bool g5p_wk_attempted(u32 did) ++{ ++ return did != G5P_WK_DID_NONE && did != G5P_WK_DID_EXCLUDED; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-window.h b/drivers/gpu/apple-g5p/render-user-window.h +new file mode 100644 +index 000000000..bfbae628d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-window.h +@@ -0,0 +1,74 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The firmware window of a render VM (docs/gpu/multi-client-design.md §6 R4, stage 3). ++ * ++ * The firmware's own GPU memory lives in one GiB, catalogue entry G5P_RU_FIRMWARE_CI: its ++ * image, the root and shared pages (rings, the PA/tag arrays, W), the MMU table alias and the ++ * PIO pages (Fender, AIC). Stage 1 copied that catalogue entry into every render VM, so any ++ * render job could address all of it. A render VM now gets, under that entry, a directory ++ * the kernel built that maps only the pages the GPU's own requesters reach through the ++ * context's slot: the PM control page (S) and the PM list and free list (F). Their leaf ++ * entries are copied from the firmware's tables, so their translation is unchanged. ++ * ++ * render_user_fw_window = 0 (G5P_RU_FW_FULL) the stage 1 mapping: the whole firmware GiB; ++ * render contexts need CAP_SYS_RAWIO ++ * 1 (G5P_RU_FW_WINDOW) the window above (default) ++ * 2 (G5P_RU_FW_NONE) no firmware page at all (an experiment) ++ * ++ * Pure helpers, host-tested (scripts/gpu/multiclient2-host/test_render_window.c). The ++ * caller provides g5p_le32() and the boot layout constants. ++ */ ++#ifndef G5P_RENDER_USER_WINDOW_H ++#define G5P_RENDER_USER_WINDOW_H ++ ++#define G5P_RU_FW_FULL 0U ++#define G5P_RU_FW_WINDOW 1U ++#define G5P_RU_FW_NONE 2U ++#define G5P_RU_FW_MODES 3U ++ ++#define G5P_RU_FW_PAGES 3U ++/* The pages of the window, all in the firmware GiB. */ ++static const u64 g5p_ru_fw_window_va[G5P_RU_FW_PAGES] = { ++ G5P_SHARED_VA + G5P_R22_PM, /* PM control and S */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE, /* PM list */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, /* PM free list F */ ++}; ++ ++static inline u32 g5p_ru_fw_di(u64 va) ++{ ++ return (va >> 21) & 511; ++} ++ ++static inline u32 g5p_ru_fw_ti(u64 va) ++{ ++ return (va >> 14) & 127; ++} ++ ++/* The firmware's 64-bit leaf entry for `va`, from its leaf pages and their directory ++ * indices; 0 when the firmware does not map it. ++ */ ++static inline u64 g5p_ru_fw_leaf(u8 *const *leaf, const u32 *leaf_di, ++ u32 leaves, u64 va) ++{ ++ for (u32 i = 0; i < leaves; i++) { ++ const u8 *e = leaf[i] + g5p_ru_fw_ti(va) * 8; ++ ++ if (leaf_di[i] == g5p_ru_fw_di(va)) ++ return (u64)g5p_le32(e + 4) << 32 | g5p_le32(e); ++ } ++ return 0; ++} ++ ++/* The window's leaf page index for `va` among `n` distinct directory indices so far, or n ++ * (a new one) when none has its directory index. ++ */ ++static inline u32 g5p_ru_fw_slot(const u32 *di, u32 n, u64 va) ++{ ++ u32 i; ++ ++ for (i = 0; i < n && di[i] != g5p_ru_fw_di(va); i++) ++ ; ++ return i; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-variants-r25.h b/drivers/gpu/apple-g5p/render-variants-r25.h +new file mode 100644 +index 000000000..8469a2f4b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-variants-r25.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_VARIANTS_R25_H ++#define G5P_RENDER_VARIANTS_R25_H ++ ++/* Finite, owned-input experiments. No raw W, register or firmware-patch API. */ ++#define G5P_R25_MAX_BLOCKS 128U ++#define G5P_R25_HEAD 0x3000U ++ ++struct g5p_r25_variants { ++ u32 pm, blocks, context, host, sku, memory, rt, slots; ++}; ++ ++static inline bool g5p_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return v->pm <= 3 && ++ (v->blocks == 0 || v->blocks == 64 || v->blocks == 128) && ++ v->context <= 3 && v->host <= 2 && v->sku <= 3 && ++ v->memory <= 4 && v->rt <= 2 && v->slots <= 1; ++} ++ ++/* Event synthesis, removed context tasks, candidate clocks and the low-address ++ * dialect test cannot qualify rendering, even if firmware stamps retire. ++ */ ++static inline bool g5p_r25_diagnostic(const struct g5p_r25_variants *v) ++{ ++ return v->sku || v->context || v->host || v->rt == 2; ++} ++ ++/* PM payload frames are physical PPNs and have no user PTE attributes. This ++ * changes only PM auxiliary mappings (and, in mode4, other private mappings). ++ * Firmware/shared META mappings and the borrowed sink are never rewritten. ++ */ ++static inline u32 g5p_r25_attributes(const struct g5p_r25_variants *v, u64 va) ++{ ++ if (!v->memory) ++ return 0xc; ++ if (v->memory != 4 && ++ (va < G5P_R22_AUX || va >= G5P_R22_AUX + 5 * G5P_GPU_PAGE)) ++ return 0xc; ++ return v->memory == 2 ? 4 : v->memory == 3 ? 8 : 0; ++} ++ ++/* Preserve the original constructor exactly for the default pool. The extra ++ * blocks fit the existing 16KiB list and free allocations, including F's last ++ * dword at128 blocks. Validate every PA before mutating either allocation. ++ */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/retire108.h b/drivers/gpu/apple-g5p/retire108.h +new file mode 100644 +index 000000000..e944b92c9 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/retire108.h +@@ -0,0 +1,434 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RETIRE108_H ++#define G5P_RETIRE108_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P outgoing retire layer (series gpu-recovery108, patch 0006). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 3. ++ * ++ * Wraps the recovery98 teardown ladder (teardown.h, g5p_td_run()) without ++ * changing it: the TD98 lines of a run are byte for byte the ones of the ++ * same run without this layer. It adds, around a run: ++ * ++ * ADMIT atomically close job admission (one cmpxchg word), then a ++ * bounded RAM-only wait for submitters already inside the ++ * admission window; still inside at the deadline = explicit ++ * ADMIT_OPEN flag, never a block. ++ * CLASSIFY the epoch class from RAM inputs only (struct g5p_r8_in). ++ * RECEIPT one TR108 line with the class and the generation, printed ++ * before the ladder makes its first AIC or R1 access. ++ * (ladder) g5p_td_run(FINAL) through a callback, unchanged. ++ * SLOTS three named post-ladder slots. Each runs only with a glue op ++ * present AND its own parameter on; every skip is named: ++ * fw-sleep validated firmware sleep before stable-off (G2: ++ * no recovered command; the glue has no op) ++ * stable-off one R1 read of isStableOff after a verified ++ * Fender stop leg (G6: default off) ++ * pmgr-gate outgoing PMGR power gate (G1: not implemented, ++ * there is no op and no parameter) ++ * RETIRE the retired latch: from here on g5p_r8_mmio_ok() refuses R0, ++ * R1 and every write except the always-on AIC ack of a late ++ * handler entry. The latch is set whatever the verdict. ++ * ++ * Generations: gen counts real runs (CAPTURE and FINAL) since boot and is ++ * monotonic; a REENTRY (a second FINAL after retire, e.g. notifier then ++ * .shutdown then remove) prints a line but does not advance it. ++ * ++ * Pure policy, like teardown.h: no register access, no kernel API except ++ * cmpxchg(), READ_ONCE()/WRITE_ONCE(), scnprintf() and the errno names. The ++ * same file is compiled by the kernel glue (teardown.inc) and by the offline ++ * tests (patches/gpu-recovery108/tests). Needs teardown.h first. ++ */ ++ ++#define G5P_R8_CLOSED 0x80000000u /* admission word: closed bit */ ++#define G5P_R8_INSIDE 0x7fffffffu /* admission word: submitters inside */ ++#define G5P_R8_POLL_US 100 ++#define G5P_R8_MAX_STEPS 512 /* ADMIT wait cap, independent of the clock */ ++#define G5P_R8_ADMIT_US 20000 /* ADMIT wait budget */ ++ ++/* Admission: one word, closed bit + count of submitters inside the window. */ ++struct g5p_r8_admit { ++ u32 word; ++}; ++ ++/* A submitter enters before its first check of driver state and leaves on ++ * every return path. False once admission is closed: the submit must fail ++ * with -ESHUTDOWN and touch nothing. ++ */ ++static inline bool g5p_r8_admit_enter(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if ((cur & G5P_R8_CLOSED) || ++ (cur & G5P_R8_INSIDE) == G5P_R8_INSIDE) ++ return false; ++ old = cmpxchg(&a->word, cur, cur + 1); ++ if (old == cur) ++ return true; ++ cur = old; ++ } ++} ++ ++static inline void g5p_r8_admit_exit(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if (!(cur & G5P_R8_INSIDE)) ++ return; /* unbalanced exit: never wrap into the closed bit */ ++ old = cmpxchg(&a->word, cur, cur - 1); ++ if (old == cur) ++ return; ++ cur = old; ++ } ++} ++ ++/* Close (idempotent). Returns the submitters inside at the instant of closing. */ ++static inline u32 g5p_r8_admit_close(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ old = cmpxchg(&a->word, cur, cur | G5P_R8_CLOSED); ++ if (old == cur) ++ return cur & G5P_R8_INSIDE; ++ cur = old; ++ } ++} ++ ++static inline u32 g5p_r8_admit_inside(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_INSIDE; ++} ++ ++static inline bool g5p_r8_admit_closed(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_CLOSED; ++} ++ ++/* Epoch classes. The dirty ones are never reported as clean (H10). */ ++enum g5p_r8_class { ++ G5P_R8C_NEVER_BOOTED, /* shared pages never mapped */ ++ G5P_R8C_UNRELEASED, /* mapped, firmware never released */ ++ G5P_R8C_IDLE, /* released and ALIVE, no context ever claimed (not-run) */ ++ G5P_R8C_CLEAN, /* claimed, every job signalled, no holder left */ ++ G5P_R8C_HELD, /* clean, but a process still holds the claimed context */ ++ G5P_R8C_PARKED, /* GPC=0 already verified by a park */ ++ G5P_R8C_IN_FLIGHT, /* a job accepted and not signalled, or a worker inside a wait */ ++ G5P_R8C_TIMEOUT, /* terminal latch = timeout (a late completion stays a timeout) */ ++ G5P_R8C_FAULT, /* failed, event failure or a fault terminal */ ++ G5P_R8C_ABORTED, /* bootstrap power abort latched (the r87b/C3 epoch) */ ++ G5P_R8C_UNKNOWN, /* released, never ALIVE; or an unverified earlier stop */ ++ G5P_R8C_N, ++}; ++ ++static const char *const g5p_r8_class_name[G5P_R8C_N] = { ++ "never-booted", "unreleased", "idle", "clean", "held", "parked", ++ "in-flight", "timeout", "fault", "aborted", "unknown", ++}; ++ ++static inline bool g5p_r8_class_dirty(enum g5p_r8_class c) ++{ ++ return c >= G5P_R8C_IN_FLIGHT; ++} ++ ++/* RAM-only inputs, read by the glue before the run (never from a register). */ ++struct g5p_r8_in { ++ bool booted; /* b->maps_ready */ ++ bool released; /* b->released */ ++ bool alive; /* b->alive before FREEZE, or ALIVE was proven this boot */ ++ bool failed; /* b->failed */ ++ bool fault_event; /* b->event_failed */ ++ int terminal; /* teardown terminal latch: 0 none, 1 complete, 2 timeout/fault */ ++ bool claimed; /* a context claimed the device (DRM quarantined_context) */ ++ bool held; /* ... and a render node file still references it */ ++ u32 queued; /* accepted render jobs not yet signalled */ ++ bool worker_busy; /* the command lock was busy: a worker may be inside a wait */ ++ bool parked; /* b->parked */ ++ bool stopped_unparked; /* b->stopped && !b->parked before this run */ ++ bool power_abort; /* r73b early_power_abort */ ++}; ++ ++static inline u32 g5p_r8_in_bits(const struct g5p_r8_in *i) ++{ ++ return i->booted | i->released << 1 | i->alive << 2 | i->failed << 3 | ++ i->fault_event << 4 | i->claimed << 5 | i->held << 6 | ++ (i->queued != 0) << 7 | i->worker_busy << 8 | i->parked << 9 | ++ i->stopped_unparked << 10 | i->power_abort << 11; ++} ++ ++/* First match wins; the order is the precedence (docs section 3.2). */ ++static inline enum g5p_r8_class g5p_r8_classify(const struct g5p_r8_in *i) ++{ ++ if (!i->booted) ++ return G5P_R8C_NEVER_BOOTED; ++ if (!i->released) ++ return G5P_R8C_UNRELEASED; ++ if (i->stopped_unparked) ++ return G5P_R8C_UNKNOWN; ++ if (i->power_abort) ++ return G5P_R8C_ABORTED; ++ if (i->failed || i->fault_event) ++ return G5P_R8C_FAULT; ++ if (i->terminal == 2) ++ return G5P_R8C_TIMEOUT; ++ if (i->queued || i->worker_busy) ++ return G5P_R8C_IN_FLIGHT; ++ if (i->parked) ++ return G5P_R8C_PARKED; ++ if (!i->alive) ++ return G5P_R8C_UNKNOWN; ++ if (i->claimed) ++ return i->held ? G5P_R8C_HELD : G5P_R8C_CLEAN; ++ return G5P_R8C_IDLE; ++} ++ ++enum g5p_r8_slot { ++ G5P_R8S_FW_SLEEP, ++ G5P_R8S_STABLE_OFF, ++ G5P_R8S_PMGR_GATE, ++ G5P_R8S_N, ++}; ++ ++enum g5p_r8_skip { ++ G5P_R8K_RAN, /* the slot ran (see its result) */ ++ G5P_R8K_NO_MECHANISM, /* the glue has no op for it */ ++ G5P_R8K_OFF, /* its parameter is off (default) */ ++ G5P_R8K_NOT_FINAL, /* CAPTURE never runs a slot */ ++ G5P_R8K_NEEDS_RT1, /* needs a verified Fender stop leg in this run */ ++ G5P_R8K_DIRTY, /* the ladder ended dirty: no register access */ ++ G5P_R8K_N, ++}; ++ ++static const char *const g5p_r8_skip_name[G5P_R8K_N] = { ++ "ran", "no-mechanism", "off", "not-final", "needs-rt1", "dirty", ++}; ++ ++#define G5P_R8F_ADMIT_OPEN \ ++ BIT(0) /* submitters still inside at the ADMIT deadline */ ++#define G5P_R8F_ADMIT_LATE BIT(1) /* ... or the step cap ended the wait */ ++#define G5P_R8F_REENTRY BIT(2) /* answered after retire: nothing ran */ ++#define G5P_R8F_BUSY BIT(3) /* a concurrent entrant was inside */ ++#define G5P_R8F_SLOT_FAIL BIT(4) /* a slot ran and failed */ ++#define G5P_R8F_STABLE_OFF BIT(5) /* stable-off slot read isStableOff == 1 */ ++/* jobs still queued at DETACH (abandoned to their worker) */ ++#define G5P_R8F_DETACH_OPEN BIT(6) ++ ++/* ++ * Ops. td_run runs the recovery98 ladder and returns its rc; td_flags/td_verdict ++ * read its result. slot[] may hold NULL (no mechanism). Only slot ops touch a ++ * register, and only after the ladder, never after RETIRE. ++ */ ++struct g5p_r8_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); ++ int (*td_run)(void *ctx, enum g5p_td_mode mode); ++ u32 (*td_flags)(void *ctx); ++ enum g5p_td_verdict (*td_verdict)(void *ctx); ++ int (*slot[G5P_R8S_N])(void *ctx, u32 *value); ++ void (*emit)(void *ctx, const char *line); ++}; ++ ++struct g5p_r8 { ++ struct g5p_r8_admit admit; ++ u32 admit_us; /* ADMIT budget */ ++ bool slot_on[G5P_R8S_N]; /* the slots' parameters (default all off) */ ++ /* latches */ ++ int running; ++ int retired; /* 0, or the generation that retired the device */ ++ u32 gen; /* monotonic: real runs since boot */ ++ /* last run */ ++ enum g5p_td_mode mode; ++ enum g5p_r8_class cls; ++ u32 in_bits, flags, inside0, inside1, steps, detach_queued; ++ enum g5p_r8_skip skip[G5P_R8S_N]; ++ int slot_rc[G5P_R8S_N]; ++ u32 slot_val[G5P_R8S_N]; ++ int rc; ++ enum g5p_td_verdict verdict; ++ u32 lines; /* TR108 lines emitted since boot */ ++}; ++ ++static inline void g5p_r8_init(struct g5p_r8 *s) ++{ ++ *s = (struct g5p_r8){ .admit_us = G5P_R8_ADMIT_US }; ++} ++ ++/* ++ * The predicate for every register access outside the ladder: after RETIRE only ++ * RAM, a PMGR read and the AIC (the always-on swirq ack of a late handler entry) ++ * remain. R0 and R1 are refused forever, whatever the verdict. ++ */ ++static inline bool g5p_r8_mmio_ok(const struct g5p_r8 *s, enum g5p_td_dom d) ++{ ++ if (d == G5P_TDD_RAM || d == G5P_TDD_PMGR || d == G5P_TDD_AIC) ++ return true; ++ return !READ_ONCE(s->retired); ++} ++ ++static inline bool g5p_r8_retired(const struct g5p_r8 *s) ++{ ++ return READ_ONCE(s->retired) != 0; ++} ++ ++/* The DRM frontend's DETACH callback reports what it left to the job workers. */ ++static inline void g5p_r8_note_detach(struct g5p_r8 *s, u32 queued) ++{ ++ WRITE_ONCE(s->detach_queued, queued); ++ if (queued) ++ s->flags |= G5P_R8F_DETACH_OPEN; ++} ++ ++static inline void g5p_r8_line(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ const char *line) ++{ ++ s->lines++; ++ if (o->emit) ++ o->emit(o->ctx, line); ++} ++ ++/* Bounded RAM-only wait for submitters inside the admission window. */ ++static inline void g5p_r8_admit_wait(struct g5p_r8 *s, ++ const struct g5p_r8_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)s->admit_us * 1000; ++ ++ s->inside0 = g5p_r8_admit_close(&s->admit); ++ s->inside1 = s->inside0; ++ s->steps = 0; ++ while (s->inside1) { ++ if (s->steps >= G5P_R8_MAX_STEPS) { ++ s->flags |= G5P_R8F_ADMIT_LATE; ++ break; ++ } ++ s->steps++; ++ if (o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_R8_POLL_US); ++ s->inside1 = g5p_r8_admit_inside(&s->admit); ++ } ++ if (s->inside1) ++ s->flags |= G5P_R8F_ADMIT_OPEN; ++} ++ ++static inline void g5p_r8_slots(struct g5p_r8 *s, const struct g5p_r8_ops *o) ++{ ++ u32 tdf = o->td_flags ? o->td_flags(o->ctx) : 0; ++ unsigned int i; ++ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ s->slot_rc[i] = 0; ++ s->slot_val[i] = 0; ++ if (s->mode != G5P_TD_FINAL) ++ s->skip[i] = G5P_R8K_NOT_FINAL; ++ else if (!o->slot[i]) ++ s->skip[i] = G5P_R8K_NO_MECHANISM; ++ else if (!s->slot_on[i]) ++ s->skip[i] = G5P_R8K_OFF; ++ else if (s->verdict == G5P_TDV_DIRTY) ++ s->skip[i] = G5P_R8K_DIRTY; ++ else if (!(tdf & G5P_TDF_RT1_OK)) ++ s->skip[i] = G5P_R8K_NEEDS_RT1; ++ else ++ s->skip[i] = G5P_R8K_RAN; ++ } ++ /* One attempt each, in slot order, no retry; a failed slot ends the slots. */ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ if (s->skip[i] != G5P_R8K_RAN) ++ continue; ++ if (s->flags & G5P_R8F_SLOT_FAIL) { ++ s->skip[i] = G5P_R8K_DIRTY; ++ continue; ++ } ++ s->slot_rc[i] = o->slot[i](o->ctx, &s->slot_val[i]); ++ if (s->slot_rc[i]) ++ s->flags |= G5P_R8F_SLOT_FAIL; ++ else if (i == G5P_R8S_STABLE_OFF && s->slot_val[i] == 1) ++ s->flags |= G5P_R8F_STABLE_OFF; ++ } ++} ++ ++/* ++ * One run. mode is the teardown mode the glue chose (teardown=1 CAPTURE, ++ * teardown=2 FINAL). Returns the ladder's rc, -EBUSY for a concurrent entrant, ++ * and the cached rc after retire. Lines: ++ * TR108 gen= phase=classify why= mode= class= dirty= in= inside= after= flags= admit_us= ++ * (TD98 lines of the ladder, unchanged) ++ * TR108 gen= phase=retired|captured verdict= class= flags= fw-sleep= stable-off= pmgr-gate= ++ * so= detach_queued= rc= ++ * TR108 gen= phase=reentry why= retired_gen= rc= (after retire; no gen step) ++ */ ++static inline int g5p_r8_run(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ enum g5p_td_mode mode, const struct g5p_r8_in *in, ++ const char *why) ++{ ++ char line[256]; ++ u64 t0; ++ int rc; ++ ++ if (cmpxchg(&s->running, 0, 1)) { ++ /* A concurrent entrant touches nothing; it does not reach the ladder either. */ ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=busy why=%s mode=%s rc=%d", ++ READ_ONCE(s->gen), why, g5p_td_mode_name[mode], ++ -EBUSY); ++ if (o->emit) ++ o->emit(o->ctx, line); ++ return -EBUSY; ++ } ++ if (READ_ONCE(s->retired)) { ++ /* The ladder answers from its cache too (REENTRY); nothing else runs. */ ++ rc = o->td_run(o->ctx, mode); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=reentry why=%s mode=%s retired_gen=%d rc=%d", ++ s->gen, why, g5p_td_mode_name[mode], ++ READ_ONCE(s->retired), rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++ } ++ s->gen++; ++ s->mode = mode; ++ s->flags = 0; ++ s->inside0 = 0; ++ s->inside1 = 0; ++ s->steps = 0; ++ WRITE_ONCE(s->detach_queued, 0); ++ t0 = o->now_ns(o->ctx); ++ if (mode == G5P_TD_FINAL) ++ g5p_r8_admit_wait(s, o); ++ s->cls = g5p_r8_classify(in); ++ s->in_bits = g5p_r8_in_bits(in); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=classify why=%s mode=%s class=%s dirty=%u in=%03x term=%d queued=%u inside=%u after=%u flags=%02x admit_us=%u", ++ s->gen, why, g5p_td_mode_name[mode], g5p_r8_class_name[s->cls], ++ g5p_r8_class_dirty(s->cls), s->in_bits, in->terminal, ++ in->queued, s->inside0, s->inside1, s->flags, ++ (u32)((o->now_ns(o->ctx) - t0) / 1000)); ++ /* The receipt is out before the ladder's first AIC or R1 access. */ ++ g5p_r8_line(s, o, line); ++ s->rc = o->td_run(o->ctx, mode); ++ s->verdict = o->td_verdict ? o->td_verdict(o->ctx) : G5P_TDV_NONE; ++ g5p_r8_slots(s, o); ++ if (mode == G5P_TD_FINAL) ++ WRITE_ONCE(s->retired, (int)s->gen); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=%s verdict=%s class=%s flags=%02x fw-sleep=%s stable-off=%s pmgr-gate=%s so=%u detach_queued=%u rc=%d", ++ s->gen, mode == G5P_TD_FINAL ? "retired" : "captured", ++ g5p_td_verdict_name[s->verdict], g5p_r8_class_name[s->cls], ++ s->flags, g5p_r8_skip_name[s->skip[G5P_R8S_FW_SLEEP]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_STABLE_OFF]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_PMGR_GATE]], ++ s->slot_val[G5P_R8S_STABLE_OFF], READ_ONCE(s->detach_queued), ++ s->rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.h b/drivers/gpu/apple-g5p/teardown.h +new file mode 100644 +index 000000000..b536f428c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.h +@@ -0,0 +1,740 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_TEARDOWN_H ++#define G5P_TEARDOWN_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P GPU teardown state machine (series gpu-recovery98). ++ * ++ * Design and evidence: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md. ++ * ++ * Pure policy: no register access, no kernel API except cmpxchg(), scnprintf() ++ * and the errno names. Every hardware or firmware-visible access goes through ++ * struct g5p_td_ops, and every call is checked against g5p_td_allow[phase][domain] ++ * before it is made. The same file is compiled by the kernel glue (teardown.inc) ++ * and by the offline fault-injection tests (patches/gpu-recovery/tests). ++ * ++ * Two modes: ++ * CAPTURE timeout / fault / attach failure, and (gpu-recovery99) any number of looks at a ++ * live epoch. Observation only: reads RAM rings, ++ * changes nothing the host could still want to export. Never writes ++ * driver or firmware-shared state, never masks the IRQ, never touches ++ * a register. Arms the quiet latch for diagnostics (g5p_td_mmio_ok()). ++ * FINAL shutdown / reboot notifier / remove. Runs the whole ladder and leaves ++ * a known state for the next kernel. ++ * ++ * Domains: RAM (DMA pages + driver structs), AIC (swirq 0x20e104000 ack and the ++ * IRQ-chip mask), R1 (Fender wrapper window), R0 (core register window), PMGR ++ * (0x20e080340 read-only). R0 has no op at all and is forbidden in every phase. ++ * PMGR is read-only in every phase. R1 is reachable only in the RESET phase. ++ * ++ * Nothing here frees or reuses memory the GPU might still access. ++ */ ++ ++#define G5P_TD_NCMD 4 /* command rings: shared objects 0x60 0x64 0x68 0x6c */ ++#define G5P_TD_NRING (G5P_TD_NCMD + 1) /* + event ring (object 0x70) */ ++#define G5P_TD_MAX_RCPT 12 ++#define G5P_TD_MAX_STEPS 4096 /* hard stop independent of the clock */ ++#define G5P_TD_POLL_US 100 ++#define G5P_TD_EV_SLACK \ ++ 16 /* events produced during the phase above this = storm */ ++ ++enum g5p_td_mode { ++ G5P_TD_CAPTURE = 1, ++ G5P_TD_FINAL = 2, ++}; ++ ++enum g5p_td_phase { ++ G5P_TDP_FREEZE, /* stopped=1 alive=0, watchdog disarmed (RAM) */ ++ G5P_TDP_ACCOUNT, /* RAM-only ring/event accounting */ ++ G5P_TDP_DRAIN, /* bounded RAM-only wait for the firmware to self-drain */ ++ G5P_TDP_EVENTS, /* bounded RAM-only event-ring quiesce */ ++ G5P_TDP_IRQ, /* AIC ack, IRQ mask, settle */ ++ G5P_TDP_DETACH, /* host detach, RAM only */ ++ G5P_TDP_RESET, /* RT1 Fender stop leg when eligible; PMGR read */ ++ G5P_TDP_DONE, ++ G5P_TDP_N, ++}; ++ ++enum g5p_td_dom { ++ G5P_TDD_RAM, ++ G5P_TDD_AIC, ++ G5P_TDD_R1, ++ G5P_TDD_R0, ++ G5P_TDD_PMGR, ++ G5P_TDD_N, ++}; ++ ++#define G5P_TDA_R 1 ++#define G5P_TDA_W 2 ++#define G5P_TDA_RW 3 ++ ++/* Hard stop boundaries. A phase may only use the domains marked here. */ ++static const unsigned char g5p_td_allow[G5P_TDP_N][G5P_TDD_N] = { ++ /* RAM AIC R1 R0 PMGR */ ++ [G5P_TDP_FREEZE] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_ACCOUNT] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_DRAIN] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_EVENTS] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_IRQ] = { G5P_TDA_R, G5P_TDA_W, 0, 0, 0 }, ++ [G5P_TDP_DETACH] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_RESET] = { G5P_TDA_R, 0, G5P_TDA_RW, 0, G5P_TDA_R }, ++ [G5P_TDP_DONE] = { G5P_TDA_R, 0, 0, 0, 0 }, ++}; ++ ++#define G5P_TDF_NOT_BOOTED \ ++ BIT(0) /* shared pages not mapped: nothing to tear down */ ++#define G5P_TDF_UNRELEASED BIT(1) /* firmware never released */ ++#define G5P_TDF_PROTOCOL \ ++ BIT(2) /* ring index > 255, read error, producer moved */ ++#define G5P_TDF_TERMINAL BIT(3) /* TA complete/timeout latched before entry */ ++#define G5P_TDF_LATE_TERMINAL BIT(4) /* ... or during the teardown (race) */ ++#define G5P_TDF_ABANDON BIT(5) /* >=1 ring pending after the drain */ ++#define G5P_TDF_SELF_DRAINED \ ++ BIT(6) /* >=1 ring consumed by the firmware, no bell */ ++#define G5P_TDF_FAULT_EVENT \ ++ BIT(7) /* event type other than 0/1 left unconsumed */ ++#define G5P_TDF_EV_UNQUIET \ ++ BIT(8) /* event ring still non-empty after the budget */ ++#define G5P_TDF_IRQ_UNSETTLED BIT(9) /* IRQ count still moving after the mask */ ++#define G5P_TDF_RT1_OK BIT(10) /* Fender stop leg verified GPC=0 */ ++#define G5P_TDF_RT1_FAIL BIT(11) /* stop leg attempted, not verified */ ++#define G5P_TDF_RT1_SKIPPED BIT(12) /* no register access; see skip reason */ ++#define G5P_TDF_OVERRUN BIT(13) /* step or time budget exhausted */ ++#define G5P_TDF_VIOLATION BIT(14) /* a gate refused an access (machine bug) */ ++#define G5P_TDF_CAPTURE_ONLY BIT(15) /* CAPTURE run: nothing was changed */ ++#define G5P_TDF_REENTRY BIT(16) /* second entry answered from the cache */ ++ ++enum g5p_td_skip { ++ G5P_TDS_NONE, ++ G5P_TDS_NOT_BOOTED, ++ G5P_TDS_NOT_RELEASED, ++ G5P_TDS_NOT_POWERED, ++ G5P_TDS_ALREADY_PARKED, /* clean: GPC=0 already verified by park */ ++ G5P_TDS_STOPPED_UNPARKED, /* dirty: an earlier stop leg did not verify */ ++ G5P_TDS_POWER_ABORT, ++ G5P_TDS_FW_POLICY, /* meta+0x6c8 != 0: firmware also writes R1+0x1000 */ ++ G5P_TDS_LOCK_BUSY, ++ G5P_TDS_CAPTURE_MODE, ++ G5P_TDS_BAD_STATE, /* protocol/overrun/unsettled IRQ: no register access */ ++ G5P_TDS_N, ++}; ++ ++enum g5p_td_verdict { ++ G5P_TDV_NONE, ++ G5P_TDV_OBSERVED, /* CAPTURE: nothing changed */ ++ G5P_TDV_UNTOUCHED, /* never booted or never released: nothing to do */ ++ G5P_TDV_QUIESCED, /* rings clear, events clear, IRQ masked, GPC=0 verified */ ++ G5P_TDV_ABANDONED, /* as QUIESCED, but >=1 command abandoned after the drain */ ++ G5P_TDV_RAM_ONLY, /* IRQ masked, no register access: GPU left as found */ ++ G5P_TDV_DIRTY, /* a step failed: warm continue forbidden, cold DFU */ ++ G5P_TDV_N, ++}; ++ ++static const char *const g5p_td_phase_name[G5P_TDP_N] = { ++ "freeze", "account", "drain", "events", ++ "irq", "detach", "reset", "done", ++}; ++ ++static const char *const g5p_td_mode_name[3] = { "none", "capture", "final" }; ++ ++static const char *const g5p_td_verdict_name[G5P_TDV_N] = { ++ "none", "observed", "untouched", "quiesced", ++ "abandoned", "ram-only", "dirty", ++}; ++ ++struct g5p_td_budget { ++ u32 drain_us; /* self-drain wait; DC3 took 2.34 ms (r87b) */ ++ u32 events_us; ++ u32 settle_us; /* IRQ count must stay constant this long after the mask */ ++ u32 events_pass_max; /* entries per pass; the ring has 256 slots */ ++ u32 events_passes; ++ u32 total_us; /* RESET is skipped when the RAM phases already overran this */ ++}; ++ ++#define G5P_TD_BUDGET_DEFAULT { 20000, 20000, 2000, 256, 4, 100000 } ++ ++/* Inputs the glue reads from RAM before the run (never from a register). */ ++struct g5p_td_pre { ++ bool booted; /* maps_ready: shared pages exist */ ++ bool released; /* firmware was released */ ++ bool powered; /* ps_gfx held on by the retained PM reference */ ++ bool fender_ready; ++ bool already_parked; /* b->parked: GPC=0 verified by park */ ++ bool stopped_unparked; /* b->stopped && !b->parked: an earlier stop did not verify */ ++ bool policy_host_owned; /* init meta+0x6c8 == 0 */ ++ bool power_abort; /* r73b early_power_abort latched */ ++ bool lock_held; /* the glue holds g->lock: no worker is inside a hardware wait */ ++}; ++ ++/* ++ * Every op is bounded by its own contract; the machine additionally bounds how ++ * often and for how long it calls them. Ops marked RAM never touch a register. ++ */ ++struct g5p_td_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); /* sleep, no register access */ ++ void (*freeze)(void *ctx); /* RAM: stopped=1 alive=0, disarm */ ++ /* RAM: idx 0..NCMD-1 command rings, NCMD = event ring. -ENODEV: not mapped. */ ++ int (*ring_read)(void *ctx, u32 idx, u32 *cons, u32 *prod); ++ /* RAM: consume at most max entries of type 0/1, stop at the first other type. */ ++ int (*event_pass)(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault); ++ bool (*terminal_seen)(void *ctx); /* RAM: TA complete/timeout latch */ ++ void (*detach)(void *ctx); /* RAM: signal/forget host jobs */ ++ void (*aic_ack)(void *ctx); /* AIC: swirq+0x94 = 0x4000 */ ++ void (*irq_mask)(void *ctx); /* AIC: ack stays, line masked */ ++ u32 (*irq_total)(void *ctx); /* RAM: handler entry counter */ ++ int (*fender_stop)(void *ctx); /* R1: g5p_fender_idle() */ ++ int (*pmgr_actual)(void *ctx, u32 *target, ++ u32 *actual); /* PMGR read; may be NULL */ ++ void (*emit)(void *ctx, const char *line); /* printk; may be NULL */ ++}; ++ ++struct g5p_td_receipt { ++ unsigned char seq, phase, mode, verdict; ++ s32 rc; ++ u32 us, steps, flags; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [dom][0]=reads [dom][1]=writes, op level */ ++ u32 a, b; /* phase specific, see g5p_td_run() */ ++}; ++ ++struct g5p_td { ++ struct g5p_td_budget budget; ++ /* latches shared with the render path; touched by cmpxchg only */ ++ int running; /* 0 idle, 1 inside g5p_td_run() */ ++ int terminal; /* 0, 1 complete, 2 timeout: first writer wins */ ++ int quiet; /* diagnostics MMIO forbidden from here on */ ++ /* 0 idle, 1 captured (informational), 2 finalized: only 2 answers from the cache */ ++ int state; ++ u32 snaps; /* CAPTURE runs since init: the n= of the TD99 snap line */ ++ /* per run */ ++ enum g5p_td_mode mode; ++ enum g5p_td_phase phase; ++ enum g5p_td_skip skip; ++ enum g5p_td_verdict verdict; ++ bool eligible; ++ u32 flags, steps, violations, seq, nrcpt; ++ u64 t0_ns; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [enum g5p_td_dom][0=reads,1=writes], op level */ ++ u32 pending_mask, pending_initial, abandoned, drained; ++ u32 ring_cons[G5P_TD_NRING], ring_prod[G5P_TD_NRING]; ++ u32 ev_consumed, ev_left, ev_initial, irq_before, irq_after; ++ u32 us_total; ++ u32 pmgr_target, pmgr_actual; ++ int rc; ++ struct g5p_td_receipt rcpt[G5P_TD_MAX_RCPT]; ++}; ++ ++static inline void g5p_td_init(struct g5p_td *t) ++{ ++ const struct g5p_td_budget b = G5P_TD_BUDGET_DEFAULT; ++ ++ *t = (struct g5p_td){ .budget = b }; ++} ++ ++/* One latch for "the TA reached a terminal state"; the first writer wins. */ ++static inline bool g5p_td_note_terminal(struct g5p_td *t, int kind) ++{ ++ if (cmpxchg(&t->terminal, 0, kind) != 0) ++ return false; ++ WRITE_ONCE(t->quiet, 1); ++ return true; ++} ++ ++/* ++ * The single predicate for diagnostic register access (R0 snapshots, end banks, ++ * exports): false once the epoch is terminal or a teardown has started. Nothing ++ * in this series calls a diagnostic; this exists so the render path can. ++ */ ++ ++/* RT1 (Fender stop leg) eligibility. Pure function of RAM-derived inputs. */ ++static inline bool g5p_td_eligible(const struct g5p_td_pre *p, ++ enum g5p_td_mode mode, enum g5p_td_skip *why) ++{ ++ enum g5p_td_skip s = G5P_TDS_NONE; ++ ++ if (!p->booted) ++ s = G5P_TDS_NOT_BOOTED; ++ else if (!p->released) ++ s = G5P_TDS_NOT_RELEASED; ++ else if (!p->powered || !p->fender_ready) ++ s = G5P_TDS_NOT_POWERED; ++ else if (p->already_parked) ++ s = G5P_TDS_ALREADY_PARKED; ++ else if (p->stopped_unparked) ++ s = G5P_TDS_STOPPED_UNPARKED; ++ else if (p->power_abort) ++ s = G5P_TDS_POWER_ABORT; ++ else if (!p->policy_host_owned) ++ s = G5P_TDS_FW_POLICY; ++ else if (!p->lock_held) ++ s = G5P_TDS_LOCK_BUSY; ++ else if (mode != G5P_TD_FINAL) ++ s = G5P_TDS_CAPTURE_MODE; ++ if (why) ++ *why = s; ++ return s == G5P_TDS_NONE; ++} ++ ++static inline int g5p_td_format(const struct g5p_td_receipt *r, char *buf, ++ size_t n) ++{ ++ return scnprintf(buf, n, ++ "TD98 seq=%u phase=%s mode=%s rc=%d us=%u steps=%u flags=%08x aic=%u/%u r1=%u/%u r0=%u/%u pmgr=%u/%u a=%u b=%u verdict=%s", ++ r->seq, g5p_td_phase_name[r->phase], g5p_td_mode_name[r->mode], ++ r->rc, r->us, r->steps, r->flags, r->cnt[G5P_TDD_AIC][0], ++ r->cnt[G5P_TDD_AIC][1], r->cnt[G5P_TDD_R1][0], ++ r->cnt[G5P_TDD_R1][1], r->cnt[G5P_TDD_R0][0], ++ r->cnt[G5P_TDD_R0][1], r->cnt[G5P_TDD_PMGR][0], ++ r->cnt[G5P_TDD_PMGR][1], r->a, r->b, ++ g5p_td_verdict_name[r->verdict]); ++} ++ ++static inline u32 g5p_td_us_since(const struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ u64 d = o->now_ns(o->ctx) - t->t0_ns; ++ ++ return d > 0xffffffffULL * 1000 ? 0xffffffffu : (u32)(d / 1000); ++} ++ ++static inline void g5p_td_receipt(struct g5p_td *t, const struct g5p_td_ops *o, ++ int rc, u32 a, u32 b) ++{ ++ struct g5p_td_receipt *r = ++ &t->rcpt[t->nrcpt < G5P_TD_MAX_RCPT ? t->nrcpt : ++ G5P_TD_MAX_RCPT - 1]; ++ char line[224]; ++ unsigned int d; ++ ++ r->seq = (unsigned char)t->seq++; ++ r->phase = t->phase; ++ r->mode = t->mode; ++ r->verdict = t->verdict; ++ r->rc = rc; ++ r->us = g5p_td_us_since(t, o); ++ r->steps = t->steps; ++ r->flags = t->flags; ++ for (d = 0; d < G5P_TDD_N; d++) { ++ r->cnt[d][0] = t->cnt[d][0]; ++ r->cnt[d][1] = t->cnt[d][1]; ++ } ++ r->a = a; ++ r->b = b; ++ if (t->nrcpt < G5P_TD_MAX_RCPT) ++ t->nrcpt++; ++ if (o->emit) { ++ g5p_td_format(r, line, sizeof(line)); ++ o->emit(o->ctx, line); ++ } ++} ++ ++/* Count and check one access. A refused access is a bug in the machine itself. */ ++static inline int g5p_td_gate(struct g5p_td *t, enum g5p_td_dom d, ++ unsigned int acc) ++{ ++ if ((g5p_td_allow[t->phase][d] & acc) != acc) { ++ t->violations++; ++ t->flags |= G5P_TDF_VIOLATION; ++ return -EACCES; ++ } ++ if ((acc & G5P_TDA_R) && t->cnt[d][0] < 255) ++ t->cnt[d][0]++; ++ if ((acc & G5P_TDA_W) && t->cnt[d][1] < 255) ++ t->cnt[d][1]++; ++ return 0; ++} ++ ++/* One loop iteration or hardware op. False once the hard step cap is hit. */ ++static inline bool g5p_td_step(struct g5p_td *t) ++{ ++ if (t->steps >= G5P_TD_MAX_STEPS) { ++ t->flags |= G5P_TDF_OVERRUN; ++ return false; ++ } ++ t->steps++; ++ return true; ++} ++ ++static inline bool g5p_td_ring_ok(u32 cons, u32 prod) ++{ ++ return cons <= 255 && prod <= 255; ++} ++ ++static inline void g5p_td_note_late(struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ if (o->terminal_seen && o->terminal_seen(o->ctx) && ++ !(t->flags & G5P_TDF_TERMINAL)) ++ t->flags |= G5P_TDF_LATE_TERMINAL; ++} ++ ++static inline int g5p_td_account(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 i, bad = 0, pending = 0, cons, prod; ++ int rc; ++ ++ for (i = 0; i < G5P_TD_NRING; i++) { ++ if (!g5p_td_step(t)) ++ return -ETIMEDOUT; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return -EACCES; ++ rc = o->ring_read(o->ctx, i, &cons, &prod); ++ if (rc == -ENODEV) { ++ t->flags |= G5P_TDF_NOT_BOOTED; ++ break; ++ } ++ if (rc || !g5p_td_ring_ok(cons, prod)) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ bad++; ++ continue; ++ } ++ t->ring_cons[i] = cons; ++ t->ring_prod[i] = prod; ++ if (i < G5P_TD_NCMD && cons != prod) { ++ t->pending_mask |= 1u << i; ++ pending++; ++ } ++ } ++ t->pending_initial = pending; ++ t->ev_left = (t->ring_prod[G5P_TD_NCMD] - t->ring_cons[G5P_TD_NCMD]) & ++ 255; ++ t->ev_initial = t->ev_left; ++ g5p_td_receipt(t, o, bad ? -EPROTO : 0, pending, bad); ++ return 0; ++} ++ ++/* ++ * gpu-recovery99 0004: one extra line per CAPTURE run, after the ACCOUNT receipt, so a live ++ * epoch can be looked at ++ * several times (S1a). RAM only: it formats what ACCOUNT stored in struct g5p_td and reads the ++ * handler entry counter ++ * (RAM) and the clock; no register, no state change. The prefix is not TD98 on purpose: receipt ++ * parsers keyed on ++ * "TD98 seq=" / "TD98 begin" ignore it. ++ * n CAPTURE runs since init (1 based) ++ * t_ns ops->now_ns(), for the IRQ rate between two lines ++ * irq handler entry counter (ops->irq_total), 0 if there is no such op ++ * term terminal latch (0 none, 1 complete, 2 timeout); quiet = the diagnostics latch ++ * own bit0 booted 1 released 2 powered 3 fender_ready 4 already_parked 5 stopped_unparked ++ * 6 policy_host_owned 7 power_abort 8 lock_held ++ * cN/pN command ring N consumer / producer; ce/pe the event ring. Zero when the shared pages ++ * are not mapped. ++ */ ++static inline void g5p_td_snap(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ char line[224]; ++ u32 own, irq = 0; ++ ++ t->snaps++; ++ if (!o->emit) ++ return; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R) && o->irq_total) ++ irq = o->irq_total(o->ctx); ++ own = pre->booted | pre->released << 1 | pre->powered << 2 | ++ pre->fender_ready << 3 | pre->already_parked << 4 | ++ pre->stopped_unparked << 5 | pre->policy_host_owned << 6 | ++ pre->power_abort << 7 | pre->lock_held << 8; ++ scnprintf(line, sizeof(line), ++ "TD99 snap n=%u t_ns=%llu irq=%u term=%d quiet=%d own=%03x c0=%u p0=%u c1=%u p1=%u c2=%u p2=%u c3=%u p3=%u ce=%u pe=%u", ++ t->snaps, (unsigned long long)o->now_ns(o->ctx), irq, ++ READ_ONCE(t->terminal), READ_ONCE(t->quiet), own, ++ t->ring_cons[0], t->ring_prod[0], t->ring_cons[1], ++ t->ring_prod[1], t->ring_cons[2], t->ring_prod[2], ++ t->ring_cons[3], t->ring_prod[3], t->ring_cons[G5P_TD_NCMD], ++ t->ring_prod[G5P_TD_NCMD]); ++ o->emit(o->ctx, line); ++} ++ ++static inline void g5p_td_drain(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.drain_us * 1000; ++ u32 i, cons, prod; ++ ++ while (t->pending_mask) { ++ if (!g5p_td_step(t)) ++ break; ++ if (o->now_ns(o->ctx) >= deadline) { ++ t->flags |= G5P_TDF_ABANDON; ++ break; ++ } ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ for (i = 0; i < G5P_TD_NCMD; i++) { ++ if (!(t->pending_mask & (1u << i))) ++ continue; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return; ++ if (o->ring_read(o->ctx, i, &cons, &prod) || ++ !g5p_td_ring_ok(cons, prod) || ++ prod != t->ring_prod[i]) { ++ /* The producer is host-written and the host is frozen. */ ++ t->flags |= G5P_TDF_PROTOCOL; ++ t->pending_mask &= ~(1u << i); ++ continue; ++ } ++ if (cons == prod) { ++ t->pending_mask &= ~(1u << i); ++ t->flags |= G5P_TDF_SELF_DRAINED; ++ } ++ } ++ } ++ for (i = 0; i < G5P_TD_NCMD; i++) ++ if (t->pending_mask & (1u << i)) ++ t->abandoned++; ++ if (t->abandoned) ++ t->flags |= G5P_TDF_ABANDON; ++ t->drained = t->pending_initial - t->abandoned; ++} ++ ++static inline void g5p_td_events(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.events_us * 1000; ++ u32 pass, consumed, left = t->ev_left; ++ bool fault = false; ++ int rc; ++ ++ for (pass = 0; pass < t->budget.events_passes && left; pass++) { ++ if (!g5p_td_step(t)) ++ break; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_RW)) ++ return; ++ consumed = 0; ++ rc = o->event_pass(o->ctx, t->budget.events_pass_max, &consumed, ++ &left, &fault); ++ t->ev_consumed += consumed; ++ if (rc) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ break; ++ } ++ if (fault) { ++ /* Retained for the host; stop consuming, the IRQ is masked next. */ ++ t->flags |= G5P_TDF_FAULT_EVENT; ++ break; ++ } ++ if (!left || o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ } ++ t->ev_left = left; ++ /* ++ * A ring holds at most 255 entries, so consuming more than the backlog plus a ++ * few ALIVE ticks means the producer outran us: an empty ring at the last ++ * look is a snapshot, not a quiesce. ++ */ ++ if (!fault && (left || (s32)(t->ev_consumed + left - t->ev_initial) > ++ G5P_TD_EV_SLACK)) ++ t->flags |= G5P_TDF_EV_UNQUIET; ++} ++ ++static inline void g5p_td_irq(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 tries, n0, n1; ++ ++ if (g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W) || ++ g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W)) ++ return; ++ t->irq_before = o->irq_total(o->ctx); ++ /* Same pair as the IRQ-cap path: ack the software source, then mask the line. */ ++ o->aic_ack(o->ctx); ++ o->irq_mask(o->ctx); ++ for (tries = 0; tries < 3; tries++) { ++ if (!g5p_td_step(t)) ++ break; ++ n0 = o->irq_total(o->ctx); ++ o->relax_us(o->ctx, t->budget.settle_us); ++ n1 = o->irq_total(o->ctx); ++ t->irq_after = n1; ++ if (n0 == n1) ++ return; ++ } ++ t->flags |= G5P_TDF_IRQ_UNSETTLED; ++} ++ ++static inline void g5p_td_reset(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ enum g5p_td_skip why = G5P_TDS_NONE; ++ bool bad = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_VIOLATION); ++ int rc = 0; ++ ++ t->eligible = g5p_td_eligible(pre, t->mode, &why); ++ if (t->eligible && ++ (bad || g5p_td_us_since(t, o) > t->budget.total_us)) { ++ t->eligible = false; ++ why = G5P_TDS_BAD_STATE; ++ } ++ t->skip = why; ++ if (!t->eligible) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ } else if (!g5p_td_step(t) || g5p_td_gate(t, G5P_TDD_R1, G5P_TDA_RW)) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ t->skip = G5P_TDS_BAD_STATE; ++ } else { ++ /* ++ * Apple stopFirmwareForPowerDown 6c9fa10, R1 only; flown here after a DC ++ * timeout (round 10). Never retried, even on failure. ++ */ ++ rc = o->fender_stop(o->ctx); ++ t->flags |= rc ? G5P_TDF_RT1_FAIL : G5P_TDF_RT1_OK; ++ } ++ if (o->pmgr_actual && !g5p_td_gate(t, G5P_TDD_PMGR, G5P_TDA_R)) ++ o->pmgr_actual(o->ctx, &t->pmgr_target, &t->pmgr_actual); ++ g5p_td_receipt(t, o, rc, t->eligible ? 0 : (u32)t->skip, ++ t->pmgr_actual); ++} ++ ++static inline void g5p_td_decide(struct g5p_td *t, const struct g5p_td_pre *pre) ++{ ++ bool dirty = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_EV_UNQUIET | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_RT1_FAIL | ++ G5P_TDF_OVERRUN | G5P_TDF_VIOLATION) || ++ t->skip == G5P_TDS_STOPPED_UNPARKED || ++ t->skip == G5P_TDS_BAD_STATE; ++ ++ if (t->mode == G5P_TD_CAPTURE) ++ t->verdict = (t->flags & G5P_TDF_VIOLATION) ? G5P_TDV_DIRTY : ++ G5P_TDV_OBSERVED; ++ else if (dirty) ++ t->verdict = G5P_TDV_DIRTY; ++ else if (t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED)) ++ t->verdict = G5P_TDV_UNTOUCHED; ++ else if (t->flags & G5P_TDF_RT1_OK || t->skip == G5P_TDS_ALREADY_PARKED) ++ t->verdict = (t->flags & G5P_TDF_ABANDON) ? G5P_TDV_ABANDONED : ++ G5P_TDV_QUIESCED; ++ else ++ t->verdict = G5P_TDV_RAM_ONLY; ++ (void)pre; ++} ++ ++/* ++ * Run the teardown once per mode. Returns 0 for QUIESCED/ABANDONED/UNTOUCHED/OBSERVED, ++ * -EAGAIN for RAM_ONLY (clean, but the GPU was left as found), -EIO for DIRTY, ++ * -EBUSY if another caller is inside the machine. Every phase ends in a receipt. ++ * Receipt a/b: account pending/bad, drain drained/abandoned, events consumed/left, ++ * irq before/after, reset skip-reason/pmgr-actual, done skip-reason/abandoned. ++ * Counters in the receipts are op-level (one count per gate): the AIC pair is ++ * 2 writes, the whole Fender stop leg is one R1 read+write op. ++ */ ++static inline int g5p_td_run(struct g5p_td *t, const struct g5p_td_ops *o, ++ enum g5p_td_mode mode, ++ const struct g5p_td_pre *pre) ++{ ++ if (cmpxchg(&t->running, 0, 1)) ++ return -EBUSY; ++ WRITE_ONCE(t->quiet, 1); ++ if (t->state == 2) { ++ /* Answered from the cache (after FINAL only; CAPTURE re-runs, gpu- ++ * recovery99 0004): no phase runs, nothing is accessed. ++ */ ++ u32 saved_flags = t->flags; ++ ++ t->t0_ns = o->now_ns(o->ctx) - (u64)t->us_total * 1000; ++ t->flags |= G5P_TDF_REENTRY; ++ t->phase = G5P_TDP_DONE; ++ g5p_td_receipt(t, o, t->rc, t->flags, t->steps); ++ t->flags = saved_flags; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++ } ++ t->mode = mode; ++ t->phase = G5P_TDP_FREEZE; ++ t->t0_ns = o->now_ns(o->ctx); ++ t->seq = 0; ++ t->nrcpt = 0; ++ t->flags = mode == G5P_TD_CAPTURE ? G5P_TDF_CAPTURE_ONLY : 0; ++ t->steps = 0; ++ t->violations = 0; ++ t->abandoned = 0; ++ t->drained = 0; ++ t->pending_mask = 0; ++ t->pending_initial = 0; ++ t->ev_consumed = 0; ++ t->ev_left = 0; ++ t->ev_initial = 0; ++ t->irq_before = 0; ++ t->irq_after = 0; ++ t->skip = G5P_TDS_NONE; ++ t->verdict = G5P_TDV_NONE; ++ memset(t->cnt, 0, sizeof(t->cnt)); ++ /* a CAPTURE may now run many times: nothing of the previous look may leak into this one */ ++ memset(t->ring_cons, 0, sizeof(t->ring_cons)); ++ memset(t->ring_prod, 0, sizeof(t->ring_prod)); ++ if (READ_ONCE(t->terminal)) ++ t->flags |= G5P_TDF_TERMINAL; ++ if (!pre->released) ++ t->flags |= G5P_TDF_UNRELEASED; ++ ++ if (mode == G5P_TD_FINAL) { ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->freeze(o->ctx); ++ g5p_td_receipt(t, o, 0, pre->released, pre->policy_host_owned); ++ } ++ t->phase = G5P_TDP_ACCOUNT; ++ if (g5p_td_account(t, o)) ++ t->flags |= G5P_TDF_OVERRUN; ++ g5p_td_note_late(t, o); ++ if (mode == G5P_TD_CAPTURE) ++ g5p_td_snap(t, o, pre); ++ ++ if (mode == G5P_TD_FINAL && ++ !(t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED | ++ G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN))) { ++ t->phase = G5P_TDP_DRAIN; ++ g5p_td_drain(t, o); ++ g5p_td_note_late(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_ABANDON) ? -ETIMEDOUT : 0, ++ t->drained, t->abandoned); ++ ++ t->phase = G5P_TDP_EVENTS; ++ g5p_td_events(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_EV_UNQUIET) ? -ETIMEDOUT : 0, ++ t->ev_consumed, t->ev_left); ++ } ++ if (mode == G5P_TD_FINAL) { ++ /* The IRQ is masked even after an overrun: ack + mask is the safe exit. */ ++ t->phase = G5P_TDP_IRQ; ++ if (!(t->flags & G5P_TDF_NOT_BOOTED)) ++ g5p_td_irq(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_IRQ_UNSETTLED) ? -ETIMEDOUT : ++ 0, ++ t->irq_before, t->irq_after); ++ ++ t->phase = G5P_TDP_DETACH; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->detach(o->ctx); ++ g5p_td_receipt(t, o, 0, 0, 0); ++ ++ t->phase = G5P_TDP_RESET; ++ g5p_td_reset(t, o, pre); ++ } ++ g5p_td_note_late(t, o); ++ t->phase = G5P_TDP_DONE; ++ g5p_td_decide(t, pre); ++ switch (t->verdict) { ++ case G5P_TDV_DIRTY: ++ t->rc = -EIO; ++ break; ++ case G5P_TDV_RAM_ONLY: ++ t->rc = -EAGAIN; ++ break; ++ default: ++ t->rc = 0; ++ } ++ t->us_total = g5p_td_us_since(t, o); ++ g5p_td_receipt(t, o, t->rc, (u32)t->skip, t->abandoned); ++ t->state = mode == G5P_TD_FINAL ? 2 : 1; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +new file mode 100644 +index 000000000..e3875bc59 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -0,0 +1,694 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the GPU teardown state machine (teardown.h). ++ * Design, evidence and device plan: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md ++ * ++ * Included from core.inc after lifecycle.inc, so it may use g5p_dc_root_word(), ++ * g5p_fender_write()/poll() and g5p_r73b_disarm(). No state is added to struct ++ * g5p_boot or struct g5p_diag: one GPU, one file-static teardown instance. ++ * ++ * Compiled with W=1 against a9/tri96 (patches/gpu-recovery/README.md); NOT run on hardware. ++ * ++ * gpu-recovery108 (0006): every run goes through the retire layer (retire108.h): ++ * admission close, RAM-only epoch class and a TR108 receipt before the ladder, the ++ * named post-ladder slots, and the retired latch after a FINAL. The TD98 lines are ++ * unchanged. docs/gpu/experiments/recovery108-driver.md. ++ */ ++#include ++#include ++#include ++#include "teardown.h" ++#include "retire108.h" ++#include "render-user-bind.h" ++#include "render-user-quiesce.h" ++ ++/* 0 off (default; the notifier and .shutdown do nothing), 1 observe-only receipts ++ * (CAPTURE: RAM reads, no change), 2 full ladder (FINAL). ++ */ ++ ++/* gpu-recovery108 G6 slot: one R1 read of isStableOff after a verified Fender stop leg. */ ++ ++/* Defined in render-r96.inc (same translation unit, included later by render-fixed.inc). */ ++ ++static struct g5p_td g5p_td_state; ++static struct g5p_diag *g5p_td_dev; ++static void (*g5p_td_detach_cb)(void *cookie); ++static void *g5p_td_detach_cookie; ++ ++/* gpu-recovery108: the retire layer around the ladder (retire108.h), one GPU, file static. */ ++static struct g5p_r8 g5p_r8_state = { .admit_us = G5P_R8_ADMIT_US }; ++static void (*g5p_r8_epoch_cb)(void *cookie, struct g5p_r8_in *in); ++static void *g5p_r8_epoch_cookie; ++ ++/* The DRM frontend reports its share of the epoch inputs (claim, holders, queued jobs), RAM only. */ ++static void __maybe_unused g5p_r8_register_epoch(void (*cb)(void *, ++ struct g5p_r8_in *), ++ void *cookie) ++{ ++ g5p_r8_epoch_cookie = cookie; ++ smp_store_release(&g5p_r8_epoch_cb, cb); ++} ++ ++/* Job admission: a submitter enters before its first look at driver state and ++ * leaves on every return; false once a FINAL teardown closed admission. ++ */ ++static bool __maybe_unused g5p_r8_enter(void) ++{ ++ return g5p_r8_admit_enter(&g5p_r8_state.admit); ++} ++ ++static void __maybe_unused g5p_r8_exit(void) ++{ ++ g5p_r8_admit_exit(&g5p_r8_state.admit); ++} ++ ++/* The post-retire register predicate for the R1 helpers outside the ladder. */ ++static bool __maybe_unused g5p_r8_r1_allowed(void) ++{ ++ return g5p_r8_mmio_ok(&g5p_r8_state, G5P_TDD_R1); ++} ++ ++static bool __maybe_unused g5p_r8_is_retired(void) ++{ ++ return g5p_r8_retired(&g5p_r8_state); ++} ++ ++/* The DRM frontend registers the callback that fails its queued/in-flight job ++ * fences (exactly once each, -ENODEV) without touching hardware. ++ */ ++static void __maybe_unused g5p_td_register_detach(void (*cb)(void *), ++ void *cookie) ++{ ++ g5p_td_detach_cookie = cookie; ++ smp_store_release(&g5p_td_detach_cb, cb); ++} ++ ++static u64 g5p_td_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_td_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++static void g5p_td_op_freeze(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ struct g5p_boot *b = g->boot; ++ ++ if (!b) ++ return; ++ /* The IRQ-side plateau banks (g5p_r80_irq_bank/g5p_r85_irq_bank) read R0 until this ++ * latch is set; arm it before anything else (RAM only, no-op without an r96 config). ++ */ ++ /* Same freeze as g5p_boot_park(): nothing may publish after this. */ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++} ++ ++static int g5p_td_op_ring_read(void *ctx, u32 idx, u32 *cons, u32 *prod) ++{ ++ static const u32 slots[G5P_TD_NRING] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *ctl; ++ ++ if (!b || !READ_ONCE(b->maps_ready)) ++ return -ENODEV; ++ ctl = (u32 *)g5p_boot_shared(b, slots[idx]); ++ *cons = READ_ONCE(ctl[0]); ++ *prod = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ return 0; ++} ++ ++/* RAM only. Consumes ALIVE (0) and stamp (1) entries; the first other type is ++ * left in place for the host, as g5p_boot_drain() does. ++ */ ++static int g5p_td_op_event_pass(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n = 0; ++ unsigned long flags; ++ int ret = 0; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ while (n < max && consumer != producer) { ++ u32 off = b->layout.offset[0x74] + consumer * 44; ++ ++ if (g5p_le32(b->shared[off / G5P_GPU_PAGE].cpu + ++ off % G5P_GPU_PAGE) > 1) { ++ *fault = true; ++ break; ++ } ++ consumer = (consumer + 1) & 255; ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ n++; ++ } ++ producer = READ_ONCE(control[4]); ++ *left = producer > 255 ? 255 : (producer - consumer) & 255; ++out: ++ *consumed = n; ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* The render path latches terminal states through g5p_td_note_terminal(); until ++ * prod98 wires that call this reports "not seen" and the race is only visible as ++ * ring state. ++ */ ++static bool g5p_td_op_terminal_seen(void *ctx) ++{ ++ return READ_ONCE(g5p_td_state.terminal) != 0; ++} ++ ++static void g5p_td_op_detach(void *ctx) ++{ ++ void (*cb)(void *) = smp_load_acquire(&g5p_td_detach_cb); ++ ++ if (cb) ++ cb(g5p_td_detach_cookie); ++} ++ ++static void g5p_td_op_aic_ack(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* The same software-source ACK the IRQ-cap path uses. */ ++ if (b && b->irq_ready && b->swirq) ++ writel(0x4000, b->swirq + 0x94); ++} ++ ++static void g5p_td_op_irq_mask(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* Line masked at the interrupt controller; the handler stays registered for ++ * a late entry, which still ACKs. Same call as the IRQ-cap path. ++ */ ++ if (b && b->irq_ready && b->irq > 0) ++ disable_irq_nosync(b->irq); ++} ++ ++static u32 g5p_td_op_irq_total(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ return b ? (u32)atomic_read(&b->irqs) : 0; ++} ++ ++static int g5p_td_op_fender_stop(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ ++ /* Apple stopFirmwareForPowerDown 6c9fa10; no retry, no PMGR off. */ ++ return g5p_fender_idle(&io); ++} ++ ++static int g5p_td_op_pmgr(void *ctx, u32 *target, u32 *actual) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) ++ return -ENODEV; ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (!ret) { ++ *target = raw & 0xf; ++ *actual = (raw >> 4) & 0xf; ++ } ++ return ret; ++} ++ ++/* dev_emerg so the receipt reaches netconsole at any console loglevel. */ ++static void g5p_td_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_diag *)ctx)->dev, "%s\n", line); ++} ++ ++static void g5p_td_pre_fill(struct g5p_diag *g, struct g5p_td_pre *p, ++ bool locked) ++{ ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(p, 0, sizeof(*p)); ++ p->lock_held = locked; ++ p->powered = READ_ONCE(g->powered); ++ p->fender_ready = READ_ONCE(g->fender_ready); ++ if (!b) ++ return; ++ p->booted = READ_ONCE(b->maps_ready); ++ p->released = READ_ONCE(b->released); ++ p->already_parked = READ_ONCE(b->parked); ++ p->stopped_unparked = READ_ONCE(b->stopped) && !READ_ONCE(b->parked); ++ p->power_abort = b->r73b && READ_ONCE(b->r73b->early_power_abort); ++ /* RAM-only read of the host-written init word (same as g5p_dc_snapshot()). */ ++ p->policy_host_owned = ++ p->booted && ++ !g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8); ++} ++ ++/* gpu-recovery108: the ladder as the retire layer's td_run op. The TD98 begin line ++ * and the ladder's own lines are exactly the recovery98/99 ones. ++ */ ++struct g5p_r8_ctx { ++ struct g5p_diag *g; ++ const struct g5p_td_ops *ops; ++ struct g5p_td_pre *pre; ++ const char *why; ++ bool locked; ++}; ++ ++static int g5p_r8_op_td_run(void *ctx, enum g5p_td_mode mode) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ dev_emerg(c->g->dev, ++ "TD98 begin why=%s mode=%s teardown=%u locked=%u\n", c->why, ++ g5p_td_mode_name[mode], (0), c->locked); ++ return g5p_td_run(&g5p_td_state, c->ops, mode, c->pre); ++} ++ ++static u32 g5p_r8_op_td_flags(void *ctx) ++{ ++ return g5p_td_state.flags; ++} ++ ++static enum g5p_td_verdict g5p_r8_op_td_verdict(void *ctx) ++{ ++ return g5p_td_state.verdict; ++} ++ ++static u64 g5p_r8_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_r8_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++/* G6: R1+0x0 bit 30 after a verified stop leg (the slot runs only then, before RETIRE). */ ++static int g5p_r8_op_stable_off(void *ctx, u32 *value) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ *value = readl(c->g->window[1]) >> 30; ++ return 0; ++} ++ ++static void g5p_r8_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_r8_ctx *)ctx)->g->dev, "%s\n", line); ++} ++ ++static void g5p_r8_in_fill(struct g5p_diag *g, struct g5p_r8_in *in, ++ bool locked, enum g5p_td_mode mode) ++{ ++ void (*cb)(void *, struct g5p_r8_in *) = ++ smp_load_acquire(&g5p_r8_epoch_cb); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(in, 0, sizeof(*in)); ++ in->terminal = READ_ONCE(g5p_td_state.terminal); ++ /* FINAL without the lock: a worker (or a sysfs writer) holds it, maybe inside a wait. */ ++ in->worker_busy = mode == G5P_TD_FINAL && !locked; ++ if (b) { ++ in->booted = READ_ONCE(b->maps_ready); ++ in->released = READ_ONCE(b->released); ++ in->alive = READ_ONCE(b->alive); ++ in->failed = READ_ONCE(b->failed); ++ in->fault_event = READ_ONCE(b->event_failed); ++ in->parked = READ_ONCE(b->parked); ++ in->stopped_unparked = READ_ONCE(b->stopped) && ++ !READ_ONCE(b->parked); ++ in->power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ if (cb) ++ cb(g5p_r8_epoch_cookie, in); ++} ++ ++static int g5p_td_execute(struct g5p_diag *g, enum g5p_td_mode mode, ++ const char *why) ++{ ++ const struct g5p_td_ops ops = { ++ .ctx = g, ++ .now_ns = g5p_td_op_now, ++ .relax_us = g5p_td_op_relax, ++ .freeze = g5p_td_op_freeze, ++ .ring_read = g5p_td_op_ring_read, ++ .event_pass = g5p_td_op_event_pass, ++ .terminal_seen = g5p_td_op_terminal_seen, ++ .detach = g5p_td_op_detach, ++ .aic_ack = g5p_td_op_aic_ack, ++ .irq_mask = g5p_td_op_irq_mask, ++ .irq_total = g5p_td_op_irq_total, ++ .fender_stop = g5p_td_op_fender_stop, ++ .pmgr_actual = g5p_td_op_pmgr, ++ .emit = g5p_td_op_emit, ++ }; ++ struct g5p_td_pre pre; ++ struct g5p_r8_in in; ++ struct g5p_r8_ctx c = { .g = g, .ops = &ops, .pre = &pre, .why = why }; ++ /* fw-sleep (G2) and pmgr-gate (G1) have no mechanism: their ops stay NULL. */ ++ const struct g5p_r8_ops r8ops = { ++ .ctx = &c, ++ .now_ns = g5p_r8_op_now, ++ .relax_us = g5p_r8_op_relax, ++ .td_run = g5p_r8_op_td_run, ++ .td_flags = g5p_r8_op_td_flags, ++ .td_verdict = g5p_r8_op_td_verdict, ++ .slot = { [G5P_R8S_STABLE_OFF] = g5p_r8_op_stable_off }, ++ .emit = g5p_r8_op_emit, ++ }; ++ bool locked = false; ++ int ret; ++ ++ /* Never queue behind a worker or sysfs writer that may be inside MMIO. CAPTURE ++ * reads RAM only and needs no lock. ++ */ ++ if (mode == G5P_TD_FINAL) ++ locked = mutex_trylock(&g->lock); ++ c.locked = locked; ++ g5p_td_pre_fill(g, &pre, locked); ++ g5p_r8_in_fill(g, &in, locked, mode); ++ g5p_r8_state.slot_on[G5P_R8S_STABLE_OFF] = (0); ++ ret = g5p_r8_run(&g5p_r8_state, &r8ops, mode, &in, why); ++ if (g5p_td_state.flags & G5P_TDF_RT1_OK && g->boot) ++ WRITE_ONCE(g->boot->parked, true); ++ if (locked) ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Observation only: arms the quiet predicate, changes nothing else. */ ++static int __maybe_unused g5p_td_capture(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_CAPTURE, why); ++} ++ ++static int g5p_td_final(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_FINAL, why); ++} ++ ++/* ++ * The quiesce before a kexec, reboot or power-off (render-user-quiesce.h). It runs from the ++ * reboot notifier whatever `teardown` is, once per boot; .shutdown and the `td-kexec` command ++ * reach the same run. It leaves the GPU powered with its firmware stopped and the boot state ++ * frozen: the next kernel boots it as after any warm kexec. ++ */ ++ ++#define G5P_KQ_LOCK_MS 2000U ++#define G5P_KQ_ADMIT_US 20000U ++#define G5P_KQ_OFF_MS 10U ++ ++/* The render frontend's share: its TYPED_V3 epoch inputs and the RAM-only unbind of slots. */ ++struct g5p_kq_frontend { ++ void (*state)(void *cookie, struct g5p_ru_kq_in *in); ++ int (*detach)(void *cookie, const u32 *slots, u32 n); ++}; ++ ++static const struct g5p_kq_frontend *g5p_kq_fe; ++static void *g5p_kq_fe_cookie; ++static atomic_t g5p_kq_ran = ATOMIC_INIT(0); ++ ++struct g5p_kq_ctx { ++ struct g5p_diag *g; ++ u32 slots[G5P_RU_USER_SLOTS]; ++ u32 held; ++}; ++ ++static void __maybe_unused g5p_kq_register(const struct g5p_kq_frontend *fe, ++ void *cookie) ++{ ++ g5p_kq_fe_cookie = cookie; ++ smp_store_release(&g5p_kq_fe, fe); ++} ++ ++static u32 g5p_kq_admit_close(void *ctx) ++{ ++ struct g5p_r8_admit *a = &g5p_r8_state.admit; ++ u64 deadline = ktime_get_ns() + (u64)G5P_KQ_ADMIT_US * NSEC_PER_USEC; ++ u32 inside = g5p_r8_admit_close(a); ++ ++ while (inside && ktime_get_ns() < deadline) { ++ usleep_range(G5P_R8_POLL_US, 2 * G5P_R8_POLL_US); ++ inside = g5p_r8_admit_inside(a); ++ } ++ return inside; ++} ++ ++/* Never queue behind a holder that may be inside MMIO: bounded, like g5p_render_run(). */ ++static bool g5p_kq_lock(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_KQ_LOCK_MS); ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ return true; ++} ++ ++static void g5p_kq_unlock(void *ctx) ++{ ++ mutex_unlock(&((struct g5p_kq_ctx *)ctx)->g->lock); ++} ++ ++/* RAM only: the boot state, the held user slots and the DC ring, then the frontend's inputs. */ ++static void g5p_kq_inputs(void *ctx, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ struct g5p_diag *g = c->g; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ u32 *ctl; ++ ++ if (!READ_ONCE(g->powered)) ++ return; /* Already idle: no detach, MMIO or second gate cycle. */ ++ in->pmgr = !!g->pmgr; ++ if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released) || ++ READ_ONCE(b->parked)) ++ return; ++ in->booted = true; ++ in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ dma_rmb(); ++ in->cons = READ_ONCE(ctl[0]); ++ in->prod = READ_ONCE(ctl[4]); ++ in->ring_idle = in->prod <= 255 && in->cons == in->prod; ++ c->held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), ++ g5p_boot_shared(b, 0x84), c->slots, ++ ARRAY_SIZE(c->slots)); ++ in->held = c->held; ++ if (fe) { ++ in->frontend = true; ++ fe->state(g5p_kq_fe_cookie, in); ++ } ++} ++ ++static int g5p_kq_detach(void *ctx) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ ++ return fe ? fe->detach(g5p_kq_fe_cookie, c->slots, c->held) : -ENODEV; ++} ++ ++static void g5p_kq_freeze(void *ctx) ++{ ++ g5p_td_op_freeze(((struct g5p_kq_ctx *)ctx)->g); ++} ++ ++/* The IRQ-cap path's AIC ack and mask, then wait out a handler already running (process ++ * context; the threaded handler drains RAM and takes no lock the quiesce holds). ++ */ ++static void g5p_kq_irq_off(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ struct g5p_boot *b = g->boot; ++ ++ g5p_td_op_aic_ack(g); ++ g5p_td_op_irq_mask(g); ++ if (b && b->irq_ready && b->irq > 0) ++ synchronize_irq(b->irq); ++} ++ ++/* PMGR only: the bounded gate cycle of the r33/dirty-start path (refuses a gate that is not ++ * plainly on), which ends with the gate requested on. ++ */ ++static int g5p_kq_cycle(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ ++ g5p_mmio_gate(g); ++ return g5p_r33_gfx_off(g->dev, g->pmgr, G5P_KQ_OFF_MS); ++} ++ ++static int g5p_kq_run(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_ru_kq_ops o = { ++ .admit_close = g5p_kq_admit_close, ++ .lock = g5p_kq_lock, ++ .unlock = g5p_kq_unlock, ++ .inputs = g5p_kq_inputs, ++ .detach = g5p_kq_detach, ++ .freeze = g5p_kq_freeze, ++ .irq_off = g5p_kq_irq_off, ++ .cycle = g5p_kq_cycle, ++ }; ++ struct g5p_ru_kq_out out; ++ struct g5p_kq_ctx *c; ++ u64 t0 = ktime_get_ns(); ++ ++ if (atomic_xchg(&g5p_kq_ran, 1)) { ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s already ran this boot; nothing done end=kq\n", ++ why); ++ return -EALREADY; ++ } ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ if (!c) { ++ dev_emerg(g->dev, "GPU-KQ why=%s ret=%d end=kq\n", why, ++ -ENOMEM); ++ return -ENOMEM; ++ } ++ c->g = g; ++ o.ctx = c; ++ g5p_ru_kq_run(&o, min((G5P_RU_KQ_FULL), G5P_RU_KQ_FULL), &out); ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s mode=%u inside=%u locked=%u booted=%u failed=%u wedged=%u " ++ "ram_only=%u exec=%u dc=%u/%u held=%u plan=%s%s skip=%#x detach=%d froze=%u " ++ "cycle=%d ms=%llu end=kq\n", ++ why, out.in.mode, out.inside, out.in.locked, out.in.booted, ++ out.in.failed, out.in.term_failed, out.in.ram_only, ++ out.in.executing, out.in.cons, out.in.prod, out.in.held, ++ out.plan & G5P_RU_KQ_DETACH ? "detach" : "-", ++ out.plan & G5P_RU_KQ_CYCLE ? "+cycle" : "", out.why, ++ out.detach_ret, out.froze, out.cycle_ret, ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC); ++ kfree(c); ++ return out.detach_ret ?: out.cycle_ret; ++} ++ ++/* sysfs `command`: td-capture / td-final / td-kexec. Runs before the command mutex, like ++ * r73b-export, so a wedged stage cannot block it. -EAGAIN (clean, GPU left as ++ * found) is reported as success; the verdict is in the TD98 lines. td-kexec is the ++ * reboot quiesce run by hand (the GPU is finished for this boot afterwards). ++ */ ++static int g5p_td_command(struct g5p_diag *g, const char *command) ++{ ++ int ret; ++ ++ if (sysfs_streq(command, "td-capture")) ++ ret = g5p_td_capture(g, "sysfs"); ++ else if (sysfs_streq(command, "td-final")) ++ ret = g5p_td_final(g, "sysfs"); ++ else if (sysfs_streq(command, "td-kexec")) ++ ret = g5p_kq_run(g, "sysfs"); ++ else ++ return -EINVAL; ++ return ret == -EAGAIN ? 0 : ret; ++} ++ ++/* Runs from kernel_restart_prepare() for reboot, halt, power-off and kexec, i.e. ++ * before device_shutdown() and its unbounded wait_for_device_probe(). Panic kexec ++ * skips both and is not covered. ++ */ ++static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, ++ void *data) ++{ ++ struct g5p_diag *g = READ_ONCE(g5p_td_dev); ++ ++ if (!g) ++ return NOTIFY_DONE; ++ if (action != SYS_RESTART && action != SYS_HALT && ++ action != SYS_POWER_OFF) ++ return NOTIFY_DONE; ++ /* The quiesce needs a live firmware for its detach, so it runs before the ladder freezes. */ ++ g5p_kq_run(g, "reboot-notifier"); ++ { ++ return NOTIFY_DONE; ++ } ++} ++ ++static struct notifier_block g5p_td_nb = { ++ .notifier_call = g5p_td_reboot_notify, ++ .priority = INT_MAX, /* before every other reboot notifier */ ++}; ++ ++/* g is devm-allocated and the DRM frontend's probe error path unwinds without ++ * g5p_remove(): the notifier must not outlive g. ++ */ ++static void g5p_td_devres_release(void *data) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ WRITE_ONCE(g5p_td_dev, NULL); ++} ++ ++static void g5p_td_attach(struct g5p_diag *g) ++{ ++ g5p_td_init(&g5p_td_state); ++ /* A probe after a retire (not reachable on the built-in driver) never reopens admission. */ ++ if (!g5p_r8_retired(&g5p_r8_state)) { ++ u32 gen = g5p_r8_state.gen; ++ ++ g5p_r8_init(&g5p_r8_state); ++ g5p_r8_state.gen = gen; /* monotonic across a re-probe */ ++ } ++ WRITE_ONCE(g5p_td_dev, g); ++ register_reboot_notifier(&g5p_td_nb); ++ /* On failure the action has already run: no notifier, teardown stays inert. */ ++ if (devm_add_action_or_reset(g->dev, g5p_td_devres_release, g)) ++ dev_warn(g->dev, "teardown not attached\n"); ++} ++ ++/* Returns true when the PM reference must be kept: a PMGR power-off was never ++ * validated on the outgoing side and the stop leg is not isStableOff. ++ */ ++static bool g5p_td_remove(struct g5p_diag *g) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ ; ++ ; ++ WRITE_ONCE(g5p_td_dev, NULL); ++ return (0) != 0; ++} ++ ++/* The .shutdown platform callback: second chance after the notifier. */ ++static void __maybe_unused g5p_td_shutdown(struct g5p_diag *g) ++{ ++ /* A second chance after the notifier (a no-op once it ran). */ ++ if (!atomic_read(&g5p_kq_ran)) ++ g5p_kq_run(g, "shutdown"); ++ { ++ ; ++ ; ++ }; ++} +diff --git a/drivers/gpu/apple-g5p/transport-budget.inc b/drivers/gpu/apple-g5p/transport-budget.inc +new file mode 100644 +index 000000000..d70e3c6e1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/transport-budget.inc +@@ -0,0 +1,6 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include "vm-mutation.h" ++static u32 g5p_transport_event_limit(void) ++{ ++ return 256 + 131072 * 4; ++} +diff --git a/drivers/gpu/apple-g5p/vm-mutation.h b/drivers/gpu/apple-g5p/vm-mutation.h +new file mode 100644 +index 000000000..4645240bd +--- /dev/null ++++ b/drivers/gpu/apple-g5p/vm-mutation.h +@@ -0,0 +1,36 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_VM_MUTATION_H ++#define G5P_VM_MUTATION_H ++struct g5p_vm_mutation { ++ bool generations; ++ u32 generation[3]; ++}; ++ ++/* Count actual changed descriptors, including clears, not dispatches. Repeated ++ * parent writes of identical bytes do not advance a generation. Packed fields ++ * follow iOS6cabf40. Publication is serialized by the existing command lock. ++ */ ++static void g5p_vm_entry(struct g5p_vm_mutation *c, void *at, u64 value, ++ u32 level) ++{ ++ u64 old = g5p_le32(at); ++ ++ if (level) ++ old |= (u64)g5p_le32(at + 4) << 32; ++ if (old == value) ++ return; ++ if (level) ++ g5p_put64(at, value); ++ else ++ g5p_put32(at, value); ++ c->generation[level]++; ++} ++ ++static u32 g5p_vm_generation(struct g5p_vm_mutation *c) ++{ ++ if (!c->generations) ++ return 0; ++ return ((c->generation[0] & 0x3ff) << 22) | ++ ((c->generation[1] & 0x3ff) << 12) | (c->generation[2] & 0xfff); ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/work-layout.h b/drivers/gpu/apple-g5p/work-layout.h +new file mode 100644 +index 000000000..b5dc73725 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/work-layout.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_WORK_LAYOUT_H ++#define G5P_WORK_LAYOUT_H ++ ++/* Experimental first CL queue probe: c89420/c88840/c8a808/c980a0. ++ * Tag 4 is emitted for dependencies by c8a808. Treating an already satisfied ++ * dependency as a NOP is inferred and must be validated after G2. ++ */ ++#define G5P_WORK_OFFSET 0x54000U ++#define G5P_WORK_BYTES 0x400U ++#define G5P_WORK_CTL 0x100U ++#define G5P_WORK_RING 0x200U ++#define G5P_WORK_CMD 0x300U ++#define G5P_WORK_STAMP 0x340U ++#define G5P_WORK_SLOTS 16U ++ ++/* Isolated visibility hypothesis, not the recovered normal host alias. */ ++ ++/* Firmware10004e94 copies notification word2 into ctl+30. */ ++ ++#endif diff --git a/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch b/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch new file mode 100644 index 00000000..8cdb117a --- /dev/null +++ b/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch @@ -0,0 +1,1027 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add buffer VM and cache visibility support + +Preserve the sealed mapping budgets, PRIME and cache policy. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/cache-policy.h | 13 + + drivers/gpu/drm/apple-g5p/cache-receipt.inc | 38 + + drivers/gpu/drm/apple-g5p/drm-vm.inc | 912 ++++++++++++++++++++ + drivers/gpu/drm/apple-g5p/vm-ranges.h | 21 + + 4 files changed, 984 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/cache-policy.h + create mode 100644 drivers/gpu/drm/apple-g5p/cache-receipt.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-vm.inc + create mode 100644 drivers/gpu/drm/apple-g5p/vm-ranges.h + +diff --git a/drivers/gpu/drm/apple-g5p/cache-policy.h b/drivers/gpu/drm/apple-g5p/cache-policy.h +new file mode 100644 +index 000000000..40f5cdfe1 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-policy.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* No caller may infer CPU caching from BO size or GPU use alone. */ ++#ifndef G5P_CACHE_POLICY_H ++#define G5P_CACHE_POLICY_H ++static inline int g5p_cache_class(int imported, int exported, int wc) ++{ ++ return imported ? 2 : (!exported && wc ? 0 : 1); ++} ++static inline int g5p_cache_omit(int enabled, int imported, int exported, int wc) ++{ ++ return enabled && g5p_cache_class(imported, exported, wc) == 0; ++} ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/cache-receipt.inc b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +new file mode 100644 +index 000000000..1efeecd0c +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Bounded RAM-only records. No register accesses, default off. */ ++#include ++#include "cache-policy.h" ++/* Qualified by kernel-cost2b and combined-bench2c; conservative off switches remain. */ ++static const bool render_user_cache_import_wc = true; ++static const bool render_user_cache_read_va = true; ++static const bool render_user_cache_private_wc = true; ++static const bool render_user_cache_receipts; ++struct g5p_cache_receipt { ++ u64 job, start_cpu, start_wall, total_cpu, total_wall; ++ u64 requested[2][3], maintained[2][3], cpu[2][3], wall[2][3]; ++ u32 pid, serial; ++ bool enabled, policy, import_policy; ++}; ++#define G5P_CACHE_RECORDS 128 ++static struct g5p_cache_receipt g5p_cache_ring[G5P_CACHE_RECORDS]; ++static DEFINE_SPINLOCK(g5p_cache_ring_lock); ++static u32 g5p_cache_head; ++static int g5p_cache_show(struct seq_file *m, void *data) ++{ ++ struct g5p_cache_receipt r; ++ u32 head = READ_ONCE(g5p_cache_head), begin = head > G5P_CACHE_RECORDS ? head - G5P_CACHE_RECORDS : 0; ++ unsigned long flags; ++ seq_puts(m, "v=1 classes=private-wc,native-other,import time=task-runtime-ns wall=elapsed-ns\n"); ++ for (u32 n = begin; n < head; n++) { ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ r = g5p_cache_ring[n % G5P_CACHE_RECORDS]; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ if (r.serial != n + 1) continue; /* overwritten during this debugfs read */ ++ seq_printf(m, "job=%llu pid=%u policy=%u total_cpu_ns=%llu total_wall_ns=%llu", r.job, r.pid, r.policy, r.total_cpu, r.total_wall); ++ for (u32 phase = 0; phase < 2; phase++) ++ for (u32 c = 0; c < 3; c++) ++ seq_printf(m, " p%uc%u=%llu,%llu,%llu,%llu", phase, c, r.requested[phase][c], r.maintained[phase][c], r.cpu[phase][c], r.wall[phase][c]); ++ seq_putc(m, '\n'); ++ } ++ return 0; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-vm.inc b/drivers/gpu/drm/apple-g5p/drm-vm.inc +new file mode 100644 +index 000000000..644554156 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-vm.inc +@@ -0,0 +1,912 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* 16 KiB VM preparation; accepted user VMs freeze until reboot. */ ++ ++/* release8 memory accounting: every live G5P GEM object (native and imported) with the task ++ * that created it. One line per context release and per file close prints the totals; ++ * debugfs g5p_bos lists each object (size, references, handles, exported/imported, VM pins, ++ * creator) and each open file's handles, so a page held after its client exited is named. ++ */ ++static LIST_HEAD(g5p_bo_live); ++static DEFINE_SPINLOCK(g5p_bo_lock); ++static u32 g5p_bo_count; ++static u64 g5p_bo_bytes; ++ ++static void g5p_gem_free(struct drm_gem_object *obj); ++ ++static const struct drm_gem_object_funcs g5p_gem_funcs = { ++ .free = g5p_gem_free, ++ .print_info = drm_gem_shmem_object_print_info, ++ .pin = drm_gem_shmem_object_pin, ++ .unpin = drm_gem_shmem_object_unpin, ++ .get_sg_table = drm_gem_shmem_object_get_sg_table, ++ .vmap = drm_gem_shmem_object_vmap, ++ .vunmap = drm_gem_shmem_object_vunmap, ++ .mmap = drm_gem_shmem_object_mmap, ++ .export = g5p_gem_prime_export, ++ .vm_ops = &drm_gem_shmem_vm_ops, ++}; ++ ++/* perf60 fault address (render_user_fault_read, drm-render-v3.inc): bit 0 reads the MMU fault ++ * banks at a TA/3D fault event, bit 1 also at a silent or timed-out phase; any nonzero value also ++ * records every VM edit (MAP, UNMAP, render-target create/destroy) in a device-wide ring, so the ++ * fault address can be named. The ring holds an opaque VM pointer, never dereferenced. ++ */ ++ ++/* faultrec: the bank read and its decode alone at a TA/3D fault event (no per-job PM lanes, no ++ * PM snapshots, no VM edit log): one licensed read per fault, at most 32 per boot. ++ */ ++ ++/* perf60-fillrate.md §11.3 cut 5, kernel half (render-user-sync.h): cache maintenance by range. */ ++static const unsigned int render_user_sync_range = 3; /* release8 default */ ++MODULE_PARM_DESC( ++ render_user_sync_range, ++ "perf60 default0 (release8: whole-BO maintenance); bit0 BO_SYNC honours " ++ "offset/size; bit1 a job's ACCESS_CACHE clean/invalidate covers each resource's " ++ "BO range [offset, offset+size), page widened, instead of the whole BO (latched " ++ "per job at submit); BO_SYNC read live"); ++ ++/* CPU cache maintenance of [lo, hi) of a BO's DMA mapping, walked over its DMA segments. */ ++static void g5p_sync_sgt_range(struct device *dev, struct sg_table *sgt, u64 lo, ++ u64 hi, bool to_cpu) ++{ ++ struct scatterlist *sg; ++ u64 pos = 0, start, n; ++ unsigned int i; ++ ++ for_each_sgtable_dma_sg(sgt, sg, i) { ++ if (pos >= hi) ++ break; ++ if (g5p_sync_clip(pos, sg_dma_len(sg), lo, hi, &start, &n)) { ++ if (to_cpu) ++ dma_sync_single_for_cpu( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_single_for_device( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ } ++ pos += sg_dma_len(sg); ++ } ++} ++ ++#define G5P_VMLOG_N 1024U ++enum { G5P_VMLOG_MAP = 1, G5P_VMLOG_UNMAP, G5P_VMLOG_RT_NEW, G5P_VMLOG_RT_DEL }; ++static const char *const g5p_vmlog_ops[] = { "?", "map", "unmap", "rt-new", ++ "rt-del" }; ++ ++struct g5p_vmlog_ent { ++ u64 ns, va, size, offset; ++ const void *vm; ++ u32 op, handle, vm_id; ++ pid_t tgid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++static struct { ++ spinlock_t lock; /* head and the entries */ ++ u32 head; ++ struct g5p_vmlog_ent e[G5P_VMLOG_N]; ++} g5p_vmlog = { .lock = __SPIN_LOCK_UNLOCKED(g5p_vmlog.lock) }; ++ ++static void g5p_vmlog_add(u32 op, const void *vm, u32 vm_id, u64 va, u64 size, ++ u32 handle, u64 offset) ++{ ++ { ++ return; ++ } ++} ++ ++/* Latency123: leaf attributes of new user mappings (0 = Apple's CC | SLC bypass). */ ++ ++/* Stage 3: the device-wide pinned-memory budget. */ ++static const unsigned int render_pinned_mib = 1024; ++MODULE_PARM_DESC( ++ render_pinned_mib, ++ "MiB of BO memory all GPU VMs together may map; an object mapped in several VMs " ++ "or files (PRIME) counts once (default 1024)"); ++ ++static u32 g5p_pte_attrs(void) ++{ ++ static const u32 attrs[] = { 0xc, 0x4, 0x8, 0x0 }; ++ ++ return attrs[min_t(u32, (0), ARRAY_SIZE(attrs) - 1)]; ++} ++ ++static struct drm_gem_object *g5p_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct g5p_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); ++ ++ if (!bo) ++ return ERR_PTR(-ENOMEM); ++ bo->shmem.base.funcs = &g5p_gem_funcs; ++ INIT_LIST_HEAD(&bo->live); ++ return &bo->shmem.base; ++} ++ ++/* A fully created object enters the live list (an object the core frees on a failed init ++ * never does). ++ */ ++static void g5p_bo_track(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ bo->pid = current->tgid; ++ strscpy(bo->comm, current->comm, sizeof(bo->comm)); ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_add_tail(&bo->live, &g5p_bo_live); ++ g5p_bo_count++; ++ g5p_bo_bytes += obj->size; ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++} ++ ++static void g5p_gem_free(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ if (!list_empty(&bo->live)) { ++ list_del_init(&bo->live); ++ g5p_bo_count--; ++ g5p_bo_bytes -= obj->size; ++ } ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ drm_gem_shmem_object_free(obj); ++} ++ ++static int g5p_bos_show(struct seq_file *m, void *data) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct drm_device *drm = entry->dev; ++ u32 exported = 0, imported = 0, nohandle = 0; ++ u64 nohandle_bytes = 0; ++ struct drm_file *file; ++ struct g5p_bo *bo; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_for_each_entry(bo, &g5p_bo_live, live) { ++ struct drm_gem_object *obj = &bo->shmem.base; ++ bool imp = drm_gem_is_imported(obj); ++ ++ exported += !!READ_ONCE(obj->dma_buf); ++ imported += imp; ++ if (!READ_ONCE(obj->handle_count)) { ++ nohandle++; ++ nohandle_bytes += obj->size; ++ } ++ seq_printf( ++ m, ++ "bo size=%zuK refs=%u handles=%u exported=%c imported=%c vm_pins=%u by=%s[%d]\n", ++ obj->size >> 10, kref_read(&obj->refcount), ++ READ_ONCE(obj->handle_count), ++ READ_ONCE(obj->dma_buf) ? 'y' : 'n', imp ? 'y' : 'n', ++ bo->vm_pins, bo->comm, bo->pid); ++ } ++ seq_printf( ++ m, ++ "live=%u bytes=%lluK exported=%u imported=%u no_handle=%u/%lluK device_pinned=%lluK\n", ++ g5p_bo_count, g5p_bo_bytes >> 10, exported, imported, nohandle, ++ nohandle_bytes >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ mutex_lock(&drm->filelist_mutex); ++ list_for_each_entry(file, &drm->filelist, lhead) { ++ struct drm_gem_object *obj; ++ struct task_struct *task; ++ u64 bytes = 0; ++ u32 handles = 0; ++ struct pid *pid; ++ int id; ++ ++ spin_lock(&file->table_lock); ++ idr_for_each_entry(&file->object_idr, obj, id) { ++ handles++; ++ bytes += obj->size; ++ } ++ spin_unlock(&file->table_lock); ++ rcu_read_lock(); ++ pid = rcu_dereference(file->pid); ++ task = pid_task(pid, PIDTYPE_TGID); ++ seq_printf(m, "file %s[%d] handles=%u bytes=%lluK\n", ++ task ? task->comm : "?", pid_vnr(pid), handles, ++ bytes >> 10); ++ rcu_read_unlock(); ++ } ++ mutex_unlock(&drm->filelist_mutex); ++ return 0; ++} ++ ++static int g5p_create_bo(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_bo *a = data; ++ struct drm_gem_shmem_object *shmem; ++ int ret; ++ ++ if (!a->size || a->size > G5P_MAX_BO_SIZE || ++ (a->size & (G5P_GPU_PAGE - 1)) || a->handle || !a->flags || ++ (a->flags & ++ ~(DRM_G5P_BO_READ | DRM_G5P_BO_WRITE | DRM_G5P_BO_EXEC))) ++ return -EINVAL; ++ shmem = drm_gem_shmem_create(drm, a->size); ++ if (IS_ERR(shmem)) ++ return PTR_ERR(shmem); ++ /* Match the DMA API's coherent/noncoherent CPU mapping choice. The GPU ++ * cache contract remains an explicit COMPUTE capability prerequisite. ++ */ ++ shmem->map_wc = !dev_is_dma_coherent(drm->dev); ++ to_g5p_bo(&shmem->base)->roles = a->flags; ++ g5p_bo_track(&shmem->base); ++ ret = drm_gem_handle_create(file, &shmem->base, &a->handle); ++ drm_gem_object_put(&shmem->base); ++ return ret; ++} ++ ++static int g5p_bo_mmap_offset(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_mmap_offset *a = data; ++ struct drm_gem_object *obj; ++ int ret; ++ ++ if (a->flags || a->offset) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ ret = drm_gem_create_mmap_offset(obj); ++ if (!ret) ++ a->offset = drm_vma_node_offset_addr(&obj->vma_node); ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_bo_sync(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_sync *a = data; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *ctx; ++ unsigned long id; ++ int ret = 0; ++ ++ if (a->op != DRM_G5P_BO_SYNC_TO_DEVICE && ++ a->op != DRM_G5P_BO_SYNC_TO_CPU) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ mutex_lock(&f->lock); ++ xa_for_each(&f->contexts, id, ctx) ++ if (ctx->serial && ++ (ctx->bad || smp_load_acquire(&ctx->pending))) { ++ mutex_unlock(&f->lock); ++ ret = -EBUSY; ++ goto out; ++ } ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ mutex_unlock(&f->lock); ++ goto out; ++ } ++ /* render_user_sync_range bit 0: the requested range; else the whole BO (stronger, allowed). */ ++ if (READ_ONCE(render_user_sync_range) & 1) { ++ u64 lo, hi; ++ ++ if (g5p_sync_window(a->offset, a->size, obj->size, &lo, &hi)) ++ g5p_sync_sgt_range(drm->dev, sgt, lo, hi, ++ a->op == DRM_G5P_BO_SYNC_TO_CPU); ++ } else if (a->op == DRM_G5P_BO_SYNC_TO_DEVICE) { ++ dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } else { ++ dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ mutex_unlock(&f->lock); ++out: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_alloc_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ p->cpu = dma_alloc_coherent(dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ if (!g5p_user_pa_valid(p->dma) || dma_to_phys(dev, p->dma) != p->dma) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ return 0; ++} ++ ++static struct g5p_pt *g5p_vm_table(struct g5p_vm *vm, struct device *dev, ++ u32 level, u32 key, bool create) ++{ ++ struct g5p_pt *pt; ++ int ret; ++ ++ list_for_each_entry(pt, &vm->tables, link) ++ if (pt->level == level && pt->key == key) ++ return pt; ++ if (!create) ++ return NULL; ++ if (vm->table_count >= G5P_MAX_PT_PAGES) ++ return ERR_PTR(-ENOSPC); ++ pt = kzalloc(sizeof(*pt), GFP_KERNEL); ++ if (!pt) ++ return ERR_PTR(-ENOMEM); ++ ret = g5p_vm_alloc_page(dev, &pt->page); ++ if (ret) { ++ kfree(pt); ++ return ERR_PTR(ret); ++ } ++ pt->level = level; ++ pt->key = key; ++ list_add_tail(&pt->link, &vm->tables); ++ vm->table_count++; ++ return pt; ++} ++ ++static u8 *g5p_vm_pte(struct g5p_vm *vm, struct device *dev, u64 va, ++ bool create) ++{ ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ struct g5p_pt *dir, *leaf; ++ ++ dir = g5p_vm_table(vm, dev, 1, ci, create); ++ if (IS_ERR_OR_NULL(dir)) ++ return (void *)dir; ++ leaf = g5p_vm_table(vm, dev, 2, (ci << 9) | di, create); ++ if (IS_ERR_OR_NULL(leaf)) ++ return (void *)leaf; ++ if (create) { ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + ci * 4, ++ g5p_catalogue_entry(dir->page.dma), 0); ++ g5p_vm_entry(&vm->mutation, dir->page.cpu + di * 8, ++ g5p_directory_entry(leaf->page.dma), 1); ++ } ++ return leaf->page.cpu + ((va >> 14) & 127) * 8; ++} ++ ++static void g5p_vm_clear(struct g5p_vm *vm, struct device *dev, u64 va, ++ u64 size) ++{ ++ for (u64 off = 0; off < size; off += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, dev, va + off, false); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, 0, 2); ++ } ++ dma_wmb(); ++} ++ ++static bool g5p_vm_table_empty(struct g5p_pt *pt) ++{ ++ u32 count = pt->level == 2 ? 128 : 512; ++ ++ for (u32 i = 0; i < count; i++) ++ if (g5p_le32(pt->page.cpu + i * 8) & 1) ++ return false; ++ return true; ++} ++ ++static void g5p_vm_prune(struct device *dev, struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt, *tmp, *dir; ++ ++ /* These tables are mutable only before firmware attachment. ++ * Remove empty leaves before directories, including failed MAP rollback. ++ */ ++ for (u32 level = 2; level; level--) ++ list_for_each_entry_safe(pt, tmp, &vm->tables, link) { ++ if (pt->level != level || !g5p_vm_table_empty(pt)) ++ continue; ++ if (level == 2) { ++ dir = g5p_vm_table(vm, dev, 1, pt->key >> 9, ++ false); ++ if (dir) ++ g5p_vm_entry(&vm->mutation, ++ dir->page.cpu + (pt->key & ++ 511) * 8, ++ 0, 1); ++ } else { ++ g5p_vm_entry(&vm->mutation, ++ vm->root.cpu + pt->key * 4, 0, 0); ++ } ++ dma_wmb(); ++ list_del(&pt->link); ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ vm->table_count--; ++ kfree(pt); ++ } ++} ++ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj); ++ ++static void g5p_vm_release(struct kref *ref) ++{ ++ struct g5p_vm *vm = container_of(ref, struct g5p_vm, ref); ++ struct device *dev = vm->dev; ++ struct g5p_mapping *m, *mtmp; ++ struct g5p_pt *pt, *ptmp; ++ u64 pinned = vm->pinned_bytes; ++ u32 maps = vm->map_count, tables = vm->table_count; ++ ++ list_for_each_entry_safe(m, mtmp, &vm->maps, link) { ++ drm_mm_remove_node(&m->node); ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ list_del(&m->link); ++ kfree(m); ++ } ++ list_for_each_entry_safe(pt, ptmp, &vm->tables, link) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ list_del(&pt->link); ++ kfree(pt); ++ } ++ dma_free_coherent(dev, G5P_GPU_PAGE, vm->root.cpu, vm->root.dma); ++ xa_destroy(&vm->objs); ++ drm_mm_remove_node(&vm->reserved); ++ drm_mm_takedown(&vm->mm); ++ /* One line per freed VM: the device's pinned total must fall back after a client exits. */ ++ dev_info( ++ dev, ++ "G5P VM release maps=%u tables=%u pinned=%lluK device_pinned=%lluK\n", ++ maps, tables, pinned >> 10, ++ READ_ONCE(vm->gpu->pinned_bytes) >> 10); ++ kfree(vm); ++} ++ ++static void g5p_vm_put(struct g5p_vm *vm) ++{ ++ kref_put(&vm->ref, g5p_vm_release); ++} ++ ++static void g5p_context_release(struct kref *ref) ++{ ++ struct g5p_context *c = container_of(ref, struct g5p_context, ref); ++ g5p_render_release(c); ++ g5p_vm_put(c->vm); ++ kfree(c); ++} ++ ++static void g5p_context_put(struct g5p_context *c) ++{ ++ kref_put(&c->ref, g5p_context_release); ++} ++ ++static int g5p_create_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ u32 id; ++ int ret; ++ ++ if (a->flags || a->handle) ++ return -EINVAL; ++ vm = kzalloc(sizeof(*vm), GFP_KERNEL); ++ if (!vm) ++ return -ENOMEM; ++ kref_init(&vm->ref); ++ vm->dev = drm->dev; ++ vm->gpu = to_g5p(drm); ++ vm->bounded = (0); ++ vm->mutation.generations = true; ++ INIT_LIST_HEAD(&vm->maps); ++ xa_init(&vm->objs); ++ INIT_LIST_HEAD(&vm->tables); ++ drm_mm_init(&vm->mm, G5P_USER_VA_START, ++ G5P_USER_VA_END - G5P_USER_VA_START); ++ vm->reserved.start = G5P_USER_RESERVED_START; ++ vm->reserved.size = G5P_USER_RESERVED_END - G5P_USER_RESERVED_START; ++ ret = drm_mm_reserve_node(&vm->mm, &vm->reserved); ++ if (ret) ++ goto err_mm; ++ ret = g5p_vm_alloc_page(drm->dev, &vm->root); ++ if (ret) ++ goto err_reserved; ++ vm->table_count = 1; ++ mutex_lock(&f->lock); ++ ret = xa_alloc(&f->vms, &id, vm, XA_LIMIT(1, G5P_MAX_VMS), GFP_KERNEL); ++ mutex_unlock(&f->lock); ++ if (ret) { ++ g5p_vm_put(vm); ++ return ret; ++ } ++ a->handle = id; ++ return 0; ++err_reserved: ++ drm_mm_remove_node(&vm->reserved); ++err_mm: ++ drm_mm_takedown(&vm->mm); ++ kfree(vm); ++ return ret; ++} ++ ++static int g5p_destroy_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->handle); ++ if (!vm) ++ ret = -ENOENT; ++ else if (vm->contexts || f->submitted || vm->frozen) ++ ret = -EBUSY; ++ else ++ xa_erase(&f->vms, a->handle); ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_vm_put(vm); ++ return ret; ++} ++ ++static unsigned long g5p_vm_obj_count(struct g5p_vm *vm, ++ struct drm_gem_object *obj) ++{ ++ void *v = xa_load(&vm->objs, (unsigned long)obj); ++ ++ return v ? xa_to_value(v) : 0; ++} ++ ++/* ++ * One more mapping of `obj` in `vm`. Its first one in this VM counts its size against the ++ * VM's pin limit, and its first one in any VM against the device budget (render_pinned_mib). ++ * Our own dma-bufs import back as the same GEM object, in any file, so memory shared between ++ * clients (a client's buffer in the compositor's VM) counts once. ++ */ ++static int g5p_vm_obj_get(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ u64 limit = (u64)READ_ONCE(render_pinned_mib) << 20; ++ unsigned long count; ++ int ret = 0; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (!count && obj->size > G5P_MAX_PINNED_BYTES - vm->pinned_bytes) ++ ret = -ENOSPC; ++ else if (!count && !bo->vm_pins && ++ obj->size > limit - min(limit, d->pinned_bytes)) ++ ret = -ENOSPC; ++ if (!ret) ++ ret = xa_err(xa_store(&vm->objs, (unsigned long)obj, ++ xa_mk_value(count + 1), GFP_KERNEL)); ++ if (!ret && !count) { ++ vm->pinned_bytes += obj->size; ++ if (!bo->vm_pins++) ++ d->pinned_bytes += obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++ return ret; ++} ++ ++/* One mapping of `obj` in `vm` less: g5p_vm_obj_get() undone. Also called by the render ++ * scheduler for an unmap deferred behind queued jobs, hence the lock around the counts. ++ */ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long count; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (count > 1) { ++ /* replaces an existing entry: no allocation */ ++ xa_store(&vm->objs, (unsigned long)obj, xa_mk_value(count - 1), ++ GFP_KERNEL); ++ } else if (!WARN_ON(!count)) { ++ xa_erase(&vm->objs, (unsigned long)obj); ++ vm->pinned_bytes -= obj->size; ++ if (!--bo->vm_pins) ++ d->pinned_bytes -= obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++} ++ ++static int g5p_vm_map_bo(struct drm_device *drm, struct g5p_vm *vm, ++ struct drm_file *file, struct drm_g5p_vm_bind *a) ++{ ++ struct drm_gem_object *obj; ++ struct drm_gem_shmem_object *shmem; ++ struct sg_table *sgt; ++ struct sg_dma_page_iter iter; ++ struct g5p_mapping *m; ++ u64 *addresses, done = 0; ++ u32 count = 0; ++ bool imported; ++ int ret; ++ ++ if (vm->map_count >= G5P_MAX_MAPS || ++ a->size > G5P_MAX_VM_BYTES - vm->mapped_bytes) ++ return -ENOSPC; ++ obj = drm_gem_object_lookup(file, a->bo); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto err_obj; ++ } ++ ret = g5p_vm_obj_get(vm, obj); ++ if (ret) ++ goto err_obj; ++ addresses = kvmalloc_array(obj->size / G5P_GPU_PAGE, sizeof(*addresses), ++ GFP_KERNEL); ++ m = kzalloc(sizeof(*m), GFP_KERNEL); ++ if (!addresses || !m) { ++ ret = -ENOMEM; ++ goto err_alloc; ++ } ++ m->node.start = a->va; ++ m->node.size = a->size; ++ ret = drm_mm_reserve_node(&vm->mm, &m->node); ++ if (ret) { ++ ret = -EINVAL; ++ goto err_alloc; ++ } ++ imported = drm_gem_is_imported(obj); ++ shmem = to_drm_gem_shmem_obj(obj); ++ /* PRIME import mapped this attachment for our device and owns it until ++ * object release. Re-locking the exporter's reservation here can return ++ * -EBUSY after the first V3 submission even though the render VM is idle. ++ */ ++ if (imported) { ++ sgt = READ_ONCE(shmem->sgt); ++ if (!sgt) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=prime-no-sgt vm=%u bo=%u bytes=%llu\n", ++ a->vm, a->bo, a->size); ++ ret = -ENXIO; ++ goto err_node; ++ } ++ } else { ++ sgt = drm_gem_shmem_get_pages_sgt(shmem); ++ } ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=sgt-error vm=%u bo=%u imported=%u err=%d\n", ++ a->vm, a->bo, imported, ret); ++ goto err_node; ++ } ++ for_each_sgtable_dma_page(sgt, &iter, 0) { ++ u64 dma = sg_page_iter_dma_address(&iter); ++ ++ if (count >= obj->size / G5P_GPU_PAGE || ++ !g5p_user_pa_valid(dma) || ++ dma_to_phys(drm->dev, dma) != dma || ++ (!imported && dma_to_phys(drm->dev, dma) != ++ page_to_phys(shmem->pages[count]))) { ++ ret = -ERANGE; ++ goto err_node; ++ } ++ addresses[count++] = dma; ++ } ++ if (count != obj->size / G5P_GPU_PAGE) { ++ ret = -EINVAL; ++ goto err_node; ++ } ++ for (; done < a->size; done += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, drm->dev, a->va + done, true); ++ ++ if (IS_ERR_OR_NULL(pte)) { ++ ret = pte ? PTR_ERR(pte) : -ENOMEM; ++ goto err_clear; ++ } ++ /* Recovered bypass bits; user permission encodings are not claimed. ++ * A render VM latched to r25 memory variant 4 uses flags 0. ++ */ ++ g5p_vm_entry( ++ &vm->mutation, pte, ++ g5p_leaf_entry( ++ addresses[(a->offset + done) / G5P_GPU_PAGE], ++ vm->zero_leaf_flags ? 0 : g5p_pte_attrs()), ++ 2); ++ } ++ dma_wmb(); ++ m->obj = obj; ++ m->offset = a->offset; ++ list_add_tail(&m->link, &vm->maps); ++ vm->mapped_bytes += a->size; ++ vm->map_count++; ++ kvfree(addresses); ++ return 0; ++err_clear: ++ g5p_vm_clear(vm, drm->dev, a->va, done); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++err_node: ++ drm_mm_remove_node(&m->node); ++err_alloc: ++ kvfree(addresses); ++ kfree(m); ++ g5p_vm_obj_put(vm, obj); ++err_obj: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_bind(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_vm_bind *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_mapping *m; ++ struct g5p_vm *vm; ++ int ret = -ENOENT; ++ bool resident = false, locked = false; ++ u32 generation[3]; ++ ++ if (a->flags || !g5p_user_range_valid(a->va, a->size) || ++ (a->op != DRM_G5P_VM_MAP && a->op != DRM_G5P_VM_UNMAP) || ++ (a->op == DRM_G5P_VM_UNMAP && (a->bo || a->offset))) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->vm); ++ if (!vm) ++ goto out; ++ /* Its render context is closing: the close item owns the tables now, and the context ++ * may be freed at any time, so it is not looked at. ++ */ ++ if (vm->resident_gone) { ++ ret = -EBUSY; ++ goto out; ++ } ++ if ((f->submitted || vm->frozen) && !vm->resident) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=no-resident submitted=%u frozen=%u vm=%u op=%u\n", ++ f->submitted, vm->frozen, a->vm, a->op); ++ ret = -EBUSY; ++ goto out; ++ } ++ if (vm->resident) { ++ ret = g5p_resident_idle(vm->resident, true); ++ if (ret) ++ goto out; ++ /* A render context's edit is RAM only and touches its own tables and ++ * slots, which the firmware reads only at that context's next kick; ++ * f->lock plus resident-idle orders it against its submit (a MAP or ++ * UNMAP may pass queued jobs, see g5p_render_vm_idle()). Other contexts' jobs ++ * may hold the core lock meanwhile, so it is not taken. Serialized ++ * compute still uses the global mutex. ++ */ ++ if (!vm->resident->render) { ++ mutex_lock(&to_g5p(drm)->core->lock); ++ locked = true; ++ } ++ resident = true; ++ memcpy(generation, vm->mutation.generation, sizeof(generation)); ++ } ++ if (a->op == DRM_G5P_VM_MAP) { ++ if (vm->bounded && !(vm->resident && vm->resident->render) && ++ g5p_user_private_overlap(a->va, a->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_vm_map_bo(drm, vm, file, a); ++ if (!ret) ++ g5p_vmlog_add(G5P_VMLOG_MAP, vm, a->vm, a->va, a->size, ++ a->bo, a->offset); ++ if (ret == -EBUSY) ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=map vm=%u bo=%u submitted=%u frozen=%u resident=%u render=%u\n", ++ a->vm, a->bo, f->submitted, vm->frozen, ++ !!vm->resident, ++ !!(vm->resident && vm->resident->render)); ++ goto out; ++ } ++ list_for_each_entry(m, &vm->maps, link) { ++ if (m->node.start != a->va || m->node.size != a->size) { ++ if (m->node.start < a->va + a->size && ++ a->va < m->node.start + m->node.size) { ++ ret = -EINVAL; ++ break; ++ } ++ continue; ++ } ++ g5p_vmlog_add(G5P_VMLOG_UNMAP, vm, a->vm, a->va, a->size, 0, 0); ++ g5p_vm_clear(vm, drm->dev, a->va, a->size); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++ else { ++ ret = g5p_resident_invalidate(vm->resident); ++ if (ret) ++ goto out; ++ memcpy(generation, vm->mutation.generation, ++ sizeof(generation)); ++ } ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ vm->mapped_bytes -= a->size; ++ /* Beside queued jobs the PTEs are gone, but the pages stay pinned until ++ * those jobs have retired. ++ */ ++ if (!vm->resident || !g5p_resident_defer(vm->resident, m)) { ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ ret = 0; ++ break; ++ } ++out: ++ if (resident) { ++ int invalidate = 0; ++ ++ if (memcmp(generation, vm->mutation.generation, ++ sizeof(generation))) ++ invalidate = g5p_resident_invalidate(vm->resident); ++ ++ if (invalidate) ++ ret = invalidate; ++ if (locked) ++ mutex_unlock(&to_g5p(drm)->core->lock); ++ } ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_create_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_context *a = data; ++ if (a->type != DRM_G5P_CONTEXT_RENDER) ++ return -EOPNOTSUPP; ++ return g5p_render_create_context(drm, file, a); ++} ++ ++static int g5p_destroy_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_context *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->handle); ++ if (!c) ++ ret = -ENOENT; ++ else ++ ret = g5p_render_destroy(c); ++ if (!ret && c) { ++ c->vm->contexts--; ++ xa_erase(&f->contexts, a->handle); ++ } ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_context_put(c); ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/vm-ranges.h b/drivers/gpu/drm/apple-g5p/vm-ranges.h +new file mode 100644 +index 000000000..25bd98ee9 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/vm-ranges.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_H ++#define G5P_COMPUTE_USER_H ++ ++/* Public geometry is fixed across VMs. The private ranges never alias a BO. */ ++ ++static const u64 g5p_private_va[] = { ++ 0x300040000ULL, 0x300014000ULL, 0x300028000ULL, 0x300044000ULL, 0, ++ 0x100080000ULL, 0x100084000ULL, 0x1000a0000ULL, 0x1000a4000ULL, ++}; ++ ++static inline bool g5p_user_private_overlap(u64 va, u64 size) ++{ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_private_va); i++) ++ if (g5p_private_va[i] && va < g5p_private_va[i] + 0x4000 && ++ g5p_private_va[i] < va + size) ++ return true; ++ return false; ++} ++ ++#endif diff --git a/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch b/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch new file mode 100644 index 00000000..93dca40f --- /dev/null +++ b/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch @@ -0,0 +1,8973 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add typed render and compute submission + +[Orchard: from brkosebay/linux d67f133590a; normal probe registration +and fixed default policy.] Keep the A10 PowerVR driver and its build +hooks unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/Kconfig | 2 + + drivers/gpu/drm/Makefile | 2 + + drivers/gpu/drm/apple-g5p/Kconfig | 14 + + drivers/gpu/drm/apple-g5p/Makefile | 2 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 742 ++++ + .../gpu/drm/apple-g5p/compute-user-validate.h | 141 + + drivers/gpu/drm/apple-g5p/drm-compute-v1.h | 17 + + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 720 ++++ + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 293 ++ + drivers/gpu/drm/apple-g5p/drm-prime.inc | 114 + + .../gpu/drm/apple-g5p/drm-render-sched.inc | 203 ++ + .../gpu/drm/apple-g5p/drm-render-unbind.inc | 488 +++ + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 62 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 3038 +++++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-render.inc | 2549 ++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-submit.inc | 29 + + .../gpu/drm/apple-g5p/render-user-validate.h | 384 +++ + drivers/gpu/drm/apple-g5p/runtime-idle.inc | 14 + + 18 files changed, 8814 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/Kconfig + create mode 100644 drivers/gpu/drm/apple-g5p/Makefile + create mode 100644 drivers/gpu/drm/apple-g5p/a9-gpu-drm.c + create mode 100644 drivers/gpu/drm/apple-g5p/compute-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-compute-v1.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-compute-v1.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-latency123.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-prime.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-sched.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-unbind.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-submit.inc + create mode 100644 drivers/gpu/drm/apple-g5p/render-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/runtime-idle.inc + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 4c74c1501..eaf71742f 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -371,3 +371,5 @@ endif + # Separate option because drm_panel_orientation_quirks.c is shared with fbdev + config DRM_PANEL_ORIENTATION_QUIRKS + tristate ++ ++source "drivers/gpu/drm/apple-g5p/Kconfig" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index d75eba399..1315aa378 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -257,3 +257,5 @@ quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + + $(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) ++ ++obj-$(CONFIG_DRM_APPLE_G5P) += apple-g5p/ +diff --git a/drivers/gpu/drm/apple-g5p/Kconfig b/drivers/gpu/drm/apple-g5p/Kconfig +new file mode 100644 +index 000000000..1838fc90f +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Kconfig +@@ -0,0 +1,14 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_APPLE_G5P ++ bool "Apple A9/G5P render driver" ++ depends on DRM=y && ARCH_APPLE && OF && PM && ARM64_16K_PAGES ++ select DRM_GEM_SHMEM_HELPER ++ select FW_LOADER ++ select CRYPTO_LIB_SHA256 ++ help ++ Render-node driver for the S8003 G5P GPU using the unmodified META ++ firmware from the device. Supports TYPED_V3 jobs, GEM buffers, PRIME, ++ virtual address spaces and syncobj synchronization for the merged ++ Mesa winsys. The GPU is brought up at probe once the user's ++ apple/agx-g5p-c0.ldr is readable. Built-in only while reset and ++ active-DMA teardown require the platform boot sequence. +diff --git a/drivers/gpu/drm/apple-g5p/Makefile b/drivers/gpu/drm/apple-g5p/Makefile +new file mode 100644 +index 000000000..843d56a04 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Makefile +@@ -0,0 +1,2 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++obj-$(CONFIG_DRM_APPLE_G5P) += a9-gpu-drm.o +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +new file mode 100644 +index 000000000..f2fb4bf2d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -0,0 +1,742 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* Experimental render frontend; default disabled, no implicit firmware boot. */ ++#include "../../apple-g5p/core.inc" ++#include "../../apple-g5p/drm-validate.h" ++#include "../../apple-g5p/render-user-evwait.h" ++#include "../../apple-g5p/render-user-sync.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define G5P_MAX_VMS 8 ++#define G5P_MAX_CONTEXTS 32 ++#define G5P_MAX_MAPS 4096 ++#define G5P_MAX_PT_PAGES 512 ++#define G5P_MAX_SYNCS 17 ++ ++struct g5p_bo { ++ struct drm_gem_shmem_object shmem; ++ u32 roles; ++ u32 vm_pins; /* VMs that map it, any file; counted once in pinned_bytes (pin_lock) */ ++ /* release8 memory accounting (drm-vm.inc g5p_bo_live): the live list, its creator */ ++ struct list_head live; ++ pid_t pid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++struct g5p_pt { ++ struct list_head link; ++ struct g5p_dma_page page; ++ u32 key; ++ u32 level; ++}; ++ ++struct g5p_mapping { ++ struct list_head link; ++ struct drm_mm_node node; ++ struct drm_gem_object *obj; ++ u64 offset; ++ u64 retire; /* unmapped beside queued jobs: freed once this fence seqno has signalled */ ++}; ++ ++struct g5p_vm { ++ struct g5p_vm_mutation mutation; ++ bool bounded, frozen, serial_context; ++ bool zero_leaf_flags; /* render context latched r25 memory variant 4 */ ++ /* Its render context's handle is gone (g5p_render_close()): no VM_BIND, no new ++ * context; the context's close item and release own the tables. ++ */ ++ bool resident_gone; ++ struct g5p_context *resident; ++ struct kref ref; ++ struct device *dev; ++ struct g5p_drm *gpu; ++ struct drm_mm mm; ++ struct drm_mm_node reserved; ++ struct list_head maps; ++ struct xarray objs; /* object -> its mappings in this VM (xa_mk_value) */ ++ struct list_head tables; ++ struct g5p_dma_page root; ++ u64 mapped_bytes, pinned_bytes; ++ u32 table_count, map_count, contexts; ++}; ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rdev; ++ ++struct g5p_context { ++ struct g5p_drm *gpu; ++ ++ bool live, bad; ++ bool serial, pending; ++ ++ struct g5p_render *render; /* render contexts only, see drm-render.inc */ ++ struct kref ref; ++ struct g5p_vm *vm; ++}; ++ ++struct g5p_file { ++ /* Serializes VM/context handles, mappings and the one-shot submit state. */ ++ struct mutex lock; ++ struct xarray vms; ++ struct xarray contexts; ++ struct xarray results; ++ struct xarray targets; /* render targets; each holds its context */ ++ u32 render_results; ++ bool submitted; ++ u64 job_id; ++}; ++ ++struct g5p_drm { ++ struct drm_device drm; ++ struct g5p_diag *core; ++ /* Protects cached IRQ event/status fields; never held for hardware IO. */ ++ spinlock_t status_lock; ++ u32 last_event[11], notifications[4], first_fault[11]; ++ u64 event_serial, fault_serial; ++ struct delayed_work idle_work; ++ bool idle_suspended; ++ struct delayed_work autoboot; /* probe-time bring-up */ ++ unsigned long autoboot_deadline; ++ u32 autoboot_phase; ++ /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ ++ wait_queue_head_t ev_wq; ++ /* TYPED_V3 3D completion-event latch (render-user-retire.h): G5P_RV3_TD_WANT | endpoint ++ * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. ++ */ ++ u32 td_want; ++ u64 td_hit_ns; ++ /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ ++ bool ta_want; ++ u64 ta_hit_ns; ++ /* wake110: type-1 completion events decoded per engine (word 1 bit 0 TA, bit 1 3D) since ++ * load, and the decode time of the first one since the executor last cleared it. ++ */ ++ u32 t1_count[2]; ++ u64 t1_first_ns[2]; ++ /* perf60 bell gates (drm-render-v3.inc): the previous TYPED_V3 job's end and context. */ ++ u64 bell_prev_end_ns; ++ const void *bell_prev_ctx; ++ u64 last_job; ++ int last_error, job_error; ++ u32 job_state; ++ ++ atomic_t accepted; ++ atomic64_t job_sequence; ++ u64 fence_context; ++ /* Attached catalogues survive fd close until proven unbind/reset. */ ++ struct g5p_context *quarantined_context; ++ /* The first accepted render job claimed the device for render, for the rest of the boot. */ ++ bool render_claimed; ++ struct g5p_rdev *rdev; /* device-wide render state, see drm-render.inc */ ++ struct workqueue_struct ++ *render_wq; /* ordered: the render scheduler's worker */ ++ /* Orders render acceptance, dma_resv fence attach and queueing. */ ++ struct mutex render_submit_lock; ++ /* recovery108 epoch inputs, RAM only: open render-node files, render jobs not yet ++ * signalled. ++ */ ++ atomic_t open_files, r8_queued; ++ /* Memory mapped into any VM, each object once (render_pinned_mib). */ ++ struct mutex pin_lock; ++ u64 pinned_bytes; ++}; ++ ++static inline struct g5p_drm *to_g5p(struct drm_device *drm) ++{ ++ return container_of(drm, struct g5p_drm, drm); ++} ++ ++static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) ++{ ++ return container_of(obj, struct g5p_bo, shmem.base); ++} ++ ++#include "vm-ranges.h" ++ ++static int g5p_resident_idle(struct g5p_context *c, bool live); ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m); ++static int g5p_resident_invalidate(struct g5p_context *c); ++static void g5p_render_release(struct g5p_context *c); ++static int g5p_render_destroy(struct g5p_context *c); ++static void g5p_render_close(struct g5p_context *c); ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a); ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags); ++ ++#include "drm-vm.inc" ++#include "drm-submit.inc" ++#include "drm-render.inc" ++#include "drm-prime.inc" ++#include "drm-compute-v1.inc" ++ ++static int g5p_get_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_info *a = data; ++ u64 caps = DRM_G5P_CAP_BO_CPU | DRM_G5P_CAP_VM_PREPARE | ++ DRM_G5P_CAP_SYNCOBJ | DRM_G5P_CAP_JOB_STATUS | ++ DRM_G5P_CAP_CONTEXT_STATUS | DRM_G5P_CAP_FAULT_INFO; ++ ++ if (a->size != sizeof(*a) || a->flags || a->reserved[0] || ++ a->reserved[1]) ++ return -EINVAL; ++ ; ++ ; ++ ; ++ ; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ a->version = DRM_G5P_UAPI_VERSION; ++ a->gpu_page_shift = 14; ++ ; ++ ; ++ ; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER | DRM_G5P_CAP_BOUNDED_RENDER | ++ DRM_G5P_CAP_TYPED_RENDER | DRM_G5P_CAP_VM_LIVE | ++ DRM_G5P_CAP_CONTEXT_LIVE | ++ DRM_G5P_CAP_CODE_BASE_VERIFIED; ++ a->code_base = 0x300000000ULL; ++ }; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY; ++ }; ++ if ((1) && g5p_render_msaa_enabled()) ++ caps |= DRM_G5P_CAP_RENDER_MSAA; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_TYPED_RENDER_V3 | ++ DRM_G5P_CAP_RENDER_MULTI_CONTEXT | ++ DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH | ++ DRM_G5P_CAP_RENDER_OCCLUSION_QUERY; ++ }; ++ { ++ caps |= DRM_G5P_CAP_RENDER_UNPRIVILEGED; ++ }; ++ ; ++ ; ++ caps |= DRM_G5P_CAP_PRIME; ++ caps |= DRM_G5P_CAP_TYPED_COMPUTE; ++ a->capabilities = caps; ++ a->va_start = G5P_USER_VA_START; ++ a->va_end = G5P_USER_VA_END; ++ a->reserved_start = G5P_USER_RESERVED_START; ++ a->reserved_end = G5P_USER_RESERVED_END; ++ a->max_bo_size = G5P_MAX_BO_SIZE; ++ a->bvnc[0] = 9; ++ a->bvnc[1] = 15; ++ a->bvnc[2] = 6; ++ a->bvnc[3] = 2; ++ return 0; ++} ++ ++/* The gated dialect uses the common B proven by the r19c numerical store. */ ++static int g5p_get_exec_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_exec_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || a->pad || ++ memchr_inv(a->reserved, 0, sizeof(a->reserved))) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ { ++ a->base_scope = DRM_G5P_BASE_VM; ++ a->usc_base = 0x300000000ULL; ++ a->pds_base = a->usc_base; ++ a->window_size = 0x34000; ++ a->allocation_alignment = G5P_GPU_PAGE; ++ a->usc_offset_bits = 32; ++ a->pds_offset_bits = 32; ++ } ++ return 0; ++} ++ ++static int g5p_get_job_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_job_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ bool owned; ++ ++ if (a->flags || a->pad || !a->job_id || a->reset_generation) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ if (xa_load(&f->results, a->job_id)) { ++ struct g5p_fence *fence = xa_load(&f->results, a->job_id); ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = fence->state; ++ a->error = fence->error; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ mutex_unlock(&f->lock); ++ return 0; ++ } ++ owned = f->job_id == a->job_id; ++ mutex_unlock(&f->lock); ++ if (!owned) ++ return -ENOENT; ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = d->job_state; ++ a->error = d->job_error; ++ /* Owner acknowledges terminal status before another fixed submission. ++ * This avoids waking a fence waiter before the device admission reopens. ++ */ ++ ; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return 0; ++} ++ ++/* c99788 classifies event4/7/8. Preserve opaque fields, never invent a VA. */ ++static int g5p_get_fault_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_fault_info *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ ++ if (a->flags || a->reserved) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->event_serial = d->fault_serial; ++ memcpy(a->raw, d->first_fault, sizeof(a->raw)); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!a->event_serial) ++ return 0; ++ switch (a->raw[0]) { ++ case 4: ++ a->kind = DRM_G5P_FAULT_RECOVERY; ++ break; ++ case 7: ++ a->kind = DRM_G5P_FAULT_PAGE_REQUEST; ++ break; ++ case 8: ++ a->kind = DRM_G5P_FAULT_CHANNEL; ++ a->valid = DRM_G5P_FAULT_VALID_CHANNEL; ++ a->channel_class = a->raw[1]; ++ a->channel_token = a->raw[3]; ++ a->channel_stamp = a->raw[4]; ++ break; ++ default: ++ a->kind = DRM_G5P_FAULT_UNKNOWN; ++ break; ++ } ++ return 0; ++} ++ ++static int g5p_get_context_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_context_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(to_g5p(drm)->core->boot); ++ struct g5p_context *c; ++ bool failed; ++ int ret = 0; ++ ++ if (a->flags || a->state || a->map_count || a->mapped_bytes || ++ a->reserved) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c) { ++ ret = -ENOENT; ++ goto out; ++ } ++ a->state = DRM_G5P_CONTEXT_HOST_ONLY; ++ if (READ_ONCE(c->live)) ++ a->state = DRM_G5P_CONTEXT_LIVE; ++ /* a failure noted mid-job is not this context's until the executor rules */ ++ failed = b && READ_ONCE(b->failed) && ++ !g5p_rdev_ruling(to_g5p(drm)->rdev); ++ if (READ_ONCE(c->bad) || failed || (b && READ_ONCE(b->stopped))) ++ a->state = DRM_G5P_CONTEXT_LOST; ++ a->map_count = c->vm->map_count; ++ a->mapped_bytes = c->vm->mapped_bytes; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static void g5p_drm_event(void *cookie, const u32 words[11]) ++{ ++ struct g5p_drm *d = cookie; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ memcpy(d->last_event, words, sizeof(d->last_event)); ++ d->event_serial++; ++ if (words[0] == 1) ++ for (u32 i = 0; i < 4; i++) ++ d->notifications[i] |= words[i + 1]; ++ if (d->td_want && !d->td_hit_ns && ++ g5p_ru_rt_event_match(words, d->td_want & 0xff)) ++ WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); ++ if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) ++ WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); ++ for (u32 e = 0; words[0] == 1 && e < 2; e++) { ++ if (!(words[1] & BIT(e))) ++ continue; ++ WRITE_ONCE(d->t1_count[e], d->t1_count[e] + 1); ++ if (!d->t1_first_ns[e]) ++ WRITE_ONCE(d->t1_first_ns[e], ktime_get_ns()); ++ } ++ if (words[0] != 0 && words[0] != 1) { ++ d->last_error = -EIO; ++ if (!d->fault_serial) { ++ d->fault_serial = d->event_serial; ++ memcpy(d->first_fault, words, sizeof(d->first_fault)); ++ } ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ wake_up(&d->ev_wq); ++} ++ ++static int g5p_idle_open(struct g5p_drm *d); ++static void g5p_idle_queue(struct g5p_drm *d); ++ ++static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = kzalloc(sizeof(*f), GFP_KERNEL); ++ ++ if (!f) ++ return -ENOMEM; ++ mutex_init(&f->lock); ++ xa_init_flags(&f->vms, XA_FLAGS_ALLOC1); ++ xa_init_flags(&f->contexts, XA_FLAGS_ALLOC1); ++ xa_init(&f->results); ++ xa_init_flags(&f->targets, XA_FLAGS_ALLOC1); ++ file->driver_priv = f; ++ { ++ int ret = g5p_idle_open(to_g5p(drm)); ++ ++ if (ret) { ++ mutex_destroy(&f->lock); ++ kfree(f); ++ file->driver_priv = NULL; ++ return ret; ++ } ++ } ++ return 0; ++} ++ ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_vm *vm; ++ unsigned long id; ++ struct g5p_rt *rt; ++ ++ ++ /* Render-target handles hold their context; drop them first. */ ++ xa_for_each(&f->targets, id, rt) ++ g5p_context_put(rt->ctx); ++ xa_destroy(&f->targets); ++ /* Jobs hold contexts, contexts hold VMs, mappings hold GEM objects. ++ * Closing the fd drops handle ownership, never a queued job's references. ++ * Host-only tables may retire at job completion: no live GPU attachment. ++ */ ++ xa_for_each(&f->contexts, id, c) { ++ /* A render context's firmware binding outlives the handle until its close ++ * item, queued behind its jobs, has unbound it. ++ */ ++ if (c->render) ++ g5p_render_close(c); ++ g5p_context_put(c); ++ } ++ xa_destroy(&f->contexts); ++ { ++ struct g5p_fence *fence; ++ ++ xa_for_each(&f->results, id, fence) ++ dma_fence_put(&fence->base); ++ xa_destroy(&f->results); ++ } ++ xa_for_each(&f->vms, id, vm) ++ g5p_vm_put(vm); ++ xa_destroy(&f->vms); ++ mutex_destroy(&f->lock); ++ kfree(f); ++ atomic_dec(&to_g5p(drm)->open_files); ++ g5p_idle_queue(to_g5p(drm)); ++ /* The file's handles are gone (drm_gem_release() ran first): what stays live is held ++ * by another file's handle, a dma-buf, a VM or a job (debugfs g5p_bos names it). ++ */ ++ dev_info(drm->dev, ++ "G5P file close live_bos=%u live_bo_bytes=%lluK device_pinned=%lluK\n", ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++} ++ ++static const struct drm_ioctl_desc g5p_ioctls[] = { ++ DRM_IOCTL_DEF_DRV(G5P_GET_INFO, g5p_get_info, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_BO, g5p_create_bo, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_MMAP_OFFSET, g5p_bo_mmap_offset, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_SYNC, g5p_bo_sync, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_VM, g5p_create_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_VM, g5p_destroy_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_VM_BIND, g5p_vm_bind, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_CONTEXT, g5p_create_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_CONTEXT, g5p_destroy_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_EXEC_INFO, g5p_get_exec_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_JOB_STATUS, g5p_get_job_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_CONTEXT_STATUS, g5p_get_context_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_FAULT_INFO, g5p_get_fault_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_RENDER_TARGET, g5p_create_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_RENDER_TARGET, g5p_destroy_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_RENDER_INFO, g5p_get_render_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_RENDER_TYPED, g5p_submit_render_typed, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_COMPUTE_TYPED, g5p_submit_compute_typed, ++ DRM_RENDER_ALLOW), ++}; ++ ++DEFINE_DRM_GEM_FOPS(g5p_drm_fops); ++ ++static const struct drm_driver g5p_drm_driver = { ++ .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | ++ DRIVER_SYNCOBJ_TIMELINE, ++ .open = g5p_drm_open, ++ .postclose = g5p_drm_postclose, ++ .fops = &g5p_drm_fops, ++ .ioctls = g5p_ioctls, ++ .num_ioctls = ARRAY_SIZE(g5p_ioctls), ++ .gem_create_object = g5p_gem_create_object, ++ .prime_handle_to_fd = g5p_prime_handle_to_fd, ++ .prime_fd_to_handle = g5p_prime_fd_to_handle, ++ .gem_prime_import = g5p_gem_prime_import, ++ .gem_prime_import_sg_table = g5p_prime_import_sg_table, ++ .name = "apple_g5p", ++ .desc = "Apple A9 G5P experimental render driver", ++ .major = 0, ++ .minor = DRM_G5P_UAPI_VERSION, ++}; ++ ++/* recovery108: the frontend's epoch inputs for the retire layer. RAM only and lock free: ++ * the claimed context is compared, never dereferenced (it may be dropped concurrently). ++ */ ++static void g5p_drm_r8_epoch(void *cookie, struct g5p_r8_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ in->claimed = READ_ONCE(d->render_claimed) || ++ READ_ONCE(d->quarantined_context); ++ in->held = in->claimed && atomic_read(&d->open_files) > 0; ++ in->queued = queued > 0 ? queued : 0; ++} ++ ++/* The teardown DETACH phase. No fence is signalled here: with admission closed and the ++ * device frozen, the render scheduler fails each queued job exactly once (-ESHUTDOWN, ++ * without waiting for its inputs), and an in-flight job ends on its own RAM-only ++ * deadline. The count left to the workers is reported as the explicit abandonment. ++ */ ++static void g5p_drm_td_detach(void *cookie) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ g5p_r8_note_detach(&g5p_r8_state, queued > 0 ? queued : 0); ++ g5p_rsched_kick(d); ++} ++ ++#include "runtime-idle.inc" ++ ++static const bool autoboot = true; ++static const unsigned int autoboot_wait_s = 120; ++ ++/* The bootstrap's order: the one-time firmware, wake and identity steps, then the stages ++ * recovery repeats (recover.inc). ++ */ ++static const char *const g5p_autoboot_phases[] = { ++ "validate", "prepare", "wake", "id", "sample", "map", "arm", "load", "release", ++ "events", "control-prepare", "control-submit", "control-wait", ++}; ++ ++static void g5p_autoboot_work(struct work_struct *work) ++{ ++ struct g5p_drm *d = container_of(to_delayed_work(work), struct g5p_drm, autoboot); ++ struct g5p_diag *g = d->core; ++ bool ran; ++ int ret; ++ ++ while (d->autoboot_phase < ARRAY_SIZE(g5p_autoboot_phases)) { ++ const char *phase = g5p_autoboot_phases[d->autoboot_phase]; ++ ++ ret = g5p_run_phase(g, phase, &ran); ++ /* Built in: the firmware is on the root file system, which may not be mounted yet. */ ++ if (ret == -ENOENT && !d->autoboot_phase && ++ time_before(jiffies, d->autoboot_deadline)) { ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 2 * HZ); ++ return; ++ } ++ dev_info(g->dev, "autoboot phase=%s result=%d\n", phase, ret); ++ if (ret) { ++ dev_warn(g->dev, "autoboot stopped at %s (%d); the command file can continue\n", ++ phase, ret); ++ return; ++ } ++ d->autoboot_phase++; ++ } ++ dev_info(g->dev, "autoboot complete: GPU up\n"); ++ g5p_idle_queue(d); ++} ++ ++static int g5p_drm_probe(struct platform_device *pdev) ++{ ++ struct g5p_drm *d; ++ struct g5p_diag *g; ++ int ret; ++ ++ ; ++ ; ++ ret = g5p_probe(pdev); ++ if (ret) ++ return ret; ++ g = platform_get_drvdata(pdev); ++ d = devm_drm_dev_alloc(&pdev->dev, &g5p_drm_driver, struct g5p_drm, ++ drm); ++ if (IS_ERR(d)) { ++ ret = PTR_ERR(d); ++ goto err_pm; ++ } ++ d->core = g; ++ d->fence_context = dma_fence_context_alloc(1); ++ spin_lock_init(&d->status_lock); ++ init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->accepted, 0); ++ atomic64_set(&d->job_sequence, 0); ++ d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", 0); ++ if (IS_ERR_OR_NULL(d->render_wq)) { ++ ret = d->render_wq ? PTR_ERR(d->render_wq) : -ENOMEM; ++ goto err_pm; ++ } ++ ret = drmm_mutex_init(&d->drm, &d->render_submit_lock); ++ if (!ret) ++ ret = drmm_mutex_init(&d->drm, &d->pin_lock); ++ if (ret) ++ goto err_pm; ++ ret = g5p_render_init(d); ++ if (ret) ++ goto err_pm; ++ g->event_cookie = d; ++ g->event_notify = g5p_drm_event; ++ atomic_set(&d->open_files, 0); ++ atomic_set(&d->r8_queued, 0); ++ drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); ++ drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); ++ INIT_DELAYED_WORK(&d->idle_work, g5p_idle_work); ++ ret = drm_dev_register(&d->drm, 0); ++ if (ret) ++ goto err_pm; ++ g5p_r8_register_epoch(g5p_drm_r8_epoch, d); ++ g5p_td_register_detach(g5p_drm_td_detach, d); ++ g5p_kq_register(&g5p_kq_drm, d); ++ dev_info(&pdev->dev, ++ "render node registered; no GPU access, COMPUTE/RENDER capabilities absent\n"); ++ INIT_DELAYED_WORK(&d->autoboot, g5p_autoboot_work); ++ if (READ_ONCE(autoboot)) { ++ d->autoboot_deadline = jiffies + secs_to_jiffies(autoboot_wait_s); ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 0); ++ } ++ return 0; ++err_pm: ++ g->event_notify = NULL; ++ pm_runtime_disable(&pdev->dev); ++ return ret; ++} ++ ++static void g5p_drm_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g->event_cookie; ++ ++ cancel_delayed_work_sync(&d->autoboot); ++ cancel_delayed_work_sync(&d->idle_work); ++ drm_dev_unplug(&d->drm); ++ g5p_remove(pdev); ++ /* recovery108: the remove-time FINAL above was the callbacks' last use; d is ++ * devres-freed next. ++ */ ++ g5p_td_register_detach(NULL, NULL); ++ g5p_r8_register_epoch(NULL, NULL); ++ g5p_kq_register(NULL, NULL); ++} ++ ++static void g5p_drm_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g ? g->event_cookie : NULL; ++ ++ if (d) { ++ cancel_delayed_work_sync(&d->autoboot); ++ cancel_delayed_work_sync(&d->idle_work); ++ } ++ g5p_shutdown(pdev); ++} ++ ++static const struct of_device_id g5p_drm_match[] = { ++ { .compatible = "apple,s8003-gpu" }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, g5p_drm_match); ++ ++static struct platform_driver g5p_drm_platform = { ++ .probe = g5p_drm_probe, ++ .remove = g5p_drm_remove, ++ .shutdown = g5p_drm_shutdown, ++ .driver = { ++ .name = "apple-g5p-drm", ++ .of_match_table = g5p_drm_match, ++ .dev_groups = g5p_groups, ++ .suppress_bind_attrs = true, ++ }, ++}; ++ ++module_platform_driver(g5p_drm_platform); ++ ++/* Built-in only until reset can prove all DMA references are quiescent. */ ++MODULE_DESCRIPTION("Apple G5P render/GEM/VM experimental driver"); ++MODULE_LICENSE("Dual MIT/GPL"); ++MODULE_FIRMWARE(G5P_FW_NAME); +diff --git a/drivers/gpu/drm/apple-g5p/compute-user-validate.h b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +new file mode 100644 +index 000000000..ac16d4d0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +@@ -0,0 +1,141 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_VALIDATE_H ++#define G5P_COMPUTE_USER_VALIDATE_H ++ ++/* ++ * TYPED_COMPUTE pass checks over already-resolved resource ++ * ranges, as render-user-validate.h does for TYPED_V3. Pure, host-tested ++ * (scripts/gpu/compute-host/test_compute.c). Nothing here is a pointer the kernel ++ * dereferences; the stream's last word is read by the caller through the BO's own pages ++ * (g5p_cu_content()). Like TYPED_V3 the kernel does not decode the CDM records, the PDS ++ * programs or the USC code they name; it bounds every address the firmware or the CDM ++ * starts from and leaves the rest to the per-context VM and fault isolation. ++ * -EINVAL flags, alignment, sizes, a non-zero terminate control word, no terminator ++ * -ERANGE a PDS reference outside [pds_base, pds_base + code_window), or a range past ++ * hw_va_limit ++ * -ENOENT no listed range wholly covers the stream, a state word or a PDS reference ++ * -EACCES the range lacks READ (stream, PDS data), WRITE (state) or EXEC (PDS code) ++ */ ++#define G5P_CU_MAX_STREAM 0x10000U ++#define G5P_CU_STREAM_MIN 4U ++#define G5P_CU_TERMINATE 0x80000000U ++ ++/* The pass as the UAPI carries it, kernel copy. */ ++struct g5p_cu_pass { ++ u64 cdm_va; ++ u32 cdm_size, cdm_end; ++ u64 state[4]; ++ u64 tail; ++ u32 config[2]; ++ u32 store[3], resume[3], term[3]; /* code, data, control */ ++}; ++ ++/* Where the content rule reads: the range holding the stream. */ ++struct g5p_cu_loc { ++ u32 stream_range; ++}; ++ ++/* One {code, data} PDS pair: code in a READ|EXEC range, data in a READ range (g5p_ru_v3_ref()). */ ++static inline int g5p_cu_pds(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ const u32 *pds) ++{ ++ int ret = g5p_ru_v3_ref(lim, r, n, pds[0], true); ++ ++ return ret ?: g5p_ru_v3_ref(lim, r, n, pds[1], false); ++} ++ ++static inline int g5p_cu_pass_check(const struct g5p_cu_pass *p, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_cu_loc *loc) ++{ ++ const u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ u32 si; ++ int ret; ++ ++ if (p->cdm_size < G5P_CU_STREAM_MIN || ++ p->cdm_size > G5P_CU_MAX_STREAM || ++ ((p->cdm_va | p->cdm_size | p->cdm_end) & 3) || ++ p->cdm_end < G5P_CU_STREAM_MIN || p->cdm_end > p->cdm_size) ++ return -EINVAL; ++ if ((p->tail & 0xf) || p->term[2]) ++ return -EINVAL; ++ for (u32 i = 0; i < 4; i++) ++ if (p->state[i] & 3) ++ return -EINVAL; ++ /* Every listed range, as TYPED_V3: the CDM and its tasks read and write past them. */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, p->cdm_va, p->cdm_size, &si)) ++ return -ENOENT; ++ if (!(r[si].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ for (u32 i = 0; i < 5; i++) { ++ u64 va = i < 4 ? p->state[i] : p->tail; ++ u32 k; ++ ++ if (!g5p_ru_cover(r, n, va, i < 4 ? 4 : 16, &k)) ++ return -ENOENT; ++ if ((r[k].access & rw) != rw) ++ return -EACCES; ++ } ++ ret = g5p_cu_pds(lim, r, n, p->store); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->resume); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->term); ++ if (ret) ++ return ret; ++ loc->stream_range = si; ++ return 0; ++} ++ ++/* The last stream dword, at cdm_va + cdm_end - 4, is the CDM terminate word. */ ++static inline int g5p_cu_content(const u8 *tail) ++{ ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ return word == G5P_CU_TERMINATE ? 0 : -EINVAL; ++} ++ ++/* The firmware's register images of a checked pass. */ ++static inline void g5p_cu_regs_of(struct g5p_cu_regs *k, ++ const struct g5p_cu_pass *p) ++{ ++ k->stream = p->cdm_va; ++ k->terminate = p->cdm_va + p->cdm_end - 4; ++ for (u32 i = 0; i < 4; i++) ++ k->state[i] = p->state[i]; ++ k->tail = p->tail; ++ k->config[0] = p->config[0]; ++ k->config[1] = p->config[1]; ++ k->store = p->store[0] | (u64)p->store[1] << 32; ++ k->resume = p->resume[0] | (u64)p->resume[1] << 32; ++ k->term = p->term[0] | (u64)p->term[1] << 32; ++ k->store1 = p->store[2]; ++ k->resume1 = p->resume[2]; ++} ++ ++/* ++ * Submit-time admission of a compute job, on top of g5p_ru_v3_admit(): the CDM rides an epoch ++ * a render job brought up (the attach rings the TA bell; a compute job never does), and its ++ * own sequence is bounded like the render one. ++ */ ++enum g5p_cu_refusal { ++ G5P_CU_OK, ++ G5P_CU_NO_EPOCH, /* no render job has brought the device up in this epoch */ ++ G5P_CU_OVERFLOW, /* the CDM sequence is exhausted */ ++}; ++ ++static inline int g5p_cu_admit(bool device_up, u32 cu_sequence) ++{ ++ if (!device_up) ++ return G5P_CU_NO_EPOCH; ++ if (cu_sequence >= G5P_CU_SEQ_MAX) ++ return G5P_CU_OVERFLOW; ++ return G5P_CU_OK; ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.h b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +new file mode 100644 +index 000000000..3531cefb7 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_COMPUTE (drm-compute-v1.inc): what the TYPED_V3 executor, the scheduler's drain and the ++ * render device's bring-up call before the file is included. ++ */ ++#ifndef G5P_DRM_COMPUTE_V1_H ++#define G5P_DRM_COMPUTE_V1_H ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rjob; ++ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job); ++static bool g5p_cu_drain(struct g5p_drm *d); ++static void g5p_cu_device_init(struct g5p_drm *d); ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +new file mode 100644 +index 000000000..dcbe34c0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -0,0 +1,720 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * TYPED_COMPUTE: compute jobs on TYPED_V3 render contexts. ++ * ++ * A compute job is a TYPED_V3 job of its render context with one phase on the device's CDM ++ * channel instead of a TA and a 3D: it is queued, scheduled, fenced, quarantined and lost ++ * exactly like a render job (g5p_render_run(), drm-render-sched.inc, render-user-fault.h). ++ * It binds the context's slot pair the RAM-only way if the context has not run a job yet, ++ * builds the CDM job (compute-user-layout.h) from the pass the submit validated ++ * (compute-user-validate.h), publishes it on the CL ring with a guarded DM2 wake, and waits RAM only ++ * for coherent channel watermarks and both completion stamps. It never brings the device up (that rings the TA bell): ++ * an epoch without a render job refuses it before the hardware. ++ * ++ * Only the licensed publish-time wake touches registers; validation, waits, ++ * samples, retirement and recovery decisions are RAM-only. ++ */ ++ ++static u8 *g5p_cu_page(struct g5p_boot *b) ++{ ++ return b->shared[G5P_CU_OFFSET / G5P_GPU_PAGE].cpu + ++ G5P_CU_OFFSET % G5P_GPU_PAGE; ++} ++ ++static u32 g5p_cu_meta(void) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + G5P_CU_OFFSET); ++} ++ ++/* The CDM's global object (the boot layout's engine-2 object, R0 group, CPU 0x49 / GPU 0x53). */ ++static u32 g5p_cu_global_meta(struct g5p_boot *b) ++{ ++ return g5p_meta_address( ++ g5p_object_va(&b->layout, 0x51 + G5P_CU_ENGINE)); ++} ++ ++/* The render device's bring-up (g5p_render_device_prepare()), after the TA/3D channel init that ++ * clears the 3D page: the CDM channel at its epoch start. RAM only. ++ */ ++static void g5p_cu_device_init(struct g5p_drm *d) ++{ ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_R22_DEP + 0x280); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_RU_SLOT1_END); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE + G5P_CU_BYTES > ++ G5P_GPU_PAGE); ++ BUILD_BUG_ON(G5P_CU_DEP + G5P_CU_DEP_BYTES > G5P_CU_BYTES); ++ BUILD_BUG_ON(G5P_CU_W + G5P_CU_W_BYTES > G5P_CU_SKU); ++ g5p_cu_channel_init(g5p_cu_page(d->core->boot), g5p_cu_meta()); ++ d->rdev->cu_sequence = 0; ++ d->rdev->cu_target = 0; ++} ++ ++/* One RAM sample of the CDM channel for job `seq` (the slots of struct g5p_rv3_ev: stamp[0] = ++ * W258 mirror, stamp[1] = W25c, ctl = the five watermarks, idle[0..2] = the CDM global's ++ * G724/728/72c). RAM only. ++ */ ++static void g5p_cu_sample(struct g5p_diag *g, u32 seq, u32 target, ++ struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_cu_page(b), *global = g5p_ru_global(b, G5P_CU_ENGINE); ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ s->stamp[0] = g5p_le32(p + G5P_CU_START); ++ s->stamp[1] = g5p_le32(p + G5P_CU_END); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < 3; i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != target || s->cons > 255 || ++ !g5p_cu_stamp_sane(p, seq); ++ s->consumed = s->cons == target; ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2]; ++ s->retired = g5p_cu_retired(p, g5p_cu_meta(), seq); ++ s->pm_ok = true; ++} ++ ++#define G5P_CU_SLOW_MS 1000U ++ ++/* The CDM phase's RAM-only wait, the TYPED_V3 wait (g5p_rv3_wait()) for engine 2: the silence ++ * clock, the fault/protocol rules and the closing sample are the same; complete is the ring ++ * consumed plus the retirement predicate (the idle words are recorded, not required: render path). ++ */ ++/* Event serial is sampled before RAM, so an IRQ between sample and sleep ++ * cannot be lost. Any firmware event wakes us; only the full RAM retirement ++ * predicate may complete the job. No render-only engine-indexed observers. ++ */ ++static void g5p_cu_pause(struct g5p_drm *d, u64 seen) ++{ ++ u32 wait_us; ++ ++ if (g5p_ew_plan(READ_ONCE(render_user_event_wake), false, &wait_us) == ++ G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ return; ++ } ++ wait_event_hrtimeout(d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us)); ++} ++ ++static u32 g5p_cu_wait(struct g5p_drm *d, struct g5p_diag *g, u32 seq, ++ u32 target, struct g5p_ru_wait *w, struct g5p_rv3_ev *s, ++ u64 *consumed_ns, int *err) ++{ ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE(d->event_serial); ++ bool fault, silent, done; ++ ++ g5p_cu_sample(g, seq, target, s); ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed || silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = s->consumed && s->retired; ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (w->result) ++ return w->result; ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_CU_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ dev_warn( ++ g->dev, ++ "compute slow seq=%u ms=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x consumed=%u retired=%u events=%u\n", ++ seq, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, ++ s->cons, s->stamp[0], s->stamp[1], s->ctl[0], ++ s->ctl[1], s->ctl[2], s->ctl[3], s->ctl[4], ++ s->idle[0], s->idle[1], s->idle[2], s->consumed, ++ s->retired, READ_ONCE(g->boot->events)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ g5p_cu_pause(d, seen); ++ } ++} ++ ++/* The guarded publish-time wake, identical ordering to TA/3D with DM2. ++ * Core lock held; only this helper may touch registers in the compute path. ++ * The event lock closes the reset/fault race. Never called from wait/drain. ++ */ ++static int g5p_cu_wake(struct g5p_diag *g, u32 seq, u32 start, u32 target) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ u32 why, fence = U32_MAX, gpc = U32_MAX; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = G5P_WK_ALWAYS, ++ .e = G5P_CU_ENGINE, ++ .seq = seq, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = target, ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_CU_START), ++ g5p_le32(p + G5P_CU_END) }, ++ }; ++ why = g5p_wk_guard(&in); ++ if (!why) ++ why = g5p_wk_ring(&io, G5P_CU_ENGINE, &fence, &gpc); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_dbg(g->dev, ++ "compute wake seq=%u did=%u power=%#x fence=%#x gpc=%#x\n", seq, ++ why, in.power, fence, gpc); ++ return why == G5P_WK_DID_RANG ? 0 : -EHOSTDOWN; ++} ++ ++/* The CL publish of job `seq`: the global ring entry {type 2, channel, endpoint}, the channel's ++ * admission watermark, then the producer, followed by the guarded DM2 wake. Refused, with nothing written, unless the ++ * ring is idle and the channel stands at the previous job's endpoint and stamps. RAM only. ++ */ ++static int g5p_cu_publish(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_cu_endpoint(seq - 1); ++ u32 endpoint = g5p_cu_endpoint(seq), why; ++ ++ why = (u32)(start > 255) | (u32)(READ_ONCE(ring[0]) != start) << 1 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 2 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 3 | ++ (u32)(g5p_le32(p + G5P_CU_START) != seq - 1 || ++ g5p_le32(p + G5P_CU_END) != seq - 1) ++ << 4; ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "compute refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x stamp=%08x/%08x: nothing published\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + G5P_CU_START), g5p_le32(p + G5P_CU_END)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x69] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 2 : ++ j == 1 ? g5p_cu_meta() : ++ j == 2 ? endpoint : ++ 0); ++ } ++ rd->cu_target = (start + 1) & 255; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], rd->cu_target); ++ dma_wmb(); ++ if (g5p_ru_v3_verbose(r->jobs, 0)) ++ dev_dbg(g->dev, ++ "compute publish slot=%u endpoint=%u ring=%u W=%08x sequence=%u kick=dm2\n", ++ r->slot[0], endpoint, start, g5p_cu_meta() + G5P_CU_W, ++ seq); ++ return g5p_cu_wake(g, seq, start, rd->cu_target); ++} ++ ++/* The CDM phase timed out with the firmware alive: lose the job and its context, block the ++ * channel behind it until the scheduler's drain (g5p_cu_drain()) sees it retire. RAM only. ++ */ ++static int g5p_cu_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = G5P_CU_ENGINE; ++ s->drain_ms = 45000; ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "compute lost seq=%u slot=%u phase_ms=%lld drain_ms=%u: context lost, device kept, channel blocked\n", ++ seq, r->slot[0], ++ g5p_rv3_us(v->ta.start_ns, v->ta.end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++static void g5p_cu_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ const struct g5p_rv3 *v = &job->rv3; ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ dev_dbg(g->dev, ++ "compute job=%llu seq=%u ret=%d slot=%u consumed_us=%lld done_us=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x lost=%u bind_failed=%u\n", ++ job->id, v->seq, ret, r->slot[0], ++ g5p_rv3_us(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_us(v->ta_pub_ns, v->ta.end_ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ v->lost, v->bind_failed); ++} ++ ++/* One compute job, core lock held (g5p_render_run() via g5p_rv3_execute()). Returns 0, -EAGAIN ++ * (no epoch: nothing touched), -ETIMEDOUT, or a fault. ++ */ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (r->stage_done) ++ return -ECANCELED; ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ why = g5p_cu_admit(READ_ONCE(rd->up), rd->cu_sequence); ++ if (why) ++ return why == G5P_CU_NO_EPOCH ? -EAGAIN : -EOVERFLOW; ++ seq = rd->cu_sequence + 1; ++ if (!r->bound) { ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ } ++ /* The firmware owns the CL ring and the channel from here on: any failure quarantines. */ ++ v->touched = true; ++ g5p_cu_job(g5p_cu_page(b), g5p_cu_meta(), g5p_cu_global_meta(b), ++ r->slot[0], seq, &job->cu); ++ g5p_render_clean_job(g); ++ rd->cu_sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ ret = g5p_cu_publish(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, v->job_deadline_ns); ++ res = g5p_cu_wait(d, g, seq, rd->cu_target, &v->ta, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ /* The TA/3D paths' MMU fault-bank read, for the CDM (engine 2). */ ++ g5p_rv3_fault_read(d, r, seq, G5P_CU_ENGINE, res, v->ta.silent); ++ loss = g5p_rv3_loss(d, r, false, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && !g5p_cu_lose(d, r, v, seq)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, false)) { ++ g5p_rv3_lose_recover(d, r, v, seq, G5P_CU_ENGINE, ++ v->ta.silent ? "cdm-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "cdm-timeout" : ++ "cdm-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal(g, ++ res == G5P_RU_WAIT_TIMEOUT ? ++ G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "cdm-timeout" : ++ "cdm-fault"); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ if (!v->last[0].idle_ok) ++ dev_warn( ++ g->dev, ++ "compute retired seq=%u with idle=%08x,%08x,%08x set\n", ++ seq, v->last[0].idle[0], v->last[0].idle[1], ++ v->last[0].idle[2]); ++ v->phase = G5P_RV3_DONE; ++ ret = 0; ++out: ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), "cdm"); ++ g5p_cu_receipt(g, r, job, ret); ++ return ret; ++} ++ ++/* The scheduler's drain for a lost compute job (g5p_rv3_drain() hands engine 2 here): wait, RAM ++ * only and for the drain bound, for the stuck job to retire. It did: the channel is clear and ++ * the held jobs run. It did not (silent, fault, bound): the device recovers if it may, else the ++ * epoch ends. Returns false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_cu_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ struct g5p_ru_wait w = {}; ++ u32 res = G5P_RU_WAIT_FAULT; ++ int err = -EIO; ++ u64 now; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ now = ktime_get_ns(); ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (!READ_ONCE(b->failed) && !READ_ONCE(b->stopped)) ++ res = g5p_cu_wait(d, g, s->seq, rd->cu_target, &w, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res == G5P_RU_WAIT_COMPLETE) { ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "compute drained seq=%u slot=%u lost_ms=%lld: channel clear, held jobs resume\n", ++ s->seq, r->slot[0], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "compute drain dead seq=%u why=%s err=%d lost_ms=%lld: the device recovers if it may, else the epoch ends\n", ++ s->seq, ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal(g, ++ fault ? G5P_RU_TERM_FAULT : ++ G5P_RU_TERM_TIMEOUT, ++ fault ? "cdm-drain-fault" : ++ "cdm-drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++static void g5p_cu_pass_from_uapi(struct g5p_cu_pass *p, ++ const struct drm_g5p_compute_pass *u) ++{ ++ *p = (struct g5p_cu_pass){ ++ .cdm_va = u->cdm_va, ++ .cdm_size = u->cdm_size, ++ .cdm_end = u->cdm_end, ++ .tail = u->state_tail_va, ++ .config = { u->config[0], u->config[1] }, ++ .store = { u->store.code, u->store.data, u->store.control }, ++ .resume = { u->resume.code, u->resume.data, u->resume.control }, ++ .term = { u->terminate.code, u->terminate.data, ++ u->terminate.control }, ++ }; ++ for (u32 i = 0; i < 4; i++) ++ p->state[i] = u->state_va[i]; ++} ++ ++static bool g5p_cu_uapi_clean(const struct drm_g5p_compute_pass *u) ++{ ++ return u->size == sizeof(*u) && !u->flags && !u->store.pad && ++ !u->resume.pad && !u->terminate.pad && ++ !memchr_inv(u->reserved, 0, sizeof(u->reserved)); ++} ++ ++/* Acceptance, as g5p_render_submit_admitted() for TYPED_V3 minus the render target: every error ++ * precedes the fence replacement and the queue. ++ */ ++static int g5p_cu_submit_admitted(struct drm_device *drm, struct drm_file *file, ++ const struct drm_g5p_submit_compute_typed *a, ++ u64 *job_id) ++{ ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_compute_pass *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_cu_pass pass; ++ struct g5p_cu_loc loc = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ u8 tail[4]; ++ u64 point = 0; ++ int ret; ++ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(a->resources), ++ a->resource_count, sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(a->syncs), a->sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ upass = memdup_user(u64_to_user_ptr(a->pass), sizeof(*upass)); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (!g5p_cu_uapi_clean(upass)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_cu_pass_from_uapi(&pass, upass); ++ ranges = kvcalloc(a->resource_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(a->resource_count, sizeof(*job->objs), ++ GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, a->sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ /* Compute rides an epoch a render job brought up; a recovery in progress is held. */ ++ if (!READ_ONCE(d->rdev->recovering) && ++ g5p_cu_admit(READ_ONCE(d->rdev->up), ++ READ_ONCE(d->rdev->cu_sequence))) { ++ ret = READ_ONCE(d->rdev->up) ? -EOVERFLOW : -EAGAIN; ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, a->resource_count, ranges, ++ job->objs, &job->obj_count); ++ if (ret) ++ goto unlock; ++ /* Structure first; the stream's last dword is read only once its range is proven backed. */ ++ ret = g5p_cu_pass_check(&pass, &lim, ranges, a->resource_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass.cdm_va + pass.cdm_end - 4, tail, ++ sizeof(tail)); ++ if (!ret) ++ ret = g5p_cu_content(tail); ++ if (ret) ++ goto unlock; ++ g5p_cu_regs_of(&job->cu, &pass); ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = true; ++ job->compute = true; ++ r->v3_accepted = true; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++static_assert(sizeof(struct drm_g5p_compute_pds) == 16); ++static_assert(sizeof(struct drm_g5p_compute_pass) == 152); ++static_assert(offsetof(struct drm_g5p_compute_pass, state_va) == 24); ++static_assert(offsetof(struct drm_g5p_compute_pass, config) == 64); ++static_assert(offsetof(struct drm_g5p_compute_pass, store) == 72); ++static_assert(offsetof(struct drm_g5p_compute_pass, reserved) == 120); ++static_assert(sizeof(struct drm_g5p_submit_compute_typed) == 72); ++static_assert(DRM_G5P_CAP_TYPED_COMPUTE == BIT_ULL(30)); ++static_assert(DRM_G5P_COMPUTE_MAX_STREAM_BYTES == G5P_CU_MAX_STREAM); ++static_assert(DRM_G5P_COMPUTE_STREAM_TERMINATE == G5P_CU_TERMINATE); ++ ++static int g5p_submit_compute_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_compute_typed *a = data; ++ u64 id = 0; ++ int ret; ++ ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_compute_pass)) ++ return -EINVAL; ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_cu_submit_admitted(drm, file, a, &id); ++ g5p_r8_exit(); ++ a->job_id = id; ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +new file mode 100644 +index 000000000..018e83a5b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -0,0 +1,293 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: runtime A/B knobs for the V3 render path, all default off and all writable ++ * through /sys/module/a9_gpu_drm/parameters on a live boot. Included from drm-render.inc ++ * before drm-render-v3.inc. The executor is serial and holds the core lock, so the state ++ * below needs no lock of its own. ++ * ++ * fw_tick_mask (boot.inc) ctl+0x04, re-applied before every job (RAM) ++ * slc_evict_us while a job is in flight, one pass over slc_evict_mib MiB of ++ * cacheable kernel memory at most every N us (the waiter's ++ * 1-2 ms sample cadence bounds it from below); slc_evict_ops: ++ * 1 read pass, 2 write pass, 4 Normal-NC read of the firmware's ++ * root+shared pages, 8 TLBI VMALLE1IS, 16 DSB SY + SEV ++ * cpu_sync 1 dma_sync every job BO (device before, CPU after), 2 clean ++ * root+shared to PoC before the publish, 4 clean+invalidate the ++ * polled pages before every sample ++ * lat123_log one RENDER-L123 line per job (times in ms from the submit ioctl) ++ * pstate_probe 1 PMGR gfx word at job start/end (always-on block), 2 also one ++ * read of Fender +0x1000 at the first sample that shows a fresh ++ * engine busy rise (engine running, so the block is powered) ++ * ++ * No other register access. Temporary: none of this is the fix. ++ */ ++ ++#include ++#include ++ ++#define G5P_L123_OP_READ BIT(0) ++#define G5P_L123_OP_WRITE BIT(1) ++#define G5P_L123_OP_FW_READ BIT(2) ++#define G5P_L123_OP_TLBI BIT(3) ++#define G5P_L123_OP_SEV BIT(4) ++#define G5P_L123_SYNC_BO BIT(0) ++#define G5P_L123_SYNC_REGIONS BIT(1) ++#define G5P_L123_SYNC_POLL BIT(2) ++#define G5P_L123_LINE 64 ++ ++static struct { ++ void *buf; ++ size_t size; ++ u64 sink; ++ /* per job */ ++ u64 exec_ns, next_ns, evict_ns, ps_ns; ++ u32 us, mib, ops, sync, passes, tick, ps; ++ u32 pmgr[2]; ++ bool seen[2], idle[2], probed; ++ /* render_user_event_wake: pauses ended by an event, the timeout, a post-event poll, legacy */ ++ u32 ew[4]; ++ u64 ew_ns; ++} g5p_l123; ++ ++/* perf60-fillrate.md §11.3 cut 1 (render-user-evwait.h): the phase wait's pause between RAM samples. */ ++static const unsigned int render_user_event_wake = 2000; /* release8 default */ ++MODULE_PARM_DESC( ++ render_user_event_wake, ++ "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " ++ "until the next firmware event (the IRQ thread's decode wakes the waiter), at " ++ "most N us (100..20000) as the safety net, and poll every 100 us once the " ++ "phase's own completion event is latched; read live"); ++ ++/* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ ++static u8 *g5p_l123_word(struct g5p_boot *b, u32 slot, u32 field) ++{ ++ u32 off = b->layout.offset[slot] + field; ++ struct g5p_dma_page *page = b->layout.group[slot] ? b->shared : b->root; ++ ++ return page[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_l123_pages(struct g5p_dma_page *page, u32 count, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ for (u32 i = 0; i < count; i++) ++ if (page[i].cpu) ++ op((unsigned long)page[i].cpu, ++ (unsigned long)page[i].cpu + G5P_GPU_PAGE); ++} ++ ++static void g5p_l123_page_of(const void *p, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ unsigned long start = (unsigned long)p & ++ ~(unsigned long)(G5P_GPU_PAGE - 1); ++ ++ op(start, start + G5P_GPU_PAGE); ++} ++ ++/* cpu_sync 4: the pages the waiter samples (rings, TA/3D/PM pages, the idle words). */ ++static void g5p_l123_poll_pages(struct g5p_boot *b, u32 e) ++{ ++ g5p_l123_page_of(g5p_boot_shared(b, 0x60), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_boot_shared(b, 0x64), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 0), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 1), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_pm_cpu(b), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_global(b, e) + 0x724, dcache_clean_inval_poc); ++ dsb(sy); ++} ++ ++static void g5p_l123_bos(struct g5p_drm *d, struct g5p_rjob *job, bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct sg_table *sgt = ++ to_drm_gem_shmem_obj(job->objs[i].obj)->sgt; ++ ++ if (!sgt) ++ continue; ++ if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ } ++} ++ ++static u32 g5p_l123_pmgr(struct g5p_diag *g) ++{ ++ u32 raw = U32_MAX; ++ ++ if (!((0) & 1) || !g->pmgr || regmap_read(g->pmgr, 0x80340, &raw)) ++ return U32_MAX; ++ return raw; ++} ++ ++/* At the executor's entry: this job's knob values and counters. */ ++static void g5p_l123_job_reset(struct g5p_diag *g) ++{ ++ g5p_l123.exec_ns = ktime_get_ns(); ++ g5p_l123.next_ns = 0; ++ g5p_l123.evict_ns = 0; ++ g5p_l123.ps_ns = 0; ++ g5p_l123.passes = 0; ++ g5p_l123.ps = U32_MAX; ++ g5p_l123.probed = false; ++ memset(g5p_l123.ew, 0, sizeof(g5p_l123.ew)); ++ g5p_l123.ew_ns = 0; ++ g5p_l123.seen[0] = false; ++ g5p_l123.seen[1] = false; ++ g5p_l123.us = (0); ++ g5p_l123.mib = clamp((16), 1U, 64U); ++ g5p_l123.ops = (1); ++ g5p_l123.sync = (0); ++ g5p_l123.pmgr[0] = g5p_l123_pmgr(g); ++ g5p_l123.pmgr[1] = U32_MAX; ++} ++ ++/* After the job's records are written, before they are cleaned and published. May sleep. */ ++static void g5p_l123_job_prepare(struct g5p_drm *d, struct g5p_rjob *job) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u8 *tick = g5p_l123_word(b, G5P_L123_CTL, 4); ++ u32 mask = (0); ++ size_t size = (size_t)g5p_l123.mib << 20; ++ ++ if (mask <= G5P_L123_TICK_MASK && g5p_le32(tick) != mask) { ++ g5p_put32(tick, mask); ++ dma_wmb(); ++ } ++ g5p_l123.tick = g5p_le32(tick); ++ ++ if (!g5p_l123.us || ++ !(g5p_l123.ops & (G5P_L123_OP_READ | G5P_L123_OP_WRITE))) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = NULL; ++ g5p_l123.size = 0; ++ } else if (g5p_l123.size != size) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = vmalloc(size); ++ g5p_l123.size = g5p_l123.buf ? size : 0; ++ if (g5p_l123.buf) ++ memset(g5p_l123.buf, 0x5a, size); ++ } ++ ++ if (g5p_l123.sync & G5P_L123_SYNC_BO) ++ g5p_l123_bos(d, job, false); ++ if (g5p_l123.sync & G5P_L123_SYNC_REGIONS) { ++ g5p_l123_pages(b->root, ARRAY_SIZE(b->root), dcache_clean_poc); ++ g5p_l123_pages(b->shared, ARRAY_SIZE(b->shared), ++ dcache_clean_poc); ++ dsb(sy); ++ } ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(b, 0); ++} ++ ++/* One streaming pass and the other per-pass operations. */ ++static void g5p_l123_evict(struct g5p_boot *b) ++{ ++ u64 t0 = ktime_get_ns(), sum = 0; ++ u32 ops = g5p_l123.ops; ++ ++ if ((ops & G5P_L123_OP_READ) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ sum += READ_ONCE(p[i]); ++ } ++ if ((ops & G5P_L123_OP_WRITE) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ WRITE_ONCE(p[i], i); ++ } ++ if (ops & G5P_L123_OP_FW_READ) { ++ struct g5p_dma_page *regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; ++ regions[r][i].cpu && at < G5P_GPU_PAGE; ++ at += G5P_L123_LINE) ++ sum += READ_ONCE(*( ++ const u32 *)(regions[r][i].cpu + ++ at)); ++ } ++ if (ops & G5P_L123_OP_TLBI) ++ flush_tlb_all(); ++ if (ops & G5P_L123_OP_SEV) { ++ dsb(sy); ++ sev(); ++ } ++ WRITE_ONCE(g5p_l123.sink, g5p_l123.sink + sum); ++ g5p_l123.passes++; ++ g5p_l123.evict_ns += ktime_get_ns() - t0; ++} ++ ++/* Replaces the waiter's sleep between two RAM samples of engine e. seen: d->event_serial as ++ * read after the drain and before this sample (render-user-evwait.h). ++ */ ++static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, ++ const struct g5p_rv3_ev *s, u64 seen) ++{ ++ u64 now = s->ns, t0; ++ enum g5p_ew_act act; ++ u32 wait_us; ++ ++ if (((0) & 2) && !g5p_l123.probed && g5p_l123.seen[e] && ++ g5p_l123.idle[e] && !s->idle_ok && !s->failed && !s->proto && ++ !g5p_ru_term_failed(&g5p_rv3_term) && !READ_ONCE(g->boot->failed)) { ++ struct g5p_mmio_ticket t; ++ ++ /* an R1-only ticket of the register licence (mmio.inc), never a bare read */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ g5p_l123.ps = t.gpc; ++ g5p_l123.ps_ns = ktime_get_ns(); ++ g5p_l123.probed = true; ++ } ++ g5p_l123.seen[e] = true; ++ g5p_l123.idle[e] = s->idle_ok; ++ if (g5p_l123.us && now >= g5p_l123.next_ns) { ++ g5p_l123_evict(g->boot); ++ g5p_l123.next_ns = now + (u64)g5p_l123.us * NSEC_PER_USEC; ++ } ++ act = g5p_ew_plan(READ_ONCE(render_user_event_wake), s->event, ++ &wait_us); ++ t0 = ktime_get_ns(); ++ if (act == G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ g5p_l123.ew[3]++; ++ } else if (act == G5P_EW_SHORT) { ++ usleep_range(wait_us, wait_us + wait_us / 2); ++ g5p_l123.ew[2]++; ++ } else if (wait_event_hrtimeout( ++ d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us))) { ++ g5p_l123.ew[1]++; /* -ETIME: the safety net */ ++ } else { ++ g5p_l123.ew[0]++; ++ } ++ g5p_l123.ew_ns += ktime_get_ns() - t0; ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(g->boot, e); ++} ++ ++/* After the receipt. One line per job while lat123_log is set. */ ++static void g5p_l123_job_end(struct g5p_drm *d, struct g5p_rjob *job, u32 seq, ++ int ret) ++{ ++ struct g5p_diag *g = d->core; ++ ++ if (!ret && (g5p_l123.sync & G5P_L123_SYNC_BO)) ++ g5p_l123_bos(d, job, true); ++ g5p_l123.pmgr[1] = g5p_l123_pmgr(g); ++ { ++ return; ++ } ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-prime.inc b/drivers/gpu/drm/apple-g5p/drm-prime.inc +new file mode 100644 +index 000000000..bd513b969 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-prime.inc +@@ -0,0 +1,114 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* PRIME for shmem GEM BOs. ++ * ++ * Export: Mesa WSI/client buffers and kmsro bookkeeping. The dma-buf adds ++ * begin/end_cpu_access, so an importer that CPU-reads (simpledrm's blit, ++ * llvmpipe) gets cache maintenance on the G5P DMA mapping. vmap/mmap reuse ++ * the shmem helpers (write-combined when the device is not DMA-coherent). ++ * ++ * Import: foreign dma-bufs, e.g. simpledrm dumb buffers that the GPU renders ++ * into for scanout. The attachment stays mapped (pinned) for the life of the ++ * GEM object; VM_BIND maps its sg_table. Render jobs listing an imported BO ++ * get whole-buffer cache maintenance and dma_resv fences (drm-render.inc). ++ */ ++ ++static int g5p_dmabuf_begin_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ /* No device mapping yet means no device-owned cache lines either. */ ++ if (sgt) ++ dma_sync_sgtable_for_cpu(obj->dev->dev, sgt, DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static int g5p_dmabuf_end_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ if (sgt) ++ dma_sync_sgtable_for_device(obj->dev->dev, sgt, ++ DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static const struct dma_buf_ops g5p_dmabuf_ops = { ++ .attach = drm_gem_map_attach, ++ .detach = drm_gem_map_detach, ++ .map_dma_buf = drm_gem_map_dma_buf, ++ .unmap_dma_buf = drm_gem_unmap_dma_buf, ++ .release = drm_gem_dmabuf_release, ++ .mmap = drm_gem_dmabuf_mmap, ++ .vmap = drm_gem_dmabuf_vmap, ++ .vunmap = drm_gem_dmabuf_vunmap, ++ .begin_cpu_access = g5p_dmabuf_begin_cpu_access, ++ .end_cpu_access = g5p_dmabuf_end_cpu_access, ++}; ++ ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags) ++{ ++ struct dma_buf_export_info info = { ++ .exp_name = KBUILD_MODNAME, ++ .owner = obj->dev->driver->fops->owner, ++ .ops = &g5p_dmabuf_ops, ++ .size = obj->size, ++ .flags = flags, ++ .priv = obj, ++ .resv = obj->resv, ++ }; ++ ++ return drm_gem_dmabuf_export(obj->dev, &info); ++} ++ ++static int g5p_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file, ++ u32 handle, u32 flags, int *fd) ++{ ++ return drm_gem_prime_handle_to_fd(dev, file, handle, flags, fd); ++} ++ ++static int g5p_prime_fd_to_handle(struct drm_device *dev, struct drm_file *file, ++ int fd, u32 *handle) ++{ ++ return drm_gem_prime_fd_to_handle(dev, file, fd, handle); ++} ++ ++/* Our own export comes back as the same GEM object (the core's identity check ++ * only knows its default ops); anything else is attached and mapped. ++ */ ++static struct drm_gem_object *g5p_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *buf) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ ++ if (buf->ops == &g5p_dmabuf_ops && obj->dev == dev) { ++ drm_gem_object_get(obj); ++ return obj; ++ } ++ return drm_gem_prime_import(dev, buf); ++} ++ ++/* Foreign buffers are GPU READ|WRITE data, never executable, and must be ++ * whole 16KiB GPU pages within the BO size limit. ++ */ ++static struct drm_gem_object * ++g5p_prime_import_sg_table(struct drm_device *dev, ++ struct dma_buf_attachment *attach, ++ struct sg_table *sgt) ++{ ++ struct drm_gem_object *obj; ++ ++ if (attach->dmabuf->size & (G5P_GPU_PAGE - 1) || ++ attach->dmabuf->size > G5P_MAX_BO_SIZE) ++ return ERR_PTR(-EINVAL); ++ obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); ++ if (!IS_ERR(obj)) { ++ to_g5p_bo(obj)->roles = DRM_G5P_BO_READ | DRM_G5P_BO_WRITE; ++ g5p_bo_track(obj); ++ } ++ return obj; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +new file mode 100644 +index 000000000..21dc64d89 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -0,0 +1,203 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * The render scheduler (docs/gpu/multi-client-design.md "Stage 2/3 as built"): one worker on ++ * the ordered render_wq serves the contexts round-robin (core/render-user-sched.h). Jobs wait ++ * for their inputs on dma-fence callbacks, not in the worker, so one client's unsignalled ++ * fence never holds up another client. Each picked job runs to completion on the one shared ++ * channel (one credit), exactly as a work item did before; a context's close item runs after ++ * its last job. Included from drm-render.inc after drm-render-v3.inc. ++ * ++ * The shape is drm_sched's (entity FIFO, run-list RR, dependency callbacks, entity teardown ++ * after the last job), so stage 4 can replace it with drm_sched without a UAPI change. ++ */ ++ ++static struct dma_fence *g5p_rjob_dep(struct g5p_rjob *job, u32 i) ++{ ++ return i < job->input_count ? job->inputs[i] : ++ job->deps[i - job->input_count]; ++} ++ ++/* A signalled input: its error, or that of any fence it wraps (array, chain), else 0. */ ++static int g5p_render_fence_status(struct dma_fence *fence) ++{ ++ struct dma_fence_unwrap cursor; ++ struct dma_fence *part; ++ u32 unwrapped = 0; ++ int ret = dma_fence_get_status(fence); ++ ++ if (ret < 0) ++ return ret; ++ dma_fence_unwrap_for_each(part, &cursor, fence) { ++ ret = dma_fence_get_status(part); ++ if (ret <= 0 || ++unwrapped > 4096) { ++ dma_fence_put(cursor.chain); ++ return ret < 0 ? ret : -ETIMEDOUT; ++ } ++ } ++ return 0; ++} ++ ++/* Fence callback (any context, the fence lock held): the head job's input signalled. */ ++static void g5p_rjob_dep_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++{ ++ struct g5p_rjob *job = container_of(cb, struct g5p_rjob, cb); ++ ++ WRITE_ONCE(job->cb_fired, true); ++ mod_delayed_work(job->gpu->render_wq, &job->gpu->rdev->sched_work, 0); ++} ++ ++/* Scheduler lock held: no callback of this job stays armed. */ ++static void g5p_rjob_disarm(struct g5p_rjob *job) ++{ ++ if (job->cb_fence) ++ dma_fence_remove_callback(job->cb_fence, &job->cb); ++ job->cb_fence = NULL; ++} ++ ++/* ++ * Scheduler lock held: may the head job of its context run now? Walks its explicit inputs, ++ * then its implicit dependencies, in order; at the first unsignalled one it arms a callback ++ * and reports the job's input deadline. A failed input, the deadline (G5P_RU_INPUT_MS after ++ * the submit, as the old in-worker wait) and closed admission make it ready with an error, so ++ * that it fails before the hardware and its fence still signals in order. ++ */ ++static bool g5p_rsched_ready(struct g5p_rs_job *node, void *cookie, ++ unsigned long *deadline) ++{ ++ struct g5p_rjob *job = container_of(node, struct g5p_rjob, node); ++ ++ if (job->setup_error || job->dep_error) ++ return true; ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ESHUTDOWN; ++ return true; ++ } ++ if (job->cb_fence) { ++ if (!READ_ONCE(job->cb_fired)) ++ goto pending; ++ job->cb_fence = NULL; ++ } ++ while (job->dep_next < job->input_count + job->dep_count) { ++ struct dma_fence *fence = g5p_rjob_dep(job, job->dep_next); ++ ++ if (dma_fence_is_signaled(fence)) { ++ job->dep_error = g5p_render_fence_status(fence); ++ if (job->dep_error) ++ return true; ++ job->dep_next++; ++ continue; ++ } ++ WRITE_ONCE(job->cb_fired, false); ++ if (dma_fence_add_callback(fence, &job->cb, g5p_rjob_dep_cb)) ++ continue; /* -ENOENT: signalled meanwhile */ ++ job->cb_fence = fence; ++ goto pending; ++ } ++ return true; ++pending: ++ if (time_after_eq(jiffies, job->deadline)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ETIMEDOUT; ++ return true; ++ } ++ *deadline = job->deadline; ++ return false; ++} ++ ++static void g5p_rsched_kick(struct g5p_drm *d) ++{ ++ mod_delayed_work(d->render_wq, &d->rdev->sched_work, 0); ++} ++ ++/* Submit, render_submit_lock held: an accepted job joins its context's FIFO. */ ++static void g5p_rsched_push(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ g5p_rs_push(&d->rdev->sched, &r->entity, &job->node); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ g5p_rsched_kick(d); ++} ++ ++/* The close item joins the context's entity behind its jobs; false if one is queued. */ ++static bool g5p_rsched_close(struct g5p_drm *d, struct g5p_render *r) ++{ ++ unsigned long flags; ++ bool queued; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ queued = g5p_rs_close(&d->rdev->sched, &r->entity); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ if (queued) ++ g5p_rsched_kick(d); ++ return queued; ++} ++ ++static void g5p_render_run(struct g5p_rjob *job); ++static void g5p_render_close_run(struct g5p_render *r); ++static void g5p_render_close_deferred(struct g5p_drm *d); ++ ++/* ++ * The worker: pick, run to completion, repeat; then sleep until a callback, a submit or the ++ * earliest input deadline wakes it. render_wq is ordered, so this never runs twice at once, ++ * and it is the only writer of the device's job state, as each job's work item was. ++ */ ++static void g5p_rsched_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = container_of(to_delayed_work(work), ++ struct g5p_rdev, sched_work); ++ struct g5p_drm *d = rd->drm; ++ ++ /* Each item drops the references it held; the device must outlive this loop. */ ++ drm_dev_get(&d->drm); ++ for (;;) { ++ struct g5p_rs_entity *e; ++ struct g5p_rs_job *node; ++ struct g5p_render *r; ++ struct g5p_context *c; ++ unsigned long flags, wake = 0; ++ bool has_wake; ++ int what; ++ ++ /* A lost job blocks the channel: hold every job until the drain is decided, and ++ * until a recovery the drain or the executor asked for has run. ++ */ ++ if (READ_ONCE(rd->stuck) || READ_ONCE(rd->recovering)) { ++ if (rd->stuck ? g5p_rv3_drain(d) : g5p_rv3_recover(d)) ++ continue; ++ queue_delayed_work(d->render_wq, &rd->sched_work, ++ msecs_to_jiffies(100)); ++ break; ++ } ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ what = g5p_rs_pick(&rd->sched, g5p_rsched_ready, d, &e, &node, ++ &wake, &has_wake); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ if (what == G5P_RS_IDLE) { ++ if (has_wake) ++ queue_delayed_work( ++ d->render_wq, &rd->sched_work, ++ max_t(long, (long)(wake - jiffies), 1)); ++ break; ++ } ++ if (what == G5P_RS_JOB) { ++ g5p_render_run( ++ container_of(node, struct g5p_rjob, node)); ++ g5p_render_close_deferred(d); ++ continue; ++ } ++ r = container_of(e, struct g5p_render, entity); ++ c = r->ctx; ++ g5p_render_close_run(r); ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ g5p_rs_closed(&rd->sched, e); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ /* The close item's own reference (g5p_render_close()); r may go with it. */ ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ drm_dev_put(&d->drm); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +new file mode 100644 +index 000000000..36550c443 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -0,0 +1,488 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Unbinding firmware slots (docs/gpu/multi-client-design.md §4.5): a closed context's pair ++ * (stage 2, core/render-user-close.h) and the render frontend's share of the reboot quiesce ++ * (core/render-user-quiesce.h, docs/gpu/experiments/mcfold2-kexec.md). ++ * Included from drm-render.inc after drm-render-v3.inc. ++ */ ++ ++/* The RAM words of `slot` and of its unbind record at ring entry `index`. */ ++static void g5p_render_slot_mem(struct g5p_boot *b, u32 slot, u32 index, ++ struct g5p_ru_slot_mem *m) ++{ ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + (index & 255) * 20 + j * 4; ++ ++ m->rec[j] = ++ b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ } ++ m->pa = g5p_boot_shared(b, 0x86) + slot * 8; ++ m->host = g5p_boot_shared(b, 0x84) + slot * 4; ++ m->cached = g5p_boot_shared(b, 0x85) + slot * 4; ++} ++ ++/* ++ * Unbind `n` firmware slots in one RAM-only DC3 batch: PA and host tag of each to zero, one ++ * record {3, slot} per slot, one producer store, then the RAM-only wait of a bind and a check ++ * that the firmware's DC3 handler copied the zero tag of every slot. No doorbell and no ++ * register access, so it is safe after a job retired and while the GPU may be idle-gated. ++ * ++ * The caller holds the core lock (the DC ring has one writer) and has retired every use of ++ * these slots: no job of theirs is queued on the firmware or in flight. On success the ++ * firmware holds, per slot, PA 0, host and cached tag 0 and CBASE 0, with the slot's PC level ++ * invalidated; the slot may be bound again by a later DC3. Its translations stay allocated: ++ * freeing them is the caller's step, after this returns 0. A batch that is not consumed in ++ * time keeps later binds and unbinds refused until the firmware catches up (dc_pending). ++ */ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_slot_sample s = {}; ++ struct g5p_ru_slot_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, cons, target, bad = 0, verified = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!g5p_ru_unbind_fits(n)) ++ return -EINVAL; ++ for (u32 i = 0; i < n; i++) ++ if (slots[i] < G5P_RU_SLOT_MIN || slots[i] > G5P_RU_SLOT_MAX) ++ return -EINVAL; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ cons = READ_ONCE(ctl[0]); ++ if (start > 255 || cons != start || ++ (rd->dc_pending && cons != rd->dc_wait)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s refused slots=%u dc=%u/%u pending=%u: ring busy; no " ++ "hardware touched end=v3\n", ++ why, n, cons, start, rd->dc_pending); ++ return -EBUSY; ++ } ++ rd->dc_pending = false; ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_unbind_publish(&m, slots[i]); ++ } ++ target = (start + n) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, why); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ for (u32 i = 0; i < n; i++) { ++ u32 one; ++ ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_slot_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ one = g5p_ru_unbind_check(&s, slots[i], target); ++ bad |= one; ++ verified += !one; ++ } ++ if (!ret && bad) ++ ret = -EUCLEAN; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s slots=%u verified=%u ret=%d consumed_us=%lld " ++ "dc=%u->%u/%u bad=%#x first=%u last=%u mode=ram-dc3 end=v3\n", ++ why, n, verified, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ s.cons, s.prod, bad, slots[0], slots[n - 1]); ++ return ret; ++} ++ ++/* ++ * render_user_close_park: bind a closed context's just-unbound pair to the park catalogue, one ++ * DC3 pair as a context's bind (g5p_render_vm_prepare()), checked the same way. The catalogue ++ * holds the firmware window's entry and the borrowed sink entry, both stable within an epoch, so ++ * rewriting them for a later park never clears what an earlier parked pair resolves through. ++ * Core lock held, between jobs (the close item), RAM only and no register: as the unbind. ++ */ ++static int g5p_rv3_park(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, target, entry, sink, bad = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!rd->park.cpu) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->park); ++ if (ret) ++ return ret; ++ memset(rd->park.cpu, 0, G5P_GPU_PAGE); ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (start > 255 || rd->dc_pending || g5p_ru_bind_busy(&s)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park refused slots=%u/%u dc=%u/%u pending=%u busy=%#x: the pair stays " ++ "unbound end=v3\n", ++ r->slot[0], r->slot[1], s.cons, start, rd->dc_pending, ++ g5p_ru_bind_busy(&s)); ++ return -EBUSY; ++ } ++ sink = b->sink_ready ? g5p_le32(b->catalogue.cpu) : 0; ++ if (g5p_le32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4) != entry) ++ g5p_put32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4, entry); ++ if (g5p_le32(rd->park.cpu) != sink) ++ g5p_put32(rd->park.cpu, sink); ++ g5p_table_clean_page(g, &rd->park); ++ dsb(sy); ++ g5p_ru_bind_publish(&m, r->slot, rd->park.dma, G5P_RU_PARK_TAG); ++ target = (start + 2) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "park"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, rd->park.dma, ++ G5P_RU_PARK_TAG, target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ rd->parks += !ret; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park slots=%u/%u ret=%d consumed_us=%lld dc=%u->%u/%u bad=%#x " ++ "parks=%u pc=%pad end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ start, s.cons, s.prod, bad, rd->parks, &rd->park.dma); ++ return ret; ++} ++ ++/* Drop what the claim of the device took (g5p_render_device_claim()); the device stays ++ * claimed for render. Returns whether a context reference was dropped. ++ */ ++static bool g5p_render_unclaim(struct g5p_drm *d, struct g5p_render *r) ++{ ++ if (!r->claimant) ++ return false; ++ r->claimant = false; ++ if (d->quarantined_context == r->ctx) ++ d->quarantined_context = NULL; ++ return true; ++} ++ ++/* ++ * The close item of a TYPED_V3 context (multi-client-design.md "Stage 2/3 as built"): the ++ * scheduler runs it after every job of the context, so every use of the pair has retired or ++ * failed. RAM only: the unbind is g5p_rv3_unbind_slots() for this pair under the core lock. ++ * After a verified unbind the binding's references go, and the last one frees the VM tables, ++ * render targets and mappings (unpinning the BOs) and returns the slots ++ * (g5p_render_release()). Anything the firmware may still hold is retained until reboot. ++ */ ++/* perf60 (docs/gpu/experiments/perf60-clock.md), corrected by the release8 bench (bench-gaps3, ++ * render-user-pmset.h): the PM writes a previous PM context back (its free list F, through that ++ * context's TA slot) not once at the switch but on the later jobs of the next context, until a ++ * third context runs. A closed context therefore keeps its slots bound and its set and pages alive ++ * while it is the last job's context or the previous distinct one; its unbind and release run once ++ * it is neither. At most those two wait. (The first version held only the last job's context and ++ * released it after one other job; 2 held one job longer, by job records: both released it while ++ * the PM still wrote to it.) RAM only: bookkeeping and the close item's references. ++ */ ++/* release8 default */ ++ ++/* ++ * A close item follows every job of this context on the ordered render worker. ++ * resident_gone, set under the file lock before queueing it, rejects all later ++ * VM_BIND and context creation. Thus the close item owns vm->maps exclusively. ++ * ++ * Idle VM_BIND UNMAP already drops a user mapping's pages after retirement. ++ * Do that here without releasing the PM's write-back route: no unbind, no ++ * table pruning, no PM set/pool/aux/RT release, no pm_last/pm_prev change. The ++ * PM free list is in the firmware window, never in vm->maps. All user mappings ++ * must pass the same range check as VM_BIND before any PTE is touched. ++ * ++ * No new kick of this detached context can occur. Clean the cleared user PTEs ++ * to PoC, but do not publish a tag, invalidate a slot or access R0. The PM can ++ * keep using its unchanged translations and pages, including cached ones. ++ * Failed/queued/terminal/shutdown epochs keep every BO, as before. ++ */ ++static void g5p_render_close_trim(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_boot *b; ++ struct g5p_ru_close_in in; ++ struct g5p_mapping *m, *tmp; ++ struct g5p_pt *pt; ++ u64 pinned; ++ u32 maps, why; ++ ++ if (!vm->map_count || !vm->resident_gone || !r->closing || ++ !r->detached || !mutex_trylock(&g->lock)) ++ return; ++ b = g->boot; ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* Match close_run: a successfully drained lost job has no user-memory use left. */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .dc_pending = d->rdev->dc_pending, ++ .queued = atomic_read(&r->queued), ++ }; ++ if (g5p_ru_close_plan(&in, &why) != G5P_RU_CLOSE_UNBIND) ++ goto out; ++ list_for_each_entry(m, &vm->maps, link) ++ if (!g5p_user_range_valid(m->node.start, m->node.size) || ++ (m->node.start < G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE && ++ G5P_RU_PRIVATE_VA < m->node.start + m->node.size)) ++ goto out; ++ ++ pinned = vm->pinned_bytes; ++ maps = vm->map_count; ++ list_for_each_entry(m, &vm->maps, link) ++ g5p_vm_clear(vm, d->drm.dev, m->node.start, m->node.size); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ list_for_each_entry_safe(m, tmp, &vm->maps, link) { ++ vm->mapped_bytes -= m->node.size; ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-trim slots=%u/%u maps=%u unpinned=%lluK remaining=%lluK " ++ "device_pinned=%lluK pm=retained end=v3\n", ++ r->slot[0], r->slot[1], maps, (pinned - vm->pinned_bytes) >> 10, ++ vm->pinned_bytes >> 10, READ_ONCE(d->pinned_bytes) >> 10); ++out: ++ mutex_unlock(&g->lock); ++} ++ ++/* render_user_close_defer: may the firmware's PM still write this context's state back? */ ++static bool g5p_render_pm_holds(struct g5p_rdev *rd, struct g5p_render *r, ++ u32 mode) ++{ ++ return mode && g5p_pmset_holds(rd->pm_last, rd->pm_prev, r); ++} ++ ++static void g5p_render_close_run(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_close_in in; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 puts = 0, why; ++ bool locked = false; ++ int plan, ret = 0; ++ u32 mode, k; ++ ++ mode = (1); ++ k = d->rdev->close_deferred[0] ? 1 : 0; ++ if (mode && !r->close_defer_run && r->bound && ++ g5p_render_pm_holds(d->rdev, r, mode) && ++ k < ARRAY_SIZE(d->rdev->close_deferred)) { ++ g5p_render_close_trim(r); ++ /* the deferred close's own references, dropped by g5p_render_close_deferred() */ ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->rdev->close_deferred[k] = r; ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer slots=%u/%u jobs=%u: the PM may still write its state " ++ "back; unbind once it has moved past it end=v3\n", ++ r->slot[0], r->slot[1], r->jobs); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer2 slots=%u/%u pm=%s pending=%u end=v3\n", ++ r->slot[0], r->slot[1], ++ d->rdev->pm_last == r ? "last" : "previous", k + 1); ++ return; ++ } ++ ++ for (;;) { ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* a lost job that drained left the firmware nothing of this context */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ /* as g5p_rv3_unbind_slots() refuses it */ ++ .dc_pending = b && d->rdev->dc_pending && ++ READ_ONCE(((u32 *)g5p_boot_shared( ++ b, 0x6c))[0]) != d->rdev->dc_wait, ++ .queued = atomic_read(&r->queued), ++ }; ++ plan = g5p_ru_close_plan(&in, &why); ++ if (plan != G5P_RU_CLOSE_UNBIND || locked) ++ break; ++ /* Bounded, as a job takes it: never wait forever behind a command in MMIO. */ ++ while (!(locked = mutex_trylock(&g->lock)) && ++ time_before(jiffies, deadline)) ++ msleep(5); ++ if (!locked) { ++ plan = G5P_RU_CLOSE_RETAIN; ++ why = G5P_RU_CLOSE_LOCK; ++ break; ++ } ++ /* Plan again under the lock: the reboot quiesce or a failure may have run. */ ++ } ++ if (plan == G5P_RU_CLOSE_UNBIND) { ++ ret = g5p_rv3_unbind_slots(d, r->slot, 2, "close"); ++ if (!ret) { ++ r->bound = false; ++ r->retained = false; ++ WRITE_ONCE(c->live, false); ++ puts++; ++ /* the PM may still write this context's free list back through its TA slot */ ++ { ++ g5p_rv3_park(d, r); ++ }; ++ } ++ } ++ if (plan == G5P_RU_CLOSE_RELEASE || ++ (plan == G5P_RU_CLOSE_UNBIND && !ret)) ++ puts += g5p_render_unclaim(d, r); ++ if (locked) ++ mutex_unlock(&g->lock); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close slots=%u/%u plan=%s why=%#x ret=%d jobs=%u claimant=%u " ++ "retained=%u end=v3\n", ++ r->slot[0], r->slot[1], g5p_ru_close_name(plan), why, ret, ++ r->jobs, r->claimant, r->retained); ++ while (puts--) { ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++} ++ ++/* After a job: a deferred close runs once the last job was another context's. */ ++static void g5p_render_close_deferred(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ u32 mode = (1); ++ ++ for (u32 k = 0; k < ARRAY_SIZE(rd->close_deferred); k++) { ++ struct g5p_render *r = rd->close_deferred[k]; ++ struct g5p_context *c; ++ ++ /* mode 0 now: a pending close still waits until the PM has moved past it */ ++ if (!r || g5p_render_pm_holds(rd, r, mode ?: 1)) ++ continue; ++ rd->close_deferred[k] = NULL; ++ c = r->ctx; ++ r->close_defer_run = true; ++ dev_info( ++ d->core->dev, ++ "RENDER-V3 close-deferred slots=%u/%u: neither the PM's last nor its previous " ++ "context end=v3\n", ++ r->slot[0], r->slot[1]); ++ g5p_render_close_run(r); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ /* keep a pending close in [0]: close_run fills [1] only behind it */ ++ if (!rd->close_deferred[0] && rd->close_deferred[1]) { ++ rd->close_deferred[0] = rd->close_deferred[1]; ++ rd->close_deferred[1] = NULL; ++ } ++} ++ ++/* ++ * The handle of a TYPED_V3 context is gone (DESTROY_CONTEXT, or its file closed): stop VM and ++ * render-target edits and queue the close item behind the context's jobs (g5p_rsched_close()). ++ * File lock held, or the file is closing. A context that never submitted is released by its ++ * last reference. ++ */ ++static void g5p_render_close(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ ++ if (!r->v3_accepted || r->closing) ++ return; ++ r->closing = true; ++ r->detached = true; ++ c->vm->resident_gone = true; ++ /* The close item's references, dropped by the scheduler once it ran. */ ++ kref_get(&c->ref); ++ drm_dev_get(&c->gpu->drm); ++ if (!g5p_rsched_close(c->gpu, r)) { ++ g5p_context_put(c); ++ drm_dev_put(&c->gpu->drm); ++ } ++} ++ ++/* Reboot quiesce, RAM only and lock free: the TYPED_V3 latch and whether its executor runs. ++ * The executor is RAM-only once the latch is quiet, unless the wake110 experiment (doorbells ++ * after retirement, off in every desktop tuple) is configured. Called after admission closed; ++ * the scheduler is kicked so that jobs waiting on an input fail now, not at their deadline. ++ */ ++static void g5p_kq_drm_state(void *cookie, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ ++ /* Admission is closed by now: queued jobs blocked on an input fail at once. */ ++ g5p_rsched_kick(d); ++ ++ /* A channel blocked behind a lost job is a wedged epoch for the quiesce. */ ++ in->term_failed = g5p_ru_term_failed(&g5p_rv3_term) || ++ READ_ONCE(d->rdev->stuck) || ++ READ_ONCE(d->rdev->recovering); ++ in->ram_only = READ_ONCE(d->rdev->v3) && ++ !g5p_ru_mmio_ok(&g5p_rv3_term) && !(2); ++ in->executing = READ_ONCE(d->rdev->executing); ++} ++ ++/* Reboot quiesce, core lock held: every held user slot, the first claimant's pair included. ++ * Nothing is freed and no context changes state: admission is closed and the boot state is ++ * frozen right after, so no job runs again on this boot. ++ */ ++static int g5p_kq_drm_detach(void *cookie, const u32 *slots, u32 n) ++{ ++ return g5p_rv3_unbind_slots(cookie, slots, n, "kexec"); ++} ++ ++static const struct g5p_kq_frontend g5p_kq_drm = { ++ .state = g5p_kq_drm_state, ++ .detach = g5p_kq_drm_detach, ++}; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +new file mode 100644 +index 000000000..ac4dd841e +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_V3 serial executor: per-job state (receipt data). Types only; code: drm-render-v3.inc. */ ++#ifndef G5P_DRM_RENDER_V3_H ++#define G5P_DRM_RENDER_V3_H ++ ++enum g5p_rv3_phase { ++ G5P_RV3_IDLE, ++ G5P_RV3_ATTACH, ++ G5P_RV3_BIND, ++ G5P_RV3_TA_PUBLISH, ++ G5P_RV3_TA_WAIT, ++ G5P_RV3_TD_PUBLISH, ++ G5P_RV3_TD_WAIT, ++ G5P_RV3_DONE, ++ G5P_RV3_PHASES, ++}; ++ ++/* g5p_drm.td_want: the 3D completion-event latch is armed for the endpoint in the low byte. */ ++#define G5P_RV3_TD_WANT 0x100U ++ ++/* One RAM sample of one engine: ring, stamps, channel watermarks, idle words, PM state, and ++ * (3D) whether the matching completion event has been latched. ++ */ ++struct g5p_rv3_ev { ++ u64 ns; ++ u32 prod, cons, stamp[2], ctl[5], idle[6], pm80; ++ bool proto, failed, consumed, retired, idle_ok, pm_ok, event; ++}; ++ ++struct g5p_rv3 { ++ struct g5p_ru_wait ta, td; ++ struct g5p_ru_touch ++ touch[2]; /* RC4a A/B (render-user-touch.h); zero when off */ ++ struct g5p_ru_retire rt; /* the 3D predicate (render-user-retire.h) */ ++ struct g5p_ru_ta tr; /* the TA predicate (render-user-retire.h) */ ++ struct g5p_rv3_ev last[2]; ++ u64 td_event_ns; /* decode time of the matching 3D event; 0 = none */ ++ u64 ta_event_ns; /* decode time of the first TA event since the publish */ ++ u64 attach_ns, job_deadline_ns, ta_pub_ns, td_pub_ns, ta_consumed_ns, ++ td_consumed_ns; ++ u32 phase, grace_end, bell; ++ u32 seq; /* the device sequence this job got; 0 = none */ ++ /* wake110 (render-user-wake.h), per engine: the decision, the power evidence word it saw ++ * (wake111), the two register words it read (U32_MAX = not read), the consumer it saw, the ++ * type-1 event counts at the phase's arm and at the job's end, the lock window and the ++ * first type-1 decode time. Zero when off. ++ */ ++ u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], ++ wk_t1_pub[2], wk_t1_end[2]; ++ u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ ++ /* The bind of this job's context published its DC3 pair and then failed: that context is ++ * lost, the device and the contexts already bound are not. ++ */ ++ bool bind_failed; ++ /* The job was lost alone (render-user-loss.h): its context is dead, the device is not; the ++ * channel stays blocked behind its stuck phase until the scheduler's drain clears it. ++ */ ++ bool lost; ++}; ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +new file mode 100644 +index 000000000..1d0a0a76d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -0,0 +1,3038 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial executor: the control flow that completed the natural TA and the ++ * natural 3D in r96 (docs/gpu/experiments/production-typed-render-kernel.md), on typed ++ * state. Included from drm-render.inc after g5p_render_execute(). ++ * ++ * What it does, in order, for each job a V3 context runs (strictly serial across every context, ++ * render-user-serial.h; only the epoch's job 1 attaches and has hardware evidence): ++ * 0. bind, the first job of every later context (multi-client-design.md §4.2): its own DC3 ++ * pair, published in RAM only, no doorbell, a RAM-only wait for the firmware to consume ++ * it. With the TEMPORARY render_user_spare_pairs instead: a pair job 1 pre-bound, taken ++ * over through the tag path. ++ * 1. attach (job 1): the typed preparation (PM, channels, catalogue entries), then the ++ * DC3 pair is published and its doorbell is owned by the r74f machinery (r87b ++ * self-drain: a closed gate is never repaired by the host). The firmware installs ++ * the page-table root and the generation tags itself (r74g); the host does NO MMU pass ++ * and checks the result in RAM only. ++ * 2. TA: record, WORK_CTL and producer, no doorbell; the r74g grace (four ALIVEs or ++ * 20 ms) lets the firmware drain it, and only if it is still pending afterwards the ++ * first ALIVE decoded owns one fenced fallback doorbell. ++ * 3. TA wait: RAM only (drain, sample, sleep), 20 s per phase by default, 30 s for the ++ * whole job (render-user-wait.h decides the deadline and closes the race with a ++ * completion). tahang1: complete on a coherent target with the PM saved and either ++ * the idle words clear, the TA completion event, or a 500 ms coherent hold ++ * (render-user-retire.h); a phase pending 1 s prints one RAM-only `slow` line. From ++ * the TA complete on, and after any timeout or fault, the host touches no register. ++ * 4. 3D: record, WORK_CTL and producer, NO doorbell (r96 stage 1: the firmware consumed ++ * it 2 ms later), RAM-only wait as above. ++ * release (render_user_touch_ms, default 0 = off): at publish + T of each phase, odd jobs read ++ * the receipt pages once and even jobs only record the instant (render-user-touch.h); RAM only. ++ * repeat109: the 3D phase is also complete when its target state is coherent and its completion ++ * event arrived, even if the engine idle words never clear (render-user-retire.h; the latched ++ * render_user_idle_settle_ms, default 0, adds a minimum coherent run); one `RENDER-V3 retire` ++ * receipt line splits the firmware's retirement from the host's decision. RAM only. ++ * wake110 (render_user_wake, default 0 = off): right after the producer store of every phase but ++ * job 1's TA, the A/B's factor phases run Apple's submit kick (R1+0x14 = 7, GPC active, R0+0xb00 = ++ * 0x10 / 0x11) and its control phases the same guards without a register (render-user-wake.h); ++ * one `RENDER-V3 wake` line per job. The only register access after the TA complete, licensed ++ * while no timeout or fault was noted. ++ * Jobs >= 2 (a compositor's next frame, or another context's): no attach, the PM pool is ++ * rebuilt, and both rings are published with no doorbell at all. ++ * Not in this path on purpose: the r74h / r80 / r85-r96 root-cause instrumentation (stream ++ * markers, PM canary, plateau and end banks, region/target dumps), host MMU passes, cache ++ * maintenance MMIO, selector reads, snapshots (they read R0), and any recovery. A timeout ++ * or fault retains everything; the context is frozen and kept until reboot. ++ * ++ * Terminal state (render-user-terminal.h): the TA complete, a timeout and a fault go through one ++ * first-writer-wins latch, g5p_rv3_terminal(). From then on g5p_ru_mmio_ok() is false, the admission ++ * refuses every job that would start behind a failure (-EIO), and the legacy typed paths that read or ++ * write registers (snapshot, invalidate, run, destroy) refuse. g5p_rv3_td_hook() is the optional ++ * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. ++ */ ++ ++static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { ++ "idle", "attach", "bind", "ta-publish", ++ "ta-wait", "3d-publish", "3d-wait", "done", ++}; ++static const char *const g5p_rv3_results[] = { "pending", "complete", "timeout", ++ "fault" }; ++static const char *const g5p_rv3_how[] = { "none", "idle-clear", "idle-stuck", ++ "hold" }; ++ ++#include "drm-latency123.inc" ++ ++/* perf60 stage 4 (render-user-pmset.h): per-context PM records, the iOS model (one ParamBuffer ++ * per work queue, 6cb9afc/6cb2378). Latched at the epoch's first attach, with boot_pm_sets: ++ * 1 every context bound after the epoch's first gets its own PM set (control + S, list, F in ++ * the firmware GiB), its own pool pages and its own aux pages at G5P_RU_PRIVATE_VA in its ++ * VM; the epoch's first context keeps the device PM. PM ID 0 everywhere. ++ * 2 as 1, and set s carries PM ID s + 1 (TA W+0x10, TA start, 3D start, tag6). ++ * RAM only, by the executor between jobs. ++ */ ++/* release8 default */ ++ ++/* The PM a context's jobs use: its set, or the device PM. */ ++static u8 *g5p_rv3_pm_cpu(struct g5p_boot *b, const struct g5p_render *r) ++{ ++ if (r->pm_set >= 0) ++ return b->pm_set[r->pm_set * G5P_PMSET_PAGES + G5P_PMSET_CONTROL] ++ .cpu; ++ return g5p_ru_pm_cpu(b); ++} ++ ++static u32 g5p_rv3_pm_meta(const struct g5p_render *r) ++{ ++ return g5p_meta_address( ++ r->pm_set >= 0 ? g5p_pmset_va(r->pm_set, G5P_PMSET_CONTROL) : ++ G5P_SHARED_VA + G5P_R22_PM); ++} ++ ++static u32 g5p_rv3_pm_id(const struct g5p_rdev *rd, const struct g5p_render *r) ++{ ++ return rd->pm_ctx >= 2 && r->pm_set >= 0 ? g5p_pmset_id(r->pm_set) : 0; ++} ++ ++static struct g5p_dma_page *g5p_rv3_pm_aux(struct g5p_rdev *rd, ++ struct g5p_render *r) ++{ ++ return r->pm_aux_mapped ? r->pm_aux : rd->aux; ++} ++ ++/* The boot tuple the V3 path is proven under (r96 params.txt), minus the instrumentation: ++ * r73b/r73c/r73e/r73f, r74 with the raised IRQ cap, r74f, r74g, r87b self-drain; no r74h, ++ * no r80..r96 (their IRQ banks read registers), no r75/r79 firmware control, and no fixed ++ * render engine (it is exclusive with the DRM one). ++ */ ++static bool g5p_rv3_tuple(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ struct g5p_r73b *x = b ? b->r73b : NULL; ++ ++ return x && g5p_r74g_on(b) && g5p_r74b_ram_only(g) && ++ x->r87b_attach == G5P_R87B_ATTACH_SELF_DRAIN; ++} ++ ++/* The admission state of a context and the device, as plain values for g5p_ru_v3_admit(). */ ++static struct g5p_ru_v3_state g5p_rv3_state(struct g5p_diag *g, ++ struct g5p_render *r) ++{ ++ const struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ ++ return (struct g5p_ru_v3_state){ ++ .tuple = g5p_rv3_tuple(g), ++ .v3_accepted = r->v3_accepted, ++ .bound = r->bound, ++ .jobs = r->jobs, ++ .device_up = READ_ONCE(rd->up), ++ .device_v3 = READ_ONCE(rd->v3), ++ .sequence = READ_ONCE(rd->sequence), ++ .slot_ta = r->slot[0], ++ .slot_3d = r->slot[1], ++ .variant = 0 || r->after_ta || 0 || r->v.pm || r->v.memory || ++ r->v.rt || r->v.slots, ++ .wait_ms = r->wait_ms, ++ .quiet = !g5p_ru_mmio_ok(&g5p_rv3_term), ++ .terminal_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ }; ++} ++ ++/* A terminal epoch is a device state (-EIO), the others are about this request. */ ++static int g5p_rv3_refusal_errno(int why) ++{ ++ if (why == G5P_RU_V3_TERMINAL) ++ return -EIO; ++ return why == G5P_RU_V3_OVERFLOW ? -EOVERFLOW : -EOPNOTSUPP; ++} ++ ++/* Submit-time admission: every V3 job refused here has touched nothing. */ ++static int g5p_rv3_admit_submit(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_ru_v3_state st; ++ int why; ++ ++ /* The epoch is being replaced: the executor admits the job again under the lock. */ ++ if (READ_ONCE(d->rdev->recovering)) ++ return 0; ++ st = g5p_rv3_state(g, r); ++ why = g5p_ru_v3_admit(&st, false); ++ ++ if (!why) ++ return 0; ++ /* per submit while admission is closed and a client retries: ratelimited */ ++ dev_info_ratelimited( ++ g->dev, "RENDER-V3 refused: %s; no hardware touched end=v3\n", ++ g5p_ru_v3_why[why]); ++ return g5p_rv3_refusal_errno(why); ++} ++ ++/* Recovery hooks (patches/gpu-recovery), compiled only when that series is applied: core.inc then includes ++ * teardown.inc before this file, and teardown.h defines G5P_TEARDOWN_H. The CAPTURE entry is observation only ++ * (RAM reads, no register access) and runs only with the recovery module parameter set, once per epoch, on a ++ * timeout or fault. Without the series both are nothing, and the prod98 kernel does not depend on it. ++ */ ++#ifdef G5P_TEARDOWN_H ++static void g5p_rv3_td_hook(struct g5p_diag *g, int kind, const char *why) ++{ ++ g5p_td_note_terminal(&g5p_td_state, g5p_ru_td_kind(kind)); ++ if ((0) && g5p_ru_term_capture_due(&g5p_rv3_term, kind)) ++ g5p_td_capture(g, why); ++} ++#else ++static inline void g5p_rv3_td_hook(struct g5p_diag *g, int kind, ++ const char *why) ++{ ++} ++#endif ++ ++/* The terminal state of the epoch: the TA complete, and any timeout or fault. First writer wins; from the first ++ * note on g5p_ru_mmio_ok() is false and the host touches no register (R0, R1, AIC, doorbell) and no bank. The r80 ++ * flag is set when an r80 block exists, so that a bank that might be latched cannot run behind this path (the V3 ++ * tuple forbids r80, so there is none today); the latch above does not depend on it. ++ */ ++static void g5p_rv3_terminal(struct g5p_diag *g, int kind, const char *why) ++{ ++ bool failed = g5p_ru_term_failed(&g5p_rv3_term); ++ ++ if (g5p_ru_term_note(&g5p_rv3_term, kind)) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s mmio=off r0=off export=ram-only end=v3\n", ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)], ++ why); ++ else if (!failed && g5p_ru_term_failed(&g5p_rv3_term)) ++ /* The epoch's TA complete took the latch long ago (job 1): a later timeout or fault ++ * only makes it sticky-failed, which closes admission for every client. Say so once. ++ */ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s latched=%s failed=1 admission=closed end=v3\n", ++ g5p_ru_term_names ++ [kind > G5P_RU_TERM_NONE && ++ kind < G5P_RU_TERM_KINDS ? ++ kind : ++ G5P_RU_TERM_FAULT], ++ why, ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)]); ++ g5p_rv3_td_hook(g, kind, why); ++} ++ ++/* "Retained boundary" of the r74f machinery -> the errno a job reports. -EINPROGRESS is ++ * never a job error (the UAPI uses it for "pending"). ++ */ ++static int g5p_rv3_errno(struct g5p_diag *g, int ret) ++{ ++ if (ret != -EINPROGRESS) ++ return ret; ++ switch (READ_ONCE(g->boot->r73b->r74f_stop)) { ++ case G5P_R74F_STOP_ATTACH_NOT_KICKED: ++ case G5P_R74F_STOP_ATTACH_NOT_CONSUMED: ++ case G5P_R74F_STOP_TA_NOT_KICKED: ++ case G5P_R74G_STOP_TA_NOT_CONSUMED: ++ case G5P_R74G_STOP_TA_WINDOW_MISSED: ++ return -ETIMEDOUT; ++ default: ++ return -EIO; ++ } ++} ++ ++/* ++ * Interim: on iOS the firmware writes back its own lines with the cache maintenance it runs on ++ * every tick (meta+0x6d4); that needs init words still being recovered (latency123.md), and ++ * this read stands in for it until then. ++ * ++ * While a phase is in flight the firmware does not see its own progress in the root and ++ * shared pages until a coherent read of those lines reaches the fabric: without one, the TA ++ * and 3D stages wait for the firmware tick, for seconds (latency123: key to frame 7.9 s ++ * median; with a Normal-NC read of both regions every 1-2 ms, 0.27 s). One 32-bit load per ++ * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU ++ * has no sampler, so it costs nothing then. ++ */ ++static void g5p_rv3_fw_touch(struct g5p_boot *b) ++{ ++ struct g5p_dma_page *const regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE( ++ *(const u32 *)(regions[r][i].cpu + at)); ++ /* perf60 stage 4: the per-context PM sets are firmware pages like the shared ones */ ++ for (u32 i = 0; i < b->pm_sets * G5P_PMSET_PAGES; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(b->pm_set[i].cpu + at)); ++} ++ ++/* perf60 bell gates, A/B knobs for the multi-context 3D fault on the bell path (render_user_wake=2 only; ++ * 0600, read at every bell, all 0 = no gate). They run between a phase's producer store and its bell; ++ * in policy state 0 the firmware does not drain a ring on its own, so a gate holds the phase back. ++ * RAM reads, sleeps and barriers only; no register access. ++ * bell_gap_us (a) the TA bell waits until this long after the previous job (any context) ended ++ * bell_idle_ms (a) the TA bell first waits, up to this, for the 3D engine idle words to read zero ++ * bell_3d_delay_us (c) mb() and this delay between the 3D producer store and the 3D bell ++ * bell_ctx_gap_us (d) mb() and this delay before the TA bell of a job of another context than the last ++ * bell_touch (c) the Normal-NC read of the firmware pages (g5p_rv3_fw_touch) right before each bell ++ */ ++ ++/* perf60 job-start gate: the same waits before the job's own host writes (the PM pool rebuild and ++ * the shared TA/3D work records, g5p_rv3_pm_rebuild() and g5p_ru_job()), which happen before any ++ * publish. The bell gates above run after them, so they cannot separate "the host rewrites shared ++ * firmware pages while the firmware still finishes the previous job" from a doorbell race. ++ * job_gap_us the job's first host write no sooner than this after the previous job ended ++ * job_idle_ms then wait up to this (RAM) for the TA and 3D engine idle words to read zero ++ */ ++/* perf60 layout (latched at the epoch's first attach, default 0 = the r22 layout): ++ * render_user_job_slots 2: job seq writes its command, W record and microsequence into slot ++ * (seq - 1) % 2 of the channel page, so a job never rewrites the record the firmware used for ++ * the job before it; the pointer ring carries each job's addresses (render-user-layout.h). ++ * render_user_pm_once 1: the PM pool, list and F are built by the epoch's first attach only and ++ * never rewritten by the host between jobs; the TA's tag6 record has the firmware rebuild F, ++ * as on iOS (PM created per queue once; docs/gpu/experiments/freelist-lifetime-production-fix.md). ++ */ ++/* release8 default */ ++/* release8 default */ ++ ++/* The record slot of job seq on this epoch's layout. */ ++static u32 g5p_rv3_js(const struct g5p_rdev *rd, u32 seq) ++{ ++ return rd->job_slots == 2 ? (seq - 1) & 1 : 0; ++} ++ ++ ++ ++ ++ ++ ++ ++ ++ ++/* RAM only. */ ++static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_ru_page(b, e), *global = g5p_ru_global(b, e), ++ *pm = g5p_rv3_pm_cpu(b, r); ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ for (u32 i = 0; i < ARRAY_SIZE(s->stamp); i++) ++ s->stamp[i] = g5p_le32(p + G5P_R22_STAMP + 4 * i); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < ARRAY_SIZE(s->idle); i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->pm80 = g5p_le32(pm + 0x80); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != r->target[e] || s->cons > 255 || ++ !g5p_ru_stamp_sane(p, seq); ++ s->consumed = s->cons == r->target[e]; ++ /* The predicates of the flown r74b (TA) and r96 (3D) waits; the 3D's PM serial is a receipt. */ ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2] && ++ (!e || (!s->idle[4] && !s->idle[5])); ++ s->retired = g5p_ru_retired_at(p, g5p_ru_meta(e), e, seq, ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ s->pm_ok = e ? s->pm80 == r->ctx->gpu->rdev->pm_serial + 1 : ++ g5p_r22_pm_saved(pm + 0x80); ++ s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns) : ++ READ_ONCE(r->ctx->gpu->ta_hit_ns)); ++} ++ ++/* TA: one step of render-user-retire.h's TA rules when ta is given, else the flown r74b ++ * predicate. 3D: one step of its 3D rules when rt is given, else the r96 predicate. ++ */ ++static bool g5p_rv3_done(u32 e, const struct g5p_rv3_ev *s, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta) ++{ ++ if (e && rt) ++ return g5p_ru_rt_step(rt, s->ns, s->consumed, s->retired, ++ s->idle_ok, ++ s->event) != G5P_RU_RT_PENDING; ++ if (!e && ta) ++ return g5p_ru_ta_step(ta, s->ns, s->consumed, s->retired, ++ s->pm_ok, s->idle_ok, ++ s->event) != G5P_RU_TA_PENDING; ++ return s->consumed && s->idle_ok && s->retired && (e || s->pm_ok); ++} ++ ++/* The TA clauses in the bits of render-user-retire.h, for the receipt. */ ++static u32 g5p_rv3_ta_unsat(const struct g5p_rv3_ev *s) ++{ ++ return (s->consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (s->retired ? 0 : G5P_RU_RT_RETIRED) | ++ (s->idle_ok ? 0 : G5P_RU_RT_IDLE) | ++ (s->pm_ok ? 0 : G5P_RU_RT_PM); ++} ++ ++/* Arm engine e's completion-event latch of g5p_drm_event() (3D: for this job's endpoint), before ++ * the phase's producer store, so an event decoded right after the publish is not missed. The ++ * latch is set under status_lock by whichever CPU decodes the event, and the wait samples it ++ * level-triggered every 1-2 ms, so there is no wakeup to lose. RAM only. ++ */ ++static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) ++{ ++ unsigned long flags; ++ ++ BUILD_BUG_ON(G5P_RU_RT_GRID != G5P_WORK_SLOTS); ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = G5P_RV3_TD_WANT | endpoint; ++ WRITE_ONCE(d->td_hit_ns, 0); ++ } else { ++ d->ta_want = true; ++ WRITE_ONCE(d->ta_hit_ns, 0); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) ++{ ++ unsigned long flags; ++ u64 hit; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = 0; ++ hit = d->td_hit_ns; ++ } else { ++ d->ta_want = false; ++ hit = d->ta_hit_ns; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return hit; ++} ++ ++/* release: the receipt's read set (g5p_rv3_receipt()'s blobs: PM-control, then W, channels and the ++ * stamp window of the TA and of the 3D page), read once with 32-bit loads, the access size of the ++ * receipt's hex dump. The words are folded, not copied: no allocation, no store to shared memory. ++ * RAM only: no register, no doorbell, no print, no latch. ++ */ ++static u32 g5p_rv3_touch_sweep(struct g5p_boot *b, u32 *bytes) ++{ ++ u8 *ta = g5p_ru_page(b, 0), *td = g5p_ru_page(b, 1); ++ const u8 *const src[] = { ++ g5p_ru_pm_cpu(b), ta + G5P_R22_W, ta, ++ ta + G5P_R22_TQ, td + G5P_R22_W, td, ++ td + G5P_R22_TQ, ++ }; ++ static const u32 len[] = { ++ 0x140, G5P_RU_TA_W_BYTES, 0x340, 0x80, G5P_RU_3D_W_BYTES, 0x340, ++ 0x80, ++ }; ++ u32 sum = 0, n = 0; ++ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(src); i++) { ++ for (u32 at = 0; at < len[i]; at += 4) ++ sum = (sum << 5 | sum >> 27) ^ ++ READ_ONCE(*(const u32 *)(src[i] + at)); ++ n += len[i]; ++ } ++ *bytes = n; ++ return sum; ++} ++ ++/* release: the instant of one phase, taken at the sample at now_ns. The touch arm reads, the control ++ * arm only records; both record the same instant. ++ */ ++static void g5p_rv3_touch(struct g5p_diag *g, struct g5p_ru_touch *t, ++ u64 now_ns) ++{ ++ u32 bytes = 0, sum = 0; ++ ++ if (t->sweep) ++ sum = g5p_rv3_touch_sweep(g->boot, &bytes); ++ g5p_ru_touch_mark(t, now_ns, t->sweep ? ktime_get_ns() : now_ns, bytes, ++ sum); ++} ++ ++/* tahang1: a phase still pending this long after its publish prints its RAM state once. */ ++#define G5P_RV3_SLOW_MS 1000U ++ ++/* The one `slow` line of a phase: the deciding sample's RAM words and the waiter's own state, so ++ * a stall that later times out is described from the first second on. RAM only. ++ */ ++static void g5p_rv3_slow(struct g5p_diag *g, u32 e, u32 seq, ++ const struct g5p_ru_wait *w, ++ const struct g5p_rv3_ev *s, u32 unsat, u64 hit_ns) ++{ ++ dev_warn( ++ g->dev, ++ "RENDER-V3 slow seq=%u engine=%u ms=%lld ring=%u/%u stamp=%08x/%08x " ++ "ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x,%08x,%08x,%08x pm80=%08x " ++ "consumed=%u retired=%u pm=%u event_ms=%lld unsat=%#x busy_ms=%lld events=%u " ++ "end=v3\n", ++ seq, e, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ s->idle[3], s->idle[4], s->idle[5], s->pm80, s->consumed, ++ s->retired, s->pm_ok, g5p_ru_rt_ms(w->start_ns, hit_ns), unsat, ++ g5p_ru_rt_ms(w->start_ns, w->busy_ns), ++ READ_ONCE(g->boot->events)); ++ g5p_np_policy(g, "slow", seq, e); ++} ++ ++/* nopatch: the policy record at a few phase ends (log-spaced) and at every timeout. RAM only. */ ++static void g5p_rv3_np_end(struct g5p_diag *g, u32 e, u32 seq, u32 result) ++{ ++ if (seq == 1 || seq == 10 || seq == 100 || seq == 1000 || ++ seq == 10000 || result == G5P_RU_WAIT_TIMEOUT) ++ g5p_np_policy(g, ++ result == G5P_RU_WAIT_TIMEOUT ? "phase-timeout" : ++ "phase-end", ++ seq, e); ++} ++ ++/* One phase's RAM-only wait. g5p_ru_wait_step()/close() decide; this loop only samples. release: the ++ * touch instant is taken after a step that left the phase undecided, before the sleep (off: never). ++ * The predicate is level-triggered: every sample, including the one right after each sleep and the ++ * closing one at the deadline, re-reads the whole RAM state (rt: the 3D, ta: the TA). ++ */ ++static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, struct g5p_ru_touch *t, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta, ++ struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) ++{ ++ struct g5p_drm *d = r->ctx->gpu; ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE( ++ d->event_serial); /* after the drain, before the sample */ ++ bool fault, silent, done; ++ ++ g5p_rv3_sample(g, r, e, seq, s); ++ /* latency101: a silent event stream during the phase is a dead firmware (never off). */ ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed) ++ *err = -EIO; ++ else if (silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = g5p_rv3_done(e, s, rt, ta); ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (rt) ++ g5p_ru_rt_note(rt, w->result); ++ if (ta) ++ g5p_ru_ta_note(ta, w->result); ++ if (w->result) { ++ g5p_rv3_np_end(g, e, seq, w->result); ++ return w->result; ++ } ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_RV3_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ g5p_rv3_slow(g, e, seq, w, s, ++ rt ? rt->unsat : ++ ta ? ta->unsat : ++ 0, ++ e ? READ_ONCE(d->td_hit_ns) : ++ READ_ONCE(d->ta_hit_ns)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ if (g5p_ru_touch_due(t, s->ns, w->result, w->expired)) ++ g5p_rv3_touch(g, t, s->ns); ++ g5p_l123_pause(d, g, e, s, seen); ++ } ++} ++ ++/* One phase's deadline: the legacy single bound, or (latency101) the soft/hard pair. */ ++static void g5p_rv3_wait_start(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_ru_wait *w, u64 pub_ns, ++ u64 job_deadline_ns) ++{ ++ if (r->hard_ms) ++ g5p_ru_wait_start_lat(w, pub_ns, r->soft_ms, r->hard_ms, ++ job_deadline_ns, ++ READ_ONCE(g->boot->events)); ++ else ++ g5p_ru_wait_start(w, pub_ns, r->wait_ms, job_deadline_ns); ++} ++ ++/* After a timeout (or fault) was decided: the one RAM look of the legacy path, or (latency101, ++ * timeout only) a RAM-only observation window that keeps draining and sampling at the wait cadence ++ * and prints nothing, so a completion that lands milliseconds after the decision is recorded ++ * (late_after/late_ns) before the receipt line and the terminal latch. It never changes the ++ * result and never touches a register. The 3D look steps a copy of the predicate's record, so the ++ * receipt keeps the deciding sample's. ++ */ ++static void g5p_rv3_observe(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, ++ const struct g5p_ru_retire *rt, ++ const struct g5p_ru_ta *ta, struct g5p_rv3_ev *late) ++{ ++ struct g5p_ru_retire copy, *c = NULL; ++ struct g5p_ru_ta tcopy, *tc = NULL; ++ ++ if (rt) { ++ copy = *rt; ++ c = © ++ } ++ if (ta) { ++ tcopy = *ta; ++ tc = &tcopy; ++ } ++ if (!r->obs_ms || w->result != G5P_RU_WAIT_TIMEOUT) { ++ g5p_rv3_sample(g, r, e, seq, late); ++ g5p_ru_wait_after(w, g5p_rv3_done(e, late, c, tc)); ++ return; ++ } ++ do { ++ g5p_boot_drain( ++ g); /* a fault latches b->failed, the result stays */ ++ g5p_rv3_sample(g, r, e, seq, late); ++ if (!g5p_ru_wait_observe(w, late->ns, ++ g5p_rv3_done(e, late, c, tc), ++ r->obs_ms)) ++ break; ++ usleep_range(1000, 2000); ++ } while (1); ++} ++ ++/* wake110: render-user-wake.h's register access, on the Fender and RGX windows. */ ++static void g5p_rv3_wk_w1(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->window[1] + reg); ++} ++ ++/* The GPC state read of g5p_wk_ring() comes right after its R1+0x14 = 7 wake: in bell mode ++ * (render_user_wake=2) it waits up to G5P_RV3_WK_SETTLE_US, re-reading every 10 us, for a settled ++ * powered pstate (bit 4 clear, nibble nonzero), as Apple's setGPCPerformanceState waits for bit 4. ++ * Reads only; the ring's own check of the final value is unchanged. ++ */ ++#define G5P_RV3_WK_SETTLE_US 2000U ++ ++static u32 g5p_rv3_wk_r1(void *ctx, u32 reg) ++{ ++ void __iomem *r1 = ((struct g5p_diag *)ctx)->window[1]; ++ u32 raw = readl(r1 + reg); ++ u64 t0; ++ ++ if (reg != G5P_WK_R1_GPC || (2) != G5P_WK_ALWAYS || ++ (!(raw & 0x10) && (raw & 0xf))) ++ return raw; ++ t0 = ktime_get_mono_fast_ns(); ++ while (((raw & 0x10) || !(raw & 0xf)) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_WK_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(r1 + reg); ++ } ++ return raw; ++} ++ ++static void g5p_rv3_wk_wmb(void *ctx) ++{ ++ dma_wmb(); ++} ++ ++static void g5p_rv3_wk_w0(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->rgx + reg); ++} ++ ++/* wake110: an r74f bell that still owns its ring (the ALIVE decoder may fire it). */ ++static bool g5p_rv3_wk_r74f_busy(struct g5p_r73b *x) ++{ ++ u32 k = READ_ONCE(x->r74f_active); ++ int s; ++ ++ if (!k || k > G5P_R74F_KINDS) ++ return false; ++ s = atomic_read_acquire(&x->r74f_bell[k - 1].state); ++ return s == G5P_R74F_PUBLISHED || s == G5P_R74F_OPEN || ++ s == G5P_R74F_FIRING; ++} ++ ++/* wake111: DS108's claim proof re-read at the publish: ALIVE proven in the current reset ++ * generation, the claim admitted in it, nothing latched failed since, no R74E power abort. ++ * Called under event_lock (interrupts off); g5p_ds_lock is a leaf (g5p_ds_event() is pure). ++ */ ++static bool g5p_rv3_wk_ds108(struct g5p_boot *b) ++{ ++ bool ok; ++ ++ spin_lock(&g5p_ds_lock); ++ ok = g5p_ds_proven(&g5p_ds_state) && ++ g5p_ds_ok(&g5p_ds_state, G5P_DSE_CLAIM); ++ spin_unlock(&g5p_ds_lock); ++ return ok && !(b->r73b && READ_ONCE(b->r73b->early_power_abort)); ++} ++ ++/* wake110: before the phase's producer store, the engine's type-1 count and a cleared first-decode ++ * time, so an event decoded right after the publish is counted. RAM only. ++ */ ++static void g5p_rv3_wake_mark(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ v->wk_t1_pub[e] = d->t1_count[e]; ++ WRITE_ONCE(d->t1_first_ns[e], 0); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* wake110: at the job's end, the type-1 counts and the first decode of each published phase. */ ++static void g5p_rv3_wake_end(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ for (u32 e = 0; e < 2; e++) { ++ v->wk_t1_end[e] = d->t1_count[e]; ++ v->wk_first_ns[e] = d->t1_first_ns[e]; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (v->phase < G5P_RV3_TA_PUBLISH) ++ v->wk_first_ns[0] = 0; ++ if (v->phase < G5P_RV3_TD_PUBLISH) ++ v->wk_first_ns[1] = 0; ++} ++ ++/* wake110: microseconds / milliseconds from a phase's publish to a later instant; -1 = none. */ ++static s64 g5p_rv3_wk_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++static s64 g5p_rv3_wk_ms(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_MSEC : ++ -1; ++} ++ ++/* wake110: one phase's publish-time decision, right after its producer store and barrier. Both ++ * arms take the event lock (the r74f owner's serialization against the decoder) and sample the ++ * same RAM; only a factor that passed every guard runs g5p_wk_ring(). Nothing is printed under ++ * the lock. Off: returns at once. ++ */ ++static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e, u32 start, u32 seq) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ struct g5p_drm *d = r->ctx->gpu; ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ u8 *p = g5p_ru_page(b, e); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&b->event_lock, flags); ++ v->wk_start_ns[e] = ktime_get_ns(); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = r->wake, ++ .seed = 0, ++ .seq = seq, ++ .e = e, ++ .attempted = g5p_wk_attempted(v->wk_did[e]), ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = r->target[e], ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4) }, ++ .type1 = READ_ONCE(d->t1_count[e]) - v->wk_t1_pub[e], ++ }; ++ v->wk_cons[e] = in.cons; ++ v->wk_power[e] = in.power; ++ v->wk_did[e] = g5p_wk_decide(&in, &io, &v->wk_fence[e], &v->wk_gpc[e]); ++ v->wk_done_ns[e] = ktime_get_ns(); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (v->wk_did[e] == G5P_WK_DID_RANG) ++ r->wake_rang++; ++} ++ ++/* The epoch's first job: the device (PM, channels) and this context's DC3 records, shared with ++ * g5p_render_attach(), then the r74f/r74g protocol of the fixed r74f_attach() on typed state. ++ */ ++static int g5p_rv3_attach(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline; ++ u32 start, tag, bad; ++ int ret; ++ ++ ret = g5p_r74f_admit(g, G5P_R74F_DC3); ++ if (ret) ++ return ret; ++ ret = g5p_render_attach_prepare(d, r, &start, &tag); ++ if (ret) ++ return ret; ++ r->dc_target = (start + 2) & 255; ++ r->bound = true; ++ rd->up = true; ++ rd->v3 = true; ++ rd->job_slots = (2) == 2 ? 2 : 1; ++ rd->pm_once = (1) == 1; ++ rd->pm_ctx = b->pm_sets ? min((3), 3U) : 0; ++ r->pm_first = r->pm_set < 0; ++ dev_info( ++ g->dev, ++ "RENDER-V3 layout job_slots=%u pm_once=%u slot1=%#x..%#x pm_ctx=%u pm_sets=%u " ++ "end=v3\n", ++ rd->job_slots, rd->pm_once, G5P_RU_SLOT1, G5P_RU_SLOT1_END, ++ rd->pm_ctx, b->pm_sets); ++ g5p_render_retain(r); ++ v->touched = true; ++ WRITE_ONCE(r->ctx->live, true); ++ dma_wmb(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ v->attach_ns = ktime_get_ns(); ++ g5p_r74_mark(g, G5P_R74_ATTACH_PUBLISHED, r->dc_target); ++ /* The producer is final from here; only the doorbell may be pending. */ ++ ret = g5p_r74f_bell(g, G5P_R74F_DC3, start, r->dc_target); ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || READ_ONCE(b->failed)) ++ return ret ?: -EIO; ++ dma_rmb(); ++ if (READ_ONCE(ctl[4]) != r->dc_target || ++ READ_ONCE(ctl[0]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(ctl[0]) == r->dc_target) { ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ bad = g5p_r74g_check(g, 0, r->slot, vm->root.dma, tag, ++ start); ++ g5p_r74_mark(g, G5P_R74_ATTACH_CONSUMED, r->dc_target); ++ g5p_r74g_dc3_line(g, 0, false); ++ if (bad) ++ return g5p_rv3_errno(g, ++ g5p_r74g_refuse(g, bad)); ++ for (u32 e = 0; e < 2; e++) ++ g5p_mmu_record( ++ g, ++ "RENDER tags slot=%u host=%08x cached=%08x expected=%08x\n", ++ r->slot[e], ++ g5p_le32(g5p_boot_shared(b, 0x84) + ++ r->slot[e] * 4), ++ g5p_le32(g5p_boot_shared(b, 0x85) + ++ r->slot[e] * 4), ++ tag); ++ g5p_mmu_record( ++ g, ++ "RENDER-USER attached TA=%u 3D=%u PC=%pad tag=%08x blocks=%u; RAM-only, firmware " ++ "DC3 installed CBASE, host MMU pass skipped\n", ++ r->slot[0], r->slot[1], &vm->root.dma, tag, ++ rd->blocks); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } while (time_before(jiffies, deadline)); ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ return g5p_rv3_errno(g, ++ g5p_r74f_stop(g, G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ READ_ONCE(ctl[0]))); ++} ++ ++/* perf60 bell mode (render_user_wake=2): a DC publication (bind, spare pairs, unbind) gets Apple's ++ * submit kick too. With the policy held in state 0 the firmware no longer drains the DC ring on ++ * its own (perf60clk-9ec288b868: a second client's RAM-only DC3 bind was never consumed). The ++ * wake path's guards (event lock, power proof, no fault or terminal, no r74f bell owning a ring, ++ * the ring holding exactly this publication), then Apple's order: R1+0x14 = 7 and its read-back, ++ * R1+0x1000 re-read every 10 us until a settled powered pstate (g5p_gpc_ok(): bit 4 clear, ++ * nibble nonzero; Apple's setGPCPerformanceState waits for bit 4 the same way), at most ++ * G5P_RV3_DC_SETTLE_US, and R0+0xb00 = 0x10 only then. A GPC that never reads on is not written: ++ * no R0 access, the publication waits as before. Off (render_user_wake != 2): returns at once. ++ */ ++#define G5P_RV3_DC_SETTLE_US 2000U ++ ++static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, ++ const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 fence = U32_MAX, gpc = U32_MAX, power, cons, n = 0; ++ const char *did = "skip-guard"; ++ unsigned long flags; ++ u64 t0; ++ ++ ; ++ ; ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ cons = READ_ONCE(ctl[0]); ++ power = g5p_wk_power_mask(READ_ONCE(g->powered), ++ READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), ++ g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ !g5p_wk_power_ready(power) || g5p_rv3_wk_r74f_busy(b->r73b) || ++ cons != start || READ_ONCE(ctl[4]) != target) ++ goto out; ++ writel(7, g->window[1] + 0x14); ++ fence = readl(g->window[1] + 0x14); ++ if (fence & ~7U) { ++ did = "skip-fence"; ++ goto out; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ gpc = readl(g->window[1] + 0x1000); ++ while (!g5p_gpc_ok(gpc) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ gpc = readl(g->window[1] + 0x1000); ++ n++; ++ } ++ if (!g5p_gpc_ok(gpc)) { ++ did = "skip-gpc"; ++ goto out; ++ } ++ /* the publication's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ wmb(); ++ writel(0x10, g->rgx + 0xb00); ++ did = "rang"; ++out: ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 dc-kick why=%s did=%s dc=%u->%u power=%02x fence=%08x gpc=%08x " ++ "settle=%u end=v3\n", ++ why, did, cons, target, power, fence, gpc, n); ++} ++ ++/* RAM only: drain the event stream, read the firmware pages (interim, see g5p_rv3_fw_touch) and ++ * sample the DC ring until its consumer reaches `target` ++ * or G5P_RU_BIND_MS pass. No doorbell and no register, whether or not a job has retired. ++ */ ++static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_RU_BIND_MS); ++ ++ for (;;) { ++ bool expired = time_after_eq(jiffies, deadline); ++ int ret = g5p_boot_drain(g); ++ ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ g5p_rv3_fw_touch(b); ++ dma_rmb(); ++ switch (g5p_ru_bind_step(READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), ++ target, expired)) { ++ case G5P_RU_BIND_CONSUMED: ++ *consumed_ns = ktime_get_ns(); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } ++} ++ ++static s64 g5p_rv3_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++/* ++ * The first job of a context on a device that is already up binds the context's slot pair ++ * with a DC3 pair published in RAM only (multi-client-design.md §4.2). No doorbell and no ++ * register access of any kind, so it is safe after a job retired and while the GPU may be ++ * idle-gated: the firmware services the DC ring in its scheduler pass, before TA and 3D, and ++ * the host waits on RAM. Core lock held, between jobs. ++ * ++ * Refused before the producer store, it touched nothing. From the store on, the firmware may ++ * hold the pair: the context is retained, and on failure it is lost (v->bind_failed) but the ++ * device and the contexts already bound are not. A pair not consumed in time keeps every later ++ * bind refused until the firmware catches up. ++ */ ++/* ++ * render_user_close_park (release8 quiet2, docs/gpu/experiments/release8-quiet2.md). The PM writes ++ * a closed context's free list back through that context's TA slot after the context is gone, and ++ * not only while it is the last or the previous distinct context (render-user-pmset.h): two bench ++ * faults wrote the free list of a context two switches back, through its slot, on the 2nd job of a ++ * third context, after the pm-prev rule had unbound it (page-invalid level 2, its PC level empty). ++ * iOS never leaves the PM's target unresolvable: its PM is the device's (shared by the work queues, ++ * AGXWorkQueue_initAndAttachPM), in the kernel's GPU address space, and a GART keeps its slots until ++ * the address space is destroyed (AGXGart_detachSlotsAndDestroyAddressSpace). Here, a closed ++ * context's pair is bound again, right after its unbind (between jobs, core lock held), to the park ++ * catalogue: a kernel page that maps only the firmware window, where the PM sets live. A late ++ * write-back then lands in the closed context's own set page (released, reused last) instead of ++ * faulting. A later bind of the pair unbinds it first. Both steps are the proven DC3 ++ * transitions (PA to zero, zero to PA); a park that fails leaves the pair unbound, as before. ++ */ ++/* release8 default */ ++ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why); ++ ++/* Before a bind, core lock held: a pair a close parked (RAM: park PA and tag on both slots) is ++ * unbound first, by its own DC3. 0 when the pair was not parked or the unbind verified. ++ */ ++static int g5p_rv3_unpark(struct g5p_drm *d, const u32 *slot) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ ++ if (!d->rdev->park.cpu) ++ return 0; ++ dma_rmb(); ++ g5p_render_bind_mem(b, slot, READ_ONCE(ctl[4]), &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!g5p_ru_bind_parked(&s, d->rdev->park.dma)) ++ return 0; ++ return g5p_rv3_unbind_slots(d, slot, 2, "unpark"); ++} ++ ++static int g5p_rv3_bind(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, tag, bad = 0; ++ int ret; ++ ++ dma_rmb(); ++ if (rd->dc_pending && READ_ONCE(ctl[0]) != rd->dc_wait) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u dc=%u/%u wait=%u: an earlier bind is " ++ "pending; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4]), rd->dc_wait); ++ return -EIO; ++ } ++ rd->dc_pending = false; ++ ret = g5p_rv3_unpark(d, r->slot); ++ if (ret) ++ return ret; ++ ret = g5p_render_vm_prepare(d, r, &start, &tag); ++ if (ret) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u ret=%d dc=%u/%u: ring busy or slot in use; " ++ "no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], ret, READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4])); ++ return ret; ++ } ++ r->dc_target = (start + 2) & 255; ++ g5p_render_retain(r); ++ v->bind_failed = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, r->dc_target, "bind"); ++ ret = g5p_rv3_dc_wait(g, r->dc_target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = r->dc_target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, vm->root.dma, tag, ++ r->dc_target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=%lld tag=%08x host=%08x/%08x " ++ "cached=%08x/%08x dc=%u->%u/%u bad=%#x pc=%pad mode=ram-dc3 end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ tag, s.host[0], s.host[1], s.cached[0], s.cached[1], start, ++ s.cons, s.prod, bad, &vm->root.dma); ++ if (ret) ++ return ret; ++ v->bind_failed = false; ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ return 0; ++} ++ ++/* ++ * TEMPORARY fallback (render_user_spare_pairs, read at the epoch's first job), for a firmware ++ * that never drains a DC3 published after a job retired. Before job 1's attach, bind that ++ * many extra slot pairs, each to a zeroed kernel-owned catalogue, in one RAM-only batch with no ++ * doorbell. The firmware drains it on its own like the self-drained attach of this boot tuple ++ * (r87b); job 1's own attach, doorbell and r74g checks then run unchanged behind it ++ * (g5p_r74f_ring() refuses a ring that holds more than that attach's pair). Later contexts ++ * take a pair over with g5p_rv3_adopt(). A pair that does not check out is never used. Core ++ * lock held, device not up, so register access is still allowed (none is made here). ++ */ ++static int g5p_rv3_spares(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 want = min((0), G5P_RU_MAX_SPARES); ++ u32 start = READ_ONCE(ctl[4]), target, n, ok = 0; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ unsigned long flags; ++ int ret; ++ ++ if (!want || rd->spares) ++ return 0; ++ rd->spares = kcalloc(want, sizeof(*rd->spares), GFP_KERNEL); ++ if (!rd->spares) ++ return -ENOMEM; ++ for (n = 0; n < want; n++) { ++ struct g5p_rspare *sp = &rd->spares[n]; ++ bool pair; ++ ++ spin_lock_irqsave(&g->slot_lock, flags); ++ pair = g5p_ru_slot_pair(g->user_slots, false, sp->slot); ++ if (pair) { ++ __set_bit(sp->slot[0], g->user_slots); ++ __set_bit(sp->slot[1], g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ if (!pair) ++ break; ++ g5p_render_bind_mem(b, sp->slot, start + 2 * n, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (!g5p_ru_bind_busy(&s) && ++ !g5p_vm_alloc_page(d->drm.dev, &sp->root)) { ++ g5p_table_clean_page(g, &sp->root); ++ continue; ++ } ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(sp->slot[0], g->user_slots); ++ __clear_bit(sp->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ break; ++ } ++ if (!n) ++ return 0; ++ dsb(sy); ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_bind_mem(b, rd->spares[i].slot, start + 2 * i, &m); ++ g5p_ru_bind_publish(&m, rd->spares[i].slot, ++ rd->spares[i].root.dma, G5P_RU_SPARE_TAG); ++ } ++ rd->spare_count = n; ++ target = (start + 2 * n) & 255; ++ v->touched = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "spares"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ for (u32 i = 0; !ret && i < n; i++) { ++ struct g5p_rspare *sp = &rd->spares[i]; ++ ++ g5p_render_bind_mem(b, sp->slot, start + 2 * i, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ sp->ok = !g5p_ru_bind_check(&s, sp->slot, sp->root.dma, ++ G5P_RU_SPARE_TAG, target); ++ ok += sp->ok; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 spares pairs=%u ok=%u ret=%d consumed_us=%lld dc=%u->%u first=%u/%u " ++ "last=%u/%u end=v3\n", ++ n, ok, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ READ_ONCE(ctl[0]), rd->spares[0].slot[0], rd->spares[0].slot[1], ++ rd->spares[n - 1].slot[0], rd->spares[n - 1].slot[1]); ++ return ret; ++} ++ ++/* ++ * TEMPORARY fallback: the first job of a context on a device whose job 1 bound spare pairs. ++ * Takes the next pair that checked out, makes its catalogue page this VM's root (a copy of the ++ * current root) and publishes the VM's tag through the proven RAM-only tag path: CBASE already ++ * names the page, and the changed tag makes the firmware invalidate the slots' PC level at this ++ * context's first job. No DC3, no register. Core lock held; the context has a job queued, so ++ * no VM edit of it runs concurrently. ++ */ ++static int g5p_rv3_adopt(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_rspare *sp = NULL; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ struct g5p_dma_page old; ++ unsigned long flags; ++ struct g5p_rt *rt; ++ u32 entry; ++ int ret; ++ ++ for (u32 i = 0; i < rd->spare_count && !sp; i++) ++ if (rd->spares[i].ok && !rd->spares[i].used) ++ sp = &rd->spares[i]; ++ if (!sp) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare: none left of %u; no hardware " ++ "touched end=v3\n", ++ r->slot[0], r->slot[1], rd->spare_count); ++ return -ENOSPC; ++ } ++ /* RAM: the pair still names its page, with the tag job 1 checked. */ ++ dma_rmb(); ++ g5p_render_bind_mem(b, sp->slot, 0, &m); ++ g5p_ru_bind_read(&m, 0, 0, &s); ++ for (u32 e = 0; e < 2; e++) { ++ if (s.pa[e] == sp->root.dma && s.host[e] == G5P_RU_SPARE_TAG && ++ s.cached[e] == G5P_RU_SPARE_TAG) ++ continue; ++ sp->ok = false; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare pair=%u/%u host=%08x cached=%08x: " ++ "pair changed; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], sp->slot[0], sp->slot[1], ++ s.host[e], s.cached[e]); ++ return -EIO; ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ memcpy(sp->root.cpu, vm->root.cpu, G5P_GPU_PAGE); ++ old = vm->root; ++ vm->root = sp->root; ++ sp->root = old; /* the VM's former root, never published */ ++ g5p_ru_free_page(d->drm.dev, &sp->root); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(r->slot[0], g->user_slots); ++ __clear_bit(r->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ r->slot[0] = sp->slot[0]; ++ r->slot[1] = sp->slot[1]; ++ sp->used = true; ++ g5p_render_retain(r); ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ /* Root and tables to PoC, then a tag unlike the cached one in both slots. */ ++ ret = g5p_render_invalidate(r->ctx); ++ v->bind_failed = ret != 0; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=0 tag=%08x cached=%08x/%08x " ++ "pc=%pad mode=spare end=v3\n", ++ r->slot[0], r->slot[1], ret, ++ g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[0] * 4), ++ s.cached[0], s.cached[1], &vm->root.dma); ++ return ret; ++} ++ ++/* r74f_publish_ta() on typed state: record, WORK_CTL, producer; then (job 1 only) the r74g grace ++ * and, only if the ring is still pending, the one ALIVE-owned fallback doorbell. Jobs >= 2 ++ * publish and return: no grace, no bell (the r74f/r74g machinery is one-shot). No canary, no marker. ++ */ ++static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *x = b->r73b; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_r74g_dc3 *dc = x->r74g_dc3; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x60); ++ u8 *p = g5p_ru_page(b, 0); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ bool first = seq == 1; ++ u32 bad; ++ int ret; ++ ++ if (!r->bound || r->submitted[0] || start > 255 || ++ READ_ONCE(ring[0]) != start || ++ g5p_le32(p + G5P_WORK_CTL) != previous || ++ g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) ++ return -EBUSY; ++ if (first) { ++ ret = g5p_r74f_admit(g, G5P_R74F_TA); ++ if (ret) ++ return ret; ++ /* g5p_r74g_ta_preflight() on typed state: DC3 still complete with the same records, ++ * tag and backing, and no page-table mutation since the attach. ++ */ ++ bad = g5p_r74g_check(g, 1, r->slot, dc[0].expected_pa, ++ dc[0].tag, dc[0].start); ++ if (g5p_vm_generation(&vm->mutation) != dc[0].tag || ++ vm->root.dma != dc[0].expected_pa || ++ r->dc_target != dc[0].target) { ++ bad |= G5P_R74G_BAD_MUTATION; ++ dc[1].bad = bad; ++ } ++ g5p_r74g_dc3_line(g, 1, false); ++ if (bad) ++ return g5p_rv3_errno(g, g5p_r74g_refuse(g, bad)); ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x61] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 0 : ++ j == 1 ? g5p_ru_meta(0) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[0] = (start + 1) & 255; ++ r->submitted[0] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_PUBLISH, start); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], r->target[0]); ++ dma_wmb(); ++ if (first) { ++ ret = g5p_r74g_ta(g, start, r->target[0]); ++ v->grace_end = READ_ONCE(x->r74g_grace_end); ++ v->bell = ret == 0; ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ } ++ /* wake110: jobs >= 2 reach this at once after the producer store (job 1: excluded). */ ++ if (!first) ++ 0; ++ g5p_rv3_wake(g, r, v, 0, start, seq); ++ /* The line the userspace lane parses (a9c_parse_publish): same format as the fixed path's. */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 0, r->slot[0], endpoint, first ? 0x10 : 0, ++ g5p_ru_meta(0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 0); ++ return 0; ++} ++ ++/* The RAM preconditions of the 3D publish, one bit per clause (0 = admitted). RAM reads only; the bits are ++ * what the refusal line prints, so a second blocker is visible in the run that meets it: ++ * 0 TA not retired 1 TA not consumed 2 3D already submitted 3 producer > 255 4 consumer != producer ++ * 5 WORK_CTL 6 WORK_CTL+0x40 7 command tag != 4 8 dependency word 9 dependency value ++ * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) ++ */ ++static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, ++ const u32 *ring, u32 start, u32 previous, u32 seq) ++{ ++ const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ ++ return (u32)!r->retired[0] | (u32)!r->consumed[0] << 1 | ++ (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | ++ (u32)(READ_ONCE(ring[0]) != start) << 4 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 5 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 6 | ++ (u32)(g5p_le32(cmd) != 4) << 7 | ++ (u32)(g5p_le32(cmd + 4) != g5p_ru_meta(0) + G5P_R22_STAMP + 4) ++ << 8 | ++ (u32)(g5p_le32(cmd + 8) != seq) << 9 | ++ (u32)(g5p_le32(p + G5P_R22_STAMP) != seq - 1 || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq - 1) ++ << 10; ++} ++ ++/* The 3D after the TA retired: record, WORK_CTL, producer, and NO doorbell. */ ++static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x64); ++ u8 *p = g5p_ru_page(b, 1); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ u32 why; ++ ++ /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one ++ * fails, and the refusal names the clauses and the raw values it read. ++ */ ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq); ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 3d-refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x cmd=%08x/%08x/%08x " ++ "stamp=%08x/%08x end=v3\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ seq))), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 4), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 8), ++ g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x65] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 1 : ++ j == 1 ? g5p_ru_meta(1) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[1] = (start + 1) & 255; ++ r->submitted[1] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->td_pub_ns = ktime_get_ns(); ++ smp_store_release(&ring[4], r->target[1]); ++ dma_wmb(); ++ 0; ++ g5p_rv3_wake(g, r, v, 1, start, seq); ++ /* The 3D twin of the TA's publish line: engine 1, kick 0 (no doorbell). */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 1, r->slot[1], endpoint, 0, ++ g5p_ru_meta(1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 1); ++ return 0; ++} ++ ++static const char *const g5p_rv3_ta_how[G5P_RU_TA_HOWS] = { "none", ++ "idle-clear", ++ "event", "hold" }; ++ ++/* tahang1: the TA decisions since load, by rule (core lock held). A TA that completed with an ++ * idle word still set prints one line with the running totals: rate-limited for rule (b), which ++ * is routine, always for rule (c), which means the event was not seen. RAM only. ++ */ ++static u32 g5p_rv3_ta_rules[G5P_RU_TA_HOWS]; ++static DEFINE_RATELIMIT_STATE(g5p_rv3_ta_rs, 5 * HZ, 4); ++ ++static void g5p_rv3_ta_count(struct g5p_diag *g, struct g5p_drm *d, ++ const struct g5p_rv3 *v) ++{ ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ u32 how = min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1); ++ u64 hit = READ_ONCE(d->ta_hit_ns); ++ ++ g5p_rv3_ta_rules[how]++; ++ if (how == G5P_RU_TA_IDLE_CLEAR) ++ return; ++ if (how == G5P_RU_TA_EVENT && !__ratelimit(&g5p_rv3_ta_rs)) ++ return; ++ dev_warn( ++ g->dev, ++ "RENDER-V3 ta-done seq=%u how=%s idle=%08x,%08x,%08x pm80=%08x coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld busy_ms=%lld rules=%u/%u/%u end=v3\n", ++ v->seq, g5p_rv3_ta_how[how], s->idle[0], s->idle[1], s->idle[2], ++ s->pm80, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, hit), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_ta_rules[G5P_RU_TA_IDLE_CLEAR], ++ g5p_rv3_ta_rules[G5P_RU_TA_EVENT], ++ g5p_rv3_ta_rules[G5P_RU_TA_HOLD]); ++} ++ ++/* A receipt line: a failed job's at KERN_EMERG as before (it survives a freeze), a completed job's at ++ * job_log (wake every job, end/lat/retire the first 8 of a context: routine at desktop rates). ++ */ ++#define g5p_rv3_receipt_printk(g, ret, fmt, ...) \ ++ do { \ ++ if (ret) \ ++ dev_emerg((g)->dev, fmt, ##__VA_ARGS__); \ ++ else \ ++ g5p_job_printk((g)->dev, fmt, ##__VA_ARGS__); \ ++ } while (0) ++ ++/* The end of a job, whatever it was: one receipt line on the console (it survives a freeze) and the ++ * RAM-only raw state in the blob. Reads RAM only. ++ */ ++static void g5p_rv3_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_r73b *x = g->boot->r73b; ++ const struct g5p_rv3_ev *t = &v->last[0], *d = &v->last[1]; ++ const struct g5p_ru_touch *k = v->touch; ++ ++ /* release: every job while the A/B is on, so both arms are complete past the verbose jobs. */ ++ if (r->touch_ms) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 touch seq=%u ret=%d arm=%s touch_ms=%u ta=%s/%lld td=%s/%lld " ++ "seen_us=%lld/%lld read_us=%lld/%lld after_us=%lld/%lld bytes=%u/%u " ++ "sum=%08x/%08x end=v3\n", ++ v->seq, ret, ++ g5p_ru_touch_arm(v->seq) ? "touch" : "control", ++ r->touch_ms, g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? (s64)(v->ta.end_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? (s64)(v->td.end_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_ru_touch_us(v->ta_pub_ns, k[0].seen_ns), ++ g5p_ru_touch_us(v->td_pub_ns, k[1].seen_ns), ++ g5p_ru_touch_us(k[0].seen_ns, k[0].done_ns), ++ g5p_ru_touch_us(k[1].seen_ns, k[1].done_ns), ++ g5p_ru_touch_after_us(&k[0], v->ta.result, ++ v->ta.end_ns), ++ g5p_ru_touch_after_us(&k[1], v->td.result, ++ v->td.end_ns), ++ k[0].bytes, k[1].bytes, k[0].sum, k[1].sum); ++ /* wake110: every job while the wake is on (the end/lat/retire lines stop at seq 8). */ ++ if (r->wake) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 wake seq=%u ret=%d mode=%u seed=%08x arm=%s/%s did=%s/%s " ++ "power=%02x/%02x fence=%08x/%08x gpc=%08x/%08x cons_at=%u/%u bell_us=%lld/%lld " ++ "lock_us=%lld/%lld ta=%s/%lld td=%s/%lld ev_ms=%lld/%lld consumed_ms=%lld/%lld " ++ "busy_ms=%lld/%lld t1_pub=%u/%u t1_end=%u/%u endpoint=%u ta_stamp=%08x/%08x " ++ "td_stamp=%08x/%08x ta_ring=%u/%u td_ring=%u/%u rang=%u end=v3\n", ++ v->seq, ret, r->wake, r->wake_seed, ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 0)], ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[0], ++ G5P_WK_DIDS - 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[1], ++ G5P_WK_DIDS - 1)], ++ v->wk_power[0], v->wk_power[1], v->wk_fence[0], ++ v->wk_fence[1], v->wk_gpc[0], v->wk_gpc[1], ++ v->wk_cons[0], v->wk_cons[1], ++ g5p_rv3_wk_us(v->ta_pub_ns, v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->td_pub_ns, v->wk_done_ns[1]), ++ g5p_rv3_wk_us(v->wk_start_ns[0], v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->wk_start_ns[1], v->wk_done_ns[1]), ++ g5p_rv3_results[v->ta.result], ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_rv3_results[v->td.result], ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->wk_first_ns[0]), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->wk_first_ns[1]), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td_consumed_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.busy_ns), ++ v->wk_t1_pub[0], v->wk_t1_pub[1], v->wk_t1_end[0], ++ v->wk_t1_end[1], g5p_ru_endpoint(v->seq), t->stamp[0], ++ t->stamp[1], d->stamp[0], d->stamp[1], t->prod, t->cons, ++ d->prod, d->cons, r->wake_rang); ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 end seq=%u ret=%d phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " ++ "cap_ms=%u ta_consumed_ms=%lld td_consumed_ms=%lld late_gap=%u/%u " ++ "late_after=%u/%u grace=%u bells=%d ta_ring=%u/%u td_ring=%u/%u " ++ "ta_stamp=%08x/%08x td_stamp=%08x/%08x pm80=%08x r0_after_ta_publish=bell-only " ++ "end=v3\n", ++ v->seq, ret, ++ g5p_rv3_phase_names[min_t(u32, v->phase, G5P_RV3_PHASES - 1)], ++ g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? ++ (s64)(v->ta.end_ns - v->ta_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? ++ (s64)(v->td.end_ns - v->td_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ r->wait_ms, G5P_RU_V3_CAP_MS, ++ v->ta_consumed_ns ? (s64)(v->ta_consumed_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td_consumed_ns ? (s64)(v->td_consumed_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_gap, v->td.late_gap, v->ta.late_after, ++ v->td.late_after, v->grace_end, atomic_read(&x->r74f_bells), ++ t->prod, t->cons, d->prod, d->cons, t->stamp[0], t->stamp[1], ++ d->stamp[0], d->stamp[1], t->pm80 ? t->pm80 : d->pm80); ++ if (r->hard_ms) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 lat seq=%u soft_ms=%u hard_ms=%u obs_ms=%u slow=%u/%u late_soft=%u/%u " ++ "silent=%u/%u soft_seen_ms=%lld/%lld busy_ms=%lld/%lld late_after_ms=%lld/%lld " ++ "events=%u end=v3\n", ++ v->seq, r->soft_ms, r->hard_ms, r->obs_ms, v->ta.slow, ++ v->td.slow, v->ta.late_soft, v->td.late_soft, ++ v->ta.silent, v->td.silent, ++ v->ta.soft_seen_ns ? ++ (s64)(v->ta.soft_seen_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.soft_seen_ns ? ++ (s64)(v->td.soft_seen_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.busy_ns ? (s64)(v->ta.busy_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.busy_ns ? (s64)(v->td.busy_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_ns ? (s64)(v->ta.late_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.late_ns ? (s64)(v->td.late_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ READ_ONCE(g->boot->events)); ++ /* repeat109: the 3D's firmware retirement (coherent_ms, event_ms) apart from the host's decision ++ * (decided_ms; held_ms = how long a coherent target waited for it), the clauses still open at the ++ * deciding or closing sample (unsat=TA/3D, render-user-retire.h bits) and that sample's 3D idle words. ++ */ ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 retire seq=%u ret=%d td=%s how=%s settle_ms=%u coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld held_ms=%lld unsat=%#x/%#x " ++ "idle=%08x,%08x,%08x,%08x,%08x,%08x run=%u breaks=%u busy_ms=%lld clear_ms=%lld " ++ "ta_how=%s ta_unsat=%#x ta_coherent_ms=%lld ta_event_ms=%lld " ++ "ta_idle=%08x,%08x,%08x end=v3\n", ++ v->seq, ret, g5p_rv3_results[v->td.result], ++ g5p_rv3_how[min_t(u32, v->rt.decided, 3)], r->settle_ms, ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td_event_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_ru_rt_ms(v->rt.coherent_ns, v->td.end_ns), ++ g5p_rv3_ta_unsat(t), v->rt.unsat, d->idle[0], d->idle[1], ++ d->idle[2], d->idle[3], d->idle[4], d->idle[5], v->rt.run, ++ v->rt.breaks, g5p_ru_rt_ms(v->td_pub_ns, v->rt.busy_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.clear_ns), ++ g5p_rv3_ta_how[min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1)], ++ v->tr.unsat, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta_event_ns), t->idle[0], ++ t->idle[1], t->idle[2]); ++ /* The raw rows once per context (its first job) and for every failure: the blob is not a ++ * frame log. ++ */ ++ if (!v->touched || (r->jobs > 1 && !ret)) ++ return; ++ g5p_blob_hex(g, "RENDER-V3 PM-control", g5p_rv3_pm_cpu(g->boot, r), ++ 0x140); ++ g5p_blob_hex(g, "RENDER-V3 TA-W", ++ g5p_ru_page(g->boot, 0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, v->seq)), ++ G5P_RU_TA_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 TA-CH", g5p_ru_page(g->boot, 0), 0x340); ++ /* The stamp words (+0x1840/+0x1844) live in this 0x80-byte window; no earlier blob held one. */ ++ g5p_blob_hex(g, "RENDER-V3 TA-STAMP", ++ g5p_ru_page(g->boot, 0) + G5P_R22_TQ, 0x80); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) { ++ g5p_blob_hex(g, "RENDER-V3 3D-W", ++ g5p_ru_page(g->boot, 1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ v->seq)), ++ G5P_RU_3D_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 3D-CH", g5p_ru_page(g->boot, 1), ++ 0x340); ++ g5p_blob_hex(g, "RENDER-V3 3D-STAMP", ++ g5p_ru_page(g->boot, 1) + G5P_R22_TQ, 0x80); ++ } ++} ++ ++/* Jobs >= 2: the legacy job>=2 PM step with policy 0 (pool, list and F rebuilt from the device ++ * pool, state S kept), whichever context the job belongs to. ++ */ ++static int g5p_rv3_pm_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks); ++ mutex_unlock(&rd->lock); ++ kvfree(physical); ++ return ret; ++} ++ ++/* render-user-loss.h for a phase that did not complete. */ ++static int g5p_rv3_loss(struct g5p_drm *d, struct g5p_render *r, bool first, ++ u32 res, const struct g5p_ru_wait *w) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 drain_ms = (45000); ++ struct g5p_ru_loss_in in = { ++ .result = res, ++ .first = first, ++ .isolate = drain_ms && ++ g5p_ru_drain_valid(drain_ms, r->hard_ms, r->obs_ms), ++ .silent = w->silent, ++ .fault = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ }; ++ ++ return g5p_ru_loss(&in); ++} ++ ++/* ++ * The phase on engine `e` of this job timed out with the firmware alive: lose the job and its ++ * context, not the device. No terminal note and no device failure: the job ends -ETIMEDOUT ++ * (g5p_render_run() marks the context bad, so its client gets -EIO from now on and its pair and ++ * memory stay retained), and the channel is blocked behind the stuck phase until the scheduler's ++ * drain (g5p_rv3_drain()) clears it. RAM only. Fails (-ENOMEM) only if the record cannot be ++ * allocated; the caller then ends the epoch as before. ++ */ ++static int g5p_rv3_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ const struct g5p_ru_wait *w = e ? &v->td : &v->ta; ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = e; ++ s->drain_ms = (45000); ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u: context " ++ "lost, device kept, channel blocked end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++/* A job that stopped the firmware may be recovered from in this boot (core recover.inc, ++ * render-user-recover.h). A refusal is one line: why the epoch ends. ++ */ ++static bool g5p_rv3_recoverable(struct g5p_drm *d, bool first) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_ru_recover_in in = { ++ .first = first, ++ .retired = g5p_r8_is_retired(), ++ .closed = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .recoveries = rd->recoveries, ++ .streak = rd->recover_streak, ++ .budget = (1), ++ .streak_max = (3), ++ }; ++ ++ if (g5p_ru_recoverable(&in)) ++ return true; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 no-recover why=%s recoveries=%u streak=%u recover=%u streak_max=%u " ++ "boot_max=%u first=%u end=v3\n", ++ g5p_ru_recover_refusal(&in), in.recoveries, in.streak, ++ in.budget, in.streak_max, G5P_RU_RECOVER_BOOT_MAX, first); ++ return false; ++} ++ ++/* The job stopped the firmware (silent stream, a fault): lose it and its context, not the ++ * device; the worker recovers the GPU before it picks anything (g5p_rv3_recover()). RAM only. ++ */ ++static void g5p_rv3_lose_recover(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e, ++ const char *why) ++{ ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_fw_recovery rec = b->fw_recovery; ++ u32 recs = READ_ONCE(b->fw_recoveries); ++ ++ v->lost = true; ++ WRITE_ONCE(d->rdev->recovering, true); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u why=%s: context lost, device " ++ "recovers end=v3\n", ++ seq, e, r->slot[0], r->slot[1], why); ++ /* The guilty job is the one waited on (one job in flight, core lock); the packet's slot ++ * names the phase the firmware stopped in (render-user-recover.h). RAM only. ++ */ ++ if (recs) ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-attrib seq=%u engine=%u packet_slot=%u match=%u code=%08x " ++ "w20=%08x w24=%08x packets=%u end=v3\n", ++ seq, e, rec.slot, rec.slot == e, rec.code, rec.w20, ++ rec.w24, recs); ++} ++ ++/* render_user_test_fault_job: this job's TA reads its control stream (W+0x98, g5p_ru_fill_ta()) ++ * from the page the context reserved in its VM and never maps, so the GPU faults in that VM. A ++ * RAM edit of the job's own W copy before it is staged; the firmware sees an ordinary job. ++ */ ++static void g5p_rv3_test_fault(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, u32 seq) ++{ ++ g5p_put64(job->ta_w + 0x98, r->fault_node.start); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-fault seq=%u job=%u slots=%u/%u va=%#llx: TA control stream " ++ "unmapped end=v3\n", ++ seq, r->jobs + 1, r->slot[0], r->slot[1], r->fault_node.start); ++} ++ ++/* One V3 job: [attach, the epoch's job 1 only | bind, a later context's first job,] TA, 3D. ++ * Core lock held. Returns 0, -ECANCELED (a latched stage ended it), -ETIMEDOUT (a phase ran out ++ * its deadline, or the bind's DC3 was not consumed), or a fault. ++ */ ++/* perf60 MMU arm, bits 1/3 (render_user_mmu_inval): before a bound context's job publishes ++ * anything, move its slots' host tag to a fresh PC field, so the firmware's tag compare at this ++ * job's TA and 3D kicks takes its PC-level invalidate (e138 = (S << 3) | 4) for each slot. RAM ++ * only. vm->mutation is left alone: an edit on another thread (VM_BIND of a render VM takes no ++ * core lock) recomputes its own tag and checks it only against the cached tags, as before. ++ */ ++static void g5p_rv3_mmu_retag(struct g5p_diag *g, struct g5p_render *r, u32 seq, ++ bool switched) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 host[2], cached[2], tag; ++ ++ dma_rmb(); ++ for (u32 e = 0; e < 2; e++) { ++ host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4); ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ } ++ tag = g5p_ru_mmu_pc_tag(host[0] ?: cached[0], cached[0], cached[1]); ++ if (tag) { ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, ++ tag); ++ dma_wmb(); ++ } ++ g5p_mmu_record( ++ g, ++ "RENDER-V3 mmu-retag seq=%u slots=%u/%u why=%s host=%08x/%08x cached=%08x/%08x " ++ "tag=%08x end=v3\n", ++ seq, r->slot[0], r->slot[1], switched ? "switch" : "every", ++ host[0], host[1], cached[0], cached[1], tag); ++} ++ ++/* perf60 A/B: the park that the never-park envelope removed, back around a context switch only. ++ * Before a bound context's job that follows another context's job, ctrl+0x1040 = 50 makes the ++ * firmware's next pass compute x < 100 (policy state 2: it parks the GPC when idle, as release7); ++ * after the window the pin goes back and the job runs in state 0 with its bells as usual. RAM ++ * only: the firmware reloads policy+0x6f4 from ctrl+0x1040 on every pass (perf-envelope.h). ++ */ ++ ++ ++ ++/* perf60: the incoming context's render targets on a switch (lat123_log), RAM. */ ++static void g5p_rv3_switch_line(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ char buf[160]; ++ struct g5p_rt *rt; ++ u32 n = 0, len = 0; ++ ++ buf[0] = 0; ++ list_for_each_entry(rt, &r->targets, link) ++ if (n++ < 4) ++ len += scnprintf(buf + len, sizeof(buf) - len, ++ " %ux%u/s%u/p%u%s", rt->width, ++ rt->height, rt->samples ?: 1, ++ rt->page_count, ++ rt->destroyed ? "/d" : ""); ++ dev_info(g->dev, ++ "RENDER-V3 switch seq=%u to=%u/%u jobs=%u rts=%u%s end=v3\n", ++ seq, r->slot[0], r->slot[1], r->jobs, n, buf); ++} ++ ++/* perf60 per-context PM state (render_user_pm_switch, default 0). The PM record's selected state ++ * S (control+0x80, 0xc0 bytes: the backing pointers a0/a8/b0 the fresh TA loads the PM context ++ * from, the lanes TA finish saves and the 3D start restores, the bc accumulator) and the five ++ * aux pages it points into (the PM context store) are built once at attach and then shared by ++ * every context; the host never rewrites them. iOS builds one PM record per work queue ++ * (6cb2378: S explicit zeros plus the four backing pointers) and keeps it per queue. ++ * 1: a context's job after another context's job starts from attach's fresh state (S rebuilt, ++ * aux zeroed); ++ * 2: each context keeps its own S + aux: saved after each of its completed jobs, restored when ++ * it follows another context's job, fresh the first time. ++ * RAM only, between jobs (the previous job retired, this one has published nothing). ++ */ ++ ++/* perf60 stage 4: (re)build context r's PM set for this epoch: grow its pool to the device's ++ * block count, zero its aux pages, build its control + S, list and F (g5p_ru_pm(), as attach ++ * builds the device PM) and clean them to PoC. rd->lock (the pool's size) held. ++ */ ++static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ const char *how) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_dma_page *page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ u32 blocks = rd->blocks; ++ u64 *physical, list_va, free_va; ++ int ret = 0; ++ ++ lockdep_assert_held(&rd->lock); ++ if (blocks > r->pm_blocks) { ++ struct g5p_dma_page *pool = ++ kvcalloc(blocks * 8, sizeof(*pool), GFP_KERNEL); ++ ++ if (!pool) ++ return -ENOMEM; ++ if (r->pm_pool) ++ memcpy(pool, r->pm_pool, ++ r->pm_blocks * 8 * sizeof(*pool)); ++ for (u32 i = r->pm_blocks * 8; !ret && i < blocks * 8; i++) ++ ret = g5p_vm_alloc_page(d->drm.dev, &pool[i]); ++ if (ret) { ++ for (u32 i = r->pm_blocks * 8; i < blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &pool[i]); ++ kvfree(pool); ++ return ret; ++ } ++ kvfree(r->pm_pool); ++ r->pm_pool = pool; ++ r->pm_blocks = blocks; ++ } ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ for (u32 i = 0; i < r->pm_blocks * 8; i++) { ++ physical[i] = r->pm_pool[i].dma; ++ memset(r->pm_pool[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm_pool[i]); ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ struct g5p_dma_page *aux = &g5p_rv3_pm_aux(rd, r)[i]; ++ ++ memset(aux->cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, aux); ++ } ++ list_va = g5p_pmset_va(r->pm_set, G5P_PMSET_LIST); ++ free_va = g5p_pmset_va(r->pm_set, G5P_PMSET_FREE); ++ ret = g5p_ru_pm(page[G5P_PMSET_CONTROL].cpu, page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, list_va, free_va, physical, ++ r->pm_blocks, G5P_RU_PRIVATE_VA); ++ kvfree(physical); ++ for (u32 k = 0; k < G5P_PMSET_PAGES; k++) ++ g5p_table_clean_page(g, &page[k]); ++ dma_wmb(); ++ if (!ret) ++ r->pm_epoch = rd->recoveries + 1; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u meta=%#x " ++ "ret=%d end=v3\n", ++ seq, r->slot[0], r->slot[1], how, r->pm_set, ++ g5p_rv3_pm_id(rd, r), r->pm_blocks, g5p_rv3_pm_meta(r), ret); ++ return ret; ++} ++ ++/* Before a context's bind (its tables are published by the bind that follows): take a free set, ++ * allocate its aux pages, point G5P_RU_PRIVATE_VA in its VM at them, and build the set. On any ++ * failure the context keeps the device PM and the shared aux pages, as before stage 4. ++ */ ++static void g5p_rv3_pmctx_assign(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ int set, ret = 0; ++ ++ if (r->pm_set >= 0) ++ return; ++ set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next); ++ for (u32 i = 0; set >= 0 && !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!r->pm_aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &r->pm_aux[i]); ++ if (set < 0 || ret) { ++ dev_warn( ++ d->core->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=none set=%d ret=%d: the device PM end=v3\n", ++ seq, r->slot[0], r->slot[1], set, ret); ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "new"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ return; ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ u64 va = G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE; ++ u8 *pte = g5p_vm_pte(vm, d->drm.dev, va, false); ++ ++ u64 leaf = g5p_leaf_entry(r->pm_aux[i].dma, ++ g5p_r25_attributes(&r->v, va)); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, leaf, 2); ++ } ++ dma_wmb(); ++ r->pm_aux_mapped = true; ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* Before each job of a context with a set: a new epoch or a grown device pool rebuilds the set; ++ * without pm_once the pool is rebuilt per job, as g5p_rv3_pm_rebuild() does the device's. ++ */ ++static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_dma_page *page; ++ u64 *physical; ++ int ret = 0; ++ ++ if (r->pm_set < 0) ++ return 0; ++ page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ mutex_lock(&rd->lock); ++ if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { ++ ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); ++ } else if (!rd->pm_once) { ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), ++ GFP_KERNEL); ++ ret = physical ? 0 : -ENOMEM; ++ for (u32 i = 0; !ret && i < r->pm_blocks * 8; i++) ++ physical[i] = r->pm_pool[i].dma; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ page[G5P_PMSET_CONTROL].cpu, ++ page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, ++ g5p_pmset_va(r->pm_set, G5P_PMSET_LIST), ++ g5p_pmset_va(r->pm_set, G5P_PMSET_FREE), ++ physical, r->pm_blocks); ++ kvfree(physical); ++ } ++ mutex_unlock(&rd->lock); ++ return ret; ++} ++ ++/* render_user_pm_ctx 3: the context that brought the epoch up (the device PM for its attach job) ++ * takes its own set before its next job; the device aux pages its VM already maps become the ++ * set's aux (no other context maps them once every later one has its own), so no PTE changes. ++ */ ++static void g5p_rv3_pmctx_take(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ int set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next), ++ ret; ++ ++ if (set < 0) { ++ r->pm_first = false; /* stays on the device PM; not retried */ ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "take"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ r->pm_first = false; ++ return; ++ } ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* perf60 stage 4 diagnostics (render_user_fault_read): whose page each PM lane in S points at ++ * after the TA saved them: the device pool or aux, a set page, a context's pool or aux. ++ */ ++ ++static void g5p_rv3_pm_lanes(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ { ++ return; ++ } ++} ++ ++/* The context is gone (g5p_render_release()): its set is free again; its pages go. */ ++static void g5p_rv3_pmctx_release(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ ++ if (r->pm_set >= 0) ++ rd->pm_set_used[r->pm_set] = false; ++ r->pm_set = -1; ++ for (u32 i = 0; r->pm_pool && i < r->pm_blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_pool[i]); ++ kvfree(r->pm_pool); ++ r->pm_pool = NULL; ++ r->pm_blocks = 0; ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_aux[i]); ++} ++ ++#define G5P_RV3_PM_S 0x80U ++#define G5P_RV3_PM_S_LEN 0xc0U ++#define G5P_RV3_PM_SAVE (G5P_RV3_PM_S_LEN + G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ ++/* perf60 fault address: PM state snapshots (S, PM control, the aux windows S points at, each aux ++ * page's head) at three phases of every job, in a ring and per slot pair; printed at a fault read. ++ * Only the executor writes them. RAM only. ++ */ ++enum { G5P_PMSNAP_START, G5P_PMSNAP_TA_DONE, G5P_PMSNAP_END }; ++struct g5p_pmsnap { ++ u64 ns, auxva[3]; ++ u32 seq, phase, slot, valid; ++ u8 s[G5P_RV3_PM_S_LEN], ctl[0x80], win[3][64], ++ head[G5P_RU_AUX_PAGES][64]; ++}; ++#define G5P_PMSNAP_RING 9U ++static struct g5p_pmsnap g5p_pmsnap_ring[G5P_PMSNAP_RING]; ++static struct g5p_pmsnap g5p_pmsnap_slot[16][3]; ++static u32 g5p_pmsnap_head; ++ ++static void g5p_rv3_pmsnap(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 phase) ++{ ++ { ++ return; ++ } ++} ++ ++static void g5p_rv3_pmsnap_print(struct g5p_diag *g, u32 seq, const char *set, ++ const struct g5p_pmsnap *x) ++{ ++ u64 now = ktime_get_ns(); ++ ++ if (!x->valid) ++ return; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pmsnap seq=%u set=%s job=%u slot=%u phase=%s ago_us=%llu " ++ "aux_va=%#llx/%#llx/%#llx end=v3\n", ++ seq, set, x->seq, x->slot, ++ x->phase == G5P_PMSNAP_START ? "start" : ++ x->phase == G5P_PMSNAP_TA_DONE ? "ta-done" : ++ "end", ++ (now - x->ns) / NSEC_PER_USEC, x->auxva[0], x->auxva[1], ++ x->auxva[2]); ++ for (u32 o = 0; o < sizeof(x->s); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap S+%02x %*ph\n", o, 64, ++ x->s + o); ++ for (u32 o = 0; o < sizeof(x->ctl); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap C+%02x %*ph\n", o, 64, ++ x->ctl + o); ++ for (u32 k = 0; k < 3; k++) ++ dev_info(g->dev, "RENDER-V3 pmsnap W%u %*ph\n", k, 64, ++ x->win[k]); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ dev_info(g->dev, "RENDER-V3 pmsnap A%u %*ph\n", i, 64, ++ x->head[i]); ++} ++ ++static void g5p_rv3_pmsnap_dump(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ for (u32 p = 0; p < 3; p++) ++ g5p_rv3_pmsnap_print(g, seq, "slot-last", ++ &g5p_pmsnap_slot[r->slot[0] & 15][p]); ++ for (u32 i = G5P_PMSNAP_RING; i; i--) ++ g5p_rv3_pmsnap_print(g, seq, "ring", ++ &g5p_pmsnap_ring[(g5p_pmsnap_head - i) % ++ G5P_PMSNAP_RING]); ++} ++ ++/* perf60 fault address: the VM edit log entries that cover the fault VA (any VM, newest first), ++ * then the faulting job's VM's latest edits. Copied under the lock, printed outside it. ++ */ ++#define G5P_VMLOG_HITS 24U ++#define G5P_VMLOG_TAIL 16U ++static void g5p_vmlog_dump(struct g5p_diag *g, u32 seq, const void *vm, u64 va, ++ bool have_va) ++{ ++ struct g5p_vmlog_ent *out = kmalloc_array( ++ G5P_VMLOG_HITS + G5P_VMLOG_TAIL, sizeof(*out), GFP_KERNEL); ++ u32 hits = 0, tail = 0, n, head; ++ unsigned long flags; ++ u64 now = ktime_get_ns(); ++ ++ if (!out) ++ return; ++ spin_lock_irqsave(&g5p_vmlog.lock, flags); ++ head = g5p_vmlog.head; ++ n = min(head, G5P_VMLOG_N); ++ for (u32 i = 1; i <= n; i++) { ++ const struct g5p_vmlog_ent *x = ++ &g5p_vmlog.e[(head - i) % G5P_VMLOG_N]; ++ ++ if (have_va && hits < G5P_VMLOG_HITS && ++ g5p_ru_mmu_fault_in(va, x->va, x->size)) ++ out[hits++] = *x; ++ else if (x->vm == vm && tail < G5P_VMLOG_TAIL) ++ out[G5P_VMLOG_HITS + tail++] = *x; ++ } ++ spin_unlock_irqrestore(&g5p_vmlog.lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u va=%#llx edits=%u covering=%u vm_tail=%u end=v3\n", ++ seq, have_va ? va : 0, head, hits, tail); ++ for (u32 i = 0; i < hits + tail; i++) { ++ const struct g5p_vmlog_ent *x = ++ &out[i < hits ? i : G5P_VMLOG_HITS + i - hits]; ++ ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u %s ago_ms=%llu op=%s pid=%d comm=%s vm=%s/%u va=%#llx " ++ "size=%#llx handle=%#x off=%#llx end=v3\n", ++ seq, i < hits ? "cover" : "tail", ++ (now - x->ns) / NSEC_PER_MSEC, ++ g5p_vmlog_ops[min_t(u32, x->op, ++ ARRAY_SIZE(g5p_vmlog_ops) - 1)], ++ x->tgid, x->comm, x->vm == vm ? "this" : "other", ++ x->vm_id, x->va, x->size, x->handle, x->offset); ++ } ++ kfree(out); ++} ++ ++/* perf60 fault address (render_user_fault_read): the MMU fault banks, read once at a failed ++ * phase, the way iOS's fault handler (6ca5c14) reads them; the firmware keeps no copy. Bell mode ++ * only. Under the event lock with the wake path's power proof (the fault already latched ++ * b->failed, so that flag is not a guard here), then the GPU register licence (mmio.inc, ++ * mmio-access.md §4): no wake (Apple's reads never write R1+0x14, G6), a refusal retried outside ++ * the locks every 20-40 us for at most G5P_RV3_DC_SETTLE_US, and the four bank reads only under a ++ * held ticket. A parked GPC records skip-parked, as Apple's report has no bif fault entry then. ++ * At most G5P_RV3_FAULT_READS per boot. It runs before the loss decision and the terminal ++ * latch, while the faulted job is still the GPU's current work. ++ */ ++#define G5P_RV3_FAULT_READS G5P_RU_RECOVER_BOOT_MAX ++static atomic_t g5p_rv3_fault_reads = ATOMIC_INIT(0); ++ ++/* faultrec: a fault VA in the PM-set window (render-user-pmset.h) names a set and its part; say ++ * which context owns that set now and how it stands against the two job records and the deferred ++ * closes (render_user_close_defer). RAM only; rdev lock for the context list. ++ */ ++static void g5p_rv3_fault_pmset(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq, u32 bank, ++ const struct g5p_ru_mmu_fault *f) ++{ ++ static const char *const parts[G5P_PMSET_PAGES] = { "control", "list", ++ "free" }; ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_render *o, *owner = NULL; ++ u32 page, set, own[2] = { U32_MAX, U32_MAX }, df[2][2]; ++ bool last = false, prev = false, closing = false, bound = false; ++ ++ if (f->va < G5P_PMSET_VA || f->va >= G5P_PMSET_END) ++ return; ++ page = (f->va - G5P_PMSET_VA) / G5P_GPU_PAGE; ++ set = page / G5P_PMSET_PAGES; ++ mutex_lock(&rd->lock); ++ list_for_each_entry(o, &rd->contexts, rdev_link) ++ if (o->pm_set == (int)set) { ++ owner = o; ++ break; ++ } ++ if (owner) { ++ own[0] = owner->slot[0]; ++ own[1] = owner->slot[1]; ++ last = rd->pm_last == owner; ++ prev = rd->pm_prev == owner; ++ closing = owner->closing; ++ bound = owner->bound; ++ } ++ for (u32 k = 0; k < 2; k++) { ++ o = rd->close_deferred[k]; ++ df[k][0] = o ? o->slot[0] : U32_MAX; ++ df[k][1] = o ? o->slot[1] : U32_MAX; ++ } ++ mutex_unlock(&rd->lock); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-pmset seq=%u bank=%u mmu_context=%u set=%u part=%s owner=%d/%d " ++ "bound=%u closing=%u last=%u prev=%u job=%u/%u deferred=%d/%d,%d/%d job_slots=%u " ++ "end=v3\n", ++ seq, bank, f->context, set, parts[page % G5P_PMSET_PAGES], ++ (int)own[0], (int)own[1], bound, closing, last, prev, ++ r->slot[0], r->slot[1], (int)df[0][0], (int)df[0][1], ++ (int)df[1][0], (int)df[1][1], rd->job_slots); ++} ++ ++static void g5p_rv3_fault_read(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 e, int res, bool silent) ++{ ++ u32 full = (0), w[4] = { 0 }; ++ u32 mode = full | ((1) ? 1 : 0); ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_mmu_fault f[2]; ++ struct g5p_mmio_ticket t = { .r1_0 = U32_MAX, ++ .gpc = U32_MAX, ++ .r1_0_end = U32_MAX, ++ .gpc_end = U32_MAX }; ++ const char *did = "skip-guard"; ++ u32 power = 0, tries = 0; ++ unsigned long flags; ++ bool fault = res == G5P_RU_WAIT_FAULT && !silent, ticket = false; ++ u64 t0; ++ int ret; ++ ++ if ((2) != G5P_WK_ALWAYS || !(mode & (fault ? 1 : 2))) ++ return; ++ if (atomic_inc_return(&g5p_rv3_fault_reads) > G5P_RV3_FAULT_READS) { ++ did = "skip-cap"; ++ goto print; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ for (;;) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->stopped) || !g5p_wk_power_ready(power)) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ break; ++ } ++ tries++; ++ ticket = true; ++ ret = g5p_mmio_hold(g, &t, 0); ++ if (!ret) { ++ w[0] = g5p_mmio_r0_read(g, &t, 0xe150); ++ w[1] = g5p_mmio_r0_read(g, &t, 0xe154); ++ w[2] = g5p_mmio_r0_read(g, &t, 0xe158); ++ w[3] = g5p_mmio_r0_read(g, &t, 0xe15c); ++ g5p_mmio_release(g, &t); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ did = g5p_mmio_did(&t); ++ if (ret != -EAGAIN || ++ ktime_get_mono_fast_ns() - t0 >= ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) ++ break; ++ usleep_range(20, 40); ++ } ++ if (ticket) ++ g5p_mmio_sample_line(g, G5P_MMS_FAULT, &t, w, t.ops ? 4 : 0, ++ true); ++print: ++ f[0] = g5p_ru_mmu_fault_decode(w[0], w[1]); ++ f[1] = g5p_ru_mmu_fault_decode(w[2], w[3]); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-read seq=%u engine=%u slots=%u/%u why=%s did=%s power=%02x " ++ "fence=none r1_0=%08x gpc=%08x tries=%u e150=%08x:%08x e158=%08x:%08x end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ fault ? "fault" : ++ silent ? "silent" : ++ "timeout", ++ did, power, t.r1_0, t.gpc, tries, w[1], w[0], w[3], w[2]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-decode seq=%u bank=%u va=%#llx %s type=%s level=%u " ++ "requester=%u/%s sideband=%#x context=%u job_slots=%u/%u end=v3\n", ++ seq, k, f[k].va, f[k].read ? "read" : "write", ++ g5p_ru_mmu_fault_types[f[k].type], f[k].level, ++ f[k].requester, ++ g5p_ru_mmu_requester(f[k].requester), ++ f[k].sideband, f[k].context, r->slot[0], ++ r->slot[1]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ g5p_rv3_fault_pmset(d, r, seq, k, &f[k]); ++ g5p_rv3_switch_line(g, r, seq); ++ if (!full) ++ return; ++ g5p_vmlog_dump(g, seq, r->ctx->vm, f[0].valid ? f[0].va : f[1].va, ++ f[0].valid || f[1].valid); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ g5p_rv3_pmsnap_dump(g, r, seq); ++} ++ ++/* GPU overclock thermal stop (core boot.inc, boot_gpu_oc_mhz): every 500 ms while the epoch runs ++ * state 5, the three SoC zones; at or above boot_gpu_oc_temp_mc (or unreadable) the overclock is off ++ * for the rest of the boot and the render scheduler recovers the GPU (g5p_rv3_recover(): gate cycle, ++ * firmware boot) between jobs, so the new epoch powers on at state 4 with record 5 cleared: iOS's ++ * park -> reboot -> new pstate, never a pstate change under a live firmware. Every 10 s one status ++ * line: the temperatures, the real Fender state (an R1-only licence ticket) and the jobs run. ++ */ ++#define G5P_OC_POLL_MS 500 ++static void g5p_rsched_kick(struct g5p_drm *d); ++ ++static void g5p_oc_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = ++ container_of(to_delayed_work(work), struct g5p_rdev, oc_work); ++ struct g5p_drm *d = rd->drm; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ u32 lim = (90000), zones; ++ const char *zone; ++ int t; ++ ++ if (!b || !0 || READ_ONCE(g5p_oc_tripped)) { ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ zones = g5p_oc_temp(&t, &zone); ++ if (zones != ARRAY_SIZE(g5p_oc_zone_names) || t >= (int)lim) { ++ WRITE_ONCE(g5p_oc_tripped, true); ++ dev_emerg( ++ g->dev, ++ "GPUPERF oc thermal-stop temp=%d zone=%s zones=%u limit=%u mhz=%u jobs=%u: the " ++ "GPU recovers at state 4 (723 MHz) end=gpuperf\n", ++ t, zone, zones, lim, b->gpu_oc_mhz, rd->oc_jobs); ++ WRITE_ONCE(rd->recovering, true); ++ g5p_rsched_kick(d); ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ if (!(++rd->oc_ticks % (10000 / G5P_OC_POLL_MS))) { ++ struct g5p_mmio_ticket tk; ++ ++ if (!g5p_mmio_hold(g, &tk, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &tk); ++ g5p_job_printk( ++ g->dev, ++ "GPUPERF oc status mhz=%u temp=%d zone=%s limit=%u r1=%s r1_1000=%08x jobs=%u " ++ "end=gpuperf\n", ++ b->gpu_oc_mhz, t, zone, lim, g5p_mmio_did(&tk), tk.gpc, ++ rd->oc_jobs); ++ } ++ queue_delayed_work(system_wq, &rd->oc_work, ++ msecs_to_jiffies(G5P_OC_POLL_MS)); ++} ++ ++ ++ ++static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ bool first = !rd->up, bound_before = r->bound; ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (job->compute) ++ return g5p_cu_execute(d, r, job); ++ if (r->stage_done) ++ return -ECANCELED; ++ g5p_l123_job_reset(g); ++ /* The submit-time admission again, now under the core lock and with the bind state: still nothing touched. */ ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ seq = rd->sequence + 1; ++ ; ++ 0; ++ if (bound_before && d->bell_prev_ctx && d->bell_prev_ctx != r->ctx) { ++ u32 ms = (0); ++ ++ ; ++ ; ++ if (ms && b->gpu_perf) ++ 0; ++ } ++ 0; ++ if (g5p_ru_v3_verbose(r->jobs + 1, 0)) ++ g5p_job_printk( ++ g->dev, ++ "RENDER-V3 start seq=%u bound_ms=%u cap_ms=%u slots=%u/%u attach=%s " ++ "ta_doorbell=%s 3d_doorbell=none wait=ram-only after_timeout=quiet end=v3\n", ++ seq, r->wait_ms, G5P_RU_V3_CAP_MS, r->slot[1], ++ r->slot[0], ++ first ? "firmware-dc3" : ++ r->bound ? "kept" : ++ rd->spare_count ? "spare" : ++ "ram-dc3", ++ first ? "grace-owned" : "none"); ++ if (first) { ++ v->phase = G5P_RV3_ATTACH; ++ ret = g5p_rv3_spares(d, v); ++ if (!ret) ++ ret = g5p_rv3_attach(d, r, v); ++ /* perf60 stage 4: a context with a set that brings a new epoch up */ ++ if (!ret) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ if (ret) ++ goto out; ++ ; ++ } else { ++ if (!r->bound) { ++ /* perf60 stage 4: its own PM set and aux pages before the bind ++ * publishes its tables ++ */ ++ if (rd->pm_ctx) ++ g5p_rv3_pmctx_assign(d, r, seq); ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ ; ++ } ++ /* The firmware owns the rings and the channels from here on: any failure quarantines. */ ++ v->touched = true; ++ if (rd->pm_ctx == 3 && r->pm_first && r->pm_set < 0) ++ g5p_rv3_pmctx_take(d, r, seq); ++ if (r->pm_set >= 0) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ else ++ ret = rd->pm_once ? 0 : g5p_rv3_pm_rebuild(d); ++ if (ret) ++ goto out; ++ ; ++ ; ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_START); ++ if (g5p_ru_mmu_retag_due((0), bound_before, ++ d->bell_prev_ctx != r->ctx)) ++ g5p_rv3_mmu_retag(g, r, seq, ++ d->bell_prev_ctx != r->ctx); ++ } ++ rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu(b, r) + 0x80); ++ if (g5p_ru_fault_due(r->fault_job, r->jobs, first)) ++ g5p_rv3_test_fault(g, r, job, seq); ++ for (u32 e = 0; e < 2; e++) { ++ g5p_ru_job_at( ++ g5p_ru_page(b, e), g5p_ru_meta(e), ++ g5p_meta_address(g5p_object_va(&b->layout, 0x51 + e)), ++ g5p_rv3_pm_meta(r), r->slot[e], e, seq, ++ e ? job->td_w : job->ta_w, ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r)); ++ r->submitted[e] = false; ++ r->consumed[e] = false; ++ r->retired[e] = false; ++ } ++ g5p_l123_job_prepare(d, job); ++ g5p_render_clean_job(g); ++ rd->sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_rv3_wake_mark(d, r, v, 0); ++ ret = g5p_rv3_publish_ta(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ if (r->stall_job && r->jobs == r->stall_job && !first) { ++ /* render_user_test_stall_job: the host gives up, the firmware runs on */ ++ g5p_ru_wait_start(&v->ta, v->ta_pub_ns, 1, v->job_deadline_ns); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-stall seq=%u job=%u slots=%u/%u: TA bound 1 ms end=v3\n", ++ seq, r->jobs, r->slot[0], r->slot[1]); ++ } else { ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, ++ v->job_deadline_ns); ++ } ++ g5p_ru_touch_start(&v->touch[0], v->ta_pub_ns, r->touch_ms, seq); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, seq, &v->ta, &v->touch[0], NULL, &v->tr, ++ &v->last[0], &v->ta_consumed_ns, &err); ++ r->consumed[0] = v->ta_consumed_ns != 0; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ /* Timeout or fault: (latency101: the RAM-only observation window first,) terminal ++ * latch, one RAM sample for the record (a completion that arrives now is evidence, ++ * never the result), no export, no register access. ++ */ ++ g5p_rv3_fault_read(d, r, seq, 0, res, v->ta.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 0)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 0, ++ v->ta.silent ? "ta-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "ta-timeout" : ++ "ta-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "ta-timeout" : "ta-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_TIMEOUT, ret); ++ goto out; ++ } ++ g5p_rv3_ta_count(g, d, v); ++ r->retired[0] = true; ++ g5p_rv3_terminal(g, G5P_RU_TERM_COMPLETE, "ta-complete"); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_TA_DONE); ++ g5p_rv3_pm_lanes(d, r, seq); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, ++ g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); ++ ; ++ v->phase = G5P_RV3_TD_PUBLISH; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample( ++ g, r, 1, seq, ++ &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ ++ goto out; ++ } ++ v->phase = G5P_RV3_TD_WAIT; ++ g5p_rv3_wait_start(g, r, &v->td, v->td_pub_ns, v->job_deadline_ns); ++ g5p_ru_touch_start(&v->touch[1], v->td_pub_ns, r->touch_ms, seq); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, ++ &v->last[1], &v->td_consumed_ns, &err); ++ r->consumed[1] = v->td_consumed_ns != 0; ++ /* mmio_selftest bit 2 (rung 5, DFU-risk on a park tuple): one licensed ticket at the 3D ++ * completion decision; always: the idle clock of bit 3. ++ */ ++ ; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ g5p_rv3_fault_read(d, r, seq, 1, res, v->td.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->td); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 1)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 1, ++ v->td.silent ? "3d-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "3d-timeout" : ++ "3d-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "3d-timeout" : "3d-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ /* Rule (c) decided: the matching event was not seen. Always one line (RAM only). */ ++ if (v->rt.decided == G5P_RU_RT_HOLD) ++ dev_warn( ++ g->dev, ++ "RENDER-V3 3d-hold seq=%u idle=%08x,%08x,%08x,%08x,%08x,%08x coherent_ms=%lld " ++ "decided_ms=%lld end=v3\n", ++ seq, v->last[1].idle[0], v->last[1].idle[1], ++ v->last[1].idle[2], v->last[1].idle[3], ++ v->last[1].idle[4], v->last[1].idle[5], ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns)); ++ r->retired[1] = true; ++ v->phase = G5P_RV3_DONE; ++ rd->recover_streak = ++ 0; /* a job finished: a later fault starts a new streak */ ++ ret = 0; ++out: ++ if (v->phase >= G5P_RV3_TA_PUBLISH) ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ g5p_rv3_wake_end(d, r, v); ++ /* Any other failure after the hardware was touched (attach, publish, drain, protocol): the same ++ * criterion g5p_render_run() quarantines the context by. A stage stop (-ECANCELED) is not one. ++ */ ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), ++ g5p_rv3_phase_names[min_t( ++ u32, v->phase, G5P_RV3_PHASES - 1)]); ++ g5p_rv3_receipt(g, r, job, ret); ++ g5p_l123_job_end(d, job, seq, ret); ++ if (!ret) { ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ ; ++ ; ++ } ++ /* render-user-pmset.h: the PM's write-back targets */ ++ if (g5p_pmset_switched(rd->pm_last, r)) { ++ rd->pm_prev = rd->pm_last; ++ rd->pm_last = r; ++ } ++ /* perf60 bell gates: the next job's gap and context switch are measured from here. */ ++ d->bell_prev_end_ns = ktime_get_ns(); ++ d->bell_prev_ctx = r->ctx; ++ return ret; ++} ++ ++/* ++ * The channel is blocked behind a lost job (g5p_rv3_lose()): wait for its stuck phase, RAM only, ++ * with the firmware's event stream watched from the first sample (silent for 1 s = dead). A late ++ * TA gets the job's 3D published (its W is staged since the job's start; no doorbell for a job ++ * other than the epoch's first), so the 3D channel's sequence and the PM state stay as the ++ * firmware expects; a late 3D clears the channel. Then the held jobs run. A silent stream, a ++ * fault, a refused 3D or the drain bound end the epoch as a timeout or fault always did, and the ++ * held jobs then fail at admission. Called by the scheduler's worker before any pick; returns ++ * false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_rv3_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 st = s->engine ? G5P_RU_DRAIN_TD : G5P_RU_DRAIN_TA, ++ res = G5P_RU_WAIT_PENDING; ++ struct g5p_ru_wait w = {}; ++ bool armed = false; ++ int act, err = -EIO; ++ ++ /* TYPED_COMPUTE: a lost CDM job has its own one-phase drain. */ ++ if (s->engine == G5P_CU_ENGINE) ++ return g5p_cu_drain(d); ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ for (;;) { ++ u32 e = st == G5P_RU_DRAIN_TA ? 0 : 1; ++ u64 now = ktime_get_ns(); ++ ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->stopped)) { ++ res = G5P_RU_WAIT_FAULT; ++ } else if (!e) { ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], ++ NULL, &v->tr, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ } else { ++ if (!armed) ++ g5p_rv3_event_arm(d, 1, ++ g5p_ru_endpoint(s->seq)); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], ++ &v->rt, NULL, &v->last[1], ++ &v->td_consumed_ns, &err); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ } ++ act = g5p_ru_drain_step(&st, res); ++ if (act != G5P_RU_DRAIN_PUBLISH_TD) ++ break; ++ r->retired[0] = true; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); ++ armed = true; ++ if (g5p_rv3_publish_3d(g, r, v, s->seq)) { ++ g5p_rv3_event_disarm(d, 1); ++ res = G5P_RU_WAIT_FAULT; ++ act = g5p_ru_drain_step(&st, res); ++ break; ++ } ++ } ++ if (act == G5P_RU_DRAIN_UNBLOCK) { ++ r->retired[1] = true; ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drained seq=%u engine=%u slots=%u/%u lost_ms=%lld: channel clear, " ++ "held jobs resume end=v3\n", ++ s->seq, s->engine, r->slot[0], r->slot[1], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drain dead seq=%u engine=%u phase=%s why=%s err=%d lost_ms=%lld: the " ++ "device recovers if it may, else the epoch ends end=v3\n", ++ s->seq, s->engine, ++ st == G5P_RU_DRAIN_DEAD && !v->td_pub_ns ? "ta" : "3d", ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal( ++ g, ++ fault ? G5P_RU_TERM_FAULT : G5P_RU_TERM_TIMEOUT, ++ fault ? "drain-fault" : "drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++/* The firmware window's leaves again from the new epoch's tables, in the same pages, so every ++ * render VM's root entry stays valid (g5p_render_fw_entry()). Core lock held. ++ */ ++static void g5p_render_fw_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_boot *b = d->core->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ ++ if (!w->built) ++ return; ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ w->pages = 0; ++ w->missing = 0; ++ for (u32 i = 0; i < G5P_RU_FW_PAGES; i++) { ++ u64 va = g5p_ru_fw_window_va[i]; ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (k == w->leaves) ++ continue; ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, ++ leaf & 1 ? leaf : 0); ++ w->pages += leaf & 1; ++ w->missing += !(leaf & 1); ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(d->core, &w->leaf[i]); ++} ++ ++/* ++ * A lost job stopped the firmware (g5p_rv3_lose_recover(), or a drain that died): cycle the GPU ++ * and boot the firmware again (core recover.inc), then start the render device over as at boot. ++ * Every context survives except the lost one: its bindings belonged to the old epoch, which the ++ * cycle ended, so each is unbound in RAM, its binding's reference dropped, and its next job binds ++ * again (the first job after the recovery runs the epoch's attach). The aux pages and the PM ++ * pool are zeroed as at their allocation. The held jobs run after this. Returns false, having ++ * done nothing, when the core lock is busy. A failed recovery leaves the device failed. ++ */ ++static bool g5p_rv3_recover(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ unsigned long deadline = jiffies + 2 * HZ, flags; ++ struct g5p_render **drop = NULL; ++ struct g5p_render *r; ++ u32 n = 0, count = 0, dropped = 0, stage; ++ int ret; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ count++; ++ } ++ mutex_unlock(&rd->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ rd->recoveries++; ++ rd->recover_streak++; ++ ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; ++ if (ret) { ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", ++ ret, rd->recoveries); ++ return true; ++ } ++ g5p_ru_term_reset(&g5p_rv3_term); ++ /* the new firmware's PM has written nothing back yet: no context is its last or previous */ ++ rd->pm_last = NULL; ++ rd->pm_prev = NULL; ++ rd->up = false; ++ rd->v3 = false; ++ rd->sequence = 0; ++ rd->pm_serial = 0; ++ rd->dc_pending = false; ++ rd->dc_wait = 0; ++ if (rd->spares) { ++ spin_lock_irqsave(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ if (!rd->spares[i].used) { ++ __clear_bit(rd->spares[i].slot[0], ++ g->user_slots); ++ __clear_bit(rd->spares[i].slot[1], ++ g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rd->spares[i].root); ++ kfree(rd->spares); ++ rd->spares = NULL; ++ rd->spare_count = 0; ++ } ++ g5p_render_fw_rebuild(d); ++ mutex_lock(&rd->lock); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ if (rd->aux[i].cpu) { ++ memset(rd->aux[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->aux[i]); ++ } ++ for (u32 i = 0; i < rd->blocks * 8; i++) { ++ memset(rd->physical[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->physical[i]); ++ } ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ r->jobs = 0; ++ r->dc_target = 0; ++ WRITE_ONCE(r->live_maps, false); ++ memset(r->submitted, 0, sizeof(r->submitted)); ++ memset(r->consumed, 0, sizeof(r->consumed)); ++ memset(r->retired, 0, sizeof(r->retired)); ++ /* nothing of the old epoch holds this context's memory any more */ ++ r->lost_drained = true; ++ WRITE_ONCE(r->ctx->live, false); ++ if (r->retained && n < count) { ++ r->retained = false; ++ drop[n++] = r; ++ } ++ } ++ mutex_unlock(&rd->lock); ++ spin_lock_irqsave(&d->status_lock, flags); ++ d->last_error = 0; ++ d->job_error = 0; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ for (u32 i = 0; i < n; i++) { ++ struct g5p_context *c = drop[i]->ctx; ++ ++ dropped++; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recovered n=%u contexts=%u unbound=%u streak=%u: held jobs resume, " ++ "the next job attaches end=v3\n", ++ rd->recoveries, count, dropped, rd->recover_streak); ++ return true; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +new file mode 100644 +index 000000000..bd86d7dff +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -0,0 +1,2549 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Userspace TA/3D render: render contexts, render targets and SUBMIT_RENDER. ++ * Builds on the r22 constructors through render-user-layout.h (byte-equal for ++ * the round16 inputs). Default off. The first accepted job claims the device ++ * for render; TYPED_V3 contexts then share it, each with its own VM and slot ++ * pair (docs/gpu/multi-client-design.md), the other profiles stay with that ++ * first context. Jobs of all contexts are serialized; no reset/recovery: any ++ * hardware failure retains everything. ++ */ ++#include "../../apple-g5p/render-user-layout.h" ++#include "../../apple-g5p/render-user-wait.h" ++#include "../../apple-g5p/render-user-touch.h" ++#include "../../apple-g5p/render-user-retire.h" ++#include "../../apple-g5p/render-user-wake.h" ++#include "../../apple-g5p/render-user-serial.h" ++#include "../../apple-g5p/render-user-terminal.h" ++#include "../../apple-g5p/render-user-bind.h" ++#include "../../apple-g5p/render-user-close.h" ++#include "../../apple-g5p/render-user-sched.h" ++#include "../../apple-g5p/render-user-window.h" ++#include "../../apple-g5p/render-user-msaa.h" ++#include "../../apple-g5p/render-user-loss.h" ++#include "../../apple-g5p/render-user-fault.h" ++#include "../../apple-g5p/render-user-mmu.h" ++#include "../../apple-g5p/render-user-faultreg.h" ++#include "render-user-validate.h" ++#include "drm-render-v3.h" ++#include "../../apple-g5p/compute-user-layout.h" ++#include "compute-user-validate.h" ++#include "drm-compute-v1.h" ++ ++/* The terminal state of the epoch (one GPU): from the first terminal note on, no register access. */ ++static struct g5p_ru_terminal g5p_rv3_term; ++ ++/* Defaults of the latched knobs. GET_RENDER_INFO reports VARIANT_ACTIVE when ++ * any knob differs, so the declarations below and that comparison share them. ++ */ ++#define G5P_RU_DEF_CACHE_VARIANT 3 ++#define G5P_RU_DEF_MMU_VARIANT 1 ++/* 20 s: the natural TA completion took 12.006 s in the r96 boots a 12 s cap turned into ++ * timeouts (render-user-wait.h); it is also the minimum of a TYPED_V3 job's per-phase bound. ++ */ ++#define G5P_RU_DEF_WAIT_MS G5P_RU_V3_WAIT_MS ++#define G5P_RU_DEF_PROFILE1_VDM_CONTROL 0x280 ++ ++static const unsigned int render_user_wait_ms = 20000; ++MODULE_PARM_DESC( ++ render_user_wait_ms, ++ "Latched per-phase TA/3D bound, 3000..30000 ms (default 20000; TYPED_V3 needs " ++ "20000..30000, absolute job cap 30000)"); ++/* latency101 (default off): host-side TYPED_V3 wait policy; nothing the firmware sees depends on it. */ ++static const unsigned int render_user_hard_ms = 60000; ++static const unsigned int render_user_soft_ms = 45000; ++static const unsigned int render_user_obs_ms = 2000; ++MODULE_PARM_DESC( ++ render_user_hard_ms, ++ "Latched TYPED_V3 hard per-phase bound soft..60000 ms, the job bound is twice " ++ "it; 0 = off, render_user_wait_ms and the 30000 job cap apply (default 0)"); ++MODULE_PARM_DESC( ++ render_user_soft_ms, ++ "Latched TYPED_V3 soft per-phase bound 20000..hard ms, 0 = 45000; needs " ++ "render_user_hard_ms"); ++MODULE_PARM_DESC( ++ render_user_obs_ms, ++ "Latched RAM-only observation window after a TYPED_V3 timeout, 0..5000 ms " ++ "(default 2000; needs render_user_hard_ms)"); ++/* release (default off): RAM-only touch A/B of the TYPED_V3 wait (render-user-touch.h). */ ++ ++/* repeat109: the TYPED_V3 3D completion without the idle words (render-user-retire.h). */ ++ ++/* wake110 (default off): Apple's submit kick at the TYPED_V3 publish (render-user-wake.h). */ ++/* release8 default */ ++ ++/* TEMPORARY (multi-client stage 1): the fallback for a firmware that never drains a DC3 ++ * published after a job retired. Remove once the device has settled that question. ++ */ ++ ++/* Stage 2 fault isolation (render-user-loss.h): how long the scheduler holds every other job ++ * while it waits, RAM only, for a lost job's stuck phase. Read at the loss. ++ */ ++ ++/* Stage 2 in-boot recovery (core recover.inc): a job that stopped the firmware loses only its ++ * context, and the GPU is power-cycled and its firmware booted again while the other contexts' ++ * jobs are held (each leaves the old epoch's firmware pages allocated); 0 = such a job ends the ++ * epoch, as before. How many: render_user_recover_streak. Off by default; the release tuple turns ++ * it on (faultiso F1-F4, 2026-11-02: two recoveries in one boot on hardware). ++ */ ++ ++/* render-user-recover.h: Apple restarts the GPU on every recovery packet. Nonzero: recover every ++ * time, but end the epoch after this many recoveries in a row with no job completing between ++ * them. 0 = render_user_recover counts recoveries per boot, as before (a boot's second fault then ++ * ends the epoch, however long the GPU ran in between). ++ */ ++ ++/* Fault-isolation test (host side only: the firmware runs the job as any other). A context ++ * created while this is N gives up on its N-th job's TA after 1 ms instead of its bound, so the ++ * host loses that job the way it loses a stuck one, while the firmware completes it late: the ++ * context is lost and the channel drained behind it. 0 = off (default). ++ */ ++ ++/* Fault-isolation test with a real GPU fault (faultiso). A context created by a CAP_SYS_RAWIO ++ * process while this is N reserves one page of its VM that is never mapped ++ * (G5P_RU_TEST_FAULT_VA), and its N-th job's TA fetches its control stream from that page ++ * instead of the validated stream: the TA reads an unmapped VA of the context's own VM. What ++ * the firmware does then (a fault event, a silent stream, or a stuck engine) is what the test ++ * records; each ends that job and loses its context. 0 = off (default). ++ */ ++ ++/* render_user_test_fault_job's page: the last GiB below the hardware VA limit, far from every ++ * kernel-private range (low sink, aux, PDS window, firmware window). ++ */ ++#define G5P_RU_TEST_FAULT_VA (G5P_RU_HW_VA_LIMIT - 0x40000000ULL) ++ ++/* Stage 3 (render-user-window.h): which firmware memory a render VM maps. Latched per context ++ * at creation. The kill switch 0 restores the stage 1 mapping and then needs CAP_SYS_RAWIO. ++ */ ++ ++/* The latency101 knobs are valid when all are 0 (off), or hard is in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000 and the dialect is TYPED_V3. The release touch ++ * is 0, or 1..20000 below the effective soft bound with latency101 on. The 3D idle settle is ++ * 0..5000. The wake110 mode is 0, or 1..2 with latency101 on and the touch off (one A/B per ++ * context). Anything else refuses the context. ++ */ ++static bool g5p_ru_lat_params_valid(void) ++{ ++ u32 hard = READ_ONCE(render_user_hard_ms), ++ soft = READ_ONCE(render_user_soft_ms); ++ ++ return g5p_ru_lat_valid(soft, hard, READ_ONCE(render_user_obs_ms)) && ++ g5p_ru_touch_valid((0), g5p_ru_lat_soft(soft, hard), hard) && ++ g5p_ru_rt_settle_valid((G5P_RU_RT_DEF_SETTLE_MS)) && ++ g5p_wk_mode_valid((2)) && (!(2) || (hard && !(0))) && ++ (!hard || (1)); ++} ++/* r25 frame-capable selections, same values as the fixed render_*_variant. */ ++ ++#define G5P_RU_MAX_QUEUED 16 ++#define G5P_RU_MAX_TARGETS 32 ++#define G5P_RU_RESULTS 1024 ++#define G5P_RU_INPUT_MS 10000 ++ ++struct g5p_rt { ++ struct list_head link; ++ struct g5p_context *ctx; ++ struct g5p_ru_rt geometry; ++ struct g5p_ru_rt_sizes sizes; ++ struct drm_mm_node node; ++ struct g5p_dma_page *pages; ++ u32 width, height, page_count; ++ u32 samples; /* 1 or 4 (CREATE_RENDER_TARGET); 0 = 1 */ ++ bool destroyed; ++}; ++ ++/* A pair job 1 bound to a kernel-owned catalogue (render_user_spare_pairs, TEMPORARY). */ ++struct g5p_rspare { ++ u32 slot[2]; /* TA, 3D */ ++ struct g5p_dma_page root; /* the catalogue CBASE points at */ ++ bool ok, used; ++}; ++ ++/* ++ * Device-wide render state (docs/gpu/multi-client-design.md §4.1): the one TA/3D channel pair ++ * and the one PM record with its pool and aux pages, shared by every render context. Jobs of ++ * all contexts run one at a time on render_wq with the core lock held, and only they write ++ * up, v3, sequence, pm_serial and the spares. `lock` guards the pool and the aux pages, which ++ * render-target creation grows while another context's job may run. The scheduler's worker ++ * is the only executor: it picks the jobs of all contexts round-robin. ++ */ ++/* The kernel-built firmware window shared by every render VM (render-user-window.h): built ++ * once from the firmware's tables, never edited, kept until reboot. ++ */ ++struct g5p_rwin { ++ bool built; ++ u32 pages, missing, leaves; ++ struct g5p_dma_page dir, leaf[G5P_RU_FW_PAGES]; ++ u32 leaf_di[G5P_RU_FW_PAGES]; ++}; ++ ++struct g5p_rdev { ++ struct mutex lock; ++ bool up, v3; /* PM and channels built; by the TYPED_V3 attach */ ++ bool dc_pending; /* a bind's DC3 pair was not consumed in time ... */ ++ u32 dc_wait; /* ... and this is the consumer it waits for */ ++ u32 sequence, pm_serial; ++ u32 cu_sequence, cu_target; /* TYPED_COMPUTE: CDM sequence, CL producer target */ ++ u32 blocks; ++ /* perf60, latched at the epoch's first TYPED_V3 attach: 2 = jobs alternate two record slots ++ * (render-user-layout.h), else 1; pm_once = the PM pool is built at that attach only. ++ */ ++ u32 job_slots; ++ bool pm_once; ++ /* perf60 stage 4, latched there too: render_user_pm_ctx (0 off, 1 per-context PM records, ++ * 2 also per-context PM IDs) and the boot-reserved sets in use (core lock). ++ */ ++ u32 pm_ctx, pm_set_next; ++ bool pm_set_used[G5P_PMSET_MAX]; ++ /* the context of the last job the executor ran and the previous distinct one, the PM's ++ * write-back targets (render-user-pmset.h; compared, never dereferenced), and the closed ++ * contexts whose unbind waits until they are neither (render_user_close_defer) ++ */ ++ struct g5p_render *pm_last, *pm_prev, *close_deferred[2]; ++ struct g5p_dma_page *physical; ++ struct g5p_dma_page aux[G5P_RU_AUX_PAGES]; ++ struct g5p_rspare *spares; ++ u32 spare_count; ++ /* render_user_close_park: the park catalogue a closed pair is bound to (allocated at the ++ * first park, kept for the device's life; rebuilt at every park) ++ */ ++ struct g5p_dma_page park; ++ u32 parks; ++ bool executing; /* a TYPED_V3 job is in g5p_rv3_execute() (reboot quiesce) */ ++ /* The scheduler (drm-render-sched.inc): its run list under sched_lock, and the one ++ * worker on render_wq that runs every job and close item. ++ */ ++ struct g5p_drm *drm; ++ spinlock_t sched_lock; ++ struct g5p_rs sched; ++ struct delayed_work sched_work; ++ /* boot_gpu_oc_mhz: the thermal stop's poll (system_wq, so a busy render_wq never starves it) */ ++ struct delayed_work oc_work; ++ unsigned long oc_armed; ++ u32 oc_ticks, oc_jobs; ++ struct g5p_rwin ++ window; /* written once, by the executor, core lock held */ ++ struct g5p_rstuck ++ *stuck; /* a lost job blocks the channel (drain), worker only */ ++ /* A lost job stopped the firmware: the worker recovers before any pick. Set by the ++ * executor, cleared by the worker; submissions and the claim pass meanwhile. ++ */ ++ bool recovering; ++ u32 recoveries; ++ u32 recover_streak; /* recoveries since a job completed (core lock) */ ++ struct list_head ++ contexts; /* every render context (lock), for the recovery */ ++}; ++ ++/* render-user-fault.h's ruling window on the device state: a failure noted while a TYPED_V3 job or ++ * a drain runs, or a recovery is pending, waits for the executor. RAM only. ++ */ ++static bool g5p_rdev_ruling(struct g5p_rdev *rd) ++{ ++ bool executing = ++ smp_load_acquire(&rd->executing); /* before `recovering` */ ++ ++ return g5p_ru_ruling(executing, READ_ONCE(rd->recovering)); ++} ++ ++/* A render context. Its firmware state is the slot pair (slot[], bound), owned here the way ++ * an Asahi queue owns its firmware context; the channels and the PM record are the device's. ++ */ ++struct g5p_render { ++ struct g5p_context *ctx; ++ u32 slot[2]; /* engine0 TA, engine1 normal3D */ ++ u32 jobs; /* jobs that started on this context */ ++ u32 dc_target; /* DC ring consumer after this context's binding */ ++ struct drm_mm private_mm; ++ struct drm_mm_node aux_node, low_node, firmware_node, private_node; ++ struct drm_mm_node ++ fault_node; /* render_user_test_fault_job's unmapped page */ ++ struct list_head targets; ++ struct g5p_rt *profile1_rt; ++ u32 target_count; ++ u64 fence_context, fence_seqno; ++ atomic_t queued; ++ /* bound: the slot pair is published (for a RAM-only bind: consumed and checked). ++ * retained: the binding holds a context and device reference until its unbind. ++ * closing: the handle is gone; the close item (g5p_render_close_run()) is queued. ++ * live_maps: a job retired on the binding, so VM_BIND MAP may run beside queued jobs. ++ */ ++ bool claimant, bound, retained, aux_mapped, detached, stage_done, ++ unhooked; ++ bool closing, live_maps; ++ bool lost_drained; /* a lost job of it drained: the firmware holds no use of it */ ++ struct g5p_rs_entity ++ entity; /* its place in the scheduler (drm-render-sched.inc) */ ++ struct list_head rdev_link; /* in rdev->contexts */ ++ /* Mappings unmapped beside queued jobs, freed as those retire (g5p_render_reap()). */ ++ spinlock_t zlock; ++ struct list_head zombies; ++ /* Latched at creation; mirrors whatever qualifies the fixed path. */ ++ u32 stage, cache_variant, mmu_variant, timestamp_variant, wait_ms; ++ u32 soft_ms, hard_ms, obs_ms; /* latency101; hard_ms == 0: off */ ++ u32 touch_ms; /* RC4a; 0: off (always 0 when hard_ms is) */ ++ u32 settle_ms; /* repeat109 minimum coherent run before the 3D event completes */ ++ u32 stall_job; /* render_user_test_stall_job; 0: off */ ++ u32 fault_job; /* render_user_test_fault_job; 0: off */ ++ u32 wake, wake_seed; /* wake110 (render-user-wake.h); wake 0: off */ ++ u32 wake_rang; /* wake110: 0xb00 writes of this context */ ++ u32 pm_policy, vdm_control; ++ u32 fw_window; /* render_user_fw_window, G5P_RU_FW_* */ ++ bool after_ta, cache_repeat; ++ struct g5p_r25_variants v; /* pm, memory, rt<=1, slots; others 0 */ ++ u32 target[2], status[2], selector[2]; ++ bool submitted[2], consumed[2], retired[2]; ++ /* TYPED_V3 (drm-render-v3.inc): serial jobs, the context is retained from its binding on. */ ++ bool v3_accepted; ++ /* perf60 render_user_pm_switch 2: this context's own PM state (S + aux) after its last job */ ++ u8 *pm_save; ++ ++ /* perf60 stage 4 (render_user_pm_ctx): its own PM set (-1: the device PM), pool pages, ++ * aux pages mapped at G5P_RU_PRIVATE_VA in its VM, and the epoch they were built for. ++ */ ++ int pm_set; ++ u32 pm_blocks, pm_epoch; ++ struct g5p_dma_page *pm_pool; ++ struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; ++ bool pm_aux_mapped; ++ bool pm_first; /* it brought the epoch up (attach): the device aux is its own */ ++ bool close_defer_run; /* render_user_close_defer: its deferred close is running */ ++}; ++ ++/* The lost job the channel waits behind (render-user-loss.h): its context (a reference held), ++ * its per-job state as the executor left it, and the drain bound read at the loss. ++ */ ++struct g5p_rstuck { ++ struct g5p_render *r; ++ struct g5p_rv3 v; ++ u32 seq, engine, drain_ms; ++ u64 lost_ns; ++}; ++ ++#include "cache-receipt.inc" ++ ++/* A listed BO that needs implicit fences and/or kernel cache maintenance. */ ++struct g5p_robj { ++ struct drm_gem_object *obj; /* reference held by the job */ ++ u32 access; ++ /* ACCESS_CACHE window (render_user_sync_range bit 1: the resource's range; else whole BO) */ ++ u64 lo, hi; ++}; ++ ++struct g5p_rjob { ++ struct g5p_cache_receipt cache; ++ struct g5p_rs_job node; /* in its context's FIFO until picked */ ++ struct g5p_drm *gpu; ++ struct g5p_context *ctx; ++ struct g5p_fence *fence; ++ struct dma_fence *inputs[G5P_MAX_SYNCS - 1]; ++ u32 input_count; ++ struct g5p_robj *objs; ++ u32 obj_count; ++ struct dma_fence **deps; /* implicit dependencies from dma_resv */ ++ u32 dep_count; ++ int setup_error; /* post-acceptance setup failure: fail without hardware */ ++ /* Inputs then deps, in order: dep_next is the first not yet seen signalled, cb_fence the ++ * one whose callback is armed (scheduler lock), dep_error a failed input or the deadline. ++ */ ++ u32 dep_next; ++ struct dma_fence *cb_fence; ++ struct dma_fence_cb cb; ++ bool cb_fired; ++ int dep_error; ++ u64 id; ++ unsigned long deadline; ++ u8 ta_w[G5P_RU_TA_W_BYTES], td_w[G5P_RU_3D_W_BYTES]; ++ bool v3; ++ bool compute; /* TYPED_COMPUTE (drm-compute-v1.inc): a CDM job */ ++ struct g5p_cu_regs cu; ++ struct g5p_rv3 rv3; ++ u64 submit_ns; /* latency123: the submit ioctl */ ++}; ++ ++static u8 *g5p_ru_page(struct g5p_boot *b, u32 engine) ++{ ++ return b->shared[(engine ? G5P_R22_3D : G5P_R22_TA) / G5P_GPU_PAGE].cpu; ++} ++ ++static u32 g5p_ru_meta(u32 engine) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + ++ (engine ? G5P_R22_3D : G5P_R22_TA)); ++} ++ ++static u8 *g5p_ru_pm_cpu(struct g5p_boot *b) ++{ ++ return b->shared[G5P_R22_PM / G5P_GPU_PAGE].cpu; ++} ++ ++static u8 *g5p_ru_global(struct g5p_boot *b, u32 engine) ++{ ++ u32 off = b->layout.offset[0x47 + engine]; ++ ++ return b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_ru_free_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ if (p->cpu) ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++} ++ ++/* Probe: the device render state and its scheduler. Its pages are allocated on demand and ++ * kept until reboot. ++ */ ++static void g5p_rsched_work(struct work_struct *work); ++static void g5p_oc_work(struct work_struct *work); ++ ++static int g5p_render_init(struct g5p_drm *d) ++{ ++ d->rdev = drmm_kzalloc(&d->drm, sizeof(*d->rdev), GFP_KERNEL); ++ if (!d->rdev) ++ return -ENOMEM; ++ d->rdev->drm = d; ++ INIT_LIST_HEAD(&d->rdev->contexts); ++ spin_lock_init(&d->rdev->sched_lock); ++ g5p_rs_init(&d->rdev->sched); ++ INIT_DELAYED_WORK(&d->rdev->sched_work, g5p_rsched_work); ++ INIT_DELAYED_WORK(&d->rdev->oc_work, g5p_oc_work); ++ return drmm_mutex_init(&d->drm, &d->rdev->lock); ++} ++ ++/* rdev->lock held: the aux pages (once) and a PM pool of at least `blocks`. The pool never ++ * shrinks; every page added reaches PoC like the bring-up pool, whether or not the device's ++ * bring-up has cleaned the pool yet. The pool, list and F are rebuilt from it before every ++ * job, so growth takes effect at the next job, and the job in flight keeps its pages. ++ */ ++static int g5p_rdev_grow(struct g5p_drm *d, u32 blocks) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_dma_page *physical; ++ u32 have = rd->blocks * 8, i; ++ int ret = 0; ++ ++ for (i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!rd->aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->aux[i]); ++ if (ret || blocks <= rd->blocks) ++ return ret; ++ if (blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ physical = kvcalloc(blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ if (have) ++ memcpy(physical, rd->physical, have * sizeof(*physical)); ++ for (i = have; !ret && i < blocks * 8; i++) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &physical[i]); ++ if (!ret) ++ g5p_table_clean_page(d->core, &physical[i]); ++ } ++ if (ret) { ++ while (i-- > have) ++ g5p_ru_free_page(d->drm.dev, &physical[i]); ++ kvfree(physical); ++ return ret; ++ } ++ kvfree(rd->physical); ++ rd->physical = physical; ++ rd->blocks = blocks; ++ return 0; ++} ++ ++/* rdev->lock held: the pool's page addresses, for the PM builders. */ ++static u64 *g5p_rdev_addresses(struct g5p_rdev *rd) ++{ ++ u64 *physical = ++ kvmalloc_array(rd->blocks * 8, sizeof(*physical), GFP_KERNEL); ++ ++ for (u32 i = 0; physical && i < rd->blocks * 8; i++) ++ physical[i] = rd->physical[i].dma; ++ return physical; ++} ++ ++/* From the moment the firmware may hold a context's slot pair, the context and everything it ++ * owns stay alive whatever its files do, until its close item has unbound the pair ++ * (g5p_render_close_run()); a pair that cannot be unbound keeps them until reboot. ++ */ ++static void g5p_render_retain(struct g5p_render *r) ++{ ++ if (r->retained) ++ return; ++ kref_get(&r->ctx->ref); ++ drm_dev_get(&r->ctx->gpu->drm); ++ r->retained = true; ++} ++ ++/* The RAM words of the binding of `slot` through the DC records at start and start + 1. */ ++static void g5p_render_bind_mem(struct g5p_boot *b, const u32 *slot, u32 start, ++ struct g5p_ru_bind_mem *m) ++{ ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + ++ ((start + i) & 255) * 20 + j * 4; ++ ++ m->rec[i][j] = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ } ++ for (u32 e = 0; e < 2; e++) { ++ m->pa[e] = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ m->host[e] = g5p_boot_shared(b, 0x84) + slot[e] * 4; ++ m->cached[e] = g5p_boot_shared(b, 0x85) + slot[e] * 4; ++ } ++} ++ ++/* ++ * Called with the file lock. Idle: no accepted job is queued or running. A VM_BIND (`live`) ++ * of a context with live_maps may also run beside its queued jobs, the way an Asahi VM_BIND ++ * does not wait for the queue: a compositor imports and releases client buffers while its ++ * previous frame is in flight. Its tag reaches the firmware at the next kick of the slots. A ++ * MAP only adds entries for VAs no accepted job references; an UNMAP clears its entries now but ++ * keeps the object pinned until the queued jobs have retired (g5p_resident_defer()), so a job ++ * that still used the range faults on the hardware instead of reaching freed memory. Not ++ * before a job has retired on the binding: the epoch's first job checks that the tables did not ++ * change since its attach, and a bind in flight computes its tag from them. Render-target ++ * edits still need the context idle. ++ */ ++static int g5p_render_vm_idle(struct g5p_context *c, bool live) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_boot *b = c->gpu->core->boot; ++ /* During a recovery no epoch may receive a tag. */ ++ bool busy = (atomic_read(&r->queued) && ++ !(live && READ_ONCE(r->live_maps))) || ++ READ_ONCE(c->gpu->rdev->recovering); ++ ++ if (c->bad || r->detached || busy) { ++ dev_warn_ratelimited( ++ c->gpu->drm.dev, ++ "G5P VM_BIND busy=render-idle bad=%u detached=%u queued=%d bound=%u\n", ++ c->bad, r->detached, atomic_read(&r->queued), r->bound); ++ return -EBUSY; ++ } ++ if (r->bound && (!b || READ_ONCE(b->failed) || READ_ONCE(b->stopped))) ++ return g5p_rdev_ruling(c->gpu->rdev) ? -EBUSY : -EIO; ++ return 0; ++} ++ ++/* perf60 MMU invalidation arm, default 0 (render-user-mmu.h): bit 0 every edit publication asks ++ * the firmware for a PC-level invalidate, bit 1 a bound context's job after another context's ++ * job retags its slots (PC), bit 2 NC-reads every table line before the tag store, bit 3 retags ++ * on every job. RAM only, like the tag path itself. ++ */ ++ ++/* An NC read of every 64-byte line of the VM's tables (g5p_rv3_fw_touch()'s visibility read). */ ++static void g5p_render_table_touch(struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt; ++ ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(vm->root.cpu + at)); ++ list_for_each_entry(pt, &vm->tables, link) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(pt->page.cpu + at)); ++} ++ ++/* File lock held, context idle (or a MAP with live_maps). After a PTE edit on a bound render ++ * VM: tables to PoC, then the packed tag to both of its slots. RAM only: no register access ++ * is safe after retirement, and the firmware reads these words only at this context's next ++ * kick, which then invalidates the slots' PC level. ++ */ ++static int g5p_render_invalidate(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_diag *g = c->gpu->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_pt *pt; ++ u32 tag, cached[2], bumps = 0, mode = (0); ++ ++ if (!READ_ONCE(r->bound) || r->detached) ++ return 0; ++ if (!b || c->bad || b->failed || b->stopped) ++ return -EIO; ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ if (mode & G5P_RU_MMU_TOUCH) ++ g5p_render_table_touch(vm); ++ tag = g5p_vm_generation(&vm->mutation); ++ for (u32 e = 0; e < 2; e++) ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ while (tag == cached[0] || tag == cached[1]) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ bumps++; ++ } ++ if (bumps) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER generation collision cached=%08x/%08x bumps=%u tag=%08x\n", ++ cached[0], cached[1], bumps, tag); ++ if (mode & G5P_RU_MMU_EDIT) { ++ /* Advance the VM's own PC counter to the PC field the helper picked. */ ++ u32 pc = g5p_ru_mmu_pc_tag(tag, cached[0], cached[1]); ++ ++ if (pc) { ++ vm->mutation.generation[0] += ++ ((pc >> 22) - (tag >> 22)) & 0x3ff; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ } ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, tag); ++ dma_wmb(); ++ if (mode) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick mmu_inval=%x " ++ "cached=%08x/%08x\n", ++ tag, r->slot[0], r->slot[1], mode, cached[0], ++ cached[1]); ++ else ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick\n", ++ tag, r->slot[0], r->slot[1]); ++ return 0; ++} ++ ++/* DC3 consumption precedes this CBASE check and selected-slot invalidation; ++ * the r22 diagnostic sequence with the user catalogue instead of a private PC. ++ */ ++ ++/* CPU-written firmware objects of one job: TA/3D pages, PM control and the ++ * list/free pages. Coherent allocations plus explicit PoC, as r22 does. ++ */ ++static void g5p_render_clean_job(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ for (u32 off = G5P_R22_TA; off <= G5P_R22_PM; off += G5P_GPU_PAGE) ++ g5p_table_clean_page(g, &b->shared[off / G5P_GPU_PAGE]); ++ for (u32 i = 0; i < 2; i++) ++ g5p_table_clean_page(g, &b->render_meta[i]); ++ dsb(sy); ++ dma_wmb(); ++} ++ ++/* RAM only: the DC ring is idle and neither slot of this context has a PA or a tag. */ ++static int g5p_render_vm_ready(struct g5p_boot *b, struct g5p_render *r) ++{ ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u32 start = READ_ONCE(ctl[4]); ++ ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ return g5p_ru_bind_busy(&s) ? -EBUSY : 0; ++} ++ ++/* The epoch's first job, core lock held, device not up: the PM record from the device pool ++ * and both channels, and every page they reach cleaned to PoC. Doing this again with the ++ * device up would reset the channels and the PM state that every context shares. ++ */ ++static int g5p_render_device_prepare(struct g5p_drm *d, ++ const struct g5p_r25_variants *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm(g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, ++ G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks, G5P_RU_PRIVATE_VA); ++ if (!ret) ++ g5p_ru_pm_variant(g5p_ru_pm_cpu(b), b->render_meta[1].cpu, ++ physical, rd->blocks, v->pm); ++ kvfree(physical); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ g5p_table_clean_page(g, &rd->aux[i]); ++ for (u32 i = 0; !ret && i < rd->blocks * 8; i++) ++ g5p_table_clean_page(g, &rd->physical[i]); ++ mutex_unlock(&rd->lock); ++ if (ret) ++ return ret; ++ rd->pm_serial = 0; ++ for (u32 e = 0; e < 2; e++) ++ g5p_ru_channel_init(g5p_ru_page(b, e), g5p_ru_meta(e)); ++ g5p_cu_device_init(d); ++ g5p_render_clean_job(g); ++ return 0; ++} ++ ++/* ++ * Core lock held, firmware up: the root entry of the firmware GiB for a VM in mode `fw_window`. ++ * The window is built at its first use from the firmware's leaf entries of the pages it keeps ++ * (render-user-window.h), and then shared, unchanged, by every render VM. RAM only. ++ */ ++static int g5p_render_fw_entry(struct g5p_drm *d, struct g5p_render *r, ++ u32 *entry) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ int ret = 0; ++ ++ *entry = 0; ++ if (r->fw_window == G5P_RU_FW_FULL) ++ *entry = g5p_le32(b->catalogue.cpu + G5P_RU_FIRMWARE_CI * 4); ++ if (r->fw_window != G5P_RU_FW_WINDOW) ++ return 0; ++ if (w->built) { ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ return 0; ++ } ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->dir); ++ /* perf60 stage 4: the per-context PM sets reserved at boot, after the device PM's pages */ ++ for (u32 i = 0; ++ !ret && i < G5P_RU_FW_PAGES + b->pm_sets * G5P_PMSET_PAGES; i++) { ++ u32 j = i - G5P_RU_FW_PAGES; ++ u64 va = i < G5P_RU_FW_PAGES ? ++ g5p_ru_fw_window_va[i] : ++ g5p_pmset_va(j / G5P_PMSET_PAGES, ++ j % G5P_PMSET_PAGES); ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (!(leaf & 1)) { ++ w->missing++; ++ continue; ++ } ++ if (k == w->leaves) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->leaf[k]); ++ if (ret) ++ break; ++ w->leaf_di[k] = g5p_ru_fw_di(va); ++ g5p_put64(w->dir.cpu + w->leaf_di[k] * 8, ++ g5p_directory_entry(w->leaf[k].dma)); ++ w->leaves++; ++ } ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, leaf); ++ w->pages++; ++ } ++ if (ret) { ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_ru_free_page(d->drm.dev, &w->leaf[i]); ++ g5p_ru_free_page(d->drm.dev, &w->dir); ++ memset(w, 0, sizeof(*w)); ++ return ret; ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(g, &w->leaf[i]); ++ g5p_table_clean_page(g, &w->dir); ++ w->built = true; ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ dev_info( ++ g->dev, ++ "RENDER-V3 fw-window pages=%u missing=%u leaves=%u dir=%pad end=v3\n", ++ w->pages, w->missing, w->leaves, &w->dir.dma); ++ return 0; ++} ++ ++/* Core lock held, between jobs: the DC3 pair that points this context's slots at its own page ++ * catalogue. Maps the firmware window (g5p_render_fw_entry()) and borrows the sink catalogue ++ * entry, cleans the tables and this ++ * context's render targets, then writes the PA and tag of both slots and the two records. ++ * RAM only, nothing is written when the ring is busy or a slot is in use; the caller stores ++ * the producer. ++ */ ++static int g5p_render_vm_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 start = READ_ONCE(ctl[4]), tag, entry; ++ struct g5p_ru_bind_mem m; ++ struct g5p_pt *pt; ++ struct g5p_rt *rt; ++ int ret; ++ ++ ret = g5p_render_vm_ready(b, r); ++ if (!ret) ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ dsb(sy); ++ tag = g5p_vm_generation(&vm->mutation); ++ /* A zero tag would read as unbound: the unbind check takes a zero cached tag as proof ++ * that the DC3 handler ran. ++ */ ++ while (!tag && vm->mutation.generations) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_publish(&m, r->slot, vm->root.dma, tag); ++ *start_out = start; ++ *tag_out = tag; ++ return 0; ++} ++ ++/* The epoch's first job, core lock held: build the device, then bind this context's pair. ++ * Any failure leaves every allocation and binding retained. ++ */ ++static int g5p_render_attach_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ int ret = g5p_render_vm_ready(d->core->boot, r); ++ ++ if (!ret) ++ ret = g5p_render_device_prepare(d, &r->v); ++ return ret ?: g5p_render_vm_prepare(d, r, start_out, tag_out); ++} ++ ++/* The legacy profiles' attach: the epoch's first job, with a doorbell and a host MMU pass. */ ++ ++/* Pointer contents, channel producer, global producer and kick, each after a ++ * barrier. The channel must have retired the previous job's endpoint. ++ */ ++ ++/* Same predicates as g5p_r22_observe() for sequence `seq`. */ ++ ++#include "drm-render-v3.inc" ++#include "drm-render-sched.inc" ++#include "drm-render-unbind.inc" ++ ++static void g5p_render_job_free(struct g5p_rjob *job) ++{ ++ for (u32 i = 0; i < job->input_count; i++) ++ dma_fence_put(job->inputs[i]); ++ for (u32 i = 0; i < job->dep_count; i++) ++ dma_fence_put(job->deps[i]); ++ kfree(job->deps); ++ for (u32 i = 0; i < job->obj_count; i++) ++ drm_gem_object_put(job->objs[i].obj); ++ kvfree(job->objs); ++ if (job->fence) ++ dma_fence_put(&job->fence->base); ++ if (job->ctx) ++ g5p_context_put(job->ctx); ++ kfree(job); ++} ++ ++/* CPU cache maintenance on the G5P mapping (for an imported BO, the attachment's sg_table): clean ++ * before the job, invalidate after retirement. Whole BO, or (render_user_sync_range bit 1, latched ++ * at submit) the resource's BO range, the same window both ways. ++ */ ++static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, ++ bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct g5p_robj *o = &job->objs[i]; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(o->obj)->sgt; ++ ++ if (!(o->access & DRM_G5P_RENDER_ACCESS_CACHE) || !sgt) ++ continue; ++ /* A buffer the job only reads (a sampled client dma-buf or wl_shm udmabuf) ++ * was cleaned before the job and the GPU wrote none of it: no invalidate, ++ * which could discard what the CPU side wrote meanwhile into lines of the ++ * pages the buffer shares (a udmabuf covers whole 16 KiB pages). ++ */ ++ if (to_cpu && !(o->access & DRM_G5P_RENDER_ACCESS_WRITE)) ++ continue; ++ struct g5p_cache_receipt *cr = &job->cache; ++ bool imported = drm_gem_is_imported(o->obj); ++ bool exported = READ_ONCE(o->obj->dma_buf) != NULL; ++ bool wc = to_drm_gem_shmem_obj(o->obj)->map_wc; ++ int cls = g5p_cache_class(imported, exported, wc); ++ u64 bytes = o->hi ? o->hi - o->lo : o->obj->size; ++ u64 cpu = 0, wall = 0; ++ if (cr->enabled) ++ cr->requested[to_cpu][cls] += bytes; ++ /* Native pages are initially DMA-mapped with no SKIP_CPU_SYNC. ++ * All later CPU views of a private map_wc object (GEM mmap/vmap, ++ * including command validation) are write-combined. There is no ++ * cacheable client alias to clean/invalidate. Shared/foreign BOs ++ * remain conservative. Barriers below still order WC accesses. ++ */ ++ bool imported_uncached = imported && ++ drm_gem_dma_buf_is_cpu_uncached(o->obj->import_attach->dmabuf); ++ if ((imported_uncached && cr->import_policy) || g5p_cache_omit(cr->policy, imported, exported, wc)) ++ continue; ++ if (cr->enabled) { cpu = task_sched_runtime(current); wall = ktime_get_ns(); } ++ if (o->hi && (o->lo || o->hi != o->obj->size)) ++ g5p_sync_sgt_range(d->drm.dev, sgt, o->lo, o->hi, ++ to_cpu); ++ else if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ if (cr->enabled) { ++ cr->maintained[to_cpu][cls] += bytes; ++ cr->cpu[to_cpu][cls] += task_sched_runtime(current) - cpu; ++ cr->wall[to_cpu][cls] += ktime_get_ns() - wall; ++ } ++ } ++ if (to_cpu) dma_rmb(); else dma_wmb(); ++} ++ ++/* After acceptance, render_submit_lock held, no file lock: one dma_resv at a ++ * time, so no ww ordering is needed. Our own context's earlier fences are ++ * already ordered by the queue. A failure fails the job before hardware. ++ */ ++static int g5p_render_implicit(struct g5p_rjob *job, u64 own_context) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct dma_resv *resv = job->objs[i].obj->resv; ++ bool write = job->objs[i].access & DRM_G5P_RENDER_ACCESS_WRITE; ++ struct dma_fence **fences = NULL, **grown; ++ unsigned int count = 0; ++ int ret; ++ ++ if (!(job->objs[i].access & DRM_G5P_RENDER_ACCESS_IMPLICIT)) ++ continue; ++ dma_resv_lock(resv, NULL); ++ ret = dma_resv_get_fences(resv, ++ write ? DMA_RESV_USAGE_READ : ++ DMA_RESV_USAGE_WRITE, ++ &count, &fences); ++ if (!ret) ++ ret = dma_resv_reserve_fences(resv, 1); ++ if (!ret) ++ dma_resv_add_fence(resv, &job->fence->base, ++ write ? DMA_RESV_USAGE_WRITE : ++ DMA_RESV_USAGE_READ); ++ dma_resv_unlock(resv); ++ grown = ret || !count ? ++ NULL : ++ krealloc_array(job->deps, ++ job->dep_count + count, ++ sizeof(*grown), GFP_KERNEL); ++ if (!ret && count && !grown) ++ ret = -ENOMEM; ++ if (grown) ++ job->deps = grown; ++ for (u32 j = 0; j < count; j++) { ++ if (!ret && fences[j]->context != own_context) ++ job->deps[job->dep_count++] = fences[j]; ++ else ++ dma_fence_put(fences[j]); ++ } ++ kfree(fences); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++/* Free the mappings unmapped beside jobs up to fence seqno `seqno` (all: every one). */ ++static void g5p_render_reap(struct g5p_render *r, u64 seqno, bool all) ++{ ++ struct g5p_mapping *m, *t; ++ unsigned long flags; ++ LIST_HEAD(done); ++ ++ spin_lock_irqsave(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &r->zombies, link) ++ if (all || m->retire <= seqno) ++ list_move_tail(&m->link, &done); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &done, link) { ++ list_del(&m->link); ++ g5p_vm_obj_put(r->ctx->vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++} ++ ++/* One job, picked by the scheduler once its inputs signalled (or failed, or timed out). */ ++static void g5p_render_run(struct g5p_rjob *job) ++{ ++ struct g5p_drm *d = job->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_render *r = job->ctx->render; ++ unsigned long lock_deadline, flags; ++ bool hardware = false; ++ int ret; ++ ++ job->cache.enabled = READ_ONCE(render_user_cache_receipts); ++ job->cache.policy = READ_ONCE(render_user_cache_private_wc); ++ job->cache.import_policy = READ_ONCE(render_user_cache_import_wc); ++ if (job->cache.enabled) { ++ job->cache.job = job->id; ++ job->cache.start_cpu = task_sched_runtime(current); ++ job->cache.start_wall = ktime_get_ns(); ++ } ++ /* Admission closes only for good (reboot quiesce, FINAL teardown): a job queued before ++ * that fails here, without waiting for its inputs and without competing for the command ++ * lock. ++ */ ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) ++ ret = -ESHUTDOWN; ++ else ++ ret = job->setup_error ?: job->dep_error; ++ if (!ret && (job->ctx->bad || READ_ONCE(g->boot->failed))) ++ ret = -EIO; ++ if (!ret) { ++ /* Bounded: never queue forever behind a command stuck in MMIO. */ ++ lock_deadline = jiffies + 2 * HZ; ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, lock_deadline)) { ++ ret = -EBUSY; ++ break; ++ } ++ msleep(5); ++ } ++ } ++ if (!ret) { ++ if (g->boot->failed || g->boot->stopped) { ++ ret = -EIO; ++ } else if (!1 && !g5p_ru_mmio_ok(&g5p_rv3_term)) { ++ /* A V3 epoch is terminal: the legacy path would read and write registers. */ ++ ret = -EIO; ++ } else { ++ /* RAM-only from the DC3 attach on: no snapshot (it reads R0), no ++ * export, no register access of any kind after a timeout. ++ */ ++ g5p_render_cache(d, job, false); ++ WRITE_ONCE(d->rdev->executing, true); ++ ret = g5p_rv3_execute(d, r, job); ++ /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ ++ smp_store_release(&d->rdev->executing, false); ++ hardware = job->rv3.touched || job->rv3.bind_failed; ++ if (!ret || ret == -ECANCELED) ++ g5p_render_cache( ++ d, job, ++ true); /* before status and fence */ ++ if (!ret && r->bound) ++ WRITE_ONCE(r->live_maps, true); ++ } ++ g->last_result = ret; ++ mutex_unlock(&g->lock); ++ } ++ spin_lock_irqsave(&d->status_lock, flags); ++ /* A failed device fails every later job; one context's lost job or failed bind does not. */ ++ if (READ_ONCE(g->boot->failed) || READ_ONCE(g->boot->event_failed)) ++ ret = ret ?: -EIO; ++ job->fence->state = DRM_G5P_JOB_COMPLETE; ++ job->fence->error = ret < 0 ? ret : 0; ++ d->last_job = job->id; ++ d->last_error = job->fence->error; ++ d->job_error = job->fence->error; ++ d->job_state = DRM_G5P_JOB_COMPLETE; ++ if (ret < 0 && ret != -ECANCELED && hardware) { ++ /* No reset protocol: quarantine the context and the device. A TYPED_V3 ++ * timeout is one exception for the device: nothing is wrong with the ++ * protocol, the firmware may still complete, and the drain must keep ++ * decoding (RAM only) so that a late completion is recorded. A failed ++ * bind is another: the DC ring is not on the path of the contexts ++ * already bound, so only this context is lost. A lost job is the last: ++ * the executor already ruled (the drain, or the recovery that replaces ++ * this epoch), and only its context is lost. ++ */ ++ WRITE_ONCE(job->ctx->bad, true); ++ if (g5p_ru_job_fails_device(1, ret, job->rv3.bind_failed, ++ job->rv3.lost)) ++ WRITE_ONCE(g->boot->failed, true); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (ret < 0) ++ dma_fence_set_error(&job->fence->base, ret); ++ atomic_dec(&r->queued); ++ atomic_dec(&d->r8_queued); ++ dma_fence_signal(&job->fence->base); ++ /* A job that may still run on the hardware (a bad context) keeps what it might use. */ ++ if (!READ_ONCE(job->ctx->bad)) ++ g5p_render_reap(r, job->fence->base.seqno, false); ++ /* a failed job (complete=-N) always prints; a completed one is routine: job_log */ ++ if (ret) ++ dev_info( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ else ++ g5p_job_printk( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ if (job->cache.enabled) { ++ job->cache.total_cpu = task_sched_runtime(current) - job->cache.start_cpu; ++ job->cache.total_wall = ktime_get_ns() - job->cache.start_wall; ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ job->cache.serial = g5p_cache_head + 1; ++ g5p_cache_ring[g5p_cache_head++ % G5P_CACHE_RECORDS] = job->cache; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ } ++ g5p_render_job_free(job); ++ ++ drm_dev_put(&d->drm); ++} ++ ++/* Private PM-aux/RT page with the latched r25 leaf flags (c by default). */ ++static int g5p_render_map_page(struct g5p_drm *d, struct g5p_render *r, u64 va, ++ struct g5p_dma_page *page) ++{ ++ u8 *pte = g5p_vm_pte(r->ctx->vm, d->drm.dev, va, true); ++ ++ if (IS_ERR_OR_NULL(pte)) ++ return pte ? PTR_ERR(pte) : -ENOMEM; ++ if (g5p_le32(pte) || g5p_le32(pte + 4)) ++ return -EBUSY; ++ g5p_vm_entry(&r->ctx->vm->mutation, pte, ++ g5p_leaf_entry(page->dma, g5p_r25_attributes(&r->v, va)), ++ 2); ++ return 0; ++} ++ ++/* clear: unmap first (file lock held, VM still hooked). */ ++static void g5p_rt_release_memory(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rt *rt, bool clear) ++{ ++ if (clear && rt->page_count) ++ g5p_vm_clear(r->ctx->vm, d->drm.dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rt->pages[i]); ++ kvfree(rt->pages); ++ rt->pages = NULL; ++ rt->page_count = 0; ++ if (drm_mm_node_allocated(&rt->node)) ++ drm_mm_remove_node(&rt->node); ++} ++ ++/* File lock held, context idle; RAM only. Main ring then secondary in one ++ * RT_ALIGN node: the first 64x64 target lands exactly on r22's ++ * 101020000/101030000. ++ */ ++static int g5p_rt_create(struct g5p_drm *d, struct g5p_render *r, ++ const struct g5p_ru_rt *geometry, u32 width, ++ u32 height, struct g5p_rt **out) ++{ ++ struct g5p_rt *rt; ++ u64 bytes; ++ int ret; ++ ++ if (r->target_count >= G5P_RU_MAX_TARGETS) ++ return -ENOSPC; ++ rt = kzalloc(sizeof(*rt), GFP_KERNEL); ++ if (!rt) ++ return -ENOMEM; ++ rt->ctx = r->ctx; ++ rt->geometry = *geometry; ++ rt->width = width; ++ rt->height = height; ++ ret = g5p_ru_geometry_check(geometry, width, height, &rt->sizes); ++ if (ret) ++ goto err_free; ++ bytes = rt->sizes.main + rt->sizes.secondary_map; ++ ret = drm_mm_insert_node_in_range( ++ &r->private_mm, &rt->node, bytes, G5P_RU_RT_ALIGN, 0, ++ G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE, ++ DRM_MM_INSERT_LOW); ++ if (ret) ++ goto err_free; ++ rt->pages = ++ kvcalloc(bytes / G5P_GPU_PAGE, sizeof(*rt->pages), GFP_KERNEL); ++ if (!rt->pages) { ++ ret = -ENOMEM; ++ goto err_node; ++ } ++ for (; rt->page_count < bytes / G5P_GPU_PAGE; rt->page_count++) { ++ struct g5p_dma_page *p = &rt->pages[rt->page_count]; ++ ++ ret = g5p_vm_alloc_page(d->drm.dev, p); ++ if (!ret) ++ ret = g5p_render_map_page( ++ d, r, ++ rt->node.start + rt->page_count * G5P_GPU_PAGE, ++ p); ++ if (ret) { ++ g5p_ru_free_page(d->drm.dev, p); ++ goto err_pages; ++ } ++ } ++ if (r->bound) { ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(d->core, &rt->pages[i]); ++ ret = g5p_render_invalidate(r->ctx); ++ if (ret) { ++ /* Hardware may hold the new tag: retain the target. */ ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ return ret; ++ } ++ } ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ *out = rt; ++ g5p_vmlog_add(G5P_VMLOG_RT_NEW, r->ctx->vm, 0, rt->node.start, bytes, ++ rt->width << 16 | rt->height, rt->samples); ++ return 0; ++err_pages: ++ g5p_rt_release_memory(d, r, rt, true); ++ goto err_free; ++err_node: ++ drm_mm_remove_node(&rt->node); ++err_free: ++ kfree(rt); ++ return ret; ++} ++ ++/* Detach every VM-side trace: private PTEs and the reserved VA nodes. File ++ * lock held, or the file is closing (no concurrent VM ioctl can exist). ++ */ ++static void g5p_render_unhook(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct device *dev = c->gpu->drm.dev; ++ struct g5p_rt *rt; ++ ++ if (r->unhooked) ++ return; ++ list_for_each_entry(rt, &r->targets, link) ++ if (rt->page_count) ++ g5p_vm_clear(c->vm, dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ if (r->aux_mapped) ++ g5p_vm_clear(c->vm, dev, G5P_RU_PRIVATE_VA, ++ G5P_RU_AUX_PAGES * G5P_GPU_PAGE); ++ for (u32 i = 0; i < 4; i++) { ++ struct drm_mm_node *n = i == 0 ? &r->low_node : ++ i == 1 ? &r->firmware_node : ++ i == 2 ? &r->private_node : ++ &r->fault_node; ++ ++ if (drm_mm_node_allocated(n)) ++ drm_mm_remove_node(n); ++ } ++ if (c->vm->resident == c) ++ c->vm->resident = NULL; ++ r->unhooked = true; ++} ++ ++/* Last context reference: only reached unbound or after a successful DC3 ++ * unbind, so no firmware or GPU reference to this memory remains. The aux ++ * pages and the PM pool are the device's. ++ */ ++static void g5p_render_release(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_rt *rt, *tmp; ++ unsigned long flags; ++ ++ g5p_render_unhook(c); ++ g5p_render_reap(r, 0, true); ++ if (!list_empty(&r->rdev_link)) { ++ mutex_lock(&d->rdev->lock); ++ list_del_init(&r->rdev_link); ++ mutex_unlock(&d->rdev->lock); ++ } ++ list_for_each_entry_safe(rt, tmp, &r->targets, link) { ++ g5p_rt_release_memory(d, r, rt, false); ++ list_del(&rt->link); ++ kfree(rt); ++ } ++ if (drm_mm_node_allocated(&r->aux_node)) ++ drm_mm_remove_node(&r->aux_node); ++ drm_mm_takedown(&r->private_mm); ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ for (u32 e = 0; e < 2; e++) ++ if (r->slot[e]) ++ __clear_bit(r->slot[e], d->core->user_slots); ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ { ++ u32 aux = 0; ++ ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ aux += !!r->pm_aux[i].cpu; ++ /* release8 accounting: what this context frees, and the device's live GEM total */ ++ dev_info( ++ d->drm.dev, ++ "RENDER-V3 release slots=%u/%u jobs=%u set=%d pool_pages=%u aux_pages=%u " ++ "vm_pinned=%lluK live_bos=%u live_bo_bytes=%lluK end=v3\n", ++ r->slot[0], r->slot[1], r->jobs, r->pm_set, ++ r->pm_pool ? r->pm_blocks * 8 : 0, aux, ++ c->vm ? c->vm->pinned_bytes >> 10 : 0, ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10); ++ } ++ g5p_rv3_pmctx_release(d, r); ++ if (d->rdev->pm_last == r) ++ d->rdev->pm_last = NULL; ++ if (d->rdev->pm_prev == r) ++ d->rdev->pm_prev = NULL; ++ kvfree(r->pm_save); ++ kfree(r); ++ c->render = NULL; ++} ++ ++/* CREATE_CONTEXT type RENDER. Exclusive VM: reserves [64KiB, 1GiB) for the ++ * borrowed VA0 sink, the whole firmware GiB and the private RT/PM window. ++ */ ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_r25_variants v = { ++ .pm = (0), ++ .memory = (0), ++ .rt = (0), ++ .slots = (0), ++ }; ++ unsigned long flags; ++ u32 id, fw_window; ++ int ret; ++ ++ BUILD_BUG_ON(G5P_USER_RESERVED_START & ((1ULL << 30) - 1)); ++ if (a->flags || a->handle) ++ return -EINVAL; ++ ; ++ ; ++ fw_window = (G5P_RU_FW_WINDOW); ++ /* Unprivileged unless the VM would map the whole firmware GiB (the kill switch). */ ++ if (fw_window >= G5P_RU_FW_MODES || ++ (fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO))) ++ return fw_window >= G5P_RU_FW_MODES ? -EINVAL : -EPERM; ++ if ((0) > 2 || ++ ((G5P_RU_DEF_CACHE_VARIANT) != 0 && ++ (G5P_RU_DEF_CACHE_VARIANT) != 3) || ++ (G5P_RU_DEF_MMU_VARIANT) > 1 || (0) > 1 || (0) > 1 || ++ READ_ONCE(render_user_wait_ms) < ((1) ? G5P_RU_V3_WAIT_MS : 3000) || ++ READ_ONCE(render_user_wait_ms) > G5P_RU_V3_CAP_MS || ++ !g5p_ru_lat_params_valid() || !(32) || (32) > G5P_RU_MAX_BLOCKS || ++ ((G5P_RU_DEF_PROFILE1_VDM_CONTROL) & 0xf) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) > G5P_RU_MAX_VDM_CONTROL || ++ !g5p_ru_r25_valid(&v)) ++ return -EINVAL; ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ r = kzalloc(sizeof(*r), GFP_KERNEL); ++ if (!c || !r) { ++ kfree(c); ++ kfree(r); ++ return -ENOMEM; ++ } ++ r->pm_set = -1; ++ c->gpu = d; ++ c->render = r; ++ r->ctx = c; ++ INIT_LIST_HEAD(&r->targets); ++ INIT_LIST_HEAD(&r->rdev_link); ++ g5p_rs_entity_init(&r->entity); ++ spin_lock_init(&r->zlock); ++ INIT_LIST_HEAD(&r->zombies); ++ atomic_set(&r->queued, 0); ++ drm_mm_init(&r->private_mm, G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_SIZE); ++ r->stage = (0); ++ r->cache_variant = (G5P_RU_DEF_CACHE_VARIANT); ++ r->mmu_variant = (G5P_RU_DEF_MMU_VARIANT); ++ r->timestamp_variant = (0); ++ r->wait_ms = READ_ONCE(render_user_wait_ms); ++ r->hard_ms = READ_ONCE(render_user_hard_ms); ++ r->soft_ms = r->hard_ms ? ++ g5p_ru_lat_soft(READ_ONCE(render_user_soft_ms), ++ r->hard_ms) : ++ 0; ++ r->obs_ms = r->hard_ms ? READ_ONCE(render_user_obs_ms) : 0; ++ r->touch_ms = r->hard_ms ? (0) : 0; ++ r->settle_ms = (G5P_RU_RT_DEF_SETTLE_MS); ++ r->stall_job = (0); ++ r->fault_job = capable(CAP_SYS_RAWIO) ? (0) : 0; ++ r->wake = r->hard_ms && !r->touch_ms ? (2) : 0; ++ r->wake = g5p_wk_mode_valid(r->wake) ? r->wake : 0; ++ r->wake_seed = (0); ++ r->after_ta = (0); ++ r->pm_policy = (0); ++ r->cache_repeat = (0); ++ r->vdm_control = (G5P_RU_DEF_PROFILE1_VDM_CONTROL); ++ r->fw_window = fw_window; ++ r->v = v; ++ r->fence_context = dma_fence_context_alloc(1); ++ mutex_lock(&f->lock); ++ c->vm = xa_load(&f->vms, a->vm); ++ if (!c->vm) { ++ ret = -ENOENT; ++ goto err_unlock; ++ } ++ if (c->vm->contexts || c->vm->resident || c->vm->resident_gone || ++ c->vm->frozen || c->vm->serial_context) { ++ ret = -EBUSY; ++ goto err_unlock; ++ } ++ /* Memory mode 4 covers every render leaf, so no BO may be mapped yet. */ ++ if (v.memory == 4 && c->vm->map_count) { ++ ret = -EINVAL; ++ goto err_unlock; ++ } ++ r->low_node.start = G5P_USER_VA_START; ++ r->low_node.size = G5P_RU_USER_VA_MIN - G5P_USER_VA_START; ++ r->private_node.start = G5P_RU_PRIVATE_VA; ++ r->private_node.size = G5P_RU_PRIVATE_SIZE; ++ r->firmware_node.start = G5P_USER_RESERVED_END; ++ r->firmware_node.size = ++ ((G5P_RU_FIRMWARE_CI + 1) << 30) - G5P_USER_RESERVED_END; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->low_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->private_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->firmware_node) || ++ g5p_le32(c->vm->root.cpu + G5P_RU_FIRMWARE_CI * 4) || ++ g5p_le32(c->vm->root.cpu)) { ++ ret = -EINVAL; /* VM already maps a range this context needs */ ++ goto err_unlock; ++ } ++ if (r->fault_job) { ++ r->fault_node.start = G5P_RU_TEST_FAULT_VA; ++ r->fault_node.size = G5P_GPU_PAGE; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->fault_node)) { ++ dev_warn( ++ d->drm.dev, ++ "RENDER-V3 test-fault off: va=%#llx already used in this VM end=v3\n", ++ G5P_RU_TEST_FAULT_VA); ++ r->fault_job = 0; ++ } ++ } ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ if (g5p_ru_slot_pair(d->core->user_slots, v.slots, r->slot)) { ++ __set_bit(r->slot[0], d->core->user_slots); ++ __set_bit(r->slot[1], d->core->user_slots); ++ } else { ++ r->slot[0] = 0; ++ r->slot[1] = 0; ++ } ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ if (!r->slot[0]) { ++ ret = -ENOSPC; ++ goto err_unlock; ++ } ++ /* The device's PM pool (at least this context's latched size) and aux pages, mapped at ++ * the same VA in every render VM, so the PM state each TA reloads is the one the ++ * previous job saved, whichever VM ran it. ++ */ ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, (32)); ++ mutex_unlock(&d->rdev->lock); ++ r->aux_node.start = G5P_RU_PRIVATE_VA; ++ r->aux_node.size = G5P_RU_AUX_PAGES * G5P_GPU_PAGE; ++ if (!ret) ++ ret = drm_mm_reserve_node(&r->private_mm, &r->aux_node); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ ret = g5p_render_map_page(d, r, ++ G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE, ++ &d->rdev->aux[i]); ++ r->aux_mapped = true; ++ } ++ if (ret) ++ goto err_unlock; ++ ret = xa_alloc(&f->contexts, &id, c, XA_LIMIT(1, G5P_MAX_CONTEXTS), ++ GFP_KERNEL); ++ if (ret) ++ goto err_unlock; ++ kref_init(&c->ref); ++ kref_get(&c->vm->ref); ++ mutex_lock(&d->rdev->lock); ++ list_add_tail(&r->rdev_link, &d->rdev->contexts); ++ mutex_unlock(&d->rdev->lock); ++ c->vm->contexts++; ++ c->vm->serial_context = true; ++ c->vm->resident = c; ++ c->vm->zero_leaf_flags = v.memory == 4; ++ a->handle = id; ++ mutex_unlock(&f->lock); ++ return 0; ++err_unlock: ++ if (c->vm) ++ g5p_render_release(c); ++ else ++ kfree(r); ++ mutex_unlock(&f->lock); ++ kfree(c); ++ return ret; ++} ++ ++/* DESTROY_CONTEXT, file lock held. A TYPED_V3 context that submitted: its close item, queued ++ * behind its jobs, unbinds and releases it (g5p_render_close()). Unbound: plain release. A ++ * legacy context that brought the device up, bound and idle: zero PA/tag and consume one DC3 ++ * per slot, as r21 detach does. Failure or a quarantined legacy context retains everything ++ * until reboot. ++ */ ++static int g5p_render_destroy(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl, start, target; ++ unsigned long deadline; ++ int ret; ++ ++ if (r->v3_accepted) { ++ g5p_render_close(c); ++ return 0; ++ } ++ if (atomic_read(&r->queued) || c->bad) ++ return -EBUSY; ++ if (!r->bound) { ++ g5p_render_unhook(c); ++ if (r->claimant) { ++ /* Claimed but never touched hardware: drop the claim's reference. The ++ * device stays claimed for render: other contexts may be queued on it. ++ */ ++ r->claimant = false; ++ d->quarantined_context = NULL; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ return 0; ++ } ++ /* The legacy unbind rings a doorbell: retained once the epoch is terminal. */ ++ if (!r->claimant || !g5p_ru_mmio_ok(&g5p_rv3_term)) ++ return -EBUSY; ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ ret = -EIO; ++ if (b->failed || b->stopped) ++ goto out; ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ start = READ_ONCE(ctl[4]); ++ ret = -EBUSY; ++ if (start > 255 || READ_ONCE(ctl[0]) != start) ++ goto out; ++ for (u32 e = 0; e < 2; e++) { ++ u32 index = (start + e) & 255; ++ ++ g5p_put64(g5p_boot_shared(b, 0x86) + r->slot[e] * 8, 0); ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, 0); ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + index * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE, ++ j == 0 ? 3 : ++ j == 1 ? r->slot[e] : ++ 0); ++ } ++ } ++ target = (start + 2) & 255; ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], target); ++ dma_wmb(); ++ ret = g5p_prepare_kick(g, "RENDER-USER-unbind"); ++ if (ret) ++ goto fail; ++ writel(0x10, g->rgx + 0xb00); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || b->failed) { ++ ret = ret ?: -EIO; ++ goto fail; ++ } ++ dma_rmb(); ++ if (READ_ONCE(ctl[0]) == target) ++ break; ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (READ_ONCE(ctl[0]) != target) { ++ ret = -ETIMEDOUT; ++ goto fail; ++ } ++ r->detached = true; ++ r->bound = false; ++ r->claimant = false; ++ WRITE_ONCE(c->live, false); ++ g5p_render_unhook(c); ++ /* One render attachment per boot: accepted stays held, so neither a new ++ * render owner nor compute can reuse the shared channel pages. ++ */ ++ d->quarantined_context = NULL; ++ g5p_mmu_record( ++ g, ++ "RENDER-USER detached slots=%u/%u; resources freed on release\n", ++ r->slot[0], r->slot[1]); ++ mutex_unlock(&g->lock); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ return 0; ++fail: ++ c->bad = true; ++ b->failed = true; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, ++ const struct drm_g5p_sync *syncs, u32 count, ++ struct drm_syncobj **signal, u64 *point, ++ struct dma_fence_chain **chain) ++{ ++ u32 signal_handle = 0; ++ int ret; ++ ++ if (!count || count > G5P_MAX_SYNCS) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) { ++ for (u32 j = 0; j < i; j++) ++ if (syncs[i].handle == syncs[j].handle && ++ syncs[i].point == syncs[j].point && ++ syncs[i].flags == syncs[j].flags) ++ return -EINVAL; ++ if (syncs[i].flags & ~DRM_G5P_SYNC_SIGNAL) ++ return -EINVAL; ++ if (syncs[i].flags) { ++ if (*signal) ++ return -EINVAL; ++ *signal = drm_syncobj_find(file, syncs[i].handle); ++ if (!*signal) ++ return -ENOENT; ++ *point = syncs[i].point; ++ signal_handle = syncs[i].handle; ++ continue; ++ } ++ if (job->input_count == ARRAY_SIZE(job->inputs)) ++ return -E2BIG; ++ ret = drm_syncobj_find_fence(file, syncs[i].handle, ++ syncs[i].point, 0, ++ &job->inputs[job->input_count]); ++ if (ret) ++ return ret; ++ job->input_count++; ++ } ++ if (!*signal) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) ++ if (!syncs[i].flags && syncs[i].handle == signal_handle && ++ (!*point || !syncs[i].point || syncs[i].point >= *point)) ++ return -EINVAL; ++ if (*point) { ++ struct dma_fence *old = drm_syncobj_fence_get(*signal); ++ ++ ret = old && (!to_dma_fence_chain(old) || ++ old->seqno >= *point) ? ++ -EINVAL : ++ 0; ++ dma_fence_put(old); ++ if (ret) ++ return ret; ++ *chain = dma_fence_chain_alloc(); ++ if (!*chain) ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++/* File lock held. One mapping of the listed BO must contain each range at a ++ * consistent BO offset; access is a subset of the BO's immutable roles. ++ */ ++static int g5p_render_resolve(struct drm_file *file, struct g5p_vm *vm, ++ const struct drm_g5p_render_resource *res, ++ u32 count, struct g5p_ru_range *ranges, ++ struct g5p_robj *objs, u32 *obj_count) ++{ ++ u32 known = DRM_G5P_RENDER_ACCESS_READ | DRM_G5P_RENDER_ACCESS_WRITE | ++ DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ ++ for (u32 i = 0; i < count; i++) { ++ struct drm_gem_object *obj; ++ struct g5p_mapping *m, *found = NULL; ++ u32 roles, access; ++ ++ for (u32 j = 0; j < i; j++) ++ if (res[i].handle == res[j].handle) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, res[i].handle); ++ if (!obj) ++ return -ENOENT; ++ roles = to_g5p_bo(obj)->roles; ++ { ++ access = res[i].kind; ++ if (!(access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE)) || ++ access & ~known || ++ ((access & G5P_RU_ACCESS_READ) && ++ !(roles & DRM_G5P_BO_READ)) || ++ ((access & G5P_RU_ACCESS_WRITE) && ++ !(roles & DRM_G5P_BO_WRITE))) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ /* A foreign buffer always gets cache maintenance and fences. */ ++ if (drm_gem_is_imported(obj)) ++ access |= DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ } ++ list_for_each_entry(m, &vm->maps, link) ++ if (m->obj == obj && res[i].va >= m->node.start && ++ res[i].size && ++ res[i].va - m->node.start < m->node.size && ++ res[i].size <= m->node.size - (res[i].va - ++ m->node.start) && ++ res[i].offset == ++ m->offset + (res[i].va - m->node.start)) ++ found = m; ++ if (!found) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ if (access & (DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE)) { ++ struct g5p_robj *o = &objs[*obj_count]; ++ ++ o->obj = obj; /* the job inherits this reference */ ++ o->access = access; ++ o->lo = 0; ++ o->hi = obj->size; ++ if ((READ_ONCE(render_user_sync_range) & 2) && ++ !g5p_sync_window(res[i].offset, res[i].size, ++ obj->size, &o->lo, &o->hi)) { ++ o->lo = 0; ++ o->hi = obj->size; ++ } ++ (*obj_count)++; ++ } else { ++ drm_gem_object_put(obj); ++ } ++ ranges[i].va = res[i].va; ++ ranges[i].size = res[i].size; ++ ranges[i].access = access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE); ++ /* The BO's EXEC role, for TYPED_V3 only; an imported BO never has it. */ ++ if ((roles & DRM_G5P_BO_EXEC)) ++ ranges[i].access |= G5P_RU_ACCESS_EXEC; ++ } ++ return 0; ++} ++ ++/* LINEAR64_V1 admits only the exact round16 immutable inputs. Writes stay ++ * privileged and caller-serialized, as for the r20 first-store dialect. ++ */ ++ ++/* File lock held. Copy `len` bytes at GPU VA `va` out of the BO the VM maps ++ * there, through the BO's own pages, as g5p_render_profile1_bytes() does. The ++ * mapping is found again by VA and its own (VM-referenced) object is read, so ++ * a handle closed or replaced since g5p_render_resolve() cannot redirect the ++ * read. Only a native BO has shmem pages: an imported one (never EXEC, so ++ * never a TA program) is refused for the stream too. ++ */ ++static int g5p_render_read_va(struct drm_device *drm, struct g5p_vm *vm, u64 va, ++ void *dst, u32 len) ++{ ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); ++ struct g5p_mapping *m, *found = NULL; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ u64 off; ++ int ret; ++ ++ list_for_each_entry(m, &vm->maps, link) ++ if (va >= m->node.start && va - m->node.start < m->node.size && ++ len <= m->node.size - (va - m->node.start)) ++ found = m; ++ if (!found) ++ return -ENOENT; ++ obj = found->obj; ++ off = found->offset + (va - found->node.start); ++ if (drm_gem_is_imported(obj) || off > obj->size || ++ len > obj->size - off) ++ return -EINVAL; ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ ret = IS_ERR(sgt) ? PTR_ERR(sgt) : drm_gem_vmap(obj, &map); ++ if (ret) ++ return ret; ++ /* Keep content validation. Native private WC has no CPU cache alias; ++ * all other supported mappings retain a bounded DMA ownership sync. ++ */ ++ bool range = READ_ONCE(render_user_cache_read_va); ++ bool omit = range && g5p_cache_omit(READ_ONCE(render_user_cache_private_wc), ++ false, READ_ONCE(obj->dma_buf) != NULL, to_drm_gem_shmem_obj(obj)->map_wc); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, true); ++ else dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_rmb(); ++ if (map.is_iomem) ++ ret = -EINVAL; ++ else ++ memcpy(dst, (const u8 *)map.vaddr + off, len); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, false); ++ else dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_wmb(); ++ drm_gem_vunmap(obj, &map); ++ return ret; ++} ++ ++static void g5p_render_pass_from_uapi(struct g5p_ru_pass *p, ++ const struct drm_g5p_render_pass *u) ++{ ++ p->vdm_va = u->vdm_va; ++ p->vdm_size = u->vdm_size; ++ p->vdm_control = u->vdm_control; ++ p->scratch_va = u->scratch_va; ++ p->scratch_size = u->scratch_size; ++ memcpy(p->scratch_ptr, u->scratch_ptr, sizeof(p->scratch_ptr)); ++ p->pool7_va = u->pool7_va; ++ p->pool7_size = u->pool7_size; ++ p->pool8_va = u->pool8_va; ++ p->pool8_size = u->pool8_size; ++ memcpy(p->context_pds, u->context_pds, sizeof(p->context_pds)); ++ memcpy(p->resume_pds, u->resume_pds, sizeof(p->resume_pds)); ++ memcpy(p->ta_pds, u->ta_pds, sizeof(p->ta_pds)); ++ memcpy(p->background, u->background, sizeof(p->background)); ++ memcpy(p->eot, u->eot, sizeof(p->eot)); ++ memcpy(p->eot_resume, u->eot_resume, sizeof(p->eot_resume)); ++ memcpy(p->restore, u->restore, sizeof(p->restore)); ++} ++ ++static void g5p_render_pass2_from_uapi(struct g5p_ru_pass2 *e, ++ const struct drm_g5p_render_pass_ext *u) ++{ ++ *e = (struct g5p_ru_pass2){ ++ .scissor_va = u->scissor_va, ++ .dbias_va = u->dbias_va, ++ .background_texture = u->background_texture, ++ .occlusion_query_va = u->occlusion_query_va, ++ .occlusion_query_size = u->occlusion_query_size, ++ .scissor_count = u->scissor_count, ++ .dbias_count = u->dbias_count, ++ .bgobj_depth = u->bgobj_depth, ++ .bgobj_vals = u->bgobj_vals, ++ .pixel_width = u->pixel_output_width, ++ .zls_mask = u->zls_address_mask, ++ }; ++ memcpy(e->zls, u->zls, sizeof(e->zls)); ++} ++ ++static void g5p_render_pass3_from_uapi(struct g5p_ru_pass3 *v, ++ const struct drm_g5p_render_pass_v3 *u) ++{ ++ *v = (struct g5p_ru_pass3){ ++ .ta_program_va = u->ta_program_va, ++ .ta_program_size = u->ta_program_size, ++ .vdm_end = u->vdm_end, ++ .pixel_width = u->pixel_output_width, ++ .pad = u->pad, ++ }; ++} ++ ++static_assert(offsetof(struct drm_g5p_render_pass, samples) == 296); ++static_assert(offsetof(struct drm_g5p_render_pass, sample_positions) == 304); ++static_assert(offsetof(struct drm_g5p_render_pass, reserved) == 312); ++static_assert(DRM_G5P_CAP_RENDER_MSAA == BIT_ULL(31)); ++static_assert(DRM_G5P_CAP_RENDER_OCCLUSION_QUERY == BIT_ULL(33)); ++static_assert(DRM_G5P_RENDER_MSAA_RESOLVE == G5P_RU_MSAA_RESOLVE); ++ ++/* CAP_RENDER_MSAA: the target's geometry is kernel-derived (it holds the sample ++ * count), and only TYPED_V3 carries the pass fields. ++ */ ++static bool g5p_render_msaa_enabled(void) ++{ ++ return (1) && (1); ++} ++ ++/* The base pass's MSAA fields: zero unless TYPED_V3 with the capability. */ ++static int g5p_render_msaa_from_uapi(struct g5p_ru_msaa *m, ++ const struct drm_g5p_render_pass *u, ++ bool v3) ++{ ++ *m = (struct g5p_ru_msaa){ ++ .samples = u->samples, ++ .flags = u->msaa_flags, ++ .positions = u->sample_positions, ++ }; ++ if ((m->samples || m->flags || m->positions) && ++ (!v3 || !g5p_render_msaa_enabled())) ++ return -EINVAL; ++ return 0; ++} ++ ++/* Render submit lock held. The first accepted job claims the device for render until reboot; ++ * same exclusions as compute/r22. Every later job, of any context, passes while the device is ++ * neither failed nor stopped, without the core lock (a job may hold it for its whole run). The ++ * claimant keeps the reference d->quarantined_context has always held. The IRQ-test term is ++ * the r74 one (render-fixed.inc, render-r72.inc): the r74 boots refuse the IRQ test, so ++ * irq_tested must stay 0 there and must be 1 on every other boot. ++ */ ++static int g5p_render_device_claim(struct g5p_drm *d, struct g5p_context *c) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_render *r = c->render; ++ int ret = -EBUSY; ++ ++ if (READ_ONCE(d->render_claimed)) { ++ /* A recovery swaps the epoch, and a failure noted mid-job may lead to one: the job ++ * waits for the ruling in the scheduler, which fails it if the epoch ends. ++ */ ++ if (g5p_rdev_ruling(d->rdev)) ++ return 0; ++ b = READ_ONCE(g->boot); ++ return !b || READ_ONCE(b->failed) || READ_ONCE(b->stopped) ? ++ -EIO : ++ 0; ++ } ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ if (b->failed || b->stopped || !b->control_complete || b->irq_tested || ++ b->user_compute_owned || d->quarantined_context || ++ !b->render_meta[0].cpu || !b->render_meta[1].cpu || ++ b->layout.used[1] > G5P_R22_TA || ++ /* recovery108: expose the GPU only with a current-generation ALIVE proof. */ ++ g5p_ds_hook_claim(g) || atomic_cmpxchg(&d->accepted, 0, 1)) ++ goto out; ++ b->user_compute_owned = true; /* blocks diagnostic commands, latched */ ++ r->claimant = true; ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->quarantined_context = c; ++ WRITE_ONCE(d->render_claimed, true); ++ ret = 0; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Evict the oldest completed result beyond the per-file bound. */ ++static int g5p_render_result_slot(struct g5p_file *f, struct g5p_drm *d) ++{ ++ unsigned long index = 0, flags; ++ struct g5p_fence *old; ++ bool done; ++ ++ if (f->render_results < G5P_RU_RESULTS) ++ return 0; ++ old = xa_find(&f->results, &index, ULONG_MAX, XA_PRESENT); ++ if (!old) ++ return 0; ++ spin_lock_irqsave(&d->status_lock, flags); ++ done = old->state == DRM_G5P_JOB_COMPLETE; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!done) ++ return -EBUSY; ++ xa_erase(&f->results, index); ++ dma_fence_put(&old->base); ++ f->render_results--; ++ return 0; ++} ++ ++static int g5p_render_submit_admitted(struct drm_device *drm, ++ struct drm_file *file, u32 context, ++ u32 profile, u32 rt_handle, u64 res_ptr, ++ u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, ++ u32 pass_size, u64 *job_id) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_render_pass_v3 *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_ru_pass pass; ++ struct g5p_ru_pass2 pass2 = {}; ++ struct g5p_ru_pass3 pass3 = {}; ++ struct g5p_ru_v3_loc loc = {}; ++ struct g5p_ru_msaa msaa = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ struct g5p_rt *rt = NULL; ++ const bool v2 = false; ++ const bool v3 = true; ++ u64 point = 0; ++ int ret; ++ ++ ; ++ ; ++ /* TYPED_V3 is unprivileged (checked per context below); the legacy profiles touch ++ * registers and stay with CAP_SYS_RAWIO. ++ */ ++ ; ++ ; ++ /* During a recovery the epoch is being replaced: accept, the scheduler holds the job. */ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ /* a failure noted mid-job waits for the executor's ruling (g5p_rdev_ruling()) */ ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(res_ptr), res_count, ++ sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(sync_ptr), sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ { ++ /* pass_size is the exact V1, V2 or V3 size; ext exists for V2 and V3, ++ * the program fields for V3 only. ++ */ ++ upass = memdup_user(u64_to_user_ptr(pass_ptr), pass_size); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (upass->base.size != pass_size || upass->base.flags || ++ memchr_inv(upass->base.reserved, 0, ++ sizeof(upass->base.reserved)) || ++ (true && memchr_inv(upass->ext.reserved, 0, ++ sizeof(upass->ext.reserved))) || ++ (v3 && ++ memchr_inv(upass->reserved, 0, sizeof(upass->reserved)))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_render_pass_from_uapi(&pass, &upass->base); ++ ret = g5p_render_msaa_from_uapi(&msaa, &upass->base, v3); ++ if (ret) ++ goto out; ++ { ++ g5p_render_pass2_from_uapi(&pass2, &upass->ext); ++ }; ++ { ++ g5p_render_pass3_from_uapi(&pass3, upass); ++ }; ++ } ++ ranges = kvcalloc(res_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(res_count, sizeof(*job->objs), GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ /* Acceptance, implicit fence attach and queueing happen in one order. */ ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ { ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, res_count, ranges, job->objs, ++ &job->obj_count); ++ if (ret) ++ goto unlock; ++ { ++ rt = xa_load(&f->targets, rt_handle); ++ if (!rt || rt->ctx != c || rt->destroyed) { ++ ret = -ENOENT; ++ goto unlock; ++ } ++ } ++ { ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ u8 head[G5P_RU_PROGRAM_MIN], tail[4]; ++ ++ /* Structure first; the program head and the stream's last dword are ++ * read only once every range is proven backed, READ and (program) EXEC. ++ */ ++ ret = g5p_ru_pass3_check(&pass, &pass2, &pass3, &lim, ranges, ++ res_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass3.ta_program_va, head, ++ sizeof(head)); ++ if (!ret) ++ ret = g5p_render_read_va( ++ drm, c->vm, pass.vdm_va + pass3.vdm_end - 4, ++ tail, sizeof(tail)); ++ if (!ret) ++ ret = g5p_ru_pass3_content(head, tail); ++ } ++ /* Every profile: a one-sample pass needs a one-sample target, and back. */ ++ if (!ret) ++ ret = g5p_ru_msaa_check(&msaa, rt->samples ? rt->samples : 1); ++ if (ret) ++ goto unlock; ++ g5p_ru_fill_ta(job->ta_w, &pass); ++ { ++ g5p_ru_fill_ta_v3(job->ta_w, loc.program_off); ++ }; ++ g5p_ru_fill_3d(job->td_w, &pass); ++ /* TYPED_V3 with an all-zero ext is the V1 dialect: the V1 3D words, plus ++ * the output width words (width 2 rewrites the frozen dialect-1 values). ++ */ ++ if (v2 || loc.dialect2) ++ g5p_ru_fill_3d_v2(job->td_w, &pass2); ++ else { ++ g5p_ru_fill_3d_width(job->td_w, loc.pixel_width); ++ }; ++ /* r25 RT variant 1 moves only TA.W28/3D.W20 one slice into the ring. */ ++ g5p_ru_rt_fill(job->ta_w, job->td_w, &rt->geometry, ++ rt->node.start + (r->v.rt == 1 ? rt->sizes.slice : 0), ++ rt->node.start + rt->sizes.main); ++ { ++ g5p_ru_fill_msaa(job->ta_w, job->td_w, &msaa, &rt->geometry); ++ }; ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ /* Unpublished, never signalled: still a clean rejection. */ ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = v3; ++ { ++ r->v3_accepted = true; ++ }; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ job->cache.pid = task_tgid_nr(current); ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); /* only ever an uninitialized fence here */ ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++/* recovery108: a submitter is inside the admission window for its whole call; once a ++ * FINAL teardown closed admission it is refused before it looks at anything. ++ */ ++static int g5p_render_submit(struct drm_device *drm, struct drm_file *file, ++ u32 context, u32 profile, u32 rt_handle, ++ u64 res_ptr, u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, u32 pass_size, ++ u64 *job_id) ++{ ++ int ret; ++ ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_render_submit_admitted(drm, file, context, profile, rt_handle, ++ res_ptr, res_count, sync_ptr, ++ sync_count, pass_ptr, pass_size, ++ job_id); ++ g5p_r8_exit(); ++ return ret; ++} ++ ++/* The layout the profile dispatch (pass_size selects the profile) and ++ * GET_RENDER_INFO rely on; a header edit that moves any of it fails here. ++ */ ++static_assert(sizeof(struct drm_g5p_render_pass) == 360); ++static_assert(offsetof(struct drm_g5p_render_pass, vdm_control) == 20); ++static_assert(sizeof(struct drm_g5p_render_pass_ext) == 128); ++static_assert(sizeof(struct drm_g5p_render_pass_v2) == 488); ++static_assert(sizeof(struct drm_g5p_render_pass_v3) == 536); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ext) == ++ sizeof(struct drm_g5p_render_pass)); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_va) == 488); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_size) == 496); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, vdm_end) == 500); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, pixel_output_width) == ++ 504); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, reserved) == 512); ++static_assert(sizeof(struct drm_g5p_submit_render_typed) == 80); ++static_assert(sizeof(struct drm_g5p_get_render_info) == 120); ++static_assert(offsetof(struct drm_g5p_get_render_info, hw_va_limit) == 104); ++static_assert(offsetof(struct drm_g5p_get_render_info, reserved1) == 112); ++static_assert(DRM_G5P_CAP_TYPED_RENDER_V3 == BIT_ULL(26)); ++static_assert(DRM_G5P_CAP_RENDER_MULTI_CONTEXT == BIT_ULL(27)); ++static_assert(DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH == BIT_ULL(29)); ++static_assert(DRM_G5P_RENDER_PROFILE_TYPED_V3 == 4); ++ ++static int g5p_submit_render_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_render_typed *a = data; ++ u64 id = 0; ++ int ret; ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || a->profile != DRM_G5P_RENDER_PROFILE_TYPED_V3 || ++ !a->render_target || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_render_pass_v3)) ++ return -EINVAL; ++ ret = g5p_render_submit(drm, file, a->context, ++ DRM_G5P_RENDER_PROFILE_TYPED_V3, ++ a->render_target, a->resources, ++ a->resource_count, a->syncs, a->sync_count, ++ a->pass, a->pass_size, &id); ++ a->job_id = id; ++ return ret; ++} ++ ++static void g5p_rt_geometry_to_uapi(struct drm_g5p_rt_geometry *u, ++ const struct g5p_ru_rt *k) ++{ ++ u->header_bytes = k->header; ++ u->region_bytes = k->region; ++ u->packed_state = k->packed; ++ u->state_word = k->state; ++ u->padded_dims = k->padded; ++ memcpy(u->scale, k->scale, sizeof(u->scale)); ++ memcpy(u->ta_state, k->ta_state, sizeof(u->ta_state)); ++ memcpy(u->ta_tiling, k->ta_tiling, sizeof(u->ta_tiling)); ++ u->ta_tiling_word = k->ta_tiling_word; ++ u->tile_count = k->tiles; ++ u->secondary_bytes = k->secondary; ++ u->secondary_align = k->secondary_align; ++} ++ ++static int g5p_create_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ const struct drm_g5p_rt_geometry *u = &a->geometry; ++ struct g5p_ru_rt geometry; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ bool user = a->flags & DRM_G5P_RT_USER_GEOMETRY; ++ bool kernel = (1); ++ u64 positions = a->samples == 4 ? G5P_RU_POSITIONS_4X : ++ G5P_RU_POSITIONS_1X; ++ u32 id, blocks = 0; ++ int ret; ++ ++ ; ++ ; ++ if (a->flags & ~DRM_G5P_RT_USER_GEOMETRY || a->handle || a->pad || ++ a->main_va || a->main_bytes || a->secondary_va || ++ a->secondary_bytes || ++ (a->samples != 1 && ++ (a->samples != 4 || !g5p_render_msaa_enabled())) || ++ a->layers != 1 || a->pm_blocks || a->pad2 || a->reserved || ++ u->reserved[0] || u->reserved[1]) ++ return -EINVAL; ++ if (user) { ++ { ++ return -EOPNOTSUPP; ++ } ++ } else if (kernel) { ++ /* 6ca6d5c in integers; equal to the leaf for every size. */ ++ if (memchr_inv(u, 0, sizeof(*u)) || ++ g5p_ru_rt_derive_samples(a->width, a->height, a->samples, ++ positions, &geometry)) ++ return -EINVAL; ++ } else { ++ /* Kernel table: only the offline-executed 6ca6d5c result. */ ++ if (a->width != 64 || a->height != 64 || ++ memchr_inv(u, 0, sizeof(*u))) ++ return -EOPNOTSUPP; ++ geometry = g5p_ru_rt64; ++ } ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto out; ++ } ++ r = c->render; ++ ret = g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ /* No core lock: another context's job may hold it for its whole run, and nothing here ++ * touches the hardware. The device pool only grows; it serves every target of every ++ * context. ++ */ ++ if (kernel) { ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, g5p_ru_pm_blocks_for(geometry.tiles, ++ d->rdev->blocks)); ++ blocks = d->rdev->blocks; ++ mutex_unlock(&d->rdev->lock); ++ } ++ if (!ret) ++ ret = g5p_rt_create(d, r, &geometry, a->width, a->height, &rt); ++ if (ret) ++ goto out; ++ rt->samples = a->samples; ++ ret = xa_alloc(&f->targets, &id, rt, ++ XA_LIMIT(1, G5P_RU_MAX_TARGETS * 8), GFP_KERNEL); ++ if (ret) { ++ /* Mapped but unpublished to a job: leave it owned by the context. */ ++ rt->destroyed = true; ++ goto out; ++ } ++ kref_get(&c->ref); ++ a->handle = id; ++ a->main_va = rt->node.start; ++ a->main_bytes = rt->sizes.main; ++ a->secondary_va = rt->node.start + rt->sizes.main; ++ a->secondary_bytes = rt->sizes.secondary_map; ++ if (!user) ++ g5p_rt_geometry_to_uapi(&a->geometry, &geometry); ++ if (kernel) ++ a->pm_blocks = blocks; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_destroy_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c = NULL; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ int ret; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ rt = xa_load(&f->targets, a->handle); ++ if (!rt) { ++ ret = -ENOENT; ++ goto out; ++ } ++ c = rt->ctx; ++ r = c->render; ++ ret = r->detached ? 0 : g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ g5p_vmlog_add(G5P_VMLOG_RT_DEL, c->vm, 0, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map, ++ rt->width << 16 | rt->height, rt->samples); ++ /* RAM only, no core lock: see g5p_create_render_target(). */ ++ if (r->bound) { ++ g5p_vm_clear(c->vm, drm->dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ ret = g5p_render_invalidate(c); ++ if (ret) ++ goto out; ++ } ++ if (!r->detached) { ++ g5p_rt_release_memory(d, r, rt, true); ++ list_del(&rt->link); ++ r->target_count--; ++ if (r->profile1_rt == rt) ++ r->profile1_rt = NULL; ++ kfree(rt); ++ } else { ++ rt->destroyed = true; ++ } ++ xa_erase(&f->targets, a->handle); ++out: ++ mutex_unlock(&f->lock); ++ if (!ret && c) ++ g5p_context_put(c); ++ return ret; ++} ++ ++/* True when a context created now would latch a hypothesis knob that differs ++ * from its default: GET_RENDER_INFO has no context to report on. ++ */ ++static bool g5p_render_variant_active(void) ++{ ++ return (0) || (G5P_RU_DEF_CACHE_VARIANT) != G5P_RU_DEF_CACHE_VARIANT || ++ (G5P_RU_DEF_MMU_VARIANT) != G5P_RU_DEF_MMU_VARIANT || (0) || ++ READ_ONCE(render_user_wait_ms) != G5P_RU_DEF_WAIT_MS || (0) || ++ (0) || (0) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) != ++ G5P_RU_DEF_PROFILE1_VDM_CONTROL || ++ (0) || (0) || (0) || (0); ++} ++ ++static int g5p_get_render_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_render_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || ++ memchr_inv(&a->profiles, 0, ++ sizeof(*a) - offsetof(struct drm_g5p_get_render_info, ++ profiles))) ++ return -EINVAL; ++ ; ++ ; ++ if (g5p_render_variant_active()) ++ a->flags = DRM_G5P_RENDER_INFO_VARIANT_ACTIVE; ++ a->profiles = BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ ; ++ ; ++ { ++ a->profiles |= BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ }; ++ { ++ a->max_render_width = G5P_RU_MAX_DIM; ++ a->max_render_height = G5P_RU_MAX_DIM; ++ a->max_pm_blocks = G5P_RU_MAX_BLOCKS; ++ } ++ a->max_render_targets = G5P_RU_MAX_TARGETS; ++ a->private_va_start = G5P_RU_PRIVATE_VA; ++ a->private_va_end = G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE; ++ a->user_va_min = G5P_RU_USER_VA_MIN; ++ a->usc_base = G5P_RU_PDS_BASE; ++ a->pds_base = a->usc_base; ++ a->code_window = G5P_RU_CODE_WINDOW; ++ a->hw_va_limit = G5P_RU_HW_VA_LIMIT; ++ a->pm_blocks = (32); ++ a->pm_block_bytes = G5P_RU_BLOCK_BYTES; ++ a->max_queued_jobs = G5P_RU_MAX_QUEUED; ++ a->max_vdm_bytes = G5P_RU_MAX_VDM; ++ /* TYPED_V3: what a waiter must outlast is the job's absolute bound (TA publish to the end of the ++ * 3D, render-user-wait.h), not the per-phase module value: a TA that takes its whole phase leaves the ++ * 3D the rest of the job cap. ++ */ ++ { ++ if (READ_ONCE(render_user_hard_ms)) ++ a->job_timeout_ms = g5p_ru_lat_job_ms( ++ READ_ONCE(render_user_hard_ms), ++ READ_ONCE(render_user_obs_ms)); ++ else ++ a->job_timeout_ms = G5P_RU_V3_CAP_MS; ++ }; ++ a->retained_results = G5P_RU_RESULTS; ++ return 0; ++} ++ ++/* VM_BIND hooks for the context resident on a VM (serial compute or render). */ ++static int g5p_resident_idle(struct g5p_context *c, bool live) ++{ ++ return g5p_render_vm_idle(c, live); ++} ++ ++/* VM_BIND UNMAP, file lock held, PTEs already cleared and the tag published: a render ++ * context with queued jobs keeps the mapping's object (its pages and its pin) until those ++ * jobs have retired (g5p_render_reap()). False: free it now. ++ */ ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m) ++{ ++ struct g5p_render *r = c->render; ++ unsigned long flags; ++ ++ if (!r || !atomic_read(&r->queued)) ++ return false; ++ m->retire = r->fence_seqno; ++ spin_lock_irqsave(&r->zlock, flags); ++ list_add_tail(&m->link, &r->zombies); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ return true; ++} ++ ++static int g5p_resident_invalidate(struct g5p_context *c) ++{ ++ return g5p_render_invalidate(c); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-submit.inc b/drivers/gpu/drm/apple-g5p/drm-submit.inc +new file mode 100644 +index 000000000..f36998e4d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-submit.inc +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Asynchronous privileged probes; payloads and firmware VM are kernel-owned. */ ++struct g5p_fence { ++ struct dma_fence base; ++ /* Owned by dma_fence; independent of queue/status locks. */ ++ spinlock_t lock; ++ u32 state; ++ int error; ++}; ++ ++static const char *g5p_fence_driver_name(struct dma_fence *f) ++{ ++ return "apple_g5p"; ++} ++ ++static const char *g5p_fence_timeline_name(struct dma_fence *f) ++{ ++ return "g5p-probe"; ++} ++ ++/* Default release uses dma_fence_free()/RCU; base is the first member. */ ++static const struct dma_fence_ops g5p_fence_ops = { ++ .get_driver_name = g5p_fence_driver_name, ++ .get_timeline_name = g5p_fence_timeline_name, ++}; ++ ++/* recovery108: the compute/probe submitters share the render path's admission window. */ ++ ++/* Legacy JOB_COMPUTE remains unsupported: v1 lacks a bounded stream length. */ +diff --git a/drivers/gpu/drm/apple-g5p/render-user-validate.h b/drivers/gpu/drm/apple-g5p/render-user-validate.h +new file mode 100644 +index 000000000..a466367bb +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/render-user-validate.h +@@ -0,0 +1,384 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_VALIDATE_H ++#define G5P_RENDER_USER_VALIDATE_H ++ ++/* Pure TYPED_V1 checks over already-resolved resource ranges. The caller ++ * proves each range is inside a mapping of the listed BO, in the submitting ++ * context's VM, with BO roles covering `access`. Nothing here is a pointer ++ * the kernel dereferences; GPU addresses are only range-checked. ++ */ ++#define G5P_RU_ACCESS_READ 1U ++#define G5P_RU_ACCESS_WRITE 2U ++/* Software admission bound; TYPED_V3 never sends a stream length to firmware. ++ * vdm_size/end only bound the listed range and the four-byte tail read. ++ */ ++#define G5P_RU_MAX_VDM 0x100000U ++#define G5P_RU_MAX_VDM_CONTROL 0x10000U ++#define G5P_RU_MAX_SCRATCH (64ULL << 20) ++ ++/* Hardware facts GET_RENDER_INFO reports and the TYPED_V3 validator enforces; ++ * one definition so the two cannot drift. Code offsets are 32-bit from the ++ * common USC/PDS base. TA/3D reads at or above 2^38 stall silently on A10 ++ * (works at 0x3_, 0x10_ and 0x20_, stalls at 0x40_ and 0x80_, no MMU fault); ++ * not flown on A9, so the limit is a statement, not a proof (see ++ * docs/gpu/experiments/typed-v3-kernel-implementation.md). ++ */ ++#define G5P_RU_PDS_BASE 0x300000000ULL ++#define G5P_RU_CODE_WINDOW 0x100000000ULL /* 2^32 */ ++#define G5P_RU_HW_VA_LIMIT 0x4000000000ULL /* 2^38 */ ++ ++/* Not a requested access: the resolver sets it, for the typed profiles only, ++ * when the listed BO carries DRM_G5P_BO_EXEC. Only TYPED_V3 reads it; every ++ * other check masks on READ/WRITE. It lives in `access` rather than in a new ++ * field so positional g5p_ru_range initialisers keep compiling. ++ */ ++#define G5P_RU_ACCESS_EXEC 16U ++ ++struct g5p_ru_range { ++ u64 va, size; ++ u32 access; ++}; ++ ++/* [va, va + size) inside one listed range that grants every access bit. */ ++static inline bool g5p_ru_in(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 access) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if ((r[i].access & access) == access && va >= r[i].va && ++ va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) ++ return true; ++ return false; ++} ++ ++/* Everything TYPED_V1 and TYPED_V2 share: all fields except the pools. */ ++static inline int g5p_ru_pass_check_common(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || ++ !g5p_ru_in(r, n, p->vdm_va, p->vdm_size, G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ /* Copied to TA W88/R0+470; meaning unresolved (r23), so bounded. */ ++ if ((p->vdm_control & 0xf) || p->vdm_control > G5P_RU_MAX_VDM_CONTROL) ++ return -EINVAL; ++ if ((p->scratch_va & 0xf) || p->scratch_size < 0x10 || ++ p->scratch_size > G5P_RU_MAX_SCRATCH || ++ !g5p_ru_in(r, n, p->scratch_va, p->scratch_size, rw)) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((p->scratch_ptr[i] & 3) || ++ !g5p_ru_in(r, n, p->scratch_ptr[i], 4, rw)) ++ return -EINVAL; ++ /* Three-word descriptors keep their fourth slot zero. */ ++ if (p->ta_pds[3] || p->eot[3] || p->eot_resume[3]) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V1 and LINEAR64_V1: the two allocator pools are page-aligned RW. */ ++static inline int g5p_ru_pass_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret = g5p_ru_pass_check_common(p, r, n); ++ ++ if (ret) ++ return ret; ++ if (((p->pool7_va | p->pool7_size | p->pool8_va | p->pool8_size) & ++ (G5P_GPU_PAGE - 1)) || ++ !g5p_ru_in(r, n, p->pool7_va, p->pool7_size, rw) || ++ !g5p_ru_in(r, n, p->pool8_va, p->pool8_size, rw)) ++ return -EINVAL; ++ return 0; ++} ++ ++#define G5P_RU_MAX_TABLE 65536U ++ ++/* TYPED_V2: the pools are the ISP scissor (8-byte entries) and depth-bias ++ * (12-byte) tables, READ and word aligned; the V1 pool fields stay zero. ++ * Flagged ZLS words are addresses into listed RW ranges; the rest is opaque. ++ */ ++static inline int g5p_ru_pass2_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret; ++ ++ if (p->pool7_va || p->pool7_size || p->pool8_va || p->pool8_size) ++ return -EINVAL; ++ ret = g5p_ru_pass_check_common(p, r, n); ++ if (ret) ++ return ret; ++ if (!e->scissor_count || e->scissor_count > G5P_RU_MAX_TABLE || ++ (e->scissor_va & 3) || ++ !g5p_ru_in(r, n, e->scissor_va, e->scissor_count * 8ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if (!e->dbias_count || e->dbias_count > G5P_RU_MAX_TABLE || ++ (e->dbias_va & 3) || ++ !g5p_ru_in(r, n, e->dbias_va, e->dbias_count * 12ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if ((e->pixel_width != 2 && e->pixel_width != 4 && ++ e->pixel_width != 8) || ++ (e->bgobj_vals & ~0x3ffU) || (e->zls_mask & ~0x3fU)) ++ return -EINVAL; ++ if (!!e->occlusion_query_va != !!e->occlusion_query_size || ++ (e->occlusion_query_va && ++ ((e->occlusion_query_va & 0xf) || e->occlusion_query_size < sizeof(u32) || ++ !g5p_ru_in(r, n, e->occlusion_query_va, e->occlusion_query_size, rw)))) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((e->zls_mask & (1U << i)) && ++ ((e->zls[i] >> 40) || ++ !g5p_ru_in(r, n, e->zls[i] & ~0xfULL, 16, rw))) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V3: the TA PDS program (W+0x88 -> R0+0x470) is an explicit reference ++ * and the VDM stream length is a separate field that never reaches W. ++ * Contract and evidence: docs/gpu/experiments/typed-v3-kernel-implementation.md. ++ * Everything here is rejected, never masked. Structural rules need only the ++ * resolved ranges; the two content rules take bytes the caller read through ++ * the BOs' own pages (g5p_ru_pass3_content()). The rules are a subset of what ++ * the userspace lane accepts (tools/mesa-a9 a9c_ctx_lint(), a9scene): nothing ++ * here is stricter than a client that passes those lints. ++ * -EINVAL stale vdm_control, malformed program/stream/descriptor fields, ++ * no terminator, or (V1 dialect) the V1 rules fail ++ * -ERANGE outside [pds_base, pds_base + code_window) or past hw_va_limit ++ * -ENOENT no listed range wholly covers the program, stream or task code ++ * -EACCES missing READ, or the program/task code range's BO lacks EXEC ++ * -ENODATA the first 16 program bytes are all zero ++ */ ++#define G5P_RU_PROGRAM_ALIGN 16U ++#define G5P_RU_PROGRAM_MIN 16U ++ ++struct g5p_ru_v3_limits { ++ u64 pds_base, code_window; ++ u64 hw_va_limit; /* 0: no statement */ ++}; ++ ++/* Where the content rules read: indexes into the ranges array. */ ++struct g5p_ru_v3_loc { ++ u64 program_off; /* the exact W+0x88 value */ ++ u32 program_range, vdm_range; ++ u32 pixel_width; /* the pass's output width, 2, 4 or 8, either dialect */ ++ bool dialect2; /* ext was non-zero: V2 3D words; else the V1 dialect */ ++}; ++ ++/* TYPED_V3 carries the V2 ext struct. An all-zero ext selects the V1 dialect ++ * (pools in base, no tables, V1 3D words): the pass tools/mesa-a9 records ++ * today, plus the explicit program. Any non-zero ext field selects V2. ++ */ ++static inline bool g5p_ru_pass2_zero(const struct g5p_ru_pass2 *e) ++{ ++ u64 any = e->scissor_va | e->dbias_va | e->background_texture | ++ e->occlusion_query_va | e->occlusion_query_size; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(e->zls); i++) ++ any |= e->zls[i]; ++ any |= e->scissor_count | e->dbias_count | e->bgobj_depth | ++ e->bgobj_vals | e->pixel_width | e->zls_mask; ++ return !any; ++} ++ ++/* First listed range that wholly contains [va, va + size). */ ++static inline bool g5p_ru_cover(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 *idx) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if (va >= r[i].va && va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) { ++ *idx = i; ++ return true; ++ } ++ return false; ++} ++ ++/* [va, va + size) ends at or below the limit; zero limit makes no statement. */ ++static inline bool g5p_ru_below(u64 limit, u64 va, u64 size) ++{ ++ return !limit || (size <= limit && va <= limit - size); ++} ++ ++/* One PDS task reference of a store/resume descriptor: a 32-bit offset from ++ * pds_base, 16-byte aligned like every captured value and W+0x88's mask, ++ * non-zero, inside the code window and the hardware limit, and starting in a ++ * listed READ range (`code`: whose BO also carries EXEC). ++ */ ++static inline int g5p_ru_v3_ref(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, u32 off, ++ bool code) ++{ ++ u64 va; ++ u32 i; ++ ++ if (!off || (off % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (lim->code_window < G5P_RU_PROGRAM_MIN || ++ off > lim->code_window - G5P_RU_PROGRAM_MIN || ++ lim->pds_base > ~0ULL - off) ++ return -ERANGE; ++ va = lim->pds_base + off; ++ if (!g5p_ru_below(lim->hw_va_limit, va, G5P_RU_PROGRAM_MIN)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, va, G5P_RU_PROGRAM_MIN, &i)) ++ return -ENOENT; ++ if (!(r[i].access & G5P_RU_ACCESS_READ) || ++ (code && !(r[i].access & G5P_RU_ACCESS_EXEC))) ++ return -EACCES; ++ return 0; ++} ++ ++/* The TYPED_V3 pixel output width. The V1 dialect takes it from the V3 field ++ * (0 = 2, the V1 dialect exactly), the V2 dialect from ext, where the V3 field ++ * must stay zero. A width above 2 is several colour targets, and a partial ++ * render must then resume with the pass's own EOT: a one-target resume EOT ++ * would store only the first target. The EOT itself is not decoded. ++ */ ++static inline int g5p_ru_pass3_width(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ bool dialect2, u32 *width) ++{ ++ u32 w = v->pixel_width; ++ ++ if (v->pad || (w && w != 2 && w != 4 && w != 8) || (dialect2 && w)) ++ return -EINVAL; ++ if (dialect2) ++ w = e->pixel_width; /* 2, 4 or 8: g5p_ru_pass2_check() */ ++ else if (!w) ++ w = 2; ++ if (w > 2 && memcmp(p->eot, p->eot_resume, sizeof(p->eot))) ++ return -EINVAL; ++ *width = w; ++ return 0; ++} ++ ++/* The V3 rules, then the TYPED_V1 rules (all-zero ext) or the TYPED_V2 rules ++ * (any ext field set; pools zero), then the output width. base.vdm_control is ++ * checked first. On success loc names the ranges to read, the dialect and the ++ * width. ++ */ ++static inline int g5p_ru_pass3_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_ru_v3_loc *loc) ++{ ++ u32 pi, vi; ++ u64 off; ++ int ret; ++ ++ if (p->vdm_control) ++ return -EINVAL; /* a stream length here is the r36/r88 defect */ ++ if (v->ta_program_size < G5P_RU_PROGRAM_MIN || ++ v->ta_program_size > DRM_G5P_RENDER_MAX_PROGRAM_BYTES || ++ (v->ta_program_size % G5P_RU_PROGRAM_ALIGN) || ++ (v->ta_program_va % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || v->vdm_end < 4 || ++ v->vdm_end > p->vdm_size || (v->vdm_end & 3)) ++ return -EINVAL; ++ if (v->ta_program_va < lim->pds_base) ++ return -ERANGE; ++ off = v->ta_program_va - lim->pds_base; ++ if (lim->code_window < v->ta_program_size || ++ off > lim->code_window - v->ta_program_size) ++ return -ERANGE; ++ if (!g5p_ru_below(lim->hw_va_limit, v->ta_program_va, ++ v->ta_program_size) || ++ !g5p_ru_below(lim->hw_va_limit, p->vdm_va, p->vdm_size)) ++ return -ERANGE; ++ /* Every listed range, not only the two above: on A10 a buffer moved past ++ * the limit stalls the TA without a fault, and all buffers moved together. ++ */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, v->ta_program_va, v->ta_program_size, &pi)) ++ return -ENOENT; ++ if (!(r[pi].access & G5P_RU_ACCESS_READ) || ++ !(r[pi].access & G5P_RU_ACCESS_EXEC)) ++ return -EACCES; ++ if (!g5p_ru_cover(r, n, p->vdm_va, p->vdm_size, &vi)) ++ return -ENOENT; ++ if (!(r[vi].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ /* Context store [0] / load [1]: code at word 2 (READ and EXEC), STREAM_OUT2 ++ * at word 5 (READ). Resume tasks: word 2 (READ only: R0+0xf060/0xf070, ++ * named stream-out by the userspace lane and aux program by r92 2.4, so ++ * its role is unresolved and EXEC is not demanded). Words 0, 1, 3 and 4 ++ * stay opaque. The spacing (store - program = 0x280, resume - store = ++ * 0x340) and a program that equals a task entry (r63 flew the store ++ * program; the role is not detectable structurally) are not enforced. ++ */ ++ for (u32 i = 0; i < 2; i++) { ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][2], true); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][5], ++ false); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->resume_pds[i][2], ++ false); ++ if (ret) ++ return ret; ++ } ++ loc->dialect2 = !g5p_ru_pass2_zero(e); ++ ret = loc->dialect2 ? g5p_ru_pass2_check(p, e, r, n) : ++ g5p_ru_pass_check(p, r, n); ++ if (!ret) ++ ret = g5p_ru_pass3_width(p, e, v, loc->dialect2, ++ &loc->pixel_width); ++ if (ret) ++ return ret; ++ loc->program_off = off; ++ loc->program_range = pi; ++ loc->vdm_range = vi; ++ return 0; ++} ++ ++/* The first 16 program bytes are not all zero (r88 published 0x370, which ++ * pointed at zeros) and the last stream dword is the terminator. `head` is ++ * G5P_RU_PROGRAM_MIN bytes, `tail` the 4 stream bytes at vdm_va + vdm_end - 4. ++ */ ++static inline int g5p_ru_pass3_content(const u8 *head, const u8 *tail) ++{ ++ u8 any = 0; ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ for (u32 i = 0; i < G5P_RU_PROGRAM_MIN; i++) ++ any |= head[i]; ++ if (!any) ++ return -ENODATA; ++ if (word != DRM_G5P_RENDER_STREAM_TERMINATE) ++ return -EINVAL; ++ return 0; ++} ++ ++/* AGXRenderTargetMemoryG5 output sanity for userspace geometry: exact one ++ * sample and the sizes the kernel will allocate. Opaque fields unchecked. ++ */ ++static inline int g5p_ru_geometry_check(const struct g5p_ru_rt *rt, u32 width, ++ u32 height, ++ struct g5p_ru_rt_sizes *sizes) ++{ ++ if (!width || !height || width > 16384 || height > 16384) ++ return -EINVAL; ++ if (rt->tiles > ((width + 31) / 32) * ((height + 31) / 32)) ++ return -EINVAL; ++ return g5p_ru_rt_sizes(rt, sizes); ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/runtime-idle.inc b/drivers/gpu/drm/apple-g5p/runtime-idle.inc +new file mode 100644 +index 000000000..a83d99ad3 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/runtime-idle.inc +@@ -0,0 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++static int g5p_idle_open(struct g5p_drm *d) ++{ ++ atomic_inc(&d->open_files); ++ return 0; ++} ++ ++static void g5p_idle_work(struct work_struct *work) ++{ ++} ++ ++static void g5p_idle_queue(struct g5p_drm *d) ++{ ++} diff --git a/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch b/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch new file mode 100644 index 00000000..b26eba8a --- /dev/null +++ b/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch @@ -0,0 +1,147 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:11:46 +0000 +Subject: [PATCH] drm: expose the native GEM exporter CPU cache contract + +[Orchard: from brkosebay/linux 78775ea3b9e; retain Orchard ADBE +implementation, adding its exporter contract only.] + +G5P PRIME imports require an immutable native cache mapping contract. +Unknown and foreign exporters stay conservative. + +Assisted-by: LLM +--- + drivers/gpu/drm/adbe/adbe_drv.c | 25 +++++++++++++++++++++++++ + drivers/gpu/drm/drm_prime.c | 19 +++++++++++++++++++ + include/drm/drm_gem.h | 8 ++++++++ + include/drm/drm_gem_dma_helper.h | 8 ++++++++ + include/drm/drm_prime.h | 2 ++ + 5 files changed, 62 insertions(+) + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..24c76fef9 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -653,11 +653,36 @@ static int adbe_dumb_create(struct drm_file *file, struct drm_device *drm, + return drm_gem_dma_dumb_create_internal(file, drm, args); + } + ++/* Native allocations: the DMA helper's defaults plus the uncached-export ++ * contract. Imported objects never make that promise. ++ */ ++static const struct drm_gem_object_funcs adbe_gem_funcs = { ++ .free = drm_gem_dma_object_free, ++ .print_info = drm_gem_dma_object_print_info, ++ .get_sg_table = drm_gem_dma_object_get_sg_table, ++ .vmap = drm_gem_dma_object_vmap, ++ .mmap = drm_gem_dma_object_mmap, ++ .vm_ops = &drm_gem_dma_vm_ops, ++ .is_cpu_uncached = drm_gem_dma_object_is_cpu_uncached, ++}; ++ ++static struct drm_gem_object *adbe_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct drm_gem_dma_object *dma_obj = kzalloc_obj(*dma_obj); ++ ++ if (!dma_obj) ++ return ERR_PTR(-ENOMEM); ++ dma_obj->base.funcs = &adbe_gem_funcs; ++ return &dma_obj->base; ++} ++ + DEFINE_DRM_GEM_DMA_FOPS(adbe_fops); + + static const struct drm_driver adbe_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .fops = &adbe_fops, ++ .gem_create_object = adbe_gem_create_object, + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(adbe_dumb_create), + DRM_FBDEV_DMA_DRIVER_OPS, + .name = "adbe", +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index 9b44c78cd..70110173d 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -835,6 +835,25 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { + .vunmap = drm_gem_dmabuf_vunmap, + }; + ++/** ++ * drm_gem_dma_buf_is_cpu_uncached - query a trusted native GEM exporter ++ * @dma_buf: exported buffer whose supported CPU aliases are queried ++ * ++ * Returns true only for the explicit native exporter contract. Non-GEM, ++ * re-exported foreign objects and unspecified exporters remain conservative. ++ */ ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf) ++{ ++ struct drm_gem_object *obj; ++ ++ if (!dma_buf || dma_buf->ops != &drm_gem_prime_dmabuf_ops) ++ return false; ++ obj = dma_buf->priv; ++ return !drm_gem_is_imported(obj) && obj->funcs && ++ obj->funcs->is_cpu_uncached && obj->funcs->is_cpu_uncached(obj); ++} ++EXPORT_SYMBOL_GPL(drm_gem_dma_buf_is_cpu_uncached); ++ + /** + * drm_prime_pages_to_sg - converts a page array into an sg list + * @dev: DRM device +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 885244e37..048193377 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -73,6 +73,14 @@ enum drm_gem_object_status { + * struct drm_gem_object_funcs - GEM object functions + */ + struct drm_gem_object_funcs { ++ /** ++ * @is_cpu_uncached: Optional exporter contract. True only when every ++ * supported CPU mapping (kernel and userspace) is uncached or WC and ++ * no cacheable alias exists. Native allocation properties must be ++ * immutable; imported memory must not inherit this promise. Default ++ * NULL is conservative. This does not remove DMA ordering barriers. ++ */ ++ bool (*is_cpu_uncached)(struct drm_gem_object *obj); + /** + * @free: + * +diff --git a/include/drm/drm_gem_dma_helper.h b/include/drm/drm_gem_dma_helper.h +index f2678e7ec..481007c13 100644 +--- a/include/drm/drm_gem_dma_helper.h ++++ b/include/drm/drm_gem_dma_helper.h +@@ -4,6 +4,7 @@ + + #include + #include ++#include + #include + + struct drm_mode_create_dumb; +@@ -55,6 +56,13 @@ extern const struct vm_operations_struct drm_gem_dma_vm_ops; + * This function wraps drm_gem_dma_free_object(). Drivers that employ the DMA helpers + * should use it as their &drm_gem_object_funcs.free handler. + */ ++static inline bool drm_gem_dma_object_is_cpu_uncached(struct drm_gem_object *obj) ++{ ++ return !drm_gem_is_imported(obj) && ++ !to_drm_gem_dma_obj(obj)->map_noncoherent && ++ !dev_is_dma_coherent(drm_dev_dma_dev(obj->dev)); ++} ++ + static inline void drm_gem_dma_object_free(struct drm_gem_object *obj) + { + struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj); +diff --git a/include/drm/drm_prime.h b/include/drm/drm_prime.h +index f50f862f0..5de957f7d 100644 +--- a/include/drm/drm_prime.h ++++ b/include/drm/drm_prime.h +@@ -116,4 +116,6 @@ int drm_prime_sg_to_page_array(struct sg_table *sgt, struct page **pages, + int drm_prime_sg_to_dma_addr_array(struct sg_table *sgt, dma_addr_t *addrs, + int max_pages); + ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf); ++ + #endif /* __DRM_PRIME_H__ */ diff --git a/src/orchard/apple/agx_g5p.py b/src/orchard/apple/agx_g5p.py new file mode 100644 index 00000000..c46ca6a7 --- /dev/null +++ b/src/orchard/apple/agx_g5p.py @@ -0,0 +1,19 @@ +"""Extract the unmodified G5P C0 META LDR from the user's prelinked kernelcache.""" +import hashlib +from orchard.apple.prelink import Kernelcache, named + +BUNDLE = 'com.apple.AGXFirmwareKextG5P' +CONST = ('__TEXT', '__const') +SIZE = 119112 +SHA256 = '33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca' + + +def extract(blob): + kernel = Kernelcache(blob) + section = named(kernel.kext_sections(BUNDLE, 'G5P firmware'), CONST)[CONST] + if section['size'] != SIZE: + raise ValueError('Unsupported G5P C0 firmware size') + data = kernel.read(section['addr'], section['size']) + if hashlib.sha256(data).hexdigest() != SHA256: + raise ValueError('Unsupported G5P C0 firmware digest') + return data diff --git a/src/orchard/recipe.py b/src/orchard/recipe.py index 74b73102..976c6666 100644 --- a/src/orchard/recipe.py +++ b/src/orchard/recipe.py @@ -8,7 +8,7 @@ import json import plistlib -from orchard.apple import agx, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet +from orchard.apple import agx, agx_g5p, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet from orchard.apple.osserialize import apple_xml, binary @@ -43,6 +43,10 @@ def _vxd(step, data, expected): return d5500.extract(data)[0] +def _agx_g5p(step, data, expected): + return agx_g5p.extract(data) + + def _agx(step, data, expected): return agx.extract(data) @@ -100,6 +104,7 @@ def _z2fw_mtfw(step, data, expected): # step@version -> (implementation, {parameter besides 'step' and 'input': type}) STEPS = { + 'agx-g5p-ldr-from-kernelcache@1': (_agx_g5p, {}), 'copy@1': (_copy, {}), 'trx-add-nvram@1': (_trx, {'nvram': str, 'placeholder_mac': str}), 'sgx543-from-kernelcache@1': (_sgx, {'build': str}), diff --git a/tests/test_agx_g5p.py b/tests/test_agx_g5p.py new file mode 100644 index 00000000..f048ef01 --- /dev/null +++ b/tests/test_agx_g5p.py @@ -0,0 +1,29 @@ +import hashlib +import unittest +from unittest.mock import Mock, patch +from orchard.apple import agx_g5p + +class G5PFirmware(unittest.TestCase): + def test_original_bytes_and_bundle(self): + data = b'synthetic firmware, not Apple data' + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=len(data))] + kernel.read.return_value = data + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), patch.object(agx_g5p, 'SIZE', len(data)), patch.object(agx_g5p, 'SHA256', hashlib.sha256(data).hexdigest()): + self.assertIs(agx_g5p.extract(b'input'), data) + kernel.kext_sections.assert_called_once_with(agx_g5p.BUNDLE, 'G5P firmware') + kernel.read.assert_called_once_with(4096, len(data)) + + def test_wrong_size_is_rejected_before_read(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=1)] + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'size'): + agx_g5p.extract(b'input') + kernel.read.assert_not_called() + + def test_wrong_digest_is_rejected(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=agx_g5p.SIZE)] + kernel.read.return_value = bytes(agx_g5p.SIZE) + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'digest'): + agx_g5p.extract(b'input') From 2c93befe5600a4ded08d7cd7eb96716d1a72db56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 05:32:56 +0100 Subject: [PATCH 27/33] m1n1: Pass the A9 AOP's boot arguments and sensor calibration The A9 AOP driver on main reads apple,aop-target, apple,chip-id and apple,chip-revision, and the motion sensor drivers read their factory calibration, from properties the bootloader fills in. Nothing on main fills them in; going by the driver's probe, the A9 AOP cannot start without them (read from the code, not run). This is the m1n1 patch that fills them in, from #101, unchanged. Assisted-by: LLM --- ...9-AOP-s-boot-arguments-and-sensor-ca.patch | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch diff --git a/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch new file mode 100644 index 00000000..6a30fc37 --- /dev/null +++ b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 2 Nov 2026 15:07:08 +0100 +Subject: [PATCH] kboot: Pass the A9 AOP's boot arguments and sensor + calibration + +The A9's AOP firmware takes more than the ECID from its loader: the AOP +target number (iop-aop-nub aop-target), the chip's ID (/chosen chip-id) +and revision (/arm-io chip-revision). Its accelerometer, gyroscope and +pressure sensor also send no reports until the driver has given them the +unit's factory calibration, which iBoot resolves from syscfg into the +sensors' nodes below iop-aop-nub. + +Copy the three numbers into the AOP's node (alias aop) as +apple,aop-target, apple,chip-id and apple,chip-revision, and each +sensor's calibration properties into the AOP's child node of the same +name, prefixed with "apple,". Unprovisioned "syscfg/" references and +nodes the device tree doesn't have are skipped. The orientation +properties are not copied: the A9 firmware rejects them and the board's +mount-matrix describes the mounting. + +The values are the unit's own and never leave the device: the kernel +sends them to the AOP and nothing else. + +Assisted-by: LLM +--- + src/kboot.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 68 insertions(+), 1 deletion(-) + +diff --git a/src/kboot.c b/src/kboot.c +index ebd6282..7ce9d72 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -1005,6 +1005,73 @@ static int dt_set_als(void) + return 0; + } + ++/* ++ * The A9's AOP firmware also takes the AOP target number and the chip's ID and ++ * revision in its boot arguments, and each motion sensor's factory ++ * calibration, which iBoot resolves from syscfg into the sensor's node below ++ * iop-aop-nub. Copy them into the AOP's node and its sensor child nodes of the ++ * same name; the calibration keeps its name with an "apple," prefix. ++ */ ++static const char *const aop_sensor_calibrations[] = { ++ "low-temp-accel-offset", ++ "accel-sensitivity-calibration", ++ "accel-range-sensitivity-cal", ++ "accel-interrupt-calibration", ++ "gyro-sensitivity-calibration", ++ "gyro-range-sensitivity-cal", ++ "gyro-interrupt-calibration", ++ "gyro-trim-calibration", ++ "pressure-offset-calibration", ++ "pressure-accel-sensitivity", ++ "pressure-heater-resistance", ++}; ++ ++static int dt_set_aop_sensors(int node) ++{ ++ int nub = adt_path_offset(adt, "/arm-io/aop/iop-aop-nub"); ++ if (nub < 0) ++ return 0; ++ ++ u32 val; ++ if (ADT_GETPROP(adt, nub, "aop-target", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,aop-target", val)) ++ bail("FDT: failed to set apple,aop-target\n"); ++ ++ int chosen = adt_path_offset(adt, "/chosen"); ++ if (chosen >= 0 && ADT_GETPROP(adt, chosen, "chip-id", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-id", val)) ++ bail("FDT: failed to set apple,chip-id\n"); ++ ++ int arm_io = adt_path_offset(adt, "/arm-io"); ++ if (arm_io >= 0 && ADT_GETPROP(adt, arm_io, "chip-revision", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-revision", val)) ++ bail("FDT: failed to set apple,chip-revision\n"); ++ ++ int sensor = nub; ++ ADT_FOREACH_CHILD(adt, sensor) ++ { ++ const char *name = adt_get_name(adt, sensor); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(aop_sensor_calibrations); i++) { ++ u32 len; ++ const u8 *blob = adt_getprop(adt, sensor, aop_sensor_calibrations[i], &len); ++ if (!blob || !len || (len >= 7 && !memcmp(blob, "syscfg/", 7))) ++ continue; ++ ++ /* Earlier properties may have moved the sensor's node. */ ++ int child = fdt_subnode_offset(dt, node, name); ++ if (child < 0) ++ break; ++ ++ char prop[64]; ++ snprintf(prop, sizeof(prop), "apple,%s", aop_sensor_calibrations[i]); ++ if (fdt_setprop(dt, child, prop, blob, len)) ++ bail("FDT: failed to set %s/%s\n", name, prop); ++ } ++ } ++ return 0; ++} ++ + /* + * The AOP's firmware carries the chip's ECID in its boot arguments. Pass it + * only to the AOP's node; the driver removes it from the tree once read. +@@ -1028,7 +1095,7 @@ static int dt_set_aop(void) + + if (fdt_setprop_u64(dt, node, "apple,ecid", ecid)) + bail("FDT: failed to set apple,ecid\n"); +- return 0; ++ return dt_set_aop_sensors(node); + } + + static int dt_set_multitouch(void) From 011ab650e4d6bde4abcea24c091bf2cf72a6439a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 05:33:17 +0100 Subject: [PATCH 28/33] kernel: Keep the AOP's firmware across kexec and reload it From #101, the part that main did not take with the sensors: the retained AOP reload across kexec (checksummed shutdown log, power-state observer, pr_flush export, warm_state attribute). The sensor parts that main carries are left out of these patches; what remains applies on top of main's AOP driver and its fresh-start order. Assisted-by: LLM --- ...select-certified-AOP-reload-through-.patch | 49 + ...lect-certified-AOP-reload-through-DT.patch | 26 + ...elect-certified-AOP-reload-through-D.patch | 127 + ...apple-select-certified-AOP-reload-th.patch | 52 + ...lect-certified-AOP-reload-through-DT.patch | 2777 +++++++++++++++++ 5 files changed, 3031 insertions(+) create mode 100644 kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch create mode 100644 kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch create mode 100644 kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch create mode 100644 kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch create mode 100644 kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch diff --git a/kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch b/kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch new file mode 100644 index 00000000..46d27545 --- /dev/null +++ b/kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch @@ -0,0 +1,49 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. The AOP itself, its firmware and +mailbox are enabled for every iPad 5 in the shared file; J71t adds the +reserved shutdown page and the power domains the reload observes. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 21 ++++++++++++++++++++ + 1 file changed, 21 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -16,6 +16,27 @@ + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; + }; + ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* A fixed tail page, never a firmware DMA grant. */ ++ aop_shutdown: aop-shutdown@8583fc000 { ++ reg = <0x8 0x583fc000 0 0x4000>; ++ no-map; ++ }; ++ }; ++}; ++ ++&aop { ++ apple,certified-reload; ++ memory-region = <&aop_shutdown>; ++ apple,observe-domains = <&ps_aop_gpio &ps_aop &ps_aop_busif ++ &ps_aop_cpu &ps_aop_filter>; ++ apple,spi0-domain = <&ps_spi0>; ++}; ++ + &touchscreen { + firmware-name = "apple/j71t-fwstart.z2fw"; + }; +-- +2.54.0 diff --git a/kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..6fd6514b --- /dev/null +++ b/kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index 3fcdf4b4e..fcc51b699 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4589,6 +4589,7 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) + { + return __pr_flush(NULL, timeout_ms, reset_on_progress); + } ++EXPORT_SYMBOL_GPL(pr_flush); + + /* + * Delayed printk version, for scheduler-internal messages: diff --git a/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch new file mode 100644 index 00000000..d9a8480b --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch @@ -0,0 +1,127 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 84 ++++++++++++++++++++++++++ + 1 file changed, 84 insertions(+) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..0ae0eda2d 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -417,6 +419,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + goto err_remove; + } + ++ platform_set_drvdata(pdev, ps); + return 0; + err_remove: + if (ps->has_genpd) { +@@ -446,3 +449,84 @@ MODULE_AUTHOR("Hector Martin "); + MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); + + module_platform_driver(apple_pmgr_ps_driver); ++ ++/* ++ * Read-only observer for bring-up. The caller's nodes follow the same ++ * child -> parent order as genpd power transitions. No domain is attached, ++ * resumed, pinned on, reset, or written by this function. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data) ++{ ++ struct platform_device *providers[5]; ++ struct apple_pmgr_ps *ps[5]; ++ unsigned long flags; ++ unsigned int i, j, found = 0; ++ int ret = 0; ++ bool accessible = true; ++ ++ if (!count || count > ARRAY_SIZE(providers) || !read) ++ return -EINVAL; ++ for (i = 0; i < count; i++) { ++ for (j = 0; j < i; j++) ++ if (nodes[i] == nodes[j]) { ++ ret = -EINVAL; ++ goto put; ++ } ++ providers[i] = of_find_device_by_node(nodes[i]); ++ if (!providers[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ device_lock(&providers[i]->dev); ++ found++; ++ if (!providers[i]->dev.driver || ++ strcmp(providers[i]->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ ps[i] = platform_get_drvdata(providers[i]); ++ if (!ps[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ } ++ ++ /* PMGR domains use GENPD_FLAG_IRQ_SAFE, i.e. genpd.slock. */ ++ local_irq_save(flags); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->genpd.slock, i); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->lock, i); ++ for (i = 0; i < count; i++) { ++ states[i].genpd_status = ps[i]->genpd.status; ++ ret = regmap_read(ps[i]->regmap, ps[i]->offset, &states[i].raw); ++ if (ret) ++ goto unlock; ++ if (states[i].genpd_status != GENPD_STATE_ON || ++ FIELD_GET(APPLE_PMGR_PS_ACTUAL, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ FIELD_GET(APPLE_PMGR_PS_TARGET, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ states[i].raw & (APPLE_PMGR_RESET | APPLE_PMGR_DEV_DISABLE | ++ APPLE_PMGR_PARENT_OFF)) ++ accessible = false; ++ } ++ if (accessible) ++ read(data); ++ else ++ ret = -EHOSTDOWN; ++unlock: ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->lock); ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->genpd.slock); ++ local_irq_restore(flags); ++put: ++ while (found) { ++ found--; ++ device_unlock(&providers[found]->dev); ++ put_device(&providers[found]->dev); ++ } ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_observe); diff --git a/kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch b/kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch new file mode 100644 index 00000000..7b738fe7 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch @@ -0,0 +1,52 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: soc-apple: select certified AOP reload through + DT + +A retained AOP must not be reset as though it were a fresh boot. +Describe the retained-reload properties: the certified-reload flag, the +reserved shutdown page and the power domains the reload observes. + +Assisted-by: LLM +--- +--- + .../bindings/soc/apple/apple,t8010-aop.yaml | 18 ++++++++++++++++++ + 1 file changed, 18 insertions(+) + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +index 32788d1c7..74586c6e0 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +@@ -18,6 +18,10 @@ description: + board-specific and is loaded by the operating system. + + properties: ++ apple,certified-reload: ++ type: boolean ++ description: Firmware supports the retained shutdown certificate for warm reload. ++ + compatible: + enum: + - apple,s8000-aop +@@ -70,6 +74,20 @@ properties: + The SoC's chip revision, filled in by the bootloader. The S8000 + firmware's boot arguments carry it. + ++ memory-region: ++ maxItems: 1 ++ description: Reserved 16 KiB shutdown transcript page for J71t warm reload. ++ ++ apple,observe-domains: ++ $ref: /schemas/types.yaml#/definitions/phandle-array ++ minItems: 5 ++ maxItems: 5 ++ description: AOP GPIO, AOP, bus interface, CPU and filter in lock order. ++ ++ apple,spi0-domain: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: Read-only SPI0 power-state provider observation. ++ + patternProperties: + "^(accel|gyro|compass|pressure)$": + type: object diff --git a/kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..a73acf3a --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,2777 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] soc: apple: aop: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/soc/apple/aop-warm-record.h | 63 ++ + drivers/soc/apple/aop-warm-spi0.c.h | 1339 ++++++++++++++++++++++++ + drivers/soc/apple/aop-warm-spi0.h | 543 ++++++++++ + drivers/soc/apple/aop.c | 528 +++++++++- + drivers/soc/apple/rtkit.c | 50 + + include/linux/soc/apple/pmgr-observe.h | 27 + + include/linux/soc/apple/rtkit.h | 2 + + 7 files changed, 2543 insertions(+), 9 deletions(-) + create mode 100644 drivers/soc/apple/aop-warm-record.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.c.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.h + create mode 100644 include/linux/soc/apple/pmgr-observe.h + +diff --git a/drivers/soc/apple/aop-warm-record.h b/drivers/soc/apple/aop-warm-record.h +new file mode 100644 +index 000000000..237d78d17 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-record.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#include ++#include ++#define A9_HO_PAGE 0x4000U ++#define A9_HO_LOG_OFFSET 0x800U ++#define A9_SL_MAGIC 0x4c533941U /* "A9SL" */ ++#define A9_SL_VERSION 1U ++ ++enum a9_ho_step { ++ A9_HO_STEP_BEGIN, ++ A9_HO_STEP_RATES, ++ A9_HO_STEP_AFK_SENT, ++ A9_HO_STEP_AFK_ACKED, ++ A9_HO_STEP_DISABLED, ++ A9_HO_STEP_AP_SENT, ++ A9_HO_STEP_AP_ACKED, ++ A9_HO_STEP_IOP_SENT, ++ A9_HO_STEP_IOP_ACKED, ++ A9_HO_STEP_RAM_STABLE, ++ A9_HO_STEP_MAILBOX_IDLE, ++ A9_HO_STEP_HALTED, ++ A9_HO_STEP_DRAINED, ++ A9_HO_STEPS, ++}; ++ ++static const char * const a9_ho_step_names[A9_HO_STEPS] = { ++ "begin", "rates-zero", "afk-sent", "afk-acked", "endpoint-disable", ++ "ap-sent", "ap-ack", "iop-sent", "iop-ack", "ram-stable", "mailbox-idle", ++ "halt", "drain", ++}; ++ ++struct a9_sl_header { ++ u32 magic, version, capacity, length; ++ u32 lines, truncated, checksum, mode; ++ u64 begin_ns; ++ u8 writer[16]; ++ char entry[64]; ++}; ++ ++_Static_assert(sizeof(struct a9_sl_header) == 120, "shutdown log ABI"); ++ ++#define A9_SL_CAPACITY (A9_HO_PAGE - A9_HO_LOG_OFFSET - sizeof(struct a9_sl_header)) ++ ++/* FNV-1a; incremental over appended text. */ ++static inline u32 a9_sl_hash(u32 hash, const char *text, size_t length) ++{ ++ size_t i; ++ ++ for (i = 0; i < length; i++) ++ hash = (hash ^ (u8)text[i]) * 16777619U; ++ return hash; ++} ++ ++#define A9_SL_HASH_SEED 2166136261U ++ ++/* A replayable previous transcript: complete header, bounded, intact. */ ++static inline bool a9_sl_valid(const struct a9_sl_header *h, const char *text) ++{ ++ return h->magic == A9_SL_MAGIC && h->version == A9_SL_VERSION && ++ h->capacity == A9_SL_CAPACITY && h->length <= h->capacity && ++ !h->entry[sizeof(h->entry) - 1] && ++ a9_sl_hash(A9_SL_HASH_SEED, text, h->length) == h->checksum; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.c.h b/drivers/soc/apple/aop-warm-spi0.c.h +new file mode 100644 +index 000000000..e9f608abb +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.c.h +@@ -0,0 +1,1339 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Ported from a9/aop-release: wake16, release6-d254a9f790. */ ++static void a9_aop_spi0_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_REGS; i++) ++ values[i] = readl(base + a9_spi0_offsets[i]); ++} ++ ++/* ++ * AOPSPIMaster_reset (0102f2f4) under the AOP-domain PMGR guard with IRQs ++ * off and the AOP CPU halted. Its provider power transition 3 is an ++ * AOP-local call with no AP register equivalent and is not replayed. A poll ++ * timeout still restores the saved control word. ++ */ ++static void a9_aop_spi0_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, s->before); ++ s->saved = s->before[A9_SPI0_I_CONTROL]; ++ s->asserted = a9_spi0_base_word(s->saved) | A9_SPI0_RESET; ++ writel(s->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ s->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(s->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(s->saved, control); ++ s->reset_done_ns = ktime_get_ns(); ++ a9_aop_spi0_snapshot(s->base, s->after); ++ if (!ret && (s->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ s->error = ret; ++} ++ ++/* ++ * Wake8 device evidence: SCM was 0 before/after the first controller reset, ++ * then became 3 at the channel-3 stop store. There is no firmware-derived ++ * SCM status write. Replay AOPSPIMaster_reset after the teardown so the ++ * controller reset, rather than an invented SCM write, is the one variable. ++ */ ++static void a9_aop_spi0_post_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, r->before); ++ r->saved = r->before[A9_SPI0_I_CONTROL]; ++ r->asserted = a9_spi0_base_word(r->saved) | A9_SPI0_RESET; ++ writel(r->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ r->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(r->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(r->saved, control); ++ a9_aop_spi0_snapshot(s->base, r->after); ++ if (!ret && (r->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ r->error = ret; ++} ++ ++static void a9_aop_spi0_ch3_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_CH3_REGS; i++) ++ values[i] = readl(base + a9_spi0_ch3_offsets[i]); ++} ++ ++static void a9_aop_spi0_trace_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int channel, i; ++ ++ s->trace.control = readl(s->base + A9_SPI0_CONTROL); ++ s->trace.clock = readl(s->base + A9_SPI0_CLOCK); ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->trace.scm[i] = readl(s->base + a9_spi0_scm_offsets[i]); ++ a9_aop_spi0_ch3_snapshot(s->base, s->trace.ch3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) ++ for (i = 0; i < A9_SPI0_CHANNEL_REGS; i++) ++ s->trace.channel[channel][i] = ++ readl(s->base + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel + ++ a9_spi0_channel_offsets[i]); ++} ++ ++/* Warm path only, CPU halted. Log outside the PMGR guard; no SCM writes. */ ++static int a9_aop_spi0_trace(struct apple_aop *aop, const char *stage) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *c = s->trace.ch3, *m = s->trace.scm; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_trace_read, s); ++ if (ret) { ++ s->intc.failed = "failed-spi0-trace-guard"; ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=%d: guard refused\n", stage, ret); ++ return ret; ++ } ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=0 ctl=%08x clk=%08x scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ stage, s->trace.control, s->trace.clock, m[0], m[1], m[2], m[3], ++ c[0], c[1], c[2], c[3], c[4], c[5], c[6]); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ v = s->trace.channel[channel]; ++ dev_info(aop->dev, "SPI0 SCM channel stage=%s ch=%02u ctl=%08x route=%08x rate=%08x cfg=%08x interval=%08x cursor=%08x desc=%08x pending0=%u pending1=%u\n", ++ stage, channel, v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ !!(c[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(c[A9_CH3_I_8D4] & BIT(2 * channel))); ++ } ++ return 0; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s); ++ ++/* ++ * Wake10 safety gate, strengthened in wake11 to retain every selected-channel ++ * teardown invariant. A fault observed by a copy sample remains terminal even ++ * if it is transient and the final read happens to clear. ++ */ ++static int a9_aop_spi0_gate(struct apple_aop *aop, const char *stage, ++ const char *check, bool include_copy) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *m, *c; ++ int ret; ++ ++ ret = a9_aop_spi0_trace(aop, stage); ++ if (ret) ++ return ret; ++ m = s->trace.scm; ++ c = s->trace.ch3; ++ if (m[0] || m[1]) ++ s->gate_failed = "failed-spi0-scm"; ++ else if (c[A9_CH3_I_8D0] || c[A9_CH3_I_8D4]) ++ s->gate_failed = "failed-spi0-fault"; ++ else if (include_copy && s->copy_faulted) ++ s->gate_failed = "failed-spi0-copy-sample"; ++ else { ++ s->gate_failed = a9_spi0_ch3_verify(c, s->teardown_stable & BIT(3)); ++ if (!s->gate_failed) ++ s->gate_failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = s->gate_failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 %s check errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x copy_faulted=%u result=%s\n", ++ check, ret, m[0], m[1], c[A9_CH3_I_8D0], c[A9_CH3_I_8D4], ++ s->copy_faulted, s->gate_failed ?: "pass"); ++ return ret; ++} ++ ++#define A9_SPI0_COPY_CHUNK 256U ++#define A9_SPI0_COPY_SAMPLE_US 5000U ++ ++struct a9_aop_spi0_copy_probe { ++ void __iomem *base; ++ u32 scm_lo, scm_hi, fault, fault_8d4; ++}; ++ ++static void a9_aop_spi0_copy_probe_read(void *data) ++{ ++ struct a9_aop_spi0_copy_probe *p = data; ++ ++ p->scm_lo = readl(p->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(p->base + A9_SPI0_SCM_HI); ++ p->fault = readl(p->base + A9_SPI0_FAULT); ++ p->fault_8d4 = readl(p->base + A9_SPI0_FAULT_8D4); ++} ++ ++/* ++ * Read-only samples spaced by at least 5 ms across both halves of the SRAM ++ * copy. Once a nonzero state is seen it remains a terminal fact, but the copy ++ * completes so later samples and the full after-copy channel snapshot expose ++ * whether the source is level, transient or moves between channels. ++ */ ++static int a9_aop_spi0_copy_sample(struct apple_aop *aop, const char *phase, ++ size_t offset, size_t size, u64 start_ns, ++ u64 *last_us, unsigned int *sample, ++ bool force) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_copy_probe p = { .base = s->base }; ++ u64 elapsed_us = div_u64(ktime_get_boottime_ns() - start_ns, ++ NSEC_PER_USEC); ++ bool first_fault, initial = !*sample; ++ int ret; ++ ++ if (!force && *sample && elapsed_us - *last_us < A9_SPI0_COPY_SAMPLE_US) ++ return 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_copy_probe_read, &p); ++ first_fault = !s->copy_faulted && ++ (p.scm_lo || p.scm_hi || p.fault || p.fault_8d4); ++ if (!ret && first_fault) ++ s->copy_faulted = true; ++ dev_info(aop->dev, "SPI0 COPY sample=%u phase=%s elapsed_us=%llu offset=0x%zx size=0x%zx scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ *sample, phase, elapsed_us, offset, size, p.scm_lo, p.scm_hi, ++ p.fault, p.fault_8d4, ret ? "guard" : s->copy_faulted ? "fault" : "pass"); ++ *last_us = elapsed_us; ++ (*sample)++; ++ if (ret) ++ return ret; ++ if (first_fault) { ++ ret = a9_aop_spi0_trace(aop, "copy-fault"); ++ if (ret) ++ return ret; ++ } ++ /* The offset-zero sample is the last no-write gate before the copy. */ ++ return !initial || !s->copy_faulted ? 0 : -EIO; ++} ++ ++static int a9_aop_copy_verify_wake10(struct apple_aop *aop, const u8 *src, ++ size_t size) ++{ ++ u32 chunk[A9_SPI0_COPY_CHUNK / 4]; ++ u64 start_ns = ktime_get_boottime_ns(), last_us = 0; ++ unsigned int sample = 0; ++ size_t done, n; ++ int ret; ++ ++ ret = a9_aop_spi0_copy_sample(aop, "write", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __iowrite32_copy(aop->sram + done, src + done, n / 4); ++ ret = a9_aop_spi0_copy_sample(aop, "write", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_copy_sample(aop, "verify", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __ioread32_copy(chunk, aop->sram + done, n / 4); ++ if (memcmp(chunk, src + done, n)) ++ return -EIO; ++ ret = a9_aop_spi0_copy_sample(aop, "verify", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void __iomem *a9_aop_spi0_channel_base(struct a9_aop_spi0 *s, ++ unsigned int channel) ++{ ++ return s->base + A9_SPI0_CHANNEL_BASE + A9_SPI0_CHANNEL_STRIDE * channel; ++} ++ ++/* ++ * Wake15: which other channels were RUN with a counting interval when this ++ * stop was issued, and which of them advanced across the stop. Read only; it ++ * separates a stop that had a ticking peer from the stop of the last live ++ * channel in the log. It never gates anything in-kernel. ++ */ ++static void a9_aop_spi0_peers_begin(struct a9_aop_spi0 *s, unsigned int channel, ++ u32 *interval, u32 *run) ++{ ++ unsigned int i; ++ u32 ctl; ++ ++ *run = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ interval[i] = 0; ++ if (i == channel) ++ continue; ++ ctl = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ interval[i] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ if ((ctl & A9_SPI0_CH3_RUN) && interval[i]) ++ *run |= BIT(i); ++ } ++} ++ ++static void a9_aop_spi0_stop_timing(struct a9_aop_spi0 *s, ++ struct a9_aop_spi0_ch3 *c, ++ const u32 *interval) ++{ ++ unsigned int i; ++ ++ c->peers_ticking = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ if ((c->peers_run & BIT(i)) && ++ readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]) != ++ interval[i]) ++ c->peers_ticking |= BIT(i); ++ } ++ c->reset_to_stop_us = c->stop_ns > s->reset_done_ns ? ++ div_u64(c->stop_ns - s->reset_done_ns, NSEC_PER_USEC) : 0; ++} ++ ++/* ++ * Wake12/13 captured channel 7, and wake13 channel 3, stopped but never idle: ++ * for 20 complete slices the control/progress word, config, interval, cursor, ++ * SCM and both fault banks were unchanged. This predicate is intentionally ++ * narrower than "RUN clear and no progress": the channel must be one of the ++ * two device-observed periodic clients, must carry that client's route, rate ++ * and config before the stop, and every sample must be byte-identical with SCM ++ * 3/0 (every earlier non-idle stop, which followed a channel-3 stop that had ++ * latched it) or 0/0 (a first stop that latches nothing), no fault, no cursor ++ * movement and a retained nonzero interval. ++ * It only permits the callback masks, descriptor clear, config acknowledge and ++ * the established controller reset; every later SCM, fault and copy gate stays. ++ */ ++static const char *a9_aop_spi0_stable_why(struct a9_aop_spi0_ch3 *c, ++ const struct a9_aop_channel_diag_slice *diag, ++ unsigned int channel) ++{ ++ const struct a9_aop_ch3_slice *first; ++ unsigned int i; ++ u32 route; ++ ++ if (channel == A9_SPI0_PHOSPHORUS_CHANNEL) ++ route = A9_SPI0_PHOSPHORUS_ROUTE; ++ else if (channel == A9_SPI0_MPU_ASYNC_CHANNEL) ++ route = A9_SPI0_MPU_ASYNC_ROUTE; ++ else ++ return "channel"; ++ if (c->poll != -ETIMEDOUT || c->slices != A9_SPI0_CH3_SLICES) ++ return "no-timeout"; ++ if (!(c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) || ++ c->before[A9_CH3_I_ROUTE] != route || ++ c->before[A9_CH3_I_RATE] != A9_SPI0_PHOSPHORUS_RATE || ++ c->before[A9_CH3_I_C3C] != A9_SPI0_PHOSPHORUS_CFG || ++ c->before[A9_CH3_I_DESC] || (c->stop & A9_SPI0_CH3_RUN)) ++ return "before-identity"; ++ if (c->before[A9_CH3_I_8D4] || (c->before[A9_CH3_I_8D0] & ~BIT(2 * channel))) ++ return "before-fault"; ++ ++ first = &c->slice[0]; ++ if ((first->c3 & A9_SPI0_STABLE_CTL_MASK) != A9_SPI0_STABLE_CTL_STOPPED || ++ (first->c3 & (A9_SPI0_CH3_RUN | A9_SPI0_CH3_IDLE | A9_SPI0_CH3_IRQ1))) ++ return "stopped-control"; ++ if ((first->scm_lo != 0 && first->scm_lo != 3) || first->scm_hi) ++ return "scm"; ++ if (first->fault || diag[0].fault_8d4) ++ return "sample-fault"; ++ if (diag[0].cfg != A9_SPI0_PHOSPHORUS_CFG || diag[0].cursor || !diag[0].interval) ++ return "cfg-cursor-interval"; ++ ++ for (i = 1; i < A9_SPI0_CH3_SLICES; i++) { ++ const struct a9_aop_ch3_slice *p = &c->slice[i]; ++ ++ if (p->c3 != first->c3 || p->scm_lo != first->scm_lo || ++ p->scm_hi != first->scm_hi || p->fault != first->fault || ++ diag[i].cfg != diag[0].cfg || ++ diag[i].interval != diag[0].interval || ++ diag[i].cursor != diag[0].cursor || ++ diag[i].fault_8d4 != diag[0].fault_8d4) ++ return "progress"; ++ } ++ if (c->slice[A9_SPI0_CH3_SLICES - 1].ns < ++ (u64)A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US * NSEC_PER_USEC) ++ return "short"; ++ return NULL; ++} ++ ++/* ++ * AOPSPIMaster_teardownChannel (0102e4f8) for channel 3, then the fault ++ * handler's acknowledge (0102f20a), under the same guard as the reset with ++ * the CPU halted. Every store is a read-modify-write except the descriptor, ++ * which the firmware zeroes. The idle wait runs in 1 ms slices, up to twice ++ * the firmware's own 10 ms SCMto bound; each slice ends with read-only ++ * samples of C3, the SCM state and +0x8d0 (wake15: and config, interval, ++ * cursor and +0x8d4). An idle timeout stops before the masks unless the ++ * stable-stop predicate holds. The acknowledge bit self-clears; +0x8d0 bit 6 ++ * shows whether it took effect. ++ */ ++static void a9_aop_spi0_ch3_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ void __iomem *c3 = s->base + A9_SPI0_CH3; ++ void __iomem *c3c = s->base + A9_SPI0_CH3_0C; ++ void __iomem *base = a9_aop_spi0_channel_base(s, 3); ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_ch3_snapshot(s->base, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(c3) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, c3); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, 3, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(c3, value, value & A9_SPI0_CH3_IDLE, ++ 1, A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->ch3_diag[c->slices]; ++ d->cfg = readl(c3c); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->ch3_diag, 3); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(3); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(c3) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, c3); ++ c->class1 = readl(c3) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, c3); ++ writel(0, s->base + A9_SPI0_CH3_DESC); ++ c->ack = readl(c3c) | A9_SPI0_CH3_ACK; ++ writel(c->ack, c3c); ++ } ++ a9_aop_spi0_ch3_snapshot(s->base, c->after); ++ c->failed = c->poll && !c->stable ? "failed-ch3-teardown-idle" : ++ a9_spi0_ch3_verify(c->after, c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++ ++static void a9_aop_spi0_channel_snapshot(struct a9_aop_spi0 *s, ++ unsigned int channel, u32 *values) ++{ ++ void __iomem *base = a9_aop_spi0_channel_base(s, channel); ++ ++ values[A9_CH3_I_C3] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ values[A9_CH3_I_ROUTE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_ROUTE]); ++ values[A9_CH3_I_RATE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_RATE]); ++ values[A9_CH3_I_C3C] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]); ++ values[A9_CH3_I_DESC] = readl(base + A9_SPI0_CHANNEL_DESC); ++ values[A9_CH3_I_8D0] = readl(s->base + A9_SPI0_FAULT); ++ values[A9_CH3_I_8D4] = readl(s->base + A9_SPI0_FAULT_8D4); ++} ++ ++static const char *a9_spi0_channel_state_verify(u32 control, u32 desc, ++ u32 fault, u32 fault_8d4, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ u32 pending = BIT(2 * channel); ++ ++ if (control & A9_SPI0_CH3_RUN) ++ return "failed-channel-teardown-run"; ++ if (control & A9_SPI0_CH3_IRQ0) ++ return "failed-channel-teardown-class0"; ++ if (control & A9_SPI0_CH3_IRQ1) ++ return "failed-channel-teardown-class1"; ++ if (!(control & A9_SPI0_CH3_IDLE) && !stable_stop) ++ return "failed-channel-teardown-idle"; ++ if (desc) ++ return "failed-channel-teardown-desc"; ++ if ((fault | fault_8d4) & pending) ++ return "failed-channel-teardown-pending"; ++ return NULL; ++} ++ ++static const char *a9_spi0_channel_verify(const u32 *after, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ return a9_spi0_channel_state_verify(after[A9_CH3_I_C3], ++ after[A9_CH3_I_DESC], ++ after[A9_CH3_I_8D0], ++ after[A9_CH3_I_8D4], channel, ++ stable_stop); ++} ++ ++/* ++ * Wake11 applies the channel-3 firmware sequence unchanged to another slot. ++ * Wake12 only appends read-only scheduler samples to that established poll. ++ */ ++static void a9_aop_spi0_channel_locked(void *data) ++{ ++ struct a9_aop_spi0_channel_op *op = data; ++ struct a9_aop_spi0 *s = op->spi0; ++ struct a9_aop_spi0_ch3 *c = op->state; ++ void __iomem *base = a9_aop_spi0_channel_base(s, op->channel); ++ void __iomem *control = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]; ++ void __iomem *config = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]; ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(control) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, control); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, op->channel, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(control, value, ++ value & A9_SPI0_CH3_IDLE, 1, ++ A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->channel_diag[c->slices]; ++ d->cfg = readl(config); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->channel_diag, op->channel); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(op->channel); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(control) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, control); ++ c->class1 = readl(control) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, control); ++ writel(0, base + A9_SPI0_CHANNEL_DESC); ++ c->ack = readl(config) | A9_SPI0_CH3_ACK; ++ writel(c->ack, config); ++ } ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->after); ++ c->failed = c->poll && !c->stable ? "failed-channel-teardown-idle" : ++ a9_spi0_channel_verify(c->after, op->channel, ++ c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++/* Called only if ps_spi0 is fully on; reads nothing. */ ++static void a9_aop_spi0_pmgr_on(void *data) ++{ ++ *(bool *)data = true; ++} ++ ++/* AP PMGR ps_spi0, read-only: never requested, resumed or written. ++ * 0 = on, -EHOSTDOWN = off (raw still filled), -ENODEV = no node. ++ */ ++static int a9_aop_spi0_pmgr(struct apple_aop *aop, u32 *raw) ++{ ++ struct apple_pmgr_observation state = {}; ++ bool on = false; ++ int ret; ++ ++ if (!aop->spi0_pmgr) ++ return -ENODEV; ++ ret = apple_pmgr_observe(&aop->spi0_pmgr, 1, &state, ++ a9_aop_spi0_pmgr_on, &on); ++ *raw = state.raw; ++ return ret ?: on ? 0 : -EIO; ++} ++ ++/* ++ * After a reset that cleared +0x8d0 bit 6, before the image copy: CPU halted, ++ * AP notifications masked. A refusal sets ch3.failed and nothing is loaded. ++ */ ++static int a9_aop_spi0_ch3_teardown(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *b = c->before, *a = c->after; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ unsigned int i; ++ bool source_progress = false; ++ int ret; ++ ++ aop->experiment_step = "spi0-ch3-teardown"; ++ dev_info(aop->dev, "SPI0 CH3TD begin pa=%llx cpu=halted: first channel-3 store follows\n", ++ A9_SPI0_PA + A9_SPI0_CH3); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, "spi0-ch3-teardown"); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_ch3_locked, s); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH3TD errno=%d ran=0: guard refused, channel 3 not touched\n", ++ ret); ++ return ret; ++ } ++ c->ran = true; ++ dev_info(aop->dev, "SPI0 CH3TD before c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH3TD idle-sample slice=%u at_us=%llu c3=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ i, div_u64(p->ns, NSEC_PER_USEC), p->c3, p->scm_lo, ++ p->scm_hi, p->fault, !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->ch3_diag[i]; ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ i, p->c3, d->cfg, d->interval, d->cursor, p->scm_lo, ++ p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->ch3_diag[0].interval || ++ d->cursor != s->ch3_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->ch3_diag[0].interval, ++ s->ch3_diag[c->slices - 1].interval, ++ s->ch3_diag[0].cursor, ++ s->ch3_diag[c->slices - 1].cursor, source_progress, ++ c->stable ? "stable-stop-reset-only" : "refused"); ++ /* beyond_fw_bound=1: idle, but later than the firmware would wait. */ ++ dev_info(aop->dev, "SPI0 CH3TD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ c->stop, c->idle, c->poll); ++ else if (c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->poll, c->class0, c->class1, c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH3TD after c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH3TD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ /* c3c_bit5 is the pulse's readback, logged only (0 at 19:50). */ ++ dev_info(aop->dev, "SPI0 CH3TD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x c3c_bit5=%u ch3_pending=%u result=%s\n", ++ c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), ++ a[A9_CH3_I_DESC], !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), c->failed ?: ++ c->stable ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static int a9_aop_spi0_channel_teardown(struct apple_aop *aop, ++ unsigned int channel) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->channel; ++ struct a9_aop_spi0_channel_op op = { ++ .spi0 = s, ++ .state = c, ++ .channel = channel, ++ }; ++ const u32 *b, *a; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ char checkpoint[32]; ++ unsigned int i; ++ bool source_progress = false; ++ bool stable_stop; ++ int ret; ++ ++ memset(c, 0, sizeof(*c)); ++ memset(s->channel_diag, 0, sizeof(s->channel_diag)); ++ s->teardown_failed_channel = channel; ++ aop->experiment_step = "spi0-channel-teardown"; ++ snprintf(checkpoint, sizeof(checkpoint), "spi0-ch%u-teardown", channel); ++ dev_info(aop->dev, "SPI0 CH%uTD begin pa=%llx cpu=halted: first channel-%u store follows\n", ++ channel, A9_SPI0_PA + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel, channel); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, checkpoint); ++ if (ret) { ++ c->failed = "failed-channel-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_channel_locked, &op); ++ if (ret) { ++ c->failed = "failed-channel-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH%uTD errno=%d ran=0: guard refused, channel %u not touched\n", ++ channel, ret, channel); ++ return ret; ++ } ++ c->ran = true; ++ stable_stop = s->teardown_stable & BIT(channel); ++ b = c->before; ++ a = c->after; ++ dev_info(aop->dev, "SPI0 CH%uTD before ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD idle-sample slice=%u at_us=%llu ctl=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ channel, i, div_u64(p->ns, NSEC_PER_USEC), p->c3, ++ p->scm_lo, p->scm_hi, p->fault, ++ !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->channel_diag[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ channel, i, p->c3, d->cfg, d->interval, d->cursor, ++ p->scm_lo, p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->channel_diag[0].interval || ++ d->cursor != s->channel_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ channel, c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->channel_diag[0].interval, ++ s->channel_diag[c->slices - 1].interval, ++ s->channel_diag[0].cursor, ++ s->channel_diag[c->slices - 1].cursor, ++ source_progress, ++ stable_stop ? "stable-stop-reset-only" : "refused"); ++ dev_info(aop->dev, "SPI0 CH%uTD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ channel, c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ channel, c->stop, c->idle, c->poll); ++ else if (stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->poll, c->class0, c->class1, ++ c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH%uTD after ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH%uTD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ channel, c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ dev_info(aop->dev, "SPI0 CH%uTD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x cfg_bit5=%u pending0=%u pending1=%u result=%s\n", ++ channel, c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), a[A9_CH3_I_DESC], ++ !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(a[A9_CH3_I_8D4] & BIT(2 * channel)), c->failed ?: ++ stable_stop ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s) ++{ ++ const u32 *v; ++ const char *failed = NULL; ++ unsigned int channel; ++ ++ if (s->teardown_selected != s->teardown_done) ++ return "failed-channel-teardown-incomplete"; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ v = s->trace.channel[channel]; ++ failed = a9_spi0_channel_state_verify(v[A9_CHANNEL_I_CTL], ++ v[A9_CHANNEL_I_DESC], ++ s->trace.ch3[A9_CH3_I_8D0], ++ s->trace.ch3[A9_CH3_I_8D4], channel, ++ s->teardown_stable & BIT(channel)); ++ if (failed) { ++ s->teardown_failed_channel = channel; ++ return failed; ++ } ++ } ++ return NULL; ++} ++ ++/* ++ * Channel 3 remains mandatory and executes first. Wake11 additionally selects ++ * every snapshot slot with run bit 0 (RUN) set and a nonzero interval, which ++ * excludes configured-but-idle slots while selecting any periodic client still ++ * owned by the old firmware. Wake16 returns to wake11's order after wake15's ++ * channel-7-first stop failed: all ten device stops fit "channel 3 idles when ++ * stopped within ~2 ms of the reset (5/5; 0/2 at 11 and 33 ms) and channel 7 ++ * never idles (0/3, at 0.3 ms, 1.9 ms and 15 ms)". Channel 3 goes first and ++ * early; channel 7 is then the stopped-without-idle channel the stable-stop ++ * fingerprint was captured from (wake11/12). ++ */ ++static int a9_aop_spi0_teardown_running(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ s->teardown_selected = BIT(3); ++ s->teardown_done = 0; ++ s->teardown_stable = 0; ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3) ++ continue; ++ v = s->trace.channel[channel]; ++ if ((v[A9_CHANNEL_I_CTL] & A9_SPI0_CH3_RUN) && ++ v[A9_CHANNEL_I_INTERVAL]) ++ s->teardown_selected |= BIT(channel); ++ } ++ dev_info(aop->dev, "SPI0 teardown select stage=after-controller-reset selected=%04x mandatory=0008 rule=ch3-or-run-and-interval order=ch3-then-others\n", ++ s->teardown_selected); ++ ret = a9_aop_spi0_ch3_teardown(aop); ++ if (ret) { ++ s->teardown_failed_channel = 3; ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=3 errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, ret, ++ s->ch3.failed ?: "failed-ch3-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ ret = a9_aop_spi0_channel_teardown(aop, channel); ++ if (ret) { ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=%u errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, channel, ret, ++ s->channel.failed ?: "failed-channel-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(channel); ++ } ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ dev_info(aop->dev, "SPI0 teardown complete selected=%04x done=%04x stable=%04x result=pass\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable); ++ return 0; ++} ++ ++/* ++ * Second copy of the firmware controller reset, after the stop has completed. ++ * The preceding after-ch3-teardown trace is the read-only before snapshot; ++ * after-post-teardown-reset is the matching read-only after snapshot. Refuse ++ * the image copy unless SCM is zero and every channel quiet-state gate holds. ++ */ ++static int a9_aop_spi0_post_reset(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ const u32 *b = r->before, *a = r->after; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.ran || ++ s->ch3.error || s->teardown_selected != s->teardown_done || r->ran) { ++ r->failed = "failed-post-reset-state"; ++ return -EPERM; ++ } ++ aop->experiment_step = "spi0-post-teardown-reset"; ++ dev_info(aop->dev, "SPI0 POSTRST begin pa=%llx reason=ch3-stop-latched-scm cpu=halted\n", ++ A9_SPI0_PA); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-post-teardown-reset"); ++ if (ret) { ++ r->failed = "failed-post-reset-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_post_reset_locked, s); ++ r->ran = !ret; ++ r->error = ret ?: r->error; ++ dev_info(aop->dev, "SPI0 POSTRST errno=%d ran=%d before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ r->error, r->ran, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 POSTRST after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], r->saved, r->asserted, ++ r->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ if (r->error) { ++ r->failed = "failed-post-reset-controller"; ++ return r->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-post-teardown-reset"); ++ if (ret) ++ return ret; ++ if (s->trace.scm[0] || s->trace.scm[1]) ++ r->failed = "failed-post-reset-scm"; ++ else { ++ r->failed = a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3)); ++ if (!r->failed) ++ r->failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = r->failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 POSTRST check errno=%d scm_lo=%08x scm_hi=%08x ch3_pending=%u result=%s\n", ++ ret, s->trace.scm[0], s->trace.scm[1], ++ !!(s->trace.ch3[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), ++ r->failed ?: "pass"); ++ dev_info(aop->dev, "SPI0 POSTRST channels selected=%04x done=%04x stable=%04x result=%s\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable, ++ r->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * The ten INTC words of a9_intc_offsets into state[@at], read only. The ++ * INTC is the AOP wrapper block (0xc1800000 in the firmware, space 2 as ++ * SPI0's verified 0xc1100000 = 0x210100000), which this driver reads ++ * unguarded for the mailbox and CPU control; no separate PMGR domain. ++ */ ++static void a9_aop_intc_snapshot(struct apple_aop *aop, unsigned int at) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ u32 *v = n->state[at]; ++ unsigned int i; ++ ++ for (i = 0; i < A9_INTC_REGS; i++) ++ v[i] = aop_rd(aop, a9_intc_offsets[i]); ++ n->states |= BIT(at); ++ dev_info(aop->dev, "AOP INTC state at=%s reset808=%08x current81c=%08x raise900=%08x ack980=%08x mask_a00=%08x mask_a04=%08x mask_a08=%08x mask_a80=%08x g7_c020=%08x g7_c024=%08x src_masked=%u view=%s\n", ++ a9_intc_at_name(at), v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ v[7], v[8], v[9], !!(v[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(v)); ++} ++ ++/* ++ * Round 4 (ds-aop-spi0-ch3-round4.md): the old session's SPI0 global IRQ ++ * 0x10014 stays latched in the AOP interrupt controller, and fresh firmware ++ * takes it as soon as SPI lifecycle event 1 enables it, before its own SPI0 ++ * reset (SCMErr:0x3,3). With the CPU halted and before the first SPI0 ++ * access, disable that source (group 1 disable bank, 0102c9ce). The bank ++ * reads back the mask, as the firmware's own reads of it show: bit 20 must ++ * read 1 after the store. ++ */ ++static int a9_aop_intc_disable(struct apple_aop *aop) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ const u32 *b = n->state[A9_INTC_AT_PRE_DISABLE]; ++ const u32 *a = n->state[A9_INTC_AT_POST_DISABLE]; ++ int ret; ++ ++ aop->experiment_step = "intc-disable"; ++ dev_info(aop->dev, "AOP INTC disable begin irq=0x10014 pa=%llx value=%08x cpu=halted: first INTC store follows\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "intc-disable"); ++ if (ret) { ++ n->failed = "failed-intc-disable-checkpoint"; ++ return ret; ++ } ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_DISABLE); ++ ret = aop_wr(aop, A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_DISABLE); ++ if (ret) { ++ n->failed = "failed-intc-disable"; ++ } else if (!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM)) { ++ n->failed = "failed-intc-disable-readback"; ++ ret = -EIO; ++ } ++ n->disabled = !ret; ++ dev_info(aop->dev, "AOP INTC disable errno=%d before=%08x value=%08x after=%08x mask_a80=%08x src_masked=%u view=%s result=%s\n", ++ ret, b[A9_INTC_I_MASK0], A9_INTC_SPI0_SCM, a[A9_INTC_I_MASK0], ++ a[A9_INTC_I_ENABLE0], !!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(a), n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Release policy for aop_spi0_reset=1. The reload is device-proven for exactly ++ * one retained state: the previous kernel quiesced and halted the firmware ++ * (wake16, 2026-10-29; decision prev-quiesce-halted). Auto-boot still set, a ++ * previous kernel that did not halt it, or an older kernel in the chain are ++ * different hardware states that no run has reloaded with the final teardown. ++ * Refuse them before the first halt or SPI0 access so the old firmware is ++ * left exactly as found; aop_spi0_untested=1 attempts them anyway. A skip ++ * (DFU cold, bad record, nothing ran) must also find the wrapper at its reset ++ * state: a record-less AOP that something already stopped is refused rather ++ * than cold-loaded. A stale record skips as before. ++ */ ++static int a9_aop_spi0_policy(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ enum a9_spi0_reason reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, ++ &aop->spi0.prev); ++ ++ if (a9_spi0_policy_allows(reason, outbox & AOP_OUTBOX_ENABLE, spi0_untested)) ++ return 0; ++ dev_info(aop->dev, "SPI0 policy decision=refuse reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d: only prev-quiesce-halted reloads and a skip needs the wrapper at its reset state block not touched\n", ++ a9_spi0_reason_name(reason), auto_boot, outbox, ++ a9_spi0_prev_name(aop->spi0.prev.state), aop->spi0.prev.hello, ++ aop->spi0.prev.halted); ++ return -EPERM; ++} ++ ++/* ++ * After the proven halt/drain, before any SRAM write. Only firmware that ran ++ * since the last SoC reset leaves SPI0 state behind: auto-boot 1, or the ++ * previous kernel's transcript bound by +0x4020 (a9_spi0_decide). Without ++ * that evidence the block is never touched. @auto_boot and @outbox are the ++ * boot snapshot from before the stop. Nonzero means: do not load or ++ * release the AOP. ++ */ ++static int a9_aop_spi0_prereset(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *b = s->before, *a = s->after; ++ const char *failed = NULL; ++ bool run; ++ int ret; ++ ++ s->auto_boot = auto_boot; ++ s->outbox = outbox; ++ s->reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, &s->prev); ++ s->decided = true; ++ run = a9_spi0_reason_runs(s->reason); ++ dev_info(aop->dev, "SPI0 reset decision=%s reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d prev_steps=0x%x prev_carry=%d%s\n", ++ run ? "run" : "skip", a9_spi0_reason_name(s->reason), auto_boot, ++ outbox, a9_spi0_prev_name(s->prev.state), s->prev.hello, ++ s->prev.halted, s->prev.steps, s->prev.carried, ++ run ? "" : ": block not touched"); ++ if (!run) { ++ s->skipped = a9_spi0_reason_name(s->reason); ++ return 0; ++ } ++ if (!s->base || !aop->force_halted || !aop->force_drained || ++ !a9_aop_force_stopped(aop)) ++ return -EPERM; ++ ret = a9_aop_intc_disable(aop); ++ if (ret) ++ return ret; ++ aop->experiment_step = "spi0-reset"; ++ s->pmgr_before_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_before); ++ /* Reaches ramoops before the first AP access to the block. */ ++ dev_info(aop->dev, "SPI0 reset begin pa=%llx cpu=halted ps_spi0=%08x errno=%d: first AP access follows\n", ++ A9_SPI0_PA, s->pmgr_before, s->pmgr_before_errno); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-reset"); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_trace(aop, "before-controller-reset"); ++ if (ret) ++ return ret; ++ /* ++ * Wake15: from the controller reset to the last channel stop nothing may ++ * drain a console in this task. Holding the console lock makes every ++ * dev_info() below only append to the log buffer; the flushed spi0-reset ++ * checkpoint above already put the begin line into ramoops, and the lines ++ * are emitted by console_unlock() once the stops are stored. ++ */ ++ console_lock(); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_reset_locked, s); ++ s->ran = !ret; ++ s->error = ret ?: s->error; ++ s->pmgr_after_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_after); ++ dev_info(aop->dev, "SPI0 reset errno=%d ran=%d pa=%llx before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ s->error, s->ran, A9_SPI0_PA, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 reset after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u provider3=not-replayed\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], s->saved, s->asserted, ++ s->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 pmgr ps_spi0 before=%08x errno=%d after=%08x errno=%d (read-only, never powered)\n", ++ s->pmgr_before, s->pmgr_before_errno, s->pmgr_after, ++ s->pmgr_after_errno); ++ if (s->error) { ++ console_unlock(); ++ return s->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-controller-reset"); ++ if (!ret) ++ ret = a9_aop_spi0_teardown_running(aop); ++ console_unlock(); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-ch3-teardown"); ++ if (!ret) ++ failed = a9_aop_spi0_selected_verify(s); ++ if (failed) { ++ s->channel.failed = failed; ++ ret = -EIO; ++ } ++ if (!ret) ++ ret = a9_aop_spi0_post_reset(aop); ++ return ret; ++} ++ ++/* Wake10 samples both SCM words and both fault classes, not only channel 3. */ ++static void a9_aop_spi0_fault_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ ++ s->prerun_scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ s->prerun_scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ s->prerun = readl(s->base + A9_SPI0_FAULT); ++ s->prerun_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ a9_aop_spi0_ch3_snapshot(s->base, s->ch3.reassert); ++} ++ ++/* ++ * Last AP look before RUN, CPU still halted, after the image copy: bit 6 ++ * must stay clear in every one of 31 guarded samples spaced >= 5 ms apart, ++ * spanning >= 150 ms. A reasserted channel-3 bit would crash the fresh ++ * image as before, so the first set sample refuses the release; nothing is ++ * retried. Any refusal sets ch3.failed. ++ */ ++static int a9_aop_spi0_recheck(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *r = c->reassert; ++ u64 start = ktime_get_boottime_ns(); ++ unsigned int i; ++ int ret = 0; ++ ++ for (i = 0; i < A9_SPI0_WINDOW_SAMPLES; i++) { ++ if (i) ++ usleep_range(A9_SPI0_WINDOW_US, A9_SPI0_WINDOW_US + 1000); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_fault_read, s); ++ c->samples = i + 1; ++ c->elapsed_ms = div_u64(ktime_get_boottime_ns() - start, NSEC_PER_MSEC); ++ if (ret) { ++ c->failed = "failed-ch3-window-guard"; ++ break; ++ } ++ if (s->prerun_scm_lo || s->prerun_scm_hi || s->prerun || ++ s->prerun_8d4) { ++ s->reasserted = c->reasserted = true; ++ c->reassert_sample = i; ++ c->reassert_ms = c->elapsed_ms; ++ c->failed = "failed-spi0-window-reasserted"; ++ ret = -EIO; ++ dev_info(aop->dev, "SPI0 WINDOW reasserted sample=%u at_ms=%llu scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ i, c->reassert_ms, s->prerun_scm_lo, s->prerun_scm_hi, ++ s->prerun, s->prerun_8d4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ dev_info(aop->dev, "SPI0 WINDOW channel3 c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ r[0], r[1], r[2], r[3], r[4], r[5], r[6]); ++ break; ++ } ++ } ++ /* usleep_range() cannot return early; this only trips on a bad clock. */ ++ if (!ret && c->elapsed_ms < A9_SPI0_WINDOW_MS) { ++ c->failed = "failed-ch3-window-short"; ++ ret = -EIO; ++ } ++ c->window_done = !ret; ++ dev_info(aop->dev, "SPI0 CH3TD window errno=%d samples=%u/%u elapsed_ms=%llu min_ms=%u result=%s\n", ++ ret, c->samples, A9_SPI0_WINDOW_SAMPLES, c->elapsed_ms, ++ A9_SPI0_WINDOW_MS, ret ? c->failed : "pass"); ++ dev_info(aop->dev, "SPI0 pre-run errno=%d 8d0=%08x ch3_pending=%u cpu=halted\n", ++ ret, s->prerun, !!(s->prerun & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 WINDOW state errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ ret, s->prerun_scm_lo, s->prerun_scm_hi, s->prerun, ++ s->prerun_8d4, ret ? c->failed : "pass"); ++ if (ret && c->reasserted) ++ a9_aop_spi0_trace(aop, "window-fault"); ++ return ret; ++} ++ ++/* SCM state and the channel-3 interval pair, read-only, into scm[scm_at]. */ ++static void a9_aop_spi0_scm_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->intc.scm[s->intc.scm_at][i] = readl(s->base + a9_spi0_scm_offsets[i]); ++} ++ ++/* ++ * After the clean pre-RUN window, CPU still halted: the firmware's group-1 ++ * pre-handler acknowledge for 0x10014 (0102c6de..0102c6f2) discards the ++ * old-session latch now that its SPI0 cause is quiet. Fresh firmware ++ * re-enables the source in event 1. The INTC words, the SCM state and the ++ * C3 interval pair are logged around it and never gated: with 0x10014 ++ * masked, no readable word shows whether it is still pending. ++ */ ++static int a9_aop_intc_ack(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ const u32 *p = n->scm[0], *q = n->scm[1]; ++ const u32 *b = n->state[A9_INTC_AT_PRE_ACK]; ++ const u32 *a = n->state[A9_INTC_AT_POST_ACK]; ++ int ret; ++ ++ aop->experiment_step = "intc-ack"; ++ n->scm_at = 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) { ++ n->failed = "failed-intc-ack-guard"; ++ dev_info(aop->dev, "AOP INTC pre-ack errno=%d: guard refused, no acknowledge\n", ++ ret); ++ return ret; ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_ACK); ++ dev_info(aop->dev, "AOP INTC pre-ack scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x cpu=halted\n", ++ p[0], p[1], p[2], p[3], b[A9_INTC_I_CURRENT]); ++ ret = aop_wr(aop, A9_INTC_ACK, A9_INTC_SPI0_SCM); ++ /* Also the posted-write barrier. */ ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_ACK); ++ n->acked = !ret; ++ dev_info(aop->dev, "AOP INTC ack errno=%d before=%08x value=%08x after=%08x\n", ++ ret, b[A9_INTC_I_ACK], A9_INTC_SPI0_SCM, a[A9_INTC_I_ACK]); ++ if (ret) { ++ n->failed = "failed-intc-ack"; ++ return ret; ++ } ++ n->scm_at = 1; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) ++ n->failed = "failed-intc-post-guard"; ++ dev_info(aop->dev, "AOP INTC post-ack errno=%d scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x result=%s\n", ++ ret, q[0], q[1], q[2], q[3], a[A9_INTC_I_CURRENT], n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Wake7 still took 0x10014 before SPI event 5 despite the targeted ack. ++ * Replay INTC lifecycle event 3 (0102c592/598/59c/5a2): 1/read/0/read at ++ * +0x808. aop_wr includes a posted-write read after each store. This resets ++ * all AOP-private interrupt state; it does not write the SCM status or MPU. ++ * No assumption that SCM must become zero. Only the final reset readback ++ * and the established channel-3 quiet conditions gate RUN. ++ */ ++static int a9_aop_intc_reset(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.window_done || ++ !n->acked || n->reset_done || ++ (aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) || readl(aop->control)) { ++ n->failed = "failed-intc-reset-state"; ++ return -EPERM; ++ } ++ ret = a9_aop_spi0_trace(aop, "pre-intc-reset"); ++ if (ret) ++ return ret; ++ aop->experiment_step = "intc-reset"; ++ dev_info(aop->dev, "AOP INTC reset begin pa=%llx sequence=1/read/0/read cpu=halted\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_RESET); ++ ret = aop_wr(aop, A9_INTC_RESET, 1); ++ n->reset_asserted = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-assert"; ++ return ret; ++ } ++ ret = aop_wr(aop, A9_INTC_RESET, 0); ++ n->reset_after = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-deassert"; ++ return ret; ++ } ++ if (n->reset_after) { ++ n->failed = "failed-intc-reset-readback"; ++ ret = -EIO; ++ } else { ++ ret = a9_aop_spi0_trace(aop, "post-intc-reset"); ++ if (!ret && a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3))) { ++ n->failed = "failed-intc-reset-ch3"; ++ ret = -EIO; ++ } else if (!ret && a9_aop_spi0_selected_verify(s)) { ++ n->failed = "failed-intc-reset-channel"; ++ ret = -EIO; ++ } ++ } ++ n->reset_done = !ret; ++ dev_info(aop->dev, "AOP INTC reset errno=%d asserted=%08x after=%08x result=%s\n", ++ ret, n->reset_asserted, n->reset_after, n->failed ?: "pass"); ++ return ret; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.h b/drivers/soc/apple/aop-warm-spi0.h +new file mode 100644 +index 000000000..2470165ce +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.h +@@ -0,0 +1,543 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * J71t AOP reload/wake experiments, both opt-in and default off. Constants ++ * and evidence types only; the MMIO lives in apple-a9-aop-active.c. ++ * Evidence: docs/periph/ds-aop-reload-crash-and-wake.md. ++ * ++ * aop_spi0_reset=1: every halt + SRAM reload of a live AOP crashed in ++ * AOPSPIMaster_channelFaultIRQ (0102f252, DFAR 0x4c): the AOP-private SPI0 ++ * controller kept channel 3's +0x8d0 bit 6 while the fresh firmware's ++ * channel table was still empty. With the CPU halted, the AP replays the ++ * firmware's own controller reset AOPSPIMaster_reset (0102f2f4), then the ++ * firmware's own channel-3 teardown, and samples bit 6 for >= 150 ms after ++ * the image copy before it releases RUN once. Round 4: the old session's ++ * SPI0 global IRQ 0x10014 stays latched in the AOP interrupt controller ++ * (SCMErr:0x3,3 at the accel STARTEP), so the AP disables that source before ++ * the reset and acknowledges it after the clean window, just before RUN. ++ * Wake7: the channel-3 idle wait gets the firmware's own 10 ms bound twice ++ * over, sampled per 1 ms slice, and the disable is judged by the INTC mask ++ * readback, which the firmware itself reads. ++ * Wake8: after the targeted ack, replay INTC event 3 (+0x808: 1/read/0/read). ++ * This resets retained interrupt delivery, not the read-only SCM state. ++ * Wake9: wake8 device evidence showed the channel-3 stop itself changes SCM ++ * from 0 to 3. Replay the same SPI controller reset once more after teardown; ++ * require the post-reset SCM state to be zero and the teardown state to hold. ++ * Wake10: the second reset clears SCM, but a halted CPU then saw SCM 3 and ++ * +0x8d0 bit 14 (channel 7) during the firmware copy. Snapshot all 16 channel ++ * slots, sample global SCM/fault state across the copy, and gate the copy and ++ * final RUN on a completely quiet readback. ++ * Wake11: that snapshot proved channels 3 and 7 were both running. Keep the ++ * channel-3 sequence unchanged, then apply the same firmware teardown to every ++ * other run-bit + nonzero-interval channel before the post-teardown reset. ++ * Wake12: channel 7 is the phosphorus/BMP284 async client. Its firmware has no ++ * owner-specific stop path, so do not invent another store when the generic ++ * stop times out. Record the remaining scheduler state read-only and refuse. ++ * Wake13/14: every device stop of channel 3 that idled (wake8..wake12) came ++ * 1.5-2.2 ms after the controller reset; the two that did not (wake13/14) came ++ * 11/33 ms after it (pr_flush in the checkpoint), with class-0 state identical to ++ * wake12's passes. Wake15 closed that window (no pr_flush between the reset and ++ * the stops, console lock held), recorded the reset-to-stop latency and the peers ++ * per stop, and accepted one identity-pinned stopped-without-idle channel (3 or ++ * 7) as wake13 did for channel 7. Wake15's channel-7-first order then showed ++ * channel 7 never idles even at 0.3 ms (it is 0/3 over wake11/12/15), with ++ * nothing ticking. Wake16 therefore returns to channel 3 first (early, the only ++ * stop that ever idled) and then channel 7, which takes the stable-stop path ++ * (SCM 0/0 or 3/0, constant); every stop logs why the fingerprint did or did not ++ * hold (stable_why=). Wake14's class-0 prelude stays removed. ++ * ++ * aop_wake_wrapper=1: before 0x220, the AppleA7IOP status-3 resume wrapper ++ * writes (b4b854..b4b8b4). No CPU RUN, no auto-boot, no NMI. ++ */ ++#ifndef _A9_AOP_RELOAD_H ++#define _A9_AOP_RELOAD_H ++ ++#include ++#include ++#include "aop-warm-record.h" ++ ++/* Firmware device spi0: IOP 0x10100000 (space 2, 0x1000) via /arm-io ranges. */ ++#define A9_SPI0_PA 0x210100000ULL ++#define A9_SPI0_SIZE 0x1000U ++#define A9_SPI0_WRAPPER_PA 0x210800000ULL /* the only layout this applies to */ ++#define A9_SPI0_CONTROL 0x000U ++#define A9_SPI0_CLOCK 0x008U /* 0102f2aa programs 0x01b71b00 here */ ++#define A9_SPI0_FAULT 0x8d0U /* bit 2n = channel n, read by 0102f20a */ ++#define A9_SPI0_FAULT_8D4 0x8d4U ++#define A9_SPI0_CHANNEL_BASE 0x100U ++#define A9_SPI0_CHANNEL_STRIDE 0x080U ++#define A9_SPI0_CHANNELS 16U ++#define A9_SPI0_CHANNEL_REGS 7U ++#define A9_SPI0_CHANNEL_DESC 0x018U ++#define A9_SPI0_CH3 0x280U /* base + 0x100 + 0x80 * 3 */ ++#define A9_SPI0_CH3_0C 0x28cU ++#define A9_SPI0_RESET 0x100U /* control bit 8 */ ++#define A9_SPI0_ENABLE 0x001U /* control bit 0 */ ++#define A9_SPI0_CH3_PENDING 0x040U ++/* The firmware spins without bound; the AP gives each poll 1 ms. */ ++#define A9_SPI0_POLL_US 1000U ++#define A9_SPI0_DELAY_US 100U /* 0102b3f8(100, 0), twice */ ++#define A9_SPI0_REGS 6 ++ ++static const u32 a9_spi0_offsets[A9_SPI0_REGS] = { ++ A9_SPI0_CONTROL, A9_SPI0_CLOCK, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++ A9_SPI0_CH3, A9_SPI0_CH3_0C, ++}; ++ ++/* Snapshot indices into a9_spi0_offsets. */ ++enum { A9_SPI0_I_CONTROL, A9_SPI0_I_CLOCK, A9_SPI0_I_FAULT }; ++ ++/* ++ * Channel-3 teardown after the reset (ds-aop-spi0-ch3-round3.md section 5). ++ * The reset leaves C3 = 0x00010447: idle but still enabled, async-armed and ++ * with the class-0 fault IRQ on, so a level source re-latched +0x8d0 bit 6 ++ * within ~105 ms with the CPU halted. AOPSPIMaster_teardownChannel ++ * (0102e4f8) stops the channel and masks both IRQ classes; the fault ++ * handler (0102f20a) then acknowledges with C3C bit 5. ++ */ ++#define A9_SPI0_CH3_ROUTE 0x284U ++#define A9_SPI0_CH3_RATE 0x288U ++#define A9_SPI0_CH3_DESC 0x298U /* channel +0x18, zeroed on teardown */ ++#define A9_SPI0_CH3_RUN 0x001U /* C3 bit 0, channel run/enable */ ++#define A9_SPI0_CH3_IDLE 0x002U /* C3 bit 1, polled after the stop */ ++#define A9_SPI0_CH3_IRQ0 0x040U /* C3 bit 6, +0x8d0 class-0/fault enable */ ++#define A9_SPI0_CH3_IRQ1 0x100U /* C3 bit 8, +0x8d4 class-1 enable */ ++#define A9_SPI0_CHANNEL_PROGRESS_SHIFT 24U /* hardware-updated, not an IRQ class */ ++#define A9_SPI0_CH3_STOP 0x411U /* clear run bit 0 and bits 4, 10 */ ++#define A9_SPI0_CH3_CLASS0 0x450U /* mask class 0: bits 4, 6, 10 */ ++#define A9_SPI0_CH3_CLASS1 0x510U /* mask class 1: bits 4, 8, 10 */ ++#define A9_SPI0_CH3_ACK 0x020U /* C3C bit 5, ORed by 0102f20a; self-clearing */ ++#define A9_SPI0_PHOSPHORUS_CHANNEL 7U ++#define A9_SPI0_PHOSPHORUS_CTL_MASK 0x00ffffffU ++#define A9_SPI0_PHOSPHORUS_CTL_RUN 0x00010445U ++#define A9_SPI0_PHOSPHORUS_CTL_STOPPED 0x00010444U ++#define A9_SPI0_PHOSPHORUS_ROUTE 0x00000040U ++#define A9_SPI0_PHOSPHORUS_RATE 0x019000c8U ++#define A9_SPI0_PHOSPHORUS_CFG 0x00200009U ++/* Channel 3 is the MPU6900 async FIFO client (route 0x10, same rate/config). */ ++#define A9_SPI0_MPU_ASYNC_CHANNEL 3U ++#define A9_SPI0_MPU_ASYNC_ROUTE 0x00000010U ++/* Stopped control word, any top byte, class-0 enable optional (0x40). */ ++#define A9_SPI0_STABLE_CTL_STOPPED 0x00010404U ++#define A9_SPI0_STABLE_CTL_MASK 0x00ffffbfU ++/* Peer sampling and the "late stop" mark are reported, never gated, in-kernel. */ ++#define A9_SPI0_LATE_STOP_US 3000U ++/* ++ * AOPSPIMaster_setChannelEnabled(false) (0102ef7c) stores C3 & ~0x411 only ++ * if bit 0 was set, then spins on bit 1 until master +0x30, and panics ++ * "SCMto:%u" (0x010398da) past it. Event 1 sets +0x30 to 0x2710 us when it ++ * is 0 (0102e6fa, as in the live-cold object), through timer +0x18 = ++ * us * Hz / 1000000 (0102ff3c): a 10 ms bound. The AP polls in 1 ms slices, ++ * up to twice that, sampling C3, the SCM state and +0x8d0 after each slice. ++ */ ++#define A9_SPI0_CH3_IDLE_FW_US 10000U ++#define A9_SPI0_CH3_SLICE_US 1000U ++#define A9_SPI0_CH3_SLICES 20U /* 20 ms */ ++/* At least 150 ms (the 16:27 re-latch took ~105 ms) of samples before RUN. */ ++#define A9_SPI0_WINDOW_SAMPLES 31U ++#define A9_SPI0_WINDOW_US 5000U /* 30 sleeps >= 5 ms each */ ++#define A9_SPI0_WINDOW_MS 150U /* required first-to-last sample span */ ++#define A9_SPI0_CH3_REGS 7 ++ ++static const u32 a9_spi0_ch3_offsets[A9_SPI0_CH3_REGS] = { ++ A9_SPI0_CH3, A9_SPI0_CH3_ROUTE, A9_SPI0_CH3_RATE, A9_SPI0_CH3_0C, ++ A9_SPI0_CH3_DESC, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++}; ++ ++/* Every firmware-visible channel slot, read only in wake10 and selected in wake11. */ ++static const u32 a9_spi0_channel_offsets[A9_SPI0_CHANNEL_REGS] = { ++ 0x00U, 0x04U, 0x08U, 0x0cU, 0x10U, 0x14U, 0x18U, ++}; ++ ++enum { ++ A9_CHANNEL_I_CTL, A9_CHANNEL_I_ROUTE, A9_CHANNEL_I_RATE, A9_CHANNEL_I_CFG, ++ A9_CHANNEL_I_INTERVAL, A9_CHANNEL_I_CURSOR, A9_CHANNEL_I_DESC, ++}; ++ ++/* Snapshot indices into a9_spi0_ch3_offsets. */ ++enum { ++ A9_CH3_I_C3, A9_CH3_I_ROUTE, A9_CH3_I_RATE, A9_CH3_I_C3C, A9_CH3_I_DESC, ++ A9_CH3_I_8D0, A9_CH3_I_8D4, ++}; ++ ++/* One idle-poll slice: C3 as the slice ended, then SCM state and +0x8d0. */ ++struct a9_aop_ch3_slice { ++ u64 ns; /* since the stop store */ ++ u32 c3, scm_lo, scm_hi, fault; ++}; ++ ++/* Wake12 read-only scheduler evidence for an additional-channel stop poll. */ ++struct a9_aop_channel_diag_slice { ++ u32 cfg, interval, cursor, fault_8d4; ++}; ++ ++struct a9_aop_spi0_ch3 { ++ u32 before[A9_SPI0_CH3_REGS], after[A9_SPI0_CH3_REGS]; ++ u32 stop, idle, class0, class1, ack; /* words written / polled */ ++ int poll; /* idle-bit poll result */ ++ bool run_before; /* C3 bit 0 before the stop */ ++ unsigned int slices; /* idle-poll slices used */ ++ struct a9_aop_ch3_slice slice[A9_SPI0_CH3_SLICES]; ++ u64 idle_us; /* stop store to the idle read */ ++ int error; ++ bool ran; /* the guarded teardown executed */ ++ const char *failed; /* distinct refusal reason, or NULL */ ++ /* Wake15: stop latency after the controller reset and the peers' state. */ ++ u64 stop_ns, reset_to_stop_us; ++ u32 peers_run, peers_ticking; /* masks of the other 15 channels */ ++ bool stable; /* accepted stopped-without-idle identity */ ++ const char *stable_why; /* first failed fingerprint condition, NULL if stable */ ++ /* Pre-RUN window after the image copy; a refusal also sets failed. */ ++ unsigned int samples; ++ u64 elapsed_ms; ++ bool window_done, reasserted; ++ unsigned int reassert_sample; ++ u64 reassert_ms; ++ u32 reassert[A9_SPI0_CH3_REGS]; ++}; ++ ++struct a9_aop_spi0; ++ ++struct a9_aop_spi0_channel_op { ++ struct a9_aop_spi0 *spi0; ++ struct a9_aop_spi0_ch3 *state; ++ unsigned int channel; ++}; ++ ++/* ++ * AOP interrupt controller, wrapper offsets (ds-aop-spi0-ch3-round4.md ++ * section 3). IRQ 0x10014 is group 1, source 0x14: bank 0, bit 20. The ++ * group-1 disable/enable banks are write-one commands whose reads return ++ * the mask, 1 = disabled: the sleep command saves +0xa00..+0xa08 (0102c5ce) ++ * before masking all (0102c5da), the restore writes saved to +0xa00 and ++ * ~saved to +0xa80 (0102c9e6/0102c9f4), and aop_interrupt_write_enabled ++ * reads the target bank for the previous state (0102c9c0). So the disable ++ * is judged by bit 20 of the +0xa00 readback (wake6: ffffffff before the ++ * store, every source already masked by the quiesce). +0x81c only reports ++ * an unmasked pending source: the sleep path masks all, writes group 7 ++ * +0xc020/+0xc024 = 2 and spins until +0x81c reads 0 (0102c654), so ++ * all-ones there means the AP is not reading the INTC. +0x808 ends 0 after ++ * the event-3 pulse. +0x900 is the group-1 raise (0102c7c6) and +0x980 its ++ * pre-handler clear (0102c6f2); the firmware never reads them, so their ++ * readbacks are logged only. No register shows a masked source pending. ++ */ ++#define A9_INTC_RESET 0x808U /* event-3 pulse 1 then 0 (0102c592) */ ++#define A9_INTC_CURRENT 0x81cU /* current unmasked source */ ++#define A9_INTC_RAISE 0x900U /* group 1 raise, 0102c7c6 */ ++#define A9_INTC_ACK 0x980U /* group 1 pre-handler ack/clear, 0102c6f2 */ ++#define A9_INTC_DISABLE 0xa00U /* group 1 disable, store at 0102c9ce */ ++#define A9_INTC_ENABLE 0xa80U /* group 1 enable; reads the same mask */ ++#define A9_INTC_G7_DISABLE 0xc020U /* group 7 disable, 2 after sleep */ ++#define A9_INTC_SPI0_SCM 0x00100000U /* IRQ 0x10014 */ ++#define A9_INTC_ALL_ONES 0xffffffffU ++#define A9_INTC_REGS 10 ++ ++static const u32 a9_intc_offsets[A9_INTC_REGS] = { ++ A9_INTC_RESET, A9_INTC_CURRENT, A9_INTC_RAISE, A9_INTC_ACK, ++ A9_INTC_DISABLE, A9_INTC_DISABLE + 4, A9_INTC_DISABLE + 8, ++ A9_INTC_ENABLE, A9_INTC_G7_DISABLE, A9_INTC_G7_DISABLE + 4, ++}; ++ ++/* Snapshot indices into a9_intc_offsets. */ ++enum { ++ A9_INTC_I_RESET, A9_INTC_I_CURRENT, A9_INTC_I_RAISE, A9_INTC_I_ACK, ++ A9_INTC_I_MASK0, A9_INTC_I_MASK1, A9_INTC_I_MASK2, A9_INTC_I_ENABLE0, ++ A9_INTC_I_G7_0, A9_INTC_I_G7_1, ++}; ++ ++/* When the AP snapshots the INTC words. */ ++enum { ++ A9_INTC_AT_PRE_DISABLE, A9_INTC_AT_POST_DISABLE, A9_INTC_AT_PRE_ACK, ++ A9_INTC_AT_POST_ACK, A9_INTC_AT_N, ++}; ++ ++static inline const char *a9_intc_at_name(unsigned int at) ++{ ++ static const char * const names[] = { ++ "pre-disable", "post-disable", "pre-ack", "post-ack", ++ }; ++ ++ return at < A9_INTC_AT_N ? names[at] : "unknown"; ++} ++ ++/* "all-ones" if +0x81c reads ffffffff, which a live INTC cannot report. */ ++static inline const char *a9_intc_view(const u32 *state) ++{ ++ return state[A9_INTC_I_CURRENT] == A9_INTC_ALL_ONES ? "all-ones" : "intc"; ++} ++/* ++ * Read only, never written: the SCM state (also per idle-poll slice) and, ++ * around the acknowledge, the C3 interval pair. ++ */ ++#define A9_SPI0_SCM_LO 0x00cU ++#define A9_SPI0_SCM_HI 0x010U ++#define A9_SPI0_CH3_IVAL 0x290U ++#define A9_SPI0_CH3_CURSOR 0x294U ++#define A9_SPI0_SCM_REGS 4 ++ ++static const u32 a9_spi0_scm_offsets[A9_SPI0_SCM_REGS] = { ++ A9_SPI0_SCM_LO, A9_SPI0_SCM_HI, A9_SPI0_CH3_IVAL, A9_SPI0_CH3_CURSOR, ++}; ++ ++struct a9_aop_intc { ++ u32 state[A9_INTC_AT_N][A9_INTC_REGS]; /* around each store */ ++ unsigned int states; /* bit n: state[n] was read */ ++ u32 scm[2][A9_SPI0_SCM_REGS]; /* before / after the ack */ ++ unsigned int scm_at; /* scm[] row the guarded read fills */ ++ bool disabled, acked; ++ bool reset_done; ++ u32 reset_asserted, reset_after; ++ const char *failed; /* distinct refusal reason, or NULL */ ++}; ++ ++/* Read-only milestones. Wake10 gates the global SCM/fault words, not channels. */ ++struct a9_aop_spi0_trace { ++ u32 control, clock, scm[A9_SPI0_SCM_REGS], ch3[A9_SPI0_CH3_REGS]; ++ u32 channel[A9_SPI0_CHANNELS][A9_SPI0_CHANNEL_REGS]; ++}; ++ ++/* Wake9's second, post-channel-stop replay of AOPSPIMaster_reset. */ ++struct a9_aop_spi0_post_reset { ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved, asserted, polled; ++ int error; ++ bool ran; ++ const char *failed; ++}; ++ ++/* ++ * First failed readback after the teardown, or NULL if all hold. C3C bit 5 ++ * is a pulse: at 19:50 the write of 0x00200029 read back 0x00200009 while ++ * +0x8d0 bit 6 went 1 -> 0, so the acknowledge is judged by bit 6 alone and ++ * the C3C readback is only logged. ++ */ ++static inline const char *a9_spi0_ch3_verify(const u32 *after, bool stable) ++{ ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) ++ return "failed-ch3-teardown-run"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0) ++ return "failed-ch3-teardown-class0"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1) ++ return "failed-ch3-teardown-class1"; ++ if (!(after[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE) && !stable) ++ return "failed-ch3-teardown-idle"; ++ if (after[A9_CH3_I_DESC]) ++ return "failed-ch3-teardown-desc"; ++ if (after[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING) ++ return "failed-ch3-teardown-pending"; ++ return NULL; ++} ++ ++/* ++ * Has firmware run since the last SoC reset? auto-boot alone cannot say: ++ * the outgoing aop_quiesce=1 halt clears it, exactly like a DFU cold start. ++ * The previous kernel's reserved-RAM shutdown transcript says whether its ++ * firmware ran. That DRAM can outlive a DFU; wrapper +0x4020 outbox enable ++ * cannot: set after reset (12/12 DFU cold snapshots), cleared by the first ++ * store of every Linux stop. No SPI0 access is needed to decide. ++ */ ++enum a9_spi0_prev_state { ++ A9_SPI0_PREV_MISSING, /* no transcript magic, or none reserved */ ++ A9_SPI0_PREV_INVALID, /* magic present, header/checksum rejected */ ++ A9_SPI0_PREV_VALID, ++}; ++ ++struct a9_spi0_prev { ++ enum a9_spi0_prev_state state; ++ bool hello; /* "SHUTDOWN begin ... hello=1": its firmware ran */ ++ bool halted; /* "QUIESCE step=complete ... halted=1" + steps bit */ ++ bool carried; /* "SPI0-CARRY retained=1": ran earlier in the chain */ ++ u32 steps; /* "QUIESCE step=complete ... steps=0x..." */ ++}; ++ ++enum a9_spi0_reason { ++ /* Run the reset. */ ++ A9_SPI0_LIVE_AUTO_BOOT, /* auto-boot 1 before the stop */ ++ A9_SPI0_PREV_HALTED, /* previous kernel quiesced and halted it */ ++ A9_SPI0_PREV_LIVE, /* previous kernel's firmware ran, no halt */ ++ A9_SPI0_PREV_CARRIED, /* an earlier kernel in the chain ran it */ ++ /* Skip: the block is not touched. */ ++ A9_SPI0_FRESH, /* no transcript, auto-boot 0: DFU cold */ ++ A9_SPI0_PREV_BAD, /* transcript present but corrupt */ ++ A9_SPI0_PREV_NO_RUN, /* transcript: nothing ran in the chain */ ++ A9_SPI0_STALE, /* run evidence, wrapper at reset state */ ++}; ++ ++static inline const char *a9_spi0_reason_name(enum a9_spi0_reason reason) ++{ ++ static const char * const names[] = { ++ "live-auto-boot", "prev-quiesce-halted", "prev-live", "prev-carried", ++ "fresh-instance", "prev-transcript-invalid", "prev-no-run", ++ "stale-transcript", ++ }; ++ ++ return reason <= A9_SPI0_STALE ? names[reason] : "unknown"; ++} ++ ++static inline const char *a9_spi0_prev_name(enum a9_spi0_prev_state state) ++{ ++ return state == A9_SPI0_PREV_VALID ? "valid" : ++ state == A9_SPI0_PREV_INVALID ? "invalid" : "missing"; ++} ++ ++static inline bool a9_spi0_reason_runs(enum a9_spi0_reason reason) ++{ ++ return reason <= A9_SPI0_PREV_CARRIED; ++} ++ ++/* ++ * Release policy: a reason that would run the reload is allowed by default ++ * only for prev-quiesce-halted, the one state device-proven end to end. A ++ * skip (fresh, bad record, nothing ran) is allowed only while the wrapper is ++ * still at its reset state; stale records skip as before. Untested lifts both. ++ */ ++static inline bool a9_spi0_policy_allows(enum a9_spi0_reason reason, bool outbox_enabled, ++ bool untested) ++{ ++ if (untested || reason == A9_SPI0_PREV_HALTED || reason == A9_SPI0_STALE) ++ return true; ++ if (a9_spi0_reason_runs(reason)) ++ return false; ++ /* ++ * A skip means "nothing ran since the SoC reset", which only holds if the ++ * wrapper is still at its reset state (+0x4020 outbox enable set, 12/12 DFU ++ * cold starts). Without it some earlier kernel stopped the firmware and its ++ * record is lost: a cold load would land on a halted, never-torn-down AOP. ++ */ ++ return outbox_enabled; ++} ++ ++/* Token search inside one transcript line, which holds no NUL. */ ++static inline bool a9_spi0_line_has(const char *line, size_t length, ++ const char *token, size_t n, ++ const char **at) ++{ ++ size_t i; ++ ++ for (i = 0; n <= length && i <= length - n; i++) { ++ if (!memcmp(line + i, token, n)) { ++ *at = line + i + n; ++ return true; ++ } ++ } ++ return false; ++} ++ ++#define a9_spi0_has(line, length, token, at) \ ++ a9_spi0_line_has(line, length, token, sizeof(token) - 1, at) ++ ++static inline bool a9_spi0_line_starts(const char *line, size_t length, ++ const char *prefix, size_t n) ++{ ++ return length >= n && !memcmp(line, prefix, n); ++} ++ ++#define a9_spi0_starts(line, length, prefix) \ ++ a9_spi0_line_starts(line, length, prefix, sizeof(prefix) - 1) ++ ++static inline u32 a9_spi0_hex(const char *p, const char *end) ++{ ++ u32 value = 0; ++ ++ for (; p < end; p++) { ++ if (*p >= '0' && *p <= '9') ++ value = value << 4 | (u32)(*p - '0'); ++ else if (*p >= 'a' && *p <= 'f') ++ value = value << 4 | (u32)(*p - 'a' + 10); ++ else ++ break; ++ } ++ return value; ++} ++ ++/* Our own checksummed transcript; only whole-line prefixes count. */ ++static inline void a9_spi0_prev_parse(struct a9_spi0_prev *p, ++ const char *text, size_t length) ++{ ++ const char *end = text + length, *line, *next, *at; ++ size_t n; ++ ++ for (line = text; line < end; line = next < end ? next + 1 : end) { ++ next = memchr(line, '\n', end - line); ++ if (!next) ++ next = end; ++ n = next - line; ++ if (a9_spi0_starts(line, n, "SHUTDOWN begin ") && ++ a9_spi0_has(line, n, " hello=1 ", &at)) ++ p->hello = true; ++ if (a9_spi0_starts(line, n, "QUIESCE step=complete ")) { ++ if (a9_spi0_has(line, n, " steps=0x", &at)) ++ p->steps = a9_spi0_hex(at, next); ++ p->halted = a9_spi0_has(line, n, " halted=1 ", &at) && ++ (p->steps & (1U << A9_HO_STEP_HALTED)); ++ } ++ if (a9_spi0_starts(line, n, "SPI0-CARRY retained=1 ")) ++ p->carried = true; ++ } ++} ++ ++/* ++ * Skips unless there is positive evidence: running the reset on a fresh ++ * block would be the first AP access to SPI0 with nothing retained to ++ * clear (and ps_spi0 may gate it), while a skip on retained state only ++ * repeats the known, non-hanging accel STARTEP crash. ++ */ ++static inline enum a9_spi0_reason a9_spi0_decide(bool auto_boot, bool outbox_enabled, ++ const struct a9_spi0_prev *p) ++{ ++ if (auto_boot) ++ return A9_SPI0_LIVE_AUTO_BOOT; ++ if (p->state == A9_SPI0_PREV_MISSING) ++ return A9_SPI0_FRESH; ++ if (p->state != A9_SPI0_PREV_VALID) ++ return A9_SPI0_PREV_BAD; ++ if (!p->hello && !p->carried) ++ return A9_SPI0_PREV_NO_RUN; ++ if (outbox_enabled) ++ return A9_SPI0_STALE; ++ if (p->hello) ++ return p->halted ? A9_SPI0_PREV_HALTED : A9_SPI0_PREV_LIVE; ++ return A9_SPI0_PREV_CARRIED; ++} ++ ++struct a9_aop_spi0 { ++ void __iomem *base; /* mapped only with aop_spi0_reset=1 */ ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved; /* control word before the reset */ ++ u32 asserted; /* saved & ~0x101 | 0x100 */ ++ u32 polled; /* control word when the polls ended */ ++ u32 prerun; /* +0x8d0 just before RUN */ ++ u32 prerun_8d4, prerun_scm_lo, prerun_scm_hi; ++ u32 pmgr_before, pmgr_after; /* AP PMGR ps_spi0 raw, read-only */ ++ int pmgr_before_errno, pmgr_after_errno; ++ int error; ++ bool ran, reasserted, copy_faulted; ++ const char *gate_failed; ++ const char *skipped; ++ struct a9_spi0_prev prev; /* parsed at probe from the transcript */ ++ u32 auto_boot, outbox; /* boot snapshot, before the stop */ ++ enum a9_spi0_reason reason; ++ bool decided; ++ struct a9_aop_spi0_ch3 ch3; /* after a clean reset only */ ++ /* Wake11 reuses one bounded scratch record for each additional channel. */ ++ struct a9_aop_spi0_ch3 channel; ++ u32 teardown_selected, teardown_done; ++ unsigned int teardown_failed_channel; ++ struct a9_aop_spi0_post_reset post_reset; /* after the stop latches SCM */ ++ struct a9_aop_intc intc; /* IRQ 0x10014 around the reset */ ++ struct a9_aop_spi0_trace trace; /* latest read-only milestone */ ++ /* Appended so every pre-wake12 field, including ch3, keeps its layout. */ ++ struct a9_aop_channel_diag_slice channel_diag[A9_SPI0_CH3_SLICES]; ++ /* Wake13: exact phosphorus stopped-without-idle exception, by channel. */ ++ u32 teardown_stable; ++ /* Wake15: channel 3 shares the sampled diagnostics; reset completion time. */ ++ struct a9_aop_channel_diag_slice ch3_diag[A9_SPI0_CH3_SLICES]; ++ u64 reset_done_ns; ++}; ++ ++/* 0102f2f4: bits 8 and 0 clear; the reset write adds bit 8. */ ++static inline u32 a9_spi0_base_word(u32 saved) ++{ ++ return saved & ~(A9_SPI0_RESET | A9_SPI0_ENABLE); ++} ++ ++#endif +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index 5c23317d1..542e9b62e 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -12,10 +12,17 @@ + * child platform device; its driver receives the service's reports through + * apple_aop_listen() and sends requests with apple_aop_call(). + * +- * The firmware cannot be restarted once stopped, so the AOP starts once per +- * boot and the driver cannot be unbound or unloaded after that. ++ * J71t can reload after a certified quiesce/halt/drain across kexec. Other ++ * retained states are refused. Unbind/unload after starting remains forbidden. + */ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include "aop-warm-spi0.h" + #include + #include + #include +@@ -95,7 +102,9 @@ + #define AFK_OPC_GET_BUF_ACK 0xa1 + #define AFK_OPC_SEND 0xa2 + #define AFK_OPC_START 0xa3 +-#define AFK_OPC_SHUTDOWN_ACK 0xc1 ++#define AFK_OPC_SHUTDOWN 0xa4 ++#define AFK_OPC_SHUTDOWN_ACK 0x87 ++#define AFK_OPC_HIBERNATE_ACK 0xc3 + + #define AFK_BLOCK_STEP 0x40 + #define AFK_MAX_BUFFER SZ_1M +@@ -183,6 +192,8 @@ struct apple_aop_ep { + + struct mutex lock; /* tx ring, seq and calls */ + u16 seq; ++ bool initialized; ++ struct completion stopped; + struct apple_aop_call *calls[APPLE_AOP_MAX_CALLS]; + }; + +@@ -240,6 +251,16 @@ struct apple_aop { + struct mutex children_lock; /* children, removing */ + struct list_head children; + bool removing; ++ u64 accel_reports; ++ int shutdown_error; ++ /* J71t only; all other boards retain their original start/stop path. */ ++ bool warm, hello, iop_crashed, force_halted, force_drained; ++ void __iomem *warm_wrapper; ++ struct device_node *domains[5], *spi0_pmgr; ++ struct a9_aop_spi0 spi0; ++ const char *experiment_step; ++ struct a9_sl_header *shutdown_log; ++ char *shutdown_text; + }; + + static bool apple_aop_is_started(struct apple_aop *aop) +@@ -261,6 +282,409 @@ static bool apple_aop_halt(struct apple_aop *aop) + APPLE_AOP_HALT_TIMEOUT_US); + } + ++static bool a9_aop_ramoops_ready(void) ++{ ++ struct console *con; ++ bool ready = false; ++ int cookie; ++ ++ cookie = console_srcu_read_lock(); ++ for_each_console_srcu(con) { ++ if (!strcmp(con->name, "ramoops") && ++ (console_srcu_read_flags(con) & (CON_ENABLED | CON_SUSPENDED)) == ++ CON_ENABLED) { ++ ready = true; ++ break; ++ } ++ } ++ console_srcu_read_unlock(cookie); ++ return ready; ++} ++ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* J71t warm reload integration. SPI0 machinery is the release6 wake16 code. */ ++#define AOP_DOMAINS 5 ++#define AOP_CPU_CONTROL 0x44 ++#define AOP_CPU_START BIT(4) ++#define AOP_OUTBOX_CONTROL 0x4020 ++#define AOP_INBOX_CONTROL 0x4008 ++#define AOP_OUTBOX_ENABLE BIT(0) ++#define AOP_MBOX_FAULTS GENMASK(19, 18) ++#define cold_defer 1 ++#define spi0_untested false ++ ++static u32 aop_rd(struct apple_aop *aop, u32 offset) ++{ ++ return readl(aop->warm_wrapper + offset); ++} ++ ++static bool a9_aop_force_stopped(struct apple_aop *aop) ++{ ++ return !(aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) && ++ !readl(aop->control) && ++ !(aop_rd(aop, AOP_OUTBOX_CONTROL) & (AOP_OUTBOX_ENABLE | AOP_MBOX_FAULTS)) && ++ !(aop_rd(aop, AOP_INBOX_CONTROL) & AOP_MBOX_FAULTS); ++} ++ ++struct aop_warm_write { void __iomem *target; u32 value; }; ++static void aop_warm_store(void *data) ++{ ++ struct aop_warm_write *w = data; ++ ++ writel(w->value, w->target); ++ readl(w->target); ++} ++ ++static int aop_wr(struct apple_aop *aop, u32 offset, u32 value) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct aop_warm_write w = { aop->warm_wrapper + offset, value }; ++ ++ if ((aop_rd(aop, AOP_INBOX_CONTROL) | aop_rd(aop, AOP_OUTBOX_CONTROL)) & ++ AOP_MBOX_FAULTS) ++ return -EIO; ++ dev_info(aop->dev, "COLD-W step=%s bank=wrapper offset=%04x before=%08x value=%08x\n", ++ aop->experiment_step ?: "warm-stop", offset, readl(w.target), value); ++ { ++ int ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, aop_warm_store, &w); ++ ++ dev_info(aop->dev, "COLD-R offset=%04x after=%08x\n", offset, readl(w.target)); ++ return ret; ++ } ++} ++ ++static int a9_aop_cold_checkpoint_noflush(struct apple_aop *aop, const char *step) ++{ ++ if (!a9_aop_ramoops_ready()) ++ return -ENODEV; ++ dev_info(aop->dev, "COLD checkpoint=%s flush=skipped\n", step); ++ return 0; ++} ++ ++static int a9_aop_cold_checkpoint(struct apple_aop *aop, const char *step) ++{ ++ int ret = a9_aop_cold_checkpoint_noflush(aop, step); ++ ++ if (ret) ++ return ret; ++ if (!pr_flush(1000, false)) ++ return -ETIMEDOUT; ++ mb(); ++ return 0; ++} ++ ++#include "aop-warm-spi0.c.h" ++static int apple_aop_send(struct apple_aop_ep *ep, u8 opcode, u64 payload); ++ ++/* No cmdline override for untested states: default is the release6 policy. */ ++static atomic_t aop_warm_attempted = ATOMIC_INIT(0); ++ ++static int apple_aop_warm_stop(struct apple_aop *aop) ++{ ++ unsigned int fifo, i; ++ u32 offset, val; ++ int ret; ++ ++ ret = aop_wr(aop, 0x4020, aop_rd(aop, 0x4020) & ~BIT(0)); ++ if (!ret) ++ ret = aop_wr(aop, 0x4000, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0xc00, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0x44, aop_rd(aop, 0x44) & ~BIT(4)); ++ if (ret) ++ return ret; ++ writel(0, aop->control); ++ msleep(10); ++ if (!a9_aop_force_stopped(aop)) ++ return -EIO; ++ aop->force_halted = true; ++ for (fifo = 0; fifo < 2; fifo++) { ++ offset = fifo ? 0x4020 : 0x4008; ++ for (i = 0; i < 64; i++) { ++ val = aop_rd(aop, offset); ++ if (!a9_aop_force_stopped(aop) || (val & AOP_MBOX_FAULTS)) ++ return -EIO; ++ if (val & BIT(17)) ++ break; ++ readq(aop->warm_wrapper + (fifo ? 0x4038 : 0x4018)); ++ } ++ if (!(aop_rd(aop, offset) & BIT(17))) ++ return -ETIMEDOUT; ++ } ++ aop->force_drained = true; ++ return 0; ++} ++ ++static void apple_aop_warm_put_node(void *node) ++{ ++ of_node_put(node); ++} ++ ++static void apple_aop_warm_unmap(void *p) ++{ ++ memunmap(p); ++} ++ ++static int apple_aop_warm_prepare(struct apple_aop *aop) ++{ ++ struct device *dev = aop->dev; ++ struct device_node *np = dev->of_node, *mem; ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct a9_sl_header header; ++ struct resource res; ++ char *text; ++ u32 auto_boot, outbox; ++ unsigned int i; ++ int ret; ++ ++ if (!of_property_read_bool(aop->dev->of_node, "apple,certified-reload") || !aop->hw->sram_window) ++ return 0; ++ /* Missing the new DT is not permission to run a retained instance. */ ++ mem = of_parse_phandle(np, "memory-region", 0); ++ if (!mem) ++ return dev_err_probe(dev, -EINVAL, "warm reload requires the matching J71t DT\n"); ++ ret = of_address_to_resource(mem, 0, &res); ++ of_node_put(mem); ++ if (ret || resource_size(&res) != A9_HO_PAGE || res.start != 0x8583fc000ULL || ++ aop->akf_base != A9_SPI0_WRAPPER_PA) ++ return -EINVAL; ++ aop->shutdown_log = memremap(res.start, A9_HO_PAGE, MEMREMAP_WB); ++ if (!aop->shutdown_log) ++ return -ENOMEM; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_unmap, aop->shutdown_log); ++ if (ret) ++ return ret; ++ /* Keep release6's transcript location and validation ABI. */ ++ aop->shutdown_log = (void *)aop->shutdown_log + A9_HO_LOG_OFFSET; ++ aop->shutdown_text = (char *)(aop->shutdown_log + 1); ++ memcpy(&header, aop->shutdown_log, sizeof(header)); ++ text = devm_kmemdup(dev, aop->shutdown_text, A9_SL_CAPACITY, GFP_KERNEL); ++ if (!text) ++ return -ENOMEM; ++ if (header.magic == A9_SL_MAGIC) { ++ aop->spi0.prev.state = A9_SPI0_PREV_INVALID; ++ if (a9_sl_valid(&header, text)) { ++ aop->spi0.prev.state = A9_SPI0_PREV_VALID; ++ a9_spi0_prev_parse(&aop->spi0.prev, text, header.length); ++ for (i = 0; i < header.length;) { ++ char *next = memchr(text + i, '\n', header.length - i); ++ unsigned int n = next ? next - text - i : header.length - i; ++ ++ dev_info(dev, "AOP-PREV %.*s\n", n, text + i); ++ i += n + 1; ++ } ++ } ++ } ++ aop->warm_wrapper = devm_ioremap(dev, aop->akf_base, 0x1c000); ++ if (!aop->warm_wrapper) ++ return -ENOMEM; ++ for (i = 0; i < AOP_DOMAINS; i++) { ++ aop->domains[i] = of_parse_phandle(np, "apple,observe-domains", i); ++ if (!aop->domains[i]) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->domains[i]); ++ if (ret) ++ return ret; ++ } ++ aop->spi0_pmgr = of_parse_phandle(np, "apple,spi0-domain", 0); ++ if (!aop->spi0_pmgr) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->spi0_pmgr); ++ if (ret) ++ return ret; ++ /* Check all domains before reading the wrapper; no power transitions. */ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, a9_aop_spi0_pmgr_on, &(bool){false}); ++ if (ret) ++ return dev_err_probe(dev, ret, "AOP domain guard refused\n"); ++ auto_boot = readl(aop->control); ++ outbox = aop_rd(aop, AOP_OUTBOX_CONTROL); ++ ret = a9_aop_spi0_policy(aop, auto_boot, outbox); ++ if (ret) ++ return ret; /* Before first halt, mask, SRAM or SPI0 write. */ ++ /* Orchard's old halt leaves +0x4020 enabled. On a known kexec, ++ * that alone cannot certify the cold wrapper baseline. Release6's ++ * wrapper discriminator was tied to its own outgoing stop routine. ++ */ ++ if (of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev) != A9_SPI0_PREV_HALTED) { ++ dev_info(dev, "SPI0 policy decision=refuse reason=uncertified-kexec auto_boot=%u out4020=%08x: block not touched\n", ++ auto_boot, outbox); ++ return -EPERM; ++ } ++ aop->warm = true; ++ aop->spi0.reason = a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev); ++ if (!a9_spi0_reason_runs(aop->spi0.reason)) { ++ /* Original cold path: no halt, drain, reset, or SPI0 access. */ ++ dev_info(dev, "SPI0 reset decision=skip reason=%s: block not touched\n", ++ a9_spi0_reason_name(aop->spi0.reason)); ++ return 0; ++ } ++ if (atomic_xchg(&aop_warm_attempted, 1)) ++ return dev_err_probe(dev, -EPERM, "refused-second-experiment-attempt\n"); ++ if (!a9_aop_ramoops_ready()) ++ return dev_err_probe(dev, -ENODEV, "warm reload requires ramoops console\n"); ++ aop->spi0.base = devm_ioremap(dev, A9_SPI0_PA, A9_SPI0_SIZE); ++ if (!aop->spi0.base) ++ return -ENOMEM; ++ /* Consume certification before the first control write. Reloading the ++ * module after a failure must not reuse it as a second attempt. ++ */ ++ aop->shutdown_log->magic = 0; ++ wmb(); ++ ret = apple_aop_warm_stop(aop); ++ if (!ret) ++ ret = a9_aop_spi0_prereset(aop, auto_boot, outbox); ++ if (ret) ++ dev_err(dev, "warm reload refused errno=%d; no copy/RUN, recover with DFU\n", ret); ++ return ret; ++} ++ ++static int apple_aop_warm_release(struct apple_aop *aop) ++{ ++ int ret; ++ ++ ret = a9_aop_spi0_recheck(aop); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-window"); ++ if (!ret) ++ ret = a9_aop_intc_ack(aop); ++ if (!ret) ++ ret = a9_aop_intc_reset(aop); ++ if (!ret) ++ ret = a9_aop_cold_checkpoint(aop, "before-run"); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "pre-run-final", "FINAL", true); ++ return ret; ++} ++ ++static void apple_aop_warm_hash(struct apple_aop *aop, u8 *digest) ++{ ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ if (aop->ext) ++ sha256_update(&ctx, aop->ext, aop->ext_size); ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) ++ if (aop->eps[i].buf) ++ sha256_update(&ctx, aop->eps[i].buf, aop->eps[i].buf_size); ++ sha256_final(&ctx, digest); ++} ++ ++static __printf(2, 3) void apple_aop_warm_log(struct apple_aop *aop, const char *fmt, ...) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ va_list args; ++ char line[512]; ++ size_t n; ++ ++ va_start(args, fmt); ++ n = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ dev_info(aop->dev, "%s", line); ++ if (n > h->capacity - h->length) { ++ h->truncated++; ++ return; ++ } ++ memcpy(aop->shutdown_text + h->length, line, n); ++ wmb(); ++ h->checksum = a9_sl_hash(h->checksum, line, n); ++ h->length += n; ++ h->lines++; ++ wmb(); ++} ++ ++static void apple_aop_warm_shutdown(struct apple_aop *aop) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ unsigned long deadline = jiffies + msecs_to_jiffies(10000), start; ++ unsigned int i, acked = 0; ++ int ret = -EIO; ++ ++ memset(h, 0, sizeof(*h)); ++ h->version = A9_SL_VERSION; ++ h->capacity = A9_SL_CAPACITY; ++ h->checksum = A9_SL_HASH_SEED; ++ h->mode = 1; ++ h->begin_ns = ktime_get_boottime_ns(); ++ wmb(); ++ h->magic = A9_SL_MAGIC; ++ apple_aop_warm_log(aop, "SHUTDOWN begin mode=quiesce hello=%d phase=stopping iop_crashed=%d\n", ++ aop->hello, aop->iop_crashed); ++ if (!aop->hello || aop->iop_crashed || aop->shutdown_error) ++ goto done; ++ /* Rates-zero has stopped reports. Stop one AFK at a time, max 2s ++ * per endpoint and 10s in total, including send time. Never force halt ++ * after a failed ACK. Keep every grant allocated for the entire hook. ++ */ ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ unsigned long limit; ++ ++ if (!ep->initialized) ++ continue; ++ if (time_after_eq(jiffies, deadline)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ reinit_completion(&ep->stopped); ++ start = jiffies; ++ ret = apple_aop_send(ep, AFK_OPC_SHUTDOWN, 0); ++ if (ret) ++ goto done; ++ limit = time_before(jiffies, deadline) ? min_t(unsigned long, ++ deadline - jiffies, msecs_to_jiffies(2000)) : 0; ++ if (!limit || !wait_for_completion_timeout(&ep->stopped, limit)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ acked++; ++ apple_aop_warm_log(aop, "QUIESCE step=afk-ack ep=0x%x state=stopped ms=%u\n", ++ ep->index, jiffies_to_msecs(jiffies - start)); ++ } ++ /* release6: complete every AFK stop before disabling endpoint routing. */ ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; ++ } ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ ++ if (!ep->initialized) ++ continue; ++ /* MGMT STARTEP bit 0 disables routing, no separate stop ACK. */ ++ ret = apple_rtkit_send_message(aop->rtk, 0, ++ (5ULL << 52) | ((u64)ep->index << 32) | 1, NULL, false); ++ if (ret) ++ goto done; ++ } ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; /* Never certify an incompletely initialized instance. */ ++ } ++ ret = apple_rtkit_shutdown_preserve(aop->rtk); ++ if (ret) ++ goto done; ++ msleep(20); ++ apple_aop_warm_hash(aop, before); ++ msleep(20); ++ apple_aop_warm_hash(aop, after); ++ ret = memcmp(before, after, sizeof(before)) ? -EBUSY : 0; ++ apple_aop_warm_log(aop, "QUIESCE step=ram-stable window_ms=20 status=%d\n", ret); ++ if (ret || aop->iop_crashed) ++ goto done; ++ ret = apple_aop_warm_stop(aop); ++done: ++ apple_aop_warm_log(aop, "QUIESCE step=complete status=%d quiesced=%d halted=%d drained=%d steps=0x%x afk_acks=%u iop_crashed=%d\n", ++ ret, !ret, aop->force_halted, aop->force_drained, ++ !ret ? 0x1bff : 0, acked, aop->iop_crashed); ++ if (ret) ++ dev_err(aop->dev, "quiesce failed; next AOP must refuse, recover with DFU\n"); ++} ++ + /* Firmware loading */ + + struct apple_aop_bootarg { +@@ -356,6 +780,7 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + __le64 ecid_le = cpu_to_le64(ecid), ext_le, wrapper_le; + __le32 target_le, chip_le, revision_le; + unsigned int count = 0; ++ size_t offset; + u8 *image; + int ret; + +@@ -424,10 +849,21 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + } + + ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, count); +- if (ret) ++ if (ret) { + dev_err(aop->dev, "unexpected boot arguments in %s: %d\n", main_name, ret); +- else ++ } else if (aop->spi0.ran) { ++ /* Warm reload: copy and verify the image with SPI0 gated around it */ ++ ret = a9_aop_spi0_gate(aop, "pre-copy-final", "PRECOPY", false); ++ if (!ret) ++ ret = a9_aop_copy_verify_wake10(aop, image, main_fw->size); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "after-copy", "AFTERCOPY", true); ++ if (!ret) ++ for (offset = main_fw->size; offset + 4 <= aop->sram_size; offset += 4) ++ writel_relaxed(0, aop->sram + offset); ++ } else { + apple_aop_write_sram(aop, image, main_fw->size); ++ } + + kfree_sensitive(image); + memzero_explicit(prng, sizeof(prng)); +@@ -571,6 +1007,8 @@ static void apple_aop_report(struct apple_aop_ep *ep, u32 channel, u16 subtype, + struct apple_aop_listener *listener; + bool handled = false; + ++ if (ep->index == 0x21 && subtype == 0xc2) ++ aop->accel_reports++; + mutex_lock(&aop->listeners_lock); + list_for_each_entry(listener, &aop->listeners, list) { + if (listener->endpoint == ep->index && listener->channel == channel) { +@@ -727,6 +1165,7 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + + switch (FIELD_GET(AFK_MSG_OPCODE, msg)) { + case AFK_OPC_INIT: ++ ep->initialized = true; + apple_aop_send(ep, AFK_OPC_INIT_ACK, 0); + break; + case AFK_OPC_GET_BUF: +@@ -747,7 +1186,10 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + break; + case AFK_OPC_INIT_UNK: + case AFK_OPC_START_ACK: ++ break; + case AFK_OPC_SHUTDOWN_ACK: ++ case AFK_OPC_HIBERNATE_ACK: ++ complete(&ep->stopped); + break; + default: + dev_dbg(aop->dev, "endpoint 0x%02x: message 0x%llx\n", endpoint, msg); +@@ -758,7 +1200,8 @@ static void apple_aop_crashed(void *cookie, const void *crashlog, size_t crashlo + { + struct apple_aop *aop = cookie; + +- dev_err(aop->dev, "firmware crashed\n"); ++ aop->iop_crashed = true; ++ dev_err(aop->dev, "firmware crashed iop_crashed=1\n"); + } + + static const struct apple_rtkit_ops apple_aop_rtkit_ops = { +@@ -863,6 +1306,8 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + + if (!ep) + return -EINVAL; ++ if (service->aop->warm && READ_ONCE(service->aop->iop_crashed)) ++ return -EIO; + init_completion(&call.done); + + mutex_lock(&ep->lock); +@@ -997,7 +1442,11 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + return ret; + +- if (apple_aop_is_started(aop)) ++ ret = apple_aop_warm_prepare(aop); ++ if (ret) ++ return ret; ++ ++ if (!aop->spi0.ran && apple_aop_is_started(aop)) + return dev_err_probe(dev, -EBUSY, + "already started; the AOP starts once per boot\n"); + +@@ -1023,6 +1472,7 @@ static int apple_aop_probe(struct platform_device *pdev) + aop->eps[i].aop = aop; + aop->eps[i].index = AFK_ENDPOINT_START + i; + mutex_init(&aop->eps[i].lock); ++ init_completion(&aop->eps[i].stopped); + } + + ret = apple_aop_load(aop, ecid); +@@ -1050,10 +1500,24 @@ static int apple_aop_probe(struct platform_device *pdev) + prop = of_find_property(np, "apple,ecid", NULL); + if (prop) + of_remove_property(np, prop); +- writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, +- aop->akf + APPLE_AOP_CPU_CONTROL); ++ if (aop->spi0.ran) { ++ ret = apple_aop_warm_release(aop); ++ if (ret) ++ goto err_halt; ++ } ++ if (aop->spi0.ran) { ++ aop->experiment_step = "warm-run"; ++ ret = aop_wr(aop, APPLE_AOP_CPU_CONTROL, ++ readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN); ++ if (ret) ++ goto err_halt; ++ } else { ++ writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, ++ aop->akf + APPLE_AOP_CPU_CONTROL); ++ } + writel(1, aop->control + APPLE_AOP_CONTROL_AUTO_BOOT); + ++ dev_info(dev, "AOP released warm=%d\n", aop->spi0.ran); + /* + * The firmware was just loaded and starts from scratch: wait for its + * hello and its own power state report, as for any freshly started +@@ -1073,6 +1537,8 @@ static int apple_aop_probe(struct platform_device *pdev) + dev_warn(dev, "failed to start endpoint 0x%02x: %d\n", i, ret); + } + ++ aop->hello = true; ++ dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); + return 0; + +@@ -1102,12 +1568,41 @@ static void apple_aop_shutdown(struct platform_device *pdev) + mutex_lock(&aop->children_lock); + aop->removing = true; + list_for_each_entry_safe(child, tmp, &aop->children, list) { ++ ++ if (aop->warm) { ++ const struct apple_aop_service *service = dev_get_platdata(&child->pdev->dev); ++ static const u32 keys[] = { 0x16, 0x5c, 0 }; ++ ++ /* release6 sets the three accelerometer rates only. */ ++ if (strcmp(service->name, "accel")) ++ continue; ++ __le32 zero = 0; ++ unsigned int i; ++ int ret; ++ ++ for (i = 0; i < ARRAY_SIZE(keys); i++) { ++ ret = apple_aop_set_property(service, keys[i], &zero, sizeof(zero)); ++ if (ret) { ++ aop->shutdown_error = ret; ++ break; ++ } ++ } ++ } ++ /* Shutdown retains listeners, children and DMA grants, as release6 does. ++ * Unregister is remove-time work and can wait in IIO/driver core. ++ */ ++ if (aop->warm) ++ continue; + platform_device_unregister(child->pdev); + list_del(&child->list); + kfree(child); + } + mutex_unlock(&aop->children_lock); + ++ if (aop->warm) { ++ apple_aop_warm_shutdown(aop); ++ return; ++ } + if (!apple_aop_halt(aop)) + dev_err(aop->dev, "failed to halt\n"); + } +@@ -1129,11 +1624,26 @@ static const struct of_device_id apple_aop_of_match[] = { + }; + MODULE_DEVICE_TABLE(of, apple_aop_of_match); + ++static ssize_t warm_state_show(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ struct apple_aop *aop = dev_get_drvdata(dev); ++ ++ if (!aop) ++ return sysfs_emit(buf, "owned=0\n"); ++ return sysfs_emit(buf, "owned=%u reload=%u iop_crashed=%u accel_reports=%llu\n", ++ aop->hello, aop->spi0.ran, READ_ONCE(aop->iop_crashed), ++ READ_ONCE(aop->accel_reports)); ++} ++static DEVICE_ATTR_RO(warm_state); ++static struct attribute *apple_aop_attrs[] = { &dev_attr_warm_state.attr, NULL }; ++ATTRIBUTE_GROUPS(apple_aop); ++ + static struct platform_driver apple_aop_driver = { + .driver = { + .name = "apple-aop", + .of_match_table = apple_aop_of_match, + .suppress_bind_attrs = true, ++ .dev_groups = apple_aop_groups, + }, + .probe = apple_aop_probe, + .shutdown = apple_aop_shutdown, +diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c +index 725ab3fa7..d7928621a 100644 +--- a/drivers/soc/apple/rtkit.c ++++ b/drivers/soc/apple/rtkit.c +@@ -4,6 +4,8 @@ + * Copyright (C) The Asahi Linux Contributors + */ + ++#include ++#include + #include "rtkit-internal.h" + + enum { +@@ -867,6 +869,54 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) + } + EXPORT_SYMBOL_GPL(apple_rtkit_boot); + ++/* Unlike shutdown(), retain all grants until the caller halts the CPU. ++ * Reinit frees DMA buffers and is unsafe before that point on S8000. ++ */ ++static void apple_rtkit_grants_hash(struct apple_rtkit *rtk, u8 *digest) ++{ ++ struct apple_rtkit_shmem *buffers[] = { ++ &rtk->ioreport_buffer, &rtk->crashlog_buffer, ++ &rtk->oslog_buffer, &rtk->syslog_buffer, ++ }; ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ for (i = 0; i < ARRAY_SIZE(buffers); i++) ++ if (buffers[i]->buffer) ++ sha256_update(&ctx, buffers[i]->buffer, buffers[i]->size); ++ sha256_final(&ctx, digest); ++} ++ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk) ++{ ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ if (rtk->crashed) ++ return -EIO; ++ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_QUIESCED); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=ap-ack state=0x10\n"); ++ ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=iop-ack state=0x201\n"); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, before); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, after); ++ if (rtk->crashed || memcmp(before, after, sizeof(before))) ++ return -EBUSY; ++ /* No more callback can race the caller's halt/drain. Do not reinit. */ ++ apple_mbox_stop(rtk->mbox); ++ flush_workqueue(rtk->wq); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_rtkit_shutdown_preserve); ++ + int apple_rtkit_shutdown(struct apple_rtkit *rtk) + { + int ret; +diff --git a/include/linux/soc/apple/pmgr-observe.h b/include/linux/soc/apple/pmgr-observe.h +new file mode 100644 +index 000000000..477b7fa22 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-observe.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++#define _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++ ++#include ++ ++struct device_node; ++ ++struct apple_pmgr_observation { ++ u32 raw; ++ int genpd_status; ++}; ++ ++/* ++ * Diagnostic only. Nodes must be in child-before-parent lock order, with no ++ * duplicates (maximum five). Takes no power/runtime-PM reference. read() must ++ * not sleep or acquire genpd locks; it runs with IRQs disabled and all domain ++ * locks held, only if every genpd and hardware actual/target state is ON and ++ * reset/disable/parent-off are clear. Hardware auto-PM remains untouched and is ++ * recorded in raw; locks exclude software transitions, not autonomous gating. ++ * -EHOSTDOWN means read() was skipped. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data); ++ ++#endif +diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h +index 1f9ebb618..c8e8df255 100644 +--- a/include/linux/soc/apple/rtkit.h ++++ b/include/linux/soc/apple/rtkit.h +@@ -124,6 +124,8 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); + * Shutdown the co-processor + */ + int apple_rtkit_shutdown(struct apple_rtkit *rtk); ++/* AP 0x10 / IOP 0x201, stable system RAM, stop RX; retain every grant. */ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk); + + /* + * Stop the mailbox of a shut-down co-processor, so that its power domain can From d32d990da878ab56f62a16943575f9393224c476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 11:10:18 +0100 Subject: [PATCH 29/33] platform: Keep every board's pristine device tree for warm restart From #110, the part main did not take with the stable USB serial: the platform layer embeds each supported board's pristine DTB and a compatible-to-file map in the initrd. Before switching root, the initrd copies the running board's tree to /run/orchard-kexec/pristine.dtb, or, after a warm restart, the retained tree the previous kernel named with orchard_warm.dtb=, and records the boot ID next to it. Warm restart boots the next kernel with that tree, never with one rebuilt from an updated /boot. No feature table is added: whether warm restart is offered is decided from the mechanisms it uses, not from a feature state. Assisted-by: LLM --- boot/initramfs/root/init | 26 ++++++++++++++++++++++++++ src/orchard/image/platform.py | 15 ++++++++++++--- tests/test_platform_layer.py | 2 ++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index 16c7913a..86cc9c65 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -89,6 +89,32 @@ mkdir -p /newroot mount ${fstype:+-t "$fstype"} -o "$mode${flags:+,$flags}" "$device" /newroot || stop "could not mount $device" [ -x "/newroot$init" ] || [ -L "/newroot$init" ] || stop "the root has no $init" +# Preserve the running boot's *pristine* tree before any package update can +# replace /boot or same-release module assets. The matching initrd embeds cold +# DTBs; warm boots name their retained stage instead. Never infer it by uname. +mkdir -p /run /newroot/run +mount -t tmpfs -o mode=0755,nosuid,nodev tmpfs /run +mkdir -p /run/orchard-kexec +warm_pristine= +for argument in $(cat /proc/cmdline); do + case "$argument" in orchard_warm.dtb=*) warm_pristine=${argument#orchard_warm.dtb=} ;; esac +done +if [ -n "$warm_pristine" ]; then + case "$warm_pristine" in + /var/tmp/orchard-warm.*/pristine.dtb) + case "$warm_pristine" in *..*) stop 'invalid warm DT provenance' ;; esac + cp "/newroot$warm_pristine" /run/orchard-kexec/pristine.dtb || stop 'missing warm pristine DT' ;; + *) stop 'invalid warm DT provenance' ;; + esac +elif [ -f /etc/orchard/pristine-map ]; then + board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + while read compatible file; do + [ "$compatible" != "$board" ] || cp "/etc/orchard/dtbs/$file" /run/orchard-kexec/pristine.dtb +done < /etc/orchard/pristine-map +fi +cat /proc/sys/kernel/random/boot_id > /run/orchard-kexec/boot-id +mount --move /run /newroot/run || stop 'cannot retain boot provenance' + # USB for the host: network (usb0) and a serial console (ttyGS0), named after # the system, so `orchard devices` sees it. What runs on them is the system's. usb_gadget() { diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index ee55ae55..997d8180 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -85,7 +85,7 @@ def tools(hardware=None, cache=None, platform='apple-16k'): return prepare_tools(hardware, cache, device=sorted(boards(platform, hardware))[0]) / 'busybox' -def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES): +def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES, pristine_dtbs=None): """orchard's initrd for installed systems: BusyBox, the storage modules and their dependencies. MODULES is {path: (data, mode)} as module_files gives it. @@ -116,6 +116,15 @@ def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_ entries['etc/orchard/modules'] = (stat.S_IFREG | 0o644, ('\n'.join(loaded) + '\n').encode()) if usb: entries['etc/orchard/usb-modules'] = (stat.S_IFREG | 0o644, ('\n'.join(usb) + '\n').encode()) + if pristine_dtbs: + entries['etc/orchard/dtbs'] = (directory, b'') + mapping = [] + for board, data in sorted(pristine_dtbs.items()): + if not board.replace('-', '').isalnum(): + raise ValueError('Invalid pristine DT board') + entries[f'etc/orchard/dtbs/{board}.dtb'] = (stat.S_IFREG | 0o644, data) + mapping.append(f'apple,{board} {board}.dtb\n') + entries['etc/orchard/pristine-map'] = (stat.S_IFREG | 0o644, ''.join(mapping).encode()) dependency_lines = [] for name in sorted(wanted): target = f'lib/modules/{release}/{name}' @@ -154,13 +163,13 @@ def files(bundle, platform, board_dtbs, *, busybox, hardware=None): base = f'usr/lib/modules/{release}/' result[base + 'vmlinuz'] = ((bundle / described['image']).read_bytes(), 0o644) modules_to_load = storage_modules(hardware, platform) - result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, - storage_modules=modules_to_load), 0o644) for dtb in sorted(set(board_dtbs.values())): path = bundle / 'dtbs' / (dtb + '.dtb') if path.is_symlink() or not path.is_file(): raise ValueError(f'Kernel bundle lacks {path.name}') result[base + 'dtb/' + path.name] = (path.read_bytes(), 0o644) + pristine = {board: result[base + 'dtb/' + dtb + '.dtb'][0] for board, dtb in board_dtbs.items()} + result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, pristine_dtbs=pristine, storage_modules=modules_to_load), 0o644) result.update(regulatory_files()) defaults = hardware / 'system/defaults' for path in sorted(defaults.rglob('*')): diff --git a/tests/test_platform_layer.py b/tests/test_platform_layer.py index 013e405a..b7029826 100644 --- a/tests/test_platform_layer.py +++ b/tests/test_platform_layer.py @@ -84,6 +84,8 @@ def test_the_initrd_carries_busybox_and_the_storage_modules_only(self): self.assertEqual(names['init'], (ROOT / 'boot/initramfs/root/init').read_bytes()) self.assertEqual(names['etc/orchard/usb-identity'], (ROOT / 'src/orchard/usb-identity').read_bytes()) self.assertEqual(names['etc/orchard/modules'], b'nvme\n') + self.assertEqual(names['etc/orchard/dtbs/j71t.dtb'], b's8003-j71t') + self.assertIn(b'apple,j71t j71t.dtb\n', names['etc/orchard/pristine-map']) self.assertEqual(sorted(name for name in names if name.endswith('.ko')), [base + 'kernel/drivers/nvme/host/nvme-core.ko', base + 'kernel/drivers/nvme/host/nvme.ko']) self.assertEqual(names[base + 'modules.dep'], From 7e6d4595c4e458318692a7ceed17fa0026b3f9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 11:11:37 +0100 Subject: [PATCH 30/33] kernel: Refuse a system sleep while the AOP firmware is not settled From #101, the suspend precheck that main did not take with the sensors. It belongs with suspend: it makes the AOP refuse a pre-sync system sleep while its firmware is starting or stopping. Assisted-by: LLM --- ...fuse-pre-sync-sleep-during-firmware-.patch | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch diff --git a/kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch b/kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch new file mode 100644 index 00000000..8a4e6613 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch @@ -0,0 +1,80 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] soc: apple: aop: refuse pre-sync sleep during firmware boot + +A suspend while the AOP firmware is still starting stops it part way +through its boot, and the sensors behind it do not come back. Register a +suspend precheck notifier that refuses sleep with -EBUSY until the +firmware has reported ready. + +Assisted-by: LLM +--- + drivers/soc/apple/aop.c | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -226,6 +227,8 @@ + }; + + struct apple_aop { ++ struct notifier_block sleep_precheck; ++ bool sleep_ready; + struct device *dev; + const struct apple_aop_hw *hw; + void __iomem *akf; +@@ -1385,6 +1388,22 @@ + dma_free_coherent(aop->dev, aop->ext_size, aop->ext, aop->ext_dma); + } + ++static int apple_aop_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_aop *aop = container_of(nb, struct apple_aop, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ return READ_ONCE(aop->sleep_ready) ? NOTIFY_OK : notifier_from_errno(-EBUSY); ++} ++ ++static void apple_aop_sleep_unregister(void *data) ++{ ++ struct apple_aop *aop = data; ++ ++ unregister_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++} ++ + static int apple_aop_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -1399,6 +1418,13 @@ + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->sleep_precheck.notifier_call = apple_aop_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++ if (ret) ++ return ret; ++ ret = devm_add_action_or_reset(dev, apple_aop_sleep_unregister, aop); ++ if (ret) ++ return ret; + aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); +@@ -1514,6 +1540,7 @@ + aop->hello = true; + dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); ++ WRITE_ONCE(aop->sleep_ready, true); + return 0; + + err_halt: From ce7d3b25bcb3d677c14c2cac9912270448703c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 11:12:01 +0100 Subject: [PATCH 31/33] kernel: Require a proven VXD firmware stop before kexec From #121, the patch main did not take with HEVC. It uses the VXD sleep stop and sleep gate this branch adds, so it sits here rather than with the earlier warm restart branch: a failed firmware stop keeps the firmware, command buffers and power, and kexec takes the cold-reset path before those pages become the next kernel's RAM. Assisted-by: LLM --- ...require-a-proven-firmware-stop-befor.patch | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch diff --git a/kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch b/kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch new file mode 100644 index 00000000..3670d596 --- /dev/null +++ b/kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch @@ -0,0 +1,120 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:51 +0100 +Subject: [PATCH] media: apple-vxd: require a proven firmware stop before kexec + +release has no shutdown callback and no B1 ProbeTerminate path. Add the +checked S8000 stop at shutdown: failed Terminate keeps firmware and +command buffers plus power, and kexec takes the cold-reset escape before +those pages become incoming RAM. Successful stop may release buffers and +force suspend. Other variants keep their shutdown behavior. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 26 ++++++++++++++ + .../media/platform/apple/vxd/apple-vxd-fw.c | 35 +++++++++++++++++++ + drivers/media/platform/apple/vxd/apple-vxd.h | 1 + + 3 files changed, 62 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index b6bf2d300..c89e7e987 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -1,3 +1,6 @@ ++#include ++#include ++ + // SPDX-License-Identifier: GPL-2.0 + /* + * Apple VXD (Imagination MSVDX) video decoder driver - core +@@ -476,9 +479,32 @@ static void apple_vxd_remove(struct platform_device *pdev) + pm_runtime_disable(&pdev->dev); + } + ++static void apple_vxd_shutdown(struct platform_device *pdev) ++{ ++ struct apple_vxd_dev *vxd = platform_get_drvdata(pdev); ++ int ret; ++ ++ if (!vxd->variant->sleep_stop) ++ return; ++ ++ cancel_delayed_work_sync(&vxd->watchdog_work); ++ flush_work(&vxd->job_done_work); ++ ret = apple_vxd_fw_shutdown_stop(vxd); ++ if (ret) { ++ /* Outgoing allocations cannot be reserved against the next kernel. */ ++ if (kexec_in_progress) ++ apple_cold_reset("VXD Terminate failed; DMA buffers cannot cross kexec"); ++ return; /* Retain every buffer and the runtime-PM reference. */ ++ } ++ ret = pm_runtime_force_suspend(&pdev->dev); ++ if (ret) ++ dev_warn(&pdev->dev, "power off at shutdown failed: %d\n", ret); ++} ++ + static struct platform_driver apple_vxd_driver = { + .probe = apple_vxd_probe, + .remove = apple_vxd_remove, ++ .shutdown = apple_vxd_shutdown, + .driver = { + .name = APPLE_VXD_DRV_NAME, + .pm = &apple_vxd_pm_ops, +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-fw.c b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +index ede737efd..bccd04259 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-fw.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +@@ -512,6 +512,41 @@ int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd) + return 0; + } + ++int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ vxd->sleep_gate = true; ++ if (!vxd->fw_loaded) { ++ mutex_unlock(&vxd->power_lock); ++ return 0; ++ } ++ ++ apple_vxd_mbox_stop(vxd); ++ vxd->hw_ready = false; ++ vxd_write(vxd, VXD_REG_FW_READY_MAGIC, 0); ++ vxd_write(vxd, VXD_REG_VECT_CFG, VXD_VECT_CFG_TERMINATE); ++ ret = vxd_known_state_v1(vxd, "Terminate"); ++ if (ret) { ++ dev_err(vxd->dev, "firmware stop at shutdown failed: %d; retaining buffers and power\n", ret); ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++ } ++ ++ apple_vxd_cmdbufs_fini(vxd); ++ apple_vxd_gart_unmap(vxd, &vxd->fw_map); ++ dma_free_coherent(vxd->dev, vxd->fw_size, vxd->fw_vaddr, vxd->fw_dma); ++ vxd->fw_vaddr = NULL; ++ vxd->fw_loaded = false; ++ mutex_unlock(&vxd->power_lock); ++ ++ /* The reference boot took; the PM core holds its own until complete. */ ++ pm_runtime_put_noidle(vxd->dev); ++ dev_info(vxd->dev, "firmware stopped at shutdown\n"); ++ return 0; ++} ++ + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd) + { + bool was_loaded; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index 7922ebb73..b601ea6b7 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -471,6 +471,7 @@ int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd); + int apple_vxd_fw_open(struct apple_vxd_dev *vxd); + void apple_vxd_fw_close(struct apple_vxd_dev *vxd); + int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *vxd); + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd); + + /* mbox.c */ From e3800e6f43ef8ddf7e3ead490ca7dbae05444415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 12:42:40 +0100 Subject: [PATCH 32/33] j71t: Ship the G5P firmware loader again Bringing this branch onto main took main's devices/j71t/firmware.json, which main rewrote with its AOP firmware, and lost the output the GPU needs: lib/firmware/apple/agx-g5p-c0.ldr, extracted from the kernelcache by agx-g5p-ldr-from-kernelcache@1 (src/orchard/recipe.py still has the step). Without it the G5P driver finds no loader and the GPU never starts ("autoboot stopped at validate (-2)"). Put the entry back after main's outputs, unchanged; the file is again main's plus this one output. Assisted-by: LLM --- devices/j71t/firmware.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index c03fb880..39e7e406 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -313,6 +313,16 @@ "coprocessor": "AOP_ext" } ] + }, + { + "destination": "lib/firmware/apple/agx-g5p-c0.ldr", + "sha256": "33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca", + "steps": [ + { + "step": "agx-g5p-ldr-from-kernelcache@1", + "input": "kernelcache" + } + ] } ] } From ee634f8d2dda9cfa1ca55fa5d0a3e00096c1a6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= Date: Fri, 13 Nov 2026 13:26:28 +0100 Subject: [PATCH 33/33] tests: Check J71t's G5P loader and that every recipe step is used J71t's agx-g5p-c0.ldr output was lost when main's firmware.json was merged in, while its recipe step stayed, and no test noticed: the GPU simply never started. Assert that J71t ships the loader, and that every step in orchard.recipe.STEPS is used by some board's firmware output, which holds on main as well. Assisted-by: LLM --- tests/test_firmware_recipes.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_firmware_recipes.py b/tests/test_firmware_recipes.py index 4aa3ad15..433b8c66 100644 --- a/tests/test_firmware_recipes.py +++ b/tests/test_firmware_recipes.py @@ -39,6 +39,23 @@ def test_every_recipe_matches_the_schema(self): with self.subTest(board=path.parent.name): jsonschema.validate(json.loads(path.read_text()), schema, cls=jsonschema.Draft202012Validator) + def test_every_recipe_step_is_used_by_a_board(self): + # A step no board's output uses is dead code, or the trace of an output + # that was lost while the step stayed. + from orchard.recipe import STEPS + used = {step['step'] for path in RECIPES for output in json.loads(path.read_text())['outputs'] + for step in output['steps']} + self.assertEqual(sorted(set(STEPS) - used), []) + + def test_j71t_ships_the_g5p_firmware_loader(self): + # The G5P driver loads it at boot; without it the GPU never starts. + recipe = json.loads((ROOT / 'devices/j71t/firmware.json').read_text()) + loader = [output for output in recipe['outputs'] + if output['destination'] == 'lib/firmware/apple/agx-g5p-c0.ldr'] + self.assertEqual(len(loader), 1) + self.assertEqual(loader[0]['steps'], [{'step': 'agx-g5p-ldr-from-kernelcache@1', 'input': 'kernelcache'}]) + self.assertIn('kernelcache', recipe['inputs']) + if __name__ == '__main__': unittest.main()