diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index 16c7913a..86cc9c65 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -89,6 +89,32 @@ mkdir -p /newroot mount ${fstype:+-t "$fstype"} -o "$mode${flags:+,$flags}" "$device" /newroot || stop "could not mount $device" [ -x "/newroot$init" ] || [ -L "/newroot$init" ] || stop "the root has no $init" +# Preserve the running boot's *pristine* tree before any package update can +# replace /boot or same-release module assets. The matching initrd embeds cold +# DTBs; warm boots name their retained stage instead. Never infer it by uname. +mkdir -p /run /newroot/run +mount -t tmpfs -o mode=0755,nosuid,nodev tmpfs /run +mkdir -p /run/orchard-kexec +warm_pristine= +for argument in $(cat /proc/cmdline); do + case "$argument" in orchard_warm.dtb=*) warm_pristine=${argument#orchard_warm.dtb=} ;; esac +done +if [ -n "$warm_pristine" ]; then + case "$warm_pristine" in + /var/tmp/orchard-warm.*/pristine.dtb) + case "$warm_pristine" in *..*) stop 'invalid warm DT provenance' ;; esac + cp "/newroot$warm_pristine" /run/orchard-kexec/pristine.dtb || stop 'missing warm pristine DT' ;; + *) stop 'invalid warm DT provenance' ;; + esac +elif [ -f /etc/orchard/pristine-map ]; then + board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + while read compatible file; do + [ "$compatible" != "$board" ] || cp "/etc/orchard/dtbs/$file" /run/orchard-kexec/pristine.dtb +done < /etc/orchard/pristine-map +fi +cat /proc/sys/kernel/random/boot_id > /run/orchard-kexec/boot-id +mount --move /run /newroot/run || stop 'cannot retain boot provenance' + # USB for the host: network (usb0) and a serial console (ttyGS0), named after # the system, so `orchard devices` sees it. What runs on them is the system's. usb_gadget() { diff --git a/boot/m1n1/README.md b/boot/m1n1/README.md index bd78f93b..0a7c6e1c 100644 --- a/boot/m1n1/README.md +++ b/boot/m1n1/README.md @@ -28,3 +28,16 @@ gives the same tree as Hoolock's tip before updating `sources.toml`. Scratch stays in ignored `boot/m1n1/.work//`. `--resume` reuses it only if the recorded patch set is unchanged. + +## Resident PSCI on A7-A9 + +Orchard patch 0015 adds the resident EL3 PSCI monitor (CPU_ON, CPU_OFF, +AFFINITY_INFO); 0016-0019 make it the default whenever m1n1 is entered at EL3 +and add SYSTEM_RESET (warm reboot back into m1n1, which boots the default BLS +entry with the last launch's DTB) and SYSTEM_OFF (halt). Any PSCI-aware +kernel reboots, powers off and kexecs through its ordinary paths; m1n1 takes +IPI-stopped CPUs back by trapping EL1 WFI/WFE to EL3. EL1/EL2 entries keep +the spin table. Payload variables (append to `m1n1.bin`, e.g. with +`psci-loader`): `psci=0` turns PSCI off, `psci_reset=0` keeps only the CPU +lifecycle (no WFI/WFE traps), `psci_reset=proxy` stops in the proxy after a +reset. 0020 marks retained hardware in the device tree after a re-entry. diff --git a/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch b/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch new file mode 100644 index 00000000..4eb87533 --- /dev/null +++ b/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch @@ -0,0 +1,880 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sat, 7 Nov 2026 15:00:00 +0100 +Subject: [PATCH] psci: Add opt-in resident EL3 CPU lifecycle for A7-A9 + +On A7-A9 m1n1 is entered at EL3 and stays resident: its image and the +per-CPU EL3 stacks are memreserved, and VBAR_EL3 points at its vectors +while Linux runs at EL1. Use that to give Linux PSCI 1.0 CPU_ON, CPU_OFF +and AFFINITY_INFO (plus VERSION, FEATURES and MIGRATE_INFO_TYPE). The +spin table has no cpu_die, so without this the kernel refuses kexec on +an SMP system, can't offline CPUs, and a crash kernel inherits CPUs +stuck in the old kernel. + +CPU_OFF does not power the core down. It cleans and invalidates its L1D, +stops its timers, and waits in WFE at EL3 (MMU off) on a one-line +mailbox. CPU_ON posts an entry point; the core leaves with a clean +I-cache and TLB, SCTLR_EL1 at its reset value, DAIF masked and SP_EL3 on +a static per-CPU stack. SYSTEM_OFF/SYSTEM_RESET, CPU_SUSPEND and +SMCCC_VERSION return NOT_SUPPORTED, so callers fall back to their own +methods and assume SMCCC 1.0. + +Opt-in with the payload variable psci=1, and only used when m1n1 itself +was entered at EL3. kboot then parks the secondaries in the pen (a core +that doesn't park is pruned), sets enable-method = "psci" on every CPU, +adds /psci (method = "smc") and records the re-entry address and the +original boot_args in /chosen. + +Re-entry: _reentry can be jumped to from a kexec'd trampoline (EL1, MMU +off, x0 = boot_args copy). It drops this image's lines from the caches, +flags the re-entry and restarts m1n1, which then skips the (stale) +payload area and runs the proxy. smp_start_secondaries() takes cores +back from the pen with CPU_ON, and the mailboxes and the psci=1 setting +live in .data so they survive the .bss clear. + +Rebase the resident boot shim onto Orchard's complete m1n1 series +without changing its runtime behavior. The payload variable is not a +kernel command-line option; a bare proxy loader also needs it appended. + +Assisted-by: LLM +--- + Makefile | 1 + + src/kboot.c | 56 +++++++++++++- + src/main.c | 8 +- + src/payload.c | 3 + + src/psci.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + src/psci.h | 78 ++++++++++++++++++++ + src/psci_asm.S | 170 ++++++++++++++++++++++++++++++++++++++++++ + src/smp.c | 73 ++++++++++++++++++ + src/smp.h | 1 + + src/start.S | 6 +- + src/startup.c | 6 +- + 11 files changed, 626 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index bb72266..4aefb37 100644 +--- a/Makefile ++++ b/Makefile +@@ -166,6 +166,7 @@ OBJECTS := \ + pcie.o \ + pmgr.o \ + proxy.o \ ++ psci.o psci_asm.o \ + ringbuffer.o \ + rtkit.o \ + sart.o \ +diff --git a/src/kboot.c b/src/kboot.c +index b75bb45..2316709 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -21,6 +21,7 @@ + #include "memory.h" + #include "pcie.h" + #include "pmgr.h" ++#include "psci.h" + #include "sep.h" + #include "sio.h" + #include "smc.h" +@@ -574,6 +575,53 @@ static int dt_set_smbios(void) + return 0; + } + ++/* ++ * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * (see psci.c) instead of the spin table, so the kernel can offline them ++ * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. ++ */ ++static int dt_set_psci(void) ++{ ++ static const char compat[] = "arm,psci-1.0\0arm,psci-0.2"; ++ int node; ++ ++ if (!psci_active()) { ++ printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ return 0; ++ } ++ ++ int cpus = fdt_path_offset(dt, "/cpus"); ++ if (cpus < 0) ++ bail("FDT: /cpus node not found in devtree\n"); ++ ++ fdt_for_each_subnode(node, dt, cpus) ++ { ++ if (strncmp(fdt_get_name(dt, node, NULL), "cpu@", 4)) ++ continue; ++ if (fdt_setprop_string(dt, node, "enable-method", "psci")) ++ bail("FDT: couldn't set enable-method\n"); ++ } ++ ++ node = fdt_path_offset(dt, "/psci"); ++ if (node < 0) ++ node = fdt_add_subnode(dt, 0, "psci"); ++ if (node < 0) ++ bail("FDT: couldn't add /psci\n"); ++ if (fdt_setprop(dt, node, "compatible", compat, sizeof(compat)) || ++ fdt_setprop_string(dt, node, "method", "smc")) ++ bail("FDT: couldn't fill /psci\n"); ++ ++ node = fdt_path_offset(dt, "/chosen"); ++ if (node < 0) ++ bail("FDT: /chosen node not found in devtree\n"); ++ if (fdt_setprop_u64(dt, node, "linux-a9,m1n1-reentry", (u64)_reentry) || ++ fdt_setprop(dt, node, "linux-a9,m1n1-boot-args", &cur_boot_args, sizeof(cur_boot_args))) ++ bail("FDT: couldn't set the m1n1 re-entry properties\n"); ++ ++ printf("PSCI: on, enable-method = \"psci\", m1n1 re-entry at %p\n", _reentry); ++ return 0; ++} ++ + static int dt_set_cpus(void) + { + int ret = 0; +@@ -587,6 +635,10 @@ static int dt_set_cpus(void) + if (!pruned_phandles) + bail("FDT: out of memory\n"); + ++ /* Secondaries wait in the PSCI pen; one that can't get there is pruned below */ ++ if (psci_active()) ++ printf("PSCI: parked %d secondary CPU(s)\n", smp_park_secondaries_psci()); ++ + /* Prune CPU nodes */ + int node, cpu = 0; + for (node = fdt_first_subnode(dt, cpus); node >= 0;) { +@@ -618,7 +670,7 @@ static int dt_set_cpus(void) + } else { + printf("FDT: Reserving stack for CPU %d 0x%lx\n", cpu, (uint64_t)secondary_stacks[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks[cpu], SECONDARY_STACK_SIZE); +- if (has_el3()) { ++ if (has_el3() && secondary_stacks_el3[cpu]) { + printf("FDT: Reserving EL3 stack for CPU %d 0x%lx\n", cpu, + (uint64_t)secondary_stacks_el3[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks_el3[cpu], SECONDARY_STACK_SIZE); +@@ -3206,6 +3258,8 @@ int kboot_prepare_dt(void *fdt) + return -1; + if (dt_set_cpus()) + return -1; ++ if (dt_set_psci()) ++ return -1; + if (dt_set_mac_addresses()) + return -1; + if (dt_set_wifi()) +diff --git a/src/main.c b/src/main.c +index d4e828c..3f32380 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -21,6 +21,7 @@ + #include "pcie.h" + #include "pmgr.h" + #include "sep.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "tps6598x.h" +@@ -117,9 +118,12 @@ void run_actions(void) + #endif + #endif + +- printf("Checking for payloads...\n"); ++ if (psci_reentered()) ++ printf("Re-entered from Linux: whatever follows m1n1 is stale, not loading payloads\n"); ++ else ++ printf("Checking for payloads...\n"); + +- if (payload_run() == 0) { ++ if (!psci_reentered() && payload_run() == 0) { + printf("Valid payload found\n"); + return; + } +diff --git a/src/payload.c b/src/payload.c +index 5b1547f..4ffc9e2 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -12,6 +12,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "utils.h" + +@@ -205,6 +206,8 @@ static bool check_var(u8 **p) + mitigations_configure(val); + } else if (IS_VAR("tso=")) { + enable_tso = val[0] == '1'; ++ } else if (IS_VAR("psci=")) { ++ psci_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +new file mode 100644 +index 0000000..2b8ca0d +--- /dev/null ++++ b/src/psci.c +@@ -0,0 +1,229 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++#include "smp.h" ++#include "types.h" ++#include "utils.h" ++ ++struct psci_cpu { ++ volatile u64 state; ++ volatile u64 entry; ++ volatile u64 context; ++ u64 pad[PSCI_CPU_SIZE / 8 - 3]; ++}; ++ ++_Static_assert(sizeof(struct psci_cpu) == PSCI_CPU_SIZE, "psci_cpu layout"); ++ ++/* ++ * Mailboxes, one cache line each, only ever accessed at EL3 with the MMU off. ++ * They live in .data rather than .bss so that a parked CPU's state survives ++ * m1n1 being re-entered from Linux (which clears .bss). ++ */ ++struct psci_cpu psci_cpus[PSCI_MAX_CPUS] ++ __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* Written with the MMU off only (_reentry sets reentry_flag, the first member) */ ++struct { ++ u64 reentry_flag; ++ u64 boot_el3; ++} psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* psci=1 from the payload, kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))); ++ ++/* EL3 stacks a CPU gets when it leaves the pen */ ++static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); ++ ++static bool psci_have_el3; ++static bool reentered; ++ ++/* EL3 side */ ++ ++static inline void psci_line_sync(volatile void *p) ++{ ++ __asm__ volatile("dc civac, %0\n\tdsb sy" ::"r"(p) : "memory"); ++} ++ ++static int psci_mpidr_index(u64 mpidr) ++{ ++ mpidr &= 0xff00ffffffUL; ++ ++ /* A7-A9: one cluster, Aff0 is the core */ ++ if (mpidr >= PSCI_MAX_CPUS) ++ return -1; ++ ++ return mpidr; ++} ++ ++static s64 psci_cpu_off(void) ++{ ++ int idx = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (idx < 0) ++ return PSCI_RET_DENIED; ++ ++ psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); ++} ++ ++static s64 psci_cpu_on(u64 mpidr, u64 entry, u64 context) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || !entry) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ struct psci_cpu *c = &psci_cpus[idx]; ++ ++ psci_line_sync(c); ++ switch (c->state) { ++ case PSCI_AFF_OFF: ++ break; ++ case PSCI_AFF_ON_PENDING: ++ return PSCI_RET_ON_PENDING; ++ default: ++ /* Running, or never seen by us (still in m1n1 or never started) */ ++ return PSCI_RET_ALREADY_ON; ++ } ++ ++ /* State before entry: the pen sets ON as soon as it sees the entry */ ++ c->context = context; ++ c->state = PSCI_AFF_ON_PENDING; ++ psci_line_sync(c); ++ c->entry = entry; ++ psci_line_sync(c); ++ sysop("sev"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++static s64 psci_affinity(u64 mpidr, u64 level) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || level) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ psci_line_sync(&psci_cpus[idx]); ++ return psci_cpus[idx].state; ++} ++ ++static s64 psci_features(u32 fid) ++{ ++ switch (fid) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ case PSCI_0_2_FN_CPU_OFF: ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return PSCI_RET_SUCCESS; ++ default: ++ /* ++ * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make ++ * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) ++{ ++ u32 fn = fid; ++ ++ /* SMC32 calls carry 32-bit arguments */ ++ if (!(fn & 0x40000000)) { ++ a1 = (u32)a1; ++ a2 = (u32)a2; ++ a3 = (u32)a3; ++ } ++ ++ switch (fn) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ return 0x10000; /* 1.0 */ ++ case PSCI_0_2_FN_CPU_OFF: ++ return psci_cpu_off(); ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ return psci_cpu_on(a1, a2, a3); ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ return psci_affinity(a1, a2); ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ return 2; /* no trusted OS, nothing to migrate */ ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return psci_features(a1); ++ default: ++ /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++/* EL1 side */ ++ ++void psci_note_boot_el(u64 el) ++{ ++ /* MMU still off: these .data accesses are uncached, like the writers' */ ++ if (psci_persist.reentry_flag) { ++ reentered = true; ++ psci_persist.reentry_flag = 0; ++ } ++ if (el == 3) ++ psci_persist.boot_el3 = 1; ++ ++ psci_have_el3 = psci_persist.boot_el3 && has_el3(); ++} ++ ++void psci_configure(const char *val) ++{ ++ psci_mode = val[0] == '1'; ++} ++ ++bool psci_active(void) ++{ ++ return psci_mode && psci_have_el3; ++} ++ ++bool psci_reentered(void) ++{ ++ return reentered; ++} ++ ++int psci_affinity_info(u64 mpidr) ++{ ++ return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); ++} ++ ++static void psci_self_off(void) ++{ ++ /* Only returns on failure; the core then stays in m1n1's loop */ ++ psci_smc(PSCI_0_2_FN_CPU_OFF, 0, 0, 0); ++} ++ ++int psci_park_cpu(int cpu, u64 mpidr) ++{ ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ ++ smp_call0(cpu, psci_self_off); ++ ++ for (int i = 0; i < 100; i++) { ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ udelay(1000); ++ } ++ ++ return -1; ++} ++ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top) ++{ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return -1; ++ ++ s64 ret = psci_smc(PSCI_0_2_FN64_CPU_ON, mpidr, (u64)psci_secondary_entry, ++ (u64)stack_top | (u64)index); ++ ++ return ret == PSCI_RET_SUCCESS ? 0 : -1; ++} +diff --git a/src/psci.h b/src/psci.h +new file mode 100644 +index 0000000..6769ebe +--- /dev/null ++++ b/src/psci.h +@@ -0,0 +1,78 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#ifndef PSCI_H ++#define PSCI_H ++ ++#ifndef __ASSEMBLER__ ++#include "types.h" ++#endif ++ ++/* ++ * Minimal PSCI 1.0 for SoCs where m1n1 owns EL3 (A7-A9 entered at EL3). ++ * Only CPU_ON/CPU_OFF/AFFINITY_INFO and the discovery calls are implemented. ++ * A CPU that calls CPU_OFF is not powered down: it waits at EL3, MMU and ++ * caches off, until CPU_ON hands it a new EL1 entry point. That is what the ++ * kernel needs to offline CPUs for kexec and CPU hotplug. ++ */ ++ ++#define PSCI_0_2_FN_PSCI_VERSION 0x84000000 ++#define PSCI_0_2_FN_CPU_SUSPEND 0x84000001 ++#define PSCI_0_2_FN_CPU_OFF 0x84000002 ++#define PSCI_0_2_FN_CPU_ON 0x84000003 ++#define PSCI_0_2_FN_AFFINITY_INFO 0x84000004 ++#define PSCI_0_2_FN_MIGRATE_INFO_TYPE 0x84000006 ++#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 ++#define PSCI_0_2_FN_SYSTEM_RESET 0x84000009 ++#define PSCI_1_0_FN_PSCI_FEATURES 0x8400000a ++#define PSCI_0_2_FN64_CPU_ON 0xc4000003 ++#define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 ++ ++#define PSCI_RET_SUCCESS 0 ++#define PSCI_RET_NOT_SUPPORTED -1 ++#define PSCI_RET_INVALID_PARAMS -2 ++#define PSCI_RET_DENIED -3 ++#define PSCI_RET_ALREADY_ON -4 ++#define PSCI_RET_ON_PENDING -5 ++#define PSCI_RET_INTERNAL_FAILURE -6 ++ ++#define PSCI_AFF_ON 0 ++#define PSCI_AFF_OFF 1 ++#define PSCI_AFF_ON_PENDING 2 ++ ++#define PSCI_MAX_CPUS 4 ++#define PSCI_STACK_SIZE 0x4000 ++ ++/* Layout shared with psci_asm.S */ ++#define PSCI_CPU_STATE 0x00 ++#define PSCI_CPU_ENTRY 0x08 ++#define PSCI_CPU_CONTEXT 0x10 ++#define PSCI_CPU_SIZE 0x80 ++ ++#ifndef __ASSEMBLER__ ++ ++/* Payload variable psci=1 */ ++void psci_configure(const char *val); ++/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++bool psci_active(void); ++/* Called once from _start_c with the EL m1n1 was entered at */ ++void psci_note_boot_el(u64 el); ++/* Set when m1n1 was re-entered from Linux (see psci_asm.S _reentry) */ ++bool psci_reentered(void); ++ ++/* EL1 side: SMC into our own EL3 */ ++s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3); ++int psci_affinity_info(u64 mpidr); ++/* Park a secondary that is spinning in smp_secondary_entry() at EL3 */ ++int psci_park_cpu(int cpu, u64 mpidr); ++/* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); ++ ++/* EL3 side */ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); ++void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_secondary_entry(void); ++void _reentry(void); ++ ++#endif ++ ++#endif +diff --git a/src/psci_asm.S b/src/psci_asm.S +new file mode 100644 +index 0000000..14f842f +--- /dev/null ++++ b/src/psci_asm.S +@@ -0,0 +1,170 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++ ++#define DCSW_OP_DCCISW 0x1 // memory.h ++ ++.extern _v_sp0_sync ++.extern _start ++.extern _cpu_reset_c ++.extern psci_handle ++.extern psci_persist ++ ++.text ++ ++/* ++ * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) ++ * goes to the PSCI dispatcher; everything else, and every exception taken ++ * at EL1/EL2, takes the usual path. ++ */ ++.globl _v_lower_sync ++.type _v_lower_sync, @function ++_v_lower_sync: ++ stp x16, x17, [sp, #-16]! ++ mrs x16, CurrentEL ++ cmp x16, #0xc ++ b.ne 1f ++ mrs x16, esr_el3 ++ lsr x17, x16, #26 ++ cmp x17, #0x17 // EC: SMC from AArch64 ++ b.ne 1f ++ tst x16, #0xffff // imm16 == 0: SMCCC ++ b.ne 1f ++ ldp x16, x17, [sp], #16 ++ b psci_smc_entry ++1: ++ ldp x16, x17, [sp], #16 ++ b _v_sp0_sync ++ ++/* ++ * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the ++ * caller. x0-x3 go straight to psci_handle(). ELR/SPSR are saved in case ++ * anything below takes an exception at EL3. ++ */ ++.type psci_smc_entry, @function ++psci_smc_entry: ++ stp x29, x30, [sp, #-16]! ++ mrs x29, elr_el3 ++ mrs x30, spsr_el3 ++ stp x29, x30, [sp, #-16]! ++ str x18, [sp, #-16]! ++ bl psci_handle ++ ldr x18, [sp], #16 ++ ldp x29, x30, [sp], #16 ++ msr elr_el3, x29 ++ msr spsr_el3, x30 ++ ldp x29, x30, [sp], #16 ++ eret ++ ++/* ++ * psci_pen(struct psci_cpu *me, u64 stack_top): CPU_OFF. Never returns. ++ * ++ * The core is not powered down. It writes its L1D back, marks itself OFF and ++ * waits in WFE at EL3 (MMU off, so every access here is non-cacheable) for ++ * CPU_ON to post an entry point. It then enters EL1 there with the MMU off, ++ * a clean I-cache and TLB, and x0 = context id, like a core coming out of ++ * reset. SP_EL3 is reset to this core's own static stack. ++ */ ++.globl psci_pen ++.type psci_pen, @function ++psci_pen: ++ mov x19, x0 ++ mov x20, x1 ++ ++ // A pending timer interrupt must not keep waking WFE. ++ msr cntp_ctl_el0, xzr ++ msr cntv_ctl_el0, xzr ++ isb ++ ++ // Clean and invalidate this core's L1D (L2 is shared and coherent). ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_level1 ++ dsb sy ++ ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_OFF ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ sev ++ ++1: ++ wfe ++ dc civac, x19 ++ dsb sy ++ ldr x2, [x19, #PSCI_CPU_ENTRY] ++ cbz x2, 1b ++ ++ ldr x0, [x19, #PSCI_CPU_CONTEXT] ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_ON ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ ++ // Nothing of the previous kernel's EL1 context may survive. ++ ic iallu ++ tlbi alle1 ++ dsb sy ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ msr elr_el3, x2 ++ mov sp, x20 ++ isb ++ eret ++ ++/* EL1 side: s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3) */ ++.globl psci_smc ++.type psci_smc, @function ++psci_smc: ++ smc #0 ++ ret ++ ++/* ++ * CPU_ON target when m1n1 takes a parked core back after a re-entry. ++ * x0 = EL1 stack top | m1n1 CPU index (stack tops are 16 KiB aligned). ++ */ ++.globl psci_secondary_entry ++.type psci_secondary_entry, @function ++psci_secondary_entry: ++ and x1, x0, #0xf ++ bic x0, x0, #0xf ++ msr tpidr_el1, x1 ++ mov sp, x0 ++ bl _cpu_reset_c ++ b . ++ ++/* ++ * Re-entry from Linux (a kexec'd trampoline, see docs/boot/warm-reboot.md): ++ * EL1, MMU and caches off, x0 = a copy of the original boot_args. ++ * Linux may hold clean lines of this image in its linear map, so drop them ++ * before m1n1 writes its .bss uncached and then reads it with caches on. ++ */ ++.globl _reentry ++.type _reentry, @function ++_reentry: ++ mov x19, x0 ++ adrp x1, _base ++ adrp x2, _end ++1: ++ dc civac, x1 ++ add x1, x1, #64 ++ cmp x1, x2 ++ b.lo 1b ++ dsb sy ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #1 ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ mov x0, x19 ++ b _start +diff --git a/src/smp.c b/src/smp.c +index 83eadff..7f5b336 100644 +--- a/src/smp.c ++++ b/src/smp.c +@@ -8,6 +8,7 @@ + #include "malloc.h" + #include "memory.h" + #include "pmgr.h" ++#include "psci.h" + #include "soc.h" + #include "string.h" + #include "types.h" +@@ -227,6 +228,74 @@ static void smp_stop_cpu(int index, int die, int cluster, int core, u64 impl, u6 + } + } + ++/* ++ * After a re-entry from Linux, a secondary that Linux offlined waits in the ++ * PSCI pen at EL3 rather than in smp_secondary_entry(). It can't be reset ++ * through the PMGR (it is running), so hand it an EL1 entry with CPU_ON. ++ */ ++static bool smp_reclaim_cpu(int index, u64 mpidr) ++{ ++ int i; ++ ++ if (index >= MAX_CPUS || spin_table[index].flag) ++ return false; ++ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return false; ++ ++ printf("Taking CPU %d back from the PSCI pen... ", index); ++ ++ memset(&spin_table[index], 0, sizeof(struct spin_table)); ++ target_cpu = index; ++ secondary_stacks[index] = memalign(0x4000, SECONDARY_STACK_SIZE); ++ ++ /* The core starts with its MMU off */ ++ dc_civac_range(&spin_table[index], sizeof(struct spin_table)); ++ dc_civac_range(&target_cpu, sizeof(target_cpu)); ++ dc_civac_range(&boot_cpu_idx, sizeof(boot_cpu_idx)); ++ dc_civac_range(&boot_cpu_mpidr, sizeof(boot_cpu_mpidr)); ++ ++ if (psci_unpark_cpu(index, mpidr, secondary_stacks[index] + SECONDARY_STACK_SIZE) < 0) { ++ printf("Failed (CPU_ON)\n"); ++ return true; ++ } ++ ++ for (i = 0; i < 100; i++) { ++ sysop("dmb ld"); ++ if (spin_table[index].flag) ++ break; ++ udelay(1000); ++ } ++ ++ printf("%s\n", i >= 100 ? "Failed!" : " Back."); ++ return true; ++} ++ ++/* ++ * Put every live secondary into the PSCI pen, for a kernel that boots them ++ * with enable-method = "psci". A CPU that doesn't make it is dropped (and ++ * then pruned from the DT by the caller). ++ */ ++int smp_park_secondaries_psci(void) ++{ ++ int parked = 0; ++ ++ for (int i = 0; i < MAX_CPUS; i++) { ++ if (i == boot_cpu_idx || !smp_is_alive(i)) ++ continue; ++ ++ u64 mpidr = spin_table[i].mpidr; ++ if (psci_park_cpu(i, mpidr) < 0) { ++ printf("PSCI: CPU %d (MPIDR 0x%lx) did not park, dropping it\n", i, mpidr); ++ memset(&spin_table[i], 0, sizeof(struct spin_table)); ++ continue; ++ } ++ parked++; ++ } ++ ++ return parked; ++} ++ + static u64 smp_get_iorvbar(int node, int cpu) + { + u64 cpu_impl_reg[2]; +@@ -405,6 +474,10 @@ void smp_start_secondaries(void) + u8 cluster = FIELD_GET(CPU_REG_CLUSTER, reg); + u8 die = FIELD_GET(CPU_REG_DIE, reg); + ++ if (psci_active() && psci_reentered() && ++ smp_reclaim_cpu(i, core | (cluster << 8) | (die << 16))) ++ continue; ++ + smp_start_cpu(i, die, cluster, core, iorvbar, pmgr_reg + cpu_start_off); + } + } +diff --git a/src/smp.h b/src/smp.h +index 0296978..6c90557 100644 +--- a/src/smp.h ++++ b/src/smp.h +@@ -18,6 +18,7 @@ void smp_secondary_prep_el3(void); + + void smp_start_secondaries(void); + void smp_stop_secondaries(bool deep_sleep); ++int smp_park_secondaries_psci(void); + + #define smp_call0(i, f) smp_call4(i, f, 0, 0, 0, 0) + #define smp_call1(i, f, a) smp_call4(i, f, a, 0, 0, 0) +diff --git a/src/start.S b/src/start.S +index aa11e18..a3f35ae 100644 +--- a/src/start.S ++++ b/src/start.S +@@ -11,6 +11,7 @@ + .extern _v_sp0_irq + .extern _v_sp0_fiq + .extern _v_sp0_serr ++.extern _v_lower_sync + .extern _reset_stack + .extern _cpu_reset_c + .extern wdt_reboot +@@ -41,7 +42,7 @@ _vectors_start: + .align 7 + b _v_sp0_serr + .align 7 +- b _v_sp0_sync ++ b _v_lower_sync + .align 7 + b _v_sp0_irq + .align 7 +@@ -103,6 +104,8 @@ _start: + + mov x0, x19 + mov x1, x20 ++ mrs x2, CurrentEL ++ lsr x2, x2, #2 + bl _start_c + b . + +@@ -115,6 +118,7 @@ start_el3: + msr tpidr_el3, xzr + adrp x6, _stack_bot + adr x7, _start_c ++ mov x2, #3 // _start_c: entered at EL3 + b el3_eret_to_el1 + + .globl exc_unk +diff --git a/src/startup.c b/src/startup.c +index 61afe16..167fae8 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -5,6 +5,7 @@ + #include "dockchannel_uart.h" + #include "exception.h" + #include "firmware.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "types.h" +@@ -159,12 +160,15 @@ void dump_boot_args(struct boot_args *ba) + } + + extern void get_device_info(void); +-void _start_c(void *boot_args, void *base) ++void _start_c(void *boot_args, void *base, u64 boot_el) + { + UNUSED(base); + u32 cpu_id = 0; + + memset64(_bss_start, 0, _bss_end - _bss_start); ++ psci_note_boot_el(boot_el); ++ if (psci_reentered()) ++ boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); + diff --git a/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch b/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch new file mode 100644 index 00000000..edfc7592 --- /dev/null +++ b/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch @@ -0,0 +1,84 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 15:56:40 +0100 +Subject: [PATCH] psci: Turn the resident EL3 PSCI on by default + +When m1n1 was entered at EL3 (A7-A9 booted by Pongo) it stays resident +and already implements CPU_ON/CPU_OFF/AFFINITY_INFO for the kernel, but +only when the payload asked for it with psci=1. Without it the kernel +gets the spin table and can neither offline CPUs nor kexec on SMP, so +every loader had to be built with the variable appended. + +Make it the default. psci=0 turns it off again (spin table, no /psci +node), psci=1 is still accepted. Nothing changes where m1n1 is entered +at EL1 or EL2: psci_active() still requires the EL3 entry. + +Assisted-by: LLM +--- + src/kboot.c | 4 ++-- + src/psci.c | 6 +++--- + src/psci.h | 4 ++-- + 3 files changed, 7 insertions(+), 7 deletions(-) + +diff --git a/src/kboot.c b/src/kboot.c +index 7123d60..a5f4238 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -576,7 +576,7 @@ static int dt_set_smbios(void) + } + + /* +- * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * When m1n1 owns EL3 (A7-A9), CPUs boot through m1n1's PSCI + * (see psci.c) instead of the spin table, so the kernel can offline them + * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. + */ +@@ -586,7 +586,7 @@ static int dt_set_psci(void) + int node; + + if (!psci_active()) { +- printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ printf("PSCI: off (m1n1 not entered at EL3, or psci=0)\n"); + return 0; + } + +diff --git a/src/psci.c b/src/psci.c +index 2b8ca0d..25e2a4f 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -28,8 +28,8 @@ struct { + u64 boot_el3; + } psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + +-/* psci=1 from the payload, kept across a re-entry */ +-static u32 psci_mode __attribute__((section(".data"))); ++/* On unless the payload says psci=0; kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))) = 1; + + /* EL3 stacks a CPU gets when it leaves the pen */ + static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); +@@ -177,7 +177,7 @@ void psci_note_boot_el(u64 el) + + void psci_configure(const char *val) + { +- psci_mode = val[0] == '1'; ++ psci_mode = val[0] != '0'; + } + + bool psci_active(void) +diff --git a/src/psci.h b/src/psci.h +index 6769ebe..9a213af 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -50,9 +50,9 @@ + + #ifndef __ASSEMBLER__ + +-/* Payload variable psci=1 */ ++/* Payload variable psci=0 turns PSCI off; anything else (psci=1) leaves it on */ + void psci_configure(const char *val); +-/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++/* m1n1 entered at EL3 and not psci=0: the kernel uses PSCI */ + bool psci_active(void); + /* Called once from _start_c with the EL m1n1 was entered at */ + void psci_note_boot_el(u64 el); diff --git a/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch b/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch new file mode 100644 index 00000000..a1c26412 --- /dev/null +++ b/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch @@ -0,0 +1,286 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 15:58:57 +0100 +Subject: [PATCH] psci: Trap EL1 WFI/WFE to EL3 for a firmware CPU rendezvous + +An ordinary arm64 reboot or power-off does not offline the secondaries +with CPU_OFF: smp_send_stop() leaves them spinning in "wfe; wfi" at EL1 +with interrupts masked, still ON as far as the monitor knows. A9 has no +way to reset a core from another one (Apple and m1n1 power cores down +from the core itself), so SYSTEM_RESET and SYSTEM_OFF need those CPUs to +come to EL3 on their own. + +Give the monitor that hook: a private SiP call, made by m1n1 on the boot +CPU right before it starts a kernel, sets SCR_EL3.TWI and TWE there and +on every CPU that later leaves the pen, and records the AIC and watchdog +bases for later EL3 use. A trapped WFE returns at once (a spurious +wake-up is architectural, and the exception return may have cleared the +exclusive monitor whose event the caller waits for); a trapped WFI waits +at EL3 with IRQ/FIQ routed there only around the wfi, so an interrupt +meant for EL1 still wakes it. While a rendezvous is pending, a trapped +CPU instead parks itself in the pen exactly as CPU_OFF would, cleaning +its L1D on the way. + +The trap enable has no user yet. The CPUs that have been in the pen are +recorded, so a rendezvous knows which ones the kernel was given. + +Assisted-by: LLM +--- + src/psci.c | 56 ++++++++++++++++++++++++++++++++++++++ + src/psci.h | 20 ++++++++++++++ + src/psci_asm.S | 73 ++++++++++++++++++++++++++++++++++++++++++++++++-- + 3 files changed, 147 insertions(+), 2 deletions(-) + +diff --git a/src/psci.c b/src/psci.c +index 25e2a4f..af4f325 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -28,6 +28,24 @@ struct { + u64 boot_el3; + } psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + ++/* ++ * Rendezvous state, one line, only accessed at EL3 with the MMU off. A CPU ++ * that wants all others in the pen (SYSTEM_RESET/SYSTEM_OFF) sets pending; ++ * every other CPU then parks itself at its next trapped WFI/WFE. ++ */ ++struct psci_rdv { ++ volatile u64 pending; ++ volatile u64 traps; /* SCR_EL3.TWI|TWE for CPUs leaving the pen */ ++ volatile u64 cpus; /* MPIDR indices that have been in the pen: handed to the kernel */ ++ u64 aic; ++ u64 wdt; ++ u64 pad[PSCI_CPU_SIZE / 8 - 5]; ++} psci_rdv __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++_Static_assert(sizeof(struct psci_rdv) == PSCI_CPU_SIZE, "psci_rdv layout"); ++_Static_assert(offsetof(struct psci_rdv, pending) == PSCI_RDV_PENDING, "psci_rdv layout"); ++_Static_assert(offsetof(struct psci_rdv, traps) == PSCI_RDV_TRAPS, "psci_rdv layout"); ++ + /* On unless the payload says psci=0; kept across a re-entry */ + static u32 psci_mode __attribute__((section(".data"))) = 1; + +@@ -62,6 +80,10 @@ static s64 psci_cpu_off(void) + if (idx < 0) + return PSCI_RET_DENIED; + ++ psci_line_sync(&psci_rdv); ++ psci_rdv.cpus |= BIT(idx); ++ psci_line_sync(&psci_rdv); ++ + psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); + } + +@@ -128,6 +150,32 @@ static s64 psci_features(u32 fid) + } + } + ++/* Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the pen */ ++static s64 psci_m1n1_handoff(u64 traps, u64 aic, u64 wdt) ++{ ++ u64 bits = traps ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ ++ psci_rdv.pending = 0; ++ psci_rdv.traps = bits; ++ psci_rdv.aic = aic; ++ psci_rdv.wdt = wdt; ++ psci_line_sync(&psci_rdv); ++ ++ msr(SCR_EL3, (mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)) | bits); ++ sysop("isb"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++/* A trapped WFI/WFE saw the rendezvous flag: become CPU_OFF right here */ ++void psci_rendezvous_off(void) ++{ ++ psci_cpu_off(); ++ /* Not a CPU we know (can't happen after the pen saw it): stay out of the way */ ++ for (;;) ++ sysop("wfe"); ++} ++ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + { + u32 fn = fid; +@@ -154,6 +202,8 @@ u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + return 2; /* no trusted OS, nothing to migrate */ + case PSCI_1_0_FN_PSCI_FEATURES: + return psci_features(a1); ++ case PSCI_M1N1_SIP_HANDOFF: ++ return psci_m1n1_handoff(a1, a2, a3); + default: + /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ + return PSCI_RET_NOT_SUPPORTED; +@@ -190,6 +240,12 @@ bool psci_reentered(void) + return reentered; + } + ++void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base) ++{ ++ if (psci_active()) ++ psci_smc(PSCI_M1N1_SIP_HANDOFF, wfx_traps, aic_base, wdt_base); ++} ++ + int psci_affinity_info(u64 mpidr) + { + return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); +diff --git a/src/psci.h b/src/psci.h +index 9a213af..1d3a88e 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -27,6 +27,13 @@ + #define PSCI_0_2_FN64_CPU_ON 0xc4000003 + #define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 + ++/* ++ * m1n1-private SiP call, made by m1n1 itself on the boot CPU right before it ++ * starts a kernel: x1 = 1 to trap EL1 WFI/WFE to EL3 on this and every CPU ++ * that later leaves the pen, x2 = AIC base, x3 = primary watchdog base. ++ */ ++#define PSCI_M1N1_SIP_HANDOFF 0xc200ff00 ++ + #define PSCI_RET_SUCCESS 0 + #define PSCI_RET_NOT_SUPPORTED -1 + #define PSCI_RET_INVALID_PARAMS -2 +@@ -48,6 +55,15 @@ + #define PSCI_CPU_CONTEXT 0x10 + #define PSCI_CPU_SIZE 0x80 + ++/* struct psci_rdv, one line, EL3 only (MMU off) */ ++#define PSCI_RDV_PENDING 0x00 ++#define PSCI_RDV_TRAPS 0x08 ++ ++#define SCR_EL3_IRQ (1 << 1) ++#define SCR_EL3_FIQ (1 << 2) ++#define SCR_EL3_TWI (1 << 12) ++#define SCR_EL3_TWE (1 << 13) ++ + #ifndef __ASSEMBLER__ + + /* Payload variable psci=0 turns PSCI off; anything else (psci=1) leaves it on */ +@@ -67,9 +83,13 @@ int psci_park_cpu(int cpu, u64 mpidr); + /* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ + int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); + ++/* EL1, boot CPU, last thing before the kernel: see PSCI_M1N1_SIP_HANDOFF */ ++void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base); ++ + /* EL3 side */ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); + void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_rendezvous_off(void) __attribute__((noreturn)); + void psci_secondary_entry(void); + void _reentry(void); + +diff --git a/src/psci_asm.S b/src/psci_asm.S +index 14f842f..ff31f3a 100644 +--- a/src/psci_asm.S ++++ b/src/psci_asm.S +@@ -9,13 +9,16 @@ + .extern _cpu_reset_c + .extern psci_handle + .extern psci_persist ++.extern psci_rdv ++.extern psci_rendezvous_off + + .text + + /* + * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) +- * goes to the PSCI dispatcher; everything else, and every exception taken +- * at EL1/EL2, takes the usual path. ++ * goes to the PSCI dispatcher and a trapped WFI/WFE to psci_wfx_entry; ++ * everything else, and every exception taken at EL1/EL2, takes the usual ++ * path. + */ + .globl _v_lower_sync + .type _v_lower_sync, @function +@@ -26,6 +29,8 @@ _v_lower_sync: + b.ne 1f + mrs x16, esr_el3 + lsr x17, x16, #26 ++ cmp x17, #0x01 // EC: trapped WFI/WFE ++ b.eq 2f + cmp x17, #0x17 // EC: SMC from AArch64 + b.ne 1f + tst x16, #0xffff // imm16 == 0: SMCCC +@@ -35,6 +40,59 @@ _v_lower_sync: + 1: + ldp x16, x17, [sp], #16 + b _v_sp0_sync ++2: ++ ldp x16, x17, [sp], #16 ++ b psci_wfx_entry ++ ++/* ++ * WFI/WFE from EL1/EL0, trapped by SCR_EL3.TWI/TWE (see PSCI_M1N1_SIP_HANDOFF). ++ * This is how firmware gets hold of a CPU that the kernel stopped with an ++ * IPI and left spinning in "wfe; wfi" with interrupts masked (an ordinary ++ * reboot does that; only kexec and hotplug use CPU_OFF): while a ++ * SYSTEM_RESET/SYSTEM_OFF rendezvous is pending, the CPU parks itself as if ++ * it had called CPU_OFF. Otherwise: ++ * - WFE completes at once. A spurious WFE wake-up is architectural, and ++ * waiting here could miss the event the caller waits for (the exception ++ * return may clear its exclusive monitor). ++ * - WFI waits here, with IRQ/FIQ routed to EL3 only around the wfi so that ++ * an interrupt for EL1 wakes it (DAIF is masked at EL3, nothing is taken). ++ * Both then return past the instruction. Only x0-x3 are used. ++ */ ++.type psci_wfx_entry, @function ++psci_wfx_entry: ++ stp x0, x1, [sp, #-16]! ++ stp x2, x3, [sp, #-16]! ++ adrp x0, psci_rdv ++ add x0, x0, :lo12:psci_rdv ++ dc civac, x0 ++ dsb sy ++ ldr x1, [x0, #PSCI_RDV_PENDING] ++ cbnz x1, 3f ++ mrs x1, esr_el3 ++ tbnz x1, #0, 2f // ISS.TI bit 0 set: WFE (or WFET) ++ mrs x2, scr_el3 ++ orr x3, x2, #(SCR_EL3_IRQ | SCR_EL3_FIQ) ++ msr scr_el3, x3 ++ isb ++ dsb sy ++ wfi ++ msr scr_el3, x2 ++ isb ++ dc civac, x0 ++ dsb sy ++ ldr x1, [x0, #PSCI_RDV_PENDING] ++ cbnz x1, 3f ++2: ++ mrs x1, elr_el3 ++ add x1, x1, #4 ++ msr elr_el3, x1 ++ ldp x2, x3, [sp], #16 ++ ldp x0, x1, [sp], #16 ++ eret ++3: ++ add sp, sp, #32 ++ bl psci_rendezvous_off ++ b . + + /* + * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the +@@ -106,6 +164,17 @@ psci_pen: + ic iallu + tlbi alle1 + dsb sy ++ ++ // WFI/WFE traps as the boot CPU set them for the kernel (psci_rdv.traps) ++ adrp x3, psci_rdv ++ add x3, x3, :lo12:psci_rdv ++ dc civac, x3 ++ dsb sy ++ ldr x4, [x3, #PSCI_RDV_TRAPS] ++ mrs x1, scr_el3 ++ bic x1, x1, #(SCR_EL3_TWI | SCR_EL3_TWE) ++ orr x1, x1, x4 ++ msr scr_el3, x1 + mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off + movk x1, #0x30d0, lsl #16 + msr sctlr_el1, x1 diff --git a/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch b/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch new file mode 100644 index 00000000..e5dd449e --- /dev/null +++ b/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch @@ -0,0 +1,497 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 16:01:53 +0100 +Subject: [PATCH] psci: Implement SYSTEM_RESET and SYSTEM_OFF with a CPU + rendezvous + +Give the kernel's ordinary reboot and power-off paths somewhere to go. +On SYSTEM_RESET or SYSTEM_OFF the calling CPU sets the rendezvous flag, +sends an event and an AIC IPI to every other CPU the kernel was given, +and waits up to a second for all of them to park in the pen through the +WFI/WFE traps. If one does not arrive, it clears the flag and returns +without having changed anything, so the kernel goes on to its next +restart or power-off method as it does today. + +Once they are parked it disarms the primary watchdog (the kernel, or +systemd's reboot watchdog, may have armed it), stops its timers and +drops the traps. SYSTEM_RESET then turns the EL1 MMU and caches off, +writes back every data cache level by set/way, invalidates the EL1 TLB +and I-cache and restarts m1n1 at EL1h with the original iBoot boot_args, +saved in .data on the first cold start before top_of_memory_alloc() +changes them. psci_reentered() and the new psci_reset_reentered() tell +the restarted m1n1 where it came from. SYSTEM_OFF parks the caller too: +there is no PMU driver here, so this halts rather than powers off. + +The payload variable psci_reset selects what SYSTEM_RESET leads to: boot +(the default, used by the next change), proxy, or 0 for NOT_SUPPORTED +with no WFI/WFE traps. It is reported by PSCI_FEATURES and armed by +kboot_boot() through the private handoff call, which also records the +boot CPU and needs an AIC v1 and a watchdog in the ADT. + +Assisted-by: LLM +--- + src/kboot.c | 1 + + src/payload.c | 2 + + src/psci.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++-- + src/psci.h | 25 +++++- + src/psci_asm.S | 44 +++++++++- + src/startup.c | 1 + + 6 files changed, 280 insertions(+), 13 deletions(-) + +diff --git a/src/kboot.c b/src/kboot.c +index a5f4238..299e388 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -3301,6 +3301,7 @@ int kboot_boot(void *kernel) + + printf("Setting SMP mode to WFE...\n"); + smp_set_wfe_mode(true); ++ psci_handoff(); + printf("Preparing to boot kernel at %p with fdt at %p\n", kernel, dt); + + next_stage.entry = kernel; +diff --git a/src/payload.c b/src/payload.c +index 4ffc9e2..0b7dc14 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -208,6 +208,8 @@ static bool check_var(u8 **p) + enable_tso = val[0] == '1'; + } else if (IS_VAR("psci=")) { + psci_configure(val); ++ } else if (IS_VAR("psci_reset=")) { ++ psci_reset_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +index af4f325..1821ebc 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -1,9 +1,19 @@ + /* SPDX-License-Identifier: MIT */ + + #include "psci.h" ++#include "adt.h" ++#include "aic_regs.h" ++#include "memory.h" + #include "smp.h" ++#include "string.h" + #include "types.h" + #include "utils.h" ++#include "xnuboot.h" ++ ++#define WDT_CTL 0x1c ++ ++/* SYSTEM_RESET/SYSTEM_OFF wait this long for the other CPUs to park */ ++#define RENDEZVOUS_TIMEOUT_US 1000000 + + struct psci_cpu { + volatile u64 state; +@@ -39,7 +49,8 @@ struct psci_rdv { + volatile u64 cpus; /* MPIDR indices that have been in the pen: handed to the kernel */ + u64 aic; + u64 wdt; +- u64 pad[PSCI_CPU_SIZE / 8 - 5]; ++ u64 reset; /* enum psci_reset_mode the kernel was started with */ ++ u64 pad[PSCI_CPU_SIZE / 8 - 6]; + } psci_rdv __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + + _Static_assert(sizeof(struct psci_rdv) == PSCI_CPU_SIZE, "psci_rdv layout"); +@@ -49,11 +60,18 @@ _Static_assert(offsetof(struct psci_rdv, traps) == PSCI_RDV_TRAPS, "psci_rdv lay + /* On unless the payload says psci=0; kept across a re-entry */ + static u32 psci_mode __attribute__((section(".data"))) = 1; + ++/* psci_reset=0|proxy|boot, kept across a re-entry */ ++static u32 psci_reset_mode __attribute__((section(".data"))) = PSCI_RESET_BOOT; ++ ++/* The iBoot boot_args m1n1 was first started with; SYSTEM_RESET starts m1n1 with them again */ ++static struct boot_args reset_boot_args __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ + /* EL3 stacks a CPU gets when it leaves the pen */ + static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); + + static bool psci_have_el3; + static bool reentered; ++static bool reset_reentered; + + /* EL3 side */ + +@@ -129,9 +147,113 @@ static s64 psci_affinity(u64 mpidr, u64 level) + return psci_cpus[idx].state; + } + ++/* ++ * Gets every other CPU the kernel was given into the pen: they park at their ++ * next trapped WFI/WFE (see psci_wfx_entry). A CPU waiting in WFI is woken ++ * with an IPI, one in WFE with an event. If one doesn't arrive in time, ++ * nothing has been changed that the kernel cares about. ++ */ ++static bool psci_rendezvous(int self) ++{ ++ psci_line_sync(&psci_rdv); ++ u64 others = psci_rdv.cpus & ~BIT(self); ++ ++ psci_rdv.pending = 1; ++ psci_line_sync(&psci_rdv); ++ sysop("sev"); ++ ++ for (int i = 0; i < PSCI_MAX_CPUS; i++) { ++ if (!(others & BIT(i))) ++ continue; ++ psci_line_sync(&psci_cpus[i]); ++ if (psci_cpus[i].state != PSCI_AFF_OFF) ++ write32(psci_rdv.aic + AIC_IPI_SEND, AIC_IPI_SEND_CPU(i)); ++ } ++ ++ u64 timeout = timeout_calculate(RENDEZVOUS_TIMEOUT_US); ++ do { ++ bool all = true; ++ ++ for (int i = 0; i < PSCI_MAX_CPUS; i++) { ++ if (!(others & BIT(i))) ++ continue; ++ psci_line_sync(&psci_cpus[i]); ++ if (psci_cpus[i].state != PSCI_AFF_OFF) ++ all = false; ++ } ++ if (all) { ++ psci_rdv.pending = 0; ++ psci_line_sync(&psci_rdv); ++ return true; ++ } ++ sysop("sev"); ++ udelay(100); ++ } while (!timeout_expired(timeout)); ++ ++ psci_rdv.pending = 0; ++ psci_line_sync(&psci_rdv); ++ return false; ++} ++ ++/* Everything that must happen before this CPU leaves the kernel for good */ ++static s64 psci_system_stop(void) ++{ ++ int self = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ psci_line_sync(&psci_rdv); ++ if (psci_rdv.reset == PSCI_RESET_OFF) ++ return PSCI_RET_NOT_SUPPORTED; ++ if (self < 0 || !psci_rendezvous(self)) ++ return PSCI_RET_INTERNAL_FAILURE; ++ ++ msr(CNTP_CTL_EL0, 0); ++ msr(CNTV_CTL_EL0, 0); ++ /* The kernel (or systemd's reboot watchdog) may have armed it */ ++ write32(psci_rdv.wdt + WDT_CTL, 0); ++ read32(psci_rdv.wdt + WDT_CTL); ++ ++ /* ++ * The WFx traps go away with the kernel; the next handoff sets them ++ * again, and the next kboot records which CPUs the next kernel gets. ++ */ ++ psci_rdv.traps = 0; ++ psci_rdv.cpus = 0; ++ psci_line_sync(&psci_rdv); ++ msr(SCR_EL3, mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)); ++ sysop("isb"); ++ return PSCI_RET_SUCCESS; ++} ++ ++/* Returns only if the other CPUs could not be stopped: the kernel tries its next method */ ++static s64 psci_system_reset(void) ++{ ++ s64 ret = psci_system_stop(); ++ ++ if (ret != PSCI_RET_SUCCESS) ++ return ret; ++ ++ psci_system_reset_entry(&reset_boot_args); ++} ++ ++/* No PMU driver here: this halts the SoC with every CPU parked, it does not power off */ ++static s64 psci_system_off(void) ++{ ++ s64 ret = psci_system_stop(); ++ ++ if (ret != PSCI_RET_SUCCESS) ++ return ret; ++ ++ for (;;) ++ sysop("wfi"); ++} ++ + static s64 psci_features(u32 fid) + { + switch (fid) { ++ case PSCI_0_2_FN_SYSTEM_OFF: ++ case PSCI_0_2_FN_SYSTEM_RESET: ++ psci_line_sync(&psci_rdv); ++ return psci_rdv.reset != PSCI_RESET_OFF ? PSCI_RET_SUCCESS : PSCI_RET_NOT_SUPPORTED; + case PSCI_0_2_FN_PSCI_VERSION: + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_CPU_ON: +@@ -144,21 +266,35 @@ static s64 psci_features(u32 fid) + default: + /* + * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make +- * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ * no other SMCs), CPU_SUSPEND and SYSTEM_RESET2. + */ + return PSCI_RET_NOT_SUPPORTED; + } + } + +-/* Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the pen */ +-static s64 psci_m1n1_handoff(u64 traps, u64 aic, u64 wdt) ++/* ++ * Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the ++ * pen, and with them SYSTEM_RESET/SYSTEM_OFF (reset = enum psci_reset_mode). ++ */ ++static s64 psci_m1n1_handoff(u64 reset, u64 aic, u64 wdt) + { +- u64 bits = traps ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ if (reset > PSCI_RESET_PROXY || (reset && (!aic || !wdt))) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ u64 bits = reset ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ int self = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (self < 0) ++ return PSCI_RET_DENIED; + ++ psci_line_sync(&psci_rdv); ++ /* The secondaries were added when kboot parked them */ ++ psci_rdv.cpus |= BIT(self); + psci_rdv.pending = 0; + psci_rdv.traps = bits; + psci_rdv.aic = aic; + psci_rdv.wdt = wdt; ++ psci_rdv.reset = reset; + psci_line_sync(&psci_rdv); + + msr(SCR_EL3, (mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)) | bits); +@@ -202,10 +338,13 @@ u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + return 2; /* no trusted OS, nothing to migrate */ + case PSCI_1_0_FN_PSCI_FEATURES: + return psci_features(a1); ++ case PSCI_0_2_FN_SYSTEM_RESET: ++ return psci_system_reset(); ++ case PSCI_0_2_FN_SYSTEM_OFF: ++ return psci_system_off(); + case PSCI_M1N1_SIP_HANDOFF: + return psci_m1n1_handoff(a1, a2, a3); + default: +- /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ + return PSCI_RET_NOT_SUPPORTED; + } + } +@@ -216,6 +355,7 @@ void psci_note_boot_el(u64 el) + { + /* MMU still off: these .data accesses are uncached, like the writers' */ + if (psci_persist.reentry_flag) { ++ reset_reentered = psci_persist.reentry_flag == PSCI_REENTRY_RESET; + reentered = true; + psci_persist.reentry_flag = 0; + } +@@ -240,10 +380,72 @@ bool psci_reentered(void) + return reentered; + } + +-void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base) ++void psci_reset_configure(const char *val) ++{ ++ if (val[0] == '0') ++ psci_reset_mode = PSCI_RESET_OFF; ++ else if (!strncmp(val, "proxy", 5)) ++ psci_reset_mode = PSCI_RESET_PROXY; ++ else ++ psci_reset_mode = PSCI_RESET_BOOT; ++} ++ ++enum psci_reset_mode psci_reset_mode_get(void) + { +- if (psci_active()) +- psci_smc(PSCI_M1N1_SIP_HANDOFF, wfx_traps, aic_base, wdt_base); ++ return psci_reset_mode; ++} ++ ++bool psci_reset_reentered(void) ++{ ++ return reset_reentered; ++} ++ ++void psci_reset_save_boot_args(void) ++{ ++ if (reentered) ++ return; ++ memcpy(&reset_boot_args, &cur_boot_args, sizeof(reset_boot_args)); ++ dc_civac_range(&reset_boot_args, sizeof(reset_boot_args)); ++} ++ ++/* ++ * The CPU rendezvous needs the AIC v1 IPI (A7-A9) and the primary watchdog ++ * (wdt-version 1 there, nothing else to stop). Without them SYSTEM_RESET and ++ * SYSTEM_OFF stay NOT_SUPPORTED and the kernel uses its own methods. ++ */ ++void psci_handoff(void) ++{ ++ enum psci_reset_mode reset = psci_reset_mode; ++ u64 aic = 0, wdt = 0; ++ int path[8]; ++ ++ if (!psci_active()) ++ return; ++ ++ if (reset != PSCI_RESET_OFF) { ++ int node = adt_path_offset_trace(adt, "/arm-io/aic", path); ++ if (node < 0 || !adt_is_compatible(adt, node, "aic,1") || ++ adt_get_reg(adt, path, "reg", 0, &aic, NULL) < 0) { ++ printf("PSCI: SYSTEM_RESET/SYSTEM_OFF unavailable: no AIC v1\n"); ++ reset = PSCI_RESET_OFF; ++ } ++ node = adt_path_offset_trace(adt, "/arm-io/wdt", path); ++ if (node < 0 || adt_get_reg(adt, path, "reg", 0, &wdt, NULL) < 0) { ++ printf("PSCI: SYSTEM_RESET/SYSTEM_OFF unavailable: no watchdog\n"); ++ reset = PSCI_RESET_OFF; ++ } ++ } ++ ++ if (psci_smc(PSCI_M1N1_SIP_HANDOFF, reset, aic, wdt) != PSCI_RET_SUCCESS) { ++ printf("PSCI: handoff refused, SYSTEM_RESET/SYSTEM_OFF unavailable\n"); ++ psci_smc(PSCI_M1N1_SIP_HANDOFF, PSCI_RESET_OFF, 0, 0); ++ return; ++ } ++ ++ static const char *const names[] = {"off", "warm reboot into the default entry", ++ "warm reboot into the proxy"}; ++ printf("PSCI: SYSTEM_RESET: %s%s\n", names[reset], ++ reset ? "; EL1 WFI/WFE trap to EL3" : ""); + } + + int psci_affinity_info(u64 mpidr) +diff --git a/src/psci.h b/src/psci.h +index 1d3a88e..7c81af7 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -29,8 +29,9 @@ + + /* + * m1n1-private SiP call, made by m1n1 itself on the boot CPU right before it +- * starts a kernel: x1 = 1 to trap EL1 WFI/WFE to EL3 on this and every CPU +- * that later leaves the pen, x2 = AIC base, x3 = primary watchdog base. ++ * starts a kernel: x1 = enum psci_reset_mode (nonzero also traps EL1 WFI/WFE ++ * to EL3 on this and every CPU that later leaves the pen), x2 = AIC base, ++ * x3 = primary watchdog base. + */ + #define PSCI_M1N1_SIP_HANDOFF 0xc200ff00 + +@@ -59,6 +60,10 @@ + #define PSCI_RDV_PENDING 0x00 + #define PSCI_RDV_TRAPS 0x08 + ++/* psci_persist.reentry_flag */ ++#define PSCI_REENTRY_KEXEC 1 ++#define PSCI_REENTRY_RESET 2 ++ + #define SCR_EL3_IRQ (1 << 1) + #define SCR_EL3_FIQ (1 << 2) + #define SCR_EL3_TWI (1 << 12) +@@ -83,13 +88,27 @@ int psci_park_cpu(int cpu, u64 mpidr); + /* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ + int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); + ++/* What SYSTEM_RESET does: payload variable psci_reset=0|proxy|boot (default boot) */ ++enum psci_reset_mode { ++ PSCI_RESET_OFF = 0, /* NOT_SUPPORTED: the kernel resets the SoC its own way */ ++ PSCI_RESET_BOOT = 1, /* back into m1n1, which boots the default boot entry */ ++ PSCI_RESET_PROXY = 2, /* back into m1n1, which only runs the proxy */ ++}; ++void psci_reset_configure(const char *val); ++enum psci_reset_mode psci_reset_mode_get(void); ++/* Set when m1n1 was re-entered by SYSTEM_RESET (psci_reentered() is too) */ ++bool psci_reset_reentered(void); ++/* _start_c: keep the first boot_args, SYSTEM_RESET starts m1n1 with them */ ++void psci_reset_save_boot_args(void); ++ + /* EL1, boot CPU, last thing before the kernel: see PSCI_M1N1_SIP_HANDOFF */ +-void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base); ++void psci_handoff(void); + + /* EL3 side */ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); + void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); + void psci_rendezvous_off(void) __attribute__((noreturn)); ++void psci_system_reset_entry(void *boot_args) __attribute__((noreturn)); + void psci_secondary_entry(void); + void _reentry(void); + +diff --git a/src/psci_asm.S b/src/psci_asm.S +index ff31f3a..61a6a02 100644 +--- a/src/psci_asm.S ++++ b/src/psci_asm.S +@@ -230,10 +230,52 @@ _reentry: + + adrp x1, psci_persist // .reentry_flag + add x1, x1, :lo12:psci_persist +- mov x2, #1 ++ mov x2, #PSCI_REENTRY_KEXEC + str x2, [x1] + dc civac, x1 + dsb sy + + mov x0, x19 + b _start ++ ++/* ++ * psci_system_reset_entry(boot_args): the end of SYSTEM_RESET, at EL3 on the ++ * calling CPU once every other CPU is in the pen (L1D clean) and nothing ++ * else runs. The kernel's EL1 MMU and caches may still be on; its stack and ++ * memory are never used again. Turn EL1 translation and caching off, write ++ * all data caches back, flag the reset and restart m1n1 at EL1h with the ++ * original boot_args, like a cold boot. _start resets the stack and reapplies ++ * the relocations; SP_EL3 goes back to the boot stack. ++ */ ++.globl psci_system_reset_entry ++.type psci_system_reset_entry, @function ++psci_system_reset_entry: ++ msr daifset, #0xf ++ mov x19, x0 ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ isb ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_all ++ tlbi alle1 ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #PSCI_REENTRY_RESET ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ adrp x1, _start ++ add x1, x1, :lo12:_start ++ msr elr_el3, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ adrp x1, _stack_bot_el3 ++ mov sp, x1 ++ mov x0, x19 ++ eret +diff --git a/src/startup.c b/src/startup.c +index 167fae8..4d9ac39 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -171,6 +171,7 @@ void _start_c(void *boot_args, void *base, u64 boot_el) + boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); ++ psci_reset_save_boot_args(); + + adt = + (void *)(((u64)cur_boot_args.devtree) - cur_boot_args.virt_base + cur_boot_args.phys_base); diff --git a/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch b/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch new file mode 100644 index 00000000..a8039549 --- /dev/null +++ b/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch @@ -0,0 +1,298 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 16:03:26 +0100 +Subject: [PATCH] bootmgr: Boot the default entry after a PSCI SYSTEM_RESET + +SYSTEM_RESET brings m1n1 back with every other CPU parked and the +original boot_args, so m1n1 can finish the reboot itself the way the +host's orchard launch starts a system: bring main storage up, take the +first bootable entry in BLS order (the same default the host suggests; +there is still no remembered default) and start it with bootmgr_start(). +The board DTB file name comes from the last launch, which m1n1 now keeps +in .data; if no entry was ever launched (a kernel from the payload, say) +m1n1 doesn't guess the board and stays in the proxy. + +A re-entered m1n1 also skips the stale payload area for the logo, keeps +the picture on screen instead of clearing it and drawing the logo (the +entry's splash replaces it), and leaves SEP alone, which was never +reset. A 96x96 square at the top left shows how far it got: cyan back in +m1n1, green loading the entry, magenta no previous launch, red storage +or entry failure. Any failure ends in the USB proxy, as a cold boot with +nothing to do does; m1n1 never turns it into a cold reset. + +With psci_reset=proxy it stops in the proxy right after the re-entry. + +Assisted-by: LLM +--- + rust/src/bls.rs | 48 +++++++++++++++++++++++++++++ + src/bootmgr.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ + src/bootmgr.h | 2 ++ + src/fb.c | 4 ++- + src/main.c | 16 +++++++--- + 5 files changed, 146 insertions(+), 5 deletions(-) + +diff --git a/rust/src/bls.rs b/rust/src/bls.rs +index 35ddc1b..4570d63 100644 +--- a/rust/src/bls.rs ++++ b/rust/src/bls.rs +@@ -629,6 +629,54 @@ pub unsafe extern "C" fn rust_boot_entries( + text.leak().as_mut_ptr() as *mut c_void + } + ++/// The entry a boot without a host starts: the first bootable one in BLS ++/// order, as the host's `orchard launch` suggests (there is no remembered ++/// default). ++pub fn default_entry(listing: &Listing) -> Option<&Entry> { ++ listing.entries.iter().find(|entry| entry.bootable()) ++} ++ ++/// Writes the id of main storage's default entry (see `default_entry`), ++/// NUL-terminated, to `id` (`size` bytes). Main storage must be up; ++/// `devicetree` may be NULL. Returns 0, or -1 with the reason on the console. ++#[no_mangle] ++pub unsafe extern "C" fn rust_boot_default_entry( ++ devicetree: *const c_char, ++ id: *mut c_char, ++ size: c_size_t, ++) -> c_int { ++ let devicetree = match devicetree.is_null() { ++ true => None, ++ false => unsafe { CStr::from_ptr(devicetree) }.to_str().ok(), ++ }; ++ let listing = read_entries(storage::MainStorage::new, devicetree); ++ log(&listing); ++ let entry = match &listing { ++ Err(error) => { ++ println!("bls: {}", error); ++ return -1; ++ } ++ Ok(listing) => match default_entry(listing) { ++ Some(entry) => entry, ++ None => { ++ println!("bls: no bootable entry"); ++ return -1; ++ } ++ }, ++ }; ++ let name = entry.id.as_bytes(); ++ if name.len() >= size || name.contains(&0) { ++ println!("bls: entry id too long"); ++ return -1; ++ } ++ unsafe { ++ core::ptr::copy_nonoverlapping(name.as_ptr(), id as *mut u8, name.len()); ++ *id.add(name.len()) = 0; ++ } ++ println!("bls: default entry {}", entry.id); ++ 0 ++} ++ + #[repr(C)] + pub struct BootFiles { + kernel: *mut c_void, +diff --git a/src/bootmgr.c b/src/bootmgr.c +index b91b6a6..a746673 100644 +--- a/src/bootmgr.c ++++ b/src/bootmgr.c +@@ -8,6 +8,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "utils.h" +@@ -32,6 +33,32 @@ struct boot_files { + + int rust_boot_entry_load(const char *id, const char *devicetree, bool splash, + struct boot_files *files); ++int rust_boot_default_entry(const char *devicetree, char *id, size_t size); ++ ++/* ++ * The board DTB file name of the last launch (the host's orchard launch ++ * passes it), kept across a SYSTEM_RESET so m1n1 can boot an entry without ++ * the host and without guessing the board. ++ */ ++static struct { ++ bool launched; ++ bool have_devicetree; ++ char devicetree[MAX_NAME]; ++} last_launch __attribute__((section(".data"))); ++ ++/* Warm reset progress, top left */ ++#define MARK_REENTERED 0x00b7ff ++#define MARK_LOADING 0x00c853 ++#define MARK_NO_LAUNCH 0xff00ff ++#define MARK_FAILED 0xff1744 ++ ++static void bootmgr_mark(u32 color) ++{ ++ if (!fb.ptr) ++ return; ++ fb_fill(0, 0, min(fb.width, 96U), min(fb.height, 96U), int2rgb(color)); ++ fb_update(); ++} + + /* Called from Rust: the entry's splash, pixels as b, g, r, 0, centred on black. */ + void bootmgr_paint_splash(const void *pixels, u32 width, u32 height) +@@ -104,6 +131,11 @@ int bootmgr_start(const char *id_arg, const char *devicetree_arg, u64 flags) + return -1; + } + ++ last_launch.launched = true; ++ last_launch.have_devicetree = devicetree_arg; ++ if (devicetree_arg) ++ memcpy(last_launch.devicetree, devicetree, sizeof(devicetree)); ++ + /* The kernel, initrd and DT are m1n1's now, past any limit the host set. */ + heapblock_set_limit(NULL); + +@@ -136,8 +168,57 @@ int bootmgr_start(const char *id_arg, const char *devicetree_arg, u64 flags) + return kboot_boot(kernel); + } + ++/* ++ * m1n1 was restarted by a PSCI SYSTEM_RESET from the kernel: every other CPU ++ * is parked, the watchdog is off and memory is as after a cold start. Do ++ * what the host's orchard launch would: bring up main storage and start the ++ * default entry with the last launch's board DTB. Returns 0 when the kernel ++ * is set up to run, -1 (proxy) otherwise. ++ */ ++int bootmgr_warm_reset(void) ++{ ++ char id[MAX_NAME]; ++ const char *devicetree = last_launch.have_devicetree ? last_launch.devicetree : NULL; ++ ++ bootmgr_mark(MARK_REENTERED); ++ if (psci_reset_mode_get() == PSCI_RESET_PROXY) { ++ printf("bootmgr: warm reset: psci_reset=proxy, not booting\n"); ++ return -1; ++ } ++ if (!last_launch.launched) { ++ printf("bootmgr: warm reset: no boot entry was launched before, not guessing the board\n"); ++ bootmgr_mark(MARK_NO_LAUNCH); ++ return -1; ++ } ++ ++ printf("bootmgr: warm reset: starting the default entry (devicetree %s)\n", ++ devicetree ? devicetree : "from the entry"); ++ if (!main_storage_init()) { ++ printf("bootmgr: warm reset: main storage did not come up\n"); ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ if (rust_boot_default_entry(devicetree, id, sizeof(id)) < 0) { ++ main_storage_shutdown(); ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ bootmgr_mark(MARK_LOADING); ++ if (bootmgr_start(id, devicetree, 0) < 0) { ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ return 0; ++} ++ + #else + ++int bootmgr_warm_reset(void) ++{ ++ printf("bootmgr: not supported in this build\n"); ++ return -1; ++} ++ + int bootmgr_start(const char *id, const char *devicetree, u64 flags) + { + UNUSED(id); +diff --git a/src/bootmgr.h b/src/bootmgr.h +index 8a03baa..f3b86ee 100644 +--- a/src/bootmgr.h ++++ b/src/bootmgr.h +@@ -8,5 +8,7 @@ + #define BOOTMGR_NO_SPLASH BIT(0) + + int bootmgr_start(const char *id, const char *devicetree, u64 flags); ++/* After a PSCI SYSTEM_RESET: boot the default entry the way the last launch did */ ++int bootmgr_warm_reset(void); + + #endif +diff --git a/src/fb.c b/src/fb.c +index ba606b2..0aa88b8 100644 +--- a/src/fb.c ++++ b/src/fb.c +@@ -1,6 +1,7 @@ + /* SPDX-License-Identifier: MIT */ + + #include "fb.h" ++#include "psci.h" + #include "assert.h" + #include "iodev.h" + #include "malloc.h" +@@ -464,14 +465,16 @@ void fb_init(bool clear) + &orig_logo); + } + ++ // After a re-entry from Linux the payload area behind m1n1 is stale ++ bool payload_ok = !psci_reentered(); + u32 *splash, splash_w, splash_h; +- if (payload_splash(&splash, &splash_w, &splash_h) && splash_w <= fb.width && ++ if (payload_ok && payload_splash(&splash, &splash_w, &splash_h) && splash_w <= fb.width && + splash_h <= fb.height) { + custom_logo.ptr = splash; + custom_logo.width = splash_w; + custom_logo.height = splash_h; + logo = &custom_logo; +- } else if (payload_logo(&custom_128, &custom_256)) { ++ } else if (payload_ok && payload_logo(&custom_128, &custom_256)) { + custom_logo = *logo; + if (custom_logo.width == 256) { + custom_logo.ptr = custom_256; +diff --git a/src/main.c b/src/main.c +index 3f32380..8966e41 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -7,6 +7,7 @@ + + #include "adt.h" + #include "aic.h" ++#include "bootmgr.h" + #include "cpufreq.h" + #include "display.h" + #include "exception.h" +@@ -127,6 +128,9 @@ void run_actions(void) + printf("Valid payload found\n"); + return; + } ++ ++ if (psci_reset_reentered() && bootmgr_warm_reset() == 0) ++ return; + fb_set_active(true); + + printf("No valid payload found\n"); +@@ -173,9 +177,11 @@ void m1n1_main(void) + // Kick DCP to sleep, so dodgy monitors which cause reconnect cycles don't cause us to lose the + // framebuffer. + display_shutdown(DCP_SLEEP_IF_EXTERNAL); +- // On idevice we need to always clear, because otherwise it looks scuffed on white devices +- fb_init(!is_mac); +- fb_display_logo(); ++ // On idevice we need to always clear, because otherwise it looks scuffed on white devices. ++ // After a warm reset, keep the picture until the next system's splash. ++ fb_init(!is_mac && !psci_reset_reentered()); ++ if (!psci_reset_reentered()) ++ fb_display_logo(); + #ifdef FB_SILENT_MODE + fb_set_active(!cur_boot_args.video.display); + #else +@@ -184,7 +190,9 @@ void m1n1_main(void) + #endif + + cpufreq_fixup(); +- sep_init(); ++ // SEP was never reset; it is still running what it ran at the first boot ++ if (!psci_reset_reentered()) ++ sep_init(); + #endif + + printf("Initialization complete.\n"); diff --git a/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch b/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch new file mode 100644 index 00000000..08935588 --- /dev/null +++ b/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch @@ -0,0 +1,38 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 12:00:00 +0100 +Subject: [PATCH] kboot: Mark retained hardware after PSCI re-entry + +SYSTEM_RESET re-enters m1n1 without resetting all peripherals. Mark the +next kernel's /chosen with the existing linux,booted-from-kexec boolean +so retained hardware is not mistaken for the cold iBoot baseline. + +Use psci_reentered(), which covers both SYSTEM_RESET and the kexec +trampoline. This also preserves the contract when kboot loads a fresh +BLS devicetree instead of inheriting the outgoing kernel's tree. + +The Orchard kernel's G5P, PMGR, PCIe/ANS and AOP consumers all use this +property to select retained-state handling, with their existing hardware +and certification guards. The only other reader is PowerPC-only. No +kernel consumer needs a separate PSCI-specific property. + +Assisted-by: LLM +--- +diff --git a/src/kboot.c b/src/kboot.c +index 1d4b26f..7dc9874 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -278,6 +278,13 @@ static int dt_set_chosen(void) + printf("FDT: %s = '%s'\n", name, value); + } + ++ /* Re-entry retains peripherals, just like a kernel-to-kernel handoff. */ ++ if (psci_reentered()) { ++ if (fdt_setprop(dt, node, "linux,booted-from-kexec", NULL, 0)) ++ bail("FDT: couldn't mark retained hardware after m1n1 re-entry\n"); ++ printf("FDT: linux,booted-from-kexec (m1n1 re-entry)\n"); ++ } ++ + if (initrd_start && initrd_size) { + if (fdt_setprop_u64(dt, node, "linux,initrd-start", (u64)initrd_start)) + bail("FDT: couldn't set chosen.linux,initrd-start property\n"); diff --git a/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch new file mode 100644 index 00000000..6a30fc37 --- /dev/null +++ b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 2 Nov 2026 15:07:08 +0100 +Subject: [PATCH] kboot: Pass the A9 AOP's boot arguments and sensor + calibration + +The A9's AOP firmware takes more than the ECID from its loader: the AOP +target number (iop-aop-nub aop-target), the chip's ID (/chosen chip-id) +and revision (/arm-io chip-revision). Its accelerometer, gyroscope and +pressure sensor also send no reports until the driver has given them the +unit's factory calibration, which iBoot resolves from syscfg into the +sensors' nodes below iop-aop-nub. + +Copy the three numbers into the AOP's node (alias aop) as +apple,aop-target, apple,chip-id and apple,chip-revision, and each +sensor's calibration properties into the AOP's child node of the same +name, prefixed with "apple,". Unprovisioned "syscfg/" references and +nodes the device tree doesn't have are skipped. The orientation +properties are not copied: the A9 firmware rejects them and the board's +mount-matrix describes the mounting. + +The values are the unit's own and never leave the device: the kernel +sends them to the AOP and nothing else. + +Assisted-by: LLM +--- + src/kboot.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 68 insertions(+), 1 deletion(-) + +diff --git a/src/kboot.c b/src/kboot.c +index ebd6282..7ce9d72 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -1005,6 +1005,73 @@ static int dt_set_als(void) + return 0; + } + ++/* ++ * The A9's AOP firmware also takes the AOP target number and the chip's ID and ++ * revision in its boot arguments, and each motion sensor's factory ++ * calibration, which iBoot resolves from syscfg into the sensor's node below ++ * iop-aop-nub. Copy them into the AOP's node and its sensor child nodes of the ++ * same name; the calibration keeps its name with an "apple," prefix. ++ */ ++static const char *const aop_sensor_calibrations[] = { ++ "low-temp-accel-offset", ++ "accel-sensitivity-calibration", ++ "accel-range-sensitivity-cal", ++ "accel-interrupt-calibration", ++ "gyro-sensitivity-calibration", ++ "gyro-range-sensitivity-cal", ++ "gyro-interrupt-calibration", ++ "gyro-trim-calibration", ++ "pressure-offset-calibration", ++ "pressure-accel-sensitivity", ++ "pressure-heater-resistance", ++}; ++ ++static int dt_set_aop_sensors(int node) ++{ ++ int nub = adt_path_offset(adt, "/arm-io/aop/iop-aop-nub"); ++ if (nub < 0) ++ return 0; ++ ++ u32 val; ++ if (ADT_GETPROP(adt, nub, "aop-target", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,aop-target", val)) ++ bail("FDT: failed to set apple,aop-target\n"); ++ ++ int chosen = adt_path_offset(adt, "/chosen"); ++ if (chosen >= 0 && ADT_GETPROP(adt, chosen, "chip-id", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-id", val)) ++ bail("FDT: failed to set apple,chip-id\n"); ++ ++ int arm_io = adt_path_offset(adt, "/arm-io"); ++ if (arm_io >= 0 && ADT_GETPROP(adt, arm_io, "chip-revision", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-revision", val)) ++ bail("FDT: failed to set apple,chip-revision\n"); ++ ++ int sensor = nub; ++ ADT_FOREACH_CHILD(adt, sensor) ++ { ++ const char *name = adt_get_name(adt, sensor); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(aop_sensor_calibrations); i++) { ++ u32 len; ++ const u8 *blob = adt_getprop(adt, sensor, aop_sensor_calibrations[i], &len); ++ if (!blob || !len || (len >= 7 && !memcmp(blob, "syscfg/", 7))) ++ continue; ++ ++ /* Earlier properties may have moved the sensor's node. */ ++ int child = fdt_subnode_offset(dt, node, name); ++ if (child < 0) ++ break; ++ ++ char prop[64]; ++ snprintf(prop, sizeof(prop), "apple,%s", aop_sensor_calibrations[i]); ++ if (fdt_setprop(dt, child, prop, blob, len)) ++ bail("FDT: failed to set %s/%s\n", name, prop); ++ } ++ } ++ return 0; ++} ++ + /* + * The AOP's firmware carries the chip's ECID in its boot arguments. Pass it + * only to the AOP's node; the driver removes it from the tree once read. +@@ -1028,7 +1095,7 @@ static int dt_set_aop(void) + + if (fdt_setprop_u64(dt, node, "apple,ecid", ecid)) + bail("FDT: failed to set apple,ecid\n"); +- return 0; ++ return dt_set_aop_sensors(node); + } + + static int dt_set_multitouch(void) diff --git a/boot/m1n1/psci-loader b/boot/m1n1/psci-loader new file mode 100755 index 00000000..c21f1801 --- /dev/null +++ b/boot/m1n1/psci-loader @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Append m1n1 PSCI payload variables to a bare m1n1.bin. + +m1n1 entered at EL3 (A7-A9 from Pongo) provides PSCI by default, including +SYSTEM_RESET as a warm reboot into the default boot entry; a bare m1n1.bin +needs no framing for that. Use this to get the other settings, before +assembling any kernel/DTB/logo payload. The output works with +orchard run/build --loader or launch's loaders/m1n1.bin. m1n1 entered at EL1 +always uses the spin table. +""" +import argparse +from pathlib import Path +import shutil + +RESET = {'boot': b'', 'proxy': b'psci_reset=proxy\n', 'off': b'psci_reset=0\n'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--device', required=True, choices=('j71t',)) + parser.add_argument('--loader', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + group = parser.add_mutually_exclusive_group() + group.add_argument('--system-reset', choices=tuple(RESET), default='boot', + help='boot: warm reboot into the default entry (default); proxy: stop in ' + 'the m1n1 proxy; off: NOT_SUPPORTED, kernel resets the SoC itself') + group.add_argument('--no-psci', action='store_true', help='psci=0: spin table, no PSCI') + args = parser.parse_args() + if not args.loader.is_file() or not args.loader.stat().st_size: + parser.error('input must be a nonempty bare m1n1.bin') + # Exclusive creation: never replace the original or another boot artifact. + with args.loader.open('rb') as source, args.output.open('xb') as output: + shutil.copyfileobj(source, output) + output.write(b'psci=0\n' if args.no_psci else b'psci=1\n' + RESET[args.system_reset]) + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index c03fb880..39e7e406 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -313,6 +313,16 @@ "coprocessor": "AOP_ext" } ] + }, + { + "destination": "lib/firmware/apple/agx-g5p-c0.ldr", + "sha256": "33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca", + "steps": [ + { + "step": "agx-g5p-ldr-from-kernelcache@1", + "input": "kernelcache" + } + ] } ] } diff --git a/docs/firmware-recipes.md b/docs/firmware-recipes.md index 6eb10f97..2d0ca8db 100644 --- a/docs/firmware-recipes.md +++ b/docs/firmware-recipes.md @@ -238,3 +238,16 @@ The decompressed N61 kernel's SHA-256 is These fields are explicit recipe parameters, including zero-valued overrides. Physical qualification of firmware loading and input events remains separate from converting this shared firmware file. + + +## S8003 G5P META firmware + +`agx-g5p-ldr-from-kernelcache@1` reads the unique `__TEXT,__const` section +of `com.apple.AGXFirmwareKextG5P` in the user's own legacy prelinked +kernelcache. It rejects any size other than 119112 bytes or any digest other +than `33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca`. +The existing J71t kernelcache input names the active device kernelcache and +the pinned restore-image fallback. The normal recipe installer writes the +verified output to `/lib/firmware/apple/agx-g5p-c0.ldr`. No payload patch, +calibration, firmware bytes or captured job is stored in the source tree. +Other C0 revisions and other boards require separate digest qualification. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index 7b7cc31d..255d4e3a 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -4152,6 +4152,7 @@ CONFIG_DRM_SIMPLEDRM=y # CONFIG_DRM_VKMS is not set # CONFIG_DRM_VMWGFX is not set CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DRM_APPLE_G5P=y # # Frame buffer Devices diff --git a/kernel/patches/devices/s8000/0013-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/devices/s8000/0013-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..dab3a029 --- /dev/null +++ b/kernel/patches/devices/s8000/0013-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,32 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..0bc19727d 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -11,6 +11,14 @@ + #include "s8003.dtsi" + #include "s800x-ipad5.dtsi" + ++&pcie0 { ++ apple,ans-kexec-handoff; ++}; ++ ++&pcie0_dart0 { ++ apple,ans-dart-handoff; ++}; ++ + / { + compatible = "apple,j71t", "apple,s8003", "apple,arm-platform"; + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; diff --git a/kernel/patches/devices/s8000/0014-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/devices/s8000/0014-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..36c9879f --- /dev/null +++ b/kernel/patches/devices/s8000/0014-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 3277a73bb..f6449a7e6 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -443,3 +443,7 @@ pmu { + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/ { ++ apple,reserve-pstore; ++}; diff --git a/kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch b/kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch new file mode 100644 index 00000000..46d27545 --- /dev/null +++ b/kernel/patches/devices/s8000/0015-arm64-dts-apple-select-certified-AOP-reload-through-.patch @@ -0,0 +1,49 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. The AOP itself, its firmware and +mailbox are enabled for every iPad 5 in the shared file; J71t adds the +reserved shutdown page and the power domains the reload observes. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 21 ++++++++++++++++++++ + 1 file changed, 21 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -16,6 +16,27 @@ + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; + }; + ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* A fixed tail page, never a firmware DMA grant. */ ++ aop_shutdown: aop-shutdown@8583fc000 { ++ reg = <0x8 0x583fc000 0 0x4000>; ++ no-map; ++ }; ++ }; ++}; ++ ++&aop { ++ apple,certified-reload; ++ memory-region = <&aop_shutdown>; ++ apple,observe-domains = <&ps_aop_gpio &ps_aop &ps_aop_busif ++ &ps_aop_cpu &ps_aop_filter>; ++ apple,spi0-domain = <&ps_spi0>; ++}; ++ + &touchscreen { + firmware-name = "apple/j71t-fwstart.z2fw"; + }; +-- +2.54.0 diff --git a/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch b/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch new file mode 100644 index 00000000..70a0484d --- /dev/null +++ b/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:20:28 +0100 +Subject: [PATCH] arm64: dts: apple: describe J71t tethered boot + +Record the board fact for the droppable watchdog reset guard. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..0df58a56f 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -24,3 +24,7 @@ &touchscreen { + &wifi { + brcm,board-type = "apple,feara"; + }; ++ ++&wdt { ++ apple,tethered-boot; ++}; diff --git a/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch b/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch new file mode 100644 index 00000000..43ae9e86 --- /dev/null +++ b/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] arm64: dts: apple: j71t: enable paired sleep and the s2idle + gadget + +Mark the J71t ANS port for paired system sleep and keep the USB gadget +connected over suspend to idle. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 3ee634e79..d03bc4ff6 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -75,3 +75,11 @@ + &wdt { + apple,tethered-boot; + }; ++ ++&pcie0 { ++ apple,paired-system-sleep; ++}; ++ ++&usbdev { ++ apple,retain-gadget-in-s2idle; ++}; diff --git a/kernel/patches/devices/s8003/0003-arm64-dts-apple-enable-G5P-on-J71t.patch b/kernel/patches/devices/s8003/0003-arm64-dts-apple-enable-G5P-on-J71t.patch new file mode 100644 index 00000000..4b13d168 --- /dev/null +++ b/kernel/patches/devices/s8003/0003-arm64-dts-apple-enable-G5P-on-J71t.patch @@ -0,0 +1,53 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] arm64: dts: apple: enable G5P on J71t + +Use chip compatible matching; only the qualified board enables the node. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + arch/arm64/boot/dts/apple/s8003.dtsi | 17 +++++++++++++++++ + 2 files changed, 21 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 5b129d695..323c27e27 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -76,6 +76,10 @@ + apple,tethered-boot; + }; + ++&gpu_diag { ++ status = "okay"; ++}; ++ + &pcie0 { + apple,paired-system-sleep; + }; +diff --git a/arch/arm64/boot/dts/apple/s8003.dtsi b/arch/arm64/boot/dts/apple/s8003.dtsi +index b51588941..b7b7a2188 100644 +--- a/arch/arm64/boot/dts/apple/s8003.dtsi ++++ b/arch/arm64/boot/dts/apple/s8003.dtsi +@@ -336,3 +336,20 @@ + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/* G5P resources; enable only on a board qualified with this driver. */ ++&{/soc} { ++ gpu_diag: gpu@201000000 { ++ status = "disabled"; ++ compatible = "apple,s8003-gpu"; ++ reg = <0x2 0x01000000 0x0 0x100000>, ++ <0x2 0x01e00000 0x0 0x100000>; ++ reg-names = "rgx", "fender"; ++ power-domains = <&ps_gfx>; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ }; ++}; diff --git a/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch b/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch new file mode 100644 index 00000000..ff7c75cc --- /dev/null +++ b/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch @@ -0,0 +1,42 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +index 2ce40b469..20de6953f 100644 +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -191,11 +191,11 @@ + interrupt-parent = <&pmu_gpio>; + /* + * The PMU line, ADC done, then the events the iPadOS +- * charger subscribes to. ++ * charger subscribes to, except RTC event 16: Linux gives that ++ * event exclusively to rtc@500, not to source identification. + */ + interrupts = <240 IRQ_TYPE_EDGE_RISING>, + <176 IRQ_TYPE_EDGE_RISING>, +- <16 IRQ_TYPE_EDGE_RISING>, + <40 IRQ_TYPE_EDGE_RISING>, + <41 IRQ_TYPE_EDGE_RISING>, + <42 IRQ_TYPE_EDGE_RISING>, +@@ -218,6 +218,9 @@ + reg = <0x500 0x10>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; ++ interrupt-parent = <&pmu_gpio>; ++ interrupts = <16 IRQ_TYPE_EDGE_RISING>; ++ wakeup-source; + }; + + /* D2365 register 0x31c bit 0 */ diff --git a/kernel/patches/devices/t8010/0029-arm64-dts-apple-retain-J71t-charger-wake.patch b/kernel/patches/devices/t8010/0029-arm64-dts-apple-retain-J71t-charger-wake.patch new file mode 100644 index 00000000..9de5ab14 --- /dev/null +++ b/kernel/patches/devices/t8010/0029-arm64-dts-apple-retain-J71t-charger-wake.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: retain charger wake after shared PMU data + +Keep the charger marked as a wake source after the shared PMU data has +been applied. Preserve the separate RTC event ownership. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -185,6 +185,7 @@ + }; + + tristar_charger: charger@180 { ++ wakeup-source; + compatible = "apple,j71t-tristar-charger", "apple,tristar-charger"; + reg = <0x180 0x10>, <0xa00 0x04>, <0x4000 0x03>; + reg-names = "status", "control", "adc"; diff --git a/kernel/patches/drivers/base/0001-core-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/base/0001-core-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..c3550de8 --- /dev/null +++ b/kernel/patches/drivers/base/0001-core-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + kernel/kexec_core.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c +index dc770b9a6..819650bcd 100644 +--- a/kernel/kexec_core.c ++++ b/kernel/kexec_core.c +@@ -53,6 +53,7 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); + + /* Flag to indicate we are going to kexec a new kernel */ + bool kexec_in_progress = false; ++EXPORT_SYMBOL_GPL(kexec_in_progress); + + bool kexec_file_dbg_print; + diff --git a/kernel/patches/drivers/base/0002-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch b/kernel/patches/drivers/base/0002-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch new file mode 100644 index 00000000..054cba10 --- /dev/null +++ b/kernel/patches/drivers/base/0002-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch @@ -0,0 +1,27 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:54:33 +0100 +Subject: [PATCH] printk: export kmsg_dump_desc for module cold-reset + diagnostics + +The VXD shutdown fail-safe is a GPL module. Allow it to dump emergency +evidence before a cold reset, matching the built-in GPU and PCIe paths. +No printk behavior changes. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index fcc51b699..a812ec1e5 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4882,6 +4882,7 @@ void kmsg_dump_desc(enum kmsg_dump_reason reason, const char *desc) + } + rcu_read_unlock(); + } ++EXPORT_SYMBOL_GPL(kmsg_dump_desc); + + /** + * kmsg_dump_get_line - retrieve one kmsg log line diff --git a/kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..6fd6514b --- /dev/null +++ b/kernel/patches/drivers/base/0003-core-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index 3fcdf4b4e..fcc51b699 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4589,6 +4589,7 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) + { + return __pr_flush(NULL, timeout_ms, reset_on_progress); + } ++EXPORT_SYMBOL_GPL(pr_flush); + + /* + * Delayed printk version, for scheduler-internal messages: diff --git a/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch b/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch new file mode 100644 index 00000000..e510724f --- /dev/null +++ b/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch @@ -0,0 +1,205 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:37 +0000 +Subject: [PATCH] PM: sleep: reserve driver admission before sync through + resume or abort + +Some drivers can only enter system sleep while no new work is admitted, +and they have to decide that before the filesystems are synced. Once +sync has run, a refusal from a suspend callback aborts the transition +late, after work that cannot be undone. There is no hook between the +start of a transition and sync for this. + +Add a small admission list to kernel/power/suspend.c. A driver registers +a prepare/complete pair. enter_state() calls every prepare under +system_transition_mutex before sync, then a blocking precheck notifier +chain on which a driver can refuse the transition with -EBUSY. complete +runs in reverse order after resume and after every abort, including a +failed prepare, and the chain is then called with +PM_SUSPEND_PRECHECK_COMPLETE so that notifier users can drop their +state. pm_suspend_admission_reserved() lets a precheck ask whether a +device's admission is held. + +Assisted-by: LLM +--- + include/linux/pm-suspend-admission.h | 29 ++++++++++ + include/linux/suspend.h | 10 ++++ + kernel/power/suspend.c | 82 ++++++++++++++++++++++++++++ + 3 files changed, 121 insertions(+) + create mode 100644 include/linux/pm-suspend-admission.h + +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +new file mode 100644 +index 000000000..f1c92e058 +--- /dev/null ++++ b/include/linux/pm-suspend-admission.h +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_PM_SUSPEND_ADMISSION_H ++#define _LINUX_PM_SUSPEND_ADMISSION_H ++#include ++#include ++ ++/* Driver-owned reversible admission gate. Registration must precede exposing ++ * hardware work. Unregister before removing the hardware or freeing this object. ++ * prepare must reserve admission without destructive MMIO; complete releases it ++ * after resume or ANY abort, including a failed prepare. No PM-op inference. ++ */ ++struct pm_suspend_admission { ++ struct list_head node; ++ struct device *dev; ++ int (*prepare)(struct device *dev); ++ void (*complete)(struct device *dev); ++ bool reserved; ++}; ++#ifdef CONFIG_SUSPEND ++int register_pm_suspend_admission(struct pm_suspend_admission *admission); ++void unregister_pm_suspend_admission(struct pm_suspend_admission *admission); ++/* Only callable from a precheck notifier, under system_transition_mutex. */ ++bool pm_suspend_admission_reserved(struct device *dev); ++#else ++static inline int register_pm_suspend_admission(struct pm_suspend_admission *a) { return 0; } ++static inline void unregister_pm_suspend_admission(struct pm_suspend_admission *a) {} ++static inline bool pm_suspend_admission_reserved(struct device *dev) { return false; } ++#endif ++#endif +diff --git a/include/linux/suspend.h b/include/linux/suspend.h +index b02876f1a..93082b990 100644 +--- a/include/linux/suspend.h ++++ b/include/linux/suspend.h +@@ -142,9 +142,19 @@ struct platform_s2idle_ops { + void (*end)(void); + }; + ++#ifdef CONFIG_SUSPEND ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb); ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb); ++#else ++static inline int register_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } ++static inline int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } ++#endif ++#define PM_SUSPEND_PRECHECK_COMPLETE 0 ++ + #ifdef CONFIG_SUSPEND + extern suspend_state_t pm_suspend_target_state; + extern suspend_state_t mem_sleep_current; ++ + extern suspend_state_t mem_sleep_default; + + /** +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 57c442686..307e6d5f3 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -9,6 +9,7 @@ + + #define pr_fmt(fmt) "PM: " fmt + ++#include + #include + #include + #include +@@ -573,8 +574,68 @@ static void suspend_finish(void) + * Fail if that's not the case. Otherwise, prepare for system suspend, make the + * system enter the given sleep state and clean up after wakeup. + */ ++static LIST_HEAD(suspend_admissions); ++ ++int register_pm_suspend_admission(struct pm_suspend_admission *a) ++{ ++ struct pm_suspend_admission *other; ++ ++ if (!a->dev || !a->prepare || !a->complete) ++ return -EINVAL; ++ mutex_lock(&system_transition_mutex); ++ list_for_each_entry(other, &suspend_admissions, node) { ++ if (other->dev == a->dev) { ++ mutex_unlock(&system_transition_mutex); ++ return -EEXIST; ++ } ++ } ++ a->reserved = false; ++ get_device(a->dev); ++ list_add_tail(&a->node, &suspend_admissions); ++ mutex_unlock(&system_transition_mutex); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_admission); ++ ++void unregister_pm_suspend_admission(struct pm_suspend_admission *a) ++{ ++ /* Serializes removal/module unload with reservation through complete. */ ++ mutex_lock(&system_transition_mutex); ++ list_del(&a->node); ++ put_device(a->dev); ++ mutex_unlock(&system_transition_mutex); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_admission); ++ ++bool pm_suspend_admission_reserved(struct device *dev) ++{ ++ struct pm_suspend_admission *a; ++ ++ lockdep_assert_held(&system_transition_mutex); ++ list_for_each_entry(a, &suspend_admissions, node) ++ if (a->dev == dev) ++ return a->reserved; ++ return false; ++} ++EXPORT_SYMBOL_GPL(pm_suspend_admission_reserved); ++ ++static BLOCKING_NOTIFIER_HEAD(suspend_precheck_chain); ++ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_register(&suspend_precheck_chain, nb); ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_precheck_notifier); ++ ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_unregister(&suspend_precheck_chain, nb); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); ++ + static int enter_state(suspend_state_t state) + { ++ struct pm_suspend_admission *admission; + int error; + + trace_suspend_resume(TPS("suspend_enter"), state, true); +@@ -594,6 +655,18 @@ static int enter_state(suspend_state_t state) + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); + ++ list_for_each_entry(admission, &suspend_admissions, node) { ++ /* Also release a provider whose prepare returned an error. */ ++ admission->reserved = true; ++ error = admission->prepare(admission->dev); ++ if (error) ++ goto Unlock; ++ } ++ error = notifier_to_errno(blocking_notifier_call_chain( ++ &suspend_precheck_chain, state, NULL)); ++ if (error) ++ goto Unlock; ++ + if (sync_on_suspend_enabled) { + trace_suspend_resume(TPS("sync_filesystems"), 0, true); + +@@ -622,6 +695,15 @@ static int enter_state(suspend_state_t state) + pm_pr_dbg("Finishing wakeup.\n"); + suspend_finish(); + Unlock: ++ list_for_each_entry_reverse(admission, &suspend_admissions, node) { ++ if (!admission->reserved) ++ continue; ++ admission->complete(admission->dev); ++ admission->reserved = false; ++ } ++ /* Release driver admission reservations on success and every abort. */ ++ blocking_notifier_call_chain(&suspend_precheck_chain, ++ PM_SUSPEND_PRECHECK_COMPLETE, NULL); + mutex_unlock(&system_transition_mutex); + return error; + } diff --git a/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch b/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch new file mode 100644 index 00000000..88392fe4 --- /dev/null +++ b/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch @@ -0,0 +1,176 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:11:36 +0000 +Subject: [PATCH] PM: sleep: report suspend admission readiness in + /sys/power/suspend_ready + +Driver admission providers and the early suspend prechecks decide inside +enter_state() whether a suspend is admitted, but userspace has no way to +ask before requesting one. A desktop that offers Suspend, or a power +daemon that would suspend on idle, can only try it and see the +transition fail. + +Add an optional non-reserving check callback to struct +pm_suspend_admission and a read-only /sys/power/suspend_ready. Reading +it takes system_transition_mutex without waiting, runs every provider's +check and then every precheck notifier for the current mem_sleep state, +and reports "ready" or "not-ready " for the first refusal. +While the query runs, pm_suspend_admission_reserved() also reports a +provider whose check passed, so prechecks that require a reserved +provider answer as they would during a suspend. A provider without a +check reports not ready. No reservation is taken, nothing is logged by +the query itself, and a later suspend reserves and checks again. + +Assisted-by: LLM +--- + Documentation/ABI/testing/sysfs-power | 18 ++++++++ + include/linux/pm-suspend-admission.h | 10 ++++- + kernel/power/suspend.c | 63 ++++++++++++++++++++++++++- + 3 files changed, 89 insertions(+), 2 deletions(-) + +diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power +index d38da0779..98cc5c2b5 100644 +--- a/Documentation/ABI/testing/sysfs-power ++++ b/Documentation/ABI/testing/sysfs-power +@@ -470,3 +470,21 @@ Description: + + Minimum value: 1 + Default value: 3 ++ ++What: /sys/power/suspend_ready ++Date: November 2026 ++Contact: Linux PM list ++Description: ++ Read-only report of whether a suspend to the current ++ mem_sleep state would be admitted now. Reading runs the ++ admission a suspend runs before syncing filesystems: each ++ registered driver admission provider's non-reserving check, ++ then each early suspend precheck. It takes no reservation ++ and changes no state; a later suspend checks again. ++ ++ The file reads "ready", or one line ++ "not-ready " naming the first refusal: a ++ device name for a provider, the precheck callback's symbol, ++ "transition" while a system transition is in progress, or ++ "state" when the mem_sleep state is not available. ++ is a negative decimal errno value. +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +index f1c92e058..0b1a1c57b 100644 +--- a/include/linux/pm-suspend-admission.h ++++ b/include/linux/pm-suspend-admission.h +@@ -8,18 +8,26 @@ + * hardware work. Unregister before removing the hardware or freeing this object. + * prepare must reserve admission without destructive MMIO; complete releases it + * after resume or ANY abort, including a failed prepare. No PM-op inference. ++ * check, optional, answers /sys/power/suspend_ready: whether prepare would ++ * admit now, without reserving or any side effect. Work prepare would drain ++ * is not a refusal. A provider without check reports not ready. + */ + struct pm_suspend_admission { + struct list_head node; + struct device *dev; + int (*prepare)(struct device *dev); + void (*complete)(struct device *dev); ++ int (*check)(struct device *dev); + bool reserved; ++ bool checked; /* check passed during a readiness query */ + }; + #ifdef CONFIG_SUSPEND + int register_pm_suspend_admission(struct pm_suspend_admission *admission); + void unregister_pm_suspend_admission(struct pm_suspend_admission *admission); +-/* Only callable from a precheck notifier, under system_transition_mutex. */ ++/* ++ * Only callable from a precheck notifier, under system_transition_mutex. ++ * During a readiness query it reports a provider whose check passed. ++ */ + bool pm_suspend_admission_reserved(struct device *dev); + #else + static inline int register_pm_suspend_admission(struct pm_suspend_admission *a) { return 0; } +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 307e6d5f3..670f50ad8 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -590,6 +590,7 @@ int register_pm_suspend_admission(struct pm_suspend_admission *a) + } + } + a->reserved = false; ++ a->checked = false; + get_device(a->dev); + list_add_tail(&a->node, &suspend_admissions); + mutex_unlock(&system_transition_mutex); +@@ -614,7 +615,7 @@ bool pm_suspend_admission_reserved(struct device *dev) + lockdep_assert_held(&system_transition_mutex); + list_for_each_entry(a, &suspend_admissions, node) + if (a->dev == dev) +- return a->reserved; ++ return a->reserved || a->checked; + return false; + } + EXPORT_SYMBOL_GPL(pm_suspend_admission_reserved); +@@ -633,6 +634,66 @@ int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) + } + EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); + ++/* ++ * The admission enter_state() would run, read only: each provider's check ++ * instead of its reservation, then each precheck notifier for the current ++ * mem_sleep state. No reservation is taken and nothing is logged here; a ++ * later suspend still reserves and checks again. ++ */ ++static ssize_t suspend_ready_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ suspend_state_t state = mem_sleep_current; ++ struct pm_suspend_admission *a; ++ struct notifier_block *nb; ++ ssize_t len = 0; ++ int error; ++ ++ if (!mutex_trylock(&system_transition_mutex)) ++ return sysfs_emit(buf, "not-ready transition %d\n", -EBUSY); ++ if (state != PM_SUSPEND_TO_IDLE && !valid_state(state)) { ++ len = sysfs_emit(buf, "not-ready state %d\n", -EINVAL); ++ goto unlock; ++ } ++ list_for_each_entry(a, &suspend_admissions, node) { ++ error = a->check ? a->check(a->dev) : -EOPNOTSUPP; ++ if (error) { ++ len = sysfs_emit(buf, "not-ready %s %d\n", ++ dev_name(a->dev), error); ++ goto clear; ++ } ++ a->checked = true; ++ } ++ /* Walk the chain to name the refusing notifier, not only its errno. */ ++ down_read(&suspend_precheck_chain.rwsem); ++ for (nb = rcu_dereference_raw(suspend_precheck_chain.head); nb; ++ nb = rcu_dereference_raw(nb->next)) { ++ error = notifier_to_errno(nb->notifier_call(nb, state, NULL)); ++ if (error) { ++ len = sysfs_emit(buf, "not-ready %ps %d\n", ++ nb->notifier_call, error); ++ break; ++ } ++ } ++ up_read(&suspend_precheck_chain.rwsem); ++ if (!len) ++ len = sysfs_emit(buf, "ready\n"); ++clear: ++ list_for_each_entry(a, &suspend_admissions, node) ++ a->checked = false; ++unlock: ++ mutex_unlock(&system_transition_mutex); ++ return len; ++} ++ ++power_attr_ro(suspend_ready); ++ ++static int __init suspend_ready_init(void) ++{ ++ return sysfs_create_file(power_kobj, &suspend_ready_attr.attr); ++} ++late_initcall(suspend_ready_init); ++ + static int enter_state(suspend_state_t state) + { + struct pm_suspend_admission *admission; diff --git a/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..2865725b --- /dev/null +++ b/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] drm: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/gpu/drm/adbe/adbe_drv.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..7ed2de3d7 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -1065,6 +1065,9 @@ static void adbe_shutdown(struct platform_device *pdev) + struct adbe *adbe = platform_get_drvdata(pdev); + + drm_atomic_helper_shutdown(&adbe->drm); ++ timer_delete_sync(&adbe->flip_timer); ++ /* release9: restore the boot buffer even if the CRTC was never enabled. */ ++ adbe_park(adbe); + writel(adbe->saved_inten, adbe->regs + ADP_INTEN); + } + diff --git a/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch b/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch new file mode 100644 index 00000000..d0acb8a6 --- /dev/null +++ b/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch @@ -0,0 +1,46 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] dt-bindings: gpu: describe Apple S8003 G5P resources + +Use chip compatible matching; only the qualified board enables the node. + +Assisted-by: LLM +--- + .../bindings/gpu/apple,s8003-g5p.yaml | 27 +++++++++++++++++++ + 1 file changed, 27 insertions(+) + create mode 100644 Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml + +diff --git a/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +new file mode 100644 +index 000000000..76aebc183 +--- /dev/null ++++ b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +@@ -0,0 +1,27 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/gpu/apple,s8003-g5p.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++title: Apple S8003 G5P GPU ++maintainers: ++ - Berkay Kösebay ++description: | ++ G5P GPU with META firmware on S8003. The compatible is Apple's inherited ++ ADT spelling used by the qualified driver; no new vendor prefix is implied. ++properties: ++ compatible: ++ const: apple,s8003-gpu ++ reg: ++ maxItems: 2 ++ reg-names: ++ items: ++ - const: rgx ++ - const: fender ++ interrupts: ++ minItems: 5 ++ maxItems: 5 ++ power-domains: ++ maxItems: 1 ++required: [compatible, reg, reg-names, interrupts, power-domains] ++additionalProperties: false diff --git a/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch b/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch new file mode 100644 index 00000000..4da6e1b6 --- /dev/null +++ b/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch @@ -0,0 +1,651 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: define the render and compute userspace + interface + +Import the sealed warmport ABI, before pipeline stage wait extensions. + +Assisted-by: LLM +--- + include/uapi/drm/apple_g5p_drm.h | 631 +++++++++++++++++++++++++++++++ + 1 file changed, 631 insertions(+) + create mode 100644 include/uapi/drm/apple_g5p_drm.h + +diff --git a/include/uapi/drm/apple_g5p_drm.h b/include/uapi/drm/apple_g5p_drm.h +new file mode 100644 +index 000000000..5e41ab533 +--- /dev/null ++++ b/include/uapi/drm/apple_g5p_drm.h +@@ -0,0 +1,631 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++#ifndef _UAPI_APPLE_G5P_DRM_H_ ++#define _UAPI_APPLE_G5P_DRM_H_ ++ ++#include "drm.h" ++ ++/* G5P merged winsys ABI. Retired ioctl numbers remain reserved. */ ++#define DRM_G5P_UAPI_VERSION 1 ++#define DRM_G5P_CAP_BO_CPU (1ULL << 0) ++#define DRM_G5P_CAP_VM_PREPARE (1ULL << 1) ++#define DRM_G5P_CAP_SYNCOBJ (1ULL << 2) ++#define DRM_G5P_CAP_RENDER (1ULL << 5) ++#define DRM_G5P_CAP_PRIME (1ULL << 6) ++#define DRM_G5P_CAP_CODE_BASE_VERIFIED (1ULL << 7) ++#define DRM_G5P_CAP_VM_LIVE (1ULL << 8) ++#define DRM_G5P_CAP_CONTEXT_LIVE (1ULL << 9) ++#define DRM_G5P_CAP_JOB_STATUS (1ULL << 10) ++#define DRM_G5P_CAP_CONTEXT_STATUS (1ULL << 11) ++#define DRM_G5P_CAP_FAULT_INFO (1ULL << 13) ++ ++/* LIVE means the owned user catalogue has completed firmware attachment. */ ++#define DRM_G5P_CONTEXT_LIVE 3 ++#define DRM_G5P_CONTEXT_HOST_ONLY 0 ++#define DRM_G5P_CONTEXT_LOST 1 ++ ++struct drm_g5p_get_context_status { ++ __u32 context; ++ __u32 flags; ++ __u32 state; ++ __u32 map_count; ++ __u64 mapped_bytes; ++ __u64 reserved; ++}; ++ ++/* First terminal firmware event, retained independently of last_event. */ ++#define DRM_G5P_FAULT_NONE 0 ++#define DRM_G5P_FAULT_RECOVERY 1 ++#define DRM_G5P_FAULT_PAGE_REQUEST 2 ++#define DRM_G5P_FAULT_CHANNEL 3 ++#define DRM_G5P_FAULT_UNKNOWN 4 ++#define DRM_G5P_FAULT_VALID_CHANNEL (1U << 0) ++struct drm_g5p_get_fault_info { ++ __u32 flags; ++ __u32 kind; ++ __u64 event_serial; /* One-based arrival order; zero if no fault. */ ++ __u32 valid; ++ __u32 channel_class; /* Event8: 0 memory failure, 1 killed, others opaque. */ ++ __u32 channel_token; /* Firmware token, not a DRM context handle. */ ++ __u32 channel_stamp; ++ __u32 raw[11]; /* Includes unknown fields; no decoded fault VA is claimed. */ ++ __u32 reserved; ++}; ++ ++struct drm_g5p_get_info { ++ __u32 size; ++ __u32 flags; ++ __u32 version; ++ __u32 gpu_page_shift; ++ __u64 capabilities; ++ __u64 va_start; ++ __u64 va_end; ++ __u64 reserved_start; ++ __u64 reserved_end; ++ __u64 code_base; /* Zero until CODE_BASE_VERIFIED. */ ++ __u64 max_bo_size; ++ __u32 bvnc[4]; /* Static target, not a new MMIO measurement. */ ++ __u64 reserved[2]; ++}; ++ ++/* GPU access roles; CPU initialization/mmap is permitted for every role. */ ++#define DRM_G5P_BO_READ (1U << 0) ++#define DRM_G5P_BO_WRITE (1U << 1) ++#define DRM_G5P_BO_EXEC (1U << 2) ++struct drm_g5p_create_bo { ++ __u64 size; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_bo_mmap_offset { ++ __u32 handle; ++ __u32 flags; ++ __u64 offset; ++}; ++ ++#define DRM_G5P_BO_SYNC_TO_DEVICE 0 ++#define DRM_G5P_BO_SYNC_TO_CPU 1 ++struct drm_g5p_bo_sync { ++ __u32 handle; ++ __u32 op; ++ __u64 offset; ++ __u64 size; ++}; ++ ++struct drm_g5p_create_vm { ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_vm { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++#define DRM_G5P_VM_MAP 0 ++#define DRM_G5P_VM_UNMAP 1 ++struct drm_g5p_vm_bind { ++ __u32 vm; ++ __u32 bo; ++ __u64 va; ++ __u64 size; ++ __u64 offset; ++ __u32 op; ++ __u32 flags; ++}; ++ ++struct drm_g5p_create_context { ++ __u32 vm; ++ __u32 type; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_context { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* A point of zero uses a binary syncobj; nonzero uses a timeline point. */ ++#define DRM_G5P_SYNC_SIGNAL (1U << 0) ++struct drm_g5p_sync { ++ __u32 handle; ++ __u32 flags; ++ __u64 point; ++}; ++ ++/* Privileged one-shot kernel fixture; all payload/BO fields must be zero. ++ * The empty user context authorizes ownership only; its VM is not attached. ++ * Does not advertise general COMPUTE or attach user-provided GPU addresses. ++ */ ++ ++/* Geometry is all zero/NONE while CODE_BASE_VERIFIED is clear. */ ++#define DRM_G5P_BASE_NONE 0 ++#define DRM_G5P_BASE_DEVICE 1 ++#define DRM_G5P_BASE_VM 2 ++#define DRM_G5P_BASE_CONTEXT 3 ++struct drm_g5p_get_exec_info { ++ __u32 size; ++ __u32 flags; ++ __u32 base_scope; ++ __u32 pad; ++ __u64 usc_base; ++ __u64 pds_base; ++ __u64 window_size; ++ __u64 allocation_alignment; ++ __u32 usc_offset_bits; ++ __u32 pds_offset_bits; ++ __u64 reserved[3]; ++}; ++ ++#define DRM_G5P_JOB_PENDING 0 ++#define DRM_G5P_JOB_COMPLETE 1 ++struct drm_g5p_get_job_status { ++ __u64 job_id; ++ __u32 flags; ++ __u32 state; ++ __s32 error; /* -EINPROGRESS while pending; immutable after completion. */ ++ __u32 pad; ++ __u64 reset_generation; /* Zero: no reset/recovery support. */ ++}; ++ ++#define DRM_G5P_GET_INFO 0x00 ++#define DRM_G5P_CREATE_BO 0x01 ++#define DRM_G5P_BO_MMAP_OFFSET 0x02 ++#define DRM_G5P_BO_SYNC 0x03 ++#define DRM_G5P_CREATE_VM 0x04 ++#define DRM_G5P_DESTROY_VM 0x05 ++#define DRM_G5P_VM_BIND 0x06 ++#define DRM_G5P_CREATE_CONTEXT 0x07 ++#define DRM_G5P_DESTROY_CONTEXT 0x08 ++#define DRM_G5P_GET_EXEC_INFO 0x0b ++#define DRM_G5P_GET_JOB_STATUS 0x0c ++#define DRM_G5P_GET_CONTEXT_STATUS 0x0d ++#define DRM_G5P_GET_FAULT_INFO 0x0e ++ ++#define DRM_IOCTL_G5P_GET_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_INFO, struct drm_g5p_get_info) ++#define DRM_IOCTL_G5P_CREATE_BO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_BO, struct drm_g5p_create_bo) ++#define DRM_IOCTL_G5P_BO_MMAP_OFFSET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_BO_MMAP_OFFSET, \ ++ struct drm_g5p_bo_mmap_offset) ++#define DRM_IOCTL_G5P_BO_SYNC \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_BO_SYNC, struct drm_g5p_bo_sync) ++#define DRM_IOCTL_G5P_CREATE_VM \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_VM, struct drm_g5p_create_vm) ++#define DRM_IOCTL_G5P_DESTROY_VM \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_VM, \ ++ struct drm_g5p_destroy_vm) ++#define DRM_IOCTL_G5P_VM_BIND \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_VM_BIND, struct drm_g5p_vm_bind) ++#define DRM_IOCTL_G5P_CREATE_CONTEXT \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_CONTEXT, \ ++ struct drm_g5p_create_context) ++#define DRM_IOCTL_G5P_DESTROY_CONTEXT \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_CONTEXT, \ ++ struct drm_g5p_destroy_context) ++ ++#define DRM_IOCTL_G5P_GET_EXEC_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_EXEC_INFO, \ ++ struct drm_g5p_get_exec_info) ++#define DRM_IOCTL_G5P_GET_JOB_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_JOB_STATUS, \ ++ struct drm_g5p_get_job_status) ++ ++#define DRM_IOCTL_G5P_GET_CONTEXT_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_CONTEXT_STATUS, \ ++ struct drm_g5p_get_context_status) ++ ++#define DRM_IOCTL_G5P_GET_FAULT_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_FAULT_INFO, \ ++ struct drm_g5p_get_fault_info) ++ ++#define DRM_G5P_CAP_IDLE_VM_BIND (1ULL << 18) ++ ++#define DRM_G5P_CAP_BOUNDED_RENDER (1ULL << 19) ++#define DRM_G5P_CAP_TYPED_RENDER (1ULL << 20) ++#define DRM_G5P_CAP_RENDER_USER_GEOMETRY (1ULL << 21) ++#define DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY (1ULL << 22) ++#define DRM_G5P_CAP_TYPED_RENDER_V2 (1ULL << 23) ++/* Bit 24 is reserved for COMPUTE_BLIT_V1 and bit 25 is FIXED_DATA_B1 on ++ * a9/release, so the next bit free in every branch is 26. ++ */ ++#define DRM_G5P_CAP_TYPED_RENDER_V3 (1ULL << 26) ++ ++/** ++ * DRM_G5P_CAP_RENDER_MULTI_CONTEXT - several render contexts share the GPU ++ * ++ * Reported together with TYPED_RENDER_V3. Any number of render contexts, in any ++ * number of files, may submit TYPED_V3 jobs. Each context has its own VM and its ++ * own firmware slot pair, and the jobs of all contexts run one at a time in ++ * submission order. Without this bit, the first context to submit owns the GPU ++ * until reboot and every other context is refused. The other render profiles ++ * stay single-context. ++ */ ++#define DRM_G5P_CAP_RENDER_MULTI_CONTEXT (1ULL << 27) ++ ++/** ++ * DRM_G5P_CAP_RENDER_UNPRIVILEGED - TYPED_V3 render needs no CAP_SYS_RAWIO ++ * ++ * Reported together with RENDER_MULTI_CONTEXT. Any opener of the render node may ++ * create render contexts and submit TYPED_V3 jobs. A render VM then maps none of ++ * the firmware's memory beyond the kernel-built window the GPU's own requesters ++ * need, and the contexts close cleanly: a closed context's firmware binding is ++ * removed and its memory freed. Pinned memory is limited device-wide. The other ++ * render profiles, compute and the diagnostic ioctls still need CAP_SYS_RAWIO. ++ */ ++#define DRM_G5P_CAP_RENDER_UNPRIVILEGED (1ULL << 28) ++ ++/** ++ * DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH - TYPED_V3 carries pixel_output_width ++ * ++ * Reported together with TYPED_RENDER_V3. struct drm_g5p_render_pass_v3 has ++ * pixel_output_width, so a TYPED_V3 pass in the V1 dialect (all-zero ext) can ++ * write several colour targets. Without this bit the field is reserved and ++ * must be zero. ++ */ ++#define DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH (1ULL << 29) ++ ++/* drm_g5p_create_context.type; flags must be zero. */ ++#define DRM_G5P_CONTEXT_RENDER 1 ++ ++#define DRM_G5P_RENDER_PROFILE_TYPED_V3 4 ++ ++/* LINEAR64_V1: kind is also the resource's position in the canonical array. */ ++ ++#define DRM_G5P_RENDER_ACCESS_READ (1U << 0) ++#define DRM_G5P_RENDER_ACCESS_WRITE (1U << 1) ++#define DRM_G5P_RENDER_ACCESS_IMPLICIT (1U << 2) ++#define DRM_G5P_RENDER_ACCESS_CACHE (1U << 3) ++ ++#define DRM_G5P_RENDER_MAX_RESOURCES 256 ++#define DRM_G5P_RENDER_MAX_SYNCS 17 ++#define DRM_G5P_RENDER_MAX_VDM_BYTES 0x10000 ++ ++/* A GPU VA range [va, va + size) inside one existing mapping of handle's BO ++ * in the context VM, at BO byte offset `offset`. Never a CPU pointer, PA or ++ * META address. LINEAR64_V1 requires offset 0 and the exact whole BO. ++ */ ++struct drm_g5p_render_resource { ++ __u32 handle; ++ __u32 kind; ++ __u64 offset; ++ __u64 size; ++ __u64 va; ++}; ++ ++/** ++ * DRM_G5P_CAP_RENDER_MSAA - 4x multisampled render targets and passes ++ * ++ * Reported with TYPED_RENDER_V3 and RENDER_KERNEL_GEOMETRY. ++ * CREATE_RENDER_TARGET accepts samples = 4 (kernel-derived geometry; with ++ * DRM_G5P_RT_USER_GEOMETRY userspace may only restate it), and a TYPED_V3 ++ * pass carries samples, msaa_flags and sample_positions in struct ++ * drm_g5p_render_pass. Without this bit those fields are reserved. ++ */ ++#define DRM_G5P_CAP_RENDER_MSAA (1ULL << 31) ++ ++/** ++ * DRM_G5P_CAP_RENDER_OCCLUSION_QUERY - TYPED_V3 carries ISP query storage ++ * ++ * The V2 extension names the query-result allocation used by ISP visibility ++ * tests. The kernel validates the complete range and writes its base into the ++ * two per-partition ISP_OCLQRY_BASE work-record slots. A zero VA and size ++ * disable visibility-result storage. ++ */ ++#define DRM_G5P_CAP_RENDER_OCCLUSION_QUERY (1ULL << 33) ++ ++/* drm_g5p_render_pass.msaa_flags: the end of tile averages each pixel's ++ * samples into the (one-sample) surface the EOT's PBE words name. ++ */ ++#define DRM_G5P_RENDER_MSAA_RESOLVE (1U << 0) ++ ++struct drm_g5p_render_pass { ++ __u32 size; /* sizeof(struct drm_g5p_render_pass) */ ++ __u32 flags; /* zero */ ++ __u64 vdm_va; /* VDM control stream, READ, 4-byte aligned */ ++ __u32 vdm_size; /* bytes, multiple of 4, <= MAX_VDM_BYTES */ ++ __u32 vdm_control; /* V1/V2: TA W+0x88 value, multiple of 16; V3: zero */ ++ __u64 scratch_va; /* TA scratch block, WRITE, 16-byte aligned */ ++ __u64 scratch_size; ++ __u64 scratch_ptr[6]; /* TA scratch/marker pointers, in a WRITE range */ ++ __u64 pool7_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool7_size; ++ __u64 pool8_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool8_size; ++ __u32 context_pds[2][6]; /* TA context store/load PDS descriptors */ ++ __u32 resume_pds[2][4]; /* TA resume PDS descriptors */ ++ __u32 ta_pds[4]; /* TA state PDS descriptor; word 3 zero */ ++ __u32 background[8]; /* 3D background (load/clear) object */ ++ __u32 eot[4]; /* 3D end-of-tile object; word 3 zero */ ++ __u32 eot_resume[4]; /* 3D partial-render EOT; word 3 zero */ ++ __u32 restore[4]; /* 3D partial-render restore words */ ++ /* DRM_G5P_CAP_RENDER_MSAA, TYPED_V3 only; zero in every other profile. ++ * samples: 0 or 1 with a one-sample target, 4 with a four-sample one. ++ * sample_positions: one byte per sample, x in [3:0] and y in [7:4] in ++ * 1/16 px (PPP_MULTISAMPLECTL); 0 takes the target's standard pattern. ++ * The kernel writes the AA mode, ISP_CTL and output-partition words ++ * for the sample count. The draws' ISPCTL.upass (0x7 at four samples, ++ * 0xf at one), the PBE/EOT words and the fragment programs are ++ * userspace's, as for one sample; see docs/gpu/experiments/msaa.md. ++ */ ++ __u32 samples; ++ __u32 msaa_flags; /* DRM_G5P_RENDER_MSAA_* */ ++ __u64 sample_positions; ++ __u64 reserved[6]; ++}; ++ ++/* ++ * TYPED_V2 additions. Each field is a 3D register image that the firmware ++ * uploads from the normal-3D work object. The CR offsets are the firmware ++ * store targets (its 3D upload puts W3c0 on ROGUE_CR_ISP_CTL 0xf38); the ++ * Rogue names are inferred, graded in docs/gpu/render-uapi.md. ++ */ ++#define DRM_G5P_RENDER_MAX_TABLE_ENTRIES 65536 /* ISPDBSC 16-bit indices */ ++#define DRM_G5P_RENDER_ZLS_REGS 6 ++ ++struct drm_g5p_render_pass_ext { ++ /* ISP_SCISSOR_BASE (CR fb0): IPF_SCISSOR_WORD_0/1 pairs, 8 bytes each, ++ * indexed by TA_STATE_ISPDBSC.scindex. READ, 4-byte aligned. ++ */ ++ __u64 scissor_va; ++ __u32 scissor_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ __u32 dbias_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ /* ISP_DBIAS_BASE (CR fb8): {constant, slope, clamp} f32 triples, ++ * 12 bytes each, indexed by ISPDBSC.dbindex. READ, 4-byte aligned. ++ */ ++ __u64 dbias_va; ++ /* PDS_BGRND1_BASE (CR 6a8): background object texture/varying data ++ * (B-relative, like background[]); LOAD_OP_LOAD's texture state. ++ */ ++ __u64 background_texture; ++ __u32 bgobj_depth; /* ISP_BGOBJDEPTH (CR f80): depth clear, ZLS format */ ++ __u32 bgobj_vals; /* ISP_BGOBJVALS (CR f88): stencil [7:0], mask [8], bgtag [9] */ ++ /* Pixel output width (2, 4 or 8): the 6ca7ae0 inputs to ISP_CTL's ++ * partition fields and the six per-core output words. 2 for one ++ * 32-bpp colour attachment (the captured value). ++ */ ++ __u32 pixel_output_width; ++ /* Bit i: zls[i] is an address ([39:4]; [63:40] zero; [3:0] flags) ++ * inside a listed READ|WRITE range. The kernel does not decode ZLS. ++ */ ++ __u32 zls_address_mask; ++ /* Opaque ZLS register images, CR f48, f50, f58, f60, f68 and fd0: ++ * Rogue ISP_ZLSCTL, ZLOAD/ZSTORE and STENCIL_LOAD/STORE bases plus ++ * one more; positions proven, order unqualified. Zero: no ZLS. ++ */ ++ __u64 zls[DRM_G5P_RENDER_ZLS_REGS]; ++ __u64 occlusion_query_va; /* ISP_OCLQRY_BASE, 16-byte aligned, READ|WRITE */ ++ __u64 occlusion_query_size; /* bytes in the listed allocation */ ++ __u64 reserved[2]; ++}; ++ ++struct drm_g5p_render_pass_v2 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++}; ++ ++#define DRM_G5P_RENDER_MAX_PROGRAM_BYTES 0x1000 ++#define DRM_G5P_RENDER_STREAM_TERMINATE 0xc0000000U ++ ++struct drm_g5p_render_pass_v3 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++ __u64 ta_program_va; /* -> W+0x88 / R0+0x470 as an offset from pds_base */ ++ __u32 ta_program_size; /* bytes, 16..MAX_PROGRAM_BYTES, multiple of 16 */ ++ __u32 vdm_end; /* bytes from vdm_va past STREAM_TERMINATE, <= vdm_size */ ++ __u32 pixel_output_width; /* V1 dialect: 0 (= 2), 2, 4 or 8 */ ++ __u32 pad; /* zero */ ++ __u64 reserved[3]; ++}; ++ ++struct drm_g5p_submit_render_typed { ++ __u32 context; ++ __u32 flags; /* zero */ ++ __u32 profile; ++ __u32 render_target; /* handle from CREATE_RENDER_TARGET */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[] */ ++ __u64 pass; /* struct drm_g5p_render_pass */ ++ __u32 pass_size; ++ __u32 pad; ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; ++}; ++ ++/* ++ * Render-target memory geometry: the AGXRenderTargetMemoryG5 arithmetic ++ * outputs (firmware round16 section 3, R-offsets in comments). The kernel ++ * computes them for its verified table (64x64x1), or with ++ * RENDER_KERNEL_GEOMETRY for any size; with RENDER_USER_GEOMETRY userspace ++ * supplies them and the kernel bounds the memory they imply (and, with ++ * RENDER_KERNEL_GEOMETRY, requires its own result). ++ */ ++struct drm_g5p_rt_geometry { ++ __u32 header_bytes; /* R10 */ ++ __u32 region_bytes; /* R20 */ ++ __u64 packed_state; /* Rb0 */ ++ __u32 state_word; /* Rb8 */ ++ __u32 padded_dims; /* Rbc */ ++ __u64 scale[3]; /* Rc0, Rc8, Rd0 packed float pairs */ ++ __u64 ta_state[2]; /* Rf0, Rf8 */ ++ __u64 ta_tiling[2]; /* R100, R108 */ ++ __u32 ta_tiling_word; /* R110 */ ++ __u32 tile_count; /* R118 */ ++ __u64 secondary_bytes; /* R138 */ ++ __u64 secondary_align; /* R178 */ ++ __u64 reserved[2]; ++}; ++ ++#define DRM_G5P_RT_USER_GEOMETRY (1U << 0) ++struct drm_g5p_create_render_target { ++ __u32 context; /* render context; RT memory lives in its VM */ ++ __u32 flags; ++ __u32 width; ++ __u32 height; ++ __u32 samples; /* 1, or 4 with DRM_G5P_CAP_RENDER_MSAA */ ++ __u32 layers; /* 1 */ ++ __u32 handle; /* output */ ++ __u32 pad; ++ __u64 main_va; /* output: kernel-private GPU VA */ ++ __u64 main_bytes; /* output */ ++ __u64 secondary_va; /* output */ ++ __u64 secondary_bytes; /* output: mapped bytes */ ++ struct drm_g5p_rt_geometry geometry; /* input if USER, else output */ ++ __u32 pm_blocks; /* output with RENDER_KERNEL_GEOMETRY: context PM pool */ ++ __u32 pad2; ++ __u64 reserved; ++}; ++ ++struct drm_g5p_destroy_render_target { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* GET_RENDER_INFO.flags, output only (input must be zero). ++ * VARIANT_ACTIVE: a render_user_* hypothesis knob (stage, cache/mmu/timestamp ++ * variant, 3d_after_ta, pm_policy, cache_repeat, wait_ms, pm/memory/rt/slot ++ * variant, profile1_vdm_control) differs from its default, so a context ++ * created now latches a non-default job behaviour. GET_RENDER_INFO has no ++ * context: it reports what CREATE_CONTEXT would latch at this moment. ++ */ ++#define DRM_G5P_RENDER_INFO_VARIANT_ACTIVE (1U << 0) ++ ++struct drm_g5p_get_render_info { ++ __u32 size; /* sizeof, exact */ ++ __u32 flags; /* in: zero; out: RENDER_INFO_* */ ++ __u32 profiles; /* bit N: profile N accepted */ ++ __u32 max_render_targets; ++ __u64 private_va_start; /* kernel window in render VMs */ ++ __u64 private_va_end; ++ __u64 user_va_min; /* lowest mappable VA in a render VM */ ++ __u64 usc_base; /* common USC/PDS base, 32-bit offsets */ ++ __u64 pds_base; ++ __u64 code_window; /* bytes above usc_base usable for code */ ++ __u32 pm_blocks; /* parameter buffer blocks per context */ ++ __u32 pm_block_bytes; ++ __u32 max_queued_jobs; ++ __u32 max_vdm_bytes; ++ __u32 job_timeout_ms; ++ __u32 retained_results; ++ /* RENDER_KERNEL_GEOMETRY only, else zero (64x64 only). */ ++ __u32 max_render_width; ++ __u32 max_render_height; ++ __u32 max_pm_blocks; /* PM pool growth bound per context */ ++ __u32 pad; ++ union { ++ __u64 reserved[2]; /* in: zero */ ++ struct { ++ /* TYPED_V3: every GPU VA the TA/3D read ends at or below ++ * this; 0 makes no statement. Out only. ++ */ ++ __u64 hw_va_limit; ++ __u64 reserved1; ++ }; ++ }; ++}; ++ ++#define DRM_G5P_CREATE_RENDER_TARGET 0x11 ++#define DRM_G5P_DESTROY_RENDER_TARGET 0x12 ++#define DRM_G5P_GET_RENDER_INFO 0x13 ++#define DRM_G5P_SUBMIT_RENDER_TYPED 0x14 ++ ++#define DRM_IOCTL_G5P_SUBMIT_RENDER_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_RENDER_TYPED, \ ++ struct drm_g5p_submit_render_typed) ++#define DRM_IOCTL_G5P_CREATE_RENDER_TARGET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_RENDER_TARGET, \ ++ struct drm_g5p_create_render_target) ++#define DRM_IOCTL_G5P_DESTROY_RENDER_TARGET \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_RENDER_TARGET, \ ++ struct drm_g5p_destroy_render_target) ++#define DRM_IOCTL_G5P_GET_RENDER_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_RENDER_INFO, \ ++ struct drm_g5p_get_render_info) ++ ++/** ++ * DRM_G5P_CAP_TYPED_COMPUTE - compute jobs on render contexts (SUBMIT_COMPUTE_TYPED) ++ * ++ * Reported together with TYPED_RENDER_V3. A ++ * render context (DRM_G5P_CONTEXT_RENDER) runs compute jobs beside its TYPED_V3 render jobs, ++ * in the same VM, in submission order, serialized with every other context's jobs. The ++ * kernel builds the firmware's compute work (W, microsequence, stamps, channel, slot); the ++ * job carries the CDM control stream and the register images iOS's userspace hands its ++ * kernel, which the kernel bounds but does not decode. Compute needs an epoch a render job ++ * brought up: on a device no render job has started, SUBMIT_COMPUTE_TYPED fails with EAGAIN. ++ */ ++#define DRM_G5P_CAP_TYPED_COMPUTE (1ULL << 30) ++ ++#define DRM_G5P_COMPUTE_MAX_STREAM_BYTES 0x10000 ++#define DRM_G5P_COMPUTE_STREAM_TERMINATE 0x80000000U ++ ++/* A PDS program reference: code and data offsets from GET_RENDER_INFO.pds_base (16-byte ++ * aligned; code in a READ resource whose BO carries DRM_G5P_BO_EXEC, data in a READ resource) ++ * and its opaque control word (Rogue's PDS1 position: sizes and temps). ++ */ ++struct drm_g5p_compute_pds { ++ __u32 code; ++ __u32 data; ++ __u32 control; ++ __u32 pad; /* zero */ ++}; ++ ++/* ++ * One compute job. Every *_va is a GPU VA inside a listed resource with the stated access. ++ * cdm_va/cdm_size the CDM control stream (W+0x70, R0+0x480): READ, 4-byte aligned, ++ * cdm_size <= MAX_STREAM_BYTES, in a native (not PRIME-imported) BO ++ * cdm_end bytes from cdm_va past the stream's terminate word (the dword at ++ * cdm_va + cdm_end - 4 is STREAM_TERMINATE), <= cdm_size ++ * state_va[4] CDM context-state words (W+0x1ec..0x204), READ|WRITE, 4-byte aligned ++ * state_tail_va CDM context-state tail (W+0x60, R0+0x498; W+0x20c), READ|WRITE, 16 bytes, ++ * 16-byte aligned ++ * config[2] CDM configuration words (W+0x10/0x14, R0+0x4c0/0x4c8), opaque ++ * store context-store program (W+0x21c/0x224, R0+0x4a8/0x4b0) ++ * resume context-resume program (W+0x22c/0x234) ++ * terminate terminate program (W+0x68, R0+0x4b8); control must be zero ++ */ ++struct drm_g5p_compute_pass { ++ __u32 size; /* sizeof(struct drm_g5p_compute_pass) */ ++ __u32 flags; /* zero */ ++ __u64 cdm_va; ++ __u32 cdm_size; ++ __u32 cdm_end; ++ __u64 state_va[4]; ++ __u64 state_tail_va; ++ __u32 config[2]; ++ struct drm_g5p_compute_pds store; ++ struct drm_g5p_compute_pds resume; ++ struct drm_g5p_compute_pds terminate; ++ __u64 reserved[4]; /* zero */ ++}; ++ ++/* Command 0x15. resources lists every BO the job references (1..RENDER_MAX_RESOURCES, ++ * TYPED_V1 access-mask kinds, IMPLICIT and CACHE included); pass_size is exactly ++ * sizeof(struct drm_g5p_compute_pass). Syncs, job ids, results and the error contract are ++ * SUBMIT_RENDER_TYPED's: every error other than EINTR/EFAULT on copy-out precedes fence ++ * replacement and hardware publication. ++ */ ++struct drm_g5p_submit_compute_typed { ++ __u32 context; /* a DRM_G5P_CONTEXT_RENDER context */ ++ __u32 flags; /* zero */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[]: exactly one SIGNAL, at most 16 inputs */ ++ __u64 pass; /* struct drm_g5p_compute_pass */ ++ __u32 pass_size; ++ __u32 pad; /* zero */ ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; /* zero */ ++}; ++ ++#define DRM_G5P_SUBMIT_COMPUTE_TYPED 0x15 ++ ++#define DRM_IOCTL_G5P_SUBMIT_COMPUTE_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_COMPUTE_TYPED, \ ++ struct drm_g5p_submit_compute_typed) ++ ++#endif diff --git a/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch b/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch new file mode 100644 index 00000000..5e33a414 --- /dev/null +++ b/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch @@ -0,0 +1,14822 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add firmware transport and recovery core + +[Orchard: from brkosebay/linux d67f133590a; fixed default policy and +removed idle experiments.] Load the original user-supplied META firmware +and retain bounded recovery. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/bits-compat.h | 20 + + drivers/gpu/apple-g5p/boot-layout.h | 306 ++++ + drivers/gpu/apple-g5p/boot-r73b.inc | 1179 +++++++++++++ + drivers/gpu/apple-g5p/boot-r74f.inc | 532 ++++++ + drivers/gpu/apple-g5p/boot-r74g.inc | 288 +++ + drivers/gpu/apple-g5p/boot.inc | 1665 ++++++++++++++++++ + drivers/gpu/apple-g5p/compute-user-layout.h | 207 +++ + drivers/gpu/apple-g5p/core.inc | 567 ++++++ + drivers/gpu/apple-g5p/diagnostics.inc | 181 ++ + drivers/gpu/apple-g5p/dirtystart108.h | 454 +++++ + drivers/gpu/apple-g5p/dirtystart108.inc | 280 +++ + drivers/gpu/apple-g5p/drm-validate.h | 40 + + drivers/gpu/apple-g5p/fender.h | 87 + + drivers/gpu/apple-g5p/gpu-oc.h | 149 ++ + drivers/gpu/apple-g5p/heartbeat116.h | 175 ++ + drivers/gpu/apple-g5p/idle-power.inc | 38 + + drivers/gpu/apple-g5p/kick.inc | 27 + + drivers/gpu/apple-g5p/latency123.h | 63 + + drivers/gpu/apple-g5p/ldr.h | 154 ++ + drivers/gpu/apple-g5p/lifecycle.inc | 1032 +++++++++++ + drivers/gpu/apple-g5p/meta71c114.h | 65 + + drivers/gpu/apple-g5p/mmio-licence.h | 148 ++ + drivers/gpu/apple-g5p/mmio.inc | 295 ++++ + drivers/gpu/apple-g5p/mmu-attrs.inc | 54 + + drivers/gpu/apple-g5p/mmu-tables.inc | 57 + + drivers/gpu/apple-g5p/pack.h | 89 + + drivers/gpu/apple-g5p/perf-envelope.h | 107 ++ + drivers/gpu/apple-g5p/power-r33.inc | 70 + + drivers/gpu/apple-g5p/power-r41.inc | 135 ++ + drivers/gpu/apple-g5p/r74e112.h | 137 ++ + drivers/gpu/apple-g5p/r87b-attach.h | 34 + + drivers/gpu/apple-g5p/readback.h | 40 + + drivers/gpu/apple-g5p/recover.inc | 101 ++ + drivers/gpu/apple-g5p/render-layout-r22.h | 54 + + drivers/gpu/apple-g5p/render-layout.h | 59 + + drivers/gpu/apple-g5p/render-user-bind.h | 285 +++ + drivers/gpu/apple-g5p/render-user-close.h | 91 + + drivers/gpu/apple-g5p/render-user-evwait.h | 52 + + drivers/gpu/apple-g5p/render-user-fault.h | 60 + + drivers/gpu/apple-g5p/render-user-faultreg.h | 124 ++ + drivers/gpu/apple-g5p/render-user-geometry.h | 144 ++ + drivers/gpu/apple-g5p/render-user-layout.h | 651 +++++++ + drivers/gpu/apple-g5p/render-user-loss.h | 120 ++ + drivers/gpu/apple-g5p/render-user-mmu.h | 60 + + drivers/gpu/apple-g5p/render-user-msaa.h | 88 + + drivers/gpu/apple-g5p/render-user-pmset.h | 77 + + drivers/gpu/apple-g5p/render-user-quiesce.h | 145 ++ + drivers/gpu/apple-g5p/render-user-recover.h | 107 ++ + drivers/gpu/apple-g5p/render-user-retire.h | 248 +++ + drivers/gpu/apple-g5p/render-user-sched.h | 155 ++ + drivers/gpu/apple-g5p/render-user-serial.h | 122 ++ + drivers/gpu/apple-g5p/render-user-sync.h | 50 + + drivers/gpu/apple-g5p/render-user-template.h | 24 + + drivers/gpu/apple-g5p/render-user-terminal.h | 103 ++ + drivers/gpu/apple-g5p/render-user-touch.h | 102 ++ + drivers/gpu/apple-g5p/render-user-wait.h | 266 +++ + drivers/gpu/apple-g5p/render-user-wake.h | 273 +++ + drivers/gpu/apple-g5p/render-user-window.h | 74 + + drivers/gpu/apple-g5p/render-variants-r25.h | 48 + + drivers/gpu/apple-g5p/retire108.h | 434 +++++ + drivers/gpu/apple-g5p/teardown.h | 740 ++++++++ + drivers/gpu/apple-g5p/teardown.inc | 694 ++++++++ + drivers/gpu/apple-g5p/transport-budget.inc | 6 + + drivers/gpu/apple-g5p/vm-mutation.h | 36 + + drivers/gpu/apple-g5p/work-layout.h | 21 + + 65 files changed, 14289 insertions(+) + create mode 100644 drivers/gpu/apple-g5p/bits-compat.h + create mode 100644 drivers/gpu/apple-g5p/boot-layout.h + create mode 100644 drivers/gpu/apple-g5p/boot-r73b.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74f.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74g.inc + create mode 100644 drivers/gpu/apple-g5p/boot.inc + create mode 100644 drivers/gpu/apple-g5p/compute-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/core.inc + create mode 100644 drivers/gpu/apple-g5p/diagnostics.inc + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.h + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.inc + create mode 100644 drivers/gpu/apple-g5p/drm-validate.h + create mode 100644 drivers/gpu/apple-g5p/fender.h + create mode 100644 drivers/gpu/apple-g5p/gpu-oc.h + create mode 100644 drivers/gpu/apple-g5p/heartbeat116.h + create mode 100644 drivers/gpu/apple-g5p/idle-power.inc + create mode 100644 drivers/gpu/apple-g5p/kick.inc + create mode 100644 drivers/gpu/apple-g5p/latency123.h + create mode 100644 drivers/gpu/apple-g5p/ldr.h + create mode 100644 drivers/gpu/apple-g5p/lifecycle.inc + create mode 100644 drivers/gpu/apple-g5p/meta71c114.h + create mode 100644 drivers/gpu/apple-g5p/mmio-licence.h + create mode 100644 drivers/gpu/apple-g5p/mmio.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-attrs.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-tables.inc + create mode 100644 drivers/gpu/apple-g5p/pack.h + create mode 100644 drivers/gpu/apple-g5p/perf-envelope.h + create mode 100644 drivers/gpu/apple-g5p/power-r33.inc + create mode 100644 drivers/gpu/apple-g5p/power-r41.inc + create mode 100644 drivers/gpu/apple-g5p/r74e112.h + create mode 100644 drivers/gpu/apple-g5p/r87b-attach.h + create mode 100644 drivers/gpu/apple-g5p/readback.h + create mode 100644 drivers/gpu/apple-g5p/recover.inc + create mode 100644 drivers/gpu/apple-g5p/render-layout-r22.h + create mode 100644 drivers/gpu/apple-g5p/render-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-bind.h + create mode 100644 drivers/gpu/apple-g5p/render-user-close.h + create mode 100644 drivers/gpu/apple-g5p/render-user-evwait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-fault.h + create mode 100644 drivers/gpu/apple-g5p/render-user-faultreg.h + create mode 100644 drivers/gpu/apple-g5p/render-user-geometry.h + create mode 100644 drivers/gpu/apple-g5p/render-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-loss.h + create mode 100644 drivers/gpu/apple-g5p/render-user-mmu.h + create mode 100644 drivers/gpu/apple-g5p/render-user-msaa.h + create mode 100644 drivers/gpu/apple-g5p/render-user-pmset.h + create mode 100644 drivers/gpu/apple-g5p/render-user-quiesce.h + create mode 100644 drivers/gpu/apple-g5p/render-user-recover.h + create mode 100644 drivers/gpu/apple-g5p/render-user-retire.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sched.h + create mode 100644 drivers/gpu/apple-g5p/render-user-serial.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sync.h + create mode 100644 drivers/gpu/apple-g5p/render-user-template.h + create mode 100644 drivers/gpu/apple-g5p/render-user-terminal.h + create mode 100644 drivers/gpu/apple-g5p/render-user-touch.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wake.h + create mode 100644 drivers/gpu/apple-g5p/render-user-window.h + create mode 100644 drivers/gpu/apple-g5p/render-variants-r25.h + create mode 100644 drivers/gpu/apple-g5p/retire108.h + create mode 100644 drivers/gpu/apple-g5p/teardown.h + create mode 100644 drivers/gpu/apple-g5p/teardown.inc + create mode 100644 drivers/gpu/apple-g5p/transport-budget.inc + create mode 100644 drivers/gpu/apple-g5p/vm-mutation.h + create mode 100644 drivers/gpu/apple-g5p/work-layout.h + +diff --git a/drivers/gpu/apple-g5p/bits-compat.h b/drivers/gpu/apple-g5p/bits-compat.h +new file mode 100644 +index 000000000..608e9b8ec +--- /dev/null ++++ b/drivers/gpu/apple-g5p/bits-compat.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_BITS_COMPAT_H ++#define G5P_BITS_COMPAT_H ++ ++#ifdef __KERNEL__ ++#include ++#include ++#else ++#ifndef BIT ++#define BIT(nr) (1UL << (nr)) ++#endif ++#ifndef BIT_ULL ++#define BIT_ULL(nr) (1ULL << (nr)) ++#endif ++#ifndef ARRAY_SIZE ++#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) ++#endif ++#endif ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-layout.h b/drivers/gpu/apple-g5p/boot-layout.h +new file mode 100644 +index 000000000..752e6873f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-layout.h +@@ -0,0 +1,306 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_BOOT_LAYOUT_H ++#define APPLE_G5P_BOOT_LAYOUT_H ++ ++#include "bits-compat.h" ++#include "pack.h" ++ ++#define G5P_GPU_PAGE 0x4000U ++#define G5P_ROOT_SIZE 0x8000U ++#define G5P_SHARED_SIZE 0x60000U ++#define G5P_IMAGE_VA 0xe1c0000000ULL ++#define G5P_SHARED_VA 0xe1c0200000ULL ++#define G5P_ROOT_VA 0xe1c3200000ULL ++#define G5P_PIO_VA 0xe1c4200000ULL ++#define G5P_PIO_SIZE 0x10000U ++ ++/* c95da0 descriptors, sizes at 5940b90; c97b6c packs on 64-byte boundaries. */ ++struct g5p_boot_layout { ++ u32 offset[0x90]; ++ u8 group[0x90]; ++ u32 used[2]; ++}; ++ ++static inline void g5p_put32(u8 *p, u32 value) ++{ ++ p[0] = value; ++ p[1] = value >> 8; ++ p[2] = value >> 16; ++ p[3] = value >> 24; ++} ++ ++static inline void g5p_put64(u8 *p, u64 value) ++{ ++ g5p_put32(p, value); ++ g5p_put32(p + 4, value >> 32); ++} ++ ++static inline u32 g5p_meta_address(u64 va) ++{ ++ /* ca0744: root range 8 bypasses META caches; other shared RAM is global. */ ++ u32 prefix = va >= G5P_ROOT_VA && va < G5P_ROOT_VA + 0x1000000 ? ++ 0x40000000 : ++ 0xc0000000; ++ ++ return ((u32)va + 0x40000000) | prefix; ++} ++ ++static inline int g5p_plan_layout(struct g5p_boot_layout *p) ++{ ++ static const struct { ++ u8 cpu, gpu, group; ++ u32 size; ++ } objects[] = { ++ { 0x44, 0x50, 0, 0x1490 }, ++ { 0x47, 0x51, 0, 0x990 }, ++ { 0x48, 0x52, 0, 0x8e0 }, ++ { 0x49, 0x53, 0, 0x8c0 }, ++ { 0x4a, 0x54, 0, 0x938 }, ++ { 0x4b, 0x55, 0, 0x28 }, ++ { 0x4c, 0x56, 0, 8 }, ++ { 0x4d, 0x57, 0, 0x78c }, ++ { 0x4e, 0x58, 0, 0x84 }, ++ { 0x5e, 0x5f, 1, 0x2058 }, ++ { 0x70, 0x78, 1, 0x20 }, ++ { 0x71, 0x79, 1, 0x80 }, ++ { 0x72, 0x7a, 1, 0x20 }, ++ { 0x73, 0x7b, 1, 0x20 }, ++ { 0x74, 0x7c, 1, 0x2c00 }, ++ { 0x75, 0x7d, 1, 0xb000 }, ++ { 0x4f, 0x59, 1, 0x36000 }, ++ { 0x76, 0x7e, 1, 0x2c00 }, ++ { 0x77, 0x7f, 1, 0x2400 }, ++ { 0x6c, 0x6e, 1, 0x20 }, ++ { 0x6d, 0x6f, 1, 0x1400 }, ++ { 0x60, 0x62, 1, 0x20 }, ++ { 0x64, 0x66, 1, 0x20 }, ++ { 0x68, 0x6a, 1, 0x20 }, ++ { 0x61, 0x63, 1, 0x1400 }, ++ { 0x65, 0x67, 1, 0x1400 }, ++ { 0x69, 0x6b, 1, 0x1400 }, ++ /* Separate c95da0 allocations, placed after the recovered group. */ ++ { 0x5c, 0x5d, 1, 8 }, ++ { 0x85, 0x88, 1, 0x400 }, ++ { 0x84, 0x87, 1, 0x400 }, ++ { 0x86, 0x89, 1, 0x800 }, ++ }; ++ unsigned int i; ++ ++ memset(p, 0, sizeof(*p)); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) { ++ u32 group = objects[i].group; ++ u32 off = (p->used[group] + 63) & ~63U; ++ ++ p->offset[objects[i].cpu] = off; ++ p->offset[objects[i].gpu] = off; ++ p->group[objects[i].cpu] = group; ++ p->group[objects[i].gpu] = group; ++ p->used[group] = off + objects[i].size; ++ } ++ return p->used[0] <= G5P_ROOT_SIZE && p->used[1] <= G5P_SHARED_SIZE ? ++ 0 : ++ -EINVAL; ++} ++ ++static inline u64 g5p_object_va(const struct g5p_boot_layout *p, u32 slot) ++{ ++ return (p->group[slot] ? G5P_SHARED_VA : G5P_ROOT_VA) + p->offset[slot]; ++} ++ ++static inline u8 *g5p_object(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 slot) ++{ ++ return (p->group[slot] ? shared : root) + p->offset[slot]; ++} ++ ++/* ca9b34 + cab44c + cac314: 40-bit PA, 16 KiB GPU pages. */ ++static inline u32 g5p_catalogue_entry(u64 pa) ++{ ++ return (u32)(pa >> 8) | 1; ++} ++ ++static inline u64 g5p_directory_entry(u64 pa) ++{ ++ /* Geometry bytes 3c..3f are 08 05 05 15: size, entry/PA/VA shifts. */ ++ return pa | 3; ++} ++ ++static inline u64 g5p_leaf_entry(u64 pa, u32 attributes) ++{ ++ /* caa6e4 range attributes -> ca9a60: META protect, CC / SLC bypass. */ ++ return (pa & 0xfffffff000ULL) | (attributes & 0xc) | ++ ((u64)(attributes & 0x10) << 58) | 1; ++} ++ ++static inline void g5p_make_boot_config(struct g5p_ldr_image *im, ++ const struct g5p_boot_layout *p) ++{ ++ static const u32 bases[] = { 0x10000000, 0x20000000, 0x40000000, ++ 0x60001000 }; ++ static const u32 sizes[] = { 0x18000, 0x4000, 0x4000, 0x4000 }; ++ static const u32 offsets[] = { 0x4000, 0x1c000, 0, 0x20000 }; ++ u8 *out = im->data + 0x200; ++ unsigned int seg, i; ++ ++ for (seg = 0; seg < 4; seg++) { ++ u64 va = G5P_IMAGE_VA + offsets[seg]; ++ u32 reg = 0x04850000 + seg * 16; ++ u32 values[] = { bases[seg] | 0xf02, sizes[seg] - 1, (u32)va, ++ (u32)(va >> 32) | 0x300000 }; ++ ++ if (seg == 2) { ++ g5p_put32(out, reg + 12); ++ g5p_put32(out + 4, (u32)(va >> 32) | 0x100); ++ out += 8; ++ continue; ++ } ++ for (i = 0; i < 4; i++, out += 8) { ++ g5p_put32(out, reg + i * 4); ++ g5p_put32(out + 4, values[i]); ++ } ++ } ++ for (i = 0; i < im->config_count; i++, out += 8) { ++ g5p_put32(out, im->config[i][0]); ++ g5p_put32(out + 4, im->config[i][1]); ++ } ++ g5p_put64(out, 0); ++ g5p_put32(out + 8, g5p_meta_address(G5P_IMAGE_VA + 0x3ff8)); ++ g5p_put32(im->data + 0x3ff8, g5p_meta_address(g5p_object_va(p, 0x50))); ++ g5p_put32(im->data + 0x3ffc, g5p_meta_address(g5p_object_va(p, 0x5f))); ++} ++ ++/* c96be8/c9edcc/ca3ccc. Unknown platform policy fields remain zero. */ ++static inline void g5p_make_init(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, const struct g5p_ldr_image *im, ++ u64 catalogue, u64 timestamp) ++{ ++ static const u8 roots[] = { ++ 0x62, 0x63, 0x66, 0x67, 0x6a, 0x6b, 0x6e, 0x6f, ++ 0x78, 0x7c, 0x79, 0x7d, 0x7a, 0x7e, 0x7b, 0x7f, ++ }; ++ static const u8 objects[] = { 0x51, 0x52, 0x53, 0x54, 0x55 }; ++ static const u8 tail[] = { 0x58, 0x56, 0x57, 0x88, 0x59 }; ++ static const u64 pio_pa[] = { 0x201e00000ULL, 0x20e100000ULL, ++ 0x20e104000ULL, 0x201000000ULL }; ++ static const u32 pio_size[] = { 0x5000, 0x4000, 0x4000, 0x20000 }; ++ static const u32 pio_offset[] = { 0, 0x8000, 0xc000, 0 }; ++ u8 *init = g5p_object(p, root, shared, 0x44); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *ctrl = g5p_object(p, root, shared, 0x5e); ++ unsigned int i; ++ ++ memset(root, 0, G5P_ROOT_SIZE); ++ memset(shared, 0, G5P_SHARED_SIZE); ++ for (i = 0; i < ARRAY_SIZE(roots); i++) ++ g5p_put32(init + i * 4, ++ g5p_meta_address(g5p_object_va(p, roots[i]))); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) ++ g5p_put32(init + 0x64 + i * 4, ++ g5p_meta_address(g5p_object_va(p, objects[i]))); ++ for (i = 0; i < ARRAY_SIZE(tail); i++) ++ g5p_put32(init + 0x44c + i * 4, ++ g5p_meta_address(g5p_object_va(p, tail[i]))); ++ for (i = 0; i < 3; i++) { ++ u8 *channel = g5p_object(p, root, shared, 0x47 + i); ++ u32 ptr = g5p_meta_address(g5p_object_va(p, 0x51 + i)); ++ ++ g5p_put32(channel + 4, ptr); ++ g5p_put32(channel + 12, ptr + 8); ++ } ++ g5p_put32(g5p_object(p, root, shared, 0x48) + 0x778, 0xffffffff); ++ g5p_put32(init + 0x480, ++ ((u32)(G5P_IMAGE_VA + 0x24000) + 0x40000000) | 0x40000000); ++ g5p_put64(init + 0x484, G5P_IMAGE_VA + 0x24000); ++ g5p_put32(init + 0x48c, 0x55dc); ++ g5p_put64(meta, 0x600000000ULL); ++ g5p_put64(meta + 8, 0xffc00000); ++ g5p_put64(meta + 0x10, 0x300000000ULL); ++ g5p_put64(meta + 0x18, 0x300000000ULL); ++ for (i = 0; i < ARRAY_SIZE(pio_pa); i++) { ++ u8 *entry = meta + 0x630 + i * 16; ++ ++ g5p_put64(entry, pio_pa[i]); ++ g5p_put32(entry + 8, i == 3 ? 0 : ++ g5p_meta_address(G5P_PIO_VA + ++ pio_offset[i])); ++ g5p_put32(entry + 12, pio_size[i]); ++ } ++ g5p_put32(meta + 0x6c0, 1); ++ g5p_put32(meta + 0x6c8, 1); ++ g5p_put64(meta + 0x6d8, 0x100000000ULL); ++ g5p_put32(meta + 0x6e0, 1); ++ g5p_put32(meta + 0x6e4, 1); ++ /* readChipInfo version 15 -> tables 5942650/5942674 -> 1,5; N=6. */ ++ g5p_put32(meta + 0x704, 1); ++ g5p_put32(meta + 0x708, 5); ++ g5p_put32(meta + 0x70c, 6); ++ g5p_put32(meta + 0x758, 0x14); ++ g5p_put32(meta + 0x75c, 0x4000); ++ g5p_put32(meta + 0x760, 0x1020); ++ g5p_put32(meta + 0x764, im->t1_pc); ++ g5p_put32(meta + 0x768, im->t1_sp); ++ g5p_put64(meta + 0x774, G5P_IMAGE_VA); ++ g5p_put64(meta + 0x77c, 0x300000); ++ g5p_put64(meta + 0x784, timestamp); ++ /* Fixed portion of c974f0; dynamic power-policy fields remain zero. */ ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 8, 0x400000000ULL); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x10, 0x3f800000); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x28, 0x100000001ULL); ++ g5p_put32(g5p_object(p, root, shared, 0x4a) + 0x38, 100); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x8f8, timestamp); ++ g5p_put32(ctrl + 0x24, 0x78); ++ g5p_put32(ctrl + 0x30, 1); ++ g5p_put32(ctrl + 0x1044, g5p_meta_address(g5p_object_va(p, 0x87))); ++ g5p_put32(ctrl + 0x1048, g5p_meta_address(g5p_object_va(p, 0x89))); ++ g5p_put32(ctrl + 0x104c, 1); ++ g5p_put64(g5p_object(p, root, shared, 0x86), catalogue); ++} ++ ++/* Explicit boot-time hypotheses; never modify a live root table. */ ++static inline void g5p_dc_init_variant(const struct g5p_boot_layout *p, ++ u8 *root, u8 *shared, u32 variant) ++{ ++ /* ca0744 uses 40 aliases for cache bypass. Applying that alias to DC ++ * is an unverified hypothesis, not recovered normal host behavior. ++ */ ++ if (variant == 1) ++ g5p_put32(root + 0x18, g5p_le32(root + 0x18) & ~0x80000000U); ++ if (variant == 1 || variant == 6) ++ g5p_put32(root + 0x1c, g5p_le32(root + 0x1c) & ~0x80000000U); ++ /* c96be8 chooses this policy from host+1af8 bits; both values exist. ++ * Whether zero is appropriate for our incomplete policy is unknown. ++ */ ++ if (variant == 5) ++ g5p_put32(shared + p->offset[0x5e] + 0x30, 0); ++ /* MINIM 10001ac8 enters policy1000c0d8 only when meta6c0/6c8 ++ * are both nonzero. Experimental pre-MTS bypass; no live length writes. ++ */ ++ if (variant == 7) ++ g5p_put32(g5p_object(p, root, shared, 0x4d) + 0x6c8, 0); ++ ++ /* Round13: recovered iOS inputs; variants 8/9 retain meta6c8=1. ++ * Neither is a complete DVFS policy (table+3c remains absent). ++ */ ++ if (variant == 8 || variant == 9) { ++ u8 *policy = g5p_object(p, root, shared, 0x4a); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *control = g5p_object(p, root, shared, 0x5e); ++ static const u32 mhz[] = { 0, 340, 474, 550, 723 }; ++ ++ g5p_put32(policy + 4, 16); ++ if (variant == 9) { ++ g5p_put32(meta + 0x710, 4); ++ g5p_put32(meta + 0x714, 5); ++ for (u32 i = 0; i < 16; i++) ++ g5p_put32(meta + 0x718 + 4 * i, ++ i < ARRAY_SIZE(mhz) ? mhz[i] : 0); ++ g5p_put32(control + 0x1028, 3348); ++ g5p_put32(control + 0x102c, 400); ++ g5p_put32(control + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++ } ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +new file mode 100644 +index 000000000..df2560f5e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -0,0 +1,1179 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R73b crash-first IRQ174 diagnostics. Included by boot.inc after struct ++ * g5p_boot. Case zero allocates nothing and keeps the r73 IRQ order. ++ * Rows live in CPU RAM only; they are formatted after the epoch, never in the ++ * IRQ service loop. The first 128 rows are retained, the last 128 roll. ++ */ ++#include ++#include ++#include ++#include ++#include "r87b-attach.h" ++#include "r74e112.h" ++#include "heartbeat116.h" ++ ++#define G5P_R73B_ROWS 256 ++#define G5P_R73B_HEAD 128 ++#define G5P_R73B_IRQ_CAP 8192 ++#define G5P_R73B_PANIC_ROWS 32 ++#define G5P_R73B_TRACE_ROWS 128 ++#define G5P_R73B_TRACE_WORDS 9 ++#define G5P_R74_IRQ_CAP 262144 ++#define G5P_R74_IRQ_CAP_MAX 1048576 ++#define G5P_R74_EVENT_FLOOR (256 + 4 * 65536) ++#define G5P_R74_STAGES 8 ++#define G5P_R74_MILESTONES 64 ++#define G5P_R74_MAX_TICKS 16 ++#define G5P_R74_RGX 9 ++#define G5P_R74_WORD1 8 ++#define G5P_R74F_KINDS 3 ++#define G5P_R74F_OFF_US 1000 ++#define G5P_R74F_ON_US 1800 ++#define G5P_R74F_BELL_MS 250 ++#define G5P_R74F_MMU_SLOTS 2 ++#define G5P_R74F_MMU_US 1000 ++#define G5P_R74G_GRACE_ALIVES 4 ++#define G5P_R74G_GRACE_US 20000 ++#define G5P_R74G_COMPANION 0x3980 ++#define G5P_R74G_RECOVERIES 0x3994 ++#define G5P_R74G_CLASS 0x3998 ++#define G5P_R74G_BAD_RING BIT(0) ++#define G5P_R74G_BAD_RECORD BIT(1) ++#define G5P_R74G_BAD_PA BIT(2) ++#define G5P_R74G_BAD_HOST BIT(3) ++#define G5P_R74G_BAD_CACHED BIT(4) ++#define G5P_R74G_BAD_FAILED BIT(5) ++#define G5P_R74G_BAD_COMPANION BIT(6) ++#define G5P_R74G_BAD_MUTATION BIT(7) ++#define G5P_R74H_TAIL 128 ++#define G5P_R74H_PHYS_MAX 1024 ++#define G5P_R74H_PTES 10 ++ ++/* First-ALIVE submit: wait (reads only) for a GPC request in flight to finish. */ ++static const unsigned int boot_alive_settle_us = 2000; ++MODULE_PARM_DESC( ++ boot_alive_settle_us, ++ "default0 (single read, as before); 1..5000: when the first-ALIVE submit reads " ++ "R1+0x1000 with bit 4 set (a GPC performance-state request in flight), re-read " ++ "it every 10 us for up to this long before the unchanged state==1 check, as " ++ "Apple's setGPCPerformanceState waits for bit 4; R1 reads only; latched at map"); ++ ++enum g5p_r73b_phase { ++ G5P_R73B_RELEASE_ENTRY = 1, ++ G5P_R73B_RELEASE_FIFO, ++ G5P_R73B_RELEASE_GPC, ++ G5P_R73B_RELEASE_EXIT, ++ G5P_R73B_IRQ, ++ G5P_R73B_DRAIN_BEFORE, ++ G5P_R73B_DRAIN_AFTER, ++ G5P_R73B_DRAIN_BUSY, ++ G5P_R73B_CAP, ++ G5P_R73B_EVENTS_EXIT, ++ G5P_R73B_DC_PUBLISHED, ++ G5P_R73B_DC_KICKED, ++ G5P_R73B_WAIT_EXIT, ++ G5P_R73B_EXPORT, ++ G5P_R73B_PHASES, ++}; ++ ++static const char *const g5p_r73b_phase_names[G5P_R73B_PHASES] = { ++ "unknown", "release-entry", "release-fifo", "release-gpc", ++ "release-exit", "irq", "drain-before", "drain-after", ++ "drain-busy", "cap", "events-exit", "dc-published", ++ "dc-kicked", "wait-exit", "export", ++}; ++ ++enum g5p_r73b_owner { ++ G5P_R73B_NONE, ++ G5P_R73B_OWNER_IRQ, ++ G5P_R73B_OWNER_EVENTS, ++ G5P_R73B_OWNER_PREPARE, ++ G5P_R73B_OWNER_PUBLISH, ++ G5P_R73B_OWNER_KICK, ++ G5P_R73B_OWNER_WAIT, ++}; ++ ++/* Row columns: event, log lane 0, statistics, trace, device control. */ ++static const u8 g5p_r73b_row_slots[5] = { 0x70, 0x71, 0x72, 0x73, 0x6c }; ++ ++struct g5p_r73b_row { ++ u32 seq; /* sequence + 1, published last; zero means empty/torn */ ++ u32 phase; ++ u64 t_ns; ++ u32 irq, ack, released, owner, cap_seen; ++ s32 result; ++ u32 cp[5][2]; ++}; ++ ++enum g5p_r74_step { ++ G5P_R74_IRQ_TEST_REFUSED, ++ G5P_R74_PREPARE_BEGIN, ++ G5P_R74_PREPARE_END, ++ G5P_R74_ATTACH_BEGIN, ++ G5P_R74_ATTACH_PUBLISHED, ++ G5P_R74_ATTACH_DOORBELL_BEGIN, ++ G5P_R74_ATTACH_DOORBELL_END, ++ G5P_R74_ATTACH_CONSUMED, ++ G5P_R74_ATTACH_END, ++ G5P_R74_TA_BEGIN, ++ G5P_R74_TA_PUBLISH, ++ G5P_R74_TA_DOORBELL_RETURNED, ++ G5P_R74_OBSERVE, ++ G5P_R74_WAIT_TICK, ++ G5P_R74_WAIT_EXIT, ++ G5P_R74_REPORT_END, ++ G5P_R74_CMD8_CONSUMED, ++ G5P_R74_TA_CONSUMED, ++ G5P_R74_TA_RAM_COMPLETE, ++ G5P_R74_TA_RAM_TIMEOUT, ++ G5P_R74F_ATTACH_ARMED, ++ G5P_R74F_ATTACH_REACTIVATED, ++ G5P_R74F_ATTACH_BELL, ++ G5P_R74F_MMU_CLOSED, ++ G5P_R74F_TA_ARMED, ++ G5P_R74F_TA_REACTIVATED, ++ G5P_R74F_TA_BELL, ++ G5P_R74F_STOP, ++ G5P_R74G_FW_MMU, ++ G5P_R74G_ACK_SKIPPED, ++ G5P_R74G_TA_GRACE, ++ G5P_R74G_TA_FALLBACK, ++ G5P_R74_STEPS, ++}; ++ ++static const char *const g5p_r74_steps[G5P_R74_STEPS] = { ++ "irq-test-refused", ++ "prepare-begin", ++ "prepare-end", ++ "attach-begin", ++ "attach-dc3-published", ++ "attach-doorbell-begin", ++ "attach-doorbell-end", ++ "attach-consumed", ++ "attach-end", ++ "ta-begin", ++ "ta-publish", ++ "ta-doorbell-returned", ++ "observe", ++ "wait-tick", ++ "wait-exit", ++ "report-end", ++ "cmd8-consumed", ++ "ta-consumed", ++ "ta-ram-complete", ++ "ta-ram-timeout", ++ "attach-bell-armed", ++ "attach-reactivated", ++ "attach-bell", ++ "first-mmu-window-closed", ++ "ta-bell-armed", ++ "ta-reactivated", ++ "ta-bell", ++ "r74f-stop", ++ "firmware-mmu-complete", ++ "r31-ack-skipped", ++ "ta-grace-end", ++ "ta-fallback", ++}; ++ ++/* R74 read-only witnesses; every address is already read elsewhere in-tree. */ ++ ++struct g5p_r74_milestone { ++ u64 t_ns; ++ u32 step, irqs, events; ++ s32 ret; ++ u32 dc[2], ta[2]; ++}; ++ ++enum g5p_r74f_kind { ++ G5P_R74F_DC3, ++ G5P_R74F_TA, ++ G5P_R74F_3D, /* r96: ring 0x64, doorbell 0x11, owned by the ALIVE decoder like the TA's */ ++}; ++ ++/* idle -> published (gate closed) -> open (reactivated) -> firing -> ++ * rang; published/open -> cancelled. FIRING lives under event_lock. ++ */ ++enum g5p_r74f_state { ++ G5P_R74F_IDLE, ++ G5P_R74F_PUBLISHED, ++ G5P_R74F_OPEN, ++ G5P_R74F_FIRING, ++ G5P_R74F_RANG, ++ G5P_R74F_CANCELLED, ++ G5P_R74F_STATES, ++}; ++ ++enum g5p_r74f_cancel { ++ G5P_R74F_CANCEL_NONE, ++ G5P_R74F_CANCEL_REACTIVATION, ++ G5P_R74F_CANCEL_NO_ALIVE, ++ G5P_R74F_CANCEL_CONSUMED, ++ G5P_R74F_CANCEL_PROTOCOL, ++ G5P_R74F_CANCEL_ALIVE_ADVANCED, ++ G5P_R74F_CANCEL_SELF_DRAIN, ++ G5P_R74F_CANCELS, ++}; ++ ++enum g5p_r74f_stop { ++ G5P_R74F_STOP_NONE, ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ G5P_R74F_STOP_MMU_CLOSED, ++ G5P_R74F_STOP_TA_REACTIVATION, ++ G5P_R74F_STOP_TA_NOT_KICKED, ++ G5P_R74G_STOP_DC3_TAGS, ++ G5P_R74G_STOP_DC3_RAM, ++ G5P_R74G_STOP_COMPANION, ++ G5P_R74G_STOP_TA_NOT_CONSUMED, ++ G5P_R74G_STOP_TA_WINDOW_MISSED, ++ G5P_R74F_STOPS, ++}; ++ ++enum g5p_r74f_path { ++ G5P_R74F_PATH_NONE, ++ G5P_R74F_PATH_IMMEDIATE, ++ G5P_R74F_PATH_DEFERRED, ++ G5P_R74F_PATHS, ++}; ++ ++/* One generation. Its owner writes the receipt, then releases state. */ ++struct g5p_r74f_bell { ++ atomic_t state; ++ u32 kind, generation, ring, start, target, path, cancel; ++ u32 arm_alive, open_alive, bell_alive; ++ u32 first_fence, first_state, fence, bell_state; ++ s32 first_ret, bell_ret, power_ret; ++ u32 reactivated, core_fence, gpc_fence, off_raw, on_raw; ++ u32 attempted, kicked, complete; ++ u32 consumer_before, consumer_after, producer_after; ++ u32 alive_done; ++ u64 arm_ns, power_start_ns, off_ns, on_ns, open_ns; ++ u64 bell_start_ns, bell_ns, done_ns; ++}; ++ ++enum g5p_r74g_grace { ++ G5P_R74G_GRACE_NONE, ++ G5P_R74G_GRACE_CONSUMED, ++ G5P_R74G_GRACE_ALIVES_END, ++ G5P_R74G_GRACE_WALL, ++ G5P_R74G_GRACE_FAULT, ++ G5P_R74G_GRACES, ++}; ++ ++/* The RAM-visible results of the firmware DC3 handler: phase 0 at the DC3 ++ * consume, phase 1 again just before the TA producer moves. ++ */ ++struct g5p_r74g_dc3 { ++ u64 pa[2], expected_pa, t_ns; ++ u32 slot[2], host[2], cached[2], rec[2][5]; ++ u32 start, target, consumer, producer, tag; ++ u32 failed, event_failed, companion, recoveries, classification, bad; ++}; ++ ++struct g5p_r73b { ++ struct device *dev; ++ struct g5p_boot *boot; ++ u32 case_id, watchdog_ms, dc_variant, meta6c8, r73e_case; ++ bool ack_first, aux, fast_enabled, fast_window; ++ bool early_release_ready, early_power_abort; ++ atomic_t early_once, early_busy; ++ u64 early_irq_ns, early_power_start_ns; ++ u64 early_off_ns, early_on_ns; ++ u32 early_state, early_after_state, early_power_raw; ++ s32 early_ret, early_power_ret; ++ /* wake112 (r74e112.h): the R74E proof deferred to the first-ALIVE owner. */ ++ atomic_t early_proof; ++ u32 early_proof_gen, early_proof_gen_now, early_proof_why; ++ s32 early_defer_ret; ++ /* boot_alive_settle_us: limit, first in-flight value seen, re-reads, time spent. */ ++ u32 settle_us, settle_first, settle_samples; ++ u64 settle_ns; ++ u64 fast_release_ns, fast_alive_ns, fast_submit_ns; ++ u32 fast_state; ++ s32 fast_ret; ++ /* nopatch: Apple's unmodified image; policy record +0/+4/+8/+c/+34 taken before the first bell. */ ++ bool nopatch; ++ u32 np_pre[5]; ++ atomic_t seq, acks, cap_seen, busy, drains, drain_errors, exported, ++ trace_saved; ++ atomic_t watchdog_state; /* 0 off, 1 armed, 2 disarmed, 3 fired, 4 terminal */ ++ int watchdog_cpu; ++ u32 owner; ++ struct mutex drain_lock; /* single auxiliary-ring owner */ ++ struct hrtimer watchdog; ++ struct g5p_r73b_row rows[G5P_R73B_ROWS]; ++ u32 trace_rows[G5P_R73B_TRACE_ROWS][G5P_R73B_TRACE_WORDS]; ++ u32 irq_cap, r74_case, r74_mask, r74_ticks, r74_word1_count; ++ u32 r74_word1[G5P_R74_WORD1]; ++ atomic_t r74_seq, r74_irq_refused; ++ struct g5p_r74_milestone r74_milestones[G5P_R74_MILESTONES]; ++ ++ /* r74f_alive and FIRING are written under event_lock only. */ ++ u32 r74f_case, r74f_alive, r74f_active, r74f_generation, r74f_stop; ++ ++ u32 r87b_attach; ++ s32 r74f_stop_value; ++ atomic_t r74f_bells; ++ struct g5p_r74f_bell r74f_bell[G5P_R74F_KINDS]; ++ ++ /* R74g: no host MMU pass, publish-only TA grace, one fallback bell. */ ++ u32 r74g_case, r74g_ack_skipped, r74g_grace_end, r74g_opened; ++ u32 r74g_grace_alive; ++ u64 r74g_grace_ns; ++ struct g5p_r74g_dc3 r74g_dc3[2]; ++ /* R74h: RAM-only trace tail and checkpoints; NULL when off. */ ++ u32 r74h_case; ++ ++ ++ ++ /* Heartbeat116 (heartbeat116.h): the R74 window detector; IRQ thread only. */ ++ bool hb_on; ++ u32 hb_seed, hb_empty, hb_cross; ++ atomic_t hb_late; ++ struct g5p_hb_cfg hb_cfg; ++ struct g5p_hb hb; ++}; ++ ++/* The ctrl+0 word installed before release. */ ++static u32 g5p_boot_ctrl0(struct g5p_boot *b) ++{ ++ return b->r78_ctrl0; ++} ++ ++ ++ ++/* render-r96.inc: the receipts at export (RAM only) */ ++ ++/* render-r94.inc: the RAM state of the record and the code BO at export, read only */ ++ ++/* boot-r80.inc: the two IRQ-owned plateau banks, read-only */ ++ ++/* render-r84b.inc: the late raw dump and the CPU visibility step, RAM only */ ++ ++static int g5p_boot_drain(struct g5p_diag *g); ++static u32 g5p_transport_event_limit(void); ++static int g5p_r73c_fast_control(struct g5p_diag *g); ++static int g5p_r73e_prepare(struct g5p_diag *g); ++static int g5p_r73e_irq_submit(struct g5p_diag *g); ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner); ++ ++static const char *g5p_r73b_name(u32 id) ++{ ++ return id == 1 ? "meta1-bounded-all-rings" : ++ id == 2 ? "meta0-bounded-all-rings-control" : ++ id == 3 ? "meta1-event-only-reference" : ++ "off"; ++} ++ ++static bool g5p_r73b_sampled(u32 n) ++{ ++ return n <= 16 || is_power_of_2(n); ++} ++ ++static u32 *g5p_r73b_ptr(struct g5p_boot *b, u32 off) ++{ ++ return (u32 *)(b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r73b_word(struct g5p_boot *b, u32 off) ++{ ++ return READ_ONCE(*g5p_r73b_ptr(b, off)); ++} ++ ++static u32 g5p_r73b_slot(u32 seq) ++{ ++ if (seq < G5P_R73B_HEAD) ++ return seq; ++ return G5P_R73B_HEAD + ++ (seq - G5P_R73B_HEAD) % (G5P_R73B_ROWS - G5P_R73B_HEAD); ++} ++ ++/* Lockless: a writer owns its slot by sequence; seq is published last. */ ++static void g5p_r73b_crumb(struct g5p_boot *b, u32 phase, u32 owner, int result) ++{ ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r73b_row *row; ++ u32 seq; ++ ++ if (!r) ++ return; ++ seq = atomic_inc_return(&r->seq) - 1; ++ row = &r->rows[g5p_r73b_slot(seq)]; ++ WRITE_ONCE(row->seq, 0); ++ /* Invalidate before rewriting so a reader never pairs old seq/new data. */ ++ smp_wmb(); ++ row->phase = phase; ++ row->t_ns = ktime_get_mono_fast_ns(); ++ row->irq = atomic_read(&b->irqs); ++ row->ack = atomic_read(&r->acks); ++ row->released = READ_ONCE(b->released); ++ row->owner = owner; ++ row->result = result; ++ row->cap_seen = atomic_read(&r->cap_seen); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r73b_row_slots); i++) { ++ u32 ctl = b->layout.offset[g5p_r73b_row_slots[i]]; ++ ++ row->cp[i][0] = g5p_r73b_word(b, ctl); ++ row->cp[i][1] = g5p_r73b_word(b, ctl + 0x10); ++ } ++ /* Publish the row contents before its sequence. */ ++ smp_wmb(); ++ WRITE_ONCE(row->seq, seq + 1); ++} ++ ++static const char *g5p_r74_name(u32 id) ++{ ++ return id == 1 ? "forced-ready-meta1-natural-ta" : ++ id == 2 ? "meta0-unpatched-natural-ta-control" : ++ "off"; ++} ++ ++static const char *g5p_r74_step_name(u32 step) ++{ ++ return step < G5P_R74_STEPS ? g5p_r74_steps[step] : "unknown"; ++} ++ ++static u32 g5p_r74_case(struct g5p_boot *b) ++{ ++ return b && b->r73b ? 1 : 0; ++} ++ ++/* R74b case 1: after cmd8 retirement, optional host MMIO must not race the ++ * firmware policy loop power-gating RGX. Functional setup/kick sites opt in ++ * explicitly; this predicate gates observations only. ++ */ ++static bool g5p_r74b_ram_only(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ ++ return g5p_r74_case(b) == 1 && b && READ_ONCE(b->released) && ++ READ_ONCE(b->r72_meta6c8); ++} ++ ++/* Process context only, never per IRQ: one console line per r74 step, so a ++ * whole-SoC freeze is localized by the last line netconsole delivered. ++ */ ++static void g5p_r74_mark(struct g5p_diag *g, u32 step, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_r74_milestone m; ++ u32 seq; ++ ++ if (!r || !1) ++ return; ++ m.t_ns = ktime_get_ns(); ++ m.step = step; ++ m.ret = ret; ++ m.irqs = atomic_read(&b->irqs); ++ m.events = READ_ONCE(b->events); ++ m.dc[0] = g5p_r73b_word(b, b->layout.offset[0x6c]); ++ m.dc[1] = g5p_r73b_word(b, b->layout.offset[0x6c] + 0x10); ++ m.ta[0] = g5p_r73b_word(b, b->layout.offset[0x60]); ++ m.ta[1] = g5p_r73b_word(b, b->layout.offset[0x60] + 0x10); ++ seq = atomic_inc_return(&r->r74_seq) - 1; ++ if (seq < G5P_R74_MILESTONES) ++ r->r74_milestones[seq] = m; ++ dev_emerg( ++ g->dev, ++ "R74 milestone seq=%u step=%s ret=%d irqs=%u events=%u dc=%u/%u ta=%u/%u " ++ "t_ns=%llu end=r74\n", ++ seq, g5p_r74_step_name(step), ret, m.irqs, m.events, m.dc[0], ++ m.dc[1], m.ta[0], m.ta[1], m.t_ns); ++} ++ ++/* R74 keeps repeated ALIVEs quiet whatever their word1: each new value is ++ * printed once (eight at most), so a moved loop cannot flood the console. ++ */ ++static bool g5p_r74_word1(struct g5p_boot *b, u32 word1) ++{ ++ struct g5p_r73b *r = b->r73b; ++ ++ ; ++ ; ++ for (u32 i = 0; i < r->r74_word1_count; i++) ++ if (r->r74_word1[i] == word1) ++ return false; ++ if (r->r74_word1_count == G5P_R74_WORD1) ++ return false; ++ r->r74_word1[r->r74_word1_count++] = word1; ++ return true; ++} ++ ++static bool g5p_r73b_copy_row(struct g5p_r73b *r, u32 seq, ++ struct g5p_r73b_row *out) ++{ ++ struct g5p_r73b_row *row = &r->rows[g5p_r73b_slot(seq)]; ++ ++ if (READ_ONCE(row->seq) != seq + 1) ++ return false; ++ /* Pairs with the writer's publication barrier. */ ++ smp_rmb(); ++ *out = *row; ++ /* Re-check that the slot was not recycled while copying. */ ++ smp_rmb(); ++ return READ_ONCE(row->seq) == seq + 1; ++} ++ ++static int g5p_r73b_format(char *buf, size_t len, ++ const struct g5p_r73b_row *row) ++{ ++ u32 phase = row->phase < G5P_R73B_PHASES ? row->phase : 0; ++ ++ return scnprintf( ++ buf, len, ++ "R73B crumb seq=%u t_ns=%llu phase=%s irq=%u ack=%u released=%u event=%u/%u " ++ "log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d cap=%u end=r73b\n", ++ row->seq - 1, row->t_ns, g5p_r73b_phase_names[phase], row->irq, ++ row->ack, row->released, row->cp[0][0], row->cp[0][1], ++ row->cp[1][0], row->cp[1][1], row->cp[2][0], row->cp[2][1], ++ row->cp[3][0], row->cp[3][1], row->cp[4][0], row->cp[4][1], ++ row->owner, row->result, row->cap_seen); ++} ++ ++/* Snapshot the producer once and consume at most that modular distance ++ * (<= 255). One consumer store returns the whole batch; a producer that ++ * advances meanwhile is left for the next pass, never chased. ++ */ ++static int g5p_r73b_drain_ring(struct g5p_boot *b, u32 ctl_slot, u32 lane, ++ u32 data_slot, u32 stride, u32 ring, ++ u32 *consumed) ++{ ++ u32 ctl = b->layout.offset[ctl_slot] + lane * 0x20; ++ u32 data = b->layout.offset[data_slot] + ++ (ctl_slot == 0x71 ? lane * 0x2c00 : 0); ++ u32 consumer = g5p_r73b_word(b, ctl); ++ u32 producer = g5p_r73b_word(b, ctl + 0x10); ++ u32 count, hash = b->r73_aux_hash[ring] ?: 2166136261U; ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ count = (producer - consumer) & 255; ++ if (!count) ++ return 0; ++ /* Producer index before the slot payloads it covers. */ ++ dma_rmb(); ++ for (u32 n = 0; n < count; n++) { ++ u32 words[G5P_R73B_TRACE_WORDS] = { 0 }; ++ u32 off = data + consumer * stride; ++ ++ for (u32 i = 0; i < stride / 4; i++) { ++ u32 value = g5p_r73b_word(b, off + i * 4); ++ ++ if (i < ARRAY_SIZE(words)) ++ words[i] = value; ++ hash = (hash ^ value) * 16777619U; ++ } ++ if (ring == 5 && ++ atomic_read(&b->r73b->trace_saved) < G5P_R73B_TRACE_ROWS) { ++ u32 saved = atomic_read(&b->r73b->trace_saved); ++ ++ memcpy(b->r73b->trace_rows[saved], words, ++ sizeof(words)); ++ /* Publish this complete row before increasing the visible count. */ ++ smp_wmb(); ++ atomic_inc(&b->r73b->trace_saved); ++ }; ++ ; ++ if (!b->r73_aux_entries[ring]) ++ memcpy(b->r73_aux_first[ring], words, ++ sizeof(b->r73_aux_first[ring])); ++ memcpy(b->r73_aux_last[ring], words, ++ sizeof(b->r73_aux_last[ring])); ++ b->r73_aux_entries[ring]++; ++ consumer = (consumer + 1) & 255; ++ } ++ b->r73_aux_hash[ring] = hash; ++ /* dma_wmb() ordering plus the slot loads above: Apple returns the ++ * consumer only after its reads, with a full barrier. ++ */ ++ dma_mb(); ++ WRITE_ONCE(*g5p_r73b_ptr(b, ctl), consumer); ++ *consumed += count; ++ return 0; ++} ++ ++/* Apple order after the ACK: four log lanes, events, statistics, trace. */ ++static int g5p_r73b_aux_pass(struct g5p_diag *g, u32 *consumed) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret = 0; ++ ++ for (u32 lane = 0; lane < 4 && !ret; lane++) ++ ret = g5p_r73b_drain_ring(b, 0x71, lane, 0x75, 44, lane, ++ consumed); ++ if (!ret) ++ ret = g5p_boot_drain(g); ++ /* R80: ring 4 has one consumer, the drain thread (retains every record). */ ++ if (!ret && !false) ++ ret = g5p_r73b_drain_ring(b, 0x72, 0, 0x76, 44, 4, consumed); ++ if (!ret) ++ ret = g5p_r73b_drain_ring(b, 0x73, 0, 0x77, 36, 5, consumed); ++ return ret; ++} ++ ++static bool g5p_r73b_pending(struct g5p_boot *b, bool aux) ++{ ++ static const u8 slots[] = { 0x70, 0x71, 0x72, 0x73 }; ++ ++ for (u32 i = 0; i < (aux ? ARRAY_SIZE(slots) : 1); i++) { ++ for (u32 lane = 0; lane < (slots[i] == 0x71 ? 4 : 1); lane++) { ++ u32 ctl = b->layout.offset[slots[i]] + lane * 0x20; ++ ++ if (g5p_r73b_word(b, ctl) != ++ g5p_r73b_word(b, ctl + 0x10)) ++ return true; ++ } ++ } ++ return false; ++} ++ ++/* The IRQ thread only trylocks the auxiliary rings; a miss still runs the ++ * spinlock-protected event decoder. Process-context paths lock normally. ++ * Drains with work, errors or sampled entries record before/after rows. ++ */ ++static int g5p_r73b_service(struct g5p_diag *g, u32 owner, bool sampled) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 consumed = 0, before; ++ bool rows, aux; ++ int ret; ++ ++ if (!r) ++ return g5p_boot_drain(g); ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* The atomic r73c window services only the event ring. Auxiliary ++ * traffic is retained until cmd8 is either published or refused. */ ++ aux = 1 && !READ_ONCE(r->fast_window); ++ if (aux && owner == G5P_R73B_OWNER_IRQ && ++ !mutex_trylock(&r->drain_lock)) { ++ atomic_inc(&r->busy); ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BUSY, READ_ONCE(r->owner), ++ -EBUSY); ++ ret = g5p_boot_drain(g); ++ goto out; ++ } ++ if (aux && owner != G5P_R73B_OWNER_IRQ) ++ mutex_lock(&r->drain_lock); ++ if (aux) ++ WRITE_ONCE(r->owner, owner); ++ atomic_inc(&r->drains); ++ rows = sampled || g5p_r73b_pending(b, aux); ++ if (rows) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BEFORE, owner, 0); ++ before = READ_ONCE(b->events); ++ ret = aux ? g5p_r73b_aux_pass(g, &consumed) : g5p_boot_drain(g); ++ consumed += READ_ONCE(b->events) - before; ++ if (rows || ret) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_AFTER, owner, ret ?: consumed); ++ if (aux) { ++ WRITE_ONCE(r->owner, G5P_R73B_NONE); ++ mutex_unlock(&r->drain_lock); ++ } ++out: ++ if (ret) { ++ atomic_inc(&r->drain_errors); ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return ret; ++} ++ ++/* Heartbeat116: an R74 epoch bounds IRQ174 and its ring entries per window. */ ++static bool g5p_hb116_on(struct g5p_boot *b) ++{ ++ return b->r73b && b->r73b->hb_on; ++} ++ ++/* Auxiliary-ring entries consumed so far (RAM, monotonic, may wrap). */ ++static u32 g5p_hb116_aux(struct g5p_boot *b) ++{ ++ u32 sum = 0; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(b->r73_aux_entries); i++) ++ sum += READ_ONCE(b->r73_aux_entries[i]); ++ return sum; ++} ++ ++static u32 g5p_hb116_progress(struct g5p_boot *b) ++{ ++ return READ_ONCE(b->events) + g5p_hb116_aux(b); ++} ++ ++/* The lifetime cap's terminal stop, once: final row, cap_seen, failed, one disable. */ ++static void g5p_hb116_trip(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb *h = &r->hb; ++ u32 prior = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (atomic_cmpxchg(&r->cap_seen, 0, 1)) ++ return; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "HB116 trip why=%s prior_failed=%u windows=%u stalled=%u ms=%llu irqs=%u " ++ "events=%u acks=%u empty=%u aux=%u total_irqs=%u total_events=%u t_ns=%llu " ++ "stop=masked late=ack-only end=hb116\n", ++ g5p_hb_why_name[h->why < G5P_HBW_N ? h->why : 0], prior, ++ h->windows, h->stalled, h->trip.ns / NSEC_PER_MSEC, ++ h->trip.irqs, h->trip.events, h->trip.acks, h->trip.empty, ++ h->trip.aux, n, READ_ONCE(b->events), h->trip_ns); ++} ++ ++/* Receipts only: the first window past each Wake112 lifetime bound; nothing trips here. */ ++static void g5p_hb116_cross(struct g5p_diag *g, u32 irqs, u32 events) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ u32 limit = g5p_transport_event_limit(); ++ ++ if (!(r->hb_cross & 1) && irqs > r->irq_cap) { ++ r->hb_cross |= 1; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=irq-cap value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ r->irq_cap, irqs, events, r->hb.windows); ++ } ++ if (!(r->hb_cross & 2) && events >= limit) { ++ r->hb_cross |= 2; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=event-limit value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ limit, irqs, events, r->hb.windows); ++ } ++} ++ ++/* After the entry's ACK and service: account it, close a window, or trip. */ ++static void g5p_hb116_irq(struct g5p_diag *g, int irq, u32 n, u32 before) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb_win *w = &r->hb.last; ++ u32 seed = r->hb_seed, events = READ_ONCE(b->events); ++ struct g5p_hb_in in; ++ ++ if (g5p_hb116_progress(b) == before) ++ r->hb_empty++; ++ in.now_ns = ktime_get_mono_fast_ns(); ++ in.irqs = n + seed; ++ in.events = events + seed; ++ in.acks = atomic_read(&r->acks) + seed; ++ in.empty = r->hb_empty + seed; ++ in.aux = g5p_hb116_aux(b) + seed; ++ switch (g5p_hb_step(&r->hb, &r->hb_cfg, &in)) { ++ case G5P_HBR_OPEN: ++ case G5P_HBR_CLOSED: ++ break; ++ case G5P_HBR_TRIP: ++ g5p_hb116_trip(g, irq, n); ++ return; ++ } ++ g5p_hb116_cross(g, n, events); ++ if (g5p_hb_report(&r->hb)) ++ g5p_job_printk( ++ g->dev, ++ "HB116 window n=%u t_ns=%llu ms=%llu irqs=%u events=%u acks=%u empty=%u aux=%u " ++ "stalled=%u peak_irqs=%u peak_events=%u total_irqs=%u total_events=%u " ++ "total_acks=%d end=hb116\n", ++ r->hb.windows, in.now_ns, w->ns / NSEC_PER_MSEC, ++ w->irqs, w->events, w->acks, w->empty, w->aux, ++ r->hb.stalled, r->hb.peak_irqs, r->hb.peak_events, n, ++ events, atomic_read(&r->acks)); ++} ++ ++/* Every entry ACKs. Over the cap: ACK, final row, cap_seen, one disable; ++ * the kernel stays alive so the runner can export the record. ++ * Heartbeat116: an R74 epoch has no lifetime cap. The window detector runs ++ * after the ACK and the service; after its trip an entry only ACKs. ++ */ ++static irqreturn_t g5p_r73b_irq(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ bool sampled = g5p_r73b_sampled(n); ++ u32 before = 0; ++ ++ if (r->hb_on && atomic_read(&r->cap_seen)) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ atomic_inc(&r->hb_late); ++ return IRQ_HANDLED; ++ } ++ if (!r->hb_on && n > r->irq_cap) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ if (!atomic_read(&r->cap_seen)) { ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (!atomic_cmpxchg(&r->cap_seen, 0, 1)) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ } ++ } ++ return IRQ_HANDLED; ++ } ++ { ++ /* KPF level6G: Apple ACKs the AIC meta interrupt before ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ } ++ if (sampled) ++ g5p_r73b_crumb(b, G5P_R73B_IRQ, G5P_R73B_OWNER_IRQ, n); ++ if (r->hb_on) ++ before = g5p_hb116_progress(b); ++ if (READ_ONCE(b->released)) ++ g5p_r73b_service(g, G5P_R73B_OWNER_IRQ, sampled); ++ if (1 && READ_ONCE(b->alive)) ++ g5p_r73e_alive_submit(g, G5P_R73B_OWNER_IRQ); ++ ; ++ ; ++ ; ++ ; ++ ; ++ if (r->hb_on) ++ g5p_hb116_irq(g, irq, n, before); ++ return IRQ_HANDLED; ++} ++ ++static void g5p_r73b_disarm(struct g5p_r73b *r) ++{ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 2) == 1) ++ hrtimer_cancel(&r->watchdog); ++} ++ ++static void g5p_r73b_stop(void *data) ++{ ++ g5p_r73b_disarm(data); ++} ++ ++/* Cached CPU RAM only: no MMIO, no locks, then a deliberate panic so a ++ * silent hang becomes a pstore/ramoops record. ++ */ ++static enum hrtimer_restart g5p_r73b_watchdog_fire(struct hrtimer *timer) ++{ ++ struct g5p_r73b *r = container_of(timer, struct g5p_r73b, watchdog); ++ struct g5p_boot *b = r->boot; ++ struct g5p_r73b_row row; ++ u32 total = atomic_read(&r->seq), first, last; ++ char line[320]; ++ bool found = false; ++ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 3) != 1) ++ return HRTIMER_NORESTART; ++ /* A contained cap or a latched failure is terminal: the IRQ only ACKs, ++ * service returns -EIO and the runner sees failed=1. Keep that host ++ * alive for the export instead of turning it into a deliberate panic. ++ */ ++ if (atomic_read(&r->cap_seen) || READ_ONCE(b->failed)) { ++ atomic_set(&r->watchdog_state, 4); ++ dev_emerg( ++ r->dev, ++ "R73B watchdog terminal case=%u irqs=%d acks=%d cap_seen=%d failed=%u rows=%u " ++ "no-panic end=r73b\n", ++ r->case_id, atomic_read(&b->irqs), ++ atomic_read(&r->acks), atomic_read(&r->cap_seen), ++ READ_ONCE(b->failed), total); ++ return HRTIMER_NORESTART; ++ } ++ dev_emerg( ++ r->dev, ++ "R73B receipt case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u release=%s end=r73b\n", ++ r->case_id, g5p_r73b_name(r->case_id), r->ack_first, r->aux, ++ r->irq_cap, r->watchdog_ms, r->dc_variant, r->meta6c8, ++ init_utsname()->release); ++ first = total > G5P_R73B_PANIC_ROWS ? total - G5P_R73B_PANIC_ROWS : 0; ++ for (u32 seq = first; seq < total; seq++) { ++ if (!g5p_r73b_copy_row(r, seq, &row)) ++ continue; ++ g5p_r73b_format(line, sizeof(line), &row); ++ dev_emerg(r->dev, "%s", line); ++ } ++ for (last = total; last > first && !found; last--) ++ found = g5p_r73b_copy_row(r, last - 1, &row); ++ if (!found) ++ memset(&row, 0, sizeof(row)); ++ panic("G5P R73B watchdog: release epoch incomplete after %ums case=%u irqs=%d acks=%d " ++ "cap_seen=%d busy=%d rows=%u last seq=%u phase=%s irq=%u ack=%u released=%u " ++ "event=%u/%u log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d " ++ "cap=%u end=r73b\n", ++ 0, 1, atomic_read(&b->irqs), atomic_read(&r->acks), ++ atomic_read(&r->cap_seen), atomic_read(&r->busy), total, ++ row.seq ? row.seq - 1 : 0, ++ g5p_r73b_phase_names[row.phase < G5P_R73B_PHASES ? row.phase : 0], ++ row.irq, row.ack, row.released, row.cp[0][0], row.cp[0][1], ++ row.cp[1][0], row.cp[1][1], row.cp[2][0], row.cp[2][1], ++ row.cp[3][0], row.cp[3][1], row.cp[4][0], row.cp[4][1], row.owner, ++ row.result, row.cap_seen); ++ return HRTIMER_NORESTART; ++} ++ ++static void g5p_r73b_release_entry(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ ++ if (!r) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_ENTRY, G5P_R73B_NONE, 0); ++ { ++ return; ++ } ++} ++ ++static void g5p_r73b_release_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_EXIT, G5P_R73B_NONE, ret); ++ if (ret) ++ g5p_r73b_disarm(b->r73b); ++} ++ ++/* The release epoch ends when the host has returned from the ALIVE wait. */ ++static void g5p_r73b_events_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_EVENTS_EXIT, G5P_R73B_OWNER_EVENTS, ret); ++ g5p_r73b_disarm(b->r73b); ++} ++ ++static int g5p_r73b_latch(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const u32 id = (1), ms = (0); ++ const u32 fast = (1); ++ const u32 early = (1); ++ const u32 r74 = (1), cap = (1048576); ++ const u32 r74f = (1); ++ const u32 r74g = (1); ++ const u32 r74h = (0); ++ ++ struct g5p_r73b *r; ++ int ret; ++ ++ ; ++ ; ++ ; ++ ; ++ ; ++ ; ++ /* nopatch replaces r73f case 1 under the whole r74/r74f/r74g path, and nothing else. */ ++ /* nopatch's receipts and its r74 gate are the TYPED_V3 path's; any other path (r74 0) runs ++ * Apple's image anyway (only r73f patches it) and gets neither. ++ */ ++ if (1 != g5p_fw_unpatched_want((1), 0, 0, 0) || ++ (1 && r74 == 1 && (0 || r74f != 1 || r74g != 1))) { ++ dev_err(g->dev, ++ "NOPATCH reject phase=map unpatched=%u/%u r73f_case=%u path=%u/%u/%u end=nopatch\n", ++ 1, (1), 0, r74, r74f, r74g); ++ return -EINVAL; ++ } ++ /* R74 rides only on an exact r73f case 1 (or nopatch) or 4 with the raised limits. */ ++ if (r74 && (!id || r74 > 2 || 0 != (r74 == 1 ? (1 ? 0 : 1) : 4) || ++ 1 != (r74 == 1) || !(1) || cap < G5P_R74_IRQ_CAP || ++ cap > G5P_R74_IRQ_CAP_MAX || ++ g5p_transport_event_limit() < G5P_R74_EVENT_FLOOR)) { ++ dev_err(g->dev, ++ "R74 reject phase=map case=%u r73b_case=%u r73f_case=%u meta6c8=%u " ++ "render_engine=%u irq_cap=%u event_limit=%u end=r74\n", ++ r74, id, 0, b->r72_meta6c8, (1), cap, ++ g5p_transport_event_limit()); ++ return -EINVAL; ++ } ++ ++ /* R74f rides only on the exact r74 case 1 forced-ready epoch. */ ++ ; ++ ++ /* R74g rides only on the exact r74f case 1 epoch. */ ++ ; ++ ++ /* R74h observes only the exact r74g case 1 epoch. */ ++ ; ++ ++ /* The bounded serialized drain supersedes r73's IRQ-context drain. */ ++ ; ++ ; ++ ; ++ ; ++ if (id > 3 || fast > 1 || (fast && id == 3) || early > 2 || ++ (early && (!fast || id != early)) || b->r73_aux_drain || 7 != 7 || ++ 1 != (id == 2 ? 0 : 1) || (ms && (ms < 1000 || ms > 30000))) ++ return -EINVAL; ++ r = devm_kzalloc(g->dev, sizeof(*r), GFP_KERNEL); ++ if (!r) ++ return -ENOMEM; ++ r->dev = g->dev; ++ r->boot = b; ++ r->case_id = id; ++ r->watchdog_ms = ms; ++ r->dc_variant = 7; ++ r->meta6c8 = 1; ++ r->aux = id != 3; ++ r->ack_first = true; ++ r->fast_enabled = fast; ++ r->r73e_case = early; ++ r->r74_case = r74; ++ r->r74f_case = r74f; ++ r->r74g_case = r74g; ++ r->r74h_case = r74h; ++ r->settle_us = min_t(u32, READ_ONCE(boot_alive_settle_us), 5000); ++ r->nopatch = 1 && r74 == 1; ++ r->irq_cap = r74 ? cap : G5P_R73B_IRQ_CAP; ++ /* Heartbeat116: in an R74 epoch the window detector replaces both lifetime bounds. */ ++ r->hb_on = r74 != 0; ++ r->hb_seed = r74 ? (0) : 0; ++ g5p_hb_cfg_default(&r->hb_cfg); ++ atomic_set(&r->early_once, 0); ++ atomic_set(&r->early_busy, 0); ++ atomic_set(&r->early_proof, G5P_PX_NONE); ++ r->watchdog_cpu = -1; ++ mutex_init(&r->drain_lock); ++ hrtimer_setup(&r->watchdog, g5p_r73b_watchdog_fire, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL_PINNED_HARD); ++ ret = devm_add_action_or_reset(g->dev, g5p_r73b_stop, r); ++ if (ret) ++ return ret; ++ b->r73_aux_drain = 1; ++ b->r73b = r; ++ dev_info( ++ g->dev, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), r->ack_first, r->aux, r->irq_cap, ms, ++ r->dc_variant, r->meta6c8, G5P_R73B_ROWS, ++ init_utsname()->release); ++ g5p_blob_printf( ++ g, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), 1, 1, r->irq_cap, ms, 7, 1, ++ G5P_R73B_ROWS, init_utsname()->release); ++ g5p_blob_printf(g, "R73C latched fast=%u case=%u meta6c8=%u end=r73c\n", ++ 1, id, 1); ++ dev_info(g->dev, "R73E latched case=%u meta6c8=%u fast=%u end=r73e\n", ++ early, r->meta6c8, r->fast_enabled); ++ ; ++ ; ++ if (r->nopatch) ++ dev_info( ++ g->dev, ++ "NOPATCH latched fw=apple-original r73f_case=0 meta6c8=%u r73e_case=%u r74=%u " ++ "r74f=%u r74g=%u force_policy_ready=0 end=nopatch\n", ++ r->meta6c8, early, r74, r74f, r74g); ++ { ++ dev_info( ++ g->dev, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, r->meta6c8, ++ r->irq_cap, g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, 1, r->irq_cap, ++ g5p_transport_event_limit()); ++ dev_info( ++ g->dev, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ g5p_blob_printf( ++ g, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ g5p_blob_printf( ++ g, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ }; ++ /* TYPED_V3 production path: the self-drain DC3 attach policy of r87b rides on r74f + r74g ++ * alone, with none of the r74h / r80 instrumentation. r74g has the firmware own the first ++ * MMU pass, which is what makes self-drain safe (r87b-attach.h); r74h only observes. Every ++ * other r8x / r9x knob is still refused below without boot_r80_case. ++ */ ++ { ++ r->r87b_attach = G5P_R87B_ATTACH_SELF_DRAIN; ++ dev_info( ++ g->dev, ++ "R87B latched policy=1 name=self-drain r74f=%u r74g=%u r74h=%u r80=0 " ++ "production_path=1 end=r87b\n", ++ r74f, r74g, r74h); ++ } ++ { ++ { ++ { ++ { ++ { ++ if ((1) && !r->r87b_attach) { ++ dev_err(g->dev, ++ "R87B reject phase=map r87b knob without boot_r80_case policy=%u end=r87b\n", ++ (1)); ++ return -EBADMSG; ++ } else { ++ { ++ { ++ ; ++ ; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ } ++ return 0; ++} ++ ++/* Process context, cached CPU RAM only; never waits for the command lock. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +new file mode 100644 +index 000000000..71cf79dbe +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -0,0 +1,532 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74f deferred doorbell. Included by boot.inc before the event decoder. ++ * Diagnostic bypass on the r74 forced-ready epoch only; not power policy. ++ * ++ * A DC3 pair or a natural TA is published once, append-only, before this ++ * code runs; its producer is never rewound, republished or torn down. If ++ * the immediate functional gate finds GPC inactive, that generation stays ++ * pending (never b->failed), the host repeats the r74e release transition ++ * once, and the first type-0 ALIVE decoded afterwards owns one doorbell ++ * attempt. The owner runs under event_lock: producer confirm, R1+14 fence, ++ * active check, dma_wmb(), doorbell 0x10. Receipts stay in RAM; console ++ * lines are printed afterwards from process context. ++ */ ++ ++static const char *const g5p_r74f_kinds[G5P_R74F_KINDS] = { "dc3", "ta", "3d" }; ++ ++static const char *const g5p_r74f_states[G5P_R74F_STATES] = { ++ "idle", "published", "open", "firing", "rang", "cancelled", ++}; ++ ++static const char *const g5p_r74f_cancels[G5P_R74F_CANCELS] = { ++ "none", "reactivation-failed", ++ "no-alive", "consumed-before-bell", ++ "protocol", "alive-during-reactivation", ++ "self-drain", ++}; ++ ++static const char *const g5p_r74f_stops[G5P_R74F_STOPS] = { ++ "none", ++ "attach-reactivation-failed", ++ "attach-published-not-kicked", ++ "attach-kicked-not-consumed", ++ "first-mmu-window-closed", ++ "ta-reactivation-failed", ++ "ta-published-not-kicked", ++ "firmware-dc3-tag-mismatch", ++ "dc3-ram-validation-refused", ++ "firmware-companion-recovery", ++ "ta-published-not-consumed", ++ "ta-window-missed", ++}; ++ ++static const char *const g5p_r74f_paths[G5P_R74F_PATHS] = { ++ "none", ++ "immediate", ++ "deferred", ++}; ++ ++static bool g5p_r74f_on(struct g5p_boot *b) ++{ ++ return b && b->r73b && 1; ++} ++ ++static u32 *g5p_r74f_ctl(struct g5p_boot *b, u32 kind) ++{ ++ return (u32 *)g5p_boot_shared(b, kind == G5P_R74F_TA ? 0x60 : ++ kind == G5P_R74F_3D ? 0x64 : ++ 0x6c); ++} ++ ++/* Owner only, event_lock held with interrupts off. Between the active check ++ * and the doorbell there is only dma_wmb(): no console, no other MMIO. ++ */ ++static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ u32 *fence, u32 *state) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 consumer = READ_ONCE(ctl[0]); ++ u32 producer = smp_load_acquire(&ctl[4]); ++ u32 queued = (f->target - f->start) & 255; ++ u32 progressed = (consumer - f->start) & 255; ++ u32 expected = f->kind == G5P_R74F_DC3 ? 2 : 1; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* Do not turn corrupted modulo-256 ring state into a doorbell. */ ++ if (consumer > 255 || producer > 255 || queued != expected || ++ progressed > queued) ++ return -EPROTO; ++ /* Acquire: pairs with the producer store published before the state. */ ++ if (producer != f->target) ++ return -EPROTO; ++ if (consumer == f->target) ++ return -EALREADY; ++ writel(7, g->window[1] + 0x14); ++ *fence = readl(g->window[1] + 0x14); ++ if (*fence & ~7U) ++ return -EPROTO; ++ *state = readl(g->window[1] + 0x1000); ++ /* perf60 (gpc_any_pstate): a GPC request in flight or a not-yet-awake GPC after the wake ++ * above; re-read for up to 2 ms, as Apple's setGPCPerformanceState waits for bit 4. Reads only. ++ */ ++ { ++ u64 t0 = ktime_get_mono_fast_ns(); ++ ++ while (!g5p_gpc_ok(*state) && ktime_get_mono_fast_ns() - t0 < ++ 2000ULL * NSEC_PER_USEC) { ++ udelay(10); ++ *state = readl(g->window[1] + 0x1000); ++ } ++ } ++ if (!g5p_gpc_ok(*state)) ++ return -EHOSTDOWN; ++ dma_wmb(); ++ writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); ++ return 0; ++} ++ ++/* Owner only: record the attempt, then publish it with the state release. */ ++static void g5p_r74f_rang(struct g5p_r73b *r, struct g5p_r74f_bell *f, int ret, ++ u32 fence, u32 state, u32 path) ++{ ++ f->bell_ns = ktime_get_mono_fast_ns(); ++ f->path = path; ++ f->fence = fence; ++ f->bell_state = state; ++ f->bell_ret = ret; ++ if (!ret) { ++ f->kicked = 1; ++ atomic_inc(&r->r74f_bells); ++ } else if (ret == -EALREADY) { ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ } else if (ret != -EHOSTDOWN) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ } ++ atomic_set_release(&f->state, G5P_R74F_RANG); ++} ++ ++/* g5p_boot_drain(), event_lock held: one type-0 ALIVE was just decoded. */ ++static void g5p_r74f_alive(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ struct g5p_r74f_bell *f; ++ u32 kind, fence = U32_MAX, state = U32_MAX; ++ int ret; ++ ++ WRITE_ONCE(r->r74f_alive, r->r74f_alive + 1); ++ kind = READ_ONCE(r->r74f_active); ++ if (!kind) ++ return; ++ f = &r->r74f_bell[kind - 1]; ++ if (atomic_read_acquire(&f->state) != G5P_R74F_OPEN || ++ r->r74f_alive <= f->open_alive) ++ return; ++ /* Sole owner: exactly one decoder moves OPEN to FIRING. */ ++ if (atomic_cmpxchg(&f->state, G5P_R74F_OPEN, G5P_R74F_FIRING) != ++ G5P_R74F_OPEN) ++ return; ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_DEFERRED); ++} ++ ++static int g5p_r74f_poll(struct g5p_diag *g, u32 alive, u32 expected, ++ u32 timeout_us, u64 *elapsed_ns, u32 *raw) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u64 start = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ *raw = U32_MAX; ++ for (;;) { ++ if (READ_ONCE(g->boot->failed)) { ++ ret = -EIO; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ /* Serialize the ordinal check and exactly one R1 read with the ++ * decoder. Once ALIVE advances, no further R1 MMIO is licensed. ++ */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ break; ++ } ++ *raw = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if ((*raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ return ret; ++} ++ ++/* Absolute freshness budget from the DC3 doorbell. The first MMU pass may ++ * touch R0 only while the same active window is still demonstrably fresh. ++ */ ++ ++/* The r74e release transition, once per generation, on r74f receipts and ++ * without console output: core request clear, GPC 0 within 1.0 ms, GPC 1 ++ * within 1.8 ms. Every R1 access is serialized against ALIVE decoding; ++ * ordinal advance cancels the generation before any further R1 MMIO. ++ */ ++static int g5p_r74f_reactivate(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u32 alive = f->arm_alive; ++ int ret; ++ ++ f->reactivated = 1; ++ f->power_start_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ f->power_ret = -EHOSTDOWN; ++ return -EHOSTDOWN; ++ } ++ writel(0, g->window[1] + 0x20000); ++ f->core_fence = readl(g->window[1] + 0x20000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, 0, G5P_R74F_OFF_US, &f->off_ns, ++ &f->off_raw); ++ if (!ret) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ goto out; ++ } ++ /* boot_gpu_perf_seed: the firmware believes pstate q; never drop the GPC under it */ ++ writel(0x10 | (b->perf_poweron ?: ++ b->perf_seed ?: ++ 1), ++ g->window[1] + 0x1000); ++ f->gpc_fence = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, ++ b->perf_poweron ?: ++ b->perf_seed ?: ++ 1, ++ G5P_R74F_ON_US, &f->on_ns, &f->on_raw); ++ } ++out: ++ f->power_ret = ret; ++ return ret; ++} ++ ++/* Process context, before any publication: no overlap with a pending ring. */ ++static int g5p_r74f_admit(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *dc = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ ++ if (!g5p_r74b_ram_only(g) || READ_ONCE(b->failed) || ++ READ_ONCE(r->r74f_stop) || ++ atomic_read(&r->r74f_bell[kind].state) != G5P_R74F_IDLE) ++ return -EPERM; ++ if (kind == G5P_R74F_DC3) ++ return READ_ONCE(r->r74f_active) ? -EBUSY : 0; ++ /* M7: the TA wakeup never shares a doorbell with a pending DC3 pair. */ ++ dma_rmb(); ++ if (READ_ONCE(r->r74f_active) != G5P_R74F_DC3 + 1 || ++ !r->r74f_bell[G5P_R74F_DC3].complete || ++ READ_ONCE(dc[0]) != READ_ONCE(dc[4])) ++ return -EBUSY; ++ return 0; ++} ++ ++static void g5p_r74f_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 state = atomic_read_acquire(&f->state); ++ char line[3][512]; ++ ++ dma_rmb(); ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74F bell kind=%s generation=%u ring=%02x start=%u target=%u state=%s path=%s " ++ "first_ret=%d first_fence=%08x first_state=%08x attempted=%u fence=%08x " ++ "bell_state=%08x bell_ret=%d kicked=%u bells=%d cancel=%s complete=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->ring, f->start, ++ f->target, g5p_r74f_states[state < G5P_R74F_STATES ? state : 0], ++ g5p_r74f_paths[f->path], f->first_ret, f->first_fence, ++ f->first_state, f->attempted, f->fence, f->bell_state, ++ f->bell_ret, f->kicked, atomic_read(&b->r73b->r74f_bells), ++ g5p_r74f_cancels[f->cancel], f->complete); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74F ring kind=%s generation=%u arm_alive=%u open_alive=%u bell_alive=%u " ++ "alive_now=%u consumer_before=%u consumer_after=%u producer_after=%u " ++ "consumer_now=%u producer_now=%u failed=%u event_failed=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->arm_alive, ++ f->open_alive, f->bell_alive, READ_ONCE(b->r73b->r74f_alive), ++ f->consumer_before, f->consumer_after, f->producer_after, ++ READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), READ_ONCE(b->failed), ++ READ_ONCE(b->event_failed)); ++ scnprintf( ++ line[2], sizeof(line[2]), ++ "R74F power kind=%s generation=%u reactivated=%u core_fence=%08x gpc_fence=%08x " ++ "off_ns=%llu on_ns=%llu off_raw=%08x on_raw=%08x power_ret=%d arm_ns=%llu " ++ "power_start_ns=%llu open_ns=%llu bell_start_ns=%llu bell_ns=%llu done_ns=%llu " ++ "end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->reactivated, ++ f->core_fence, f->gpc_fence, f->off_ns, f->on_ns, f->off_raw, ++ f->on_raw, f->power_ret, f->arm_ns, f->power_start_ns, ++ f->open_ns, f->bell_start_ns, f->bell_ns, f->done_ns); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++/* r87b receipt: the DC3 attach policy and what the host did after the publish. ++ * RAM reads only; one line per DC3 generation (console at the consume/timeout ++ * point, blob at export). ++ */ ++static void g5p_r87b_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 policy = READ_ONCE(b->r73b->r87b_attach); ++ char line[640]; ++ ++ dma_rmb(); ++ scnprintf( ++ line, sizeof(line), ++ "R87B attach policy=%u name=%s generation=%u gate_first_ret=%d " ++ "gate_first_state=%08x gate_closed=%u path=%s cancel=%s bell_rung=%u bells=%d " ++ "reactivated=%u cons_before=%u prod_before=%u prod_pub=%u cons_after=%u " ++ "prod_after=%u consumed=%u arm_alive=%u alive_done=%u alive_delta=%u " ++ "pub_to_done_ns=%llu irqs=%d events=%u end=r87b\n", ++ policy, policy ? "self-drain" : "original", f->generation, ++ f->first_ret, f->first_state, f->first_ret == -EHOSTDOWN, ++ g5p_r74f_paths[f->path], g5p_r74f_cancels[f->cancel], f->kicked, ++ atomic_read(&b->r73b->r74f_bells), f->reactivated, ++ f->consumer_before, f->start, f->target, f->consumer_after, ++ f->producer_after, f->complete, f->arm_alive, f->alive_done, ++ f->alive_done - f->arm_alive, ++ f->done_ns >= f->arm_ns ? f->done_ns - f->arm_ns : 0, ++ atomic_read(&b->irqs), READ_ONCE(b->events)); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* A retained boundary: the publication stays, nothing is failed. */ ++static int g5p_r74f_stop(struct g5p_diag *g, u32 stop, int value) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ ++ r->r74f_stop_value = value; ++ WRITE_ONCE(r->r74f_stop, stop); ++ g5p_r74_mark(g, G5P_R74F_STOP, stop); ++ dev_emerg(g->dev, "R74F stop reason=%s value=%d failed=%u end=r74f\n", ++ g5p_r74f_stops[stop], value, READ_ONCE(g->boot->failed)); ++ return -EINPROGRESS; ++} ++ ++/* After the reactivation: open the generation, wait (bounded) for the ++ * ALIVE owner while the event drain stays alive, then cancel if unrung. ++ */ ++static int g5p_r74f_open(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags, deadline; ++ int ret; ++ ++ ret = g5p_r74f_reactivate(g, f); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (ret) { ++ f->cancel = ret == -EHOSTDOWN ? G5P_R74F_CANCEL_ALIVE_ADVANCED : ++ G5P_R74F_CANCEL_REACTIVATION; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ f->open_alive = r->r74f_alive; ++ f->open_ns = ktime_get_mono_fast_ns(); ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (ret) ++ return 0; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ /* The IRQ path is the sole event consumer and bell owner. */ ++ if (READ_ONCE(b->failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* Process context, command mutex held, producer already published. Returns ++ * 0 after one doorbell, 1 if consumed without one, -EINPROGRESS when the ++ * publication is retained unrung, else a fatal protocol error. ++ */ ++static int g5p_r74f_bell(struct g5p_diag *g, u32 kind, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(b, kind); ++ u32 fence = U32_MAX, state = U32_MAX, now; ++ unsigned long flags; ++ int ret, drained = 0; ++ bool ta = kind == G5P_R74F_TA; ++ bool self_drain; ++ ++ if (kind == G5P_R74F_3D) ++ return -EINVAL; /* the 3D bell is armed by g5p_r96_3d_arm(): never the DC3 / self-drain path */ ++ f->kind = kind; ++ f->ring = ta ? 0x60 : 0x6c; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ /* The immediate functional gate r74e used, without console output. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, kind + 1); ++ atomic_set(&f->state, G5P_R74F_FIRING); ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ f->first_ret = ret; ++ f->first_fence = fence; ++ f->first_state = state; ++ /* r87b self-drain: a closed DC3 gate is only a phase sample of the ++ * firmware's GPC window. The host never repairs it: no R1+20000 clear, no ++ * GPC off/on, no bell. The caller waits RAM-only for the firmware to ++ * consume the pair. Only the DC3 kind; an active gate rang above. ++ */ ++ self_drain = ret == -EHOSTDOWN && kind == G5P_R74F_DC3 && ++ READ_ONCE(r->r87b_attach) == G5P_R87B_ATTACH_SELF_DRAIN; ++ if (self_drain) { ++ f->cancel = G5P_R74F_CANCEL_SELF_DRAIN; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (ret == -EHOSTDOWN) { ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ } else { ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_IMMEDIATE); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (self_drain) { ++ drained = 1; ++ g5p_r74_mark(g, G5P_R74F_ATTACH_ARMED, state); ++ } else if (ret == -EHOSTDOWN) { ++ drained = g5p_r74f_open(g, f); ++ /* Functional reactivation precedes all console/milestone output. */ ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_ARMED : G5P_R74F_ATTACH_ARMED, ++ state); ++ g5p_r74_mark(g, ++ ta ? G5P_R74F_TA_REACTIVATED : ++ G5P_R74F_ATTACH_REACTIVATED, ++ f->power_ret); ++ } ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_BELL : G5P_R74F_ATTACH_BELL, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ if (drained) ++ return drained; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_REACTIVATION || ++ f->cancel == G5P_R74F_CANCEL_ALIVE_ADVANCED) ++ return g5p_r74f_stop( ++ g, ++ ta ? G5P_R74F_STOP_TA_REACTIVATION : ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ f->power_ret); ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* RAM only: the consumer this generation reached, at consume or timeout. */ ++static void g5p_r74f_consumed(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_r74f_bell *f = &g->boot->r73b->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(g->boot, kind); ++ ++ dma_rmb(); ++ f->consumer_after = READ_ONCE(ctl[0]); ++ f->producer_after = READ_ONCE(ctl[4]); ++ f->complete = f->consumer_after == f->target; ++ f->done_ns = ktime_get_mono_fast_ns(); ++ f->alive_done = READ_ONCE(g->boot->r73b->r74f_alive); ++ if (kind == G5P_R74F_DC3) ++ g5p_r87b_line(g, f, false); ++} ++ ++/* g5p_r22_mmu(): attach's first pass only; records the gate it already read. */ ++ ++/* g5p_r22_command(): an r74f boundary keeps the epoch, drain and backing. */ ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74g.inc b/drivers/gpu/apple-g5p/boot-r74g.inc +new file mode 100644 +index 000000000..6ff269317 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74g.inc +@@ -0,0 +1,288 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74g. Included by boot.inc after boot-r74f.inc; r74f case 1 only. It is a ++ * diagnostic bypass on the forced-ready epoch, not power policy. ++ * ++ * The firmware DC3 handler (0x1000218c..0x10002218) writes e140/e148 for ++ * slot N, copies host_generation[N] to cached_generation[N], then issues ++ * the e138 (N << 3) | 4 catalogue invalidation and waits for it. The DC ++ * dispatcher advances the consumer only after the handler returns, so a ++ * consumed DC3 pair is the completion receipt. r74g checks the RAM-visible ++ * results and issues no host MMU MMIO. The natural TA is then published ++ * without a doorbell. A RAM-only grace (four type-0 ALIVEs, 20 ms cap) ++ * lets the firmware drain it on its own. If it stays pending, the r74f ++ * ALIVE owner gets one fenced fallback doorbell. Receipts print afterwards. ++ */ ++ ++/* perf60: with the policy held in state 0 the firmware is quiet (no ALIVE stream, no self-drain), so ++ * job 1's TA bell, owned by "the first ALIVE after the grace", would never ring. ++ */ ++/* release8 default */ ++ ++static const char *const g5p_r74g_graces[G5P_R74G_GRACES] = { ++ "none", "consumed", "alives", "wall", "fault", ++}; ++ ++static bool g5p_r74g_on(struct g5p_boot *b) ++{ ++ return g5p_r74f_on(b) && 1; ++} ++ ++static u32 g5p_r74g_root(struct g5p_boot *b, u32 off) ++{ ++ return g5p_le32(b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r74g_shared(struct g5p_boot *b, u32 at) ++{ ++ return g5p_le32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE); ++} ++ ++/* RAM only: the DC3 ring, both records in attach order (3D, then TA), the ++ * PA[N] entries, host and cached tags, both failure latches and the ++ * root+3980 companion word, whose value 2 is firmware recovery/escape. ++ */ ++static u32 g5p_r74g_check(struct g5p_diag *g, u32 phase, const u32 *slot, ++ u64 pa, u32 tag, u32 start) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r74g_dc3 *d = &b->r73b->r74g_dc3[phase]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ u32 bad = 0; ++ ++ dma_rmb(); ++ d->start = start; ++ d->target = (start + 2) & 255; ++ d->tag = tag; ++ d->expected_pa = pa; ++ d->consumer = READ_ONCE(ctl[0]); ++ d->producer = READ_ONCE(ctl[4]); ++ if (d->consumer != d->target || d->producer != d->target) ++ bad |= G5P_R74G_BAD_RING; ++ for (u32 i = 0; i < 2; i++) { ++ u32 at = b->layout.offset[0x6d] + ((start + i) & 255) * 20; ++ ++ for (u32 j = 0; j < 5; j++) { ++ d->rec[i][j] = g5p_r74g_shared(b, at + j * 4); ++ if (d->rec[i][j] != (j == 0 ? 3 : ++ j == 1 ? slot[1 - i] : ++ 0)) ++ bad |= G5P_R74G_BAD_RECORD; ++ } ++ } ++ for (u32 e = 0; e < 2; e++) { ++ u8 *p = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ ++ d->slot[e] = slot[e]; ++ d->pa[e] = (u64)g5p_le32(p + 4) << 32 | g5p_le32(p); ++ d->host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + slot[e] * 4); ++ d->cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + slot[e] * 4); ++ if (d->pa[e] != pa) ++ bad |= G5P_R74G_BAD_PA; ++ if (d->host[e] != tag) ++ bad |= G5P_R74G_BAD_HOST; ++ if (d->cached[e] != tag) ++ bad |= G5P_R74G_BAD_CACHED; ++ } ++ d->failed = READ_ONCE(b->failed); ++ d->event_failed = READ_ONCE(b->event_failed); ++ if (d->failed || d->event_failed) ++ bad |= G5P_R74G_BAD_FAILED; ++ d->companion = g5p_r74g_root(b, G5P_R74G_COMPANION); ++ d->recoveries = g5p_r74g_root(b, G5P_R74G_RECOVERIES); ++ d->classification = g5p_r74g_root(b, G5P_R74G_CLASS); ++ if (d->companion == 2) ++ bad |= G5P_R74G_BAD_COMPANION; ++ d->bad = bad; ++ d->t_ns = ktime_get_mono_fast_ns(); ++ return bad; ++} ++ ++/* A refused check is retained, never b->failed: companion 2 is firmware ++ * recovery, a cached-tag-only mismatch is the firmware tag copy, and ++ * anything else (a host tag included) is a refused RAM validation. ++ */ ++static int g5p_r74g_refuse(struct g5p_diag *g, u32 bad) ++{ ++ if (bad & G5P_R74G_BAD_COMPANION) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_COMPANION, bad); ++ if (bad == G5P_R74G_BAD_CACHED) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_TAGS, bad); ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_RAM, bad); ++} ++ ++/* Two bounded lines: every field at full width still ends in end=r74g. */ ++static void g5p_r74g_dc3_line(struct g5p_diag *g, u32 phase, bool blob) ++{ ++ const struct g5p_r74g_dc3 *d = &g->boot->r73b->r74g_dc3[phase]; ++ const char *name = phase ? "ta-preflight" : "dc3-consume"; ++ char line[2][512]; ++ ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74G dc3 phase=%s bad=%02x start=%u target=%u consumer=%u producer=%u tag=%08x " ++ "slot=%u/%u host=%08x/%08x cached=%08x/%08x failed=%u event_failed=%u " ++ "companion=%u recoveries=%u class=%u host_mmu=skipped t_ns=%llu end=r74g\n", ++ name, d->bad, d->start, d->target, d->consumer, d->producer, ++ d->tag, d->slot[0], d->slot[1], d->host[0], d->host[1], ++ d->cached[0], d->cached[1], d->failed, d->event_failed, ++ d->companion, d->recoveries, d->classification, d->t_ns); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74G dc3-ram phase=%s pa=%llx/%llx expected_pa=%llx rec0=%x,%x,%x,%x,%x " ++ "rec1=%x,%x,%x,%x,%x end=r74g\n", ++ name, d->pa[0], d->pa[1], d->expected_pa, d->rec[0][0], ++ d->rec[0][1], d->rec[0][2], d->rec[0][3], d->rec[0][4], ++ d->rec[1][0], d->rec[1][1], d->rec[1][2], d->rec[1][3], ++ d->rec[1][4]); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++static void g5p_r74g_ta_line(struct g5p_diag *g, bool blob) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ const struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 end = r->r74g_grace_end; ++ char line[256]; ++ ++ scnprintf( ++ line, sizeof(line), ++ "R74G ta grace_end=%s arm_alive=%u grace_alive=%u grace_ns=%llu opened=%u " ++ "open_alive=%u ack_skipped=%u end=r74g\n", ++ g5p_r74g_graces[end < G5P_R74G_GRACES ? end : 0], f->arm_alive, ++ r->r74g_grace_alive, r->r74g_grace_ns - f->arm_ns, ++ r->r74g_opened, f->open_alive, r->r74g_ack_skipped); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* Process context, command mutex held, TA producer already published once. ++ * Grace: RAM reads and sleeps only, no MMIO and no doorbell, while the IRQ ++ * thread keeps decoding. It ends at the consume, four type-0 ALIVEs after ++ * the arm, or the 20 ms wall cap. If the TA is still pending, the ++ * generation opens, and the first ALIVE decoded after that owns the single ++ * r74f fallback attempt. Returns 0 after one doorbell, 1 if consumed without ++ * one, -EINPROGRESS for a retained boundary, else a fatal error. ++ */ ++static int g5p_r74g_ta(struct g5p_diag *g, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_TA); ++ unsigned long flags, deadline; ++ u32 end = G5P_R74G_GRACE_NONE, now; ++ int ret = 0; ++ ++ f->kind = G5P_R74F_TA; ++ f->ring = 0x60; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, G5P_R74F_TA + 1); ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ while (end == G5P_R74G_GRACE_NONE) { ++ ; ++ dma_rmb(); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) ++ end = G5P_R74G_GRACE_FAULT; ++ else if (READ_ONCE(ctl[0]) == target) ++ end = G5P_R74G_GRACE_CONSUMED; ++ else if (READ_ONCE(r->r74f_alive) - f->arm_alive >= ++ G5P_R74G_GRACE_ALIVES) ++ end = G5P_R74G_GRACE_ALIVES_END; ++ else if (ktime_get_mono_fast_ns() - f->arm_ns >= ++ (u64)G5P_R74G_GRACE_US * 1000) ++ end = G5P_R74G_GRACE_WALL; ++ else ++ usleep_range(100, 200); ++ } ++ spin_lock_irqsave(&b->event_lock, flags); ++ r->r74g_grace_end = end; ++ r->r74g_grace_alive = r->r74f_alive; ++ r->r74g_grace_ns = ktime_get_mono_fast_ns(); ++ dma_rmb(); ++ if (end == G5P_R74G_GRACE_FAULT) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (READ_ONCE(ctl[0]) == target) { ++ /* Drained by the firmware policy loop: never ring. */ ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ r->r74g_opened = 1; ++ f->open_alive = r->r74f_alive; ++ f->open_ns = r->r74g_grace_ns; ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ /* ta1_host_bell: the host owns it instead, in the same lock section a decoder would. */ ++ if ((1) && atomic_cmpxchg(&f->state, G5P_R74F_OPEN, ++ G5P_R74F_FIRING) == G5P_R74F_OPEN) { ++ u32 fence = U32_MAX, state = U32_MAX; ++ int rung; ++ ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ rung = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, rung, fence, state, ++ G5P_R74F_PATH_IMMEDIATE); ++ } ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (end == G5P_R74G_GRACE_FAULT) ++ ret = -EIO; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, G5P_R74G_TA_GRACE, end); ++ if (r->r74g_opened) ++ g5p_r74_mark(g, G5P_R74G_TA_FALLBACK, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ g5p_r74g_ta_line(g, false); ++ if (ret) ++ return ret; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_CONSUMED) ++ return 1; ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_NOT_CONSUMED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_WINDOW_MISSED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +new file mode 100644 +index 000000000..6a62f2eb0 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -0,0 +1,1665 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Experimental G2 stages. Included after the bounded META slave primitives. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "boot-layout.h" ++#include "render-user-pmset.h" ++#include "render-user-recover.h" ++#include "fender.h" ++#include "readback.h" ++#include "meta71c114.h" ++#include "latency123.h" ++#include "perf-envelope.h" ++ ++/* Boot deltas are selected before map; no live render-time reconfiguration. */ ++ ++ ++ ++/* R46 (diag-fable51 T7): the first SOFT_RESET write of the host reset FIFO ++ * also asserts bits 56-59, VBS/VCE/TE3/TILING_CORE in the PBE2_IN_XE ++ * layout. Apple's mask 00e7fffffffffc1d is exactly the other layout's ++ * MASKFULL, in which those bits do not exist and TE/GPP/UVS/TLA/PM/VDM are ++ * already asserted; expected to be a no-op. The unchanged second write ++ * releases them. The FIFO forbids a readback here. ++ */ ++#define G5P_BOOT_RESET1 0x00e7fffffffffc1dULL ++#define G5P_BOOT_R46_RESET_BITS (0xfULL << 56) ++ ++/* R72b: isolate Apple meta+0x6c8 while retaining dc7 transport. */ ++ ++/* The event ring's stamp events (type 1) dump once per epoch, then a count at each power of two. */ ++ ++/* R73: mirror Apple's auxiliary firmware-ring returns on IRQ174. */ ++ ++/* Latency114: the pstate-1 frequency word meta+0x71c (meta71c114.h). */ ++ ++/* Latency123 (latency123.h): arm A boot words, and the live ctl+0x04 tick-service mask. */ ++ ++/* perf60 clock (perf-envelope.h): the firmware's own pstate request, pinned before release. */ ++/* release8 default */ ++/* GPU overclock (gpu-oc.h, docs/power/ds-gpu-dvfs.md §9 R5): PSINFO record 5 = stock record 4 ++ * with only the PLL multiplier moved (no voltage change), written before release while the GPC is at ++ * state 1, the firmware's table extended to state 5, the release powering on at 5. Needs ++ * boot_gpu_perf 500 and boot_gpu_perf_poweron 1. A thermal stop (drm-render-v3.inc) reboots the ++ * firmware at state 4 when a SoC zone reaches boot_gpu_oc_temp_mc. Record 5 is volatile PMGR ++ * state (LLB rewrites the table only at SoC reset; iBoot reads it once to build the ADT before us) ++ * and is cleared on every boot that does not overclock. ++ */ ++ ++static bool ++ g5p_oc_tripped; /* the thermal stop fired: every later epoch of this boot runs state 4 */ ++ ++static const char *const g5p_oc_zone_names[] = { "apple-a9-soc0", ++ "apple-a9-soc1", ++ "apple-a9-tcc2" }; ++ ++/* The hottest of the three SoC zones (millidegrees); the number of zones read. */ ++static u32 g5p_oc_temp(int *max_mc, const char **zone) ++{ ++ u32 n = 0; ++ ++ *max_mc = INT_MIN; ++ *zone = "none"; ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_zone_names); i++) { ++ struct thermal_zone_device *tz = ++ thermal_zone_get_zone_by_name(g5p_oc_zone_names[i]); ++ int t; ++ ++ if (IS_ERR_OR_NULL(tz) || thermal_zone_get_temp(tz, &t)) ++ continue; ++ n++; ++ if (t > *max_mc) { ++ *max_mc = t; ++ *zone = g5p_oc_zone_names[i]; ++ } ++ } ++ return n; ++} ++ ++static int g5p_oc_read(struct g5p_diag *g, u32 n, u32 w[4]) ++{ ++ for (u32 i = 0; i < 4; i++) ++ if (regmap_read(g->pmgr, G5P_PSINFO_REC(n) + 4 * i, &w[i])) ++ return -EIO; ++ return 0; ++} ++ ++/* Write record 5 from the live record 4 and read it back; the GPC is at state 1 (map time). ++ * Slot 5 is read first: nonzero and not a record this driver writes -> refused, left untouched. ++ */ ++ ++/* A boot without the overclock clears a record 5 this driver wrote; anything else stays. */ ++static void g5p_oc_clear(struct g5p_diag *g) ++{ ++ u32 r4[4], r5[4]; ++ ++ if (!g->pmgr || g5p_oc_read(g, 4, r4) || ++ g5p_oc_read(g, G5P_OC_STATE, r5) || ++ !(r5[0] || r5[1] || r5[2] || r5[3])) ++ return; ++ if (!g5p_oc_is_ours(r4, r5)) { ++ dev_info( ++ g->dev, ++ "GPUPERF oc slot 5 not ours, left: %08x %08x %08x %08x end=gpuperf\n", ++ r5[0], r5[1], r5[2], r5[3]); ++ return; ++ } ++ for (u32 i = 0; i < 4; i++) ++ regmap_write(g->pmgr, G5P_PSINFO_REC(G5P_OC_STATE) + 4 * i, 0); ++ g5p_oc_read(g, G5P_OC_STATE, r5); ++ dev_info( ++ g->dev, ++ "GPUPERF oc record 5 cleared (no overclock this epoch) word0=%08x end=gpuperf\n", ++ r5[0]); ++} ++ ++/* perf60 pin >100, the host's own route: power the GPC on at q (R1+0x1000 = 0x10 | q) in the release, ++ * as the GPC1 request the kext and we already make, with q in place of 1; the policy record carries ++ * the seed-1 words (+0x28 = +0x2c = q) and stays "never sampled" (+0 = 0), so the boot sampler only ++ * initialises; it derives +0x34 and its clock entry priv+0x2c4 from Fender (0x1001c838). ++ */ ++/* release8 default */ ++/* perf60 pin >100 (perf-envelope.h g5p_perf_seed): the boot sampler applies the pin before ALIVE. */ ++ ++/* perf60 stage 4 (render-user-pmset.h): firmware-GiB pages for per-context PM records. */ ++/* release8 default */ ++/* perf60 A/B (docs/gpu/experiments/perf60-clock.md): the firmware's per-tick full flush alone. */ ++ ++/* Latched by map, before firmware sees any pointers. Zero preserves round 9. */ ++ ++/* Explicit decoded VA input only: zero is a valid address when valid=true. ++ * Never infer an address or context from revision-dependent e150 bits. ++ */ ++static unsigned long long mmu_fault_va; ++ ++static const bool boot_experiment = 1; ++ ++struct g5p_dma_page { ++ u8 *cpu; ++ dma_addr_t dma; ++}; ++ ++struct g5p_boot { ++ u32 r27_variant, r27_mode_writes, r46_reset, r50_mode, r50_writes; ++ ++ bool table_clean, preinstall_context, compute_boot_vm; ++ u32 compute_slot; ++ ++ bool compute_numerical, user_compute_owned; ++ ++ struct g5p_boot_layout layout; ++ struct g5p_dma_page image[G5P_IMAGE_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page root[G5P_ROOT_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page shared[G5P_SHARED_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page render_meta[2]; ++ /* perf60 stage 4 (render-user-pmset.h): boot_pm_sets per-context PM sets, 3 pages each */ ++ u32 pm_sets; ++ struct g5p_dma_page pm_set[G5P_PMSET_MAX * G5P_PMSET_PAGES]; ++ struct g5p_dma_page catalogue, directory, zero, leaf[4]; ++ struct g5p_dma_page sink_directory, sink_leaf, sink_page; ++ u32 setup_variant; ++ bool sink_ready, sink_dumped; ++ u32 leaf_index[4]; ++ u32 leaves, pages, mapped, fifo, events, alive_word; ++ u32 fender_raw[ARRAY_SIZE(g5p_fender_regs)]; ++ u32 rgx_raw[ARRAY_SIZE(g5p_rgx_regs)]; ++ const char *snapshot_stage; ++ bool snapshot_valid, rgx_valid; ++ u32 rgx_count; ++ int fifo_error; ++ int irq; ++ atomic_t irqs; ++ spinlock_t event_lock; ++ void __iomem *swirq; ++ bool maps_ready, armed, copied, released, alive, failed, irq_ready; ++ bool release_ack, release_ack_attempted; ++ bool fault_va_valid; ++ u64 fault_va; ++ bool work_ready, work_submitted, work_complete; ++ u32 work_target, control_target; ++ u32 dc_variant, work_variant; ++ u32 r72_meta6c8; ++ u32 meta71c; ++ u32 fw_power_words; ++ u32 gpu_perf; ++ u32 perf_seed; /* boot_gpu_perf_seed: the seeded pstate q, 0 = not seeded */ ++ u32 perf_seed_mode; /* boot_gpu_perf_seed as latched (1..5) */ ++ u32 perf_poweron; /* boot_gpu_perf_poweron: the pstate q the release requests, 0 = 1 as before */ ++ u32 gpu_oc_mhz; /* boot_gpu_oc_mhz as applied (state 5), 0 = no overclock */ ++ /* record 5's voltage codes */ ++ u64 seed_stamp, ++ seed_gpc1; /* seed 3/4: the stamp and the GPC1 write (arch timer) */ ++ u32 meta6d4; ++ u32 r78_ctrl0; ++ u32 r73_aux_drain; ++ u32 r73_aux_entries[6]; ++ u32 r73_aux_hash[6]; ++ u32 r73_aux_first[6][3]; ++ u32 r73_aux_last[6][3]; ++ struct g5p_r73b *r73b; ++ bool irq_capped; ++ bool dc_observe; ++ ++ bool control_ready, control_submitted, control_complete; ++ bool irq_tested, stopped, parked; ++ bool control_timed_out, work_timed_out, event_failed; ++ bool context_submitted, context_complete, context_timed_out; ++ u32 context_target; ++ /* The epoch's last recovery packet (render-user-recover.h), under event_lock. */ ++ struct g5p_fw_recovery fw_recovery; ++ u32 fw_recoveries; ++ u32 event_n[10]; /* events drained this epoch by type (9: 9 and above) */ ++}; ++ ++#include "boot-r73b.inc" ++ ++/* The fault-status subset is read-only, including while META reset is held. ++ * Never access R0 with GPC off/busy; no slave-port transaction is involved. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage); ++ ++static int g5p_boot_alloc_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ p->cpu = dmam_alloc_coherent(g->dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ /* There is no GPU bus translation in the recovered GART path. */ ++ if ((p->dma & (G5P_GPU_PAGE - 1)) || p->dma >= BIT_ULL(40) || ++ dma_to_phys(g->dev, p->dma) != p->dma) { ++ dev_err(g->dev, ++ "unsupported DMA translation/alignment at %pad\n", ++ &p->dma); ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ g->boot->pages++; ++ return 0; ++} ++ ++/* Deliberately map one16KiB page only, not the iOS all-address fallback. ++ * Own distinct directory/leaf/backing; do not alias firmware tables or e160. ++ * No live table surgery: this helper runs only during first map construction. ++ */ ++static int g5p_boot_sink_init(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ; ++ ; ++ if (!b->catalogue.cpu || b->maps_ready || b->armed || b->released || ++ b->sink_ready || b->sink_directory.cpu || b->sink_leaf.cpu || ++ b->sink_page.cpu) ++ return -EPERM; ++ if (g5p_le32(b->catalogue.cpu)) ++ return -EEXIST; ++ ret = g5p_boot_alloc_page(g, &b->sink_directory); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_leaf); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_page); ++ if (ret) ++ return ret; ++ /* Round12 iOS optional dummy-tree leaf flags0: valid writable PA|1. ++ * Only cat[0]/dir[0]/leaf[0] are installed; every other hole stays absent. ++ */ ++ g5p_put64(b->sink_leaf.cpu, g5p_leaf_entry(b->sink_page.dma, 0)); ++ g5p_put64(b->sink_directory.cpu, g5p_directory_entry(b->sink_leaf.dma)); ++ dma_wmb(); ++ g5p_put32(b->catalogue.cpu, g5p_catalogue_entry(b->sink_directory.dma)); ++ dma_wmb(); ++ b->mapped++; ++ b->sink_ready = true; ++ g5p_mmu_record( ++ g, ++ "MMU sink-installed VA=0 bytes=4000 context=0 leaf_flags=0; pre-release " ++ "containment experiment\n"); ++ g5p_blob_printf( ++ g, ++ "MMU sink PC=%pad cat0=%08x dir=%pad dir0=%016llx leaf=%pad pte0=%016llx " ++ "backing=%pad; initially zero\n", ++ &b->catalogue.dma, g5p_catalogue_entry(b->sink_directory.dma), ++ &b->sink_directory.dma, g5p_directory_entry(b->sink_leaf.dma), ++ &b->sink_leaf.dma, g5p_leaf_entry(b->sink_page.dma, 0), ++ &b->sink_page.dma); ++ return 0; ++} ++ ++static int g5p_boot_map_page(struct g5p_diag *g, u64 va, u64 pa, u32 attributes) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ u32 ti = (va >> 14) & 127, i; ++ int ret; ++ ++ if ((va | pa) & (G5P_GPU_PAGE - 1)) ++ return -EINVAL; ++ /* This experiment only owns the recovered firmware heap's catalogue slot. */ ++ if (ci != ((G5P_IMAGE_VA >> 30) & 1023)) ++ return -ERANGE; ++ for (i = 0; i < b->leaves; i++) ++ if (b->leaf_index[i] == di) ++ break; ++ if (i == b->leaves) { ++ if (i == ARRAY_SIZE(b->leaf)) ++ return -ENOSPC; ++ ret = g5p_boot_alloc_page(g, &b->leaf[i]); ++ if (ret) ++ return ret; ++ b->leaf_index[i] = di; ++ g5p_put64(b->directory.cpu + di * 8, ++ g5p_directory_entry(b->leaf[i].dma)); ++ b->leaves++; ++ } ++ if (g5p_le32(b->leaf[i].cpu + ti * 8)) ++ return -EEXIST; ++ g5p_put64(b->leaf[i].cpu + ti * 8, g5p_leaf_entry(pa, attributes)); ++ g5p_put32(b->catalogue.cpu + ci * 4, ++ g5p_catalogue_entry(b->directory.dma)); ++ b->mapped++; ++ return 0; ++} ++ ++#include "mmu-attrs.inc" ++#include "mmu-tables.inc" ++ ++static int g5p_boot_region(struct g5p_diag *g, struct g5p_dma_page *p, u32 size, ++ u64 va) ++{ ++ u32 offset; ++ u32 attributes = va == G5P_IMAGE_VA ? 0x10 : 0x14; ++ int ret; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) { ++ ret = g5p_boot_alloc_page(g, p); ++ if (ret) ++ return ret; ++ ret = g5p_boot_map_page(g, va + offset, p->dma, attributes); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void g5p_boot_copy(struct g5p_dma_page *p, const u8 *data, u32 size) ++{ ++ u32 offset; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) ++ memcpy(p->cpu, data + offset, G5P_GPU_PAGE); ++} ++ ++static u8 *g5p_boot_shared(struct g5p_boot *b, u32 slot) ++{ ++ u32 off = b->layout.offset[slot]; ++ ++ return b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static int g5p_boot_maps(struct g5p_diag *g) ++{ ++ struct g5p_boot *b; ++ u8 *root = NULL, *shared = NULL; ++ bool m71c_wrote; ++ u32 i, m71c, seed; ++ int ret; ++ ++ if (!g->loaded || !g->id_checked || PAGE_SIZE != G5P_GPU_PAGE) ++ return -EPERM; ++ if (g->core_id != 0x011b0602 || g->core_revision != 0x00090f00) ++ return -ENODEV; ++ if (g->boot) ++ return g->boot->maps_ready ? 0 : -EIO; ++ b = devm_kzalloc(g->dev, sizeof(*b), GFP_KERNEL); ++ if (!b) ++ return -ENOMEM; ++ g->boot = b; ++ b->r50_mode = (0); ++ ; ++ b->r27_variant = (0); ++ ; ++ b->r46_reset = (0); ++ ; ++ b->r72_meta6c8 = (1); ++ b->r78_ctrl0 = (0); ++ if (b->r78_ctrl0 && b->r78_ctrl0 != 0x5f) { ++ b->failed = true; ++ return -EINVAL; ++ }; ++ b->r73_aux_drain = (0); ++ if (b->r73_aux_drain > 1 || (b->r73_aux_drain && !1)) { ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->release_ack = (0); ++ b->table_clean = (1); ++ b->compute_boot_vm = (0); ++ b->compute_slot = 0 ? 0 : (7); ++ b->preinstall_context = (0) || 0; ++ ; ++ b->setup_variant = (1); ++ ; ++ b->fault_va = READ_ONCE(mmu_fault_va); ++ b->fault_va_valid = (0); ++ ; ++ b->dc_variant = (7); ++ b->work_variant = (0); ++ b->dc_observe = (0); ++ ; ++ /* iOS: meta+0x71c = 340, the pstate-1 frequency in MHz. */ ++ b->meta71c = (340); ++ m71c = g5p_m71c_check(b->meta71c, 7); ++ if (m71c != G5P_M71C_OK) { ++ dev_err(g->dev, ++ "META71C refused requested=%u variant=%u why=%s before_release=1 end=meta71c\n", ++ b->meta71c, b->dc_variant, g5p_m71c_whys[m71c]); ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->fw_power_words = (0); ++ b->gpu_perf = 400; ++ b->meta6d4 = false; ++ seed = (0); ++ b->gpu_oc_mhz = 0; ++ ; ++ b->perf_poweron = ++ (1) ? (0 ? G5P_OC_STATE : g5p_perf_seed_q(b->gpu_perf)) : 0; ++ b->perf_seed = seed && !b->perf_poweron ? g5p_perf_seed_q(b->gpu_perf) : ++ 0; ++ b->perf_seed_mode = b->perf_seed ? seed : 0; ++ if (!(g5p_perf_valid(b->gpu_perf) || ++ (0 && b->gpu_perf == G5P_OC_PIN)) || ++ (b->gpu_perf && b->fw_power_words) || seed > 5 || (1) > 1 || ++ (b->gpu_perf > 100 && !(1)) || b->meta6d4 > 1 || ++ (b->meta6d4 && !b->gpu_perf)) { ++ dev_err(g->dev, ++ "GPUPERF refused pin=%u power_words=%u gpc_any_pstate=%u meta6d4=%u seed=%u " ++ "before_release=1 end=gpuperf\n", ++ b->gpu_perf, b->fw_power_words, (1), b->meta6d4, seed); ++ b->failed = true; ++ return -EINVAL; ++ } ++ if (b->fw_power_words > G5P_L123_FULL || (0) > G5P_L123_TICK_MASK) { ++ dev_err(g->dev, ++ "L123 refused power_words=%u tick_mask=%u before_release=1 end=l123\n", ++ b->fw_power_words, (0)); ++ b->failed = true; ++ return -EINVAL; ++ } ++ ret = g5p_r73b_latch(g); ++ if (ret) { ++ b->failed = true; ++ return ret; ++ }; ++ dev_info( ++ g->dev, ++ "DC matrix latched variant=%u observe=%u (changes require reboot)\n", ++ b->dc_variant, b->dc_observe); ++ dev_info( ++ g->dev, ++ "COMPUTE VM latched slot=%u boot_vm=%u preinstall=%u (changes require reboot)\n", ++ b->compute_slot, b->compute_boot_vm, b->preinstall_context); ++ spin_lock_init(&b->event_lock); ++ atomic_set(&b->irqs, 0); ++ ret = g5p_plan_layout(&b->layout); ++ if (ret) ++ goto out; ++ root = kvzalloc(G5P_ROOT_SIZE, GFP_KERNEL); ++ shared = kvzalloc(G5P_SHARED_SIZE, GFP_KERNEL); ++ if (!root || !shared) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_boot_alloc_page(g, &b->catalogue); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->directory); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->zero); ++ if (ret) ++ goto out; ++ ret = g5p_boot_sink_init(g); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->image, G5P_IMAGE_SIZE, G5P_IMAGE_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->root, G5P_ROOT_SIZE, G5P_ROOT_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->shared, G5P_SHARED_SIZE, G5P_SHARED_VA); ++ if (ret) ++ goto out; ++ /* R22 metadata mappings are installed before release, only behind its ++ * default-off gate. Compute/boot defaults allocate no additional pages. ++ */ ++ { ++ ret = g5p_boot_region(g, b->render_meta, 2 * G5P_GPU_PAGE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE); ++ if (ret) ++ goto out; ++ b->pm_sets = min((8), G5P_PMSET_MAX); ++ if (b->pm_sets) { ++ ret = g5p_boot_region(g, b->pm_set, ++ b->pm_sets * G5P_PMSET_PAGES * ++ G5P_GPU_PAGE, ++ G5P_PMSET_VA); ++ if (ret) ++ goto out; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-sets boot sets=%u va=%#llx..%#llx before_release=1 end=v3\n", ++ b->pm_sets, G5P_PMSET_VA, ++ g5p_pmset_va(b->pm_sets, 0)); ++ } ++ } ++ ret = 0; ++ if (ret) ++ goto out; ++ /* c80758: Fender (2 pages), AIC timer, AIC software interrupt. */ ++ for (i = 0; i < 4; i++) { ++ u64 pa = i < 2 ? 0x201e00000ULL + i * G5P_GPU_PAGE : ++ i == 2 ? 0x20e100000ULL : ++ 0x20e104000ULL; ++ ++ ret = g5p_boot_map_page(g, G5P_PIO_VA + i * G5P_GPU_PAGE, pa, ++ 0x18); ++ if (ret) ++ goto out; ++ } ++ g5p_make_boot_config(&g->image, &b->layout); ++ g5p_make_init(&b->layout, root, shared, &g->image, b->catalogue.dma, ++ arch_timer_read_counter()); ++ g5p_dc_init_variant(&b->layout, root, shared, 7); ++ /* meta+0x784 is the abstime OFFSET the firmware adds to its timebase (0x100221e8 = PIO-1 ++ * counter + priv+688, loaded from meta+0x784 at 0x1001ff58); iOS writes ++ * ml_get_abstime_offset(). Linux's timebase is the counter itself: 0. (make_init's absolute ++ * stamp put the firmware's clock ~uptime ahead of every host stamp in policy+0x8f8.) ++ */ ++ g5p_put64(g5p_object(&b->layout, root, shared, 0x4d) + 0x784, 0); ++ if (g5p_boot_ctrl0(b)) ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x5e), ++ g5p_boot_ctrl0(b)); ++ dev_info( ++ g->dev, ++ "R78 ctrl0 requested=%08x installed=%08x before_release=1 end=r78\n", ++ g5p_boot_ctrl0(b), ++ g5p_le32(g5p_object(&b->layout, root, shared, 0x5e))); ++ ; ++ ; ++ { ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x4d) + 0x6c8, ++ 1); ++ }; ++ m71c = g5p_m71c_install(&b->layout, root, shared, b->meta71c, ++ &m71c_wrote); ++ dev_info( ++ g->dev, ++ "META71C requested=%u installed=%08x variant=%u wrote=%u before_release=1 " ++ "end=meta71c\n", ++ b->meta71c, m71c, b->dc_variant, m71c_wrote); ++ if (m71c != b->meta71c) { ++ ret = -EIO; ++ goto out; ++ } ++ if (b->fw_power_words) ++ g5p_l123_arm_a(&b->layout, root, shared, b->fw_power_words); ++ if (b->gpu_perf) { ++ u8 *pol = g5p_object(&b->layout, root, shared, G5P_PERF_POLICY); ++ u8 *mt = g5p_object(&b->layout, root, shared, G5P_PERF_META); ++ u8 *ct = g5p_object(&b->layout, root, shared, G5P_PERF_CTL); ++ ++ g5p_perf_install(&b->layout, root, shared, b->gpu_perf); ++ { ++ g5p_oc_clear(g); ++ } ++ if (b->meta6d4) ++ g5p_put32(mt + 0x6d4, 1); ++ if (b->perf_poweron) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_poweron, ++ false); ++ dev_info( ++ g->dev, ++ "GPUPERF poweron q=%u policy28=%u/%u policy0=%u policy4=%u seed=%u ignored " ++ "before_release=1 end=gpuperf\n", ++ b->perf_poweron, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), seed); ++ } else { ++ dev_info( ++ g->dev, ++ "GPUPERF poweron ignored pin=%u (pins 200..400 only) before_release=1 end=gpuperf\n", ++ b->gpu_perf); ++ } ++ if (seed && !b->perf_seed && !b->perf_poweron) ++ dev_info( ++ g->dev, ++ "GPUPERF seed ignored pin=%u (seeds pins 200..400 only) before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf); ++ if (b->perf_seed) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_seed, ++ seed >= 2); ++ /* seed 5: meta+0x784 is the abstime OFFSET the firmware adds to its timebase ++ * (0x100221e8 = PIO-1 counter + priv+688, loaded from meta+0x784 at 0x1001ff58); ++ * iOS writes ml_get_abstime_offset(), make_init the absolute counter, so every ++ * policy+0x8f8 stamp lay ~counter in the firmware's past. Linux has no offset. ++ */ ++ if (seed == 5) ++ g5p_put64(mt + 0x784, 0); ++ dev_info( ++ g->dev, ++ "GPUPERF seed mode=%u q=%u policy28=%u/%u policy30=%u policy8=%u policy0=%u " ++ "policy4=%u meta784=%llx before_release=1 end=gpuperf\n", ++ seed, b->perf_seed, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0x30), ++ g5p_le32(pol + 8), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), ++ (u64)g5p_le32(mt + 0x784) | ++ (u64)g5p_le32(mt + 0x788) << 32); ++ } ++ dev_info( ++ g->dev, ++ "GPUPERF boot pin=%u table=%u/%u/%u,%u,%u,%u,%u limits=%u/%u/%u ovr=%u/%u " ++ "policy688=%u/%u policy6f0=%u/%u policy4=%u policy8=%u/%u q=%u mhz=%u " ++ "gpc_any_pstate=%u seed=%u/%u policy0=%u policy28=%u/%u before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf, g5p_le32(mt + 0x710), g5p_le32(mt + 0x714), ++ g5p_le32(mt + 0x718), g5p_le32(mt + 0x71c), ++ g5p_le32(mt + 0x720), g5p_le32(mt + 0x724), ++ g5p_le32(mt + 0x728), g5p_le32(ct + 0x1028), ++ g5p_le32(ct + 0x102c), g5p_le32(ct + 0x1030), ++ g5p_le32(ct + 0x1038), g5p_le32(ct + 0x1040), ++ g5p_le32(pol + 0x688), g5p_le32(pol + 0x68c), ++ g5p_le32(pol + 0x6f0), g5p_le32(pol + 0x6f4), ++ g5p_le32(pol + 4), g5p_le32(pol + 8), ++ g5p_le32(pol + 0xc), b->gpu_perf / 100, ++ b->gpu_perf / 100 < ARRAY_SIZE(g5p_perf_mhz) ? ++ g5p_perf_mhz[b->gpu_perf / 100] : ++ b->gpu_oc_mhz, ++ (1), seed, b->perf_seed, g5p_le32(pol + 0), ++ g5p_le32(pol + 0x28), g5p_le32(pol + 0x2c)); ++ } ++ g5p_put32(g5p_object(&b->layout, root, shared, G5P_L123_CTL) + 4, (0)); ++ { ++ u8 *meta = g5p_object(&b->layout, root, shared, G5P_L123_META); ++ ++ dev_info( ++ g->dev, ++ "L123 boot power_words=%u meta6d4=%u table=%u/%u/%u,%u,%u,%u,%u policy4=%u " ++ "ctl04=%u limits=%u/%u/%u before_release=1 end=l123\n", ++ b->fw_power_words, g5p_le32(meta + 0x6d4), ++ g5p_le32(meta + 0x710), g5p_le32(meta + 0x714), ++ g5p_le32(meta + 0x718), g5p_le32(meta + 0x71c), ++ g5p_le32(meta + 0x720), g5p_le32(meta + 0x724), ++ g5p_le32(meta + 0x728), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_POLICY) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1028), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x102c), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1030)); ++ /* the accessor's licence bound (mmio-access.md §4.2 step 4): meta+0x710 as written */ ++ WRITE_ONCE(g->mmio_max_pstate, ++ g5p_mmio_max_pstate(g5p_le32(meta + 0x710))); ++ }; ++ ; ++ g5p_boot_copy(b->image, g->image.data, G5P_IMAGE_SIZE); ++ g5p_boot_copy(b->root, root, G5P_ROOT_SIZE); ++ g5p_boot_copy(b->shared, shared, G5P_SHARED_SIZE); ++ dma_wmb(); ++ ret = 0; ++ if (ret) ++ goto out; ++ g5p_tables_clean(g, "boot-before-release", NULL, 0); ++ b->maps_ready = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 maps: %u x 16KiB allocations; %u PTEs, %u leaves, PC=%pad\n", ++ b->pages, b->mapped, b->leaves, &b->catalogue.dma); ++ dev_info( ++ g->dev, ++ "G2 init: root=%llx control=%llx event-ctl=%llx event-data=%llx\n", ++ g5p_object_va(&b->layout, 0x50), ++ g5p_object_va(&b->layout, 0x5f), ++ g5p_object_va(&b->layout, 0x78), ++ g5p_object_va(&b->layout, 0x7c)); ++ dev_info( ++ g->dev, ++ "G2 experimental zero policy fields; fault observation only in map\n"); ++out: ++ kvfree(root); ++ kvfree(shared); ++ if (ret) ++ b->failed = true; ++ g5p_ds_hook_maps(g, ret); ++ return ret; ++} ++ ++#include "boot-r74f.inc" ++#include "boot-r74g.inc" ++ ++static u32 g5p_transport_event_limit(void); ++ ++static int g5p_boot_drain(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ dev_err(g->dev, "G2 invalid event indices %u/%u\n", consumer, ++ producer); ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ for (n = 0; n < 16 && consumer != producer; n++) { ++ u32 words[11], off = b->layout.offset[0x74] + consumer * 44, t, ++ n; ++ u8 *src; ++ unsigned int i; ++ bool want; ++ ++ /* Heartbeat116: an R74 epoch bounds ring entries per window ++ * in the IRQ thread instead (heartbeat116.h EVENTS). ++ */ ++ if (!g5p_hb116_on(b) && ++ b->events >= g5p_transport_event_limit()) { ++ ret = -EOVERFLOW; ++ goto out; ++ } ++ /* Entries may straddle DMA pages. */ ++ for (i = 0; i < ARRAY_SIZE(words); i++) { ++ u32 at = off + i * 4; ++ ++ src = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ words[i] = g5p_le32(src); ++ } ++ /* A healthy meta6c8 setup loop can emit thousands of identical ++ * ALIVEs. Keep its first four packets and the exact aggregate, ++ * without making printk throughput part of the experiment. ++ * R74 also quiets a moved loop: each new word1 prints once. ++ */ ++ want = !b->r73b || words[0] || b->events < 4 || ++ (words[1] != 0x8000a120 && g5p_r74_word1(b, words[1])); ++ /* Stamp events (type 1, two per job) dump once per epoch, then only count: ++ * at a few jobs per frame they were most of the log and evicted the faults ++ * (release8 bench, 306 k lines). Every other type dumps as before. ++ */ ++ t = min_t(u32, words[0], ARRAY_SIZE(b->event_n) - 1); ++ n = ++b->event_n[t]; ++ if (words[0] == 1 && n > 1 && !(0)) { ++ if (is_power_of_2(n)) ++ g5p_job_printk( ++ g->dev, ++ "G2 events type=1 n=%u: stamp events counted, dumped once (boot_event_dump 1: each)\n", ++ n); ++ want = false; ++ } ++ /* Alive (0) and stamp (1) events are routine, one or more per job: job_log. Every other ++ * type (recovery, page request, channel error, fault) prints as before. ++ */ ++ if (want && words[0] > 1) { ++ dev_info(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ print_hex_dump(KERN_INFO, ++ "G2 event raw: ", DUMP_PREFIX_NONE, 16, ++ 4, words, sizeof(words), false); ++ } else if (want) { ++ g5p_job_printk(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ g5p_job_hex("G2 event raw: ", words, sizeof(words)); ++ }; ++ if (g->event_notify) ++ g->event_notify(g->event_cookie, words); ++ /* c99788 routes these to recovery/page-request/channel-error paths. */ ++ if (words[0] != 0 && words[0] != 1) { ++ WRITE_ONCE(b->failed, true); ++ switch (words[0]) { ++ case G5P_FW_EVENT_RECOVERY: { ++ /* The kext's fields (render-user-recover.h). */ ++ struct g5p_fw_recovery rec = ++ g5p_fw_recovery_decode(words); ++ ++ b->fw_recovery = rec; ++ b->fw_recoveries++; ++ dev_err(g->dev, ++ "G2 recovery event slot=%u flag=%u code=%08x w18=%08x w20=%08x w24=%08x n=%u: " ++ "the firmware stopped; iOS restarts the GPU\n", ++ rec.slot, rec.flag, rec.code, rec.w18, ++ rec.w20, rec.w24, b->fw_recoveries); ++ break; ++ } ++ case 7: ++ dev_err(g->dev, ++ "G2 page-request event payload=%08x/%08x/%08x; address encoding unresolved\n", ++ words[1], words[2], words[3]); ++ break; ++ case 8: ++ dev_err(g->dev, ++ "G2 channel-error event kind=%u engine=%u token=%08x stamp=%08x; no " ++ "acknowledgment command guessed\n", ++ words[1], words[2], words[3], words[4]); ++ break; ++ } ++ dev_err(g->dev, ++ "G2 unsupported/fault event=%u; epoch failed, entry retained (the render device " ++ "recovers it if it may)\n", ++ words[0]); ++ ret = -EIO; ++ break; ++ } ++ if (words[0] == 0 && READ_ONCE(b->released) && ++ !READ_ONCE(b->stopped)) { ++ b->alive_word = words[1]; ++ WRITE_ONCE(b->alive, true); ++ } ++ b->events++; ++ consumer = (consumer + 1) & 255; ++ /* c99788 uses a full barrier: finish slot reads before returning it. */ ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ /* Return the ALIVE slot before ringing work. No logging or ++ * other MMIO may be inserted between these two operations. ++ */ ++ if (!words[0] && g5p_r74f_on(b) && ++ READ_ONCE(b->r73b->owner) == G5P_R73B_OWNER_IRQ && ++ READ_ONCE(b->released) && !READ_ONCE(b->stopped)) ++ g5p_r74f_alive(g); ++ } ++out: ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++static irqreturn_t g5p_boot_irq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ struct g5p_boot *b = g->boot; ++ u32 n = atomic_inc_return(&b->irqs); ++ int ret = 0; ++ ++ if (b->r73b) ++ return g5p_r73b_irq(g, irq, n); ++ if (n > g5p_transport_event_limit()) { ++ /* Never leave the software source asserted, even when capped. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (!b->irq_capped) { ++ b->irq_capped = true; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "G2 IRQ174 storm capped; reboot required\n"); ++ } ++ return IRQ_HANDLED; ++ } ++ /* KPF level6G: Apple uses this AIC ACK before all ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (READ_ONCE(b->released)) ++ ret = g5p_boot_drain(g); ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return IRQ_HANDLED; ++} ++ ++static int g5p_boot_irq_setup(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b->irq_ready) ++ return 0; ++ b->swirq = devm_ioremap(g->dev, 0x20e104000ULL, 0x100); ++ if (!b->swirq) ++ return -ENOMEM; ++ b->irq = platform_get_irq(to_platform_device(g->dev), 4); ++ if (b->irq < 0) ++ return b->irq; ++ ret = devm_request_threaded_irq(g->dev, b->irq, NULL, g5p_boot_irq, ++ IRQF_ONESHOT, "g5p-meta", g); ++ if (ret) ++ return ret; ++ b->irq_ready = true; ++ dev_info(g->dev, "G2 IRQ174 registered as Linux IRQ %d\n", b->irq); ++ return 0; ++} ++ ++/* Cache first, then report: sysfs status never issues GPU MMIO. */ ++static void g5p_boot_snapshot(struct g5p_diag *g, const char *stage, ++ enum g5p_readback_state mode) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 i, state; ++ ++ b->snapshot_stage = stage; ++ b->snapshot_valid = false; ++ b->rgx_valid = false; ++ b->rgx_count = 0; ++ g5p_power_snapshot(g, stage); ++ dev_info(g->dev, "G2 snapshot %s: R1 first\n", stage); ++ for (i = 0; i < ARRAY_SIZE(g5p_fender_regs); i++) { ++ b->fender_raw[i] = readl(g->window[1] + g5p_fender_regs[i]); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x\n", g5p_fender_regs[i], ++ b->fender_raw[i]); ++ } ++ b->snapshot_valid = true; ++ state = b->fender_raw[4]; ++ /* GPC=1 did not make META a18 safe while bit32 was held. */ ++ b->rgx_count = g5p_readback_rgx_count(state, mode); ++ if (!b->rgx_count) { ++ dev_info(g->dev, "G2 R0 snapshot skipped: R1+1000=%08x\n", ++ state); ++ return; ++ } ++ for (i = 0; i < b->rgx_count; i++) { ++ dev_info(g->dev, "G2 R R0+%05x\n", g5p_rgx_regs[i]); ++ b->rgx_raw[i] = readl(g->rgx + g5p_rgx_regs[i]); ++ dev_info(g->dev, "G2 raw R0+%05x=%08x\n", g5p_rgx_regs[i], ++ b->rgx_raw[i]); ++ } ++ b->rgx_valid = true; ++} ++ ++static int g5p_fender_write(void *ctx, u32 reg, u32 value) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ ++ /* recovery108: never an R1 access after a FINAL teardown retired the device. */ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 W R1+%05x=%08x\n", reg, value); ++ writel(value, g->window[1] + reg); ++ raw = readl(g->window[1] + reg); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x (write fence)\n", reg, raw); ++ return 0; ++} ++ ++static int g5p_fender_poll(void *ctx, u32 reg, u32 mask, u32 expected) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 poll R1+%05x mask=%08x expected=%08x (100ms)\n", ++ reg, mask, expected); ++ ret = readl_poll_timeout(g->window[1] + reg, raw, ++ (raw & mask) == expected, 10, 100000); ++ dev_info(g->dev, "G2 poll raw R1+%05x=%08x ret=%d\n", reg, raw, ret); ++ return ret; ++} ++ ++static int g5p_r74e_power_poll(struct g5p_diag *g, u32 expected, u32 timeout_us, ++ u64 *elapsed_ns, u32 *samples) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u64 start = ktime_get_mono_fast_ns(); ++ u32 raw = U32_MAX, n = 0; ++ int ret; ++ ++ for (;;) { ++ if (READ_ONCE(b->alive)) { ++ ret = -EHOSTDOWN; ++ break; ++ } ++ raw = readl(g->window[1] + 0x1000); ++ n++; ++ if ((raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ /* wake112: the reads taken; 0 means raw is still the U32_MAX initializer, not a read. */ ++ *samples = n; ++ r->early_power_raw = raw; ++ return ret; ++} ++ ++/* wake112 (r74e112.h): a failed deferred proof is the cold power abort, absorbing. */ ++static void g5p_r74e_fail(struct g5p_r73b *r, u32 why) ++{ ++ r->early_proof_why = why; ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ atomic_set_release(&r->early_proof, G5P_PX_FAILED); ++} ++ ++/* The once-owner takes the deferred proof right before its one R1+0x1000 read. ++ * false: that read must not happen, and the proof has failed. ++ */ ++static bool g5p_r74e_take(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_px_pre p; ++ u32 why; ++ ++ if (atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, G5P_PX_TAKEN) != ++ G5P_PX_DEFERRED) ++ return false; ++ p = (struct g5p_px_pre){ ++ .gen = r->early_proof_gen, ++ .gen_now = g5p_ds_gen(), ++ .alive = READ_ONCE(b->alive), ++ .released = READ_ONCE(b->released), ++ .stopped = READ_ONCE(b->stopped), ++ .failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ .r1_allowed = g5p_r8_r1_allowed(), ++ }; ++ r->early_proof_gen_now = p.gen_now; ++ why = g5p_px_check(&p); ++ if (why) ++ g5p_r74e_fail(r, why); ++ return !why; ++} ++ ++/* The owner's read decides: only the state field 1 licenses the release. */ ++static void g5p_r74e_settle(struct g5p_r73b *r, u32 raw) ++{ ++ if (!g5p_px_on(raw)) { ++ g5p_r74e_fail(r, G5P_PXW_STATE); ++ return; ++ } ++ WRITE_ONCE(r->early_release_ready, true); ++ atomic_set_release(&r->early_proof, G5P_PX_PROVEN); ++} ++ ++/* No ALIVE wait ends with the proof still owed: close it, no register access. ++ * true: it was still owed. ++ */ ++static bool g5p_r74e_close(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ ++ if (!r || atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, ++ G5P_PX_TAKEN) != G5P_PX_DEFERRED) ++ return false; ++ g5p_r74e_fail(r, G5P_PXW_UNTAKEN); ++ return true; ++} ++ ++/* boot_gpu_perf_seed 3/4: the sampler's last-sample time policy+0x8f8 = now. Seed 3 stamps at the ++ * start of the release (flown 2026-11-03: 38 ms before the stream's end, the window was missed); ++ * seed 4 stamps as the last host write before the one that lets META run, the GPC1 request ++ * R1+0x1000 = 0x11 (writel orders the RAM store before it). ++ */ ++static void g5p_perf_seed_stamp(struct g5p_boot *b, u32 mode) ++{ ++ u32 o; ++ ++ if (mode == 4 ? b->perf_seed_mode < 4 : b->perf_seed_mode != mode) ++ return; /* the GPC1 stamp serves seeds 4 and 5 */ ++ o = b->layout.offset[G5P_PERF_POLICY] + 0x8f8; ++ b->seed_stamp = arch_timer_read_counter(); ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)b->seed_stamp)); ++ o += 4; ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)(b->seed_stamp >> 32))); ++ wmb(); ++} ++ ++static int g5p_r74e_fender_start(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxp latch; ++ bool on_ran = false; ++ u32 samples = ++ 0; /* wake112: the reads of the poll that ran last (the printed raw) */ ++ int ret; ++ ++ r->early_power_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_r74e_power_poll(g, 0, 1000, &r->early_off_ns, ++ &samples); ++ if (!ret) { ++ g5p_perf_seed_stamp(b, 4); ++ ret = g5p_fender_write(g, 0x1000, ++ 0x10 | (b->perf_poweron ?: 1)); ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ if (!ret) { ++ on_ran = true; ++ /* poweron: the first request above pstate 1, a voltage step; a longer bound */ ++ ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, ++ b->perf_poweron ? 10000 : 1800, ++ &r->early_on_ns, &samples); ++ if (b->perf_poweron) ++ dev_emerg( ++ g->dev, ++ "GPUPERF poweron q=%u r1_1000=%08x on_ns=%llu ret=%d end=gpuperf\n", ++ b->perf_poweron, r->early_power_raw, ++ r->early_on_ns, ret); ++ } ++ /* wake112: only ALIVE preempting the on-poll before its first read defers; the ++ * first-ALIVE owner owes the proof. A sampled on-poll (0x11, ...) aborts as before. ++ */ ++ latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples); ++ if (latch == G5P_PXP_DEFER) { ++ r->early_defer_ret = ret; ++ r->early_proof_gen = g5p_ds_gen(); ++ ret = 0; ++ atomic_set_release(&r->early_proof, G5P_PX_DEFERRED); ++ } ++ r->early_power_ret = ret; ++ if (latch == G5P_PXP_ABORT) { ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ } else if (latch == G5P_PXP_READY) { ++ WRITE_ONCE(r->early_release_ready, true); ++ } ++ dev_emerg( ++ g->dev, ++ "R74E power off_ns=%llu on_ns=%llu raw=%08x ret=%d release_ready=%u alive=%u " ++ "sampled=%u end=r74e\n", ++ r->early_off_ns, r->early_on_ns, r->early_power_raw, ret, ++ r->early_release_ready, READ_ONCE(b->alive), samples); ++ if (latch == G5P_PXP_DEFER) ++ dev_emerg( ++ g->dev, ++ "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u end=r74e112\n", ++ r->early_proof_gen, r->early_defer_ret, ++ r->early_power_raw, READ_ONCE(b->alive)); ++ return ret; ++} ++ ++static void g5p_fifo_word(struct g5p_diag *g, u32 word) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 raw; ++ ++ if (b->fifo_error) ++ return; ++ if (b->fifo < 0x10000 || b->fifo >= 0x10200) { ++ b->fifo_error = -EOVERFLOW; ++ return; ++ } ++ writel(word, g->window[1] + b->fifo); ++ raw = readl(g->window[1] + b->fifo); ++ if (raw != word) { ++ dev_err(g->dev, ++ "G2 FIFO verify R1+%05x actual=%08x expected=%08x\n", ++ b->fifo, raw, word); ++ b->fifo_error = -EIO; ++ } ++ b->fifo += 4; ++} ++ ++static void g5p_fifo32(struct g5p_diag *g, u32 reg, u32 value, u32 flags) ++{ ++ dev_info(g->dev, "G2 FIFO R0+%05x=%08x flags=%02x\n", reg, value, ++ flags); ++ g5p_fifo_word(g, (reg & 0xff00ffff) | (flags & 0xff) << 16); ++ g5p_fifo_word(g, value); ++} ++ ++static void g5p_fifo64(struct g5p_diag *g, u32 reg, u64 value) ++{ ++ g5p_fifo32(g, reg + 4, value >> 32, 0); ++ g5p_fifo32(g, reg, value, 0xff); ++} ++ ++static int g5p_boot_fifo_dump(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off, raw, end = min(b->fifo, 0x10200U); ++ ++ if (end <= 0x10000) ++ return -ENODATA; ++ dev_info(g->dev, "G2 FIFO dump R1+10000..%05x (<=128 words)\n", ++ end - 4); ++ for (off = 0x10000; off < end; off += 4) { ++ raw = readl(g->window[1] + off); ++ dev_info(g->dev, "G2 FIFO raw R1+%05x=%08x\n", off, raw); ++ } ++ return 0; ++} ++ ++static u64 g5p_snapshot64(struct g5p_boot *b, u32 reg) ++{ ++ u32 i; ++ ++ for (i = 0; i + 1 < b->rgx_count; i++) ++ if (g5p_rgx_regs[i] == reg && g5p_rgx_regs[i + 1] == reg + 4) ++ return (u64)b->rgx_raw[i + 1] << 32 | b->rgx_raw[i]; ++ return 0; ++} ++ ++static void g5p_release_observe(void *ctx, const char *stage) ++{ ++ ; ++ ; ++} ++ ++/* A diagnostic notification ACK only, never engine repair or a retry loop. ++ * Snapshot first, require the active wrapper, ACK only200 once, fence and ++ * re-sample. A cleared event alone is not evidence the underlying fault healed. ++ */ ++static int g5p_release_ack(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 state, event, after; ++ ++ if (!b->release_ack) ++ return 0; ++ if (!b->released || b->stopped || b->release_ack_attempted) ++ return -EPERM; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_fault_snapshot(g, "release-ack-before"); ++ g5p_mmu_progress(g, "release-ack-event", 0x130); ++ event = readl(g->rgx + 0x130); ++ b->release_ack_attempted = true; ++ if (event & 0x200) { ++ g5p_mmu_progress(g, "release-ack-write", 0x138); ++ writel(0x200, g->rgx + 0x138); ++ /* Ordered same-device read fences the single write. */ ++ g5p_mmu_progress(g, "release-ack-fence", 0x130); ++ after = readl(g->rgx + 0x130); ++ g5p_mmu_record( ++ g, ++ "MMU release-ack mask=00000200 before=%08x after=%08x; notification only\n", ++ event, after); ++ } else { ++ g5p_mmu_record(g, ++ "MMU release-ack skipped=no-event before=%08x\n", ++ event); ++ } ++ g5p_fault_snapshot(g, "release-ack-after"); ++ return 0; ++} ++ ++static int g5p_boot_program_stream(struct g5p_diag *g, bool release) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ u64 reset; ++ int ret; ++ ++ if (b->armed && !release) ++ return -EALREADY; ++ ret = g5p_boot_irq_setup(g); ++ if (ret) ++ return ret; ++ g5p_boot_snapshot(g, release ? "before-release" : "before-arm", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ ; ++ ret = 0; ++ if (ret) ++ goto out; ++ /* Slave-port access remains forbidden for the rest of this boot. */ ++ g->reset_transition = true; ++ /* c9fa10: core request false, GPC 0. Establish an actual state edge; ++ * wake left GPC at 1, so writing 1 again did not prove FIFO execution. ++ * No R0 accesses between this point and the successful active-state poll. ++ */ ++ dev_info(g->dev, "G2 Fender park: core=0, GPC=0\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ goto out; ++ /* c9fc10: core request true before prepareBoot c9feb4. */ ++ dev_info(g->dev, "G2 Fender prepare: core=20, require GPC=0\n"); ++ ret = g5p_fender_hold(&io); ++ if (ret) ++ goto out; ++ dev_info(g->dev, ++ "G2 init stream: master boot, release=%u; FIFO reset\n", ++ release); ++ ret = g5p_fender_write(g, 0x10, 0); ++ if (ret) ++ goto out; ++ /* setMode(3), after Fender prepare cleared +10, before firmware runs. ++ * Both arm and release execute prepare; never reapply after ALIVE. ++ */ ++ ; ++ ; ++ ; ++ g5p_fender_write(g, 0, 0x40000000); ++ ; ++ ; ++ b->fifo = 0x10000; ++ b->fifo_error = 0; ++ ; ++ ; ++ g5p_fifo64(g, 0x100, ++ G5P_BOOT_RESET1 | (0 ? G5P_BOOT_R46_RESET_BITS : 0)); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x00e7fe071ffffc1dULL); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x100000000ULL); ++ ; ++ ; ++ g5p_fifo32(g, 0xbf8, 1, 0xff); ++ ; ++ ; ++ g5p_fifo32(g, 0xe140, 0, 0xff); ++ ; ++ ; ++ g5p_fifo64(g, 0xe148, b->catalogue.dma >> 12); ++ ; ++ ; ++ g5p_fifo64(g, 0xe200, 3); ++ ; ++ ; ++ g5p_fifo64(g, 0xe208, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fifo64(g, 0xe210, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fender_write(g, 0x2010, 0xbf); ++ ; ++ ; ++ g5p_fender_write(g, 0x2014, 0x233); ++ ; ++ ; ++ g5p_fifo64(g, 0xe160, b->zero.dma); ++ ; ++ ; ++ g5p_fifo64(g, 0xbb8, 0); ++ ; ++ ; ++ if (release) { ++ g5p_fifo64(g, 0x100, 0); ++ ; ++ } ++ g5p_fifo_word(g, 0xffff0000); ++ ; ++ ; ++ ret = b->fifo_error; ++ if (ret) ++ goto out; ++ dma_wmb(); ++ dev_info( ++ g->dev, ++ "G2 FIFO stored and verified: %u words (not execution proof)\n", ++ (b->fifo - 0x10000) / 4); ++ if (release) { ++ WRITE_ONCE(b->released, true); ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_FIFO, G5P_R73B_NONE, 0); ++ ret = 0; ++ if (ret) ++ goto out; ++ } ++ /* c9f954: core request false, then GPC 1. */ ++ dev_info(g->dev, "G2 Fender execute: core=0, GPC=1 (from 0)\n"); ++ if (release && !(b->r73b && 1)) { ++ g5p_perf_seed_stamp( ++ b, ++ 4); /* the non-r74e start: as close as this path gets */ ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ ret = release && b->r73b && 1 ? g5p_r74e_fender_start(g) : ++ release ? g5p_fender_start_observed_q(&io, g5p_release_observe, ++ b->perf_poweron ?: 1) : ++ g5p_fender_start(&io); ++ if (release) ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_GPC, G5P_R73B_NONE, ret); ++ if (ret) ++ goto out; ++ /* r74e: after GPC1 is active, wait for ALIVE without optional MMIO. */ ++ if (release && b->r73b && 1) ++ return 0; ++ usleep_range(10, 20); ++ g5p_boot_snapshot(g, release ? "after-release" : "after-arm", ++ release ? G5P_READBACK_WRAPPER_ONLY : ++ G5P_READBACK_RESET_HELD); ++ ; ++ /* Only the fault-status subset above is sampled after release. */ ++ if (release) ++ return g5p_release_ack(g); ++ if (!b->rgx_valid) ++ return -EIO; ++ reset = g5p_snapshot64(b, 0x100); ++ dev_info(g->dev, "G2 verify reset actual=%016llx expected=%016llx\n", ++ reset, 0x100000000ULL); ++ dev_info( ++ g->dev, ++ "G2 expected catalogue=%llx scratch=%llx SLC=3 (not read: state safety)\n", ++ (u64)b->catalogue.dma >> 12, (u64)b->zero.dma); ++ if (reset != 0x100000000ULL) ++ return -EIO; ++ dev_info( ++ g->dev, ++ "G2 reset-held gate verified; remaining FIFO effects await firmware event\n"); ++ b->armed = true; ++ return 0; ++out: ++ /* A slow r74e epoch is scoreable without another unsafe MMIO read. */ ++ if (release && b->r73b && 1 && READ_ONCE(b->r73b->early_power_abort)) ++ return ret; ++ /* Wrapper-only snapshot if GPC remains off/busy. */ ++ g5p_boot_snapshot(g, "transition-or-FIFO-failed", ++ G5P_READBACK_WRAPPER_ONLY); ++ return ret; ++} ++ ++static int g5p_boot_arm(struct g5p_diag *g) ++{ ++ return g5p_boot_program_stream(g, false); ++} ++ ++static int g5p_boot_load(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off; ++ ++ /* c9f1bc/ca43b0: segment and LDR configuration pairs are supplied in ++ * mapped RAM. Polling MSLVCTRL1 in the reset-held state locks this SoC. ++ */ ++ for (off = 0; off < G5P_IMAGE_SIZE; off += G5P_GPU_PAGE) ++ if (memcmp(b->image[off / G5P_GPU_PAGE].cpu, ++ g->image.data + off, G5P_GPU_PAGE)) ++ return -EBADMSG; ++ dma_wmb(); ++ b->copied = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 RAM load verified bytes=%x segment_pairs=13 LDR_pairs=27; no slave-port MMIO\n", ++ G5P_IMAGE_SIZE); ++ return 0; ++} ++ ++static int g5p_boot_release(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u64 seed_ts = 0; ++ int ret; ++ ++ if (b->released) ++ return -EALREADY; ++ /* recovery108: fail closed before the release touches anything. */ ++ ret = g5p_ds_hook_release_gate(g); ++ if (ret) ++ return ret; ++ g5p_r73b_release_entry(g); ++ if (b->r73b && 1) { ++ ret = g5p_r73e_prepare(g); ++ if (ret) ++ return ret; ++ } ++ dev_info(g->dev, ++ "G2 release: META master boot; no GPU job submitted\n"); ++ /* Reproduce the complete original stream, including its final reset release. ++ * Do not assume the experimental arm stage preserves a FIFO read cursor. ++ */ ++ /* boot_gpu_perf_seed 3: the sampler's last-sample time = now, as close to the release as the ++ * host gets (perf-envelope.h); the firmware divides (now - policy+0x8f8) by 24000 * policy+4. ++ */ ++ if (b->perf_seed_mode >= 3) { ++ seed_ts = arch_timer_read_counter(); ++ g5p_perf_seed_stamp(b, 3); ++ } ++ ret = g5p_boot_program_stream(g, true); ++ if (seed_ts) { ++ u64 end = arch_timer_read_counter(), ++ rate = arch_timer_get_rate(); ++ ++ /* always printed on a seed 3/4 boot, before ALIVE is awaited */ ++ dev_emerg( ++ g->dev, ++ "GPUPERF seed mode=%u stamp=%llx release_to_gpc1_us=%llu stamp_to_gpc1_us=%llu " ++ "gpc1_to_stream_end_us=%llu stamp_to_stream_end_us=%llu period_us=16000 " ++ "before_alive=1 end=gpuperf\n", ++ b->perf_seed_mode, b->seed_stamp, ++ b->seed_gpc1 ? ++ div_u64((b->seed_gpc1 - seed_ts) * 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? div_u64((b->seed_gpc1 - b->seed_stamp) * ++ 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? ++ div_u64((end - b->seed_gpc1) * 1000000ULL, ++ rate) : ++ 0, ++ div_u64((end - b->seed_stamp) * 1000000ULL, rate)); ++ } ++ g5p_r73b_release_exit(g, ret); ++ g5p_ds_hook_release(g, ret); ++ if (ret) ++ return ret; ++ if (b->r73b && 1) ++ return g5p_r73c_fast_control(g); ++ return 0; ++} ++ ++static int g5p_boot_events_once(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 3 * HZ; ++ u64 submit_wait_start = 0; ++ u32 pass = 0, submit_wait_passes = 0; ++ int ret; ++ ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_EVENTS, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->alive)) { ++ if (b->r73b && 1) { ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ if (ret == -EAGAIN) ++ submit_wait_start = ++ ktime_get_mono_fast_ns(); ++ while (ret == -EAGAIN && ++ ktime_get_mono_fast_ns() - ++ submit_wait_start < ++ 100ULL * NSEC_PER_MSEC) { ++ usleep_range(50, 100); ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ submit_wait_passes++; ++ } ++ if (submit_wait_passes) ++ dev_info( ++ g->dev, ++ "R73C alive-submit synchronize wait_ns=%llu passes=%u ret=%d submitted=%u end=r73c\n", ++ ktime_get_mono_fast_ns() - ++ submit_wait_start, ++ submit_wait_passes, ret, ++ READ_ONCE( ++ b->control_submitted)); ++ if (ret) ++ return ret; ++ } ++ dev_info( ++ g->dev, ++ "G2 ALIVE type=0 word1=%08x events=%u IRQ174=%d\n", ++ b->alive_word, b->events, ++ atomic_read(&b->irqs)); ++ g5p_r41_bridge_firmware(g); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, ++ "G2 no alive event: events=%u IRQ174=%d; preserve baseline\n", ++ b->events, atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++/* recovery108: every ALIVE wait (release fast path and the events command) is an ALIVE event. */ ++static int g5p_boot_events(struct g5p_diag *g) ++{ ++ int ret = g5p_boot_events_once(g); ++ ++ /* wake112: no ALIVE wait ends with an R74E proof still owed (r74e112.h). */ ++ if (g5p_r74e_close(g)) { ++ dev_emerg( ++ g->dev, ++ "R74E112 close gen=%u why=untaken ret=%d end=r74e112\n", ++ g->boot->r73b->early_proof_gen, ret); ++ if (!ret) ++ ret = -EHOSTDOWN; ++ } ++ g5p_ds_hook_alive(g, ret); ++ return ret; ++} ++ ++static int g5p_boot_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b && b->stopped) ++ return -ESHUTDOWN; ++ if (sysfs_streq(command, "map")) ++ return g5p_boot_maps(g); ++ /* Read-only diagnostics remain available after a latched boot failure. */ ++ if (b && g->powered && g->fender_ready && g->id_checked) { ++ if (sysfs_streq(command, "snapshot")) { ++ g5p_boot_snapshot(g, "explicit", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ return 0; ++ } ++ if (sysfs_streq(command, "fifo")) ++ return g5p_boot_fifo_dump(g); ++ } ++ if (!boot_experiment || !b || !b->maps_ready || b->failed) ++ return -EPERM; ++ if (sysfs_streq(command, "arm")) { ++ ret = g5p_boot_arm(g); ++ } else if (sysfs_streq(command, "load") && b->armed && !b->released && ++ !b->copied) { ++ ret = g5p_boot_load(g); ++ g5p_ds_hook_load(g, ret); ++ } else if (sysfs_streq(command, "release") && b->copied) { ++ ret = g5p_boot_release(g); ++ } else if (sysfs_streq(command, "events") && b->released) { ++ ret = g5p_boot_events(g); ++ g5p_r73b_events_exit(g, ret); ++ } else { ++ return -EPERM; ++ } ++ if (ret && ret != -EALREADY) ++ b->failed = true; ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/compute-user-layout.h b/drivers/gpu/apple-g5p/compute-user-layout.h +new file mode 100644 +index 000000000..78b459832 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/compute-user-layout.h +@@ -0,0 +1,207 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CDM channel and per-job firmware objects. Layout helpers perform no I/O. ++ * The kernel owns the channel, control watermarks, pointer ring, work record, ++ * completion microsequence, timestamp queue, stamps and dependency object. ++ * GPU program instructions are provided by userspace, never embedded here. ++ * ++ * Compute uses the context's TA MMU slot. The scheduler serializes TA and CDM ++ * work across contexts, and a lost CDM job blocks the channel until retirement ++ * or recovery. Stamp identity zero is safe because the TA event latch is armed ++ * only while executing a TA phase; completion also requires coherent stamps. ++ */ ++#ifndef G5P_COMPUTE_USER_LAYOUT_H ++#define G5P_COMPUTE_USER_LAYOUT_H ++ ++#define G5P_CU_ENGINE 2U ++#define G5P_CU_OFFSET (G5P_R22_3D + 0x3000U) /* in the shared region */ ++#define G5P_CU_BYTES 0x1000U ++#define G5P_CU_W 0x400U ++#define G5P_CU_W_BYTES 0x290U /* through the timestamps at W+0x280..0x288 */ ++#define G5P_CU_SKU 0x800U ++#define G5P_CU_SKU_BYTES 0xc0U ++#define G5P_CU_TQ 0x900U ++#define G5P_CU_START 0x940U /* W258: the retirement mirror */ ++#define G5P_CU_END 0x944U /* W25c: finish33's stamp */ ++#define G5P_CU_DEP 0x980U ++#define G5P_CU_DEP_BYTES 0x280U ++#define G5P_CU_IDENTITY 0U ++#define G5P_CU_SEQ_MAX 0x7fffffffU ++ ++/* Firmware configuration W+0x244 and compute record type. */ ++#define G5P_CU_W244 0x00a00004U ++#define G5P_CU_TAG 3U ++ ++/* The validated register images of one job (compute-user-validate.h fills it). */ ++struct g5p_cu_regs { ++ u64 stream; /* W+0x70 -> R0+0x480 */ ++ u64 terminate; /* the stream's terminate word: W+0x214, W+0x23c */ ++ u64 state[4]; /* W+0x1ec..0x204 */ ++ u64 tail; /* W+0x60 -> R0+0x498, W+0x20c */ ++ u32 config[2]; /* W+0x10, W+0x14 -> R0+0x4c0, 0x4c8 */ ++ /* {code | data << 32}: W+0x21c, W+0x22c, W+0x68 -> R0+0x4b8 */ ++ u64 store, resume, term; ++ u32 store1, resume1; /* W+0x224 -> R0+0x4b0, W+0x234 */ ++}; ++ ++/* Job `seq` (1-based) occupies pointer-ring entry (seq - 1) % 16 and publishes seq % 16. */ ++static inline u32 g5p_cu_ring_index(u32 seq) ++{ ++ return (seq - 1) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_cu_endpoint(u32 seq) ++{ ++ return seq % G5P_WORK_SLOTS; ++} ++ ++/* The channel at its epoch start: header, ring capacity, timestamp queue, dependency object ++ * (g5p_compute_dependency()); watermarks, ring and stamps zero. `p` is the channel's ++ * G5P_CU_BYTES, `meta` its META address. ++ */ ++static inline void g5p_cu_channel_init(u8 *p, u32 meta) ++{ ++ u8 *d = p + G5P_CU_DEP; ++ ++ memset(p, 0, G5P_CU_BYTES); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_CU_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_CU_TQ + 4, meta + G5P_CU_TQ); ++ g5p_put64(d + 0x30, 1); ++ g5p_put64(d + 0x48, ~0ULL); ++ g5p_put32(d + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(d + off + 8, 0xf); ++} ++ ++/* The SKU microsequence of job `seq`: start32, timestamp, wait COMPUTE_FINISHED (event 4), ++ * wait (R0+0x4a0 & 1) == 1, timestamp, finish33 (W25c = W260), the backlink, end. ++ * Each sequence reference uses the current job number. ++ */ ++static inline u32 g5p_cu_sku(u8 *sku, u32 meta, u32 global, u32 slot, u32 seq) ++{ ++ u32 w = meta + G5P_CU_W, end = meta + G5P_CU_END; ++ const u32 words[] = { ++ 0x32, ++ w + 0x10, ++ w + 0x60, ++ global + 0x720, ++ meta, ++ slot, ++ 0, ++ 0, ++ 0, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ 0x80000018, ++ w + 0x278, ++ w + 0x280, ++ w + 0x280, ++ seq, ++ 0x101, ++ 0x004a000a, ++ 1, ++ 1, ++ 0x18, ++ w + 0x278, ++ w + 0x280, ++ w + 0x284, ++ seq, ++ 0x33, ++ global + 0x720, ++ meta, ++ slot, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ end, ++ seq, ++ (u32)-0x6c, ++ 0x40000017, ++ }; ++ ++ memset(sku, 0, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(words); i++) ++ g5p_put32(sku + i * 4, words[i]); ++ return ARRAY_SIZE(words) * 4; ++} ++ ++/* Every per-job object of job `seq` on slot `slot`: W, SKU and its pointer-ring entry. The ++ * channel header, the dependency object and the timestamp queue stay as initialised. ++ */ ++static inline void g5p_cu_job(u8 *p, u32 meta, u32 global, u32 slot, u32 seq, ++ const struct g5p_cu_regs *k) ++{ ++ u8 *w = p + G5P_CU_W; ++ ++ memset(w, 0, G5P_CU_W_BYTES); ++ g5p_put32(w, G5P_CU_TAG); ++ g5p_put32(w + 8, slot); ++ g5p_put32(w + 0xc, meta + G5P_CU_DEP); ++ g5p_put32(w + 0x10, k->config[0]); ++ g5p_put32(w + 0x14, k->config[1]); ++ g5p_put64(w + 0x60, k->tail); ++ g5p_put64(w + 0x68, k->term); ++ g5p_put64(w + 0x70, k->stream); ++ /* The six selected-resource entries at W+0x78..0xa7 stay zero (disabled). */ ++ g5p_put32(w + 0x1e0, meta + G5P_CU_SKU); ++ g5p_put32(w + 0x1e4, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x1ec + i * 8, k->state[i]); ++ g5p_put64(w + 0x20c, k->tail); ++ g5p_put64(w + 0x214, k->terminate); ++ g5p_put64(w + 0x21c, k->store); ++ g5p_put32(w + 0x224, k->store1); ++ g5p_put64(w + 0x22c, k->resume); ++ g5p_put32(w + 0x234, k->resume1); ++ g5p_put64(w + 0x23c, k->terminate); ++ g5p_put32(w + 0x244, G5P_CU_W244); ++ g5p_put32(w + 0x258, meta + G5P_CU_START); ++ g5p_put32(w + 0x25c, meta + G5P_CU_END); ++ g5p_put32(w + 0x260, seq); ++ g5p_put32(w + 0x264, G5P_CU_IDENTITY); ++ g5p_put32(w + 0x270, seq); ++ g5p_cu_sku(p + G5P_CU_SKU, meta, global, slot, seq); ++ g5p_put32(p + G5P_WORK_RING + g5p_cu_ring_index(seq) * 4, ++ meta + G5P_CU_W); ++} ++ ++/* Retirement of job `seq` (the r19d predicate, RAM only): W still names our stamps and ++ * sequence, finish33's stamp and the retirement mirror both at seq, all five channel ++ * watermarks at its endpoint. ++ */ ++static inline bool g5p_cu_retired(const u8 *p, u32 meta, u32 seq) ++{ ++ const u8 *w = p + G5P_CU_W; ++ u32 endpoint = g5p_cu_endpoint(seq); ++ ++ if (g5p_le32(w + 0x258) != meta + G5P_CU_START || ++ g5p_le32(w + 0x25c) != meta + G5P_CU_END || ++ g5p_le32(w + 0x260) != seq || g5p_le32(p + G5P_CU_START) != seq || ++ g5p_le32(p + G5P_CU_END) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* The stamps are the previous job's or ours, and no watermark runs past our endpoint's ++ * predecessor: anything else is a protocol error (another writer, or the wrong channel). ++ */ ++static inline bool g5p_cu_stamp_sane(const u8 *p, u32 seq) ++{ ++ u32 start = g5p_le32(p + G5P_CU_START), end = g5p_le32(p + G5P_CU_END); ++ ++ return (start == seq - 1 || start == seq) && ++ (end == seq - 1 || end == seq); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +new file mode 100644 +index 000000000..709efed16 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -0,0 +1,567 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared private bring-up implementation for mutually exclusive frontends. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++/* perf60 clock: which GPC states license an R0 access (a bell, a TLB pass, a register sample). */ ++/* release8 default */ ++ ++/* The GPC is powered and not mid-transition: state 1 only, or (gpc_any_pstate) any nonzero ++ * pstate nibble with no request in flight. A parked GPC (nibble 0) never qualifies. ++ */ ++static inline bool g5p_gpc_ok(u32 raw) ++{ ++ ; ++ ; ++ return !(raw & 0x10) && (raw & 0xf); ++} ++ ++/* The pure headers (readback.h, r74e112.h, render-user-wake.h, dirtystart108.h) judge the GPC state ++ * through G5P_GPC_ON(); their default is the old == 1, for the host tests. The kernel takes the rule. ++ */ ++#define G5P_GPC_ON(raw) g5p_gpc_ok(raw) ++ ++#include "pack.h" ++#include "work-layout.h" ++#include "mmio-licence.h" ++#include "gpu-oc.h" ++#include ++#include "readback.h" ++ ++/* nopatch: the TYPED_V3 boot path (r74/r74f/r74g, meta6c8=1) on Apple's original image. The release8 ++ * default since its device pass (2026-11-03, release8-nopatch a10a8556b9): an explicit firmware patch ++ * request (boot_r73f_case 1..4, or the r75/r79 cases that ride on it) takes the patched path instead, ++ * so the patched fallback is boot_r73f_case 1 alone (g5p_fw_unpatched_want()). ++ */ ++/* release8 default */ ++ ++/* The effective nopatch latch: the parameter, unless a patched image is asked for. */ ++static u32 g5p_fw_unpatched_want(u32 param, u32 r73f, u32 r75, u32 r79) ++{ ++ return param == 1 && !r73f && !r75 && !r79; ++} ++ ++#define G5P_MSLVDATAX 0x0a00 ++#define G5P_MSLVDATAT 0x0a08 ++#define G5P_MSLVCTRL0 0x0a10 ++#define G5P_MSLVCTRL1 0x0a18 ++#define G5P_MSLVIRQSTATUS 0x0ac8 ++#define G5P_SP_READY (BIT(24) | BIT(26)) ++#define G5P_FW_NAME "apple/agx-g5p-c0.ldr" ++ ++struct g5p_boot; ++ ++struct g5p_diag { ++ struct g5p_diagnostics *diagnostics; ++ struct g5p_boot *boot; ++ ++ spinlock_t slot_lock; ++ DECLARE_BITMAP(user_slots, 256); ++ struct device *dev; ++ void __iomem *window[2]; ++ void __iomem *rgx; ++ struct regmap *pmgr; ++ /* Serializes commands and all hardware access. */ ++ struct mutex lock; ++ bool powered; ++ bool loaded; ++ ++ struct g5p_ldr_image image; ++ bool fender_ready; ++ bool id_checked; ++ int last_result; ++ u32 words_written; ++ u32 core_id, core_revision; ++ u32 last_sp_addr; ++ bool reset_transition; ++ void (*event_notify)(void *cookie, const u32 words[11]); ++ void *event_cookie; ++ /* The GPU register licence (mmio.inc, docs/gpu/experiments/mmio-access.md §4). mmio_lock is ++ * a leaf: taken inside event_lock where both are held, never the reverse, nothing inside. ++ * mmio_off forbids R1/R0 through the accessor from before any PMGR gfx gate until Fender is ++ * up again (g5p_mmio_gate(), cleared by g5p_fender_wake_once()). ++ */ ++ spinlock_t mmio_lock; ++ bool mmio_off; ++ u32 mmio_max_pstate; ++ struct g5p_mmio_self *mmio_self; ++}; ++ ++/* Before every PMGR gfx gate (boot R33 off, dirty-start, kexec quiesce and in-boot recovery ++ * cycles): no accessor MMIO from here until g5p_fender_wake_once() brings Fender up again. ++ * Apple clears fender+0xc inside setPlatformPower(0), before its gate (G5). ++ */ ++static void g5p_mmio_gate(struct g5p_diag *g) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = true; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++} ++ ++/* gpu-recovery108: retire predicates (teardown.inc) and dirty-start hooks (dirtystart108.inc). */ ++static bool g5p_r8_r1_allowed(void); ++static bool g5p_r8_is_retired(void); ++static void g5p_ds_hook_probe(struct g5p_diag *g); ++static void g5p_ds_hook_power(struct g5p_diag *g); ++static int g5p_ds_hook_reset(struct g5p_diag *g); ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret); ++static int g5p_ds_hook_release_gate(struct g5p_diag *g); ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret); ++/* wake112: the reset generation of a deferred R74E proof (dirtystart108.inc). */ ++static u32 g5p_ds_gen(void); ++ ++#include "diagnostics.inc" ++ ++/* The GPU ADT clock/power gate124 resolves to ps-regs[4]+index8*8. ++ * Read via its existing syscon, sharing the provider mapping without writes. ++ */ ++static void g5p_power_observer_init(struct g5p_diag *g) ++{ ++ struct device_node *domain, *parent; ++ u32 offset; ++ ++ domain = of_parse_phandle(g->dev->of_node, "power-domains", 0); ++ if (!domain) ++ return; ++ parent = of_get_parent(domain); ++ if (of_device_is_compatible(domain, "apple,s8000-pmgr-pwrstate") && ++ !of_property_read_u32(domain, "reg", &offset) && ++ offset == 0x80340 && parent && ++ of_device_is_compatible(parent, "apple,s8000-pmgr")) { ++ g->pmgr = syscon_node_to_regmap(parent); ++ if (IS_ERR(g->pmgr)) ++ g->pmgr = NULL; ++ } ++ of_node_put(parent); ++ of_node_put(domain); ++} ++ ++/* pmgr-pwrstate.c masks; history flags are sticky, not current clock state. */ ++static void g5p_power_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) { ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx unavailable: provider not resolved\n", ++ stage); ++ return; ++ } ++ dev_info(g->dev, "POWER %s reading PMGR gfx20e080340 gate124\n", stage); ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (ret) { ++ dev_info(g->dev, "POWER %s PMGR gfx read error=%d\n", stage, ++ ret); ++ return; ++ } ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx20e080340=%08x target=%x actual=%x auto=%u auto_state=%x " ++ "min=%x reset=%u parent_off=%u disabled=%u; current outer-domain fields only\n", ++ stage, raw, raw & 0xf, (raw >> 4) & 0xf, (raw >> 28) & 1, ++ (raw >> 24) & 0xf, (raw >> 16) & 0xf, raw >> 31, ++ (raw >> 11) & 1, (raw >> 10) & 1); ++ dev_info( ++ g->dev, ++ "POWER %s PMGR history was_clkgated=%u was_pwrgated=%u; sticky history, not " ++ "current state\n", ++ stage, (raw >> 9) & 1, (raw >> 8) & 1); ++} ++ ++static int g5p_power(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->powered) ++ return 0; ++ dev_info(g->dev, "request gfx power domain\n"); ++ ret = pm_runtime_resume_and_get(g->dev); ++ if (ret < 0) ++ return ret; ++ g->powered = true; ++ dev_info(g->dev, "gfx runtime PM active\n"); ++ g5p_ds_hook_power(g); ++ return 0; ++} ++ ++#include "power-r33.inc" ++#include "power-r41.inc" ++ ++static bool boot_gfx_off_done; ++ ++static int g5p_fender_wake_once(struct g5p_diag *g) ++{ ++ void __iomem *fender = g->window[1]; ++ u32 value; ++ int ret; ++ ++ ret = g5p_power(g); ++ if (ret) ++ return ret; ++ /* recovery108 mode 3: the one bounded incoming cycle, before any GPU MMIO. */ ++ ret = g5p_ds_hook_reset(g); ++ if (ret) ++ return ret; ++ /* R33: once per boot, before the first Fender write (no GPU MMIO yet). */ ++ ; ++ ret = g5p_r41_bridge_prepare(g); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "W R1+10000=ffff0000 (Apple ...6c93fe8)\n"); ++ writel(0xffff0000, fender + 0x10000); ++ dev_info(g->dev, "W R1+00000=40000000 (Apple ...6c93ff0)\n"); ++ writel(0x40000000, fender); ++ dev_info(g->dev, "W R1+01000=00000011 (Apple ...6c93ff8)\n"); ++ writel(0x11, fender + 0x1000); ++ dev_info(g->dev, "poll R1+01000 bit4 clear\n"); ++ ret = readl_poll_timeout(fender + 0x1000, value, !(value & BIT(4)), 10, ++ 100000); ++ dev_info(g->dev, "Fender wake status=%08x result=%d\n", value, ret); ++ if (ret) ++ return ret; ++ usleep_range(10, 20); ++ g->fender_ready = true; ++ { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = false; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++ } ++ return 0; ++} ++ ++static int g5p_fender_wake(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->fender_ready) ++ return 0; ++ ret = g5p_fender_wake_once(g); ++ g5p_ds_hook_wake(g, ret); ++ return ret; ++} ++ ++static int g5p_sp_wait(struct g5p_diag *g) ++{ ++ u32 value; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = readl_poll_timeout(g->rgx + G5P_MSLVCTRL1, value, ++ (value & G5P_SP_READY) == G5P_SP_READY, 10, ++ 100000); ++ if (ret) ++ dev_err(g->dev, "slave port timeout CTRL1=%08x words=%u\n", ++ value, g->words_written); ++ if (ret) ++ dev_err(g->dev, "slave target=%08x\n", g->last_sp_addr); ++ return ret; ++} ++ ++static int g5p_meta_read(struct g5p_diag *g, u32 addr, u32 *out) ++{ ++ int ret; ++ ++ g->last_sp_addr = addr; ++ ret = g5p_sp_wait(g); ++ ++ if (ret) ++ return ret; ++ writel(addr | 1, g->rgx + G5P_MSLVCTRL0); ++ readl(g->rgx + G5P_MSLVCTRL0); ++ ret = g5p_sp_wait(g); ++ if (ret) ++ return ret; ++ *out = readl(g->rgx + G5P_MSLVDATAX); ++ return 0; ++} ++ ++static int g5p_firmware(struct g5p_diag *g, bool prepare) ++{ ++ static const u8 expected[SHA256_DIGEST_SIZE] = { ++ 0x33, 0xae, 0x20, 0xcb, 0xbe, 0x24, 0x98, 0xe4, ++ 0xf1, 0x19, 0xba, 0x4e, 0x9a, 0x55, 0x46, 0x7e, ++ 0x4e, 0x01, 0x92, 0xd4, 0xc8, 0x6d, 0x5b, 0xf1, ++ 0x90, 0x3a, 0xdc, 0xe6, 0x3f, 0xfa, 0x5d, 0xca, ++ }; ++ /* SHA-256 of the packed image of that LDR, unmodified (nopatch's receipt) */ ++ static const u8 g5p_fw_packed_sha[SHA256_DIGEST_SIZE] = { ++ 0x48, 0xea, 0x03, 0x14, 0x08, 0x23, 0x96, 0x23, ++ 0xd8, 0x6f, 0x79, 0xb8, 0xa5, 0xda, 0x55, 0x7e, ++ 0x25, 0x3d, 0xb1, 0x6f, 0xb6, 0x23, 0x47, 0x95, ++ 0xe8, 0xed, 0xba, 0xa5, 0xcd, 0x9b, 0xb9, 0x1a, ++ }; ++ const struct firmware *fw; ++ u8 digest[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ ret = request_firmware_direct(&fw, G5P_FW_NAME, g->dev); ++ if (ret) ++ return ret; ++ sha256(fw->data, fw->size, digest); ++ if (fw->size != 119112 || memcmp(digest, expected, sizeof(expected))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, NULL, NULL); ++ if (ret) ++ goto out; ++ if (prepare && !g->loaded) { ++ g->image.data = kvzalloc(G5P_IMAGE_SIZE, GFP_KERNEL); ++ if (!g->image.data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, g5p_pack_record, ++ &g->image); ++ if (!ret && g->image.config_count != G5P_CONFIG_WRITES) ++ ret = -EINVAL; ++ if (!ret) { ++ sha256(g->image.data, G5P_IMAGE_SIZE, digest); ++ if (memcmp(digest, g5p_fw_packed_sha, sizeof(digest))) ++ ret = -EBADMSG; ++ } ++ if (ret) { ++ kvfree(g->image.data); ++ memset(&g->image, 0, sizeof(g->image)); ++ goto out; ++ } ++ g->loaded = true; ++ dev_info( ++ g->dev, ++ "NOPATCH packed apple_original=1 packed_sha256=%32phN end=nopatch\n", ++ digest); ++ } ++out: ++ release_firmware(fw); ++ return ret; ++} ++ ++static int g5p_sample(struct g5p_diag *g) ++{ ++ u32 value; ++ unsigned int thread, reg; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ if (!g->rgx) ++ return -EINVAL; ++ dev_info(g->dev, "RGX CTRL1=%08x IRQSTATUS=%08x\n", ++ readl(g->rgx + G5P_MSLVCTRL1), ++ readl(g->rgx + G5P_MSLVIRQSTATUS)); ++ for (thread = 0; thread < 2; thread++) { ++ for (reg = 0; reg <= 0x10; reg += 0x10) { ++ ret = g5p_meta_read( ++ g, 0x04800000 + thread * 0x1000 + reg, &value); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "META T%u %s=%08x\n", thread, ++ reg ? "STATUS" : "ENABLE", value); ++ } ++ } ++ return 0; ++} ++ ++static int g5p_read_id(struct g5p_diag *g) ++{ ++ u32 words[2], version; ++ static const u32 offsets[] = { 0x20, 0x18 }; ++ unsigned int i; ++ ++ if (!g->fender_ready || READ_ONCE(g->reset_transition)) ++ return -EPERM; ++ g->id_checked = false; ++ for (i = 0; i < ARRAY_SIZE(offsets); i++) { ++ dev_info(g->dev, "R R0+%05x\n", offsets[i]); ++ words[i] = readl(g->rgx + offsets[i]); ++ dev_info(g->dev, "R0+%05x=%08x\n", offsets[i], words[i]); ++ } ++ g->core_revision = words[0]; ++ g->core_id = words[1]; ++ if ((!words[0] && !words[1]) || words[0] == ~0U || words[1] == ~0U) ++ return -ENODEV; ++ /* Match Apple readChipInfo at ...6ca644c before deeper META accesses. */ ++ version = (words[0] >> 8) & 0xff; ++ if ((words[0] & 0xff0000) != 0x90000 || ++ (version != 7 && version != 10 && version != 13 && version != 15)) ++ return -ENODEV; ++ g->id_checked = true; ++ return 0; ++} ++ ++#include "boot.inc" ++#include "mmio.inc" ++#include "kick.inc" ++#include "lifecycle.inc" ++#include "teardown.inc" ++#include "dirtystart108.inc" ++#include "recover.inc" ++#include "idle-power.inc" ++#include "transport-budget.inc" ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++ ++static ssize_t result_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ return sysfs_emit( ++ buf, ++ "last=%d loaded=%u maps=%u armed=%u copied=%u released=%u alive=%u " ++ "control_complete=%u failed=%u\n", ++ READ_ONCE(g->last_result), READ_ONCE(g->loaded), ++ b && b->maps_ready, b && b->armed, b && b->copied, ++ b && b->released, b && b->alive, b && b->control_complete, ++ b && b->failed); ++} ++static DEVICE_ATTR_RO(result); ++ ++/* Cached raw words from the last explicit/automatic snapshot: no GPU MMIO. */ ++ ++/* One bring-up phase, as the command file and the probe-time bring-up run it. *ran is set ++ * when the phase itself ran (and its result was recorded). ++ */ ++static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) ++{ ++ int ret; ++ ++ *ran = false; ++ if (g5p_r8_is_retired()) ++ return -ESHUTDOWN; ++ guard(mutex) ++ (&g->lock); ++ if (g->boot && (g->boot->stopped || g->boot->user_compute_owned)) ++ return -EBUSY; ++ if (sysfs_streq(buf, "validate") || sysfs_streq(buf, "prepare")) ++ ret = g5p_firmware(g, sysfs_streq(buf, "prepare")); ++ else if (sysfs_streq(buf, "wake")) ++ ret = g5p_fender_wake(g); ++ else if (sysfs_streq(buf, "id")) ++ ret = g5p_read_id(g); ++ else if (sysfs_streq(buf, "sample")) ++ ret = g->id_checked ? g5p_sample(g) : -EPERM; ++ else if (sysfs_streq(buf, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(buf, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(buf, "control-wait")) ++ ret = g5p_control_wait(g); ++ else if (sysfs_streq(buf, "map") || sysfs_streq(buf, "arm") || ++ sysfs_streq(buf, "load") || sysfs_streq(buf, "release") || ++ sysfs_streq(buf, "events")) ++ ret = g5p_boot_command(g, buf); ++ else ++ return -EOPNOTSUPP; ++ g->last_result = ret; ++ *ran = true; ++ return ret; ++} ++ ++static ssize_t command_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ bool ran; ++ int ret; ++ ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (sysfs_streq(buf, "td-kexec")) ++ return g5p_td_command(g, buf) ?: count; ++ ret = g5p_run_phase(g, buf, &ran); ++ if (ran) ++ dev_info(dev, "command %.*s result=%d\n", (int)count, buf, ret); ++ return ret ? ret : count; ++} ++static DEVICE_ATTR_WO(command); ++ ++static struct attribute *g5p_attrs[] = { ++ &dev_attr_command.attr, ++ &dev_attr_result.attr, ++ &dev_attr_recovery_status.attr, ++ &dev_attr_idle_power_status.attr, ++ NULL, ++}; ++static const struct attribute_group g5p_group = { .attrs = g5p_attrs }; ++__ATTRIBUTE_GROUPS(g5p); ++ ++static int g5p_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct g5p_diag *g; ++ unsigned int i; ++ ++ g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL); ++ if (!g) ++ return -ENOMEM; ++ g->dev = dev; ++ if (g5p_diagnostics_init(g)) ++ return -ENOMEM; ++ mutex_init(&g->lock); ++ spin_lock_init(&g->slot_lock); ++ spin_lock_init(&g->mmio_lock); ++ g->mmio_max_pstate = 1; ++ bitmap_set(g->user_slots, 0, 7); ++ for (i = 0; i < 2; i++) { ++ g->window[i] = devm_platform_ioremap_resource(pdev, i); ++ if (IS_ERR(g->window[i])) ++ return PTR_ERR(g->window[i]); ++ } ++ g->rgx = g->window[0]; ++ g5p_power_observer_init(g); ++ platform_set_drvdata(pdev, g); ++ if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40))) ++ return -ENODEV; ++ pm_runtime_enable(dev); ++ g5p_td_attach(g); ++ g5p_ds_hook_probe(g); ++ ; ++ dev_info(dev, "mapped both ADT windows; awaiting explicit command\n"); ++ return 0; ++} ++ ++static void g5p_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ /* With teardown on, keep the PM reference: an outgoing PMGR off is unvalidated. */ ++ bool keep_power; ++ ++ ; ++ keep_power = g5p_td_remove(g); ++ if (g->powered && !keep_power) { ++ g5p_mmio_gate(g); ++ pm_runtime_put_sync(g->dev); ++ } ++ pm_runtime_disable(g->dev); ++ kvfree(g->image.data); ++} ++ +diff --git a/drivers/gpu/apple-g5p/diagnostics.inc b/drivers/gpu/apple-g5p/diagnostics.inc +new file mode 100644 +index 000000000..184ce954b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/diagnostics.inc +@@ -0,0 +1,181 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Cached append-only text. Readers never take the command lock or touch MMIO. ++ * Preserve earliest records on overflow, expose dropped bytes explicitly. ++ */ ++#define G5P_BLOB_SIZE SZ_4M ++#define G5P_MMU_HISTORY_SIZE SZ_32K ++ ++struct g5p_diagnostics { ++ spinlock_t lock; ++ u32 used, history_used; ++ u64 dropped, history_dropped; ++ char stage[40]; ++ u32 next_register; ++ bool reading; ++ char history[G5P_MMU_HISTORY_SIZE]; ++ char blob[G5P_BLOB_SIZE]; ++}; ++ ++static void g5p_diagnostics_free(void *data) ++{ ++ kvfree(data); ++} ++ ++static int g5p_diagnostics_init(struct g5p_diag *g) ++{ ++ g->diagnostics = kvzalloc(sizeof(*g->diagnostics), GFP_KERNEL); ++ if (!g->diagnostics) ++ return -ENOMEM; ++ spin_lock_init(&g->diagnostics->lock); ++ return devm_add_action_or_reset(g->dev, g5p_diagnostics_free, ++ g->diagnostics); ++} ++ ++static void g5p_diagnostic_append(struct g5p_diag *g, bool history, ++ const char *text, size_t len) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ u32 *used = history ? &d->history_used : &d->used; ++ u64 *dropped = history ? &d->history_dropped : &d->dropped; ++ u32 capacity = history ? G5P_MMU_HISTORY_SIZE : G5P_BLOB_SIZE; ++ char *dst = history ? d->history : d->blob; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ /* Keep complete lines; once full, never replace earlier evidence. */ ++ if (*dropped || len > capacity - *used) { ++ *dropped += len; ++ } else { ++ memcpy(dst + *used, text, len); ++ *used += len; ++ } ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_blob_printf(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ char line[512]; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, false, line, len); ++} ++ ++static void g5p_blob_hex(struct g5p_diag *g, const char *name, const u8 *data, ++ u32 len) ++{ ++ for (u32 off = 0; off < len; off += 16) { ++ char text[80]; ++ ++ hex_dump_to_buffer(data + off, min_t(u32, len - off, 16), 16, 4, ++ text, sizeof(text), false); ++ g5p_blob_printf(g, "%s: %04x: %s\n", name, off, text); ++ } ++} ++ ++/* R1+1000 uses synthetic address11000; other offsets denote R0. */ ++static void g5p_mmu_progress(struct g5p_diag *g, const char *stage, u32 next) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ strscpy(d->stage, stage, sizeof(d->stage)); ++ d->next_register = next; ++ d->reading = true; ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_mmu_record(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ dev_info(g->dev, "%s", line); ++} ++ ++/* The routine per-job and per-event lines (a job's publish, wake and success receipt, the firmware's ++ * alive and stamp events). At desktop job rates they are hundreds of lines a second, more than netconsole ++ * can send: its queue grew to 700 MB of unreclaimable slab (release8, 55 min). Faults, losses, ++ * recoveries, terminal latches and every failed job keep their own levels and always print. ++ * job_log 0 dev_dbg (dynamic debug: off unless enabled in dynamic_debug/control) ++ * 1 KERN_DEBUG: in dmesg for the parsers (RENDER-USER job=, RENDER publish), below the ++ * default console loglevel, so not on a console or netconsole ++ * 2 KERN_INFO as before (short netconsole traces only: floods at desktop job rates) ++ */ ++static unsigned int job_log = 0; ++module_param(job_log, uint, 0600); ++MODULE_PARM_DESC( ++ job_log, ++ "default0 routine per-job/per-event lines are dev_dbg; 1 KERN_DEBUG (dmesg, no " ++ "console); 2 KERN_INFO as before (floods netconsole)"); ++ ++#define g5p_job_printk(dev, fmt, ...) \ ++ do { \ ++ switch (READ_ONCE(job_log)) { \ ++ case 0: \ ++ dev_dbg(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ case 1: \ ++ dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ default: \ ++ dev_info(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ } \ ++ } while (0) ++ ++/* g5p_mmu_record for a routine per-job line: the blob keeps it, the console sees it only at job_log 2. */ ++static __printf(2, 3) __maybe_unused ++ void g5p_mmu_record_job(struct g5p_diag *g, const char *fmt, ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ g5p_job_printk(g->dev, "%s", line); ++} ++ ++/* A routine event's raw words at the job_log level. */ ++static __maybe_unused void g5p_job_hex(const char *prefix, const void *buf, ++ size_t len) ++{ ++ switch (READ_ONCE(job_log)) { ++ case 0: ++ print_hex_dump_debug(prefix, DUMP_PREFIX_NONE, 16, 4, buf, len, ++ false); ++ break; ++ case 1: ++ print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ default: ++ print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ } ++} +diff --git a/drivers/gpu/apple-g5p/dirtystart108.h b/drivers/gpu/apple-g5p/dirtystart108.h +new file mode 100644 +index 000000000..fcf7432fa +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.h +@@ -0,0 +1,454 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DIRTYSTART108_H ++#define G5P_DIRTYSTART108_H ++ ++#include "bits-compat.h" ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * G5P incoming dirty-start state machine (series gpu-recovery108, patch 0007). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 4. ++ * ++ * The bootstrap is stepped by userspace (map, load, release, events, ++ * control), so this is an event-driven machine, not one run: the glue feeds ++ * one event per driver step with the step's result and RAM-only inputs, and ++ * asks the machine at two gates (before release, at the DRM claim). It ++ * assumes nothing pristine: the origin of the power state is classified from ++ * the kexec flag, the requested incoming PMGR cycle and the PMGR word, and ++ * every step's own result is evidence. ++ * ++ * PROBE origin inputs: booted from kexec, kexec_gpu_cycle requested ++ * POWER the PMGR gfx word, read once after the PM reference (PMGR read) ++ * RESET the bounded incoming power cycle (g5p_r33_gfx_off(), one attempt ++ * per boot, only in mode 3, only before the first wrapper wake) ++ * WAKE the Fender wake result ++ * MAPS fresh shared state: every ring control word in range (after the wake: ++ * map needs the core id, read through the woken wrapper) ++ * LOAD the firmware image memcmp of the RAM copy (b->copied) ++ * RELEASE the reset-stream/release result, the R74E power result ++ * ALIVE the first ALIVE wait result and, when read, the first-ALIVE ++ * GPC word (r73e); 0x10/0x11 is the C3 closed gate ++ * CLAIM the DRM claim asks; answers from the ALIVE proof ++ * ++ * Modes (parameter dirty_start): ++ * 0 observe: receipts only, both gates open (combined107 behaviour) ++ * 1 gate (default): fail closed on evidence the driver itself produced (a ++ * failed or out-of-order step, a power abort, a GPC word other than ++ * 0x01 at the first ALIVE, a ring out of range); a claim needs ALIVE ++ * proven in the current reset generation. Equal to combined107 on every ++ * bootstrap that succeeds. ++ * 2 strict: also refuse release and claim for an origin that is not cold ++ * and not a witnessed cycle (warm-unproven, warm-uncycled, unknown), ++ * unless a reset succeeded in this generation ++ * 3 strict + reset: as 2, and the glue may spend the one RESET attempt ++ * on such an origin before the wrapper wake (destructive: PMGR write) ++ * ++ * Generations: gen starts at 1 at PROBE and steps once per RESET attempt. A ++ * reset voids every proof of the previous generation; ALIVE proven in an ++ * earlier generation never exposes. gen never decreases. ++ * ++ * Pure policy: no register access, no kernel API except scnprintf(). Each ++ * event emits exactly one DS108 line through the emit callback. ++ */ ++ ++enum g5p_ds_ev { ++ G5P_DSE_PROBE, ++ G5P_DSE_POWER, ++ G5P_DSE_RESET, ++ G5P_DSE_WAKE, ++ G5P_DSE_MAPS, ++ G5P_DSE_LOAD, ++ G5P_DSE_RELEASE, ++ G5P_DSE_ALIVE, ++ G5P_DSE_CLAIM, ++ G5P_DSE_RECOVER, /* an in-boot recovery cycled the gate after a proven epoch */ ++ G5P_DSE_N, ++}; ++ ++static const char *const g5p_ds_ev_name[G5P_DSE_N] = { ++ "probe", "power", "reset", "wake", "maps", ++ "load", "release", "alive", "claim", "recover", ++}; ++ ++enum g5p_ds_origin { ++ G5P_DSO_UNKNOWN, /* not probed, or the PMGR word could not be read */ ++ G5P_DSO_COLD, /* not booted from kexec */ ++ G5P_DSO_WARM_CYCLED, /* kexec, incoming cycle requested and witnessed (was_pwrgated) */ ++ G5P_DSO_WARM_UNPROVEN, /* kexec, cycle requested, no witness in the PMGR word */ ++ G5P_DSO_WARM_UNCYCLED, /* kexec, no incoming cycle requested */ ++ G5P_DSO_N, ++}; ++ ++static const char *const g5p_ds_origin_name[G5P_DSO_N] = { ++ "unknown", "cold", "warm-cycled", "warm-unproven", "warm-uncycled", ++}; ++ ++enum g5p_ds_why { ++ G5P_DSW_NONE, ++ G5P_DSW_PROTOCOL, /* a step out of order (its predecessor not ok in this gen) */ ++ G5P_DSW_WAKE, /* the wrapper wake failed */ ++ G5P_DSW_MAPS, /* map failed or a ring control word out of range */ ++ G5P_DSW_LOAD, /* the RAM image is not the firmware (memcmp) */ ++ G5P_DSW_RELEASE, /* release / reset stream failed */ ++ G5P_DSW_POWER_ABORT, /* the R74E power poll or the first-ALIVE gate aborted */ ++ G5P_DSW_GPC, /* first-ALIVE GPC word not 0x01 (0x10/0x11: C3) */ ++ G5P_DSW_ALIVE, /* no ALIVE, or the ALIVE wait failed */ ++ G5P_DSW_RESET, /* the incoming power cycle failed: power state unknown */ ++ G5P_DSW_N, ++}; ++ ++static const char *const g5p_ds_why_name[G5P_DSW_N] = { ++ "none", "protocol", "wake", "maps", "load", ++ "release", "power-abort", "gpc", "alive", "reset", ++}; ++ ++/* Gate answers (not latched): why a gate is closed right now. */ ++enum g5p_ds_gate { ++ G5P_DSG_OPEN, ++ G5P_DSG_FAILED, /* the fail-closed latch */ ++ G5P_DSG_NOT_PROVEN, /* no ALIVE proof in the current generation */ ++ G5P_DSG_ORIGIN, /* strict: origin neither cold nor a witnessed cycle */ ++ G5P_DSG_STATE, /* the claim snapshot contradicts the proof */ ++ G5P_DSG_N, ++}; ++ ++static const char *const g5p_ds_gate_name[G5P_DSG_N] = { ++ "open", "failed", "not-proven", "origin", "state", ++}; ++ ++#define G5P_DS_MODE_MAX 3 ++#define G5P_DS_GPC_ON 0x01u ++#define G5P_DS_PMGR_PWRGATED \ ++ BIT(8) /* sticky was_pwrgated history bit of 0x80340 */ ++ ++#define G5P_DSF_RINGS_BUSY \ ++ BIT(0) /* a ring not empty at map (data, not a failure) */ ++#define G5P_DSF_ALIVE_LATE \ ++ BIT(1) /* ALIVE first seen at the claim (no ALIVE event) */ ++#define G5P_DSF_RESET_RAN BIT(2) /* a RESET attempt ran this boot */ ++#define G5P_DSF_RESET_OK \ ++ BIT(3) /* ... and succeeded (gen of the success = gen) */ ++/* a second attempt or one after the wake was refused */ ++#define G5P_DSF_RESET_REFUSED BIT(4) ++#define G5P_DSF_DUP BIT(5) /* an event repeated (recorded, not a failure) */ ++#define G5P_DSF_PMGR_UNREAD BIT(6) /* the PMGR word could not be read */ ++ ++/* One flat input record; each event reads only its own fields (RAM or step results). */ ++struct g5p_ds_in { ++ bool kexec; /* PROBE: chosen linux,booted-from-kexec */ ++ bool cycle_requested; /* PROBE: apple_pmgr.kexec_gpu_cycle=1 on the cmdline */ ++ bool pmgr_ok; /* POWER: the PMGR read succeeded */ ++ u32 pmgr_raw; /* POWER: the gfx word */ ++ int ret; /* RESET/WAKE/MAPS/LOAD/RELEASE/ALIVE: the step's return */ ++ u32 rings_bad; /* MAPS: control words > 255 */ ++ u32 rings_busy; /* MAPS: rings not empty */ ++ int power_ret; /* RELEASE: R74E power result */ ++ u32 power_raw; /* RELEASE: its last R1+0x1000 sample */ ++ bool power_abort; /* RELEASE/ALIVE/CLAIM: r73b early_power_abort */ ++ bool gpc_valid; /* ALIVE: the r73e first-ALIVE word was read */ ++ u32 gpc; /* ALIVE: that word */ ++ bool alive, released, failed, ++ control_complete; /* ALIVE/CLAIM snapshot */ ++}; ++ ++struct g5p_ds { ++ u32 gen; /* reset generation, monotonic, 0 before PROBE */ ++ u32 seen, ok; /* event bitmasks, current generation */ ++ u32 flags; ++ enum g5p_ds_origin origin; ++ bool kexec, cycle; /* PROBE inputs, kept for POWER */ ++ u32 pmgr_raw; ++ u32 resets; /* RESET attempts this boot (at most one runs) */ ++ u32 reset_ok_gen; /* gen in which a RESET succeeded, 0 never */ ++ u32 alive_gen; /* gen in which ALIVE was proven, 0 never */ ++ bool failed; /* fail-closed latch (absorbing) */ ++ enum g5p_ds_why why; ++ u32 events, lines, claims, refused; ++ enum g5p_ds_ev last; ++ enum g5p_ds_gate last_gate; ++}; ++ ++static inline bool g5p_ds_ok(const struct g5p_ds *d, enum g5p_ds_ev e) ++{ ++ return d->ok & (1u << e); ++} ++ ++static inline void g5p_ds_fail(struct g5p_ds *d, enum g5p_ds_why why) ++{ ++ if (!d->failed) ++ d->why = why; ++ d->failed = true; ++} ++ ++static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool cycle, ++ bool pmgr_ok, u32 raw) ++{ ++ if (!pmgr_ok) ++ return G5P_DSO_UNKNOWN; ++ if (!kexec) ++ return G5P_DSO_COLD; ++ if (!cycle) ++ return G5P_DSO_WARM_UNCYCLED; ++ return (raw & G5P_DS_PMGR_PWRGATED) ? G5P_DSO_WARM_CYCLED : ++ G5P_DSO_WARM_UNPROVEN; ++} ++ ++/* The origin is trusted (cold, witnessed cycle) or a reset succeeded in this generation. */ ++static inline bool g5p_ds_origin_trusted(const struct g5p_ds *d) ++{ ++ return d->origin == G5P_DSO_COLD || d->origin == G5P_DSO_WARM_CYCLED || ++ (d->reset_ok_gen && d->reset_ok_gen == d->gen); ++} ++ ++/* Should the glue spend the one RESET attempt now (mode 3, before the wake)? */ ++static inline bool g5p_ds_reset_wanted(const struct g5p_ds *d, u32 mode) ++{ ++ return mode >= 3 && !d->failed && !d->resets && ++ !(d->seen & (1u << G5P_DSE_WAKE)) && ++ (d->seen & (1u << G5P_DSE_POWER)) && !g5p_ds_origin_trusted(d); ++} ++ ++/* The release gate (asked by the glue right before the reset stream runs). */ ++static inline enum g5p_ds_gate g5p_ds_release_gate(const struct g5p_ds *d, ++ u32 mode) ++{ ++ if (!mode) ++ return G5P_DSG_OPEN; ++ if (d->failed) ++ return G5P_DSG_FAILED; ++ if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ return G5P_DSG_ORIGIN; ++ return G5P_DSG_OPEN; ++} ++ ++/* The ALIVE proof the claim needs: proven in this generation, nothing failed since. */ ++static inline bool g5p_ds_proven(const struct g5p_ds *d) ++{ ++ return !d->failed && d->alive_gen && d->alive_gen == d->gen && ++ g5p_ds_ok(d, G5P_DSE_RELEASE) && g5p_ds_ok(d, G5P_DSE_LOAD); ++} ++ ++static inline int g5p_ds_line(struct g5p_ds *d, char *buf, size_t n, ++ enum g5p_ds_ev e, int ret, u32 a, u32 b) ++{ ++ d->lines++; ++ return scnprintf(buf, n, ++ "DS108 gen=%u ev=%s ret=%d origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x pmgr=%08x a=%x b=%x gate=%s", ++ d->gen, g5p_ds_ev_name[e], ret, g5p_ds_origin_name[d->origin], ++ d->failed, g5p_ds_why_name[d->why], d->seen, d->ok, d->flags, ++ d->pmgr_raw, a, b, g5p_ds_gate_name[d->last_gate]); ++} ++ ++/* ++ * Feed one event. Returns, for CLAIM, 0 (expose) or -EAGAIN/-EIO (refuse); ++ * for every other event 0. mode is the current dirty_start parameter; at mode 0 ++ * the claim answers 0 whatever the state (the legacy driver decides alone). ++ * line/n receive the event's receipt (always written). ++ */ ++static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in, u32 mode, char *line, ++ size_t n) ++{ ++ u32 bit = 1u << e, a = 0, b = 0; ++ int ret = 0; ++ ++ if (mode > G5P_DS_MODE_MAX) ++ mode = G5P_DS_MODE_MAX; ++ d->events++; ++ d->last = e; ++ if ((d->seen & bit) && e != G5P_DSE_CLAIM) ++ d->flags |= G5P_DSF_DUP; ++ switch (e) { ++ case G5P_DSE_PROBE: ++ /* The only event that may start a generation: probe runs once per boot. */ ++ if (!d->gen) ++ d->gen = 1; ++ d->origin = G5P_DSO_UNKNOWN; ++ a = in->kexec; ++ b = in->cycle_requested; ++ d->kexec = in->kexec; ++ d->cycle = in->cycle_requested; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_POWER: ++ if (!(d->seen & (1u << G5P_DSE_PROBE))) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->origin = g5p_ds_origin_of(d->kexec, d->cycle, in->pmgr_ok, ++ in->pmgr_raw); ++ if (!in->pmgr_ok) ++ d->flags |= G5P_DSF_PMGR_UNREAD; ++ d->pmgr_raw = in->pmgr_ok ? in->pmgr_raw : 0; ++ a = in->pmgr_ok; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RESET: ++ a = in->ret; ++ if (d->resets || (d->seen & (1u << G5P_DSE_WAKE))) { ++ /* The glue must not have run it; record a refused attempt and fail ++ * closed. ++ */ ++ d->flags |= G5P_DSF_RESET_REFUSED; ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->resets++; ++ d->flags |= G5P_DSF_RESET_RAN; ++ /* A reset voids the previous generation's proofs (power epoch boundary). */ ++ d->gen++; ++ /* Unreachable today (a reset runs before the wake, so before every later ++ * step); kept so that a reordered glue cannot carry a proof across it. ++ */ ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->flags |= G5P_DSF_RESET_OK; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_WAKE: ++ d->seen |= bit; ++ if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_WAKE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_MAPS: ++ d->seen |= bit; ++ a = in->rings_bad; ++ b = in->rings_busy; ++ if (in->rings_busy) ++ d->flags |= G5P_DSF_RINGS_BUSY; ++ /* map needs the core id, read through the woken wrapper */ ++ if (!g5p_ds_ok(d, G5P_DSE_WAKE)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret || in->rings_bad) ++ g5p_ds_fail(d, G5P_DSW_MAPS); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_LOAD: ++ d->seen |= bit; ++ if (!g5p_ds_ok(d, G5P_DSE_MAPS)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_LOAD); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RELEASE: ++ d->seen |= bit; ++ a = (u32)in->power_ret; ++ b = in->power_raw; ++ if (!g5p_ds_ok(d, G5P_DSE_LOAD)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->power_abort || in->power_ret) ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_RELEASE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_ALIVE: ++ d->seen |= bit; ++ a = in->gpc_valid; ++ b = in->gpc; ++ if (!g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ } else if (in->power_abort) { ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ } else if (in->gpc_valid && !G5P_GPC_ON(in->gpc)) { ++ g5p_ds_fail(d, G5P_DSW_GPC); ++ } else if (in->ret || !in->alive || in->failed) { ++ g5p_ds_fail(d, G5P_DSW_ALIVE); ++ } else { ++ d->ok |= bit; ++ d->alive_gen = d->gen; ++ } ++ break; ++ case G5P_DSE_RECOVER: ++ /* The render frontend's in-boot recovery (recover.inc): the gate was cycled ++ * after this generation's release, with the firmware frozen. Like a RESET it ++ * starts a generation that voids every proof of the last one, and it is a ++ * witnessed cycle for the origin gate; the bootstrap's steps then run again ++ * and must each succeed in the new generation. Not after a failure, and not ++ * before a release: there is no epoch to recover then. ++ */ ++ a = in->ret; ++ if (d->failed || !g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->gen++; ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ /* The old ALIVE proof is void; the claim's late path may prove the new one. */ ++ d->alive_gen = 0; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_CLAIM: ++ d->claims++; ++ d->seen |= bit; ++ a = in->alive | in->released << 1 | in->failed << 2 | ++ in->control_complete << 3 | in->power_abort << 4; ++ /* ALIVE reached through a path without the ALIVE event: accept the RAM ++ * latch as the proof only after a successful release of this generation. ++ */ ++ if (!d->failed && !d->alive_gen && in->alive && !in->failed && ++ !in->power_abort && g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ d->alive_gen = d->gen; ++ d->ok |= 1u << G5P_DSE_ALIVE; ++ d->flags |= G5P_DSF_ALIVE_LATE; ++ } ++ if (!mode) ++ d->last_gate = G5P_DSG_OPEN; ++ else if (d->failed) ++ d->last_gate = G5P_DSG_FAILED; ++ else if (!g5p_ds_proven(d)) ++ d->last_gate = G5P_DSG_NOT_PROVEN; ++ else if (!in->alive || !in->released || in->failed || ++ in->power_abort || !in->control_complete) ++ d->last_gate = G5P_DSG_STATE; ++ else if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ d->last_gate = G5P_DSG_ORIGIN; ++ else ++ d->last_gate = G5P_DSG_OPEN; ++ if (d->last_gate == G5P_DSG_OPEN) { ++ d->ok |= bit; ++ } else { ++ d->refused++; ++ ret = d->last_gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++ } ++ break; ++ default: ++ break; ++ } ++ g5p_ds_line(d, line, n, e, e == G5P_DSE_CLAIM ? ret : in->ret, a, b); ++ return ret; ++} ++ ++ ++ ++static inline void g5p_ds_init(struct g5p_ds *d) ++{ ++ *d = (struct g5p_ds){ .last = G5P_DSE_N }; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +new file mode 100644 +index 000000000..be023e7e7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -0,0 +1,280 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the incoming dirty-start machine (dirtystart108.h), series ++ * gpu-recovery108 (0007). Design: docs/gpu/experiments/recovery108-driver.md. ++ * ++ * Included from core.inc after teardown.inc, so it may use the teardown ring ++ * read (RAM), g5p_r33_gfx_off() and the retire layer's state. Patch 0008 calls ++ * the hooks from the driver steps; until then nothing here runs. ++ * ++ * Register access added by this file: one PMGR read of the gfx word after the ++ * PM reference (POWER, the same read g5p_power_snapshot() makes), and in mode 3 ++ * only one bounded PMGR cycle through g5p_r33_gfx_off() before the first ++ * wrapper wake. Every other input is RAM or a step's own return value. ++ */ ++#include ++#include "dirtystart108.h" ++ ++static struct g5p_ds g5p_ds_state = { .last = G5P_DSE_N }; ++static DEFINE_SPINLOCK(g5p_ds_lock); ++ ++/* Latched from /chosen at probe, before POWER or the first wrapper wake. */ ++static u32 g5p_ds_boot_mode = 1; ++ ++static u32 g5p_ds_mode(void) ++{ ++ return READ_ONCE(g5p_ds_boot_mode); ++} ++ ++/* One event, one DS108 line (dev_emerg: netconsole at any loglevel). */ ++static int g5p_ds_feed(struct g5p_diag *g, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in) ++{ ++ unsigned long flags; ++ char line[224]; ++ int ret; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ ret = g5p_ds_event(&g5p_ds_state, e, in, g5p_ds_mode(), line, ++ sizeof(line)); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "%s\n", line); ++ return ret; ++} ++ ++/* wake112: the current reset generation, recorded with a deferred R74E proof (RAM, leaf lock). */ ++static u32 g5p_ds_gen(void) ++{ ++ unsigned long flags; ++ u32 gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return gen; ++} ++ ++/* The last apple_pmgr.kexec_gpu_cycle= token of the chosen bootargs (the one the kernel obeys). */ ++static bool g5p_ds_cycle_requested(struct device_node *chosen) ++{ ++ static const char key[] = "apple_pmgr.kexec_gpu_cycle="; ++ const char *args, *p, *last = NULL; ++ char tok[8]; ++ bool v = false; ++ size_t n; ++ ++ if (!chosen || of_property_read_string(chosen, "bootargs", &args)) ++ return false; ++ for (p = strstr(args, key); p; p = strstr(p + 1, key)) ++ if (p == args || p[-1] == ' ') ++ last = p; ++ if (!last) ++ return false; ++ last += sizeof(key) - 1; ++ n = strcspn(last, " "); ++ if (!n || n >= sizeof(tok)) ++ return false; ++ memcpy(tok, last, n); ++ tok[n] = 0; ++ return !kstrtobool(tok, &v) && v; ++} ++ ++static void g5p_ds_hook_probe(struct g5p_diag *g) ++{ ++ struct device_node *chosen = of_find_node_by_path("/chosen"); ++ struct g5p_ds_in in = {}; ++ ++ in.kexec = chosen && ++ of_property_read_bool(chosen, "linux,booted-from-kexec"); ++ in.cycle_requested = g5p_ds_cycle_requested(chosen); ++ of_node_put(chosen); ++ /* Retained warm state needs strict origin checks and the bounded reset. */ ++ WRITE_ONCE(g5p_ds_boot_mode, in.kexec ? 3 : 1); ++ dev_info(g->dev, "DS108 boot mode=%u kexec=%u\n", ++ g5p_ds_mode(), in.kexec); ++ g5p_ds_feed(g, G5P_DSE_PROBE, &in); ++} ++ ++/* After the PM reference is taken: one PMGR read (the gfx word), no write. */ ++static void g5p_ds_hook_power(struct g5p_diag *g) ++{ ++ struct g5p_ds_in in = {}; ++ u32 raw = 0; ++ ++ in.pmgr_ok = g->pmgr && !regmap_read(g->pmgr, 0x80340, &raw); ++ in.pmgr_raw = raw; ++ g5p_ds_feed(g, G5P_DSE_POWER, &in); ++} ++ ++/* Before the first wrapper wake. Mode 3 only, one attempt per boot, through the ++ * r33 helper (its own refusal rules: min/reset/auto or not on; every exit leaves ++ * the gate requested on). Returns the cycle's error; the wake then fails closed. ++ */ ++static int g5p_ds_hook_reset(struct g5p_diag *g) ++{ ++ u32 ms = clamp_t(u32, (10), 1, G5P_R33_GFX_OFF_MAX_MS); ++ struct g5p_ds_in in = {}; ++ unsigned long flags; ++ bool want; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ want = g5p_ds_reset_wanted(&g5p_ds_state, g5p_ds_mode()); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ if (!want) ++ return 0; ++ g5p_mmio_gate(g); ++ in.ret = g5p_r33_gfx_off(g->dev, g->pmgr, ms); ++ g5p_ds_feed(g, G5P_DSE_RESET, &in); ++ boot_gfx_off_done = true; /* one PMGR cycle per boot, whoever runs it */ ++ return in.ret; ++} ++ ++/* The legacy boot_gfx_off_ms cycle (R33) is the same mechanism: record it as the reset. */ ++ ++/* An in-boot recovery's gate cycle (recover.inc): a new generation, or a failed machine. */ ++static int g5p_ds_hook_recover(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ unsigned long flags; ++ bool failed; ++ ++ g5p_ds_feed(g, G5P_DSE_RECOVER, &in); ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ failed = g5p_ds_state.failed; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return failed ? -EIO : 0; ++} ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_WAKE, &in); ++} ++ ++/* Fresh shared state: every ring control word in range (RAM, the teardown ring read). */ ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ u32 i, cons, prod; ++ ++ for (i = 0; !ret && i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ in.rings_bad++; ++ continue; ++ } ++ if (cons > 255 || prod > 255) ++ in.rings_bad++; ++ else if (cons != prod) ++ in.rings_busy++; ++ } ++ g5p_ds_feed(g, G5P_DSE_MAPS, &in); ++} ++ ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_LOAD, &in); ++} ++ ++/* Asked before the release touches anything. -EIO: the fail-closed latch; -EAGAIN: strict origin. */ ++static int g5p_ds_hook_release_gate(struct g5p_diag *g) ++{ ++ enum g5p_ds_gate gate; ++ unsigned long flags; ++ u32 mode = g5p_ds_mode(), gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gate = g5p_ds_release_gate(&g5p_ds_state, mode); ++ g5p_ds_state.last_gate = gate; ++ if (gate != G5P_DSG_OPEN) ++ g5p_ds_state.refused++; ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "DS108 gen=%u gate=release mode=%u answer=%s\n", gen, ++ mode, g5p_ds_gate_name[gate]); ++ if (gate == G5P_DSG_OPEN) ++ return 0; ++ return gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++} ++ ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (r) { ++ in.power_ret = r->early_power_ret; ++ in.power_raw = r->early_power_raw; ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ } ++ g5p_ds_feed(g, G5P_DSE_RELEASE, &in); ++} ++ ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed); ++ } ++ if (r) { ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ /* r73e reads R1+0x1000 at the first ALIVE; 0 means not read. */ ++ in.gpc = READ_ONCE(r->early_state); ++ in.gpc_valid = 1 && in.gpc; ++ } ++ g5p_ds_feed(g, G5P_DSE_ALIVE, &in); ++} ++ ++/* The DRM claim: 0 expose, -EAGAIN not proven / strict origin, -EIO the fail-closed latch. */ ++static int __maybe_unused g5p_ds_hook_claim(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_ds_in in = {}; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ in.control_complete = READ_ONCE(b->control_complete); ++ in.power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ return g5p_ds_feed(g, G5P_DSE_CLAIM, &in); ++} ++ ++/* RAM only: the retire layer and the dirty-start machine, for capture without a log. */ ++static ssize_t recovery_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct g5p_r8 *s = &g5p_r8_state; ++ unsigned long flags; ++ struct g5p_ds d; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ d = g5p_ds_state; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return sysfs_emit( ++ buf, ++ "r8 gen=%u retired=%d mode=%s class=%s verdict=%s flags=%02x admit_closed=%u " ++ "inside=%u detach_queued=%u lines=%u\n" ++ "ds gen=%u mode=%u origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x " ++ "pmgr=%08x alive_gen=%u resets=%u reset_ok_gen=%u events=%u claims=%u refused=%u " ++ "gate=%s\n", ++ READ_ONCE(s->gen), READ_ONCE(s->retired), ++ g5p_td_mode_name[s->mode], g5p_r8_class_name[s->cls], ++ g5p_td_verdict_name[s->verdict], s->flags, ++ g5p_r8_admit_closed(&s->admit), g5p_r8_admit_inside(&s->admit), ++ READ_ONCE(s->detach_queued), s->lines, d.gen, g5p_ds_mode(), ++ g5p_ds_origin_name[d.origin], d.failed, g5p_ds_why_name[d.why], ++ d.seen, d.ok, d.flags, d.pmgr_raw, d.alive_gen, d.resets, ++ d.reset_ok_gen, d.events, d.claims, d.refused, ++ g5p_ds_gate_name[d.last_gate]); ++} ++static DEVICE_ATTR_RO(recovery_status); +diff --git a/drivers/gpu/apple-g5p/drm-validate.h b/drivers/gpu/apple-g5p/drm-validate.h +new file mode 100644 +index 000000000..671e13ab8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/drm-validate.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DRM_VALIDATE_H ++#define G5P_DRM_VALIDATE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_USER_VA_START 0x10000ULL ++#define G5P_USER_VA_END 0x10000000000ULL ++#define G5P_USER_RESERVED_START G5P_IMAGE_VA ++#define G5P_USER_RESERVED_END (G5P_PIO_VA + G5P_PIO_SIZE) ++#define G5P_MAX_BO_SIZE (64ULL << 20) ++/* VA bookkeeping: the sum of mapping sizes in one VM. A GEM bound twice ++ * (GBM alloc, then the EGL import of its own dma-buf) counts twice here. ++ */ ++#define G5P_MAX_VM_BYTES BIT_ULL(30) ++/* Memory behind one VM's mappings, counted once per GEM object. */ ++#define G5P_MAX_PINNED_BYTES (768ULL << 20) ++ ++static inline bool g5p_user_pa_valid(u64 pa) ++{ ++ return pa < 0x10000000000ULL && !(pa & (G5P_GPU_PAGE - 1)); ++} ++ ++static inline bool g5p_user_range_valid(u64 va, u64 size) ++{ ++ if (!size || ((va | size) & (G5P_GPU_PAGE - 1)) || ++ va < G5P_USER_VA_START || va >= G5P_USER_VA_END || ++ size > G5P_USER_VA_END - va) ++ return false; ++ return va >= G5P_USER_RESERVED_END || ++ va + size <= G5P_USER_RESERVED_START; ++} ++ ++static inline bool g5p_bo_range_valid(u64 offset, u64 size, u64 bo_size) ++{ ++ return size && !((offset | size) & (G5P_GPU_PAGE - 1)) && ++ offset < bo_size && size <= bo_size - offset; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/fender.h b/drivers/gpu/apple-g5p/fender.h +new file mode 100644 +index 000000000..cbdf203c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/fender.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_FENDER_H ++#define G5P_FENDER_H ++ ++/* Every poll callback must enforce a deadline and retain its final raw word. ++ * Apple 20H350: stop c9fa10, prepare c9fc10, boot c9f954; ++ * Fender helpers ca4ccc (GPC state) and ca6544 (core-power request). ++ * Hardware interpretation of the 0 -> 1 transition remains experimental. ++ */ ++struct g5p_fender_io { ++ void *ctx; ++ int (*write)(void *ctx, u32 reg, u32 value); ++ int (*poll)(void *ctx, u32 reg, u32 mask, u32 expected); ++}; ++ ++static inline int g5p_fender_idle(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x1000, 0x10, 0); ++ if (ret) ++ return ret; ++ ret = io->write(io->ctx, 0x1000, 0x10); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++static inline int g5p_fender_hold(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0x20); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x20000, 0x20, 0x20); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++/* q: the pstate the GPC is requested at (R1+0x1000 = 0x10 | q, poll for q); 1 as before, ++ * or boot_gpu_perf_poweron's pin (boot.inc). ++ */ ++static inline int ++g5p_fender_start_observed_q(const struct g5p_fender_io *io, ++ void (*observe)(void *, const char *), u32 q) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-core-clear"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc0-verified"); ++ ret = io->write(io->ctx, 0x1000, 0x10 | q); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc1-requested"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, q); ++ if (!ret && observe) ++ observe(io->ctx, "release-gpc1-active"); ++ return ret; ++} ++ ++static inline int g5p_fender_start_observed(const struct g5p_fender_io *io, ++ void (*observe)(void *, ++ const char *)) ++{ ++ return g5p_fender_start_observed_q(io, observe, 1); ++} ++ ++static inline int g5p_fender_start(const struct g5p_fender_io *io) ++{ ++ return g5p_fender_start_observed(io, NULL); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/gpu-oc.h b/drivers/gpu/apple-g5p/gpu-oc.h +new file mode 100644 +index 000000000..733aba9d4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/gpu-oc.h +@@ -0,0 +1,149 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * GPU overclock record (boot_gpu_oc_mhz): docs/power/ds-gpu-dvfs.md §3-4, §9 R5; ++ * docs/gpu/experiments/clock-mechanisms.md §2. Host test: scripts/gpu/perf60-host/test_oc.c. ++ * ++ * The hardware applies PSINFO record n (PMGR+0x70000 + 0x10*n, written by LLB 180395fd8 for ++ * slots 0-4 once per SoC reset, volatile) when Fender is asked for pstate n. Word 0 = core-rail ++ * DAC code [31:24] | PLL valid [21] | multiplier [20:12] | predivider [8:4] | postdivider-1 [3:0]; ++ * word 1 [7:0] = buck8 code; word 2 = timing selectors. ++ * ++ * PLL: VCO = 24 MHz * mult / prediv, f = VCO / postdiv. Apple's GPU-domain (0x17) descriptors ++ * (LLB 0x1803bc718, all three chip sections) run VCOs from 800 to 2160 MHz (2160 = id 0x20 of ++ * section 0x2f0001ff, 90/1/3); the live states use 948..1650. The overclock keeps postdiv 2 (every ++ * live state at or above 474 MHz uses 2 or 3) and the VCO inside Apple's range: f <= 1080 MHz. ++ * Apple's own descriptors at 804 (67/1/2), 850 (425/6/2), 900 (75/1/2); elsewhere record 4's ++ * dividers (prediv 4, postdiv 2: f = 3 MHz * mult) on a 3 MHz grid. ++ * ++ * Voltage: record 5 keeps record 4's timing word; its core (rail 1, PMIC 0x11, 450 mV + 3.125 mV * ++ * code, GPU only) and buck8 (rail 8, PMIC 0x18, 600 mV + 3.125 mV * code) codes default to record ++ * 4's (138 = 881.25 mV, 124 = 987.5 mV), never lower, raised only on request in whole 3.125 mV ++ * codes, ++ * and never above the ceilings: the boot parameters (default state 4 + 50 mV, the CPU overclock's ++ * rail-offset cap) bounded by the compiled maxima below (state 4 + 150 mV; raising those needs a ++ * rebuild). The encoding itself reaches 1296 mV on both rails; that is not an operating point. ++ */ ++#ifndef G5P_GPU_OC_H ++#define G5P_GPU_OC_H ++ ++#include "bits-compat.h" ++ ++#define G5P_PSINFO_BASE 0x70000U /* PMGR syscon offset */ ++#define G5P_PSINFO_REC(n) (G5P_PSINFO_BASE + 0x10U * (n)) ++#define G5P_OC_STATE 5U ++#define G5P_OC_PIN 500U /* boot_gpu_perf for state 5 */ ++#define G5P_OC_STOCK4 \ ++ 0x8a2f1041U /* record 4 word 0 as LLB writes it on this chip */ ++#define G5P_OC_STOCK4_BUCK8 124U /* record 4 word 1 [7:0] */ ++#define G5P_OC_MIN_MHZ 726U /* the first grid step above 723 */ ++#define G5P_OC_VCO_MAX 2160U /* Apple's highest GPU-domain VCO */ ++#define G5P_OC_MAX_MHZ (G5P_OC_VCO_MAX / 2) /* postdiv 2: 1080 */ ++#define G5P_OC_VCO_STOCK_MAX \ ++ 1650U /* the highest VCO of a live state (550 MHz) */ ++#define G5P_OC_VALID BIT(21) ++#define G5P_OC_PLL_MASK 0x001ff1ffU /* mult, prediv, postdiv-1 */ ++#define G5P_OC_UV_STEP 3125U ++#define G5P_OC_CORE_BASE_UV 450000U ++#define G5P_OC_BUCK8_BASE_UV 600000U ++/* 881250 */ ++#define G5P_OC_CORE_STOCK_UV \ ++ (G5P_OC_CORE_BASE_UV + G5P_OC_UV_STEP * (G5P_OC_STOCK4 >> 24)) ++/* 987500 */ ++#define G5P_OC_BUCK8_STOCK_UV \ ++ (G5P_OC_BUCK8_BASE_UV + G5P_OC_UV_STEP * G5P_OC_STOCK4_BUCK8) ++/* the boot parameters' default ceiling: P4 + 50 mV */ ++#define G5P_OC_DEFAULT_HEADROOM_UV 50000U ++#define G5P_OC_ABS_HEADROOM_UV \ ++ 150000U /* compiled: P4 + 150 mV, no parameter goes above */ ++ ++struct g5p_oc_pll { ++ u32 mhz, mult, prediv, postdiv; ++ bool llb; /* Apple's own descriptor */ ++}; ++ ++static const struct g5p_oc_pll g5p_oc_llb[] = { ++ { 804, 67, 1, 2, true }, /* LLB id 0x22, all three GPU sections */ ++ { 850, 425, 6, 2, true }, /* LLB id 0x23, the live section */ ++ { 900, 75, 1, 2, true }, /* LLB id 0x24, the live section */ ++}; ++ ++/* ++ * The PLL for a requested MHz: Apple's descriptor at 804/850/900 (the request or its 3 MHz ++ * rounding), ++ * else the 3 MHz grid (rounded down) with record 4's dividers; false outside 726..1080. ++ */ ++static inline bool g5p_oc_pll(u32 want, struct g5p_oc_pll *p) ++{ ++ u32 mhz = want / 3 * 3; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_llb); i++) ++ if (want == g5p_oc_llb[i].mhz || mhz == g5p_oc_llb[i].mhz) { ++ *p = g5p_oc_llb[i]; ++ return true; ++ } ++ if (mhz < G5P_OC_MIN_MHZ || mhz > G5P_OC_MAX_MHZ) ++ return false; ++ *p = (struct g5p_oc_pll){ mhz, mhz / 3, 4, 2, false }; ++ return true; ++} ++ ++/* A rail voltage as a code: exact multiples of 3.125 mV above the base only. */ ++static inline bool g5p_oc_code(u32 uv, u32 base, u32 *code) ++{ ++ if (uv < base || (uv - base) % G5P_OC_UV_STEP || ++ (uv - base) / G5P_OC_UV_STEP > 0xff) ++ return false; ++ *code = (uv - base) / G5P_OC_UV_STEP; ++ return true; ++} ++ ++/* The voltage codes for a request (0 = state 4's), bounded below by state 4 and above by the ++ * ceilings (each already clamped to the compiled maximum by g5p_oc_ceiling()). ++ */ ++ ++/* Record 5's words 0 and 1 from the live record 4 (stock only), a PLL and the voltage codes. */ ++static inline bool g5p_oc_words(const u32 r4[4], u32 want, u32 core, u32 buck8, ++ u32 w[4], struct g5p_oc_pll *p) ++{ ++ if (r4[0] != G5P_OC_STOCK4 || (r4[1] & 0xff) != G5P_OC_STOCK4_BUCK8 || ++ !g5p_oc_pll(want, p) || core < (G5P_OC_STOCK4 >> 24) || ++ core > 0xff || buck8 < G5P_OC_STOCK4_BUCK8 || buck8 > 0xff) ++ return false; ++ w[0] = core << 24 | G5P_OC_VALID | p->mult << 12 | p->prediv << 4 | ++ (p->postdiv - 1); ++ w[1] = (r4[1] & ~0xffU) | buck8; ++ w[2] = r4[2]; ++ w[3] = r4[3]; ++ return true; ++} ++ ++/* Hz of a record word 0, as iBoot 87001c3ac decodes it (0 if the predivider is 0). */ ++static inline u64 g5p_oc_hz(u32 w0) ++{ ++ u32 mult = (w0 >> 12) & 0x1ff, prediv = (w0 >> 4) & 0x1f, ++ postdiv = (w0 & 0xf) + 1; ++ ++ return prediv ? 24000000ULL * mult / prediv / postdiv : 0; ++} ++ ++/* A nonzero record 5 that this driver writes: record 4's timing and word-3 words, valid, a PLL ++ * g5p_oc_pll() produces, voltage codes between state 4's and the compiled maxima. Anything else in ++ * slot 5 is not ours: never overwritten or cleared. ++ */ ++static inline bool g5p_oc_is_ours(const u32 r4[4], const u32 r5[4]) ++{ ++ u32 core = r5[0] >> 24, buck8 = r5[1] & 0xff, w[4], cmax, bmax; ++ struct g5p_oc_pll p; ++ u64 hz = g5p_oc_hz(r5[0]); ++ ++ if (!g5p_oc_code(G5P_OC_CORE_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_CORE_BASE_UV, &cmax) || ++ !g5p_oc_code(G5P_OC_BUCK8_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_BUCK8_BASE_UV, &bmax) || ++ core > cmax || buck8 > bmax || !hz || hz % 1000000ULL || ++ !g5p_oc_words(r4, (u32)(hz / 1000000ULL), core, buck8, w, &p)) ++ return false; ++ return w[0] == r5[0] && w[1] == r5[1] && w[2] == r5[2] && w[3] == r5[3]; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/heartbeat116.h b/drivers/gpu/apple-g5p/heartbeat116.h +new file mode 100644 +index 000000000..428146764 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/heartbeat116.h +@@ -0,0 +1,175 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_HEARTBEAT116_H ++#define G5P_HEARTBEAT116_H ++ ++/* ++ * Heartbeat116: the R74 IRQ174 storm bound as a window rate, not a lifetime count. ++ * ++ * A healthy R74 epoch takes IRQ174 at about 575/s and consumes about 290 event-ring ++ * entries/s, idle or not: one type-0 ALIVE per firmware scheduler iteration, and two IRQ ++ * entries per ring entry (irqs = 2 * events - 18 over the Warm113 C4-prep R74 milestones; ++ * r81 measured 577/289 per s). Wake112 bounded both counts over the epoch's lifetime: ++ * IRQ entries against boot_r74_irq_cap (1048576) and ring entries against the compute event ++ * limit (524544). Either bound latched failed/event_failed, so an idle, healthy GPU faulted ++ * after about 30 minutes. ++ * ++ * Here both bounds are rates. A window opens at the first IRQ entry and closes at the first ++ * entry at least window_ns later, then re-bases on the totals of that entry. Totals are u32 ++ * and every in-window count is a u32 difference, so a wrap of a total is harmless. ++ * ++ * RATE more than irq_ceiling IRQ entries in the open window (checked on every entry) ++ * EVENTS more than event_ceiling ring entries in the open window (checked on every entry) ++ * STALL stall_windows consecutive closed windows in which ++ * irqs > ratio * events + slack. Healthy traffic is 2:1. A latch that re-fires ++ * without ring progress, or a heartbeat after a fault stopped the drain, is not. ++ * ++ * A trip is absorbing. Nothing here clears a failure; the glue adds the same terminal stop ++ * as the lifetime cap had (cap_seen, failed, event_failed, one disable_irq_nosync). ++ * ++ * Pure policy: no register access, no lock, no kernel API. The caller serializes (the R73B ++ * IRQ thread is the only caller). ++ */ ++ ++#define G5P_HB_WINDOW_NS 1000000000ULL ++#define G5P_HB_IRQ_CEILING 16384U ++#define G5P_HB_EVENT_CEILING 8192U ++#define G5P_HB_RATIO 8U ++#define G5P_HB_SLACK 512U ++#define G5P_HB_STALL_WINDOWS 3U ++ ++enum g5p_hb_why { ++ G5P_HBW_NONE, ++ G5P_HBW_RATE, ++ G5P_HBW_EVENTS, ++ G5P_HBW_STALL, ++ G5P_HBW_N, ++}; ++ ++static const char *const g5p_hb_why_name[G5P_HBW_N] = { ++ "none", ++ "rate", ++ "events", ++ "stall", ++}; ++ ++enum g5p_hb_res { ++ G5P_HBR_OPEN, /* counted in the open window */ ++ G5P_HBR_CLOSED, /* this entry closed a window (h->last) */ ++ G5P_HBR_TRIP, /* tripped now or earlier */ ++}; ++ ++struct g5p_hb_cfg { ++ u64 window_ns; ++ u32 irq_ceiling, event_ceiling, ratio, slack, stall_windows; ++}; ++ ++/* The totals the IRQ thread samples after servicing one entry. */ ++struct g5p_hb_in { ++ u64 now_ns; ++ u32 irqs; /* IRQ174 entries (b->irqs) */ ++ u32 events; /* event-ring entries consumed (b->events) */ ++ u32 acks; /* AIC ACKs written by the R73B handler */ ++ u32 empty; /* entries whose service consumed no ring entry */ ++ u32 aux; /* auxiliary-ring entries consumed */ ++}; ++ ++/* One window: its start and length and its in-window counts. */ ++struct g5p_hb_win { ++ u64 start_ns, ns; ++ u32 irqs, events, acks, empty, aux; ++}; ++ ++struct g5p_hb { ++ u32 open; /* zero until the first entry (kzalloc) */ ++ u32 why; /* enum g5p_hb_why, written once */ ++ u32 windows; /* windows closed */ ++ u32 stalled; /* consecutive unproductive closed windows */ ++ u32 peak_irqs, peak_events; ++ u64 start_ns; ++ struct g5p_hb_in base; /* totals at the window's first entry */ ++ struct g5p_hb_win last; /* the last closed window */ ++ struct g5p_hb_win trip; /* the open window at the trip */ ++ u64 trip_ns; ++}; ++ ++static inline void g5p_hb_cfg_default(struct g5p_hb_cfg *c) ++{ ++ c->window_ns = G5P_HB_WINDOW_NS; ++ c->irq_ceiling = G5P_HB_IRQ_CEILING; ++ c->event_ceiling = G5P_HB_EVENT_CEILING; ++ c->ratio = G5P_HB_RATIO; ++ c->slack = G5P_HB_SLACK; ++ c->stall_windows = G5P_HB_STALL_WINDOWS; ++} ++ ++static inline void g5p_hb_count(const struct g5p_hb *h, ++ const struct g5p_hb_in *in, ++ struct g5p_hb_win *w) ++{ ++ w->start_ns = h->start_ns; ++ w->ns = in->now_ns >= h->start_ns ? in->now_ns - h->start_ns : 0; ++ w->irqs = in->irqs - h->base.irqs; ++ w->events = in->events - h->base.events; ++ w->acks = in->acks - h->base.acks; ++ w->empty = in->empty - h->base.empty; ++ w->aux = in->aux - h->base.aux; ++} ++ ++static inline bool g5p_hb_unproductive(const struct g5p_hb_cfg *c, ++ const struct g5p_hb_win *w) ++{ ++ return (u64)w->irqs > (u64)c->ratio * w->events + c->slack; ++} ++ ++static inline enum g5p_hb_res ++g5p_hb_trip(struct g5p_hb *h, u32 why, const struct g5p_hb_win *w, u64 now_ns) ++{ ++ h->why = why; ++ h->trip = *w; ++ h->trip_ns = now_ns; ++ return G5P_HBR_TRIP; ++} ++ ++/* One serviced IRQ entry. */ ++static inline enum g5p_hb_res g5p_hb_step(struct g5p_hb *h, ++ const struct g5p_hb_cfg *c, ++ const struct g5p_hb_in *in) ++{ ++ struct g5p_hb_win w; ++ ++ if (h->why) ++ return G5P_HBR_TRIP; ++ if (!h->open) { ++ h->open = 1; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ return G5P_HBR_OPEN; ++ } ++ g5p_hb_count(h, in, &w); ++ if (w.irqs > c->irq_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_RATE, &w, in->now_ns); ++ if (w.events > c->event_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_EVENTS, &w, in->now_ns); ++ if (w.ns < c->window_ns) ++ return G5P_HBR_OPEN; ++ h->last = w; ++ h->windows++; ++ if (w.irqs > h->peak_irqs) ++ h->peak_irqs = w.irqs; ++ if (w.events > h->peak_events) ++ h->peak_events = w.events; ++ h->stalled = g5p_hb_unproductive(c, &w) ? h->stalled + 1 : 0; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ if (h->stalled >= c->stall_windows) ++ return g5p_hb_trip(h, G5P_HBW_STALL, &w, in->now_ns); ++ return G5P_HBR_CLOSED; ++} ++ ++/* Receipts: the first four closed windows, then every 64th. */ ++static inline bool g5p_hb_report(const struct g5p_hb *h) ++{ ++ return h->windows <= 4 || !(h->windows % 64); ++} ++ ++#endif /* G5P_HEARTBEAT116_H */ +diff --git a/drivers/gpu/apple-g5p/idle-power.inc b/drivers/gpu/apple-g5p/idle-power.inc +new file mode 100644 +index 000000000..d2ce71879 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/idle-power.inc +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Software/shared-RAM observation only. Never wake or probe GPU registers. */ ++static ssize_t idle_power_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b; ++ u32 p, state, target, mirror, period; ++ int ret; ++ ++ if (!mutex_trylock(&g->lock)) ++ return sysfs_emit(buf, "available=0 why=busy mmio=0\n"); ++ b = g->boot; ++ if (!b || !b->maps_ready) { ++ ret = sysfs_emit(buf, "available=0 why=no-maps mmio=0\n"); ++ goto out; ++ } ++ p = b->layout.offset[0x4a]; ++ dma_rmb(); ++ period = g5p_dc_root_word(b, p + 4); ++ state = g5p_dc_root_word(b, p + 8); ++ target = g5p_dc_root_word(b, p + 0x28); ++ mirror = g5p_dc_root_word(b, p + 0x34); ++ /* +34 is both a sampled Fender nibble and the setter's requested index. ++ * It is not a completed transition or stable-off certificate. Stale RAM ++ * remains readable across a stopped/failed epoch; label it explicitly. ++ */ ++ ret = sysfs_emit(buf, ++ "available=1 gen=%u alive=%u failed=%u stopped=%u powered=%u " ++ "policy_state=%u target=%u policy_pstate=%u sample_ms=%u " ++ "meta71c=%u source=shared-ram atomic=0 hw_ack=0 mmio=0\n", ++ g5p_ds_gen(), b->alive, b->failed, b->stopped, ++ g->powered, state, target, mirror, period, b->meta71c); ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++static DEVICE_ATTR_RO(idle_power_status); +diff --git a/drivers/gpu/apple-g5p/kick.inc b/drivers/gpu/apple-g5p/kick.inc +new file mode 100644 +index 000000000..6ef7395c6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/kick.inc +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared pre-kick step recovered from ca0358/ca05d4 and ca6644. */ ++static int g5p_prepare_kick(struct g5p_diag *g, const char *name) ++{ ++ u32 raw, state; ++ ++ if (gfx_bridge_replay && !strcmp(name, "RENDER-TA")) ++ g5p_r41_bridge_ta(g); ++ /* ca05d4 always invokes Fender +80 before inspecting GPC state. ++ * ca6644: W R1+14=7, then read fence. Hardware bit names unknown. ++ * This is distinct from acknowledging the host AIC interrupt. ++ */ ++ dev_info(g->dev, "%s Fender W R1+00014=00000007 before MTS kick\n", ++ name); ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ dev_info(g->dev, "%s Fender raw R1+00014=%08x (write fence)\n", name, ++ raw); ++ if (raw & ~7U) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ if (READ_ONCE(g->boot->failed)) ++ return -EIO; ++ return 0; ++} +diff --git a/drivers/gpu/apple-g5p/latency123.h b/drivers/gpu/apple-g5p/latency123.h +new file mode 100644 +index 000000000..164dc8e68 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/latency123.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: firmware words a powered iOS GPU has and our bootstrap leaves zero ++ * (docs/gpu/firmware-re/wake-latency-re.md sections 3.4 and 4.2). ++ * ++ * Arm A. The firmware copies meta+0x6c0..0x758 into its private block once, at init ++ * (MINIM 0x100009ac -> 0x10003ca4), so these words only act from the boot image: ++ * meta+0x6d4 = 1 per-tick cache maintenance 0x1001b638 (688/1720/e230/e138) ++ * meta+0x710/0x714 pstate table max index / count ++ * meta+0x718.. pstate MHz; the tick period is table[Fender+0x1000 & 0xf] ++ * policy+0x4 = 16 governor sample period, ms (the governor reads it live) ++ * 1 = capped: table { 0, 340 }, max index 1. The firmware's own pstate setter ++ * (0x1001c88c, Fender+0x1000 = index | 0x10) can then only ask for pstate 1, the state ++ * the host already requests, so the clock does not move. ++ * 2 = full: table { 0, 340, 474, 550, 723 }, max index 4, plus the limits README rank 2 ++ * pairs with it (control+0x1028/0x102c/0x1030 = 3348/400/400, policy+0x7cc/0x6fc = ++ * 400). The firmware may then raise the GPU clock itself; nothing on Linux programs ++ * a GPU voltage table, so this arm runs after the capped one. ++ * ++ * Arm B. ctl+0x04 is the per-engine tick-service mask (iOS 7). Every tick re-copies ++ * ctl+0x00..0x20 (0x10001328), so the word is live; the boot image gets the value the ++ * parameter holds at map and the render path re-applies it before each job. ++ * ++ * RAM only. Temporary A/B code: the winning words become unconditional init writes. ++ */ ++#ifndef APPLE_G5P_LATENCY123_H ++#define APPLE_G5P_LATENCY123_H ++ ++#include "bits-compat.h" ++#define G5P_L123_POLICY 0x4aU ++#define G5P_L123_META 0x4dU ++#define G5P_L123_CTL 0x5eU ++#define G5P_L123_TICK_MASK 7U ++#define G5P_L123_CAPPED 1U ++#define G5P_L123_FULL 2U ++ ++static const u32 g5p_l123_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline void g5p_l123_arm_a(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 arm) ++{ ++ u8 *meta = g5p_object(p, root, shared, G5P_L123_META); ++ u8 *policy = g5p_object(p, root, shared, G5P_L123_POLICY); ++ u8 *ctl = g5p_object(p, root, shared, G5P_L123_CTL); ++ u32 count = arm == G5P_L123_FULL ? ARRAY_SIZE(g5p_l123_mhz) : 2; ++ u32 i; ++ ++ g5p_put32(meta + 0x6d4, 1); ++ g5p_put32(meta + 0x710, count - 1); ++ g5p_put32(meta + 0x714, count); ++ for (i = 0; i < count; i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_l123_mhz[i]); ++ g5p_put32(policy + 4, 16); ++ if (arm != G5P_L123_FULL) ++ return; ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, 400); ++ g5p_put32(ctl + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/ldr.h b/drivers/gpu/apple-g5p/ldr.h +new file mode 100644 +index 000000000..165aacf85 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/ldr.h +@@ -0,0 +1,154 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Deliberately small Apple G5P META LDR reader. No allocation or MMIO here. */ ++#ifndef APPLE_G5P_LDR_H ++#define APPLE_G5P_LDR_H ++ ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_LDR_MAX_RECORDS 128 ++#define G5P_LDR_MAX_BYTES (256 * 1024) ++ ++enum g5p_ldr_op { G5P_LOAD, G5P_ZERO, G5P_CONFIG, G5P_START }; ++ ++typedef int (*g5p_ldr_emit)(void *ctx, enum g5p_ldr_op op, u32 address, ++ u32 size, const u8 *data); ++ ++static inline u32 g5p_le32(const u8 *p) ++{ ++ return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; ++} ++ ++static inline u16 g5p_le16(const u8 *p) ++{ ++ return (u16)p[0] | (u16)p[1] << 8; ++} ++ ++static inline bool g5p_ldr_bounds(size_t len, u32 off, u32 size) ++{ ++ return off <= len && size <= len - off; ++} ++ ++static inline int g5p_ldr_l2(const u8 *fw, size_t len, u32 off, u16 tag, ++ const u8 **data, u32 *size) ++{ ++ u16 bytes; ++ ++ if (!g5p_ldr_bounds(len, off, 4)) ++ return -EINVAL; ++ bytes = g5p_le16(fw + off + 2); ++ if (bytes < 6 || !g5p_ldr_bounds(len, off, bytes) || ++ g5p_le16(fw + off) != tag) ++ return -EINVAL; ++ *data = fw + off + 4; ++ *size = bytes - 6; ++ return 0; ++} ++ ++/* Validate in a first pass (emit=NULL); only then use a hardware callback. */ ++static inline int g5p_ldr_walk(const u8 *fw, size_t len, g5p_ldr_emit emit, ++ void *ctx) ++{ ++ u32 pos, seen[G5P_LDR_MAX_RECORDS], total = 0; ++ unsigned int nr = 0, starts = 0; ++ ++ if (len != 119112 || g5p_le32(fw) != 0x01aa5500 || ++ !g5p_ldr_bounds(len, g5p_le32(fw + 4), 1)) ++ return -EINVAL; ++ pos = g5p_le32(fw + 8); ++ while (pos != 0xffffffff) { ++ const u8 *p, *data; ++ u32 command, bytes, addr = 0, size = 0, l2, next; ++ enum g5p_ldr_op op; ++ unsigned int i; ++ int ret; ++ ++ if (nr == G5P_LDR_MAX_RECORDS || !g5p_ldr_bounds(len, pos, 12)) ++ return -EINVAL; ++ for (i = 0; i < nr; i++) ++ if (seen[i] == pos) ++ return -EINVAL; ++ seen[nr++] = pos; ++ p = fw + pos; ++ command = g5p_le16(p); ++ bytes = g5p_le16(p + 2); ++ next = g5p_le32(p + 4); ++ if (bytes < 12 || (bytes & 3) || ++ !g5p_ldr_bounds(len, pos, bytes)) ++ return -EINVAL; ++ if (command & 0x10) { ++ if (command != 0x10 || bytes != 16) ++ return -EINVAL; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), 0x8010, ++ &data, &size); ++ if (ret) ++ return ret; ++ goto next_record; ++ } ++ switch (command) { ++ case 0: ++ if (bytes != 20) ++ return -EINVAL; ++ op = G5P_LOAD; ++ addr = g5p_le32(p + 8); ++ l2 = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, l2, 0x8000, &data, &size); ++ if (ret) ++ return ret; ++ break; ++ case 4: ++ if (bytes != 24) ++ return -EINVAL; ++ op = G5P_ZERO; ++ addr = g5p_le32(p + 8); ++ size = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 16), 0x8004, ++ &data, &l2); ++ if (ret || l2) ++ return -EINVAL; ++ break; ++ case 5: ++ case 3: ++ if (bytes != 16) ++ return -EINVAL; ++ op = command == 5 ? G5P_CONFIG : G5P_START; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), ++ 0x8000 | command, &data, &size); ++ if (ret || !size || size % (command == 5 ? 12 : 16)) ++ return -EINVAL; ++ if (command == 5) { ++ for (i = 0; i < size; i += 12) ++ if (g5p_le32(data + i) != 2 || ++ (g5p_le32(data + i + 4) & 3)) ++ return -EINVAL; ++ } else { ++ if (++starts != 1 || size != 32 || ++ next != 0xffffffff || g5p_le32(data) != 1 || ++ g5p_le32(data + 16) != 0) ++ return -EINVAL; ++ } ++ break; ++ default: ++ return -EINVAL; ++ } ++ if (op == G5P_LOAD || op == G5P_ZERO) { ++ if ((addr & 3) || (size & 3) || !size || ++ size > G5P_LDR_MAX_BYTES || ++ addr > (u32)~0U - size || ++ total > G5P_LDR_MAX_BYTES - size) ++ return -EINVAL; ++ total += size; ++ } ++ if (emit) { ++ ret = emit(ctx, op, addr, size, data); ++ if (ret) ++ return ret; ++ } ++next_record: ++ pos = next; ++ } ++ return nr == 94 && starts == 1 ? 0 : -EINVAL; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +new file mode 100644 +index 000000000..9a896569c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -0,0 +1,1032 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Post-alive transport and park experiments; never implicit recovery. */ ++static const bool control_experiment = 1; ++static const bool irq_experiment = 0; ++static const bool teardown_experiment = 1; ++ ++/* A complete sparse image: first sample omits only zero words, later samples ++ * emit every changed word (including transitions to zero). All allocated ++ * root/shared memory is covered, including unknown status/trace buffers. ++ * These are CPU coherent RAM observations, never proof of META cache contents. ++ * IRQs can update memory concurrently; this is deliberately not an atomic dump. ++ */ ++static u32 g5p_dc_root_word(struct g5p_boot *b, u32 offset) ++{ ++ return READ_ONCE(*(u32 *)(b->root[offset / G5P_GPU_PAGE].cpu + ++ offset % G5P_GPU_PAGE)); ++} ++ ++/* nopatch: the policy record (slot 0x4a) words 0x1000d044 decides readiness from: +8 state, ++ * +0xc substate, +0x34 pstate mirror; +0 and +4 (sample period) for context. iOS keeps +8 at 0. ++ */ ++static const u32 g5p_np_words[5] = { 0, 4, 8, 0xc, 0x34 }; ++ ++static void g5p_np_read(struct g5p_boot *b, u32 *w) ++{ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_np_words); i++) ++ w[i] = g5p_dc_root_word(b, b->layout.offset[0x4a] + ++ g5p_np_words[i]); ++} ++ ++static void g5p_np_print(struct g5p_diag *g, const char *stage, u32 seq, u32 e, ++ const u32 *w) ++{ ++ dev_info( ++ g->dev, ++ "NOPATCH policy stage=%s seq=%u engine=%u +0=%08x +4=%08x +8=%08x +c=%08x " ++ "+34=%08x events=%u end=nopatch\n", ++ stage, seq, e, w[0], w[1], w[2], w[3], w[4], ++ READ_ONCE(g->boot->events)); ++} ++ ++/* RAM only; nothing unless the unpatched image is latched. */ ++static void g5p_np_policy(struct g5p_diag *g, const char *stage, u32 seq, u32 e) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 w[ARRAY_SIZE(g5p_np_words)]; ++ ++ if (!b || !b->r73b || !b->r73b->nopatch) ++ return; ++ g5p_np_read(b, w); ++ g5p_np_print(g, stage, seq, e, w); ++} ++ ++/* Read fixed allocated policy objects, never follow firmware-owned pointers. ++ * 1000d044/1000d504 test these states;1001c900 stores requested pstate. ++ * These are sampled boundary predicates, NOT entry/return breadcrumbs or PC. ++ */ ++ ++/* Firmware MMU recovery uses event mask200 and reads e150/e158. ++ * Read raw qword halves without ACK/clear; concurrent updates can tear them. ++ * An absent sampled event does not exclude an earlier firmware-handled fault. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 event, lo, hi, lo2, hi2; ++ ++ g5p_mmu_progress(g, stage, 0x130); ++ event = readl(g->rgx + 0x130); ++ g5p_mmu_progress(g, stage, 0xe150); ++ lo = readl(g->rgx + 0xe150); ++ g5p_mmu_progress(g, stage, 0xe154); ++ hi = readl(g->rgx + 0xe154); ++ g5p_mmu_progress(g, stage, 0xe158); ++ lo2 = readl(g->rgx + 0xe158); ++ g5p_mmu_progress(g, stage, 0xe15c); ++ hi2 = readl(g->rgx + 0xe15c); ++ g5p_mmu_record( ++ g, ++ "MMU %s event130=%08x pending=%u e150=%08x:%08x e158=%08x:%08x raw-split read-only\n", ++ stage, event, !!(event & 0x200), hi, lo, hi2, lo2); ++} ++ ++/* Same active-wrapper gate as CDM observations. Never write/ACK a register. ++ * 1001c9b8 samples R0+164/160/164; bound host sampling even on rollover. ++ * A split sample with different high words is explicitly unstable. ++ */ ++ ++static void g5p_dc_snapshot(struct g5p_diag *g, const char *stage, bool first) ++{ ++ { ++ return; ++ } ++} ++ ++static int g5p_control_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 *work = (u32 *)g5p_boot_shared(b, 0x68); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ u32 ctl_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6e)); ++ u32 data_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6f)); ++ u8 *init = b->root[0].cpu; ++ u8 *handoff = b->image[0].cpu + 0x3ff8; ++ u32 *events = (u32 *)g5p_boot_shared(b, 0x70); ++ ++ /* c9f868/c9f884: validate the loaded boot handoff, not just its builder. */ ++ if (g5p_le32(handoff) != g5p_meta_address(G5P_ROOT_VA) || ++ g5p_le32(handoff + 4) != ++ g5p_meta_address(g5p_object_va(&b->layout, 0x5f))) ++ return -EPROTO; ++ /* Drain under event_lock before checking that ALIVE has been returned. */ ++ if (g5p_r73b_service(g, G5P_R73B_OWNER_PREPARE, true)) ++ return -EIO; ++ if (READ_ONCE(events[0]) > 255 || READ_ONCE(events[4]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(events[0]) != READ_ONCE(events[4])) ++ return -EBUSY; ++ dev_info(g->dev, ++ "DC boot handoff verified; event consumer=%u producer=%u\n", ++ READ_ONCE(events[0]), READ_ONCE(events[4])); ++ ++ ; ++ ; ++ ; ++ ; ++ /* c96be8: init+18/+1c are META addresses, never CPU/DMA pointers. */ ++ if (g5p_le32(init + 0x18) != ctl_meta || ++ g5p_le32(init + 0x1c) != data_meta) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC mapping ctl GPU=%llx META=%08x data GPU=%llx META=%08x; init verified\n", ++ g5p_object_va(&b->layout, 0x6e), ctl_meta, ++ g5p_object_va(&b->layout, 0x6f), data_meta); ++ ++ if (b->work_ready) ++ return -EBUSY; ++ if (b->control_ready) ++ return b->r73b && 1 ? 0 : -EALREADY; ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (consumer != producer || READ_ONCE(work[0]) != READ_ONCE(work[4]) || ++ (b->work_submitted && !b->work_complete)) ++ return -EBUSY; ++ /* c993f8, size=0: no payload copy, ctrl+1020=0, command={8,0,0,0,0}. ++ * No claim that this is an opcode-independent firmware NOP. ++ */ ++ g5p_put32(g5p_boot_shared(b, 0x5e) + 0x1020, 0); ++ b->control_target = (producer + 1) & 255; ++ b->control_ready = true; ++ dev_info( ++ g->dev, ++ "DC prepare cmd=8 payload_words=0 consumer=%u producer=%u target=%u; RAM only\n", ++ consumer, producer, b->control_target); ++ return 0; ++} ++ ++static int g5p_r73e_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ret = g5p_control_prepare(g); ++ if (!ret) ++ WRITE_ONCE(b->r73b->fast_window, true); ++ dev_info( ++ g->dev, ++ "R73E cmd8 prepared-before-release ret=%d target=%u end=r73e\n", ++ ret, b->control_target); ++ return ret; ++} ++ ++/* Apple's GPC precondition (AGXMetaFender_setGPCPerformanceState, ...6ca4ccc): a request in ++ * device test shows R1+0x1000 bit 4; wait for it to clear before judging the state. R1 reads only, ++ * bounded by boot_alive_settle_us; with 0 the first read stands, as before. ++ */ ++static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw) ++{ ++ u64 start; ++ ++ if (!r->settle_us || !(raw & 0x10)) ++ return raw; ++ if (!r->settle_first) ++ r->settle_first = raw; ++ start = ktime_get_mono_fast_ns(); ++ while ((raw & 0x10) && ktime_get_mono_fast_ns() - start < ++ (u64)r->settle_us * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(g->window[1] + 0x1000); ++ r->settle_samples++; ++ } ++ r->settle_ns += ktime_get_mono_fast_ns() - start; ++ return raw; ++} ++ ++static int g5p_r73e_irq_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 off, raw; ++ enum g5p_pxg gate; ++ ++ if (!b->control_ready || b->control_submitted) ++ return -EPERM; ++ /* wake112: release_ready, or a deferred R74E proof that the read below settles. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate != G5P_PXG_READY && gate != G5P_PXG_PROVE) ++ return -EHOSTDOWN; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ if (gate == G5P_PXG_PROVE && !g5p_r74e_take(g)) ++ return -EHOSTDOWN; ++ r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (gate == G5P_PXG_PROVE) ++ g5p_r74e_settle(r, r->early_state); ++ if (!g5p_gpc_ok(r->early_state)) ++ return -EHOSTDOWN; ++ /* nopatch: five RAM loads, the state the readiness gate will see before this bell. */ ++ if (r->nopatch) ++ g5p_np_read(b, r->np_pre); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_IRQ, 0); ++ /* Functional pre-kick sequence only; no optional diagnostics here. */ ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ if (raw & ~7U) ++ return -EPROTO; ++ r->early_after_state = ++ g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (!g5p_gpc_ok(r->early_after_state)) ++ return -EHOSTDOWN; ++ dev_emerg( ++ g->dev, ++ "R73E doorbell begin first-ALIVE W0+00b00 value=00000010 end=r73e\n"); ++ writel(0x10, g->rgx + 0xb00); ++ dev_emerg(g->dev, "R73E doorbell end first-ALIVE W0+00b00 end=r73e\n"); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_IRQ, 0); ++ return 0; ++} ++ ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxg gate; ++ u32 px; ++ int ret; ++ ++ /* wake112: a deferred R74E proof admits the once-owner as release_ready does. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate == G5P_PXG_REFUSE) ++ return -EHOSTDOWN; ++ if (gate == G5P_PXG_WAIT) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_busy, 0, 1)) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_once, 0, 1)) { ++ ret = READ_ONCE(r->early_ret); ++ goto out; ++ } ++ r->early_irq_ns = ktime_get_mono_fast_ns(); ++ r->early_ret = g5p_r73e_irq_submit(g); ++ ret = r->early_ret; ++ WRITE_ONCE(r->fast_window, false); ++ if (ret == -EHOSTDOWN) ++ WRITE_ONCE(r->early_power_abort, true); ++ dev_emerg( ++ g->dev, ++ "R74E ALIVE submit case=%u owner=%u state=%08x after=%08x ret=%d " ++ "release_ready=%u power_abort=%u submitted=%u end=r74e\n", ++ r->r73e_case, owner, r->early_state, r->early_after_state, ret, ++ r->early_release_ready, r->early_power_abort, ++ READ_ONCE(b->control_submitted)); ++ if (r->settle_first) ++ dev_emerg( ++ g->dev, ++ "R74E settle first=%08x state=%08x after=%08x samples=%u ns=%llu limit_us=%u " ++ "ret=%d end=r74e\n", ++ r->settle_first, r->early_state, r->early_after_state, ++ r->settle_samples, r->settle_ns, r->settle_us, ret); ++ if (r->nopatch && READ_ONCE(b->control_submitted)) ++ g5p_np_print(g, "pre-bell", 0, 0, r->np_pre); ++ px = atomic_read(&r->early_proof); ++ /* raw: the proof read, or U32_MAX when it did not happen (a refusal or no take). */ ++ if (px != G5P_PX_NONE) ++ dev_emerg( ++ g->dev, ++ "R74E112 proof owner=%u state=%s why=%s gen=%u/%u raw=%08x release_ready=%u " ++ "power_abort=%u end=r74e112\n", ++ owner, g5p_px_name[min_t(u32, px, G5P_PX_N - 1)], ++ g5p_px_why_name[min_t(u32, r->early_proof_why, ++ G5P_PXW_N - 1)], ++ r->early_proof_gen, r->early_proof_gen_now, ++ px == G5P_PX_PROVEN || ++ r->early_proof_why == G5P_PXW_STATE ? ++ r->early_state : ++ U32_MAX, ++ r->early_release_ready, r->early_power_abort); ++out: ++ atomic_set(&r->early_busy, 0); ++ return ret; ++} ++ ++static int g5p_control_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 state, off; ++ int ret; ++ ++ if (b->control_submitted) ++ return b->r73b && 1 ? 0 : -EPERM; ++ if (!b->control_ready) ++ return -EPERM; ++ /* R73 synchronous pre-publish pass closes the race where trace records ++ * do not raise a final IRQ. A negative result is scoreable only when the ++ * factor reaches cmd8 with every outbound auxiliary consumer caught up. ++ */ ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_PUBLISH, true); ++ if (ret) ++ return ret; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ dev_info(g->dev, "DC raw R1+01000=%08x (require active before kick)\n", ++ state); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_dc_snapshot(g, "before-publish", true); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ /* ca05d4: full dmb ish after producer, before Fender and MTS. */ ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_PUBLISH, 0); ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *src = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ if (g5p_le32(src) != (i ? 0 : 8)) ++ return -EBADMSG; ++ } ++ if (READ_ONCE(ctl[4]) != b->control_target) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC slot verified words=00000008/0/0/0/0 producer=%u; full barrier\n", ++ b->control_target); ++ ret = g5p_prepare_kick(g, "DC"); ++ if (ret) ++ return ret; ++ ; ++ if (b->r73_aux_drain) { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_KICK, true); ++ if (ret) ++ return ret; ++ } ++ g5p_dc_snapshot(g, "before-kick", false); ++ dev_info( ++ g->dev, ++ "DC publish cmd=8 slot=%u target=%u; W R0+00b00=%08x variant=%u\n", ++ producer, b->control_target, b->dc_variant == 4 ? 0x12 : 0x10, ++ b->dc_variant); ++ /* 0x12 is verified for CL, NOT for DC: variant 4 tests dispatch routing. */ ++ writel(7 == 4 ? 0x12 : 0x10, g->rgx + 0xb00); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_KICK, 0); ++ g5p_dc_snapshot(g, "after-kick", false); ++ return 0; ++} ++ ++/* perf60 pin >100 (seed 1-5, poweron): the policy record as the firmware left it, from RAM, plus the ++ * real Fender state through an R1-only ticket of the register licence (any pstate, bit 29 allowed). ++ * The sampler 0x1000aa80 stores its own "now" at policy+0x918 on every call (0x1000ab44) and the ++ * current state at +0x34 (0x1000ab18), so fw_now against the host's arch timer names the ++ * firmware's clock, and +0x34 / +0 / +0x8f8 name the branch it took. ++ */ ++static void g5p_perf_readback(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 pol = b->layout.offset[0x4a], meta = b->layout.offset[0x4d]; ++ u64 host = arch_timer_read_counter(), rate = arch_timer_get_rate(); ++ u64 p8f8, p918, p920, m784; ++ struct g5p_mmio_ticket t; ++ ++ if (!b->gpu_perf || (!b->perf_seed && !b->perf_poweron)) ++ return; ++ dma_rmb(); ++ p8f8 = g5p_dc_root_word(b, pol + 0x8f8) | ++ (u64)g5p_dc_root_word(b, pol + 0x8fc) << 32; ++ p918 = g5p_dc_root_word(b, pol + 0x918) | ++ (u64)g5p_dc_root_word(b, pol + 0x91c) << 32; ++ p920 = g5p_dc_root_word(b, pol + 0x920) | ++ (u64)g5p_dc_root_word(b, pol + 0x924) << 32; ++ m784 = g5p_dc_root_word(b, meta + 0x784) | ++ (u64)g5p_dc_root_word(b, meta + 0x788) << 32; ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ dev_emerg( ++ g->dev, ++ "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " ++ "pol28=%u pol2c=%u pol30=%u pol34=%u p8f8=%llx fw_now918=%llx p920=%llx " ++ "meta784=%llx host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " ++ "host_minus_fw_counter_us=%lld r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", ++ why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, ++ g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30), ++ g5p_dc_root_word(b, pol + 0x34), p8f8, p918, p920, m784, host, ++ b->seed_stamp, ++ b->seed_stamp ? ++ div_s64((s64)(p918 - m784 - b->seed_stamp) * 1000000LL, ++ rate) : ++ 0, ++ div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate), ++ g5p_mmio_did(&t), t.gpc, t.r1_0); ++} ++ ++static int g5p_r73c_fast_control(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ int ret; ++ ++ r->fast_release_ns = ktime_get_mono_fast_ns(); ++ WRITE_ONCE(r->fast_window, true); ++ ret = g5p_boot_events(g); ++ r->fast_alive_ns = ktime_get_mono_fast_ns(); ++ if (!ret) ++ ret = g5p_control_prepare(g); ++ if (!ret) { ++ r->fast_state = readl(g->window[1] + 0x1000); ++ ret = g5p_control_submit(g); ++ } ++ r->fast_submit_ns = ktime_get_mono_fast_ns(); ++ r->fast_ret = ret; ++ WRITE_ONCE(r->fast_window, false); ++ g5p_r73b_events_exit(g, ret); ++ dev_info( ++ g->dev, ++ "R73C fast release_to_alive_ns=%llu release_to_submit_ns=%llu state=%08x ret=%d " ++ "control_submitted=%u end=r73c\n", ++ r->fast_alive_ns - r->fast_release_ns, ++ r->fast_submit_ns - r->fast_release_ns, r->fast_state, ret, ++ READ_ONCE(b->control_submitted)); ++ g5p_perf_readback(g, ret ? "alive-timeout" : "alive"); ++ return ret; ++} ++ ++/* perf60 pin 200 (docs/gpu/experiments/perf60-clock.md "Pin 200: the guarded drain read" and ++ * "Clock-change drain RE" §4): at a boot control timeout with boot_gpu_perf set (before any job, ++ * the firmware alive), every read goes through the GPU register licence (mmio.inc) and nothing ++ * wakes the GPU (no R1+0x14 write, mmio-access.md G6): ++ * r1: R1 +0, +8, +0x14, +0x1000, +0x1004, +0x20000 (one R1-only ticket, any pstate) ++ * drain: 11f00 688 1720 1728 e230 e238 e138, then the clock words R0+0x0/0x4/0x8/0xc ++ * timer: R0+0x160/0x164, twice, 1 ms apart ++ * tx: T0/T1 TXENABLE/TXSTATUS (0x04800000/10, 0x04801000/10) through the META slave port ++ * pc: (bit 1 only) T0 PC: RQ 0x0480fff8 = 0x10005, poll bit 31, read 0x0480fff0; x8, 100 us ++ * apart. This is the driver's first slave-port write (CTRL0 = addr, DATAT = value). ++ * One ticket per group or slave-port op (<= G5P_MMIO_MAX_OPS), a refusal retried <= 2 ms. ++ */ ++ ++#define G5P_GPP_BUDGET_US 2000U ++ ++/* One META slave-port read in one ticket: ready, CTRL0 = addr | 1, read back, ready (<= 4 ++ * polls), DATAX: 8 ops. -EBUSY when the port is not ready within the ticket. ++ */ ++static int g5p_mmio_sp_read(struct g5p_diag *g, u32 addr, u32 *out, u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1 = 0; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr | 1)) { ++ g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL0); ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ *out = g5p_mmio_r0_read(g, &t, G5P_MSLVDATAX); ++ ret = 0; ++ break; ++ } ++ } ++ } ++ if (!g5p_mmio_release(g, &t) && !ret) ++ ret = -ESTALE; ++ return ret; ++} ++ ++/* One META slave-port write in one ticket: ready, CTRL0 = addr, DATAT = value, ready (<= 4). */ ++static int g5p_mmio_sp_write(struct g5p_diag *g, u32 addr, u32 value, ++ u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr) && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVDATAT, value)) { ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ ret = 0; ++ break; ++ } ++ } ++ } ++ g5p_mmio_release(g, &t); ++ return ret; ++} ++ ++/* A group of plain R0 reads (<= G5P_MMIO_MAX_OPS) under one ticket; returns the ticket's did. */ ++static const char *g5p_gpp_r0(struct g5p_diag *g, const u32 *off, u32 *v, u32 n, ++ u32 *tries, u32 *gpc) ++{ ++ struct g5p_mmio_ticket t; ++ ++ if (g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries)) { ++ *gpc = t.gpc; ++ return g5p_mmio_did(&t); ++ } ++ for (u32 i = 0; i < n; i++) ++ v[i] = g5p_mmio_r0_read(g, &t, off[i]); ++ g5p_mmio_release(g, &t); ++ *gpc = t.gpc; ++ g5p_mmio_sample_line(g, G5P_MMS_PROBE, &t, v, n, true); ++ return g5p_mmio_did(&t); ++} ++ ++/* What a slave-port op's return means, for the dc-probe lines. */ ++static const char *g5p_sp_why(int ret) ++{ ++ switch (ret) { ++ case 0: ++ return "ok"; ++ case -ENODATA: ++ return "not-run"; ++ case -EPERM: ++ return "forbidden-after-release"; /* g->reset_transition: boot.inc's post-release ban */ ++ case -ENODEV: ++ return "licence-off"; ++ case -EAGAIN: ++ return "licence-refused"; ++ case -EBUSY: ++ return "port-not-ready"; ++ case -ESTALE: ++ return "stale"; ++ case -ETIMEDOUT: ++ return "rq-no-ready"; ++ default: ++ return "error"; ++ } ++} ++ ++static void g5p_gpu_perf_probe(struct g5p_diag *g) ++{ ++ static const u32 drain[] = { 0x11f00, 0x688, 0x1720, 0x1728, ++ 0xe230, 0xe238, 0xe138 }; ++ static const u32 clk[] = { 0x0, 0x4, 0x8, 0xc }; ++ static const u32 timer[] = { 0x160, 0x164 }; ++ static const u32 r1off[] = { 0x0, 0x8, 0x14, 0x1000, 0x1004, 0x20000 }; ++ static const u32 tx[] = { 0x04800000, 0x04800010, 0x04801000, ++ 0x04801010 }; ++ u32 mode = (0); ++ struct g5p_boot *b = g->boot; ++ u32 vd[ARRAY_SIZE(drain)], vc[ARRAY_SIZE(clk)], vt[2][2], ++ r1[ARRAY_SIZE(r1off)]; ++ u32 vx[ARRAY_SIZE(tx)], pc[8], pcrq[8], gate = U32_MAX, tries = 0, ++ gpc = U32_MAX, gpc_x; ++ int txr[ARRAY_SIZE(tx)], pcr[8]; ++ const char *d_r1 = "skip-guard", *d_drain = "skip-guard", ++ *d_clk = "skip-guard"; ++ const char *d_t0 = "skip-guard", *d_t1 = "skip-guard"; ++ struct g5p_mmio_ticket t; ++ ++ if (!mode || !b->gpu_perf) ++ return; ++ memset(vd, 0xff, sizeof(vd)); ++ memset(vc, 0xff, sizeof(vc)); ++ memset(vt, 0xff, sizeof(vt)); ++ memset(r1, 0xff, sizeof(r1)); ++ memset(vx, 0xff, sizeof(vx)); ++ memset(pc, 0xff, sizeof(pc)); ++ memset(pcrq, 0xff, sizeof(pcrq)); ++ for (u32 i = 0; i < ARRAY_SIZE(txr); i++) ++ txr[i] = -ENODATA; ++ for (u32 i = 0; i < ARRAY_SIZE(pcr); i++) ++ pcr[i] = -ENODATA; ++ if (g->pmgr && regmap_read(g->pmgr, 0x80340, &gate)) ++ gate = U32_MAX; ++ if (READ_ONCE(b->failed)) ++ goto print; ++ /* r1: before anything else, R1-only (valid at any pstate and with bit 29 set) */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) { ++ for (u32 i = 0; i < ARRAY_SIZE(r1off); i++) ++ r1[i] = g5p_mmio_r1_read(g, &t, r1off[i]); ++ g5p_mmio_release(g, &t); ++ } ++ d_r1 = g5p_mmio_did(&t); ++ /* gpc: the drain group's licence read; every group's ticket also has its own sample line */ ++ d_drain = g5p_gpp_r0(g, drain, vd, ARRAY_SIZE(drain), &tries, &gpc); ++ d_clk = g5p_gpp_r0(g, clk, vc, ARRAY_SIZE(clk), &tries, &gpc_x); ++ d_t0 = g5p_gpp_r0(g, timer, vt[0], 2, &tries, &gpc_x); ++ usleep_range(1000, 1100); ++ d_t1 = g5p_gpp_r0(g, timer, vt[1], 2, &tries, &gpc_x); ++ for (u32 i = 0; i < ARRAY_SIZE(tx); i++) ++ txr[i] = g5p_mmio_sp_read(g, tx[i], &vx[i], &tries); ++ if (mode & 2) { ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) { ++ pcr[i] = g5p_mmio_sp_write(g, 0x0480fff8, 0x10005, ++ &tries); ++ for (u32 k = 0; !pcr[i] && k < 4; k++) { ++ pcr[i] = g5p_mmio_sp_read(g, 0x0480fff8, ++ &pcrq[i], &tries); ++ if (pcr[i] || (pcrq[i] & BIT(31))) ++ break; ++ udelay(10); ++ } ++ if (!pcr[i]) ++ pcr[i] = pcrq[i] & BIT(31) ? ++ g5p_mmio_sp_read(g, 0x0480fff0, ++ &pc[i], ++ &tries) : ++ -ETIMEDOUT; ++ udelay(100); ++ } ++ } ++print: ++ dev_err(g->dev, ++ "GPUPERF dc-probe did=%s gate=%08x r1=%s r1_0=%08x r1_8=%08x r1_14=%08x " ++ "r1_1000=%08x r1_1004=%08x r1_20000=%08x gpc=%08x tries=%u end=gpuperf\n", ++ d_drain, gate, d_r1, r1[0], r1[1], r1[2], r1[3], r1[4], r1[5], ++ gpc, tries); ++ dev_err(g->dev, ++ "GPUPERF dc-probe drain=%s 11f00=%08x 688=%08x 1720=%08x 1728=%08x e230=%08x " ++ "e238=%08x e138=%08x clk=%s clk=%08x:%08x clkstat=%08x:%08x end=gpuperf\n", ++ d_drain, vd[0], vd[1], vd[2], vd[3], vd[4], vd[5], vd[6], d_clk, ++ vc[1], vc[0], vc[3], vc[2]); ++ dev_err(g->dev, ++ "GPUPERF dc-probe timer=%s/%s 160=%08x/%08x 164=%08x/%08x gap_us=1000 " ++ "txen0=%08x/%s txst0=%08x/%s txen1=%08x/%s txst1=%08x/%s end=gpuperf\n", ++ d_t0, d_t1, vt[0][0], vt[1][0], vt[0][1], vt[1][1], vx[0], ++ g5p_sp_why(txr[0]), vx[1], g5p_sp_why(txr[1]), vx[2], ++ g5p_sp_why(txr[2]), vx[3], g5p_sp_why(txr[3])); ++ if (mode & 2) ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) ++ dev_err(g->dev, ++ "GPUPERF dc-probe pc i=%u ret=%s rq=%08x t0pc=%08x end=gpuperf\n", ++ i, g5p_sp_why(pcr[i]), pcrq[i], pc[i]); ++} ++ ++static int g5p_control_wait(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long start = jiffies, deadline = start + 3 * HZ; ++ bool sampled = false, rekicked = false; ++ u32 consumer, producer, pass = 0; ++ int ret; ++ ++ if (!b->control_submitted || b->control_complete) ++ return -EPERM; ++ do { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_WAIT, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ consumer = READ_ONCE(ctl[0]); ++ producer = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ if (consumer > 255 || producer != b->control_target || ++ (consumer != b->control_target && ++ consumer != ((b->control_target - 1) & 255))) ++ return -EPROTO; ++ if (consumer == b->control_target) { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (g5p_r74_case(b) == 1) { ++ WRITE_ONCE(b->control_complete, true); ++ g5p_r74_mark(g, G5P_R74_CMD8_CONSUMED, ++ consumer); ++ dev_emerg( ++ g->dev, ++ "R74B cmd8-consumed consumer=%u producer=%u IRQ174=%d ram_only=1 end=r74b\n", ++ consumer, producer, ++ atomic_read(&b->irqs)); ++ g5p_np_policy(g, "cmd8-consumed", consumer, 0); ++ return 0; ++ } ++ g5p_dc_snapshot(g, "consumed", false); ++ b->control_complete = true; ++ ; ++ dev_info( ++ g->dev, ++ "DC consumed cmd=8 consumer=%u producer=%u IRQ174=%d; transport only\n", ++ consumer, producer, atomic_read(&b->irqs)); ++ return 0; ++ } ++ if (!sampled && !time_before(jiffies, start + HZ / 10)) { ++ sampled = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "100ms", false); ++ g5p_np_policy(g, "100ms", consumer, 0); ++ } ++ if (7 == 3 && !rekicked && !time_before(jiffies, start + HZ)) { ++ /* Re-notify the SAME slot once; never publish a second command. */ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_prepare_kick(g, "DC re-kick"); ++ if (ret) ++ return ret; ++ mb(); ++ writel(0x10, g->rgx + 0xb00); ++ rekicked = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "re-kick", false); ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "timeout", false); ++ g5p_np_policy(g, "timeout", consumer, 0); ++ dev_err(g->dev, ++ "DC timeout consumer=%u producer=%u target=%u; retain RAM, no retry\n", ++ consumer, producer, b->control_target); ++ /* perf60 (boot_gpu_perf): the policy-state words at the stall, RAM only. pol34 = the pstate the ++ * firmware last requested (its setter stores it before the Fender write): q means it reached the ++ * setter, 1 that it stopped before (the clock-change drain 0x1001cec8 -> 0x1001b638). ++ */ ++ if (b->gpu_perf) { ++ u32 pol = b->layout.offset[0x4a]; ++ ++ dma_rmb(); ++ dev_err(g->dev, ++ "GPUPERF dc-timeout pin=%u pol8=%u/%u q=%u x=%u pol34=%u req=%u/%u ceil=%u/%u " ++ "seed=%u pol0=%u pol4=%u pol28=%u pol2c=%u pol30=%u end=gpuperf\n", ++ b->gpu_perf, g5p_dc_root_word(b, pol + 8), ++ g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x38), ++ g5p_dc_root_word(b, pol + 0x34), ++ g5p_dc_root_word(b, pol + 0x6f0), ++ g5p_dc_root_word(b, pol + 0x6f4), ++ g5p_dc_root_word(b, pol + 0x688), ++ g5p_dc_root_word(b, pol + 0x68c), b->perf_seed, ++ g5p_dc_root_word(b, pol + 0), ++ g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30)); ++ g5p_gpu_perf_probe(g); ++ g5p_perf_readback(g, "dc-timeout"); ++ } ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ b->control_timed_out = true; ++ return -ETIMEDOUT; ++} ++ ++static int g5p_irq_test(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + HZ / 10; ++ int before = atomic_read(&b->irqs), ret; ++ ++ if (b->irq_tested || !b->irq_ready || g5p_r74_case(b)) ++ return -EPERM; ++ /* ADT set/ack addresses, not a GPU slave-port transaction. Firmware may ++ * also interrupt concurrently: this tests delivery, not event provenance. ++ */ ++ dev_info(g->dev, "IRQ test baseline=%d W AIC+014=00004000\n", before); ++ writel(0x4000, b->swirq + 0x14); ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (atomic_read(&b->irqs) > before) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ b->irq_tested = true; ++ ; ++ dev_info( ++ g->dev, ++ "IRQ delivery observed before=%d after=%d; no shader completion implied\n", ++ before, atomic_read(&b->irqs)); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, "IRQ delivery timeout before=%d after=%d\n", before, ++ atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++static int g5p_boot_park(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ static const u32 slots[] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ bool failed_control = READ_ONCE(b->failed) && b->control_timed_out; ++ bool failed_work = READ_ONCE(b->failed) && b->work_timed_out; ++ bool failed_context = READ_ONCE(b->failed) && b->context_timed_out; ++ bool failed_timeout = failed_control || failed_work || failed_context; ++ int ret; ++ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ if (READ_ONCE(b->event_failed) || ++ (b->control_submitted && !b->control_complete && !failed_control) || ++ (b->work_submitted && !b->work_complete && !failed_work) || ++ (b->context_submitted && !b->context_complete && !failed_context)) ++ return -EBUSY; ++ /* A failed drain must never be bypassed. On the sole permitted failure ++ * path require the event ring empty below without consuming anything. ++ */ ++ if (!failed_timeout) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(slots); i++) { ++ u32 *ctl = (u32 *)g5p_boot_shared(b, slots[i]); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (((failed_control || failed_context) && slots[i] == 0x6c) || ++ (failed_work && slots[i] == 0x68)) { ++ u32 target = failed_context && slots[i] == 0x6c ? ++ b->context_target : ++ slots[i] == 0x6c ? b->control_target : ++ b->work_target; ++ ++ if (producer != target || ++ (consumer != producer && ++ consumer != ((producer - 1) & 255))) ++ return -EPROTO; ++ } else if (consumer != producer) { ++ return -EBUSY; ++ } ++ } ++ if (READ_ONCE(b->event_failed)) ++ return -EIO; ++ if (failed_timeout) ++ dev_info( ++ g->dev, ++ "PARK failed %s timeout; abandon command, preserve failure and RAM\n", ++ failed_context ? "CONTEXT" : ++ failed_work ? "CL" : ++ "DC"); ++ /* Freeze commands before the first transition; no retry even if it fails. ++ * c9fa10 enabled-Fender branch: core=0, GPC=0, no a18 access. ++ */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ dev_info( ++ g->dev, ++ "PARK begin core=0 GPC=0; wrapper only, DMA memory retained\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ return ret; ++ /* Keep the IRQ/ack handler and every allocation alive until reboot. ++ * GPC=0 alone is not proof that outstanding bus traffic has drained. ++ */ ++ b->parked = true; ++ dev_info( ++ g->dev, ++ "PARK GPC=0 verified; quarantine until reboot, no PMGR off or DMA free\n"); ++ return 0; ++} ++ ++static int g5p_lifecycle_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (!boot_experiment || !b || !b->alive || b->stopped) ++ return -EPERM; ++ if (b->failed && ++ !(sysfs_streq(command, "park") && teardown_experiment && ++ (b->control_timed_out || b->work_timed_out || ++ b->context_timed_out))) ++ return -EPERM; ++ if (!strncmp(command, "control-", 8) && control_experiment) { ++ if (sysfs_streq(command, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(command, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(command, "control-wait")) ++ ret = g5p_control_wait(g); ++ else ++ return -EINVAL; ++ } else if (sysfs_streq(command, "irq-test") && ++ (g5p_r74_case(b) || (1))) { ++ /* R74 never runs the AIC self-test r73f case 01 stopped in. */ ++ if (b->r73b) ++ atomic_inc(&b->r73b->r74_irq_refused); ++ g5p_r74_mark(g, G5P_R74_IRQ_TEST_REFUSED, -EPERM); ++ dev_err(g->dev, ++ "R74 refuse irq-test case=%u irq_tested=%u end=r74\n", ++ g5p_r74_case(b), b->irq_tested); ++ return -EPERM; ++ } else if (sysfs_streq(command, "irq-test") && irq_experiment) { ++ ret = g5p_irq_test(g); ++ } else if (sysfs_streq(command, "park") && teardown_experiment) { ++ ret = g5p_boot_park(g); ++ } else { ++ return -EPERM; ++ } ++ if (sysfs_streq(command, "control-wait")) ++ g5p_r73b_crumb(b, G5P_R73B_WAIT_EXIT, G5P_R73B_OWNER_WAIT, ret); ++ if (ret && ret != -EALREADY && ret != -EPERM) ++ WRITE_ONCE(b->failed, true); ++ return ret; ++} ++ ++/* Outgoing-side kexec/reboot quiesce; see docs/gpu/experiments/gpu-reset-recovery-production.md ++ * section 6.10. NOT a reset: Fender core=0/GPC=0 only, no PMGR off, no DMA free and ++ * no R0 access. It acts only when the command lock is free (no job or sysfs command is ++ * inside a hardware wait), the firmware was released and the pstate policy is ++ * host-owned (init meta+0x6c8 == 0); every other state is left alone. Unlike ++ * g5p_boot_park() it does not require drained rings: round 10 flew park after an ++ * unconsumed DC command, and at shutdown there is nothing left to drain for. ++ * GPC=0 is not a proof that bus traffic drained (see g5p_boot_park). ++ */ ++ ++static void g5p_td_shutdown(struct g5p_diag *g); ++ ++static void g5p_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ struct g5p_boot *b; ++ int ret; ++ ++ if (!g) ++ return; ++ /* Idempotent second chance after the reboot notifier; a no-op unless teardown != 0. */ ++ g5p_td_shutdown(g); ++ { ++ return; ++ }; ++ /* A worker or sysfs writer stuck in MMIO holds g->lock: never queue behind it. */ ++ if (!mutex_trylock(&g->lock)) { ++ dev_warn(g->dev, "SHUTDOWN park skipped: command lock busy\n"); ++ return; ++ } ++ b = g->boot; ++ /* R1 is touched only with ps_gfx held on (the retained runtime-PM reference), ++ * the wrapper woken and the firmware released. ++ */ ++ if (!b || !g->powered || !g->fender_ready || !b->maps_ready || ++ !b->released || b->stopped || b->parked) { ++ dev_info(g->dev, "SHUTDOWN park not needed\n"); ++ goto out; ++ } ++ /* With firmware-owned policy (init meta+0x6c8 != 0) the firmware also writes ++ * R1+0x1000, so a host write would race it (dsx-r74b). RAM-only read of the ++ * host-written init word, the same one g5p_dc_snapshot() reports. ++ */ ++ if (g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8)) { ++ dev_info( ++ g->dev, ++ "SHUTDOWN park refused: firmware-owned policy (meta+0x6c8 != 0)\n"); ++ goto out; ++ } ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ /* Freeze commands before the first transition; no retry even if it fails. */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ ret = g5p_fender_idle(&io); ++ if (!ret) ++ WRITE_ONCE(b->parked, true); ++ dev_info( ++ g->dev, ++ "SHUTDOWN park core=0 GPC=0 result=%d; DMA memory retained, no PMGR off\n", ++ ret); ++out: ++ mutex_unlock(&g->lock); ++} +diff --git a/drivers/gpu/apple-g5p/meta71c114.h b/drivers/gpu/apple-g5p/meta71c114.h +new file mode 100644 +index 000000000..cc84dbd82 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/meta71c114.h +@@ -0,0 +1,65 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency114: one RAM dword, meta+0x71c, in the bootstrap image. ++ * ++ * meta (object 0x4d) +0x718 is the firmware's pstate frequency table in MHz. ++ * Its init copies the table to private+0x26c at boot (MINIM 0x1000122c..127c), ++ * and the scheduler loop reloads its timer from entry 1 on both branches ++ * (0x8000a1a4 via private+0x2c4 at pstate 1, 0x8000a338 via private+0x270). ++ * Linux leaves the table zero; Apple has { 0, 340, 474, 550, 723 }. The A/B ++ * installs entry 1 only: 0 is the control (nothing is stored, variant 7 leaves ++ * the word zero), 340 the factor. Everything else is refused at map, before ++ * any store. The store goes into the staging buffer, before g5p_boot_copy() ++ * publishes the image and long before the firmware is released. There is no ++ * register access here. ++ */ ++#ifndef APPLE_G5P_META71C114_H ++#define APPLE_G5P_META71C114_H ++ ++#define G5P_M71C_SLOT 0x4dU ++#define G5P_M71C_OFFSET 0x71cU ++#define G5P_M71C_CONTROL 0U ++#define G5P_M71C_FACTOR 340U ++#define G5P_M71C_VARIANT 7U ++ ++enum { ++ G5P_M71C_OK, ++ G5P_M71C_BAD_VALUE, ++ G5P_M71C_BAD_VARIANT, ++ G5P_M71C_WHYS, ++}; ++ ++static const char *const g5p_m71c_whys[G5P_M71C_WHYS] = { ++ "none", ++ "value", ++ "variant", ++}; ++ ++/* Latched at map: only the two pre-registered arms, the factor on dc7 only. */ ++static inline u32 g5p_m71c_check(u32 requested, u32 variant) ++{ ++ if (requested != G5P_M71C_CONTROL && requested != G5P_M71C_FACTOR) ++ return G5P_M71C_BAD_VALUE; ++ if (requested == G5P_M71C_FACTOR && variant != G5P_M71C_VARIANT) ++ return G5P_M71C_BAD_VARIANT; ++ return G5P_M71C_OK; ++} ++ ++/* ++ * One store into the staging image (the factor only) and its read-back. The ++ * caller fails the map unless the read-back equals the request. ++ */ ++static inline u32 g5p_m71c_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 requested, bool *wrote) ++{ ++ u8 *word = g5p_object(p, root, shared, G5P_M71C_SLOT) + G5P_M71C_OFFSET; ++ ++ *wrote = false; ++ if (requested == G5P_M71C_FACTOR) { ++ g5p_put32(word, requested); ++ *wrote = true; ++ } ++ return g5p_le32(word); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio-licence.h b/drivers/gpu/apple-g5p/mmio-licence.h +new file mode 100644 +index 000000000..d9d23ca2a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio-licence.h +@@ -0,0 +1,148 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU register licence (docs/gpu/experiments/mmio-access.md §4), pure so the host test ++ * (scripts/gpu/perf60-host/test_mmio.c) drives the same code. iOS's ++ * AGXMetaFirmware_isGPURegisterAccessible (fffffff006ca10ec) licenses an R0 access when PMGR gfx ++ * is on and the wrapper is up (fender+0xc), R1+0 bit 29 is clear (the firmware's power-down / ++ * clock-change routine 0x1001cec8 raises it before the GPC goes to 0) and the GPC pstate nibble is ++ * 1..meta+0x710. We are stricter: R1+0x1000 bit 4 (a transition in flight) must be clear and ++ * R1+0 == 0 (a dead wrapper, Apple's ca05d4 vt+0x70 branch) refuses too. ++ * ++ * Order, and nothing else touches the bus: the RAM terms (no MMIO when any fails), then R1+0, then ++ * R1+0x1000, then at most G5P_MMIO_MAX_OPS R0 accesses, then R1+0x1000 and R1+0 again at release ++ * to account a sample whose licence changed under it (stale: data loss, not exclusion). ++ * A refusal never waits and never wakes (no R1+0x14 write): the caller decides whether to retry. ++ */ ++#ifndef G5P_MMIO_LICENCE_H ++#define G5P_MMIO_LICENCE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_MMIO_MAX_OPS 8U /* R0 accesses per ticket */ ++#define G5P_MMIO_R1_CTRL \ ++ 0x0U /* R1+0: bit 29 = firmware power-down in progress */ ++#define G5P_MMIO_R1_GPC \ ++ 0x1000U /* R1+0x1000: bit 4 transition, nibble = pstate */ ++#define G5P_MMIO_BIT29 BIT(29) ++ ++enum g5p_mmio_why { ++ G5P_MMIO_OK, ++ G5P_MMIO_OFF, /* RAM: power epoch not readable (no MMIO done) -> -ENODEV */ ++ G5P_MMIO_BUSY, /* the lock was held (try mode, IRQ census) -> -EAGAIN, no MMIO */ ++ G5P_MMIO_DEAD, /* R1+0 == 0 */ ++ G5P_MMIO_FWOFF, /* R1+0 bit 29 */ ++ G5P_MMIO_TRANS, /* R1+0x1000 bit 4 */ ++ G5P_MMIO_PARKED, /* nibble 0 */ ++ G5P_MMIO_OVERMAX, /* nibble > max pstate */ ++ G5P_MMIO_PARKEPOCH, /* R0 in a park epoch (g5p_mmio_r0_regime_ok) */ ++ G5P_MMIO_NR, ++}; ++ ++static const char *const g5p_mmio_why_names[G5P_MMIO_NR] = { ++ "read", "skip-off", "skip-busy", ++ "skip-dead", "skip-bit29", "skip-transition", ++ "skip-parked", "skip-overmax", "skip-park-epoch", ++}; ++ ++/* The RAM preconditions (§4.2 step 2). b->failed is deliberately not one: the fault read runs ++ * after it latches, as Apple's does. ++ */ ++struct g5p_mmio_ram { ++ bool powered, fender_ready, mmio_off, have_boot, released, stopped; ++}; ++ ++static inline bool g5p_mmio_ram_ok(const struct g5p_mmio_ram *r) ++{ ++ return r->powered && r->fender_ready && !r->mmio_off && r->have_boot && ++ r->released && !r->stopped; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_r1_0(u32 r1_0) ++{ ++ if (!r1_0) ++ return G5P_MMIO_DEAD; ++ if (r1_0 & G5P_MMIO_BIT29) ++ return G5P_MMIO_FWOFF; ++ return G5P_MMIO_OK; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_gpc(u32 gpc, u32 max) ++{ ++ if (gpc & 0x10) ++ return G5P_MMIO_TRANS; ++ if (!(gpc & 0xf)) ++ return G5P_MMIO_PARKED; ++ if ((gpc & 0xf) > max) ++ return G5P_MMIO_OVERMAX; ++ return G5P_MMIO_OK; ++} ++ ++/* ++ * R0 only where the firmware cannot park under it. The governor's park does not raise R1+0 ++ * bit 29 (mmio-access.md §2.3: only the power-down routine 0x1001cec8 does, a few calls before it ++ * drops the GPC), so against a park the licence's R1 checks are a race: a check that passes can be ++ * followed by the GPC going to 0 before the R0 load. The never-park envelope (boot_gpu_perf >= ++ * 100, latched at map: the policy writer's x >= 100, q >= 1, the governor never selects pstate 0) ++ * has no such window; at pin 400/500 the 3D-completion and clock samples saw bit 29 0 times, while ++ * the park tuple's rung 5 (2026-11-03) saw it on 749 of 983 3D completions and then froze the SoC. ++ * iOS reads R0 only at a fault event, never at idle. `override` = mmio_r0_park (DFU-class rungs ++ * only). ++ */ ++static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool override) ++{ ++ return override || gpu_perf >= 100; ++} ++ ++/* ++ * meta+0x710 as written into the boot image; 0 (no table) means the firmware stays at pstate 1. ++ */ ++static inline u32 g5p_mmio_max_pstate(u32 meta710) ++{ ++ return meta710 < 1 ? 1 : meta710 > 15 ? 15 : meta710; ++} ++ ++typedef u32 (*g5p_mmio_rd_t)(void *ctx, u32 off); ++ ++/* §4.2 steps 2-4 under the caller's lock: no bus access unless the RAM terms pass, R1+0 before ++ * R1+0x1000, nothing after a refusal. *r1_0 / *gpc stay U32_MAX when not read. ++ */ ++static inline enum g5p_mmio_why g5p_mmio_licence(const struct g5p_mmio_ram *ram, ++ u32 max, g5p_mmio_rd_t rd, ++ void *ctx, u32 *r1_0, u32 *gpc) ++{ ++ enum g5p_mmio_why why; ++ ++ *r1_0 = 0xffffffffU; ++ *gpc = 0xffffffffU; ++ if (!g5p_mmio_ram_ok(ram)) ++ return G5P_MMIO_OFF; ++ *r1_0 = rd(ctx, G5P_MMIO_R1_CTRL); ++ why = g5p_mmio_check_r1_0(*r1_0); ++ if (why) ++ return why; ++ *gpc = rd(ctx, G5P_MMIO_R1_GPC); ++ return g5p_mmio_check_gpc(*gpc, max); ++} ++ ++/* An R0 op is allowed while ops < G5P_MMIO_MAX_OPS; the counter counts refusals too. */ ++static inline bool g5p_mmio_op_take(u32 *ops) ++{ ++ return (*ops)++ < G5P_MMIO_MAX_OPS; ++} ++ ++/* §4.3 release: the licence fields (nibble, bit 4, bit 29) unchanged and still licensing. */ ++static inline bool g5p_mmio_stale(u32 r1_0, u32 gpc, u32 r1_0_end, u32 gpc_end, ++ u32 max) ++{ ++ if ((gpc & 0x1f) != (gpc_end & 0x1f) || ++ (r1_0 & G5P_MMIO_BIT29) != (r1_0_end & G5P_MMIO_BIT29)) ++ return true; ++ return g5p_mmio_check_r1_0(r1_0_end) || ++ g5p_mmio_check_gpc(gpc_end, max); ++} ++ ++/* mmio_selftest rung 3/5 expectations: CORE_REVISION and CORE_ID as read at bring-up. */ ++#define G5P_MMIO_EXPECT_20 0x00090f00U ++#define G5P_MMIO_EXPECT_18 0x011b0602U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio.inc b/drivers/gpu/apple-g5p/mmio.inc +new file mode 100644 +index 000000000..b8c25bd8d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio.inc +@@ -0,0 +1,295 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The one GPU register accessor (docs/gpu/experiments/mmio-access.md §4), Apple's licence from ++ * AGXMetaFirmware_isGPURegisterAccessible (mmio-licence.h) under g->mmio_lock: ++ * ++ * if (!g5p_mmio_hold(g, &t, 0)) { ++ * v = g5p_mmio_r0_read(g, &t, off); // <= G5P_MMIO_MAX_OPS, nothing else in between ++ * g5p_mmio_release(g, &t); // false: the licence changed under the sample ++ * } ++ * ++ * hold takes mmio_lock with IRQs off and returns with it held on success. It refuses instead of ++ * waiting and never wakes the GPU (no R1+0x14 write): -ENODEV when the power epoch is not ++ * readable (RAM only, no MMIO done), -EAGAIN when parked, bit 29, in transition, above the ++ * pstate table, or (G5P_MMIO_TRY) when the lock is taken; -EPERM for an R0 ticket in an epoch ++ * the firmware may park (boot_gpu_perf < 100, unless mmio_r0_park). Callable from hard IRQ, ++ * threaded IRQ and process context, inside event_lock and g->lock. Between hold and release: no ++ * printk, no other MMIO, no sleep, no allocation. ++ * ++ * mmio_selftest (0600, default 0, live) drives the proof rungs of §6: ++ * bit 0 rung 1/1b: 1 kHz hrtimer census, R1+0 and R1+0x1000 only (no R0) ++ * bit 1 rung 3/4: every 100 ms hold -> R0+0x20, R0+0x18 -> release (DFU-risk with parks: rung 4) ++ * bit 2 rung 5: the same at every 3D completion decision (drm side; DFU-risk, park tuple) ++ * bit 3 idle check: after 5, 30 and 120 s with no job, the PMGR gfx word, then (gfx on only) ++ * R1+0x1000 and R1+0; never R0 ++ * Lines: GPU-MMIO sample (per sample, at most one per source per second unless the outcome ++ * changes or the sample is stale/bad), GPU-MMIO idle, GPU-MMIO stats (every 10 s while on and ++ * once when it goes back to 0). ++ */ ++#define G5P_MMIO_TRY \ ++ BIT(0) /* trylock: refuse busy instead of spinning (the IRQ census) */ ++/* An R1-only ticket, counted in the census class: held whenever the RAM terms pass (R1 is ++ * readable while PMGR gfx is on, mmio-access.md §1.3), R1+0 and R1+0x1000 both read even when the ++ * licence refuses; R0 only if the licence passed (t->r0_ok). ++ */ ++#define G5P_MMIO_CENSUS BIT(1) ++ ++struct g5p_mmio_ticket { ++ u32 r1_0, gpc, r1_0_end, gpc_end, flags, ops; ++ u64 t_ns, t_end_ns; ++ unsigned long irq; ++ enum g5p_mmio_why why; ++ bool held, stale, r0_ok; ++}; ++ ++enum { ++ G5P_MMS_CENSUS, ++ G5P_MMS_IDLE, ++ G5P_MMS_3D, ++ G5P_MMS_FAULT, ++ G5P_MMS_PROBE, ++ G5P_MMS_NR ++}; ++static const char *const g5p_mms_names[G5P_MMS_NR] = { ++ "census", "idle-100ms", "3d-done", "fault-read", "dc-probe", ++}; ++ ++#define G5P_MMIO_RING 32 ++struct g5p_mmio_rec { ++ u64 t_ns; ++ u32 r1_0, gpc, us; ++ u8 why, stale; ++}; ++ ++struct g5p_mmio_self { ++ struct g5p_diag *g; ++ struct hrtimer timer; ++ struct delayed_work work; ++ bool census_on, was_on; ++ u64 stats_ns; ++ /* counters: [0] R1-only census tickets, [1] R0 tickets */ ++ atomic_t n[2][G5P_MMIO_NR]; ++ atomic_t stale[2], r0_ops, bad; ++ /* census transitions, written by the hrtimer, drained by the work */ ++ spinlock_t ring_lock; ++ struct g5p_mmio_rec ring[G5P_MMIO_RING]; ++ u32 ring_head, ring_tail, ring_drop; ++ struct g5p_mmio_rec census_last; ++ u8 census_prev_why, census_prev_stale; ++ /* per-source print limiter */ ++ u64 printed_ns[G5P_MMS_NR]; ++ u8 printed_why[G5P_MMS_NR]; ++ /* idle check */ ++ u64 job_ns; ++ u32 idle_done; ++ /* bit 4, the GPU clock: per Fender pstate nibble, the highest rate seen and the sample counts */ ++ u32 clk_max_khz[16], clk_cr_khz[16], clk_n[16], clk_busy[16], clk_reset; ++ u64 clk_printed_ns; ++}; ++ ++/* mmio_selftest bit 4: the gap between the two tickets of one clock sample */ ++ ++/* R0 tickets in a park epoch (boot_gpu_perf < 100): refused unless 1 (g5p_mmio_r0_regime_ok). */ ++ ++static u32 g5p_mmio_rd_r1(void *ctx, u32 off) ++{ ++ return readl((void __iomem *)ctx + off); ++} ++ ++static void g5p_mmio_count(struct g5p_diag *g, const struct g5p_mmio_ticket *t) ++{ ++ struct g5p_mmio_self *s = READ_ONCE(g->mmio_self); ++ u32 cls = t->flags & G5P_MMIO_CENSUS ? 0 : 1; ++ ++ if (!s) ++ return; ++ atomic_inc(&s->n[cls][t->why]); ++ if (t->stale) ++ atomic_inc(&s->stale[cls]); ++ if (t->ops && cls) ++ atomic_add(min_t(u32, t->ops, G5P_MMIO_MAX_OPS), &s->r0_ops); ++} ++ ++static int g5p_mmio_hold(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags) ++{ ++ struct g5p_mmio_ram ram = { 0 }; ++ struct g5p_boot *b; ++ unsigned long irq; ++ ++ memset(t, 0, sizeof(*t)); ++ t->flags = flags; ++ t->r1_0 = t->gpc = t->r1_0_end = t->gpc_end = U32_MAX; ++ if (flags & G5P_MMIO_TRY) { ++ if (!spin_trylock_irqsave(&g->mmio_lock, irq)) { ++ t->why = G5P_MMIO_BUSY; ++ goto refused; ++ } ++ } else { ++ spin_lock_irqsave(&g->mmio_lock, irq); ++ } ++ t->t_ns = ktime_get_mono_fast_ns(); ++ ram.mmio_off = g->mmio_off; ++ ram.powered = READ_ONCE(g->powered); ++ ram.fender_ready = READ_ONCE(g->fender_ready); ++ /* mmio_off first: a boot struct behind a gate is never dereferenced */ ++ b = ram.mmio_off ? NULL : READ_ONCE(g->boot); ++ if (b) { ++ ram.have_boot = true; ++ ram.released = READ_ONCE(b->released); ++ ram.stopped = READ_ONCE(b->stopped); ++ } ++ /* before any bus access: an R0 ticket in a park epoch is refused outright */ ++ if (!(flags & G5P_MMIO_CENSUS) && b && ++ !g5p_mmio_r0_regime_ok(b->gpu_perf, (0))) { ++ t->why = G5P_MMIO_PARKEPOCH; ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ t->why = g5p_mmio_licence(&ram, READ_ONCE(g->mmio_max_pstate), ++ g5p_mmio_rd_r1, g->window[1], &t->r1_0, ++ &t->gpc); ++ if (t->why && (!(flags & G5P_MMIO_CENSUS) || t->why == G5P_MMIO_OFF)) { ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ if (t->gpc == U32_MAX) ++ t->gpc = readl( ++ g->window[1] + ++ G5P_MMIO_R1_GPC); /* census: after an R1+0 refusal */ ++ t->r0_ok = !t->why; ++ t->irq = irq; ++ t->held = true; ++ return 0; ++refused: ++ g5p_mmio_count(g, t); ++ /* a park epoch does not change by waiting: -EPERM, never retried (g5p_mmio_hold_wait) */ ++ return t->why == G5P_MMIO_OFF ? -ENODEV : ++ t->why == G5P_MMIO_PARKEPOCH ? -EPERM : ++ -EAGAIN; ++} ++ ++/* Only between a successful hold and its release; past G5P_MMIO_MAX_OPS no bus access. */ ++static u32 g5p_mmio_r0_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->rgx + off); ++} ++ ++/* R0 writes count as ops too (the slave-port request words of the pin-200 probe). */ ++static bool g5p_mmio_r0_write(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off, u32 v) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return false; ++ writel(v, g->rgx + off); ++ return true; ++} ++ ++/* Any R1 word under a held ticket (census or licensed); counts as an op. */ ++static u32 g5p_mmio_r1_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->window[1] + off); ++} ++ ++/* Process context: refusals (-EAGAIN) retried outside the lock every 20-40 us, <= budget_us. */ ++static int g5p_mmio_hold_wait(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags, u32 budget_us, u32 *tries) ++{ ++ u64 t0 = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ might_sleep(); ++ for (;;) { ++ (*tries)++; ++ ret = g5p_mmio_hold(g, t, flags); ++ if (ret != -EAGAIN || ktime_get_mono_fast_ns() - t0 >= ++ (u64)budget_us * NSEC_PER_USEC) ++ return ret; ++ usleep_range(20, 40); ++ } ++} ++ ++static bool g5p_mmio_release(struct g5p_diag *g, struct g5p_mmio_ticket *t) ++{ ++ if (!t->held) ++ return false; ++ t->gpc_end = readl(g->window[1] + G5P_MMIO_R1_GPC); ++ t->r1_0_end = readl(g->window[1] + G5P_MMIO_R1_CTRL); ++ t->stale = t->r0_ok && ++ g5p_mmio_stale(t->r1_0, t->gpc, t->r1_0_end, t->gpc_end, ++ READ_ONCE(g->mmio_max_pstate)); ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ t->held = false; ++ spin_unlock_irqrestore(&g->mmio_lock, t->irq); ++ g5p_mmio_count(g, t); ++ return !t->stale; ++} ++ ++static const char *g5p_mmio_did(const struct g5p_mmio_ticket *t) ++{ ++ return t->stale ? "stale" : g5p_mmio_why_names[t->why]; ++} ++ ++static u32 g5p_mmio_us(const struct g5p_mmio_ticket *t) ++{ ++ return t->t_end_ns > t->t_ns ? ++ (u32)div_u64(t->t_end_ns - t->t_ns, NSEC_PER_USEC) : ++ 0; ++} ++ ++/* The per-sample line, after release. force: print regardless of the per-source limiter. */ ++static void g5p_mmio_sample_line(struct g5p_diag *g, u32 src, ++ const struct g5p_mmio_ticket *t, u32 *v, ++ u32 nv, bool force) ++{ ++ struct g5p_mmio_self *s = g->mmio_self; ++ u64 now = ktime_get_mono_fast_ns(); ++ char vs[9 * G5P_MMIO_MAX_OPS + 1] = "-"; ++ u32 len = 0; ++ ++ if (s && !force && !t->stale && s->printed_why[src] == t->why && ++ now - s->printed_ns[src] < NSEC_PER_SEC) ++ return; ++ if (s) { ++ s->printed_ns[src] = now; ++ s->printed_why[src] = t->why; ++ } ++ for (u32 i = 0; i < nv && i < G5P_MMIO_MAX_OPS; i++) ++ len += scnprintf(vs + len, sizeof(vs) - len, "%s%08x", ++ i ? "," : "", v[i]); ++ dev_info( ++ g->dev, ++ "GPU-MMIO sample why=%s did=%s r1_0=%08x gpc=%08x r1_0_end=%08x gpc_end=%08x " ++ "max=%u v=%s us=%u end=mmio\n", ++ g5p_mms_names[src], g5p_mmio_did(t), t->r1_0, t->gpc, ++ t->r1_0_end, t->gpc_end, READ_ONCE(g->mmio_max_pstate), vs, ++ g5p_mmio_us(t)); ++} ++ ++/* Rung 3/5: one licensed ticket, CORE_REVISION and CORE_ID. */ ++ ++/* drm side: rung 5 at the 3D completion decision, and the idle clock for bit 3. */ ++ ++/* Bit 4: the GPU core clock. CR_TIMER (R0+0x160) ticks once per 256 core clocks whether or not a ++ * phase is open (r3clk: 2823 kHz at pstate 4 = 722.7 MHz, 3703 kHz at 948 MHz), so CR_TIMER over ++ * host time x 256 is the clock at any moment. The Rogue PERF_TA_CYCLE / PERF_3D_CYCLE / ++ * PERF_TA_OR_3D_CYCLE counters (R0+0x6020/0x6028/0x6038, 32 bit) advance at the core clock while a ++ * TA or 3D phase is open (340.0 MHz through open TA phases, docs/gpu/experiments/46-device-tests.md): ++ * over host time, the highest rate per pstate (a window busy all through) is the clock again. One ++ * sample = two licensed tickets, mmio_clock_gap_us apart, each reading the three counters and ++ * CR_TIMER's low word back to back. A cycle rate above 2 GHz is a firmware counter clear ++ * (R0+0x6000 = 1, 0x1001ab00) inside the window and is dropped. ++ */ ++ ++/* Bit 3: does the GPC power down at full idle in state 0? The PMGR word is always readable; ++ * R1 only while the gate reads on (actual == target == 0xf), through a census ticket so the RAM ++ * terms and the lock still apply. A refused licence still returns the R1 words it read. ++ */ +diff --git a/drivers/gpu/apple-g5p/mmu-attrs.inc b/drivers/gpu/apple-g5p/mmu-attrs.inc +new file mode 100644 +index 000000000..694552016 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-attrs.inc +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include ++#include ++ ++/* Read the actual stable kernel mapping of a retained DMA allocation. ++ * DMA memory may be a vmalloc PTE or a cached linear-map block. ++ */ ++static u64 g5p_table_cpu_descriptor(unsigned long va) ++{ ++ pgd_t *pgd = pgd_offset_k(va); ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ ++ if (pgd_none(READ_ONCE(*pgd)) || pgd_bad(READ_ONCE(*pgd))) ++ return 0; ++ p4d = p4d_offset(pgd, va); ++ if (p4d_none(READ_ONCE(*p4d)) || p4d_bad(READ_ONCE(*p4d))) ++ return 0; ++ pud = pud_offset(p4d, va); ++ if (pud_leaf(READ_ONCE(*pud))) ++ return pud_val(READ_ONCE(*pud)); ++ if (pud_none(READ_ONCE(*pud)) || pud_bad(READ_ONCE(*pud))) ++ return 0; ++ pmd = pmd_offset(pud, va); ++ if (pmd_leaf(READ_ONCE(*pmd))) ++ return pmd_val(READ_ONCE(*pmd)); ++ if (pmd_none(READ_ONCE(*pmd)) || pmd_bad(READ_ONCE(*pmd))) ++ return 0; ++ return pte_val(READ_ONCE(*pte_offset_kernel(pmd, va))); ++} ++ ++static void g5p_table_attrs(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *p) ++{ ++ u64 desc, mair = read_sysreg(mair_el1); ++ u32 index, attr; ++ ++ if (!p->cpu) ++ return; ++ desc = g5p_table_cpu_descriptor((unsigned long)p->cpu); ++ index = (desc & PTE_ATTRINDX_MASK) >> 2; ++ attr = (mair >> (index * 8)) & 0xff; ++ g5p_mmu_record( ++ g, ++ "MMU allocation %s DMA=%pad coherent=%u cpu_desc=%016llx AttrIndx=%u " ++ "MAIR=%016llx attr=%02x table_memtype=%s\n", ++ stage, &p->dma, dev_is_dma_coherent(g->dev), desc, index, mair, ++ attr, ++ !desc ? "unknown" : ++ attr == 0xff ? "Normal-WB" : ++ attr == 0x44 ? "Normal-NC" : ++ "other"); ++} +diff --git a/drivers/gpu/apple-g5p/mmu-tables.inc b/drivers/gpu/apple-g5p/mmu-tables.inc +new file mode 100644 +index 000000000..94ae6ed8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-tables.inc +@@ -0,0 +1,57 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Fixed, pre-release aliases only. No firmware patch or live table surgery. */ ++#define G5P_TABLE_ALIAS_VA (G5P_ROOT_VA + 0x100000ULL) ++#define G5P_TABLE_COOKIE 0x3ffcU ++ ++static void g5p_table_clean_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ unsigned long start; ++ ++ if (!p->cpu) ++ return; ++ start = (unsigned long)p->cpu; ++ /* Explicit architecture PoC operation, even if DT calls this coherent. ++ * dma_sync_single_for_device is not an API for coherent allocations. ++ */ ++ dcache_clean_poc(start, start + G5P_GPU_PAGE); ++} ++ ++static void g5p_tables_clean(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *user, u32 count) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ g5p_mmu_record( ++ g, ++ "MMU tables %s clean_poc=%u dma_coherent=%u dma_pgprot=%016llx; coherent DMA " ++ "API, extra clean is experimental\n", ++ stage, 1, dev_is_dma_coherent(g->dev), ++ (u64)pgprot_val(dma_pgprot(g->dev, PAGE_KERNEL, 0))); ++ for (u32 i = 0; i < count; i++) ++ g5p_table_attrs(g, stage, &user[i]); ++ g5p_table_attrs(g, stage, &b->catalogue); ++ g5p_table_attrs(g, stage, &b->directory); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_attrs(g, stage, &b->leaf[i]); ++ ; ++ ; ++ for (u32 i = 0; i < count; i++) ++ g5p_table_clean_page(g, &user[i]); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_clean_page(g, &b->leaf[i]); ++ g5p_table_clean_page(g, &b->sink_leaf); ++ g5p_table_clean_page(g, &b->sink_directory); ++ g5p_table_clean_page(g, &b->directory); ++ g5p_table_clean_page(g, &b->catalogue); ++ dsb(sy); ++ dma_wmb(); ++ g5p_mmu_record( ++ g, ++ "MMU tables %s PoC complete user_pages=%u boot_leaves=%u; dsb sy before publication\n", ++ stage, count, b->leaves); ++} ++ ++/* Deliberately separate from g5p_meta_read(): the generic slave guard stays ++ * latched after reset. This opt-in reader can address only the preinstalled ++ * table aliases and one immutable root control word, never private META RAM. ++ */ +diff --git a/drivers/gpu/apple-g5p/pack.h b/drivers/gpu/apple-g5p/pack.h +new file mode 100644 +index 000000000..942b44ba3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/pack.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_PACK_H ++#define APPLE_G5P_PACK_H ++ ++#include "bits-compat.h" ++#include "ldr.h" ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_IMAGE_SIZE 0x30000 ++#define G5P_CONFIG_WRITES 27 ++ ++struct g5p_ldr_image { ++ u8 *data; ++ u32 config[G5P_CONFIG_WRITES][2]; ++ u32 config_count; ++ u32 t1_pc; ++ u32 t1_sp; ++}; ++ ++/* Exact C0 ranges derived from AGXMetaFirmware_layoutAndLoadLDR ...6c9f1bc. ++ * GPU virtual mappings and the boot config at +0x200 are intentionally separate. ++ */ ++static inline int g5p_image_offset(u32 addr, u32 size, u32 *offset) ++{ ++ static const struct { ++ u32 base, size, packed; ++ } segments[] = { ++ { 0x10000000, 0x18000, 0x04000 }, ++ { 0x20000000, 0x04000, 0x1c000 }, ++ { 0xc0000000, 0x04000, 0x00000 }, ++ { 0xe0001000, 0x04000, 0x20000 }, ++ { 0x80000080, 0x055dc, 0x24000 }, ++ }; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(segments); i++) { ++ if (addr >= segments[i].base && size <= segments[i].size && ++ addr - segments[i].base <= segments[i].size - size) { ++ *offset = segments[i].packed + addr - segments[i].base; ++ return 0; ++ } ++ } ++ return -EINVAL; ++} ++ ++static inline int g5p_pack_record(void *ctx, enum g5p_ldr_op op, u32 addr, ++ u32 size, const u8 *data) ++{ ++ struct g5p_ldr_image *image = ctx; ++ u32 i, offset; ++ int ret; ++ ++ switch (op) { ++ case G5P_LOAD: ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memcpy(image->data + offset, data, size); ++ break; ++ case G5P_ZERO: ++ /* Apple does not apply the local/core-memory ZeroMem records here. */ ++ if (addr >= 0x80000000 && addr < 0x88000000) ++ break; ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memset(image->data + offset, 0, size); ++ break; ++ case G5P_CONFIG: ++ for (i = 0; i < size; i += 12) { ++ if (image->config_count == G5P_CONFIG_WRITES) ++ return -EINVAL; ++ image->config[image->config_count][0] = ++ g5p_le32(data + i + 4); ++ image->config[image->config_count++][1] = ++ g5p_le32(data + i + 8); ++ } ++ break; ++ case G5P_START: ++ image->t1_sp = g5p_le32(data + 4); ++ image->t1_pc = g5p_le32(data + 8); ++ break; ++ } ++ return 0; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/perf-envelope.h b/drivers/gpu/apple-g5p/perf-envelope.h +new file mode 100644 +index 000000000..57387bcba +--- /dev/null ++++ b/drivers/gpu/apple-g5p/perf-envelope.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 clock: the inputs of the firmware's policy-state writer (0x10008218) and of its governor, ++ * so the firmware itself moves the GPC from pstate 1 to q = pin / 100. Both compute ++ * x = min(policy+0x6f0, policy+0x688, ctrl+0x102c) ++ * (META CS is unsigned lower; iOS's live policy+0x38 equals this min in every sample), store ++ * policy+8 = 0 when x % 100 == 0 (2 when x < 100), and request pstate q = x / 100 (policy+0x28). ++ * +0x6f0 and +0x688 are recomputed by the firmware from its coefficient block unless their ++ * overrides +0x6f4 / +0x68c are nonzero, and the firmware itself reloads those overrides from ++ * the control object on every pass, whatever the record held: +0x6f4 = ctrl+0x1040 (0x1000a1e4, ++ * 0x1000b388, 0x1000c568) and +0x68c = ctrl+0x1038 (0x1000a74c, 0x1000b10c, 0x1000c2fc). iOS ++ * leaves both control words 0 and arms the sampler (policy+4 = 16); this boot pins them ++ * (ctrl+0x1040 = pin, ctrl+0x1038 = 400) and leaves the sampler off (policy+4 = 0), so there is ++ * no idle parking at pstate 0. The pstate table and limits are iOS's (live j71t capture, ++ * 2026-11-01): meta+0x710/0x714 = 4/5, MHz {0, 340, 474, 550, 723}, ctrl+0x1028/0x1030 = ++ * 3348/400; the floor ctrl+0x102c is the pin. RAM only, before release. ++ */ ++#ifndef APPLE_G5P_PERF_ENVELOPE_H ++#define APPLE_G5P_PERF_ENVELOPE_H ++ ++#include "bits-compat.h" ++#define G5P_PERF_POLICY 0x4aU ++#define G5P_PERF_META 0x4dU ++#define G5P_PERF_CTL 0x5eU ++#define G5P_PERF_TOP 400U ++ ++static const u32 g5p_perf_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline bool g5p_perf_valid(u32 pin) ++{ ++ return pin <= G5P_PERF_TOP && !(pin % 100); ++} ++ ++static inline void g5p_perf_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 pin) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ u8 *meta = g5p_object(p, root, shared, G5P_PERF_META); ++ u8 *ctl = g5p_object(p, root, shared, G5P_PERF_CTL); ++ u32 i; ++ ++ g5p_put32(meta + 0x710, ARRAY_SIZE(g5p_perf_mhz) - 1); ++ g5p_put32(meta + 0x714, ARRAY_SIZE(g5p_perf_mhz)); ++ for (i = 0; i < ARRAY_SIZE(g5p_perf_mhz); i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_perf_mhz[i]); ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, pin); ++ g5p_put32(ctl + 0x1030, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1038, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1040, pin); ++ g5p_put32(policy + 0x688, G5P_PERF_TOP); ++ g5p_put32(policy + 0x68c, G5P_PERF_TOP); ++ g5p_put32(policy + 0x6f0, pin); ++ g5p_put32(policy + 0x6f4, pin); ++} ++ ++/* boot_gpu_oc_mhz (gpu-oc.h): state 5 = the overclock record; the firmware's table, ceilings and ++ * policy limits extended to index 5 so the pin 500 and the power-on request of state 5 are in range ++ * (the firmware derives its clock entry priv+0x2c4 = meta+0x718[state] from Fender, 0x1001c838). ++ */ ++ ++/* ++ * boot_gpu_perf_seed (perf60-clock.md "Clock-change drain RE" §3): in policy state 0 the ++ * firmware ++ * never changes pstate while powered; only its boot sampler (0x1000aa80, before ALIVE) moves the ++ * GPC, when policy+0x28 != Fender & 0xf. Seeding the policy record with the firmware's own ++ * steady-state output for the pin, +0x28 = +0x2c = q, +0x30 = 0, +8 = 0, makes that sampler ++ * request q through its setter (0x1001c9b8 -> 0x1001c88c: pol34 = q, R1+0x1000 = q | 0x10), ++ * which is ++ * iOS's park -> reboot -> sampler path. q = pin / 100 for pins 200..400, else 0 (no seed). ++ */ ++static inline u32 g5p_perf_seed_q(u32 pin) ++{ ++ return pin >= 200 && pin <= G5P_PERF_TOP && !(pin % 100) ? pin / 100 : ++ 0; ++} ++ ++/* Seed 2 adds what the sampler needs to get past its first-call init: 0x1000aa80 at 0x1000abb0 ++ * takes policy+0 == 0 as "never sampled" and only initialises (0x1000ae88: +0 = 1, its time words) ++ * without reaching the target compare at 0x1000ada4 or the setter; its sampling path divides by ++ * policy+4 (0x10022f88 at 0x1000ac28). The boot call at 0x10000b54 is its only caller, so a fresh ++ * record never moves Fender. iOS's record persists across firmware reboots: +0 = 1 and +4 = 16 in ++ * every sample of the 2026-11-01 capture (idle and load). Seed 2 writes those two as iOS has them. ++ * Seed 2 on hardware (2026-11-03): no ALIVE, Fender 0. With +0 = 1 the sampler takes its sampling ++ * path when (now - policy+0x8f8) / 24000 / policy+4 >= 1 (0x1000ac10..ac5c): +0x8f8 is the host's ++ * map-time stamp, ~3 s old at release, so it runs the load governor (0x1000afa4 -> 0x1000b050, busy ++ * accumulators policy+0x62c..0x68c, all 0 on a fresh record) and lands on pstate 0. Seed 3 ++ * (boot.inc ++ * g5p_boot_release) restamps +0x8f8 right before the release stream: < 1 period elapsed, the load ++ * update is skipped (0x1000ac64 -> 0x1000acb0 -> 0x1000ada4) and the compare against +0x28 = q ++ * reaches the setter (0x1000ae68 -> 0x1001c9b8). ++ */ ++static inline void g5p_perf_seed(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 q, bool ios_record) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ ++ g5p_put64(policy + 0x28, (u64)q << 32 | q); ++ g5p_put32(policy + 0x30, 0); ++ g5p_put32(policy + 8, 0); ++ if (ios_record) { ++ g5p_put32(policy + 0, 1); ++ g5p_put32(policy + 4, 16); ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/power-r33.inc b/drivers/gpu/apple-g5p/power-r33.inc +new file mode 100644 +index 000000000..ce8a18f3c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r33.inc +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R33 (ds-r33-b knob A): the kexec GFX cycle holds ps_gfx off for ~10 us, and ++ * R0+0x418/0x41c (no reset) keeps the previous boot's value through it. At the ++ * first Fender wake, before any GPU MMIO and before any GPU IRQ is requested, ++ * hold the same PMGR gate (0x80340, gate124) off for boot_gfx_off_ms, then ++ * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset, ++ * auto). Every exit leaves the gate requested on. ++ */ ++#define G5P_R33_GFX_PS 0x80340U ++#define G5P_R33_GFX_OFF_MAX_MS 2000U ++#define G5P_R33_GFX_POLL_US 10000U ++ ++static inline bool g5p_r33_gfx_on(u32 raw) ++{ ++ return (raw & 0xf) == 0xf && ((raw >> 4) & 0xf) == 0xf; ++} ++ ++/* Only a plain, fully-on, software-controlled gate may be cycled. */ ++static inline bool g5p_r33_gfx_cyclable(u32 raw) ++{ ++ return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31) && ++ !((raw >> 28) & 1); ++} ++ ++static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) ++{ ++ u32 before, held, v = 0, after = 0; ++ u64 t0; ++ int ret, on; ++ ++ if (!ms) ++ return 0; ++ if (ms > G5P_R33_GFX_OFF_MAX_MS || !pmgr) ++ return -EINVAL; ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &before); ++ if (ret) ++ return ret; ++ if (!g5p_r33_gfx_cyclable(before)) { ++ dev_info( ++ dev, ++ "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset/auto or not on\n", ++ ms, before); ++ return -EBUSY; ++ } ++ ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0); ++ if (!ret) ++ ret = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, v, ++ !((v >> 4) & 0xf), 10, ++ G5P_R33_GFX_POLL_US); ++ t0 = ktime_get_ns(); ++ if (!ret) ++ msleep(ms); ++ held = (u32)((ktime_get_ns() - t0) / 1000); ++ if (!ret) ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &v); ++ /* Never leave the GPU domain off, whatever happened above. */ ++ on = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0xf); ++ if (!on) ++ on = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, after, ++ g5p_r33_gfx_on(after), 10, ++ G5P_R33_GFX_POLL_US); ++ dev_info( ++ dev, ++ "POWER gfx-off ms=%u held_us=%u before=%08x end=%08x after=%08x off_ret=%d " ++ "on_ret=%d\n", ++ ms, held, before, v, after, ret, on); ++ if (!ret && (v >> 4) & 0xf) ++ ret = -EIO; ++ return ret ?: on; ++} +diff --git a/drivers/gpu/apple-g5p/power-r41.inc b/drivers/gpu/apple-g5p/power-r41.inc +new file mode 100644 +index 000000000..9e5c1e668 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r41.inc +@@ -0,0 +1,135 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* J71t live ADT + ApplePMGR_restoreDeviceBridgeSettings (5a39918). ++ * Volatile fabric MMIO only. GFX selects bridge 8, then bridge 9; the two ++ * version-0 counter controls follow bridge 8's table, before bridge 9. ++ * Keep devm_ioremap/readl/writel identical to boot.inc's swirq mapping. ++ */ ++static const unsigned int gfx_bridge_replay = 0; ++ ++struct g5p_r41_write { ++ u32 bridge, offset, value; ++}; ++ ++static const struct g5p_r41_write g5p_r41_writes[] = { ++ { 8, 0x0000, 0x00000120 }, { 8, 0x0018, 0x80010005 }, ++ { 8, 0x001c, 0x00010001 }, { 8, 0x410c, 0x00000003 }, ++ { 8, 0x4100, 0x00000003 }, { 9, 0x0000, 0x00697101 }, ++ { 9, 0x0004, 0x00697101 }, { 9, 0x0008, 0x00697101 }, ++ { 9, 0x000c, 0x00697101 }, { 9, 0x0010, 0x00697101 }, ++ { 9, 0x0014, 0x00697101 }, { 9, 0x0018, 0x00697101 }, ++ { 9, 0x001c, 0x00697101 }, { 9, 0x0020, 0x00697101 }, ++ { 9, 0x0024, 0x00697101 }, { 9, 0x0028, 0x00697101 }, ++ { 9, 0x002c, 0x00697101 }, { 9, 0x0030, 0x00697101 }, ++ { 9, 0x0034, 0x00697101 }, { 9, 0x0038, 0x00697101 }, ++ { 9, 0x003c, 0x00697101 }, { 9, 0x0040, 0x00697101 }, ++ { 9, 0x0044, 0x00697101 }, { 9, 0x0048, 0x00697101 }, ++ { 9, 0x004c, 0x00697101 }, { 9, 0x0050, 0x00697101 }, ++ { 9, 0x0054, 0x00697101 }, { 9, 0x0058, 0x00697101 }, ++ { 9, 0x005c, 0x00697101 }, { 9, 0x0060, 0x00697101 }, ++ { 9, 0x0064, 0x00697101 }, { 9, 0x0068, 0x00697101 }, ++ { 9, 0x006c, 0x00697101 }, { 9, 0x0070, 0x00697101 }, ++ { 9, 0x0074, 0x00697101 }, { 9, 0x0078, 0x00697101 }, ++ { 9, 0x007c, 0x00697101 }, ++}; ++ ++static struct { ++ void __iomem *window[2]; ++ unsigned int mode; ++ int result; ++ bool attempted, ready, firmware_logged, ta_logged; ++} g5p_r41_bridge; ++ ++/* Reject anything outside the exact write allowlist, including unaligned ++ * offsets and other registers inside the two mapped ADT ranges. ++ */ ++static bool g5p_r41_valid(const struct g5p_r41_write *w) ++{ ++ if (w->offset & 3) ++ return false; ++ if (w->bridge == 8) ++ return w->offset < 0x5000 && ++ (w->offset == 0 || w->offset == 0x18 || ++ w->offset == 0x1c || w->offset == 0x410c || ++ w->offset == 0x4100); ++ return w->bridge == 9 && w->offset <= 0x7c; ++} ++ ++static void g5p_r41_bridge_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ u32 value = ++ readl(g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ ++ dev_info( ++ g->dev, ++ "BRIDGE r41 stage=%s mode=%u index=%u bridge=%u off=%04x value=%08x want=%08x\n", ++ stage, g5p_r41_bridge.mode, i, w->bridge, w->offset, ++ value, w->value); ++ } ++} ++ ++/* Called under g->lock after ACTIVE (and any r33 off/on), before Fender. ++ * Latch errors too: never retry a partial experiment in the same boot. ++ */ ++static int g5p_r41_bridge_prepare(struct g5p_diag *g) ++{ ++ unsigned int i, mode = gfx_bridge_replay; ++ ++ if (!mode) ++ return 0; ++ if (g5p_r41_bridge.attempted) ++ return g5p_r41_bridge.result; ++ g5p_r41_bridge.attempted = true; ++ g5p_r41_bridge.mode = mode; ++ g5p_r41_bridge.result = -EINVAL; ++ if (mode > 4) ++ return -EINVAL; ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) ++ if (!g5p_r41_valid(&g5p_r41_writes[i])) ++ return -EINVAL; ++ g5p_r41_bridge.result = -ENOMEM; ++ g5p_r41_bridge.window[0] = devm_ioremap(g->dev, 0x201f00000ULL, 0x5000); ++ if (!g5p_r41_bridge.window[0]) ++ return -ENOMEM; ++ g5p_r41_bridge.window[1] = devm_ioremap(g->dev, 0x201f80000ULL, 0x1000); ++ if (!g5p_r41_bridge.window[1]) ++ return -ENOMEM; ++ /* Read all 37 even in split modes; never read unmapped/partial state. */ ++ g5p_r41_bridge_snapshot(g, "before"); ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ ++ if (mode == 1 || (mode == 3 && w->bridge != 8) || ++ (mode == 4 && w->bridge != 9)) ++ continue; ++ writel(w->value, ++ g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ dev_info( ++ g->dev, ++ "BRIDGE r41 write mode=%u index=%u bridge=%u off=%04x value=%08x\n", ++ mode, i, w->bridge, w->offset, w->value); ++ } ++ g5p_r41_bridge_snapshot(g, "after"); ++ g5p_r41_bridge.ready = true; ++ g5p_r41_bridge.result = 0; ++ return 0; ++} ++ ++static void g5p_r41_bridge_firmware(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.firmware_logged) { ++ g5p_r41_bridge.firmware_logged = true; ++ g5p_r41_bridge_snapshot(g, "firmware-alive"); ++ } ++} ++ ++static void g5p_r41_bridge_ta(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.ta_logged) { ++ g5p_r41_bridge.ta_logged = true; ++ g5p_r41_bridge_snapshot(g, "first-TA-kick"); ++ } ++} +diff --git a/drivers/gpu/apple-g5p/r74e112.h b/drivers/gpu/apple-g5p/r74e112.h +new file mode 100644 +index 000000000..cacac73ac +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r74e112.h +@@ -0,0 +1,137 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R74E112_H ++#define G5P_R74E112_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * Wake112: the R74E GPC1 power proof when ALIVE preempts the on-poll. ++ * ++ * R74E (boot.inc) requests GPC=1 and polls R1+0x1000 until the state field reads 1. ++ * The poll stops as soon as ALIVE is decoded: after ALIVE only the first-ALIVE owner ++ * may read R1. On a warm, kexec-cycled boot the firmware can reach ALIVE before the ++ * poll's first read (Wake111 20261031T201022Z: off_ns=1500 on_ns=375 raw=ffffffff, ++ * the U32_MAX "not read" sentinel). Wake111 then latched a power abort although the ++ * state was never sampled. ++ * ++ * Wake112 defers that one proof to the owner instead, but only when the on-poll took ++ * no sample: the poll counts its reads, and the U32_MAX raw is never taken as "not read". ++ * The owner's existing first-ALIVE read of R1+0x1000 (g5p_r73e_irq_submit, before any ++ * register write) is the proof. State 1 sets release_ready and the unchanged cold ++ * sequence continues. Any other value (0, 0x10, 0x11, ffffffff, ...) is an absorbing ++ * power abort. An on-poll that reads state 1 never defers. An on-poll that sampled ++ * before ALIVE (0x11 in rounds 74e..79b, or any other word) and every other R74E failure ++ * (off-poll, timeout, GPC1 write refused) abort exactly as before. ++ * ++ * States (r73b->early_proof): ++ * NONE no deferral: the poll decided, or the release has not run ++ * DEFERRED ALIVE preempted the on-poll; one owner read is owed (generation recorded) ++ * TAKEN the once-owner claimed it; it settles with no exit in between ++ * PROVEN that read had state 1: release_ready ++ * FAILED any other value, another generation, a lost precondition, or an ALIVE ++ * wait that ended with the proof still owed: power abort, absorbing ++ * ++ * Pure policy: no register access, no lock, no kernel API. ++ */ ++ ++#define G5P_PX_STATE_MASK 0x1fU ++#define G5P_PX_STATE_ON 0x01U ++ ++enum g5p_px { ++ G5P_PX_NONE, ++ G5P_PX_DEFERRED, ++ G5P_PX_TAKEN, ++ G5P_PX_PROVEN, ++ G5P_PX_FAILED, ++ G5P_PX_N, ++}; ++ ++static const char *const g5p_px_name[G5P_PX_N] = { ++ "none", "deferred", "taken", "proven", "failed", ++}; ++ ++/* Why a deferred proof failed (NONE: it did not). */ ++enum g5p_px_why { ++ G5P_PXW_NONE, ++ G5P_PXW_STATE, /* the read's state field is not 1 */ ++ G5P_PXW_STALE, /* deferred in another reset generation */ ++ G5P_PXW_LOST, /* ALIVE or the release lost, stopped or failed since */ ++ G5P_PXW_RETIRED, /* a FINAL teardown forbids R1 */ ++ G5P_PXW_UNTAKEN, /* the ALIVE wait ended with the proof still owed */ ++ G5P_PXW_N, ++}; ++ ++static const char *const g5p_px_why_name[G5P_PXW_N] = { ++ "none", "state", "stale", "lost", "retired", "untaken", ++}; ++ ++/* What the R74E release path latches. */ ++enum g5p_pxp { ++ G5P_PXP_READY, /* the on-poll read state 1: release_ready (cold) */ ++ G5P_PXP_DEFER, /* ALIVE preempted the on-poll before its first read */ ++ G5P_PXP_ABORT, /* anything else: power abort (cold) */ ++}; ++ ++/* ++ * on_ran: the core request, the off-poll and the GPC1 write succeeded and the on-poll ++ * ran. ret: the last step's result. preempted: the on-poll stopped on ALIVE. ++ * samples: the on-poll's R1+0x1000 reads. Only a preemption with no read defers (no ++ * abort, no release_ready); a sampled one saw the state and aborts. ++ */ ++static inline enum g5p_pxp g5p_px_poll(bool on_ran, int ret, bool preempted, ++ u32 samples) ++{ ++ if (!on_ran) ++ return G5P_PXP_ABORT; ++ if (!ret) ++ return G5P_PXP_READY; ++ return preempted && !samples ? G5P_PXP_DEFER : G5P_PXP_ABORT; ++} ++ ++/* The first-ALIVE owner's admission. */ ++enum g5p_pxg { ++ G5P_PXG_READY, /* release_ready: the cold owner path */ ++ G5P_PXG_PROVE, /* a deferred proof is owed: the once-owner's read decides */ ++ G5P_PXG_WAIT, /* nothing proven yet, or another owner holds the proof */ ++ G5P_PXG_REFUSE, /* power abort or a failed proof: no register access */ ++}; ++ ++static inline enum g5p_pxg g5p_px_gate(bool power_abort, bool release_ready, ++ u32 px) ++{ ++ if (power_abort || px == G5P_PX_FAILED) ++ return G5P_PXG_REFUSE; ++ if (release_ready) ++ return G5P_PXG_READY; ++ if (px == G5P_PX_DEFERRED) ++ return G5P_PXG_PROVE; ++ return G5P_PXG_WAIT; ++} ++ ++/* RAM facts the once-owner samples after taking the proof, before its one read. */ ++struct g5p_px_pre { ++ u32 gen, gen_now; /* DS108 generation at the deferral and now */ ++ bool alive, released, stopped, failed; ++ bool r1_allowed; /* g5p_r8_r1_allowed() */ ++}; ++ ++static inline enum g5p_px_why g5p_px_check(const struct g5p_px_pre *p) ++{ ++ if (!p->gen || p->gen != p->gen_now) ++ return G5P_PXW_STALE; ++ if (!p->alive || !p->released || p->stopped || p->failed) ++ return G5P_PXW_LOST; ++ if (!p->r1_allowed) ++ return G5P_PXW_RETIRED; ++ return G5P_PXW_NONE; ++} ++ ++/* The proof: only the state field 1 is power-on. */ ++static inline bool g5p_px_on(u32 raw) ++{ ++ return G5P_GPC_ON(raw); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/r87b-attach.h b/drivers/gpu/apple-g5p/r87b-attach.h +new file mode 100644 +index 000000000..81600ac06 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r87b-attach.h +@@ -0,0 +1,34 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R87B_ATTACH_H ++#define G5P_R87B_ATTACH_H ++/* R87b DIAGNOSTIC ONLY, never production-safe: the DC3 attach policy selector. ++ * (docs/gpu/experiments/87b-build-report.md) ++ * ++ * Two campaigns (r85 stage 2, r87 stage 1) never reached GPU work: the DC3 attach pair was ++ * published, the immediate gate (R1+0x1000 & 0x1f == 1) read 0x10/0x11, the host GPC off/on ++ * repair timed out (-110) and attach stopped with -115 before the RAM-only consume wait ran, ++ * after which the machine froze. Every retained instance of that failure shows the firmware ++ * consuming the pair on its own (dc=3/3 at export, no bell rung). boot_r87b_attach selects: ++ * 0 the original attach policy, exactly r87: a closed gate is repaired by the host (R1+0x20000 ++ * clear, GPC off/on, ALIVE-owned doorbell). The matched control. ++ * 1 self-drain (the r82b design, docs/gpu/experiments/82-attach-reactivation-triage.md): a ++ * closed DC3 gate is only a phase sample of the firmware's GPC window. The host does no ++ * repair and rings no bell; it keeps the RAM-only 3 s consume wait of render-r74f.inc and ++ * continues through the firmware-MMU attach path (r74g) when the firmware consumed the pair. ++ * An active gate still rings the bell at once (the unchanged 80 % path). The TA kind is ++ * never affected. ++ * The selector is latched at map and read only through the latched copy. A non-zero value needs ++ * the r80 epoch (which implies r74h, r74g and r74f case 1): self-drain is only safe because r74g ++ * has the firmware own the first MMU pass, so no host R0 access is licensed by a rung bell. ++ * The TYPED_V3 production path latches policy 1 with boot_r74f_case = boot_r74g_case = 1 and NO ++ * r80 / r74h (boot-r73b.inc, "production path"); g5p_r87b_attach_valid() describes the r80 ++ * epoch only. ++ */ ++#include ++#include ++ ++#define G5P_R87B_ATTACH_ORIGINAL 0U ++#define G5P_R87B_ATTACH_SELF_DRAIN 1U ++#define G5P_R87B_ATTACH_MAX 1U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/readback.h b/drivers/gpu/apple-g5p/readback.h +new file mode 100644 +index 000000000..fdd71ab30 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/readback.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_READBACK_H ++#define G5P_READBACK_H ++ ++#include "bits-compat.h" ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++enum g5p_readback_state { ++ G5P_READBACK_WRAPPER_ONLY, ++ G5P_READBACK_BASELINE, ++ G5P_READBACK_RESET_HELD, ++}; ++ ++/* Returning wrapper registers in exp03; never a META slave-port register. */ ++static const u32 g5p_fender_regs[] = { ++ 0, 8, 0x10, 0x14, 0x1000, 0x1004, 0x2010, 0x2014, 0x20000, ++}; ++ ++/* Reset halves first: the only post-arm R0 readback needed by the gate. */ ++static const u32 g5p_rgx_regs[] = { ++ 0x100, 0x104, 0, 4, 8, 0xc, 0x18, 0x20, 0x80, 0x84, 0x88, 0x8c, ++}; ++ ++static inline u32 g5p_readback_rgx_count(u32 gpc, enum g5p_readback_state state) ++{ ++ if (!G5P_GPC_ON(gpc)) ++ return 0; ++ if (state == G5P_READBACK_RESET_HELD) ++ return 2; ++ return state == G5P_READBACK_BASELINE ? ARRAY_SIZE(g5p_rgx_regs) : 0; ++} ++ ++static inline bool g5p_slave_access_allowed(bool reset_transition) ++{ ++ return !reset_transition; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/recover.inc b/drivers/gpu/apple-g5p/recover.inc +new file mode 100644 +index 000000000..f550d1010 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/recover.inc +@@ -0,0 +1,101 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * In-boot GPU recovery (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault isolation). ++ * ++ * A render job that stopped the firmware (its event stream went silent, or a fault) blocks the ++ * one TA/3D channel for every client, and the firmware has no abort. iOS ends that state with a ++ * firmware stop, a platform power cycle and a new firmware boot (kext 6c9e070, firmware-re README ++ * "Fault recovery"). This is the same, built from two proven pieces: ++ * - the reboot quiesce's freeze, IRQ mask and PMGR gfx gate cycle (teardown.inc g5p_kq_*), which ++ * on hardware cycled a wedged, failed device safely (deskfold reboot notifier: plan=-+cycle, ++ * then the next kernel came up warm-cycled); ++ * - the bootstrap's own stage functions (wake, map, arm, load, release, events, control), the ++ * same code the sysfs commands run after the next kernel's cycle on every warm hop. ++ * The old struct g5p_boot is frozen and left allocated (devm; a few hundred KiB per recovery), ++ * so a stale pointer anywhere still reads valid memory; a new one is built by the stages. id and ++ * sample are not rerun (they refuse once the boot saw a reset transition; map needs only the ++ * cached core id), nor is irq-test (refused in an r74 epoch). The firmware image is reused. ++ * ++ * Core lock held, no job in flight (the render scheduler's worker). Returns 0 with a live new ++ * epoch, or an error with the device left frozen and failed. ++ */ ++ ++/* The bootstrap's stages after the wake, as the sysfs runner orders them. */ ++static const char *const g5p_recover_stages[] = { ++ "map", ++ "arm", ++ "load", ++ "release", ++ "events", ++ "control-prepare", ++ "control-submit", ++ "control-wait", ++}; ++ ++static int __maybe_unused g5p_recover(struct g5p_diag *g, const char *why, ++ u32 *stage_out) ++{ ++ struct g5p_boot *old = g->boot; ++ struct g5p_kq_ctx kq = { .g = g }; ++ u64 t0 = ktime_get_ns(); ++ u32 stage = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ *stage_out = 0; ++ if (!old || !READ_ONCE(old->released) || g5p_r8_is_retired() || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || !g->pmgr || ++ !g->id_checked) ++ return -EPERM; ++ /* The quiesce's freeze and IRQ mask, then the line is given back for the new epoch. */ ++ g5p_kq_freeze(&kq); ++ g5p_kq_irq_off(&kq); ++ if (old->irq_ready && old->irq > 0) ++ devm_free_irq(g->dev, old->irq, g); ++ /* mmio-access.md G5: the GPU is marked unusable before its power is cut (Apple clears ++ * fender+0xc inside setPlatformPower(0), before the gate); g5p_kq_cycle() also sets mmio_off. ++ */ ++ WRITE_ONCE(g->fender_ready, false); ++ ret = g5p_kq_cycle(&kq); ++ ret = g5p_ds_hook_recover(g, ret) ?: ret; ++ if (ret) ++ goto out; ++ /* A new epoch: the stages build a new struct g5p_boot from here. */ ++ WRITE_ONCE(g->boot, NULL); ++ stage = 1; ++ ret = g5p_fender_wake(g); ++ for (u32 i = 0; !ret && i < ARRAY_SIZE(g5p_recover_stages); i++) { ++ const char *s = g5p_recover_stages[i]; ++ ++ stage = i + 2; ++ ret = strncmp(s, "control-", 8) ? g5p_boot_command(g, s) : ++ g5p_lifecycle_command(g, s); ++ if (ret == -EALREADY) ++ ret = 0; ++ } ++ if (!ret && (!g->boot || !READ_ONCE(g->boot->alive) || ++ !READ_ONCE(g->boot->control_complete))) ++ ret = -EIO; ++ /* The claim gate again: the held jobs need ALIVE proven in the new generation. */ ++ if (!ret) ++ ret = g5p_ds_hook_claim(g); ++ if (!ret) { ++ /* The render claim is kept: no diagnostic command may reset this epoch. */ ++ WRITE_ONCE(g->boot->user_compute_owned, true); ++ stage = 0; ++ } ++out: ++ if (ret && g->boot) ++ WRITE_ONCE(g->boot->failed, true); ++ g->last_result = ret; ++ *stage_out = stage; ++ dev_emerg( ++ g->dev, ++ "GPU-RECOVER why=%s ret=%d stage=%s ms=%llu gen=%u end=recover\n", ++ why, ret, ++ !stage ? (ret ? "cycle" : "done") : ++ stage == 1 ? "wake" : ++ g5p_recover_stages[stage - 2], ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC, g5p_ds_gen()); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/render-layout-r22.h b/drivers/gpu/apple-g5p/render-layout-r22.h +new file mode 100644 +index 000000000..8391a8320 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout-r22.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_R22_H ++#define G5P_RENDER_LAYOUT_R22_H ++ ++#include "render-layout.h" ++ ++#define G5P_R22_TA 0x54000U ++#define G5P_R22_3D 0x58000U ++#define G5P_R22_PM 0x5c000U ++#define G5P_R22_W 0x400U ++#define G5P_R22_TA_SKU 0xa00U ++#define G5P_R22_3D_SKU 0xd00U ++#define G5P_R22_TQ 0x1800U ++#define G5P_R22_STAMP 0x1840U ++#define G5P_R22_DEP 0x1900U ++#define G5P_R22_BLOCKS 32U ++#define G5P_R22_PHYS_PAGES (G5P_R22_BLOCKS * 8) ++#define G5P_R22_AUX 0x101000000ULL ++#define G5P_R22_RT 0x101020000ULL ++#define G5P_R22_RT_AUX 0x101030000ULL ++ ++/* 6cb124c/13f4/14dc: PA PPNs, GPU F and META links are distinct. ++ * Caller provides complete 16KiB control/list/F allocations. Check the whole ++ * physical input before writing anything. No synthesized PM catalogue encoder. ++ */ ++ ++/* 6ca6d5c arithmetic oracle + 6cba480/6c7b64c. Native legacy mapping ++ * dialect has bit36 clear: L=H, Laux=Haux. No PA substitution or float math. ++ */ ++ ++static const u32 g5p_r22_wait_words[] = { ++ 0x201, 0x138204, 0x10, 0x8388204, 0, 0x104000a, 0x1f, ++ 0x1f, 0x8388204, 1, 0x104000a, 0x1f, 0x1f, 0xf48008, ++ 0x10000015, 4, 0, 0x14, 0x109f, 0x13000a, 0x4000, ++ 0x4000, 0x138204, 0x4000, 0x8388204, 0, 0x104000a, 8, ++ 8, 0x8388204, 1, 0x104000a, 8, 8, ++}; ++ ++/* Full normal timestamped recipes, independent stamp identities TA0/3D1. ++ * k is dependency+4 (zero on the proven fresh initializer), not MMU tag. ++ */ ++ ++/* PM bank0 save writes runtime qwords before the a0/a8/b0 backing anchors. */ ++static inline bool g5p_r22_pm_saved(const u8 *state) ++{ ++ for (u32 off = 8; off < 0xa0; off += 4) ++ if (g5p_le32(state + off)) ++ return true; ++ return false; ++} ++ ++/* Never trust a firmware pointer until it equals our retained stamp word. */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-layout.h b/drivers/gpu/apple-g5p/render-layout.h +new file mode 100644 +index 000000000..f2a0373da +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_H ++#define G5P_RENDER_LAYOUT_H ++ ++#include "boot-layout.h" ++ ++/* Offline construction primitives only. No TA/3D publication path exists. ++ * 6caf9f8/6cb2378: caller must recover N/S1/S2/S3, not infer them from USC ++ * count or target dimensions. Sizes are bounded to u32 allocation extents. ++ */ ++struct g5p_pm_sizes { ++ u32 control, page_lists, backing; ++}; ++ ++static inline int g5p_pm_sizes(u32 n, u32 blocks, u64 s1, u64 s2, u64 s3, ++ struct g5p_pm_sizes *out) ++{ ++ u64 limit = 0xffffffffULL; ++ u64 a, b; ++ ++ if (!n || !blocks || !s1 || !s2 || !s3) ++ return -EINVAL; ++ if (n > (limit - 0x80) / 0xc0 || blocks > (limit - 0x44) / 0x24 || ++ s1 > limit / n || s2 > limit / n || s3 > limit) ++ return -EOVERFLOW; ++ a = s1 * n; ++ b = s2 * n; ++ if (a > limit - b || a + b > limit - s3) ++ return -EOVERFLOW; ++ out->control = 0x80 + n * 0xc0; ++ out->page_lists = 0x44 + blocks * 0x24; ++ out->backing = a + b + s3; ++ return 0; ++} ++ ++/* Only four recovered backing-pointer writes, not a complete PM record. ++ * Caller owns a fully initialized0xc0 record and validates GPU mappings. ++ */ ++static inline int g5p_pm_record_backing(u8 *record, u32 index, u32 n, u64 base, ++ u64 s1, u64 s2, u64 s3, u32 meta) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (index >= n) ++ return -EINVAL; ++ ret = g5p_pm_sizes(n, 1, s1, s2, s3, &sizes); ++ if (ret) ++ return ret; ++ if (base > ~0ULL - sizes.backing) ++ return -EOVERFLOW; ++ g5p_put64(record + 0xa8, base + index * s1); ++ g5p_put64(record + 0xa0, base + n * s1 + index * s2); ++ g5p_put64(record + 0xb0, base + n * s1 + n * s2); ++ g5p_put32(record + 0xb8, meta); ++ return 0; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-bind.h b/drivers/gpu/apple-g5p/render-user-bind.h +new file mode 100644 +index 000000000..3bbbb1e0f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-bind.h +@@ -0,0 +1,285 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Firmware slot pairs of render contexts (docs/gpu/multi-client-design.md, stage 1). ++ * ++ * Each render context owns a TA and a 3D firmware slot in 7..255, both pointing at its own ++ * page catalogue. The pair is bound once by two DC3 records {3, slot, 0, 0, 0}, 3D first, ++ * then TA, with the catalogue PA in 0x86[slot] and the host tag in 0x84[slot]. The firmware's ++ * DC3 handler installs CBASE and copies the host tag to 0x85[slot]. ++ * ++ * These helpers only read and write those RAM words, so they cannot reach a register and are ++ * safe after a job has retired. Pure functions, host-tested ++ * (scripts/gpu/multiclient1-host/test_render_bind.c). The caller provides g5p_le32(), ++ * g5p_put32() and g5p_put64(). ++ */ ++#ifndef G5P_RENDER_USER_BIND_H ++#define G5P_RENDER_USER_BIND_H ++ ++#define G5P_RU_SLOT_MIN 7U ++#define G5P_RU_SLOT_MAX 255U ++/* RAM-only wait for the firmware to consume a published DC3 batch. */ ++#define G5P_RU_BIND_MS 3000U ++/* Host tag of a pre-bound spare pair (render_user_spare_pairs): never zero. */ ++#define G5P_RU_SPARE_TAG 0x00400000U ++/* Host tag of a parked pair (render_user_close_park): never zero, never a spare's. */ ++#define G5P_RU_PARK_TAG 0x00500000U ++#define G5P_RU_MAX_SPARES 32U ++ ++/* The words one pair's binding touches. Pointers, because the DC records may straddle the ++ * shared pages. Index [e]: 0 TA slot, 1 3D slot. ++ */ ++struct g5p_ru_bind_mem { ++ u8 *rec[2][5]; /* the records at start and start + 1 */ ++ u8 *pa[2]; /* 0x86[slot], 64 bit */ ++ u8 *host[2]; /* 0x84[slot] */ ++ u8 *cached[2]; /* 0x85[slot] */ ++}; ++ ++/* One RAM sample of those words and of the DC ring control. */ ++struct g5p_ru_bind_sample { ++ u32 cons, prod; ++ u32 rec[2][5]; ++ u64 pa[2]; ++ u32 host[2], cached[2]; ++}; ++ ++/* g5p_ru_bind_busy() */ ++#define G5P_RU_BIND_BUSY_RING 0x1U /* the DC ring is not idle */ ++#define G5P_RU_BIND_BUSY_SLOT 0x2U /* a slot has a PA or a host tag */ ++ ++/* g5p_ru_bind_check(), the r74g criteria */ ++#define G5P_RU_BIND_BAD_RING 0x1U ++#define G5P_RU_BIND_BAD_RECORD 0x2U ++#define G5P_RU_BIND_BAD_PA 0x4U ++#define G5P_RU_BIND_BAD_HOST 0x8U ++#define G5P_RU_BIND_BAD_CACHED 0x10U ++ ++enum g5p_ru_bind_wait { ++ G5P_RU_BIND_PENDING, ++ G5P_RU_BIND_CONSUMED, ++ G5P_RU_BIND_PROTO, ++ G5P_RU_BIND_TIMEOUT, ++}; ++ ++static inline bool g5p_ru_slot_used(const unsigned long *map, u32 slot) ++{ ++ return (map[slot / BITS_PER_LONG] >> (slot % BITS_PER_LONG)) & 1; ++} ++ ++/* The slot pair of a new render context: the two lowest free slots in 7..255. The lower one ++ * is the 3D slot, or the TA slot with ta_first (render_user_slot_variant 1). False when fewer ++ * than two are free. slot[0] is TA, slot[1] is 3D. ++ */ ++static inline bool g5p_ru_slot_pair(const unsigned long *map, bool ta_first, ++ u32 *slot) ++{ ++ u32 pick[2], n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < 2; s++) ++ if (!g5p_ru_slot_used(map, s)) ++ pick[n++] = s; ++ if (n < 2) ++ return false; ++ slot[0] = pick[ta_first ? 0 : 1]; ++ slot[1] = pick[ta_first ? 1 : 0]; ++ return true; ++} ++ ++/* Word j of record i of a pair: {3, slot, 0, 0, 0}, the 3D slot's record first. */ ++static inline u32 g5p_ru_bind_record(const u32 *slot, u32 i, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot[1 - i] : 0; ++} ++ ++static inline void g5p_ru_bind_read(const struct g5p_ru_bind_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_bind_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ s->rec[i][j] = g5p_le32(m->rec[i][j]); ++ for (u32 e = 0; e < 2; e++) { ++ s->pa[e] = (u64)g5p_le32(m->pa[e] + 4) << 32 | ++ g5p_le32(m->pa[e]); ++ s->host[e] = g5p_le32(m->host[e]); ++ s->cached[e] = g5p_le32(m->cached[e]); ++ } ++} ++ ++/* Before a publish: the ring must be idle and both slots unused (zero PA and host tag). */ ++static inline u32 g5p_ru_bind_busy(const struct g5p_ru_bind_sample *s) ++{ ++ u32 busy = 0; ++ ++ if (s->prod > 255 || s->cons != s->prod) ++ busy |= G5P_RU_BIND_BUSY_RING; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] || s->host[e]) ++ busy |= G5P_RU_BIND_BUSY_SLOT; ++ return busy; ++} ++ ++/* ++ * A parked pair (render_user_close_park, release8 quiet2): a closed context's slots, bound to the ++ * kernel's park catalogue (it maps only the firmware window) with the park tag, so that the PM's ++ * late write-back of that context's free list resolves instead of faulting on an empty PC level. ++ * Both slots on `park`, both host tags the park tag. park 0: nothing is parked. ++ */ ++static inline bool g5p_ru_bind_parked(const struct g5p_ru_bind_sample *s, ++ u64 park) ++{ ++ if (!park) ++ return false; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] != park || s->host[e] != G5P_RU_PARK_TAG) ++ return false; ++ return true; ++} ++ ++/* PA and tag of both slots, then the two records. The caller cleans the catalogue first and ++ * stores the producer afterwards, each behind a barrier. ++ */ ++static inline void g5p_ru_bind_publish(const struct g5p_ru_bind_mem *m, ++ const u32 *slot, u64 root, u32 tag) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ u32 e = 1 - i; ++ ++ g5p_put64(m->pa[e], root); ++ g5p_put32(m->host[e], tag); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[i][j], g5p_ru_bind_record(slot, i, j)); ++ } ++} ++ ++/* After the ring reached `target`: the records intact, both slots on `root`, and the firmware ++ * tag equal to the host tag (the DC3 handler ran for both). ++ */ ++static inline u32 g5p_ru_bind_check(const struct g5p_ru_bind_sample *s, ++ const u32 *slot, u64 root, u32 tag, ++ u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[i][j] != g5p_ru_bind_record(slot, i, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ for (u32 e = 0; e < 2; e++) { ++ if (s->pa[e] != root) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host[e] != tag) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached[e] != tag) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ } ++ return bad; ++} ++ ++/* One sample of a published batch. The caller takes `expired` before reading the ring, so ++ * the sample taken at the deadline still counts. ++ */ ++static inline int g5p_ru_bind_step(u32 cons, u32 prod, u32 target, bool expired) ++{ ++ if (prod != target || cons > 255) ++ return G5P_RU_BIND_PROTO; ++ if (cons == target) ++ return G5P_RU_BIND_CONSUMED; ++ return expired ? G5P_RU_BIND_TIMEOUT : G5P_RU_BIND_PENDING; ++} ++ ++/* ++ * Unbind (iOS AGXGart_detachSlotsAndDestroyAddressSpace, firmware-re README round 15 §1): PA ++ * and host tag of a slot go to zero and one DC3 record {3, slot, 0, 0, 0} follows. The DC3 ++ * handler then installs CBASE 0, copies the zero tag to 0x85[slot] and invalidates the slot's ++ * PC level. The slots of one batch take consecutive ring entries, one record each, behind a ++ * single producer store. The caller must have retired every use of the slots first. ++ */ ++#define G5P_RU_USER_SLOTS (G5P_RU_SLOT_MAX - G5P_RU_SLOT_MIN + 1) ++ ++struct g5p_ru_slot_mem { ++ u8 *rec[5]; /* the slot's record in the batch */ ++ u8 *pa, *host, ++ *cached; /* 0x86[slot] (64 bit), 0x84[slot], 0x85[slot] */ ++}; ++ ++struct g5p_ru_slot_sample { ++ u32 cons, prod; ++ u32 rec[5]; ++ u64 pa; ++ u32 host, cached; ++}; ++ ++/* The user slots the firmware may hold: those with a PA or a host tag. At most `max`. */ ++static inline u32 g5p_ru_held_slots(const u8 *pa, const u8 *host, u32 *slots, ++ u32 max) ++{ ++ u32 n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < max; s++) ++ if (g5p_le32(pa + s * 8) || g5p_le32(pa + s * 8 + 4) || ++ g5p_le32(host + s * 4)) ++ slots[n++] = s; ++ return n; ++} ++ ++/* A batch of n records fits the 256-entry ring behind an idle consumer. */ ++static inline bool g5p_ru_unbind_fits(u32 n) ++{ ++ return n && n <= G5P_RU_USER_SLOTS; ++} ++ ++static inline u32 g5p_ru_unbind_record(u32 slot, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot : 0; ++} ++ ++static inline void g5p_ru_slot_read(const struct g5p_ru_slot_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_slot_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 j = 0; j < 5; j++) ++ s->rec[j] = g5p_le32(m->rec[j]); ++ s->pa = (u64)g5p_le32(m->pa + 4) << 32 | g5p_le32(m->pa); ++ s->host = g5p_le32(m->host); ++ s->cached = g5p_le32(m->cached); ++} ++ ++/* PA and host tag to zero, then the record. The caller stores the producer behind a barrier. */ ++static inline void g5p_ru_unbind_publish(const struct g5p_ru_slot_mem *m, ++ u32 slot) ++{ ++ g5p_put64(m->pa, 0); ++ g5p_put32(m->host, 0); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[j], g5p_ru_unbind_record(slot, j)); ++} ++ ++/* After the ring reached `target`: the record intact, PA and host tag still zero, and the ++ * firmware's tag zero too, which only its DC3 handler writes (the slot was bound with a ++ * non-zero tag). The G5P_RU_BIND_BAD_* bits. ++ */ ++static inline u32 g5p_ru_unbind_check(const struct g5p_ru_slot_sample *s, ++ u32 slot, u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[j] != g5p_ru_unbind_record(slot, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ if (s->pa) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ return bad; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-close.h b/drivers/gpu/apple-g5p/render-user-close.h +new file mode 100644 +index 000000000..2aa461bce +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-close.h +@@ -0,0 +1,91 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Release of a closed render context (docs/gpu/multi-client-design.md, stage 2). ++ * ++ * A TYPED_V3 context whose handle is gone (DESTROY_CONTEXT or file close) gets one close item. ++ * It runs after the context's last job, in the job order, and decides from plain values what ++ * the firmware may still hold: ++ * ++ * RELEASE the firmware never held the pair: drop the references, free everything; ++ * UNBIND the pair is bound and every use retired: RAM-only DC3 unbind ++ * (g5p_rv3_unbind_slots()), then free on success, retain on failure; ++ * RETAIN the firmware may still use the pair or its memory: keep the context, its ++ * slots and its memory until reboot (the kexec quiesce detaches them then). ++ * ++ * The rule follows iOS AGXGart_detachSlotsAndDestroyAddressSpace (firmware-re README round 15 ++ * §1, "Linux sequence" step 8): stop new jobs, wait for every engine use, publish PA and tag ++ * zero, consume the DC3, and only then recycle the slots and free the translations. A slot ++ * returns to the allocator only after its unbind was consumed and verified, so an old unbind ++ * can never clear a new owner's PA, and the lowest free pair may be reused at once (iOS pops ++ * a freed slot off its free stack first). ++ * ++ * Pure function, host-tested (scripts/gpu/multiclient2-host/test_render_close.c). ++ */ ++#ifndef G5P_RENDER_USER_CLOSE_H ++#define G5P_RENDER_USER_CLOSE_H ++ ++enum g5p_ru_close_plan { ++ G5P_RU_CLOSE_RELEASE, ++ G5P_RU_CLOSE_UNBIND, ++ G5P_RU_CLOSE_RETAIN, ++}; ++ ++/* Why a bound or published pair is retained (bits). */ ++#define G5P_RU_CLOSE_UNVERIFIED \ ++ 0x01U /* a bind was published but never checked out */ ++#define G5P_RU_CLOSE_BAD \ ++ 0x02U /* a job of this context failed on the hardware */ ++#define G5P_RU_CLOSE_TERMINAL 0x04U /* the epoch noted a timeout or fault */ ++#define G5P_RU_CLOSE_FAILED 0x08U /* the device failed */ ++#define G5P_RU_CLOSE_STOPPED 0x10U /* the device stopped (teardown, quiesce) */ ++#define G5P_RU_CLOSE_SHUTDOWN \ ++ 0x20U /* admission closed for good (reboot quiesce) */ ++#define G5P_RU_CLOSE_QUEUED 0x40U /* a job of this context is still queued */ ++#define G5P_RU_CLOSE_PENDING 0x80U /* an earlier DC batch was never consumed */ ++#define G5P_RU_CLOSE_LOCK 0x100U /* the command lock could not be taken */ ++ ++struct g5p_ru_close_in { ++ bool retained; /* the firmware may hold the pair: a DC3 bind was published */ ++ bool bound; /* that bind was consumed and checked */ ++ bool bad, term_failed, dev_failed, stopped, shutdown, dc_pending; ++ u32 queued; ++}; ++ ++static inline const char *g5p_ru_close_name(int plan) ++{ ++ return plan == G5P_RU_CLOSE_RELEASE ? "release" : ++ plan == G5P_RU_CLOSE_UNBIND ? "unbind" : ++ "retain"; ++} ++ ++/* The plan, and in *why the G5P_RU_CLOSE_* bits that made it RETAIN (0 otherwise). */ ++static inline int g5p_ru_close_plan(const struct g5p_ru_close_in *in, u32 *why) ++{ ++ u32 w = 0; ++ ++ *why = 0; ++ if (!in->retained) ++ return in->queued ? (*why = G5P_RU_CLOSE_QUEUED, ++ G5P_RU_CLOSE_RETAIN) : ++ G5P_RU_CLOSE_RELEASE; ++ if (!in->bound) ++ w |= G5P_RU_CLOSE_UNVERIFIED; ++ if (in->bad) ++ w |= G5P_RU_CLOSE_BAD; ++ if (in->term_failed) ++ w |= G5P_RU_CLOSE_TERMINAL; ++ if (in->dev_failed) ++ w |= G5P_RU_CLOSE_FAILED; ++ if (in->stopped) ++ w |= G5P_RU_CLOSE_STOPPED; ++ if (in->shutdown) ++ w |= G5P_RU_CLOSE_SHUTDOWN; ++ if (in->queued) ++ w |= G5P_RU_CLOSE_QUEUED; ++ if (in->dc_pending) ++ w |= G5P_RU_CLOSE_PENDING; ++ *why = w; ++ return w ? G5P_RU_CLOSE_RETAIN : G5P_RU_CLOSE_UNBIND; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +new file mode 100644 +index 000000000..ccb7d2c8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The phase wait's pause between two RAM samples (perf60-fillrate.md §11.3 cut 1): woken by the ++ * firmware event, not a fixed 1-2 ms sleep. Host test: scripts/gpu/perf60-host/test_evwait.c. ++ * ++ * Protocol (g5p_rv3_wait): drain -> seen = event_serial -> sample RAM -> decide -> pause. ++ * The IRQ thread's drain decodes each firmware event, bumps event_serial under status_lock and ++ * wakes the queue. The pause sleeps until event_serial != seen or the safety timeout, so an ++ * event decoded at any point after `seen` was read (before, during or after the sample) ends the ++ * pause at once: no lost wakeup. The timeout is the safety net for a RAM state that changes ++ * without an event; the predicate stays level-triggered on every sample, as before. ++ * Once the phase's own completion event is latched (s->event) but the RAM rules still wait ++ * (idle words, retire stamp, PM receipt land a little after the event), the pause is a short ++ * G5P_EW_POST_US poll: there is no further event to wait for. ++ */ ++#ifndef G5P_RENDER_USER_EVWAIT_H ++#define G5P_RENDER_USER_EVWAIT_H ++ ++#define G5P_EW_POST_US \ ++ 100U /* after the phase's own event: completion is imminent */ ++#define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ ++ ++enum g5p_ew_act { ++ G5P_EW_LEGACY, /* render_user_event_wake 0: usleep_range(1000, 2000), release8 */ ++ G5P_EW_SHORT, /* the phase's event seen: sleep G5P_EW_POST_US */ ++ G5P_EW_EVENT, /* sleep until any event since `seen`, at most *wait_us */ ++}; ++ ++static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, ++ u32 *wait_us) ++{ ++ if (!knob_us) { ++ *wait_us = 0; ++ return G5P_EW_LEGACY; ++ } ++ if (phase_event) { ++ *wait_us = G5P_EW_POST_US; ++ return G5P_EW_SHORT; ++ } ++ *wait_us = knob_us < G5P_EW_POST_US ? G5P_EW_POST_US : ++ knob_us > G5P_EW_MAX_US ? G5P_EW_MAX_US : ++ knob_us; ++ return G5P_EW_EVENT; ++} ++ ++/* The wait condition: an event was decoded since `seen` was read. */ ++static inline bool g5p_ew_woken(u64 seen, u64 serial) ++{ ++ return serial != seen; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-fault.h b/drivers/gpu/apple-g5p/render-user-fault.h +new file mode 100644 +index 000000000..2af9f7513 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-fault.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * A client job that faults (docs/gpu/experiments/faultiso.md). Pure decisions, host-tested ++ * (scripts/gpu/faultiso-host/test_fault_iso.c). ++ * ++ * On this firmware a GPU page fault is not survivable in place. The firmware acknowledges the ++ * MMU fault itself (R0+0x138), reports it as event 4 and parks (firmware-re README "Fault ++ * ownership"); iOS answers event 4 only with its restart tail (kext 6c81dac -> 6c834a0: firmware ++ * stop, context and dependency reset, shared state rebuilt, firmware boot), and has no ++ * per-context ACK, channel reset or slot unbind for it. The no-reset path iOS does have, the ++ * firmware's own job kill (event 8, then device-control command 0xa), is not a page fault. So a ++ * job that stops the firmware, by a fault event or by a silent event stream, loses its context ++ * and the GPU is recovered in the boot (render_user_recover), or the epoch ends. ++ */ ++#ifndef G5P_RENDER_USER_FAULT_H ++#define G5P_RENDER_USER_FAULT_H ++ ++/* The errno a lost job reports: a timeout's own, else the fault's (-EIO for a fault event or a ++ * silent stream, -EPROTO, or the drain's error). `result` is the phase's G5P_RU_WAIT_*. ++ */ ++static inline int g5p_ru_lost_errno(u32 result, int err) ++{ ++ return result == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++} ++ ++/* ++ * Does a job that failed after touching the hardware fail the device too? The legacy profiles ++ * always do. A TYPED_V3 job does not when it timed out (nothing is wrong with the protocol and ++ * the firmware may still finish), when its bind failed (the DC ring is not on the path of the ++ * contexts already bound), or when it was lost (the executor ruled: the channel drains, or a ++ * recovery replaces the epoch). Its context is quarantined in every case. ++ */ ++static inline bool g5p_ru_job_fails_device(bool v3, int ret, bool bind_failed, ++ bool lost) ++{ ++ return !v3 || (ret != -ETIMEDOUT && !bind_failed && !lost); ++} ++ ++/* ++ * A device failure noted while a job or a drain runs (a fault event decoded mid-phase), or while ++ * a recovery is pending, is not yet ruled on: the executor either loses that job's context and ++ * recovers, or ends the epoch. Until then the other contexts wait (submit accepted and held, ++ * VM_BIND busy, context status not lost) instead of failing. The executor sets `recovering` ++ * before it clears `executing`, so a reader that loads `executing` with acquire and then ++ * `recovering` always sees one of them through a recovery. ++ */ ++static inline bool g5p_ru_ruling(bool executing, bool recovering) ++{ ++ return executing || recovering; ++} ++ ++/* render_user_test_fault_job: the context's job about to start (it has run `jobs` before) gets ++ * the unmapped control stream. Never the epoch's first job: its attach and bell are special. ++ */ ++static inline bool g5p_ru_fault_due(u32 fault_job, u32 jobs, bool first) ++{ ++ return fault_job && jobs + 1 == fault_job && !first; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-faultreg.h b/drivers/gpu/apple-g5p/render-user-faultreg.h +new file mode 100644 +index 000000000..83cac51f7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-faultreg.h +@@ -0,0 +1,124 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 fault address (render_user_fault_read, drm-render-v3.inc): decode of a GPU MMU fault ++ * status bank, R0+0xe150/e154 (bank 0) or R0+0xe158/e15c (bank 1), as high32:low32. ++ * ++ * The firmware's fault path (source 4 -> 1001a494) reads both banks with a 64-bit load, keeps ++ * only a class in its companion record and posts event 4 without the status, so the address ++ * exists only in these registers. iOS reads them itself in its fault handler (kext 6ca5c14, ++ * accelerator virtual+740) with exactly these fields (docs/gpu/firmware-re/README.md "Complete ++ * e150 decode"): bit 0 valid, 2:1 type (0 page invalid, 2 read-only, 3 PM/META protected), ++ * 3 read (0 write), 5:4 level (3 = catalogue), 11:6 requester (iOS table 6ff5ed8), 19:12 ++ * sideband, 27:20 MMU context (the slot), VA = (status >> 24) & 0xfffffffff0 (16-byte precision). ++ * Pure, host-tested (scripts/gpu/perf60-host/test_fault_decode.c). ++ */ ++#ifndef G5P_RENDER_USER_FAULTREG_H ++#define G5P_RENDER_USER_FAULTREG_H ++ ++#include "bits-compat.h" ++struct g5p_ru_mmu_fault { ++ bool valid, read; ++ u8 type, level, requester, sideband, context; ++ u64 va; ++}; ++ ++static inline struct g5p_ru_mmu_fault g5p_ru_mmu_fault_decode(u32 lo, u32 hi) ++{ ++ u64 s = (u64)hi << 32 | lo; ++ struct g5p_ru_mmu_fault f = { ++ .valid = s & 1, ++ .type = (s >> 1) & 3, ++ .read = (s >> 3) & 1, ++ .level = (s >> 4) & 3, ++ .requester = (s >> 6) & 0x3f, ++ .sideband = (s >> 12) & 0xff, ++ .context = (s >> 20) & 0xff, ++ .va = (s >> 24) & 0xfffffffff0ULL, ++ }; ++ ++ return f; ++} ++ ++static const char *const g5p_ru_mmu_fault_types[4] = { ++ "page-invalid", ++ "type1", ++ "read-only", ++ "pm-meta-protected", ++}; ++ ++/* Requester names, iOS kext table 6ff5ed8 (16-byte {name, class} entries; read from the j71t ++ * 16.7.10 kernelcache, 2026-11-03). 56 entries; a larger index prints its number only. ++ */ ++static const char *const g5p_ru_mmu_requesters[] = { ++ "META", ++ "TLA", ++ "HOST", ++ "VDMM", ++ "CDM", ++ "IPP", ++ "PM", ++ "TILING", ++ "MCU_TEXAS0", ++ "PDS_TEXAS0", ++ "PBE0_TEXAS0", ++ "PBE1_TEXAS0", ++ "VDMS_BLACKPEARL0", ++ "IPF_BLACKPEARL0", ++ "ISP_BLACKPEARL0", ++ "TPF_BLACKPEARL0", ++ "USCS_BLACKPEARL0", ++ "PPP_BLACKPEARL0", ++ "VCE_BLACKPEARL0", ++ "FBCDC_BLACKPEARL0", ++ "MCU_TEXAS1", ++ "PDS_TEXAS1", ++ "PBE0_TEXAS1", ++ "PBE1_TEXAS1", ++ "MCU_TEXAS2", ++ "PDS_TEXAS2", ++ "PBE0_TEXAS2", ++ "PBE1_TEXAS2", ++ "VDMS_BLACKPEARL1", ++ "IPF_BLACKPEARL1", ++ "ISP_BLACKPEARL1", ++ "TPF_BLACKPEARL1", ++ "USCS_BLACKPEARL1", ++ "PPP_BLACKPEARL1", ++ "VCE_BLACKPEARL1", ++ "FBCDC_BLACKPEARL1", ++ "MCU_TEXAS3", ++ "PDS_TEXAS3", ++ "PBE0_TEXAS3", ++ "PBE1_TEXAS3", ++ "MCU_TEXAS4", ++ "PDS_TEXAS4", ++ "PBE0_TEXAS4", ++ "PBE1_TEXAS4", ++ "VDMS_BLACKPEARL2", ++ "IPF_BLACKPEARL2", ++ "ISP_BLACKPEARL2", ++ "TPF_BLACKPEARL2", ++ "USCS_BLACKPEARL2", ++ "PPP_BLACKPEARL2", ++ "VCE_BLACKPEARL2", ++ "FBCDC_BLACKPEARL2", ++ "MCU_TEXAS5", ++ "PDS_TEXAS5", ++ "PBE0_TEXAS5", ++ "PBE1_TEXAS5", ++}; ++ ++static inline const char *g5p_ru_mmu_requester(u32 requester) ++{ ++ if (requester >= ARRAY_SIZE(g5p_ru_mmu_requesters)) ++ return "?"; ++ return g5p_ru_mmu_requesters[requester]; ++} ++ ++/* The range test of the VM edit log: does [va, va + size) hold the fault address? */ ++static inline bool g5p_ru_mmu_fault_in(u64 fault_va, u64 va, u64 size) ++{ ++ return fault_va >= va && fault_va - va < size; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-geometry.h b/drivers/gpu/apple-g5p/render-user-geometry.h +new file mode 100644 +index 000000000..2117d567b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-geometry.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_GEOMETRY_H ++#define G5P_RENDER_USER_GEOMETRY_H ++ ++/* Kernel-derived render-target geometry: AGXRenderTargetMemoryG5's arithmetic ++ * leaf 6ca6d5c (firmware round16 section 3) for one sample at the pixel ++ * centre (8,8) or four samples (docs/gpu/experiments/msaa.md), in integer ++ * arithmetic only. It is Rogue's non-simple ++ * parameter format: 32x32 tiles, 4x4 macrotiles of ++ * ALIGN(DIV_ROUND_UP(tiles, 4), 4) tiles per axis, five-byte region headers ++ * per padded tile, a tail pointer cache of next_pow2(max(padded tiles))^2 ++ * eight-byte entries. The scale pairs are the leaf's IEEE singles ++ * 0x3e89309c / n, 0x3fddb3d9 / n (fdiv, round to nearest even) and n * 16. ++ * tools/gpu/test_rt_oracle.c compares every output with the unmodified leaf ++ * for every width and height in 1..G5P_RU_MAX_DIM. ++ * ++ * G5P_RU_MAX_DIM is rogue_get_render_size_max() for this tiling class; it ++ * is a memory bound (16 MiB private window, 8 MiB main ring), not a field ++ * limit: the 9-bit TE macrotile and 10-bit padded-size fields would hold ++ * twice as much. ++ */ ++#define G5P_RU_MAX_DIM 8192U ++#define G5P_RU_SCALE_A 0x3e89309cU ++#define G5P_RU_SCALE_B 0x3fddb3d9U ++ ++static inline u32 g5p_ru_fls(u32 v) ++{ ++ return v ? 32 - __builtin_clz(v) : 0; ++} ++ ++/* ucvtf of 1 <= n < 2^24: exact. */ ++static inline u32 g5p_ru_f32(u32 n) ++{ ++ u32 bits = g5p_ru_fls(n); ++ ++ return ((126 + bits) << 23) | ((n << (24 - bits)) & 0x7fffff); ++} ++ ++/* fdiv of a normal positive single by 1 <= n < 2^24, round to nearest even. ++ * The quotient q of the 24-bit significand shifted by fls(n) + 3 has 27 or ++ * 28 bits; s of them are rounding bits, the remainder is the sticky bit. ++ * Results stay normal for the two leaf constants and n <= G5P_RU_MAX_DIM. ++ */ ++static inline u32 g5p_ru_fdiv(u32 a, u32 n) ++{ ++ u32 shift = g5p_ru_fls(n) + 3; ++ u64 num = (u64)((a & 0x7fffff) | 0x800000) << shift; ++ u32 q = num / n, sticky = num % n != 0; ++ u32 s = g5p_ru_fls(q) - 24, m = q >> s, low = q & ((1U << s) - 1); ++ ++ if (low > 1U << (s - 1) || ++ (low == 1U << (s - 1) && (sticky || (m & 1)))) ++ m++; ++ if (m >> 24) { ++ m >>= 1; ++ s++; ++ } ++ return ((((a >> 23) & 0xff) - shift + s) << 23) | (m & 0x7fffff); ++} ++ ++/* Sample positions, one byte per sample: x in [3:0], y in [7:4], 1/16 px. ++ * One sample sits at the pixel centre; four use the standard Vulkan/D3D ++ * pattern, which is also Metal's (record+0x69c..0x6b8 of the iOS ++ * msaa4-resolve/-store captures: (6,2) (14,6) (2,10) (10,14)). ++ */ ++#define G5P_RU_POSITIONS_1X 0x88ULL ++#define G5P_RU_POSITIONS_4X 0xeaa26e26ULL ++ ++/* The leaf for one or four samples (6ca6d5c's 1, 2, 4, 8 table, of which ++ * Linux uses 1 and 4). Four samples are 2x2 per pixel: the region headers ++ * and the tail pointer cache are sized for twice the tiles in x and y, the ++ * ISP macrotile size (Rbc) doubles in y, Rb8 is log2(samples) (5940f40) ++ * and the TE word Rf0 is 5. Rb0 and Rf8 are the positions word, packed by ++ * the leaf as x0 | y0 << 4 | x1 << 8 | ... | y3 << 28. The scales, the TE ++ * tiling words and the tile count do not depend on the samples. ++ */ ++static inline int g5p_ru_rt_derive_samples(u32 width, u32 height, u32 samples, ++ u64 positions, struct g5p_ru_rt *rt) ++{ ++ u32 tx = (width + 31) / 32, ty = (height + 31) / 32; ++ u32 mx = (((tx + 3) >> 2) + 3) & ~3U, my = (((ty + 3) >> 2) + 3) & ~3U; ++ u32 s = samples == 4 ? 2 : 1; ++ u32 px = mx * s * 4, py = my * s * 4, tpc; ++ ++ if (!width || !height || width > G5P_RU_MAX_DIM || ++ height > G5P_RU_MAX_DIM || (samples != 1 && samples != 4) || ++ positions >> (samples * 8)) ++ return -EINVAL; ++ tpc = 1U << g5p_ru_fls((px > py ? px : py) - 1); ++ *rt = (struct g5p_ru_rt) { ++ .header = px * py * 5, ++ .region = mx * my * s * s * 5, ++ .packed = positions, ++ .state = s - 1 ? 2 : 0, ++ .padded = (mx & 0x3ff) << 16 | ((my * s) & 0x3ff), ++ .scale = { ++ g5p_ru_fdiv(G5P_RU_SCALE_A, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_A, height) << 32, ++ g5p_ru_fdiv(G5P_RU_SCALE_B, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_B, height) << 32, ++ (g5p_ru_f32(width) + (4U << 23)) | ++ (u64)(g5p_ru_f32(height) + (4U << 23)) << 32, ++ }, ++ .ta_state = { s - 1 ? 5 : 0, positions }, ++ .ta_tiling = { ++ (u64)((ty - 1) << 12 | (tx - 1)) << 32 | ++ ((height - 1) << 16 | (width - 1)), ++ (u64)(my << 18 | my << 10 | 3 * my) << 32 | ++ (mx << 18 | mx << 10 | 3 * mx), ++ }, ++ .ta_tiling_word = mx * my, ++ .tiles = tx * ty, ++ .secondary = (u64)tpc * tpc * 8, ++ .secondary_align = 0x40, ++ }; ++ return 0; ++} ++ ++/* g5p_ru_rt64 is this function's result for 64x64. */ ++ ++/* PM pool for a kernel-derived target. Linux policy, not hardware: Rogue's ++ * minimum free list (50 4KiB pages, non-XE cores) plus one 4KiB PM page per ++ * tile, in 128KiB blocks, never below the latched pool and at most one F ++ * page. 64x64 (4 tiles) needs 2, so the latched 32 stays. There is no PM ++ * growth during a job: a frame that exhausts the pool fails like any OOM. ++ */ ++static inline u32 g5p_ru_pm_blocks_for(u32 tiles, u32 latched) ++{ ++ u64 need = ((50ULL + tiles) * 0x1000 + G5P_RU_BLOCK_BYTES - 1) / ++ G5P_RU_BLOCK_BYTES; ++ ++ if (need < latched) ++ need = latched; ++ return need > G5P_RU_MAX_BLOCKS ? G5P_RU_MAX_BLOCKS : need; ++} ++ ++ ++ ++static inline int g5p_ru_rt_derive(u32 width, u32 height, struct g5p_ru_rt *rt) ++{ ++ return g5p_ru_rt_derive_samples(width, height, 1, G5P_RU_POSITIONS_1X, rt); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +new file mode 100644 +index 000000000..822709ed8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -0,0 +1,651 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_LAYOUT_H ++#define G5P_RENDER_USER_LAYOUT_H ++ ++#include "bits-compat.h" ++/* Parameterized forms of the r22 fixed constructors for userspace render. ++ * Pure byte builders: no allocation, MMIO, locking or firmware publication. ++ * With r22's inputs every builder reproduces render-layout-r22.h exactly; ++ * tools/gpu/test_render_user.c checks that equivalence byte for byte. ++ */ ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++#include "render-user-template.h" ++ ++/* Kernel-private window in every render VM. r22's aux/RT addresses are the ++ * first allocations, so a 64x64 target keeps the fixed path's W anchors. ++ * Root entry 0 carries the boot VA0 sink, so user maps start at 1 GiB. ++ */ ++#define G5P_RU_PRIVATE_VA G5P_R22_AUX ++#define G5P_RU_PRIVATE_SIZE 0x1000000ULL ++#define G5P_RU_USER_VA_MIN 0x40000000ULL ++/* Catalogue entry of the firmware GiB, borrowed from the boot tables. */ ++#define G5P_RU_FIRMWARE_CI ((G5P_IMAGE_VA >> 30) & 1023) ++#define G5P_RU_AUX_BYTES 0x10680U ++#define G5P_RU_AUX_PAGES 5U ++#define G5P_RU_RT_ALIGN 0x10000ULL ++#define G5P_RU_MAX_BLOCKS 128U ++#define G5P_RU_BLOCK_BYTES 0x20000U ++#define G5P_RU_MAX_HEADER 0x80000U ++#define G5P_RU_MAX_SECONDARY 0x100000ULL ++#define G5P_RU_TA_W_BYTES 0x560U ++#define G5P_RU_3D_W_BYTES 0x8c0U ++#define G5P_RU_CMD_BYTES 0x40U ++ ++/* Kernel mirror of drm_g5p_rt_geometry (firmware round16 R-offsets). */ ++struct g5p_ru_rt { ++ u32 header, region; /* R10, R20 */ ++ u64 packed; /* Rb0 */ ++ u32 state, padded; /* Rb8, Rbc */ ++ u64 scale[3]; /* Rc0, Rc8, Rd0 */ ++ u64 ta_state[2]; /* Rf0, Rf8 */ ++ u64 ta_tiling[2]; /* R100, R108 */ ++ u32 ta_tiling_word, tiles; /* R110, R118 */ ++ u64 secondary, secondary_align; /* R138, R178 */ ++}; ++ ++/* 6ca6d5c executed offline for 64,64,1,8,8 (round16 section 3). */ ++static const struct g5p_ru_rt g5p_ru_rt64 = { ++ .header = 0x500, ++ .region = 0x50, ++ .packed = 0x88, ++ .state = 0, ++ .padded = 0x40004, ++ .scale = { 0x3b89309c3b89309cULL, 0x3cddb3d93cddb3d9ULL, ++ 0x4480000044800000ULL }, ++ .ta_state = { 0, 0x88 }, ++ .ta_tiling = { 0x00001001003f003fULL, 0x0010100c0010100cULL }, ++ .ta_tiling_word = 0x10, ++ .tiles = 4, ++ .secondary = 0x800, ++ .secondary_align = 0x40, ++}; ++ ++#include "render-user-geometry.h" ++ ++struct g5p_ru_rt_sizes { ++ u64 slice, main, secondary_map; ++}; ++ ++/* 6cb3d68: slice=R10 rounded to 4KiB; main ring is the power-of-two capacity ++ * for sixteen slices. 6cb40e0 places the secondary extent at offset 0 of ++ * its own mapping. Bounds are Linux allocation policy, not hardware limits. ++ */ ++static inline int g5p_ru_rt_sizes(const struct g5p_ru_rt *rt, ++ struct g5p_ru_rt_sizes *out) ++{ ++ u64 main = 0x1000; ++ ++ if (!rt->header || rt->header > G5P_RU_MAX_HEADER || !rt->secondary || ++ rt->secondary > G5P_RU_MAX_SECONDARY || !rt->secondary_align || ++ (rt->secondary_align & (rt->secondary_align - 1)) || ++ rt->secondary_align > G5P_GPU_PAGE || !rt->tiles) ++ return -EINVAL; ++ out->slice = ((u64)rt->header + 0xfff) & ~0xfffULL; ++ while (main < out->slice * 16) ++ main <<= 1; ++ out->main = main < G5P_GPU_PAGE ? G5P_GPU_PAGE : main; ++ out->secondary_map = (rt->secondary + G5P_GPU_PAGE - 1) & ++ ~(u64)(G5P_GPU_PAGE - 1); ++ return 0; ++} ++ ++/* 6cba480/6c7b64c anchors. Native mapping dialect: L=H, Laux=Haux. ++ * Equal to g5p_r22_rt() for g5p_ru_rt64 at the r22 addresses. 3D W58 and ++ * Wa8 are the importer's literal Rb0 and R20>>2 for the verified geometry. ++ */ ++static inline void g5p_ru_rt_fill(u8 *ta, u8 *td, const struct g5p_ru_rt *rt, ++ u64 main_va, u64 secondary_va) ++{ ++ g5p_put64(ta + 0x28, main_va); ++ g5p_put64(ta + 0x30, secondary_va); ++ g5p_put64(ta + 0x38, rt->ta_state[0]); ++ g5p_put64(ta + 0x40, rt->ta_state[1]); ++ g5p_put64(ta + 0x444, secondary_va); ++ g5p_put32(ta + 0x44c, rt->secondary); ++ g5p_put64(ta + 0x3ac, rt->ta_tiling[0]); ++ g5p_put64(ta + 0x3b4, rt->ta_tiling[1]); ++ g5p_put32(ta + 0x3bc, rt->ta_tiling_word); ++ g5p_put64(td + 0x20, main_va); ++ g5p_put64(td + 0x28, rt->packed); ++ g5p_put32(td + 0x30, rt->state); ++ g5p_put32(td + 0x34, rt->padded); ++ g5p_put32(td + 0x38, rt->region); ++ g5p_put64(td + 0x3c, rt->scale[0]); ++ g5p_put64(td + 0x44, rt->scale[1]); ++ g5p_put64(td + 0x4c, rt->scale[2]); ++ g5p_put64(td + 0x58, rt->packed); ++ g5p_put32(td + 0xa8, rt->region >> 2); ++ g5p_put32(td + 0x3a0, rt->padded); ++ g5p_put64(td + 0x3d0, rt->scale[0]); ++ g5p_put64(td + 0x3d8, rt->scale[1]); ++ g5p_put32(td + 0x3e0, rt->scale[2]); ++ g5p_put32(td + 0x3e4, rt->scale[0]); ++} ++ ++/* Validated typed render inputs, already range-checked by the caller. */ ++struct g5p_ru_pass { ++ u64 vdm_va; ++ u32 vdm_size, vdm_control; ++ u64 scratch_va, scratch_size, scratch_ptr[6]; ++ u64 pool7_va, pool7_size, pool8_va, pool8_size; ++ u32 context_pds[2][6], resume_pds[2][4], ta_pds[4]; ++ u32 background[8], eot[4], eot_resume[4], restore[4]; ++}; ++ ++/* Importer chains 6cba8a0/6c7b100/6c7b34c -> 6ca9710/6ca7ae0 (offline ++ * differential map in make-render-user-template.py). Every user VA field ++ * and code-relative descriptor lands here; the rest of W is the frozen ++ * dialect-1 scalar table or a kernel RT/channel field. ++ */ ++static inline void g5p_ru_fill_ta(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_TA_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_ta_fixed); i++) ++ g5p_put32(w + g5p_ru_ta_fixed[i][0], g5p_ru_ta_fixed[i][1]); ++ g5p_put64(w + 0x48, p->scratch_ptr[4]); ++ g5p_put64(w + 0x50, p->scratch_ptr[5]); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put32(w + 0x58 + i * 4, p->context_pds[0][i]); ++ g5p_put32(w + 0x70 + i * 4, p->context_pds[1][i]); ++ } ++ g5p_put64(w + 0x88, p->vdm_control & ~0xfULL); ++ g5p_put64(w + 0x90, p->scratch_va & ~7ULL); ++ g5p_put64(w + 0x98, p->vdm_va & ~3ULL); ++ for (u32 i = 0; i < 4; i++) { ++ g5p_put32(w + 0xa0 + i * 4, p->resume_pds[0][i]); ++ g5p_put32(w + 0xb0 + i * 4, p->resume_pds[1][i]); ++ } ++ for (u32 i = 0; i < 3; i++) ++ g5p_put32(w + 0xc0 + i * 4, p->ta_pds[i]); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x468 + i * 8, p->scratch_ptr[i]); ++ g5p_put64(w + 0x488, p->scratch_ptr[4]); ++} ++ ++static inline void g5p_ru_fill_3d(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_3D_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_3d_fixed); i++) ++ g5p_put32(w + g5p_ru_3d_fixed[i][0], g5p_ru_3d_fixed[i][1]); ++ g5p_put64(w + 0x60, p->pool8_va & ~3ULL); ++ g5p_put64(w + 0x68, p->pool7_va & ~3ULL); ++ g5p_put64(w + 0x490, p->pool7_va); ++ g5p_put64(w + 0x4a0, p->pool8_va); ++ for (u32 i = 0; i < 8; i++) ++ g5p_put32(w + 0x5b8 + i * 4, p->background[i]); ++ g5p_put32(w + 0xb0, p->background[0]); ++ g5p_put32(w + 0xb4, p->background[1]); ++ for (u32 i = 4; i < 8; i++) ++ g5p_put32(w + 0xc0 + (i - 4) * 4, p->background[i]); ++ /* 6ca7ae0 swaps the first EOT qword's halves in its B upload. */ ++ g5p_put32(w + 0x3b4, p->eot[1]); ++ g5p_put32(w + 0x3b8, p->eot[0]); ++ g5p_put32(w + 0x3bc, p->eot[2]); ++ for (u32 i = 0; i < 3; i++) { ++ g5p_put32(w + 0x680 + i * 4, p->eot[i]); ++ g5p_put32(w + 0x694 + i * 4, p->eot_resume[i]); ++ } ++ g5p_put32(w + 0x600, p->restore[0]); ++ g5p_put32(w + 0x604, p->restore[1]); ++ g5p_put32(w + 0x610, p->restore[2]); ++ g5p_put32(w + 0x614, p->restore[3]); ++} ++ ++/* TYPED_V2 additions, already validated by g5p_ru_pass2_check(). */ ++struct g5p_ru_pass2 { ++ u64 scissor_va, dbias_va, background_texture, zls[6]; ++ u64 occlusion_query_va, occlusion_query_size; ++ u32 scissor_count, dbias_count, bgobj_depth, bgobj_vals; ++ u32 pixel_width, zls_mask; ++}; ++ ++/* Dialect-2 words over g5p_ru_fill_3d() (TYPED_V2 has zero pools). Each ++ * register image goes to its B-upload word and to the normal builder's ++ * copy (6c7a06c); make-render-user-dialect2.py proves every position by ++ * perturbing record 2d8/2e0 (pools), 2f0/310/338/360/388/308 (ZLS), 508 ++ * (background texture) and 4d0 (clears) through the r22 importer. ++ * Pixel output: 6ca7ae0 for D2f8 = width, one sample (D6e0 = 0), D630 = ++ * D6b2 = 0, six cores. Widths 2/4/8 keep two or more partitions, so W3c0 ++ * and W3c4 keep their dialect-1 values; width 2 is dialect 1 exactly. ++ */ ++static inline void g5p_ru_fill_3d_width(u8 *w, u32 width) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by D2f8 - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u32 count = 0x4000 / (width << 10); ++ u32 code = width_code[width - 2]; ++ ++ g5p_put32(w + 0x3c8, count & 0x1f); ++ g5p_put32(w + 0x3cc, code >> 4 | 0xf8); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(w + 0x108 + i * 8, code | (count - 1) << 7 | 0x43); ++ g5p_put32(w + 0x6b0, width); ++} ++ ++static inline void g5p_ru_fill_3d_v2(u8 *w, const struct g5p_ru_pass2 *e) ++{ ++ static const u16 zls_w[6][2] = { ++ { 0x80, 0x618 }, { 0x88, 0x628 }, { 0x90, 0x630 }, ++ { 0x98, 0x648 }, { 0xa0, 0x650 }, { 0x138, 0x6d0 }, ++ }; ++ ++ g5p_put64(w + 0x60, e->scissor_va & ~3ULL); ++ g5p_put64(w + 0x4a0, e->scissor_va); ++ g5p_put64(w + 0x68, e->dbias_va & ~3ULL); ++ g5p_put64(w + 0x490, e->dbias_va); ++ /* ++ * D318 is copied to both partition records. The normal 3D upload masks ++ * the low nibble before programming ISP_OCLQRY_BASE. ++ */ ++ g5p_put64(w + 0x70, e->occlusion_query_va); ++ g5p_put64(w + 0x78, e->occlusion_query_va); ++ g5p_put64(w + 0xb8, e->background_texture); ++ g5p_put64(w + 0x5f0, e->background_texture); ++ g5p_put32(w + 0x3a4, e->bgobj_depth); ++ g5p_put32(w + 0x6a8, e->bgobj_depth); ++ g5p_put32(w + 0x3a8, e->bgobj_vals); ++ g5p_put32(w + 0x6ac, e->bgobj_vals); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put64(w + zls_w[i][0], e->zls[i]); ++ g5p_put64(w + zls_w[i][1], e->zls[i]); ++ } ++ g5p_ru_fill_3d_width(w, e->pixel_width); ++} ++ ++/* TYPED_V3 additions, already validated by g5p_ru_pass3_check(). */ ++struct g5p_ru_pass3 { ++ u64 ta_program_va; ++ u32 ta_program_size, vdm_end; ++ u32 pixel_width, pad; /* V1 dialect output width, 0 = 2 */ ++}; ++ ++/* Dialect-3 word over g5p_ru_fill_ta(): W+0x88 (-> R0+0x470) is the exact ++ * byte offset of the TA PDS program from pds_base, a plain 64-bit store with ++ * no mask. g5p_ru_fill_ta() leaves it zero for a V3 pass (vdm_control is ++ * zero), so V1/V2 bytes are untouched; the stream length never reaches W. ++ */ ++static inline void g5p_ru_fill_ta_v3(u8 *w, u64 program_off) ++{ ++ g5p_put64(w + 0x88, program_off); ++} ++ ++/* 6cb2378 selected state S at control+80: explicit zeros plus the four ++ * recovered backing pointers. N=1, PM ID0/record0. aux_va covers 10680. ++ */ ++static inline int g5p_ru_pm_state(u8 *control, u64 aux_va, u64 list_va) ++{ ++ memset(control + 0x80, 0, 0xc0); ++ return g5p_pm_record_backing(control + 0x80, 0, 1, aux_va, 0x80, ++ 0x10000, 0x600, ++ g5p_meta_address(list_va + 0x40)); ++} ++ ++/* 6cb124c/13f4/14dc pool for `blocks` resident = maximum blocks. Rewrites ++ * control+0..7f, the whole list and F; S at control+80 is not touched. ++ * physical[] holds blocks*8 16KiB PAs. Validate everything before writing. ++ */ ++static inline int g5p_ru_pm_check(u64 list_va, u64 free_va, const u64 *physical, ++ u32 blocks) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (!blocks || blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ ret = g5p_pm_sizes(1, blocks, 0x80, 0x10000, 0x600, &sizes); ++ if (ret) ++ return ret; ++ if ((list_va | free_va) & (G5P_GPU_PAGE - 1) || ++ list_va >= (1ULL << 40) || free_va >= (1ULL << 40) || ++ 0x80 + blocks * 0x24 > G5P_GPU_PAGE || ++ blocks * 32 * 4 > G5P_GPU_PAGE) ++ return -EINVAL; ++ for (u32 i = 0; i < blocks * 8; i++) ++ if ((physical[i] & (G5P_GPU_PAGE - 1)) || ++ physical[i] >= (1ULL << 40)) ++ return -ERANGE; ++ return 0; ++} ++ ++static inline int g5p_ru_pm_pool(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks) ++{ ++ u32 previous = 0; ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ memset(control, 0, 0x80); ++ memset(list, 0, G5P_GPU_PAGE); ++ memset(free, 0, G5P_GPU_PAGE); ++ for (u32 j = 0; j < blocks; j++) { ++ u32 at = 0x80 + j * 0x24; ++ ++ g5p_put32(list + at, previous); ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[j * 8 + k] >> 12; ++ ++ g5p_put32(list + at + 4 + k * 4, ppn); ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * 4, ++ ppn + p); ++ } ++ previous = g5p_meta_address(list_va + at); ++ } ++ g5p_put32(list + 0x60, 1); ++ g5p_put32(control + 0x10, previous); ++ g5p_put64(control + 0x14, free_va); ++ g5p_put32(control + 0x1c, blocks * 32 * 4); ++ g5p_put32(control + 0x20, blocks * 32); ++ g5p_put32(control + 0x24, blocks * 32 - 1); ++ g5p_put32(control + 0x34, G5P_GPU_PAGE); ++ g5p_put32(control + 0x40, g5p_meta_address(list_va)); ++ g5p_put32(control + 0x54, blocks * 32); ++ g5p_put32(control + 0x58, blocks * 32); ++ return 0; ++} ++ ++/* Complete fresh PM object: g5p_r22_pm() when blocks=32, aux=G5P_R22_AUX. */ ++static inline int g5p_ru_pm(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks, ++ u64 aux_va) ++{ ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ if ((aux_va & (G5P_GPU_PAGE - 1)) || ++ aux_va > (1ULL << 40) - G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ return -EINVAL; ++ memset(control, 0, G5P_GPU_PAGE); ++ ret = g5p_ru_pm_pool(control, list, free, list_va, free_va, physical, ++ blocks); ++ return ret ?: g5p_ru_pm_state(control, aux_va, list_va); ++} ++ ++/* The r25 latched selections that can still yield a frame. Context/host/SKU ++ * probes and the heap-relative RT dialect are diagnostic-only in the fixed ++ * path (pair rejected) and are not offered here; blocks is pm_blocks. ++ */ ++static inline bool g5p_ru_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return g5p_r25_valid(v) && !g5p_r25_diagnostic(v) && !v->blocks; ++} ++ ++/* r25 render_pm_variant 2/3 on a pool from g5p_ru_pm()/g5p_ru_pm_pool(): ++ * reverse only F's block order, or report F's whole mapped page as its ++ * extent. Equal to g5p_r25_pool() for 32, 64 and 128 blocks. ++ */ ++static inline void g5p_ru_pm_variant(u8 *control, u8 *free, const u64 *physical, ++ u32 blocks, u32 variant) ++{ ++ if (variant == 2) ++ for (u32 j = 0; j < blocks; j++) ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[(blocks - 1 - j) * 8 + k] >> ++ 12; ++ ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * ++ 4, ++ ppn + p); ++ } ++ if (variant == 3) ++ g5p_put32(control + 0x1c, G5P_GPU_PAGE); ++} ++ ++/* r25 render_pm_variant 1, after g5p_ru_job() built the TA page: tag6+10 ++ * names an owned holder with the linked head (outside CH/ring/W/SKU/TQ/ ++ * dependency); F/count/head start clear and last=ffffffff, so 10005368 ++ * rebuilds them. Equal to g5p_r25_inputs() for the same pool. ++ */ ++ ++/* Persistent per-channel objects (6c89420 layout): written once at attach. ++ * Control indices, TQ and the dependency object stay firmware-owned after. ++ */ ++static inline void g5p_ru_channel_init(u8 *p, u32 meta) ++{ ++ memset(p, 0, G5P_GPU_PAGE); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_R22_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_R22_TQ + 4, meta + G5P_R22_TQ); ++ g5p_put64(p + G5P_R22_DEP + 0x30, 1); ++ g5p_put64(p + G5P_R22_DEP + 0x48, ~0ULL); ++ g5p_put32(p + G5P_R22_DEP + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(p + G5P_R22_DEP + off + 8, 0xf); ++} ++ ++/* Pointer-ring positions of job `seq` (1-based): two entries per job. */ ++static inline u32 g5p_ru_ring_index(u32 seq) ++{ ++ return (2 * (seq - 1)) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_ru_endpoint(u32 seq) ++{ ++ return (2 * seq) % G5P_WORK_SLOTS; ++} ++ ++/* perf60 job slots (render_user_job_slots, drm-render-v3.inc). A job's command, W record and ++ * microsequence ("SKU") live in slot 0 (the r22 offsets) or slot 1, in the channel page's free ++ * tail; the firmware reaches both through the job's two pointer-ring entries. The channel header, ++ * control, pointer ring, TQ, stamps and dependency object stay shared. Slot 1 clears TQ/STAMP/DEP ++ * (0x1800..0x1b70) and the r25 holder (G5P_R25_HEAD, 0x3000). ++ */ ++#define G5P_RU_SLOT1 0x1c00U ++#define G5P_RU_SLOT1_END \ ++ (G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES + 0x180U) ++ ++static inline u32 g5p_ru_cmd_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 : G5P_WORK_CMD; ++} ++ ++static inline u32 g5p_ru_w_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 + G5P_RU_CMD_BYTES : G5P_R22_W; ++} ++ ++static inline u32 g5p_ru_sku_at(u32 js, bool td) ++{ ++ if (js) ++ return G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES; ++ return td ? G5P_R22_3D_SKU : G5P_R22_TA_SKU; ++} ++ ++/* Per-job objects for sequence `seq`: W image (fixed+typed+RT), channel W ++ * fields, the full timestamped microsequence, the tag6 PM bootstrap (TA) or ++ * tag4 TA-finish dependency (3D) and both pointer-ring entries. With seq=1 ++ * this equals g5p_r22_channel(). Stamp target and sequence are both seq. ++ * ta_finish is M(TA stamp word addressed by TA.W530), used by 3D only. ++ */ ++static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, ++ bool td, u32 seq, const u8 *image, ++ u32 ta_finish, bool zero_timestamps, u32 js, ++ u32 pmid) ++{ ++ u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; ++ u32 finish = meta + G5P_R22_STAMP + 4; ++ u32 sku = g5p_ru_sku_at(js, td), cmd = g5p_ru_cmd_at(js); ++ u32 ring = g5p_ru_ring_index(seq); ++ u8 *work = p + g5p_ru_w_at(js); ++ u32 cursor = sku, starts, finishes; ++ ++ memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); ++ memset(p + sku, 0, td ? 0x180 : 0xc0); ++ g5p_put32(work, td ? 1 : 0); ++ g5p_put32(work + 4, slot); ++ g5p_put32(work + (td ? 0x14 : 0xc), meta + G5P_R22_DEP); ++ g5p_put32(work + (td ? 0x18 : 0x14), pm); ++ if (!td && pmid) ++ g5p_put32(work + 0x10, ++ pmid); /* TA W+10: the PM identity (iOS Dcb0) */ ++ g5p_put32(work + (td ? 0x1c : 0x18), state); ++ g5p_put32(work + (td ? 0xc : 0x454), meta + sku); ++ g5p_put32(work + (td ? 0x10 : 0x458), td ? 0x180 : 0xc0); ++ g5p_put32(work + (td ? 0x888 : 0x52c), finish - 4); ++ g5p_put32(work + (td ? 0x88c : 0x530), finish); ++ g5p_put32(work + (td ? 0x890 : 0x534), seq); ++ g5p_put32(work + (td ? 0x894 : 0x538), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x898 : 0x53c), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x8a0 : 0x544), seq); ++ { ++ u32 ta_start[] = { ++ 0x23, ++ w + 0x3a4, ++ w + 0x20, ++ state, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ pmid, ++ 0, ++ 0, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ 0, ++ seq, ++ }; ++ u32 td_start[] = { ++ 0x25, ++ w + 0x3a0, ++ w + 0x58, ++ state, ++ global + 0x724, ++ w + 0x7f0, ++ w + 0x7f4, ++ w + 0x878, ++ meta, ++ w, ++ slot, ++ 0, ++ pmid, ++ g5p_le32(work + 0x800), ++ 0, ++ 1, ++ w + 0x6d8, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ 0, ++ seq, ++ }; ++ u32 ta_finish[] = { ++ 0x24, ++ state, ++ pm, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ seq, ++ finish, ++ seq, ++ (u32)-0x70, ++ }; ++ u32 td_finish[] = { ++ 0x26, ++ seq, ++ finish, ++ seq, ++ state, ++ pm, ++ 1, ++ global + 0x724, ++ w + 0x878, ++ w + 0x7f0, ++ meta, ++ w, ++ slot, ++ w + 0x6d8, ++ 0, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ (u32)-0x104, ++ }; ++ u32 result = w + (td ? 0x8ac : 0x550); ++ u32 timestamp = w + (td ? 0x8b4 : 0x558); ++ u32 ts0[] = { 0x80000018, result, timestamp, timestamp, seq }; ++ u32 ts1[] = { 0x18, result, timestamp, timestamp + 4, seq }; ++ ++ starts = td ? ARRAY_SIZE(td_start) : ARRAY_SIZE(ta_start); ++ finishes = td ? ARRAY_SIZE(td_finish) : ARRAY_SIZE(ta_finish); ++ for (u32 i = 0; i < starts; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_start[i] : ta_start[i]); ++ for (u32 i = 0; i < ARRAY_SIZE(ts0); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts0[i]); ++ if (td) { ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r22_wait_words); ++ i++, cursor += 4) ++ g5p_put32(p + cursor, g5p_r22_wait_words[i]); ++ } else { ++ g5p_put32(p + cursor, 0x801); ++ cursor += 4; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(ts1); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts1[i]); ++ for (u32 i = 0; i < finishes; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_finish[i] : ta_finish[i]); ++ g5p_put32(p + cursor, 0x40000017); ++ } ++ memset(p + cmd, 0, G5P_RU_CMD_BYTES); ++ g5p_put32(p + cmd, td ? 4 : 6); ++ g5p_put32(p + cmd + 4, td ? ta_finish : slot); ++ g5p_put32(p + cmd + 8, ++ td ? seq : pmid); /* tag6 {6, STA, PMid, M(control)} */ ++ g5p_put32(p + cmd + 12, td ? seq & 0x7fffffff : pm); ++ g5p_put32(p + G5P_WORK_RING + ring * 4, meta + cmd); ++ g5p_put32(p + G5P_WORK_RING + ((ring + 1) % G5P_WORK_SLOTS) * 4, w); ++} ++ ++/* Retirement of job `seq`: both W pointers still ours, finish word and ++ * retirement mirror at seq, all five channel watermarks at its endpoint. ++ * g5p_r22_retired() is the seq=1 case. ++ */ ++static inline bool g5p_ru_retired_at(const u8 *p, u32 meta, bool td, u32 seq, ++ u32 js) ++{ ++ const u8 *w = p + g5p_ru_w_at(js); ++ u32 endpoint = g5p_ru_endpoint(seq); ++ ++ if (g5p_le32(w + (td ? 0x888 : 0x52c)) != meta + G5P_R22_STAMP || ++ g5p_le32(w + (td ? 0x88c : 0x530)) != meta + G5P_R22_STAMP + 4 || ++ g5p_le32(w + (td ? 0x890 : 0x534)) != seq || ++ g5p_le32(p + G5P_R22_STAMP) != seq || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* A stamp beyond the published target, or a watermark not on the even ++ * two-entry grid, is a protocol error rather than "not yet retired". ++ */ ++static inline bool g5p_ru_stamp_sane(const u8 *p, u32 seq) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ s32 ahead = (s32)(g5p_le32(p + G5P_R22_STAMP + i * 4) - seq); ++ ++ if (ahead > 0) ++ return false; ++ } ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) >= G5P_WORK_SLOTS) ++ return false; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-loss.h b/drivers/gpu/apple-g5p/render-user-loss.h +new file mode 100644 +index 000000000..2aad49d88 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-loss.h +@@ -0,0 +1,120 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context fault isolation (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault ++ * isolation). Pure decisions, host-tested (scripts/gpu/multiclient2-host/test_render_loss.c). ++ * ++ * The hardware gives no way to abandon a job: the firmware has no abort (iOS ends a ++ * non-progressing engine with a firmware stop, a platform power cycle and a new firmware boot, ++ * kext 6c9e070, firmware-re README "Fault recovery"), the TA and 3D channels are FIFO, and the ++ * one TA and one 3D engine serve every context. So one context's job can be *lost* (its client ++ * gets an error, its context is dead) without ending the device only while the firmware is ++ * alive, and only if the stuck phase still finishes: until then no other job may be published ++ * behind it (it would wait behind the same engine and be lost in turn). ++ * ++ * loss at the end of a phase that did not complete: ++ * NONE it completed; ++ * CONTEXT a phase timeout on a job other than the epoch's first, with the firmware ++ * alive (its event stream never went silent, no fault event, no protocol ++ * error) and isolation on: the job and its context are lost, the device is ++ * not; the channel is blocked behind the stuck phase; ++ * DEVICE anything else: the epoch is terminal, as before. ++ * drain while blocked, the scheduler runs no job and waits, RAM only, for the stuck phase: ++ * a late TA gets its 3D published (the channel and PM state then stay consistent), ++ * a late 3D ends it; the channel is then clear and every held job runs. A silent ++ * event stream, a fault, a refused 3D or the drain bound make the epoch terminal. ++ */ ++#ifndef G5P_RENDER_USER_LOSS_H ++#define G5P_RENDER_USER_LOSS_H ++ ++enum g5p_ru_loss { ++ G5P_RU_LOSS_NONE, ++ G5P_RU_LOSS_CONTEXT, ++ G5P_RU_LOSS_DEVICE, ++}; ++ ++struct g5p_ru_loss_in { ++ u32 result; /* G5P_RU_WAIT_* of the phase */ ++ bool first; /* the epoch's first job (its attach and bell) */ ++ bool isolate; /* render_user_drain_ms != 0 */ ++ bool silent; /* the event stream stopped: the firmware is dead */ ++ bool fault; /* a fault event, a failed boot, a protocol or drain error */ ++}; ++ ++static inline int g5p_ru_loss(const struct g5p_ru_loss_in *in) ++{ ++ if (in->result == G5P_RU_WAIT_COMPLETE) ++ return G5P_RU_LOSS_NONE; ++ if (in->result != G5P_RU_WAIT_TIMEOUT || in->first || !in->isolate || ++ in->silent || in->fault) ++ return G5P_RU_LOSS_DEVICE; ++ return G5P_RU_LOSS_CONTEXT; ++} ++ ++/* The drain of a blocked channel. */ ++enum g5p_ru_drain_state { ++ G5P_RU_DRAIN_TA, /* waiting for the stuck TA */ ++ G5P_RU_DRAIN_TD, /* waiting for the stuck job's 3D */ ++ G5P_RU_DRAIN_CLEAR, /* done: the channel is clear */ ++ G5P_RU_DRAIN_DEAD, /* the epoch is terminal */ ++}; ++ ++/* What the drain does next. */ ++enum g5p_ru_drain_action { ++ G5P_RU_DRAIN_WAIT, /* sample again */ ++ G5P_RU_DRAIN_PUBLISH_TD, /* publish the stuck job's 3D, wait for it */ ++ G5P_RU_DRAIN_UNBLOCK, /* release the held jobs */ ++ G5P_RU_DRAIN_TERMINAL, /* note the epoch terminal, release the held jobs (they fail) */ ++}; ++ ++/* One step from the result of the phase wait (G5P_RU_WAIT_*) in state *st. A refused 3D ++ * publish is reported by the caller as G5P_RU_WAIT_FAULT on the TD state. ++ */ ++static inline int g5p_ru_drain_step(u32 *st, u32 result) ++{ ++ if (*st >= G5P_RU_DRAIN_CLEAR) ++ return *st == G5P_RU_DRAIN_CLEAR ? G5P_RU_DRAIN_UNBLOCK : ++ G5P_RU_DRAIN_TERMINAL; ++ if (result == G5P_RU_WAIT_PENDING) ++ return G5P_RU_DRAIN_WAIT; ++ if (result != G5P_RU_WAIT_COMPLETE) { ++ *st = G5P_RU_DRAIN_DEAD; ++ return G5P_RU_DRAIN_TERMINAL; ++ } ++ if (*st == G5P_RU_DRAIN_TA) { ++ *st = G5P_RU_DRAIN_TD; ++ return G5P_RU_DRAIN_PUBLISH_TD; ++ } ++ *st = G5P_RU_DRAIN_CLEAR; ++ return G5P_RU_DRAIN_UNBLOCK; ++} ++ ++/* ++ * The drain bound must leave the held clients' own wait intact. A client waits for its job up to ++ * the job_timeout_ms GET_RENDER_INFO reports plus a margin (the ICD: + 3 s, so 2 * hard + obs + 3 s ++ * = 125 s with latency101 on, 33 s with it off), and a held job has already waited out the stuck ++ * phase (up to the hard bound, or render_user_wait_ms) before the drain starts, then runs after ++ * it. Valid: 0 (isolation off), or from 1000 up to hard + obs - 10000 with latency101 on (hard ++ * 60000, obs 2000: 52000), and up to 8000 with it off. An invalid value at the time of a loss ++ * turns isolation off for it. ++ */ ++static inline bool g5p_ru_drain_valid(u32 drain_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!drain_ms) ++ return true; ++ if (drain_ms < 1000) ++ return false; ++ if (!hard_ms) ++ return drain_ms <= 8000; ++ return hard_ms + obs_ms >= 11000 && ++ drain_ms <= hard_ms + obs_ms - 10000; ++} ++ ++ ++ ++static inline const char *g5p_ru_loss_name(int loss) ++{ ++ return loss == G5P_RU_LOSS_NONE ? "none" : ++ loss == G5P_RU_LOSS_CONTEXT ? "context" : "device"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-mmu.h b/drivers/gpu/apple-g5p/render-user-mmu.h +new file mode 100644 +index 000000000..5a05dccbf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-mmu.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 MMU invalidation arm (render_user_mmu_inval, drm-render-v3.inc), pure helpers. ++ * ++ * The only invalidation of a user MMU context besides DC3 is the firmware's per-kick tag compare ++ * (10017ac4): diff = host_tag[S] ^ cached_tag[S]; bits 31:22 set -> e138 = (S << 3) | 4 (PC), ++ * else bits 21:12 -> | 2 (PD), else bits 11:0 -> | 1 (PT); then cached = host. One invalidate per ++ * kick, the highest changed field (docs/gpu/firmware-re/README.md "generation producer"). A PC ++ * field change therefore asks for the firmware's PC-level invalidate of that context, the level ++ * Apple's own producer (6cabe70) and DC3 (1000218c) use as the strongest one. ++ * ++ * g5p_ru_mmu_pc_tag(): a host tag whose PC field differs from both slots' cached PC fields, so ++ * the next kick of each slot takes the PC branch. The PD and PT fields are the given tag's. ++ * Never 0 (0 is the unbound tag) and never equal to either cached tag. Host-tested ++ * (scripts/gpu/perf60-host/test_mmu_inval.c). ++ */ ++#ifndef G5P_RENDER_USER_MMU_H ++#define G5P_RENDER_USER_MMU_H ++ ++#define G5P_RU_MMU_PC_MASK 0xffc00000U ++#define G5P_RU_MMU_PC_ONE 0x00400000U ++ ++/* render_user_mmu_inval bits */ ++#define G5P_RU_MMU_EDIT \ ++ 0x1U /* every VM edit publication (and spare adoption) asks for PC */ ++#define G5P_RU_MMU_SWITCH \ ++ 0x2U /* a bound context's job after another context's job: PC retag */ ++#define G5P_RU_MMU_TOUCH \ ++ 0x4U /* NC read of every table line before the tag store */ ++#define G5P_RU_MMU_EVERY \ ++ 0x8U /* every job of a bound context: PC retag (superset of SWITCH) */ ++ ++static inline u32 g5p_ru_mmu_pc_tag(u32 tag, u32 cached0, u32 cached1) ++{ ++ u32 pc = (tag & G5P_RU_MMU_PC_MASK) + G5P_RU_MMU_PC_ONE; ++ ++ /* At most two values are excluded, so three candidates always contain a free one. */ ++ for (u32 i = 0; i < 3; i++, pc += G5P_RU_MMU_PC_ONE) { ++ u32 t = (pc & G5P_RU_MMU_PC_MASK) | (tag & ~G5P_RU_MMU_PC_MASK); ++ ++ if (((t ^ cached0) & G5P_RU_MMU_PC_MASK) && ++ ((t ^ cached1) & G5P_RU_MMU_PC_MASK) && t) ++ return t; ++ } ++ return 0; /* unreachable: kept for the host test */ ++} ++ ++/* ++ * Retag on a job: the published host tag, or the cached one if none, moved to a fresh PC field. ++ */ ++static inline bool g5p_ru_mmu_retag_due(u32 mode, bool bound_before, ++ bool switched) ++{ ++ if (!bound_before) ++ return false; /* this job's DC3 or adoption publishes its own tag */ ++ return (mode & G5P_RU_MMU_EVERY) || ++ ((mode & G5P_RU_MMU_SWITCH) && switched); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-msaa.h b/drivers/gpu/apple-g5p/render-user-msaa.h +new file mode 100644 +index 000000000..f27f197a6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-msaa.h +@@ -0,0 +1,88 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_MSAA_H ++#define G5P_RENDER_USER_MSAA_H ++ ++#include "bits-compat.h" ++/* TYPED_V3 4x multisampling: the per-job words a pass with a 4-sample render ++ * target needs on top of the V1/V2 3D words and the target's geometry ++ * (g5p_ru_rt_derive_samples()). Pure byte builders over the kernel's own ++ * work records: no allocation, MMIO, locking or firmware publication. ++ * ++ * Evidence (docs/gpu/experiments/msaa.md): the iOS gles3/gles3b captures ++ * 20261101T175626Z, msaa4-resolve, msaa4-store, g3b-a2c and g3b-samplemask, ++ * against g3-base, and the importer's 3D/TA fillers 6ca7ae0/6ca9710 executed ++ * with the 4-sample descriptor (build/gpu/msaa/filler-msaa.c): ++ * record+0x4d8 0xf -> 0x7 D6a8 -> 3D W3c0[3:0] (ISP_CTL; the draws' ++ * ISPCTL.upass drops 0xf -> 0x7 with it) ++ * record+0x640 0 -> 1 D6ad -> 3D W3c0 bit 17 and W86c (resolve only; ++ * bit 17 is ISP_CTL.PROCESS_EMPTY_TILES, which ++ * Linux's template sets for every pass since ++ * 0e05f993cf, so only W86c is new) ++ * RT Rb8 0 -> 2 D6e0 -> 3D Wd0 (AA mode, CR f30), and the ++ * pixel-output shift 10 -> 11 in W3c0 [21], ++ * W3c4, W3c8 and the six W108 words ++ * RT Rb0 / Rf8 3D W58 (CR fa8) / TA W40 (PPP_MULTISAMPLECTL) ++ * The rest of the target's geometry reaches W through g5p_ru_rt_fill(). ++ */ ++#define G5P_RU_MSAA_RESOLVE 1U /* DRM_G5P_RENDER_MSAA_RESOLVE */ ++ ++struct g5p_ru_msaa { ++ u32 samples, flags; /* 0 = 1 */ ++ u64 positions; /* 0 = the target's */ ++}; ++ ++/* samples 0 or 1 is the one-sample pass and must use a one-sample target; ++ * 4 must use a four-sample target. Positions are one byte per sample, so a ++ * word with bits above the sample count is malformed; zero takes the ++ * target's (the standard pattern). Resolve exists only with four samples. ++ * Nothing here is a hardware hazard: positions are a 4-bit grid and the ++ * resolve bit changes what the PBE writes, not where. ++ */ ++static inline int g5p_ru_msaa_check(const struct g5p_ru_msaa *m, u32 rt_samples) ++{ ++ u32 s = m->samples ? m->samples : 1; ++ ++ if ((s != 1 && s != 4) || s != rt_samples || ++ m->flags & ~G5P_RU_MSAA_RESOLVE || (m->flags && s == 1) || ++ m->positions >> (s * 8)) ++ return -EINVAL; ++ return 0; ++} ++ ++/* After g5p_ru_fill_3d(), the dialect's width words and g5p_ru_rt_fill(). ++ * A one-sample pass is left as it is, unless it names its position. ++ */ ++static inline void g5p_ru_fill_msaa(u8 *ta, u8 *td, const struct g5p_ru_msaa *m, ++ const struct g5p_ru_rt *rt) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by width - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u64 positions = m->positions ? m->positions : rt->packed; ++ bool resolve = m->flags & G5P_RU_MSAA_RESOLVE; ++ u32 width, count, mode, w3c0; ++ ++ if (m->samples != 4 && !m->positions) ++ return; ++ g5p_put64(ta + 0x40, positions); ++ g5p_put64(td + 0x58, positions); ++ if (m->samples != 4) ++ return; ++ /* The width the dialect wrote (2, 4 or 8); 6ca7ae0 shifts by 11 with an ++ * AA mode, so each width has half the output partitions. ++ */ ++ width = g5p_le32(td + 0x6b0); ++ count = 0x4000 / (width << 11); ++ mode = count > 1 ? 2 : count; ++ w3c0 = g5p_le32(td + 0x3c0) & ~(0xfU | BIT(21)); ++ g5p_put32(td + 0x3c0, ++ w3c0 | 0x7 | (resolve ? BIT(17) : 0) | (mode - 1) << 21); ++ g5p_put32(td + 0x3c4, mode * 0x204); ++ g5p_put32(td + 0x3c8, count & 0x1f); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(td + 0x108 + i * 8, ++ width_code[width - 2] | (count - 1) << 7 | 0x43); ++ g5p_put64(td + 0xd0, rt->state); ++ g5p_put32(td + 0x86c, resolve); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-pmset.h b/drivers/gpu/apple-g5p/render-user-pmset.h +new file mode 100644 +index 000000000..170738f1f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-pmset.h +@@ -0,0 +1,77 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context PM records (perf60 stage 4, docs/gpu/multi-client-design.md risk R3). ++ * ++ * iOS acquires a ParamBuffer ID per prepare from a 127-entry pool (6cb9afc) and builds a PM ++ * control/state/aux object and its block list per PM (6cb2378); one PM is never shared by two ++ * work queues. Linux built one device PM (control+S at shared+G5P_R22_PM, list and F in the ++ * two render_meta pages, aux at G5P_RU_PRIVATE_VA) for every render context; with the GPU ++ * never parked, one context's job then started from the PM state another context's job left. ++ * ++ * boot_pm_sets reserves N sets in the firmware GiB at boot, right after render_meta: set s is ++ * three pages {control + S, list, F} at G5P_PMSET_VA + s * 3 pages, mapped like the shared ++ * pages (they are firmware objects the firmware and the PM requesters address by meta VA). ++ * A render context takes one set, its own pool pages and its own aux pages. PM ID: 0 stays ++ * the device PM's; set s may carry s + 1. ++ * ++ * Pure helpers, host-tested (scripts/gpu/perf60-host/test_pmset.c). ++ */ ++#ifndef G5P_RENDER_USER_PMSET_H ++#define G5P_RENDER_USER_PMSET_H ++ ++#define G5P_PMSET_MAX 8U ++#define G5P_PMSET_PAGES 3U /* control + S, list, F */ ++#define G5P_PMSET_VA (G5P_SHARED_VA + G5P_SHARED_SIZE + 2 * G5P_GPU_PAGE) ++#define G5P_PMSET_END \ ++ (G5P_PMSET_VA + G5P_PMSET_MAX * G5P_PMSET_PAGES * G5P_GPU_PAGE) ++ ++enum { G5P_PMSET_CONTROL, G5P_PMSET_LIST, G5P_PMSET_FREE }; ++ ++static inline u64 g5p_pmset_va(u32 set, u32 part) ++{ ++ return G5P_PMSET_VA + ++ ((u64)set * G5P_PMSET_PAGES + part) * G5P_GPU_PAGE; ++} ++ ++static inline u32 g5p_pmset_id(u32 set) ++{ ++ return set + 1; ++} ++ ++/* ++ * The PM's write-back target (release8 bench-gaps3, 2026-11-03). After the GPU switches from ++ * context A's jobs to context B's, the firmware's PM keeps writing A's free list F back, through ++ * A's TA slot (its MMU context), on B's later jobs, until a job of a third context runs: two ++ * faults, one on the 2nd and one on the 3rd job of B after A's last, both the first job after A ++ * was unbound (page-invalid level 2 in A's MMU context, A's set F page; with A still bound the ++ * same writes landed). So a context's slots and set stay bound while it is the context of the ++ * last job or the previous *distinct* one: g5p_pmset_track() at every job end, g5p_pmset_holds() ++ * at close. That bounds when the slots may go, not when the PM stops: two later faults (release8 ++ * quiet2, seq 858 and 893) wrote the free list of a context two switches back, after this rule had ++ * released it. So a released pair is parked rather than left empty (render_user_close_park, ++ * drm-render-v3.inc), and a released set is reused last (g5p_pmset_pick()). ++ */ ++static inline bool g5p_pmset_switched(const void *last, const void *job) ++{ ++ return last != job; ++} ++ ++static inline bool g5p_pmset_holds(const void *last, const void *prev, ++ const void *ctx) ++{ ++ return ctx && (ctx == last || ctx == prev); ++} ++ ++/* A free set, round-robin from `next` (a released set is reused last). -1: none free. */ ++static inline int g5p_pmset_pick(const bool *used, u32 n, u32 next) ++{ ++ for (u32 i = 0; i < n; i++) { ++ u32 s = (next + i) % n; ++ ++ if (!used[s]) ++ return s; ++ } ++ return -1; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-quiesce.h b/drivers/gpu/apple-g5p/render-user-quiesce.h +new file mode 100644 +index 000000000..3671e415e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-quiesce.h +@@ -0,0 +1,145 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU quiesce before a kexec, reboot or power-off (docs/gpu/experiments/mcfold2-kexec.md). ++ * ++ * Without it the firmware keeps running from the outgoing kernel's RAM across the relocation: ++ * it still writes its event rings, and the MMU still holds a CBASE for every bound render slot ++ * pointing at page tables the next kernel reuses, until the next kernel's PMGR GFX cycle about ++ * 0.4 s into its boot. The quiesce ends that window on the outgoing side, in this order: ++ * ++ * 1 close job admission and wait (bounded) for submitters inside; ++ * 2 take the command lock (bounded trylock), so no job or command is in flight; ++ * 3 DETACH: every user slot the firmware may hold is unbound by a RAM-only DC3 batch ++ * (render-user-bind.h), only when the lock is ours, the firmware is healthy, no timeout ++ * or fault was noted and the DC ring is idle; ++ * 4 freeze the boot state and mask the GPU IRQ (no host register access after this); ++ * 5 CYCLE: one bounded PMGR GFX off/on, the same gate cycle the next kernel performs with ++ * apple_pmgr.kexec_gpu_cycle=1, only when no host thread can be inside a GPU register ++ * access: the lock is ours, or its holder is the TYPED_V3 executor in its RAM-only regime. ++ * ++ * Pure functions over the ops below, host-tested (scripts/gpu/mcfold2-host/test_kexec_quiesce.c). ++ */ ++#ifndef G5P_RENDER_USER_QUIESCE_H ++#define G5P_RENDER_USER_QUIESCE_H ++ ++/* kexec_quiesce module parameter */ ++#define G5P_RU_KQ_OFF 0U ++#define G5P_RU_KQ_RAM 1U /* detach only, no PMGR access */ ++#define G5P_RU_KQ_FULL 2U /* detach and cycle (default) */ ++ ++/* Plan */ ++#define G5P_RU_KQ_DETACH 0x1U ++#define G5P_RU_KQ_CYCLE 0x2U ++ ++/* Why a step was skipped */ ++#define G5P_RU_KQ_WHY_OFF 0x001U /* kexec_quiesce=0 */ ++#define G5P_RU_KQ_WHY_DOWN 0x002U /* the firmware never ran on this boot */ ++#define G5P_RU_KQ_WHY_BUSY 0x004U /* the command lock is not ours: no detach */ ++#define G5P_RU_KQ_WHY_FAILED \ ++ 0x008U /* fault or protocol failure: firmware state unknown */ ++#define G5P_RU_KQ_WHY_WEDGED \ ++ 0x010U /* a TYPED_V3 timeout or fault: a job may hold its slots */ ++#define G5P_RU_KQ_WHY_RING 0x020U /* the DC ring is not idle */ ++#define G5P_RU_KQ_WHY_NONE 0x040U /* no user slot is held */ ++#define G5P_RU_KQ_WHY_HOLDER \ ++ 0x080U /* unknown lock holder, maybe inside MMIO: no cycle */ ++#define G5P_RU_KQ_WHY_NO_PMGR 0x100U ++#define G5P_RU_KQ_WHY_RAM 0x200U /* kexec_quiesce=1: no cycle */ ++#define G5P_RU_KQ_WHY_NO_DRM 0x400U /* no render frontend: nobody can detach */ ++ ++struct g5p_ru_kq_in { ++ u32 mode; /* kexec_quiesce */ ++ bool booted; /* the firmware was released on this boot */ ++ bool locked; /* the command lock is ours */ ++ bool failed; /* b->failed or b->event_failed */ ++ bool frontend; /* a render frontend that can detach is registered */ ++ bool term_failed; /* the TYPED_V3 latch noted a timeout or fault */ ++ bool ram_only; /* the TYPED_V3 latch is quiet: its executor touches no register */ ++ bool executing; /* the TYPED_V3 executor is inside a job */ ++ bool ring_idle; /* DC consumer == producer <= 255 */ ++ bool pmgr; /* the PMGR regmap is present */ ++ u32 held; /* user slots with a PA or a host tag */ ++ u32 cons, prod; /* the DC ring when sampled */ ++}; ++ ++static inline u32 g5p_ru_kq_plan(const struct g5p_ru_kq_in *in, u32 *why) ++{ ++ u32 plan = 0, w = 0; ++ ++ if (in->mode == G5P_RU_KQ_OFF || !in->booted) { ++ *why = in->mode == G5P_RU_KQ_OFF ? G5P_RU_KQ_WHY_OFF : ++ G5P_RU_KQ_WHY_DOWN; ++ return 0; ++ } ++ if (!in->locked) ++ w |= G5P_RU_KQ_WHY_BUSY; ++ if (in->failed) ++ w |= G5P_RU_KQ_WHY_FAILED; ++ if (in->term_failed) ++ w |= G5P_RU_KQ_WHY_WEDGED; ++ if (!in->ring_idle) ++ w |= G5P_RU_KQ_WHY_RING; ++ if (!in->held) ++ w |= G5P_RU_KQ_WHY_NONE; ++ if (!in->frontend) ++ w |= G5P_RU_KQ_WHY_NO_DRM; ++ if (!w) ++ plan |= G5P_RU_KQ_DETACH; ++ if (in->mode < G5P_RU_KQ_FULL) ++ w |= G5P_RU_KQ_WHY_RAM; ++ else if (!in->pmgr) ++ w |= G5P_RU_KQ_WHY_NO_PMGR; ++ else if (!in->locked && !(in->executing && in->ram_only)) ++ w |= G5P_RU_KQ_WHY_HOLDER; ++ else ++ plan |= G5P_RU_KQ_CYCLE; ++ *why = w; ++ return plan; ++} ++ ++struct g5p_ru_kq_ops { ++ void *ctx; ++ u32 (*admit_close)(void *ctx); /* close admission; submitters still inside */ ++ bool (*lock)(void *ctx); /* bounded trylock of the command lock */ ++ void (*unlock)(void *ctx); ++ void (*inputs)(void *ctx, struct g5p_ru_kq_in *in); /* RAM only */ ++ int (*detach)(void *ctx); /* RAM only */ ++ void (*freeze)(void *ctx); /* RAM only */ ++ void (*irq_off)(void *ctx); /* AIC ack, mask, wait for the handler */ ++ int (*cycle)(void *ctx); /* PMGR only */ ++}; ++ ++struct g5p_ru_kq_out { ++ struct g5p_ru_kq_in in; ++ u32 inside, plan, why; ++ int detach_ret, cycle_ret; ++ bool froze; ++}; ++ ++/* One quiesce. detach_ret/cycle_ret stay 0 for a step that did not run (see plan). */ ++static inline void g5p_ru_kq_run(const struct g5p_ru_kq_ops *o, u32 mode, ++ struct g5p_ru_kq_out *out) ++{ ++ *out = (struct g5p_ru_kq_out){ .in.mode = mode }; ++ if (mode == G5P_RU_KQ_OFF) { ++ out->why = G5P_RU_KQ_WHY_OFF; ++ return; ++ } ++ out->inside = o->admit_close(o->ctx); ++ out->in.locked = o->lock(o->ctx); ++ o->inputs(o->ctx, &out->in); ++ out->plan = g5p_ru_kq_plan(&out->in, &out->why); ++ if (out->plan & G5P_RU_KQ_DETACH) ++ out->detach_ret = o->detach(o->ctx); ++ if (out->in.booted) { ++ o->freeze(o->ctx); ++ o->irq_off(o->ctx); ++ out->froze = true; ++ } ++ if (out->plan & G5P_RU_KQ_CYCLE) ++ out->cycle_ret = o->cycle(o->ctx); ++ if (out->in.locked) ++ o->unlock(o->ctx); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-recover.h b/drivers/gpu/apple-g5p/render-user-recover.h +new file mode 100644 +index 000000000..51982c5c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-recover.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * In-boot GPU recovery: the firmware's recovery packet and the recovery decision. Pure, ++ * host-tested (scripts/gpu/perf60-host/test_recover.c). ++ * ++ * The recovery packet is firmware event type 4 (44 bytes). The AGX kext's event drain ++ * (AGXFirmware_drainFirmwareEventRing 6c99788, case 4) reads +0x0c (the faulting slot, 0x80 for ++ * -1), +0x10 (a flag, accel+0x1af8 bit 3), +0x18, +0x20 and +0x24 (accel+0x1a1c/0x1af4/0x1af0), ++ * marks a recovery pending (accel+0x1a24 = 2) and calls IOGPUScheduler_signalHardwareError(sched, ++ * 1) (6c67b4c): "GPURestartSignaled", the scheduler enqueues one GPU restart (a second signal ++ * before it runs is "Redundant hardware error"). The restart is ++ * AGXMetaFirmware_stopForRecoveryOptionalPowerCycle (6ca1e40) and ++ * AGXMetaFirmware_rebootAfterRecovery (6ca1efc), which panics only if the firmware does not boot ++ * again. There is no per-boot limit: every recovery packet restarts the GPU. The ++ * kext does not read +0x04. ++ * ++ * On A9/Linux the slot matches the phase the host waited in: 0 on every TA fault, 1 on the 3D fault ++ * of cf7 (2026-11-02); +0x20 = 8 and +0x24 = 0x81818105 (TA) / 0x81810581 (3D) so far. ++ */ ++#ifndef G5P_RENDER_USER_RECOVER_H ++#define G5P_RENDER_USER_RECOVER_H ++ ++#define G5P_FW_EVENT_RECOVERY 4 ++#define G5P_FW_RECOVERY_NOSLOT 0x80 ++ ++struct g5p_fw_recovery { ++ u32 code; /* +0x04, not read by the kext */ ++ u32 slot; /* +0x0c, G5P_FW_RECOVERY_NOSLOT for -1 */ ++ u32 flag; /* +0x10 != 0 */ ++ u32 w18, w20, w24; ++}; ++ ++/* `w` is the 11-word event entry, type 4. */ ++static inline struct g5p_fw_recovery g5p_fw_recovery_decode(const u32 *w) ++{ ++ struct g5p_fw_recovery r = { ++ .code = w[1], ++ .slot = w[3] == 0xffffffffU ? G5P_FW_RECOVERY_NOSLOT : w[3], ++ .flag = w[4] != 0, ++ .w18 = w[6], ++ .w20 = w[8], ++ .w24 = w[9], ++ }; ++ ++ return r; ++} ++ ++/* ++ * May a job that stopped the firmware (a recovery packet, a silent event stream, a hard timeout) ++ * be recovered from in this boot, the job's context lost and every other kept? ++ * ++ * budget == 0 never (render_user_recover 0: the epoch ends, as before); ++ * retired/closed never (the boot is shutting the GPU down); ++ * streak_max == 0 the old rule: `budget` recoveries per boot, never on the epoch's first job; ++ * streak_max != 0 Apple's rule, every time, with two guards: at most streak_max recoveries in a ++ * row with no job completing between them (a GPU that faults again before it ++ * finishes anything is not recovering), and at most G5P_RU_RECOVER_BOOT_MAX per ++ * boot (recover.inc leaves each old epoch allocated, ~0.8 MiB: 50 x 16 KiB pages ++ * and its struct, so a stale pointer still reads valid memory; this bounds that ++ * to ~26 MiB). The boot's first epoch's first job stays terminal (a failed ++ * bring-up, not a client); the first job of a later epoch (the attach after a ++ * recovery) is recoverable like any other. ++ */ ++#define G5P_RU_RECOVER_BOOT_MAX 32 ++ ++struct g5p_ru_recover_in { ++ bool first; /* the job runs its epoch's attach */ ++ bool retired; /* the GPU was retired (kexec / reboot quiesce) */ ++ bool closed; /* admission closed */ ++ u32 recoveries; /* recoveries this boot */ ++ u32 streak; /* recoveries since the last job completed */ ++ u32 budget; /* render_user_recover */ ++ u32 streak_max; /* render_user_recover_streak */ ++}; ++ ++static inline bool g5p_ru_recoverable(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget || in->retired || in->closed) ++ return false; ++ if (!in->streak_max) ++ return !in->first && in->recoveries < in->budget; ++ if (in->first && !in->recoveries) ++ return false; ++ return in->streak < in->streak_max && ++ in->recoveries < G5P_RU_RECOVER_BOOT_MAX; ++} ++ ++/* Why g5p_ru_recoverable() said no (for the terminal line), or "-" when it said yes. */ ++static inline const char * ++g5p_ru_recover_refusal(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget) ++ return "recover-off"; ++ if (in->retired || in->closed) ++ return "shutdown"; ++ if (!in->streak_max) ++ return in->first ? "first-job" : ++ in->recoveries >= in->budget ? "per-boot-budget" : ++ "-"; ++ if (in->first && !in->recoveries) ++ return "first-job"; ++ if (in->streak >= in->streak_max) ++ return "streak"; ++ return in->recoveries >= G5P_RU_RECOVER_BOOT_MAX ? "boot-max" : "-"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-retire.h b/drivers/gpu/apple-g5p/render-user-retire.h +new file mode 100644 +index 000000000..6e5729003 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-retire.h +@@ -0,0 +1,248 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 3D completion: target coherence and engine idle, decided separately. Pure ++ * functions, no kernel services beyond the integer types, so the host suite compiles this ++ * file as it is. Include render-user-wait.h first. ++ * ++ * combined107 P4R seq 4 (build/gpu/combined107/raw/20261031T145710Z): the 3D target state ++ * was coherent (ring consumer = producer = target 4, both stamps 4, W+0x888/0x88c ours, ++ * W+0x890 = 4, all five channel watermarks at endpoint 8, and the firmware's type-1 3D ++ * completion event with the endpoint pair 7/8 decoded at publish + 19999 ms), yet the ++ * wait timed out at 60002 ms. The wait is a 1-2 ms RAM poll (no waitqueue, no wake), so ++ * this was not a lost edge: the done predicate also required the 3D global's idle words ++ * (G+0x724/0x728/0x72c/0x734/0x738) to be zero, and they never cleared. ++ * ++ * The 3D phase is complete when either ++ * (a) the target is coherent and the idle words are zero (the flown r96 predicate), or ++ * (b) the target is coherent and the matching 3D completion event was decoded after this ++ * job's 3D publish (idle_stuck). The event is the firmware's completion report, the ++ * signal a fence follows on Asahi too; the idle words lag it by an arbitrary time. ++ * settle_ms (default 0) only adds a minimum length of the coherent run, or ++ * (c) the target has been coherent, unbroken, for G5P_RU_RT_HOLD_MS (at least settle_ms): ++ * tahang1, as the TA's rule (c), so a lost or late event does not hold a job whose ++ * stamps landed. The PM state needs no clause here: the TA phase, which the 3D ++ * publish waits for, required it saved. ++ * Target coherent = ring consumer == target && g5p_ru_retired(): both stamps == seq, the W ++ * stamp pointers ours, the W sequence word == seq, every channel watermark == endpoint. ++ * The stamps carry the identity (31-bit sequence, one per job); the watermarks and the ++ * event's endpoint pair repeat every eight jobs and only corroborate. The TA phase has its ++ * own rules (tahang1, below). ++ */ ++#ifndef G5P_RENDER_USER_RETIRE_H ++#define G5P_RENDER_USER_RETIRE_H ++ ++#define G5P_RU_RT_DEF_SETTLE_MS 0U /* default of render_user_idle_settle_ms */ ++#define G5P_RU_RT_MAX_SETTLE_MS \ ++ 5000U /* far below the smallest per-phase bound (20 s) */ ++#define G5P_RU_RT_GRID 16U /* the channel watermark grid, G5P_WORK_SLOTS */ ++#define G5P_RU_RT_HOLD_MS \ ++ 500U /* rule (c), raised to settle_ms when that is longer */ ++ ++/* Clauses of the 3D predicate, as the receipt's `unsat` bits (set = not satisfied). */ ++#define G5P_RU_RT_CONSUMED 0x1U /* ring consumer != target */ ++#define G5P_RU_RT_RETIRED 0x2U /* g5p_ru_retired() false */ ++#define G5P_RU_RT_IDLE 0x4U /* an idle word is non-zero */ ++#define G5P_RU_RT_EVENT 0x8U /* no matching completion event latched */ ++#define G5P_RU_RT_SETTLE \ ++ 0x10U /* coherent, run shorter than the settle or hold window */ ++#define G5P_RU_RT_PM 0x20U /* TA receipt only: the PM state is not saved */ ++ ++enum g5p_ru_rt_how { ++ G5P_RU_RT_PENDING, ++ G5P_RU_RT_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_RT_IDLE_STUCK, /* rule (b) */ ++ G5P_RU_RT_HOLD, /* rule (c) */ ++}; ++ ++struct g5p_ru_retire { ++ u64 settle_ns; /* minimum coherent run before rule (b); 0 = none */ ++ u64 hold_ns; /* rule (c): max(settle_ns, G5P_RU_RT_HOLD_MS) */ ++ u64 run_ns; /* first sample of the current coherent run; 0 = none */ ++ u64 coherent_ns; /* first coherent sample of the phase */ ++ u64 event_ns; /* first sample that saw the matching event latched */ ++ u64 busy_ns; /* first sample with an idle word set */ ++ u64 clear_ns; /* first all-clear sample after busy_ns */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample */ ++ u32 how; /* enum g5p_ru_rt_how of the last sample */ ++ u32 decided; /* enum g5p_ru_rt_how of the sample that decided COMPLETE */ ++}; ++ ++static inline bool g5p_ru_rt_settle_valid(u32 settle_ms) ++{ ++ return settle_ms <= G5P_RU_RT_MAX_SETTLE_MS; ++} ++ ++static inline void g5p_ru_rt_start(struct g5p_ru_retire *t, u32 settle_ms) ++{ ++ u32 hold_ms = settle_ms > G5P_RU_RT_HOLD_MS ? settle_ms : ++ G5P_RU_RT_HOLD_MS; ++ ++ *t = (struct g5p_ru_retire){ ++ .settle_ns = (u64)settle_ms * 1000000ULL, ++ .hold_ns = (u64)hold_ms * 1000000ULL, ++ }; ++} ++ ++/* The firmware's 3D completion is a type-1 event with notification bit 1 (engine 1) whose ++ * words 8/9 are the channel endpoint pair (2 seq - 1, 2 seq): 1/2, 3/4, 5/6, 7/8 for ++ * combined107 seq 1..4. The grid wraps at G5P_RU_RT_GRID, so the pair is compared modulo ++ * the grid whether the firmware reports it wrapped or as a running count: endpoint 0 ++ * (seq 8) matches 15/0 and 15/16 alike. ++ */ ++static inline bool g5p_ru_rt_event_match(const u32 words[11], u32 endpoint) ++{ ++ return words[0] == 1 && (words[1] & 2) && ++ words[9] % G5P_RU_RT_GRID == endpoint % G5P_RU_RT_GRID && ++ (words[9] - words[8]) % G5P_RU_RT_GRID == 1; ++} ++ ++/* One RAM sample of the 3D phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_rt_how); the caller passes `!= G5P_RU_RT_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_rt_step(struct g5p_ru_retire *t, u64 now_ns, ++ bool consumed, bool retired, bool idle, ++ bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ if (!idle && !t->busy_ns) ++ t->busy_ns = now_ns; ++ if (idle && t->busy_ns && !t->clear_ns) ++ t->clear_ns = now_ns; ++ t->how = G5P_RU_RT_PENDING; ++ if (!consumed || !retired) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ t->run_ns = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_RT_IDLE_CLEAR; ++ else if (event && now_ns - t->run_ns >= t->settle_ns) ++ t->how = G5P_RU_RT_IDLE_STUCK; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_RT_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_rt_note(struct g5p_ru_retire *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++/* ms from `from_ns` to `to_ns` for the receipt; -1 when either is missing or out of order. */ ++static inline s64 g5p_ru_rt_ms(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000000ULL); ++} ++ ++/* ++ * tahang1: the TA phase. mcfold1 desktop seq 251 (after T1) and seq 339 (one client): ring ++ * consumed at 2 ms, the firmware's type-1 TA completion event decoded at 7.7-8.4 ms, both ++ * stamps, the W words and the watermarks at the job and the PM state saved, yet a TA idle word ++ * (G+0x724/0x728/0x72c) set from 7-8 ms on never cleared, and the r74b predicate (which needs ++ * the idle words zero) timed out at 60 s (`unsat=0x4`). On 49 of 241 good TAs of the same ++ * boot that word was set at 7-12 ms and cleared 2 ms later. The 3D engine shows the same lag ++ * (rule (b) above). ++ * ++ * The TA phase is complete when the target is coherent (ring consumer == target and ++ * g5p_ru_retired()) with the PM state saved, and ++ * (a) the idle words are zero (the flown r74b predicate), or ++ * (b) a type-1 TA completion event was decoded after this job's TA publish, or ++ * (c) the target has stayed coherent, unbroken, for G5P_RU_TA_HOLD_MS. ++ * The stamps carry the job's identity; the TA event does not (its words are the same every ++ * job), it only reports that the firmware finished a TA after the publish. Rule (c) keeps a ++ * lost or late event from holding a job whose stamps already landed. ++ */ ++#define G5P_RU_TA_HOLD_MS 500U ++ ++enum g5p_ru_ta_how { ++ G5P_RU_TA_PENDING, ++ G5P_RU_TA_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_TA_EVENT, /* rule (b) */ ++ G5P_RU_TA_HOLD, /* rule (c) */ ++ G5P_RU_TA_HOWS, ++}; ++ ++struct g5p_ru_ta { ++ u64 hold_ns; /* rule (c) */ ++ u64 run_ns; /* first sample of the current coherent run */ ++ u64 coherent_ns; /* first coherent sample of the phase; 0 = none */ ++ u64 event_ns; /* first sample that saw the event latched; 0 = none */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample (SETTLE: run below the hold) */ ++ u32 how; /* enum g5p_ru_ta_how of the last sample */ ++ u32 decided; /* enum g5p_ru_ta_how of the sample that decided COMPLETE */ ++}; ++ ++static inline void g5p_ru_ta_start(struct g5p_ru_ta *t) ++{ ++ *t = (struct g5p_ru_ta){ .hold_ns = ++ (u64)G5P_RU_TA_HOLD_MS * 1000000ULL }; ++} ++ ++/* The firmware's TA completion: a type-1 event with notification bit 0 (engine 0). */ ++static inline bool g5p_ru_ta_event_match(const u32 words[11]) ++{ ++ return words[0] == 1 && (words[1] & 1); ++} ++ ++/* One RAM sample of the TA phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_ta_how); the caller passes `!= G5P_RU_TA_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_ta_step(struct g5p_ru_ta *t, u64 now_ns, bool consumed, ++ bool retired, bool pm, bool idle, bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT) | ++ (pm ? 0 : G5P_RU_RT_PM); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ t->how = G5P_RU_TA_PENDING; ++ if (!consumed || !retired || !pm) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_TA_IDLE_CLEAR; ++ else if (event) ++ t->how = G5P_RU_TA_EVENT; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_TA_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_ta_note(struct g5p_ru_ta *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sched.h b/drivers/gpu/apple-g5p/render-user-sched.h +new file mode 100644 +index 000000000..174a29e23 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sched.h +@@ -0,0 +1,155 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Render job scheduling across contexts (docs/gpu/multi-client-design.md, stage 2), shaped ++ * like drm_sched so that stage 4 can swap drm_sched in without a UAPI or structural change: ++ * ++ * entity = one render context (drm_sched_entity): a FIFO of its accepted jobs, and at ++ * most one close item, which runs after its last job (drm_sched_entity_destroy()); ++ * run list = the entities with work, served round-robin: each pick takes the first entity ++ * whose head job is ready and moves it to the tail (drm_sched's RR policy); ++ * ready = every dependency of the head job has signalled, it failed before the hardware, ++ * its input deadline passed, or admission closed for good (the job then fails ++ * without the hardware). Readiness is the caller's predicate; the kernel's adds ++ * dma-fence callbacks (drm_sched_job_add_dependency()) so that an unsignalled ++ * input of one client never blocks another client's jobs. ++ * ++ * One credit: the single shared TA/3D channel and PM record run one job at a time, so the ++ * caller runs each picked item to completion before the next pick. Only the head of an ++ * entity is looked at, so each context's fences signal in submission order. ++ * ++ * The lists are plain list_heads; the caller holds its scheduler lock around every call. ++ * Host-tested (scripts/gpu/multiclient2-host/test_render_sched.c). ++ */ ++#ifndef G5P_RENDER_USER_SCHED_H ++#define G5P_RENDER_USER_SCHED_H ++ ++struct g5p_rs_job { ++ struct list_head link; ++}; ++ ++struct g5p_rs_entity { ++ struct list_head link; /* in g5p_rs.run while it has work */ ++ struct list_head jobs; /* FIFO of g5p_rs_job */ ++ bool in_run, close_pending; ++}; ++ ++struct g5p_rs { ++ struct list_head run; ++}; ++ ++/* g5p_rs_pick() result */ ++enum g5p_rs_pick { ++ G5P_RS_IDLE, /* nothing ready; *wake holds the earliest deadline if any job waits */ ++ G5P_RS_JOB, /* *job was popped from *entity */ ++ G5P_RS_CLOSE, /* *entity has no job left and its close item is due */ ++}; ++ ++/* Head-job predicate: true when it may run now. *deadline is the time at which it becomes ++ * ready on its own (its input deadline); the predicate leaves it alone if there is none. ++ */ ++typedef bool (*g5p_rs_ready_fn)(struct g5p_rs_job *job, void *cookie, ++ unsigned long *deadline); ++ ++static inline void g5p_rs_init(struct g5p_rs *s) ++{ ++ INIT_LIST_HEAD(&s->run); ++} ++ ++static inline void g5p_rs_entity_init(struct g5p_rs_entity *e) ++{ ++ INIT_LIST_HEAD(&e->link); ++ INIT_LIST_HEAD(&e->jobs); ++ e->in_run = false; ++ e->close_pending = false; ++} ++ ++static inline void g5p_rs_enlist(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (!e->in_run) { ++ list_add_tail(&e->link, &s->run); ++ e->in_run = true; ++ } ++} ++ ++/* An accepted job joins the end of its entity's FIFO. */ ++static inline void g5p_rs_push(struct g5p_rs *s, struct g5p_rs_entity *e, ++ struct g5p_rs_job *job) ++{ ++ list_add_tail(&job->link, &e->jobs); ++ g5p_rs_enlist(s, e); ++} ++ ++/* The entity's close item: due once its FIFO is empty. False if one is already queued. */ ++static inline bool g5p_rs_close(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (e->close_pending) ++ return false; ++ e->close_pending = true; ++ g5p_rs_enlist(s, e); ++ return true; ++} ++ ++/* After the close item ran: the entity leaves the scheduler for good. */ ++static inline void g5p_rs_closed(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ (void)s; ++ e->close_pending = false; ++ if (e->in_run) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } ++} ++ ++/* ++ * The next item to run, round-robin over the entities with work. A picked entity moves to ++ * the tail; one left without a job and without a close item leaves the run list. Entities ++ * whose head is not ready keep their place. `has_wake` reports whether *wake is set. ++ */ ++static inline int g5p_rs_pick(struct g5p_rs *s, g5p_rs_ready_fn ready, ++ void *cookie, struct g5p_rs_entity **entity, ++ struct g5p_rs_job **job, unsigned long *wake, ++ bool *has_wake) ++{ ++ struct g5p_rs_entity *e; ++ ++ *has_wake = false; ++ list_for_each_entry(e, &s->run, link) { ++ struct g5p_rs_job *head; ++ unsigned long deadline; ++ bool timed = false; ++ ++ if (list_empty(&e->jobs)) { ++ /* Without a close item it left the run list at its last pick. */ ++ if (!e->close_pending) ++ continue; ++ *entity = e; ++ *job = NULL; ++ list_move_tail(&e->link, &s->run); ++ return G5P_RS_CLOSE; ++ } ++ head = list_first_entry(&e->jobs, struct g5p_rs_job, link); ++ deadline = 0; ++ if (!ready(head, cookie, &deadline)) { ++ timed = deadline != 0; ++ if (timed && ++ (!*has_wake || (long)(deadline - *wake) < 0)) { ++ *wake = deadline; ++ *has_wake = true; ++ } ++ continue; ++ } ++ list_del_init(&head->link); ++ *entity = e; ++ *job = head; ++ if (list_empty(&e->jobs) && !e->close_pending) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } else { ++ list_move_tail(&e->link, &s->run); ++ } ++ return G5P_RS_JOB; ++ } ++ return G5P_RS_IDLE; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-serial.h b/drivers/gpu/apple-g5p/render-user-serial.h +new file mode 100644 +index 000000000..e5cffa4e3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-serial.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial jobs of every render context (a compositor and its clients: one TA+3D job ++ * per frame each), on the device's one TA/3D channel pair. ++ * Pure functions, host-tested (scripts/gpu/multiclient1-host/test_render_admit.c). ++ * ++ * Semantics: ++ * - jobs of all contexts run strictly one after another: the queue is FIFO, each job is ++ * TA publish -> TA complete -> 3D publish -> 3D complete before the next job starts, and ++ * there is never a TA(N+1) in flight with a 3D(N); ++ * - the epoch's first job brings the device up: PM, channels and its context's DC3 attach ++ * with the r74f/r74g bells (one-shot). Every later context binds its own slot pair at its ++ * first job with a DC3 pair published in RAM only (render-user-bind.h), no doorbell; ++ * - every job after the first only rebuilds the PM pool, builds its pages for the device's ++ * sequence number, publishes with NO doorbell and waits RAM-only; ++ * - the first job of a context that does not complete (timeout, fault, protocol error) ++ * marks that context bad: every job still queued behind it fails with -EIO without ++ * touching the hardware, and every later submit is refused (-EIO); ++ * - TYPED_V3 never mixes with the other profiles, on a context or on the device; ++ * - after any timeout or fault (render-user-terminal.h `failed`) no job starts at all, and ++ * once the epoch is quiet the device cannot be brought up (that rings a doorbell); binding ++ * another context is RAM only and stays allowed; ++ * - the sequence number is the device's, 31 bit; the channel watermark grid wraps ++ * (g5p_ru_endpoint). ++ */ ++#ifndef G5P_RENDER_USER_SERIAL_H ++#define G5P_RENDER_USER_SERIAL_H ++ ++enum g5p_ru_v3_refusal { ++ G5P_RU_V3_OK, ++ G5P_RU_V3_NO_TUPLE, /* boot tuple (r74g + r87b self-drain, no r74h/r80) absent */ ++ G5P_RU_V3_MIXED, /* another profile ran on this context or brought the device up */ ++ G5P_RU_V3_OVERFLOW, /* sequence space exhausted */ ++ G5P_RU_V3_SLOTS, /* not two distinct slots in 7..255 */ ++ G5P_RU_V3_VARIANT, /* r22/r25 variant, 3d_after_ta, timestamps or pm_policy */ ++ G5P_RU_V3_BOUND, /* per-phase bound outside 20000..30000 ms */ ++ G5P_RU_V3_STATE, /* executor only: bind state and job counts disagree */ ++ G5P_RU_V3_TERMINAL, /* terminal epoch: timeout/fault, or quiet and the device down */ ++ G5P_RU_V3_REFUSALS, ++}; ++ ++static const char *const g5p_ru_v3_why[G5P_RU_V3_REFUSALS] = { ++ "", ++ "transport is not ready for TYPED_V3 rendering", ++ "another profile ran on this context or brought the device up", ++ "sequence space exhausted", ++ "slots are not two distinct slots in 7..255", ++ "an r22/r25 variant, 3d_after_ta, timestamps or pm_policy is latched", ++ "per-phase bound outside 20000..30000 ms", ++ "bind state and job counts disagree", ++ "terminal epoch (timeout or fault, or quiet and the device not up): no hardware access", ++}; ++ ++/* Everything the admission needs, as plain values. */ ++struct g5p_ru_v3_state { ++ bool tuple; /* boot tuple present */ ++ bool v3_accepted; /* a V3 job was accepted on this context */ ++ bool bound; /* this context's slot pair was published */ ++ u32 jobs; /* jobs that started on this context */ ++ bool device_up; /* the epoch's first job built PM and channels */ ++ bool device_v3; /* ... through the TYPED_V3 attach */ ++ u32 sequence; /* the device's last job that started (0 = none) */ ++ u32 slot_ta, slot_3d; ++ bool variant; ++ u32 wait_ms; ++ bool quiet; /* a terminal note was made: mmio_ok is false */ ++ bool terminal_failed; /* ... and it was, or a later one is, a timeout or fault */ ++}; ++ ++/* Any two distinct user slots: unbinds leave holes, so a pair need not be adjacent. */ ++static inline bool g5p_ru_v3_slots_valid(u32 slot_ta, u32 slot_3d) ++{ ++ return slot_ta >= 7 && slot_3d >= 7 && slot_ta <= 255 && ++ slot_3d <= 255 && slot_ta != slot_3d; ++} ++ ++#define G5P_RU_V3_SEQ_MAX 0x7fffffffU ++ ++/* At submit time and again in the executor (`executing`): a refusal here touched nothing. */ ++static inline int g5p_ru_v3_admit(const struct g5p_ru_v3_state *s, ++ bool executing) ++{ ++ if (!s->tuple) ++ return G5P_RU_V3_NO_TUPLE; ++ /* After any timeout or fault nothing new starts (closes the window between a failing job's ++ * unlock and the context being marked bad). Once quiet, the device cannot be brought up, ++ * because that rings a doorbell; a job on an up device publishes with no doorbell and ++ * touches no register, and so does the bind of a new context. ++ */ ++ if (s->terminal_failed || (s->quiet && !s->device_up)) ++ return G5P_RU_V3_TERMINAL; ++ /* A job of another profile ran on this context, or another profile brought the device ++ * up: only V3 may follow V3. ++ */ ++ if ((s->jobs && !s->v3_accepted) || (s->device_up && !s->device_v3)) ++ return G5P_RU_V3_MIXED; ++ if (s->sequence >= G5P_RU_V3_SEQ_MAX) ++ return G5P_RU_V3_OVERFLOW; ++ if (!g5p_ru_v3_slots_valid(s->slot_ta, s->slot_3d)) ++ return G5P_RU_V3_SLOTS; ++ if (s->variant) ++ return G5P_RU_V3_VARIANT; ++ if (!g5p_ru_v3_bound_valid(s->wait_ms)) ++ return G5P_RU_V3_BOUND; ++ /* The executor: the device is up exactly when a job has started on it, and a context is ++ * bound exactly when it has run a job, always on an up device. ++ */ ++ if (executing && (s->device_up != !!s->sequence || ++ s->bound != !!s->jobs || (s->bound && !s->device_up))) ++ return G5P_RU_V3_STATE; ++ return G5P_RU_V3_OK; ++} ++ ++/* Verbose receipts for each context's first jobs and every failure; a compositor prints no ++ * line per frame. ++ */ ++static inline bool g5p_ru_v3_verbose(u32 jobs, int ret) ++{ ++ return jobs <= 8 || ret; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sync.h b/drivers/gpu/apple-g5p/render-user-sync.h +new file mode 100644 +index 000000000..d212d3199 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sync.h +@@ -0,0 +1,50 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CPU cache maintenance by range (perf60-fillrate.md §11.3 cut 5, kernel half). Host test: ++ * scripts/gpu/perf60-host/test_sync_range.c. ++ * ++ * render_user_sync_range bit 0: BO_SYNC honours offset/size (already validated page aligned by ++ * g5p_bo_range_valid) instead of syncing the whole BO. Bit 1: a job's ACCESS_CACHE maintenance ++ * (clean before, invalidate after retirement) covers the resource's BO range [offset, offset + ++ * size), the bytes the job can reach through that resource, instead of the whole BO. The window ++ * is widened to whole GPU pages and clamped to the BO. Off: the whole BO, as release8. ++ * The window is walked over the sg_table's DMA segments in order (their lengths concatenate to ++ * the BO's bytes); each segment gets the intersection, relative to the segment. ++ */ ++#ifndef G5P_RENDER_USER_SYNC_H ++#define G5P_RENDER_USER_SYNC_H ++ ++#define G5P_SYNC_ALIGN 0x4000ULL ++ ++/* ++ * [lo, hi) = [off, off + size) widened to pages, clamped to the BO; false if empty or outside. ++ */ ++static inline bool g5p_sync_window(u64 off, u64 size, u64 bo_size, u64 *lo, ++ u64 *hi) ++{ ++ u64 end; ++ ++ if (!size || off >= bo_size) ++ return false; ++ end = size > bo_size - off ? bo_size : off + size; ++ *lo = off & ~(G5P_SYNC_ALIGN - 1); ++ *hi = (end + G5P_SYNC_ALIGN - 1) & ~(G5P_SYNC_ALIGN - 1); ++ if (*hi > bo_size) ++ *hi = bo_size; ++ return *hi > *lo; ++} ++ ++/* The part of segment [pos, pos + len) inside [lo, hi): *start relative to the segment. */ ++static inline bool g5p_sync_clip(u64 pos, u64 len, u64 lo, u64 hi, u64 *start, ++ u64 *n) ++{ ++ u64 a = pos > lo ? pos : lo, b = pos + len < hi ? pos + len : hi; ++ ++ if (b <= a) ++ return false; ++ *start = a - pos; ++ *n = b - a; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-template.h b/drivers/gpu/apple-g5p/render-user-template.h +new file mode 100644 +index 000000000..92bd1604a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-template.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Generated by make-render-user-template.py from the hash-pinned r22 importer. */ ++#ifndef G5P_RENDER_USER_TEMPLATE_H ++#define G5P_RENDER_USER_TEMPLATE_H ++ ++/* Frozen dialect-1 W scalars {offset, value}: color-only, one sample, ++ * normal 3D, no depth/resource arrays. Typed and kernel fields excluded. ++ */ ++static const u32 g5p_ru_ta_fixed[][2] = { ++ { 0x020, 0x1e3ce508 }, { 0x024, 0x1e3ce508 }, { 0x3a4, 0x00000088 }, ++ { 0x3a8, 0x00000002 }, { 0x3c0, 0x00020000 }, { 0x514, 0x08002358 }, ++ { 0x528, 0x00000001 }, ++}; ++ ++static const u32 g5p_ru_3d_fixed[][2] = { ++ { 0x108, 0x000003c3 }, { 0x110, 0x000003c3 }, { 0x118, 0x000003c3 }, ++ { 0x120, 0x000003c3 }, { 0x128, 0x000003c3 }, { 0x130, 0x000003c3 }, ++ { 0x3a4, 0x3f800000 }, { 0x3a8, 0x00000300 }, { 0x3b0, 0x0000001f }, ++ { 0x3c0, 0x0023000f }, { 0x3c4, 0x00000408 }, { 0x3c8, 0x00000008 }, ++ { 0x3cc, 0x000000f8 }, { 0x6a8, 0x3f800000 }, { 0x6ac, 0x00000300 }, ++ { 0x6b0, 0x00000002 }, { 0x868, 0x08002358 }, ++}; ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-terminal.h b/drivers/gpu/apple-g5p/render-user-terminal.h +new file mode 100644 +index 000000000..4ced26eaf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-terminal.h +@@ -0,0 +1,103 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 terminal state of the epoch. Pure functions over one small struct; the ++ * kernel supplies READ_ONCE/WRITE_ONCE/cmpxchg and the host suite shims them, so the ++ * host compiles this file as it is. ++ * ++ * One latch, first writer wins. The first terminal event of the epoch (the natural TA ++ * complete, a timeout, a fault or a protocol error) fixes `kind`; from then on `quiet` ++ * is set and never cleared, and g5p_ru_mmio_ok() is false: the host touches no register ++ * (R0, R1, AIC, doorbell) and no diagnostic bank. RAM stays accessible. A later ++ * failure (the 3D after a completed TA, job N+1 after job N) cannot change `kind` but ++ * sets the sticky `failed`, which is what the admission uses: after any timeout or fault ++ * no new job starts, from any queue, even one that passed the pre-lock context checks ++ * before the failing job marked the context bad. ++ * ++ * The recovery series (patches/gpu-recovery) is independent of this file. Its ++ * g5p_td_note_terminal() takes {1 complete, 2 timeout}; g5p_ru_td_kind() maps ours. ++ */ ++#ifndef G5P_RENDER_USER_TERMINAL_H ++#define G5P_RENDER_USER_TERMINAL_H ++ ++enum g5p_ru_term_kind { ++ G5P_RU_TERM_NONE, ++ G5P_RU_TERM_COMPLETE, /* the natural TA completed (RAM) */ ++ G5P_RU_TERM_TIMEOUT, /* a phase ran out its deadline */ ++ G5P_RU_TERM_FAULT, /* protocol error, drain error, attach or publish failure */ ++ G5P_RU_TERM_KINDS, ++}; ++ ++static const char *const g5p_ru_term_names[G5P_RU_TERM_KINDS] = { ++ "none", ++ "complete", ++ "timeout", ++ "fault", ++}; ++ ++struct g5p_ru_terminal { ++ int kind; /* first writer wins */ ++ int quiet; /* set with the first note, never cleared */ ++ int failed; /* sticky: any timeout or fault was noted */ ++ int captured; /* the CAPTURE entry (recovery hook) was taken, once */ ++}; ++ ++/* A new firmware epoch (the in-boot recovery, recover.inc) starts a new latch; nothing else ++ * clears it. ++ */ ++static inline void g5p_ru_term_reset(struct g5p_ru_terminal *t) ++{ ++ WRITE_ONCE(t->kind, 0); ++ WRITE_ONCE(t->quiet, 0); ++ WRITE_ONCE(t->failed, 0); ++ WRITE_ONCE(t->captured, 0); ++} ++ ++/* The kind a job's return value stands for (only called for ret < 0, ret != -ECANCELED). */ ++static inline int g5p_ru_term_from_ret(int ret) ++{ ++ return ret == -ETIMEDOUT ? G5P_RU_TERM_TIMEOUT : G5P_RU_TERM_FAULT; ++} ++ ++/* Returns true for the first note of the epoch. */ ++static inline bool g5p_ru_term_note(struct g5p_ru_terminal *t, int kind) ++{ ++ bool first; ++ ++ if (kind <= G5P_RU_TERM_NONE || kind >= G5P_RU_TERM_KINDS) ++ kind = G5P_RU_TERM_FAULT; ++ first = cmpxchg(&t->kind, G5P_RU_TERM_NONE, kind) == G5P_RU_TERM_NONE; ++ if (kind >= G5P_RU_TERM_TIMEOUT) ++ WRITE_ONCE(t->failed, 1); ++ WRITE_ONCE(t->quiet, 1); ++ return first; ++} ++ ++static inline int g5p_ru_term_kind(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->kind); ++} ++ ++static inline bool g5p_ru_term_failed(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->failed); ++} ++ ++/* The single predicate for register access: false from the first terminal note on. */ ++static inline bool g5p_ru_mmio_ok(const struct g5p_ru_terminal *t) ++{ ++ return !READ_ONCE(t->quiet); ++} ++ ++/* True once per epoch for a timeout or fault: the caller may take the recovery CAPTURE entry. */ ++static inline bool g5p_ru_term_capture_due(struct g5p_ru_terminal *t, int kind) ++{ ++ return kind >= G5P_RU_TERM_TIMEOUT && cmpxchg(&t->captured, 0, 1) == 0; ++} ++ ++/* Recovery's terminal encoding: 1 complete, 2 timeout (a fault is a no-completion exit: 2). */ ++static inline int g5p_ru_td_kind(int kind) ++{ ++ return kind == G5P_RU_TERM_COMPLETE ? 1 : 2; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-touch.h b/drivers/gpu/apple-g5p/render-user-touch.h +new file mode 100644 +index 000000000..145bdbf2d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-touch.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 RAM-touch latency A/B (release). Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. Include render-user-wait.h first. ++ * ++ * Question (docs/gpu/experiments/latency101-device-analysis.md section 9): is the host's bulk read ++ * of the receipt pages what releases a stalled phase (the +6 ms atom after the job-1 receipt)? ++ * ++ * render_user_touch_ms = T, default 0 = off. On, every phase of every job records the first wait ++ * sample at or after publish + T that left the phase undecided (still pending, deadline not ++ * reached). Odd sequences are the touch arm: right after that sample the host reads the ++ * receipt's read set once (RAM only, folded into a sum). Even sequences are the control arm: the ++ * same instant is recorded and nothing is read. Neither arm changes a sample, the sleep, the ++ * deadline, a latch or the result, and neither prints before the job ends. ++ * ++ * Off: g5p_ru_touch_start() leaves the record zero and g5p_ru_touch_due() is never true, so the ++ * wait loop runs the latency101 instructions unchanged. ++ */ ++#ifndef G5P_RENDER_USER_TOUCH_H ++#define G5P_RENDER_USER_TOUCH_H ++ ++#define G5P_RU_TOUCH_MAX_MS \ ++ 20000U /* never at or past the smallest soft bound */ ++ ++struct g5p_ru_touch { ++ u64 at_ns; /* publish + T; 0 = off for this phase */ ++ u64 seen_ns; /* the undecided sample at or after at_ns that took it; 0 = not reached */ ++ u64 done_ns; /* end of the read (touch arm), seen_ns (control arm) */ ++ u32 bytes; /* bytes read: 0 on the control arm */ ++ u32 sum; /* fold of the words read: the loads cannot be elided */ ++ bool sweep; /* this phase is in the touch arm */ ++}; ++ ++/* 0 (off), or 1..G5P_RU_TOUCH_MAX_MS with latency101 on (hard_ms != 0) and strictly before the ++ * effective soft bound (g5p_ru_lat_soft()), so the instant always falls in the plain wait. ++ */ ++static inline bool g5p_ru_touch_valid(u32 touch_ms, u32 soft_ms, u32 hard_ms) ++{ ++ if (!touch_ms) ++ return true; ++ return hard_ms && touch_ms <= G5P_RU_TOUCH_MAX_MS && touch_ms < soft_ms; ++} ++ ++/* Odd sequences are the touch arm, even ones the control. */ ++static inline bool g5p_ru_touch_arm(u32 seq) ++{ ++ return seq & 1; ++} ++ ++static inline void g5p_ru_touch_start(struct g5p_ru_touch *t, u64 start_ns, ++ u32 touch_ms, u32 seq) ++{ ++ *t = (struct g5p_ru_touch){ 0 }; ++ if (!touch_ms) ++ return; ++ t->at_ns = start_ns + (u64)touch_ms * 1000000ULL; ++ t->sweep = g5p_ru_touch_arm(seq); ++} ++ ++/* ++ * Asked after g5p_ru_wait_step() of the sample taken at now_ns: true once, for the first sample ++ * at ++ * or after the instant whose step left the phase undecided and not expired. A completion, a fault ++ * or the deadline in the same sample wins, and then the instant is never taken. ++ */ ++static inline bool g5p_ru_touch_due(const struct g5p_ru_touch *t, u64 now_ns, ++ u32 result, bool expired) ++{ ++ return t->at_ns && !t->seen_ns && !result && !expired && ++ now_ns >= t->at_ns; ++} ++ ++/* The instant was taken at seen_ns (a sample time, never 0); done_ns is when the read ended. */ ++static inline void g5p_ru_touch_mark(struct g5p_ru_touch *t, u64 seen_ns, ++ u64 done_ns, u32 bytes, u32 sum) ++{ ++ t->seen_ns = seen_ns; ++ t->done_ns = done_ns < seen_ns ? seen_ns : done_ns; ++ t->bytes = bytes; ++ t->sum = sum; ++} ++ ++/* Receipt helper: to - from in microseconds, -1 when an end is missing or the order is wrong. */ ++static inline s64 g5p_ru_touch_us(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000); ++} ++ ++/* Receipt: how long after the instant (the end of the read) the phase's deciding sample saw it ++ * complete; -1 when the instant was not taken or the phase did not complete. ++ */ ++static inline s64 g5p_ru_touch_after_us(const struct g5p_ru_touch *t, ++ u32 result, u64 end_ns) ++{ ++ if (!t->seen_ns || result != G5P_RU_WAIT_COMPLETE) ++ return -1; ++ return g5p_ru_touch_us(t->done_ns, end_ns); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wait.h b/drivers/gpu/apple-g5p/render-user-wait.h +new file mode 100644 +index 000000000..b9b5f1c49 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wait.h +@@ -0,0 +1,266 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial wait policy. Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. ++ * ++ * Evidence behind the numbers (r96 raw timelines, 2026-10-30): ++ * - the natural TA completion event arrived +12.0060 s and +12.0062 s after the ++ * publish in the two boots that a 12000 ms cap turned into timeouts; the whole ++ * delay is inside the firmware's start23, and once it returns the hardware and ++ * the retirement take less than ~1 ms; ++ * - nothing the host does shortens the delay, so the wait is RAM-only: after the ++ * DC3 attach the host touches no register until the job has ended. ++ * ++ * Policy: ++ * - one phase (TA, then 3D) waits at most `bound` from its own publish: 20 s by ++ * default, never below 20 s, never above the absolute cap; ++ * - the whole job (TA publish to the end of the 3D phase) never exceeds the ++ * absolute cap of 30 s; ++ * - the deadline is decided on RAM samples. A completion seen in the sample that ++ * reaches the deadline wins; a deadline is only final after one closing sample ++ * taken at or after it, which closes the gap between the last poll and the ++ * expiry; a completion after that is evidence only (`late_after`), never the ++ * result; ++ * - a fault wins over a completion in the same sample and over the deadline. ++ */ ++#ifndef G5P_RENDER_USER_WAIT_H ++#define G5P_RENDER_USER_WAIT_H ++ ++#define G5P_RU_V3_WAIT_MS 20000U /* default and minimum per-phase bound */ ++#define G5P_RU_V3_CAP_MS \ ++ 30000U /* absolute job bound, and the largest per-phase bound */ ++ ++/* ++ * latency101 (default off: render_user_soft_ms = render_user_hard_ms = 0 leaves every number above, ++ * and the behaviour of every function below, exactly as it was). When on, the per-phase deadline ++ * is the HARD bound and the job bound is two hard bounds; the SOFT bound marks a phase "slow" ++ * (RAM-only record, no print, no latch). A silent event stream means a dead firmware from the ++ * phase's publish on (faultiso; before it, only from the soft bound). See ++ * docs/gpu/experiments/prod98c-bimodal-latency.md and docs/gpu/experiments/faultiso.md. ++ */ ++#define G5P_RU_LAT_MIN_SOFT_MS \ ++ 20000U /* never below the proven per-phase bound */ ++#define G5P_RU_LAT_DEF_SOFT_MS 45000U ++#define G5P_RU_LAT_MAX_HARD_MS \ ++ 60000U /* largest hard bound; the job bound is twice this */ ++#define G5P_RU_LAT_DEF_OBS_MS \ ++ 2000U /* RAM-only look after a timeout, pre-receipt */ ++#define G5P_RU_LAT_MAX_OBS_MS 5000U ++#define G5P_RU_LAT_SILENT_MS \ ++ 1000U /* no event decoded for this long in a phase = silent */ ++/* The silence clock runs from the publish (the host suites key on this). */ ++#define G5P_RU_LAT_SILENT_FROM_PUBLISH 1 ++ ++enum g5p_ru_wait_result { ++ G5P_RU_WAIT_PENDING, ++ G5P_RU_WAIT_COMPLETE, ++ G5P_RU_WAIT_TIMEOUT, ++ G5P_RU_WAIT_FAULT, ++}; ++ ++struct g5p_ru_wait { ++ u64 start_ns; /* the publish (producer store) of this phase */ ++ u64 deadline_ns; /* start + bound, clamped to the job's absolute deadline */ ++ u64 end_ns; /* the sample that decided */ ++ u32 result; ++ bool expired; /* a sample at or after the deadline has been seen */ ++ bool late_gap; /* completion first seen by the closing sample */ ++ bool late_after; /* completion seen after a timeout was decided: evidence only */ ++ /* latency101; all zero (and never read) unless soft_ns != 0 */ ++ u64 soft_ns; /* start + soft bound; 0 = the grace is off */ ++ u64 soft_seen_ns; /* the first sample at or after soft_ns */ ++ u64 late_ns; /* the first retired sample after the timeout was decided */ ++ u64 event_ns; /* sample time of the last change of the decoded-event counter */ ++ u64 busy_ns; /* the first sample whose idle words showed the engine busy */ ++ u32 events; /* the decoded-event counter at that sample */ ++ bool slow; /* the phase passed its soft bound and is still pending */ ++ bool silent; /* the event stream stopped during the phase: firmware dead */ ++ bool late_soft; /* completed after the soft bound: a success, never a failure */ ++}; ++ ++/* latency101 parameters. Off is hard = soft = 0. On: hard in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000. The job bound the waiters must outlast is two ++ * hard bounds plus one observation window. ++ */ ++static inline u32 g5p_ru_lat_soft(u32 soft_ms, u32 hard_ms) ++{ ++ if (soft_ms) ++ return soft_ms; ++ return hard_ms > G5P_RU_LAT_DEF_SOFT_MS ? G5P_RU_LAT_DEF_SOFT_MS : ++ hard_ms; ++} ++ ++static inline bool g5p_ru_lat_valid(u32 soft_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!hard_ms) ++ return !soft_ms; ++ soft_ms = g5p_ru_lat_soft(soft_ms, hard_ms); ++ return hard_ms <= G5P_RU_LAT_MAX_HARD_MS && ++ soft_ms >= G5P_RU_LAT_MIN_SOFT_MS && soft_ms <= hard_ms && ++ obs_ms <= G5P_RU_LAT_MAX_OBS_MS; ++} ++ ++static inline u32 g5p_ru_lat_job_ms(u32 hard_ms, u32 obs_ms) ++{ ++ return 2 * hard_ms + obs_ms; ++} ++ ++/* A per-phase bound is valid when it is in [G5P_RU_V3_WAIT_MS, G5P_RU_V3_CAP_MS]. */ ++static inline bool g5p_ru_v3_bound_valid(u32 ms) ++{ ++ return ms >= G5P_RU_V3_WAIT_MS && ms <= G5P_RU_V3_CAP_MS; ++} ++ ++/* The deadline of a phase: its own bound, clamped to the job's absolute deadline. ++ * A job deadline of 0 means "no job deadline yet" (the first phase). ++ */ ++static inline u64 g5p_ru_wait_deadline(u64 start_ns, u32 bound_ms, ++ u64 job_deadline_ns) ++{ ++ u64 d = start_ns + (u64)bound_ms * 1000000ULL; ++ ++ return job_deadline_ns && job_deadline_ns < d ? job_deadline_ns : d; ++} ++ ++static inline void g5p_ru_wait_start(struct g5p_ru_wait *w, u64 start_ns, ++ u32 bound_ms, u64 job_deadline_ns) ++{ ++ *w = (struct g5p_ru_wait){ ++ .start_ns = start_ns, ++ .deadline_ns = g5p_ru_wait_deadline(start_ns, bound_ms, ++ job_deadline_ns), ++ }; ++} ++ ++/* ++ * latency101: hard_ms is the per-phase deadline (clamped to the job's deadline as above), ++ * soft_ms ++ * (never past the deadline) the point where a still-pending phase is called slow. `events` seeds ++ * the silence clock. ++ */ ++static inline void g5p_ru_wait_start_lat(struct g5p_ru_wait *w, u64 start_ns, ++ u32 soft_ms, u32 hard_ms, ++ u64 job_deadline_ns, u32 events) ++{ ++ g5p_ru_wait_start(w, start_ns, hard_ms, job_deadline_ns); ++ w->soft_ns = start_ns + (u64)soft_ms * 1000000ULL; ++ if (w->soft_ns > w->deadline_ns) ++ w->soft_ns = w->deadline_ns; ++ w->events = events; ++ w->event_ns = start_ns; ++} ++ ++/* The first sample whose idle words (G+0x724..) show the engine busy, i.e. the firmware's ++ * start path has run. Recorded once; it splits a late start (busy_ns close to the ++ * completion) from slow hardware (busy_ns early). A busy window shorter than the poll ++ * period can be missed: 0 = not seen. ++ */ ++static inline void g5p_ru_wait_busy(struct g5p_ru_wait *w, u64 now_ns, ++ bool busy) ++{ ++ if (busy && !w->busy_ns) ++ w->busy_ns = now_ns; ++} ++ ++/* One look at the decoded-event counter, taken with every RAM sample, before g5p_ru_wait_step(). ++ * The ALIVE stream runs at about 290 events/s whatever the work does (r96c: 29442 polls, no gap ++ * over 3.4 ms; latency101: 287 events/s through every stall of 0.55-33 s), so a stream that has ++ * been silent for G5P_RU_LAT_SILENT_MS is a dead firmware, not a slow one, at any point of the ++ * phase. Returns true then; the caller reports it as a fault. The silence clock starts at the ++ * publish (vpds rung vk, 2026-11-01: the counter froze 1 s after the TA's publish, and a clock ++ * that only ran from the soft bound called it at 45 s, with every other client held behind it). ++ * The caller drains the ring before each sample, so a host that slept longer than the window ++ * sees the events it missed and is not silent. Never true when latency101 is off. ++ */ ++static inline bool g5p_ru_wait_grace(struct g5p_ru_wait *w, u64 now_ns, ++ u32 events) ++{ ++ if (!w->soft_ns) ++ return false; ++ if (events != w->events) { ++ w->events = events; ++ w->event_ns = now_ns; ++ } ++ if (now_ns >= w->soft_ns && !w->soft_seen_ns) ++ w->soft_seen_ns = now_ns; ++ w->silent = now_ns - w->event_ns >= ++ (u64)G5P_RU_LAT_SILENT_MS * 1000000ULL; ++ return w->silent; ++} ++ ++/* One RAM sample taken at `now_ns`. Returns the result so far (PENDING until ++ * decided). A sample at or after the deadline only marks the wait expired unless ++ * it shows completion or a fault; the caller must then take one closing sample and ++ * call g5p_ru_wait_close(). ++ */ ++static inline u32 g5p_ru_wait_step(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (w->soft_ns && now_ns >= w->soft_ns) ++ w->slow = true; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ w->end_ns = now_ns; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->end_ns = now_ns; ++ w->late_soft = w->slow; ++ } else if (now_ns >= w->deadline_ns) { ++ w->expired = true; ++ } ++ return w->result; ++} ++ ++/* The closing sample, taken after the loop left because of the deadline. This is ++ * the only place a timeout is decided. ++ */ ++static inline u32 g5p_ru_wait_close(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->late_gap = true; ++ w->late_soft = w->slow; ++ } else { ++ w->result = G5P_RU_WAIT_TIMEOUT; ++ } ++ w->end_ns = now_ns; ++ return w->result; ++} ++ ++/* A sample after the result was decided: a completion after a timeout is recorded ++ * and nothing else happens (no 3D, no recovery, no register access). ++ */ ++static inline void g5p_ru_wait_after(struct g5p_ru_wait *w, bool complete) ++{ ++ if (w->result == G5P_RU_WAIT_TIMEOUT && complete) ++ w->late_after = true; ++} ++ ++/* latency101: the RAM-only observation window after a timeout was decided, before the receipt is ++ * printed and the terminal latch is taken. g5p_ru_wait_after() looks once, microseconds after the ++ * decision; a completion that arrives milliseconds later (prod98c seq 3: 5.49 ms) was never ++ * recorded, so `late_after=0/0` was a false negative. Returns true while the window is open. A ++ * retired sample ends it and is recorded in late_after/late_ns; it never changes the result (a ++ * deadline stays final). obs_ms = 0 closes it at once (the legacy behaviour). ++ */ ++static inline bool g5p_ru_wait_observe(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, u32 obs_ms) ++{ ++ if (w->result != G5P_RU_WAIT_TIMEOUT || w->late_after) ++ return false; ++ if (complete) { ++ w->late_after = true; ++ w->late_ns = now_ns; ++ return false; ++ } ++ return now_ns < w->end_ns + (u64)obs_ms * 1000000ULL; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +new file mode 100644 +index 000000000..64a51eaf4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -0,0 +1,273 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 publish-time MTS wake (wake110). Pure functions over plain values; the one ++ * register sequence goes through a callback table (the fender.h pattern), so the host ++ * suite compiles this file as it is and records every access in order. ++ * ++ * Question (docs/gpu/experiments/wake110-apple-mts-dispatch.md): is the 0.5-33 s TA/3D ++ * start latency caused by the missing Apple submit kick? Apple's TA and 3D submit ++ * wrappers (iPad6,11 kernelcache 0xfffffff006ca00a0 / 0xfffffff006ca01f8) do, every time: ++ * ring entry, dmb ish, Fender vt+0x80 (R1+0x14 = 7, read back), Fender vt+0x18 ++ * (R1+0x1000 & 0xf), then R0+0xb00 = 0x10 (TA, DM0|COUNTED) or 0x11 (3D, DM1|COUNTED). ++ * The MTS store is reached from every arm; when the GPC nibble is 0 Apple first runs a ++ * power arm (up to forcePowerCycleAndReboot). Linux V3 publishes job N >= 2's TA and ++ * every 3D with no kick at all. ++ * ++ * render_user_wake (latched per context): 0 off (default: no call has any effect), ++ * 1 the A/B, 2 every eligible phase is a factor (the candidate production fix, for use ++ * only after the A/B proved it). The TA of job 1 is never a phase of this file (the r74g ++ * grace and its fenced fallback bell own it). In the A/B each engine's phases are paired ++ * in publish order, (0,1), (2,3), ...; exactly one of each pair is the factor, which one ++ * is fixed by render_user_wake_seed before the context exists, and bit 31 of the seed ++ * inverts the whole schedule (the mirror stage). The verdict recomputes the arm of every ++ * receipt from the seed. ++ * ++ * At the publish, right after the producer store and its barrier, both arms take the same ++ * lock and read the same RAM state and run the same guards; the control stops there (no ++ * register). The factor then runs the Apple order: R1+0x14 = 7, read back (any bit ++ * outside 7: stop), R1+0x1000 (state & 0x1f != 1, the GPC is not active: stop, never ++ * repaired, never a power arm), dma_wmb(), R0+0xb00 = 0x10 | engine. At most one attempt ++ * per phase. Never after a timeout or fault (the terminal latch's sticky `failed`), never ++ * with the boot failed or stopped, never unpowered, never with an r74f bell in flight, ++ * never once the phase shows any sign of retirement, never on a ring that is not exactly ++ * this publish. A consumer already at the target does not cancel (Apple never checks ++ * it); it is recorded. The natural TA completion (terminal kind COMPLETE, the epoch's ++ * first note) is not a failure: the factor's sequence is the one licensed exception to the ++ * r96 "no register after the TA complete" envelope, default off. ++ */ ++#ifndef G5P_RENDER_USER_WAKE_H ++#define G5P_RENDER_USER_WAKE_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++#define G5P_WK_OFF 0U ++#define G5P_WK_AB 1U ++#define G5P_WK_ALWAYS 2U ++#define G5P_WK_MIRROR 0x80000000U /* seed bit 31: every arm inverted */ ++ ++#define G5P_WK_R1_FENDER 0x14U /* Fender vt+0x80: write 7, read back */ ++#define G5P_WK_R1_GPC 0x1000U /* GPC state: & 0x1f == 1 is active */ ++#define G5P_WK_R0_MTS 0xb00U /* MTS_SCHEDULE, write only */ ++#define G5P_WK_MTS_TA 0x10U /* DM0 | COUNTED */ ++#define G5P_WK_MTS_3D 0x11U /* DM1 | COUNTED */ ++ ++/* wake111: the publish-time power evidence, one bit per RAM fact, sampled under the event lock ++ * and printed per phase (power=TT/DD). Every required bit must be set. reset_transition is ++ * the permanent post-release slave-port lockout (set right before the release, never ++ * cleared), not a live reset state: Wake110 vetoed on it, so every phase skipped. It is ++ * recorded (SLAVE_LOCK) and is neither required nor a veto; the slave-port ban is unchanged. ++ */ ++#define G5P_WK_PWR_CORE 0x01U /* g->powered */ ++#define G5P_WK_PWR_FENDER 0x02U /* g->fender_ready */ ++#define G5P_WK_PWR_RELEASED 0x04U /* b->released */ ++#define G5P_WK_PWR_ALIVE 0x08U /* b->alive */ ++#define G5P_WK_PWR_CONTROL 0x10U /* b->control_complete */ ++#define G5P_WK_PWR_DS108 \ ++ 0x20U /* DS108 claim proof, current generation, no power abort */ ++#define G5P_WK_PWR_SLAVE_LOCK 0x40U /* g->reset_transition: evidence only */ ++#define G5P_WK_PWR_REQUIRED \ ++ (G5P_WK_PWR_CORE | G5P_WK_PWR_FENDER | G5P_WK_PWR_RELEASED | \ ++ G5P_WK_PWR_ALIVE | G5P_WK_PWR_CONTROL | G5P_WK_PWR_DS108) ++ ++enum g5p_wk_arm { ++ G5P_WK_ARM_NONE, /* wake off */ ++ G5P_WK_ARM_EXCLUDED, /* the TA of job 1 (r74g) */ ++ G5P_WK_ARM_CONTROL, ++ G5P_WK_ARM_FACTOR, ++ G5P_WK_ARMS, ++}; ++ ++enum g5p_wk_did { ++ G5P_WK_DID_NONE, /* off, or never reached */ ++ G5P_WK_DID_RANG, /* factor: fence, GPC active, 0xb00 written */ ++ G5P_WK_DID_SHAM, /* control: every guard passed, no register */ ++ G5P_WK_DID_EXCLUDED, ++ G5P_WK_SKIP_TWICE, /* this phase was already decided */ ++ G5P_WK_SKIP_LICENSE, /* a timeout or fault was noted (terminal failed) */ ++ G5P_WK_SKIP_FAULT, /* boot failed, event failed or stopped */ ++ G5P_WK_SKIP_POWER, /* one of G5P_WK_PWR_REQUIRED is absent */ ++ G5P_WK_SKIP_R74F, /* an r74f bell is published, open or firing */ ++ G5P_WK_SKIP_RING, /* the ring is not exactly this publish */ ++ G5P_WK_SKIP_RETIRED, /* a stamp == seq or a completion event since the publish */ ++ G5P_WK_SKIP_FENCE, /* R1+0x14 read back outside 7: no further register */ ++ G5P_WK_SKIP_GPC, /* GPC not active: no 0xb00 */ ++ G5P_WK_DIDS, ++}; ++ ++static const char *const g5p_wk_arm_names[G5P_WK_ARMS] = { ++ "none", ++ "excluded", ++ "control", ++ "factor", ++}; ++ ++static const char *const g5p_wk_did_names[G5P_WK_DIDS] = { ++ "none", "rang", "sham", "excluded", "skip-twice", ++ "skip-license", "skip-fault", "skip-power", "skip-r74f", "skip-ring", ++ "skip-retired", "skip-fence", "skip-gpc", ++}; ++ ++/* Everything the decision reads, sampled under the event lock right after the producer store. */ ++struct g5p_wk_in { ++ u32 mode, seed, seq, e; ++ bool attempted; /* this phase already went through g5p_wk_decide() */ ++ bool term_failed; /* g5p_ru_term_failed() */ ++ bool boot_failed; /* b->failed || b->event_failed || b->stopped */ ++ u32 power; /* g5p_wk_power_mask(); the slave lock is evidence, not a veto */ ++ bool r74f_pending; /* an r74f bell in PUBLISHED, OPEN or FIRING */ ++ u32 start, target, prod, ++ cons; /* ring: producer before this publish, our target, now */ ++ u32 stamp[2]; /* this engine's stamp words */ ++ u32 type1; /* this engine's completion events decoded since the phase armed */ ++}; ++ ++/* The register access of the factor, in order. The kernel binds them to writel/readl on the ++ * Fender window and the RGX window and to dma_wmb(); the host suite records them. ++ */ ++struct g5p_wk_io { ++ void *ctx; ++ void (*w1)(void *ctx, u32 reg, u32 value); ++ u32 (*r1)(void *ctx, u32 reg); ++ void (*wbarrier)(void *ctx); ++ void (*w0)(void *ctx, u32 reg, u32 value); ++}; ++ ++static inline bool g5p_wk_mode_valid(u32 mode) ++{ ++ return mode <= G5P_WK_ALWAYS; ++} ++ ++static inline u32 g5p_wk_mix(u32 x) ++{ ++ x ^= x >> 16; ++ x *= 0x7feb352dU; ++ x ^= x >> 15; ++ x *= 0x846ca68bU; ++ x ^= x >> 16; ++ return x; ++} ++ ++/* The phase's index among its engine's eligible phases: TA of job seq >= 2 -> seq - 2, ++ * 3D of job seq >= 1 -> seq - 1. ++ */ ++static inline u32 g5p_wk_index(u32 seq, u32 e) ++{ ++ return e ? seq - 1 : seq - 2; ++} ++ ++static inline u32 g5p_wk_arm(u32 mode, u32 seed, u32 seq, u32 e) ++{ ++ u32 i, flip; ++ ++ if (!mode || !g5p_wk_mode_valid(mode) || !seq || e > 1) ++ return G5P_WK_ARM_NONE; ++ if (!e && seq == 1) ++ return G5P_WK_ARM_EXCLUDED; ++ if (mode == G5P_WK_ALWAYS) ++ return G5P_WK_ARM_FACTOR; ++ i = g5p_wk_index(seq, e); ++ flip = g5p_wk_mix(g5p_wk_mix(seed & ~G5P_WK_MIRROR) + (e << 24) + ++ (i >> 1)) & ++ 1; ++ flip ^= !!(seed & G5P_WK_MIRROR); ++ return ((i & 1) ^ flip) ? G5P_WK_ARM_FACTOR : G5P_WK_ARM_CONTROL; ++} ++ ++static inline u32 g5p_wk_mts(u32 e) ++{ ++ return G5P_WK_MTS_TA | e; ++} ++ ++/* wake111: the evidence word, one bit per fact (the caller samples each fact once). */ ++static inline u32 g5p_wk_power_mask(bool powered, bool fender_ready, ++ bool released, bool alive, ++ bool control_complete, bool ds108, ++ bool slave_lock) ++{ ++ return (powered ? G5P_WK_PWR_CORE : 0) | ++ (fender_ready ? G5P_WK_PWR_FENDER : 0) | ++ (released ? G5P_WK_PWR_RELEASED : 0) | ++ (alive ? G5P_WK_PWR_ALIVE : 0) | ++ (control_complete ? G5P_WK_PWR_CONTROL : 0) | ++ (ds108 ? G5P_WK_PWR_DS108 : 0) | ++ (slave_lock ? G5P_WK_PWR_SLAVE_LOCK : 0); ++} ++ ++static inline bool g5p_wk_power_ready(u32 power) ++{ ++ return (power & G5P_WK_PWR_REQUIRED) == G5P_WK_PWR_REQUIRED; ++} ++ ++/* The RAM guards, common to both arms, in priority order. 0 = pass. */ ++static inline u32 g5p_wk_guard(const struct g5p_wk_in *in) ++{ ++ if (in->attempted) ++ return G5P_WK_SKIP_TWICE; ++ if (in->term_failed) ++ return G5P_WK_SKIP_LICENSE; ++ if (in->boot_failed) ++ return G5P_WK_SKIP_FAULT; ++ if (!g5p_wk_power_ready(in->power)) ++ return G5P_WK_SKIP_POWER; ++ if (in->r74f_pending) ++ return G5P_WK_SKIP_R74F; ++ if (in->start > 255 || in->target > 255 || in->prod != in->target || ++ in->cons > 255 || ((in->target - in->start) & 255) != 1 || ++ ((in->cons - in->start) & 255) > 1) ++ return G5P_WK_SKIP_RING; ++ if (in->stamp[0] == in->seq || in->stamp[1] == in->seq || in->type1) ++ return G5P_WK_SKIP_RETIRED; ++ return 0; ++} ++ ++/* The factor's register sequence (Apple's order). The caller set *fence and *gpc to U32_MAX ++ * (not read). ++ */ ++static inline u32 g5p_wk_ring(const struct g5p_wk_io *io, u32 e, u32 *fence, ++ u32 *gpc) ++{ ++ io->w1(io->ctx, G5P_WK_R1_FENDER, 7); ++ *fence = io->r1(io->ctx, G5P_WK_R1_FENDER); ++ if (*fence & ~7U) ++ return G5P_WK_SKIP_FENCE; ++ *gpc = io->r1(io->ctx, G5P_WK_R1_GPC); ++ if (!G5P_GPC_ON(*gpc)) ++ return G5P_WK_SKIP_GPC; ++ /* the publish's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ io->wbarrier(io->ctx); ++ io->w0(io->ctx, G5P_WK_R0_MTS, g5p_wk_mts(e)); ++ return G5P_WK_DID_RANG; ++} ++ ++/* One phase's decision and, for a factor that passed every guard, its register sequence ++ * (the control arm never touches io). ++ */ ++static inline u32 g5p_wk_decide(const struct g5p_wk_in *in, ++ const struct g5p_wk_io *io, u32 *fence, ++ u32 *gpc) ++{ ++ u32 arm = g5p_wk_arm(in->mode, in->seed, in->seq, in->e), skip; ++ ++ *fence = U32_MAX; ++ *gpc = U32_MAX; ++ if (arm == G5P_WK_ARM_NONE) ++ return G5P_WK_DID_NONE; ++ if (arm == G5P_WK_ARM_EXCLUDED) ++ return G5P_WK_DID_EXCLUDED; ++ skip = g5p_wk_guard(in); ++ if (skip) ++ return skip; ++ if (arm == G5P_WK_ARM_CONTROL) ++ return G5P_WK_DID_SHAM; ++ return g5p_wk_ring(io, in->e, fence, gpc); ++} ++ ++/* A decided phase never decides again (the caller stores this before it unlocks). */ ++static inline bool g5p_wk_attempted(u32 did) ++{ ++ return did != G5P_WK_DID_NONE && did != G5P_WK_DID_EXCLUDED; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-window.h b/drivers/gpu/apple-g5p/render-user-window.h +new file mode 100644 +index 000000000..bfbae628d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-window.h +@@ -0,0 +1,74 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The firmware window of a render VM (docs/gpu/multi-client-design.md §6 R4, stage 3). ++ * ++ * The firmware's own GPU memory lives in one GiB, catalogue entry G5P_RU_FIRMWARE_CI: its ++ * image, the root and shared pages (rings, the PA/tag arrays, W), the MMU table alias and the ++ * PIO pages (Fender, AIC). Stage 1 copied that catalogue entry into every render VM, so any ++ * render job could address all of it. A render VM now gets, under that entry, a directory ++ * the kernel built that maps only the pages the GPU's own requesters reach through the ++ * context's slot: the PM control page (S) and the PM list and free list (F). Their leaf ++ * entries are copied from the firmware's tables, so their translation is unchanged. ++ * ++ * render_user_fw_window = 0 (G5P_RU_FW_FULL) the stage 1 mapping: the whole firmware GiB; ++ * render contexts need CAP_SYS_RAWIO ++ * 1 (G5P_RU_FW_WINDOW) the window above (default) ++ * 2 (G5P_RU_FW_NONE) no firmware page at all (an experiment) ++ * ++ * Pure helpers, host-tested (scripts/gpu/multiclient2-host/test_render_window.c). The ++ * caller provides g5p_le32() and the boot layout constants. ++ */ ++#ifndef G5P_RENDER_USER_WINDOW_H ++#define G5P_RENDER_USER_WINDOW_H ++ ++#define G5P_RU_FW_FULL 0U ++#define G5P_RU_FW_WINDOW 1U ++#define G5P_RU_FW_NONE 2U ++#define G5P_RU_FW_MODES 3U ++ ++#define G5P_RU_FW_PAGES 3U ++/* The pages of the window, all in the firmware GiB. */ ++static const u64 g5p_ru_fw_window_va[G5P_RU_FW_PAGES] = { ++ G5P_SHARED_VA + G5P_R22_PM, /* PM control and S */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE, /* PM list */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, /* PM free list F */ ++}; ++ ++static inline u32 g5p_ru_fw_di(u64 va) ++{ ++ return (va >> 21) & 511; ++} ++ ++static inline u32 g5p_ru_fw_ti(u64 va) ++{ ++ return (va >> 14) & 127; ++} ++ ++/* The firmware's 64-bit leaf entry for `va`, from its leaf pages and their directory ++ * indices; 0 when the firmware does not map it. ++ */ ++static inline u64 g5p_ru_fw_leaf(u8 *const *leaf, const u32 *leaf_di, ++ u32 leaves, u64 va) ++{ ++ for (u32 i = 0; i < leaves; i++) { ++ const u8 *e = leaf[i] + g5p_ru_fw_ti(va) * 8; ++ ++ if (leaf_di[i] == g5p_ru_fw_di(va)) ++ return (u64)g5p_le32(e + 4) << 32 | g5p_le32(e); ++ } ++ return 0; ++} ++ ++/* The window's leaf page index for `va` among `n` distinct directory indices so far, or n ++ * (a new one) when none has its directory index. ++ */ ++static inline u32 g5p_ru_fw_slot(const u32 *di, u32 n, u64 va) ++{ ++ u32 i; ++ ++ for (i = 0; i < n && di[i] != g5p_ru_fw_di(va); i++) ++ ; ++ return i; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-variants-r25.h b/drivers/gpu/apple-g5p/render-variants-r25.h +new file mode 100644 +index 000000000..8469a2f4b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-variants-r25.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_VARIANTS_R25_H ++#define G5P_RENDER_VARIANTS_R25_H ++ ++/* Finite, owned-input experiments. No raw W, register or firmware-patch API. */ ++#define G5P_R25_MAX_BLOCKS 128U ++#define G5P_R25_HEAD 0x3000U ++ ++struct g5p_r25_variants { ++ u32 pm, blocks, context, host, sku, memory, rt, slots; ++}; ++ ++static inline bool g5p_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return v->pm <= 3 && ++ (v->blocks == 0 || v->blocks == 64 || v->blocks == 128) && ++ v->context <= 3 && v->host <= 2 && v->sku <= 3 && ++ v->memory <= 4 && v->rt <= 2 && v->slots <= 1; ++} ++ ++/* Event synthesis, removed context tasks, candidate clocks and the low-address ++ * dialect test cannot qualify rendering, even if firmware stamps retire. ++ */ ++static inline bool g5p_r25_diagnostic(const struct g5p_r25_variants *v) ++{ ++ return v->sku || v->context || v->host || v->rt == 2; ++} ++ ++/* PM payload frames are physical PPNs and have no user PTE attributes. This ++ * changes only PM auxiliary mappings (and, in mode4, other private mappings). ++ * Firmware/shared META mappings and the borrowed sink are never rewritten. ++ */ ++static inline u32 g5p_r25_attributes(const struct g5p_r25_variants *v, u64 va) ++{ ++ if (!v->memory) ++ return 0xc; ++ if (v->memory != 4 && ++ (va < G5P_R22_AUX || va >= G5P_R22_AUX + 5 * G5P_GPU_PAGE)) ++ return 0xc; ++ return v->memory == 2 ? 4 : v->memory == 3 ? 8 : 0; ++} ++ ++/* Preserve the original constructor exactly for the default pool. The extra ++ * blocks fit the existing 16KiB list and free allocations, including F's last ++ * dword at128 blocks. Validate every PA before mutating either allocation. ++ */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/retire108.h b/drivers/gpu/apple-g5p/retire108.h +new file mode 100644 +index 000000000..e944b92c9 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/retire108.h +@@ -0,0 +1,434 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RETIRE108_H ++#define G5P_RETIRE108_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P outgoing retire layer (series gpu-recovery108, patch 0006). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 3. ++ * ++ * Wraps the recovery98 teardown ladder (teardown.h, g5p_td_run()) without ++ * changing it: the TD98 lines of a run are byte for byte the ones of the ++ * same run without this layer. It adds, around a run: ++ * ++ * ADMIT atomically close job admission (one cmpxchg word), then a ++ * bounded RAM-only wait for submitters already inside the ++ * admission window; still inside at the deadline = explicit ++ * ADMIT_OPEN flag, never a block. ++ * CLASSIFY the epoch class from RAM inputs only (struct g5p_r8_in). ++ * RECEIPT one TR108 line with the class and the generation, printed ++ * before the ladder makes its first AIC or R1 access. ++ * (ladder) g5p_td_run(FINAL) through a callback, unchanged. ++ * SLOTS three named post-ladder slots. Each runs only with a glue op ++ * present AND its own parameter on; every skip is named: ++ * fw-sleep validated firmware sleep before stable-off (G2: ++ * no recovered command; the glue has no op) ++ * stable-off one R1 read of isStableOff after a verified ++ * Fender stop leg (G6: default off) ++ * pmgr-gate outgoing PMGR power gate (G1: not implemented, ++ * there is no op and no parameter) ++ * RETIRE the retired latch: from here on g5p_r8_mmio_ok() refuses R0, ++ * R1 and every write except the always-on AIC ack of a late ++ * handler entry. The latch is set whatever the verdict. ++ * ++ * Generations: gen counts real runs (CAPTURE and FINAL) since boot and is ++ * monotonic; a REENTRY (a second FINAL after retire, e.g. notifier then ++ * .shutdown then remove) prints a line but does not advance it. ++ * ++ * Pure policy, like teardown.h: no register access, no kernel API except ++ * cmpxchg(), READ_ONCE()/WRITE_ONCE(), scnprintf() and the errno names. The ++ * same file is compiled by the kernel glue (teardown.inc) and by the offline ++ * tests (patches/gpu-recovery108/tests). Needs teardown.h first. ++ */ ++ ++#define G5P_R8_CLOSED 0x80000000u /* admission word: closed bit */ ++#define G5P_R8_INSIDE 0x7fffffffu /* admission word: submitters inside */ ++#define G5P_R8_POLL_US 100 ++#define G5P_R8_MAX_STEPS 512 /* ADMIT wait cap, independent of the clock */ ++#define G5P_R8_ADMIT_US 20000 /* ADMIT wait budget */ ++ ++/* Admission: one word, closed bit + count of submitters inside the window. */ ++struct g5p_r8_admit { ++ u32 word; ++}; ++ ++/* A submitter enters before its first check of driver state and leaves on ++ * every return path. False once admission is closed: the submit must fail ++ * with -ESHUTDOWN and touch nothing. ++ */ ++static inline bool g5p_r8_admit_enter(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if ((cur & G5P_R8_CLOSED) || ++ (cur & G5P_R8_INSIDE) == G5P_R8_INSIDE) ++ return false; ++ old = cmpxchg(&a->word, cur, cur + 1); ++ if (old == cur) ++ return true; ++ cur = old; ++ } ++} ++ ++static inline void g5p_r8_admit_exit(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if (!(cur & G5P_R8_INSIDE)) ++ return; /* unbalanced exit: never wrap into the closed bit */ ++ old = cmpxchg(&a->word, cur, cur - 1); ++ if (old == cur) ++ return; ++ cur = old; ++ } ++} ++ ++/* Close (idempotent). Returns the submitters inside at the instant of closing. */ ++static inline u32 g5p_r8_admit_close(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ old = cmpxchg(&a->word, cur, cur | G5P_R8_CLOSED); ++ if (old == cur) ++ return cur & G5P_R8_INSIDE; ++ cur = old; ++ } ++} ++ ++static inline u32 g5p_r8_admit_inside(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_INSIDE; ++} ++ ++static inline bool g5p_r8_admit_closed(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_CLOSED; ++} ++ ++/* Epoch classes. The dirty ones are never reported as clean (H10). */ ++enum g5p_r8_class { ++ G5P_R8C_NEVER_BOOTED, /* shared pages never mapped */ ++ G5P_R8C_UNRELEASED, /* mapped, firmware never released */ ++ G5P_R8C_IDLE, /* released and ALIVE, no context ever claimed (not-run) */ ++ G5P_R8C_CLEAN, /* claimed, every job signalled, no holder left */ ++ G5P_R8C_HELD, /* clean, but a process still holds the claimed context */ ++ G5P_R8C_PARKED, /* GPC=0 already verified by a park */ ++ G5P_R8C_IN_FLIGHT, /* a job accepted and not signalled, or a worker inside a wait */ ++ G5P_R8C_TIMEOUT, /* terminal latch = timeout (a late completion stays a timeout) */ ++ G5P_R8C_FAULT, /* failed, event failure or a fault terminal */ ++ G5P_R8C_ABORTED, /* bootstrap power abort latched (the r87b/C3 epoch) */ ++ G5P_R8C_UNKNOWN, /* released, never ALIVE; or an unverified earlier stop */ ++ G5P_R8C_N, ++}; ++ ++static const char *const g5p_r8_class_name[G5P_R8C_N] = { ++ "never-booted", "unreleased", "idle", "clean", "held", "parked", ++ "in-flight", "timeout", "fault", "aborted", "unknown", ++}; ++ ++static inline bool g5p_r8_class_dirty(enum g5p_r8_class c) ++{ ++ return c >= G5P_R8C_IN_FLIGHT; ++} ++ ++/* RAM-only inputs, read by the glue before the run (never from a register). */ ++struct g5p_r8_in { ++ bool booted; /* b->maps_ready */ ++ bool released; /* b->released */ ++ bool alive; /* b->alive before FREEZE, or ALIVE was proven this boot */ ++ bool failed; /* b->failed */ ++ bool fault_event; /* b->event_failed */ ++ int terminal; /* teardown terminal latch: 0 none, 1 complete, 2 timeout/fault */ ++ bool claimed; /* a context claimed the device (DRM quarantined_context) */ ++ bool held; /* ... and a render node file still references it */ ++ u32 queued; /* accepted render jobs not yet signalled */ ++ bool worker_busy; /* the command lock was busy: a worker may be inside a wait */ ++ bool parked; /* b->parked */ ++ bool stopped_unparked; /* b->stopped && !b->parked before this run */ ++ bool power_abort; /* r73b early_power_abort */ ++}; ++ ++static inline u32 g5p_r8_in_bits(const struct g5p_r8_in *i) ++{ ++ return i->booted | i->released << 1 | i->alive << 2 | i->failed << 3 | ++ i->fault_event << 4 | i->claimed << 5 | i->held << 6 | ++ (i->queued != 0) << 7 | i->worker_busy << 8 | i->parked << 9 | ++ i->stopped_unparked << 10 | i->power_abort << 11; ++} ++ ++/* First match wins; the order is the precedence (docs section 3.2). */ ++static inline enum g5p_r8_class g5p_r8_classify(const struct g5p_r8_in *i) ++{ ++ if (!i->booted) ++ return G5P_R8C_NEVER_BOOTED; ++ if (!i->released) ++ return G5P_R8C_UNRELEASED; ++ if (i->stopped_unparked) ++ return G5P_R8C_UNKNOWN; ++ if (i->power_abort) ++ return G5P_R8C_ABORTED; ++ if (i->failed || i->fault_event) ++ return G5P_R8C_FAULT; ++ if (i->terminal == 2) ++ return G5P_R8C_TIMEOUT; ++ if (i->queued || i->worker_busy) ++ return G5P_R8C_IN_FLIGHT; ++ if (i->parked) ++ return G5P_R8C_PARKED; ++ if (!i->alive) ++ return G5P_R8C_UNKNOWN; ++ if (i->claimed) ++ return i->held ? G5P_R8C_HELD : G5P_R8C_CLEAN; ++ return G5P_R8C_IDLE; ++} ++ ++enum g5p_r8_slot { ++ G5P_R8S_FW_SLEEP, ++ G5P_R8S_STABLE_OFF, ++ G5P_R8S_PMGR_GATE, ++ G5P_R8S_N, ++}; ++ ++enum g5p_r8_skip { ++ G5P_R8K_RAN, /* the slot ran (see its result) */ ++ G5P_R8K_NO_MECHANISM, /* the glue has no op for it */ ++ G5P_R8K_OFF, /* its parameter is off (default) */ ++ G5P_R8K_NOT_FINAL, /* CAPTURE never runs a slot */ ++ G5P_R8K_NEEDS_RT1, /* needs a verified Fender stop leg in this run */ ++ G5P_R8K_DIRTY, /* the ladder ended dirty: no register access */ ++ G5P_R8K_N, ++}; ++ ++static const char *const g5p_r8_skip_name[G5P_R8K_N] = { ++ "ran", "no-mechanism", "off", "not-final", "needs-rt1", "dirty", ++}; ++ ++#define G5P_R8F_ADMIT_OPEN \ ++ BIT(0) /* submitters still inside at the ADMIT deadline */ ++#define G5P_R8F_ADMIT_LATE BIT(1) /* ... or the step cap ended the wait */ ++#define G5P_R8F_REENTRY BIT(2) /* answered after retire: nothing ran */ ++#define G5P_R8F_BUSY BIT(3) /* a concurrent entrant was inside */ ++#define G5P_R8F_SLOT_FAIL BIT(4) /* a slot ran and failed */ ++#define G5P_R8F_STABLE_OFF BIT(5) /* stable-off slot read isStableOff == 1 */ ++/* jobs still queued at DETACH (abandoned to their worker) */ ++#define G5P_R8F_DETACH_OPEN BIT(6) ++ ++/* ++ * Ops. td_run runs the recovery98 ladder and returns its rc; td_flags/td_verdict ++ * read its result. slot[] may hold NULL (no mechanism). Only slot ops touch a ++ * register, and only after the ladder, never after RETIRE. ++ */ ++struct g5p_r8_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); ++ int (*td_run)(void *ctx, enum g5p_td_mode mode); ++ u32 (*td_flags)(void *ctx); ++ enum g5p_td_verdict (*td_verdict)(void *ctx); ++ int (*slot[G5P_R8S_N])(void *ctx, u32 *value); ++ void (*emit)(void *ctx, const char *line); ++}; ++ ++struct g5p_r8 { ++ struct g5p_r8_admit admit; ++ u32 admit_us; /* ADMIT budget */ ++ bool slot_on[G5P_R8S_N]; /* the slots' parameters (default all off) */ ++ /* latches */ ++ int running; ++ int retired; /* 0, or the generation that retired the device */ ++ u32 gen; /* monotonic: real runs since boot */ ++ /* last run */ ++ enum g5p_td_mode mode; ++ enum g5p_r8_class cls; ++ u32 in_bits, flags, inside0, inside1, steps, detach_queued; ++ enum g5p_r8_skip skip[G5P_R8S_N]; ++ int slot_rc[G5P_R8S_N]; ++ u32 slot_val[G5P_R8S_N]; ++ int rc; ++ enum g5p_td_verdict verdict; ++ u32 lines; /* TR108 lines emitted since boot */ ++}; ++ ++static inline void g5p_r8_init(struct g5p_r8 *s) ++{ ++ *s = (struct g5p_r8){ .admit_us = G5P_R8_ADMIT_US }; ++} ++ ++/* ++ * The predicate for every register access outside the ladder: after RETIRE only ++ * RAM, a PMGR read and the AIC (the always-on swirq ack of a late handler entry) ++ * remain. R0 and R1 are refused forever, whatever the verdict. ++ */ ++static inline bool g5p_r8_mmio_ok(const struct g5p_r8 *s, enum g5p_td_dom d) ++{ ++ if (d == G5P_TDD_RAM || d == G5P_TDD_PMGR || d == G5P_TDD_AIC) ++ return true; ++ return !READ_ONCE(s->retired); ++} ++ ++static inline bool g5p_r8_retired(const struct g5p_r8 *s) ++{ ++ return READ_ONCE(s->retired) != 0; ++} ++ ++/* The DRM frontend's DETACH callback reports what it left to the job workers. */ ++static inline void g5p_r8_note_detach(struct g5p_r8 *s, u32 queued) ++{ ++ WRITE_ONCE(s->detach_queued, queued); ++ if (queued) ++ s->flags |= G5P_R8F_DETACH_OPEN; ++} ++ ++static inline void g5p_r8_line(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ const char *line) ++{ ++ s->lines++; ++ if (o->emit) ++ o->emit(o->ctx, line); ++} ++ ++/* Bounded RAM-only wait for submitters inside the admission window. */ ++static inline void g5p_r8_admit_wait(struct g5p_r8 *s, ++ const struct g5p_r8_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)s->admit_us * 1000; ++ ++ s->inside0 = g5p_r8_admit_close(&s->admit); ++ s->inside1 = s->inside0; ++ s->steps = 0; ++ while (s->inside1) { ++ if (s->steps >= G5P_R8_MAX_STEPS) { ++ s->flags |= G5P_R8F_ADMIT_LATE; ++ break; ++ } ++ s->steps++; ++ if (o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_R8_POLL_US); ++ s->inside1 = g5p_r8_admit_inside(&s->admit); ++ } ++ if (s->inside1) ++ s->flags |= G5P_R8F_ADMIT_OPEN; ++} ++ ++static inline void g5p_r8_slots(struct g5p_r8 *s, const struct g5p_r8_ops *o) ++{ ++ u32 tdf = o->td_flags ? o->td_flags(o->ctx) : 0; ++ unsigned int i; ++ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ s->slot_rc[i] = 0; ++ s->slot_val[i] = 0; ++ if (s->mode != G5P_TD_FINAL) ++ s->skip[i] = G5P_R8K_NOT_FINAL; ++ else if (!o->slot[i]) ++ s->skip[i] = G5P_R8K_NO_MECHANISM; ++ else if (!s->slot_on[i]) ++ s->skip[i] = G5P_R8K_OFF; ++ else if (s->verdict == G5P_TDV_DIRTY) ++ s->skip[i] = G5P_R8K_DIRTY; ++ else if (!(tdf & G5P_TDF_RT1_OK)) ++ s->skip[i] = G5P_R8K_NEEDS_RT1; ++ else ++ s->skip[i] = G5P_R8K_RAN; ++ } ++ /* One attempt each, in slot order, no retry; a failed slot ends the slots. */ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ if (s->skip[i] != G5P_R8K_RAN) ++ continue; ++ if (s->flags & G5P_R8F_SLOT_FAIL) { ++ s->skip[i] = G5P_R8K_DIRTY; ++ continue; ++ } ++ s->slot_rc[i] = o->slot[i](o->ctx, &s->slot_val[i]); ++ if (s->slot_rc[i]) ++ s->flags |= G5P_R8F_SLOT_FAIL; ++ else if (i == G5P_R8S_STABLE_OFF && s->slot_val[i] == 1) ++ s->flags |= G5P_R8F_STABLE_OFF; ++ } ++} ++ ++/* ++ * One run. mode is the teardown mode the glue chose (teardown=1 CAPTURE, ++ * teardown=2 FINAL). Returns the ladder's rc, -EBUSY for a concurrent entrant, ++ * and the cached rc after retire. Lines: ++ * TR108 gen= phase=classify why= mode= class= dirty= in= inside= after= flags= admit_us= ++ * (TD98 lines of the ladder, unchanged) ++ * TR108 gen= phase=retired|captured verdict= class= flags= fw-sleep= stable-off= pmgr-gate= ++ * so= detach_queued= rc= ++ * TR108 gen= phase=reentry why= retired_gen= rc= (after retire; no gen step) ++ */ ++static inline int g5p_r8_run(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ enum g5p_td_mode mode, const struct g5p_r8_in *in, ++ const char *why) ++{ ++ char line[256]; ++ u64 t0; ++ int rc; ++ ++ if (cmpxchg(&s->running, 0, 1)) { ++ /* A concurrent entrant touches nothing; it does not reach the ladder either. */ ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=busy why=%s mode=%s rc=%d", ++ READ_ONCE(s->gen), why, g5p_td_mode_name[mode], ++ -EBUSY); ++ if (o->emit) ++ o->emit(o->ctx, line); ++ return -EBUSY; ++ } ++ if (READ_ONCE(s->retired)) { ++ /* The ladder answers from its cache too (REENTRY); nothing else runs. */ ++ rc = o->td_run(o->ctx, mode); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=reentry why=%s mode=%s retired_gen=%d rc=%d", ++ s->gen, why, g5p_td_mode_name[mode], ++ READ_ONCE(s->retired), rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++ } ++ s->gen++; ++ s->mode = mode; ++ s->flags = 0; ++ s->inside0 = 0; ++ s->inside1 = 0; ++ s->steps = 0; ++ WRITE_ONCE(s->detach_queued, 0); ++ t0 = o->now_ns(o->ctx); ++ if (mode == G5P_TD_FINAL) ++ g5p_r8_admit_wait(s, o); ++ s->cls = g5p_r8_classify(in); ++ s->in_bits = g5p_r8_in_bits(in); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=classify why=%s mode=%s class=%s dirty=%u in=%03x term=%d queued=%u inside=%u after=%u flags=%02x admit_us=%u", ++ s->gen, why, g5p_td_mode_name[mode], g5p_r8_class_name[s->cls], ++ g5p_r8_class_dirty(s->cls), s->in_bits, in->terminal, ++ in->queued, s->inside0, s->inside1, s->flags, ++ (u32)((o->now_ns(o->ctx) - t0) / 1000)); ++ /* The receipt is out before the ladder's first AIC or R1 access. */ ++ g5p_r8_line(s, o, line); ++ s->rc = o->td_run(o->ctx, mode); ++ s->verdict = o->td_verdict ? o->td_verdict(o->ctx) : G5P_TDV_NONE; ++ g5p_r8_slots(s, o); ++ if (mode == G5P_TD_FINAL) ++ WRITE_ONCE(s->retired, (int)s->gen); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=%s verdict=%s class=%s flags=%02x fw-sleep=%s stable-off=%s pmgr-gate=%s so=%u detach_queued=%u rc=%d", ++ s->gen, mode == G5P_TD_FINAL ? "retired" : "captured", ++ g5p_td_verdict_name[s->verdict], g5p_r8_class_name[s->cls], ++ s->flags, g5p_r8_skip_name[s->skip[G5P_R8S_FW_SLEEP]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_STABLE_OFF]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_PMGR_GATE]], ++ s->slot_val[G5P_R8S_STABLE_OFF], READ_ONCE(s->detach_queued), ++ s->rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.h b/drivers/gpu/apple-g5p/teardown.h +new file mode 100644 +index 000000000..b536f428c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.h +@@ -0,0 +1,740 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_TEARDOWN_H ++#define G5P_TEARDOWN_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P GPU teardown state machine (series gpu-recovery98). ++ * ++ * Design and evidence: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md. ++ * ++ * Pure policy: no register access, no kernel API except cmpxchg(), scnprintf() ++ * and the errno names. Every hardware or firmware-visible access goes through ++ * struct g5p_td_ops, and every call is checked against g5p_td_allow[phase][domain] ++ * before it is made. The same file is compiled by the kernel glue (teardown.inc) ++ * and by the offline fault-injection tests (patches/gpu-recovery/tests). ++ * ++ * Two modes: ++ * CAPTURE timeout / fault / attach failure, and (gpu-recovery99) any number of looks at a ++ * live epoch. Observation only: reads RAM rings, ++ * changes nothing the host could still want to export. Never writes ++ * driver or firmware-shared state, never masks the IRQ, never touches ++ * a register. Arms the quiet latch for diagnostics (g5p_td_mmio_ok()). ++ * FINAL shutdown / reboot notifier / remove. Runs the whole ladder and leaves ++ * a known state for the next kernel. ++ * ++ * Domains: RAM (DMA pages + driver structs), AIC (swirq 0x20e104000 ack and the ++ * IRQ-chip mask), R1 (Fender wrapper window), R0 (core register window), PMGR ++ * (0x20e080340 read-only). R0 has no op at all and is forbidden in every phase. ++ * PMGR is read-only in every phase. R1 is reachable only in the RESET phase. ++ * ++ * Nothing here frees or reuses memory the GPU might still access. ++ */ ++ ++#define G5P_TD_NCMD 4 /* command rings: shared objects 0x60 0x64 0x68 0x6c */ ++#define G5P_TD_NRING (G5P_TD_NCMD + 1) /* + event ring (object 0x70) */ ++#define G5P_TD_MAX_RCPT 12 ++#define G5P_TD_MAX_STEPS 4096 /* hard stop independent of the clock */ ++#define G5P_TD_POLL_US 100 ++#define G5P_TD_EV_SLACK \ ++ 16 /* events produced during the phase above this = storm */ ++ ++enum g5p_td_mode { ++ G5P_TD_CAPTURE = 1, ++ G5P_TD_FINAL = 2, ++}; ++ ++enum g5p_td_phase { ++ G5P_TDP_FREEZE, /* stopped=1 alive=0, watchdog disarmed (RAM) */ ++ G5P_TDP_ACCOUNT, /* RAM-only ring/event accounting */ ++ G5P_TDP_DRAIN, /* bounded RAM-only wait for the firmware to self-drain */ ++ G5P_TDP_EVENTS, /* bounded RAM-only event-ring quiesce */ ++ G5P_TDP_IRQ, /* AIC ack, IRQ mask, settle */ ++ G5P_TDP_DETACH, /* host detach, RAM only */ ++ G5P_TDP_RESET, /* RT1 Fender stop leg when eligible; PMGR read */ ++ G5P_TDP_DONE, ++ G5P_TDP_N, ++}; ++ ++enum g5p_td_dom { ++ G5P_TDD_RAM, ++ G5P_TDD_AIC, ++ G5P_TDD_R1, ++ G5P_TDD_R0, ++ G5P_TDD_PMGR, ++ G5P_TDD_N, ++}; ++ ++#define G5P_TDA_R 1 ++#define G5P_TDA_W 2 ++#define G5P_TDA_RW 3 ++ ++/* Hard stop boundaries. A phase may only use the domains marked here. */ ++static const unsigned char g5p_td_allow[G5P_TDP_N][G5P_TDD_N] = { ++ /* RAM AIC R1 R0 PMGR */ ++ [G5P_TDP_FREEZE] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_ACCOUNT] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_DRAIN] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_EVENTS] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_IRQ] = { G5P_TDA_R, G5P_TDA_W, 0, 0, 0 }, ++ [G5P_TDP_DETACH] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_RESET] = { G5P_TDA_R, 0, G5P_TDA_RW, 0, G5P_TDA_R }, ++ [G5P_TDP_DONE] = { G5P_TDA_R, 0, 0, 0, 0 }, ++}; ++ ++#define G5P_TDF_NOT_BOOTED \ ++ BIT(0) /* shared pages not mapped: nothing to tear down */ ++#define G5P_TDF_UNRELEASED BIT(1) /* firmware never released */ ++#define G5P_TDF_PROTOCOL \ ++ BIT(2) /* ring index > 255, read error, producer moved */ ++#define G5P_TDF_TERMINAL BIT(3) /* TA complete/timeout latched before entry */ ++#define G5P_TDF_LATE_TERMINAL BIT(4) /* ... or during the teardown (race) */ ++#define G5P_TDF_ABANDON BIT(5) /* >=1 ring pending after the drain */ ++#define G5P_TDF_SELF_DRAINED \ ++ BIT(6) /* >=1 ring consumed by the firmware, no bell */ ++#define G5P_TDF_FAULT_EVENT \ ++ BIT(7) /* event type other than 0/1 left unconsumed */ ++#define G5P_TDF_EV_UNQUIET \ ++ BIT(8) /* event ring still non-empty after the budget */ ++#define G5P_TDF_IRQ_UNSETTLED BIT(9) /* IRQ count still moving after the mask */ ++#define G5P_TDF_RT1_OK BIT(10) /* Fender stop leg verified GPC=0 */ ++#define G5P_TDF_RT1_FAIL BIT(11) /* stop leg attempted, not verified */ ++#define G5P_TDF_RT1_SKIPPED BIT(12) /* no register access; see skip reason */ ++#define G5P_TDF_OVERRUN BIT(13) /* step or time budget exhausted */ ++#define G5P_TDF_VIOLATION BIT(14) /* a gate refused an access (machine bug) */ ++#define G5P_TDF_CAPTURE_ONLY BIT(15) /* CAPTURE run: nothing was changed */ ++#define G5P_TDF_REENTRY BIT(16) /* second entry answered from the cache */ ++ ++enum g5p_td_skip { ++ G5P_TDS_NONE, ++ G5P_TDS_NOT_BOOTED, ++ G5P_TDS_NOT_RELEASED, ++ G5P_TDS_NOT_POWERED, ++ G5P_TDS_ALREADY_PARKED, /* clean: GPC=0 already verified by park */ ++ G5P_TDS_STOPPED_UNPARKED, /* dirty: an earlier stop leg did not verify */ ++ G5P_TDS_POWER_ABORT, ++ G5P_TDS_FW_POLICY, /* meta+0x6c8 != 0: firmware also writes R1+0x1000 */ ++ G5P_TDS_LOCK_BUSY, ++ G5P_TDS_CAPTURE_MODE, ++ G5P_TDS_BAD_STATE, /* protocol/overrun/unsettled IRQ: no register access */ ++ G5P_TDS_N, ++}; ++ ++enum g5p_td_verdict { ++ G5P_TDV_NONE, ++ G5P_TDV_OBSERVED, /* CAPTURE: nothing changed */ ++ G5P_TDV_UNTOUCHED, /* never booted or never released: nothing to do */ ++ G5P_TDV_QUIESCED, /* rings clear, events clear, IRQ masked, GPC=0 verified */ ++ G5P_TDV_ABANDONED, /* as QUIESCED, but >=1 command abandoned after the drain */ ++ G5P_TDV_RAM_ONLY, /* IRQ masked, no register access: GPU left as found */ ++ G5P_TDV_DIRTY, /* a step failed: warm continue forbidden, cold DFU */ ++ G5P_TDV_N, ++}; ++ ++static const char *const g5p_td_phase_name[G5P_TDP_N] = { ++ "freeze", "account", "drain", "events", ++ "irq", "detach", "reset", "done", ++}; ++ ++static const char *const g5p_td_mode_name[3] = { "none", "capture", "final" }; ++ ++static const char *const g5p_td_verdict_name[G5P_TDV_N] = { ++ "none", "observed", "untouched", "quiesced", ++ "abandoned", "ram-only", "dirty", ++}; ++ ++struct g5p_td_budget { ++ u32 drain_us; /* self-drain wait; DC3 took 2.34 ms (r87b) */ ++ u32 events_us; ++ u32 settle_us; /* IRQ count must stay constant this long after the mask */ ++ u32 events_pass_max; /* entries per pass; the ring has 256 slots */ ++ u32 events_passes; ++ u32 total_us; /* RESET is skipped when the RAM phases already overran this */ ++}; ++ ++#define G5P_TD_BUDGET_DEFAULT { 20000, 20000, 2000, 256, 4, 100000 } ++ ++/* Inputs the glue reads from RAM before the run (never from a register). */ ++struct g5p_td_pre { ++ bool booted; /* maps_ready: shared pages exist */ ++ bool released; /* firmware was released */ ++ bool powered; /* ps_gfx held on by the retained PM reference */ ++ bool fender_ready; ++ bool already_parked; /* b->parked: GPC=0 verified by park */ ++ bool stopped_unparked; /* b->stopped && !b->parked: an earlier stop did not verify */ ++ bool policy_host_owned; /* init meta+0x6c8 == 0 */ ++ bool power_abort; /* r73b early_power_abort latched */ ++ bool lock_held; /* the glue holds g->lock: no worker is inside a hardware wait */ ++}; ++ ++/* ++ * Every op is bounded by its own contract; the machine additionally bounds how ++ * often and for how long it calls them. Ops marked RAM never touch a register. ++ */ ++struct g5p_td_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); /* sleep, no register access */ ++ void (*freeze)(void *ctx); /* RAM: stopped=1 alive=0, disarm */ ++ /* RAM: idx 0..NCMD-1 command rings, NCMD = event ring. -ENODEV: not mapped. */ ++ int (*ring_read)(void *ctx, u32 idx, u32 *cons, u32 *prod); ++ /* RAM: consume at most max entries of type 0/1, stop at the first other type. */ ++ int (*event_pass)(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault); ++ bool (*terminal_seen)(void *ctx); /* RAM: TA complete/timeout latch */ ++ void (*detach)(void *ctx); /* RAM: signal/forget host jobs */ ++ void (*aic_ack)(void *ctx); /* AIC: swirq+0x94 = 0x4000 */ ++ void (*irq_mask)(void *ctx); /* AIC: ack stays, line masked */ ++ u32 (*irq_total)(void *ctx); /* RAM: handler entry counter */ ++ int (*fender_stop)(void *ctx); /* R1: g5p_fender_idle() */ ++ int (*pmgr_actual)(void *ctx, u32 *target, ++ u32 *actual); /* PMGR read; may be NULL */ ++ void (*emit)(void *ctx, const char *line); /* printk; may be NULL */ ++}; ++ ++struct g5p_td_receipt { ++ unsigned char seq, phase, mode, verdict; ++ s32 rc; ++ u32 us, steps, flags; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [dom][0]=reads [dom][1]=writes, op level */ ++ u32 a, b; /* phase specific, see g5p_td_run() */ ++}; ++ ++struct g5p_td { ++ struct g5p_td_budget budget; ++ /* latches shared with the render path; touched by cmpxchg only */ ++ int running; /* 0 idle, 1 inside g5p_td_run() */ ++ int terminal; /* 0, 1 complete, 2 timeout: first writer wins */ ++ int quiet; /* diagnostics MMIO forbidden from here on */ ++ /* 0 idle, 1 captured (informational), 2 finalized: only 2 answers from the cache */ ++ int state; ++ u32 snaps; /* CAPTURE runs since init: the n= of the TD99 snap line */ ++ /* per run */ ++ enum g5p_td_mode mode; ++ enum g5p_td_phase phase; ++ enum g5p_td_skip skip; ++ enum g5p_td_verdict verdict; ++ bool eligible; ++ u32 flags, steps, violations, seq, nrcpt; ++ u64 t0_ns; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [enum g5p_td_dom][0=reads,1=writes], op level */ ++ u32 pending_mask, pending_initial, abandoned, drained; ++ u32 ring_cons[G5P_TD_NRING], ring_prod[G5P_TD_NRING]; ++ u32 ev_consumed, ev_left, ev_initial, irq_before, irq_after; ++ u32 us_total; ++ u32 pmgr_target, pmgr_actual; ++ int rc; ++ struct g5p_td_receipt rcpt[G5P_TD_MAX_RCPT]; ++}; ++ ++static inline void g5p_td_init(struct g5p_td *t) ++{ ++ const struct g5p_td_budget b = G5P_TD_BUDGET_DEFAULT; ++ ++ *t = (struct g5p_td){ .budget = b }; ++} ++ ++/* One latch for "the TA reached a terminal state"; the first writer wins. */ ++static inline bool g5p_td_note_terminal(struct g5p_td *t, int kind) ++{ ++ if (cmpxchg(&t->terminal, 0, kind) != 0) ++ return false; ++ WRITE_ONCE(t->quiet, 1); ++ return true; ++} ++ ++/* ++ * The single predicate for diagnostic register access (R0 snapshots, end banks, ++ * exports): false once the epoch is terminal or a teardown has started. Nothing ++ * in this series calls a diagnostic; this exists so the render path can. ++ */ ++ ++/* RT1 (Fender stop leg) eligibility. Pure function of RAM-derived inputs. */ ++static inline bool g5p_td_eligible(const struct g5p_td_pre *p, ++ enum g5p_td_mode mode, enum g5p_td_skip *why) ++{ ++ enum g5p_td_skip s = G5P_TDS_NONE; ++ ++ if (!p->booted) ++ s = G5P_TDS_NOT_BOOTED; ++ else if (!p->released) ++ s = G5P_TDS_NOT_RELEASED; ++ else if (!p->powered || !p->fender_ready) ++ s = G5P_TDS_NOT_POWERED; ++ else if (p->already_parked) ++ s = G5P_TDS_ALREADY_PARKED; ++ else if (p->stopped_unparked) ++ s = G5P_TDS_STOPPED_UNPARKED; ++ else if (p->power_abort) ++ s = G5P_TDS_POWER_ABORT; ++ else if (!p->policy_host_owned) ++ s = G5P_TDS_FW_POLICY; ++ else if (!p->lock_held) ++ s = G5P_TDS_LOCK_BUSY; ++ else if (mode != G5P_TD_FINAL) ++ s = G5P_TDS_CAPTURE_MODE; ++ if (why) ++ *why = s; ++ return s == G5P_TDS_NONE; ++} ++ ++static inline int g5p_td_format(const struct g5p_td_receipt *r, char *buf, ++ size_t n) ++{ ++ return scnprintf(buf, n, ++ "TD98 seq=%u phase=%s mode=%s rc=%d us=%u steps=%u flags=%08x aic=%u/%u r1=%u/%u r0=%u/%u pmgr=%u/%u a=%u b=%u verdict=%s", ++ r->seq, g5p_td_phase_name[r->phase], g5p_td_mode_name[r->mode], ++ r->rc, r->us, r->steps, r->flags, r->cnt[G5P_TDD_AIC][0], ++ r->cnt[G5P_TDD_AIC][1], r->cnt[G5P_TDD_R1][0], ++ r->cnt[G5P_TDD_R1][1], r->cnt[G5P_TDD_R0][0], ++ r->cnt[G5P_TDD_R0][1], r->cnt[G5P_TDD_PMGR][0], ++ r->cnt[G5P_TDD_PMGR][1], r->a, r->b, ++ g5p_td_verdict_name[r->verdict]); ++} ++ ++static inline u32 g5p_td_us_since(const struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ u64 d = o->now_ns(o->ctx) - t->t0_ns; ++ ++ return d > 0xffffffffULL * 1000 ? 0xffffffffu : (u32)(d / 1000); ++} ++ ++static inline void g5p_td_receipt(struct g5p_td *t, const struct g5p_td_ops *o, ++ int rc, u32 a, u32 b) ++{ ++ struct g5p_td_receipt *r = ++ &t->rcpt[t->nrcpt < G5P_TD_MAX_RCPT ? t->nrcpt : ++ G5P_TD_MAX_RCPT - 1]; ++ char line[224]; ++ unsigned int d; ++ ++ r->seq = (unsigned char)t->seq++; ++ r->phase = t->phase; ++ r->mode = t->mode; ++ r->verdict = t->verdict; ++ r->rc = rc; ++ r->us = g5p_td_us_since(t, o); ++ r->steps = t->steps; ++ r->flags = t->flags; ++ for (d = 0; d < G5P_TDD_N; d++) { ++ r->cnt[d][0] = t->cnt[d][0]; ++ r->cnt[d][1] = t->cnt[d][1]; ++ } ++ r->a = a; ++ r->b = b; ++ if (t->nrcpt < G5P_TD_MAX_RCPT) ++ t->nrcpt++; ++ if (o->emit) { ++ g5p_td_format(r, line, sizeof(line)); ++ o->emit(o->ctx, line); ++ } ++} ++ ++/* Count and check one access. A refused access is a bug in the machine itself. */ ++static inline int g5p_td_gate(struct g5p_td *t, enum g5p_td_dom d, ++ unsigned int acc) ++{ ++ if ((g5p_td_allow[t->phase][d] & acc) != acc) { ++ t->violations++; ++ t->flags |= G5P_TDF_VIOLATION; ++ return -EACCES; ++ } ++ if ((acc & G5P_TDA_R) && t->cnt[d][0] < 255) ++ t->cnt[d][0]++; ++ if ((acc & G5P_TDA_W) && t->cnt[d][1] < 255) ++ t->cnt[d][1]++; ++ return 0; ++} ++ ++/* One loop iteration or hardware op. False once the hard step cap is hit. */ ++static inline bool g5p_td_step(struct g5p_td *t) ++{ ++ if (t->steps >= G5P_TD_MAX_STEPS) { ++ t->flags |= G5P_TDF_OVERRUN; ++ return false; ++ } ++ t->steps++; ++ return true; ++} ++ ++static inline bool g5p_td_ring_ok(u32 cons, u32 prod) ++{ ++ return cons <= 255 && prod <= 255; ++} ++ ++static inline void g5p_td_note_late(struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ if (o->terminal_seen && o->terminal_seen(o->ctx) && ++ !(t->flags & G5P_TDF_TERMINAL)) ++ t->flags |= G5P_TDF_LATE_TERMINAL; ++} ++ ++static inline int g5p_td_account(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 i, bad = 0, pending = 0, cons, prod; ++ int rc; ++ ++ for (i = 0; i < G5P_TD_NRING; i++) { ++ if (!g5p_td_step(t)) ++ return -ETIMEDOUT; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return -EACCES; ++ rc = o->ring_read(o->ctx, i, &cons, &prod); ++ if (rc == -ENODEV) { ++ t->flags |= G5P_TDF_NOT_BOOTED; ++ break; ++ } ++ if (rc || !g5p_td_ring_ok(cons, prod)) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ bad++; ++ continue; ++ } ++ t->ring_cons[i] = cons; ++ t->ring_prod[i] = prod; ++ if (i < G5P_TD_NCMD && cons != prod) { ++ t->pending_mask |= 1u << i; ++ pending++; ++ } ++ } ++ t->pending_initial = pending; ++ t->ev_left = (t->ring_prod[G5P_TD_NCMD] - t->ring_cons[G5P_TD_NCMD]) & ++ 255; ++ t->ev_initial = t->ev_left; ++ g5p_td_receipt(t, o, bad ? -EPROTO : 0, pending, bad); ++ return 0; ++} ++ ++/* ++ * gpu-recovery99 0004: one extra line per CAPTURE run, after the ACCOUNT receipt, so a live ++ * epoch can be looked at ++ * several times (S1a). RAM only: it formats what ACCOUNT stored in struct g5p_td and reads the ++ * handler entry counter ++ * (RAM) and the clock; no register, no state change. The prefix is not TD98 on purpose: receipt ++ * parsers keyed on ++ * "TD98 seq=" / "TD98 begin" ignore it. ++ * n CAPTURE runs since init (1 based) ++ * t_ns ops->now_ns(), for the IRQ rate between two lines ++ * irq handler entry counter (ops->irq_total), 0 if there is no such op ++ * term terminal latch (0 none, 1 complete, 2 timeout); quiet = the diagnostics latch ++ * own bit0 booted 1 released 2 powered 3 fender_ready 4 already_parked 5 stopped_unparked ++ * 6 policy_host_owned 7 power_abort 8 lock_held ++ * cN/pN command ring N consumer / producer; ce/pe the event ring. Zero when the shared pages ++ * are not mapped. ++ */ ++static inline void g5p_td_snap(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ char line[224]; ++ u32 own, irq = 0; ++ ++ t->snaps++; ++ if (!o->emit) ++ return; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R) && o->irq_total) ++ irq = o->irq_total(o->ctx); ++ own = pre->booted | pre->released << 1 | pre->powered << 2 | ++ pre->fender_ready << 3 | pre->already_parked << 4 | ++ pre->stopped_unparked << 5 | pre->policy_host_owned << 6 | ++ pre->power_abort << 7 | pre->lock_held << 8; ++ scnprintf(line, sizeof(line), ++ "TD99 snap n=%u t_ns=%llu irq=%u term=%d quiet=%d own=%03x c0=%u p0=%u c1=%u p1=%u c2=%u p2=%u c3=%u p3=%u ce=%u pe=%u", ++ t->snaps, (unsigned long long)o->now_ns(o->ctx), irq, ++ READ_ONCE(t->terminal), READ_ONCE(t->quiet), own, ++ t->ring_cons[0], t->ring_prod[0], t->ring_cons[1], ++ t->ring_prod[1], t->ring_cons[2], t->ring_prod[2], ++ t->ring_cons[3], t->ring_prod[3], t->ring_cons[G5P_TD_NCMD], ++ t->ring_prod[G5P_TD_NCMD]); ++ o->emit(o->ctx, line); ++} ++ ++static inline void g5p_td_drain(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.drain_us * 1000; ++ u32 i, cons, prod; ++ ++ while (t->pending_mask) { ++ if (!g5p_td_step(t)) ++ break; ++ if (o->now_ns(o->ctx) >= deadline) { ++ t->flags |= G5P_TDF_ABANDON; ++ break; ++ } ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ for (i = 0; i < G5P_TD_NCMD; i++) { ++ if (!(t->pending_mask & (1u << i))) ++ continue; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return; ++ if (o->ring_read(o->ctx, i, &cons, &prod) || ++ !g5p_td_ring_ok(cons, prod) || ++ prod != t->ring_prod[i]) { ++ /* The producer is host-written and the host is frozen. */ ++ t->flags |= G5P_TDF_PROTOCOL; ++ t->pending_mask &= ~(1u << i); ++ continue; ++ } ++ if (cons == prod) { ++ t->pending_mask &= ~(1u << i); ++ t->flags |= G5P_TDF_SELF_DRAINED; ++ } ++ } ++ } ++ for (i = 0; i < G5P_TD_NCMD; i++) ++ if (t->pending_mask & (1u << i)) ++ t->abandoned++; ++ if (t->abandoned) ++ t->flags |= G5P_TDF_ABANDON; ++ t->drained = t->pending_initial - t->abandoned; ++} ++ ++static inline void g5p_td_events(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.events_us * 1000; ++ u32 pass, consumed, left = t->ev_left; ++ bool fault = false; ++ int rc; ++ ++ for (pass = 0; pass < t->budget.events_passes && left; pass++) { ++ if (!g5p_td_step(t)) ++ break; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_RW)) ++ return; ++ consumed = 0; ++ rc = o->event_pass(o->ctx, t->budget.events_pass_max, &consumed, ++ &left, &fault); ++ t->ev_consumed += consumed; ++ if (rc) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ break; ++ } ++ if (fault) { ++ /* Retained for the host; stop consuming, the IRQ is masked next. */ ++ t->flags |= G5P_TDF_FAULT_EVENT; ++ break; ++ } ++ if (!left || o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ } ++ t->ev_left = left; ++ /* ++ * A ring holds at most 255 entries, so consuming more than the backlog plus a ++ * few ALIVE ticks means the producer outran us: an empty ring at the last ++ * look is a snapshot, not a quiesce. ++ */ ++ if (!fault && (left || (s32)(t->ev_consumed + left - t->ev_initial) > ++ G5P_TD_EV_SLACK)) ++ t->flags |= G5P_TDF_EV_UNQUIET; ++} ++ ++static inline void g5p_td_irq(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 tries, n0, n1; ++ ++ if (g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W) || ++ g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W)) ++ return; ++ t->irq_before = o->irq_total(o->ctx); ++ /* Same pair as the IRQ-cap path: ack the software source, then mask the line. */ ++ o->aic_ack(o->ctx); ++ o->irq_mask(o->ctx); ++ for (tries = 0; tries < 3; tries++) { ++ if (!g5p_td_step(t)) ++ break; ++ n0 = o->irq_total(o->ctx); ++ o->relax_us(o->ctx, t->budget.settle_us); ++ n1 = o->irq_total(o->ctx); ++ t->irq_after = n1; ++ if (n0 == n1) ++ return; ++ } ++ t->flags |= G5P_TDF_IRQ_UNSETTLED; ++} ++ ++static inline void g5p_td_reset(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ enum g5p_td_skip why = G5P_TDS_NONE; ++ bool bad = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_VIOLATION); ++ int rc = 0; ++ ++ t->eligible = g5p_td_eligible(pre, t->mode, &why); ++ if (t->eligible && ++ (bad || g5p_td_us_since(t, o) > t->budget.total_us)) { ++ t->eligible = false; ++ why = G5P_TDS_BAD_STATE; ++ } ++ t->skip = why; ++ if (!t->eligible) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ } else if (!g5p_td_step(t) || g5p_td_gate(t, G5P_TDD_R1, G5P_TDA_RW)) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ t->skip = G5P_TDS_BAD_STATE; ++ } else { ++ /* ++ * Apple stopFirmwareForPowerDown 6c9fa10, R1 only; flown here after a DC ++ * timeout (round 10). Never retried, even on failure. ++ */ ++ rc = o->fender_stop(o->ctx); ++ t->flags |= rc ? G5P_TDF_RT1_FAIL : G5P_TDF_RT1_OK; ++ } ++ if (o->pmgr_actual && !g5p_td_gate(t, G5P_TDD_PMGR, G5P_TDA_R)) ++ o->pmgr_actual(o->ctx, &t->pmgr_target, &t->pmgr_actual); ++ g5p_td_receipt(t, o, rc, t->eligible ? 0 : (u32)t->skip, ++ t->pmgr_actual); ++} ++ ++static inline void g5p_td_decide(struct g5p_td *t, const struct g5p_td_pre *pre) ++{ ++ bool dirty = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_EV_UNQUIET | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_RT1_FAIL | ++ G5P_TDF_OVERRUN | G5P_TDF_VIOLATION) || ++ t->skip == G5P_TDS_STOPPED_UNPARKED || ++ t->skip == G5P_TDS_BAD_STATE; ++ ++ if (t->mode == G5P_TD_CAPTURE) ++ t->verdict = (t->flags & G5P_TDF_VIOLATION) ? G5P_TDV_DIRTY : ++ G5P_TDV_OBSERVED; ++ else if (dirty) ++ t->verdict = G5P_TDV_DIRTY; ++ else if (t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED)) ++ t->verdict = G5P_TDV_UNTOUCHED; ++ else if (t->flags & G5P_TDF_RT1_OK || t->skip == G5P_TDS_ALREADY_PARKED) ++ t->verdict = (t->flags & G5P_TDF_ABANDON) ? G5P_TDV_ABANDONED : ++ G5P_TDV_QUIESCED; ++ else ++ t->verdict = G5P_TDV_RAM_ONLY; ++ (void)pre; ++} ++ ++/* ++ * Run the teardown once per mode. Returns 0 for QUIESCED/ABANDONED/UNTOUCHED/OBSERVED, ++ * -EAGAIN for RAM_ONLY (clean, but the GPU was left as found), -EIO for DIRTY, ++ * -EBUSY if another caller is inside the machine. Every phase ends in a receipt. ++ * Receipt a/b: account pending/bad, drain drained/abandoned, events consumed/left, ++ * irq before/after, reset skip-reason/pmgr-actual, done skip-reason/abandoned. ++ * Counters in the receipts are op-level (one count per gate): the AIC pair is ++ * 2 writes, the whole Fender stop leg is one R1 read+write op. ++ */ ++static inline int g5p_td_run(struct g5p_td *t, const struct g5p_td_ops *o, ++ enum g5p_td_mode mode, ++ const struct g5p_td_pre *pre) ++{ ++ if (cmpxchg(&t->running, 0, 1)) ++ return -EBUSY; ++ WRITE_ONCE(t->quiet, 1); ++ if (t->state == 2) { ++ /* Answered from the cache (after FINAL only; CAPTURE re-runs, gpu- ++ * recovery99 0004): no phase runs, nothing is accessed. ++ */ ++ u32 saved_flags = t->flags; ++ ++ t->t0_ns = o->now_ns(o->ctx) - (u64)t->us_total * 1000; ++ t->flags |= G5P_TDF_REENTRY; ++ t->phase = G5P_TDP_DONE; ++ g5p_td_receipt(t, o, t->rc, t->flags, t->steps); ++ t->flags = saved_flags; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++ } ++ t->mode = mode; ++ t->phase = G5P_TDP_FREEZE; ++ t->t0_ns = o->now_ns(o->ctx); ++ t->seq = 0; ++ t->nrcpt = 0; ++ t->flags = mode == G5P_TD_CAPTURE ? G5P_TDF_CAPTURE_ONLY : 0; ++ t->steps = 0; ++ t->violations = 0; ++ t->abandoned = 0; ++ t->drained = 0; ++ t->pending_mask = 0; ++ t->pending_initial = 0; ++ t->ev_consumed = 0; ++ t->ev_left = 0; ++ t->ev_initial = 0; ++ t->irq_before = 0; ++ t->irq_after = 0; ++ t->skip = G5P_TDS_NONE; ++ t->verdict = G5P_TDV_NONE; ++ memset(t->cnt, 0, sizeof(t->cnt)); ++ /* a CAPTURE may now run many times: nothing of the previous look may leak into this one */ ++ memset(t->ring_cons, 0, sizeof(t->ring_cons)); ++ memset(t->ring_prod, 0, sizeof(t->ring_prod)); ++ if (READ_ONCE(t->terminal)) ++ t->flags |= G5P_TDF_TERMINAL; ++ if (!pre->released) ++ t->flags |= G5P_TDF_UNRELEASED; ++ ++ if (mode == G5P_TD_FINAL) { ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->freeze(o->ctx); ++ g5p_td_receipt(t, o, 0, pre->released, pre->policy_host_owned); ++ } ++ t->phase = G5P_TDP_ACCOUNT; ++ if (g5p_td_account(t, o)) ++ t->flags |= G5P_TDF_OVERRUN; ++ g5p_td_note_late(t, o); ++ if (mode == G5P_TD_CAPTURE) ++ g5p_td_snap(t, o, pre); ++ ++ if (mode == G5P_TD_FINAL && ++ !(t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED | ++ G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN))) { ++ t->phase = G5P_TDP_DRAIN; ++ g5p_td_drain(t, o); ++ g5p_td_note_late(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_ABANDON) ? -ETIMEDOUT : 0, ++ t->drained, t->abandoned); ++ ++ t->phase = G5P_TDP_EVENTS; ++ g5p_td_events(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_EV_UNQUIET) ? -ETIMEDOUT : 0, ++ t->ev_consumed, t->ev_left); ++ } ++ if (mode == G5P_TD_FINAL) { ++ /* The IRQ is masked even after an overrun: ack + mask is the safe exit. */ ++ t->phase = G5P_TDP_IRQ; ++ if (!(t->flags & G5P_TDF_NOT_BOOTED)) ++ g5p_td_irq(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_IRQ_UNSETTLED) ? -ETIMEDOUT : ++ 0, ++ t->irq_before, t->irq_after); ++ ++ t->phase = G5P_TDP_DETACH; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->detach(o->ctx); ++ g5p_td_receipt(t, o, 0, 0, 0); ++ ++ t->phase = G5P_TDP_RESET; ++ g5p_td_reset(t, o, pre); ++ } ++ g5p_td_note_late(t, o); ++ t->phase = G5P_TDP_DONE; ++ g5p_td_decide(t, pre); ++ switch (t->verdict) { ++ case G5P_TDV_DIRTY: ++ t->rc = -EIO; ++ break; ++ case G5P_TDV_RAM_ONLY: ++ t->rc = -EAGAIN; ++ break; ++ default: ++ t->rc = 0; ++ } ++ t->us_total = g5p_td_us_since(t, o); ++ g5p_td_receipt(t, o, t->rc, (u32)t->skip, t->abandoned); ++ t->state = mode == G5P_TD_FINAL ? 2 : 1; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +new file mode 100644 +index 000000000..e3875bc59 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -0,0 +1,694 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the GPU teardown state machine (teardown.h). ++ * Design, evidence and device plan: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md ++ * ++ * Included from core.inc after lifecycle.inc, so it may use g5p_dc_root_word(), ++ * g5p_fender_write()/poll() and g5p_r73b_disarm(). No state is added to struct ++ * g5p_boot or struct g5p_diag: one GPU, one file-static teardown instance. ++ * ++ * Compiled with W=1 against a9/tri96 (patches/gpu-recovery/README.md); NOT run on hardware. ++ * ++ * gpu-recovery108 (0006): every run goes through the retire layer (retire108.h): ++ * admission close, RAM-only epoch class and a TR108 receipt before the ladder, the ++ * named post-ladder slots, and the retired latch after a FINAL. The TD98 lines are ++ * unchanged. docs/gpu/experiments/recovery108-driver.md. ++ */ ++#include ++#include ++#include ++#include "teardown.h" ++#include "retire108.h" ++#include "render-user-bind.h" ++#include "render-user-quiesce.h" ++ ++/* 0 off (default; the notifier and .shutdown do nothing), 1 observe-only receipts ++ * (CAPTURE: RAM reads, no change), 2 full ladder (FINAL). ++ */ ++ ++/* gpu-recovery108 G6 slot: one R1 read of isStableOff after a verified Fender stop leg. */ ++ ++/* Defined in render-r96.inc (same translation unit, included later by render-fixed.inc). */ ++ ++static struct g5p_td g5p_td_state; ++static struct g5p_diag *g5p_td_dev; ++static void (*g5p_td_detach_cb)(void *cookie); ++static void *g5p_td_detach_cookie; ++ ++/* gpu-recovery108: the retire layer around the ladder (retire108.h), one GPU, file static. */ ++static struct g5p_r8 g5p_r8_state = { .admit_us = G5P_R8_ADMIT_US }; ++static void (*g5p_r8_epoch_cb)(void *cookie, struct g5p_r8_in *in); ++static void *g5p_r8_epoch_cookie; ++ ++/* The DRM frontend reports its share of the epoch inputs (claim, holders, queued jobs), RAM only. */ ++static void __maybe_unused g5p_r8_register_epoch(void (*cb)(void *, ++ struct g5p_r8_in *), ++ void *cookie) ++{ ++ g5p_r8_epoch_cookie = cookie; ++ smp_store_release(&g5p_r8_epoch_cb, cb); ++} ++ ++/* Job admission: a submitter enters before its first look at driver state and ++ * leaves on every return; false once a FINAL teardown closed admission. ++ */ ++static bool __maybe_unused g5p_r8_enter(void) ++{ ++ return g5p_r8_admit_enter(&g5p_r8_state.admit); ++} ++ ++static void __maybe_unused g5p_r8_exit(void) ++{ ++ g5p_r8_admit_exit(&g5p_r8_state.admit); ++} ++ ++/* The post-retire register predicate for the R1 helpers outside the ladder. */ ++static bool __maybe_unused g5p_r8_r1_allowed(void) ++{ ++ return g5p_r8_mmio_ok(&g5p_r8_state, G5P_TDD_R1); ++} ++ ++static bool __maybe_unused g5p_r8_is_retired(void) ++{ ++ return g5p_r8_retired(&g5p_r8_state); ++} ++ ++/* The DRM frontend registers the callback that fails its queued/in-flight job ++ * fences (exactly once each, -ENODEV) without touching hardware. ++ */ ++static void __maybe_unused g5p_td_register_detach(void (*cb)(void *), ++ void *cookie) ++{ ++ g5p_td_detach_cookie = cookie; ++ smp_store_release(&g5p_td_detach_cb, cb); ++} ++ ++static u64 g5p_td_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_td_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++static void g5p_td_op_freeze(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ struct g5p_boot *b = g->boot; ++ ++ if (!b) ++ return; ++ /* The IRQ-side plateau banks (g5p_r80_irq_bank/g5p_r85_irq_bank) read R0 until this ++ * latch is set; arm it before anything else (RAM only, no-op without an r96 config). ++ */ ++ /* Same freeze as g5p_boot_park(): nothing may publish after this. */ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++} ++ ++static int g5p_td_op_ring_read(void *ctx, u32 idx, u32 *cons, u32 *prod) ++{ ++ static const u32 slots[G5P_TD_NRING] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *ctl; ++ ++ if (!b || !READ_ONCE(b->maps_ready)) ++ return -ENODEV; ++ ctl = (u32 *)g5p_boot_shared(b, slots[idx]); ++ *cons = READ_ONCE(ctl[0]); ++ *prod = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ return 0; ++} ++ ++/* RAM only. Consumes ALIVE (0) and stamp (1) entries; the first other type is ++ * left in place for the host, as g5p_boot_drain() does. ++ */ ++static int g5p_td_op_event_pass(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n = 0; ++ unsigned long flags; ++ int ret = 0; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ while (n < max && consumer != producer) { ++ u32 off = b->layout.offset[0x74] + consumer * 44; ++ ++ if (g5p_le32(b->shared[off / G5P_GPU_PAGE].cpu + ++ off % G5P_GPU_PAGE) > 1) { ++ *fault = true; ++ break; ++ } ++ consumer = (consumer + 1) & 255; ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ n++; ++ } ++ producer = READ_ONCE(control[4]); ++ *left = producer > 255 ? 255 : (producer - consumer) & 255; ++out: ++ *consumed = n; ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* The render path latches terminal states through g5p_td_note_terminal(); until ++ * prod98 wires that call this reports "not seen" and the race is only visible as ++ * ring state. ++ */ ++static bool g5p_td_op_terminal_seen(void *ctx) ++{ ++ return READ_ONCE(g5p_td_state.terminal) != 0; ++} ++ ++static void g5p_td_op_detach(void *ctx) ++{ ++ void (*cb)(void *) = smp_load_acquire(&g5p_td_detach_cb); ++ ++ if (cb) ++ cb(g5p_td_detach_cookie); ++} ++ ++static void g5p_td_op_aic_ack(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* The same software-source ACK the IRQ-cap path uses. */ ++ if (b && b->irq_ready && b->swirq) ++ writel(0x4000, b->swirq + 0x94); ++} ++ ++static void g5p_td_op_irq_mask(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* Line masked at the interrupt controller; the handler stays registered for ++ * a late entry, which still ACKs. Same call as the IRQ-cap path. ++ */ ++ if (b && b->irq_ready && b->irq > 0) ++ disable_irq_nosync(b->irq); ++} ++ ++static u32 g5p_td_op_irq_total(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ return b ? (u32)atomic_read(&b->irqs) : 0; ++} ++ ++static int g5p_td_op_fender_stop(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ ++ /* Apple stopFirmwareForPowerDown 6c9fa10; no retry, no PMGR off. */ ++ return g5p_fender_idle(&io); ++} ++ ++static int g5p_td_op_pmgr(void *ctx, u32 *target, u32 *actual) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) ++ return -ENODEV; ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (!ret) { ++ *target = raw & 0xf; ++ *actual = (raw >> 4) & 0xf; ++ } ++ return ret; ++} ++ ++/* dev_emerg so the receipt reaches netconsole at any console loglevel. */ ++static void g5p_td_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_diag *)ctx)->dev, "%s\n", line); ++} ++ ++static void g5p_td_pre_fill(struct g5p_diag *g, struct g5p_td_pre *p, ++ bool locked) ++{ ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(p, 0, sizeof(*p)); ++ p->lock_held = locked; ++ p->powered = READ_ONCE(g->powered); ++ p->fender_ready = READ_ONCE(g->fender_ready); ++ if (!b) ++ return; ++ p->booted = READ_ONCE(b->maps_ready); ++ p->released = READ_ONCE(b->released); ++ p->already_parked = READ_ONCE(b->parked); ++ p->stopped_unparked = READ_ONCE(b->stopped) && !READ_ONCE(b->parked); ++ p->power_abort = b->r73b && READ_ONCE(b->r73b->early_power_abort); ++ /* RAM-only read of the host-written init word (same as g5p_dc_snapshot()). */ ++ p->policy_host_owned = ++ p->booted && ++ !g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8); ++} ++ ++/* gpu-recovery108: the ladder as the retire layer's td_run op. The TD98 begin line ++ * and the ladder's own lines are exactly the recovery98/99 ones. ++ */ ++struct g5p_r8_ctx { ++ struct g5p_diag *g; ++ const struct g5p_td_ops *ops; ++ struct g5p_td_pre *pre; ++ const char *why; ++ bool locked; ++}; ++ ++static int g5p_r8_op_td_run(void *ctx, enum g5p_td_mode mode) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ dev_emerg(c->g->dev, ++ "TD98 begin why=%s mode=%s teardown=%u locked=%u\n", c->why, ++ g5p_td_mode_name[mode], (0), c->locked); ++ return g5p_td_run(&g5p_td_state, c->ops, mode, c->pre); ++} ++ ++static u32 g5p_r8_op_td_flags(void *ctx) ++{ ++ return g5p_td_state.flags; ++} ++ ++static enum g5p_td_verdict g5p_r8_op_td_verdict(void *ctx) ++{ ++ return g5p_td_state.verdict; ++} ++ ++static u64 g5p_r8_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_r8_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++/* G6: R1+0x0 bit 30 after a verified stop leg (the slot runs only then, before RETIRE). */ ++static int g5p_r8_op_stable_off(void *ctx, u32 *value) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ *value = readl(c->g->window[1]) >> 30; ++ return 0; ++} ++ ++static void g5p_r8_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_r8_ctx *)ctx)->g->dev, "%s\n", line); ++} ++ ++static void g5p_r8_in_fill(struct g5p_diag *g, struct g5p_r8_in *in, ++ bool locked, enum g5p_td_mode mode) ++{ ++ void (*cb)(void *, struct g5p_r8_in *) = ++ smp_load_acquire(&g5p_r8_epoch_cb); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(in, 0, sizeof(*in)); ++ in->terminal = READ_ONCE(g5p_td_state.terminal); ++ /* FINAL without the lock: a worker (or a sysfs writer) holds it, maybe inside a wait. */ ++ in->worker_busy = mode == G5P_TD_FINAL && !locked; ++ if (b) { ++ in->booted = READ_ONCE(b->maps_ready); ++ in->released = READ_ONCE(b->released); ++ in->alive = READ_ONCE(b->alive); ++ in->failed = READ_ONCE(b->failed); ++ in->fault_event = READ_ONCE(b->event_failed); ++ in->parked = READ_ONCE(b->parked); ++ in->stopped_unparked = READ_ONCE(b->stopped) && ++ !READ_ONCE(b->parked); ++ in->power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ if (cb) ++ cb(g5p_r8_epoch_cookie, in); ++} ++ ++static int g5p_td_execute(struct g5p_diag *g, enum g5p_td_mode mode, ++ const char *why) ++{ ++ const struct g5p_td_ops ops = { ++ .ctx = g, ++ .now_ns = g5p_td_op_now, ++ .relax_us = g5p_td_op_relax, ++ .freeze = g5p_td_op_freeze, ++ .ring_read = g5p_td_op_ring_read, ++ .event_pass = g5p_td_op_event_pass, ++ .terminal_seen = g5p_td_op_terminal_seen, ++ .detach = g5p_td_op_detach, ++ .aic_ack = g5p_td_op_aic_ack, ++ .irq_mask = g5p_td_op_irq_mask, ++ .irq_total = g5p_td_op_irq_total, ++ .fender_stop = g5p_td_op_fender_stop, ++ .pmgr_actual = g5p_td_op_pmgr, ++ .emit = g5p_td_op_emit, ++ }; ++ struct g5p_td_pre pre; ++ struct g5p_r8_in in; ++ struct g5p_r8_ctx c = { .g = g, .ops = &ops, .pre = &pre, .why = why }; ++ /* fw-sleep (G2) and pmgr-gate (G1) have no mechanism: their ops stay NULL. */ ++ const struct g5p_r8_ops r8ops = { ++ .ctx = &c, ++ .now_ns = g5p_r8_op_now, ++ .relax_us = g5p_r8_op_relax, ++ .td_run = g5p_r8_op_td_run, ++ .td_flags = g5p_r8_op_td_flags, ++ .td_verdict = g5p_r8_op_td_verdict, ++ .slot = { [G5P_R8S_STABLE_OFF] = g5p_r8_op_stable_off }, ++ .emit = g5p_r8_op_emit, ++ }; ++ bool locked = false; ++ int ret; ++ ++ /* Never queue behind a worker or sysfs writer that may be inside MMIO. CAPTURE ++ * reads RAM only and needs no lock. ++ */ ++ if (mode == G5P_TD_FINAL) ++ locked = mutex_trylock(&g->lock); ++ c.locked = locked; ++ g5p_td_pre_fill(g, &pre, locked); ++ g5p_r8_in_fill(g, &in, locked, mode); ++ g5p_r8_state.slot_on[G5P_R8S_STABLE_OFF] = (0); ++ ret = g5p_r8_run(&g5p_r8_state, &r8ops, mode, &in, why); ++ if (g5p_td_state.flags & G5P_TDF_RT1_OK && g->boot) ++ WRITE_ONCE(g->boot->parked, true); ++ if (locked) ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Observation only: arms the quiet predicate, changes nothing else. */ ++static int __maybe_unused g5p_td_capture(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_CAPTURE, why); ++} ++ ++static int g5p_td_final(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_FINAL, why); ++} ++ ++/* ++ * The quiesce before a kexec, reboot or power-off (render-user-quiesce.h). It runs from the ++ * reboot notifier whatever `teardown` is, once per boot; .shutdown and the `td-kexec` command ++ * reach the same run. It leaves the GPU powered with its firmware stopped and the boot state ++ * frozen: the next kernel boots it as after any warm kexec. ++ */ ++ ++#define G5P_KQ_LOCK_MS 2000U ++#define G5P_KQ_ADMIT_US 20000U ++#define G5P_KQ_OFF_MS 10U ++ ++/* The render frontend's share: its TYPED_V3 epoch inputs and the RAM-only unbind of slots. */ ++struct g5p_kq_frontend { ++ void (*state)(void *cookie, struct g5p_ru_kq_in *in); ++ int (*detach)(void *cookie, const u32 *slots, u32 n); ++}; ++ ++static const struct g5p_kq_frontend *g5p_kq_fe; ++static void *g5p_kq_fe_cookie; ++static atomic_t g5p_kq_ran = ATOMIC_INIT(0); ++ ++struct g5p_kq_ctx { ++ struct g5p_diag *g; ++ u32 slots[G5P_RU_USER_SLOTS]; ++ u32 held; ++}; ++ ++static void __maybe_unused g5p_kq_register(const struct g5p_kq_frontend *fe, ++ void *cookie) ++{ ++ g5p_kq_fe_cookie = cookie; ++ smp_store_release(&g5p_kq_fe, fe); ++} ++ ++static u32 g5p_kq_admit_close(void *ctx) ++{ ++ struct g5p_r8_admit *a = &g5p_r8_state.admit; ++ u64 deadline = ktime_get_ns() + (u64)G5P_KQ_ADMIT_US * NSEC_PER_USEC; ++ u32 inside = g5p_r8_admit_close(a); ++ ++ while (inside && ktime_get_ns() < deadline) { ++ usleep_range(G5P_R8_POLL_US, 2 * G5P_R8_POLL_US); ++ inside = g5p_r8_admit_inside(a); ++ } ++ return inside; ++} ++ ++/* Never queue behind a holder that may be inside MMIO: bounded, like g5p_render_run(). */ ++static bool g5p_kq_lock(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_KQ_LOCK_MS); ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ return true; ++} ++ ++static void g5p_kq_unlock(void *ctx) ++{ ++ mutex_unlock(&((struct g5p_kq_ctx *)ctx)->g->lock); ++} ++ ++/* RAM only: the boot state, the held user slots and the DC ring, then the frontend's inputs. */ ++static void g5p_kq_inputs(void *ctx, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ struct g5p_diag *g = c->g; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ u32 *ctl; ++ ++ if (!READ_ONCE(g->powered)) ++ return; /* Already idle: no detach, MMIO or second gate cycle. */ ++ in->pmgr = !!g->pmgr; ++ if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released) || ++ READ_ONCE(b->parked)) ++ return; ++ in->booted = true; ++ in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ dma_rmb(); ++ in->cons = READ_ONCE(ctl[0]); ++ in->prod = READ_ONCE(ctl[4]); ++ in->ring_idle = in->prod <= 255 && in->cons == in->prod; ++ c->held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), ++ g5p_boot_shared(b, 0x84), c->slots, ++ ARRAY_SIZE(c->slots)); ++ in->held = c->held; ++ if (fe) { ++ in->frontend = true; ++ fe->state(g5p_kq_fe_cookie, in); ++ } ++} ++ ++static int g5p_kq_detach(void *ctx) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ ++ return fe ? fe->detach(g5p_kq_fe_cookie, c->slots, c->held) : -ENODEV; ++} ++ ++static void g5p_kq_freeze(void *ctx) ++{ ++ g5p_td_op_freeze(((struct g5p_kq_ctx *)ctx)->g); ++} ++ ++/* The IRQ-cap path's AIC ack and mask, then wait out a handler already running (process ++ * context; the threaded handler drains RAM and takes no lock the quiesce holds). ++ */ ++static void g5p_kq_irq_off(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ struct g5p_boot *b = g->boot; ++ ++ g5p_td_op_aic_ack(g); ++ g5p_td_op_irq_mask(g); ++ if (b && b->irq_ready && b->irq > 0) ++ synchronize_irq(b->irq); ++} ++ ++/* PMGR only: the bounded gate cycle of the r33/dirty-start path (refuses a gate that is not ++ * plainly on), which ends with the gate requested on. ++ */ ++static int g5p_kq_cycle(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ ++ g5p_mmio_gate(g); ++ return g5p_r33_gfx_off(g->dev, g->pmgr, G5P_KQ_OFF_MS); ++} ++ ++static int g5p_kq_run(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_ru_kq_ops o = { ++ .admit_close = g5p_kq_admit_close, ++ .lock = g5p_kq_lock, ++ .unlock = g5p_kq_unlock, ++ .inputs = g5p_kq_inputs, ++ .detach = g5p_kq_detach, ++ .freeze = g5p_kq_freeze, ++ .irq_off = g5p_kq_irq_off, ++ .cycle = g5p_kq_cycle, ++ }; ++ struct g5p_ru_kq_out out; ++ struct g5p_kq_ctx *c; ++ u64 t0 = ktime_get_ns(); ++ ++ if (atomic_xchg(&g5p_kq_ran, 1)) { ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s already ran this boot; nothing done end=kq\n", ++ why); ++ return -EALREADY; ++ } ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ if (!c) { ++ dev_emerg(g->dev, "GPU-KQ why=%s ret=%d end=kq\n", why, ++ -ENOMEM); ++ return -ENOMEM; ++ } ++ c->g = g; ++ o.ctx = c; ++ g5p_ru_kq_run(&o, min((G5P_RU_KQ_FULL), G5P_RU_KQ_FULL), &out); ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s mode=%u inside=%u locked=%u booted=%u failed=%u wedged=%u " ++ "ram_only=%u exec=%u dc=%u/%u held=%u plan=%s%s skip=%#x detach=%d froze=%u " ++ "cycle=%d ms=%llu end=kq\n", ++ why, out.in.mode, out.inside, out.in.locked, out.in.booted, ++ out.in.failed, out.in.term_failed, out.in.ram_only, ++ out.in.executing, out.in.cons, out.in.prod, out.in.held, ++ out.plan & G5P_RU_KQ_DETACH ? "detach" : "-", ++ out.plan & G5P_RU_KQ_CYCLE ? "+cycle" : "", out.why, ++ out.detach_ret, out.froze, out.cycle_ret, ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC); ++ kfree(c); ++ return out.detach_ret ?: out.cycle_ret; ++} ++ ++/* sysfs `command`: td-capture / td-final / td-kexec. Runs before the command mutex, like ++ * r73b-export, so a wedged stage cannot block it. -EAGAIN (clean, GPU left as ++ * found) is reported as success; the verdict is in the TD98 lines. td-kexec is the ++ * reboot quiesce run by hand (the GPU is finished for this boot afterwards). ++ */ ++static int g5p_td_command(struct g5p_diag *g, const char *command) ++{ ++ int ret; ++ ++ if (sysfs_streq(command, "td-capture")) ++ ret = g5p_td_capture(g, "sysfs"); ++ else if (sysfs_streq(command, "td-final")) ++ ret = g5p_td_final(g, "sysfs"); ++ else if (sysfs_streq(command, "td-kexec")) ++ ret = g5p_kq_run(g, "sysfs"); ++ else ++ return -EINVAL; ++ return ret == -EAGAIN ? 0 : ret; ++} ++ ++/* Runs from kernel_restart_prepare() for reboot, halt, power-off and kexec, i.e. ++ * before device_shutdown() and its unbounded wait_for_device_probe(). Panic kexec ++ * skips both and is not covered. ++ */ ++static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, ++ void *data) ++{ ++ struct g5p_diag *g = READ_ONCE(g5p_td_dev); ++ ++ if (!g) ++ return NOTIFY_DONE; ++ if (action != SYS_RESTART && action != SYS_HALT && ++ action != SYS_POWER_OFF) ++ return NOTIFY_DONE; ++ /* The quiesce needs a live firmware for its detach, so it runs before the ladder freezes. */ ++ g5p_kq_run(g, "reboot-notifier"); ++ { ++ return NOTIFY_DONE; ++ } ++} ++ ++static struct notifier_block g5p_td_nb = { ++ .notifier_call = g5p_td_reboot_notify, ++ .priority = INT_MAX, /* before every other reboot notifier */ ++}; ++ ++/* g is devm-allocated and the DRM frontend's probe error path unwinds without ++ * g5p_remove(): the notifier must not outlive g. ++ */ ++static void g5p_td_devres_release(void *data) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ WRITE_ONCE(g5p_td_dev, NULL); ++} ++ ++static void g5p_td_attach(struct g5p_diag *g) ++{ ++ g5p_td_init(&g5p_td_state); ++ /* A probe after a retire (not reachable on the built-in driver) never reopens admission. */ ++ if (!g5p_r8_retired(&g5p_r8_state)) { ++ u32 gen = g5p_r8_state.gen; ++ ++ g5p_r8_init(&g5p_r8_state); ++ g5p_r8_state.gen = gen; /* monotonic across a re-probe */ ++ } ++ WRITE_ONCE(g5p_td_dev, g); ++ register_reboot_notifier(&g5p_td_nb); ++ /* On failure the action has already run: no notifier, teardown stays inert. */ ++ if (devm_add_action_or_reset(g->dev, g5p_td_devres_release, g)) ++ dev_warn(g->dev, "teardown not attached\n"); ++} ++ ++/* Returns true when the PM reference must be kept: a PMGR power-off was never ++ * validated on the outgoing side and the stop leg is not isStableOff. ++ */ ++static bool g5p_td_remove(struct g5p_diag *g) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ ; ++ ; ++ WRITE_ONCE(g5p_td_dev, NULL); ++ return (0) != 0; ++} ++ ++/* The .shutdown platform callback: second chance after the notifier. */ ++static void __maybe_unused g5p_td_shutdown(struct g5p_diag *g) ++{ ++ /* A second chance after the notifier (a no-op once it ran). */ ++ if (!atomic_read(&g5p_kq_ran)) ++ g5p_kq_run(g, "shutdown"); ++ { ++ ; ++ ; ++ }; ++} +diff --git a/drivers/gpu/apple-g5p/transport-budget.inc b/drivers/gpu/apple-g5p/transport-budget.inc +new file mode 100644 +index 000000000..d70e3c6e1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/transport-budget.inc +@@ -0,0 +1,6 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include "vm-mutation.h" ++static u32 g5p_transport_event_limit(void) ++{ ++ return 256 + 131072 * 4; ++} +diff --git a/drivers/gpu/apple-g5p/vm-mutation.h b/drivers/gpu/apple-g5p/vm-mutation.h +new file mode 100644 +index 000000000..4645240bd +--- /dev/null ++++ b/drivers/gpu/apple-g5p/vm-mutation.h +@@ -0,0 +1,36 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_VM_MUTATION_H ++#define G5P_VM_MUTATION_H ++struct g5p_vm_mutation { ++ bool generations; ++ u32 generation[3]; ++}; ++ ++/* Count actual changed descriptors, including clears, not dispatches. Repeated ++ * parent writes of identical bytes do not advance a generation. Packed fields ++ * follow iOS6cabf40. Publication is serialized by the existing command lock. ++ */ ++static void g5p_vm_entry(struct g5p_vm_mutation *c, void *at, u64 value, ++ u32 level) ++{ ++ u64 old = g5p_le32(at); ++ ++ if (level) ++ old |= (u64)g5p_le32(at + 4) << 32; ++ if (old == value) ++ return; ++ if (level) ++ g5p_put64(at, value); ++ else ++ g5p_put32(at, value); ++ c->generation[level]++; ++} ++ ++static u32 g5p_vm_generation(struct g5p_vm_mutation *c) ++{ ++ if (!c->generations) ++ return 0; ++ return ((c->generation[0] & 0x3ff) << 22) | ++ ((c->generation[1] & 0x3ff) << 12) | (c->generation[2] & 0xfff); ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/work-layout.h b/drivers/gpu/apple-g5p/work-layout.h +new file mode 100644 +index 000000000..b5dc73725 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/work-layout.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_WORK_LAYOUT_H ++#define G5P_WORK_LAYOUT_H ++ ++/* Experimental first CL queue probe: c89420/c88840/c8a808/c980a0. ++ * Tag 4 is emitted for dependencies by c8a808. Treating an already satisfied ++ * dependency as a NOP is inferred and must be validated after G2. ++ */ ++#define G5P_WORK_OFFSET 0x54000U ++#define G5P_WORK_BYTES 0x400U ++#define G5P_WORK_CTL 0x100U ++#define G5P_WORK_RING 0x200U ++#define G5P_WORK_CMD 0x300U ++#define G5P_WORK_STAMP 0x340U ++#define G5P_WORK_SLOTS 16U ++ ++/* Isolated visibility hypothesis, not the recovered normal host alias. */ ++ ++/* Firmware10004e94 copies notification word2 into ctl+30. */ ++ ++#endif diff --git a/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch b/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch new file mode 100644 index 00000000..8cdb117a --- /dev/null +++ b/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch @@ -0,0 +1,1027 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add buffer VM and cache visibility support + +Preserve the sealed mapping budgets, PRIME and cache policy. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/cache-policy.h | 13 + + drivers/gpu/drm/apple-g5p/cache-receipt.inc | 38 + + drivers/gpu/drm/apple-g5p/drm-vm.inc | 912 ++++++++++++++++++++ + drivers/gpu/drm/apple-g5p/vm-ranges.h | 21 + + 4 files changed, 984 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/cache-policy.h + create mode 100644 drivers/gpu/drm/apple-g5p/cache-receipt.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-vm.inc + create mode 100644 drivers/gpu/drm/apple-g5p/vm-ranges.h + +diff --git a/drivers/gpu/drm/apple-g5p/cache-policy.h b/drivers/gpu/drm/apple-g5p/cache-policy.h +new file mode 100644 +index 000000000..40f5cdfe1 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-policy.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* No caller may infer CPU caching from BO size or GPU use alone. */ ++#ifndef G5P_CACHE_POLICY_H ++#define G5P_CACHE_POLICY_H ++static inline int g5p_cache_class(int imported, int exported, int wc) ++{ ++ return imported ? 2 : (!exported && wc ? 0 : 1); ++} ++static inline int g5p_cache_omit(int enabled, int imported, int exported, int wc) ++{ ++ return enabled && g5p_cache_class(imported, exported, wc) == 0; ++} ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/cache-receipt.inc b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +new file mode 100644 +index 000000000..1efeecd0c +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Bounded RAM-only records. No register accesses, default off. */ ++#include ++#include "cache-policy.h" ++/* Qualified by kernel-cost2b and combined-bench2c; conservative off switches remain. */ ++static const bool render_user_cache_import_wc = true; ++static const bool render_user_cache_read_va = true; ++static const bool render_user_cache_private_wc = true; ++static const bool render_user_cache_receipts; ++struct g5p_cache_receipt { ++ u64 job, start_cpu, start_wall, total_cpu, total_wall; ++ u64 requested[2][3], maintained[2][3], cpu[2][3], wall[2][3]; ++ u32 pid, serial; ++ bool enabled, policy, import_policy; ++}; ++#define G5P_CACHE_RECORDS 128 ++static struct g5p_cache_receipt g5p_cache_ring[G5P_CACHE_RECORDS]; ++static DEFINE_SPINLOCK(g5p_cache_ring_lock); ++static u32 g5p_cache_head; ++static int g5p_cache_show(struct seq_file *m, void *data) ++{ ++ struct g5p_cache_receipt r; ++ u32 head = READ_ONCE(g5p_cache_head), begin = head > G5P_CACHE_RECORDS ? head - G5P_CACHE_RECORDS : 0; ++ unsigned long flags; ++ seq_puts(m, "v=1 classes=private-wc,native-other,import time=task-runtime-ns wall=elapsed-ns\n"); ++ for (u32 n = begin; n < head; n++) { ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ r = g5p_cache_ring[n % G5P_CACHE_RECORDS]; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ if (r.serial != n + 1) continue; /* overwritten during this debugfs read */ ++ seq_printf(m, "job=%llu pid=%u policy=%u total_cpu_ns=%llu total_wall_ns=%llu", r.job, r.pid, r.policy, r.total_cpu, r.total_wall); ++ for (u32 phase = 0; phase < 2; phase++) ++ for (u32 c = 0; c < 3; c++) ++ seq_printf(m, " p%uc%u=%llu,%llu,%llu,%llu", phase, c, r.requested[phase][c], r.maintained[phase][c], r.cpu[phase][c], r.wall[phase][c]); ++ seq_putc(m, '\n'); ++ } ++ return 0; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-vm.inc b/drivers/gpu/drm/apple-g5p/drm-vm.inc +new file mode 100644 +index 000000000..644554156 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-vm.inc +@@ -0,0 +1,912 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* 16 KiB VM preparation; accepted user VMs freeze until reboot. */ ++ ++/* release8 memory accounting: every live G5P GEM object (native and imported) with the task ++ * that created it. One line per context release and per file close prints the totals; ++ * debugfs g5p_bos lists each object (size, references, handles, exported/imported, VM pins, ++ * creator) and each open file's handles, so a page held after its client exited is named. ++ */ ++static LIST_HEAD(g5p_bo_live); ++static DEFINE_SPINLOCK(g5p_bo_lock); ++static u32 g5p_bo_count; ++static u64 g5p_bo_bytes; ++ ++static void g5p_gem_free(struct drm_gem_object *obj); ++ ++static const struct drm_gem_object_funcs g5p_gem_funcs = { ++ .free = g5p_gem_free, ++ .print_info = drm_gem_shmem_object_print_info, ++ .pin = drm_gem_shmem_object_pin, ++ .unpin = drm_gem_shmem_object_unpin, ++ .get_sg_table = drm_gem_shmem_object_get_sg_table, ++ .vmap = drm_gem_shmem_object_vmap, ++ .vunmap = drm_gem_shmem_object_vunmap, ++ .mmap = drm_gem_shmem_object_mmap, ++ .export = g5p_gem_prime_export, ++ .vm_ops = &drm_gem_shmem_vm_ops, ++}; ++ ++/* perf60 fault address (render_user_fault_read, drm-render-v3.inc): bit 0 reads the MMU fault ++ * banks at a TA/3D fault event, bit 1 also at a silent or timed-out phase; any nonzero value also ++ * records every VM edit (MAP, UNMAP, render-target create/destroy) in a device-wide ring, so the ++ * fault address can be named. The ring holds an opaque VM pointer, never dereferenced. ++ */ ++ ++/* faultrec: the bank read and its decode alone at a TA/3D fault event (no per-job PM lanes, no ++ * PM snapshots, no VM edit log): one licensed read per fault, at most 32 per boot. ++ */ ++ ++/* perf60-fillrate.md §11.3 cut 5, kernel half (render-user-sync.h): cache maintenance by range. */ ++static const unsigned int render_user_sync_range = 3; /* release8 default */ ++MODULE_PARM_DESC( ++ render_user_sync_range, ++ "perf60 default0 (release8: whole-BO maintenance); bit0 BO_SYNC honours " ++ "offset/size; bit1 a job's ACCESS_CACHE clean/invalidate covers each resource's " ++ "BO range [offset, offset+size), page widened, instead of the whole BO (latched " ++ "per job at submit); BO_SYNC read live"); ++ ++/* CPU cache maintenance of [lo, hi) of a BO's DMA mapping, walked over its DMA segments. */ ++static void g5p_sync_sgt_range(struct device *dev, struct sg_table *sgt, u64 lo, ++ u64 hi, bool to_cpu) ++{ ++ struct scatterlist *sg; ++ u64 pos = 0, start, n; ++ unsigned int i; ++ ++ for_each_sgtable_dma_sg(sgt, sg, i) { ++ if (pos >= hi) ++ break; ++ if (g5p_sync_clip(pos, sg_dma_len(sg), lo, hi, &start, &n)) { ++ if (to_cpu) ++ dma_sync_single_for_cpu( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_single_for_device( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ } ++ pos += sg_dma_len(sg); ++ } ++} ++ ++#define G5P_VMLOG_N 1024U ++enum { G5P_VMLOG_MAP = 1, G5P_VMLOG_UNMAP, G5P_VMLOG_RT_NEW, G5P_VMLOG_RT_DEL }; ++static const char *const g5p_vmlog_ops[] = { "?", "map", "unmap", "rt-new", ++ "rt-del" }; ++ ++struct g5p_vmlog_ent { ++ u64 ns, va, size, offset; ++ const void *vm; ++ u32 op, handle, vm_id; ++ pid_t tgid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++static struct { ++ spinlock_t lock; /* head and the entries */ ++ u32 head; ++ struct g5p_vmlog_ent e[G5P_VMLOG_N]; ++} g5p_vmlog = { .lock = __SPIN_LOCK_UNLOCKED(g5p_vmlog.lock) }; ++ ++static void g5p_vmlog_add(u32 op, const void *vm, u32 vm_id, u64 va, u64 size, ++ u32 handle, u64 offset) ++{ ++ { ++ return; ++ } ++} ++ ++/* Latency123: leaf attributes of new user mappings (0 = Apple's CC | SLC bypass). */ ++ ++/* Stage 3: the device-wide pinned-memory budget. */ ++static const unsigned int render_pinned_mib = 1024; ++MODULE_PARM_DESC( ++ render_pinned_mib, ++ "MiB of BO memory all GPU VMs together may map; an object mapped in several VMs " ++ "or files (PRIME) counts once (default 1024)"); ++ ++static u32 g5p_pte_attrs(void) ++{ ++ static const u32 attrs[] = { 0xc, 0x4, 0x8, 0x0 }; ++ ++ return attrs[min_t(u32, (0), ARRAY_SIZE(attrs) - 1)]; ++} ++ ++static struct drm_gem_object *g5p_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct g5p_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); ++ ++ if (!bo) ++ return ERR_PTR(-ENOMEM); ++ bo->shmem.base.funcs = &g5p_gem_funcs; ++ INIT_LIST_HEAD(&bo->live); ++ return &bo->shmem.base; ++} ++ ++/* A fully created object enters the live list (an object the core frees on a failed init ++ * never does). ++ */ ++static void g5p_bo_track(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ bo->pid = current->tgid; ++ strscpy(bo->comm, current->comm, sizeof(bo->comm)); ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_add_tail(&bo->live, &g5p_bo_live); ++ g5p_bo_count++; ++ g5p_bo_bytes += obj->size; ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++} ++ ++static void g5p_gem_free(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ if (!list_empty(&bo->live)) { ++ list_del_init(&bo->live); ++ g5p_bo_count--; ++ g5p_bo_bytes -= obj->size; ++ } ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ drm_gem_shmem_object_free(obj); ++} ++ ++static int g5p_bos_show(struct seq_file *m, void *data) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct drm_device *drm = entry->dev; ++ u32 exported = 0, imported = 0, nohandle = 0; ++ u64 nohandle_bytes = 0; ++ struct drm_file *file; ++ struct g5p_bo *bo; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_for_each_entry(bo, &g5p_bo_live, live) { ++ struct drm_gem_object *obj = &bo->shmem.base; ++ bool imp = drm_gem_is_imported(obj); ++ ++ exported += !!READ_ONCE(obj->dma_buf); ++ imported += imp; ++ if (!READ_ONCE(obj->handle_count)) { ++ nohandle++; ++ nohandle_bytes += obj->size; ++ } ++ seq_printf( ++ m, ++ "bo size=%zuK refs=%u handles=%u exported=%c imported=%c vm_pins=%u by=%s[%d]\n", ++ obj->size >> 10, kref_read(&obj->refcount), ++ READ_ONCE(obj->handle_count), ++ READ_ONCE(obj->dma_buf) ? 'y' : 'n', imp ? 'y' : 'n', ++ bo->vm_pins, bo->comm, bo->pid); ++ } ++ seq_printf( ++ m, ++ "live=%u bytes=%lluK exported=%u imported=%u no_handle=%u/%lluK device_pinned=%lluK\n", ++ g5p_bo_count, g5p_bo_bytes >> 10, exported, imported, nohandle, ++ nohandle_bytes >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ mutex_lock(&drm->filelist_mutex); ++ list_for_each_entry(file, &drm->filelist, lhead) { ++ struct drm_gem_object *obj; ++ struct task_struct *task; ++ u64 bytes = 0; ++ u32 handles = 0; ++ struct pid *pid; ++ int id; ++ ++ spin_lock(&file->table_lock); ++ idr_for_each_entry(&file->object_idr, obj, id) { ++ handles++; ++ bytes += obj->size; ++ } ++ spin_unlock(&file->table_lock); ++ rcu_read_lock(); ++ pid = rcu_dereference(file->pid); ++ task = pid_task(pid, PIDTYPE_TGID); ++ seq_printf(m, "file %s[%d] handles=%u bytes=%lluK\n", ++ task ? task->comm : "?", pid_vnr(pid), handles, ++ bytes >> 10); ++ rcu_read_unlock(); ++ } ++ mutex_unlock(&drm->filelist_mutex); ++ return 0; ++} ++ ++static int g5p_create_bo(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_bo *a = data; ++ struct drm_gem_shmem_object *shmem; ++ int ret; ++ ++ if (!a->size || a->size > G5P_MAX_BO_SIZE || ++ (a->size & (G5P_GPU_PAGE - 1)) || a->handle || !a->flags || ++ (a->flags & ++ ~(DRM_G5P_BO_READ | DRM_G5P_BO_WRITE | DRM_G5P_BO_EXEC))) ++ return -EINVAL; ++ shmem = drm_gem_shmem_create(drm, a->size); ++ if (IS_ERR(shmem)) ++ return PTR_ERR(shmem); ++ /* Match the DMA API's coherent/noncoherent CPU mapping choice. The GPU ++ * cache contract remains an explicit COMPUTE capability prerequisite. ++ */ ++ shmem->map_wc = !dev_is_dma_coherent(drm->dev); ++ to_g5p_bo(&shmem->base)->roles = a->flags; ++ g5p_bo_track(&shmem->base); ++ ret = drm_gem_handle_create(file, &shmem->base, &a->handle); ++ drm_gem_object_put(&shmem->base); ++ return ret; ++} ++ ++static int g5p_bo_mmap_offset(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_mmap_offset *a = data; ++ struct drm_gem_object *obj; ++ int ret; ++ ++ if (a->flags || a->offset) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ ret = drm_gem_create_mmap_offset(obj); ++ if (!ret) ++ a->offset = drm_vma_node_offset_addr(&obj->vma_node); ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_bo_sync(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_sync *a = data; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *ctx; ++ unsigned long id; ++ int ret = 0; ++ ++ if (a->op != DRM_G5P_BO_SYNC_TO_DEVICE && ++ a->op != DRM_G5P_BO_SYNC_TO_CPU) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ mutex_lock(&f->lock); ++ xa_for_each(&f->contexts, id, ctx) ++ if (ctx->serial && ++ (ctx->bad || smp_load_acquire(&ctx->pending))) { ++ mutex_unlock(&f->lock); ++ ret = -EBUSY; ++ goto out; ++ } ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ mutex_unlock(&f->lock); ++ goto out; ++ } ++ /* render_user_sync_range bit 0: the requested range; else the whole BO (stronger, allowed). */ ++ if (READ_ONCE(render_user_sync_range) & 1) { ++ u64 lo, hi; ++ ++ if (g5p_sync_window(a->offset, a->size, obj->size, &lo, &hi)) ++ g5p_sync_sgt_range(drm->dev, sgt, lo, hi, ++ a->op == DRM_G5P_BO_SYNC_TO_CPU); ++ } else if (a->op == DRM_G5P_BO_SYNC_TO_DEVICE) { ++ dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } else { ++ dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ mutex_unlock(&f->lock); ++out: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_alloc_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ p->cpu = dma_alloc_coherent(dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ if (!g5p_user_pa_valid(p->dma) || dma_to_phys(dev, p->dma) != p->dma) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ return 0; ++} ++ ++static struct g5p_pt *g5p_vm_table(struct g5p_vm *vm, struct device *dev, ++ u32 level, u32 key, bool create) ++{ ++ struct g5p_pt *pt; ++ int ret; ++ ++ list_for_each_entry(pt, &vm->tables, link) ++ if (pt->level == level && pt->key == key) ++ return pt; ++ if (!create) ++ return NULL; ++ if (vm->table_count >= G5P_MAX_PT_PAGES) ++ return ERR_PTR(-ENOSPC); ++ pt = kzalloc(sizeof(*pt), GFP_KERNEL); ++ if (!pt) ++ return ERR_PTR(-ENOMEM); ++ ret = g5p_vm_alloc_page(dev, &pt->page); ++ if (ret) { ++ kfree(pt); ++ return ERR_PTR(ret); ++ } ++ pt->level = level; ++ pt->key = key; ++ list_add_tail(&pt->link, &vm->tables); ++ vm->table_count++; ++ return pt; ++} ++ ++static u8 *g5p_vm_pte(struct g5p_vm *vm, struct device *dev, u64 va, ++ bool create) ++{ ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ struct g5p_pt *dir, *leaf; ++ ++ dir = g5p_vm_table(vm, dev, 1, ci, create); ++ if (IS_ERR_OR_NULL(dir)) ++ return (void *)dir; ++ leaf = g5p_vm_table(vm, dev, 2, (ci << 9) | di, create); ++ if (IS_ERR_OR_NULL(leaf)) ++ return (void *)leaf; ++ if (create) { ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + ci * 4, ++ g5p_catalogue_entry(dir->page.dma), 0); ++ g5p_vm_entry(&vm->mutation, dir->page.cpu + di * 8, ++ g5p_directory_entry(leaf->page.dma), 1); ++ } ++ return leaf->page.cpu + ((va >> 14) & 127) * 8; ++} ++ ++static void g5p_vm_clear(struct g5p_vm *vm, struct device *dev, u64 va, ++ u64 size) ++{ ++ for (u64 off = 0; off < size; off += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, dev, va + off, false); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, 0, 2); ++ } ++ dma_wmb(); ++} ++ ++static bool g5p_vm_table_empty(struct g5p_pt *pt) ++{ ++ u32 count = pt->level == 2 ? 128 : 512; ++ ++ for (u32 i = 0; i < count; i++) ++ if (g5p_le32(pt->page.cpu + i * 8) & 1) ++ return false; ++ return true; ++} ++ ++static void g5p_vm_prune(struct device *dev, struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt, *tmp, *dir; ++ ++ /* These tables are mutable only before firmware attachment. ++ * Remove empty leaves before directories, including failed MAP rollback. ++ */ ++ for (u32 level = 2; level; level--) ++ list_for_each_entry_safe(pt, tmp, &vm->tables, link) { ++ if (pt->level != level || !g5p_vm_table_empty(pt)) ++ continue; ++ if (level == 2) { ++ dir = g5p_vm_table(vm, dev, 1, pt->key >> 9, ++ false); ++ if (dir) ++ g5p_vm_entry(&vm->mutation, ++ dir->page.cpu + (pt->key & ++ 511) * 8, ++ 0, 1); ++ } else { ++ g5p_vm_entry(&vm->mutation, ++ vm->root.cpu + pt->key * 4, 0, 0); ++ } ++ dma_wmb(); ++ list_del(&pt->link); ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ vm->table_count--; ++ kfree(pt); ++ } ++} ++ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj); ++ ++static void g5p_vm_release(struct kref *ref) ++{ ++ struct g5p_vm *vm = container_of(ref, struct g5p_vm, ref); ++ struct device *dev = vm->dev; ++ struct g5p_mapping *m, *mtmp; ++ struct g5p_pt *pt, *ptmp; ++ u64 pinned = vm->pinned_bytes; ++ u32 maps = vm->map_count, tables = vm->table_count; ++ ++ list_for_each_entry_safe(m, mtmp, &vm->maps, link) { ++ drm_mm_remove_node(&m->node); ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ list_del(&m->link); ++ kfree(m); ++ } ++ list_for_each_entry_safe(pt, ptmp, &vm->tables, link) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ list_del(&pt->link); ++ kfree(pt); ++ } ++ dma_free_coherent(dev, G5P_GPU_PAGE, vm->root.cpu, vm->root.dma); ++ xa_destroy(&vm->objs); ++ drm_mm_remove_node(&vm->reserved); ++ drm_mm_takedown(&vm->mm); ++ /* One line per freed VM: the device's pinned total must fall back after a client exits. */ ++ dev_info( ++ dev, ++ "G5P VM release maps=%u tables=%u pinned=%lluK device_pinned=%lluK\n", ++ maps, tables, pinned >> 10, ++ READ_ONCE(vm->gpu->pinned_bytes) >> 10); ++ kfree(vm); ++} ++ ++static void g5p_vm_put(struct g5p_vm *vm) ++{ ++ kref_put(&vm->ref, g5p_vm_release); ++} ++ ++static void g5p_context_release(struct kref *ref) ++{ ++ struct g5p_context *c = container_of(ref, struct g5p_context, ref); ++ g5p_render_release(c); ++ g5p_vm_put(c->vm); ++ kfree(c); ++} ++ ++static void g5p_context_put(struct g5p_context *c) ++{ ++ kref_put(&c->ref, g5p_context_release); ++} ++ ++static int g5p_create_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ u32 id; ++ int ret; ++ ++ if (a->flags || a->handle) ++ return -EINVAL; ++ vm = kzalloc(sizeof(*vm), GFP_KERNEL); ++ if (!vm) ++ return -ENOMEM; ++ kref_init(&vm->ref); ++ vm->dev = drm->dev; ++ vm->gpu = to_g5p(drm); ++ vm->bounded = (0); ++ vm->mutation.generations = true; ++ INIT_LIST_HEAD(&vm->maps); ++ xa_init(&vm->objs); ++ INIT_LIST_HEAD(&vm->tables); ++ drm_mm_init(&vm->mm, G5P_USER_VA_START, ++ G5P_USER_VA_END - G5P_USER_VA_START); ++ vm->reserved.start = G5P_USER_RESERVED_START; ++ vm->reserved.size = G5P_USER_RESERVED_END - G5P_USER_RESERVED_START; ++ ret = drm_mm_reserve_node(&vm->mm, &vm->reserved); ++ if (ret) ++ goto err_mm; ++ ret = g5p_vm_alloc_page(drm->dev, &vm->root); ++ if (ret) ++ goto err_reserved; ++ vm->table_count = 1; ++ mutex_lock(&f->lock); ++ ret = xa_alloc(&f->vms, &id, vm, XA_LIMIT(1, G5P_MAX_VMS), GFP_KERNEL); ++ mutex_unlock(&f->lock); ++ if (ret) { ++ g5p_vm_put(vm); ++ return ret; ++ } ++ a->handle = id; ++ return 0; ++err_reserved: ++ drm_mm_remove_node(&vm->reserved); ++err_mm: ++ drm_mm_takedown(&vm->mm); ++ kfree(vm); ++ return ret; ++} ++ ++static int g5p_destroy_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->handle); ++ if (!vm) ++ ret = -ENOENT; ++ else if (vm->contexts || f->submitted || vm->frozen) ++ ret = -EBUSY; ++ else ++ xa_erase(&f->vms, a->handle); ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_vm_put(vm); ++ return ret; ++} ++ ++static unsigned long g5p_vm_obj_count(struct g5p_vm *vm, ++ struct drm_gem_object *obj) ++{ ++ void *v = xa_load(&vm->objs, (unsigned long)obj); ++ ++ return v ? xa_to_value(v) : 0; ++} ++ ++/* ++ * One more mapping of `obj` in `vm`. Its first one in this VM counts its size against the ++ * VM's pin limit, and its first one in any VM against the device budget (render_pinned_mib). ++ * Our own dma-bufs import back as the same GEM object, in any file, so memory shared between ++ * clients (a client's buffer in the compositor's VM) counts once. ++ */ ++static int g5p_vm_obj_get(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ u64 limit = (u64)READ_ONCE(render_pinned_mib) << 20; ++ unsigned long count; ++ int ret = 0; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (!count && obj->size > G5P_MAX_PINNED_BYTES - vm->pinned_bytes) ++ ret = -ENOSPC; ++ else if (!count && !bo->vm_pins && ++ obj->size > limit - min(limit, d->pinned_bytes)) ++ ret = -ENOSPC; ++ if (!ret) ++ ret = xa_err(xa_store(&vm->objs, (unsigned long)obj, ++ xa_mk_value(count + 1), GFP_KERNEL)); ++ if (!ret && !count) { ++ vm->pinned_bytes += obj->size; ++ if (!bo->vm_pins++) ++ d->pinned_bytes += obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++ return ret; ++} ++ ++/* One mapping of `obj` in `vm` less: g5p_vm_obj_get() undone. Also called by the render ++ * scheduler for an unmap deferred behind queued jobs, hence the lock around the counts. ++ */ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long count; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (count > 1) { ++ /* replaces an existing entry: no allocation */ ++ xa_store(&vm->objs, (unsigned long)obj, xa_mk_value(count - 1), ++ GFP_KERNEL); ++ } else if (!WARN_ON(!count)) { ++ xa_erase(&vm->objs, (unsigned long)obj); ++ vm->pinned_bytes -= obj->size; ++ if (!--bo->vm_pins) ++ d->pinned_bytes -= obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++} ++ ++static int g5p_vm_map_bo(struct drm_device *drm, struct g5p_vm *vm, ++ struct drm_file *file, struct drm_g5p_vm_bind *a) ++{ ++ struct drm_gem_object *obj; ++ struct drm_gem_shmem_object *shmem; ++ struct sg_table *sgt; ++ struct sg_dma_page_iter iter; ++ struct g5p_mapping *m; ++ u64 *addresses, done = 0; ++ u32 count = 0; ++ bool imported; ++ int ret; ++ ++ if (vm->map_count >= G5P_MAX_MAPS || ++ a->size > G5P_MAX_VM_BYTES - vm->mapped_bytes) ++ return -ENOSPC; ++ obj = drm_gem_object_lookup(file, a->bo); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto err_obj; ++ } ++ ret = g5p_vm_obj_get(vm, obj); ++ if (ret) ++ goto err_obj; ++ addresses = kvmalloc_array(obj->size / G5P_GPU_PAGE, sizeof(*addresses), ++ GFP_KERNEL); ++ m = kzalloc(sizeof(*m), GFP_KERNEL); ++ if (!addresses || !m) { ++ ret = -ENOMEM; ++ goto err_alloc; ++ } ++ m->node.start = a->va; ++ m->node.size = a->size; ++ ret = drm_mm_reserve_node(&vm->mm, &m->node); ++ if (ret) { ++ ret = -EINVAL; ++ goto err_alloc; ++ } ++ imported = drm_gem_is_imported(obj); ++ shmem = to_drm_gem_shmem_obj(obj); ++ /* PRIME import mapped this attachment for our device and owns it until ++ * object release. Re-locking the exporter's reservation here can return ++ * -EBUSY after the first V3 submission even though the render VM is idle. ++ */ ++ if (imported) { ++ sgt = READ_ONCE(shmem->sgt); ++ if (!sgt) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=prime-no-sgt vm=%u bo=%u bytes=%llu\n", ++ a->vm, a->bo, a->size); ++ ret = -ENXIO; ++ goto err_node; ++ } ++ } else { ++ sgt = drm_gem_shmem_get_pages_sgt(shmem); ++ } ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=sgt-error vm=%u bo=%u imported=%u err=%d\n", ++ a->vm, a->bo, imported, ret); ++ goto err_node; ++ } ++ for_each_sgtable_dma_page(sgt, &iter, 0) { ++ u64 dma = sg_page_iter_dma_address(&iter); ++ ++ if (count >= obj->size / G5P_GPU_PAGE || ++ !g5p_user_pa_valid(dma) || ++ dma_to_phys(drm->dev, dma) != dma || ++ (!imported && dma_to_phys(drm->dev, dma) != ++ page_to_phys(shmem->pages[count]))) { ++ ret = -ERANGE; ++ goto err_node; ++ } ++ addresses[count++] = dma; ++ } ++ if (count != obj->size / G5P_GPU_PAGE) { ++ ret = -EINVAL; ++ goto err_node; ++ } ++ for (; done < a->size; done += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, drm->dev, a->va + done, true); ++ ++ if (IS_ERR_OR_NULL(pte)) { ++ ret = pte ? PTR_ERR(pte) : -ENOMEM; ++ goto err_clear; ++ } ++ /* Recovered bypass bits; user permission encodings are not claimed. ++ * A render VM latched to r25 memory variant 4 uses flags 0. ++ */ ++ g5p_vm_entry( ++ &vm->mutation, pte, ++ g5p_leaf_entry( ++ addresses[(a->offset + done) / G5P_GPU_PAGE], ++ vm->zero_leaf_flags ? 0 : g5p_pte_attrs()), ++ 2); ++ } ++ dma_wmb(); ++ m->obj = obj; ++ m->offset = a->offset; ++ list_add_tail(&m->link, &vm->maps); ++ vm->mapped_bytes += a->size; ++ vm->map_count++; ++ kvfree(addresses); ++ return 0; ++err_clear: ++ g5p_vm_clear(vm, drm->dev, a->va, done); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++err_node: ++ drm_mm_remove_node(&m->node); ++err_alloc: ++ kvfree(addresses); ++ kfree(m); ++ g5p_vm_obj_put(vm, obj); ++err_obj: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_bind(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_vm_bind *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_mapping *m; ++ struct g5p_vm *vm; ++ int ret = -ENOENT; ++ bool resident = false, locked = false; ++ u32 generation[3]; ++ ++ if (a->flags || !g5p_user_range_valid(a->va, a->size) || ++ (a->op != DRM_G5P_VM_MAP && a->op != DRM_G5P_VM_UNMAP) || ++ (a->op == DRM_G5P_VM_UNMAP && (a->bo || a->offset))) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->vm); ++ if (!vm) ++ goto out; ++ /* Its render context is closing: the close item owns the tables now, and the context ++ * may be freed at any time, so it is not looked at. ++ */ ++ if (vm->resident_gone) { ++ ret = -EBUSY; ++ goto out; ++ } ++ if ((f->submitted || vm->frozen) && !vm->resident) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=no-resident submitted=%u frozen=%u vm=%u op=%u\n", ++ f->submitted, vm->frozen, a->vm, a->op); ++ ret = -EBUSY; ++ goto out; ++ } ++ if (vm->resident) { ++ ret = g5p_resident_idle(vm->resident, true); ++ if (ret) ++ goto out; ++ /* A render context's edit is RAM only and touches its own tables and ++ * slots, which the firmware reads only at that context's next kick; ++ * f->lock plus resident-idle orders it against its submit (a MAP or ++ * UNMAP may pass queued jobs, see g5p_render_vm_idle()). Other contexts' jobs ++ * may hold the core lock meanwhile, so it is not taken. Serialized ++ * compute still uses the global mutex. ++ */ ++ if (!vm->resident->render) { ++ mutex_lock(&to_g5p(drm)->core->lock); ++ locked = true; ++ } ++ resident = true; ++ memcpy(generation, vm->mutation.generation, sizeof(generation)); ++ } ++ if (a->op == DRM_G5P_VM_MAP) { ++ if (vm->bounded && !(vm->resident && vm->resident->render) && ++ g5p_user_private_overlap(a->va, a->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_vm_map_bo(drm, vm, file, a); ++ if (!ret) ++ g5p_vmlog_add(G5P_VMLOG_MAP, vm, a->vm, a->va, a->size, ++ a->bo, a->offset); ++ if (ret == -EBUSY) ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=map vm=%u bo=%u submitted=%u frozen=%u resident=%u render=%u\n", ++ a->vm, a->bo, f->submitted, vm->frozen, ++ !!vm->resident, ++ !!(vm->resident && vm->resident->render)); ++ goto out; ++ } ++ list_for_each_entry(m, &vm->maps, link) { ++ if (m->node.start != a->va || m->node.size != a->size) { ++ if (m->node.start < a->va + a->size && ++ a->va < m->node.start + m->node.size) { ++ ret = -EINVAL; ++ break; ++ } ++ continue; ++ } ++ g5p_vmlog_add(G5P_VMLOG_UNMAP, vm, a->vm, a->va, a->size, 0, 0); ++ g5p_vm_clear(vm, drm->dev, a->va, a->size); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++ else { ++ ret = g5p_resident_invalidate(vm->resident); ++ if (ret) ++ goto out; ++ memcpy(generation, vm->mutation.generation, ++ sizeof(generation)); ++ } ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ vm->mapped_bytes -= a->size; ++ /* Beside queued jobs the PTEs are gone, but the pages stay pinned until ++ * those jobs have retired. ++ */ ++ if (!vm->resident || !g5p_resident_defer(vm->resident, m)) { ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ ret = 0; ++ break; ++ } ++out: ++ if (resident) { ++ int invalidate = 0; ++ ++ if (memcmp(generation, vm->mutation.generation, ++ sizeof(generation))) ++ invalidate = g5p_resident_invalidate(vm->resident); ++ ++ if (invalidate) ++ ret = invalidate; ++ if (locked) ++ mutex_unlock(&to_g5p(drm)->core->lock); ++ } ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_create_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_context *a = data; ++ if (a->type != DRM_G5P_CONTEXT_RENDER) ++ return -EOPNOTSUPP; ++ return g5p_render_create_context(drm, file, a); ++} ++ ++static int g5p_destroy_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_context *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->handle); ++ if (!c) ++ ret = -ENOENT; ++ else ++ ret = g5p_render_destroy(c); ++ if (!ret && c) { ++ c->vm->contexts--; ++ xa_erase(&f->contexts, a->handle); ++ } ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_context_put(c); ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/vm-ranges.h b/drivers/gpu/drm/apple-g5p/vm-ranges.h +new file mode 100644 +index 000000000..25bd98ee9 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/vm-ranges.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_H ++#define G5P_COMPUTE_USER_H ++ ++/* Public geometry is fixed across VMs. The private ranges never alias a BO. */ ++ ++static const u64 g5p_private_va[] = { ++ 0x300040000ULL, 0x300014000ULL, 0x300028000ULL, 0x300044000ULL, 0, ++ 0x100080000ULL, 0x100084000ULL, 0x1000a0000ULL, 0x1000a4000ULL, ++}; ++ ++static inline bool g5p_user_private_overlap(u64 va, u64 size) ++{ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_private_va); i++) ++ if (g5p_private_va[i] && va < g5p_private_va[i] + 0x4000 && ++ g5p_private_va[i] < va + size) ++ return true; ++ return false; ++} ++ ++#endif diff --git a/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch b/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch new file mode 100644 index 00000000..93dca40f --- /dev/null +++ b/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch @@ -0,0 +1,8973 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add typed render and compute submission + +[Orchard: from brkosebay/linux d67f133590a; normal probe registration +and fixed default policy.] Keep the A10 PowerVR driver and its build +hooks unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/Kconfig | 2 + + drivers/gpu/drm/Makefile | 2 + + drivers/gpu/drm/apple-g5p/Kconfig | 14 + + drivers/gpu/drm/apple-g5p/Makefile | 2 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 742 ++++ + .../gpu/drm/apple-g5p/compute-user-validate.h | 141 + + drivers/gpu/drm/apple-g5p/drm-compute-v1.h | 17 + + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 720 ++++ + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 293 ++ + drivers/gpu/drm/apple-g5p/drm-prime.inc | 114 + + .../gpu/drm/apple-g5p/drm-render-sched.inc | 203 ++ + .../gpu/drm/apple-g5p/drm-render-unbind.inc | 488 +++ + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 62 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 3038 +++++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-render.inc | 2549 ++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-submit.inc | 29 + + .../gpu/drm/apple-g5p/render-user-validate.h | 384 +++ + drivers/gpu/drm/apple-g5p/runtime-idle.inc | 14 + + 18 files changed, 8814 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/Kconfig + create mode 100644 drivers/gpu/drm/apple-g5p/Makefile + create mode 100644 drivers/gpu/drm/apple-g5p/a9-gpu-drm.c + create mode 100644 drivers/gpu/drm/apple-g5p/compute-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-compute-v1.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-compute-v1.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-latency123.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-prime.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-sched.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-unbind.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-submit.inc + create mode 100644 drivers/gpu/drm/apple-g5p/render-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/runtime-idle.inc + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 4c74c1501..eaf71742f 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -371,3 +371,5 @@ endif + # Separate option because drm_panel_orientation_quirks.c is shared with fbdev + config DRM_PANEL_ORIENTATION_QUIRKS + tristate ++ ++source "drivers/gpu/drm/apple-g5p/Kconfig" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index d75eba399..1315aa378 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -257,3 +257,5 @@ quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + + $(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) ++ ++obj-$(CONFIG_DRM_APPLE_G5P) += apple-g5p/ +diff --git a/drivers/gpu/drm/apple-g5p/Kconfig b/drivers/gpu/drm/apple-g5p/Kconfig +new file mode 100644 +index 000000000..1838fc90f +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Kconfig +@@ -0,0 +1,14 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_APPLE_G5P ++ bool "Apple A9/G5P render driver" ++ depends on DRM=y && ARCH_APPLE && OF && PM && ARM64_16K_PAGES ++ select DRM_GEM_SHMEM_HELPER ++ select FW_LOADER ++ select CRYPTO_LIB_SHA256 ++ help ++ Render-node driver for the S8003 G5P GPU using the unmodified META ++ firmware from the device. Supports TYPED_V3 jobs, GEM buffers, PRIME, ++ virtual address spaces and syncobj synchronization for the merged ++ Mesa winsys. The GPU is brought up at probe once the user's ++ apple/agx-g5p-c0.ldr is readable. Built-in only while reset and ++ active-DMA teardown require the platform boot sequence. +diff --git a/drivers/gpu/drm/apple-g5p/Makefile b/drivers/gpu/drm/apple-g5p/Makefile +new file mode 100644 +index 000000000..843d56a04 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Makefile +@@ -0,0 +1,2 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++obj-$(CONFIG_DRM_APPLE_G5P) += a9-gpu-drm.o +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +new file mode 100644 +index 000000000..f2fb4bf2d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -0,0 +1,742 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* Experimental render frontend; default disabled, no implicit firmware boot. */ ++#include "../../apple-g5p/core.inc" ++#include "../../apple-g5p/drm-validate.h" ++#include "../../apple-g5p/render-user-evwait.h" ++#include "../../apple-g5p/render-user-sync.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define G5P_MAX_VMS 8 ++#define G5P_MAX_CONTEXTS 32 ++#define G5P_MAX_MAPS 4096 ++#define G5P_MAX_PT_PAGES 512 ++#define G5P_MAX_SYNCS 17 ++ ++struct g5p_bo { ++ struct drm_gem_shmem_object shmem; ++ u32 roles; ++ u32 vm_pins; /* VMs that map it, any file; counted once in pinned_bytes (pin_lock) */ ++ /* release8 memory accounting (drm-vm.inc g5p_bo_live): the live list, its creator */ ++ struct list_head live; ++ pid_t pid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++struct g5p_pt { ++ struct list_head link; ++ struct g5p_dma_page page; ++ u32 key; ++ u32 level; ++}; ++ ++struct g5p_mapping { ++ struct list_head link; ++ struct drm_mm_node node; ++ struct drm_gem_object *obj; ++ u64 offset; ++ u64 retire; /* unmapped beside queued jobs: freed once this fence seqno has signalled */ ++}; ++ ++struct g5p_vm { ++ struct g5p_vm_mutation mutation; ++ bool bounded, frozen, serial_context; ++ bool zero_leaf_flags; /* render context latched r25 memory variant 4 */ ++ /* Its render context's handle is gone (g5p_render_close()): no VM_BIND, no new ++ * context; the context's close item and release own the tables. ++ */ ++ bool resident_gone; ++ struct g5p_context *resident; ++ struct kref ref; ++ struct device *dev; ++ struct g5p_drm *gpu; ++ struct drm_mm mm; ++ struct drm_mm_node reserved; ++ struct list_head maps; ++ struct xarray objs; /* object -> its mappings in this VM (xa_mk_value) */ ++ struct list_head tables; ++ struct g5p_dma_page root; ++ u64 mapped_bytes, pinned_bytes; ++ u32 table_count, map_count, contexts; ++}; ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rdev; ++ ++struct g5p_context { ++ struct g5p_drm *gpu; ++ ++ bool live, bad; ++ bool serial, pending; ++ ++ struct g5p_render *render; /* render contexts only, see drm-render.inc */ ++ struct kref ref; ++ struct g5p_vm *vm; ++}; ++ ++struct g5p_file { ++ /* Serializes VM/context handles, mappings and the one-shot submit state. */ ++ struct mutex lock; ++ struct xarray vms; ++ struct xarray contexts; ++ struct xarray results; ++ struct xarray targets; /* render targets; each holds its context */ ++ u32 render_results; ++ bool submitted; ++ u64 job_id; ++}; ++ ++struct g5p_drm { ++ struct drm_device drm; ++ struct g5p_diag *core; ++ /* Protects cached IRQ event/status fields; never held for hardware IO. */ ++ spinlock_t status_lock; ++ u32 last_event[11], notifications[4], first_fault[11]; ++ u64 event_serial, fault_serial; ++ struct delayed_work idle_work; ++ bool idle_suspended; ++ struct delayed_work autoboot; /* probe-time bring-up */ ++ unsigned long autoboot_deadline; ++ u32 autoboot_phase; ++ /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ ++ wait_queue_head_t ev_wq; ++ /* TYPED_V3 3D completion-event latch (render-user-retire.h): G5P_RV3_TD_WANT | endpoint ++ * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. ++ */ ++ u32 td_want; ++ u64 td_hit_ns; ++ /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ ++ bool ta_want; ++ u64 ta_hit_ns; ++ /* wake110: type-1 completion events decoded per engine (word 1 bit 0 TA, bit 1 3D) since ++ * load, and the decode time of the first one since the executor last cleared it. ++ */ ++ u32 t1_count[2]; ++ u64 t1_first_ns[2]; ++ /* perf60 bell gates (drm-render-v3.inc): the previous TYPED_V3 job's end and context. */ ++ u64 bell_prev_end_ns; ++ const void *bell_prev_ctx; ++ u64 last_job; ++ int last_error, job_error; ++ u32 job_state; ++ ++ atomic_t accepted; ++ atomic64_t job_sequence; ++ u64 fence_context; ++ /* Attached catalogues survive fd close until proven unbind/reset. */ ++ struct g5p_context *quarantined_context; ++ /* The first accepted render job claimed the device for render, for the rest of the boot. */ ++ bool render_claimed; ++ struct g5p_rdev *rdev; /* device-wide render state, see drm-render.inc */ ++ struct workqueue_struct ++ *render_wq; /* ordered: the render scheduler's worker */ ++ /* Orders render acceptance, dma_resv fence attach and queueing. */ ++ struct mutex render_submit_lock; ++ /* recovery108 epoch inputs, RAM only: open render-node files, render jobs not yet ++ * signalled. ++ */ ++ atomic_t open_files, r8_queued; ++ /* Memory mapped into any VM, each object once (render_pinned_mib). */ ++ struct mutex pin_lock; ++ u64 pinned_bytes; ++}; ++ ++static inline struct g5p_drm *to_g5p(struct drm_device *drm) ++{ ++ return container_of(drm, struct g5p_drm, drm); ++} ++ ++static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) ++{ ++ return container_of(obj, struct g5p_bo, shmem.base); ++} ++ ++#include "vm-ranges.h" ++ ++static int g5p_resident_idle(struct g5p_context *c, bool live); ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m); ++static int g5p_resident_invalidate(struct g5p_context *c); ++static void g5p_render_release(struct g5p_context *c); ++static int g5p_render_destroy(struct g5p_context *c); ++static void g5p_render_close(struct g5p_context *c); ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a); ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags); ++ ++#include "drm-vm.inc" ++#include "drm-submit.inc" ++#include "drm-render.inc" ++#include "drm-prime.inc" ++#include "drm-compute-v1.inc" ++ ++static int g5p_get_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_info *a = data; ++ u64 caps = DRM_G5P_CAP_BO_CPU | DRM_G5P_CAP_VM_PREPARE | ++ DRM_G5P_CAP_SYNCOBJ | DRM_G5P_CAP_JOB_STATUS | ++ DRM_G5P_CAP_CONTEXT_STATUS | DRM_G5P_CAP_FAULT_INFO; ++ ++ if (a->size != sizeof(*a) || a->flags || a->reserved[0] || ++ a->reserved[1]) ++ return -EINVAL; ++ ; ++ ; ++ ; ++ ; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ a->version = DRM_G5P_UAPI_VERSION; ++ a->gpu_page_shift = 14; ++ ; ++ ; ++ ; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER | DRM_G5P_CAP_BOUNDED_RENDER | ++ DRM_G5P_CAP_TYPED_RENDER | DRM_G5P_CAP_VM_LIVE | ++ DRM_G5P_CAP_CONTEXT_LIVE | ++ DRM_G5P_CAP_CODE_BASE_VERIFIED; ++ a->code_base = 0x300000000ULL; ++ }; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY; ++ }; ++ if ((1) && g5p_render_msaa_enabled()) ++ caps |= DRM_G5P_CAP_RENDER_MSAA; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_TYPED_RENDER_V3 | ++ DRM_G5P_CAP_RENDER_MULTI_CONTEXT | ++ DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH | ++ DRM_G5P_CAP_RENDER_OCCLUSION_QUERY; ++ }; ++ { ++ caps |= DRM_G5P_CAP_RENDER_UNPRIVILEGED; ++ }; ++ ; ++ ; ++ caps |= DRM_G5P_CAP_PRIME; ++ caps |= DRM_G5P_CAP_TYPED_COMPUTE; ++ a->capabilities = caps; ++ a->va_start = G5P_USER_VA_START; ++ a->va_end = G5P_USER_VA_END; ++ a->reserved_start = G5P_USER_RESERVED_START; ++ a->reserved_end = G5P_USER_RESERVED_END; ++ a->max_bo_size = G5P_MAX_BO_SIZE; ++ a->bvnc[0] = 9; ++ a->bvnc[1] = 15; ++ a->bvnc[2] = 6; ++ a->bvnc[3] = 2; ++ return 0; ++} ++ ++/* The gated dialect uses the common B proven by the r19c numerical store. */ ++static int g5p_get_exec_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_exec_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || a->pad || ++ memchr_inv(a->reserved, 0, sizeof(a->reserved))) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ { ++ a->base_scope = DRM_G5P_BASE_VM; ++ a->usc_base = 0x300000000ULL; ++ a->pds_base = a->usc_base; ++ a->window_size = 0x34000; ++ a->allocation_alignment = G5P_GPU_PAGE; ++ a->usc_offset_bits = 32; ++ a->pds_offset_bits = 32; ++ } ++ return 0; ++} ++ ++static int g5p_get_job_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_job_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ bool owned; ++ ++ if (a->flags || a->pad || !a->job_id || a->reset_generation) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ if (xa_load(&f->results, a->job_id)) { ++ struct g5p_fence *fence = xa_load(&f->results, a->job_id); ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = fence->state; ++ a->error = fence->error; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ mutex_unlock(&f->lock); ++ return 0; ++ } ++ owned = f->job_id == a->job_id; ++ mutex_unlock(&f->lock); ++ if (!owned) ++ return -ENOENT; ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = d->job_state; ++ a->error = d->job_error; ++ /* Owner acknowledges terminal status before another fixed submission. ++ * This avoids waking a fence waiter before the device admission reopens. ++ */ ++ ; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return 0; ++} ++ ++/* c99788 classifies event4/7/8. Preserve opaque fields, never invent a VA. */ ++static int g5p_get_fault_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_fault_info *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ ++ if (a->flags || a->reserved) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->event_serial = d->fault_serial; ++ memcpy(a->raw, d->first_fault, sizeof(a->raw)); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!a->event_serial) ++ return 0; ++ switch (a->raw[0]) { ++ case 4: ++ a->kind = DRM_G5P_FAULT_RECOVERY; ++ break; ++ case 7: ++ a->kind = DRM_G5P_FAULT_PAGE_REQUEST; ++ break; ++ case 8: ++ a->kind = DRM_G5P_FAULT_CHANNEL; ++ a->valid = DRM_G5P_FAULT_VALID_CHANNEL; ++ a->channel_class = a->raw[1]; ++ a->channel_token = a->raw[3]; ++ a->channel_stamp = a->raw[4]; ++ break; ++ default: ++ a->kind = DRM_G5P_FAULT_UNKNOWN; ++ break; ++ } ++ return 0; ++} ++ ++static int g5p_get_context_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_context_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(to_g5p(drm)->core->boot); ++ struct g5p_context *c; ++ bool failed; ++ int ret = 0; ++ ++ if (a->flags || a->state || a->map_count || a->mapped_bytes || ++ a->reserved) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c) { ++ ret = -ENOENT; ++ goto out; ++ } ++ a->state = DRM_G5P_CONTEXT_HOST_ONLY; ++ if (READ_ONCE(c->live)) ++ a->state = DRM_G5P_CONTEXT_LIVE; ++ /* a failure noted mid-job is not this context's until the executor rules */ ++ failed = b && READ_ONCE(b->failed) && ++ !g5p_rdev_ruling(to_g5p(drm)->rdev); ++ if (READ_ONCE(c->bad) || failed || (b && READ_ONCE(b->stopped))) ++ a->state = DRM_G5P_CONTEXT_LOST; ++ a->map_count = c->vm->map_count; ++ a->mapped_bytes = c->vm->mapped_bytes; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static void g5p_drm_event(void *cookie, const u32 words[11]) ++{ ++ struct g5p_drm *d = cookie; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ memcpy(d->last_event, words, sizeof(d->last_event)); ++ d->event_serial++; ++ if (words[0] == 1) ++ for (u32 i = 0; i < 4; i++) ++ d->notifications[i] |= words[i + 1]; ++ if (d->td_want && !d->td_hit_ns && ++ g5p_ru_rt_event_match(words, d->td_want & 0xff)) ++ WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); ++ if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) ++ WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); ++ for (u32 e = 0; words[0] == 1 && e < 2; e++) { ++ if (!(words[1] & BIT(e))) ++ continue; ++ WRITE_ONCE(d->t1_count[e], d->t1_count[e] + 1); ++ if (!d->t1_first_ns[e]) ++ WRITE_ONCE(d->t1_first_ns[e], ktime_get_ns()); ++ } ++ if (words[0] != 0 && words[0] != 1) { ++ d->last_error = -EIO; ++ if (!d->fault_serial) { ++ d->fault_serial = d->event_serial; ++ memcpy(d->first_fault, words, sizeof(d->first_fault)); ++ } ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ wake_up(&d->ev_wq); ++} ++ ++static int g5p_idle_open(struct g5p_drm *d); ++static void g5p_idle_queue(struct g5p_drm *d); ++ ++static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = kzalloc(sizeof(*f), GFP_KERNEL); ++ ++ if (!f) ++ return -ENOMEM; ++ mutex_init(&f->lock); ++ xa_init_flags(&f->vms, XA_FLAGS_ALLOC1); ++ xa_init_flags(&f->contexts, XA_FLAGS_ALLOC1); ++ xa_init(&f->results); ++ xa_init_flags(&f->targets, XA_FLAGS_ALLOC1); ++ file->driver_priv = f; ++ { ++ int ret = g5p_idle_open(to_g5p(drm)); ++ ++ if (ret) { ++ mutex_destroy(&f->lock); ++ kfree(f); ++ file->driver_priv = NULL; ++ return ret; ++ } ++ } ++ return 0; ++} ++ ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_vm *vm; ++ unsigned long id; ++ struct g5p_rt *rt; ++ ++ ++ /* Render-target handles hold their context; drop them first. */ ++ xa_for_each(&f->targets, id, rt) ++ g5p_context_put(rt->ctx); ++ xa_destroy(&f->targets); ++ /* Jobs hold contexts, contexts hold VMs, mappings hold GEM objects. ++ * Closing the fd drops handle ownership, never a queued job's references. ++ * Host-only tables may retire at job completion: no live GPU attachment. ++ */ ++ xa_for_each(&f->contexts, id, c) { ++ /* A render context's firmware binding outlives the handle until its close ++ * item, queued behind its jobs, has unbound it. ++ */ ++ if (c->render) ++ g5p_render_close(c); ++ g5p_context_put(c); ++ } ++ xa_destroy(&f->contexts); ++ { ++ struct g5p_fence *fence; ++ ++ xa_for_each(&f->results, id, fence) ++ dma_fence_put(&fence->base); ++ xa_destroy(&f->results); ++ } ++ xa_for_each(&f->vms, id, vm) ++ g5p_vm_put(vm); ++ xa_destroy(&f->vms); ++ mutex_destroy(&f->lock); ++ kfree(f); ++ atomic_dec(&to_g5p(drm)->open_files); ++ g5p_idle_queue(to_g5p(drm)); ++ /* The file's handles are gone (drm_gem_release() ran first): what stays live is held ++ * by another file's handle, a dma-buf, a VM or a job (debugfs g5p_bos names it). ++ */ ++ dev_info(drm->dev, ++ "G5P file close live_bos=%u live_bo_bytes=%lluK device_pinned=%lluK\n", ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++} ++ ++static const struct drm_ioctl_desc g5p_ioctls[] = { ++ DRM_IOCTL_DEF_DRV(G5P_GET_INFO, g5p_get_info, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_BO, g5p_create_bo, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_MMAP_OFFSET, g5p_bo_mmap_offset, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_SYNC, g5p_bo_sync, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_VM, g5p_create_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_VM, g5p_destroy_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_VM_BIND, g5p_vm_bind, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_CONTEXT, g5p_create_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_CONTEXT, g5p_destroy_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_EXEC_INFO, g5p_get_exec_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_JOB_STATUS, g5p_get_job_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_CONTEXT_STATUS, g5p_get_context_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_FAULT_INFO, g5p_get_fault_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_RENDER_TARGET, g5p_create_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_RENDER_TARGET, g5p_destroy_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_RENDER_INFO, g5p_get_render_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_RENDER_TYPED, g5p_submit_render_typed, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_COMPUTE_TYPED, g5p_submit_compute_typed, ++ DRM_RENDER_ALLOW), ++}; ++ ++DEFINE_DRM_GEM_FOPS(g5p_drm_fops); ++ ++static const struct drm_driver g5p_drm_driver = { ++ .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | ++ DRIVER_SYNCOBJ_TIMELINE, ++ .open = g5p_drm_open, ++ .postclose = g5p_drm_postclose, ++ .fops = &g5p_drm_fops, ++ .ioctls = g5p_ioctls, ++ .num_ioctls = ARRAY_SIZE(g5p_ioctls), ++ .gem_create_object = g5p_gem_create_object, ++ .prime_handle_to_fd = g5p_prime_handle_to_fd, ++ .prime_fd_to_handle = g5p_prime_fd_to_handle, ++ .gem_prime_import = g5p_gem_prime_import, ++ .gem_prime_import_sg_table = g5p_prime_import_sg_table, ++ .name = "apple_g5p", ++ .desc = "Apple A9 G5P experimental render driver", ++ .major = 0, ++ .minor = DRM_G5P_UAPI_VERSION, ++}; ++ ++/* recovery108: the frontend's epoch inputs for the retire layer. RAM only and lock free: ++ * the claimed context is compared, never dereferenced (it may be dropped concurrently). ++ */ ++static void g5p_drm_r8_epoch(void *cookie, struct g5p_r8_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ in->claimed = READ_ONCE(d->render_claimed) || ++ READ_ONCE(d->quarantined_context); ++ in->held = in->claimed && atomic_read(&d->open_files) > 0; ++ in->queued = queued > 0 ? queued : 0; ++} ++ ++/* The teardown DETACH phase. No fence is signalled here: with admission closed and the ++ * device frozen, the render scheduler fails each queued job exactly once (-ESHUTDOWN, ++ * without waiting for its inputs), and an in-flight job ends on its own RAM-only ++ * deadline. The count left to the workers is reported as the explicit abandonment. ++ */ ++static void g5p_drm_td_detach(void *cookie) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ g5p_r8_note_detach(&g5p_r8_state, queued > 0 ? queued : 0); ++ g5p_rsched_kick(d); ++} ++ ++#include "runtime-idle.inc" ++ ++static const bool autoboot = true; ++static const unsigned int autoboot_wait_s = 120; ++ ++/* The bootstrap's order: the one-time firmware, wake and identity steps, then the stages ++ * recovery repeats (recover.inc). ++ */ ++static const char *const g5p_autoboot_phases[] = { ++ "validate", "prepare", "wake", "id", "sample", "map", "arm", "load", "release", ++ "events", "control-prepare", "control-submit", "control-wait", ++}; ++ ++static void g5p_autoboot_work(struct work_struct *work) ++{ ++ struct g5p_drm *d = container_of(to_delayed_work(work), struct g5p_drm, autoboot); ++ struct g5p_diag *g = d->core; ++ bool ran; ++ int ret; ++ ++ while (d->autoboot_phase < ARRAY_SIZE(g5p_autoboot_phases)) { ++ const char *phase = g5p_autoboot_phases[d->autoboot_phase]; ++ ++ ret = g5p_run_phase(g, phase, &ran); ++ /* Built in: the firmware is on the root file system, which may not be mounted yet. */ ++ if (ret == -ENOENT && !d->autoboot_phase && ++ time_before(jiffies, d->autoboot_deadline)) { ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 2 * HZ); ++ return; ++ } ++ dev_info(g->dev, "autoboot phase=%s result=%d\n", phase, ret); ++ if (ret) { ++ dev_warn(g->dev, "autoboot stopped at %s (%d); the command file can continue\n", ++ phase, ret); ++ return; ++ } ++ d->autoboot_phase++; ++ } ++ dev_info(g->dev, "autoboot complete: GPU up\n"); ++ g5p_idle_queue(d); ++} ++ ++static int g5p_drm_probe(struct platform_device *pdev) ++{ ++ struct g5p_drm *d; ++ struct g5p_diag *g; ++ int ret; ++ ++ ; ++ ; ++ ret = g5p_probe(pdev); ++ if (ret) ++ return ret; ++ g = platform_get_drvdata(pdev); ++ d = devm_drm_dev_alloc(&pdev->dev, &g5p_drm_driver, struct g5p_drm, ++ drm); ++ if (IS_ERR(d)) { ++ ret = PTR_ERR(d); ++ goto err_pm; ++ } ++ d->core = g; ++ d->fence_context = dma_fence_context_alloc(1); ++ spin_lock_init(&d->status_lock); ++ init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->accepted, 0); ++ atomic64_set(&d->job_sequence, 0); ++ d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", 0); ++ if (IS_ERR_OR_NULL(d->render_wq)) { ++ ret = d->render_wq ? PTR_ERR(d->render_wq) : -ENOMEM; ++ goto err_pm; ++ } ++ ret = drmm_mutex_init(&d->drm, &d->render_submit_lock); ++ if (!ret) ++ ret = drmm_mutex_init(&d->drm, &d->pin_lock); ++ if (ret) ++ goto err_pm; ++ ret = g5p_render_init(d); ++ if (ret) ++ goto err_pm; ++ g->event_cookie = d; ++ g->event_notify = g5p_drm_event; ++ atomic_set(&d->open_files, 0); ++ atomic_set(&d->r8_queued, 0); ++ drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); ++ drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); ++ INIT_DELAYED_WORK(&d->idle_work, g5p_idle_work); ++ ret = drm_dev_register(&d->drm, 0); ++ if (ret) ++ goto err_pm; ++ g5p_r8_register_epoch(g5p_drm_r8_epoch, d); ++ g5p_td_register_detach(g5p_drm_td_detach, d); ++ g5p_kq_register(&g5p_kq_drm, d); ++ dev_info(&pdev->dev, ++ "render node registered; no GPU access, COMPUTE/RENDER capabilities absent\n"); ++ INIT_DELAYED_WORK(&d->autoboot, g5p_autoboot_work); ++ if (READ_ONCE(autoboot)) { ++ d->autoboot_deadline = jiffies + secs_to_jiffies(autoboot_wait_s); ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 0); ++ } ++ return 0; ++err_pm: ++ g->event_notify = NULL; ++ pm_runtime_disable(&pdev->dev); ++ return ret; ++} ++ ++static void g5p_drm_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g->event_cookie; ++ ++ cancel_delayed_work_sync(&d->autoboot); ++ cancel_delayed_work_sync(&d->idle_work); ++ drm_dev_unplug(&d->drm); ++ g5p_remove(pdev); ++ /* recovery108: the remove-time FINAL above was the callbacks' last use; d is ++ * devres-freed next. ++ */ ++ g5p_td_register_detach(NULL, NULL); ++ g5p_r8_register_epoch(NULL, NULL); ++ g5p_kq_register(NULL, NULL); ++} ++ ++static void g5p_drm_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g ? g->event_cookie : NULL; ++ ++ if (d) { ++ cancel_delayed_work_sync(&d->autoboot); ++ cancel_delayed_work_sync(&d->idle_work); ++ } ++ g5p_shutdown(pdev); ++} ++ ++static const struct of_device_id g5p_drm_match[] = { ++ { .compatible = "apple,s8003-gpu" }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, g5p_drm_match); ++ ++static struct platform_driver g5p_drm_platform = { ++ .probe = g5p_drm_probe, ++ .remove = g5p_drm_remove, ++ .shutdown = g5p_drm_shutdown, ++ .driver = { ++ .name = "apple-g5p-drm", ++ .of_match_table = g5p_drm_match, ++ .dev_groups = g5p_groups, ++ .suppress_bind_attrs = true, ++ }, ++}; ++ ++module_platform_driver(g5p_drm_platform); ++ ++/* Built-in only until reset can prove all DMA references are quiescent. */ ++MODULE_DESCRIPTION("Apple G5P render/GEM/VM experimental driver"); ++MODULE_LICENSE("Dual MIT/GPL"); ++MODULE_FIRMWARE(G5P_FW_NAME); +diff --git a/drivers/gpu/drm/apple-g5p/compute-user-validate.h b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +new file mode 100644 +index 000000000..ac16d4d0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +@@ -0,0 +1,141 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_VALIDATE_H ++#define G5P_COMPUTE_USER_VALIDATE_H ++ ++/* ++ * TYPED_COMPUTE pass checks over already-resolved resource ++ * ranges, as render-user-validate.h does for TYPED_V3. Pure, host-tested ++ * (scripts/gpu/compute-host/test_compute.c). Nothing here is a pointer the kernel ++ * dereferences; the stream's last word is read by the caller through the BO's own pages ++ * (g5p_cu_content()). Like TYPED_V3 the kernel does not decode the CDM records, the PDS ++ * programs or the USC code they name; it bounds every address the firmware or the CDM ++ * starts from and leaves the rest to the per-context VM and fault isolation. ++ * -EINVAL flags, alignment, sizes, a non-zero terminate control word, no terminator ++ * -ERANGE a PDS reference outside [pds_base, pds_base + code_window), or a range past ++ * hw_va_limit ++ * -ENOENT no listed range wholly covers the stream, a state word or a PDS reference ++ * -EACCES the range lacks READ (stream, PDS data), WRITE (state) or EXEC (PDS code) ++ */ ++#define G5P_CU_MAX_STREAM 0x10000U ++#define G5P_CU_STREAM_MIN 4U ++#define G5P_CU_TERMINATE 0x80000000U ++ ++/* The pass as the UAPI carries it, kernel copy. */ ++struct g5p_cu_pass { ++ u64 cdm_va; ++ u32 cdm_size, cdm_end; ++ u64 state[4]; ++ u64 tail; ++ u32 config[2]; ++ u32 store[3], resume[3], term[3]; /* code, data, control */ ++}; ++ ++/* Where the content rule reads: the range holding the stream. */ ++struct g5p_cu_loc { ++ u32 stream_range; ++}; ++ ++/* One {code, data} PDS pair: code in a READ|EXEC range, data in a READ range (g5p_ru_v3_ref()). */ ++static inline int g5p_cu_pds(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ const u32 *pds) ++{ ++ int ret = g5p_ru_v3_ref(lim, r, n, pds[0], true); ++ ++ return ret ?: g5p_ru_v3_ref(lim, r, n, pds[1], false); ++} ++ ++static inline int g5p_cu_pass_check(const struct g5p_cu_pass *p, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_cu_loc *loc) ++{ ++ const u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ u32 si; ++ int ret; ++ ++ if (p->cdm_size < G5P_CU_STREAM_MIN || ++ p->cdm_size > G5P_CU_MAX_STREAM || ++ ((p->cdm_va | p->cdm_size | p->cdm_end) & 3) || ++ p->cdm_end < G5P_CU_STREAM_MIN || p->cdm_end > p->cdm_size) ++ return -EINVAL; ++ if ((p->tail & 0xf) || p->term[2]) ++ return -EINVAL; ++ for (u32 i = 0; i < 4; i++) ++ if (p->state[i] & 3) ++ return -EINVAL; ++ /* Every listed range, as TYPED_V3: the CDM and its tasks read and write past them. */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, p->cdm_va, p->cdm_size, &si)) ++ return -ENOENT; ++ if (!(r[si].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ for (u32 i = 0; i < 5; i++) { ++ u64 va = i < 4 ? p->state[i] : p->tail; ++ u32 k; ++ ++ if (!g5p_ru_cover(r, n, va, i < 4 ? 4 : 16, &k)) ++ return -ENOENT; ++ if ((r[k].access & rw) != rw) ++ return -EACCES; ++ } ++ ret = g5p_cu_pds(lim, r, n, p->store); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->resume); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->term); ++ if (ret) ++ return ret; ++ loc->stream_range = si; ++ return 0; ++} ++ ++/* The last stream dword, at cdm_va + cdm_end - 4, is the CDM terminate word. */ ++static inline int g5p_cu_content(const u8 *tail) ++{ ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ return word == G5P_CU_TERMINATE ? 0 : -EINVAL; ++} ++ ++/* The firmware's register images of a checked pass. */ ++static inline void g5p_cu_regs_of(struct g5p_cu_regs *k, ++ const struct g5p_cu_pass *p) ++{ ++ k->stream = p->cdm_va; ++ k->terminate = p->cdm_va + p->cdm_end - 4; ++ for (u32 i = 0; i < 4; i++) ++ k->state[i] = p->state[i]; ++ k->tail = p->tail; ++ k->config[0] = p->config[0]; ++ k->config[1] = p->config[1]; ++ k->store = p->store[0] | (u64)p->store[1] << 32; ++ k->resume = p->resume[0] | (u64)p->resume[1] << 32; ++ k->term = p->term[0] | (u64)p->term[1] << 32; ++ k->store1 = p->store[2]; ++ k->resume1 = p->resume[2]; ++} ++ ++/* ++ * Submit-time admission of a compute job, on top of g5p_ru_v3_admit(): the CDM rides an epoch ++ * a render job brought up (the attach rings the TA bell; a compute job never does), and its ++ * own sequence is bounded like the render one. ++ */ ++enum g5p_cu_refusal { ++ G5P_CU_OK, ++ G5P_CU_NO_EPOCH, /* no render job has brought the device up in this epoch */ ++ G5P_CU_OVERFLOW, /* the CDM sequence is exhausted */ ++}; ++ ++static inline int g5p_cu_admit(bool device_up, u32 cu_sequence) ++{ ++ if (!device_up) ++ return G5P_CU_NO_EPOCH; ++ if (cu_sequence >= G5P_CU_SEQ_MAX) ++ return G5P_CU_OVERFLOW; ++ return G5P_CU_OK; ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.h b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +new file mode 100644 +index 000000000..3531cefb7 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_COMPUTE (drm-compute-v1.inc): what the TYPED_V3 executor, the scheduler's drain and the ++ * render device's bring-up call before the file is included. ++ */ ++#ifndef G5P_DRM_COMPUTE_V1_H ++#define G5P_DRM_COMPUTE_V1_H ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rjob; ++ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job); ++static bool g5p_cu_drain(struct g5p_drm *d); ++static void g5p_cu_device_init(struct g5p_drm *d); ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +new file mode 100644 +index 000000000..dcbe34c0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -0,0 +1,720 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * TYPED_COMPUTE: compute jobs on TYPED_V3 render contexts. ++ * ++ * A compute job is a TYPED_V3 job of its render context with one phase on the device's CDM ++ * channel instead of a TA and a 3D: it is queued, scheduled, fenced, quarantined and lost ++ * exactly like a render job (g5p_render_run(), drm-render-sched.inc, render-user-fault.h). ++ * It binds the context's slot pair the RAM-only way if the context has not run a job yet, ++ * builds the CDM job (compute-user-layout.h) from the pass the submit validated ++ * (compute-user-validate.h), publishes it on the CL ring with a guarded DM2 wake, and waits RAM only ++ * for coherent channel watermarks and both completion stamps. It never brings the device up (that rings the TA bell): ++ * an epoch without a render job refuses it before the hardware. ++ * ++ * Only the licensed publish-time wake touches registers; validation, waits, ++ * samples, retirement and recovery decisions are RAM-only. ++ */ ++ ++static u8 *g5p_cu_page(struct g5p_boot *b) ++{ ++ return b->shared[G5P_CU_OFFSET / G5P_GPU_PAGE].cpu + ++ G5P_CU_OFFSET % G5P_GPU_PAGE; ++} ++ ++static u32 g5p_cu_meta(void) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + G5P_CU_OFFSET); ++} ++ ++/* The CDM's global object (the boot layout's engine-2 object, R0 group, CPU 0x49 / GPU 0x53). */ ++static u32 g5p_cu_global_meta(struct g5p_boot *b) ++{ ++ return g5p_meta_address( ++ g5p_object_va(&b->layout, 0x51 + G5P_CU_ENGINE)); ++} ++ ++/* The render device's bring-up (g5p_render_device_prepare()), after the TA/3D channel init that ++ * clears the 3D page: the CDM channel at its epoch start. RAM only. ++ */ ++static void g5p_cu_device_init(struct g5p_drm *d) ++{ ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_R22_DEP + 0x280); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_RU_SLOT1_END); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE + G5P_CU_BYTES > ++ G5P_GPU_PAGE); ++ BUILD_BUG_ON(G5P_CU_DEP + G5P_CU_DEP_BYTES > G5P_CU_BYTES); ++ BUILD_BUG_ON(G5P_CU_W + G5P_CU_W_BYTES > G5P_CU_SKU); ++ g5p_cu_channel_init(g5p_cu_page(d->core->boot), g5p_cu_meta()); ++ d->rdev->cu_sequence = 0; ++ d->rdev->cu_target = 0; ++} ++ ++/* One RAM sample of the CDM channel for job `seq` (the slots of struct g5p_rv3_ev: stamp[0] = ++ * W258 mirror, stamp[1] = W25c, ctl = the five watermarks, idle[0..2] = the CDM global's ++ * G724/728/72c). RAM only. ++ */ ++static void g5p_cu_sample(struct g5p_diag *g, u32 seq, u32 target, ++ struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_cu_page(b), *global = g5p_ru_global(b, G5P_CU_ENGINE); ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ s->stamp[0] = g5p_le32(p + G5P_CU_START); ++ s->stamp[1] = g5p_le32(p + G5P_CU_END); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < 3; i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != target || s->cons > 255 || ++ !g5p_cu_stamp_sane(p, seq); ++ s->consumed = s->cons == target; ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2]; ++ s->retired = g5p_cu_retired(p, g5p_cu_meta(), seq); ++ s->pm_ok = true; ++} ++ ++#define G5P_CU_SLOW_MS 1000U ++ ++/* The CDM phase's RAM-only wait, the TYPED_V3 wait (g5p_rv3_wait()) for engine 2: the silence ++ * clock, the fault/protocol rules and the closing sample are the same; complete is the ring ++ * consumed plus the retirement predicate (the idle words are recorded, not required: render path). ++ */ ++/* Event serial is sampled before RAM, so an IRQ between sample and sleep ++ * cannot be lost. Any firmware event wakes us; only the full RAM retirement ++ * predicate may complete the job. No render-only engine-indexed observers. ++ */ ++static void g5p_cu_pause(struct g5p_drm *d, u64 seen) ++{ ++ u32 wait_us; ++ ++ if (g5p_ew_plan(READ_ONCE(render_user_event_wake), false, &wait_us) == ++ G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ return; ++ } ++ wait_event_hrtimeout(d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us)); ++} ++ ++static u32 g5p_cu_wait(struct g5p_drm *d, struct g5p_diag *g, u32 seq, ++ u32 target, struct g5p_ru_wait *w, struct g5p_rv3_ev *s, ++ u64 *consumed_ns, int *err) ++{ ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE(d->event_serial); ++ bool fault, silent, done; ++ ++ g5p_cu_sample(g, seq, target, s); ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed || silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = s->consumed && s->retired; ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (w->result) ++ return w->result; ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_CU_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ dev_warn( ++ g->dev, ++ "compute slow seq=%u ms=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x consumed=%u retired=%u events=%u\n", ++ seq, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, ++ s->cons, s->stamp[0], s->stamp[1], s->ctl[0], ++ s->ctl[1], s->ctl[2], s->ctl[3], s->ctl[4], ++ s->idle[0], s->idle[1], s->idle[2], s->consumed, ++ s->retired, READ_ONCE(g->boot->events)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ g5p_cu_pause(d, seen); ++ } ++} ++ ++/* The guarded publish-time wake, identical ordering to TA/3D with DM2. ++ * Core lock held; only this helper may touch registers in the compute path. ++ * The event lock closes the reset/fault race. Never called from wait/drain. ++ */ ++static int g5p_cu_wake(struct g5p_diag *g, u32 seq, u32 start, u32 target) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ u32 why, fence = U32_MAX, gpc = U32_MAX; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = G5P_WK_ALWAYS, ++ .e = G5P_CU_ENGINE, ++ .seq = seq, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = target, ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_CU_START), ++ g5p_le32(p + G5P_CU_END) }, ++ }; ++ why = g5p_wk_guard(&in); ++ if (!why) ++ why = g5p_wk_ring(&io, G5P_CU_ENGINE, &fence, &gpc); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_dbg(g->dev, ++ "compute wake seq=%u did=%u power=%#x fence=%#x gpc=%#x\n", seq, ++ why, in.power, fence, gpc); ++ return why == G5P_WK_DID_RANG ? 0 : -EHOSTDOWN; ++} ++ ++/* The CL publish of job `seq`: the global ring entry {type 2, channel, endpoint}, the channel's ++ * admission watermark, then the producer, followed by the guarded DM2 wake. Refused, with nothing written, unless the ++ * ring is idle and the channel stands at the previous job's endpoint and stamps. RAM only. ++ */ ++static int g5p_cu_publish(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_cu_endpoint(seq - 1); ++ u32 endpoint = g5p_cu_endpoint(seq), why; ++ ++ why = (u32)(start > 255) | (u32)(READ_ONCE(ring[0]) != start) << 1 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 2 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 3 | ++ (u32)(g5p_le32(p + G5P_CU_START) != seq - 1 || ++ g5p_le32(p + G5P_CU_END) != seq - 1) ++ << 4; ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "compute refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x stamp=%08x/%08x: nothing published\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + G5P_CU_START), g5p_le32(p + G5P_CU_END)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x69] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 2 : ++ j == 1 ? g5p_cu_meta() : ++ j == 2 ? endpoint : ++ 0); ++ } ++ rd->cu_target = (start + 1) & 255; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], rd->cu_target); ++ dma_wmb(); ++ if (g5p_ru_v3_verbose(r->jobs, 0)) ++ dev_dbg(g->dev, ++ "compute publish slot=%u endpoint=%u ring=%u W=%08x sequence=%u kick=dm2\n", ++ r->slot[0], endpoint, start, g5p_cu_meta() + G5P_CU_W, ++ seq); ++ return g5p_cu_wake(g, seq, start, rd->cu_target); ++} ++ ++/* The CDM phase timed out with the firmware alive: lose the job and its context, block the ++ * channel behind it until the scheduler's drain (g5p_cu_drain()) sees it retire. RAM only. ++ */ ++static int g5p_cu_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = G5P_CU_ENGINE; ++ s->drain_ms = 45000; ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "compute lost seq=%u slot=%u phase_ms=%lld drain_ms=%u: context lost, device kept, channel blocked\n", ++ seq, r->slot[0], ++ g5p_rv3_us(v->ta.start_ns, v->ta.end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++static void g5p_cu_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ const struct g5p_rv3 *v = &job->rv3; ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ dev_dbg(g->dev, ++ "compute job=%llu seq=%u ret=%d slot=%u consumed_us=%lld done_us=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x lost=%u bind_failed=%u\n", ++ job->id, v->seq, ret, r->slot[0], ++ g5p_rv3_us(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_us(v->ta_pub_ns, v->ta.end_ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ v->lost, v->bind_failed); ++} ++ ++/* One compute job, core lock held (g5p_render_run() via g5p_rv3_execute()). Returns 0, -EAGAIN ++ * (no epoch: nothing touched), -ETIMEDOUT, or a fault. ++ */ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (r->stage_done) ++ return -ECANCELED; ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ why = g5p_cu_admit(READ_ONCE(rd->up), rd->cu_sequence); ++ if (why) ++ return why == G5P_CU_NO_EPOCH ? -EAGAIN : -EOVERFLOW; ++ seq = rd->cu_sequence + 1; ++ if (!r->bound) { ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ } ++ /* The firmware owns the CL ring and the channel from here on: any failure quarantines. */ ++ v->touched = true; ++ g5p_cu_job(g5p_cu_page(b), g5p_cu_meta(), g5p_cu_global_meta(b), ++ r->slot[0], seq, &job->cu); ++ g5p_render_clean_job(g); ++ rd->cu_sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ ret = g5p_cu_publish(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, v->job_deadline_ns); ++ res = g5p_cu_wait(d, g, seq, rd->cu_target, &v->ta, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ /* The TA/3D paths' MMU fault-bank read, for the CDM (engine 2). */ ++ g5p_rv3_fault_read(d, r, seq, G5P_CU_ENGINE, res, v->ta.silent); ++ loss = g5p_rv3_loss(d, r, false, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && !g5p_cu_lose(d, r, v, seq)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, false)) { ++ g5p_rv3_lose_recover(d, r, v, seq, G5P_CU_ENGINE, ++ v->ta.silent ? "cdm-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "cdm-timeout" : ++ "cdm-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal(g, ++ res == G5P_RU_WAIT_TIMEOUT ? ++ G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "cdm-timeout" : ++ "cdm-fault"); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ if (!v->last[0].idle_ok) ++ dev_warn( ++ g->dev, ++ "compute retired seq=%u with idle=%08x,%08x,%08x set\n", ++ seq, v->last[0].idle[0], v->last[0].idle[1], ++ v->last[0].idle[2]); ++ v->phase = G5P_RV3_DONE; ++ ret = 0; ++out: ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), "cdm"); ++ g5p_cu_receipt(g, r, job, ret); ++ return ret; ++} ++ ++/* The scheduler's drain for a lost compute job (g5p_rv3_drain() hands engine 2 here): wait, RAM ++ * only and for the drain bound, for the stuck job to retire. It did: the channel is clear and ++ * the held jobs run. It did not (silent, fault, bound): the device recovers if it may, else the ++ * epoch ends. Returns false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_cu_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ struct g5p_ru_wait w = {}; ++ u32 res = G5P_RU_WAIT_FAULT; ++ int err = -EIO; ++ u64 now; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ now = ktime_get_ns(); ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (!READ_ONCE(b->failed) && !READ_ONCE(b->stopped)) ++ res = g5p_cu_wait(d, g, s->seq, rd->cu_target, &w, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res == G5P_RU_WAIT_COMPLETE) { ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "compute drained seq=%u slot=%u lost_ms=%lld: channel clear, held jobs resume\n", ++ s->seq, r->slot[0], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "compute drain dead seq=%u why=%s err=%d lost_ms=%lld: the device recovers if it may, else the epoch ends\n", ++ s->seq, ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal(g, ++ fault ? G5P_RU_TERM_FAULT : ++ G5P_RU_TERM_TIMEOUT, ++ fault ? "cdm-drain-fault" : ++ "cdm-drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++static void g5p_cu_pass_from_uapi(struct g5p_cu_pass *p, ++ const struct drm_g5p_compute_pass *u) ++{ ++ *p = (struct g5p_cu_pass){ ++ .cdm_va = u->cdm_va, ++ .cdm_size = u->cdm_size, ++ .cdm_end = u->cdm_end, ++ .tail = u->state_tail_va, ++ .config = { u->config[0], u->config[1] }, ++ .store = { u->store.code, u->store.data, u->store.control }, ++ .resume = { u->resume.code, u->resume.data, u->resume.control }, ++ .term = { u->terminate.code, u->terminate.data, ++ u->terminate.control }, ++ }; ++ for (u32 i = 0; i < 4; i++) ++ p->state[i] = u->state_va[i]; ++} ++ ++static bool g5p_cu_uapi_clean(const struct drm_g5p_compute_pass *u) ++{ ++ return u->size == sizeof(*u) && !u->flags && !u->store.pad && ++ !u->resume.pad && !u->terminate.pad && ++ !memchr_inv(u->reserved, 0, sizeof(u->reserved)); ++} ++ ++/* Acceptance, as g5p_render_submit_admitted() for TYPED_V3 minus the render target: every error ++ * precedes the fence replacement and the queue. ++ */ ++static int g5p_cu_submit_admitted(struct drm_device *drm, struct drm_file *file, ++ const struct drm_g5p_submit_compute_typed *a, ++ u64 *job_id) ++{ ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_compute_pass *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_cu_pass pass; ++ struct g5p_cu_loc loc = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ u8 tail[4]; ++ u64 point = 0; ++ int ret; ++ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(a->resources), ++ a->resource_count, sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(a->syncs), a->sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ upass = memdup_user(u64_to_user_ptr(a->pass), sizeof(*upass)); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (!g5p_cu_uapi_clean(upass)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_cu_pass_from_uapi(&pass, upass); ++ ranges = kvcalloc(a->resource_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(a->resource_count, sizeof(*job->objs), ++ GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, a->sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ /* Compute rides an epoch a render job brought up; a recovery in progress is held. */ ++ if (!READ_ONCE(d->rdev->recovering) && ++ g5p_cu_admit(READ_ONCE(d->rdev->up), ++ READ_ONCE(d->rdev->cu_sequence))) { ++ ret = READ_ONCE(d->rdev->up) ? -EOVERFLOW : -EAGAIN; ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, a->resource_count, ranges, ++ job->objs, &job->obj_count); ++ if (ret) ++ goto unlock; ++ /* Structure first; the stream's last dword is read only once its range is proven backed. */ ++ ret = g5p_cu_pass_check(&pass, &lim, ranges, a->resource_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass.cdm_va + pass.cdm_end - 4, tail, ++ sizeof(tail)); ++ if (!ret) ++ ret = g5p_cu_content(tail); ++ if (ret) ++ goto unlock; ++ g5p_cu_regs_of(&job->cu, &pass); ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = true; ++ job->compute = true; ++ r->v3_accepted = true; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++static_assert(sizeof(struct drm_g5p_compute_pds) == 16); ++static_assert(sizeof(struct drm_g5p_compute_pass) == 152); ++static_assert(offsetof(struct drm_g5p_compute_pass, state_va) == 24); ++static_assert(offsetof(struct drm_g5p_compute_pass, config) == 64); ++static_assert(offsetof(struct drm_g5p_compute_pass, store) == 72); ++static_assert(offsetof(struct drm_g5p_compute_pass, reserved) == 120); ++static_assert(sizeof(struct drm_g5p_submit_compute_typed) == 72); ++static_assert(DRM_G5P_CAP_TYPED_COMPUTE == BIT_ULL(30)); ++static_assert(DRM_G5P_COMPUTE_MAX_STREAM_BYTES == G5P_CU_MAX_STREAM); ++static_assert(DRM_G5P_COMPUTE_STREAM_TERMINATE == G5P_CU_TERMINATE); ++ ++static int g5p_submit_compute_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_compute_typed *a = data; ++ u64 id = 0; ++ int ret; ++ ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_compute_pass)) ++ return -EINVAL; ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_cu_submit_admitted(drm, file, a, &id); ++ g5p_r8_exit(); ++ a->job_id = id; ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +new file mode 100644 +index 000000000..018e83a5b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -0,0 +1,293 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: runtime A/B knobs for the V3 render path, all default off and all writable ++ * through /sys/module/a9_gpu_drm/parameters on a live boot. Included from drm-render.inc ++ * before drm-render-v3.inc. The executor is serial and holds the core lock, so the state ++ * below needs no lock of its own. ++ * ++ * fw_tick_mask (boot.inc) ctl+0x04, re-applied before every job (RAM) ++ * slc_evict_us while a job is in flight, one pass over slc_evict_mib MiB of ++ * cacheable kernel memory at most every N us (the waiter's ++ * 1-2 ms sample cadence bounds it from below); slc_evict_ops: ++ * 1 read pass, 2 write pass, 4 Normal-NC read of the firmware's ++ * root+shared pages, 8 TLBI VMALLE1IS, 16 DSB SY + SEV ++ * cpu_sync 1 dma_sync every job BO (device before, CPU after), 2 clean ++ * root+shared to PoC before the publish, 4 clean+invalidate the ++ * polled pages before every sample ++ * lat123_log one RENDER-L123 line per job (times in ms from the submit ioctl) ++ * pstate_probe 1 PMGR gfx word at job start/end (always-on block), 2 also one ++ * read of Fender +0x1000 at the first sample that shows a fresh ++ * engine busy rise (engine running, so the block is powered) ++ * ++ * No other register access. Temporary: none of this is the fix. ++ */ ++ ++#include ++#include ++ ++#define G5P_L123_OP_READ BIT(0) ++#define G5P_L123_OP_WRITE BIT(1) ++#define G5P_L123_OP_FW_READ BIT(2) ++#define G5P_L123_OP_TLBI BIT(3) ++#define G5P_L123_OP_SEV BIT(4) ++#define G5P_L123_SYNC_BO BIT(0) ++#define G5P_L123_SYNC_REGIONS BIT(1) ++#define G5P_L123_SYNC_POLL BIT(2) ++#define G5P_L123_LINE 64 ++ ++static struct { ++ void *buf; ++ size_t size; ++ u64 sink; ++ /* per job */ ++ u64 exec_ns, next_ns, evict_ns, ps_ns; ++ u32 us, mib, ops, sync, passes, tick, ps; ++ u32 pmgr[2]; ++ bool seen[2], idle[2], probed; ++ /* render_user_event_wake: pauses ended by an event, the timeout, a post-event poll, legacy */ ++ u32 ew[4]; ++ u64 ew_ns; ++} g5p_l123; ++ ++/* perf60-fillrate.md §11.3 cut 1 (render-user-evwait.h): the phase wait's pause between RAM samples. */ ++static const unsigned int render_user_event_wake = 2000; /* release8 default */ ++MODULE_PARM_DESC( ++ render_user_event_wake, ++ "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " ++ "until the next firmware event (the IRQ thread's decode wakes the waiter), at " ++ "most N us (100..20000) as the safety net, and poll every 100 us once the " ++ "phase's own completion event is latched; read live"); ++ ++/* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ ++static u8 *g5p_l123_word(struct g5p_boot *b, u32 slot, u32 field) ++{ ++ u32 off = b->layout.offset[slot] + field; ++ struct g5p_dma_page *page = b->layout.group[slot] ? b->shared : b->root; ++ ++ return page[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_l123_pages(struct g5p_dma_page *page, u32 count, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ for (u32 i = 0; i < count; i++) ++ if (page[i].cpu) ++ op((unsigned long)page[i].cpu, ++ (unsigned long)page[i].cpu + G5P_GPU_PAGE); ++} ++ ++static void g5p_l123_page_of(const void *p, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ unsigned long start = (unsigned long)p & ++ ~(unsigned long)(G5P_GPU_PAGE - 1); ++ ++ op(start, start + G5P_GPU_PAGE); ++} ++ ++/* cpu_sync 4: the pages the waiter samples (rings, TA/3D/PM pages, the idle words). */ ++static void g5p_l123_poll_pages(struct g5p_boot *b, u32 e) ++{ ++ g5p_l123_page_of(g5p_boot_shared(b, 0x60), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_boot_shared(b, 0x64), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 0), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 1), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_pm_cpu(b), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_global(b, e) + 0x724, dcache_clean_inval_poc); ++ dsb(sy); ++} ++ ++static void g5p_l123_bos(struct g5p_drm *d, struct g5p_rjob *job, bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct sg_table *sgt = ++ to_drm_gem_shmem_obj(job->objs[i].obj)->sgt; ++ ++ if (!sgt) ++ continue; ++ if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ } ++} ++ ++static u32 g5p_l123_pmgr(struct g5p_diag *g) ++{ ++ u32 raw = U32_MAX; ++ ++ if (!((0) & 1) || !g->pmgr || regmap_read(g->pmgr, 0x80340, &raw)) ++ return U32_MAX; ++ return raw; ++} ++ ++/* At the executor's entry: this job's knob values and counters. */ ++static void g5p_l123_job_reset(struct g5p_diag *g) ++{ ++ g5p_l123.exec_ns = ktime_get_ns(); ++ g5p_l123.next_ns = 0; ++ g5p_l123.evict_ns = 0; ++ g5p_l123.ps_ns = 0; ++ g5p_l123.passes = 0; ++ g5p_l123.ps = U32_MAX; ++ g5p_l123.probed = false; ++ memset(g5p_l123.ew, 0, sizeof(g5p_l123.ew)); ++ g5p_l123.ew_ns = 0; ++ g5p_l123.seen[0] = false; ++ g5p_l123.seen[1] = false; ++ g5p_l123.us = (0); ++ g5p_l123.mib = clamp((16), 1U, 64U); ++ g5p_l123.ops = (1); ++ g5p_l123.sync = (0); ++ g5p_l123.pmgr[0] = g5p_l123_pmgr(g); ++ g5p_l123.pmgr[1] = U32_MAX; ++} ++ ++/* After the job's records are written, before they are cleaned and published. May sleep. */ ++static void g5p_l123_job_prepare(struct g5p_drm *d, struct g5p_rjob *job) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u8 *tick = g5p_l123_word(b, G5P_L123_CTL, 4); ++ u32 mask = (0); ++ size_t size = (size_t)g5p_l123.mib << 20; ++ ++ if (mask <= G5P_L123_TICK_MASK && g5p_le32(tick) != mask) { ++ g5p_put32(tick, mask); ++ dma_wmb(); ++ } ++ g5p_l123.tick = g5p_le32(tick); ++ ++ if (!g5p_l123.us || ++ !(g5p_l123.ops & (G5P_L123_OP_READ | G5P_L123_OP_WRITE))) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = NULL; ++ g5p_l123.size = 0; ++ } else if (g5p_l123.size != size) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = vmalloc(size); ++ g5p_l123.size = g5p_l123.buf ? size : 0; ++ if (g5p_l123.buf) ++ memset(g5p_l123.buf, 0x5a, size); ++ } ++ ++ if (g5p_l123.sync & G5P_L123_SYNC_BO) ++ g5p_l123_bos(d, job, false); ++ if (g5p_l123.sync & G5P_L123_SYNC_REGIONS) { ++ g5p_l123_pages(b->root, ARRAY_SIZE(b->root), dcache_clean_poc); ++ g5p_l123_pages(b->shared, ARRAY_SIZE(b->shared), ++ dcache_clean_poc); ++ dsb(sy); ++ } ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(b, 0); ++} ++ ++/* One streaming pass and the other per-pass operations. */ ++static void g5p_l123_evict(struct g5p_boot *b) ++{ ++ u64 t0 = ktime_get_ns(), sum = 0; ++ u32 ops = g5p_l123.ops; ++ ++ if ((ops & G5P_L123_OP_READ) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ sum += READ_ONCE(p[i]); ++ } ++ if ((ops & G5P_L123_OP_WRITE) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ WRITE_ONCE(p[i], i); ++ } ++ if (ops & G5P_L123_OP_FW_READ) { ++ struct g5p_dma_page *regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; ++ regions[r][i].cpu && at < G5P_GPU_PAGE; ++ at += G5P_L123_LINE) ++ sum += READ_ONCE(*( ++ const u32 *)(regions[r][i].cpu + ++ at)); ++ } ++ if (ops & G5P_L123_OP_TLBI) ++ flush_tlb_all(); ++ if (ops & G5P_L123_OP_SEV) { ++ dsb(sy); ++ sev(); ++ } ++ WRITE_ONCE(g5p_l123.sink, g5p_l123.sink + sum); ++ g5p_l123.passes++; ++ g5p_l123.evict_ns += ktime_get_ns() - t0; ++} ++ ++/* Replaces the waiter's sleep between two RAM samples of engine e. seen: d->event_serial as ++ * read after the drain and before this sample (render-user-evwait.h). ++ */ ++static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, ++ const struct g5p_rv3_ev *s, u64 seen) ++{ ++ u64 now = s->ns, t0; ++ enum g5p_ew_act act; ++ u32 wait_us; ++ ++ if (((0) & 2) && !g5p_l123.probed && g5p_l123.seen[e] && ++ g5p_l123.idle[e] && !s->idle_ok && !s->failed && !s->proto && ++ !g5p_ru_term_failed(&g5p_rv3_term) && !READ_ONCE(g->boot->failed)) { ++ struct g5p_mmio_ticket t; ++ ++ /* an R1-only ticket of the register licence (mmio.inc), never a bare read */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ g5p_l123.ps = t.gpc; ++ g5p_l123.ps_ns = ktime_get_ns(); ++ g5p_l123.probed = true; ++ } ++ g5p_l123.seen[e] = true; ++ g5p_l123.idle[e] = s->idle_ok; ++ if (g5p_l123.us && now >= g5p_l123.next_ns) { ++ g5p_l123_evict(g->boot); ++ g5p_l123.next_ns = now + (u64)g5p_l123.us * NSEC_PER_USEC; ++ } ++ act = g5p_ew_plan(READ_ONCE(render_user_event_wake), s->event, ++ &wait_us); ++ t0 = ktime_get_ns(); ++ if (act == G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ g5p_l123.ew[3]++; ++ } else if (act == G5P_EW_SHORT) { ++ usleep_range(wait_us, wait_us + wait_us / 2); ++ g5p_l123.ew[2]++; ++ } else if (wait_event_hrtimeout( ++ d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us))) { ++ g5p_l123.ew[1]++; /* -ETIME: the safety net */ ++ } else { ++ g5p_l123.ew[0]++; ++ } ++ g5p_l123.ew_ns += ktime_get_ns() - t0; ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(g->boot, e); ++} ++ ++/* After the receipt. One line per job while lat123_log is set. */ ++static void g5p_l123_job_end(struct g5p_drm *d, struct g5p_rjob *job, u32 seq, ++ int ret) ++{ ++ struct g5p_diag *g = d->core; ++ ++ if (!ret && (g5p_l123.sync & G5P_L123_SYNC_BO)) ++ g5p_l123_bos(d, job, true); ++ g5p_l123.pmgr[1] = g5p_l123_pmgr(g); ++ { ++ return; ++ } ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-prime.inc b/drivers/gpu/drm/apple-g5p/drm-prime.inc +new file mode 100644 +index 000000000..bd513b969 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-prime.inc +@@ -0,0 +1,114 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* PRIME for shmem GEM BOs. ++ * ++ * Export: Mesa WSI/client buffers and kmsro bookkeeping. The dma-buf adds ++ * begin/end_cpu_access, so an importer that CPU-reads (simpledrm's blit, ++ * llvmpipe) gets cache maintenance on the G5P DMA mapping. vmap/mmap reuse ++ * the shmem helpers (write-combined when the device is not DMA-coherent). ++ * ++ * Import: foreign dma-bufs, e.g. simpledrm dumb buffers that the GPU renders ++ * into for scanout. The attachment stays mapped (pinned) for the life of the ++ * GEM object; VM_BIND maps its sg_table. Render jobs listing an imported BO ++ * get whole-buffer cache maintenance and dma_resv fences (drm-render.inc). ++ */ ++ ++static int g5p_dmabuf_begin_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ /* No device mapping yet means no device-owned cache lines either. */ ++ if (sgt) ++ dma_sync_sgtable_for_cpu(obj->dev->dev, sgt, DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static int g5p_dmabuf_end_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ if (sgt) ++ dma_sync_sgtable_for_device(obj->dev->dev, sgt, ++ DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static const struct dma_buf_ops g5p_dmabuf_ops = { ++ .attach = drm_gem_map_attach, ++ .detach = drm_gem_map_detach, ++ .map_dma_buf = drm_gem_map_dma_buf, ++ .unmap_dma_buf = drm_gem_unmap_dma_buf, ++ .release = drm_gem_dmabuf_release, ++ .mmap = drm_gem_dmabuf_mmap, ++ .vmap = drm_gem_dmabuf_vmap, ++ .vunmap = drm_gem_dmabuf_vunmap, ++ .begin_cpu_access = g5p_dmabuf_begin_cpu_access, ++ .end_cpu_access = g5p_dmabuf_end_cpu_access, ++}; ++ ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags) ++{ ++ struct dma_buf_export_info info = { ++ .exp_name = KBUILD_MODNAME, ++ .owner = obj->dev->driver->fops->owner, ++ .ops = &g5p_dmabuf_ops, ++ .size = obj->size, ++ .flags = flags, ++ .priv = obj, ++ .resv = obj->resv, ++ }; ++ ++ return drm_gem_dmabuf_export(obj->dev, &info); ++} ++ ++static int g5p_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file, ++ u32 handle, u32 flags, int *fd) ++{ ++ return drm_gem_prime_handle_to_fd(dev, file, handle, flags, fd); ++} ++ ++static int g5p_prime_fd_to_handle(struct drm_device *dev, struct drm_file *file, ++ int fd, u32 *handle) ++{ ++ return drm_gem_prime_fd_to_handle(dev, file, fd, handle); ++} ++ ++/* Our own export comes back as the same GEM object (the core's identity check ++ * only knows its default ops); anything else is attached and mapped. ++ */ ++static struct drm_gem_object *g5p_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *buf) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ ++ if (buf->ops == &g5p_dmabuf_ops && obj->dev == dev) { ++ drm_gem_object_get(obj); ++ return obj; ++ } ++ return drm_gem_prime_import(dev, buf); ++} ++ ++/* Foreign buffers are GPU READ|WRITE data, never executable, and must be ++ * whole 16KiB GPU pages within the BO size limit. ++ */ ++static struct drm_gem_object * ++g5p_prime_import_sg_table(struct drm_device *dev, ++ struct dma_buf_attachment *attach, ++ struct sg_table *sgt) ++{ ++ struct drm_gem_object *obj; ++ ++ if (attach->dmabuf->size & (G5P_GPU_PAGE - 1) || ++ attach->dmabuf->size > G5P_MAX_BO_SIZE) ++ return ERR_PTR(-EINVAL); ++ obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); ++ if (!IS_ERR(obj)) { ++ to_g5p_bo(obj)->roles = DRM_G5P_BO_READ | DRM_G5P_BO_WRITE; ++ g5p_bo_track(obj); ++ } ++ return obj; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +new file mode 100644 +index 000000000..21dc64d89 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -0,0 +1,203 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * The render scheduler (docs/gpu/multi-client-design.md "Stage 2/3 as built"): one worker on ++ * the ordered render_wq serves the contexts round-robin (core/render-user-sched.h). Jobs wait ++ * for their inputs on dma-fence callbacks, not in the worker, so one client's unsignalled ++ * fence never holds up another client. Each picked job runs to completion on the one shared ++ * channel (one credit), exactly as a work item did before; a context's close item runs after ++ * its last job. Included from drm-render.inc after drm-render-v3.inc. ++ * ++ * The shape is drm_sched's (entity FIFO, run-list RR, dependency callbacks, entity teardown ++ * after the last job), so stage 4 can replace it with drm_sched without a UAPI change. ++ */ ++ ++static struct dma_fence *g5p_rjob_dep(struct g5p_rjob *job, u32 i) ++{ ++ return i < job->input_count ? job->inputs[i] : ++ job->deps[i - job->input_count]; ++} ++ ++/* A signalled input: its error, or that of any fence it wraps (array, chain), else 0. */ ++static int g5p_render_fence_status(struct dma_fence *fence) ++{ ++ struct dma_fence_unwrap cursor; ++ struct dma_fence *part; ++ u32 unwrapped = 0; ++ int ret = dma_fence_get_status(fence); ++ ++ if (ret < 0) ++ return ret; ++ dma_fence_unwrap_for_each(part, &cursor, fence) { ++ ret = dma_fence_get_status(part); ++ if (ret <= 0 || ++unwrapped > 4096) { ++ dma_fence_put(cursor.chain); ++ return ret < 0 ? ret : -ETIMEDOUT; ++ } ++ } ++ return 0; ++} ++ ++/* Fence callback (any context, the fence lock held): the head job's input signalled. */ ++static void g5p_rjob_dep_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++{ ++ struct g5p_rjob *job = container_of(cb, struct g5p_rjob, cb); ++ ++ WRITE_ONCE(job->cb_fired, true); ++ mod_delayed_work(job->gpu->render_wq, &job->gpu->rdev->sched_work, 0); ++} ++ ++/* Scheduler lock held: no callback of this job stays armed. */ ++static void g5p_rjob_disarm(struct g5p_rjob *job) ++{ ++ if (job->cb_fence) ++ dma_fence_remove_callback(job->cb_fence, &job->cb); ++ job->cb_fence = NULL; ++} ++ ++/* ++ * Scheduler lock held: may the head job of its context run now? Walks its explicit inputs, ++ * then its implicit dependencies, in order; at the first unsignalled one it arms a callback ++ * and reports the job's input deadline. A failed input, the deadline (G5P_RU_INPUT_MS after ++ * the submit, as the old in-worker wait) and closed admission make it ready with an error, so ++ * that it fails before the hardware and its fence still signals in order. ++ */ ++static bool g5p_rsched_ready(struct g5p_rs_job *node, void *cookie, ++ unsigned long *deadline) ++{ ++ struct g5p_rjob *job = container_of(node, struct g5p_rjob, node); ++ ++ if (job->setup_error || job->dep_error) ++ return true; ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ESHUTDOWN; ++ return true; ++ } ++ if (job->cb_fence) { ++ if (!READ_ONCE(job->cb_fired)) ++ goto pending; ++ job->cb_fence = NULL; ++ } ++ while (job->dep_next < job->input_count + job->dep_count) { ++ struct dma_fence *fence = g5p_rjob_dep(job, job->dep_next); ++ ++ if (dma_fence_is_signaled(fence)) { ++ job->dep_error = g5p_render_fence_status(fence); ++ if (job->dep_error) ++ return true; ++ job->dep_next++; ++ continue; ++ } ++ WRITE_ONCE(job->cb_fired, false); ++ if (dma_fence_add_callback(fence, &job->cb, g5p_rjob_dep_cb)) ++ continue; /* -ENOENT: signalled meanwhile */ ++ job->cb_fence = fence; ++ goto pending; ++ } ++ return true; ++pending: ++ if (time_after_eq(jiffies, job->deadline)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ETIMEDOUT; ++ return true; ++ } ++ *deadline = job->deadline; ++ return false; ++} ++ ++static void g5p_rsched_kick(struct g5p_drm *d) ++{ ++ mod_delayed_work(d->render_wq, &d->rdev->sched_work, 0); ++} ++ ++/* Submit, render_submit_lock held: an accepted job joins its context's FIFO. */ ++static void g5p_rsched_push(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ g5p_rs_push(&d->rdev->sched, &r->entity, &job->node); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ g5p_rsched_kick(d); ++} ++ ++/* The close item joins the context's entity behind its jobs; false if one is queued. */ ++static bool g5p_rsched_close(struct g5p_drm *d, struct g5p_render *r) ++{ ++ unsigned long flags; ++ bool queued; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ queued = g5p_rs_close(&d->rdev->sched, &r->entity); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ if (queued) ++ g5p_rsched_kick(d); ++ return queued; ++} ++ ++static void g5p_render_run(struct g5p_rjob *job); ++static void g5p_render_close_run(struct g5p_render *r); ++static void g5p_render_close_deferred(struct g5p_drm *d); ++ ++/* ++ * The worker: pick, run to completion, repeat; then sleep until a callback, a submit or the ++ * earliest input deadline wakes it. render_wq is ordered, so this never runs twice at once, ++ * and it is the only writer of the device's job state, as each job's work item was. ++ */ ++static void g5p_rsched_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = container_of(to_delayed_work(work), ++ struct g5p_rdev, sched_work); ++ struct g5p_drm *d = rd->drm; ++ ++ /* Each item drops the references it held; the device must outlive this loop. */ ++ drm_dev_get(&d->drm); ++ for (;;) { ++ struct g5p_rs_entity *e; ++ struct g5p_rs_job *node; ++ struct g5p_render *r; ++ struct g5p_context *c; ++ unsigned long flags, wake = 0; ++ bool has_wake; ++ int what; ++ ++ /* A lost job blocks the channel: hold every job until the drain is decided, and ++ * until a recovery the drain or the executor asked for has run. ++ */ ++ if (READ_ONCE(rd->stuck) || READ_ONCE(rd->recovering)) { ++ if (rd->stuck ? g5p_rv3_drain(d) : g5p_rv3_recover(d)) ++ continue; ++ queue_delayed_work(d->render_wq, &rd->sched_work, ++ msecs_to_jiffies(100)); ++ break; ++ } ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ what = g5p_rs_pick(&rd->sched, g5p_rsched_ready, d, &e, &node, ++ &wake, &has_wake); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ if (what == G5P_RS_IDLE) { ++ if (has_wake) ++ queue_delayed_work( ++ d->render_wq, &rd->sched_work, ++ max_t(long, (long)(wake - jiffies), 1)); ++ break; ++ } ++ if (what == G5P_RS_JOB) { ++ g5p_render_run( ++ container_of(node, struct g5p_rjob, node)); ++ g5p_render_close_deferred(d); ++ continue; ++ } ++ r = container_of(e, struct g5p_render, entity); ++ c = r->ctx; ++ g5p_render_close_run(r); ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ g5p_rs_closed(&rd->sched, e); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ /* The close item's own reference (g5p_render_close()); r may go with it. */ ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ drm_dev_put(&d->drm); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +new file mode 100644 +index 000000000..36550c443 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -0,0 +1,488 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Unbinding firmware slots (docs/gpu/multi-client-design.md §4.5): a closed context's pair ++ * (stage 2, core/render-user-close.h) and the render frontend's share of the reboot quiesce ++ * (core/render-user-quiesce.h, docs/gpu/experiments/mcfold2-kexec.md). ++ * Included from drm-render.inc after drm-render-v3.inc. ++ */ ++ ++/* The RAM words of `slot` and of its unbind record at ring entry `index`. */ ++static void g5p_render_slot_mem(struct g5p_boot *b, u32 slot, u32 index, ++ struct g5p_ru_slot_mem *m) ++{ ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + (index & 255) * 20 + j * 4; ++ ++ m->rec[j] = ++ b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ } ++ m->pa = g5p_boot_shared(b, 0x86) + slot * 8; ++ m->host = g5p_boot_shared(b, 0x84) + slot * 4; ++ m->cached = g5p_boot_shared(b, 0x85) + slot * 4; ++} ++ ++/* ++ * Unbind `n` firmware slots in one RAM-only DC3 batch: PA and host tag of each to zero, one ++ * record {3, slot} per slot, one producer store, then the RAM-only wait of a bind and a check ++ * that the firmware's DC3 handler copied the zero tag of every slot. No doorbell and no ++ * register access, so it is safe after a job retired and while the GPU may be idle-gated. ++ * ++ * The caller holds the core lock (the DC ring has one writer) and has retired every use of ++ * these slots: no job of theirs is queued on the firmware or in flight. On success the ++ * firmware holds, per slot, PA 0, host and cached tag 0 and CBASE 0, with the slot's PC level ++ * invalidated; the slot may be bound again by a later DC3. Its translations stay allocated: ++ * freeing them is the caller's step, after this returns 0. A batch that is not consumed in ++ * time keeps later binds and unbinds refused until the firmware catches up (dc_pending). ++ */ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_slot_sample s = {}; ++ struct g5p_ru_slot_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, cons, target, bad = 0, verified = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!g5p_ru_unbind_fits(n)) ++ return -EINVAL; ++ for (u32 i = 0; i < n; i++) ++ if (slots[i] < G5P_RU_SLOT_MIN || slots[i] > G5P_RU_SLOT_MAX) ++ return -EINVAL; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ cons = READ_ONCE(ctl[0]); ++ if (start > 255 || cons != start || ++ (rd->dc_pending && cons != rd->dc_wait)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s refused slots=%u dc=%u/%u pending=%u: ring busy; no " ++ "hardware touched end=v3\n", ++ why, n, cons, start, rd->dc_pending); ++ return -EBUSY; ++ } ++ rd->dc_pending = false; ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_unbind_publish(&m, slots[i]); ++ } ++ target = (start + n) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, why); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ for (u32 i = 0; i < n; i++) { ++ u32 one; ++ ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_slot_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ one = g5p_ru_unbind_check(&s, slots[i], target); ++ bad |= one; ++ verified += !one; ++ } ++ if (!ret && bad) ++ ret = -EUCLEAN; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s slots=%u verified=%u ret=%d consumed_us=%lld " ++ "dc=%u->%u/%u bad=%#x first=%u last=%u mode=ram-dc3 end=v3\n", ++ why, n, verified, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ s.cons, s.prod, bad, slots[0], slots[n - 1]); ++ return ret; ++} ++ ++/* ++ * render_user_close_park: bind a closed context's just-unbound pair to the park catalogue, one ++ * DC3 pair as a context's bind (g5p_render_vm_prepare()), checked the same way. The catalogue ++ * holds the firmware window's entry and the borrowed sink entry, both stable within an epoch, so ++ * rewriting them for a later park never clears what an earlier parked pair resolves through. ++ * Core lock held, between jobs (the close item), RAM only and no register: as the unbind. ++ */ ++static int g5p_rv3_park(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, target, entry, sink, bad = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!rd->park.cpu) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->park); ++ if (ret) ++ return ret; ++ memset(rd->park.cpu, 0, G5P_GPU_PAGE); ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (start > 255 || rd->dc_pending || g5p_ru_bind_busy(&s)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park refused slots=%u/%u dc=%u/%u pending=%u busy=%#x: the pair stays " ++ "unbound end=v3\n", ++ r->slot[0], r->slot[1], s.cons, start, rd->dc_pending, ++ g5p_ru_bind_busy(&s)); ++ return -EBUSY; ++ } ++ sink = b->sink_ready ? g5p_le32(b->catalogue.cpu) : 0; ++ if (g5p_le32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4) != entry) ++ g5p_put32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4, entry); ++ if (g5p_le32(rd->park.cpu) != sink) ++ g5p_put32(rd->park.cpu, sink); ++ g5p_table_clean_page(g, &rd->park); ++ dsb(sy); ++ g5p_ru_bind_publish(&m, r->slot, rd->park.dma, G5P_RU_PARK_TAG); ++ target = (start + 2) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "park"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, rd->park.dma, ++ G5P_RU_PARK_TAG, target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ rd->parks += !ret; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park slots=%u/%u ret=%d consumed_us=%lld dc=%u->%u/%u bad=%#x " ++ "parks=%u pc=%pad end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ start, s.cons, s.prod, bad, rd->parks, &rd->park.dma); ++ return ret; ++} ++ ++/* Drop what the claim of the device took (g5p_render_device_claim()); the device stays ++ * claimed for render. Returns whether a context reference was dropped. ++ */ ++static bool g5p_render_unclaim(struct g5p_drm *d, struct g5p_render *r) ++{ ++ if (!r->claimant) ++ return false; ++ r->claimant = false; ++ if (d->quarantined_context == r->ctx) ++ d->quarantined_context = NULL; ++ return true; ++} ++ ++/* ++ * The close item of a TYPED_V3 context (multi-client-design.md "Stage 2/3 as built"): the ++ * scheduler runs it after every job of the context, so every use of the pair has retired or ++ * failed. RAM only: the unbind is g5p_rv3_unbind_slots() for this pair under the core lock. ++ * After a verified unbind the binding's references go, and the last one frees the VM tables, ++ * render targets and mappings (unpinning the BOs) and returns the slots ++ * (g5p_render_release()). Anything the firmware may still hold is retained until reboot. ++ */ ++/* perf60 (docs/gpu/experiments/perf60-clock.md), corrected by the release8 bench (bench-gaps3, ++ * render-user-pmset.h): the PM writes a previous PM context back (its free list F, through that ++ * context's TA slot) not once at the switch but on the later jobs of the next context, until a ++ * third context runs. A closed context therefore keeps its slots bound and its set and pages alive ++ * while it is the last job's context or the previous distinct one; its unbind and release run once ++ * it is neither. At most those two wait. (The first version held only the last job's context and ++ * released it after one other job; 2 held one job longer, by job records: both released it while ++ * the PM still wrote to it.) RAM only: bookkeeping and the close item's references. ++ */ ++/* release8 default */ ++ ++/* ++ * A close item follows every job of this context on the ordered render worker. ++ * resident_gone, set under the file lock before queueing it, rejects all later ++ * VM_BIND and context creation. Thus the close item owns vm->maps exclusively. ++ * ++ * Idle VM_BIND UNMAP already drops a user mapping's pages after retirement. ++ * Do that here without releasing the PM's write-back route: no unbind, no ++ * table pruning, no PM set/pool/aux/RT release, no pm_last/pm_prev change. The ++ * PM free list is in the firmware window, never in vm->maps. All user mappings ++ * must pass the same range check as VM_BIND before any PTE is touched. ++ * ++ * No new kick of this detached context can occur. Clean the cleared user PTEs ++ * to PoC, but do not publish a tag, invalidate a slot or access R0. The PM can ++ * keep using its unchanged translations and pages, including cached ones. ++ * Failed/queued/terminal/shutdown epochs keep every BO, as before. ++ */ ++static void g5p_render_close_trim(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_boot *b; ++ struct g5p_ru_close_in in; ++ struct g5p_mapping *m, *tmp; ++ struct g5p_pt *pt; ++ u64 pinned; ++ u32 maps, why; ++ ++ if (!vm->map_count || !vm->resident_gone || !r->closing || ++ !r->detached || !mutex_trylock(&g->lock)) ++ return; ++ b = g->boot; ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* Match close_run: a successfully drained lost job has no user-memory use left. */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .dc_pending = d->rdev->dc_pending, ++ .queued = atomic_read(&r->queued), ++ }; ++ if (g5p_ru_close_plan(&in, &why) != G5P_RU_CLOSE_UNBIND) ++ goto out; ++ list_for_each_entry(m, &vm->maps, link) ++ if (!g5p_user_range_valid(m->node.start, m->node.size) || ++ (m->node.start < G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE && ++ G5P_RU_PRIVATE_VA < m->node.start + m->node.size)) ++ goto out; ++ ++ pinned = vm->pinned_bytes; ++ maps = vm->map_count; ++ list_for_each_entry(m, &vm->maps, link) ++ g5p_vm_clear(vm, d->drm.dev, m->node.start, m->node.size); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ list_for_each_entry_safe(m, tmp, &vm->maps, link) { ++ vm->mapped_bytes -= m->node.size; ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-trim slots=%u/%u maps=%u unpinned=%lluK remaining=%lluK " ++ "device_pinned=%lluK pm=retained end=v3\n", ++ r->slot[0], r->slot[1], maps, (pinned - vm->pinned_bytes) >> 10, ++ vm->pinned_bytes >> 10, READ_ONCE(d->pinned_bytes) >> 10); ++out: ++ mutex_unlock(&g->lock); ++} ++ ++/* render_user_close_defer: may the firmware's PM still write this context's state back? */ ++static bool g5p_render_pm_holds(struct g5p_rdev *rd, struct g5p_render *r, ++ u32 mode) ++{ ++ return mode && g5p_pmset_holds(rd->pm_last, rd->pm_prev, r); ++} ++ ++static void g5p_render_close_run(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_close_in in; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 puts = 0, why; ++ bool locked = false; ++ int plan, ret = 0; ++ u32 mode, k; ++ ++ mode = (1); ++ k = d->rdev->close_deferred[0] ? 1 : 0; ++ if (mode && !r->close_defer_run && r->bound && ++ g5p_render_pm_holds(d->rdev, r, mode) && ++ k < ARRAY_SIZE(d->rdev->close_deferred)) { ++ g5p_render_close_trim(r); ++ /* the deferred close's own references, dropped by g5p_render_close_deferred() */ ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->rdev->close_deferred[k] = r; ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer slots=%u/%u jobs=%u: the PM may still write its state " ++ "back; unbind once it has moved past it end=v3\n", ++ r->slot[0], r->slot[1], r->jobs); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer2 slots=%u/%u pm=%s pending=%u end=v3\n", ++ r->slot[0], r->slot[1], ++ d->rdev->pm_last == r ? "last" : "previous", k + 1); ++ return; ++ } ++ ++ for (;;) { ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* a lost job that drained left the firmware nothing of this context */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ /* as g5p_rv3_unbind_slots() refuses it */ ++ .dc_pending = b && d->rdev->dc_pending && ++ READ_ONCE(((u32 *)g5p_boot_shared( ++ b, 0x6c))[0]) != d->rdev->dc_wait, ++ .queued = atomic_read(&r->queued), ++ }; ++ plan = g5p_ru_close_plan(&in, &why); ++ if (plan != G5P_RU_CLOSE_UNBIND || locked) ++ break; ++ /* Bounded, as a job takes it: never wait forever behind a command in MMIO. */ ++ while (!(locked = mutex_trylock(&g->lock)) && ++ time_before(jiffies, deadline)) ++ msleep(5); ++ if (!locked) { ++ plan = G5P_RU_CLOSE_RETAIN; ++ why = G5P_RU_CLOSE_LOCK; ++ break; ++ } ++ /* Plan again under the lock: the reboot quiesce or a failure may have run. */ ++ } ++ if (plan == G5P_RU_CLOSE_UNBIND) { ++ ret = g5p_rv3_unbind_slots(d, r->slot, 2, "close"); ++ if (!ret) { ++ r->bound = false; ++ r->retained = false; ++ WRITE_ONCE(c->live, false); ++ puts++; ++ /* the PM may still write this context's free list back through its TA slot */ ++ { ++ g5p_rv3_park(d, r); ++ }; ++ } ++ } ++ if (plan == G5P_RU_CLOSE_RELEASE || ++ (plan == G5P_RU_CLOSE_UNBIND && !ret)) ++ puts += g5p_render_unclaim(d, r); ++ if (locked) ++ mutex_unlock(&g->lock); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close slots=%u/%u plan=%s why=%#x ret=%d jobs=%u claimant=%u " ++ "retained=%u end=v3\n", ++ r->slot[0], r->slot[1], g5p_ru_close_name(plan), why, ret, ++ r->jobs, r->claimant, r->retained); ++ while (puts--) { ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++} ++ ++/* After a job: a deferred close runs once the last job was another context's. */ ++static void g5p_render_close_deferred(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ u32 mode = (1); ++ ++ for (u32 k = 0; k < ARRAY_SIZE(rd->close_deferred); k++) { ++ struct g5p_render *r = rd->close_deferred[k]; ++ struct g5p_context *c; ++ ++ /* mode 0 now: a pending close still waits until the PM has moved past it */ ++ if (!r || g5p_render_pm_holds(rd, r, mode ?: 1)) ++ continue; ++ rd->close_deferred[k] = NULL; ++ c = r->ctx; ++ r->close_defer_run = true; ++ dev_info( ++ d->core->dev, ++ "RENDER-V3 close-deferred slots=%u/%u: neither the PM's last nor its previous " ++ "context end=v3\n", ++ r->slot[0], r->slot[1]); ++ g5p_render_close_run(r); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ /* keep a pending close in [0]: close_run fills [1] only behind it */ ++ if (!rd->close_deferred[0] && rd->close_deferred[1]) { ++ rd->close_deferred[0] = rd->close_deferred[1]; ++ rd->close_deferred[1] = NULL; ++ } ++} ++ ++/* ++ * The handle of a TYPED_V3 context is gone (DESTROY_CONTEXT, or its file closed): stop VM and ++ * render-target edits and queue the close item behind the context's jobs (g5p_rsched_close()). ++ * File lock held, or the file is closing. A context that never submitted is released by its ++ * last reference. ++ */ ++static void g5p_render_close(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ ++ if (!r->v3_accepted || r->closing) ++ return; ++ r->closing = true; ++ r->detached = true; ++ c->vm->resident_gone = true; ++ /* The close item's references, dropped by the scheduler once it ran. */ ++ kref_get(&c->ref); ++ drm_dev_get(&c->gpu->drm); ++ if (!g5p_rsched_close(c->gpu, r)) { ++ g5p_context_put(c); ++ drm_dev_put(&c->gpu->drm); ++ } ++} ++ ++/* Reboot quiesce, RAM only and lock free: the TYPED_V3 latch and whether its executor runs. ++ * The executor is RAM-only once the latch is quiet, unless the wake110 experiment (doorbells ++ * after retirement, off in every desktop tuple) is configured. Called after admission closed; ++ * the scheduler is kicked so that jobs waiting on an input fail now, not at their deadline. ++ */ ++static void g5p_kq_drm_state(void *cookie, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ ++ /* Admission is closed by now: queued jobs blocked on an input fail at once. */ ++ g5p_rsched_kick(d); ++ ++ /* A channel blocked behind a lost job is a wedged epoch for the quiesce. */ ++ in->term_failed = g5p_ru_term_failed(&g5p_rv3_term) || ++ READ_ONCE(d->rdev->stuck) || ++ READ_ONCE(d->rdev->recovering); ++ in->ram_only = READ_ONCE(d->rdev->v3) && ++ !g5p_ru_mmio_ok(&g5p_rv3_term) && !(2); ++ in->executing = READ_ONCE(d->rdev->executing); ++} ++ ++/* Reboot quiesce, core lock held: every held user slot, the first claimant's pair included. ++ * Nothing is freed and no context changes state: admission is closed and the boot state is ++ * frozen right after, so no job runs again on this boot. ++ */ ++static int g5p_kq_drm_detach(void *cookie, const u32 *slots, u32 n) ++{ ++ return g5p_rv3_unbind_slots(cookie, slots, n, "kexec"); ++} ++ ++static const struct g5p_kq_frontend g5p_kq_drm = { ++ .state = g5p_kq_drm_state, ++ .detach = g5p_kq_drm_detach, ++}; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +new file mode 100644 +index 000000000..ac4dd841e +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_V3 serial executor: per-job state (receipt data). Types only; code: drm-render-v3.inc. */ ++#ifndef G5P_DRM_RENDER_V3_H ++#define G5P_DRM_RENDER_V3_H ++ ++enum g5p_rv3_phase { ++ G5P_RV3_IDLE, ++ G5P_RV3_ATTACH, ++ G5P_RV3_BIND, ++ G5P_RV3_TA_PUBLISH, ++ G5P_RV3_TA_WAIT, ++ G5P_RV3_TD_PUBLISH, ++ G5P_RV3_TD_WAIT, ++ G5P_RV3_DONE, ++ G5P_RV3_PHASES, ++}; ++ ++/* g5p_drm.td_want: the 3D completion-event latch is armed for the endpoint in the low byte. */ ++#define G5P_RV3_TD_WANT 0x100U ++ ++/* One RAM sample of one engine: ring, stamps, channel watermarks, idle words, PM state, and ++ * (3D) whether the matching completion event has been latched. ++ */ ++struct g5p_rv3_ev { ++ u64 ns; ++ u32 prod, cons, stamp[2], ctl[5], idle[6], pm80; ++ bool proto, failed, consumed, retired, idle_ok, pm_ok, event; ++}; ++ ++struct g5p_rv3 { ++ struct g5p_ru_wait ta, td; ++ struct g5p_ru_touch ++ touch[2]; /* RC4a A/B (render-user-touch.h); zero when off */ ++ struct g5p_ru_retire rt; /* the 3D predicate (render-user-retire.h) */ ++ struct g5p_ru_ta tr; /* the TA predicate (render-user-retire.h) */ ++ struct g5p_rv3_ev last[2]; ++ u64 td_event_ns; /* decode time of the matching 3D event; 0 = none */ ++ u64 ta_event_ns; /* decode time of the first TA event since the publish */ ++ u64 attach_ns, job_deadline_ns, ta_pub_ns, td_pub_ns, ta_consumed_ns, ++ td_consumed_ns; ++ u32 phase, grace_end, bell; ++ u32 seq; /* the device sequence this job got; 0 = none */ ++ /* wake110 (render-user-wake.h), per engine: the decision, the power evidence word it saw ++ * (wake111), the two register words it read (U32_MAX = not read), the consumer it saw, the ++ * type-1 event counts at the phase's arm and at the job's end, the lock window and the ++ * first type-1 decode time. Zero when off. ++ */ ++ u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], ++ wk_t1_pub[2], wk_t1_end[2]; ++ u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ ++ /* The bind of this job's context published its DC3 pair and then failed: that context is ++ * lost, the device and the contexts already bound are not. ++ */ ++ bool bind_failed; ++ /* The job was lost alone (render-user-loss.h): its context is dead, the device is not; the ++ * channel stays blocked behind its stuck phase until the scheduler's drain clears it. ++ */ ++ bool lost; ++}; ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +new file mode 100644 +index 000000000..1d0a0a76d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -0,0 +1,3038 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial executor: the control flow that completed the natural TA and the ++ * natural 3D in r96 (docs/gpu/experiments/production-typed-render-kernel.md), on typed ++ * state. Included from drm-render.inc after g5p_render_execute(). ++ * ++ * What it does, in order, for each job a V3 context runs (strictly serial across every context, ++ * render-user-serial.h; only the epoch's job 1 attaches and has hardware evidence): ++ * 0. bind, the first job of every later context (multi-client-design.md §4.2): its own DC3 ++ * pair, published in RAM only, no doorbell, a RAM-only wait for the firmware to consume ++ * it. With the TEMPORARY render_user_spare_pairs instead: a pair job 1 pre-bound, taken ++ * over through the tag path. ++ * 1. attach (job 1): the typed preparation (PM, channels, catalogue entries), then the ++ * DC3 pair is published and its doorbell is owned by the r74f machinery (r87b ++ * self-drain: a closed gate is never repaired by the host). The firmware installs ++ * the page-table root and the generation tags itself (r74g); the host does NO MMU pass ++ * and checks the result in RAM only. ++ * 2. TA: record, WORK_CTL and producer, no doorbell; the r74g grace (four ALIVEs or ++ * 20 ms) lets the firmware drain it, and only if it is still pending afterwards the ++ * first ALIVE decoded owns one fenced fallback doorbell. ++ * 3. TA wait: RAM only (drain, sample, sleep), 20 s per phase by default, 30 s for the ++ * whole job (render-user-wait.h decides the deadline and closes the race with a ++ * completion). tahang1: complete on a coherent target with the PM saved and either ++ * the idle words clear, the TA completion event, or a 500 ms coherent hold ++ * (render-user-retire.h); a phase pending 1 s prints one RAM-only `slow` line. From ++ * the TA complete on, and after any timeout or fault, the host touches no register. ++ * 4. 3D: record, WORK_CTL and producer, NO doorbell (r96 stage 1: the firmware consumed ++ * it 2 ms later), RAM-only wait as above. ++ * release (render_user_touch_ms, default 0 = off): at publish + T of each phase, odd jobs read ++ * the receipt pages once and even jobs only record the instant (render-user-touch.h); RAM only. ++ * repeat109: the 3D phase is also complete when its target state is coherent and its completion ++ * event arrived, even if the engine idle words never clear (render-user-retire.h; the latched ++ * render_user_idle_settle_ms, default 0, adds a minimum coherent run); one `RENDER-V3 retire` ++ * receipt line splits the firmware's retirement from the host's decision. RAM only. ++ * wake110 (render_user_wake, default 0 = off): right after the producer store of every phase but ++ * job 1's TA, the A/B's factor phases run Apple's submit kick (R1+0x14 = 7, GPC active, R0+0xb00 = ++ * 0x10 / 0x11) and its control phases the same guards without a register (render-user-wake.h); ++ * one `RENDER-V3 wake` line per job. The only register access after the TA complete, licensed ++ * while no timeout or fault was noted. ++ * Jobs >= 2 (a compositor's next frame, or another context's): no attach, the PM pool is ++ * rebuilt, and both rings are published with no doorbell at all. ++ * Not in this path on purpose: the r74h / r80 / r85-r96 root-cause instrumentation (stream ++ * markers, PM canary, plateau and end banks, region/target dumps), host MMU passes, cache ++ * maintenance MMIO, selector reads, snapshots (they read R0), and any recovery. A timeout ++ * or fault retains everything; the context is frozen and kept until reboot. ++ * ++ * Terminal state (render-user-terminal.h): the TA complete, a timeout and a fault go through one ++ * first-writer-wins latch, g5p_rv3_terminal(). From then on g5p_ru_mmio_ok() is false, the admission ++ * refuses every job that would start behind a failure (-EIO), and the legacy typed paths that read or ++ * write registers (snapshot, invalidate, run, destroy) refuse. g5p_rv3_td_hook() is the optional ++ * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. ++ */ ++ ++static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { ++ "idle", "attach", "bind", "ta-publish", ++ "ta-wait", "3d-publish", "3d-wait", "done", ++}; ++static const char *const g5p_rv3_results[] = { "pending", "complete", "timeout", ++ "fault" }; ++static const char *const g5p_rv3_how[] = { "none", "idle-clear", "idle-stuck", ++ "hold" }; ++ ++#include "drm-latency123.inc" ++ ++/* perf60 stage 4 (render-user-pmset.h): per-context PM records, the iOS model (one ParamBuffer ++ * per work queue, 6cb9afc/6cb2378). Latched at the epoch's first attach, with boot_pm_sets: ++ * 1 every context bound after the epoch's first gets its own PM set (control + S, list, F in ++ * the firmware GiB), its own pool pages and its own aux pages at G5P_RU_PRIVATE_VA in its ++ * VM; the epoch's first context keeps the device PM. PM ID 0 everywhere. ++ * 2 as 1, and set s carries PM ID s + 1 (TA W+0x10, TA start, 3D start, tag6). ++ * RAM only, by the executor between jobs. ++ */ ++/* release8 default */ ++ ++/* The PM a context's jobs use: its set, or the device PM. */ ++static u8 *g5p_rv3_pm_cpu(struct g5p_boot *b, const struct g5p_render *r) ++{ ++ if (r->pm_set >= 0) ++ return b->pm_set[r->pm_set * G5P_PMSET_PAGES + G5P_PMSET_CONTROL] ++ .cpu; ++ return g5p_ru_pm_cpu(b); ++} ++ ++static u32 g5p_rv3_pm_meta(const struct g5p_render *r) ++{ ++ return g5p_meta_address( ++ r->pm_set >= 0 ? g5p_pmset_va(r->pm_set, G5P_PMSET_CONTROL) : ++ G5P_SHARED_VA + G5P_R22_PM); ++} ++ ++static u32 g5p_rv3_pm_id(const struct g5p_rdev *rd, const struct g5p_render *r) ++{ ++ return rd->pm_ctx >= 2 && r->pm_set >= 0 ? g5p_pmset_id(r->pm_set) : 0; ++} ++ ++static struct g5p_dma_page *g5p_rv3_pm_aux(struct g5p_rdev *rd, ++ struct g5p_render *r) ++{ ++ return r->pm_aux_mapped ? r->pm_aux : rd->aux; ++} ++ ++/* The boot tuple the V3 path is proven under (r96 params.txt), minus the instrumentation: ++ * r73b/r73c/r73e/r73f, r74 with the raised IRQ cap, r74f, r74g, r87b self-drain; no r74h, ++ * no r80..r96 (their IRQ banks read registers), no r75/r79 firmware control, and no fixed ++ * render engine (it is exclusive with the DRM one). ++ */ ++static bool g5p_rv3_tuple(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ struct g5p_r73b *x = b ? b->r73b : NULL; ++ ++ return x && g5p_r74g_on(b) && g5p_r74b_ram_only(g) && ++ x->r87b_attach == G5P_R87B_ATTACH_SELF_DRAIN; ++} ++ ++/* The admission state of a context and the device, as plain values for g5p_ru_v3_admit(). */ ++static struct g5p_ru_v3_state g5p_rv3_state(struct g5p_diag *g, ++ struct g5p_render *r) ++{ ++ const struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ ++ return (struct g5p_ru_v3_state){ ++ .tuple = g5p_rv3_tuple(g), ++ .v3_accepted = r->v3_accepted, ++ .bound = r->bound, ++ .jobs = r->jobs, ++ .device_up = READ_ONCE(rd->up), ++ .device_v3 = READ_ONCE(rd->v3), ++ .sequence = READ_ONCE(rd->sequence), ++ .slot_ta = r->slot[0], ++ .slot_3d = r->slot[1], ++ .variant = 0 || r->after_ta || 0 || r->v.pm || r->v.memory || ++ r->v.rt || r->v.slots, ++ .wait_ms = r->wait_ms, ++ .quiet = !g5p_ru_mmio_ok(&g5p_rv3_term), ++ .terminal_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ }; ++} ++ ++/* A terminal epoch is a device state (-EIO), the others are about this request. */ ++static int g5p_rv3_refusal_errno(int why) ++{ ++ if (why == G5P_RU_V3_TERMINAL) ++ return -EIO; ++ return why == G5P_RU_V3_OVERFLOW ? -EOVERFLOW : -EOPNOTSUPP; ++} ++ ++/* Submit-time admission: every V3 job refused here has touched nothing. */ ++static int g5p_rv3_admit_submit(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_ru_v3_state st; ++ int why; ++ ++ /* The epoch is being replaced: the executor admits the job again under the lock. */ ++ if (READ_ONCE(d->rdev->recovering)) ++ return 0; ++ st = g5p_rv3_state(g, r); ++ why = g5p_ru_v3_admit(&st, false); ++ ++ if (!why) ++ return 0; ++ /* per submit while admission is closed and a client retries: ratelimited */ ++ dev_info_ratelimited( ++ g->dev, "RENDER-V3 refused: %s; no hardware touched end=v3\n", ++ g5p_ru_v3_why[why]); ++ return g5p_rv3_refusal_errno(why); ++} ++ ++/* Recovery hooks (patches/gpu-recovery), compiled only when that series is applied: core.inc then includes ++ * teardown.inc before this file, and teardown.h defines G5P_TEARDOWN_H. The CAPTURE entry is observation only ++ * (RAM reads, no register access) and runs only with the recovery module parameter set, once per epoch, on a ++ * timeout or fault. Without the series both are nothing, and the prod98 kernel does not depend on it. ++ */ ++#ifdef G5P_TEARDOWN_H ++static void g5p_rv3_td_hook(struct g5p_diag *g, int kind, const char *why) ++{ ++ g5p_td_note_terminal(&g5p_td_state, g5p_ru_td_kind(kind)); ++ if ((0) && g5p_ru_term_capture_due(&g5p_rv3_term, kind)) ++ g5p_td_capture(g, why); ++} ++#else ++static inline void g5p_rv3_td_hook(struct g5p_diag *g, int kind, ++ const char *why) ++{ ++} ++#endif ++ ++/* The terminal state of the epoch: the TA complete, and any timeout or fault. First writer wins; from the first ++ * note on g5p_ru_mmio_ok() is false and the host touches no register (R0, R1, AIC, doorbell) and no bank. The r80 ++ * flag is set when an r80 block exists, so that a bank that might be latched cannot run behind this path (the V3 ++ * tuple forbids r80, so there is none today); the latch above does not depend on it. ++ */ ++static void g5p_rv3_terminal(struct g5p_diag *g, int kind, const char *why) ++{ ++ bool failed = g5p_ru_term_failed(&g5p_rv3_term); ++ ++ if (g5p_ru_term_note(&g5p_rv3_term, kind)) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s mmio=off r0=off export=ram-only end=v3\n", ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)], ++ why); ++ else if (!failed && g5p_ru_term_failed(&g5p_rv3_term)) ++ /* The epoch's TA complete took the latch long ago (job 1): a later timeout or fault ++ * only makes it sticky-failed, which closes admission for every client. Say so once. ++ */ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s latched=%s failed=1 admission=closed end=v3\n", ++ g5p_ru_term_names ++ [kind > G5P_RU_TERM_NONE && ++ kind < G5P_RU_TERM_KINDS ? ++ kind : ++ G5P_RU_TERM_FAULT], ++ why, ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)]); ++ g5p_rv3_td_hook(g, kind, why); ++} ++ ++/* "Retained boundary" of the r74f machinery -> the errno a job reports. -EINPROGRESS is ++ * never a job error (the UAPI uses it for "pending"). ++ */ ++static int g5p_rv3_errno(struct g5p_diag *g, int ret) ++{ ++ if (ret != -EINPROGRESS) ++ return ret; ++ switch (READ_ONCE(g->boot->r73b->r74f_stop)) { ++ case G5P_R74F_STOP_ATTACH_NOT_KICKED: ++ case G5P_R74F_STOP_ATTACH_NOT_CONSUMED: ++ case G5P_R74F_STOP_TA_NOT_KICKED: ++ case G5P_R74G_STOP_TA_NOT_CONSUMED: ++ case G5P_R74G_STOP_TA_WINDOW_MISSED: ++ return -ETIMEDOUT; ++ default: ++ return -EIO; ++ } ++} ++ ++/* ++ * Interim: on iOS the firmware writes back its own lines with the cache maintenance it runs on ++ * every tick (meta+0x6d4); that needs init words still being recovered (latency123.md), and ++ * this read stands in for it until then. ++ * ++ * While a phase is in flight the firmware does not see its own progress in the root and ++ * shared pages until a coherent read of those lines reaches the fabric: without one, the TA ++ * and 3D stages wait for the firmware tick, for seconds (latency123: key to frame 7.9 s ++ * median; with a Normal-NC read of both regions every 1-2 ms, 0.27 s). One 32-bit load per ++ * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU ++ * has no sampler, so it costs nothing then. ++ */ ++static void g5p_rv3_fw_touch(struct g5p_boot *b) ++{ ++ struct g5p_dma_page *const regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE( ++ *(const u32 *)(regions[r][i].cpu + at)); ++ /* perf60 stage 4: the per-context PM sets are firmware pages like the shared ones */ ++ for (u32 i = 0; i < b->pm_sets * G5P_PMSET_PAGES; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(b->pm_set[i].cpu + at)); ++} ++ ++/* perf60 bell gates, A/B knobs for the multi-context 3D fault on the bell path (render_user_wake=2 only; ++ * 0600, read at every bell, all 0 = no gate). They run between a phase's producer store and its bell; ++ * in policy state 0 the firmware does not drain a ring on its own, so a gate holds the phase back. ++ * RAM reads, sleeps and barriers only; no register access. ++ * bell_gap_us (a) the TA bell waits until this long after the previous job (any context) ended ++ * bell_idle_ms (a) the TA bell first waits, up to this, for the 3D engine idle words to read zero ++ * bell_3d_delay_us (c) mb() and this delay between the 3D producer store and the 3D bell ++ * bell_ctx_gap_us (d) mb() and this delay before the TA bell of a job of another context than the last ++ * bell_touch (c) the Normal-NC read of the firmware pages (g5p_rv3_fw_touch) right before each bell ++ */ ++ ++/* perf60 job-start gate: the same waits before the job's own host writes (the PM pool rebuild and ++ * the shared TA/3D work records, g5p_rv3_pm_rebuild() and g5p_ru_job()), which happen before any ++ * publish. The bell gates above run after them, so they cannot separate "the host rewrites shared ++ * firmware pages while the firmware still finishes the previous job" from a doorbell race. ++ * job_gap_us the job's first host write no sooner than this after the previous job ended ++ * job_idle_ms then wait up to this (RAM) for the TA and 3D engine idle words to read zero ++ */ ++/* perf60 layout (latched at the epoch's first attach, default 0 = the r22 layout): ++ * render_user_job_slots 2: job seq writes its command, W record and microsequence into slot ++ * (seq - 1) % 2 of the channel page, so a job never rewrites the record the firmware used for ++ * the job before it; the pointer ring carries each job's addresses (render-user-layout.h). ++ * render_user_pm_once 1: the PM pool, list and F are built by the epoch's first attach only and ++ * never rewritten by the host between jobs; the TA's tag6 record has the firmware rebuild F, ++ * as on iOS (PM created per queue once; docs/gpu/experiments/freelist-lifetime-production-fix.md). ++ */ ++/* release8 default */ ++/* release8 default */ ++ ++/* The record slot of job seq on this epoch's layout. */ ++static u32 g5p_rv3_js(const struct g5p_rdev *rd, u32 seq) ++{ ++ return rd->job_slots == 2 ? (seq - 1) & 1 : 0; ++} ++ ++ ++ ++ ++ ++ ++ ++ ++ ++/* RAM only. */ ++static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_ru_page(b, e), *global = g5p_ru_global(b, e), ++ *pm = g5p_rv3_pm_cpu(b, r); ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ for (u32 i = 0; i < ARRAY_SIZE(s->stamp); i++) ++ s->stamp[i] = g5p_le32(p + G5P_R22_STAMP + 4 * i); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < ARRAY_SIZE(s->idle); i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->pm80 = g5p_le32(pm + 0x80); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != r->target[e] || s->cons > 255 || ++ !g5p_ru_stamp_sane(p, seq); ++ s->consumed = s->cons == r->target[e]; ++ /* The predicates of the flown r74b (TA) and r96 (3D) waits; the 3D's PM serial is a receipt. */ ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2] && ++ (!e || (!s->idle[4] && !s->idle[5])); ++ s->retired = g5p_ru_retired_at(p, g5p_ru_meta(e), e, seq, ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ s->pm_ok = e ? s->pm80 == r->ctx->gpu->rdev->pm_serial + 1 : ++ g5p_r22_pm_saved(pm + 0x80); ++ s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns) : ++ READ_ONCE(r->ctx->gpu->ta_hit_ns)); ++} ++ ++/* TA: one step of render-user-retire.h's TA rules when ta is given, else the flown r74b ++ * predicate. 3D: one step of its 3D rules when rt is given, else the r96 predicate. ++ */ ++static bool g5p_rv3_done(u32 e, const struct g5p_rv3_ev *s, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta) ++{ ++ if (e && rt) ++ return g5p_ru_rt_step(rt, s->ns, s->consumed, s->retired, ++ s->idle_ok, ++ s->event) != G5P_RU_RT_PENDING; ++ if (!e && ta) ++ return g5p_ru_ta_step(ta, s->ns, s->consumed, s->retired, ++ s->pm_ok, s->idle_ok, ++ s->event) != G5P_RU_TA_PENDING; ++ return s->consumed && s->idle_ok && s->retired && (e || s->pm_ok); ++} ++ ++/* The TA clauses in the bits of render-user-retire.h, for the receipt. */ ++static u32 g5p_rv3_ta_unsat(const struct g5p_rv3_ev *s) ++{ ++ return (s->consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (s->retired ? 0 : G5P_RU_RT_RETIRED) | ++ (s->idle_ok ? 0 : G5P_RU_RT_IDLE) | ++ (s->pm_ok ? 0 : G5P_RU_RT_PM); ++} ++ ++/* Arm engine e's completion-event latch of g5p_drm_event() (3D: for this job's endpoint), before ++ * the phase's producer store, so an event decoded right after the publish is not missed. The ++ * latch is set under status_lock by whichever CPU decodes the event, and the wait samples it ++ * level-triggered every 1-2 ms, so there is no wakeup to lose. RAM only. ++ */ ++static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) ++{ ++ unsigned long flags; ++ ++ BUILD_BUG_ON(G5P_RU_RT_GRID != G5P_WORK_SLOTS); ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = G5P_RV3_TD_WANT | endpoint; ++ WRITE_ONCE(d->td_hit_ns, 0); ++ } else { ++ d->ta_want = true; ++ WRITE_ONCE(d->ta_hit_ns, 0); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) ++{ ++ unsigned long flags; ++ u64 hit; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = 0; ++ hit = d->td_hit_ns; ++ } else { ++ d->ta_want = false; ++ hit = d->ta_hit_ns; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return hit; ++} ++ ++/* release: the receipt's read set (g5p_rv3_receipt()'s blobs: PM-control, then W, channels and the ++ * stamp window of the TA and of the 3D page), read once with 32-bit loads, the access size of the ++ * receipt's hex dump. The words are folded, not copied: no allocation, no store to shared memory. ++ * RAM only: no register, no doorbell, no print, no latch. ++ */ ++static u32 g5p_rv3_touch_sweep(struct g5p_boot *b, u32 *bytes) ++{ ++ u8 *ta = g5p_ru_page(b, 0), *td = g5p_ru_page(b, 1); ++ const u8 *const src[] = { ++ g5p_ru_pm_cpu(b), ta + G5P_R22_W, ta, ++ ta + G5P_R22_TQ, td + G5P_R22_W, td, ++ td + G5P_R22_TQ, ++ }; ++ static const u32 len[] = { ++ 0x140, G5P_RU_TA_W_BYTES, 0x340, 0x80, G5P_RU_3D_W_BYTES, 0x340, ++ 0x80, ++ }; ++ u32 sum = 0, n = 0; ++ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(src); i++) { ++ for (u32 at = 0; at < len[i]; at += 4) ++ sum = (sum << 5 | sum >> 27) ^ ++ READ_ONCE(*(const u32 *)(src[i] + at)); ++ n += len[i]; ++ } ++ *bytes = n; ++ return sum; ++} ++ ++/* release: the instant of one phase, taken at the sample at now_ns. The touch arm reads, the control ++ * arm only records; both record the same instant. ++ */ ++static void g5p_rv3_touch(struct g5p_diag *g, struct g5p_ru_touch *t, ++ u64 now_ns) ++{ ++ u32 bytes = 0, sum = 0; ++ ++ if (t->sweep) ++ sum = g5p_rv3_touch_sweep(g->boot, &bytes); ++ g5p_ru_touch_mark(t, now_ns, t->sweep ? ktime_get_ns() : now_ns, bytes, ++ sum); ++} ++ ++/* tahang1: a phase still pending this long after its publish prints its RAM state once. */ ++#define G5P_RV3_SLOW_MS 1000U ++ ++/* The one `slow` line of a phase: the deciding sample's RAM words and the waiter's own state, so ++ * a stall that later times out is described from the first second on. RAM only. ++ */ ++static void g5p_rv3_slow(struct g5p_diag *g, u32 e, u32 seq, ++ const struct g5p_ru_wait *w, ++ const struct g5p_rv3_ev *s, u32 unsat, u64 hit_ns) ++{ ++ dev_warn( ++ g->dev, ++ "RENDER-V3 slow seq=%u engine=%u ms=%lld ring=%u/%u stamp=%08x/%08x " ++ "ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x,%08x,%08x,%08x pm80=%08x " ++ "consumed=%u retired=%u pm=%u event_ms=%lld unsat=%#x busy_ms=%lld events=%u " ++ "end=v3\n", ++ seq, e, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ s->idle[3], s->idle[4], s->idle[5], s->pm80, s->consumed, ++ s->retired, s->pm_ok, g5p_ru_rt_ms(w->start_ns, hit_ns), unsat, ++ g5p_ru_rt_ms(w->start_ns, w->busy_ns), ++ READ_ONCE(g->boot->events)); ++ g5p_np_policy(g, "slow", seq, e); ++} ++ ++/* nopatch: the policy record at a few phase ends (log-spaced) and at every timeout. RAM only. */ ++static void g5p_rv3_np_end(struct g5p_diag *g, u32 e, u32 seq, u32 result) ++{ ++ if (seq == 1 || seq == 10 || seq == 100 || seq == 1000 || ++ seq == 10000 || result == G5P_RU_WAIT_TIMEOUT) ++ g5p_np_policy(g, ++ result == G5P_RU_WAIT_TIMEOUT ? "phase-timeout" : ++ "phase-end", ++ seq, e); ++} ++ ++/* One phase's RAM-only wait. g5p_ru_wait_step()/close() decide; this loop only samples. release: the ++ * touch instant is taken after a step that left the phase undecided, before the sleep (off: never). ++ * The predicate is level-triggered: every sample, including the one right after each sleep and the ++ * closing one at the deadline, re-reads the whole RAM state (rt: the 3D, ta: the TA). ++ */ ++static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, struct g5p_ru_touch *t, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta, ++ struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) ++{ ++ struct g5p_drm *d = r->ctx->gpu; ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE( ++ d->event_serial); /* after the drain, before the sample */ ++ bool fault, silent, done; ++ ++ g5p_rv3_sample(g, r, e, seq, s); ++ /* latency101: a silent event stream during the phase is a dead firmware (never off). */ ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed) ++ *err = -EIO; ++ else if (silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = g5p_rv3_done(e, s, rt, ta); ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (rt) ++ g5p_ru_rt_note(rt, w->result); ++ if (ta) ++ g5p_ru_ta_note(ta, w->result); ++ if (w->result) { ++ g5p_rv3_np_end(g, e, seq, w->result); ++ return w->result; ++ } ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_RV3_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ g5p_rv3_slow(g, e, seq, w, s, ++ rt ? rt->unsat : ++ ta ? ta->unsat : ++ 0, ++ e ? READ_ONCE(d->td_hit_ns) : ++ READ_ONCE(d->ta_hit_ns)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ if (g5p_ru_touch_due(t, s->ns, w->result, w->expired)) ++ g5p_rv3_touch(g, t, s->ns); ++ g5p_l123_pause(d, g, e, s, seen); ++ } ++} ++ ++/* One phase's deadline: the legacy single bound, or (latency101) the soft/hard pair. */ ++static void g5p_rv3_wait_start(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_ru_wait *w, u64 pub_ns, ++ u64 job_deadline_ns) ++{ ++ if (r->hard_ms) ++ g5p_ru_wait_start_lat(w, pub_ns, r->soft_ms, r->hard_ms, ++ job_deadline_ns, ++ READ_ONCE(g->boot->events)); ++ else ++ g5p_ru_wait_start(w, pub_ns, r->wait_ms, job_deadline_ns); ++} ++ ++/* After a timeout (or fault) was decided: the one RAM look of the legacy path, or (latency101, ++ * timeout only) a RAM-only observation window that keeps draining and sampling at the wait cadence ++ * and prints nothing, so a completion that lands milliseconds after the decision is recorded ++ * (late_after/late_ns) before the receipt line and the terminal latch. It never changes the ++ * result and never touches a register. The 3D look steps a copy of the predicate's record, so the ++ * receipt keeps the deciding sample's. ++ */ ++static void g5p_rv3_observe(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, ++ const struct g5p_ru_retire *rt, ++ const struct g5p_ru_ta *ta, struct g5p_rv3_ev *late) ++{ ++ struct g5p_ru_retire copy, *c = NULL; ++ struct g5p_ru_ta tcopy, *tc = NULL; ++ ++ if (rt) { ++ copy = *rt; ++ c = © ++ } ++ if (ta) { ++ tcopy = *ta; ++ tc = &tcopy; ++ } ++ if (!r->obs_ms || w->result != G5P_RU_WAIT_TIMEOUT) { ++ g5p_rv3_sample(g, r, e, seq, late); ++ g5p_ru_wait_after(w, g5p_rv3_done(e, late, c, tc)); ++ return; ++ } ++ do { ++ g5p_boot_drain( ++ g); /* a fault latches b->failed, the result stays */ ++ g5p_rv3_sample(g, r, e, seq, late); ++ if (!g5p_ru_wait_observe(w, late->ns, ++ g5p_rv3_done(e, late, c, tc), ++ r->obs_ms)) ++ break; ++ usleep_range(1000, 2000); ++ } while (1); ++} ++ ++/* wake110: render-user-wake.h's register access, on the Fender and RGX windows. */ ++static void g5p_rv3_wk_w1(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->window[1] + reg); ++} ++ ++/* The GPC state read of g5p_wk_ring() comes right after its R1+0x14 = 7 wake: in bell mode ++ * (render_user_wake=2) it waits up to G5P_RV3_WK_SETTLE_US, re-reading every 10 us, for a settled ++ * powered pstate (bit 4 clear, nibble nonzero), as Apple's setGPCPerformanceState waits for bit 4. ++ * Reads only; the ring's own check of the final value is unchanged. ++ */ ++#define G5P_RV3_WK_SETTLE_US 2000U ++ ++static u32 g5p_rv3_wk_r1(void *ctx, u32 reg) ++{ ++ void __iomem *r1 = ((struct g5p_diag *)ctx)->window[1]; ++ u32 raw = readl(r1 + reg); ++ u64 t0; ++ ++ if (reg != G5P_WK_R1_GPC || (2) != G5P_WK_ALWAYS || ++ (!(raw & 0x10) && (raw & 0xf))) ++ return raw; ++ t0 = ktime_get_mono_fast_ns(); ++ while (((raw & 0x10) || !(raw & 0xf)) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_WK_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(r1 + reg); ++ } ++ return raw; ++} ++ ++static void g5p_rv3_wk_wmb(void *ctx) ++{ ++ dma_wmb(); ++} ++ ++static void g5p_rv3_wk_w0(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->rgx + reg); ++} ++ ++/* wake110: an r74f bell that still owns its ring (the ALIVE decoder may fire it). */ ++static bool g5p_rv3_wk_r74f_busy(struct g5p_r73b *x) ++{ ++ u32 k = READ_ONCE(x->r74f_active); ++ int s; ++ ++ if (!k || k > G5P_R74F_KINDS) ++ return false; ++ s = atomic_read_acquire(&x->r74f_bell[k - 1].state); ++ return s == G5P_R74F_PUBLISHED || s == G5P_R74F_OPEN || ++ s == G5P_R74F_FIRING; ++} ++ ++/* wake111: DS108's claim proof re-read at the publish: ALIVE proven in the current reset ++ * generation, the claim admitted in it, nothing latched failed since, no R74E power abort. ++ * Called under event_lock (interrupts off); g5p_ds_lock is a leaf (g5p_ds_event() is pure). ++ */ ++static bool g5p_rv3_wk_ds108(struct g5p_boot *b) ++{ ++ bool ok; ++ ++ spin_lock(&g5p_ds_lock); ++ ok = g5p_ds_proven(&g5p_ds_state) && ++ g5p_ds_ok(&g5p_ds_state, G5P_DSE_CLAIM); ++ spin_unlock(&g5p_ds_lock); ++ return ok && !(b->r73b && READ_ONCE(b->r73b->early_power_abort)); ++} ++ ++/* wake110: before the phase's producer store, the engine's type-1 count and a cleared first-decode ++ * time, so an event decoded right after the publish is counted. RAM only. ++ */ ++static void g5p_rv3_wake_mark(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ v->wk_t1_pub[e] = d->t1_count[e]; ++ WRITE_ONCE(d->t1_first_ns[e], 0); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* wake110: at the job's end, the type-1 counts and the first decode of each published phase. */ ++static void g5p_rv3_wake_end(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ for (u32 e = 0; e < 2; e++) { ++ v->wk_t1_end[e] = d->t1_count[e]; ++ v->wk_first_ns[e] = d->t1_first_ns[e]; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (v->phase < G5P_RV3_TA_PUBLISH) ++ v->wk_first_ns[0] = 0; ++ if (v->phase < G5P_RV3_TD_PUBLISH) ++ v->wk_first_ns[1] = 0; ++} ++ ++/* wake110: microseconds / milliseconds from a phase's publish to a later instant; -1 = none. */ ++static s64 g5p_rv3_wk_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++static s64 g5p_rv3_wk_ms(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_MSEC : ++ -1; ++} ++ ++/* wake110: one phase's publish-time decision, right after its producer store and barrier. Both ++ * arms take the event lock (the r74f owner's serialization against the decoder) and sample the ++ * same RAM; only a factor that passed every guard runs g5p_wk_ring(). Nothing is printed under ++ * the lock. Off: returns at once. ++ */ ++static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e, u32 start, u32 seq) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ struct g5p_drm *d = r->ctx->gpu; ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ u8 *p = g5p_ru_page(b, e); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&b->event_lock, flags); ++ v->wk_start_ns[e] = ktime_get_ns(); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = r->wake, ++ .seed = 0, ++ .seq = seq, ++ .e = e, ++ .attempted = g5p_wk_attempted(v->wk_did[e]), ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = r->target[e], ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4) }, ++ .type1 = READ_ONCE(d->t1_count[e]) - v->wk_t1_pub[e], ++ }; ++ v->wk_cons[e] = in.cons; ++ v->wk_power[e] = in.power; ++ v->wk_did[e] = g5p_wk_decide(&in, &io, &v->wk_fence[e], &v->wk_gpc[e]); ++ v->wk_done_ns[e] = ktime_get_ns(); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (v->wk_did[e] == G5P_WK_DID_RANG) ++ r->wake_rang++; ++} ++ ++/* The epoch's first job: the device (PM, channels) and this context's DC3 records, shared with ++ * g5p_render_attach(), then the r74f/r74g protocol of the fixed r74f_attach() on typed state. ++ */ ++static int g5p_rv3_attach(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline; ++ u32 start, tag, bad; ++ int ret; ++ ++ ret = g5p_r74f_admit(g, G5P_R74F_DC3); ++ if (ret) ++ return ret; ++ ret = g5p_render_attach_prepare(d, r, &start, &tag); ++ if (ret) ++ return ret; ++ r->dc_target = (start + 2) & 255; ++ r->bound = true; ++ rd->up = true; ++ rd->v3 = true; ++ rd->job_slots = (2) == 2 ? 2 : 1; ++ rd->pm_once = (1) == 1; ++ rd->pm_ctx = b->pm_sets ? min((3), 3U) : 0; ++ r->pm_first = r->pm_set < 0; ++ dev_info( ++ g->dev, ++ "RENDER-V3 layout job_slots=%u pm_once=%u slot1=%#x..%#x pm_ctx=%u pm_sets=%u " ++ "end=v3\n", ++ rd->job_slots, rd->pm_once, G5P_RU_SLOT1, G5P_RU_SLOT1_END, ++ rd->pm_ctx, b->pm_sets); ++ g5p_render_retain(r); ++ v->touched = true; ++ WRITE_ONCE(r->ctx->live, true); ++ dma_wmb(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ v->attach_ns = ktime_get_ns(); ++ g5p_r74_mark(g, G5P_R74_ATTACH_PUBLISHED, r->dc_target); ++ /* The producer is final from here; only the doorbell may be pending. */ ++ ret = g5p_r74f_bell(g, G5P_R74F_DC3, start, r->dc_target); ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || READ_ONCE(b->failed)) ++ return ret ?: -EIO; ++ dma_rmb(); ++ if (READ_ONCE(ctl[4]) != r->dc_target || ++ READ_ONCE(ctl[0]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(ctl[0]) == r->dc_target) { ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ bad = g5p_r74g_check(g, 0, r->slot, vm->root.dma, tag, ++ start); ++ g5p_r74_mark(g, G5P_R74_ATTACH_CONSUMED, r->dc_target); ++ g5p_r74g_dc3_line(g, 0, false); ++ if (bad) ++ return g5p_rv3_errno(g, ++ g5p_r74g_refuse(g, bad)); ++ for (u32 e = 0; e < 2; e++) ++ g5p_mmu_record( ++ g, ++ "RENDER tags slot=%u host=%08x cached=%08x expected=%08x\n", ++ r->slot[e], ++ g5p_le32(g5p_boot_shared(b, 0x84) + ++ r->slot[e] * 4), ++ g5p_le32(g5p_boot_shared(b, 0x85) + ++ r->slot[e] * 4), ++ tag); ++ g5p_mmu_record( ++ g, ++ "RENDER-USER attached TA=%u 3D=%u PC=%pad tag=%08x blocks=%u; RAM-only, firmware " ++ "DC3 installed CBASE, host MMU pass skipped\n", ++ r->slot[0], r->slot[1], &vm->root.dma, tag, ++ rd->blocks); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } while (time_before(jiffies, deadline)); ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ return g5p_rv3_errno(g, ++ g5p_r74f_stop(g, G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ READ_ONCE(ctl[0]))); ++} ++ ++/* perf60 bell mode (render_user_wake=2): a DC publication (bind, spare pairs, unbind) gets Apple's ++ * submit kick too. With the policy held in state 0 the firmware no longer drains the DC ring on ++ * its own (perf60clk-9ec288b868: a second client's RAM-only DC3 bind was never consumed). The ++ * wake path's guards (event lock, power proof, no fault or terminal, no r74f bell owning a ring, ++ * the ring holding exactly this publication), then Apple's order: R1+0x14 = 7 and its read-back, ++ * R1+0x1000 re-read every 10 us until a settled powered pstate (g5p_gpc_ok(): bit 4 clear, ++ * nibble nonzero; Apple's setGPCPerformanceState waits for bit 4 the same way), at most ++ * G5P_RV3_DC_SETTLE_US, and R0+0xb00 = 0x10 only then. A GPC that never reads on is not written: ++ * no R0 access, the publication waits as before. Off (render_user_wake != 2): returns at once. ++ */ ++#define G5P_RV3_DC_SETTLE_US 2000U ++ ++static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, ++ const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 fence = U32_MAX, gpc = U32_MAX, power, cons, n = 0; ++ const char *did = "skip-guard"; ++ unsigned long flags; ++ u64 t0; ++ ++ ; ++ ; ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ cons = READ_ONCE(ctl[0]); ++ power = g5p_wk_power_mask(READ_ONCE(g->powered), ++ READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), ++ g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ !g5p_wk_power_ready(power) || g5p_rv3_wk_r74f_busy(b->r73b) || ++ cons != start || READ_ONCE(ctl[4]) != target) ++ goto out; ++ writel(7, g->window[1] + 0x14); ++ fence = readl(g->window[1] + 0x14); ++ if (fence & ~7U) { ++ did = "skip-fence"; ++ goto out; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ gpc = readl(g->window[1] + 0x1000); ++ while (!g5p_gpc_ok(gpc) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ gpc = readl(g->window[1] + 0x1000); ++ n++; ++ } ++ if (!g5p_gpc_ok(gpc)) { ++ did = "skip-gpc"; ++ goto out; ++ } ++ /* the publication's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ wmb(); ++ writel(0x10, g->rgx + 0xb00); ++ did = "rang"; ++out: ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 dc-kick why=%s did=%s dc=%u->%u power=%02x fence=%08x gpc=%08x " ++ "settle=%u end=v3\n", ++ why, did, cons, target, power, fence, gpc, n); ++} ++ ++/* RAM only: drain the event stream, read the firmware pages (interim, see g5p_rv3_fw_touch) and ++ * sample the DC ring until its consumer reaches `target` ++ * or G5P_RU_BIND_MS pass. No doorbell and no register, whether or not a job has retired. ++ */ ++static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_RU_BIND_MS); ++ ++ for (;;) { ++ bool expired = time_after_eq(jiffies, deadline); ++ int ret = g5p_boot_drain(g); ++ ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ g5p_rv3_fw_touch(b); ++ dma_rmb(); ++ switch (g5p_ru_bind_step(READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), ++ target, expired)) { ++ case G5P_RU_BIND_CONSUMED: ++ *consumed_ns = ktime_get_ns(); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } ++} ++ ++static s64 g5p_rv3_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++/* ++ * The first job of a context on a device that is already up binds the context's slot pair ++ * with a DC3 pair published in RAM only (multi-client-design.md §4.2). No doorbell and no ++ * register access of any kind, so it is safe after a job retired and while the GPU may be ++ * idle-gated: the firmware services the DC ring in its scheduler pass, before TA and 3D, and ++ * the host waits on RAM. Core lock held, between jobs. ++ * ++ * Refused before the producer store, it touched nothing. From the store on, the firmware may ++ * hold the pair: the context is retained, and on failure it is lost (v->bind_failed) but the ++ * device and the contexts already bound are not. A pair not consumed in time keeps every later ++ * bind refused until the firmware catches up. ++ */ ++/* ++ * render_user_close_park (release8 quiet2, docs/gpu/experiments/release8-quiet2.md). The PM writes ++ * a closed context's free list back through that context's TA slot after the context is gone, and ++ * not only while it is the last or the previous distinct context (render-user-pmset.h): two bench ++ * faults wrote the free list of a context two switches back, through its slot, on the 2nd job of a ++ * third context, after the pm-prev rule had unbound it (page-invalid level 2, its PC level empty). ++ * iOS never leaves the PM's target unresolvable: its PM is the device's (shared by the work queues, ++ * AGXWorkQueue_initAndAttachPM), in the kernel's GPU address space, and a GART keeps its slots until ++ * the address space is destroyed (AGXGart_detachSlotsAndDestroyAddressSpace). Here, a closed ++ * context's pair is bound again, right after its unbind (between jobs, core lock held), to the park ++ * catalogue: a kernel page that maps only the firmware window, where the PM sets live. A late ++ * write-back then lands in the closed context's own set page (released, reused last) instead of ++ * faulting. A later bind of the pair unbinds it first. Both steps are the proven DC3 ++ * transitions (PA to zero, zero to PA); a park that fails leaves the pair unbound, as before. ++ */ ++/* release8 default */ ++ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why); ++ ++/* Before a bind, core lock held: a pair a close parked (RAM: park PA and tag on both slots) is ++ * unbound first, by its own DC3. 0 when the pair was not parked or the unbind verified. ++ */ ++static int g5p_rv3_unpark(struct g5p_drm *d, const u32 *slot) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ ++ if (!d->rdev->park.cpu) ++ return 0; ++ dma_rmb(); ++ g5p_render_bind_mem(b, slot, READ_ONCE(ctl[4]), &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!g5p_ru_bind_parked(&s, d->rdev->park.dma)) ++ return 0; ++ return g5p_rv3_unbind_slots(d, slot, 2, "unpark"); ++} ++ ++static int g5p_rv3_bind(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, tag, bad = 0; ++ int ret; ++ ++ dma_rmb(); ++ if (rd->dc_pending && READ_ONCE(ctl[0]) != rd->dc_wait) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u dc=%u/%u wait=%u: an earlier bind is " ++ "pending; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4]), rd->dc_wait); ++ return -EIO; ++ } ++ rd->dc_pending = false; ++ ret = g5p_rv3_unpark(d, r->slot); ++ if (ret) ++ return ret; ++ ret = g5p_render_vm_prepare(d, r, &start, &tag); ++ if (ret) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u ret=%d dc=%u/%u: ring busy or slot in use; " ++ "no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], ret, READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4])); ++ return ret; ++ } ++ r->dc_target = (start + 2) & 255; ++ g5p_render_retain(r); ++ v->bind_failed = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, r->dc_target, "bind"); ++ ret = g5p_rv3_dc_wait(g, r->dc_target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = r->dc_target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, vm->root.dma, tag, ++ r->dc_target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=%lld tag=%08x host=%08x/%08x " ++ "cached=%08x/%08x dc=%u->%u/%u bad=%#x pc=%pad mode=ram-dc3 end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ tag, s.host[0], s.host[1], s.cached[0], s.cached[1], start, ++ s.cons, s.prod, bad, &vm->root.dma); ++ if (ret) ++ return ret; ++ v->bind_failed = false; ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ return 0; ++} ++ ++/* ++ * TEMPORARY fallback (render_user_spare_pairs, read at the epoch's first job), for a firmware ++ * that never drains a DC3 published after a job retired. Before job 1's attach, bind that ++ * many extra slot pairs, each to a zeroed kernel-owned catalogue, in one RAM-only batch with no ++ * doorbell. The firmware drains it on its own like the self-drained attach of this boot tuple ++ * (r87b); job 1's own attach, doorbell and r74g checks then run unchanged behind it ++ * (g5p_r74f_ring() refuses a ring that holds more than that attach's pair). Later contexts ++ * take a pair over with g5p_rv3_adopt(). A pair that does not check out is never used. Core ++ * lock held, device not up, so register access is still allowed (none is made here). ++ */ ++static int g5p_rv3_spares(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 want = min((0), G5P_RU_MAX_SPARES); ++ u32 start = READ_ONCE(ctl[4]), target, n, ok = 0; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ unsigned long flags; ++ int ret; ++ ++ if (!want || rd->spares) ++ return 0; ++ rd->spares = kcalloc(want, sizeof(*rd->spares), GFP_KERNEL); ++ if (!rd->spares) ++ return -ENOMEM; ++ for (n = 0; n < want; n++) { ++ struct g5p_rspare *sp = &rd->spares[n]; ++ bool pair; ++ ++ spin_lock_irqsave(&g->slot_lock, flags); ++ pair = g5p_ru_slot_pair(g->user_slots, false, sp->slot); ++ if (pair) { ++ __set_bit(sp->slot[0], g->user_slots); ++ __set_bit(sp->slot[1], g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ if (!pair) ++ break; ++ g5p_render_bind_mem(b, sp->slot, start + 2 * n, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (!g5p_ru_bind_busy(&s) && ++ !g5p_vm_alloc_page(d->drm.dev, &sp->root)) { ++ g5p_table_clean_page(g, &sp->root); ++ continue; ++ } ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(sp->slot[0], g->user_slots); ++ __clear_bit(sp->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ break; ++ } ++ if (!n) ++ return 0; ++ dsb(sy); ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_bind_mem(b, rd->spares[i].slot, start + 2 * i, &m); ++ g5p_ru_bind_publish(&m, rd->spares[i].slot, ++ rd->spares[i].root.dma, G5P_RU_SPARE_TAG); ++ } ++ rd->spare_count = n; ++ target = (start + 2 * n) & 255; ++ v->touched = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "spares"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ for (u32 i = 0; !ret && i < n; i++) { ++ struct g5p_rspare *sp = &rd->spares[i]; ++ ++ g5p_render_bind_mem(b, sp->slot, start + 2 * i, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ sp->ok = !g5p_ru_bind_check(&s, sp->slot, sp->root.dma, ++ G5P_RU_SPARE_TAG, target); ++ ok += sp->ok; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 spares pairs=%u ok=%u ret=%d consumed_us=%lld dc=%u->%u first=%u/%u " ++ "last=%u/%u end=v3\n", ++ n, ok, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ READ_ONCE(ctl[0]), rd->spares[0].slot[0], rd->spares[0].slot[1], ++ rd->spares[n - 1].slot[0], rd->spares[n - 1].slot[1]); ++ return ret; ++} ++ ++/* ++ * TEMPORARY fallback: the first job of a context on a device whose job 1 bound spare pairs. ++ * Takes the next pair that checked out, makes its catalogue page this VM's root (a copy of the ++ * current root) and publishes the VM's tag through the proven RAM-only tag path: CBASE already ++ * names the page, and the changed tag makes the firmware invalidate the slots' PC level at this ++ * context's first job. No DC3, no register. Core lock held; the context has a job queued, so ++ * no VM edit of it runs concurrently. ++ */ ++static int g5p_rv3_adopt(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_rspare *sp = NULL; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ struct g5p_dma_page old; ++ unsigned long flags; ++ struct g5p_rt *rt; ++ u32 entry; ++ int ret; ++ ++ for (u32 i = 0; i < rd->spare_count && !sp; i++) ++ if (rd->spares[i].ok && !rd->spares[i].used) ++ sp = &rd->spares[i]; ++ if (!sp) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare: none left of %u; no hardware " ++ "touched end=v3\n", ++ r->slot[0], r->slot[1], rd->spare_count); ++ return -ENOSPC; ++ } ++ /* RAM: the pair still names its page, with the tag job 1 checked. */ ++ dma_rmb(); ++ g5p_render_bind_mem(b, sp->slot, 0, &m); ++ g5p_ru_bind_read(&m, 0, 0, &s); ++ for (u32 e = 0; e < 2; e++) { ++ if (s.pa[e] == sp->root.dma && s.host[e] == G5P_RU_SPARE_TAG && ++ s.cached[e] == G5P_RU_SPARE_TAG) ++ continue; ++ sp->ok = false; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare pair=%u/%u host=%08x cached=%08x: " ++ "pair changed; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], sp->slot[0], sp->slot[1], ++ s.host[e], s.cached[e]); ++ return -EIO; ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ memcpy(sp->root.cpu, vm->root.cpu, G5P_GPU_PAGE); ++ old = vm->root; ++ vm->root = sp->root; ++ sp->root = old; /* the VM's former root, never published */ ++ g5p_ru_free_page(d->drm.dev, &sp->root); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(r->slot[0], g->user_slots); ++ __clear_bit(r->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ r->slot[0] = sp->slot[0]; ++ r->slot[1] = sp->slot[1]; ++ sp->used = true; ++ g5p_render_retain(r); ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ /* Root and tables to PoC, then a tag unlike the cached one in both slots. */ ++ ret = g5p_render_invalidate(r->ctx); ++ v->bind_failed = ret != 0; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=0 tag=%08x cached=%08x/%08x " ++ "pc=%pad mode=spare end=v3\n", ++ r->slot[0], r->slot[1], ret, ++ g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[0] * 4), ++ s.cached[0], s.cached[1], &vm->root.dma); ++ return ret; ++} ++ ++/* r74f_publish_ta() on typed state: record, WORK_CTL, producer; then (job 1 only) the r74g grace ++ * and, only if the ring is still pending, the one ALIVE-owned fallback doorbell. Jobs >= 2 ++ * publish and return: no grace, no bell (the r74f/r74g machinery is one-shot). No canary, no marker. ++ */ ++static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *x = b->r73b; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_r74g_dc3 *dc = x->r74g_dc3; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x60); ++ u8 *p = g5p_ru_page(b, 0); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ bool first = seq == 1; ++ u32 bad; ++ int ret; ++ ++ if (!r->bound || r->submitted[0] || start > 255 || ++ READ_ONCE(ring[0]) != start || ++ g5p_le32(p + G5P_WORK_CTL) != previous || ++ g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) ++ return -EBUSY; ++ if (first) { ++ ret = g5p_r74f_admit(g, G5P_R74F_TA); ++ if (ret) ++ return ret; ++ /* g5p_r74g_ta_preflight() on typed state: DC3 still complete with the same records, ++ * tag and backing, and no page-table mutation since the attach. ++ */ ++ bad = g5p_r74g_check(g, 1, r->slot, dc[0].expected_pa, ++ dc[0].tag, dc[0].start); ++ if (g5p_vm_generation(&vm->mutation) != dc[0].tag || ++ vm->root.dma != dc[0].expected_pa || ++ r->dc_target != dc[0].target) { ++ bad |= G5P_R74G_BAD_MUTATION; ++ dc[1].bad = bad; ++ } ++ g5p_r74g_dc3_line(g, 1, false); ++ if (bad) ++ return g5p_rv3_errno(g, g5p_r74g_refuse(g, bad)); ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x61] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 0 : ++ j == 1 ? g5p_ru_meta(0) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[0] = (start + 1) & 255; ++ r->submitted[0] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_PUBLISH, start); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], r->target[0]); ++ dma_wmb(); ++ if (first) { ++ ret = g5p_r74g_ta(g, start, r->target[0]); ++ v->grace_end = READ_ONCE(x->r74g_grace_end); ++ v->bell = ret == 0; ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ } ++ /* wake110: jobs >= 2 reach this at once after the producer store (job 1: excluded). */ ++ if (!first) ++ 0; ++ g5p_rv3_wake(g, r, v, 0, start, seq); ++ /* The line the userspace lane parses (a9c_parse_publish): same format as the fixed path's. */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 0, r->slot[0], endpoint, first ? 0x10 : 0, ++ g5p_ru_meta(0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 0); ++ return 0; ++} ++ ++/* The RAM preconditions of the 3D publish, one bit per clause (0 = admitted). RAM reads only; the bits are ++ * what the refusal line prints, so a second blocker is visible in the run that meets it: ++ * 0 TA not retired 1 TA not consumed 2 3D already submitted 3 producer > 255 4 consumer != producer ++ * 5 WORK_CTL 6 WORK_CTL+0x40 7 command tag != 4 8 dependency word 9 dependency value ++ * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) ++ */ ++static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, ++ const u32 *ring, u32 start, u32 previous, u32 seq) ++{ ++ const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ ++ return (u32)!r->retired[0] | (u32)!r->consumed[0] << 1 | ++ (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | ++ (u32)(READ_ONCE(ring[0]) != start) << 4 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 5 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 6 | ++ (u32)(g5p_le32(cmd) != 4) << 7 | ++ (u32)(g5p_le32(cmd + 4) != g5p_ru_meta(0) + G5P_R22_STAMP + 4) ++ << 8 | ++ (u32)(g5p_le32(cmd + 8) != seq) << 9 | ++ (u32)(g5p_le32(p + G5P_R22_STAMP) != seq - 1 || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq - 1) ++ << 10; ++} ++ ++/* The 3D after the TA retired: record, WORK_CTL, producer, and NO doorbell. */ ++static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x64); ++ u8 *p = g5p_ru_page(b, 1); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ u32 why; ++ ++ /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one ++ * fails, and the refusal names the clauses and the raw values it read. ++ */ ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq); ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 3d-refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x cmd=%08x/%08x/%08x " ++ "stamp=%08x/%08x end=v3\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ seq))), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 4), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 8), ++ g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x65] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 1 : ++ j == 1 ? g5p_ru_meta(1) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[1] = (start + 1) & 255; ++ r->submitted[1] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->td_pub_ns = ktime_get_ns(); ++ smp_store_release(&ring[4], r->target[1]); ++ dma_wmb(); ++ 0; ++ g5p_rv3_wake(g, r, v, 1, start, seq); ++ /* The 3D twin of the TA's publish line: engine 1, kick 0 (no doorbell). */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 1, r->slot[1], endpoint, 0, ++ g5p_ru_meta(1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 1); ++ return 0; ++} ++ ++static const char *const g5p_rv3_ta_how[G5P_RU_TA_HOWS] = { "none", ++ "idle-clear", ++ "event", "hold" }; ++ ++/* tahang1: the TA decisions since load, by rule (core lock held). A TA that completed with an ++ * idle word still set prints one line with the running totals: rate-limited for rule (b), which ++ * is routine, always for rule (c), which means the event was not seen. RAM only. ++ */ ++static u32 g5p_rv3_ta_rules[G5P_RU_TA_HOWS]; ++static DEFINE_RATELIMIT_STATE(g5p_rv3_ta_rs, 5 * HZ, 4); ++ ++static void g5p_rv3_ta_count(struct g5p_diag *g, struct g5p_drm *d, ++ const struct g5p_rv3 *v) ++{ ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ u32 how = min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1); ++ u64 hit = READ_ONCE(d->ta_hit_ns); ++ ++ g5p_rv3_ta_rules[how]++; ++ if (how == G5P_RU_TA_IDLE_CLEAR) ++ return; ++ if (how == G5P_RU_TA_EVENT && !__ratelimit(&g5p_rv3_ta_rs)) ++ return; ++ dev_warn( ++ g->dev, ++ "RENDER-V3 ta-done seq=%u how=%s idle=%08x,%08x,%08x pm80=%08x coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld busy_ms=%lld rules=%u/%u/%u end=v3\n", ++ v->seq, g5p_rv3_ta_how[how], s->idle[0], s->idle[1], s->idle[2], ++ s->pm80, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, hit), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_ta_rules[G5P_RU_TA_IDLE_CLEAR], ++ g5p_rv3_ta_rules[G5P_RU_TA_EVENT], ++ g5p_rv3_ta_rules[G5P_RU_TA_HOLD]); ++} ++ ++/* A receipt line: a failed job's at KERN_EMERG as before (it survives a freeze), a completed job's at ++ * job_log (wake every job, end/lat/retire the first 8 of a context: routine at desktop rates). ++ */ ++#define g5p_rv3_receipt_printk(g, ret, fmt, ...) \ ++ do { \ ++ if (ret) \ ++ dev_emerg((g)->dev, fmt, ##__VA_ARGS__); \ ++ else \ ++ g5p_job_printk((g)->dev, fmt, ##__VA_ARGS__); \ ++ } while (0) ++ ++/* The end of a job, whatever it was: one receipt line on the console (it survives a freeze) and the ++ * RAM-only raw state in the blob. Reads RAM only. ++ */ ++static void g5p_rv3_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_r73b *x = g->boot->r73b; ++ const struct g5p_rv3_ev *t = &v->last[0], *d = &v->last[1]; ++ const struct g5p_ru_touch *k = v->touch; ++ ++ /* release: every job while the A/B is on, so both arms are complete past the verbose jobs. */ ++ if (r->touch_ms) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 touch seq=%u ret=%d arm=%s touch_ms=%u ta=%s/%lld td=%s/%lld " ++ "seen_us=%lld/%lld read_us=%lld/%lld after_us=%lld/%lld bytes=%u/%u " ++ "sum=%08x/%08x end=v3\n", ++ v->seq, ret, ++ g5p_ru_touch_arm(v->seq) ? "touch" : "control", ++ r->touch_ms, g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? (s64)(v->ta.end_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? (s64)(v->td.end_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_ru_touch_us(v->ta_pub_ns, k[0].seen_ns), ++ g5p_ru_touch_us(v->td_pub_ns, k[1].seen_ns), ++ g5p_ru_touch_us(k[0].seen_ns, k[0].done_ns), ++ g5p_ru_touch_us(k[1].seen_ns, k[1].done_ns), ++ g5p_ru_touch_after_us(&k[0], v->ta.result, ++ v->ta.end_ns), ++ g5p_ru_touch_after_us(&k[1], v->td.result, ++ v->td.end_ns), ++ k[0].bytes, k[1].bytes, k[0].sum, k[1].sum); ++ /* wake110: every job while the wake is on (the end/lat/retire lines stop at seq 8). */ ++ if (r->wake) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 wake seq=%u ret=%d mode=%u seed=%08x arm=%s/%s did=%s/%s " ++ "power=%02x/%02x fence=%08x/%08x gpc=%08x/%08x cons_at=%u/%u bell_us=%lld/%lld " ++ "lock_us=%lld/%lld ta=%s/%lld td=%s/%lld ev_ms=%lld/%lld consumed_ms=%lld/%lld " ++ "busy_ms=%lld/%lld t1_pub=%u/%u t1_end=%u/%u endpoint=%u ta_stamp=%08x/%08x " ++ "td_stamp=%08x/%08x ta_ring=%u/%u td_ring=%u/%u rang=%u end=v3\n", ++ v->seq, ret, r->wake, r->wake_seed, ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 0)], ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[0], ++ G5P_WK_DIDS - 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[1], ++ G5P_WK_DIDS - 1)], ++ v->wk_power[0], v->wk_power[1], v->wk_fence[0], ++ v->wk_fence[1], v->wk_gpc[0], v->wk_gpc[1], ++ v->wk_cons[0], v->wk_cons[1], ++ g5p_rv3_wk_us(v->ta_pub_ns, v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->td_pub_ns, v->wk_done_ns[1]), ++ g5p_rv3_wk_us(v->wk_start_ns[0], v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->wk_start_ns[1], v->wk_done_ns[1]), ++ g5p_rv3_results[v->ta.result], ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_rv3_results[v->td.result], ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->wk_first_ns[0]), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->wk_first_ns[1]), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td_consumed_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.busy_ns), ++ v->wk_t1_pub[0], v->wk_t1_pub[1], v->wk_t1_end[0], ++ v->wk_t1_end[1], g5p_ru_endpoint(v->seq), t->stamp[0], ++ t->stamp[1], d->stamp[0], d->stamp[1], t->prod, t->cons, ++ d->prod, d->cons, r->wake_rang); ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 end seq=%u ret=%d phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " ++ "cap_ms=%u ta_consumed_ms=%lld td_consumed_ms=%lld late_gap=%u/%u " ++ "late_after=%u/%u grace=%u bells=%d ta_ring=%u/%u td_ring=%u/%u " ++ "ta_stamp=%08x/%08x td_stamp=%08x/%08x pm80=%08x r0_after_ta_publish=bell-only " ++ "end=v3\n", ++ v->seq, ret, ++ g5p_rv3_phase_names[min_t(u32, v->phase, G5P_RV3_PHASES - 1)], ++ g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? ++ (s64)(v->ta.end_ns - v->ta_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? ++ (s64)(v->td.end_ns - v->td_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ r->wait_ms, G5P_RU_V3_CAP_MS, ++ v->ta_consumed_ns ? (s64)(v->ta_consumed_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td_consumed_ns ? (s64)(v->td_consumed_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_gap, v->td.late_gap, v->ta.late_after, ++ v->td.late_after, v->grace_end, atomic_read(&x->r74f_bells), ++ t->prod, t->cons, d->prod, d->cons, t->stamp[0], t->stamp[1], ++ d->stamp[0], d->stamp[1], t->pm80 ? t->pm80 : d->pm80); ++ if (r->hard_ms) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 lat seq=%u soft_ms=%u hard_ms=%u obs_ms=%u slow=%u/%u late_soft=%u/%u " ++ "silent=%u/%u soft_seen_ms=%lld/%lld busy_ms=%lld/%lld late_after_ms=%lld/%lld " ++ "events=%u end=v3\n", ++ v->seq, r->soft_ms, r->hard_ms, r->obs_ms, v->ta.slow, ++ v->td.slow, v->ta.late_soft, v->td.late_soft, ++ v->ta.silent, v->td.silent, ++ v->ta.soft_seen_ns ? ++ (s64)(v->ta.soft_seen_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.soft_seen_ns ? ++ (s64)(v->td.soft_seen_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.busy_ns ? (s64)(v->ta.busy_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.busy_ns ? (s64)(v->td.busy_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_ns ? (s64)(v->ta.late_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.late_ns ? (s64)(v->td.late_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ READ_ONCE(g->boot->events)); ++ /* repeat109: the 3D's firmware retirement (coherent_ms, event_ms) apart from the host's decision ++ * (decided_ms; held_ms = how long a coherent target waited for it), the clauses still open at the ++ * deciding or closing sample (unsat=TA/3D, render-user-retire.h bits) and that sample's 3D idle words. ++ */ ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 retire seq=%u ret=%d td=%s how=%s settle_ms=%u coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld held_ms=%lld unsat=%#x/%#x " ++ "idle=%08x,%08x,%08x,%08x,%08x,%08x run=%u breaks=%u busy_ms=%lld clear_ms=%lld " ++ "ta_how=%s ta_unsat=%#x ta_coherent_ms=%lld ta_event_ms=%lld " ++ "ta_idle=%08x,%08x,%08x end=v3\n", ++ v->seq, ret, g5p_rv3_results[v->td.result], ++ g5p_rv3_how[min_t(u32, v->rt.decided, 3)], r->settle_ms, ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td_event_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_ru_rt_ms(v->rt.coherent_ns, v->td.end_ns), ++ g5p_rv3_ta_unsat(t), v->rt.unsat, d->idle[0], d->idle[1], ++ d->idle[2], d->idle[3], d->idle[4], d->idle[5], v->rt.run, ++ v->rt.breaks, g5p_ru_rt_ms(v->td_pub_ns, v->rt.busy_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.clear_ns), ++ g5p_rv3_ta_how[min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1)], ++ v->tr.unsat, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta_event_ns), t->idle[0], ++ t->idle[1], t->idle[2]); ++ /* The raw rows once per context (its first job) and for every failure: the blob is not a ++ * frame log. ++ */ ++ if (!v->touched || (r->jobs > 1 && !ret)) ++ return; ++ g5p_blob_hex(g, "RENDER-V3 PM-control", g5p_rv3_pm_cpu(g->boot, r), ++ 0x140); ++ g5p_blob_hex(g, "RENDER-V3 TA-W", ++ g5p_ru_page(g->boot, 0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, v->seq)), ++ G5P_RU_TA_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 TA-CH", g5p_ru_page(g->boot, 0), 0x340); ++ /* The stamp words (+0x1840/+0x1844) live in this 0x80-byte window; no earlier blob held one. */ ++ g5p_blob_hex(g, "RENDER-V3 TA-STAMP", ++ g5p_ru_page(g->boot, 0) + G5P_R22_TQ, 0x80); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) { ++ g5p_blob_hex(g, "RENDER-V3 3D-W", ++ g5p_ru_page(g->boot, 1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ v->seq)), ++ G5P_RU_3D_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 3D-CH", g5p_ru_page(g->boot, 1), ++ 0x340); ++ g5p_blob_hex(g, "RENDER-V3 3D-STAMP", ++ g5p_ru_page(g->boot, 1) + G5P_R22_TQ, 0x80); ++ } ++} ++ ++/* Jobs >= 2: the legacy job>=2 PM step with policy 0 (pool, list and F rebuilt from the device ++ * pool, state S kept), whichever context the job belongs to. ++ */ ++static int g5p_rv3_pm_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks); ++ mutex_unlock(&rd->lock); ++ kvfree(physical); ++ return ret; ++} ++ ++/* render-user-loss.h for a phase that did not complete. */ ++static int g5p_rv3_loss(struct g5p_drm *d, struct g5p_render *r, bool first, ++ u32 res, const struct g5p_ru_wait *w) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 drain_ms = (45000); ++ struct g5p_ru_loss_in in = { ++ .result = res, ++ .first = first, ++ .isolate = drain_ms && ++ g5p_ru_drain_valid(drain_ms, r->hard_ms, r->obs_ms), ++ .silent = w->silent, ++ .fault = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ }; ++ ++ return g5p_ru_loss(&in); ++} ++ ++/* ++ * The phase on engine `e` of this job timed out with the firmware alive: lose the job and its ++ * context, not the device. No terminal note and no device failure: the job ends -ETIMEDOUT ++ * (g5p_render_run() marks the context bad, so its client gets -EIO from now on and its pair and ++ * memory stay retained), and the channel is blocked behind the stuck phase until the scheduler's ++ * drain (g5p_rv3_drain()) clears it. RAM only. Fails (-ENOMEM) only if the record cannot be ++ * allocated; the caller then ends the epoch as before. ++ */ ++static int g5p_rv3_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ const struct g5p_ru_wait *w = e ? &v->td : &v->ta; ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = e; ++ s->drain_ms = (45000); ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u: context " ++ "lost, device kept, channel blocked end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++/* A job that stopped the firmware may be recovered from in this boot (core recover.inc, ++ * render-user-recover.h). A refusal is one line: why the epoch ends. ++ */ ++static bool g5p_rv3_recoverable(struct g5p_drm *d, bool first) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_ru_recover_in in = { ++ .first = first, ++ .retired = g5p_r8_is_retired(), ++ .closed = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .recoveries = rd->recoveries, ++ .streak = rd->recover_streak, ++ .budget = (1), ++ .streak_max = (3), ++ }; ++ ++ if (g5p_ru_recoverable(&in)) ++ return true; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 no-recover why=%s recoveries=%u streak=%u recover=%u streak_max=%u " ++ "boot_max=%u first=%u end=v3\n", ++ g5p_ru_recover_refusal(&in), in.recoveries, in.streak, ++ in.budget, in.streak_max, G5P_RU_RECOVER_BOOT_MAX, first); ++ return false; ++} ++ ++/* The job stopped the firmware (silent stream, a fault): lose it and its context, not the ++ * device; the worker recovers the GPU before it picks anything (g5p_rv3_recover()). RAM only. ++ */ ++static void g5p_rv3_lose_recover(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e, ++ const char *why) ++{ ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_fw_recovery rec = b->fw_recovery; ++ u32 recs = READ_ONCE(b->fw_recoveries); ++ ++ v->lost = true; ++ WRITE_ONCE(d->rdev->recovering, true); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u why=%s: context lost, device " ++ "recovers end=v3\n", ++ seq, e, r->slot[0], r->slot[1], why); ++ /* The guilty job is the one waited on (one job in flight, core lock); the packet's slot ++ * names the phase the firmware stopped in (render-user-recover.h). RAM only. ++ */ ++ if (recs) ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-attrib seq=%u engine=%u packet_slot=%u match=%u code=%08x " ++ "w20=%08x w24=%08x packets=%u end=v3\n", ++ seq, e, rec.slot, rec.slot == e, rec.code, rec.w20, ++ rec.w24, recs); ++} ++ ++/* render_user_test_fault_job: this job's TA reads its control stream (W+0x98, g5p_ru_fill_ta()) ++ * from the page the context reserved in its VM and never maps, so the GPU faults in that VM. A ++ * RAM edit of the job's own W copy before it is staged; the firmware sees an ordinary job. ++ */ ++static void g5p_rv3_test_fault(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, u32 seq) ++{ ++ g5p_put64(job->ta_w + 0x98, r->fault_node.start); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-fault seq=%u job=%u slots=%u/%u va=%#llx: TA control stream " ++ "unmapped end=v3\n", ++ seq, r->jobs + 1, r->slot[0], r->slot[1], r->fault_node.start); ++} ++ ++/* One V3 job: [attach, the epoch's job 1 only | bind, a later context's first job,] TA, 3D. ++ * Core lock held. Returns 0, -ECANCELED (a latched stage ended it), -ETIMEDOUT (a phase ran out ++ * its deadline, or the bind's DC3 was not consumed), or a fault. ++ */ ++/* perf60 MMU arm, bits 1/3 (render_user_mmu_inval): before a bound context's job publishes ++ * anything, move its slots' host tag to a fresh PC field, so the firmware's tag compare at this ++ * job's TA and 3D kicks takes its PC-level invalidate (e138 = (S << 3) | 4) for each slot. RAM ++ * only. vm->mutation is left alone: an edit on another thread (VM_BIND of a render VM takes no ++ * core lock) recomputes its own tag and checks it only against the cached tags, as before. ++ */ ++static void g5p_rv3_mmu_retag(struct g5p_diag *g, struct g5p_render *r, u32 seq, ++ bool switched) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 host[2], cached[2], tag; ++ ++ dma_rmb(); ++ for (u32 e = 0; e < 2; e++) { ++ host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4); ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ } ++ tag = g5p_ru_mmu_pc_tag(host[0] ?: cached[0], cached[0], cached[1]); ++ if (tag) { ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, ++ tag); ++ dma_wmb(); ++ } ++ g5p_mmu_record( ++ g, ++ "RENDER-V3 mmu-retag seq=%u slots=%u/%u why=%s host=%08x/%08x cached=%08x/%08x " ++ "tag=%08x end=v3\n", ++ seq, r->slot[0], r->slot[1], switched ? "switch" : "every", ++ host[0], host[1], cached[0], cached[1], tag); ++} ++ ++/* perf60 A/B: the park that the never-park envelope removed, back around a context switch only. ++ * Before a bound context's job that follows another context's job, ctrl+0x1040 = 50 makes the ++ * firmware's next pass compute x < 100 (policy state 2: it parks the GPC when idle, as release7); ++ * after the window the pin goes back and the job runs in state 0 with its bells as usual. RAM ++ * only: the firmware reloads policy+0x6f4 from ctrl+0x1040 on every pass (perf-envelope.h). ++ */ ++ ++ ++ ++/* perf60: the incoming context's render targets on a switch (lat123_log), RAM. */ ++static void g5p_rv3_switch_line(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ char buf[160]; ++ struct g5p_rt *rt; ++ u32 n = 0, len = 0; ++ ++ buf[0] = 0; ++ list_for_each_entry(rt, &r->targets, link) ++ if (n++ < 4) ++ len += scnprintf(buf + len, sizeof(buf) - len, ++ " %ux%u/s%u/p%u%s", rt->width, ++ rt->height, rt->samples ?: 1, ++ rt->page_count, ++ rt->destroyed ? "/d" : ""); ++ dev_info(g->dev, ++ "RENDER-V3 switch seq=%u to=%u/%u jobs=%u rts=%u%s end=v3\n", ++ seq, r->slot[0], r->slot[1], r->jobs, n, buf); ++} ++ ++/* perf60 per-context PM state (render_user_pm_switch, default 0). The PM record's selected state ++ * S (control+0x80, 0xc0 bytes: the backing pointers a0/a8/b0 the fresh TA loads the PM context ++ * from, the lanes TA finish saves and the 3D start restores, the bc accumulator) and the five ++ * aux pages it points into (the PM context store) are built once at attach and then shared by ++ * every context; the host never rewrites them. iOS builds one PM record per work queue ++ * (6cb2378: S explicit zeros plus the four backing pointers) and keeps it per queue. ++ * 1: a context's job after another context's job starts from attach's fresh state (S rebuilt, ++ * aux zeroed); ++ * 2: each context keeps its own S + aux: saved after each of its completed jobs, restored when ++ * it follows another context's job, fresh the first time. ++ * RAM only, between jobs (the previous job retired, this one has published nothing). ++ */ ++ ++/* perf60 stage 4: (re)build context r's PM set for this epoch: grow its pool to the device's ++ * block count, zero its aux pages, build its control + S, list and F (g5p_ru_pm(), as attach ++ * builds the device PM) and clean them to PoC. rd->lock (the pool's size) held. ++ */ ++static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ const char *how) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_dma_page *page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ u32 blocks = rd->blocks; ++ u64 *physical, list_va, free_va; ++ int ret = 0; ++ ++ lockdep_assert_held(&rd->lock); ++ if (blocks > r->pm_blocks) { ++ struct g5p_dma_page *pool = ++ kvcalloc(blocks * 8, sizeof(*pool), GFP_KERNEL); ++ ++ if (!pool) ++ return -ENOMEM; ++ if (r->pm_pool) ++ memcpy(pool, r->pm_pool, ++ r->pm_blocks * 8 * sizeof(*pool)); ++ for (u32 i = r->pm_blocks * 8; !ret && i < blocks * 8; i++) ++ ret = g5p_vm_alloc_page(d->drm.dev, &pool[i]); ++ if (ret) { ++ for (u32 i = r->pm_blocks * 8; i < blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &pool[i]); ++ kvfree(pool); ++ return ret; ++ } ++ kvfree(r->pm_pool); ++ r->pm_pool = pool; ++ r->pm_blocks = blocks; ++ } ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ for (u32 i = 0; i < r->pm_blocks * 8; i++) { ++ physical[i] = r->pm_pool[i].dma; ++ memset(r->pm_pool[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm_pool[i]); ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ struct g5p_dma_page *aux = &g5p_rv3_pm_aux(rd, r)[i]; ++ ++ memset(aux->cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, aux); ++ } ++ list_va = g5p_pmset_va(r->pm_set, G5P_PMSET_LIST); ++ free_va = g5p_pmset_va(r->pm_set, G5P_PMSET_FREE); ++ ret = g5p_ru_pm(page[G5P_PMSET_CONTROL].cpu, page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, list_va, free_va, physical, ++ r->pm_blocks, G5P_RU_PRIVATE_VA); ++ kvfree(physical); ++ for (u32 k = 0; k < G5P_PMSET_PAGES; k++) ++ g5p_table_clean_page(g, &page[k]); ++ dma_wmb(); ++ if (!ret) ++ r->pm_epoch = rd->recoveries + 1; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u meta=%#x " ++ "ret=%d end=v3\n", ++ seq, r->slot[0], r->slot[1], how, r->pm_set, ++ g5p_rv3_pm_id(rd, r), r->pm_blocks, g5p_rv3_pm_meta(r), ret); ++ return ret; ++} ++ ++/* Before a context's bind (its tables are published by the bind that follows): take a free set, ++ * allocate its aux pages, point G5P_RU_PRIVATE_VA in its VM at them, and build the set. On any ++ * failure the context keeps the device PM and the shared aux pages, as before stage 4. ++ */ ++static void g5p_rv3_pmctx_assign(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ int set, ret = 0; ++ ++ if (r->pm_set >= 0) ++ return; ++ set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next); ++ for (u32 i = 0; set >= 0 && !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!r->pm_aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &r->pm_aux[i]); ++ if (set < 0 || ret) { ++ dev_warn( ++ d->core->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=none set=%d ret=%d: the device PM end=v3\n", ++ seq, r->slot[0], r->slot[1], set, ret); ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "new"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ return; ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ u64 va = G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE; ++ u8 *pte = g5p_vm_pte(vm, d->drm.dev, va, false); ++ ++ u64 leaf = g5p_leaf_entry(r->pm_aux[i].dma, ++ g5p_r25_attributes(&r->v, va)); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, leaf, 2); ++ } ++ dma_wmb(); ++ r->pm_aux_mapped = true; ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* Before each job of a context with a set: a new epoch or a grown device pool rebuilds the set; ++ * without pm_once the pool is rebuilt per job, as g5p_rv3_pm_rebuild() does the device's. ++ */ ++static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_dma_page *page; ++ u64 *physical; ++ int ret = 0; ++ ++ if (r->pm_set < 0) ++ return 0; ++ page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ mutex_lock(&rd->lock); ++ if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { ++ ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); ++ } else if (!rd->pm_once) { ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), ++ GFP_KERNEL); ++ ret = physical ? 0 : -ENOMEM; ++ for (u32 i = 0; !ret && i < r->pm_blocks * 8; i++) ++ physical[i] = r->pm_pool[i].dma; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ page[G5P_PMSET_CONTROL].cpu, ++ page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, ++ g5p_pmset_va(r->pm_set, G5P_PMSET_LIST), ++ g5p_pmset_va(r->pm_set, G5P_PMSET_FREE), ++ physical, r->pm_blocks); ++ kvfree(physical); ++ } ++ mutex_unlock(&rd->lock); ++ return ret; ++} ++ ++/* render_user_pm_ctx 3: the context that brought the epoch up (the device PM for its attach job) ++ * takes its own set before its next job; the device aux pages its VM already maps become the ++ * set's aux (no other context maps them once every later one has its own), so no PTE changes. ++ */ ++static void g5p_rv3_pmctx_take(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ int set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next), ++ ret; ++ ++ if (set < 0) { ++ r->pm_first = false; /* stays on the device PM; not retried */ ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "take"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ r->pm_first = false; ++ return; ++ } ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* perf60 stage 4 diagnostics (render_user_fault_read): whose page each PM lane in S points at ++ * after the TA saved them: the device pool or aux, a set page, a context's pool or aux. ++ */ ++ ++static void g5p_rv3_pm_lanes(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ { ++ return; ++ } ++} ++ ++/* The context is gone (g5p_render_release()): its set is free again; its pages go. */ ++static void g5p_rv3_pmctx_release(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ ++ if (r->pm_set >= 0) ++ rd->pm_set_used[r->pm_set] = false; ++ r->pm_set = -1; ++ for (u32 i = 0; r->pm_pool && i < r->pm_blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_pool[i]); ++ kvfree(r->pm_pool); ++ r->pm_pool = NULL; ++ r->pm_blocks = 0; ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_aux[i]); ++} ++ ++#define G5P_RV3_PM_S 0x80U ++#define G5P_RV3_PM_S_LEN 0xc0U ++#define G5P_RV3_PM_SAVE (G5P_RV3_PM_S_LEN + G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ ++/* perf60 fault address: PM state snapshots (S, PM control, the aux windows S points at, each aux ++ * page's head) at three phases of every job, in a ring and per slot pair; printed at a fault read. ++ * Only the executor writes them. RAM only. ++ */ ++enum { G5P_PMSNAP_START, G5P_PMSNAP_TA_DONE, G5P_PMSNAP_END }; ++struct g5p_pmsnap { ++ u64 ns, auxva[3]; ++ u32 seq, phase, slot, valid; ++ u8 s[G5P_RV3_PM_S_LEN], ctl[0x80], win[3][64], ++ head[G5P_RU_AUX_PAGES][64]; ++}; ++#define G5P_PMSNAP_RING 9U ++static struct g5p_pmsnap g5p_pmsnap_ring[G5P_PMSNAP_RING]; ++static struct g5p_pmsnap g5p_pmsnap_slot[16][3]; ++static u32 g5p_pmsnap_head; ++ ++static void g5p_rv3_pmsnap(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 phase) ++{ ++ { ++ return; ++ } ++} ++ ++static void g5p_rv3_pmsnap_print(struct g5p_diag *g, u32 seq, const char *set, ++ const struct g5p_pmsnap *x) ++{ ++ u64 now = ktime_get_ns(); ++ ++ if (!x->valid) ++ return; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pmsnap seq=%u set=%s job=%u slot=%u phase=%s ago_us=%llu " ++ "aux_va=%#llx/%#llx/%#llx end=v3\n", ++ seq, set, x->seq, x->slot, ++ x->phase == G5P_PMSNAP_START ? "start" : ++ x->phase == G5P_PMSNAP_TA_DONE ? "ta-done" : ++ "end", ++ (now - x->ns) / NSEC_PER_USEC, x->auxva[0], x->auxva[1], ++ x->auxva[2]); ++ for (u32 o = 0; o < sizeof(x->s); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap S+%02x %*ph\n", o, 64, ++ x->s + o); ++ for (u32 o = 0; o < sizeof(x->ctl); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap C+%02x %*ph\n", o, 64, ++ x->ctl + o); ++ for (u32 k = 0; k < 3; k++) ++ dev_info(g->dev, "RENDER-V3 pmsnap W%u %*ph\n", k, 64, ++ x->win[k]); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ dev_info(g->dev, "RENDER-V3 pmsnap A%u %*ph\n", i, 64, ++ x->head[i]); ++} ++ ++static void g5p_rv3_pmsnap_dump(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ for (u32 p = 0; p < 3; p++) ++ g5p_rv3_pmsnap_print(g, seq, "slot-last", ++ &g5p_pmsnap_slot[r->slot[0] & 15][p]); ++ for (u32 i = G5P_PMSNAP_RING; i; i--) ++ g5p_rv3_pmsnap_print(g, seq, "ring", ++ &g5p_pmsnap_ring[(g5p_pmsnap_head - i) % ++ G5P_PMSNAP_RING]); ++} ++ ++/* perf60 fault address: the VM edit log entries that cover the fault VA (any VM, newest first), ++ * then the faulting job's VM's latest edits. Copied under the lock, printed outside it. ++ */ ++#define G5P_VMLOG_HITS 24U ++#define G5P_VMLOG_TAIL 16U ++static void g5p_vmlog_dump(struct g5p_diag *g, u32 seq, const void *vm, u64 va, ++ bool have_va) ++{ ++ struct g5p_vmlog_ent *out = kmalloc_array( ++ G5P_VMLOG_HITS + G5P_VMLOG_TAIL, sizeof(*out), GFP_KERNEL); ++ u32 hits = 0, tail = 0, n, head; ++ unsigned long flags; ++ u64 now = ktime_get_ns(); ++ ++ if (!out) ++ return; ++ spin_lock_irqsave(&g5p_vmlog.lock, flags); ++ head = g5p_vmlog.head; ++ n = min(head, G5P_VMLOG_N); ++ for (u32 i = 1; i <= n; i++) { ++ const struct g5p_vmlog_ent *x = ++ &g5p_vmlog.e[(head - i) % G5P_VMLOG_N]; ++ ++ if (have_va && hits < G5P_VMLOG_HITS && ++ g5p_ru_mmu_fault_in(va, x->va, x->size)) ++ out[hits++] = *x; ++ else if (x->vm == vm && tail < G5P_VMLOG_TAIL) ++ out[G5P_VMLOG_HITS + tail++] = *x; ++ } ++ spin_unlock_irqrestore(&g5p_vmlog.lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u va=%#llx edits=%u covering=%u vm_tail=%u end=v3\n", ++ seq, have_va ? va : 0, head, hits, tail); ++ for (u32 i = 0; i < hits + tail; i++) { ++ const struct g5p_vmlog_ent *x = ++ &out[i < hits ? i : G5P_VMLOG_HITS + i - hits]; ++ ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u %s ago_ms=%llu op=%s pid=%d comm=%s vm=%s/%u va=%#llx " ++ "size=%#llx handle=%#x off=%#llx end=v3\n", ++ seq, i < hits ? "cover" : "tail", ++ (now - x->ns) / NSEC_PER_MSEC, ++ g5p_vmlog_ops[min_t(u32, x->op, ++ ARRAY_SIZE(g5p_vmlog_ops) - 1)], ++ x->tgid, x->comm, x->vm == vm ? "this" : "other", ++ x->vm_id, x->va, x->size, x->handle, x->offset); ++ } ++ kfree(out); ++} ++ ++/* perf60 fault address (render_user_fault_read): the MMU fault banks, read once at a failed ++ * phase, the way iOS's fault handler (6ca5c14) reads them; the firmware keeps no copy. Bell mode ++ * only. Under the event lock with the wake path's power proof (the fault already latched ++ * b->failed, so that flag is not a guard here), then the GPU register licence (mmio.inc, ++ * mmio-access.md §4): no wake (Apple's reads never write R1+0x14, G6), a refusal retried outside ++ * the locks every 20-40 us for at most G5P_RV3_DC_SETTLE_US, and the four bank reads only under a ++ * held ticket. A parked GPC records skip-parked, as Apple's report has no bif fault entry then. ++ * At most G5P_RV3_FAULT_READS per boot. It runs before the loss decision and the terminal ++ * latch, while the faulted job is still the GPU's current work. ++ */ ++#define G5P_RV3_FAULT_READS G5P_RU_RECOVER_BOOT_MAX ++static atomic_t g5p_rv3_fault_reads = ATOMIC_INIT(0); ++ ++/* faultrec: a fault VA in the PM-set window (render-user-pmset.h) names a set and its part; say ++ * which context owns that set now and how it stands against the two job records and the deferred ++ * closes (render_user_close_defer). RAM only; rdev lock for the context list. ++ */ ++static void g5p_rv3_fault_pmset(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq, u32 bank, ++ const struct g5p_ru_mmu_fault *f) ++{ ++ static const char *const parts[G5P_PMSET_PAGES] = { "control", "list", ++ "free" }; ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_render *o, *owner = NULL; ++ u32 page, set, own[2] = { U32_MAX, U32_MAX }, df[2][2]; ++ bool last = false, prev = false, closing = false, bound = false; ++ ++ if (f->va < G5P_PMSET_VA || f->va >= G5P_PMSET_END) ++ return; ++ page = (f->va - G5P_PMSET_VA) / G5P_GPU_PAGE; ++ set = page / G5P_PMSET_PAGES; ++ mutex_lock(&rd->lock); ++ list_for_each_entry(o, &rd->contexts, rdev_link) ++ if (o->pm_set == (int)set) { ++ owner = o; ++ break; ++ } ++ if (owner) { ++ own[0] = owner->slot[0]; ++ own[1] = owner->slot[1]; ++ last = rd->pm_last == owner; ++ prev = rd->pm_prev == owner; ++ closing = owner->closing; ++ bound = owner->bound; ++ } ++ for (u32 k = 0; k < 2; k++) { ++ o = rd->close_deferred[k]; ++ df[k][0] = o ? o->slot[0] : U32_MAX; ++ df[k][1] = o ? o->slot[1] : U32_MAX; ++ } ++ mutex_unlock(&rd->lock); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-pmset seq=%u bank=%u mmu_context=%u set=%u part=%s owner=%d/%d " ++ "bound=%u closing=%u last=%u prev=%u job=%u/%u deferred=%d/%d,%d/%d job_slots=%u " ++ "end=v3\n", ++ seq, bank, f->context, set, parts[page % G5P_PMSET_PAGES], ++ (int)own[0], (int)own[1], bound, closing, last, prev, ++ r->slot[0], r->slot[1], (int)df[0][0], (int)df[0][1], ++ (int)df[1][0], (int)df[1][1], rd->job_slots); ++} ++ ++static void g5p_rv3_fault_read(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 e, int res, bool silent) ++{ ++ u32 full = (0), w[4] = { 0 }; ++ u32 mode = full | ((1) ? 1 : 0); ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_mmu_fault f[2]; ++ struct g5p_mmio_ticket t = { .r1_0 = U32_MAX, ++ .gpc = U32_MAX, ++ .r1_0_end = U32_MAX, ++ .gpc_end = U32_MAX }; ++ const char *did = "skip-guard"; ++ u32 power = 0, tries = 0; ++ unsigned long flags; ++ bool fault = res == G5P_RU_WAIT_FAULT && !silent, ticket = false; ++ u64 t0; ++ int ret; ++ ++ if ((2) != G5P_WK_ALWAYS || !(mode & (fault ? 1 : 2))) ++ return; ++ if (atomic_inc_return(&g5p_rv3_fault_reads) > G5P_RV3_FAULT_READS) { ++ did = "skip-cap"; ++ goto print; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ for (;;) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->stopped) || !g5p_wk_power_ready(power)) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ break; ++ } ++ tries++; ++ ticket = true; ++ ret = g5p_mmio_hold(g, &t, 0); ++ if (!ret) { ++ w[0] = g5p_mmio_r0_read(g, &t, 0xe150); ++ w[1] = g5p_mmio_r0_read(g, &t, 0xe154); ++ w[2] = g5p_mmio_r0_read(g, &t, 0xe158); ++ w[3] = g5p_mmio_r0_read(g, &t, 0xe15c); ++ g5p_mmio_release(g, &t); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ did = g5p_mmio_did(&t); ++ if (ret != -EAGAIN || ++ ktime_get_mono_fast_ns() - t0 >= ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) ++ break; ++ usleep_range(20, 40); ++ } ++ if (ticket) ++ g5p_mmio_sample_line(g, G5P_MMS_FAULT, &t, w, t.ops ? 4 : 0, ++ true); ++print: ++ f[0] = g5p_ru_mmu_fault_decode(w[0], w[1]); ++ f[1] = g5p_ru_mmu_fault_decode(w[2], w[3]); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-read seq=%u engine=%u slots=%u/%u why=%s did=%s power=%02x " ++ "fence=none r1_0=%08x gpc=%08x tries=%u e150=%08x:%08x e158=%08x:%08x end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ fault ? "fault" : ++ silent ? "silent" : ++ "timeout", ++ did, power, t.r1_0, t.gpc, tries, w[1], w[0], w[3], w[2]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-decode seq=%u bank=%u va=%#llx %s type=%s level=%u " ++ "requester=%u/%s sideband=%#x context=%u job_slots=%u/%u end=v3\n", ++ seq, k, f[k].va, f[k].read ? "read" : "write", ++ g5p_ru_mmu_fault_types[f[k].type], f[k].level, ++ f[k].requester, ++ g5p_ru_mmu_requester(f[k].requester), ++ f[k].sideband, f[k].context, r->slot[0], ++ r->slot[1]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ g5p_rv3_fault_pmset(d, r, seq, k, &f[k]); ++ g5p_rv3_switch_line(g, r, seq); ++ if (!full) ++ return; ++ g5p_vmlog_dump(g, seq, r->ctx->vm, f[0].valid ? f[0].va : f[1].va, ++ f[0].valid || f[1].valid); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ g5p_rv3_pmsnap_dump(g, r, seq); ++} ++ ++/* GPU overclock thermal stop (core boot.inc, boot_gpu_oc_mhz): every 500 ms while the epoch runs ++ * state 5, the three SoC zones; at or above boot_gpu_oc_temp_mc (or unreadable) the overclock is off ++ * for the rest of the boot and the render scheduler recovers the GPU (g5p_rv3_recover(): gate cycle, ++ * firmware boot) between jobs, so the new epoch powers on at state 4 with record 5 cleared: iOS's ++ * park -> reboot -> new pstate, never a pstate change under a live firmware. Every 10 s one status ++ * line: the temperatures, the real Fender state (an R1-only licence ticket) and the jobs run. ++ */ ++#define G5P_OC_POLL_MS 500 ++static void g5p_rsched_kick(struct g5p_drm *d); ++ ++static void g5p_oc_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = ++ container_of(to_delayed_work(work), struct g5p_rdev, oc_work); ++ struct g5p_drm *d = rd->drm; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ u32 lim = (90000), zones; ++ const char *zone; ++ int t; ++ ++ if (!b || !0 || READ_ONCE(g5p_oc_tripped)) { ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ zones = g5p_oc_temp(&t, &zone); ++ if (zones != ARRAY_SIZE(g5p_oc_zone_names) || t >= (int)lim) { ++ WRITE_ONCE(g5p_oc_tripped, true); ++ dev_emerg( ++ g->dev, ++ "GPUPERF oc thermal-stop temp=%d zone=%s zones=%u limit=%u mhz=%u jobs=%u: the " ++ "GPU recovers at state 4 (723 MHz) end=gpuperf\n", ++ t, zone, zones, lim, b->gpu_oc_mhz, rd->oc_jobs); ++ WRITE_ONCE(rd->recovering, true); ++ g5p_rsched_kick(d); ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ if (!(++rd->oc_ticks % (10000 / G5P_OC_POLL_MS))) { ++ struct g5p_mmio_ticket tk; ++ ++ if (!g5p_mmio_hold(g, &tk, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &tk); ++ g5p_job_printk( ++ g->dev, ++ "GPUPERF oc status mhz=%u temp=%d zone=%s limit=%u r1=%s r1_1000=%08x jobs=%u " ++ "end=gpuperf\n", ++ b->gpu_oc_mhz, t, zone, lim, g5p_mmio_did(&tk), tk.gpc, ++ rd->oc_jobs); ++ } ++ queue_delayed_work(system_wq, &rd->oc_work, ++ msecs_to_jiffies(G5P_OC_POLL_MS)); ++} ++ ++ ++ ++static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ bool first = !rd->up, bound_before = r->bound; ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (job->compute) ++ return g5p_cu_execute(d, r, job); ++ if (r->stage_done) ++ return -ECANCELED; ++ g5p_l123_job_reset(g); ++ /* The submit-time admission again, now under the core lock and with the bind state: still nothing touched. */ ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ seq = rd->sequence + 1; ++ ; ++ 0; ++ if (bound_before && d->bell_prev_ctx && d->bell_prev_ctx != r->ctx) { ++ u32 ms = (0); ++ ++ ; ++ ; ++ if (ms && b->gpu_perf) ++ 0; ++ } ++ 0; ++ if (g5p_ru_v3_verbose(r->jobs + 1, 0)) ++ g5p_job_printk( ++ g->dev, ++ "RENDER-V3 start seq=%u bound_ms=%u cap_ms=%u slots=%u/%u attach=%s " ++ "ta_doorbell=%s 3d_doorbell=none wait=ram-only after_timeout=quiet end=v3\n", ++ seq, r->wait_ms, G5P_RU_V3_CAP_MS, r->slot[1], ++ r->slot[0], ++ first ? "firmware-dc3" : ++ r->bound ? "kept" : ++ rd->spare_count ? "spare" : ++ "ram-dc3", ++ first ? "grace-owned" : "none"); ++ if (first) { ++ v->phase = G5P_RV3_ATTACH; ++ ret = g5p_rv3_spares(d, v); ++ if (!ret) ++ ret = g5p_rv3_attach(d, r, v); ++ /* perf60 stage 4: a context with a set that brings a new epoch up */ ++ if (!ret) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ if (ret) ++ goto out; ++ ; ++ } else { ++ if (!r->bound) { ++ /* perf60 stage 4: its own PM set and aux pages before the bind ++ * publishes its tables ++ */ ++ if (rd->pm_ctx) ++ g5p_rv3_pmctx_assign(d, r, seq); ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ ; ++ } ++ /* The firmware owns the rings and the channels from here on: any failure quarantines. */ ++ v->touched = true; ++ if (rd->pm_ctx == 3 && r->pm_first && r->pm_set < 0) ++ g5p_rv3_pmctx_take(d, r, seq); ++ if (r->pm_set >= 0) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ else ++ ret = rd->pm_once ? 0 : g5p_rv3_pm_rebuild(d); ++ if (ret) ++ goto out; ++ ; ++ ; ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_START); ++ if (g5p_ru_mmu_retag_due((0), bound_before, ++ d->bell_prev_ctx != r->ctx)) ++ g5p_rv3_mmu_retag(g, r, seq, ++ d->bell_prev_ctx != r->ctx); ++ } ++ rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu(b, r) + 0x80); ++ if (g5p_ru_fault_due(r->fault_job, r->jobs, first)) ++ g5p_rv3_test_fault(g, r, job, seq); ++ for (u32 e = 0; e < 2; e++) { ++ g5p_ru_job_at( ++ g5p_ru_page(b, e), g5p_ru_meta(e), ++ g5p_meta_address(g5p_object_va(&b->layout, 0x51 + e)), ++ g5p_rv3_pm_meta(r), r->slot[e], e, seq, ++ e ? job->td_w : job->ta_w, ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r)); ++ r->submitted[e] = false; ++ r->consumed[e] = false; ++ r->retired[e] = false; ++ } ++ g5p_l123_job_prepare(d, job); ++ g5p_render_clean_job(g); ++ rd->sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_rv3_wake_mark(d, r, v, 0); ++ ret = g5p_rv3_publish_ta(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ if (r->stall_job && r->jobs == r->stall_job && !first) { ++ /* render_user_test_stall_job: the host gives up, the firmware runs on */ ++ g5p_ru_wait_start(&v->ta, v->ta_pub_ns, 1, v->job_deadline_ns); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-stall seq=%u job=%u slots=%u/%u: TA bound 1 ms end=v3\n", ++ seq, r->jobs, r->slot[0], r->slot[1]); ++ } else { ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, ++ v->job_deadline_ns); ++ } ++ g5p_ru_touch_start(&v->touch[0], v->ta_pub_ns, r->touch_ms, seq); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, seq, &v->ta, &v->touch[0], NULL, &v->tr, ++ &v->last[0], &v->ta_consumed_ns, &err); ++ r->consumed[0] = v->ta_consumed_ns != 0; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ /* Timeout or fault: (latency101: the RAM-only observation window first,) terminal ++ * latch, one RAM sample for the record (a completion that arrives now is evidence, ++ * never the result), no export, no register access. ++ */ ++ g5p_rv3_fault_read(d, r, seq, 0, res, v->ta.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 0)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 0, ++ v->ta.silent ? "ta-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "ta-timeout" : ++ "ta-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "ta-timeout" : "ta-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_TIMEOUT, ret); ++ goto out; ++ } ++ g5p_rv3_ta_count(g, d, v); ++ r->retired[0] = true; ++ g5p_rv3_terminal(g, G5P_RU_TERM_COMPLETE, "ta-complete"); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_TA_DONE); ++ g5p_rv3_pm_lanes(d, r, seq); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, ++ g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); ++ ; ++ v->phase = G5P_RV3_TD_PUBLISH; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample( ++ g, r, 1, seq, ++ &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ ++ goto out; ++ } ++ v->phase = G5P_RV3_TD_WAIT; ++ g5p_rv3_wait_start(g, r, &v->td, v->td_pub_ns, v->job_deadline_ns); ++ g5p_ru_touch_start(&v->touch[1], v->td_pub_ns, r->touch_ms, seq); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, ++ &v->last[1], &v->td_consumed_ns, &err); ++ r->consumed[1] = v->td_consumed_ns != 0; ++ /* mmio_selftest bit 2 (rung 5, DFU-risk on a park tuple): one licensed ticket at the 3D ++ * completion decision; always: the idle clock of bit 3. ++ */ ++ ; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ g5p_rv3_fault_read(d, r, seq, 1, res, v->td.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->td); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 1)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 1, ++ v->td.silent ? "3d-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "3d-timeout" : ++ "3d-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "3d-timeout" : "3d-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ /* Rule (c) decided: the matching event was not seen. Always one line (RAM only). */ ++ if (v->rt.decided == G5P_RU_RT_HOLD) ++ dev_warn( ++ g->dev, ++ "RENDER-V3 3d-hold seq=%u idle=%08x,%08x,%08x,%08x,%08x,%08x coherent_ms=%lld " ++ "decided_ms=%lld end=v3\n", ++ seq, v->last[1].idle[0], v->last[1].idle[1], ++ v->last[1].idle[2], v->last[1].idle[3], ++ v->last[1].idle[4], v->last[1].idle[5], ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns)); ++ r->retired[1] = true; ++ v->phase = G5P_RV3_DONE; ++ rd->recover_streak = ++ 0; /* a job finished: a later fault starts a new streak */ ++ ret = 0; ++out: ++ if (v->phase >= G5P_RV3_TA_PUBLISH) ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ g5p_rv3_wake_end(d, r, v); ++ /* Any other failure after the hardware was touched (attach, publish, drain, protocol): the same ++ * criterion g5p_render_run() quarantines the context by. A stage stop (-ECANCELED) is not one. ++ */ ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), ++ g5p_rv3_phase_names[min_t( ++ u32, v->phase, G5P_RV3_PHASES - 1)]); ++ g5p_rv3_receipt(g, r, job, ret); ++ g5p_l123_job_end(d, job, seq, ret); ++ if (!ret) { ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ ; ++ ; ++ } ++ /* render-user-pmset.h: the PM's write-back targets */ ++ if (g5p_pmset_switched(rd->pm_last, r)) { ++ rd->pm_prev = rd->pm_last; ++ rd->pm_last = r; ++ } ++ /* perf60 bell gates: the next job's gap and context switch are measured from here. */ ++ d->bell_prev_end_ns = ktime_get_ns(); ++ d->bell_prev_ctx = r->ctx; ++ return ret; ++} ++ ++/* ++ * The channel is blocked behind a lost job (g5p_rv3_lose()): wait for its stuck phase, RAM only, ++ * with the firmware's event stream watched from the first sample (silent for 1 s = dead). A late ++ * TA gets the job's 3D published (its W is staged since the job's start; no doorbell for a job ++ * other than the epoch's first), so the 3D channel's sequence and the PM state stay as the ++ * firmware expects; a late 3D clears the channel. Then the held jobs run. A silent stream, a ++ * fault, a refused 3D or the drain bound end the epoch as a timeout or fault always did, and the ++ * held jobs then fail at admission. Called by the scheduler's worker before any pick; returns ++ * false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_rv3_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 st = s->engine ? G5P_RU_DRAIN_TD : G5P_RU_DRAIN_TA, ++ res = G5P_RU_WAIT_PENDING; ++ struct g5p_ru_wait w = {}; ++ bool armed = false; ++ int act, err = -EIO; ++ ++ /* TYPED_COMPUTE: a lost CDM job has its own one-phase drain. */ ++ if (s->engine == G5P_CU_ENGINE) ++ return g5p_cu_drain(d); ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ for (;;) { ++ u32 e = st == G5P_RU_DRAIN_TA ? 0 : 1; ++ u64 now = ktime_get_ns(); ++ ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->stopped)) { ++ res = G5P_RU_WAIT_FAULT; ++ } else if (!e) { ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], ++ NULL, &v->tr, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ } else { ++ if (!armed) ++ g5p_rv3_event_arm(d, 1, ++ g5p_ru_endpoint(s->seq)); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], ++ &v->rt, NULL, &v->last[1], ++ &v->td_consumed_ns, &err); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ } ++ act = g5p_ru_drain_step(&st, res); ++ if (act != G5P_RU_DRAIN_PUBLISH_TD) ++ break; ++ r->retired[0] = true; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); ++ armed = true; ++ if (g5p_rv3_publish_3d(g, r, v, s->seq)) { ++ g5p_rv3_event_disarm(d, 1); ++ res = G5P_RU_WAIT_FAULT; ++ act = g5p_ru_drain_step(&st, res); ++ break; ++ } ++ } ++ if (act == G5P_RU_DRAIN_UNBLOCK) { ++ r->retired[1] = true; ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drained seq=%u engine=%u slots=%u/%u lost_ms=%lld: channel clear, " ++ "held jobs resume end=v3\n", ++ s->seq, s->engine, r->slot[0], r->slot[1], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drain dead seq=%u engine=%u phase=%s why=%s err=%d lost_ms=%lld: the " ++ "device recovers if it may, else the epoch ends end=v3\n", ++ s->seq, s->engine, ++ st == G5P_RU_DRAIN_DEAD && !v->td_pub_ns ? "ta" : "3d", ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal( ++ g, ++ fault ? G5P_RU_TERM_FAULT : G5P_RU_TERM_TIMEOUT, ++ fault ? "drain-fault" : "drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++/* The firmware window's leaves again from the new epoch's tables, in the same pages, so every ++ * render VM's root entry stays valid (g5p_render_fw_entry()). Core lock held. ++ */ ++static void g5p_render_fw_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_boot *b = d->core->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ ++ if (!w->built) ++ return; ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ w->pages = 0; ++ w->missing = 0; ++ for (u32 i = 0; i < G5P_RU_FW_PAGES; i++) { ++ u64 va = g5p_ru_fw_window_va[i]; ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (k == w->leaves) ++ continue; ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, ++ leaf & 1 ? leaf : 0); ++ w->pages += leaf & 1; ++ w->missing += !(leaf & 1); ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(d->core, &w->leaf[i]); ++} ++ ++/* ++ * A lost job stopped the firmware (g5p_rv3_lose_recover(), or a drain that died): cycle the GPU ++ * and boot the firmware again (core recover.inc), then start the render device over as at boot. ++ * Every context survives except the lost one: its bindings belonged to the old epoch, which the ++ * cycle ended, so each is unbound in RAM, its binding's reference dropped, and its next job binds ++ * again (the first job after the recovery runs the epoch's attach). The aux pages and the PM ++ * pool are zeroed as at their allocation. The held jobs run after this. Returns false, having ++ * done nothing, when the core lock is busy. A failed recovery leaves the device failed. ++ */ ++static bool g5p_rv3_recover(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ unsigned long deadline = jiffies + 2 * HZ, flags; ++ struct g5p_render **drop = NULL; ++ struct g5p_render *r; ++ u32 n = 0, count = 0, dropped = 0, stage; ++ int ret; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ count++; ++ } ++ mutex_unlock(&rd->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ rd->recoveries++; ++ rd->recover_streak++; ++ ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; ++ if (ret) { ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", ++ ret, rd->recoveries); ++ return true; ++ } ++ g5p_ru_term_reset(&g5p_rv3_term); ++ /* the new firmware's PM has written nothing back yet: no context is its last or previous */ ++ rd->pm_last = NULL; ++ rd->pm_prev = NULL; ++ rd->up = false; ++ rd->v3 = false; ++ rd->sequence = 0; ++ rd->pm_serial = 0; ++ rd->dc_pending = false; ++ rd->dc_wait = 0; ++ if (rd->spares) { ++ spin_lock_irqsave(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ if (!rd->spares[i].used) { ++ __clear_bit(rd->spares[i].slot[0], ++ g->user_slots); ++ __clear_bit(rd->spares[i].slot[1], ++ g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rd->spares[i].root); ++ kfree(rd->spares); ++ rd->spares = NULL; ++ rd->spare_count = 0; ++ } ++ g5p_render_fw_rebuild(d); ++ mutex_lock(&rd->lock); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ if (rd->aux[i].cpu) { ++ memset(rd->aux[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->aux[i]); ++ } ++ for (u32 i = 0; i < rd->blocks * 8; i++) { ++ memset(rd->physical[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->physical[i]); ++ } ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ r->jobs = 0; ++ r->dc_target = 0; ++ WRITE_ONCE(r->live_maps, false); ++ memset(r->submitted, 0, sizeof(r->submitted)); ++ memset(r->consumed, 0, sizeof(r->consumed)); ++ memset(r->retired, 0, sizeof(r->retired)); ++ /* nothing of the old epoch holds this context's memory any more */ ++ r->lost_drained = true; ++ WRITE_ONCE(r->ctx->live, false); ++ if (r->retained && n < count) { ++ r->retained = false; ++ drop[n++] = r; ++ } ++ } ++ mutex_unlock(&rd->lock); ++ spin_lock_irqsave(&d->status_lock, flags); ++ d->last_error = 0; ++ d->job_error = 0; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ for (u32 i = 0; i < n; i++) { ++ struct g5p_context *c = drop[i]->ctx; ++ ++ dropped++; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recovered n=%u contexts=%u unbound=%u streak=%u: held jobs resume, " ++ "the next job attaches end=v3\n", ++ rd->recoveries, count, dropped, rd->recover_streak); ++ return true; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +new file mode 100644 +index 000000000..bd86d7dff +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -0,0 +1,2549 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Userspace TA/3D render: render contexts, render targets and SUBMIT_RENDER. ++ * Builds on the r22 constructors through render-user-layout.h (byte-equal for ++ * the round16 inputs). Default off. The first accepted job claims the device ++ * for render; TYPED_V3 contexts then share it, each with its own VM and slot ++ * pair (docs/gpu/multi-client-design.md), the other profiles stay with that ++ * first context. Jobs of all contexts are serialized; no reset/recovery: any ++ * hardware failure retains everything. ++ */ ++#include "../../apple-g5p/render-user-layout.h" ++#include "../../apple-g5p/render-user-wait.h" ++#include "../../apple-g5p/render-user-touch.h" ++#include "../../apple-g5p/render-user-retire.h" ++#include "../../apple-g5p/render-user-wake.h" ++#include "../../apple-g5p/render-user-serial.h" ++#include "../../apple-g5p/render-user-terminal.h" ++#include "../../apple-g5p/render-user-bind.h" ++#include "../../apple-g5p/render-user-close.h" ++#include "../../apple-g5p/render-user-sched.h" ++#include "../../apple-g5p/render-user-window.h" ++#include "../../apple-g5p/render-user-msaa.h" ++#include "../../apple-g5p/render-user-loss.h" ++#include "../../apple-g5p/render-user-fault.h" ++#include "../../apple-g5p/render-user-mmu.h" ++#include "../../apple-g5p/render-user-faultreg.h" ++#include "render-user-validate.h" ++#include "drm-render-v3.h" ++#include "../../apple-g5p/compute-user-layout.h" ++#include "compute-user-validate.h" ++#include "drm-compute-v1.h" ++ ++/* The terminal state of the epoch (one GPU): from the first terminal note on, no register access. */ ++static struct g5p_ru_terminal g5p_rv3_term; ++ ++/* Defaults of the latched knobs. GET_RENDER_INFO reports VARIANT_ACTIVE when ++ * any knob differs, so the declarations below and that comparison share them. ++ */ ++#define G5P_RU_DEF_CACHE_VARIANT 3 ++#define G5P_RU_DEF_MMU_VARIANT 1 ++/* 20 s: the natural TA completion took 12.006 s in the r96 boots a 12 s cap turned into ++ * timeouts (render-user-wait.h); it is also the minimum of a TYPED_V3 job's per-phase bound. ++ */ ++#define G5P_RU_DEF_WAIT_MS G5P_RU_V3_WAIT_MS ++#define G5P_RU_DEF_PROFILE1_VDM_CONTROL 0x280 ++ ++static const unsigned int render_user_wait_ms = 20000; ++MODULE_PARM_DESC( ++ render_user_wait_ms, ++ "Latched per-phase TA/3D bound, 3000..30000 ms (default 20000; TYPED_V3 needs " ++ "20000..30000, absolute job cap 30000)"); ++/* latency101 (default off): host-side TYPED_V3 wait policy; nothing the firmware sees depends on it. */ ++static const unsigned int render_user_hard_ms = 60000; ++static const unsigned int render_user_soft_ms = 45000; ++static const unsigned int render_user_obs_ms = 2000; ++MODULE_PARM_DESC( ++ render_user_hard_ms, ++ "Latched TYPED_V3 hard per-phase bound soft..60000 ms, the job bound is twice " ++ "it; 0 = off, render_user_wait_ms and the 30000 job cap apply (default 0)"); ++MODULE_PARM_DESC( ++ render_user_soft_ms, ++ "Latched TYPED_V3 soft per-phase bound 20000..hard ms, 0 = 45000; needs " ++ "render_user_hard_ms"); ++MODULE_PARM_DESC( ++ render_user_obs_ms, ++ "Latched RAM-only observation window after a TYPED_V3 timeout, 0..5000 ms " ++ "(default 2000; needs render_user_hard_ms)"); ++/* release (default off): RAM-only touch A/B of the TYPED_V3 wait (render-user-touch.h). */ ++ ++/* repeat109: the TYPED_V3 3D completion without the idle words (render-user-retire.h). */ ++ ++/* wake110 (default off): Apple's submit kick at the TYPED_V3 publish (render-user-wake.h). */ ++/* release8 default */ ++ ++/* TEMPORARY (multi-client stage 1): the fallback for a firmware that never drains a DC3 ++ * published after a job retired. Remove once the device has settled that question. ++ */ ++ ++/* Stage 2 fault isolation (render-user-loss.h): how long the scheduler holds every other job ++ * while it waits, RAM only, for a lost job's stuck phase. Read at the loss. ++ */ ++ ++/* Stage 2 in-boot recovery (core recover.inc): a job that stopped the firmware loses only its ++ * context, and the GPU is power-cycled and its firmware booted again while the other contexts' ++ * jobs are held (each leaves the old epoch's firmware pages allocated); 0 = such a job ends the ++ * epoch, as before. How many: render_user_recover_streak. Off by default; the release tuple turns ++ * it on (faultiso F1-F4, 2026-11-02: two recoveries in one boot on hardware). ++ */ ++ ++/* render-user-recover.h: Apple restarts the GPU on every recovery packet. Nonzero: recover every ++ * time, but end the epoch after this many recoveries in a row with no job completing between ++ * them. 0 = render_user_recover counts recoveries per boot, as before (a boot's second fault then ++ * ends the epoch, however long the GPU ran in between). ++ */ ++ ++/* Fault-isolation test (host side only: the firmware runs the job as any other). A context ++ * created while this is N gives up on its N-th job's TA after 1 ms instead of its bound, so the ++ * host loses that job the way it loses a stuck one, while the firmware completes it late: the ++ * context is lost and the channel drained behind it. 0 = off (default). ++ */ ++ ++/* Fault-isolation test with a real GPU fault (faultiso). A context created by a CAP_SYS_RAWIO ++ * process while this is N reserves one page of its VM that is never mapped ++ * (G5P_RU_TEST_FAULT_VA), and its N-th job's TA fetches its control stream from that page ++ * instead of the validated stream: the TA reads an unmapped VA of the context's own VM. What ++ * the firmware does then (a fault event, a silent stream, or a stuck engine) is what the test ++ * records; each ends that job and loses its context. 0 = off (default). ++ */ ++ ++/* render_user_test_fault_job's page: the last GiB below the hardware VA limit, far from every ++ * kernel-private range (low sink, aux, PDS window, firmware window). ++ */ ++#define G5P_RU_TEST_FAULT_VA (G5P_RU_HW_VA_LIMIT - 0x40000000ULL) ++ ++/* Stage 3 (render-user-window.h): which firmware memory a render VM maps. Latched per context ++ * at creation. The kill switch 0 restores the stage 1 mapping and then needs CAP_SYS_RAWIO. ++ */ ++ ++/* The latency101 knobs are valid when all are 0 (off), or hard is in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000 and the dialect is TYPED_V3. The release touch ++ * is 0, or 1..20000 below the effective soft bound with latency101 on. The 3D idle settle is ++ * 0..5000. The wake110 mode is 0, or 1..2 with latency101 on and the touch off (one A/B per ++ * context). Anything else refuses the context. ++ */ ++static bool g5p_ru_lat_params_valid(void) ++{ ++ u32 hard = READ_ONCE(render_user_hard_ms), ++ soft = READ_ONCE(render_user_soft_ms); ++ ++ return g5p_ru_lat_valid(soft, hard, READ_ONCE(render_user_obs_ms)) && ++ g5p_ru_touch_valid((0), g5p_ru_lat_soft(soft, hard), hard) && ++ g5p_ru_rt_settle_valid((G5P_RU_RT_DEF_SETTLE_MS)) && ++ g5p_wk_mode_valid((2)) && (!(2) || (hard && !(0))) && ++ (!hard || (1)); ++} ++/* r25 frame-capable selections, same values as the fixed render_*_variant. */ ++ ++#define G5P_RU_MAX_QUEUED 16 ++#define G5P_RU_MAX_TARGETS 32 ++#define G5P_RU_RESULTS 1024 ++#define G5P_RU_INPUT_MS 10000 ++ ++struct g5p_rt { ++ struct list_head link; ++ struct g5p_context *ctx; ++ struct g5p_ru_rt geometry; ++ struct g5p_ru_rt_sizes sizes; ++ struct drm_mm_node node; ++ struct g5p_dma_page *pages; ++ u32 width, height, page_count; ++ u32 samples; /* 1 or 4 (CREATE_RENDER_TARGET); 0 = 1 */ ++ bool destroyed; ++}; ++ ++/* A pair job 1 bound to a kernel-owned catalogue (render_user_spare_pairs, TEMPORARY). */ ++struct g5p_rspare { ++ u32 slot[2]; /* TA, 3D */ ++ struct g5p_dma_page root; /* the catalogue CBASE points at */ ++ bool ok, used; ++}; ++ ++/* ++ * Device-wide render state (docs/gpu/multi-client-design.md §4.1): the one TA/3D channel pair ++ * and the one PM record with its pool and aux pages, shared by every render context. Jobs of ++ * all contexts run one at a time on render_wq with the core lock held, and only they write ++ * up, v3, sequence, pm_serial and the spares. `lock` guards the pool and the aux pages, which ++ * render-target creation grows while another context's job may run. The scheduler's worker ++ * is the only executor: it picks the jobs of all contexts round-robin. ++ */ ++/* The kernel-built firmware window shared by every render VM (render-user-window.h): built ++ * once from the firmware's tables, never edited, kept until reboot. ++ */ ++struct g5p_rwin { ++ bool built; ++ u32 pages, missing, leaves; ++ struct g5p_dma_page dir, leaf[G5P_RU_FW_PAGES]; ++ u32 leaf_di[G5P_RU_FW_PAGES]; ++}; ++ ++struct g5p_rdev { ++ struct mutex lock; ++ bool up, v3; /* PM and channels built; by the TYPED_V3 attach */ ++ bool dc_pending; /* a bind's DC3 pair was not consumed in time ... */ ++ u32 dc_wait; /* ... and this is the consumer it waits for */ ++ u32 sequence, pm_serial; ++ u32 cu_sequence, cu_target; /* TYPED_COMPUTE: CDM sequence, CL producer target */ ++ u32 blocks; ++ /* perf60, latched at the epoch's first TYPED_V3 attach: 2 = jobs alternate two record slots ++ * (render-user-layout.h), else 1; pm_once = the PM pool is built at that attach only. ++ */ ++ u32 job_slots; ++ bool pm_once; ++ /* perf60 stage 4, latched there too: render_user_pm_ctx (0 off, 1 per-context PM records, ++ * 2 also per-context PM IDs) and the boot-reserved sets in use (core lock). ++ */ ++ u32 pm_ctx, pm_set_next; ++ bool pm_set_used[G5P_PMSET_MAX]; ++ /* the context of the last job the executor ran and the previous distinct one, the PM's ++ * write-back targets (render-user-pmset.h; compared, never dereferenced), and the closed ++ * contexts whose unbind waits until they are neither (render_user_close_defer) ++ */ ++ struct g5p_render *pm_last, *pm_prev, *close_deferred[2]; ++ struct g5p_dma_page *physical; ++ struct g5p_dma_page aux[G5P_RU_AUX_PAGES]; ++ struct g5p_rspare *spares; ++ u32 spare_count; ++ /* render_user_close_park: the park catalogue a closed pair is bound to (allocated at the ++ * first park, kept for the device's life; rebuilt at every park) ++ */ ++ struct g5p_dma_page park; ++ u32 parks; ++ bool executing; /* a TYPED_V3 job is in g5p_rv3_execute() (reboot quiesce) */ ++ /* The scheduler (drm-render-sched.inc): its run list under sched_lock, and the one ++ * worker on render_wq that runs every job and close item. ++ */ ++ struct g5p_drm *drm; ++ spinlock_t sched_lock; ++ struct g5p_rs sched; ++ struct delayed_work sched_work; ++ /* boot_gpu_oc_mhz: the thermal stop's poll (system_wq, so a busy render_wq never starves it) */ ++ struct delayed_work oc_work; ++ unsigned long oc_armed; ++ u32 oc_ticks, oc_jobs; ++ struct g5p_rwin ++ window; /* written once, by the executor, core lock held */ ++ struct g5p_rstuck ++ *stuck; /* a lost job blocks the channel (drain), worker only */ ++ /* A lost job stopped the firmware: the worker recovers before any pick. Set by the ++ * executor, cleared by the worker; submissions and the claim pass meanwhile. ++ */ ++ bool recovering; ++ u32 recoveries; ++ u32 recover_streak; /* recoveries since a job completed (core lock) */ ++ struct list_head ++ contexts; /* every render context (lock), for the recovery */ ++}; ++ ++/* render-user-fault.h's ruling window on the device state: a failure noted while a TYPED_V3 job or ++ * a drain runs, or a recovery is pending, waits for the executor. RAM only. ++ */ ++static bool g5p_rdev_ruling(struct g5p_rdev *rd) ++{ ++ bool executing = ++ smp_load_acquire(&rd->executing); /* before `recovering` */ ++ ++ return g5p_ru_ruling(executing, READ_ONCE(rd->recovering)); ++} ++ ++/* A render context. Its firmware state is the slot pair (slot[], bound), owned here the way ++ * an Asahi queue owns its firmware context; the channels and the PM record are the device's. ++ */ ++struct g5p_render { ++ struct g5p_context *ctx; ++ u32 slot[2]; /* engine0 TA, engine1 normal3D */ ++ u32 jobs; /* jobs that started on this context */ ++ u32 dc_target; /* DC ring consumer after this context's binding */ ++ struct drm_mm private_mm; ++ struct drm_mm_node aux_node, low_node, firmware_node, private_node; ++ struct drm_mm_node ++ fault_node; /* render_user_test_fault_job's unmapped page */ ++ struct list_head targets; ++ struct g5p_rt *profile1_rt; ++ u32 target_count; ++ u64 fence_context, fence_seqno; ++ atomic_t queued; ++ /* bound: the slot pair is published (for a RAM-only bind: consumed and checked). ++ * retained: the binding holds a context and device reference until its unbind. ++ * closing: the handle is gone; the close item (g5p_render_close_run()) is queued. ++ * live_maps: a job retired on the binding, so VM_BIND MAP may run beside queued jobs. ++ */ ++ bool claimant, bound, retained, aux_mapped, detached, stage_done, ++ unhooked; ++ bool closing, live_maps; ++ bool lost_drained; /* a lost job of it drained: the firmware holds no use of it */ ++ struct g5p_rs_entity ++ entity; /* its place in the scheduler (drm-render-sched.inc) */ ++ struct list_head rdev_link; /* in rdev->contexts */ ++ /* Mappings unmapped beside queued jobs, freed as those retire (g5p_render_reap()). */ ++ spinlock_t zlock; ++ struct list_head zombies; ++ /* Latched at creation; mirrors whatever qualifies the fixed path. */ ++ u32 stage, cache_variant, mmu_variant, timestamp_variant, wait_ms; ++ u32 soft_ms, hard_ms, obs_ms; /* latency101; hard_ms == 0: off */ ++ u32 touch_ms; /* RC4a; 0: off (always 0 when hard_ms is) */ ++ u32 settle_ms; /* repeat109 minimum coherent run before the 3D event completes */ ++ u32 stall_job; /* render_user_test_stall_job; 0: off */ ++ u32 fault_job; /* render_user_test_fault_job; 0: off */ ++ u32 wake, wake_seed; /* wake110 (render-user-wake.h); wake 0: off */ ++ u32 wake_rang; /* wake110: 0xb00 writes of this context */ ++ u32 pm_policy, vdm_control; ++ u32 fw_window; /* render_user_fw_window, G5P_RU_FW_* */ ++ bool after_ta, cache_repeat; ++ struct g5p_r25_variants v; /* pm, memory, rt<=1, slots; others 0 */ ++ u32 target[2], status[2], selector[2]; ++ bool submitted[2], consumed[2], retired[2]; ++ /* TYPED_V3 (drm-render-v3.inc): serial jobs, the context is retained from its binding on. */ ++ bool v3_accepted; ++ /* perf60 render_user_pm_switch 2: this context's own PM state (S + aux) after its last job */ ++ u8 *pm_save; ++ ++ /* perf60 stage 4 (render_user_pm_ctx): its own PM set (-1: the device PM), pool pages, ++ * aux pages mapped at G5P_RU_PRIVATE_VA in its VM, and the epoch they were built for. ++ */ ++ int pm_set; ++ u32 pm_blocks, pm_epoch; ++ struct g5p_dma_page *pm_pool; ++ struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; ++ bool pm_aux_mapped; ++ bool pm_first; /* it brought the epoch up (attach): the device aux is its own */ ++ bool close_defer_run; /* render_user_close_defer: its deferred close is running */ ++}; ++ ++/* The lost job the channel waits behind (render-user-loss.h): its context (a reference held), ++ * its per-job state as the executor left it, and the drain bound read at the loss. ++ */ ++struct g5p_rstuck { ++ struct g5p_render *r; ++ struct g5p_rv3 v; ++ u32 seq, engine, drain_ms; ++ u64 lost_ns; ++}; ++ ++#include "cache-receipt.inc" ++ ++/* A listed BO that needs implicit fences and/or kernel cache maintenance. */ ++struct g5p_robj { ++ struct drm_gem_object *obj; /* reference held by the job */ ++ u32 access; ++ /* ACCESS_CACHE window (render_user_sync_range bit 1: the resource's range; else whole BO) */ ++ u64 lo, hi; ++}; ++ ++struct g5p_rjob { ++ struct g5p_cache_receipt cache; ++ struct g5p_rs_job node; /* in its context's FIFO until picked */ ++ struct g5p_drm *gpu; ++ struct g5p_context *ctx; ++ struct g5p_fence *fence; ++ struct dma_fence *inputs[G5P_MAX_SYNCS - 1]; ++ u32 input_count; ++ struct g5p_robj *objs; ++ u32 obj_count; ++ struct dma_fence **deps; /* implicit dependencies from dma_resv */ ++ u32 dep_count; ++ int setup_error; /* post-acceptance setup failure: fail without hardware */ ++ /* Inputs then deps, in order: dep_next is the first not yet seen signalled, cb_fence the ++ * one whose callback is armed (scheduler lock), dep_error a failed input or the deadline. ++ */ ++ u32 dep_next; ++ struct dma_fence *cb_fence; ++ struct dma_fence_cb cb; ++ bool cb_fired; ++ int dep_error; ++ u64 id; ++ unsigned long deadline; ++ u8 ta_w[G5P_RU_TA_W_BYTES], td_w[G5P_RU_3D_W_BYTES]; ++ bool v3; ++ bool compute; /* TYPED_COMPUTE (drm-compute-v1.inc): a CDM job */ ++ struct g5p_cu_regs cu; ++ struct g5p_rv3 rv3; ++ u64 submit_ns; /* latency123: the submit ioctl */ ++}; ++ ++static u8 *g5p_ru_page(struct g5p_boot *b, u32 engine) ++{ ++ return b->shared[(engine ? G5P_R22_3D : G5P_R22_TA) / G5P_GPU_PAGE].cpu; ++} ++ ++static u32 g5p_ru_meta(u32 engine) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + ++ (engine ? G5P_R22_3D : G5P_R22_TA)); ++} ++ ++static u8 *g5p_ru_pm_cpu(struct g5p_boot *b) ++{ ++ return b->shared[G5P_R22_PM / G5P_GPU_PAGE].cpu; ++} ++ ++static u8 *g5p_ru_global(struct g5p_boot *b, u32 engine) ++{ ++ u32 off = b->layout.offset[0x47 + engine]; ++ ++ return b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_ru_free_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ if (p->cpu) ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++} ++ ++/* Probe: the device render state and its scheduler. Its pages are allocated on demand and ++ * kept until reboot. ++ */ ++static void g5p_rsched_work(struct work_struct *work); ++static void g5p_oc_work(struct work_struct *work); ++ ++static int g5p_render_init(struct g5p_drm *d) ++{ ++ d->rdev = drmm_kzalloc(&d->drm, sizeof(*d->rdev), GFP_KERNEL); ++ if (!d->rdev) ++ return -ENOMEM; ++ d->rdev->drm = d; ++ INIT_LIST_HEAD(&d->rdev->contexts); ++ spin_lock_init(&d->rdev->sched_lock); ++ g5p_rs_init(&d->rdev->sched); ++ INIT_DELAYED_WORK(&d->rdev->sched_work, g5p_rsched_work); ++ INIT_DELAYED_WORK(&d->rdev->oc_work, g5p_oc_work); ++ return drmm_mutex_init(&d->drm, &d->rdev->lock); ++} ++ ++/* rdev->lock held: the aux pages (once) and a PM pool of at least `blocks`. The pool never ++ * shrinks; every page added reaches PoC like the bring-up pool, whether or not the device's ++ * bring-up has cleaned the pool yet. The pool, list and F are rebuilt from it before every ++ * job, so growth takes effect at the next job, and the job in flight keeps its pages. ++ */ ++static int g5p_rdev_grow(struct g5p_drm *d, u32 blocks) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_dma_page *physical; ++ u32 have = rd->blocks * 8, i; ++ int ret = 0; ++ ++ for (i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!rd->aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->aux[i]); ++ if (ret || blocks <= rd->blocks) ++ return ret; ++ if (blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ physical = kvcalloc(blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ if (have) ++ memcpy(physical, rd->physical, have * sizeof(*physical)); ++ for (i = have; !ret && i < blocks * 8; i++) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &physical[i]); ++ if (!ret) ++ g5p_table_clean_page(d->core, &physical[i]); ++ } ++ if (ret) { ++ while (i-- > have) ++ g5p_ru_free_page(d->drm.dev, &physical[i]); ++ kvfree(physical); ++ return ret; ++ } ++ kvfree(rd->physical); ++ rd->physical = physical; ++ rd->blocks = blocks; ++ return 0; ++} ++ ++/* rdev->lock held: the pool's page addresses, for the PM builders. */ ++static u64 *g5p_rdev_addresses(struct g5p_rdev *rd) ++{ ++ u64 *physical = ++ kvmalloc_array(rd->blocks * 8, sizeof(*physical), GFP_KERNEL); ++ ++ for (u32 i = 0; physical && i < rd->blocks * 8; i++) ++ physical[i] = rd->physical[i].dma; ++ return physical; ++} ++ ++/* From the moment the firmware may hold a context's slot pair, the context and everything it ++ * owns stay alive whatever its files do, until its close item has unbound the pair ++ * (g5p_render_close_run()); a pair that cannot be unbound keeps them until reboot. ++ */ ++static void g5p_render_retain(struct g5p_render *r) ++{ ++ if (r->retained) ++ return; ++ kref_get(&r->ctx->ref); ++ drm_dev_get(&r->ctx->gpu->drm); ++ r->retained = true; ++} ++ ++/* The RAM words of the binding of `slot` through the DC records at start and start + 1. */ ++static void g5p_render_bind_mem(struct g5p_boot *b, const u32 *slot, u32 start, ++ struct g5p_ru_bind_mem *m) ++{ ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + ++ ((start + i) & 255) * 20 + j * 4; ++ ++ m->rec[i][j] = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ } ++ for (u32 e = 0; e < 2; e++) { ++ m->pa[e] = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ m->host[e] = g5p_boot_shared(b, 0x84) + slot[e] * 4; ++ m->cached[e] = g5p_boot_shared(b, 0x85) + slot[e] * 4; ++ } ++} ++ ++/* ++ * Called with the file lock. Idle: no accepted job is queued or running. A VM_BIND (`live`) ++ * of a context with live_maps may also run beside its queued jobs, the way an Asahi VM_BIND ++ * does not wait for the queue: a compositor imports and releases client buffers while its ++ * previous frame is in flight. Its tag reaches the firmware at the next kick of the slots. A ++ * MAP only adds entries for VAs no accepted job references; an UNMAP clears its entries now but ++ * keeps the object pinned until the queued jobs have retired (g5p_resident_defer()), so a job ++ * that still used the range faults on the hardware instead of reaching freed memory. Not ++ * before a job has retired on the binding: the epoch's first job checks that the tables did not ++ * change since its attach, and a bind in flight computes its tag from them. Render-target ++ * edits still need the context idle. ++ */ ++static int g5p_render_vm_idle(struct g5p_context *c, bool live) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_boot *b = c->gpu->core->boot; ++ /* During a recovery no epoch may receive a tag. */ ++ bool busy = (atomic_read(&r->queued) && ++ !(live && READ_ONCE(r->live_maps))) || ++ READ_ONCE(c->gpu->rdev->recovering); ++ ++ if (c->bad || r->detached || busy) { ++ dev_warn_ratelimited( ++ c->gpu->drm.dev, ++ "G5P VM_BIND busy=render-idle bad=%u detached=%u queued=%d bound=%u\n", ++ c->bad, r->detached, atomic_read(&r->queued), r->bound); ++ return -EBUSY; ++ } ++ if (r->bound && (!b || READ_ONCE(b->failed) || READ_ONCE(b->stopped))) ++ return g5p_rdev_ruling(c->gpu->rdev) ? -EBUSY : -EIO; ++ return 0; ++} ++ ++/* perf60 MMU invalidation arm, default 0 (render-user-mmu.h): bit 0 every edit publication asks ++ * the firmware for a PC-level invalidate, bit 1 a bound context's job after another context's ++ * job retags its slots (PC), bit 2 NC-reads every table line before the tag store, bit 3 retags ++ * on every job. RAM only, like the tag path itself. ++ */ ++ ++/* An NC read of every 64-byte line of the VM's tables (g5p_rv3_fw_touch()'s visibility read). */ ++static void g5p_render_table_touch(struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt; ++ ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(vm->root.cpu + at)); ++ list_for_each_entry(pt, &vm->tables, link) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(pt->page.cpu + at)); ++} ++ ++/* File lock held, context idle (or a MAP with live_maps). After a PTE edit on a bound render ++ * VM: tables to PoC, then the packed tag to both of its slots. RAM only: no register access ++ * is safe after retirement, and the firmware reads these words only at this context's next ++ * kick, which then invalidates the slots' PC level. ++ */ ++static int g5p_render_invalidate(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_diag *g = c->gpu->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_pt *pt; ++ u32 tag, cached[2], bumps = 0, mode = (0); ++ ++ if (!READ_ONCE(r->bound) || r->detached) ++ return 0; ++ if (!b || c->bad || b->failed || b->stopped) ++ return -EIO; ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ if (mode & G5P_RU_MMU_TOUCH) ++ g5p_render_table_touch(vm); ++ tag = g5p_vm_generation(&vm->mutation); ++ for (u32 e = 0; e < 2; e++) ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ while (tag == cached[0] || tag == cached[1]) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ bumps++; ++ } ++ if (bumps) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER generation collision cached=%08x/%08x bumps=%u tag=%08x\n", ++ cached[0], cached[1], bumps, tag); ++ if (mode & G5P_RU_MMU_EDIT) { ++ /* Advance the VM's own PC counter to the PC field the helper picked. */ ++ u32 pc = g5p_ru_mmu_pc_tag(tag, cached[0], cached[1]); ++ ++ if (pc) { ++ vm->mutation.generation[0] += ++ ((pc >> 22) - (tag >> 22)) & 0x3ff; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ } ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, tag); ++ dma_wmb(); ++ if (mode) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick mmu_inval=%x " ++ "cached=%08x/%08x\n", ++ tag, r->slot[0], r->slot[1], mode, cached[0], ++ cached[1]); ++ else ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick\n", ++ tag, r->slot[0], r->slot[1]); ++ return 0; ++} ++ ++/* DC3 consumption precedes this CBASE check and selected-slot invalidation; ++ * the r22 diagnostic sequence with the user catalogue instead of a private PC. ++ */ ++ ++/* CPU-written firmware objects of one job: TA/3D pages, PM control and the ++ * list/free pages. Coherent allocations plus explicit PoC, as r22 does. ++ */ ++static void g5p_render_clean_job(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ for (u32 off = G5P_R22_TA; off <= G5P_R22_PM; off += G5P_GPU_PAGE) ++ g5p_table_clean_page(g, &b->shared[off / G5P_GPU_PAGE]); ++ for (u32 i = 0; i < 2; i++) ++ g5p_table_clean_page(g, &b->render_meta[i]); ++ dsb(sy); ++ dma_wmb(); ++} ++ ++/* RAM only: the DC ring is idle and neither slot of this context has a PA or a tag. */ ++static int g5p_render_vm_ready(struct g5p_boot *b, struct g5p_render *r) ++{ ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u32 start = READ_ONCE(ctl[4]); ++ ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ return g5p_ru_bind_busy(&s) ? -EBUSY : 0; ++} ++ ++/* The epoch's first job, core lock held, device not up: the PM record from the device pool ++ * and both channels, and every page they reach cleaned to PoC. Doing this again with the ++ * device up would reset the channels and the PM state that every context shares. ++ */ ++static int g5p_render_device_prepare(struct g5p_drm *d, ++ const struct g5p_r25_variants *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm(g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, ++ G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks, G5P_RU_PRIVATE_VA); ++ if (!ret) ++ g5p_ru_pm_variant(g5p_ru_pm_cpu(b), b->render_meta[1].cpu, ++ physical, rd->blocks, v->pm); ++ kvfree(physical); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ g5p_table_clean_page(g, &rd->aux[i]); ++ for (u32 i = 0; !ret && i < rd->blocks * 8; i++) ++ g5p_table_clean_page(g, &rd->physical[i]); ++ mutex_unlock(&rd->lock); ++ if (ret) ++ return ret; ++ rd->pm_serial = 0; ++ for (u32 e = 0; e < 2; e++) ++ g5p_ru_channel_init(g5p_ru_page(b, e), g5p_ru_meta(e)); ++ g5p_cu_device_init(d); ++ g5p_render_clean_job(g); ++ return 0; ++} ++ ++/* ++ * Core lock held, firmware up: the root entry of the firmware GiB for a VM in mode `fw_window`. ++ * The window is built at its first use from the firmware's leaf entries of the pages it keeps ++ * (render-user-window.h), and then shared, unchanged, by every render VM. RAM only. ++ */ ++static int g5p_render_fw_entry(struct g5p_drm *d, struct g5p_render *r, ++ u32 *entry) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ int ret = 0; ++ ++ *entry = 0; ++ if (r->fw_window == G5P_RU_FW_FULL) ++ *entry = g5p_le32(b->catalogue.cpu + G5P_RU_FIRMWARE_CI * 4); ++ if (r->fw_window != G5P_RU_FW_WINDOW) ++ return 0; ++ if (w->built) { ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ return 0; ++ } ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->dir); ++ /* perf60 stage 4: the per-context PM sets reserved at boot, after the device PM's pages */ ++ for (u32 i = 0; ++ !ret && i < G5P_RU_FW_PAGES + b->pm_sets * G5P_PMSET_PAGES; i++) { ++ u32 j = i - G5P_RU_FW_PAGES; ++ u64 va = i < G5P_RU_FW_PAGES ? ++ g5p_ru_fw_window_va[i] : ++ g5p_pmset_va(j / G5P_PMSET_PAGES, ++ j % G5P_PMSET_PAGES); ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (!(leaf & 1)) { ++ w->missing++; ++ continue; ++ } ++ if (k == w->leaves) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->leaf[k]); ++ if (ret) ++ break; ++ w->leaf_di[k] = g5p_ru_fw_di(va); ++ g5p_put64(w->dir.cpu + w->leaf_di[k] * 8, ++ g5p_directory_entry(w->leaf[k].dma)); ++ w->leaves++; ++ } ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, leaf); ++ w->pages++; ++ } ++ if (ret) { ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_ru_free_page(d->drm.dev, &w->leaf[i]); ++ g5p_ru_free_page(d->drm.dev, &w->dir); ++ memset(w, 0, sizeof(*w)); ++ return ret; ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(g, &w->leaf[i]); ++ g5p_table_clean_page(g, &w->dir); ++ w->built = true; ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ dev_info( ++ g->dev, ++ "RENDER-V3 fw-window pages=%u missing=%u leaves=%u dir=%pad end=v3\n", ++ w->pages, w->missing, w->leaves, &w->dir.dma); ++ return 0; ++} ++ ++/* Core lock held, between jobs: the DC3 pair that points this context's slots at its own page ++ * catalogue. Maps the firmware window (g5p_render_fw_entry()) and borrows the sink catalogue ++ * entry, cleans the tables and this ++ * context's render targets, then writes the PA and tag of both slots and the two records. ++ * RAM only, nothing is written when the ring is busy or a slot is in use; the caller stores ++ * the producer. ++ */ ++static int g5p_render_vm_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 start = READ_ONCE(ctl[4]), tag, entry; ++ struct g5p_ru_bind_mem m; ++ struct g5p_pt *pt; ++ struct g5p_rt *rt; ++ int ret; ++ ++ ret = g5p_render_vm_ready(b, r); ++ if (!ret) ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ dsb(sy); ++ tag = g5p_vm_generation(&vm->mutation); ++ /* A zero tag would read as unbound: the unbind check takes a zero cached tag as proof ++ * that the DC3 handler ran. ++ */ ++ while (!tag && vm->mutation.generations) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_publish(&m, r->slot, vm->root.dma, tag); ++ *start_out = start; ++ *tag_out = tag; ++ return 0; ++} ++ ++/* The epoch's first job, core lock held: build the device, then bind this context's pair. ++ * Any failure leaves every allocation and binding retained. ++ */ ++static int g5p_render_attach_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ int ret = g5p_render_vm_ready(d->core->boot, r); ++ ++ if (!ret) ++ ret = g5p_render_device_prepare(d, &r->v); ++ return ret ?: g5p_render_vm_prepare(d, r, start_out, tag_out); ++} ++ ++/* The legacy profiles' attach: the epoch's first job, with a doorbell and a host MMU pass. */ ++ ++/* Pointer contents, channel producer, global producer and kick, each after a ++ * barrier. The channel must have retired the previous job's endpoint. ++ */ ++ ++/* Same predicates as g5p_r22_observe() for sequence `seq`. */ ++ ++#include "drm-render-v3.inc" ++#include "drm-render-sched.inc" ++#include "drm-render-unbind.inc" ++ ++static void g5p_render_job_free(struct g5p_rjob *job) ++{ ++ for (u32 i = 0; i < job->input_count; i++) ++ dma_fence_put(job->inputs[i]); ++ for (u32 i = 0; i < job->dep_count; i++) ++ dma_fence_put(job->deps[i]); ++ kfree(job->deps); ++ for (u32 i = 0; i < job->obj_count; i++) ++ drm_gem_object_put(job->objs[i].obj); ++ kvfree(job->objs); ++ if (job->fence) ++ dma_fence_put(&job->fence->base); ++ if (job->ctx) ++ g5p_context_put(job->ctx); ++ kfree(job); ++} ++ ++/* CPU cache maintenance on the G5P mapping (for an imported BO, the attachment's sg_table): clean ++ * before the job, invalidate after retirement. Whole BO, or (render_user_sync_range bit 1, latched ++ * at submit) the resource's BO range, the same window both ways. ++ */ ++static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, ++ bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct g5p_robj *o = &job->objs[i]; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(o->obj)->sgt; ++ ++ if (!(o->access & DRM_G5P_RENDER_ACCESS_CACHE) || !sgt) ++ continue; ++ /* A buffer the job only reads (a sampled client dma-buf or wl_shm udmabuf) ++ * was cleaned before the job and the GPU wrote none of it: no invalidate, ++ * which could discard what the CPU side wrote meanwhile into lines of the ++ * pages the buffer shares (a udmabuf covers whole 16 KiB pages). ++ */ ++ if (to_cpu && !(o->access & DRM_G5P_RENDER_ACCESS_WRITE)) ++ continue; ++ struct g5p_cache_receipt *cr = &job->cache; ++ bool imported = drm_gem_is_imported(o->obj); ++ bool exported = READ_ONCE(o->obj->dma_buf) != NULL; ++ bool wc = to_drm_gem_shmem_obj(o->obj)->map_wc; ++ int cls = g5p_cache_class(imported, exported, wc); ++ u64 bytes = o->hi ? o->hi - o->lo : o->obj->size; ++ u64 cpu = 0, wall = 0; ++ if (cr->enabled) ++ cr->requested[to_cpu][cls] += bytes; ++ /* Native pages are initially DMA-mapped with no SKIP_CPU_SYNC. ++ * All later CPU views of a private map_wc object (GEM mmap/vmap, ++ * including command validation) are write-combined. There is no ++ * cacheable client alias to clean/invalidate. Shared/foreign BOs ++ * remain conservative. Barriers below still order WC accesses. ++ */ ++ bool imported_uncached = imported && ++ drm_gem_dma_buf_is_cpu_uncached(o->obj->import_attach->dmabuf); ++ if ((imported_uncached && cr->import_policy) || g5p_cache_omit(cr->policy, imported, exported, wc)) ++ continue; ++ if (cr->enabled) { cpu = task_sched_runtime(current); wall = ktime_get_ns(); } ++ if (o->hi && (o->lo || o->hi != o->obj->size)) ++ g5p_sync_sgt_range(d->drm.dev, sgt, o->lo, o->hi, ++ to_cpu); ++ else if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ if (cr->enabled) { ++ cr->maintained[to_cpu][cls] += bytes; ++ cr->cpu[to_cpu][cls] += task_sched_runtime(current) - cpu; ++ cr->wall[to_cpu][cls] += ktime_get_ns() - wall; ++ } ++ } ++ if (to_cpu) dma_rmb(); else dma_wmb(); ++} ++ ++/* After acceptance, render_submit_lock held, no file lock: one dma_resv at a ++ * time, so no ww ordering is needed. Our own context's earlier fences are ++ * already ordered by the queue. A failure fails the job before hardware. ++ */ ++static int g5p_render_implicit(struct g5p_rjob *job, u64 own_context) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct dma_resv *resv = job->objs[i].obj->resv; ++ bool write = job->objs[i].access & DRM_G5P_RENDER_ACCESS_WRITE; ++ struct dma_fence **fences = NULL, **grown; ++ unsigned int count = 0; ++ int ret; ++ ++ if (!(job->objs[i].access & DRM_G5P_RENDER_ACCESS_IMPLICIT)) ++ continue; ++ dma_resv_lock(resv, NULL); ++ ret = dma_resv_get_fences(resv, ++ write ? DMA_RESV_USAGE_READ : ++ DMA_RESV_USAGE_WRITE, ++ &count, &fences); ++ if (!ret) ++ ret = dma_resv_reserve_fences(resv, 1); ++ if (!ret) ++ dma_resv_add_fence(resv, &job->fence->base, ++ write ? DMA_RESV_USAGE_WRITE : ++ DMA_RESV_USAGE_READ); ++ dma_resv_unlock(resv); ++ grown = ret || !count ? ++ NULL : ++ krealloc_array(job->deps, ++ job->dep_count + count, ++ sizeof(*grown), GFP_KERNEL); ++ if (!ret && count && !grown) ++ ret = -ENOMEM; ++ if (grown) ++ job->deps = grown; ++ for (u32 j = 0; j < count; j++) { ++ if (!ret && fences[j]->context != own_context) ++ job->deps[job->dep_count++] = fences[j]; ++ else ++ dma_fence_put(fences[j]); ++ } ++ kfree(fences); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++/* Free the mappings unmapped beside jobs up to fence seqno `seqno` (all: every one). */ ++static void g5p_render_reap(struct g5p_render *r, u64 seqno, bool all) ++{ ++ struct g5p_mapping *m, *t; ++ unsigned long flags; ++ LIST_HEAD(done); ++ ++ spin_lock_irqsave(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &r->zombies, link) ++ if (all || m->retire <= seqno) ++ list_move_tail(&m->link, &done); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &done, link) { ++ list_del(&m->link); ++ g5p_vm_obj_put(r->ctx->vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++} ++ ++/* One job, picked by the scheduler once its inputs signalled (or failed, or timed out). */ ++static void g5p_render_run(struct g5p_rjob *job) ++{ ++ struct g5p_drm *d = job->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_render *r = job->ctx->render; ++ unsigned long lock_deadline, flags; ++ bool hardware = false; ++ int ret; ++ ++ job->cache.enabled = READ_ONCE(render_user_cache_receipts); ++ job->cache.policy = READ_ONCE(render_user_cache_private_wc); ++ job->cache.import_policy = READ_ONCE(render_user_cache_import_wc); ++ if (job->cache.enabled) { ++ job->cache.job = job->id; ++ job->cache.start_cpu = task_sched_runtime(current); ++ job->cache.start_wall = ktime_get_ns(); ++ } ++ /* Admission closes only for good (reboot quiesce, FINAL teardown): a job queued before ++ * that fails here, without waiting for its inputs and without competing for the command ++ * lock. ++ */ ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) ++ ret = -ESHUTDOWN; ++ else ++ ret = job->setup_error ?: job->dep_error; ++ if (!ret && (job->ctx->bad || READ_ONCE(g->boot->failed))) ++ ret = -EIO; ++ if (!ret) { ++ /* Bounded: never queue forever behind a command stuck in MMIO. */ ++ lock_deadline = jiffies + 2 * HZ; ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, lock_deadline)) { ++ ret = -EBUSY; ++ break; ++ } ++ msleep(5); ++ } ++ } ++ if (!ret) { ++ if (g->boot->failed || g->boot->stopped) { ++ ret = -EIO; ++ } else if (!1 && !g5p_ru_mmio_ok(&g5p_rv3_term)) { ++ /* A V3 epoch is terminal: the legacy path would read and write registers. */ ++ ret = -EIO; ++ } else { ++ /* RAM-only from the DC3 attach on: no snapshot (it reads R0), no ++ * export, no register access of any kind after a timeout. ++ */ ++ g5p_render_cache(d, job, false); ++ WRITE_ONCE(d->rdev->executing, true); ++ ret = g5p_rv3_execute(d, r, job); ++ /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ ++ smp_store_release(&d->rdev->executing, false); ++ hardware = job->rv3.touched || job->rv3.bind_failed; ++ if (!ret || ret == -ECANCELED) ++ g5p_render_cache( ++ d, job, ++ true); /* before status and fence */ ++ if (!ret && r->bound) ++ WRITE_ONCE(r->live_maps, true); ++ } ++ g->last_result = ret; ++ mutex_unlock(&g->lock); ++ } ++ spin_lock_irqsave(&d->status_lock, flags); ++ /* A failed device fails every later job; one context's lost job or failed bind does not. */ ++ if (READ_ONCE(g->boot->failed) || READ_ONCE(g->boot->event_failed)) ++ ret = ret ?: -EIO; ++ job->fence->state = DRM_G5P_JOB_COMPLETE; ++ job->fence->error = ret < 0 ? ret : 0; ++ d->last_job = job->id; ++ d->last_error = job->fence->error; ++ d->job_error = job->fence->error; ++ d->job_state = DRM_G5P_JOB_COMPLETE; ++ if (ret < 0 && ret != -ECANCELED && hardware) { ++ /* No reset protocol: quarantine the context and the device. A TYPED_V3 ++ * timeout is one exception for the device: nothing is wrong with the ++ * protocol, the firmware may still complete, and the drain must keep ++ * decoding (RAM only) so that a late completion is recorded. A failed ++ * bind is another: the DC ring is not on the path of the contexts ++ * already bound, so only this context is lost. A lost job is the last: ++ * the executor already ruled (the drain, or the recovery that replaces ++ * this epoch), and only its context is lost. ++ */ ++ WRITE_ONCE(job->ctx->bad, true); ++ if (g5p_ru_job_fails_device(1, ret, job->rv3.bind_failed, ++ job->rv3.lost)) ++ WRITE_ONCE(g->boot->failed, true); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (ret < 0) ++ dma_fence_set_error(&job->fence->base, ret); ++ atomic_dec(&r->queued); ++ atomic_dec(&d->r8_queued); ++ dma_fence_signal(&job->fence->base); ++ /* A job that may still run on the hardware (a bad context) keeps what it might use. */ ++ if (!READ_ONCE(job->ctx->bad)) ++ g5p_render_reap(r, job->fence->base.seqno, false); ++ /* a failed job (complete=-N) always prints; a completed one is routine: job_log */ ++ if (ret) ++ dev_info( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ else ++ g5p_job_printk( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ if (job->cache.enabled) { ++ job->cache.total_cpu = task_sched_runtime(current) - job->cache.start_cpu; ++ job->cache.total_wall = ktime_get_ns() - job->cache.start_wall; ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ job->cache.serial = g5p_cache_head + 1; ++ g5p_cache_ring[g5p_cache_head++ % G5P_CACHE_RECORDS] = job->cache; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ } ++ g5p_render_job_free(job); ++ ++ drm_dev_put(&d->drm); ++} ++ ++/* Private PM-aux/RT page with the latched r25 leaf flags (c by default). */ ++static int g5p_render_map_page(struct g5p_drm *d, struct g5p_render *r, u64 va, ++ struct g5p_dma_page *page) ++{ ++ u8 *pte = g5p_vm_pte(r->ctx->vm, d->drm.dev, va, true); ++ ++ if (IS_ERR_OR_NULL(pte)) ++ return pte ? PTR_ERR(pte) : -ENOMEM; ++ if (g5p_le32(pte) || g5p_le32(pte + 4)) ++ return -EBUSY; ++ g5p_vm_entry(&r->ctx->vm->mutation, pte, ++ g5p_leaf_entry(page->dma, g5p_r25_attributes(&r->v, va)), ++ 2); ++ return 0; ++} ++ ++/* clear: unmap first (file lock held, VM still hooked). */ ++static void g5p_rt_release_memory(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rt *rt, bool clear) ++{ ++ if (clear && rt->page_count) ++ g5p_vm_clear(r->ctx->vm, d->drm.dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rt->pages[i]); ++ kvfree(rt->pages); ++ rt->pages = NULL; ++ rt->page_count = 0; ++ if (drm_mm_node_allocated(&rt->node)) ++ drm_mm_remove_node(&rt->node); ++} ++ ++/* File lock held, context idle; RAM only. Main ring then secondary in one ++ * RT_ALIGN node: the first 64x64 target lands exactly on r22's ++ * 101020000/101030000. ++ */ ++static int g5p_rt_create(struct g5p_drm *d, struct g5p_render *r, ++ const struct g5p_ru_rt *geometry, u32 width, ++ u32 height, struct g5p_rt **out) ++{ ++ struct g5p_rt *rt; ++ u64 bytes; ++ int ret; ++ ++ if (r->target_count >= G5P_RU_MAX_TARGETS) ++ return -ENOSPC; ++ rt = kzalloc(sizeof(*rt), GFP_KERNEL); ++ if (!rt) ++ return -ENOMEM; ++ rt->ctx = r->ctx; ++ rt->geometry = *geometry; ++ rt->width = width; ++ rt->height = height; ++ ret = g5p_ru_geometry_check(geometry, width, height, &rt->sizes); ++ if (ret) ++ goto err_free; ++ bytes = rt->sizes.main + rt->sizes.secondary_map; ++ ret = drm_mm_insert_node_in_range( ++ &r->private_mm, &rt->node, bytes, G5P_RU_RT_ALIGN, 0, ++ G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE, ++ DRM_MM_INSERT_LOW); ++ if (ret) ++ goto err_free; ++ rt->pages = ++ kvcalloc(bytes / G5P_GPU_PAGE, sizeof(*rt->pages), GFP_KERNEL); ++ if (!rt->pages) { ++ ret = -ENOMEM; ++ goto err_node; ++ } ++ for (; rt->page_count < bytes / G5P_GPU_PAGE; rt->page_count++) { ++ struct g5p_dma_page *p = &rt->pages[rt->page_count]; ++ ++ ret = g5p_vm_alloc_page(d->drm.dev, p); ++ if (!ret) ++ ret = g5p_render_map_page( ++ d, r, ++ rt->node.start + rt->page_count * G5P_GPU_PAGE, ++ p); ++ if (ret) { ++ g5p_ru_free_page(d->drm.dev, p); ++ goto err_pages; ++ } ++ } ++ if (r->bound) { ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(d->core, &rt->pages[i]); ++ ret = g5p_render_invalidate(r->ctx); ++ if (ret) { ++ /* Hardware may hold the new tag: retain the target. */ ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ return ret; ++ } ++ } ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ *out = rt; ++ g5p_vmlog_add(G5P_VMLOG_RT_NEW, r->ctx->vm, 0, rt->node.start, bytes, ++ rt->width << 16 | rt->height, rt->samples); ++ return 0; ++err_pages: ++ g5p_rt_release_memory(d, r, rt, true); ++ goto err_free; ++err_node: ++ drm_mm_remove_node(&rt->node); ++err_free: ++ kfree(rt); ++ return ret; ++} ++ ++/* Detach every VM-side trace: private PTEs and the reserved VA nodes. File ++ * lock held, or the file is closing (no concurrent VM ioctl can exist). ++ */ ++static void g5p_render_unhook(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct device *dev = c->gpu->drm.dev; ++ struct g5p_rt *rt; ++ ++ if (r->unhooked) ++ return; ++ list_for_each_entry(rt, &r->targets, link) ++ if (rt->page_count) ++ g5p_vm_clear(c->vm, dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ if (r->aux_mapped) ++ g5p_vm_clear(c->vm, dev, G5P_RU_PRIVATE_VA, ++ G5P_RU_AUX_PAGES * G5P_GPU_PAGE); ++ for (u32 i = 0; i < 4; i++) { ++ struct drm_mm_node *n = i == 0 ? &r->low_node : ++ i == 1 ? &r->firmware_node : ++ i == 2 ? &r->private_node : ++ &r->fault_node; ++ ++ if (drm_mm_node_allocated(n)) ++ drm_mm_remove_node(n); ++ } ++ if (c->vm->resident == c) ++ c->vm->resident = NULL; ++ r->unhooked = true; ++} ++ ++/* Last context reference: only reached unbound or after a successful DC3 ++ * unbind, so no firmware or GPU reference to this memory remains. The aux ++ * pages and the PM pool are the device's. ++ */ ++static void g5p_render_release(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_rt *rt, *tmp; ++ unsigned long flags; ++ ++ g5p_render_unhook(c); ++ g5p_render_reap(r, 0, true); ++ if (!list_empty(&r->rdev_link)) { ++ mutex_lock(&d->rdev->lock); ++ list_del_init(&r->rdev_link); ++ mutex_unlock(&d->rdev->lock); ++ } ++ list_for_each_entry_safe(rt, tmp, &r->targets, link) { ++ g5p_rt_release_memory(d, r, rt, false); ++ list_del(&rt->link); ++ kfree(rt); ++ } ++ if (drm_mm_node_allocated(&r->aux_node)) ++ drm_mm_remove_node(&r->aux_node); ++ drm_mm_takedown(&r->private_mm); ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ for (u32 e = 0; e < 2; e++) ++ if (r->slot[e]) ++ __clear_bit(r->slot[e], d->core->user_slots); ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ { ++ u32 aux = 0; ++ ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ aux += !!r->pm_aux[i].cpu; ++ /* release8 accounting: what this context frees, and the device's live GEM total */ ++ dev_info( ++ d->drm.dev, ++ "RENDER-V3 release slots=%u/%u jobs=%u set=%d pool_pages=%u aux_pages=%u " ++ "vm_pinned=%lluK live_bos=%u live_bo_bytes=%lluK end=v3\n", ++ r->slot[0], r->slot[1], r->jobs, r->pm_set, ++ r->pm_pool ? r->pm_blocks * 8 : 0, aux, ++ c->vm ? c->vm->pinned_bytes >> 10 : 0, ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10); ++ } ++ g5p_rv3_pmctx_release(d, r); ++ if (d->rdev->pm_last == r) ++ d->rdev->pm_last = NULL; ++ if (d->rdev->pm_prev == r) ++ d->rdev->pm_prev = NULL; ++ kvfree(r->pm_save); ++ kfree(r); ++ c->render = NULL; ++} ++ ++/* CREATE_CONTEXT type RENDER. Exclusive VM: reserves [64KiB, 1GiB) for the ++ * borrowed VA0 sink, the whole firmware GiB and the private RT/PM window. ++ */ ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_r25_variants v = { ++ .pm = (0), ++ .memory = (0), ++ .rt = (0), ++ .slots = (0), ++ }; ++ unsigned long flags; ++ u32 id, fw_window; ++ int ret; ++ ++ BUILD_BUG_ON(G5P_USER_RESERVED_START & ((1ULL << 30) - 1)); ++ if (a->flags || a->handle) ++ return -EINVAL; ++ ; ++ ; ++ fw_window = (G5P_RU_FW_WINDOW); ++ /* Unprivileged unless the VM would map the whole firmware GiB (the kill switch). */ ++ if (fw_window >= G5P_RU_FW_MODES || ++ (fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO))) ++ return fw_window >= G5P_RU_FW_MODES ? -EINVAL : -EPERM; ++ if ((0) > 2 || ++ ((G5P_RU_DEF_CACHE_VARIANT) != 0 && ++ (G5P_RU_DEF_CACHE_VARIANT) != 3) || ++ (G5P_RU_DEF_MMU_VARIANT) > 1 || (0) > 1 || (0) > 1 || ++ READ_ONCE(render_user_wait_ms) < ((1) ? G5P_RU_V3_WAIT_MS : 3000) || ++ READ_ONCE(render_user_wait_ms) > G5P_RU_V3_CAP_MS || ++ !g5p_ru_lat_params_valid() || !(32) || (32) > G5P_RU_MAX_BLOCKS || ++ ((G5P_RU_DEF_PROFILE1_VDM_CONTROL) & 0xf) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) > G5P_RU_MAX_VDM_CONTROL || ++ !g5p_ru_r25_valid(&v)) ++ return -EINVAL; ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ r = kzalloc(sizeof(*r), GFP_KERNEL); ++ if (!c || !r) { ++ kfree(c); ++ kfree(r); ++ return -ENOMEM; ++ } ++ r->pm_set = -1; ++ c->gpu = d; ++ c->render = r; ++ r->ctx = c; ++ INIT_LIST_HEAD(&r->targets); ++ INIT_LIST_HEAD(&r->rdev_link); ++ g5p_rs_entity_init(&r->entity); ++ spin_lock_init(&r->zlock); ++ INIT_LIST_HEAD(&r->zombies); ++ atomic_set(&r->queued, 0); ++ drm_mm_init(&r->private_mm, G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_SIZE); ++ r->stage = (0); ++ r->cache_variant = (G5P_RU_DEF_CACHE_VARIANT); ++ r->mmu_variant = (G5P_RU_DEF_MMU_VARIANT); ++ r->timestamp_variant = (0); ++ r->wait_ms = READ_ONCE(render_user_wait_ms); ++ r->hard_ms = READ_ONCE(render_user_hard_ms); ++ r->soft_ms = r->hard_ms ? ++ g5p_ru_lat_soft(READ_ONCE(render_user_soft_ms), ++ r->hard_ms) : ++ 0; ++ r->obs_ms = r->hard_ms ? READ_ONCE(render_user_obs_ms) : 0; ++ r->touch_ms = r->hard_ms ? (0) : 0; ++ r->settle_ms = (G5P_RU_RT_DEF_SETTLE_MS); ++ r->stall_job = (0); ++ r->fault_job = capable(CAP_SYS_RAWIO) ? (0) : 0; ++ r->wake = r->hard_ms && !r->touch_ms ? (2) : 0; ++ r->wake = g5p_wk_mode_valid(r->wake) ? r->wake : 0; ++ r->wake_seed = (0); ++ r->after_ta = (0); ++ r->pm_policy = (0); ++ r->cache_repeat = (0); ++ r->vdm_control = (G5P_RU_DEF_PROFILE1_VDM_CONTROL); ++ r->fw_window = fw_window; ++ r->v = v; ++ r->fence_context = dma_fence_context_alloc(1); ++ mutex_lock(&f->lock); ++ c->vm = xa_load(&f->vms, a->vm); ++ if (!c->vm) { ++ ret = -ENOENT; ++ goto err_unlock; ++ } ++ if (c->vm->contexts || c->vm->resident || c->vm->resident_gone || ++ c->vm->frozen || c->vm->serial_context) { ++ ret = -EBUSY; ++ goto err_unlock; ++ } ++ /* Memory mode 4 covers every render leaf, so no BO may be mapped yet. */ ++ if (v.memory == 4 && c->vm->map_count) { ++ ret = -EINVAL; ++ goto err_unlock; ++ } ++ r->low_node.start = G5P_USER_VA_START; ++ r->low_node.size = G5P_RU_USER_VA_MIN - G5P_USER_VA_START; ++ r->private_node.start = G5P_RU_PRIVATE_VA; ++ r->private_node.size = G5P_RU_PRIVATE_SIZE; ++ r->firmware_node.start = G5P_USER_RESERVED_END; ++ r->firmware_node.size = ++ ((G5P_RU_FIRMWARE_CI + 1) << 30) - G5P_USER_RESERVED_END; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->low_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->private_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->firmware_node) || ++ g5p_le32(c->vm->root.cpu + G5P_RU_FIRMWARE_CI * 4) || ++ g5p_le32(c->vm->root.cpu)) { ++ ret = -EINVAL; /* VM already maps a range this context needs */ ++ goto err_unlock; ++ } ++ if (r->fault_job) { ++ r->fault_node.start = G5P_RU_TEST_FAULT_VA; ++ r->fault_node.size = G5P_GPU_PAGE; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->fault_node)) { ++ dev_warn( ++ d->drm.dev, ++ "RENDER-V3 test-fault off: va=%#llx already used in this VM end=v3\n", ++ G5P_RU_TEST_FAULT_VA); ++ r->fault_job = 0; ++ } ++ } ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ if (g5p_ru_slot_pair(d->core->user_slots, v.slots, r->slot)) { ++ __set_bit(r->slot[0], d->core->user_slots); ++ __set_bit(r->slot[1], d->core->user_slots); ++ } else { ++ r->slot[0] = 0; ++ r->slot[1] = 0; ++ } ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ if (!r->slot[0]) { ++ ret = -ENOSPC; ++ goto err_unlock; ++ } ++ /* The device's PM pool (at least this context's latched size) and aux pages, mapped at ++ * the same VA in every render VM, so the PM state each TA reloads is the one the ++ * previous job saved, whichever VM ran it. ++ */ ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, (32)); ++ mutex_unlock(&d->rdev->lock); ++ r->aux_node.start = G5P_RU_PRIVATE_VA; ++ r->aux_node.size = G5P_RU_AUX_PAGES * G5P_GPU_PAGE; ++ if (!ret) ++ ret = drm_mm_reserve_node(&r->private_mm, &r->aux_node); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ ret = g5p_render_map_page(d, r, ++ G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE, ++ &d->rdev->aux[i]); ++ r->aux_mapped = true; ++ } ++ if (ret) ++ goto err_unlock; ++ ret = xa_alloc(&f->contexts, &id, c, XA_LIMIT(1, G5P_MAX_CONTEXTS), ++ GFP_KERNEL); ++ if (ret) ++ goto err_unlock; ++ kref_init(&c->ref); ++ kref_get(&c->vm->ref); ++ mutex_lock(&d->rdev->lock); ++ list_add_tail(&r->rdev_link, &d->rdev->contexts); ++ mutex_unlock(&d->rdev->lock); ++ c->vm->contexts++; ++ c->vm->serial_context = true; ++ c->vm->resident = c; ++ c->vm->zero_leaf_flags = v.memory == 4; ++ a->handle = id; ++ mutex_unlock(&f->lock); ++ return 0; ++err_unlock: ++ if (c->vm) ++ g5p_render_release(c); ++ else ++ kfree(r); ++ mutex_unlock(&f->lock); ++ kfree(c); ++ return ret; ++} ++ ++/* DESTROY_CONTEXT, file lock held. A TYPED_V3 context that submitted: its close item, queued ++ * behind its jobs, unbinds and releases it (g5p_render_close()). Unbound: plain release. A ++ * legacy context that brought the device up, bound and idle: zero PA/tag and consume one DC3 ++ * per slot, as r21 detach does. Failure or a quarantined legacy context retains everything ++ * until reboot. ++ */ ++static int g5p_render_destroy(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl, start, target; ++ unsigned long deadline; ++ int ret; ++ ++ if (r->v3_accepted) { ++ g5p_render_close(c); ++ return 0; ++ } ++ if (atomic_read(&r->queued) || c->bad) ++ return -EBUSY; ++ if (!r->bound) { ++ g5p_render_unhook(c); ++ if (r->claimant) { ++ /* Claimed but never touched hardware: drop the claim's reference. The ++ * device stays claimed for render: other contexts may be queued on it. ++ */ ++ r->claimant = false; ++ d->quarantined_context = NULL; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ return 0; ++ } ++ /* The legacy unbind rings a doorbell: retained once the epoch is terminal. */ ++ if (!r->claimant || !g5p_ru_mmio_ok(&g5p_rv3_term)) ++ return -EBUSY; ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ ret = -EIO; ++ if (b->failed || b->stopped) ++ goto out; ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ start = READ_ONCE(ctl[4]); ++ ret = -EBUSY; ++ if (start > 255 || READ_ONCE(ctl[0]) != start) ++ goto out; ++ for (u32 e = 0; e < 2; e++) { ++ u32 index = (start + e) & 255; ++ ++ g5p_put64(g5p_boot_shared(b, 0x86) + r->slot[e] * 8, 0); ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, 0); ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + index * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE, ++ j == 0 ? 3 : ++ j == 1 ? r->slot[e] : ++ 0); ++ } ++ } ++ target = (start + 2) & 255; ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], target); ++ dma_wmb(); ++ ret = g5p_prepare_kick(g, "RENDER-USER-unbind"); ++ if (ret) ++ goto fail; ++ writel(0x10, g->rgx + 0xb00); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || b->failed) { ++ ret = ret ?: -EIO; ++ goto fail; ++ } ++ dma_rmb(); ++ if (READ_ONCE(ctl[0]) == target) ++ break; ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (READ_ONCE(ctl[0]) != target) { ++ ret = -ETIMEDOUT; ++ goto fail; ++ } ++ r->detached = true; ++ r->bound = false; ++ r->claimant = false; ++ WRITE_ONCE(c->live, false); ++ g5p_render_unhook(c); ++ /* One render attachment per boot: accepted stays held, so neither a new ++ * render owner nor compute can reuse the shared channel pages. ++ */ ++ d->quarantined_context = NULL; ++ g5p_mmu_record( ++ g, ++ "RENDER-USER detached slots=%u/%u; resources freed on release\n", ++ r->slot[0], r->slot[1]); ++ mutex_unlock(&g->lock); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ return 0; ++fail: ++ c->bad = true; ++ b->failed = true; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, ++ const struct drm_g5p_sync *syncs, u32 count, ++ struct drm_syncobj **signal, u64 *point, ++ struct dma_fence_chain **chain) ++{ ++ u32 signal_handle = 0; ++ int ret; ++ ++ if (!count || count > G5P_MAX_SYNCS) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) { ++ for (u32 j = 0; j < i; j++) ++ if (syncs[i].handle == syncs[j].handle && ++ syncs[i].point == syncs[j].point && ++ syncs[i].flags == syncs[j].flags) ++ return -EINVAL; ++ if (syncs[i].flags & ~DRM_G5P_SYNC_SIGNAL) ++ return -EINVAL; ++ if (syncs[i].flags) { ++ if (*signal) ++ return -EINVAL; ++ *signal = drm_syncobj_find(file, syncs[i].handle); ++ if (!*signal) ++ return -ENOENT; ++ *point = syncs[i].point; ++ signal_handle = syncs[i].handle; ++ continue; ++ } ++ if (job->input_count == ARRAY_SIZE(job->inputs)) ++ return -E2BIG; ++ ret = drm_syncobj_find_fence(file, syncs[i].handle, ++ syncs[i].point, 0, ++ &job->inputs[job->input_count]); ++ if (ret) ++ return ret; ++ job->input_count++; ++ } ++ if (!*signal) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) ++ if (!syncs[i].flags && syncs[i].handle == signal_handle && ++ (!*point || !syncs[i].point || syncs[i].point >= *point)) ++ return -EINVAL; ++ if (*point) { ++ struct dma_fence *old = drm_syncobj_fence_get(*signal); ++ ++ ret = old && (!to_dma_fence_chain(old) || ++ old->seqno >= *point) ? ++ -EINVAL : ++ 0; ++ dma_fence_put(old); ++ if (ret) ++ return ret; ++ *chain = dma_fence_chain_alloc(); ++ if (!*chain) ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++/* File lock held. One mapping of the listed BO must contain each range at a ++ * consistent BO offset; access is a subset of the BO's immutable roles. ++ */ ++static int g5p_render_resolve(struct drm_file *file, struct g5p_vm *vm, ++ const struct drm_g5p_render_resource *res, ++ u32 count, struct g5p_ru_range *ranges, ++ struct g5p_robj *objs, u32 *obj_count) ++{ ++ u32 known = DRM_G5P_RENDER_ACCESS_READ | DRM_G5P_RENDER_ACCESS_WRITE | ++ DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ ++ for (u32 i = 0; i < count; i++) { ++ struct drm_gem_object *obj; ++ struct g5p_mapping *m, *found = NULL; ++ u32 roles, access; ++ ++ for (u32 j = 0; j < i; j++) ++ if (res[i].handle == res[j].handle) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, res[i].handle); ++ if (!obj) ++ return -ENOENT; ++ roles = to_g5p_bo(obj)->roles; ++ { ++ access = res[i].kind; ++ if (!(access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE)) || ++ access & ~known || ++ ((access & G5P_RU_ACCESS_READ) && ++ !(roles & DRM_G5P_BO_READ)) || ++ ((access & G5P_RU_ACCESS_WRITE) && ++ !(roles & DRM_G5P_BO_WRITE))) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ /* A foreign buffer always gets cache maintenance and fences. */ ++ if (drm_gem_is_imported(obj)) ++ access |= DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ } ++ list_for_each_entry(m, &vm->maps, link) ++ if (m->obj == obj && res[i].va >= m->node.start && ++ res[i].size && ++ res[i].va - m->node.start < m->node.size && ++ res[i].size <= m->node.size - (res[i].va - ++ m->node.start) && ++ res[i].offset == ++ m->offset + (res[i].va - m->node.start)) ++ found = m; ++ if (!found) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ if (access & (DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE)) { ++ struct g5p_robj *o = &objs[*obj_count]; ++ ++ o->obj = obj; /* the job inherits this reference */ ++ o->access = access; ++ o->lo = 0; ++ o->hi = obj->size; ++ if ((READ_ONCE(render_user_sync_range) & 2) && ++ !g5p_sync_window(res[i].offset, res[i].size, ++ obj->size, &o->lo, &o->hi)) { ++ o->lo = 0; ++ o->hi = obj->size; ++ } ++ (*obj_count)++; ++ } else { ++ drm_gem_object_put(obj); ++ } ++ ranges[i].va = res[i].va; ++ ranges[i].size = res[i].size; ++ ranges[i].access = access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE); ++ /* The BO's EXEC role, for TYPED_V3 only; an imported BO never has it. */ ++ if ((roles & DRM_G5P_BO_EXEC)) ++ ranges[i].access |= G5P_RU_ACCESS_EXEC; ++ } ++ return 0; ++} ++ ++/* LINEAR64_V1 admits only the exact round16 immutable inputs. Writes stay ++ * privileged and caller-serialized, as for the r20 first-store dialect. ++ */ ++ ++/* File lock held. Copy `len` bytes at GPU VA `va` out of the BO the VM maps ++ * there, through the BO's own pages, as g5p_render_profile1_bytes() does. The ++ * mapping is found again by VA and its own (VM-referenced) object is read, so ++ * a handle closed or replaced since g5p_render_resolve() cannot redirect the ++ * read. Only a native BO has shmem pages: an imported one (never EXEC, so ++ * never a TA program) is refused for the stream too. ++ */ ++static int g5p_render_read_va(struct drm_device *drm, struct g5p_vm *vm, u64 va, ++ void *dst, u32 len) ++{ ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); ++ struct g5p_mapping *m, *found = NULL; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ u64 off; ++ int ret; ++ ++ list_for_each_entry(m, &vm->maps, link) ++ if (va >= m->node.start && va - m->node.start < m->node.size && ++ len <= m->node.size - (va - m->node.start)) ++ found = m; ++ if (!found) ++ return -ENOENT; ++ obj = found->obj; ++ off = found->offset + (va - found->node.start); ++ if (drm_gem_is_imported(obj) || off > obj->size || ++ len > obj->size - off) ++ return -EINVAL; ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ ret = IS_ERR(sgt) ? PTR_ERR(sgt) : drm_gem_vmap(obj, &map); ++ if (ret) ++ return ret; ++ /* Keep content validation. Native private WC has no CPU cache alias; ++ * all other supported mappings retain a bounded DMA ownership sync. ++ */ ++ bool range = READ_ONCE(render_user_cache_read_va); ++ bool omit = range && g5p_cache_omit(READ_ONCE(render_user_cache_private_wc), ++ false, READ_ONCE(obj->dma_buf) != NULL, to_drm_gem_shmem_obj(obj)->map_wc); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, true); ++ else dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_rmb(); ++ if (map.is_iomem) ++ ret = -EINVAL; ++ else ++ memcpy(dst, (const u8 *)map.vaddr + off, len); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, false); ++ else dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_wmb(); ++ drm_gem_vunmap(obj, &map); ++ return ret; ++} ++ ++static void g5p_render_pass_from_uapi(struct g5p_ru_pass *p, ++ const struct drm_g5p_render_pass *u) ++{ ++ p->vdm_va = u->vdm_va; ++ p->vdm_size = u->vdm_size; ++ p->vdm_control = u->vdm_control; ++ p->scratch_va = u->scratch_va; ++ p->scratch_size = u->scratch_size; ++ memcpy(p->scratch_ptr, u->scratch_ptr, sizeof(p->scratch_ptr)); ++ p->pool7_va = u->pool7_va; ++ p->pool7_size = u->pool7_size; ++ p->pool8_va = u->pool8_va; ++ p->pool8_size = u->pool8_size; ++ memcpy(p->context_pds, u->context_pds, sizeof(p->context_pds)); ++ memcpy(p->resume_pds, u->resume_pds, sizeof(p->resume_pds)); ++ memcpy(p->ta_pds, u->ta_pds, sizeof(p->ta_pds)); ++ memcpy(p->background, u->background, sizeof(p->background)); ++ memcpy(p->eot, u->eot, sizeof(p->eot)); ++ memcpy(p->eot_resume, u->eot_resume, sizeof(p->eot_resume)); ++ memcpy(p->restore, u->restore, sizeof(p->restore)); ++} ++ ++static void g5p_render_pass2_from_uapi(struct g5p_ru_pass2 *e, ++ const struct drm_g5p_render_pass_ext *u) ++{ ++ *e = (struct g5p_ru_pass2){ ++ .scissor_va = u->scissor_va, ++ .dbias_va = u->dbias_va, ++ .background_texture = u->background_texture, ++ .occlusion_query_va = u->occlusion_query_va, ++ .occlusion_query_size = u->occlusion_query_size, ++ .scissor_count = u->scissor_count, ++ .dbias_count = u->dbias_count, ++ .bgobj_depth = u->bgobj_depth, ++ .bgobj_vals = u->bgobj_vals, ++ .pixel_width = u->pixel_output_width, ++ .zls_mask = u->zls_address_mask, ++ }; ++ memcpy(e->zls, u->zls, sizeof(e->zls)); ++} ++ ++static void g5p_render_pass3_from_uapi(struct g5p_ru_pass3 *v, ++ const struct drm_g5p_render_pass_v3 *u) ++{ ++ *v = (struct g5p_ru_pass3){ ++ .ta_program_va = u->ta_program_va, ++ .ta_program_size = u->ta_program_size, ++ .vdm_end = u->vdm_end, ++ .pixel_width = u->pixel_output_width, ++ .pad = u->pad, ++ }; ++} ++ ++static_assert(offsetof(struct drm_g5p_render_pass, samples) == 296); ++static_assert(offsetof(struct drm_g5p_render_pass, sample_positions) == 304); ++static_assert(offsetof(struct drm_g5p_render_pass, reserved) == 312); ++static_assert(DRM_G5P_CAP_RENDER_MSAA == BIT_ULL(31)); ++static_assert(DRM_G5P_CAP_RENDER_OCCLUSION_QUERY == BIT_ULL(33)); ++static_assert(DRM_G5P_RENDER_MSAA_RESOLVE == G5P_RU_MSAA_RESOLVE); ++ ++/* CAP_RENDER_MSAA: the target's geometry is kernel-derived (it holds the sample ++ * count), and only TYPED_V3 carries the pass fields. ++ */ ++static bool g5p_render_msaa_enabled(void) ++{ ++ return (1) && (1); ++} ++ ++/* The base pass's MSAA fields: zero unless TYPED_V3 with the capability. */ ++static int g5p_render_msaa_from_uapi(struct g5p_ru_msaa *m, ++ const struct drm_g5p_render_pass *u, ++ bool v3) ++{ ++ *m = (struct g5p_ru_msaa){ ++ .samples = u->samples, ++ .flags = u->msaa_flags, ++ .positions = u->sample_positions, ++ }; ++ if ((m->samples || m->flags || m->positions) && ++ (!v3 || !g5p_render_msaa_enabled())) ++ return -EINVAL; ++ return 0; ++} ++ ++/* Render submit lock held. The first accepted job claims the device for render until reboot; ++ * same exclusions as compute/r22. Every later job, of any context, passes while the device is ++ * neither failed nor stopped, without the core lock (a job may hold it for its whole run). The ++ * claimant keeps the reference d->quarantined_context has always held. The IRQ-test term is ++ * the r74 one (render-fixed.inc, render-r72.inc): the r74 boots refuse the IRQ test, so ++ * irq_tested must stay 0 there and must be 1 on every other boot. ++ */ ++static int g5p_render_device_claim(struct g5p_drm *d, struct g5p_context *c) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_render *r = c->render; ++ int ret = -EBUSY; ++ ++ if (READ_ONCE(d->render_claimed)) { ++ /* A recovery swaps the epoch, and a failure noted mid-job may lead to one: the job ++ * waits for the ruling in the scheduler, which fails it if the epoch ends. ++ */ ++ if (g5p_rdev_ruling(d->rdev)) ++ return 0; ++ b = READ_ONCE(g->boot); ++ return !b || READ_ONCE(b->failed) || READ_ONCE(b->stopped) ? ++ -EIO : ++ 0; ++ } ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ if (b->failed || b->stopped || !b->control_complete || b->irq_tested || ++ b->user_compute_owned || d->quarantined_context || ++ !b->render_meta[0].cpu || !b->render_meta[1].cpu || ++ b->layout.used[1] > G5P_R22_TA || ++ /* recovery108: expose the GPU only with a current-generation ALIVE proof. */ ++ g5p_ds_hook_claim(g) || atomic_cmpxchg(&d->accepted, 0, 1)) ++ goto out; ++ b->user_compute_owned = true; /* blocks diagnostic commands, latched */ ++ r->claimant = true; ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->quarantined_context = c; ++ WRITE_ONCE(d->render_claimed, true); ++ ret = 0; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Evict the oldest completed result beyond the per-file bound. */ ++static int g5p_render_result_slot(struct g5p_file *f, struct g5p_drm *d) ++{ ++ unsigned long index = 0, flags; ++ struct g5p_fence *old; ++ bool done; ++ ++ if (f->render_results < G5P_RU_RESULTS) ++ return 0; ++ old = xa_find(&f->results, &index, ULONG_MAX, XA_PRESENT); ++ if (!old) ++ return 0; ++ spin_lock_irqsave(&d->status_lock, flags); ++ done = old->state == DRM_G5P_JOB_COMPLETE; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!done) ++ return -EBUSY; ++ xa_erase(&f->results, index); ++ dma_fence_put(&old->base); ++ f->render_results--; ++ return 0; ++} ++ ++static int g5p_render_submit_admitted(struct drm_device *drm, ++ struct drm_file *file, u32 context, ++ u32 profile, u32 rt_handle, u64 res_ptr, ++ u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, ++ u32 pass_size, u64 *job_id) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_render_pass_v3 *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_ru_pass pass; ++ struct g5p_ru_pass2 pass2 = {}; ++ struct g5p_ru_pass3 pass3 = {}; ++ struct g5p_ru_v3_loc loc = {}; ++ struct g5p_ru_msaa msaa = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ struct g5p_rt *rt = NULL; ++ const bool v2 = false; ++ const bool v3 = true; ++ u64 point = 0; ++ int ret; ++ ++ ; ++ ; ++ /* TYPED_V3 is unprivileged (checked per context below); the legacy profiles touch ++ * registers and stay with CAP_SYS_RAWIO. ++ */ ++ ; ++ ; ++ /* During a recovery the epoch is being replaced: accept, the scheduler holds the job. */ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ /* a failure noted mid-job waits for the executor's ruling (g5p_rdev_ruling()) */ ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(res_ptr), res_count, ++ sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(sync_ptr), sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ { ++ /* pass_size is the exact V1, V2 or V3 size; ext exists for V2 and V3, ++ * the program fields for V3 only. ++ */ ++ upass = memdup_user(u64_to_user_ptr(pass_ptr), pass_size); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (upass->base.size != pass_size || upass->base.flags || ++ memchr_inv(upass->base.reserved, 0, ++ sizeof(upass->base.reserved)) || ++ (true && memchr_inv(upass->ext.reserved, 0, ++ sizeof(upass->ext.reserved))) || ++ (v3 && ++ memchr_inv(upass->reserved, 0, sizeof(upass->reserved)))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_render_pass_from_uapi(&pass, &upass->base); ++ ret = g5p_render_msaa_from_uapi(&msaa, &upass->base, v3); ++ if (ret) ++ goto out; ++ { ++ g5p_render_pass2_from_uapi(&pass2, &upass->ext); ++ }; ++ { ++ g5p_render_pass3_from_uapi(&pass3, upass); ++ }; ++ } ++ ranges = kvcalloc(res_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(res_count, sizeof(*job->objs), GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ /* Acceptance, implicit fence attach and queueing happen in one order. */ ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ { ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, res_count, ranges, job->objs, ++ &job->obj_count); ++ if (ret) ++ goto unlock; ++ { ++ rt = xa_load(&f->targets, rt_handle); ++ if (!rt || rt->ctx != c || rt->destroyed) { ++ ret = -ENOENT; ++ goto unlock; ++ } ++ } ++ { ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ u8 head[G5P_RU_PROGRAM_MIN], tail[4]; ++ ++ /* Structure first; the program head and the stream's last dword are ++ * read only once every range is proven backed, READ and (program) EXEC. ++ */ ++ ret = g5p_ru_pass3_check(&pass, &pass2, &pass3, &lim, ranges, ++ res_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass3.ta_program_va, head, ++ sizeof(head)); ++ if (!ret) ++ ret = g5p_render_read_va( ++ drm, c->vm, pass.vdm_va + pass3.vdm_end - 4, ++ tail, sizeof(tail)); ++ if (!ret) ++ ret = g5p_ru_pass3_content(head, tail); ++ } ++ /* Every profile: a one-sample pass needs a one-sample target, and back. */ ++ if (!ret) ++ ret = g5p_ru_msaa_check(&msaa, rt->samples ? rt->samples : 1); ++ if (ret) ++ goto unlock; ++ g5p_ru_fill_ta(job->ta_w, &pass); ++ { ++ g5p_ru_fill_ta_v3(job->ta_w, loc.program_off); ++ }; ++ g5p_ru_fill_3d(job->td_w, &pass); ++ /* TYPED_V3 with an all-zero ext is the V1 dialect: the V1 3D words, plus ++ * the output width words (width 2 rewrites the frozen dialect-1 values). ++ */ ++ if (v2 || loc.dialect2) ++ g5p_ru_fill_3d_v2(job->td_w, &pass2); ++ else { ++ g5p_ru_fill_3d_width(job->td_w, loc.pixel_width); ++ }; ++ /* r25 RT variant 1 moves only TA.W28/3D.W20 one slice into the ring. */ ++ g5p_ru_rt_fill(job->ta_w, job->td_w, &rt->geometry, ++ rt->node.start + (r->v.rt == 1 ? rt->sizes.slice : 0), ++ rt->node.start + rt->sizes.main); ++ { ++ g5p_ru_fill_msaa(job->ta_w, job->td_w, &msaa, &rt->geometry); ++ }; ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ /* Unpublished, never signalled: still a clean rejection. */ ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = v3; ++ { ++ r->v3_accepted = true; ++ }; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ job->cache.pid = task_tgid_nr(current); ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); /* only ever an uninitialized fence here */ ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++/* recovery108: a submitter is inside the admission window for its whole call; once a ++ * FINAL teardown closed admission it is refused before it looks at anything. ++ */ ++static int g5p_render_submit(struct drm_device *drm, struct drm_file *file, ++ u32 context, u32 profile, u32 rt_handle, ++ u64 res_ptr, u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, u32 pass_size, ++ u64 *job_id) ++{ ++ int ret; ++ ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_render_submit_admitted(drm, file, context, profile, rt_handle, ++ res_ptr, res_count, sync_ptr, ++ sync_count, pass_ptr, pass_size, ++ job_id); ++ g5p_r8_exit(); ++ return ret; ++} ++ ++/* The layout the profile dispatch (pass_size selects the profile) and ++ * GET_RENDER_INFO rely on; a header edit that moves any of it fails here. ++ */ ++static_assert(sizeof(struct drm_g5p_render_pass) == 360); ++static_assert(offsetof(struct drm_g5p_render_pass, vdm_control) == 20); ++static_assert(sizeof(struct drm_g5p_render_pass_ext) == 128); ++static_assert(sizeof(struct drm_g5p_render_pass_v2) == 488); ++static_assert(sizeof(struct drm_g5p_render_pass_v3) == 536); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ext) == ++ sizeof(struct drm_g5p_render_pass)); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_va) == 488); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_size) == 496); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, vdm_end) == 500); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, pixel_output_width) == ++ 504); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, reserved) == 512); ++static_assert(sizeof(struct drm_g5p_submit_render_typed) == 80); ++static_assert(sizeof(struct drm_g5p_get_render_info) == 120); ++static_assert(offsetof(struct drm_g5p_get_render_info, hw_va_limit) == 104); ++static_assert(offsetof(struct drm_g5p_get_render_info, reserved1) == 112); ++static_assert(DRM_G5P_CAP_TYPED_RENDER_V3 == BIT_ULL(26)); ++static_assert(DRM_G5P_CAP_RENDER_MULTI_CONTEXT == BIT_ULL(27)); ++static_assert(DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH == BIT_ULL(29)); ++static_assert(DRM_G5P_RENDER_PROFILE_TYPED_V3 == 4); ++ ++static int g5p_submit_render_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_render_typed *a = data; ++ u64 id = 0; ++ int ret; ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || a->profile != DRM_G5P_RENDER_PROFILE_TYPED_V3 || ++ !a->render_target || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_render_pass_v3)) ++ return -EINVAL; ++ ret = g5p_render_submit(drm, file, a->context, ++ DRM_G5P_RENDER_PROFILE_TYPED_V3, ++ a->render_target, a->resources, ++ a->resource_count, a->syncs, a->sync_count, ++ a->pass, a->pass_size, &id); ++ a->job_id = id; ++ return ret; ++} ++ ++static void g5p_rt_geometry_to_uapi(struct drm_g5p_rt_geometry *u, ++ const struct g5p_ru_rt *k) ++{ ++ u->header_bytes = k->header; ++ u->region_bytes = k->region; ++ u->packed_state = k->packed; ++ u->state_word = k->state; ++ u->padded_dims = k->padded; ++ memcpy(u->scale, k->scale, sizeof(u->scale)); ++ memcpy(u->ta_state, k->ta_state, sizeof(u->ta_state)); ++ memcpy(u->ta_tiling, k->ta_tiling, sizeof(u->ta_tiling)); ++ u->ta_tiling_word = k->ta_tiling_word; ++ u->tile_count = k->tiles; ++ u->secondary_bytes = k->secondary; ++ u->secondary_align = k->secondary_align; ++} ++ ++static int g5p_create_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ const struct drm_g5p_rt_geometry *u = &a->geometry; ++ struct g5p_ru_rt geometry; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ bool user = a->flags & DRM_G5P_RT_USER_GEOMETRY; ++ bool kernel = (1); ++ u64 positions = a->samples == 4 ? G5P_RU_POSITIONS_4X : ++ G5P_RU_POSITIONS_1X; ++ u32 id, blocks = 0; ++ int ret; ++ ++ ; ++ ; ++ if (a->flags & ~DRM_G5P_RT_USER_GEOMETRY || a->handle || a->pad || ++ a->main_va || a->main_bytes || a->secondary_va || ++ a->secondary_bytes || ++ (a->samples != 1 && ++ (a->samples != 4 || !g5p_render_msaa_enabled())) || ++ a->layers != 1 || a->pm_blocks || a->pad2 || a->reserved || ++ u->reserved[0] || u->reserved[1]) ++ return -EINVAL; ++ if (user) { ++ { ++ return -EOPNOTSUPP; ++ } ++ } else if (kernel) { ++ /* 6ca6d5c in integers; equal to the leaf for every size. */ ++ if (memchr_inv(u, 0, sizeof(*u)) || ++ g5p_ru_rt_derive_samples(a->width, a->height, a->samples, ++ positions, &geometry)) ++ return -EINVAL; ++ } else { ++ /* Kernel table: only the offline-executed 6ca6d5c result. */ ++ if (a->width != 64 || a->height != 64 || ++ memchr_inv(u, 0, sizeof(*u))) ++ return -EOPNOTSUPP; ++ geometry = g5p_ru_rt64; ++ } ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto out; ++ } ++ r = c->render; ++ ret = g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ /* No core lock: another context's job may hold it for its whole run, and nothing here ++ * touches the hardware. The device pool only grows; it serves every target of every ++ * context. ++ */ ++ if (kernel) { ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, g5p_ru_pm_blocks_for(geometry.tiles, ++ d->rdev->blocks)); ++ blocks = d->rdev->blocks; ++ mutex_unlock(&d->rdev->lock); ++ } ++ if (!ret) ++ ret = g5p_rt_create(d, r, &geometry, a->width, a->height, &rt); ++ if (ret) ++ goto out; ++ rt->samples = a->samples; ++ ret = xa_alloc(&f->targets, &id, rt, ++ XA_LIMIT(1, G5P_RU_MAX_TARGETS * 8), GFP_KERNEL); ++ if (ret) { ++ /* Mapped but unpublished to a job: leave it owned by the context. */ ++ rt->destroyed = true; ++ goto out; ++ } ++ kref_get(&c->ref); ++ a->handle = id; ++ a->main_va = rt->node.start; ++ a->main_bytes = rt->sizes.main; ++ a->secondary_va = rt->node.start + rt->sizes.main; ++ a->secondary_bytes = rt->sizes.secondary_map; ++ if (!user) ++ g5p_rt_geometry_to_uapi(&a->geometry, &geometry); ++ if (kernel) ++ a->pm_blocks = blocks; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_destroy_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c = NULL; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ int ret; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ rt = xa_load(&f->targets, a->handle); ++ if (!rt) { ++ ret = -ENOENT; ++ goto out; ++ } ++ c = rt->ctx; ++ r = c->render; ++ ret = r->detached ? 0 : g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ g5p_vmlog_add(G5P_VMLOG_RT_DEL, c->vm, 0, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map, ++ rt->width << 16 | rt->height, rt->samples); ++ /* RAM only, no core lock: see g5p_create_render_target(). */ ++ if (r->bound) { ++ g5p_vm_clear(c->vm, drm->dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ ret = g5p_render_invalidate(c); ++ if (ret) ++ goto out; ++ } ++ if (!r->detached) { ++ g5p_rt_release_memory(d, r, rt, true); ++ list_del(&rt->link); ++ r->target_count--; ++ if (r->profile1_rt == rt) ++ r->profile1_rt = NULL; ++ kfree(rt); ++ } else { ++ rt->destroyed = true; ++ } ++ xa_erase(&f->targets, a->handle); ++out: ++ mutex_unlock(&f->lock); ++ if (!ret && c) ++ g5p_context_put(c); ++ return ret; ++} ++ ++/* True when a context created now would latch a hypothesis knob that differs ++ * from its default: GET_RENDER_INFO has no context to report on. ++ */ ++static bool g5p_render_variant_active(void) ++{ ++ return (0) || (G5P_RU_DEF_CACHE_VARIANT) != G5P_RU_DEF_CACHE_VARIANT || ++ (G5P_RU_DEF_MMU_VARIANT) != G5P_RU_DEF_MMU_VARIANT || (0) || ++ READ_ONCE(render_user_wait_ms) != G5P_RU_DEF_WAIT_MS || (0) || ++ (0) || (0) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) != ++ G5P_RU_DEF_PROFILE1_VDM_CONTROL || ++ (0) || (0) || (0) || (0); ++} ++ ++static int g5p_get_render_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_render_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || ++ memchr_inv(&a->profiles, 0, ++ sizeof(*a) - offsetof(struct drm_g5p_get_render_info, ++ profiles))) ++ return -EINVAL; ++ ; ++ ; ++ if (g5p_render_variant_active()) ++ a->flags = DRM_G5P_RENDER_INFO_VARIANT_ACTIVE; ++ a->profiles = BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ ; ++ ; ++ { ++ a->profiles |= BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ }; ++ { ++ a->max_render_width = G5P_RU_MAX_DIM; ++ a->max_render_height = G5P_RU_MAX_DIM; ++ a->max_pm_blocks = G5P_RU_MAX_BLOCKS; ++ } ++ a->max_render_targets = G5P_RU_MAX_TARGETS; ++ a->private_va_start = G5P_RU_PRIVATE_VA; ++ a->private_va_end = G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE; ++ a->user_va_min = G5P_RU_USER_VA_MIN; ++ a->usc_base = G5P_RU_PDS_BASE; ++ a->pds_base = a->usc_base; ++ a->code_window = G5P_RU_CODE_WINDOW; ++ a->hw_va_limit = G5P_RU_HW_VA_LIMIT; ++ a->pm_blocks = (32); ++ a->pm_block_bytes = G5P_RU_BLOCK_BYTES; ++ a->max_queued_jobs = G5P_RU_MAX_QUEUED; ++ a->max_vdm_bytes = G5P_RU_MAX_VDM; ++ /* TYPED_V3: what a waiter must outlast is the job's absolute bound (TA publish to the end of the ++ * 3D, render-user-wait.h), not the per-phase module value: a TA that takes its whole phase leaves the ++ * 3D the rest of the job cap. ++ */ ++ { ++ if (READ_ONCE(render_user_hard_ms)) ++ a->job_timeout_ms = g5p_ru_lat_job_ms( ++ READ_ONCE(render_user_hard_ms), ++ READ_ONCE(render_user_obs_ms)); ++ else ++ a->job_timeout_ms = G5P_RU_V3_CAP_MS; ++ }; ++ a->retained_results = G5P_RU_RESULTS; ++ return 0; ++} ++ ++/* VM_BIND hooks for the context resident on a VM (serial compute or render). */ ++static int g5p_resident_idle(struct g5p_context *c, bool live) ++{ ++ return g5p_render_vm_idle(c, live); ++} ++ ++/* VM_BIND UNMAP, file lock held, PTEs already cleared and the tag published: a render ++ * context with queued jobs keeps the mapping's object (its pages and its pin) until those ++ * jobs have retired (g5p_render_reap()). False: free it now. ++ */ ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m) ++{ ++ struct g5p_render *r = c->render; ++ unsigned long flags; ++ ++ if (!r || !atomic_read(&r->queued)) ++ return false; ++ m->retire = r->fence_seqno; ++ spin_lock_irqsave(&r->zlock, flags); ++ list_add_tail(&m->link, &r->zombies); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ return true; ++} ++ ++static int g5p_resident_invalidate(struct g5p_context *c) ++{ ++ return g5p_render_invalidate(c); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-submit.inc b/drivers/gpu/drm/apple-g5p/drm-submit.inc +new file mode 100644 +index 000000000..f36998e4d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-submit.inc +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Asynchronous privileged probes; payloads and firmware VM are kernel-owned. */ ++struct g5p_fence { ++ struct dma_fence base; ++ /* Owned by dma_fence; independent of queue/status locks. */ ++ spinlock_t lock; ++ u32 state; ++ int error; ++}; ++ ++static const char *g5p_fence_driver_name(struct dma_fence *f) ++{ ++ return "apple_g5p"; ++} ++ ++static const char *g5p_fence_timeline_name(struct dma_fence *f) ++{ ++ return "g5p-probe"; ++} ++ ++/* Default release uses dma_fence_free()/RCU; base is the first member. */ ++static const struct dma_fence_ops g5p_fence_ops = { ++ .get_driver_name = g5p_fence_driver_name, ++ .get_timeline_name = g5p_fence_timeline_name, ++}; ++ ++/* recovery108: the compute/probe submitters share the render path's admission window. */ ++ ++/* Legacy JOB_COMPUTE remains unsupported: v1 lacks a bounded stream length. */ +diff --git a/drivers/gpu/drm/apple-g5p/render-user-validate.h b/drivers/gpu/drm/apple-g5p/render-user-validate.h +new file mode 100644 +index 000000000..a466367bb +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/render-user-validate.h +@@ -0,0 +1,384 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_VALIDATE_H ++#define G5P_RENDER_USER_VALIDATE_H ++ ++/* Pure TYPED_V1 checks over already-resolved resource ranges. The caller ++ * proves each range is inside a mapping of the listed BO, in the submitting ++ * context's VM, with BO roles covering `access`. Nothing here is a pointer ++ * the kernel dereferences; GPU addresses are only range-checked. ++ */ ++#define G5P_RU_ACCESS_READ 1U ++#define G5P_RU_ACCESS_WRITE 2U ++/* Software admission bound; TYPED_V3 never sends a stream length to firmware. ++ * vdm_size/end only bound the listed range and the four-byte tail read. ++ */ ++#define G5P_RU_MAX_VDM 0x100000U ++#define G5P_RU_MAX_VDM_CONTROL 0x10000U ++#define G5P_RU_MAX_SCRATCH (64ULL << 20) ++ ++/* Hardware facts GET_RENDER_INFO reports and the TYPED_V3 validator enforces; ++ * one definition so the two cannot drift. Code offsets are 32-bit from the ++ * common USC/PDS base. TA/3D reads at or above 2^38 stall silently on A10 ++ * (works at 0x3_, 0x10_ and 0x20_, stalls at 0x40_ and 0x80_, no MMU fault); ++ * not flown on A9, so the limit is a statement, not a proof (see ++ * docs/gpu/experiments/typed-v3-kernel-implementation.md). ++ */ ++#define G5P_RU_PDS_BASE 0x300000000ULL ++#define G5P_RU_CODE_WINDOW 0x100000000ULL /* 2^32 */ ++#define G5P_RU_HW_VA_LIMIT 0x4000000000ULL /* 2^38 */ ++ ++/* Not a requested access: the resolver sets it, for the typed profiles only, ++ * when the listed BO carries DRM_G5P_BO_EXEC. Only TYPED_V3 reads it; every ++ * other check masks on READ/WRITE. It lives in `access` rather than in a new ++ * field so positional g5p_ru_range initialisers keep compiling. ++ */ ++#define G5P_RU_ACCESS_EXEC 16U ++ ++struct g5p_ru_range { ++ u64 va, size; ++ u32 access; ++}; ++ ++/* [va, va + size) inside one listed range that grants every access bit. */ ++static inline bool g5p_ru_in(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 access) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if ((r[i].access & access) == access && va >= r[i].va && ++ va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) ++ return true; ++ return false; ++} ++ ++/* Everything TYPED_V1 and TYPED_V2 share: all fields except the pools. */ ++static inline int g5p_ru_pass_check_common(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || ++ !g5p_ru_in(r, n, p->vdm_va, p->vdm_size, G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ /* Copied to TA W88/R0+470; meaning unresolved (r23), so bounded. */ ++ if ((p->vdm_control & 0xf) || p->vdm_control > G5P_RU_MAX_VDM_CONTROL) ++ return -EINVAL; ++ if ((p->scratch_va & 0xf) || p->scratch_size < 0x10 || ++ p->scratch_size > G5P_RU_MAX_SCRATCH || ++ !g5p_ru_in(r, n, p->scratch_va, p->scratch_size, rw)) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((p->scratch_ptr[i] & 3) || ++ !g5p_ru_in(r, n, p->scratch_ptr[i], 4, rw)) ++ return -EINVAL; ++ /* Three-word descriptors keep their fourth slot zero. */ ++ if (p->ta_pds[3] || p->eot[3] || p->eot_resume[3]) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V1 and LINEAR64_V1: the two allocator pools are page-aligned RW. */ ++static inline int g5p_ru_pass_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret = g5p_ru_pass_check_common(p, r, n); ++ ++ if (ret) ++ return ret; ++ if (((p->pool7_va | p->pool7_size | p->pool8_va | p->pool8_size) & ++ (G5P_GPU_PAGE - 1)) || ++ !g5p_ru_in(r, n, p->pool7_va, p->pool7_size, rw) || ++ !g5p_ru_in(r, n, p->pool8_va, p->pool8_size, rw)) ++ return -EINVAL; ++ return 0; ++} ++ ++#define G5P_RU_MAX_TABLE 65536U ++ ++/* TYPED_V2: the pools are the ISP scissor (8-byte entries) and depth-bias ++ * (12-byte) tables, READ and word aligned; the V1 pool fields stay zero. ++ * Flagged ZLS words are addresses into listed RW ranges; the rest is opaque. ++ */ ++static inline int g5p_ru_pass2_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret; ++ ++ if (p->pool7_va || p->pool7_size || p->pool8_va || p->pool8_size) ++ return -EINVAL; ++ ret = g5p_ru_pass_check_common(p, r, n); ++ if (ret) ++ return ret; ++ if (!e->scissor_count || e->scissor_count > G5P_RU_MAX_TABLE || ++ (e->scissor_va & 3) || ++ !g5p_ru_in(r, n, e->scissor_va, e->scissor_count * 8ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if (!e->dbias_count || e->dbias_count > G5P_RU_MAX_TABLE || ++ (e->dbias_va & 3) || ++ !g5p_ru_in(r, n, e->dbias_va, e->dbias_count * 12ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if ((e->pixel_width != 2 && e->pixel_width != 4 && ++ e->pixel_width != 8) || ++ (e->bgobj_vals & ~0x3ffU) || (e->zls_mask & ~0x3fU)) ++ return -EINVAL; ++ if (!!e->occlusion_query_va != !!e->occlusion_query_size || ++ (e->occlusion_query_va && ++ ((e->occlusion_query_va & 0xf) || e->occlusion_query_size < sizeof(u32) || ++ !g5p_ru_in(r, n, e->occlusion_query_va, e->occlusion_query_size, rw)))) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((e->zls_mask & (1U << i)) && ++ ((e->zls[i] >> 40) || ++ !g5p_ru_in(r, n, e->zls[i] & ~0xfULL, 16, rw))) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V3: the TA PDS program (W+0x88 -> R0+0x470) is an explicit reference ++ * and the VDM stream length is a separate field that never reaches W. ++ * Contract and evidence: docs/gpu/experiments/typed-v3-kernel-implementation.md. ++ * Everything here is rejected, never masked. Structural rules need only the ++ * resolved ranges; the two content rules take bytes the caller read through ++ * the BOs' own pages (g5p_ru_pass3_content()). The rules are a subset of what ++ * the userspace lane accepts (tools/mesa-a9 a9c_ctx_lint(), a9scene): nothing ++ * here is stricter than a client that passes those lints. ++ * -EINVAL stale vdm_control, malformed program/stream/descriptor fields, ++ * no terminator, or (V1 dialect) the V1 rules fail ++ * -ERANGE outside [pds_base, pds_base + code_window) or past hw_va_limit ++ * -ENOENT no listed range wholly covers the program, stream or task code ++ * -EACCES missing READ, or the program/task code range's BO lacks EXEC ++ * -ENODATA the first 16 program bytes are all zero ++ */ ++#define G5P_RU_PROGRAM_ALIGN 16U ++#define G5P_RU_PROGRAM_MIN 16U ++ ++struct g5p_ru_v3_limits { ++ u64 pds_base, code_window; ++ u64 hw_va_limit; /* 0: no statement */ ++}; ++ ++/* Where the content rules read: indexes into the ranges array. */ ++struct g5p_ru_v3_loc { ++ u64 program_off; /* the exact W+0x88 value */ ++ u32 program_range, vdm_range; ++ u32 pixel_width; /* the pass's output width, 2, 4 or 8, either dialect */ ++ bool dialect2; /* ext was non-zero: V2 3D words; else the V1 dialect */ ++}; ++ ++/* TYPED_V3 carries the V2 ext struct. An all-zero ext selects the V1 dialect ++ * (pools in base, no tables, V1 3D words): the pass tools/mesa-a9 records ++ * today, plus the explicit program. Any non-zero ext field selects V2. ++ */ ++static inline bool g5p_ru_pass2_zero(const struct g5p_ru_pass2 *e) ++{ ++ u64 any = e->scissor_va | e->dbias_va | e->background_texture | ++ e->occlusion_query_va | e->occlusion_query_size; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(e->zls); i++) ++ any |= e->zls[i]; ++ any |= e->scissor_count | e->dbias_count | e->bgobj_depth | ++ e->bgobj_vals | e->pixel_width | e->zls_mask; ++ return !any; ++} ++ ++/* First listed range that wholly contains [va, va + size). */ ++static inline bool g5p_ru_cover(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 *idx) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if (va >= r[i].va && va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) { ++ *idx = i; ++ return true; ++ } ++ return false; ++} ++ ++/* [va, va + size) ends at or below the limit; zero limit makes no statement. */ ++static inline bool g5p_ru_below(u64 limit, u64 va, u64 size) ++{ ++ return !limit || (size <= limit && va <= limit - size); ++} ++ ++/* One PDS task reference of a store/resume descriptor: a 32-bit offset from ++ * pds_base, 16-byte aligned like every captured value and W+0x88's mask, ++ * non-zero, inside the code window and the hardware limit, and starting in a ++ * listed READ range (`code`: whose BO also carries EXEC). ++ */ ++static inline int g5p_ru_v3_ref(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, u32 off, ++ bool code) ++{ ++ u64 va; ++ u32 i; ++ ++ if (!off || (off % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (lim->code_window < G5P_RU_PROGRAM_MIN || ++ off > lim->code_window - G5P_RU_PROGRAM_MIN || ++ lim->pds_base > ~0ULL - off) ++ return -ERANGE; ++ va = lim->pds_base + off; ++ if (!g5p_ru_below(lim->hw_va_limit, va, G5P_RU_PROGRAM_MIN)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, va, G5P_RU_PROGRAM_MIN, &i)) ++ return -ENOENT; ++ if (!(r[i].access & G5P_RU_ACCESS_READ) || ++ (code && !(r[i].access & G5P_RU_ACCESS_EXEC))) ++ return -EACCES; ++ return 0; ++} ++ ++/* The TYPED_V3 pixel output width. The V1 dialect takes it from the V3 field ++ * (0 = 2, the V1 dialect exactly), the V2 dialect from ext, where the V3 field ++ * must stay zero. A width above 2 is several colour targets, and a partial ++ * render must then resume with the pass's own EOT: a one-target resume EOT ++ * would store only the first target. The EOT itself is not decoded. ++ */ ++static inline int g5p_ru_pass3_width(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ bool dialect2, u32 *width) ++{ ++ u32 w = v->pixel_width; ++ ++ if (v->pad || (w && w != 2 && w != 4 && w != 8) || (dialect2 && w)) ++ return -EINVAL; ++ if (dialect2) ++ w = e->pixel_width; /* 2, 4 or 8: g5p_ru_pass2_check() */ ++ else if (!w) ++ w = 2; ++ if (w > 2 && memcmp(p->eot, p->eot_resume, sizeof(p->eot))) ++ return -EINVAL; ++ *width = w; ++ return 0; ++} ++ ++/* The V3 rules, then the TYPED_V1 rules (all-zero ext) or the TYPED_V2 rules ++ * (any ext field set; pools zero), then the output width. base.vdm_control is ++ * checked first. On success loc names the ranges to read, the dialect and the ++ * width. ++ */ ++static inline int g5p_ru_pass3_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_ru_v3_loc *loc) ++{ ++ u32 pi, vi; ++ u64 off; ++ int ret; ++ ++ if (p->vdm_control) ++ return -EINVAL; /* a stream length here is the r36/r88 defect */ ++ if (v->ta_program_size < G5P_RU_PROGRAM_MIN || ++ v->ta_program_size > DRM_G5P_RENDER_MAX_PROGRAM_BYTES || ++ (v->ta_program_size % G5P_RU_PROGRAM_ALIGN) || ++ (v->ta_program_va % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || v->vdm_end < 4 || ++ v->vdm_end > p->vdm_size || (v->vdm_end & 3)) ++ return -EINVAL; ++ if (v->ta_program_va < lim->pds_base) ++ return -ERANGE; ++ off = v->ta_program_va - lim->pds_base; ++ if (lim->code_window < v->ta_program_size || ++ off > lim->code_window - v->ta_program_size) ++ return -ERANGE; ++ if (!g5p_ru_below(lim->hw_va_limit, v->ta_program_va, ++ v->ta_program_size) || ++ !g5p_ru_below(lim->hw_va_limit, p->vdm_va, p->vdm_size)) ++ return -ERANGE; ++ /* Every listed range, not only the two above: on A10 a buffer moved past ++ * the limit stalls the TA without a fault, and all buffers moved together. ++ */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, v->ta_program_va, v->ta_program_size, &pi)) ++ return -ENOENT; ++ if (!(r[pi].access & G5P_RU_ACCESS_READ) || ++ !(r[pi].access & G5P_RU_ACCESS_EXEC)) ++ return -EACCES; ++ if (!g5p_ru_cover(r, n, p->vdm_va, p->vdm_size, &vi)) ++ return -ENOENT; ++ if (!(r[vi].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ /* Context store [0] / load [1]: code at word 2 (READ and EXEC), STREAM_OUT2 ++ * at word 5 (READ). Resume tasks: word 2 (READ only: R0+0xf060/0xf070, ++ * named stream-out by the userspace lane and aux program by r92 2.4, so ++ * its role is unresolved and EXEC is not demanded). Words 0, 1, 3 and 4 ++ * stay opaque. The spacing (store - program = 0x280, resume - store = ++ * 0x340) and a program that equals a task entry (r63 flew the store ++ * program; the role is not detectable structurally) are not enforced. ++ */ ++ for (u32 i = 0; i < 2; i++) { ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][2], true); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][5], ++ false); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->resume_pds[i][2], ++ false); ++ if (ret) ++ return ret; ++ } ++ loc->dialect2 = !g5p_ru_pass2_zero(e); ++ ret = loc->dialect2 ? g5p_ru_pass2_check(p, e, r, n) : ++ g5p_ru_pass_check(p, r, n); ++ if (!ret) ++ ret = g5p_ru_pass3_width(p, e, v, loc->dialect2, ++ &loc->pixel_width); ++ if (ret) ++ return ret; ++ loc->program_off = off; ++ loc->program_range = pi; ++ loc->vdm_range = vi; ++ return 0; ++} ++ ++/* The first 16 program bytes are not all zero (r88 published 0x370, which ++ * pointed at zeros) and the last stream dword is the terminator. `head` is ++ * G5P_RU_PROGRAM_MIN bytes, `tail` the 4 stream bytes at vdm_va + vdm_end - 4. ++ */ ++static inline int g5p_ru_pass3_content(const u8 *head, const u8 *tail) ++{ ++ u8 any = 0; ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ for (u32 i = 0; i < G5P_RU_PROGRAM_MIN; i++) ++ any |= head[i]; ++ if (!any) ++ return -ENODATA; ++ if (word != DRM_G5P_RENDER_STREAM_TERMINATE) ++ return -EINVAL; ++ return 0; ++} ++ ++/* AGXRenderTargetMemoryG5 output sanity for userspace geometry: exact one ++ * sample and the sizes the kernel will allocate. Opaque fields unchecked. ++ */ ++static inline int g5p_ru_geometry_check(const struct g5p_ru_rt *rt, u32 width, ++ u32 height, ++ struct g5p_ru_rt_sizes *sizes) ++{ ++ if (!width || !height || width > 16384 || height > 16384) ++ return -EINVAL; ++ if (rt->tiles > ((width + 31) / 32) * ((height + 31) / 32)) ++ return -EINVAL; ++ return g5p_ru_rt_sizes(rt, sizes); ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/runtime-idle.inc b/drivers/gpu/drm/apple-g5p/runtime-idle.inc +new file mode 100644 +index 000000000..a83d99ad3 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/runtime-idle.inc +@@ -0,0 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++static int g5p_idle_open(struct g5p_drm *d) ++{ ++ atomic_inc(&d->open_files); ++ return 0; ++} ++ ++static void g5p_idle_work(struct work_struct *work) ++{ ++} ++ ++static void g5p_idle_queue(struct g5p_drm *d) ++{ ++} diff --git a/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch b/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch new file mode 100644 index 00000000..b26eba8a --- /dev/null +++ b/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch @@ -0,0 +1,147 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:11:46 +0000 +Subject: [PATCH] drm: expose the native GEM exporter CPU cache contract + +[Orchard: from brkosebay/linux 78775ea3b9e; retain Orchard ADBE +implementation, adding its exporter contract only.] + +G5P PRIME imports require an immutable native cache mapping contract. +Unknown and foreign exporters stay conservative. + +Assisted-by: LLM +--- + drivers/gpu/drm/adbe/adbe_drv.c | 25 +++++++++++++++++++++++++ + drivers/gpu/drm/drm_prime.c | 19 +++++++++++++++++++ + include/drm/drm_gem.h | 8 ++++++++ + include/drm/drm_gem_dma_helper.h | 8 ++++++++ + include/drm/drm_prime.h | 2 ++ + 5 files changed, 62 insertions(+) + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..24c76fef9 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -653,11 +653,36 @@ static int adbe_dumb_create(struct drm_file *file, struct drm_device *drm, + return drm_gem_dma_dumb_create_internal(file, drm, args); + } + ++/* Native allocations: the DMA helper's defaults plus the uncached-export ++ * contract. Imported objects never make that promise. ++ */ ++static const struct drm_gem_object_funcs adbe_gem_funcs = { ++ .free = drm_gem_dma_object_free, ++ .print_info = drm_gem_dma_object_print_info, ++ .get_sg_table = drm_gem_dma_object_get_sg_table, ++ .vmap = drm_gem_dma_object_vmap, ++ .mmap = drm_gem_dma_object_mmap, ++ .vm_ops = &drm_gem_dma_vm_ops, ++ .is_cpu_uncached = drm_gem_dma_object_is_cpu_uncached, ++}; ++ ++static struct drm_gem_object *adbe_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct drm_gem_dma_object *dma_obj = kzalloc_obj(*dma_obj); ++ ++ if (!dma_obj) ++ return ERR_PTR(-ENOMEM); ++ dma_obj->base.funcs = &adbe_gem_funcs; ++ return &dma_obj->base; ++} ++ + DEFINE_DRM_GEM_DMA_FOPS(adbe_fops); + + static const struct drm_driver adbe_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .fops = &adbe_fops, ++ .gem_create_object = adbe_gem_create_object, + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(adbe_dumb_create), + DRM_FBDEV_DMA_DRIVER_OPS, + .name = "adbe", +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index 9b44c78cd..70110173d 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -835,6 +835,25 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { + .vunmap = drm_gem_dmabuf_vunmap, + }; + ++/** ++ * drm_gem_dma_buf_is_cpu_uncached - query a trusted native GEM exporter ++ * @dma_buf: exported buffer whose supported CPU aliases are queried ++ * ++ * Returns true only for the explicit native exporter contract. Non-GEM, ++ * re-exported foreign objects and unspecified exporters remain conservative. ++ */ ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf) ++{ ++ struct drm_gem_object *obj; ++ ++ if (!dma_buf || dma_buf->ops != &drm_gem_prime_dmabuf_ops) ++ return false; ++ obj = dma_buf->priv; ++ return !drm_gem_is_imported(obj) && obj->funcs && ++ obj->funcs->is_cpu_uncached && obj->funcs->is_cpu_uncached(obj); ++} ++EXPORT_SYMBOL_GPL(drm_gem_dma_buf_is_cpu_uncached); ++ + /** + * drm_prime_pages_to_sg - converts a page array into an sg list + * @dev: DRM device +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 885244e37..048193377 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -73,6 +73,14 @@ enum drm_gem_object_status { + * struct drm_gem_object_funcs - GEM object functions + */ + struct drm_gem_object_funcs { ++ /** ++ * @is_cpu_uncached: Optional exporter contract. True only when every ++ * supported CPU mapping (kernel and userspace) is uncached or WC and ++ * no cacheable alias exists. Native allocation properties must be ++ * immutable; imported memory must not inherit this promise. Default ++ * NULL is conservative. This does not remove DMA ordering barriers. ++ */ ++ bool (*is_cpu_uncached)(struct drm_gem_object *obj); + /** + * @free: + * +diff --git a/include/drm/drm_gem_dma_helper.h b/include/drm/drm_gem_dma_helper.h +index f2678e7ec..481007c13 100644 +--- a/include/drm/drm_gem_dma_helper.h ++++ b/include/drm/drm_gem_dma_helper.h +@@ -4,6 +4,7 @@ + + #include + #include ++#include + #include + + struct drm_mode_create_dumb; +@@ -55,6 +56,13 @@ extern const struct vm_operations_struct drm_gem_dma_vm_ops; + * This function wraps drm_gem_dma_free_object(). Drivers that employ the DMA helpers + * should use it as their &drm_gem_object_funcs.free handler. + */ ++static inline bool drm_gem_dma_object_is_cpu_uncached(struct drm_gem_object *obj) ++{ ++ return !drm_gem_is_imported(obj) && ++ !to_drm_gem_dma_obj(obj)->map_noncoherent && ++ !dev_is_dma_coherent(drm_dev_dma_dev(obj->dev)); ++} ++ + static inline void drm_gem_dma_object_free(struct drm_gem_object *obj) + { + struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj); +diff --git a/include/drm/drm_prime.h b/include/drm/drm_prime.h +index f50f862f0..5de957f7d 100644 +--- a/include/drm/drm_prime.h ++++ b/include/drm/drm_prime.h +@@ -116,4 +116,6 @@ int drm_prime_sg_to_page_array(struct sg_table *sgt, struct page **pages, + int drm_prime_sg_to_dma_addr_array(struct sg_table *sgt, dma_addr_t *addrs, + int max_pages); + ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf); ++ + #endif /* __DRM_PRIME_H__ */ diff --git a/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch b/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch new file mode 100644 index 00000000..88cd9ce1 --- /dev/null +++ b/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch @@ -0,0 +1,48 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] Input: apple-z2: stop the touch controller on shutdown + +A kexec hands the next kernel an S8000 SPI bus that the touch controller +may still be using: its interrupt thread can run and the SPI core can +still hold its messages. + +Add a shutdown callback that stops the controller and waits for its +interrupt, also when the threaded handler has already masked itself. SPI +children shut down before their controller, so the SPI driver can then +drain its queue. + +Assisted-by: LLM +--- + drivers/input/touchscreen/apple_z2.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/drivers/input/touchscreen/apple_z2.c b/drivers/input/touchscreen/apple_z2.c +index 92a04edcd..5dcf3579d 100644 +--- a/drivers/input/touchscreen/apple_z2.c ++++ b/drivers/input/touchscreen/apple_z2.c +@@ -4599,6 +4599,16 @@ static struct spi_device_id apple_z2_of_id[] = { + }; + MODULE_DEVICE_TABLE(spi, apple_z2_of_id); + ++/* Children stop before the SPI controller drains its core queue. */ ++static void apple_z2_shutdown(struct spi_device *spi) ++{ ++ struct apple_z2 *z2 = spi_get_drvdata(spi); ++ ++ apple_z2_stop(z2); ++ /* Also wait when the IRQ thread already masked itself. */ ++ synchronize_irq(spi->irq); ++} ++ + static struct spi_driver apple_z2_driver = { + .driver = { + .name = "apple-z2", +@@ -4609,6 +4619,7 @@ static struct spi_driver apple_z2_driver = { + .id_table = apple_z2_of_id, + .probe = apple_z2_probe, + .remove = apple_z2_remove, ++ .shutdown = apple_z2_shutdown, + }; + + module_spi_driver(apple_z2_driver); diff --git a/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch b/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch new file mode 100644 index 00000000..e442e50e --- /dev/null +++ b/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: iommu: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/iommu/apple,dart.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml +index df72722fc..adcbd21fc 100644 +--- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml ++++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml +@@ -21,6 +21,10 @@ description: |+ + fault conditions. + + properties: ++ apple,ans-dart-handoff: ++ type: boolean ++ description: Allow the storage endpoint to retire and rebuild this DART stream during kexec. ++ + compatible: + oneOf: + - enum: diff --git a/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch b/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch new file mode 100644 index 00000000..e5204e73 --- /dev/null +++ b/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch @@ -0,0 +1,56 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] iommu/apple-dart: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + drivers/iommu/apple-dart.c | 24 ++++++++++++++++++++++++ + 1 file changed, 24 insertions(+) + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 1ee6add55..5aff4bc3e 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -35,6 +35,7 @@ + #include + + #include ++#include + + #include "dma-iommu.h" + +@@ -1009,6 +1010,29 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++int apple_ans_dart_stop(struct device *dev) ++{ ++ struct iommu_domain *domain = iommu_get_domain_for_dev(dev); ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ int i; ++ ++ if (!domain || !cfg || ++ !cfg->stream_maps[0].dart || ++ !of_property_read_bool(cfg->stream_maps[0].dart->dev->of_node, ++ "apple,ans-dart-handoff") || ++ (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) ++ return -ENODEV; ++ for_each_stream_map(i, cfg, map) { ++ if (map->dart->hw->type != DART_S5L8960X) ++ return -EOPNOTSUPP; ++ apple_dart_hw_disable_dma(map); ++ } ++ dev_info(dev, "Apple ANS: DART stopped for handoff\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_stop); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { diff --git a/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch b/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch new file mode 100644 index 00000000..711841f3 --- /dev/null +++ b/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch @@ -0,0 +1,97 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] iommu/apple-dart: save and rebuild translations around + certified ANS stop + +apple_ans_dart_stop() stops the ANS DART before a kexec handoff. System +sleep needs the same stop and its reverse. + +Save the DART's translation state when apple_ans_dart_stop() stops it, +and add apple_ans_dart_start() to restore it. The regular suspend +callback skips a DART stopped this way, which would otherwise save its +state a second time, and resume clears the flag. The NVMe sleep path +uses the pair to stop the ANS translations before sleep and rebuild them +on resume. + +Assisted-by: LLM +--- + drivers/iommu/apple-dart.c | 29 +++++++++++++++++++++++++++++ + include/linux/apple-ans.h | 2 ++ + 2 files changed, 31 insertions(+) + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 5aff4bc3e..5c28d4e4b 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -217,6 +217,7 @@ enum dart_type { + * @iommu: iommu core device + */ + struct apple_dart { ++ bool ans_stopped; + struct device *dev; + const struct apple_dart_hw *hw; + +@@ -1010,6 +1011,9 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++static __maybe_unused int apple_dart_suspend(struct device *dev); ++static __maybe_unused int apple_dart_resume(struct device *dev); ++ + int apple_ans_dart_stop(struct device *dev) + { + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); +@@ -1026,6 +1030,8 @@ int apple_ans_dart_stop(struct device *dev) + for_each_stream_map(i, cfg, map) { + if (map->dart->hw->type != DART_S5L8960X) + return -EOPNOTSUPP; ++ apple_dart_suspend(map->dart->dev); ++ map->dart->ans_stopped = true; + apple_dart_hw_disable_dma(map); + } + dev_info(dev, "Apple ANS: DART stopped for handoff\n"); +@@ -1033,6 +1039,25 @@ int apple_ans_dart_stop(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_ans_dart_stop); + ++int apple_ans_dart_start(struct device *dev) ++{ ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ int i, ret; ++ ++ if (!cfg) ++ return -ENODEV; ++ for_each_stream_map(i, cfg, map) { ++ if (!map->dart->ans_stopped) ++ continue; ++ ret = apple_dart_resume(map->dart->dev); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_start); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { +@@ -2017,6 +2042,9 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + ++ if (dart->ans_stopped) ++ return 0; ++ + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = dart->hw->read_tcr(dart, sid); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) +@@ -2046,6 +2074,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) + dart->hw->write_tcr(dart, sid, dart->save_tcr[sid]); + } + ++ dart->ans_stopped = false; + return 0; + } + diff --git a/kernel/patches/drivers/media/0011-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch b/kernel/patches/drivers/media/0011-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch new file mode 100644 index 00000000..0c782ee2 --- /dev/null +++ b/kernel/patches/drivers/media/0011-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch @@ -0,0 +1,332 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:23:32 +0000 +Subject: [PATCH] media: apple-vxd: stop the A9 firmware for system sleep + +The DEVA firmware boots on the first open of the video node and stays +resident until the driver is unbound. udev and media daemons open the +node at boot, so the firmware is resident in practice. Over s2idle the +decoder power domain goes off and the resident firmware does not survive +it, and the J71t sleep precheck refuses any bound decoder without a +reserved sleep admission. + +Follow AppleD5500 instead, which stops the core and frees the firmware +when its last client closes (terminateDeviceInternal, fffffff00628a604: +clear the ready magic, write 1 to 0x200, known state v1, free, power +down). Register a sleep admission provider for the A9 variant. Its +prepare does no MMIO: it refuses while a file is open and otherwise +closes the open gate until complete, after resume or any abort. The +system suspend callback then stops the firmware in that order and frees +what it used; the next open boots it again, as the first open after +probe does. If the core does not reach its known state, every buffer it +may still read is kept and the sleep is refused. + +The T8010 variant registers no provider and keeps its behaviour. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 63 ++++++++++++++- + .../media/platform/apple/vxd/apple-vxd-fw.c | 77 +++++++++++++++++++ + .../media/platform/apple/vxd/apple-vxd-regs.h | 6 ++ + .../media/platform/apple/vxd/apple-vxd-v4l2.c | 9 ++- + drivers/media/platform/apple/vxd/apple-vxd.h | 12 +++ + 5 files changed, 164 insertions(+), 3 deletions(-) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index 2940978ff..16051e6c6 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -245,7 +245,51 @@ static int apple_vxd_runtime_suspend(struct device *dev) + return 0; + } + ++/* ++ * Sleep admission: no MMIO. Refuse while a file is open (its contexts and ++ * any queued pictures belong to the running firmware); otherwise close the ++ * open gate until complete, after resume or any abort. ++ */ ++static int apple_vxd_sleep_prepare(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ int ret = 0; ++ ++ mutex_lock(&vxd->power_lock); ++ if (vxd->users) ++ ret = -EBUSY; ++ else ++ vxd->sleep_gate = true; ++ mutex_unlock(&vxd->power_lock); ++ if (ret) ++ dev_info(dev, "system sleep refused: decoder in use\n"); ++ return ret; ++} ++ ++static void apple_vxd_sleep_complete(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ ++ mutex_lock(&vxd->power_lock); ++ vxd->sleep_gate = false; ++ mutex_unlock(&vxd->power_lock); ++} ++ ++static int apple_vxd_suspend(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ ++ if (!vxd->variant->sleep_stop) ++ return 0; ++ ++ /* No open file: nothing can queue a picture, let the last one finish. */ ++ flush_work(&vxd->job_done_work); ++ cancel_delayed_work_sync(&vxd->watchdog_work); ++ return apple_vxd_fw_sleep_stop(vxd); ++} ++ + static const struct dev_pm_ops apple_vxd_pm_ops = { ++ .suspend = pm_sleep_ptr(apple_vxd_suspend), + RUNTIME_PM_OPS(apple_vxd_runtime_suspend, apple_vxd_runtime_resume, NULL) + }; + +@@ -260,6 +304,7 @@ static const struct apple_vxd_variant s8000_vxd_variant = { + .num_gart_pts = 512, /* "setting 2GB mapping" [E] */ + /* HEVCSupported = ((rev - 7) < 4) && ((0xD >> (rev - 7)) & 1) [E] */ + .hevc_supported = true, ++ .sleep_stop = true, + .hevc = { + .buf_mult = 1, + .tail_7474 = true, +@@ -369,16 +414,29 @@ static int apple_vxd_probe(struct platform_device *pdev) + if (ret) + goto err_disable_runtime_pm; + ++ /* Before the video node exists: nothing can open it unadmitted. */ ++ if (variant->sleep_stop) { ++ vxd->sleep_admission.dev = &pdev->dev; ++ vxd->sleep_admission.prepare = apple_vxd_sleep_prepare; ++ vxd->sleep_admission.complete = apple_vxd_sleep_complete; ++ ret = register_pm_suspend_admission(&vxd->sleep_admission); ++ if (ret) ++ goto err_gart_fini; ++ } ++ + ret = apple_vxd_h264_register(vxd); + if (!ret) + ret = apple_vxd_hevc_register(vxd); + if (!ret) + ret = apple_vxd_v4l2_init(vxd); + if (ret) +- goto err_gart_fini; ++ goto err_admission; + + return 0; + ++err_admission: ++ if (variant->sleep_stop) ++ unregister_pm_suspend_admission(&vxd->sleep_admission); + err_gart_fini: + apple_vxd_gart_fini(vxd); + err_disable_runtime_pm: +@@ -391,6 +449,9 @@ static void apple_vxd_remove(struct platform_device *pdev) + { + struct apple_vxd_dev *vxd = platform_get_drvdata(pdev); + ++ /* Before any teardown; also waits out a transition in progress. */ ++ if (vxd->variant->sleep_stop) ++ unregister_pm_suspend_admission(&vxd->sleep_admission); + cancel_delayed_work_sync(&vxd->watchdog_work); + apple_vxd_v4l2_cleanup(vxd); + apple_vxd_fw_cleanup(vxd); +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-fw.c b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +index 753ca4944..ede737efd 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-fw.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +@@ -435,6 +435,83 @@ out_put: + return ret; + } + ++/* ++ * One reference per open file. A sleep that has been admitted closes the ++ * gate first, so an open between admission and the device's suspend ++ * callback fails instead of booting the firmware behind it. ++ */ ++int apple_vxd_fw_open(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ if (vxd->sleep_gate) { ++ mutex_unlock(&vxd->power_lock); ++ return -EBUSY; ++ } ++ vxd->users++; ++ mutex_unlock(&vxd->power_lock); ++ ++ ret = apple_vxd_boot_firmware(vxd); ++ if (ret) ++ apple_vxd_fw_close(vxd); ++ return ret; ++} ++ ++void apple_vxd_fw_close(struct apple_vxd_dev *vxd) ++{ ++ mutex_lock(&vxd->power_lock); ++ if (!WARN_ON(!vxd->users)) ++ vxd->users--; ++ mutex_unlock(&vxd->power_lock); ++} ++ ++/* ++ * System sleep, after admission and with no open file: stop the MTX in ++ * AppleD5500's terminate order (clear the ready magic, the 0x200 write, ++ * known state v1), then free what the running firmware used. The next ++ * open boots it again, as the first open after probe does. If the core ++ * does not reach its known state, keep every buffer it may still read ++ * and refuse the sleep; the decoder then stays failed (-EIO) until rebind. ++ */ ++int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ if (!vxd->sleep_gate || vxd->users) { ++ mutex_unlock(&vxd->power_lock); ++ return -EBUSY; ++ } ++ if (!vxd->fw_loaded) { ++ mutex_unlock(&vxd->power_lock); ++ return 0; ++ } ++ ++ apple_vxd_mbox_stop(vxd); ++ vxd->hw_ready = false; ++ vxd_write(vxd, VXD_REG_FW_READY_MAGIC, 0); ++ vxd_write(vxd, VXD_REG_VECT_CFG, VXD_VECT_CFG_TERMINATE); ++ ret = vxd_known_state_v1(vxd, "Terminate"); ++ if (ret) { ++ dev_err(vxd->dev, "firmware stop for sleep failed: %d\n", ret); ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++ } ++ ++ apple_vxd_cmdbufs_fini(vxd); ++ apple_vxd_gart_unmap(vxd, &vxd->fw_map); ++ dma_free_coherent(vxd->dev, vxd->fw_size, vxd->fw_vaddr, vxd->fw_dma); ++ vxd->fw_vaddr = NULL; ++ vxd->fw_loaded = false; ++ mutex_unlock(&vxd->power_lock); ++ ++ /* The reference boot took; the PM core holds its own until complete. */ ++ pm_runtime_put_noidle(vxd->dev); ++ dev_info(vxd->dev, "firmware stopped for system sleep\n"); ++ return 0; ++} ++ + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd) + { + bool was_loaded; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-regs.h b/drivers/media/platform/apple/vxd/apple-vxd-regs.h +index edeccb8e0..fdbe48903 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-regs.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd-regs.h +@@ -89,6 +89,12 @@ + + /* VEC block and entropy decoder pipes */ + #define VXD_REG_VECT_CFG 0x0200 ++/* ++ * AppleD5500's terminate path (fffffff00628c474 -> fffffff00629972c [E]) ++ * writes 1 here after clearing the ready magic and before known state v1. ++ * Nothing beyond that order is known about the write. ++ */ ++#define VXD_VECT_CFG_TERMINATE 0x00000001 + #define VXD_REG_VECT_CFG2 0x0208 + #define VXD_VECT_CFG2_MSGS_MINUS1 0x000000c7 /* core[0xb4] - 1 */ + #define VXD_REG_ENTDEC_0_SIZE 0x0340 +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +index 9664bb802..c5e429df7 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +@@ -897,13 +897,15 @@ static int apple_vxd_open(struct file *filp) + * can be read, and keeps running (and the block powered) until the + * driver is unbound. Later opens find it running. + */ +- ret = apple_vxd_boot_firmware(vxd); ++ ret = apple_vxd_fw_open(vxd); + if (ret) + return ret; + + ctx = kzalloc_obj(*ctx); +- if (!ctx) ++ if (!ctx) { ++ apple_vxd_fw_close(vxd); + return -ENOMEM; ++ } + + ctx->vxd = vxd; + v4l2_fh_init(&ctx->fh, vdev); +@@ -912,6 +914,7 @@ static int apple_vxd_open(struct file *filp) + if (ret) { + v4l2_fh_exit(&ctx->fh); + kfree(ctx); ++ apple_vxd_fw_close(vxd); + return ret; + } + +@@ -922,11 +925,13 @@ static int apple_vxd_open(struct file *filp) + static int apple_vxd_release(struct file *filp) + { + struct apple_vxd_ctx *ctx = file_to_vxd_ctx(filp); ++ struct apple_vxd_dev *vxd = ctx->vxd; + + v4l2_fh_del(&ctx->fh, filp); + apple_vxd_ctx_fini(ctx); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); ++ apple_vxd_fw_close(vxd); + + return 0; + } +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index 1db55c1ca..7922ebb73 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -71,6 +72,11 @@ struct apple_vxd_variant { + u32 num_gart_pts; /* mapped page tables (512 = 2 GiB) */ + bool hevc_supported; + bool has_integration; /* second reg range, see VXD_INTEG_CTRL */ ++ /* ++ * Stop the firmware for system sleep the way AppleD5500 does when its ++ * last client closes, and boot it again on the next open. ++ */ ++ bool sleep_stop; + /* HEVC buffer multiplier and command quirks; zero leaves HEVC off. */ + struct apple_vxd_hevc_caps hevc; + }; +@@ -294,6 +300,9 @@ struct apple_vxd_dev { + /* Core state. */ + struct mutex power_lock; /* boot vs. runtime suspend */ + bool hw_ready; /* firmware took its init message */ ++ unsigned int users; /* open files, under power_lock */ ++ bool sleep_gate; /* sleep admitted: no new open */ ++ struct pm_suspend_admission sleep_admission; + u32 core_id; + u32 core_rev; + u32 num_entropy_pipes; +@@ -459,6 +468,9 @@ void apple_vxd_dev_buf_free(struct apple_vxd_dev *vxd, + + /* fw.c */ + int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_open(struct apple_vxd_dev *vxd); ++void apple_vxd_fw_close(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd); + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd); + + /* mbox.c */ diff --git a/kernel/patches/drivers/media/0012-media-apple-vxd-answer-the-suspend-readiness-query.patch b/kernel/patches/drivers/media/0012-media-apple-vxd-answer-the-suspend-readiness-query.patch new file mode 100644 index 00000000..fdae4cdb --- /dev/null +++ b/kernel/patches/drivers/media/0012-media-apple-vxd-answer-the-suspend-readiness-query.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:11:36 +0000 +Subject: [PATCH] media: apple-vxd: answer the suspend readiness query + +The A9 decoder's sleep admission refuses while a file is open and +otherwise closes its open gate until complete. Provide the matching +check: report -EBUSY while a file is open, without touching the gate. + +Assisted-by: LLM +--- + drivers/media/platform/apple/vxd/apple-vxd-core.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index 31a024233..dff75927d 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -266,6 +266,18 @@ static int apple_vxd_sleep_prepare(struct device *dev) + return ret; + } + ++/* Readiness query: what prepare would decide, without closing the gate. */ ++static int apple_vxd_sleep_check(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ ret = vxd->users ? -EBUSY : 0; ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++} ++ + static void apple_vxd_sleep_complete(struct device *dev) + { + struct apple_vxd_dev *vxd = dev_get_drvdata(dev); +@@ -403,6 +415,7 @@ static int apple_vxd_probe(struct platform_device *pdev) + vxd->sleep_admission.dev = &pdev->dev; + vxd->sleep_admission.prepare = apple_vxd_sleep_prepare; + vxd->sleep_admission.complete = apple_vxd_sleep_complete; ++ vxd->sleep_admission.check = apple_vxd_sleep_check; + ret = register_pm_suspend_admission(&vxd->sleep_admission); + if (ret) + goto err_gart_fini; diff --git a/kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch b/kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch new file mode 100644 index 00000000..3670d596 --- /dev/null +++ b/kernel/patches/drivers/media/0013-media-apple-vxd-require-a-proven-firmware-stop-befor.patch @@ -0,0 +1,120 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:51 +0100 +Subject: [PATCH] media: apple-vxd: require a proven firmware stop before kexec + +release has no shutdown callback and no B1 ProbeTerminate path. Add the +checked S8000 stop at shutdown: failed Terminate keeps firmware and +command buffers plus power, and kexec takes the cold-reset escape before +those pages become incoming RAM. Successful stop may release buffers and +force suspend. Other variants keep their shutdown behavior. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 26 ++++++++++++++ + .../media/platform/apple/vxd/apple-vxd-fw.c | 35 +++++++++++++++++++ + drivers/media/platform/apple/vxd/apple-vxd.h | 1 + + 3 files changed, 62 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index b6bf2d300..c89e7e987 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -1,3 +1,6 @@ ++#include ++#include ++ + // SPDX-License-Identifier: GPL-2.0 + /* + * Apple VXD (Imagination MSVDX) video decoder driver - core +@@ -476,9 +479,32 @@ static void apple_vxd_remove(struct platform_device *pdev) + pm_runtime_disable(&pdev->dev); + } + ++static void apple_vxd_shutdown(struct platform_device *pdev) ++{ ++ struct apple_vxd_dev *vxd = platform_get_drvdata(pdev); ++ int ret; ++ ++ if (!vxd->variant->sleep_stop) ++ return; ++ ++ cancel_delayed_work_sync(&vxd->watchdog_work); ++ flush_work(&vxd->job_done_work); ++ ret = apple_vxd_fw_shutdown_stop(vxd); ++ if (ret) { ++ /* Outgoing allocations cannot be reserved against the next kernel. */ ++ if (kexec_in_progress) ++ apple_cold_reset("VXD Terminate failed; DMA buffers cannot cross kexec"); ++ return; /* Retain every buffer and the runtime-PM reference. */ ++ } ++ ret = pm_runtime_force_suspend(&pdev->dev); ++ if (ret) ++ dev_warn(&pdev->dev, "power off at shutdown failed: %d\n", ret); ++} ++ + static struct platform_driver apple_vxd_driver = { + .probe = apple_vxd_probe, + .remove = apple_vxd_remove, ++ .shutdown = apple_vxd_shutdown, + .driver = { + .name = APPLE_VXD_DRV_NAME, + .pm = &apple_vxd_pm_ops, +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-fw.c b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +index ede737efd..bccd04259 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-fw.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +@@ -512,6 +512,41 @@ int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd) + return 0; + } + ++int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ vxd->sleep_gate = true; ++ if (!vxd->fw_loaded) { ++ mutex_unlock(&vxd->power_lock); ++ return 0; ++ } ++ ++ apple_vxd_mbox_stop(vxd); ++ vxd->hw_ready = false; ++ vxd_write(vxd, VXD_REG_FW_READY_MAGIC, 0); ++ vxd_write(vxd, VXD_REG_VECT_CFG, VXD_VECT_CFG_TERMINATE); ++ ret = vxd_known_state_v1(vxd, "Terminate"); ++ if (ret) { ++ dev_err(vxd->dev, "firmware stop at shutdown failed: %d; retaining buffers and power\n", ret); ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++ } ++ ++ apple_vxd_cmdbufs_fini(vxd); ++ apple_vxd_gart_unmap(vxd, &vxd->fw_map); ++ dma_free_coherent(vxd->dev, vxd->fw_size, vxd->fw_vaddr, vxd->fw_dma); ++ vxd->fw_vaddr = NULL; ++ vxd->fw_loaded = false; ++ mutex_unlock(&vxd->power_lock); ++ ++ /* The reference boot took; the PM core holds its own until complete. */ ++ pm_runtime_put_noidle(vxd->dev); ++ dev_info(vxd->dev, "firmware stopped at shutdown\n"); ++ return 0; ++} ++ + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd) + { + bool was_loaded; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index 7922ebb73..b601ea6b7 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -471,6 +471,7 @@ int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd); + int apple_vxd_fw_open(struct apple_vxd_dev *vxd); + void apple_vxd_fw_close(struct apple_vxd_dev *vxd); + int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *vxd); + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd); + + /* mbox.c */ diff --git a/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..64a11814 --- /dev/null +++ b/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,35 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] of: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/of/of_reserved_mem.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c +index 8c9d6395d..76cf3e5a4 100644 +--- a/drivers/of/of_reserved_mem.c ++++ b/drivers/of/of_reserved_mem.c +@@ -275,6 +275,16 @@ void __init fdt_scan_reserved_mem_late(void) + return; + } + ++ /* Early architecture fixups may have added static reservations. */ ++ total_reserved_mem_cnt = reserved_mem_count; ++ fdt_for_each_subnode(child, fdt, node) { ++ int len; ++ ++ if (of_fdt_device_is_available(fdt, child) && ++ fdt_getprop(fdt, child, "reg", &len) && len) ++ total_reserved_mem_cnt++; ++ } ++ + /* Attempt dynamic allocation of a new reserved_mem array */ + if (alloc_reserved_mem_array()) + return; diff --git a/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..c4b7b0ee --- /dev/null +++ b/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: pci: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +index f6497a4e1..f9b72ce26 100644 +--- a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml ++++ b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +@@ -140,6 +140,10 @@ allOf: + - apple,nvme-hmb + + properties: ++ apple,ans-kexec-handoff: ++ type: boolean ++ description: Retain storage port power across a certified stopped-DMA kexec handoff. ++ + compatible: + oneOf: + - enum: diff --git a/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..e030657f --- /dev/null +++ b/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,668 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] PCI: apple-h9p: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +The storage port is handed over powered, with its reference clock running, +PERST# asserted and the LTSSM stopped: the state the boot loader leaves it +in, except that the link is down. The incoming kernel brings it up as on a +cold boot, with the full port power cycle, NVMe mode and the port enable +sequence before the link is trained. iOS trains this link only after that +sequence and never re-trains a powered port by toggling PERST# alone; +doing so after a kexec leaves the root port in Detect.Quiet with the LTSSM +enable not latched. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 459 +++++++++++++++++++++--- + include/linux/apple-ans.h | 23 ++ + 2 files changed, 432 insertions(+), 50 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 15f7a0cbe..f0095394a 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -15,6 +15,8 @@ + + #include + #include ++#include ++#include + #include + #include + #include +@@ -38,12 +40,15 @@ + #include + #include + #include ++#include + #include + #include + #include ++#include + + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -248,7 +253,7 @@ + #define S8003_PORT_LINKCDMSTS_LTSSM GENMASK(14, 9) + #define S8003_LTSSM_L0 0x11 + +-/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (j71t) */ ++/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (S8003) */ + #define S8003_PERST_TO_REFCLK_US 60 + #define S8003_REFCLK_TO_PERST_US 100 + #define S8003_PERST_HOLD_MS 100 +@@ -269,7 +274,7 @@ + + /* + * NVMMU invalidate FIFO status, in the port 0 block (iBoot sets the live ADT +- * nvme-mmu0 invalidate-fifo-workaround = 1 on j71t). iOS (AppleNVMeMMU, ++ * nvme-mmu0 invalidate-fifo-workaround = 1 on S8003). iOS (AppleNVMeMMU, + * 0xfffffff005eebc9c) waits up to 20 ms for it to drain before every + * invalidate and never reads the invalidate register back. + */ +@@ -429,7 +434,7 @@ static const struct apple_h9p_tunable t7000_port_tunables[] = { + }; + + /* +- * S8003 tunables, which iBoot puts in the live ADT; copied from the j71t ++ * S8003 tunables, which iBoot puts in the live ADT; copied from the S8003 + * live ADT (iOS 16.7.10 iBoot), where both bridges carry the same values. + * apcie-phy-tunables and apcie-common-tunables apply after the common init, + * in that order. +@@ -582,6 +587,7 @@ struct apple_h9p_tunables { + struct apple_h9p_nvmmu { + struct apple_h9p_pcie *pcie; + void __iomem *base; ++ bool poisoned; + /* Programmed: the ANS glue may map through it */ + bool ready; + /* Serializes the invalidate FIFO wait and the invalidate write */ +@@ -711,6 +717,7 @@ struct apple_h9p_pcie { + u32 late_ports; + unsigned int late_tries; + struct delayed_work late_work; ++ bool ans_handoff_stopped; + + struct apple_h9p_nvmmu nvmmu; + +@@ -1554,6 +1561,64 @@ static irqreturn_t apple_h9p_nvmmu_irq(int irq, void *data) + return IRQ_HANDLED; + } + ++static int apple_h9p_nvmmu_program(struct apple_h9p_pcie *pcie) ++{ ++ struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; ++ u32 val; ++ int ret; ++ ++ if (pcie->hw->nvmmu_inv_fifo) { ++ /* ++ * iOS's S8003 order: the invalidate FIFO drained, the old SART ++ * aperture off before its address changes, the zeroed scratch ++ * page and TCBs visible, then scratch page, TCB base, init, SART. ++ */ ++ ret = readl_poll_timeout(pcie->base_port[0] + ++ S8003_PORT_NVMMU_INV_FIFO, val, ++ val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ dma_wmb(); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ } else { ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ } ++ h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, ++ nvmmu->base + H9P_NVMMU_TCB_CTRL); ++ ++ ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, ++ pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ if (ret) ++ return ret; ++ ++ h9p_writel_flush(nvmmu->va_base - 0x80000000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_BASE); ++ h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, ++ H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_END); ++ h9p_writel_flush(nvmmu->pa_base >> 20, ++ nvmmu->base + H9P_NVMMU_SART_PA_BASE); ++ h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ ++ return 0; ++} ++ + static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + { + struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; +@@ -1561,7 +1626,7 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + struct device_node *mem_np; + struct resource res; + resource_size_t size; +- u32 iova, val; ++ u32 iova; + int irq; + int ret; + +@@ -1630,55 +1695,10 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + } + + spin_lock_init(&nvmmu->inv_lock); +- if (pcie->hw->nvmmu_inv_fifo) { +- /* +- * iOS's S8003 order: the invalidate FIFO drained, the old SART +- * aperture off before its address changes, the zeroed scratch +- * page and TCBs visible, then scratch page, TCB base, init, SART. +- */ +- ret = readl_poll_timeout(pcie->base_port[0] + +- S8003_PORT_NVMMU_INV_FIFO, val, +- val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, +- 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); +- if (ret) +- goto out_put_node; +- h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); +- dma_wmb(); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- } else { +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- } +- h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, +- nvmmu->base + H9P_NVMMU_TCB_CTRL); +- +- ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, +- pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ ret = apple_h9p_nvmmu_program(pcie); + if (ret) + goto out_put_node; + +- h9p_writel_flush(nvmmu->va_base - 0x80000000U, +- nvmmu->base + H9P_NVMMU_SART_VA_BASE); +- h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, +- H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, +- nvmmu->base + H9P_NVMMU_SART_VA_END); +- h9p_writel_flush(nvmmu->pa_base >> 20, +- nvmmu->base + H9P_NVMMU_SART_PA_BASE); +- h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); +- + irq = platform_get_irq_byname_optional(pcie->pdev, "nvmmu"); + if (irq > 0) { + ret = devm_request_irq(dev, irq, apple_h9p_nvmmu_irq, 0, +@@ -1726,6 +1746,7 @@ static int apple_h9p_nvmmu_invalidate(struct apple_h9p_pcie *pcie, + spin_unlock_irqrestore(&nvmmu->inv_lock, flags); + + if (ret) { ++ WRITE_ONCE(nvmmu->poisoned, true); + dev_err_ratelimited(pcie->dev, + "NVMMU invalidate FIFO not empty (%#x), tag %u\n", + val, tag); +@@ -1781,6 +1802,8 @@ int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + nvmmu = &pcie->nvmmu; + if (!nvmmu->ready) + return -EOPNOTSUPP; ++ if (READ_ONCE(nvmmu->poisoned)) ++ return -EIO; + + tcb = (u32 *)nvmmu->tcb + tag * H9P_NVMMU_TCB_DWORDS; + sgl = (u32 *)nvmmu->tcb_sgl + tag * H9P_NVMMU_SGL_WORDS; +@@ -2705,6 +2728,8 @@ static int apple_s8003_power_cycle_port(struct apple_h9p_pcie *pcie, + return ret; + } + ++ if (!port && pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); + apple_s8003_port_off(pcie, port); + usleep_range(S8003_POWER_OFF_SETTLE_US, S8003_POWER_OFF_SETTLE_US + 1000); + ret = apple_h9p_wait(rc + S8003_RC_PORT_CTL(port), +@@ -2887,6 +2912,18 @@ static int apple_s8003_start_link(struct apple_h9p_pcie *pcie, + return ret; + } + ++static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *rc = pcie->base_rc; ++ ++ return (readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_ENABLE) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_ON) && ++ (readl(rc + S8003_RC_PORT_REFCLK(0)) & S8003_RC_PORT_REFCLK_ON) && ++ !(readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_RESET) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_REFCLK_NOGATE) && ++ (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -3059,6 +3096,11 @@ static void apple_s8003_late_ports_work(struct work_struct *work) + unsigned int port; + int ret; + ++ /* Lab 0019: no secondary bus publication without storage bus 1. */ ++ if (of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") && ++ !(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return; ++ + for_each_set_bit(port, &pending, H9P_NUM_PORTS) { + ret = apple_s8003_setup_endpoint_port(pcie, port); + if (ret == -EPROBE_DEFER) { +@@ -3418,6 +3460,314 @@ static void apple_h9p_pcie_pwrctrl_cleanup(void *data) + pci_pwrctrl_destroy_devices(pcie->dev); + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static struct apple_h9p_pcie *apple_ans_endpoint_host(struct device *dev) ++{ ++ struct device *host = dev; ++ struct apple_h9p_pcie *pcie; ++ ++ while (host && host->bus == dev->bus) ++ host = host->parent; ++ if (!host || !host->parent) ++ return NULL; ++ pcie = apple_h9p_pcie_lookup(host->parent); ++ if (!pcie || !of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") || ++ apple_h9p_pcie_device_port(pcie, dev) != 0) ++ return NULL; ++ return pcie; ++} ++ ++ ++/* release6: caller froze and synchronized I/O, before SHN. */ ++/* Lab 0018: caller synchronized queues and stopped CC/RDY before this. */ ++int apple_ans_nvmmu_reset(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ struct apple_h9p_nvmmu *nvmmu; ++ int ret; ++ ++ if (!pcie || !(pcie->enabled_ports & BIT(0))) ++ return -ENODEV; ++ nvmmu = &pcie->nvmmu; ++ if (!nvmmu->ready) ++ return -ENODEV; ++ memset(nvmmu->tcb, 0, nvmmu->tcb_size); ++ memset(nvmmu->tcb_table, 0, nvmmu->tcb_table_size); ++ memset(nvmmu->tcb_sgl, 0, nvmmu->tcb_sgl_size); ++ ret = apple_h9p_nvmmu_program(pcie); ++ WRITE_ONCE(nvmmu->poisoned, !!ret); ++ if (ret) ++ return ret; ++ bitmap_zero(nvmmu->mapped, APPLE_ANS_NVMMU_MAX_REQS); ++ dev_info(dev, "NVMMU: stopped-controller cache/SART rebuilt\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_reset); ++ ++bool apple_ans_is_s8003(struct device *dev) ++{ ++ return apple_ans_endpoint_host(dev) != NULL; ++} ++EXPORT_SYMBOL_GPL(apple_ans_is_s8003); ++ ++int apple_ans_link_up(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -ENODEV; ++ if (!(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return -ENOLINK; ++ return !!(readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & ++ S8003_PORT_LINKSTS_UP); ++} ++EXPORT_SYMBOL_GPL(apple_ans_link_up); ++ ++/* Host-only evidence, no endpoint config/BAR reads or invalidate register. */ ++void apple_ans_host_dump(struct device *dev, const char *reason) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return; ++ dev_info(dev, "ANS host %s: ports %#x link %#x LTSSM %#x FIFO %#x poisoned %u mapped %*pb\n", ++ reason, pcie->enabled_ports, ++ readl(pcie->base_port[0] + H9P_PORT_LINKSTS), ++ readl(pcie->base_port[0] + H9P_PORT_LTSSMCTL), ++ readl(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO), ++ READ_ONCE(pcie->nvmmu.poisoned), APPLE_ANS_NVMMU_MAX_REQS, ++ pcie->nvmmu.mapped); ++} ++EXPORT_SYMBOL_GPL(apple_ans_host_dump); ++ ++int apple_ans_nvmmu_drain(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ u32 fifo; ++ int ret; ++ ++ if (!pcie || !pcie->nvmmu.ready) ++ return -ENODEV; ++ if (!bitmap_empty(pcie->nvmmu.mapped, APPLE_ANS_NVMMU_MAX_REQS)) ++ return -EBUSY; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ dev_info(dev, "Apple ANS: outgoing NVMMU FIFO empty check (20ms) result %d\n", ret); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_drain); ++ ++int apple_ans_handoff_stopped(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ unsigned int tag; ++ u32 fifo; ++ int ret; ++ ++ if (!pcie) ++ return -ENODEV; ++ /* All queues are drained and BME is clear before retiring translations. */ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } ++ /* The last invalidate also has to leave the FIFO before certification. */ ++ ret = readl_poll_timeout_atomic(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 1, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ ret = apple_ans_dart_stop(dev); ++ if (!ret) ++ WRITE_ONCE(pcie->ans_handoff_stopped, true); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_stopped); ++ ++int apple_ans_handoff_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_handoff_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); ++ ++static void apple_h9p_s8003_shutdown(struct platform_device *pdev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ unsigned int port; ++ ++ if (!kexec_in_progress || !pcie || ++ !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) ++ return; ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!READ_ONCE(pcie->ans_handoff_stopped) || ++ (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && ++ !apple_s8003_port_powered(pcie))) ++ panic("Apple ANS: refusing uncertified powered handoff"); ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; ++ ++ if (!(pcie->enabled_ports & BIT(port))) ++ continue; ++ /* release6: block root routing, retain refclk, then isolate. */ ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[port], 1)) ++ panic("Apple PCIe: PERST failed during handoff"); ++ if (apple_s8003_port_stop(pcie, port)) ++ panic("Apple PCIe: LTSSM stop failed during handoff"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (!port && pcie->nvmmu.ready) { ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ } ++ if (!port && !of_property_read_bool(pcie->dev->of_node, ++ "apple,ans-kexec-handoff")) ++ apple_s8003_port_off(pcie, port); ++ if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && ++ (!apple_s8003_port_powered(pcie) || ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) ++ panic("Apple ANS: outgoing powered port certificate failed"); ++ pcie->enabled_ports &= ~BIT(port); ++ } ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++} ++ ++static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *cfg = pcie->base_config; ++ ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!(pcie->enabled_ports & BIT(0))) ++ return; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[0], 1) || ++ apple_s8003_port_stop(pcie, 0)) ++ panic("ANS capture isolation failed"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ pcie->enabled_ports &= ~BIT(0); ++} ++ ++struct apple_h9p_crash_state { ++ struct apple_h9p_pcie *pcie; ++ int result; ++ bool busy; ++ bool found; ++}; ++ ++/* No NVMe driver, queues, SHN, or old DMA buffers are needed in phoenix. */ ++static int apple_h9p_crash_stop_ans(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_crash_state *state = data; ++ struct apple_h9p_pcie *pcie = state->pcie; ++ void __iomem *bar; ++ u32 csts = 0; ++ int i, ret; ++ ++ if (pdev->vendor != PCI_VENDOR_ID_APPLE || ++ ((pdev->class >> 8) != PCI_CLASS_STORAGE_EXPRESS && ++ (pdev->class >> 8) != PCI_CLASS_STORAGE_OTHER)) ++ return 0; ++ state->found = true; ++ if (pdev->driver) { ++ state->busy = true; ++ return 1; ++ } ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ state->result = -ENOLINK; ++ return 0; ++ } ++ ret = pci_enable_device_mem(pdev); ++ if (ret) { ++ state->result = ret; ++ return 0; ++ } ++ bar = pci_iomap_range(pdev, 0, 0, SZ_8K); ++ if (!bar) { ++ ret = -ENOMEM; ++ goto disable; ++ } ++ /* An unknown queue must never be rung or sent a shutdown notification. */ ++ dev_info(pcie->dev, "phoenix ANS: CC %#x CSTS %#x, clearing EN without SHN\n", ++ readl(bar + NVME_REG_CC), readl(bar + NVME_REG_CSTS)); ++ writel(0, bar + NVME_REG_CC); ++ ret = -ETIMEDOUT; ++ for (i = 0; i < 2000; i++) { ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ ret = -ENOLINK; ++ break; ++ } ++ csts = readl(bar + NVME_REG_CSTS); ++ if (csts == U32_MAX) { ++ ret = -EIO; ++ break; ++ } ++ if (!(csts & NVME_CSTS_RDY)) { ++ ret = 0; ++ break; ++ } ++ usleep_range(1000, 1100); ++ } ++ dev_info(pcie->dev, "phoenix ANS: disable result %d CSTS %#x; blocking bus master\n", ++ ret, csts); ++ pci_iounmap(pdev, bar); ++disable: ++ pci_clear_master(pdev); ++ pci_disable_device(pdev); ++ state->result = ret; ++ return 0; ++} ++ ++static ssize_t crash_quiesce_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ struct apple_h9p_crash_state state = { .pcie = pcie }; ++ ++ if (!pcie || !is_kdump_kernel() || ++ !of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !sysfs_streq(buf, "1")) ++ return -EPERM; ++ /* The endpoint gate must stay closed; don't park under a live driver. */ ++ if (pcie->late_ports || work_busy(&pcie->late_work.work)) ++ return -EBUSY; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_crash_stop_ans, &state); ++ if (state.busy) ++ return -EBUSY; ++ /* Isolation is still required after a bounded RDY timeout. */ ++ /* Wi-Fi stays linked until its own shutdown callback has run. */ ++ apple_h9p_capture_park(pcie); ++ dev_info(dev, "phoenix: ANS/PCIe quiesced (controller found %d result %d)\n", ++ state.found, state.result); ++ return count; ++} ++static DEVICE_ATTR_WO(crash_quiesce); ++ ++static struct attribute *apple_h9p_crash_attrs[] = { ++ &dev_attr_crash_quiesce.attr, ++ NULL, ++}; ++ ++static const struct attribute_group apple_h9p_crash_group = { ++ .attrs = apple_h9p_crash_attrs, ++}; ++ ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3426,6 +3776,7 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + u32 doorbell; + int ret; + ++ + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; +@@ -3578,6 +3929,13 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (is_kdump_kernel() && of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) { ++ ret = devm_device_add_group(dev, &apple_h9p_crash_group); ++ if (ret) ++ return ret; ++ } ++ ++ + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +@@ -3660,6 +4018,7 @@ MODULE_DEVICE_TABLE(of, apple_h9p_pcie_of_match); + + static struct platform_driver apple_h9p_pcie_driver = { + .probe = apple_h9p_pcie_probe, ++ .shutdown = apple_h9p_s8003_shutdown, + .driver = { + .name = "pcie-apple-h9p", + .of_match_table = apple_h9p_pcie_of_match, +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index a59861010..10be3ddef 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -17,11 +17,34 @@ struct device; + * Map @npages pages for FlatDMA request @tag, or retire the tag if @npages is + * zero. @to_memory: the controller writes the pages (an NVMe read). + */ ++#if IS_REACHABLE(CONFIG_APPLE_DART) ++int apple_ans_dart_stop(struct device *dev); ++#else ++static inline int apple_ans_dart_stop(struct device *dev) ++{ ++ return -EOPNOTSUPP; ++} ++#endif ++ + #if IS_REACHABLE(CONFIG_PCIE_APPLE_H9P) ++int apple_ans_handoff_begin(struct device *dev); ++int apple_ans_handoff_stopped(struct device *dev); ++int apple_ans_nvmmu_drain(struct device *dev); ++int apple_ans_nvmmu_reset(struct device *dev); ++int apple_ans_link_up(struct device *dev); ++bool apple_ans_is_s8003(struct device *dev); ++void apple_ans_host_dump(struct device *dev, const char *reason); + int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova); + #else ++static inline int apple_ans_handoff_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_handoff_stopped(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_drain(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_reset(struct device *dev) { return -EOPNOTSUPP; } ++static inline int apple_ans_link_up(struct device *dev) { return 1; } ++static inline bool apple_ans_is_s8003(struct device *dev) { return false; } ++static inline void apple_ans_host_dump(struct device *dev, const char *reason) {} + static inline int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova) diff --git a/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..62d5bda6 --- /dev/null +++ b/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,337 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] nvme-pci: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + drivers/nvme/host/pci-apple-ans.c | 33 +++++++- + drivers/nvme/host/pci-internal.h | 2 + + drivers/nvme/host/pci.c | 128 ++++++++++++++++++++++++++++-- + 3 files changed, 155 insertions(+), 8 deletions(-) + +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index 5ba239e47..fdd9fa091 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -84,6 +84,7 @@ struct apple_ans_nvme { + u32 cc; + unsigned long cc_jiffies; + bool cc_written; ++ bool host_fault; + dma_addr_t hmb_dma; + u32 hmb_size; + struct apple_ans_nvme_req req[APPLE_ANS_NVMMU_MAX_REQS]; +@@ -218,13 +219,20 @@ static int nvme_pci_apple_ans_reset_dma(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long flags; +- unsigned int tag; + int ret; + +- for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { +- ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ans->host == &apple_ans_host_s8003) { ++ ret = apple_ans_nvmmu_reset(dev->dev); + if (ret) + return ret; ++ } else { ++ unsigned int tag; ++ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } + } + + spin_lock_irqsave(&ans->req_lock, flags); +@@ -326,10 +334,24 @@ static int nvme_pci_apple_ans_prepare_enable(struct nvme_dev *dev) + return 0; + } + ++static int nvme_pci_apple_ans_check_link(struct nvme_dev *dev) ++{ ++ struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); ++ ++ if (ans && ans->host == &apple_ans_host_s8003 && ++ (ans->host_fault || apple_ans_link_up(dev->dev) <= 0)) { ++ ans->host_fault = true; ++ return -EIO; ++ } ++ return 0; ++} ++ + static int nvme_pci_apple_ans_write_cc(struct nvme_dev *dev, u32 cc) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + writel(cc, dev->bar + NVME_REG_CC); + if (!ans) + return 0; +@@ -349,6 +371,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long deadline; + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + *csts = readl(dev->bar + NVME_REG_CSTS); + if (!ans || !ans->cc_written || !ans->host->rdy_timeout_ms || + (ans->cc & NVME_CC_SHN_MASK)) +@@ -360,6 +384,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + time_after(jiffies, deadline)) { + dev_err(dev->dev, "ANS RDY timeout (CC %#x, CSTS %#x)\n", + ans->cc, *csts); ++ ans->host_fault = true; ++ apple_ans_host_dump(dev->dev, "RDY deadline"); + return -ETIMEDOUT; + } + +@@ -419,6 +445,7 @@ static int nvme_pci_apple_ans_free_req(struct nvme_dev *dev, + dev_err_ratelimited(dev->dev, + "failed to release ANS DMA slot %u: %d\n", + tag, ret); ++ nvme_reset_ctrl(&dev->ctrl); + return ret; + } + +diff --git a/drivers/nvme/host/pci-internal.h b/drivers/nvme/host/pci-internal.h +index a62777abd..d3fe17867 100644 +--- a/drivers/nvme/host/pci-internal.h ++++ b/drivers/nvme/host/pci-internal.h +@@ -74,6 +74,8 @@ struct nvme_dev { + struct nvme_ctrl ctrl; + u32 last_ps; + bool hmb; ++ unsigned long apple_ans_flags; ++ atomic_t apple_ans_irq_count; + struct sg_table *hmb_sgt; + mempool_t *dmavec_mempool; + const struct nvme_pci_dma_ops *dma_ops; +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 9ffe2fb74..40b2448d1 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -5,6 +5,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -17,6 +18,10 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + #include + #include + #include +@@ -1559,6 +1564,9 @@ static irqreturn_t nvme_irq(int irq, void *data) + DEFINE_IO_COMP_BATCH(iob); + bool found; + ++ if (!nvmeq->qid && dev->dma_ops == &nvme_pci_apple_ans_ops) ++ atomic_inc(&dev->apple_ans_irq_count); ++ + if (nvme_apple_ans_shared_vector(dev)) + found = nvme_apple_ans_irq_vector(dev, nvmeq->cq_vector, &iob); + else +@@ -1836,6 +1844,40 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) + "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n"); + } + ++/* No endpoint MMIO/config or doorbell reads: safe first evidence for ANS. */ ++static void nvme_apple_ans_dump(struct nvme_dev *dev, const char *reason) ++{ ++ unsigned int i; ++ ++ dev_info(dev->dev, "Apple ANS: snapshot %s state %d online %u allocated %u vectors %u recovery_used %u\n", ++ reason, nvme_ctrl_state(&dev->ctrl), dev->online_queues, ++ dev->ctrl.queue_count, dev->num_vecs, ++ test_bit(0, &dev->apple_ans_flags)); ++ apple_ans_host_dump(dev->dev, reason); ++ for (i = 0; i < dev->ctrl.queue_count; i++) { ++ struct nvme_queue *q = &dev->queues[i]; ++ u16 head = READ_ONCE(q->cq_head); ++ struct nvme_completion *cqe; ++ int irq = -1; ++ ++ if (!q->cqes || head >= q->q_depth) ++ continue; ++ cqe = &q->cqes[head]; ++ if (test_bit(NVMEQ_ENABLED, &q->flags) && ++ !test_bit(NVMEQ_POLLED, &q->flags)) ++ irq = pci_irq_vector(to_pci_dev(dev->dev), q->cq_vector); ++ dev_info(dev->dev, "Apple ANS: q%u flags %#lx depth %u SQ tail %u/%u CQ head %u phase %u next status %#x cid %#x sqhead %u vector %u irq %d ANS admin IRQ actions %u SQ %pad CQ %pad\n", ++ i, READ_ONCE(q->flags), q->q_depth, ++ READ_ONCE(q->sq_tail), READ_ONCE(q->last_sq_tail), ++ head, READ_ONCE(q->cq_phase), ++ le16_to_cpu(READ_ONCE(cqe->status)), ++ le16_to_cpu(READ_ONCE(cqe->command_id)), ++ le16_to_cpu(READ_ONCE(cqe->sq_head)), q->cq_vector, ++ irq, atomic_read(&dev->apple_ans_irq_count), ++ &q->sq_dma_addr, &q->cq_dma_addr); ++ } ++} ++ + static enum blk_eh_timer_return nvme_timeout(struct request *req) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); +@@ -1844,8 +1886,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + struct request *abort_req; + struct nvme_command cmd = { }; + struct pci_dev *pdev = to_pci_dev(dev->dev); +- u32 csts = readl(dev->bar + NVME_REG_CSTS); ++ u32 csts = ~0U; + u8 opcode; ++ bool ans = dev->dma_ops == &nvme_pci_apple_ans_ops && ++ apple_ans_is_s8003(dev->dev); ++ ++ if (ans) { ++ nvme_apple_ans_dump(dev, "timeout before CSTS"); ++ if (dev->dma_ops->read_csts(dev, &csts)) ++ goto disable; ++ } else { ++ csts = readl(dev->bar + NVME_REG_CSTS); ++ } + + /* + * Shutdown the device immediately if we see it is disconnected. This +@@ -1921,6 +1973,11 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + * returned to the driver, or if this is the admin queue. + */ + opcode = nvme_req(req)->cmd->common.opcode; ++ if (ans) { ++ if (test_and_set_bit(0, &dev->apple_ans_flags)) ++ nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); ++ goto disable; ++ } + if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) { + dev_warn(dev->ctrl.device, + "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n", +@@ -2804,6 +2861,7 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, + return a->mode; + } + ++ + static struct attribute *nvme_pci_attrs[] = { + &dev_attr_cmb.attr, + &dev_attr_cmbloc.attr, +@@ -3283,11 +3341,46 @@ static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev) + return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY); + } + +-static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++/* Certify stopped DMA before the kexec PCIe handoff. */ ++static int nvme_apple_ans_handoff_disable(struct nvme_dev *dev, int shutdown_ret) ++{ ++ struct pci_dev *pdev = to_pci_dev(dev->dev); ++ u32 csts = ~0U, cc, expected; ++ int ret, err; ++ ++ ret = dev->dma_ops->read_csts(dev, &csts); ++ if (shutdown_ret || ret || (csts & NVME_CSTS_CFS) || ++ (csts & NVME_CSTS_SHST_MASK) != NVME_CSTS_SHST_CMPLT) { ++ pci_clear_master(pdev); ++ return -EBUSY; ++ } ++ dev->ctrl.ctrl_config &= ~(NVME_CC_ENABLE | NVME_CC_SHN_MASK); ++ expected = dev->ctrl.ctrl_config; ++ ret = dev->dma_ops->write_cc(dev, expected); ++ if (!ret) { ++ ret = read_poll_timeout(dev->dma_ops->read_csts, err, ++ err || !(csts & NVME_CSTS_RDY), ++ 1000, 2000000, false, dev, &csts); ++ if (!ret) ++ ret = err; ++ } ++ if (ret) ++ goto block_dma; ++ cc = readl(dev->bar + NVME_REG_CC); ++ if ((csts & NVME_CSTS_CFS) || ++ (cc & (NVME_CC_ENABLE | NVME_CC_SHN_MASK))) ++ ret = -EBUSY; ++block_dma: ++ pci_clear_master(pdev); ++ return ret; ++} ++ ++static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool paired) + { + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); + dead = nvme_pci_ctrl_is_dead(dev); +@@ -3298,15 +3391,26 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown) +- nvme_wait_freeze_timeout(&dev->ctrl); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) ++ panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); + } + + nvme_quiesce_io_queues(&dev->ctrl); + ++ if (paired && (dead || !dev->ctrl.queue_count)) ++ panic("Apple ANS: dead outgoing controller; refusing kexec"); + if (!dead && dev->ctrl.queue_count > 0) { ++ if (paired) { ++ nvme_sync_io_queues(&dev->ctrl); ++ ret = apple_ans_nvmmu_drain(dev->dev); ++ if (ret) ++ panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ } + nvme_delete_io_queues(dev); +- nvme_disable_ctrl(&dev->ctrl, shutdown); ++ dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); ++ ret = nvme_disable_ctrl(&dev->ctrl, shutdown); ++ if (paired) ++ ret = nvme_apple_ans_handoff_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } + nvme_suspend_io_queues(dev); +@@ -3329,7 +3433,15 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) + nvme_unquiesce_admin_queue(&dev->ctrl); + } ++ if (paired && !ret) ++ ret = apple_ans_handoff_stopped(dev->dev); + mutex_unlock(&dev->shutdown_lock); ++ return ret; ++} ++ ++static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++{ ++ nvme_dev_disable_paired(dev, shutdown, false); + } + + static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) +@@ -3920,6 +4032,12 @@ static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); + ++ if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) ++ panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + nvme_disable_prepare_reset(dev, true); + } + diff --git a/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch b/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch new file mode 100644 index 00000000..f82f0350 --- /dev/null +++ b/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch @@ -0,0 +1,85 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:07 +0100 +Subject: [PATCH] PCI: apple: cold-reset after an uncertified kexec handoff + +The warm guard sets panic_timeout to zero. Neither an uncertified ANS +stop nor failed PERST, LTSSM stop or powered-port certificate can safely +continue into the incoming kernel. Flush the reason and request a cold +watchdog reset instead of hanging in panic. Return without further port +writes if restart unexpectedly returns. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 9 +++++---- + include/linux/soc/apple/cold-reset.h | 19 +++++++++++++++++++ + 2 files changed, 24 insertions(+), 4 deletions(-) + create mode 100644 include/linux/soc/apple/cold-reset.h + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index f0095394a..28305b0ee 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -49,6 +49,7 @@ + #include + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -3609,7 +3610,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!READ_ONCE(pcie->ans_handoff_stopped) || + (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + !apple_s8003_port_powered(pcie))) +- panic("Apple ANS: refusing uncertified powered handoff"); ++ return apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); + for (port = 0; port < H9P_NUM_PORTS; port++) { + void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; + +@@ -3621,9 +3622,9 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, + S8003_RC_PORT_PWR_REFCLK_NOGATE); + if (gpiod_direction_output(pcie->perst[port], 1)) +- panic("Apple PCIe: PERST failed during handoff"); ++ return apple_cold_reset("Apple PCIe: PERST failed during handoff"); + if (apple_s8003_port_stop(pcie, port)) +- panic("Apple PCIe: LTSSM stop failed during handoff"); ++ return apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + udelay(S8003_PERST_TO_REFCLK_US); + if (!port && pcie->nvmmu.ready) { + h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); +@@ -3635,7 +3636,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || + gpiod_get_value_cansleep(pcie->perst[0]) != 1)) +- panic("Apple ANS: outgoing powered port certificate failed"); ++ return apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); + pcie->enabled_ports &= ~BIT(port); + } + dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); +diff --git a/include/linux/soc/apple/cold-reset.h b/include/linux/soc/apple/cold-reset.h +new file mode 100644 +index 000000000..34467be50 +--- /dev/null ++++ b/include/linux/soc/apple/cold-reset.h +@@ -0,0 +1,19 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_COLD_RESET_H ++#define _LINUX_SOC_APPLE_COLD_RESET_H ++ ++#include ++#include ++#include ++ ++/* Shutdown has no safe return to a warm kernel after an unproven DMA stop. */ ++static inline void apple_cold_reset(const char *reason) ++{ ++ pr_emerg("Apple unsafe handover: %s; forcing cold watchdog reset\n", reason); ++ pr_flush(1000, true); ++ kmsg_dump_desc(KMSG_DUMP_PANIC, reason); ++ WRITE_ONCE(reboot_mode, REBOOT_COLD); ++ emergency_restart(); ++} ++ ++#endif diff --git a/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch b/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch new file mode 100644 index 00000000..5075181b --- /dev/null +++ b/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch @@ -0,0 +1,76 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:08:51 +0100 +Subject: [PATCH] nvme-pci: cold-reset after unsafe ANS kexec shutdown + +An unsafe ANS shutdown cannot panic under a zero panic timeout. Use the +shared logged and flushed cold-reset path for each failed certificate. +If reset unexpectedly returns, leave without further device operations. + +Assisted-by: LLM +--- + drivers/nvme/host/pci.c | 29 +++++++++++++++++++++-------- + 1 file changed, 21 insertions(+), 8 deletions(-) + +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 40b2448d1..2f356a8ce 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -3391,20 +3392,30 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) +- panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) { ++ apple_cold_reset("Apple ANS: outgoing I/O did not drain"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + + nvme_quiesce_io_queues(&dev->ctrl); + +- if (paired && (dead || !dev->ctrl.queue_count)) +- panic("Apple ANS: dead outgoing controller; refusing kexec"); ++ if (paired && (dead || !dev->ctrl.queue_count)) { ++ apple_cold_reset("Apple ANS: dead outgoing controller"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + if (!dead && dev->ctrl.queue_count > 0) { + if (paired) { + nvme_sync_io_queues(&dev->ctrl); + ret = apple_ans_nvmmu_drain(dev->dev); +- if (ret) +- panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ if (ret) { ++ dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); ++ apple_cold_reset("Apple ANS: outgoing drain failed"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + nvme_delete_io_queues(dev); + dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); +@@ -4034,8 +4045,10 @@ static void nvme_shutdown(struct pci_dev *pdev) + + if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && + dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { +- if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) +- panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) { ++ apple_cold_reset("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + return; + } + nvme_disable_prepare_reset(dev, true); diff --git a/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch b/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch new file mode 100644 index 00000000..5b49b55d --- /dev/null +++ b/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch @@ -0,0 +1,62 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:08:51 +0100 +Subject: [PATCH] PCI: apple: use explicit returns after cold reset + +Use standard C void returns while retaining the early exit if the reset +helper unexpectedly returns. Successful handoff ordering is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 24 ++++++++++++++++-------- + 1 file changed, 16 insertions(+), 8 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 28305b0ee..5890d632a 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3609,8 +3609,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + cancel_delayed_work_sync(&pcie->late_work); + if (!READ_ONCE(pcie->ans_handoff_stopped) || + (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && +- !apple_s8003_port_powered(pcie))) +- return apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); ++ !apple_s8003_port_powered(pcie))) { ++ apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); ++ return; ++ } + for (port = 0; port < H9P_NUM_PORTS; port++) { + void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; + +@@ -3621,10 +3623,14 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); + h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, + S8003_RC_PORT_PWR_REFCLK_NOGATE); +- if (gpiod_direction_output(pcie->perst[port], 1)) +- return apple_cold_reset("Apple PCIe: PERST failed during handoff"); +- if (apple_s8003_port_stop(pcie, port)) +- return apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); ++ if (gpiod_direction_output(pcie->perst[port], 1)) { ++ apple_cold_reset("Apple PCIe: PERST failed during handoff"); ++ return; ++ } ++ if (apple_s8003_port_stop(pcie, port)) { ++ apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); ++ return; ++ } + udelay(S8003_PERST_TO_REFCLK_US); + if (!port && pcie->nvmmu.ready) { + h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); +@@ -3635,8 +3641,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_s8003_port_off(pcie, port); + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || +- gpiod_get_value_cansleep(pcie->perst[0]) != 1)) +- return apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) { ++ apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); ++ return; ++ } + pcie->enabled_ports &= ~BIT(port); + } + dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); diff --git a/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch b/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch new file mode 100644 index 00000000..462bcde9 --- /dev/null +++ b/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch @@ -0,0 +1,27 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:00:00 +0100 +Subject: [PATCH] apple-ans: declare retained DART resume after handoff API + +Declare the retained DART resume operation after the ANS interface has +been introduced, so the paired sleep consumer can call it. + +Assisted-by: LLM +--- + include/linux/apple-ans.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index 10be3ddef..a312a683c 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -19,7 +19,9 @@ struct device; + */ + #if IS_REACHABLE(CONFIG_APPLE_DART) + int apple_ans_dart_stop(struct device *dev); ++int apple_ans_dart_start(struct device *dev); + #else ++static inline int apple_ans_dart_start(struct device *dev) { return -EOPNOTSUPP; } + static inline int apple_ans_dart_stop(struct device *dev) + { + return -EOPNOTSUPP; diff --git a/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch b/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch new file mode 100644 index 00000000..210631ae --- /dev/null +++ b/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch @@ -0,0 +1,163 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] nvme-pci: certify ANS sleep stop and rebuild through + warm-storage APIs + +On S8003 the NVMe controller (ANS) sits behind a PCIe port that keeps +power and reference clock over system sleep, while its DART loses its +translations. The generic nvme_suspend() path neither drains the paired +controller nor rebuilds the DART, so storage did not come back after +suspend to idle. + +For the paired ANS controller, nvme_suspend() now begins the handoff, +moves the controller to RESETTING and shuts it down through +nvme_dev_disable_paired(). preinit restores the DART with +apple_ans_dart_start(). In nvme_dev_disable_paired() a failure while +preparing for sleep unfreezes the queues and returns -EBUSY, so the +suspend is aborted instead of taking the cold-reset path used for kexec. +Resume returns -ENODEV when the PCIe link was lost over sleep. + +Assisted-by: LLM +--- + drivers/nvme/host/pci-apple-ans.c | 7 ++++ + drivers/nvme/host/pci.c | 54 +++++++++++++++++++++++++------ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index fdd9fa091..bdd9f991f 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -6,6 +6,7 @@ + * Copyright (C) 2026 Paul Praschl + */ + ++#include + #include + #include + #include +@@ -276,6 +277,12 @@ static int nvme_pci_apple_ans_preinit(struct nvme_dev *dev) + if (!ans) + return 0; + ++ if (apple_ans_is_s8003(dev->dev)) { ++ ret = apple_ans_dart_start(dev->dev); ++ if (ret) ++ return ret; ++ } ++ + core_mask = readl(dev->bar + APPLE_ANS_REG_CORE_MASK); + log_size = readl(dev->bar + APPLE_ANS_REG_LOG_SIZE); + hmb_size = readl(dev->bar + APPLE_ANS_REG_HMB_SIZE_REQ); +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 2f356a8ce..6868811d5 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -3381,6 +3381,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ bool for_sleep = paired && !kexec_in_progress; + int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); +@@ -3393,28 +3394,40 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + * if doing a safe shutdown. + */ + if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) { +- apple_cold_reset("Apple ANS: outgoing I/O did not drain"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (for_sleep) { ++ nvme_unfreeze(&dev->ctrl); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EBUSY; ++ } else { ++ apple_cold_reset("Apple ANS: outgoing I/O did not drain"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + } + + nvme_quiesce_io_queues(&dev->ctrl); + + if (paired && (dead || !dev->ctrl.queue_count)) { +- apple_cold_reset("Apple ANS: dead outgoing controller"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (!for_sleep) { ++ apple_cold_reset("Apple ANS: dead outgoing controller"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } ++ goto sleep_failed; + } + if (!dead && dev->ctrl.queue_count > 0) { + if (paired) { + nvme_sync_io_queues(&dev->ctrl); + ret = apple_ans_nvmmu_drain(dev->dev); + if (ret) { +- dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); +- apple_cold_reset("Apple ANS: outgoing drain failed"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (!for_sleep) { ++ dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); ++ apple_cold_reset("Apple ANS: outgoing drain failed"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } ++ goto sleep_failed; + } + } + nvme_delete_io_queues(dev); +@@ -3424,6 +3437,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + ret = nvme_apple_ans_handoff_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } ++sleep_failed: + nvme_suspend_io_queues(dev); + nvme_suspend_queue(dev, 0); + pci_free_irq_vectors(pdev); +@@ -4101,6 +4115,9 @@ static int nvme_resume(struct device *dev) + struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); + struct nvme_ctrl *ctrl = &ndev->ctrl; + ++ if (pci_dev_is_disconnected(to_pci_dev(dev))) ++ return -ENODEV; ++ + if (ndev->last_ps == U32_MAX || + nvme_set_power_state(ctrl, ndev->last_ps) != 0) + goto reset; +@@ -4120,6 +4137,20 @@ static int nvme_suspend(struct device *dev) + int ret = -EBUSY; + + ndev->last_ps = U32_MAX; ++ if (ndev->dma_ops == &nvme_pci_apple_ans_ops && apple_ans_is_s8003(dev)) { ++ ret = apple_ans_handoff_begin(dev); ++ if (ret) ++ return ret; ++ if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || ++ !nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) ++ return -EBUSY; ++ ret = nvme_dev_disable_paired(ndev, true, true); ++ if (ret) { ++ nvme_try_sched_reset(ctrl); ++ return -EBUSY; ++ } ++ return 0; ++ } + + /* + * The platform does not remove power for a kernel managed suspend so +@@ -4201,6 +4232,9 @@ static int nvme_simple_resume(struct device *dev) + struct pci_dev *pdev = to_pci_dev(dev); + struct nvme_dev *ndev = pci_get_drvdata(pdev); + ++ if (pci_dev_is_disconnected(to_pci_dev(dev))) ++ return -ENODEV; ++ + return nvme_try_sched_reset(&ndev->ctrl); + } + diff --git a/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch b/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch new file mode 100644 index 00000000..5d551f21 --- /dev/null +++ b/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch @@ -0,0 +1,39 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] dt-bindings: pci: describe paired sleep and optional + handoff link flush + +Document two boolean properties of the S8003 H9P controller. +apple,paired-system-sleep marks an ANS controller whose port keeps power +across system sleep and is stopped and restarted together with it. +apple,ans-kexec-link-flush makes a warm handoff consume a stale link-up +status under PERST before the link is stopped. + +Assisted-by: LLM +--- + .../devicetree/bindings/pci/apple,t8010-pcie.yaml | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +index f9b72ce26..cebf60d8a 100644 +--- a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml ++++ b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +@@ -231,6 +231,17 @@ properties: + without exposing a root bridge. This is used when an enabled port shares + PHY state with a firmware-enabled sibling port. + ++ apple,ans-kexec-link-flush: ++ type: boolean ++ description: ++ Consume stale ANS link status under PERST during warm handoff. ++ ++ apple,paired-system-sleep: ++ type: boolean ++ description: ++ The ANS controller is paired with retained-power port reset over system ++ sleep. ++ + interrupt-controller: true + + '#interrupt-cells': diff --git a/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch b/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch new file mode 100644 index 00000000..8ad218f5 --- /dev/null +++ b/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch @@ -0,0 +1,402 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] PCI: apple-h9p: pair s2idle with retained ANS power and + early admission + +System sleep on J71t has to keep the ANS port powered: removing its +power or reference clock loses the NVMe controller until a cold boot. + +Add sleep support to the S8003 path of the H9P driver, enabled by +apple,paired-system-sleep. prepare pins the endpoints and holds off +config accesses. suspend asserts PERST and stops the link but keeps +power and refclk, and a failure before the first destructive write rolls +back with power kept. resume brings the ports back through the powered +reset and marks endpoints that did not return as disconnected. The +powered reset (PERST, LTSSM stop and a fresh link training, without a +power cycle) is added here with its only caller; the common block is not +reinitialized across s2idle. + +A suspend precheck refuses with -EBUSY, before sync, anything but +suspend to idle, and also refuses while a filesystem is mounted from an +NBD device, port bring-up work is pending, the port is not powered, or a +bound GPU, VXD or SEP driver holds no reserved sleep admission. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 278 +++++++++++++++++++++++- + 1 file changed, 277 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 5890d632a..7a0a832e3 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -51,6 +51,11 @@ + #include + #include + ++#include ++#include ++#include ++#include ++#include "../pci.h" + #include "pci-host-common.h" + + #define H9P_NUM_PORTS 4 +@@ -719,6 +724,13 @@ struct apple_h9p_pcie { + unsigned int late_tries; + struct delayed_work late_work; + bool ans_handoff_stopped; ++ bool ans_sleep_powered; ++ u32 resume_ports; ++ struct pci_dev *sleep_pinned[32]; ++ unsigned int sleep_npinned; ++ bool sleep_pin_overflow; ++ struct notifier_block sleep_precheck; ++ + + struct apple_h9p_nvmmu nvmmu; + +@@ -1094,6 +1106,9 @@ static int apple_h9p_pcie_attach_genpd(struct apple_h9p_pcie *pcie) + return PTR_ERR(pcie->pd_dev[i]); + } + ++ if (of_property_read_bool(dev->of_node, "apple,paired-system-sleep")) ++ dev_pm_syscore_device(pcie->pd_dev[i], true); ++ + pcie->pd_link[i] = device_link_add(dev, pcie->pd_dev[i], + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | +@@ -1239,6 +1254,20 @@ static void apple_h9p_limit_config_field(u32 *val, int where, int size, + *val = (*val & ~mask) | (value & mask); + } + ++static bool apple_h9p_sleep_config_blocked(struct pci_bus *bus) ++{ ++ struct pci_config_window *cfg = bus->sysdata; ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(cfg->parent); ++ struct pci_bus *root = bus; ++ ++ if (!pcie || bus->number == cfg->busr.start) ++ return false; ++ while (root->parent && root->parent->parent) ++ root = root->parent; ++ return root->self && !(READ_ONCE(pcie->enabled_ports) & ++ BIT(PCI_SLOT(root->self->devfn))); ++} ++ + static int apple_h9p_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) + { +@@ -1258,6 +1287,10 @@ static int apple_h9p_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + pcie = NULL; + } + ++ if (apple_h9p_sleep_config_blocked(bus)) { ++ *val = ~0U; ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ } + ret = pci_generic_config_read(bus, devfn, where, size, val); + if (ret != PCIBIOS_SUCCESSFUL || !pcie) + return ret; +@@ -1295,6 +1328,9 @@ static int apple_h9p_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + return PCIBIOS_DEVICE_NOT_FOUND; + } + ++ if (apple_h9p_sleep_config_blocked(bus)) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ + if (where <= PCI_INTERRUPT_LINE && where + size > PCI_INTERRUPT_LINE) + val |= 0xffu << ((PCI_INTERRUPT_LINE - where) << 3); + +@@ -2925,6 +2961,33 @@ static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) + (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); + } + ++static int apple_s8003_powered_reset(struct apple_h9p_pcie *pcie) ++{ ++ int ret; ++ ++ if (!apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (!ret && READ_ONCE(pcie->ans_sleep_powered) && ++ (readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & S8003_PORT_LINKSTS_UP)) { ++ u32 val; ++ ++ h9p_rmw(pcie->base_port[0] + H9P_PORT_LTSSMCTL, 0, ++ H9P_PORT_LTSSM_ENABLE); ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ ret = apple_s8003_port_stop(pcie, 0); ++ } ++ if (!ret) ++ ret = apple_s8003_start_link(pcie, 0); ++ /* A failed powered reset stays under PERST; no full-cycle fallback. */ ++ if (ret) ++ gpiod_direction_output(pcie->perst[0], 1); ++ return ret; ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -2940,6 +3003,8 @@ static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + * validated on it, the other ports the restart sequence that is the + * only one validated on the Wi-Fi port. + */ ++ if (!port && READ_ONCE(pcie->ans_sleep_powered)) ++ return apple_s8003_powered_reset(pcie); + if (pcie->hw->nvme_mode_ports & BIT(port)) + ret = apple_s8003_power_cycle_port(pcie, port); + else +@@ -3598,6 +3663,208 @@ int apple_ans_handoff_begin(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); + ++ ++static bool apple_h9p_sleep_enabled(struct apple_h9p_pcie *pcie) ++{ ++ return of_property_read_bool(pcie->dev->of_node, "apple,paired-system-sleep"); ++} ++ ++static int apple_h9p_sleep_pin(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ if (pdev->dev_flags & PCI_DEV_FLAGS_NO_D3) ++ return 0; ++ if (pcie->sleep_npinned == ARRAY_SIZE(pcie->sleep_pinned)) { ++ pcie->sleep_pin_overflow = true; ++ return 1; ++ } ++ pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; ++ pcie->sleep_pinned[pcie->sleep_npinned++] = pci_dev_get(pdev); ++ return 0; ++} ++ ++static void apple_h9p_sleep_complete(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ ++ while (pcie && pcie->sleep_npinned) { ++ struct pci_dev *pdev = pcie->sleep_pinned[--pcie->sleep_npinned]; ++ ++ pdev->dev_flags &= ~PCI_DEV_FLAGS_NO_D3; ++ pci_dev_put(pdev); ++ } ++ if (pcie) ++ pcie->ans_sleep_powered = false; ++} ++ ++static int apple_h9p_sleep_prepare(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ pcie->ans_handoff_stopped = false; ++ pcie->ans_sleep_powered = false; ++ pcie->sleep_pin_overflow = false; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_sleep_pin, pcie); ++ ret = pcie->sleep_pin_overflow ? -EBUSY : 0; ++ if (ret) ++ apple_h9p_sleep_complete(dev); ++ return ret; ++} ++ ++static void apple_h9p_sleep_cut_off(struct apple_h9p_pcie *pcie, unsigned int port) ++{ ++ struct pci_dev *rp; ++ ++ gpiod_direction_output(pcie->perst[port], 1); ++ apple_s8003_port_stop(pcie, port); ++ /* Never remove ANS power/refclk, even after failed resume. */ ++ pcie->enabled_ports &= ~BIT(port); ++ rp = pci_get_slot(pcie->bridge->bus, PCI_DEVFN(port, 0)); ++ if (!rp) ++ return; ++ if (rp->subordinate) ++ pci_walk_bus(rp->subordinate, pci_dev_set_disconnected, NULL); ++ pci_dev_set_disconnected(rp, NULL); ++ pci_dev_put(rp); ++} ++ ++static int apple_h9p_sleep_resume(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ unsigned int port; ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ struct pci_dev *rp; ++ ++ if (!(pcie->resume_ports & BIT(port))) ++ continue; ++ if (port && !gpiod_get_value_cansleep(pcie->perst[port]) && ++ apple_h9p_link_up(pcie, port)) ++ continue; ++ ret = port ? apple_s8003_setup_port_tries(pcie, port) : ++ apple_s8003_powered_reset(pcie); ++ if (ret) { ++ apple_h9p_sleep_cut_off(pcie, port); ++ dev_err(dev, "port %u lost after sleep: %d; endpoints disconnected\n", port, ret); ++ continue; ++ } ++ pcie->enabled_ports |= BIT(port); ++ msleep(PCIE_RESET_CONFIG_WAIT_MS); ++ rp = pci_get_slot(pcie->bridge->bus, PCI_DEVFN(port, 0)); ++ if (rp) { ++ pci_restore_state(rp); ++ pci_dev_put(rp); ++ } ++ } ++ if (pcie->enabled_ports & BIT(0)) { ++ ret = apple_h9p_nvmmu_program(pcie); ++ if (ret) ++ apple_h9p_sleep_cut_off(pcie, 0); ++ } ++ pcie->resume_ports = 0; ++ return 0; ++} ++ ++static int apple_h9p_sleep_suspend(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ void __iomem *cfg; ++ u32 val; ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ cfg = pcie->base_config; ++ if (!READ_ONCE(pcie->ans_handoff_stopped) || !apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ pcie->resume_ports = pcie->enabled_ports; ++ /* Latch before the first destructive write; rollback must retain power. */ ++ pcie->ans_sleep_powered = true; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) { ++ /* r1e: consume the falling link before stopping LTSSM. */ ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ ret = apple_s8003_port_stop(pcie, 0); ++ } ++ if (ret) { ++ apple_h9p_sleep_resume(dev); ++ return ret; ++ } ++ udelay(S8003_PERST_TO_REFCLK_US); ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->enabled_ports &= ~BIT(0); ++ return 0; ++} ++ ++static void apple_h9p_sleep_nbd(struct super_block *sb, void *data) ++{ ++ if (sb->s_bdev && !strncmp(sb->s_bdev->bd_disk->disk_name, "nbd", 3)) ++ *(bool *)data = true; ++} ++ ++/* Until GS provides a reserved admission capability, bound accelerators fail closed. */ ++static int apple_h9p_sleep_stack(struct device *dev, void *data) ++{ ++ const char *name; ++ ++ if (!dev->driver) ++ return 0; ++ name = dev->driver->name; ++ if (strstr(name, "g5p") || strstr(name, "powervr") || ++ strstr(name, "vxd") || strstr(name, "sep")) { ++ if (pm_suspend_admission_reserved(dev)) ++ return 0; ++ dev_warn(dev, "system sleep refused: no reserved safe sleep capability\n"); ++ return -EBUSY; ++ } ++ return 0; ++} ++ ++static int apple_h9p_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_h9p_pcie *pcie = container_of(nb, struct apple_h9p_pcie, sleep_precheck); ++ bool nbd = false; ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ if (!apple_h9p_sleep_enabled(pcie)) ++ return NOTIFY_DONE; ++ if (action != PM_SUSPEND_TO_IDLE) ++ return notifier_from_errno(-EBUSY); ++ iterate_supers(apple_h9p_sleep_nbd, &nbd); ++ if (nbd || pcie->late_ports || work_busy(&pcie->late_work.work) || ++ !pcie->pd_dev || !apple_s8003_port_powered(pcie) || ++ bus_for_each_dev(&platform_bus_type, NULL, NULL, apple_h9p_sleep_stack)) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_OK; ++} ++ ++static void apple_h9p_sleep_unregister(void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ unregister_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++} ++ ++static const struct dev_pm_ops apple_h9p_sleep_ops = { ++ .prepare = apple_h9p_sleep_prepare, ++ .complete = apple_h9p_sleep_complete, ++ .suspend_noirq = apple_h9p_sleep_suspend, ++ .resume_noirq = apple_h9p_sleep_resume, ++}; ++ + static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + { + struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); +@@ -3627,6 +3894,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_cold_reset("Apple PCIe: PERST failed during handoff"); + return; + } ++ + if (apple_s8003_port_stop(pcie, port)) { + apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + return; +@@ -3777,6 +4045,9 @@ static const struct attribute_group apple_h9p_crash_group = { + }; + + ++static int apple_h9p_sleep_precheck(struct notifier_block *, unsigned long, void *); ++static void apple_h9p_sleep_unregister(void *); ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3948,7 +4219,11 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +- return 0; ++ pcie->sleep_precheck.notifier_call = apple_h9p_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++ if (ret) ++ return ret; ++ return devm_add_action_or_reset(dev, apple_h9p_sleep_unregister, pcie); + } + + static const struct apple_h9p_hw apple_h9p_hw_t7000 = { +@@ -4032,6 +4307,7 @@ static struct platform_driver apple_h9p_pcie_driver = { + .name = "pcie-apple-h9p", + .of_match_table = apple_h9p_pcie_of_match, + .suppress_bind_attrs = true, ++ .pm = pm_sleep_ptr(&apple_h9p_sleep_ops), + }, + }; + module_platform_driver(apple_h9p_pcie_driver); diff --git a/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch b/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch new file mode 100644 index 00000000..bd2908f3 --- /dev/null +++ b/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] PCI: apple-h9p: make the handoff link flush a separate + opt-in + +Waiting for the link-up status to fall under PERST before LTSSM is +stopped was tied to the sleep path. A warm handoff can see the same +stale status, but not every board needs the wait. + +Do the flush in the outgoing kexec shutdown path only, and only when +apple,ans-kexec-link-flush is set, so it stays off unless a board asks +for it. The incoming kernel restarts the storage port with the cold-boot +power cycle, which needs no flush. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 7a0a832e3..46c24f2b5 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3894,7 +3894,12 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_cold_reset("Apple PCIe: PERST failed during handoff"); + return; + } ++ if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-link-flush")) { ++ u32 val; + ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ } + if (apple_s8003_port_stop(pcie, port)) { + apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + return; diff --git a/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch b/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch new file mode 100644 index 00000000..9e84e7ac --- /dev/null +++ b/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch @@ -0,0 +1,61 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:15:38 +0100 +Subject: [PATCH] nvme-pci: stop Apple ANS before a restart as before a kexec + +A restart that re-enters the boot loader (PSCI SYSTEM_RESET to m1n1) +leaves the S8003 ANS controller to a loader that brings its port up again +and reads from it. On a plain shutdown nvme_disable_prepare_reset() sets +SHN but keeps CC.EN, and power-cycling the port in that state hangs the +SoC. + +Take the paired, certified stop that a kexec handoff already uses on every +restart as well: drain, SHN, CC.EN and RDY clear, bus mastering off and the +NVMMU/DART retired. Such a failure is not a sleep refusal, so it no longer +takes the sleep branch; it ends in a cold watchdog reset, as for kexec. +Power-off and halt keep the existing shutdown. + +Assisted-by: LLM +--- + drivers/nvme/host/pci.c | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 6868811d5..9fe60de0e 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -3381,7 +3381,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; +- bool for_sleep = paired && !kexec_in_progress; ++ bool for_sleep = paired && !kexec_in_progress && system_state != SYSTEM_RESTART; + int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); +@@ -4053,14 +4053,23 @@ static void nvme_reset_done(struct pci_dev *pdev) + flush_work(&dev->ctrl.reset_work); + } + ++/* ++ * A restart re-enters the boot loader, which starts the storage port again ++ * and reads from the controller. It needs the controller stopped as iBoot ++ * leaves it (SHN complete, CC.EN and RDY clear, DMA certified stopped): the ++ * stop a kexec handoff takes. Anything less ends in a cold reset. ++ */ + static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); ++ bool restart = !kexec_in_progress && system_state == SYSTEM_RESTART; + +- if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ if ((kexec_in_progress || restart) && apple_ans_is_s8003(dev->dev) && + dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { + if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) { +- apple_cold_reset("Apple ANS: refusing unsafe powered kexec handoff"); ++ apple_cold_reset(restart ? ++ "Apple ANS: unsafe controller stop before restart" : ++ "Apple ANS: refusing unsafe powered kexec handoff"); + return; + } + return; diff --git a/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch b/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch new file mode 100644 index 00000000..179866ce --- /dev/null +++ b/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch @@ -0,0 +1,70 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:15:39 +0100 +Subject: [PATCH] PCI: apple-h9p: hand off S8003 ports on restart and power off + Wi-Fi + +On a restart that re-enters the boot loader the S8003 host's shutdown did +nothing, so the Wi-Fi function kept its link and bus mastering. Its +firmware can keep writing into memory that the boot loader and the next +kernel reuse, and the boot loader does not touch ports other than storage. + +Run the kexec handoff on restart as well: block root routing, assert PERST, +stop the LTSSM and disable the NVMMU SART, with the same certificate and +cold-reset fallback. The storage port then ends powered with PERST held, +as for kexec. Ports other than storage are additionally powered off with +the iOS port power-down, the state the boot loader leaves them in and from +which probe restarts them. The kexec handoff itself is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 5d6e717ee..90491965d 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3891,12 +3891,20 @@ static const struct dev_pm_ops apple_h9p_sleep_ops = { + .resume_noirq = apple_h9p_sleep_resume, + }; + ++/* ++ * A kexec and a restart take the same handoff. After a restart the boot ++ * loader starts the storage port again from this state, as it does from ++ * iBoot's, but it does not stop an endpoint left on any other port: Wi-Fi ++ * would keep DMAing into memory the next kernel reuses. Those ports are ++ * therefore powered off after the isolation, as the boot loader leaves them. ++ */ + static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + { + struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ bool restart = !kexec_in_progress && system_state == SYSTEM_RESTART; + unsigned int port; + +- if (!kexec_in_progress || !pcie || ++ if ((!kexec_in_progress && !restart) || !pcie || + !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) + return; + cancel_delayed_work_sync(&pcie->late_work); +@@ -3938,6 +3946,8 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!port && !of_property_read_bool(pcie->dev->of_node, + "apple,ans-kexec-handoff")) + apple_s8003_port_off(pcie, port); ++ if (port && restart) ++ apple_s8003_port_off(pcie, port); + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || + gpiod_get_value_cansleep(pcie->perst[0]) != 1)) { +@@ -3946,7 +3956,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + } + pcie->enabled_ports &= ~BIT(port); + } +- dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++ if (restart) ++ dev_info(pcie->dev, "restart: PCIe PERST asserted, LTSSM stopped, ports 1+ powered off; NVMMU SART disabled\n"); ++ else ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); + } + + static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) diff --git a/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch b/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch new file mode 100644 index 00000000..dc5b66a6 --- /dev/null +++ b/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch @@ -0,0 +1,130 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pinctrl: apple-gpio: retire inherited GPIO interrupt modes + after kexec + +After a kexec, or after a panic that ran no shutdown callbacks, GPIO +pins can still be in an interrupt mode set up by the previous kernel. +Once the chained parent interrupts are enabled, such a pin raises an +interrupt that nobody maps or acknowledges, and it keeps firing. + +On apple,s8000-pinctrl, switch every GPIO interrupt mode off at probe, +before the parent interrupts are enabled, and keep outputs, mux, pulls, +data and grouping. Consumers arm their own pins again later. In the +handler, a pending pin without a mapping is masked and its group latch +cleared, with a rate-limited warning. + +Assisted-by: LLM +--- + drivers/pinctrl/pinctrl-apple-gpio.c | 62 +++++++++++++++++++++++++++- + 1 file changed, 60 insertions(+), 2 deletions(-) + +diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c +index b97023c97..766fcf5bc 100644 +--- a/drivers/pinctrl/pinctrl-apple-gpio.c ++++ b/drivers/pinctrl/pinctrl-apple-gpio.c +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -46,6 +47,7 @@ struct apple_gpio_pinctrl { + + void __iomem *base; + struct regmap *map; ++ bool retire_irqs; + + struct pinctrl_desc pinctrl_desc; + struct gpio_chip gpio_chip; +@@ -379,8 +381,33 @@ static void apple_gpio_irq_handler(struct irq_desc *desc) + chained_irq_enter(chip, desc); + for (pinh = 0; pinh < gc->ngpio; pinh += 32) { + pending = readl_relaxed(pctl->base + REG_IRQ(*grpp, pinh)); +- for_each_set_bit(pinl, &pending, 32) +- generic_handle_domain_irq(gc->irq.domain, pinh + pinl); ++ for_each_set_bit(pinl, &pending, 32) { ++ unsigned int pin = pinh + pinl; ++ int ret; ++ ++ ret = pin < gc->ngpio ? ++ generic_handle_domain_irq(gc->irq.domain, pin) : -EINVAL; ++ if (ret && pctl->retire_irqs) { ++ /* An inherited, unmapped source has nobody to ack it. ++ * Mask only this GPIO, then clear this group's latch. ++ */ ++ if (pin < gc->ngpio) { ++ u32 reg = apple_gpio_get_reg(pctl, pin); ++ u32 mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ u32 off = FIELD_PREP(REG_GPIOx_MODE, ++ REG_GPIOx_IN_IRQ_OFF); ++ ++ if (!(reg & REG_GPIOx_PERIPH) && ++ mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, off); ++ } ++ writel(BIT(pinl), pctl->base + REG_IRQ(*grpp, pinh)); ++ dev_warn_ratelimited(pctl->dev, ++ "masked orphan inherited GPIO IRQ group=%u pin=%u\n", ++ *grpp, pin); ++ } ++ } + } + chained_irq_exit(chip, desc); + } +@@ -403,6 +430,35 @@ static const struct irq_chip apple_gpio_irqchip = { + + /* Probe & register */ + ++static void apple_gpio_quiesce_irqs(struct apple_gpio_pinctrl *pctl) ++{ ++ struct gpio_chip *gc = &pctl->gpio_chip; ++ unsigned int pin, group, mode, masked = 0; ++ u32 reg; ++ ++ if (!pctl->retire_irqs) ++ return; ++ /* Before chained parent IRQs are enabled, retire old GPIO IRQ modes. ++ * Preserve outputs, mux, pulls, data and grouping. Consumers rearm their ++ * own pins later; panic entry need not have run any shutdown callbacks. ++ */ ++ for (pin = 0; pin < gc->ngpio; pin++) { ++ reg = apple_gpio_get_reg(pctl, pin); ++ mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ if (!(reg & REG_GPIOx_PERIPH) && mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) { ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, ++ FIELD_PREP(REG_GPIOx_MODE, REG_GPIOx_IN_IRQ_OFF)); ++ masked++; ++ } ++ } ++ for (group = 0; group < gc->irq.num_parents; group++) ++ for (pin = 0; pin < gc->ngpio; pin += 32) ++ writel(GENMASK(min(32U, gc->ngpio - pin) - 1, 0), ++ pctl->base + REG_IRQ(group, pin)); ++ dev_info(pctl->dev, "inherited GPIO IRQ baseline: masked %u inherited inputs\n", masked); ++} ++ + static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + { + struct gpio_irq_chip *girq = &pctl->gpio_chip.irq; +@@ -452,6 +508,7 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + girq->handler = handle_level_irq; + } + ++ apple_gpio_quiesce_irqs(pctl); + ret = devm_gpiochip_add_data(pctl->dev, &pctl->gpio_chip, pctl); + + out_free_irq_data: +@@ -485,6 +542,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) + if (!pctl) + return -ENOMEM; + pctl->dev = &pdev->dev; ++ pctl->retire_irqs = of_device_is_compatible(pdev->dev.of_node, "apple,s8000-pinctrl"); + pctl->soc = of_device_get_match_data(&pdev->dev); + pctl->gpio_chip.irq.num_parents = nirqs; + dev_set_drvdata(&pdev->dev, pctl); diff --git a/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..0ae20f3c --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,113 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 34 +++++++++++++++++++++----- + 1 file changed, 28 insertions(+), 6 deletions(-) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..7995d3b71 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -58,6 +58,8 @@ struct apple_pmgr_ps { + u32 min_state; + bool has_genpd; + bool manual_pm; ++ bool warm; ++ bool keep_on; + }; + + #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) +@@ -83,17 +85,21 @@ static int apple_pmgr_ps_set(struct apple_pmgr_ps *ps, u32 pstate, bool auto_ena + + dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", ps->name, pstate, reg); + +- regmap_write(ps->regmap, ps->offset, reg); ++ ret = regmap_write(ps->regmap, ps->offset, reg); ++ if (ret) ++ return ret; + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, reg, + (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, + APPLE_PMGR_PS_SET_TIMEOUT); +- if (ret < 0) ++ if (ret < 0) { + dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", + ps->name, pstate, reg); ++ return ret; ++ } + +- if (auto_enable) { ++ if (auto_enable && !ps->keep_on) { + /* Not all devices implement this; this is a no-op where not implemented. */ + reg &= ~APPLE_PMGR_FLAGS; + reg |= APPLE_PMGR_AUTO_ENABLE; +@@ -136,6 +142,9 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) + unsigned long flags; + int ret; + ++ if (ps->keep_on) ++ return -EBUSY; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_PWRGATE, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -168,6 +177,10 @@ static void apple_pmgr_clk_disable(struct clk_hw *hw) + unsigned long flags; + int ret; + ++ /* The lab has no separate clk gate sharing this power-state word. */ ++ if (ps->keep_on) ++ return; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, gate->off_state, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -306,6 +319,10 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + return ret; + } + ++ ps->warm = of_device_is_compatible(node, "apple,s8000-pmgr-pwrstate") && ++ of_property_read_bool(of_chosen, "linux,booted-from-kexec"); ++ ps->keep_on = ps->warm; ++ ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); + clock_only = of_property_read_bool(node, "apple,clock-gate-only"); + if (!clock_only) { + ps->genpd.flags |= GENPD_FLAG_IRQ_SAFE; +@@ -315,12 +332,17 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); +- if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) ++ if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && ++ !(ps->warm && ps->manual_pm)) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, + FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + + active = apple_pmgr_ps_is_active(ps); +- ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); ++ if (ps->warm && active && !ps->manual_pm) { ++ ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_ACTIVE, false); ++ if (ret) ++ return ret; ++ } + if (!clock_only && of_property_read_bool(node, "apple,always-on")) { + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + if (!active) { +@@ -331,7 +353,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + /* Turn on auto-PM if the domain is already on */ +- if (active) ++ if (active && !ps->warm) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, + ps->manual_pm ? 0 : APPLE_PMGR_AUTO_ENABLE); + diff --git a/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch new file mode 100644 index 00000000..d9a8480b --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-select-certified-AOP-reload-through-D.patch @@ -0,0 +1,127 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 84 ++++++++++++++++++++++++++ + 1 file changed, 84 insertions(+) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..0ae0eda2d 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -417,6 +419,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + goto err_remove; + } + ++ platform_set_drvdata(pdev, ps); + return 0; + err_remove: + if (ps->has_genpd) { +@@ -446,3 +449,84 @@ MODULE_AUTHOR("Hector Martin "); + MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); + + module_platform_driver(apple_pmgr_ps_driver); ++ ++/* ++ * Read-only observer for bring-up. The caller's nodes follow the same ++ * child -> parent order as genpd power transitions. No domain is attached, ++ * resumed, pinned on, reset, or written by this function. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data) ++{ ++ struct platform_device *providers[5]; ++ struct apple_pmgr_ps *ps[5]; ++ unsigned long flags; ++ unsigned int i, j, found = 0; ++ int ret = 0; ++ bool accessible = true; ++ ++ if (!count || count > ARRAY_SIZE(providers) || !read) ++ return -EINVAL; ++ for (i = 0; i < count; i++) { ++ for (j = 0; j < i; j++) ++ if (nodes[i] == nodes[j]) { ++ ret = -EINVAL; ++ goto put; ++ } ++ providers[i] = of_find_device_by_node(nodes[i]); ++ if (!providers[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ device_lock(&providers[i]->dev); ++ found++; ++ if (!providers[i]->dev.driver || ++ strcmp(providers[i]->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ ps[i] = platform_get_drvdata(providers[i]); ++ if (!ps[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ } ++ ++ /* PMGR domains use GENPD_FLAG_IRQ_SAFE, i.e. genpd.slock. */ ++ local_irq_save(flags); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->genpd.slock, i); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->lock, i); ++ for (i = 0; i < count; i++) { ++ states[i].genpd_status = ps[i]->genpd.status; ++ ret = regmap_read(ps[i]->regmap, ps[i]->offset, &states[i].raw); ++ if (ret) ++ goto unlock; ++ if (states[i].genpd_status != GENPD_STATE_ON || ++ FIELD_GET(APPLE_PMGR_PS_ACTUAL, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ FIELD_GET(APPLE_PMGR_PS_TARGET, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ states[i].raw & (APPLE_PMGR_RESET | APPLE_PMGR_DEV_DISABLE | ++ APPLE_PMGR_PARENT_OFF)) ++ accessible = false; ++ } ++ if (accessible) ++ read(data); ++ else ++ ret = -EHOSTDOWN; ++unlock: ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->lock); ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->genpd.slock); ++ local_irq_restore(flags); ++put: ++ while (found) { ++ found--; ++ device_unlock(&providers[found]->dev); ++ put_device(&providers[found]->dev); ++ } ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_observe); diff --git a/kernel/patches/drivers/power/0031-power-supply-apple-tristar-select-J71t-USB-wake-even.patch b/kernel/patches/drivers/power/0031-power-supply-apple-tristar-select-J71t-USB-wake-even.patch new file mode 100644 index 00000000..7a81d907 --- /dev/null +++ b/kernel/patches/drivers/power/0031-power-supply-apple-tristar-select-J71t-USB-wake-even.patch @@ -0,0 +1,56 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] power: supply: apple-tristar: select J71t USB wake event + from match data + +The charger treats one TriStar event as the plug event that may wake the +system. J71t reports plug changes on event 0x49 instead of the default +one. Take the plug event from the match data and give +apple,j71t-tristar-charger 0x49. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index a953cedcb..8a869725d 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -124,6 +124,7 @@ struct tristar_charger { + struct completion adc_done; + int irqs[TRISTAR_IRQS_MAX]; + unsigned int nirqs; ++ unsigned int plug_event; + int plug_irq; /* wakes the system, 0 without one */ + u32 status_base; + u32 control_base; +@@ -1321,13 +1322,15 @@ static int tristar_probe(struct platform_device *pdev) + if (ret <= TRISTAR_IRQ_SOURCE || ret > TRISTAR_IRQS_MAX) + return dev_err_probe(dev, -EINVAL, + "expected PMU line, ADC and source event interrupts\n"); ++ chg->plug_event = device_get_match_data(dev) ? ++ *(const unsigned int *)device_get_match_data(dev) : TRISTAR_EVENT_PLUG; + chg->nirqs = ret; + for (i = 0; i < chg->nirqs; i++) { + chg->irqs[i] = platform_get_irq(pdev, i); + if (chg->irqs[i] < 0) + return chg->irqs[i]; + if (i >= TRISTAR_IRQ_SOURCE && +- irqd_to_hwirq(irq_get_irq_data(chg->irqs[i])) == TRISTAR_EVENT_PLUG) ++ irqd_to_hwirq(irq_get_irq_data(chg->irqs[i])) == chg->plug_event) + chg->plug_irq = chg->irqs[i]; + } + +@@ -1454,7 +1457,10 @@ static int tristar_resume(struct device *dev) + + static DEFINE_SIMPLE_DEV_PM_OPS(tristar_pm_ops, tristar_suspend, tristar_resume); + ++static const unsigned int j71t_plug_event = 0x49; ++ + static const struct of_device_id tristar_of_match[] = { ++ { .compatible = "apple,j71t-tristar-charger", .data = &j71t_plug_event }, + { .compatible = "apple,tristar-charger" }, + { } + }; diff --git a/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch b/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch new file mode 100644 index 00000000..af7dc5ff --- /dev/null +++ b/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch @@ -0,0 +1,68 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: pstore: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + .../devicetree/bindings/arm/apple,pstore.yaml | 31 +++++++++++++++++++ + .../bindings/reserved-memory/ramoops.yaml | 4 +++ + 2 files changed, 35 insertions(+) + create mode 100644 Documentation/devicetree/bindings/arm/apple,pstore.yaml + +diff --git a/Documentation/devicetree/bindings/arm/apple,pstore.yaml b/Documentation/devicetree/bindings/arm/apple,pstore.yaml +new file mode 100644 +index 000000000..241dee447 +--- /dev/null ++++ b/Documentation/devicetree/bindings/arm/apple,pstore.yaml +@@ -0,0 +1,31 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/arm/apple,pstore.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: Apple retained pstore allocation ++ ++maintainers: ++ - brkosebay ++ ++description: | ++ Select early allocation on the root node. The generated ramoops node ++ carries apple,preserve-in-capture so a capture kernel leaves evidence intact. ++ Its address is carried in the kexec FDT, never reallocated on capture boots. ++ ++select: ++ properties: ++ compatible: ++ contains: ++ enum: ++ - apple,s8000 ++ - apple,s8003 ++ required: ++ - compatible ++properties: ++ apple,reserve-pstore: ++ type: boolean ++ description: Reserve an aligned 2 MiB no-map ramoops area before paging. ++ ++additionalProperties: true +diff --git a/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml b/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml +index 45cc39ecc..0b349c0ba 100644 +--- a/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml ++++ b/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml +@@ -30,6 +30,10 @@ allOf: + - $ref: reserved-memory.yaml + + properties: ++ apple,preserve-in-capture: ++ type: boolean ++ description: Preserve the previous kernel records by refusing capture-kernel probe. ++ + compatible: + const: ramoops + diff --git a/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..1bac3a8f --- /dev/null +++ b/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,180 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pstore: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + arch/arm64/include/asm/mmu.h | 1 + + arch/arm64/kernel/setup.c | 1 + + arch/arm64/mm/init.c | 88 ++++++++++++++++++++++++++++++++++++ + fs/pstore/ram.c | 8 ++++ + 4 files changed, 98 insertions(+) + +diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h +index 5e1211c54..bd22a913d 100644 +--- a/arch/arm64/include/asm/mmu.h ++++ b/arch/arm64/include/asm/mmu.h +@@ -62,6 +62,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void) + + extern void arm64_memblock_init(void); + extern void paging_init(void); ++extern void apple_soc_publish_pstore(void); + extern void bootmem_init(void); + extern void create_mapping_noalloc(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot); +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index 29c6100f0..45c23e3d9 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -332,6 +332,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) + arm64_memblock_init(); + + paging_init(); ++ apple_soc_publish_pstore(); + + acpi_table_upgrade(); + +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index fbf215ecc..3e458f58e 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -22,6 +22,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -193,6 +195,91 @@ static int __init early_mem(char *p) + } + early_param("mem", early_mem); + ++static phys_addr_t apple_soc_pstore_fdt __initdata; ++ ++/* ++ * Reserve before paging/CMA/crashkernel, without touching the linear map: ++ * paging_init() has not built it yet. Prepare the replacement FDT through a ++ * temporary early mapping and publish its linear alias only after paging. ++ * Never choose a fresh address in kdump; reuse the first kernel's region. ++ */ ++static void __init apple_soc_reserve_pstore(void) ++{ ++ void *fdt = initial_boot_params, *copy; ++ phys_addr_t base, copy_pa; ++ fdt64_t reg[2]; ++ char name[40]; ++ int parent, node, size, ret; ++ ++ if (!IS_BUILTIN(CONFIG_PSTORE_RAM) || !fdt || ++ !fdt_getprop(fdt, 0, "apple,reserve-pstore", NULL)) ++ return; ++ if (fdt_node_offset_by_compatible(fdt, -1, "ramoops") >= 0) ++ return; ++ if (is_kdump_kernel()) { ++ pr_warn("Apple pstore: no inherited region in capture kernel\n"); ++ return; ++ } ++ parent = fdt_path_offset(fdt, "/reserved-memory"); ++ if (parent < 0 || fdt_address_cells(fdt, parent) != 2 || ++ fdt_size_cells(fdt, parent) != 2) ++ return; ++ ++ /* Existing firmware, kernel and initrd reservations are already known. */ ++ base = memblock_phys_alloc(SZ_2M, SZ_2M); ++ if (!base) ++ return; ++ size = fdt_totalsize(fdt) + SZ_4K; ++ copy_pa = memblock_phys_alloc(size, SMP_CACHE_BYTES); ++ if (!copy_pa) ++ goto free_region; ++ copy = early_memremap(copy_pa, size); ++ if (!copy) ++ goto free_copy; ++ ret = fdt_open_into(fdt, copy, size); ++ if (ret) ++ goto unmap_copy; ++ parent = fdt_path_offset(copy, "/reserved-memory"); ++ snprintf(name, sizeof(name), "ramoops@%llx", (u64)base); ++ node = fdt_add_subnode(copy, parent, name); ++ if (node < 0) ++ goto unmap_copy; ++ reg[0] = cpu_to_fdt64(base); ++ reg[1] = cpu_to_fdt64(SZ_2M); ++ if (fdt_setprop(copy, node, "compatible", "ramoops", sizeof("ramoops")) || ++ fdt_setprop(copy, node, "reg", reg, sizeof(reg)) || ++ fdt_setprop(copy, node, "no-map", NULL, 0) || ++ fdt_setprop_u32(copy, node, "record-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "console-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "max-reason", 2) || ++ fdt_setprop(copy, node, "apple,preserve-in-capture", NULL, 0) || ++ fdt_add_mem_rsv(copy, base, SZ_2M)) ++ goto unmap_copy; ++ /* No cached linear alias of ramoops' write-combined persistent mapping. */ ++ if (memblock_mark_nomap(base, SZ_2M)) ++ goto unmap_copy; ++ early_memunmap(copy, size); ++ apple_soc_pstore_fdt = copy_pa; ++ pr_info("Apple pstore: reserved 2 MiB at %pa, carried in kexec FDT\n", &base); ++ return; ++ ++unmap_copy: ++ early_memunmap(copy, size); ++free_copy: ++ memblock_phys_free(copy_pa, size); ++free_region: ++ memblock_phys_free(base, SZ_2M); ++ pr_warn("Apple pstore: reservation failed\n"); ++} ++ ++/* The linear map now exists; unflattening must see the prepared FDT. */ ++void __init apple_soc_publish_pstore(void) ++{ ++ if (apple_soc_pstore_fdt) ++ early_init_dt_verify(phys_to_virt(apple_soc_pstore_fdt), ++ apple_soc_pstore_fdt); ++} ++ + void __init arm64_memblock_init(void) + { + s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); +@@ -298,6 +385,7 @@ void __init arm64_memblock_init(void) + } + + early_init_fdt_scan_reserved_mem(); ++ apple_soc_reserve_pstore(); + } + + void __init bootmem_init(void) +diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c +index 2eb0d4fa2..b26a3a0d5 100644 +--- a/fs/pstore/ram.c ++++ b/fs/pstore/ram.c +@@ -9,6 +9,7 @@ + #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + + #include ++#include + #include + #include + #include +@@ -729,6 +730,13 @@ static int ramoops_probe(struct platform_device *pdev) + phys_addr_t paddr; + int err = -EINVAL; + ++ /* Leave the original panic/console records intact across phoenix. */ ++ if (is_kdump_kernel() && of_property_read_bool(pdev->dev.of_node, ++ "apple,preserve-in-capture")) { ++ pr_info("capture kernel: preserving Apple records for the full boot\n"); ++ return -ENODEV; ++ } ++ + /* + * Only a single ramoops area allowed at a time, so fail extra + * probes. diff --git a/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch b/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch new file mode 100644 index 00000000..e1ea9505 --- /dev/null +++ b/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch @@ -0,0 +1,206 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] rtc: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + drivers/rtc/rtc-apple-pmic.c | 127 ++++++++++++++++++++++++++++++++++- + 1 file changed, 126 insertions(+), 1 deletion(-) + +diff --git a/drivers/rtc/rtc-apple-pmic.c b/drivers/rtc/rtc-apple-pmic.c +index ee56618c8..d31d4c716 100644 +--- a/drivers/rtc/rtc-apple-pmic.c ++++ b/drivers/rtc/rtc-apple-pmic.c +@@ -12,6 +12,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -24,22 +25,30 @@ + #include + + #define RTC_COUNTER_SIZE 6 ++#define RTC_ALARM_SIZE 4 ++#define RTC_CONTROL_RUN BIT(0) ++#define RTC_CONTROL_ALARM BIT(6) + + /** + * struct apple_pmic_rtc_regs - RTC Hardware registers + * @control: RTC Control + * @counter: Time Counter ++ * @alarm: Alarm seconds counter ++ * @has_alarm: Alarm registers are present + */ + + struct apple_pmic_rtc_regs { + u32 control; + u32 counter; ++ u32 alarm; ++ bool has_alarm; + }; + + /** + * struct apple_pmic_rtc - RTC driver internal structure + * @rtc: RTC device + * @regmap: regmap used to access registers ++ * @dev: device used for logging and wake setup + * @nvmem_cell: nvmem cell for offset + * @regs: hardware registers + * @base: RTC base address +@@ -183,17 +192,107 @@ static int apple_pmic_rtc_read_time(struct device *dev, struct rtc_time *tm) + return 0; + } + ++static int apple_pmic_rtc_alarm_irq_enable(struct device *dev, ++ unsigned int enabled) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ ++ /* AppleD2365PMU 0xfffffff006bbbc1c writes these exact values. */ ++ return regmap_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_RUN | ++ (enabled ? RTC_CONTROL_ALARM : 0)); ++} ++ ++static int apple_pmic_rtc_read_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ unsigned int control; ++ time64_t secs; ++ int rc; ++ ++ rc = regmap_bulk_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ rc = regmap_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, &control); ++ if (rc) ++ return rc; ++ ++ secs = (time64_t)get_unaligned_le32(buf) + ap_rtc->offset; ++ rtc_time64_to_tm(secs, &alarm->time); ++ alarm->enabled = !!(control & RTC_CONTROL_ALARM); ++ alarm->pending = 0; ++ return 0; ++} ++ ++static int apple_pmic_rtc_set_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ time64_t secs; ++ int rc; ++ ++ if (!alarm->enabled) ++ return apple_pmic_rtc_alarm_irq_enable(dev, 0); ++ ++ secs = rtc_tm_to_time64(&alarm->time) - ap_rtc->offset; ++ if (secs <= 0 || secs > U32_MAX) ++ return -ERANGE; ++ put_unaligned_le32(secs, buf); ++ ++ /* iOS programs 0x508 first, then enables the one-shot at 0x500. */ ++ rc = regmap_bulk_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ return apple_pmic_rtc_alarm_irq_enable(dev, 1); ++} ++ ++static irqreturn_t apple_pmic_rtc_alarm_irq(int irq, void *data) ++{ ++ struct apple_pmic_rtc *ap_rtc = data; ++ int rc; ++ ++ /* The PMU event read is the acknowledge; iOS then clears alarm bit 6. */ ++ rc = regmap_update_bits(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_ALARM, 0); ++ if (rc) ++ dev_warn_ratelimited(ap_rtc->dev, ++ "failed to disable fired alarm: %d\n", rc); ++ rtc_update_irq(ap_rtc->rtc, 1, RTC_IRQF | RTC_AF); ++ return IRQ_HANDLED; ++} ++ + static const struct rtc_class_ops apple_pmic_rtc_ops = { + .read_time = apple_pmic_rtc_read_time, + .set_time = apple_pmic_rtc_set_time, + }; + ++static const struct rtc_class_ops apple_pmic_rtc_alarm_ops = { ++ .read_time = apple_pmic_rtc_read_time, ++ .set_time = apple_pmic_rtc_set_time, ++ .read_alarm = apple_pmic_rtc_read_alarm, ++ .set_alarm = apple_pmic_rtc_set_alarm, ++ .alarm_irq_enable = apple_pmic_rtc_alarm_irq_enable, ++}; ++ + static const struct apple_pmic_rtc_regs amber_regs = { + .counter = 0x6 + }; + + static const struct apple_pmic_rtc_regs antigua_regs = { +- .counter = 0x2 ++ .control = 0x0, ++ .counter = 0x2, ++ .alarm = 0x8, ++ .has_alarm = true, + }; + + static const struct apple_pmic_rtc_regs calpe_regs = { +@@ -230,6 +329,7 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + struct device_node *node = pdev->dev.of_node; + const struct of_device_id *match; + struct apple_pmic_rtc *ap_rtc; ++ int irq; + int rc; + + match = of_match_node(apple_pmic_id_table, node); +@@ -264,6 +364,31 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + ap_rtc->rtc->ops = &apple_pmic_rtc_ops; + ap_rtc->rtc->range_min = INT_MIN; + ap_rtc->rtc->range_max = INT_MAX; ++ if (ap_rtc->regs->has_alarm) { ++ irq = platform_get_irq_optional(pdev, 0); ++ if (irq == -EPROBE_DEFER) ++ return irq; ++ if (irq > 0) { ++ rc = devm_request_threaded_irq(&pdev->dev, irq, NULL, ++ apple_pmic_rtc_alarm_irq, ++ IRQF_ONESHOT, ++ "apple-pmic-rtc", ap_rtc); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot request alarm IRQ\n"); ++ device_init_wakeup(&pdev->dev, true); ++ rc = devm_pm_set_wake_irq(&pdev->dev, irq); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot set alarm wake IRQ\n"); ++ ap_rtc->rtc->ops = &apple_pmic_rtc_alarm_ops; ++ dev_info(&pdev->dev, ++ "D2365 alarm at 0x%03x on PMU event IRQ %d\n", ++ ap_rtc->base + ap_rtc->regs->alarm, irq); ++ } else if (irq != -ENXIO) { ++ return irq; ++ } ++ } + + return devm_rtc_register_device(ap_rtc->rtc); + } diff --git a/kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch b/kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch new file mode 100644 index 00000000..7b738fe7 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0022-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch @@ -0,0 +1,52 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: soc-apple: select certified AOP reload through + DT + +A retained AOP must not be reset as though it were a fresh boot. +Describe the retained-reload properties: the certified-reload flag, the +reserved shutdown page and the power domains the reload observes. + +Assisted-by: LLM +--- +--- + .../bindings/soc/apple/apple,t8010-aop.yaml | 18 ++++++++++++++++++ + 1 file changed, 18 insertions(+) + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +index 32788d1c7..74586c6e0 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +@@ -18,6 +18,10 @@ description: + board-specific and is loaded by the operating system. + + properties: ++ apple,certified-reload: ++ type: boolean ++ description: Firmware supports the retained shutdown certificate for warm reload. ++ + compatible: + enum: + - apple,s8000-aop +@@ -70,6 +74,20 @@ properties: + The SoC's chip revision, filled in by the bootloader. The S8000 + firmware's boot arguments carry it. + ++ memory-region: ++ maxItems: 1 ++ description: Reserved 16 KiB shutdown transcript page for J71t warm reload. ++ ++ apple,observe-domains: ++ $ref: /schemas/types.yaml#/definitions/phandle-array ++ minItems: 5 ++ maxItems: 5 ++ description: AOP GPIO, AOP, bus interface, CPU and filter in lock order. ++ ++ apple,spi0-domain: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: Read-only SPI0 power-state provider observation. ++ + patternProperties: + "^(accel|gyro|compass|pressure)$": + type: object diff --git a/kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..a73acf3a --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0023-soc-apple-aop-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,2777 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] soc: apple: aop: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/soc/apple/aop-warm-record.h | 63 ++ + drivers/soc/apple/aop-warm-spi0.c.h | 1339 ++++++++++++++++++++++++ + drivers/soc/apple/aop-warm-spi0.h | 543 ++++++++++ + drivers/soc/apple/aop.c | 528 +++++++++- + drivers/soc/apple/rtkit.c | 50 + + include/linux/soc/apple/pmgr-observe.h | 27 + + include/linux/soc/apple/rtkit.h | 2 + + 7 files changed, 2543 insertions(+), 9 deletions(-) + create mode 100644 drivers/soc/apple/aop-warm-record.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.c.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.h + create mode 100644 include/linux/soc/apple/pmgr-observe.h + +diff --git a/drivers/soc/apple/aop-warm-record.h b/drivers/soc/apple/aop-warm-record.h +new file mode 100644 +index 000000000..237d78d17 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-record.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#include ++#include ++#define A9_HO_PAGE 0x4000U ++#define A9_HO_LOG_OFFSET 0x800U ++#define A9_SL_MAGIC 0x4c533941U /* "A9SL" */ ++#define A9_SL_VERSION 1U ++ ++enum a9_ho_step { ++ A9_HO_STEP_BEGIN, ++ A9_HO_STEP_RATES, ++ A9_HO_STEP_AFK_SENT, ++ A9_HO_STEP_AFK_ACKED, ++ A9_HO_STEP_DISABLED, ++ A9_HO_STEP_AP_SENT, ++ A9_HO_STEP_AP_ACKED, ++ A9_HO_STEP_IOP_SENT, ++ A9_HO_STEP_IOP_ACKED, ++ A9_HO_STEP_RAM_STABLE, ++ A9_HO_STEP_MAILBOX_IDLE, ++ A9_HO_STEP_HALTED, ++ A9_HO_STEP_DRAINED, ++ A9_HO_STEPS, ++}; ++ ++static const char * const a9_ho_step_names[A9_HO_STEPS] = { ++ "begin", "rates-zero", "afk-sent", "afk-acked", "endpoint-disable", ++ "ap-sent", "ap-ack", "iop-sent", "iop-ack", "ram-stable", "mailbox-idle", ++ "halt", "drain", ++}; ++ ++struct a9_sl_header { ++ u32 magic, version, capacity, length; ++ u32 lines, truncated, checksum, mode; ++ u64 begin_ns; ++ u8 writer[16]; ++ char entry[64]; ++}; ++ ++_Static_assert(sizeof(struct a9_sl_header) == 120, "shutdown log ABI"); ++ ++#define A9_SL_CAPACITY (A9_HO_PAGE - A9_HO_LOG_OFFSET - sizeof(struct a9_sl_header)) ++ ++/* FNV-1a; incremental over appended text. */ ++static inline u32 a9_sl_hash(u32 hash, const char *text, size_t length) ++{ ++ size_t i; ++ ++ for (i = 0; i < length; i++) ++ hash = (hash ^ (u8)text[i]) * 16777619U; ++ return hash; ++} ++ ++#define A9_SL_HASH_SEED 2166136261U ++ ++/* A replayable previous transcript: complete header, bounded, intact. */ ++static inline bool a9_sl_valid(const struct a9_sl_header *h, const char *text) ++{ ++ return h->magic == A9_SL_MAGIC && h->version == A9_SL_VERSION && ++ h->capacity == A9_SL_CAPACITY && h->length <= h->capacity && ++ !h->entry[sizeof(h->entry) - 1] && ++ a9_sl_hash(A9_SL_HASH_SEED, text, h->length) == h->checksum; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.c.h b/drivers/soc/apple/aop-warm-spi0.c.h +new file mode 100644 +index 000000000..e9f608abb +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.c.h +@@ -0,0 +1,1339 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Ported from a9/aop-release: wake16, release6-d254a9f790. */ ++static void a9_aop_spi0_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_REGS; i++) ++ values[i] = readl(base + a9_spi0_offsets[i]); ++} ++ ++/* ++ * AOPSPIMaster_reset (0102f2f4) under the AOP-domain PMGR guard with IRQs ++ * off and the AOP CPU halted. Its provider power transition 3 is an ++ * AOP-local call with no AP register equivalent and is not replayed. A poll ++ * timeout still restores the saved control word. ++ */ ++static void a9_aop_spi0_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, s->before); ++ s->saved = s->before[A9_SPI0_I_CONTROL]; ++ s->asserted = a9_spi0_base_word(s->saved) | A9_SPI0_RESET; ++ writel(s->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ s->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(s->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(s->saved, control); ++ s->reset_done_ns = ktime_get_ns(); ++ a9_aop_spi0_snapshot(s->base, s->after); ++ if (!ret && (s->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ s->error = ret; ++} ++ ++/* ++ * Wake8 device evidence: SCM was 0 before/after the first controller reset, ++ * then became 3 at the channel-3 stop store. There is no firmware-derived ++ * SCM status write. Replay AOPSPIMaster_reset after the teardown so the ++ * controller reset, rather than an invented SCM write, is the one variable. ++ */ ++static void a9_aop_spi0_post_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, r->before); ++ r->saved = r->before[A9_SPI0_I_CONTROL]; ++ r->asserted = a9_spi0_base_word(r->saved) | A9_SPI0_RESET; ++ writel(r->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ r->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(r->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(r->saved, control); ++ a9_aop_spi0_snapshot(s->base, r->after); ++ if (!ret && (r->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ r->error = ret; ++} ++ ++static void a9_aop_spi0_ch3_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_CH3_REGS; i++) ++ values[i] = readl(base + a9_spi0_ch3_offsets[i]); ++} ++ ++static void a9_aop_spi0_trace_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int channel, i; ++ ++ s->trace.control = readl(s->base + A9_SPI0_CONTROL); ++ s->trace.clock = readl(s->base + A9_SPI0_CLOCK); ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->trace.scm[i] = readl(s->base + a9_spi0_scm_offsets[i]); ++ a9_aop_spi0_ch3_snapshot(s->base, s->trace.ch3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) ++ for (i = 0; i < A9_SPI0_CHANNEL_REGS; i++) ++ s->trace.channel[channel][i] = ++ readl(s->base + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel + ++ a9_spi0_channel_offsets[i]); ++} ++ ++/* Warm path only, CPU halted. Log outside the PMGR guard; no SCM writes. */ ++static int a9_aop_spi0_trace(struct apple_aop *aop, const char *stage) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *c = s->trace.ch3, *m = s->trace.scm; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_trace_read, s); ++ if (ret) { ++ s->intc.failed = "failed-spi0-trace-guard"; ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=%d: guard refused\n", stage, ret); ++ return ret; ++ } ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=0 ctl=%08x clk=%08x scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ stage, s->trace.control, s->trace.clock, m[0], m[1], m[2], m[3], ++ c[0], c[1], c[2], c[3], c[4], c[5], c[6]); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ v = s->trace.channel[channel]; ++ dev_info(aop->dev, "SPI0 SCM channel stage=%s ch=%02u ctl=%08x route=%08x rate=%08x cfg=%08x interval=%08x cursor=%08x desc=%08x pending0=%u pending1=%u\n", ++ stage, channel, v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ !!(c[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(c[A9_CH3_I_8D4] & BIT(2 * channel))); ++ } ++ return 0; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s); ++ ++/* ++ * Wake10 safety gate, strengthened in wake11 to retain every selected-channel ++ * teardown invariant. A fault observed by a copy sample remains terminal even ++ * if it is transient and the final read happens to clear. ++ */ ++static int a9_aop_spi0_gate(struct apple_aop *aop, const char *stage, ++ const char *check, bool include_copy) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *m, *c; ++ int ret; ++ ++ ret = a9_aop_spi0_trace(aop, stage); ++ if (ret) ++ return ret; ++ m = s->trace.scm; ++ c = s->trace.ch3; ++ if (m[0] || m[1]) ++ s->gate_failed = "failed-spi0-scm"; ++ else if (c[A9_CH3_I_8D0] || c[A9_CH3_I_8D4]) ++ s->gate_failed = "failed-spi0-fault"; ++ else if (include_copy && s->copy_faulted) ++ s->gate_failed = "failed-spi0-copy-sample"; ++ else { ++ s->gate_failed = a9_spi0_ch3_verify(c, s->teardown_stable & BIT(3)); ++ if (!s->gate_failed) ++ s->gate_failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = s->gate_failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 %s check errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x copy_faulted=%u result=%s\n", ++ check, ret, m[0], m[1], c[A9_CH3_I_8D0], c[A9_CH3_I_8D4], ++ s->copy_faulted, s->gate_failed ?: "pass"); ++ return ret; ++} ++ ++#define A9_SPI0_COPY_CHUNK 256U ++#define A9_SPI0_COPY_SAMPLE_US 5000U ++ ++struct a9_aop_spi0_copy_probe { ++ void __iomem *base; ++ u32 scm_lo, scm_hi, fault, fault_8d4; ++}; ++ ++static void a9_aop_spi0_copy_probe_read(void *data) ++{ ++ struct a9_aop_spi0_copy_probe *p = data; ++ ++ p->scm_lo = readl(p->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(p->base + A9_SPI0_SCM_HI); ++ p->fault = readl(p->base + A9_SPI0_FAULT); ++ p->fault_8d4 = readl(p->base + A9_SPI0_FAULT_8D4); ++} ++ ++/* ++ * Read-only samples spaced by at least 5 ms across both halves of the SRAM ++ * copy. Once a nonzero state is seen it remains a terminal fact, but the copy ++ * completes so later samples and the full after-copy channel snapshot expose ++ * whether the source is level, transient or moves between channels. ++ */ ++static int a9_aop_spi0_copy_sample(struct apple_aop *aop, const char *phase, ++ size_t offset, size_t size, u64 start_ns, ++ u64 *last_us, unsigned int *sample, ++ bool force) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_copy_probe p = { .base = s->base }; ++ u64 elapsed_us = div_u64(ktime_get_boottime_ns() - start_ns, ++ NSEC_PER_USEC); ++ bool first_fault, initial = !*sample; ++ int ret; ++ ++ if (!force && *sample && elapsed_us - *last_us < A9_SPI0_COPY_SAMPLE_US) ++ return 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_copy_probe_read, &p); ++ first_fault = !s->copy_faulted && ++ (p.scm_lo || p.scm_hi || p.fault || p.fault_8d4); ++ if (!ret && first_fault) ++ s->copy_faulted = true; ++ dev_info(aop->dev, "SPI0 COPY sample=%u phase=%s elapsed_us=%llu offset=0x%zx size=0x%zx scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ *sample, phase, elapsed_us, offset, size, p.scm_lo, p.scm_hi, ++ p.fault, p.fault_8d4, ret ? "guard" : s->copy_faulted ? "fault" : "pass"); ++ *last_us = elapsed_us; ++ (*sample)++; ++ if (ret) ++ return ret; ++ if (first_fault) { ++ ret = a9_aop_spi0_trace(aop, "copy-fault"); ++ if (ret) ++ return ret; ++ } ++ /* The offset-zero sample is the last no-write gate before the copy. */ ++ return !initial || !s->copy_faulted ? 0 : -EIO; ++} ++ ++static int a9_aop_copy_verify_wake10(struct apple_aop *aop, const u8 *src, ++ size_t size) ++{ ++ u32 chunk[A9_SPI0_COPY_CHUNK / 4]; ++ u64 start_ns = ktime_get_boottime_ns(), last_us = 0; ++ unsigned int sample = 0; ++ size_t done, n; ++ int ret; ++ ++ ret = a9_aop_spi0_copy_sample(aop, "write", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __iowrite32_copy(aop->sram + done, src + done, n / 4); ++ ret = a9_aop_spi0_copy_sample(aop, "write", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_copy_sample(aop, "verify", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __ioread32_copy(chunk, aop->sram + done, n / 4); ++ if (memcmp(chunk, src + done, n)) ++ return -EIO; ++ ret = a9_aop_spi0_copy_sample(aop, "verify", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void __iomem *a9_aop_spi0_channel_base(struct a9_aop_spi0 *s, ++ unsigned int channel) ++{ ++ return s->base + A9_SPI0_CHANNEL_BASE + A9_SPI0_CHANNEL_STRIDE * channel; ++} ++ ++/* ++ * Wake15: which other channels were RUN with a counting interval when this ++ * stop was issued, and which of them advanced across the stop. Read only; it ++ * separates a stop that had a ticking peer from the stop of the last live ++ * channel in the log. It never gates anything in-kernel. ++ */ ++static void a9_aop_spi0_peers_begin(struct a9_aop_spi0 *s, unsigned int channel, ++ u32 *interval, u32 *run) ++{ ++ unsigned int i; ++ u32 ctl; ++ ++ *run = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ interval[i] = 0; ++ if (i == channel) ++ continue; ++ ctl = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ interval[i] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ if ((ctl & A9_SPI0_CH3_RUN) && interval[i]) ++ *run |= BIT(i); ++ } ++} ++ ++static void a9_aop_spi0_stop_timing(struct a9_aop_spi0 *s, ++ struct a9_aop_spi0_ch3 *c, ++ const u32 *interval) ++{ ++ unsigned int i; ++ ++ c->peers_ticking = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ if ((c->peers_run & BIT(i)) && ++ readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]) != ++ interval[i]) ++ c->peers_ticking |= BIT(i); ++ } ++ c->reset_to_stop_us = c->stop_ns > s->reset_done_ns ? ++ div_u64(c->stop_ns - s->reset_done_ns, NSEC_PER_USEC) : 0; ++} ++ ++/* ++ * Wake12/13 captured channel 7, and wake13 channel 3, stopped but never idle: ++ * for 20 complete slices the control/progress word, config, interval, cursor, ++ * SCM and both fault banks were unchanged. This predicate is intentionally ++ * narrower than "RUN clear and no progress": the channel must be one of the ++ * two device-observed periodic clients, must carry that client's route, rate ++ * and config before the stop, and every sample must be byte-identical with SCM ++ * 3/0 (every earlier non-idle stop, which followed a channel-3 stop that had ++ * latched it) or 0/0 (a first stop that latches nothing), no fault, no cursor ++ * movement and a retained nonzero interval. ++ * It only permits the callback masks, descriptor clear, config acknowledge and ++ * the established controller reset; every later SCM, fault and copy gate stays. ++ */ ++static const char *a9_aop_spi0_stable_why(struct a9_aop_spi0_ch3 *c, ++ const struct a9_aop_channel_diag_slice *diag, ++ unsigned int channel) ++{ ++ const struct a9_aop_ch3_slice *first; ++ unsigned int i; ++ u32 route; ++ ++ if (channel == A9_SPI0_PHOSPHORUS_CHANNEL) ++ route = A9_SPI0_PHOSPHORUS_ROUTE; ++ else if (channel == A9_SPI0_MPU_ASYNC_CHANNEL) ++ route = A9_SPI0_MPU_ASYNC_ROUTE; ++ else ++ return "channel"; ++ if (c->poll != -ETIMEDOUT || c->slices != A9_SPI0_CH3_SLICES) ++ return "no-timeout"; ++ if (!(c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) || ++ c->before[A9_CH3_I_ROUTE] != route || ++ c->before[A9_CH3_I_RATE] != A9_SPI0_PHOSPHORUS_RATE || ++ c->before[A9_CH3_I_C3C] != A9_SPI0_PHOSPHORUS_CFG || ++ c->before[A9_CH3_I_DESC] || (c->stop & A9_SPI0_CH3_RUN)) ++ return "before-identity"; ++ if (c->before[A9_CH3_I_8D4] || (c->before[A9_CH3_I_8D0] & ~BIT(2 * channel))) ++ return "before-fault"; ++ ++ first = &c->slice[0]; ++ if ((first->c3 & A9_SPI0_STABLE_CTL_MASK) != A9_SPI0_STABLE_CTL_STOPPED || ++ (first->c3 & (A9_SPI0_CH3_RUN | A9_SPI0_CH3_IDLE | A9_SPI0_CH3_IRQ1))) ++ return "stopped-control"; ++ if ((first->scm_lo != 0 && first->scm_lo != 3) || first->scm_hi) ++ return "scm"; ++ if (first->fault || diag[0].fault_8d4) ++ return "sample-fault"; ++ if (diag[0].cfg != A9_SPI0_PHOSPHORUS_CFG || diag[0].cursor || !diag[0].interval) ++ return "cfg-cursor-interval"; ++ ++ for (i = 1; i < A9_SPI0_CH3_SLICES; i++) { ++ const struct a9_aop_ch3_slice *p = &c->slice[i]; ++ ++ if (p->c3 != first->c3 || p->scm_lo != first->scm_lo || ++ p->scm_hi != first->scm_hi || p->fault != first->fault || ++ diag[i].cfg != diag[0].cfg || ++ diag[i].interval != diag[0].interval || ++ diag[i].cursor != diag[0].cursor || ++ diag[i].fault_8d4 != diag[0].fault_8d4) ++ return "progress"; ++ } ++ if (c->slice[A9_SPI0_CH3_SLICES - 1].ns < ++ (u64)A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US * NSEC_PER_USEC) ++ return "short"; ++ return NULL; ++} ++ ++/* ++ * AOPSPIMaster_teardownChannel (0102e4f8) for channel 3, then the fault ++ * handler's acknowledge (0102f20a), under the same guard as the reset with ++ * the CPU halted. Every store is a read-modify-write except the descriptor, ++ * which the firmware zeroes. The idle wait runs in 1 ms slices, up to twice ++ * the firmware's own 10 ms SCMto bound; each slice ends with read-only ++ * samples of C3, the SCM state and +0x8d0 (wake15: and config, interval, ++ * cursor and +0x8d4). An idle timeout stops before the masks unless the ++ * stable-stop predicate holds. The acknowledge bit self-clears; +0x8d0 bit 6 ++ * shows whether it took effect. ++ */ ++static void a9_aop_spi0_ch3_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ void __iomem *c3 = s->base + A9_SPI0_CH3; ++ void __iomem *c3c = s->base + A9_SPI0_CH3_0C; ++ void __iomem *base = a9_aop_spi0_channel_base(s, 3); ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_ch3_snapshot(s->base, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(c3) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, c3); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, 3, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(c3, value, value & A9_SPI0_CH3_IDLE, ++ 1, A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->ch3_diag[c->slices]; ++ d->cfg = readl(c3c); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->ch3_diag, 3); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(3); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(c3) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, c3); ++ c->class1 = readl(c3) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, c3); ++ writel(0, s->base + A9_SPI0_CH3_DESC); ++ c->ack = readl(c3c) | A9_SPI0_CH3_ACK; ++ writel(c->ack, c3c); ++ } ++ a9_aop_spi0_ch3_snapshot(s->base, c->after); ++ c->failed = c->poll && !c->stable ? "failed-ch3-teardown-idle" : ++ a9_spi0_ch3_verify(c->after, c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++ ++static void a9_aop_spi0_channel_snapshot(struct a9_aop_spi0 *s, ++ unsigned int channel, u32 *values) ++{ ++ void __iomem *base = a9_aop_spi0_channel_base(s, channel); ++ ++ values[A9_CH3_I_C3] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ values[A9_CH3_I_ROUTE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_ROUTE]); ++ values[A9_CH3_I_RATE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_RATE]); ++ values[A9_CH3_I_C3C] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]); ++ values[A9_CH3_I_DESC] = readl(base + A9_SPI0_CHANNEL_DESC); ++ values[A9_CH3_I_8D0] = readl(s->base + A9_SPI0_FAULT); ++ values[A9_CH3_I_8D4] = readl(s->base + A9_SPI0_FAULT_8D4); ++} ++ ++static const char *a9_spi0_channel_state_verify(u32 control, u32 desc, ++ u32 fault, u32 fault_8d4, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ u32 pending = BIT(2 * channel); ++ ++ if (control & A9_SPI0_CH3_RUN) ++ return "failed-channel-teardown-run"; ++ if (control & A9_SPI0_CH3_IRQ0) ++ return "failed-channel-teardown-class0"; ++ if (control & A9_SPI0_CH3_IRQ1) ++ return "failed-channel-teardown-class1"; ++ if (!(control & A9_SPI0_CH3_IDLE) && !stable_stop) ++ return "failed-channel-teardown-idle"; ++ if (desc) ++ return "failed-channel-teardown-desc"; ++ if ((fault | fault_8d4) & pending) ++ return "failed-channel-teardown-pending"; ++ return NULL; ++} ++ ++static const char *a9_spi0_channel_verify(const u32 *after, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ return a9_spi0_channel_state_verify(after[A9_CH3_I_C3], ++ after[A9_CH3_I_DESC], ++ after[A9_CH3_I_8D0], ++ after[A9_CH3_I_8D4], channel, ++ stable_stop); ++} ++ ++/* ++ * Wake11 applies the channel-3 firmware sequence unchanged to another slot. ++ * Wake12 only appends read-only scheduler samples to that established poll. ++ */ ++static void a9_aop_spi0_channel_locked(void *data) ++{ ++ struct a9_aop_spi0_channel_op *op = data; ++ struct a9_aop_spi0 *s = op->spi0; ++ struct a9_aop_spi0_ch3 *c = op->state; ++ void __iomem *base = a9_aop_spi0_channel_base(s, op->channel); ++ void __iomem *control = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]; ++ void __iomem *config = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]; ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(control) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, control); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, op->channel, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(control, value, ++ value & A9_SPI0_CH3_IDLE, 1, ++ A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->channel_diag[c->slices]; ++ d->cfg = readl(config); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->channel_diag, op->channel); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(op->channel); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(control) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, control); ++ c->class1 = readl(control) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, control); ++ writel(0, base + A9_SPI0_CHANNEL_DESC); ++ c->ack = readl(config) | A9_SPI0_CH3_ACK; ++ writel(c->ack, config); ++ } ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->after); ++ c->failed = c->poll && !c->stable ? "failed-channel-teardown-idle" : ++ a9_spi0_channel_verify(c->after, op->channel, ++ c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++/* Called only if ps_spi0 is fully on; reads nothing. */ ++static void a9_aop_spi0_pmgr_on(void *data) ++{ ++ *(bool *)data = true; ++} ++ ++/* AP PMGR ps_spi0, read-only: never requested, resumed or written. ++ * 0 = on, -EHOSTDOWN = off (raw still filled), -ENODEV = no node. ++ */ ++static int a9_aop_spi0_pmgr(struct apple_aop *aop, u32 *raw) ++{ ++ struct apple_pmgr_observation state = {}; ++ bool on = false; ++ int ret; ++ ++ if (!aop->spi0_pmgr) ++ return -ENODEV; ++ ret = apple_pmgr_observe(&aop->spi0_pmgr, 1, &state, ++ a9_aop_spi0_pmgr_on, &on); ++ *raw = state.raw; ++ return ret ?: on ? 0 : -EIO; ++} ++ ++/* ++ * After a reset that cleared +0x8d0 bit 6, before the image copy: CPU halted, ++ * AP notifications masked. A refusal sets ch3.failed and nothing is loaded. ++ */ ++static int a9_aop_spi0_ch3_teardown(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *b = c->before, *a = c->after; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ unsigned int i; ++ bool source_progress = false; ++ int ret; ++ ++ aop->experiment_step = "spi0-ch3-teardown"; ++ dev_info(aop->dev, "SPI0 CH3TD begin pa=%llx cpu=halted: first channel-3 store follows\n", ++ A9_SPI0_PA + A9_SPI0_CH3); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, "spi0-ch3-teardown"); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_ch3_locked, s); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH3TD errno=%d ran=0: guard refused, channel 3 not touched\n", ++ ret); ++ return ret; ++ } ++ c->ran = true; ++ dev_info(aop->dev, "SPI0 CH3TD before c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH3TD idle-sample slice=%u at_us=%llu c3=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ i, div_u64(p->ns, NSEC_PER_USEC), p->c3, p->scm_lo, ++ p->scm_hi, p->fault, !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->ch3_diag[i]; ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ i, p->c3, d->cfg, d->interval, d->cursor, p->scm_lo, ++ p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->ch3_diag[0].interval || ++ d->cursor != s->ch3_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->ch3_diag[0].interval, ++ s->ch3_diag[c->slices - 1].interval, ++ s->ch3_diag[0].cursor, ++ s->ch3_diag[c->slices - 1].cursor, source_progress, ++ c->stable ? "stable-stop-reset-only" : "refused"); ++ /* beyond_fw_bound=1: idle, but later than the firmware would wait. */ ++ dev_info(aop->dev, "SPI0 CH3TD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ c->stop, c->idle, c->poll); ++ else if (c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->poll, c->class0, c->class1, c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH3TD after c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH3TD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ /* c3c_bit5 is the pulse's readback, logged only (0 at 19:50). */ ++ dev_info(aop->dev, "SPI0 CH3TD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x c3c_bit5=%u ch3_pending=%u result=%s\n", ++ c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), ++ a[A9_CH3_I_DESC], !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), c->failed ?: ++ c->stable ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static int a9_aop_spi0_channel_teardown(struct apple_aop *aop, ++ unsigned int channel) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->channel; ++ struct a9_aop_spi0_channel_op op = { ++ .spi0 = s, ++ .state = c, ++ .channel = channel, ++ }; ++ const u32 *b, *a; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ char checkpoint[32]; ++ unsigned int i; ++ bool source_progress = false; ++ bool stable_stop; ++ int ret; ++ ++ memset(c, 0, sizeof(*c)); ++ memset(s->channel_diag, 0, sizeof(s->channel_diag)); ++ s->teardown_failed_channel = channel; ++ aop->experiment_step = "spi0-channel-teardown"; ++ snprintf(checkpoint, sizeof(checkpoint), "spi0-ch%u-teardown", channel); ++ dev_info(aop->dev, "SPI0 CH%uTD begin pa=%llx cpu=halted: first channel-%u store follows\n", ++ channel, A9_SPI0_PA + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel, channel); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, checkpoint); ++ if (ret) { ++ c->failed = "failed-channel-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_channel_locked, &op); ++ if (ret) { ++ c->failed = "failed-channel-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH%uTD errno=%d ran=0: guard refused, channel %u not touched\n", ++ channel, ret, channel); ++ return ret; ++ } ++ c->ran = true; ++ stable_stop = s->teardown_stable & BIT(channel); ++ b = c->before; ++ a = c->after; ++ dev_info(aop->dev, "SPI0 CH%uTD before ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD idle-sample slice=%u at_us=%llu ctl=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ channel, i, div_u64(p->ns, NSEC_PER_USEC), p->c3, ++ p->scm_lo, p->scm_hi, p->fault, ++ !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->channel_diag[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ channel, i, p->c3, d->cfg, d->interval, d->cursor, ++ p->scm_lo, p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->channel_diag[0].interval || ++ d->cursor != s->channel_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ channel, c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->channel_diag[0].interval, ++ s->channel_diag[c->slices - 1].interval, ++ s->channel_diag[0].cursor, ++ s->channel_diag[c->slices - 1].cursor, ++ source_progress, ++ stable_stop ? "stable-stop-reset-only" : "refused"); ++ dev_info(aop->dev, "SPI0 CH%uTD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ channel, c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ channel, c->stop, c->idle, c->poll); ++ else if (stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->poll, c->class0, c->class1, ++ c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH%uTD after ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH%uTD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ channel, c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ dev_info(aop->dev, "SPI0 CH%uTD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x cfg_bit5=%u pending0=%u pending1=%u result=%s\n", ++ channel, c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), a[A9_CH3_I_DESC], ++ !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(a[A9_CH3_I_8D4] & BIT(2 * channel)), c->failed ?: ++ stable_stop ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s) ++{ ++ const u32 *v; ++ const char *failed = NULL; ++ unsigned int channel; ++ ++ if (s->teardown_selected != s->teardown_done) ++ return "failed-channel-teardown-incomplete"; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ v = s->trace.channel[channel]; ++ failed = a9_spi0_channel_state_verify(v[A9_CHANNEL_I_CTL], ++ v[A9_CHANNEL_I_DESC], ++ s->trace.ch3[A9_CH3_I_8D0], ++ s->trace.ch3[A9_CH3_I_8D4], channel, ++ s->teardown_stable & BIT(channel)); ++ if (failed) { ++ s->teardown_failed_channel = channel; ++ return failed; ++ } ++ } ++ return NULL; ++} ++ ++/* ++ * Channel 3 remains mandatory and executes first. Wake11 additionally selects ++ * every snapshot slot with run bit 0 (RUN) set and a nonzero interval, which ++ * excludes configured-but-idle slots while selecting any periodic client still ++ * owned by the old firmware. Wake16 returns to wake11's order after wake15's ++ * channel-7-first stop failed: all ten device stops fit "channel 3 idles when ++ * stopped within ~2 ms of the reset (5/5; 0/2 at 11 and 33 ms) and channel 7 ++ * never idles (0/3, at 0.3 ms, 1.9 ms and 15 ms)". Channel 3 goes first and ++ * early; channel 7 is then the stopped-without-idle channel the stable-stop ++ * fingerprint was captured from (wake11/12). ++ */ ++static int a9_aop_spi0_teardown_running(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ s->teardown_selected = BIT(3); ++ s->teardown_done = 0; ++ s->teardown_stable = 0; ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3) ++ continue; ++ v = s->trace.channel[channel]; ++ if ((v[A9_CHANNEL_I_CTL] & A9_SPI0_CH3_RUN) && ++ v[A9_CHANNEL_I_INTERVAL]) ++ s->teardown_selected |= BIT(channel); ++ } ++ dev_info(aop->dev, "SPI0 teardown select stage=after-controller-reset selected=%04x mandatory=0008 rule=ch3-or-run-and-interval order=ch3-then-others\n", ++ s->teardown_selected); ++ ret = a9_aop_spi0_ch3_teardown(aop); ++ if (ret) { ++ s->teardown_failed_channel = 3; ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=3 errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, ret, ++ s->ch3.failed ?: "failed-ch3-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ ret = a9_aop_spi0_channel_teardown(aop, channel); ++ if (ret) { ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=%u errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, channel, ret, ++ s->channel.failed ?: "failed-channel-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(channel); ++ } ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ dev_info(aop->dev, "SPI0 teardown complete selected=%04x done=%04x stable=%04x result=pass\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable); ++ return 0; ++} ++ ++/* ++ * Second copy of the firmware controller reset, after the stop has completed. ++ * The preceding after-ch3-teardown trace is the read-only before snapshot; ++ * after-post-teardown-reset is the matching read-only after snapshot. Refuse ++ * the image copy unless SCM is zero and every channel quiet-state gate holds. ++ */ ++static int a9_aop_spi0_post_reset(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ const u32 *b = r->before, *a = r->after; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.ran || ++ s->ch3.error || s->teardown_selected != s->teardown_done || r->ran) { ++ r->failed = "failed-post-reset-state"; ++ return -EPERM; ++ } ++ aop->experiment_step = "spi0-post-teardown-reset"; ++ dev_info(aop->dev, "SPI0 POSTRST begin pa=%llx reason=ch3-stop-latched-scm cpu=halted\n", ++ A9_SPI0_PA); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-post-teardown-reset"); ++ if (ret) { ++ r->failed = "failed-post-reset-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_post_reset_locked, s); ++ r->ran = !ret; ++ r->error = ret ?: r->error; ++ dev_info(aop->dev, "SPI0 POSTRST errno=%d ran=%d before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ r->error, r->ran, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 POSTRST after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], r->saved, r->asserted, ++ r->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ if (r->error) { ++ r->failed = "failed-post-reset-controller"; ++ return r->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-post-teardown-reset"); ++ if (ret) ++ return ret; ++ if (s->trace.scm[0] || s->trace.scm[1]) ++ r->failed = "failed-post-reset-scm"; ++ else { ++ r->failed = a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3)); ++ if (!r->failed) ++ r->failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = r->failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 POSTRST check errno=%d scm_lo=%08x scm_hi=%08x ch3_pending=%u result=%s\n", ++ ret, s->trace.scm[0], s->trace.scm[1], ++ !!(s->trace.ch3[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), ++ r->failed ?: "pass"); ++ dev_info(aop->dev, "SPI0 POSTRST channels selected=%04x done=%04x stable=%04x result=%s\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable, ++ r->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * The ten INTC words of a9_intc_offsets into state[@at], read only. The ++ * INTC is the AOP wrapper block (0xc1800000 in the firmware, space 2 as ++ * SPI0's verified 0xc1100000 = 0x210100000), which this driver reads ++ * unguarded for the mailbox and CPU control; no separate PMGR domain. ++ */ ++static void a9_aop_intc_snapshot(struct apple_aop *aop, unsigned int at) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ u32 *v = n->state[at]; ++ unsigned int i; ++ ++ for (i = 0; i < A9_INTC_REGS; i++) ++ v[i] = aop_rd(aop, a9_intc_offsets[i]); ++ n->states |= BIT(at); ++ dev_info(aop->dev, "AOP INTC state at=%s reset808=%08x current81c=%08x raise900=%08x ack980=%08x mask_a00=%08x mask_a04=%08x mask_a08=%08x mask_a80=%08x g7_c020=%08x g7_c024=%08x src_masked=%u view=%s\n", ++ a9_intc_at_name(at), v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ v[7], v[8], v[9], !!(v[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(v)); ++} ++ ++/* ++ * Round 4 (ds-aop-spi0-ch3-round4.md): the old session's SPI0 global IRQ ++ * 0x10014 stays latched in the AOP interrupt controller, and fresh firmware ++ * takes it as soon as SPI lifecycle event 1 enables it, before its own SPI0 ++ * reset (SCMErr:0x3,3). With the CPU halted and before the first SPI0 ++ * access, disable that source (group 1 disable bank, 0102c9ce). The bank ++ * reads back the mask, as the firmware's own reads of it show: bit 20 must ++ * read 1 after the store. ++ */ ++static int a9_aop_intc_disable(struct apple_aop *aop) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ const u32 *b = n->state[A9_INTC_AT_PRE_DISABLE]; ++ const u32 *a = n->state[A9_INTC_AT_POST_DISABLE]; ++ int ret; ++ ++ aop->experiment_step = "intc-disable"; ++ dev_info(aop->dev, "AOP INTC disable begin irq=0x10014 pa=%llx value=%08x cpu=halted: first INTC store follows\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "intc-disable"); ++ if (ret) { ++ n->failed = "failed-intc-disable-checkpoint"; ++ return ret; ++ } ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_DISABLE); ++ ret = aop_wr(aop, A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_DISABLE); ++ if (ret) { ++ n->failed = "failed-intc-disable"; ++ } else if (!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM)) { ++ n->failed = "failed-intc-disable-readback"; ++ ret = -EIO; ++ } ++ n->disabled = !ret; ++ dev_info(aop->dev, "AOP INTC disable errno=%d before=%08x value=%08x after=%08x mask_a80=%08x src_masked=%u view=%s result=%s\n", ++ ret, b[A9_INTC_I_MASK0], A9_INTC_SPI0_SCM, a[A9_INTC_I_MASK0], ++ a[A9_INTC_I_ENABLE0], !!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(a), n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Release policy for aop_spi0_reset=1. The reload is device-proven for exactly ++ * one retained state: the previous kernel quiesced and halted the firmware ++ * (wake16, 2026-10-29; decision prev-quiesce-halted). Auto-boot still set, a ++ * previous kernel that did not halt it, or an older kernel in the chain are ++ * different hardware states that no run has reloaded with the final teardown. ++ * Refuse them before the first halt or SPI0 access so the old firmware is ++ * left exactly as found; aop_spi0_untested=1 attempts them anyway. A skip ++ * (DFU cold, bad record, nothing ran) must also find the wrapper at its reset ++ * state: a record-less AOP that something already stopped is refused rather ++ * than cold-loaded. A stale record skips as before. ++ */ ++static int a9_aop_spi0_policy(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ enum a9_spi0_reason reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, ++ &aop->spi0.prev); ++ ++ if (a9_spi0_policy_allows(reason, outbox & AOP_OUTBOX_ENABLE, spi0_untested)) ++ return 0; ++ dev_info(aop->dev, "SPI0 policy decision=refuse reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d: only prev-quiesce-halted reloads and a skip needs the wrapper at its reset state block not touched\n", ++ a9_spi0_reason_name(reason), auto_boot, outbox, ++ a9_spi0_prev_name(aop->spi0.prev.state), aop->spi0.prev.hello, ++ aop->spi0.prev.halted); ++ return -EPERM; ++} ++ ++/* ++ * After the proven halt/drain, before any SRAM write. Only firmware that ran ++ * since the last SoC reset leaves SPI0 state behind: auto-boot 1, or the ++ * previous kernel's transcript bound by +0x4020 (a9_spi0_decide). Without ++ * that evidence the block is never touched. @auto_boot and @outbox are the ++ * boot snapshot from before the stop. Nonzero means: do not load or ++ * release the AOP. ++ */ ++static int a9_aop_spi0_prereset(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *b = s->before, *a = s->after; ++ const char *failed = NULL; ++ bool run; ++ int ret; ++ ++ s->auto_boot = auto_boot; ++ s->outbox = outbox; ++ s->reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, &s->prev); ++ s->decided = true; ++ run = a9_spi0_reason_runs(s->reason); ++ dev_info(aop->dev, "SPI0 reset decision=%s reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d prev_steps=0x%x prev_carry=%d%s\n", ++ run ? "run" : "skip", a9_spi0_reason_name(s->reason), auto_boot, ++ outbox, a9_spi0_prev_name(s->prev.state), s->prev.hello, ++ s->prev.halted, s->prev.steps, s->prev.carried, ++ run ? "" : ": block not touched"); ++ if (!run) { ++ s->skipped = a9_spi0_reason_name(s->reason); ++ return 0; ++ } ++ if (!s->base || !aop->force_halted || !aop->force_drained || ++ !a9_aop_force_stopped(aop)) ++ return -EPERM; ++ ret = a9_aop_intc_disable(aop); ++ if (ret) ++ return ret; ++ aop->experiment_step = "spi0-reset"; ++ s->pmgr_before_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_before); ++ /* Reaches ramoops before the first AP access to the block. */ ++ dev_info(aop->dev, "SPI0 reset begin pa=%llx cpu=halted ps_spi0=%08x errno=%d: first AP access follows\n", ++ A9_SPI0_PA, s->pmgr_before, s->pmgr_before_errno); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-reset"); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_trace(aop, "before-controller-reset"); ++ if (ret) ++ return ret; ++ /* ++ * Wake15: from the controller reset to the last channel stop nothing may ++ * drain a console in this task. Holding the console lock makes every ++ * dev_info() below only append to the log buffer; the flushed spi0-reset ++ * checkpoint above already put the begin line into ramoops, and the lines ++ * are emitted by console_unlock() once the stops are stored. ++ */ ++ console_lock(); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_reset_locked, s); ++ s->ran = !ret; ++ s->error = ret ?: s->error; ++ s->pmgr_after_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_after); ++ dev_info(aop->dev, "SPI0 reset errno=%d ran=%d pa=%llx before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ s->error, s->ran, A9_SPI0_PA, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 reset after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u provider3=not-replayed\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], s->saved, s->asserted, ++ s->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 pmgr ps_spi0 before=%08x errno=%d after=%08x errno=%d (read-only, never powered)\n", ++ s->pmgr_before, s->pmgr_before_errno, s->pmgr_after, ++ s->pmgr_after_errno); ++ if (s->error) { ++ console_unlock(); ++ return s->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-controller-reset"); ++ if (!ret) ++ ret = a9_aop_spi0_teardown_running(aop); ++ console_unlock(); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-ch3-teardown"); ++ if (!ret) ++ failed = a9_aop_spi0_selected_verify(s); ++ if (failed) { ++ s->channel.failed = failed; ++ ret = -EIO; ++ } ++ if (!ret) ++ ret = a9_aop_spi0_post_reset(aop); ++ return ret; ++} ++ ++/* Wake10 samples both SCM words and both fault classes, not only channel 3. */ ++static void a9_aop_spi0_fault_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ ++ s->prerun_scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ s->prerun_scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ s->prerun = readl(s->base + A9_SPI0_FAULT); ++ s->prerun_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ a9_aop_spi0_ch3_snapshot(s->base, s->ch3.reassert); ++} ++ ++/* ++ * Last AP look before RUN, CPU still halted, after the image copy: bit 6 ++ * must stay clear in every one of 31 guarded samples spaced >= 5 ms apart, ++ * spanning >= 150 ms. A reasserted channel-3 bit would crash the fresh ++ * image as before, so the first set sample refuses the release; nothing is ++ * retried. Any refusal sets ch3.failed. ++ */ ++static int a9_aop_spi0_recheck(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *r = c->reassert; ++ u64 start = ktime_get_boottime_ns(); ++ unsigned int i; ++ int ret = 0; ++ ++ for (i = 0; i < A9_SPI0_WINDOW_SAMPLES; i++) { ++ if (i) ++ usleep_range(A9_SPI0_WINDOW_US, A9_SPI0_WINDOW_US + 1000); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_fault_read, s); ++ c->samples = i + 1; ++ c->elapsed_ms = div_u64(ktime_get_boottime_ns() - start, NSEC_PER_MSEC); ++ if (ret) { ++ c->failed = "failed-ch3-window-guard"; ++ break; ++ } ++ if (s->prerun_scm_lo || s->prerun_scm_hi || s->prerun || ++ s->prerun_8d4) { ++ s->reasserted = c->reasserted = true; ++ c->reassert_sample = i; ++ c->reassert_ms = c->elapsed_ms; ++ c->failed = "failed-spi0-window-reasserted"; ++ ret = -EIO; ++ dev_info(aop->dev, "SPI0 WINDOW reasserted sample=%u at_ms=%llu scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ i, c->reassert_ms, s->prerun_scm_lo, s->prerun_scm_hi, ++ s->prerun, s->prerun_8d4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ dev_info(aop->dev, "SPI0 WINDOW channel3 c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ r[0], r[1], r[2], r[3], r[4], r[5], r[6]); ++ break; ++ } ++ } ++ /* usleep_range() cannot return early; this only trips on a bad clock. */ ++ if (!ret && c->elapsed_ms < A9_SPI0_WINDOW_MS) { ++ c->failed = "failed-ch3-window-short"; ++ ret = -EIO; ++ } ++ c->window_done = !ret; ++ dev_info(aop->dev, "SPI0 CH3TD window errno=%d samples=%u/%u elapsed_ms=%llu min_ms=%u result=%s\n", ++ ret, c->samples, A9_SPI0_WINDOW_SAMPLES, c->elapsed_ms, ++ A9_SPI0_WINDOW_MS, ret ? c->failed : "pass"); ++ dev_info(aop->dev, "SPI0 pre-run errno=%d 8d0=%08x ch3_pending=%u cpu=halted\n", ++ ret, s->prerun, !!(s->prerun & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 WINDOW state errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ ret, s->prerun_scm_lo, s->prerun_scm_hi, s->prerun, ++ s->prerun_8d4, ret ? c->failed : "pass"); ++ if (ret && c->reasserted) ++ a9_aop_spi0_trace(aop, "window-fault"); ++ return ret; ++} ++ ++/* SCM state and the channel-3 interval pair, read-only, into scm[scm_at]. */ ++static void a9_aop_spi0_scm_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->intc.scm[s->intc.scm_at][i] = readl(s->base + a9_spi0_scm_offsets[i]); ++} ++ ++/* ++ * After the clean pre-RUN window, CPU still halted: the firmware's group-1 ++ * pre-handler acknowledge for 0x10014 (0102c6de..0102c6f2) discards the ++ * old-session latch now that its SPI0 cause is quiet. Fresh firmware ++ * re-enables the source in event 1. The INTC words, the SCM state and the ++ * C3 interval pair are logged around it and never gated: with 0x10014 ++ * masked, no readable word shows whether it is still pending. ++ */ ++static int a9_aop_intc_ack(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ const u32 *p = n->scm[0], *q = n->scm[1]; ++ const u32 *b = n->state[A9_INTC_AT_PRE_ACK]; ++ const u32 *a = n->state[A9_INTC_AT_POST_ACK]; ++ int ret; ++ ++ aop->experiment_step = "intc-ack"; ++ n->scm_at = 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) { ++ n->failed = "failed-intc-ack-guard"; ++ dev_info(aop->dev, "AOP INTC pre-ack errno=%d: guard refused, no acknowledge\n", ++ ret); ++ return ret; ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_ACK); ++ dev_info(aop->dev, "AOP INTC pre-ack scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x cpu=halted\n", ++ p[0], p[1], p[2], p[3], b[A9_INTC_I_CURRENT]); ++ ret = aop_wr(aop, A9_INTC_ACK, A9_INTC_SPI0_SCM); ++ /* Also the posted-write barrier. */ ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_ACK); ++ n->acked = !ret; ++ dev_info(aop->dev, "AOP INTC ack errno=%d before=%08x value=%08x after=%08x\n", ++ ret, b[A9_INTC_I_ACK], A9_INTC_SPI0_SCM, a[A9_INTC_I_ACK]); ++ if (ret) { ++ n->failed = "failed-intc-ack"; ++ return ret; ++ } ++ n->scm_at = 1; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) ++ n->failed = "failed-intc-post-guard"; ++ dev_info(aop->dev, "AOP INTC post-ack errno=%d scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x result=%s\n", ++ ret, q[0], q[1], q[2], q[3], a[A9_INTC_I_CURRENT], n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Wake7 still took 0x10014 before SPI event 5 despite the targeted ack. ++ * Replay INTC lifecycle event 3 (0102c592/598/59c/5a2): 1/read/0/read at ++ * +0x808. aop_wr includes a posted-write read after each store. This resets ++ * all AOP-private interrupt state; it does not write the SCM status or MPU. ++ * No assumption that SCM must become zero. Only the final reset readback ++ * and the established channel-3 quiet conditions gate RUN. ++ */ ++static int a9_aop_intc_reset(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.window_done || ++ !n->acked || n->reset_done || ++ (aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) || readl(aop->control)) { ++ n->failed = "failed-intc-reset-state"; ++ return -EPERM; ++ } ++ ret = a9_aop_spi0_trace(aop, "pre-intc-reset"); ++ if (ret) ++ return ret; ++ aop->experiment_step = "intc-reset"; ++ dev_info(aop->dev, "AOP INTC reset begin pa=%llx sequence=1/read/0/read cpu=halted\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_RESET); ++ ret = aop_wr(aop, A9_INTC_RESET, 1); ++ n->reset_asserted = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-assert"; ++ return ret; ++ } ++ ret = aop_wr(aop, A9_INTC_RESET, 0); ++ n->reset_after = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-deassert"; ++ return ret; ++ } ++ if (n->reset_after) { ++ n->failed = "failed-intc-reset-readback"; ++ ret = -EIO; ++ } else { ++ ret = a9_aop_spi0_trace(aop, "post-intc-reset"); ++ if (!ret && a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3))) { ++ n->failed = "failed-intc-reset-ch3"; ++ ret = -EIO; ++ } else if (!ret && a9_aop_spi0_selected_verify(s)) { ++ n->failed = "failed-intc-reset-channel"; ++ ret = -EIO; ++ } ++ } ++ n->reset_done = !ret; ++ dev_info(aop->dev, "AOP INTC reset errno=%d asserted=%08x after=%08x result=%s\n", ++ ret, n->reset_asserted, n->reset_after, n->failed ?: "pass"); ++ return ret; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.h b/drivers/soc/apple/aop-warm-spi0.h +new file mode 100644 +index 000000000..2470165ce +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.h +@@ -0,0 +1,543 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * J71t AOP reload/wake experiments, both opt-in and default off. Constants ++ * and evidence types only; the MMIO lives in apple-a9-aop-active.c. ++ * Evidence: docs/periph/ds-aop-reload-crash-and-wake.md. ++ * ++ * aop_spi0_reset=1: every halt + SRAM reload of a live AOP crashed in ++ * AOPSPIMaster_channelFaultIRQ (0102f252, DFAR 0x4c): the AOP-private SPI0 ++ * controller kept channel 3's +0x8d0 bit 6 while the fresh firmware's ++ * channel table was still empty. With the CPU halted, the AP replays the ++ * firmware's own controller reset AOPSPIMaster_reset (0102f2f4), then the ++ * firmware's own channel-3 teardown, and samples bit 6 for >= 150 ms after ++ * the image copy before it releases RUN once. Round 4: the old session's ++ * SPI0 global IRQ 0x10014 stays latched in the AOP interrupt controller ++ * (SCMErr:0x3,3 at the accel STARTEP), so the AP disables that source before ++ * the reset and acknowledges it after the clean window, just before RUN. ++ * Wake7: the channel-3 idle wait gets the firmware's own 10 ms bound twice ++ * over, sampled per 1 ms slice, and the disable is judged by the INTC mask ++ * readback, which the firmware itself reads. ++ * Wake8: after the targeted ack, replay INTC event 3 (+0x808: 1/read/0/read). ++ * This resets retained interrupt delivery, not the read-only SCM state. ++ * Wake9: wake8 device evidence showed the channel-3 stop itself changes SCM ++ * from 0 to 3. Replay the same SPI controller reset once more after teardown; ++ * require the post-reset SCM state to be zero and the teardown state to hold. ++ * Wake10: the second reset clears SCM, but a halted CPU then saw SCM 3 and ++ * +0x8d0 bit 14 (channel 7) during the firmware copy. Snapshot all 16 channel ++ * slots, sample global SCM/fault state across the copy, and gate the copy and ++ * final RUN on a completely quiet readback. ++ * Wake11: that snapshot proved channels 3 and 7 were both running. Keep the ++ * channel-3 sequence unchanged, then apply the same firmware teardown to every ++ * other run-bit + nonzero-interval channel before the post-teardown reset. ++ * Wake12: channel 7 is the phosphorus/BMP284 async client. Its firmware has no ++ * owner-specific stop path, so do not invent another store when the generic ++ * stop times out. Record the remaining scheduler state read-only and refuse. ++ * Wake13/14: every device stop of channel 3 that idled (wake8..wake12) came ++ * 1.5-2.2 ms after the controller reset; the two that did not (wake13/14) came ++ * 11/33 ms after it (pr_flush in the checkpoint), with class-0 state identical to ++ * wake12's passes. Wake15 closed that window (no pr_flush between the reset and ++ * the stops, console lock held), recorded the reset-to-stop latency and the peers ++ * per stop, and accepted one identity-pinned stopped-without-idle channel (3 or ++ * 7) as wake13 did for channel 7. Wake15's channel-7-first order then showed ++ * channel 7 never idles even at 0.3 ms (it is 0/3 over wake11/12/15), with ++ * nothing ticking. Wake16 therefore returns to channel 3 first (early, the only ++ * stop that ever idled) and then channel 7, which takes the stable-stop path ++ * (SCM 0/0 or 3/0, constant); every stop logs why the fingerprint did or did not ++ * hold (stable_why=). Wake14's class-0 prelude stays removed. ++ * ++ * aop_wake_wrapper=1: before 0x220, the AppleA7IOP status-3 resume wrapper ++ * writes (b4b854..b4b8b4). No CPU RUN, no auto-boot, no NMI. ++ */ ++#ifndef _A9_AOP_RELOAD_H ++#define _A9_AOP_RELOAD_H ++ ++#include ++#include ++#include "aop-warm-record.h" ++ ++/* Firmware device spi0: IOP 0x10100000 (space 2, 0x1000) via /arm-io ranges. */ ++#define A9_SPI0_PA 0x210100000ULL ++#define A9_SPI0_SIZE 0x1000U ++#define A9_SPI0_WRAPPER_PA 0x210800000ULL /* the only layout this applies to */ ++#define A9_SPI0_CONTROL 0x000U ++#define A9_SPI0_CLOCK 0x008U /* 0102f2aa programs 0x01b71b00 here */ ++#define A9_SPI0_FAULT 0x8d0U /* bit 2n = channel n, read by 0102f20a */ ++#define A9_SPI0_FAULT_8D4 0x8d4U ++#define A9_SPI0_CHANNEL_BASE 0x100U ++#define A9_SPI0_CHANNEL_STRIDE 0x080U ++#define A9_SPI0_CHANNELS 16U ++#define A9_SPI0_CHANNEL_REGS 7U ++#define A9_SPI0_CHANNEL_DESC 0x018U ++#define A9_SPI0_CH3 0x280U /* base + 0x100 + 0x80 * 3 */ ++#define A9_SPI0_CH3_0C 0x28cU ++#define A9_SPI0_RESET 0x100U /* control bit 8 */ ++#define A9_SPI0_ENABLE 0x001U /* control bit 0 */ ++#define A9_SPI0_CH3_PENDING 0x040U ++/* The firmware spins without bound; the AP gives each poll 1 ms. */ ++#define A9_SPI0_POLL_US 1000U ++#define A9_SPI0_DELAY_US 100U /* 0102b3f8(100, 0), twice */ ++#define A9_SPI0_REGS 6 ++ ++static const u32 a9_spi0_offsets[A9_SPI0_REGS] = { ++ A9_SPI0_CONTROL, A9_SPI0_CLOCK, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++ A9_SPI0_CH3, A9_SPI0_CH3_0C, ++}; ++ ++/* Snapshot indices into a9_spi0_offsets. */ ++enum { A9_SPI0_I_CONTROL, A9_SPI0_I_CLOCK, A9_SPI0_I_FAULT }; ++ ++/* ++ * Channel-3 teardown after the reset (ds-aop-spi0-ch3-round3.md section 5). ++ * The reset leaves C3 = 0x00010447: idle but still enabled, async-armed and ++ * with the class-0 fault IRQ on, so a level source re-latched +0x8d0 bit 6 ++ * within ~105 ms with the CPU halted. AOPSPIMaster_teardownChannel ++ * (0102e4f8) stops the channel and masks both IRQ classes; the fault ++ * handler (0102f20a) then acknowledges with C3C bit 5. ++ */ ++#define A9_SPI0_CH3_ROUTE 0x284U ++#define A9_SPI0_CH3_RATE 0x288U ++#define A9_SPI0_CH3_DESC 0x298U /* channel +0x18, zeroed on teardown */ ++#define A9_SPI0_CH3_RUN 0x001U /* C3 bit 0, channel run/enable */ ++#define A9_SPI0_CH3_IDLE 0x002U /* C3 bit 1, polled after the stop */ ++#define A9_SPI0_CH3_IRQ0 0x040U /* C3 bit 6, +0x8d0 class-0/fault enable */ ++#define A9_SPI0_CH3_IRQ1 0x100U /* C3 bit 8, +0x8d4 class-1 enable */ ++#define A9_SPI0_CHANNEL_PROGRESS_SHIFT 24U /* hardware-updated, not an IRQ class */ ++#define A9_SPI0_CH3_STOP 0x411U /* clear run bit 0 and bits 4, 10 */ ++#define A9_SPI0_CH3_CLASS0 0x450U /* mask class 0: bits 4, 6, 10 */ ++#define A9_SPI0_CH3_CLASS1 0x510U /* mask class 1: bits 4, 8, 10 */ ++#define A9_SPI0_CH3_ACK 0x020U /* C3C bit 5, ORed by 0102f20a; self-clearing */ ++#define A9_SPI0_PHOSPHORUS_CHANNEL 7U ++#define A9_SPI0_PHOSPHORUS_CTL_MASK 0x00ffffffU ++#define A9_SPI0_PHOSPHORUS_CTL_RUN 0x00010445U ++#define A9_SPI0_PHOSPHORUS_CTL_STOPPED 0x00010444U ++#define A9_SPI0_PHOSPHORUS_ROUTE 0x00000040U ++#define A9_SPI0_PHOSPHORUS_RATE 0x019000c8U ++#define A9_SPI0_PHOSPHORUS_CFG 0x00200009U ++/* Channel 3 is the MPU6900 async FIFO client (route 0x10, same rate/config). */ ++#define A9_SPI0_MPU_ASYNC_CHANNEL 3U ++#define A9_SPI0_MPU_ASYNC_ROUTE 0x00000010U ++/* Stopped control word, any top byte, class-0 enable optional (0x40). */ ++#define A9_SPI0_STABLE_CTL_STOPPED 0x00010404U ++#define A9_SPI0_STABLE_CTL_MASK 0x00ffffbfU ++/* Peer sampling and the "late stop" mark are reported, never gated, in-kernel. */ ++#define A9_SPI0_LATE_STOP_US 3000U ++/* ++ * AOPSPIMaster_setChannelEnabled(false) (0102ef7c) stores C3 & ~0x411 only ++ * if bit 0 was set, then spins on bit 1 until master +0x30, and panics ++ * "SCMto:%u" (0x010398da) past it. Event 1 sets +0x30 to 0x2710 us when it ++ * is 0 (0102e6fa, as in the live-cold object), through timer +0x18 = ++ * us * Hz / 1000000 (0102ff3c): a 10 ms bound. The AP polls in 1 ms slices, ++ * up to twice that, sampling C3, the SCM state and +0x8d0 after each slice. ++ */ ++#define A9_SPI0_CH3_IDLE_FW_US 10000U ++#define A9_SPI0_CH3_SLICE_US 1000U ++#define A9_SPI0_CH3_SLICES 20U /* 20 ms */ ++/* At least 150 ms (the 16:27 re-latch took ~105 ms) of samples before RUN. */ ++#define A9_SPI0_WINDOW_SAMPLES 31U ++#define A9_SPI0_WINDOW_US 5000U /* 30 sleeps >= 5 ms each */ ++#define A9_SPI0_WINDOW_MS 150U /* required first-to-last sample span */ ++#define A9_SPI0_CH3_REGS 7 ++ ++static const u32 a9_spi0_ch3_offsets[A9_SPI0_CH3_REGS] = { ++ A9_SPI0_CH3, A9_SPI0_CH3_ROUTE, A9_SPI0_CH3_RATE, A9_SPI0_CH3_0C, ++ A9_SPI0_CH3_DESC, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++}; ++ ++/* Every firmware-visible channel slot, read only in wake10 and selected in wake11. */ ++static const u32 a9_spi0_channel_offsets[A9_SPI0_CHANNEL_REGS] = { ++ 0x00U, 0x04U, 0x08U, 0x0cU, 0x10U, 0x14U, 0x18U, ++}; ++ ++enum { ++ A9_CHANNEL_I_CTL, A9_CHANNEL_I_ROUTE, A9_CHANNEL_I_RATE, A9_CHANNEL_I_CFG, ++ A9_CHANNEL_I_INTERVAL, A9_CHANNEL_I_CURSOR, A9_CHANNEL_I_DESC, ++}; ++ ++/* Snapshot indices into a9_spi0_ch3_offsets. */ ++enum { ++ A9_CH3_I_C3, A9_CH3_I_ROUTE, A9_CH3_I_RATE, A9_CH3_I_C3C, A9_CH3_I_DESC, ++ A9_CH3_I_8D0, A9_CH3_I_8D4, ++}; ++ ++/* One idle-poll slice: C3 as the slice ended, then SCM state and +0x8d0. */ ++struct a9_aop_ch3_slice { ++ u64 ns; /* since the stop store */ ++ u32 c3, scm_lo, scm_hi, fault; ++}; ++ ++/* Wake12 read-only scheduler evidence for an additional-channel stop poll. */ ++struct a9_aop_channel_diag_slice { ++ u32 cfg, interval, cursor, fault_8d4; ++}; ++ ++struct a9_aop_spi0_ch3 { ++ u32 before[A9_SPI0_CH3_REGS], after[A9_SPI0_CH3_REGS]; ++ u32 stop, idle, class0, class1, ack; /* words written / polled */ ++ int poll; /* idle-bit poll result */ ++ bool run_before; /* C3 bit 0 before the stop */ ++ unsigned int slices; /* idle-poll slices used */ ++ struct a9_aop_ch3_slice slice[A9_SPI0_CH3_SLICES]; ++ u64 idle_us; /* stop store to the idle read */ ++ int error; ++ bool ran; /* the guarded teardown executed */ ++ const char *failed; /* distinct refusal reason, or NULL */ ++ /* Wake15: stop latency after the controller reset and the peers' state. */ ++ u64 stop_ns, reset_to_stop_us; ++ u32 peers_run, peers_ticking; /* masks of the other 15 channels */ ++ bool stable; /* accepted stopped-without-idle identity */ ++ const char *stable_why; /* first failed fingerprint condition, NULL if stable */ ++ /* Pre-RUN window after the image copy; a refusal also sets failed. */ ++ unsigned int samples; ++ u64 elapsed_ms; ++ bool window_done, reasserted; ++ unsigned int reassert_sample; ++ u64 reassert_ms; ++ u32 reassert[A9_SPI0_CH3_REGS]; ++}; ++ ++struct a9_aop_spi0; ++ ++struct a9_aop_spi0_channel_op { ++ struct a9_aop_spi0 *spi0; ++ struct a9_aop_spi0_ch3 *state; ++ unsigned int channel; ++}; ++ ++/* ++ * AOP interrupt controller, wrapper offsets (ds-aop-spi0-ch3-round4.md ++ * section 3). IRQ 0x10014 is group 1, source 0x14: bank 0, bit 20. The ++ * group-1 disable/enable banks are write-one commands whose reads return ++ * the mask, 1 = disabled: the sleep command saves +0xa00..+0xa08 (0102c5ce) ++ * before masking all (0102c5da), the restore writes saved to +0xa00 and ++ * ~saved to +0xa80 (0102c9e6/0102c9f4), and aop_interrupt_write_enabled ++ * reads the target bank for the previous state (0102c9c0). So the disable ++ * is judged by bit 20 of the +0xa00 readback (wake6: ffffffff before the ++ * store, every source already masked by the quiesce). +0x81c only reports ++ * an unmasked pending source: the sleep path masks all, writes group 7 ++ * +0xc020/+0xc024 = 2 and spins until +0x81c reads 0 (0102c654), so ++ * all-ones there means the AP is not reading the INTC. +0x808 ends 0 after ++ * the event-3 pulse. +0x900 is the group-1 raise (0102c7c6) and +0x980 its ++ * pre-handler clear (0102c6f2); the firmware never reads them, so their ++ * readbacks are logged only. No register shows a masked source pending. ++ */ ++#define A9_INTC_RESET 0x808U /* event-3 pulse 1 then 0 (0102c592) */ ++#define A9_INTC_CURRENT 0x81cU /* current unmasked source */ ++#define A9_INTC_RAISE 0x900U /* group 1 raise, 0102c7c6 */ ++#define A9_INTC_ACK 0x980U /* group 1 pre-handler ack/clear, 0102c6f2 */ ++#define A9_INTC_DISABLE 0xa00U /* group 1 disable, store at 0102c9ce */ ++#define A9_INTC_ENABLE 0xa80U /* group 1 enable; reads the same mask */ ++#define A9_INTC_G7_DISABLE 0xc020U /* group 7 disable, 2 after sleep */ ++#define A9_INTC_SPI0_SCM 0x00100000U /* IRQ 0x10014 */ ++#define A9_INTC_ALL_ONES 0xffffffffU ++#define A9_INTC_REGS 10 ++ ++static const u32 a9_intc_offsets[A9_INTC_REGS] = { ++ A9_INTC_RESET, A9_INTC_CURRENT, A9_INTC_RAISE, A9_INTC_ACK, ++ A9_INTC_DISABLE, A9_INTC_DISABLE + 4, A9_INTC_DISABLE + 8, ++ A9_INTC_ENABLE, A9_INTC_G7_DISABLE, A9_INTC_G7_DISABLE + 4, ++}; ++ ++/* Snapshot indices into a9_intc_offsets. */ ++enum { ++ A9_INTC_I_RESET, A9_INTC_I_CURRENT, A9_INTC_I_RAISE, A9_INTC_I_ACK, ++ A9_INTC_I_MASK0, A9_INTC_I_MASK1, A9_INTC_I_MASK2, A9_INTC_I_ENABLE0, ++ A9_INTC_I_G7_0, A9_INTC_I_G7_1, ++}; ++ ++/* When the AP snapshots the INTC words. */ ++enum { ++ A9_INTC_AT_PRE_DISABLE, A9_INTC_AT_POST_DISABLE, A9_INTC_AT_PRE_ACK, ++ A9_INTC_AT_POST_ACK, A9_INTC_AT_N, ++}; ++ ++static inline const char *a9_intc_at_name(unsigned int at) ++{ ++ static const char * const names[] = { ++ "pre-disable", "post-disable", "pre-ack", "post-ack", ++ }; ++ ++ return at < A9_INTC_AT_N ? names[at] : "unknown"; ++} ++ ++/* "all-ones" if +0x81c reads ffffffff, which a live INTC cannot report. */ ++static inline const char *a9_intc_view(const u32 *state) ++{ ++ return state[A9_INTC_I_CURRENT] == A9_INTC_ALL_ONES ? "all-ones" : "intc"; ++} ++/* ++ * Read only, never written: the SCM state (also per idle-poll slice) and, ++ * around the acknowledge, the C3 interval pair. ++ */ ++#define A9_SPI0_SCM_LO 0x00cU ++#define A9_SPI0_SCM_HI 0x010U ++#define A9_SPI0_CH3_IVAL 0x290U ++#define A9_SPI0_CH3_CURSOR 0x294U ++#define A9_SPI0_SCM_REGS 4 ++ ++static const u32 a9_spi0_scm_offsets[A9_SPI0_SCM_REGS] = { ++ A9_SPI0_SCM_LO, A9_SPI0_SCM_HI, A9_SPI0_CH3_IVAL, A9_SPI0_CH3_CURSOR, ++}; ++ ++struct a9_aop_intc { ++ u32 state[A9_INTC_AT_N][A9_INTC_REGS]; /* around each store */ ++ unsigned int states; /* bit n: state[n] was read */ ++ u32 scm[2][A9_SPI0_SCM_REGS]; /* before / after the ack */ ++ unsigned int scm_at; /* scm[] row the guarded read fills */ ++ bool disabled, acked; ++ bool reset_done; ++ u32 reset_asserted, reset_after; ++ const char *failed; /* distinct refusal reason, or NULL */ ++}; ++ ++/* Read-only milestones. Wake10 gates the global SCM/fault words, not channels. */ ++struct a9_aop_spi0_trace { ++ u32 control, clock, scm[A9_SPI0_SCM_REGS], ch3[A9_SPI0_CH3_REGS]; ++ u32 channel[A9_SPI0_CHANNELS][A9_SPI0_CHANNEL_REGS]; ++}; ++ ++/* Wake9's second, post-channel-stop replay of AOPSPIMaster_reset. */ ++struct a9_aop_spi0_post_reset { ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved, asserted, polled; ++ int error; ++ bool ran; ++ const char *failed; ++}; ++ ++/* ++ * First failed readback after the teardown, or NULL if all hold. C3C bit 5 ++ * is a pulse: at 19:50 the write of 0x00200029 read back 0x00200009 while ++ * +0x8d0 bit 6 went 1 -> 0, so the acknowledge is judged by bit 6 alone and ++ * the C3C readback is only logged. ++ */ ++static inline const char *a9_spi0_ch3_verify(const u32 *after, bool stable) ++{ ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) ++ return "failed-ch3-teardown-run"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0) ++ return "failed-ch3-teardown-class0"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1) ++ return "failed-ch3-teardown-class1"; ++ if (!(after[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE) && !stable) ++ return "failed-ch3-teardown-idle"; ++ if (after[A9_CH3_I_DESC]) ++ return "failed-ch3-teardown-desc"; ++ if (after[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING) ++ return "failed-ch3-teardown-pending"; ++ return NULL; ++} ++ ++/* ++ * Has firmware run since the last SoC reset? auto-boot alone cannot say: ++ * the outgoing aop_quiesce=1 halt clears it, exactly like a DFU cold start. ++ * The previous kernel's reserved-RAM shutdown transcript says whether its ++ * firmware ran. That DRAM can outlive a DFU; wrapper +0x4020 outbox enable ++ * cannot: set after reset (12/12 DFU cold snapshots), cleared by the first ++ * store of every Linux stop. No SPI0 access is needed to decide. ++ */ ++enum a9_spi0_prev_state { ++ A9_SPI0_PREV_MISSING, /* no transcript magic, or none reserved */ ++ A9_SPI0_PREV_INVALID, /* magic present, header/checksum rejected */ ++ A9_SPI0_PREV_VALID, ++}; ++ ++struct a9_spi0_prev { ++ enum a9_spi0_prev_state state; ++ bool hello; /* "SHUTDOWN begin ... hello=1": its firmware ran */ ++ bool halted; /* "QUIESCE step=complete ... halted=1" + steps bit */ ++ bool carried; /* "SPI0-CARRY retained=1": ran earlier in the chain */ ++ u32 steps; /* "QUIESCE step=complete ... steps=0x..." */ ++}; ++ ++enum a9_spi0_reason { ++ /* Run the reset. */ ++ A9_SPI0_LIVE_AUTO_BOOT, /* auto-boot 1 before the stop */ ++ A9_SPI0_PREV_HALTED, /* previous kernel quiesced and halted it */ ++ A9_SPI0_PREV_LIVE, /* previous kernel's firmware ran, no halt */ ++ A9_SPI0_PREV_CARRIED, /* an earlier kernel in the chain ran it */ ++ /* Skip: the block is not touched. */ ++ A9_SPI0_FRESH, /* no transcript, auto-boot 0: DFU cold */ ++ A9_SPI0_PREV_BAD, /* transcript present but corrupt */ ++ A9_SPI0_PREV_NO_RUN, /* transcript: nothing ran in the chain */ ++ A9_SPI0_STALE, /* run evidence, wrapper at reset state */ ++}; ++ ++static inline const char *a9_spi0_reason_name(enum a9_spi0_reason reason) ++{ ++ static const char * const names[] = { ++ "live-auto-boot", "prev-quiesce-halted", "prev-live", "prev-carried", ++ "fresh-instance", "prev-transcript-invalid", "prev-no-run", ++ "stale-transcript", ++ }; ++ ++ return reason <= A9_SPI0_STALE ? names[reason] : "unknown"; ++} ++ ++static inline const char *a9_spi0_prev_name(enum a9_spi0_prev_state state) ++{ ++ return state == A9_SPI0_PREV_VALID ? "valid" : ++ state == A9_SPI0_PREV_INVALID ? "invalid" : "missing"; ++} ++ ++static inline bool a9_spi0_reason_runs(enum a9_spi0_reason reason) ++{ ++ return reason <= A9_SPI0_PREV_CARRIED; ++} ++ ++/* ++ * Release policy: a reason that would run the reload is allowed by default ++ * only for prev-quiesce-halted, the one state device-proven end to end. A ++ * skip (fresh, bad record, nothing ran) is allowed only while the wrapper is ++ * still at its reset state; stale records skip as before. Untested lifts both. ++ */ ++static inline bool a9_spi0_policy_allows(enum a9_spi0_reason reason, bool outbox_enabled, ++ bool untested) ++{ ++ if (untested || reason == A9_SPI0_PREV_HALTED || reason == A9_SPI0_STALE) ++ return true; ++ if (a9_spi0_reason_runs(reason)) ++ return false; ++ /* ++ * A skip means "nothing ran since the SoC reset", which only holds if the ++ * wrapper is still at its reset state (+0x4020 outbox enable set, 12/12 DFU ++ * cold starts). Without it some earlier kernel stopped the firmware and its ++ * record is lost: a cold load would land on a halted, never-torn-down AOP. ++ */ ++ return outbox_enabled; ++} ++ ++/* Token search inside one transcript line, which holds no NUL. */ ++static inline bool a9_spi0_line_has(const char *line, size_t length, ++ const char *token, size_t n, ++ const char **at) ++{ ++ size_t i; ++ ++ for (i = 0; n <= length && i <= length - n; i++) { ++ if (!memcmp(line + i, token, n)) { ++ *at = line + i + n; ++ return true; ++ } ++ } ++ return false; ++} ++ ++#define a9_spi0_has(line, length, token, at) \ ++ a9_spi0_line_has(line, length, token, sizeof(token) - 1, at) ++ ++static inline bool a9_spi0_line_starts(const char *line, size_t length, ++ const char *prefix, size_t n) ++{ ++ return length >= n && !memcmp(line, prefix, n); ++} ++ ++#define a9_spi0_starts(line, length, prefix) \ ++ a9_spi0_line_starts(line, length, prefix, sizeof(prefix) - 1) ++ ++static inline u32 a9_spi0_hex(const char *p, const char *end) ++{ ++ u32 value = 0; ++ ++ for (; p < end; p++) { ++ if (*p >= '0' && *p <= '9') ++ value = value << 4 | (u32)(*p - '0'); ++ else if (*p >= 'a' && *p <= 'f') ++ value = value << 4 | (u32)(*p - 'a' + 10); ++ else ++ break; ++ } ++ return value; ++} ++ ++/* Our own checksummed transcript; only whole-line prefixes count. */ ++static inline void a9_spi0_prev_parse(struct a9_spi0_prev *p, ++ const char *text, size_t length) ++{ ++ const char *end = text + length, *line, *next, *at; ++ size_t n; ++ ++ for (line = text; line < end; line = next < end ? next + 1 : end) { ++ next = memchr(line, '\n', end - line); ++ if (!next) ++ next = end; ++ n = next - line; ++ if (a9_spi0_starts(line, n, "SHUTDOWN begin ") && ++ a9_spi0_has(line, n, " hello=1 ", &at)) ++ p->hello = true; ++ if (a9_spi0_starts(line, n, "QUIESCE step=complete ")) { ++ if (a9_spi0_has(line, n, " steps=0x", &at)) ++ p->steps = a9_spi0_hex(at, next); ++ p->halted = a9_spi0_has(line, n, " halted=1 ", &at) && ++ (p->steps & (1U << A9_HO_STEP_HALTED)); ++ } ++ if (a9_spi0_starts(line, n, "SPI0-CARRY retained=1 ")) ++ p->carried = true; ++ } ++} ++ ++/* ++ * Skips unless there is positive evidence: running the reset on a fresh ++ * block would be the first AP access to SPI0 with nothing retained to ++ * clear (and ps_spi0 may gate it), while a skip on retained state only ++ * repeats the known, non-hanging accel STARTEP crash. ++ */ ++static inline enum a9_spi0_reason a9_spi0_decide(bool auto_boot, bool outbox_enabled, ++ const struct a9_spi0_prev *p) ++{ ++ if (auto_boot) ++ return A9_SPI0_LIVE_AUTO_BOOT; ++ if (p->state == A9_SPI0_PREV_MISSING) ++ return A9_SPI0_FRESH; ++ if (p->state != A9_SPI0_PREV_VALID) ++ return A9_SPI0_PREV_BAD; ++ if (!p->hello && !p->carried) ++ return A9_SPI0_PREV_NO_RUN; ++ if (outbox_enabled) ++ return A9_SPI0_STALE; ++ if (p->hello) ++ return p->halted ? A9_SPI0_PREV_HALTED : A9_SPI0_PREV_LIVE; ++ return A9_SPI0_PREV_CARRIED; ++} ++ ++struct a9_aop_spi0 { ++ void __iomem *base; /* mapped only with aop_spi0_reset=1 */ ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved; /* control word before the reset */ ++ u32 asserted; /* saved & ~0x101 | 0x100 */ ++ u32 polled; /* control word when the polls ended */ ++ u32 prerun; /* +0x8d0 just before RUN */ ++ u32 prerun_8d4, prerun_scm_lo, prerun_scm_hi; ++ u32 pmgr_before, pmgr_after; /* AP PMGR ps_spi0 raw, read-only */ ++ int pmgr_before_errno, pmgr_after_errno; ++ int error; ++ bool ran, reasserted, copy_faulted; ++ const char *gate_failed; ++ const char *skipped; ++ struct a9_spi0_prev prev; /* parsed at probe from the transcript */ ++ u32 auto_boot, outbox; /* boot snapshot, before the stop */ ++ enum a9_spi0_reason reason; ++ bool decided; ++ struct a9_aop_spi0_ch3 ch3; /* after a clean reset only */ ++ /* Wake11 reuses one bounded scratch record for each additional channel. */ ++ struct a9_aop_spi0_ch3 channel; ++ u32 teardown_selected, teardown_done; ++ unsigned int teardown_failed_channel; ++ struct a9_aop_spi0_post_reset post_reset; /* after the stop latches SCM */ ++ struct a9_aop_intc intc; /* IRQ 0x10014 around the reset */ ++ struct a9_aop_spi0_trace trace; /* latest read-only milestone */ ++ /* Appended so every pre-wake12 field, including ch3, keeps its layout. */ ++ struct a9_aop_channel_diag_slice channel_diag[A9_SPI0_CH3_SLICES]; ++ /* Wake13: exact phosphorus stopped-without-idle exception, by channel. */ ++ u32 teardown_stable; ++ /* Wake15: channel 3 shares the sampled diagnostics; reset completion time. */ ++ struct a9_aop_channel_diag_slice ch3_diag[A9_SPI0_CH3_SLICES]; ++ u64 reset_done_ns; ++}; ++ ++/* 0102f2f4: bits 8 and 0 clear; the reset write adds bit 8. */ ++static inline u32 a9_spi0_base_word(u32 saved) ++{ ++ return saved & ~(A9_SPI0_RESET | A9_SPI0_ENABLE); ++} ++ ++#endif +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index 5c23317d1..542e9b62e 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -12,10 +12,17 @@ + * child platform device; its driver receives the service's reports through + * apple_aop_listen() and sends requests with apple_aop_call(). + * +- * The firmware cannot be restarted once stopped, so the AOP starts once per +- * boot and the driver cannot be unbound or unloaded after that. ++ * J71t can reload after a certified quiesce/halt/drain across kexec. Other ++ * retained states are refused. Unbind/unload after starting remains forbidden. + */ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include "aop-warm-spi0.h" + #include + #include + #include +@@ -95,7 +102,9 @@ + #define AFK_OPC_GET_BUF_ACK 0xa1 + #define AFK_OPC_SEND 0xa2 + #define AFK_OPC_START 0xa3 +-#define AFK_OPC_SHUTDOWN_ACK 0xc1 ++#define AFK_OPC_SHUTDOWN 0xa4 ++#define AFK_OPC_SHUTDOWN_ACK 0x87 ++#define AFK_OPC_HIBERNATE_ACK 0xc3 + + #define AFK_BLOCK_STEP 0x40 + #define AFK_MAX_BUFFER SZ_1M +@@ -183,6 +192,8 @@ struct apple_aop_ep { + + struct mutex lock; /* tx ring, seq and calls */ + u16 seq; ++ bool initialized; ++ struct completion stopped; + struct apple_aop_call *calls[APPLE_AOP_MAX_CALLS]; + }; + +@@ -240,6 +251,16 @@ struct apple_aop { + struct mutex children_lock; /* children, removing */ + struct list_head children; + bool removing; ++ u64 accel_reports; ++ int shutdown_error; ++ /* J71t only; all other boards retain their original start/stop path. */ ++ bool warm, hello, iop_crashed, force_halted, force_drained; ++ void __iomem *warm_wrapper; ++ struct device_node *domains[5], *spi0_pmgr; ++ struct a9_aop_spi0 spi0; ++ const char *experiment_step; ++ struct a9_sl_header *shutdown_log; ++ char *shutdown_text; + }; + + static bool apple_aop_is_started(struct apple_aop *aop) +@@ -261,6 +282,409 @@ static bool apple_aop_halt(struct apple_aop *aop) + APPLE_AOP_HALT_TIMEOUT_US); + } + ++static bool a9_aop_ramoops_ready(void) ++{ ++ struct console *con; ++ bool ready = false; ++ int cookie; ++ ++ cookie = console_srcu_read_lock(); ++ for_each_console_srcu(con) { ++ if (!strcmp(con->name, "ramoops") && ++ (console_srcu_read_flags(con) & (CON_ENABLED | CON_SUSPENDED)) == ++ CON_ENABLED) { ++ ready = true; ++ break; ++ } ++ } ++ console_srcu_read_unlock(cookie); ++ return ready; ++} ++ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* J71t warm reload integration. SPI0 machinery is the release6 wake16 code. */ ++#define AOP_DOMAINS 5 ++#define AOP_CPU_CONTROL 0x44 ++#define AOP_CPU_START BIT(4) ++#define AOP_OUTBOX_CONTROL 0x4020 ++#define AOP_INBOX_CONTROL 0x4008 ++#define AOP_OUTBOX_ENABLE BIT(0) ++#define AOP_MBOX_FAULTS GENMASK(19, 18) ++#define cold_defer 1 ++#define spi0_untested false ++ ++static u32 aop_rd(struct apple_aop *aop, u32 offset) ++{ ++ return readl(aop->warm_wrapper + offset); ++} ++ ++static bool a9_aop_force_stopped(struct apple_aop *aop) ++{ ++ return !(aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) && ++ !readl(aop->control) && ++ !(aop_rd(aop, AOP_OUTBOX_CONTROL) & (AOP_OUTBOX_ENABLE | AOP_MBOX_FAULTS)) && ++ !(aop_rd(aop, AOP_INBOX_CONTROL) & AOP_MBOX_FAULTS); ++} ++ ++struct aop_warm_write { void __iomem *target; u32 value; }; ++static void aop_warm_store(void *data) ++{ ++ struct aop_warm_write *w = data; ++ ++ writel(w->value, w->target); ++ readl(w->target); ++} ++ ++static int aop_wr(struct apple_aop *aop, u32 offset, u32 value) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct aop_warm_write w = { aop->warm_wrapper + offset, value }; ++ ++ if ((aop_rd(aop, AOP_INBOX_CONTROL) | aop_rd(aop, AOP_OUTBOX_CONTROL)) & ++ AOP_MBOX_FAULTS) ++ return -EIO; ++ dev_info(aop->dev, "COLD-W step=%s bank=wrapper offset=%04x before=%08x value=%08x\n", ++ aop->experiment_step ?: "warm-stop", offset, readl(w.target), value); ++ { ++ int ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, aop_warm_store, &w); ++ ++ dev_info(aop->dev, "COLD-R offset=%04x after=%08x\n", offset, readl(w.target)); ++ return ret; ++ } ++} ++ ++static int a9_aop_cold_checkpoint_noflush(struct apple_aop *aop, const char *step) ++{ ++ if (!a9_aop_ramoops_ready()) ++ return -ENODEV; ++ dev_info(aop->dev, "COLD checkpoint=%s flush=skipped\n", step); ++ return 0; ++} ++ ++static int a9_aop_cold_checkpoint(struct apple_aop *aop, const char *step) ++{ ++ int ret = a9_aop_cold_checkpoint_noflush(aop, step); ++ ++ if (ret) ++ return ret; ++ if (!pr_flush(1000, false)) ++ return -ETIMEDOUT; ++ mb(); ++ return 0; ++} ++ ++#include "aop-warm-spi0.c.h" ++static int apple_aop_send(struct apple_aop_ep *ep, u8 opcode, u64 payload); ++ ++/* No cmdline override for untested states: default is the release6 policy. */ ++static atomic_t aop_warm_attempted = ATOMIC_INIT(0); ++ ++static int apple_aop_warm_stop(struct apple_aop *aop) ++{ ++ unsigned int fifo, i; ++ u32 offset, val; ++ int ret; ++ ++ ret = aop_wr(aop, 0x4020, aop_rd(aop, 0x4020) & ~BIT(0)); ++ if (!ret) ++ ret = aop_wr(aop, 0x4000, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0xc00, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0x44, aop_rd(aop, 0x44) & ~BIT(4)); ++ if (ret) ++ return ret; ++ writel(0, aop->control); ++ msleep(10); ++ if (!a9_aop_force_stopped(aop)) ++ return -EIO; ++ aop->force_halted = true; ++ for (fifo = 0; fifo < 2; fifo++) { ++ offset = fifo ? 0x4020 : 0x4008; ++ for (i = 0; i < 64; i++) { ++ val = aop_rd(aop, offset); ++ if (!a9_aop_force_stopped(aop) || (val & AOP_MBOX_FAULTS)) ++ return -EIO; ++ if (val & BIT(17)) ++ break; ++ readq(aop->warm_wrapper + (fifo ? 0x4038 : 0x4018)); ++ } ++ if (!(aop_rd(aop, offset) & BIT(17))) ++ return -ETIMEDOUT; ++ } ++ aop->force_drained = true; ++ return 0; ++} ++ ++static void apple_aop_warm_put_node(void *node) ++{ ++ of_node_put(node); ++} ++ ++static void apple_aop_warm_unmap(void *p) ++{ ++ memunmap(p); ++} ++ ++static int apple_aop_warm_prepare(struct apple_aop *aop) ++{ ++ struct device *dev = aop->dev; ++ struct device_node *np = dev->of_node, *mem; ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct a9_sl_header header; ++ struct resource res; ++ char *text; ++ u32 auto_boot, outbox; ++ unsigned int i; ++ int ret; ++ ++ if (!of_property_read_bool(aop->dev->of_node, "apple,certified-reload") || !aop->hw->sram_window) ++ return 0; ++ /* Missing the new DT is not permission to run a retained instance. */ ++ mem = of_parse_phandle(np, "memory-region", 0); ++ if (!mem) ++ return dev_err_probe(dev, -EINVAL, "warm reload requires the matching J71t DT\n"); ++ ret = of_address_to_resource(mem, 0, &res); ++ of_node_put(mem); ++ if (ret || resource_size(&res) != A9_HO_PAGE || res.start != 0x8583fc000ULL || ++ aop->akf_base != A9_SPI0_WRAPPER_PA) ++ return -EINVAL; ++ aop->shutdown_log = memremap(res.start, A9_HO_PAGE, MEMREMAP_WB); ++ if (!aop->shutdown_log) ++ return -ENOMEM; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_unmap, aop->shutdown_log); ++ if (ret) ++ return ret; ++ /* Keep release6's transcript location and validation ABI. */ ++ aop->shutdown_log = (void *)aop->shutdown_log + A9_HO_LOG_OFFSET; ++ aop->shutdown_text = (char *)(aop->shutdown_log + 1); ++ memcpy(&header, aop->shutdown_log, sizeof(header)); ++ text = devm_kmemdup(dev, aop->shutdown_text, A9_SL_CAPACITY, GFP_KERNEL); ++ if (!text) ++ return -ENOMEM; ++ if (header.magic == A9_SL_MAGIC) { ++ aop->spi0.prev.state = A9_SPI0_PREV_INVALID; ++ if (a9_sl_valid(&header, text)) { ++ aop->spi0.prev.state = A9_SPI0_PREV_VALID; ++ a9_spi0_prev_parse(&aop->spi0.prev, text, header.length); ++ for (i = 0; i < header.length;) { ++ char *next = memchr(text + i, '\n', header.length - i); ++ unsigned int n = next ? next - text - i : header.length - i; ++ ++ dev_info(dev, "AOP-PREV %.*s\n", n, text + i); ++ i += n + 1; ++ } ++ } ++ } ++ aop->warm_wrapper = devm_ioremap(dev, aop->akf_base, 0x1c000); ++ if (!aop->warm_wrapper) ++ return -ENOMEM; ++ for (i = 0; i < AOP_DOMAINS; i++) { ++ aop->domains[i] = of_parse_phandle(np, "apple,observe-domains", i); ++ if (!aop->domains[i]) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->domains[i]); ++ if (ret) ++ return ret; ++ } ++ aop->spi0_pmgr = of_parse_phandle(np, "apple,spi0-domain", 0); ++ if (!aop->spi0_pmgr) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->spi0_pmgr); ++ if (ret) ++ return ret; ++ /* Check all domains before reading the wrapper; no power transitions. */ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, a9_aop_spi0_pmgr_on, &(bool){false}); ++ if (ret) ++ return dev_err_probe(dev, ret, "AOP domain guard refused\n"); ++ auto_boot = readl(aop->control); ++ outbox = aop_rd(aop, AOP_OUTBOX_CONTROL); ++ ret = a9_aop_spi0_policy(aop, auto_boot, outbox); ++ if (ret) ++ return ret; /* Before first halt, mask, SRAM or SPI0 write. */ ++ /* Orchard's old halt leaves +0x4020 enabled. On a known kexec, ++ * that alone cannot certify the cold wrapper baseline. Release6's ++ * wrapper discriminator was tied to its own outgoing stop routine. ++ */ ++ if (of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev) != A9_SPI0_PREV_HALTED) { ++ dev_info(dev, "SPI0 policy decision=refuse reason=uncertified-kexec auto_boot=%u out4020=%08x: block not touched\n", ++ auto_boot, outbox); ++ return -EPERM; ++ } ++ aop->warm = true; ++ aop->spi0.reason = a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev); ++ if (!a9_spi0_reason_runs(aop->spi0.reason)) { ++ /* Original cold path: no halt, drain, reset, or SPI0 access. */ ++ dev_info(dev, "SPI0 reset decision=skip reason=%s: block not touched\n", ++ a9_spi0_reason_name(aop->spi0.reason)); ++ return 0; ++ } ++ if (atomic_xchg(&aop_warm_attempted, 1)) ++ return dev_err_probe(dev, -EPERM, "refused-second-experiment-attempt\n"); ++ if (!a9_aop_ramoops_ready()) ++ return dev_err_probe(dev, -ENODEV, "warm reload requires ramoops console\n"); ++ aop->spi0.base = devm_ioremap(dev, A9_SPI0_PA, A9_SPI0_SIZE); ++ if (!aop->spi0.base) ++ return -ENOMEM; ++ /* Consume certification before the first control write. Reloading the ++ * module after a failure must not reuse it as a second attempt. ++ */ ++ aop->shutdown_log->magic = 0; ++ wmb(); ++ ret = apple_aop_warm_stop(aop); ++ if (!ret) ++ ret = a9_aop_spi0_prereset(aop, auto_boot, outbox); ++ if (ret) ++ dev_err(dev, "warm reload refused errno=%d; no copy/RUN, recover with DFU\n", ret); ++ return ret; ++} ++ ++static int apple_aop_warm_release(struct apple_aop *aop) ++{ ++ int ret; ++ ++ ret = a9_aop_spi0_recheck(aop); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-window"); ++ if (!ret) ++ ret = a9_aop_intc_ack(aop); ++ if (!ret) ++ ret = a9_aop_intc_reset(aop); ++ if (!ret) ++ ret = a9_aop_cold_checkpoint(aop, "before-run"); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "pre-run-final", "FINAL", true); ++ return ret; ++} ++ ++static void apple_aop_warm_hash(struct apple_aop *aop, u8 *digest) ++{ ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ if (aop->ext) ++ sha256_update(&ctx, aop->ext, aop->ext_size); ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) ++ if (aop->eps[i].buf) ++ sha256_update(&ctx, aop->eps[i].buf, aop->eps[i].buf_size); ++ sha256_final(&ctx, digest); ++} ++ ++static __printf(2, 3) void apple_aop_warm_log(struct apple_aop *aop, const char *fmt, ...) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ va_list args; ++ char line[512]; ++ size_t n; ++ ++ va_start(args, fmt); ++ n = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ dev_info(aop->dev, "%s", line); ++ if (n > h->capacity - h->length) { ++ h->truncated++; ++ return; ++ } ++ memcpy(aop->shutdown_text + h->length, line, n); ++ wmb(); ++ h->checksum = a9_sl_hash(h->checksum, line, n); ++ h->length += n; ++ h->lines++; ++ wmb(); ++} ++ ++static void apple_aop_warm_shutdown(struct apple_aop *aop) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ unsigned long deadline = jiffies + msecs_to_jiffies(10000), start; ++ unsigned int i, acked = 0; ++ int ret = -EIO; ++ ++ memset(h, 0, sizeof(*h)); ++ h->version = A9_SL_VERSION; ++ h->capacity = A9_SL_CAPACITY; ++ h->checksum = A9_SL_HASH_SEED; ++ h->mode = 1; ++ h->begin_ns = ktime_get_boottime_ns(); ++ wmb(); ++ h->magic = A9_SL_MAGIC; ++ apple_aop_warm_log(aop, "SHUTDOWN begin mode=quiesce hello=%d phase=stopping iop_crashed=%d\n", ++ aop->hello, aop->iop_crashed); ++ if (!aop->hello || aop->iop_crashed || aop->shutdown_error) ++ goto done; ++ /* Rates-zero has stopped reports. Stop one AFK at a time, max 2s ++ * per endpoint and 10s in total, including send time. Never force halt ++ * after a failed ACK. Keep every grant allocated for the entire hook. ++ */ ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ unsigned long limit; ++ ++ if (!ep->initialized) ++ continue; ++ if (time_after_eq(jiffies, deadline)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ reinit_completion(&ep->stopped); ++ start = jiffies; ++ ret = apple_aop_send(ep, AFK_OPC_SHUTDOWN, 0); ++ if (ret) ++ goto done; ++ limit = time_before(jiffies, deadline) ? min_t(unsigned long, ++ deadline - jiffies, msecs_to_jiffies(2000)) : 0; ++ if (!limit || !wait_for_completion_timeout(&ep->stopped, limit)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ acked++; ++ apple_aop_warm_log(aop, "QUIESCE step=afk-ack ep=0x%x state=stopped ms=%u\n", ++ ep->index, jiffies_to_msecs(jiffies - start)); ++ } ++ /* release6: complete every AFK stop before disabling endpoint routing. */ ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; ++ } ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ ++ if (!ep->initialized) ++ continue; ++ /* MGMT STARTEP bit 0 disables routing, no separate stop ACK. */ ++ ret = apple_rtkit_send_message(aop->rtk, 0, ++ (5ULL << 52) | ((u64)ep->index << 32) | 1, NULL, false); ++ if (ret) ++ goto done; ++ } ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; /* Never certify an incompletely initialized instance. */ ++ } ++ ret = apple_rtkit_shutdown_preserve(aop->rtk); ++ if (ret) ++ goto done; ++ msleep(20); ++ apple_aop_warm_hash(aop, before); ++ msleep(20); ++ apple_aop_warm_hash(aop, after); ++ ret = memcmp(before, after, sizeof(before)) ? -EBUSY : 0; ++ apple_aop_warm_log(aop, "QUIESCE step=ram-stable window_ms=20 status=%d\n", ret); ++ if (ret || aop->iop_crashed) ++ goto done; ++ ret = apple_aop_warm_stop(aop); ++done: ++ apple_aop_warm_log(aop, "QUIESCE step=complete status=%d quiesced=%d halted=%d drained=%d steps=0x%x afk_acks=%u iop_crashed=%d\n", ++ ret, !ret, aop->force_halted, aop->force_drained, ++ !ret ? 0x1bff : 0, acked, aop->iop_crashed); ++ if (ret) ++ dev_err(aop->dev, "quiesce failed; next AOP must refuse, recover with DFU\n"); ++} ++ + /* Firmware loading */ + + struct apple_aop_bootarg { +@@ -356,6 +780,7 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + __le64 ecid_le = cpu_to_le64(ecid), ext_le, wrapper_le; + __le32 target_le, chip_le, revision_le; + unsigned int count = 0; ++ size_t offset; + u8 *image; + int ret; + +@@ -424,10 +849,21 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + } + + ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, count); +- if (ret) ++ if (ret) { + dev_err(aop->dev, "unexpected boot arguments in %s: %d\n", main_name, ret); +- else ++ } else if (aop->spi0.ran) { ++ /* Warm reload: copy and verify the image with SPI0 gated around it */ ++ ret = a9_aop_spi0_gate(aop, "pre-copy-final", "PRECOPY", false); ++ if (!ret) ++ ret = a9_aop_copy_verify_wake10(aop, image, main_fw->size); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "after-copy", "AFTERCOPY", true); ++ if (!ret) ++ for (offset = main_fw->size; offset + 4 <= aop->sram_size; offset += 4) ++ writel_relaxed(0, aop->sram + offset); ++ } else { + apple_aop_write_sram(aop, image, main_fw->size); ++ } + + kfree_sensitive(image); + memzero_explicit(prng, sizeof(prng)); +@@ -571,6 +1007,8 @@ static void apple_aop_report(struct apple_aop_ep *ep, u32 channel, u16 subtype, + struct apple_aop_listener *listener; + bool handled = false; + ++ if (ep->index == 0x21 && subtype == 0xc2) ++ aop->accel_reports++; + mutex_lock(&aop->listeners_lock); + list_for_each_entry(listener, &aop->listeners, list) { + if (listener->endpoint == ep->index && listener->channel == channel) { +@@ -727,6 +1165,7 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + + switch (FIELD_GET(AFK_MSG_OPCODE, msg)) { + case AFK_OPC_INIT: ++ ep->initialized = true; + apple_aop_send(ep, AFK_OPC_INIT_ACK, 0); + break; + case AFK_OPC_GET_BUF: +@@ -747,7 +1186,10 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + break; + case AFK_OPC_INIT_UNK: + case AFK_OPC_START_ACK: ++ break; + case AFK_OPC_SHUTDOWN_ACK: ++ case AFK_OPC_HIBERNATE_ACK: ++ complete(&ep->stopped); + break; + default: + dev_dbg(aop->dev, "endpoint 0x%02x: message 0x%llx\n", endpoint, msg); +@@ -758,7 +1200,8 @@ static void apple_aop_crashed(void *cookie, const void *crashlog, size_t crashlo + { + struct apple_aop *aop = cookie; + +- dev_err(aop->dev, "firmware crashed\n"); ++ aop->iop_crashed = true; ++ dev_err(aop->dev, "firmware crashed iop_crashed=1\n"); + } + + static const struct apple_rtkit_ops apple_aop_rtkit_ops = { +@@ -863,6 +1306,8 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + + if (!ep) + return -EINVAL; ++ if (service->aop->warm && READ_ONCE(service->aop->iop_crashed)) ++ return -EIO; + init_completion(&call.done); + + mutex_lock(&ep->lock); +@@ -997,7 +1442,11 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + return ret; + +- if (apple_aop_is_started(aop)) ++ ret = apple_aop_warm_prepare(aop); ++ if (ret) ++ return ret; ++ ++ if (!aop->spi0.ran && apple_aop_is_started(aop)) + return dev_err_probe(dev, -EBUSY, + "already started; the AOP starts once per boot\n"); + +@@ -1023,6 +1472,7 @@ static int apple_aop_probe(struct platform_device *pdev) + aop->eps[i].aop = aop; + aop->eps[i].index = AFK_ENDPOINT_START + i; + mutex_init(&aop->eps[i].lock); ++ init_completion(&aop->eps[i].stopped); + } + + ret = apple_aop_load(aop, ecid); +@@ -1050,10 +1500,24 @@ static int apple_aop_probe(struct platform_device *pdev) + prop = of_find_property(np, "apple,ecid", NULL); + if (prop) + of_remove_property(np, prop); +- writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, +- aop->akf + APPLE_AOP_CPU_CONTROL); ++ if (aop->spi0.ran) { ++ ret = apple_aop_warm_release(aop); ++ if (ret) ++ goto err_halt; ++ } ++ if (aop->spi0.ran) { ++ aop->experiment_step = "warm-run"; ++ ret = aop_wr(aop, APPLE_AOP_CPU_CONTROL, ++ readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN); ++ if (ret) ++ goto err_halt; ++ } else { ++ writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, ++ aop->akf + APPLE_AOP_CPU_CONTROL); ++ } + writel(1, aop->control + APPLE_AOP_CONTROL_AUTO_BOOT); + ++ dev_info(dev, "AOP released warm=%d\n", aop->spi0.ran); + /* + * The firmware was just loaded and starts from scratch: wait for its + * hello and its own power state report, as for any freshly started +@@ -1073,6 +1537,8 @@ static int apple_aop_probe(struct platform_device *pdev) + dev_warn(dev, "failed to start endpoint 0x%02x: %d\n", i, ret); + } + ++ aop->hello = true; ++ dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); + return 0; + +@@ -1102,12 +1568,41 @@ static void apple_aop_shutdown(struct platform_device *pdev) + mutex_lock(&aop->children_lock); + aop->removing = true; + list_for_each_entry_safe(child, tmp, &aop->children, list) { ++ ++ if (aop->warm) { ++ const struct apple_aop_service *service = dev_get_platdata(&child->pdev->dev); ++ static const u32 keys[] = { 0x16, 0x5c, 0 }; ++ ++ /* release6 sets the three accelerometer rates only. */ ++ if (strcmp(service->name, "accel")) ++ continue; ++ __le32 zero = 0; ++ unsigned int i; ++ int ret; ++ ++ for (i = 0; i < ARRAY_SIZE(keys); i++) { ++ ret = apple_aop_set_property(service, keys[i], &zero, sizeof(zero)); ++ if (ret) { ++ aop->shutdown_error = ret; ++ break; ++ } ++ } ++ } ++ /* Shutdown retains listeners, children and DMA grants, as release6 does. ++ * Unregister is remove-time work and can wait in IIO/driver core. ++ */ ++ if (aop->warm) ++ continue; + platform_device_unregister(child->pdev); + list_del(&child->list); + kfree(child); + } + mutex_unlock(&aop->children_lock); + ++ if (aop->warm) { ++ apple_aop_warm_shutdown(aop); ++ return; ++ } + if (!apple_aop_halt(aop)) + dev_err(aop->dev, "failed to halt\n"); + } +@@ -1129,11 +1624,26 @@ static const struct of_device_id apple_aop_of_match[] = { + }; + MODULE_DEVICE_TABLE(of, apple_aop_of_match); + ++static ssize_t warm_state_show(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ struct apple_aop *aop = dev_get_drvdata(dev); ++ ++ if (!aop) ++ return sysfs_emit(buf, "owned=0\n"); ++ return sysfs_emit(buf, "owned=%u reload=%u iop_crashed=%u accel_reports=%llu\n", ++ aop->hello, aop->spi0.ran, READ_ONCE(aop->iop_crashed), ++ READ_ONCE(aop->accel_reports)); ++} ++static DEVICE_ATTR_RO(warm_state); ++static struct attribute *apple_aop_attrs[] = { &dev_attr_warm_state.attr, NULL }; ++ATTRIBUTE_GROUPS(apple_aop); ++ + static struct platform_driver apple_aop_driver = { + .driver = { + .name = "apple-aop", + .of_match_table = apple_aop_of_match, + .suppress_bind_attrs = true, ++ .dev_groups = apple_aop_groups, + }, + .probe = apple_aop_probe, + .shutdown = apple_aop_shutdown, +diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c +index 725ab3fa7..d7928621a 100644 +--- a/drivers/soc/apple/rtkit.c ++++ b/drivers/soc/apple/rtkit.c +@@ -4,6 +4,8 @@ + * Copyright (C) The Asahi Linux Contributors + */ + ++#include ++#include + #include "rtkit-internal.h" + + enum { +@@ -867,6 +869,54 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) + } + EXPORT_SYMBOL_GPL(apple_rtkit_boot); + ++/* Unlike shutdown(), retain all grants until the caller halts the CPU. ++ * Reinit frees DMA buffers and is unsafe before that point on S8000. ++ */ ++static void apple_rtkit_grants_hash(struct apple_rtkit *rtk, u8 *digest) ++{ ++ struct apple_rtkit_shmem *buffers[] = { ++ &rtk->ioreport_buffer, &rtk->crashlog_buffer, ++ &rtk->oslog_buffer, &rtk->syslog_buffer, ++ }; ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ for (i = 0; i < ARRAY_SIZE(buffers); i++) ++ if (buffers[i]->buffer) ++ sha256_update(&ctx, buffers[i]->buffer, buffers[i]->size); ++ sha256_final(&ctx, digest); ++} ++ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk) ++{ ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ if (rtk->crashed) ++ return -EIO; ++ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_QUIESCED); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=ap-ack state=0x10\n"); ++ ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=iop-ack state=0x201\n"); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, before); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, after); ++ if (rtk->crashed || memcmp(before, after, sizeof(before))) ++ return -EBUSY; ++ /* No more callback can race the caller's halt/drain. Do not reinit. */ ++ apple_mbox_stop(rtk->mbox); ++ flush_workqueue(rtk->wq); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_rtkit_shutdown_preserve); ++ + int apple_rtkit_shutdown(struct apple_rtkit *rtk) + { + int ret; +diff --git a/include/linux/soc/apple/pmgr-observe.h b/include/linux/soc/apple/pmgr-observe.h +new file mode 100644 +index 000000000..477b7fa22 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-observe.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++#define _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++ ++#include ++ ++struct device_node; ++ ++struct apple_pmgr_observation { ++ u32 raw; ++ int genpd_status; ++}; ++ ++/* ++ * Diagnostic only. Nodes must be in child-before-parent lock order, with no ++ * duplicates (maximum five). Takes no power/runtime-PM reference. read() must ++ * not sleep or acquire genpd locks; it runs with IRQs disabled and all domain ++ * locks held, only if every genpd and hardware actual/target state is ON and ++ * reset/disable/parent-off are clear. Hardware auto-PM remains untouched and is ++ * recorded in raw; locks exclude software transitions, not autonomous gating. ++ * -EHOSTDOWN means read() was skipped. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data); ++ ++#endif +diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h +index 1f9ebb618..c8e8df255 100644 +--- a/include/linux/soc/apple/rtkit.h ++++ b/include/linux/soc/apple/rtkit.h +@@ -124,6 +124,8 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); + * Shutdown the co-processor + */ + int apple_rtkit_shutdown(struct apple_rtkit *rtk); ++/* AP 0x10 / IOP 0x201, stable system RAM, stop RX; retain every grant. */ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk); + + /* + * Stop the mailbox of a shut-down co-processor, so that its power domain can diff --git a/kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch b/kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch new file mode 100644 index 00000000..8a4e6613 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0024-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch @@ -0,0 +1,80 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] soc: apple: aop: refuse pre-sync sleep during firmware boot + +A suspend while the AOP firmware is still starting stops it part way +through its boot, and the sensors behind it do not come back. Register a +suspend precheck notifier that refuses sleep with -EBUSY until the +firmware has reported ready. + +Assisted-by: LLM +--- + drivers/soc/apple/aop.c | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -226,6 +227,8 @@ + }; + + struct apple_aop { ++ struct notifier_block sleep_precheck; ++ bool sleep_ready; + struct device *dev; + const struct apple_aop_hw *hw; + void __iomem *akf; +@@ -1385,6 +1388,22 @@ + dma_free_coherent(aop->dev, aop->ext_size, aop->ext, aop->ext_dma); + } + ++static int apple_aop_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_aop *aop = container_of(nb, struct apple_aop, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ return READ_ONCE(aop->sleep_ready) ? NOTIFY_OK : notifier_from_errno(-EBUSY); ++} ++ ++static void apple_aop_sleep_unregister(void *data) ++{ ++ struct apple_aop *aop = data; ++ ++ unregister_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++} ++ + static int apple_aop_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -1399,6 +1418,13 @@ + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->sleep_precheck.notifier_call = apple_aop_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++ if (ret) ++ return ret; ++ ret = devm_add_action_or_reset(dev, apple_aop_sleep_unregister, aop); ++ if (ret) ++ return ret; + aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); +@@ -1514,6 +1540,7 @@ + aop->hello = true; + dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); ++ WRITE_ONCE(aop->sleep_ready, true); + return 0; + + err_halt: diff --git a/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch b/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch new file mode 100644 index 00000000..daa284cf --- /dev/null +++ b/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch @@ -0,0 +1,122 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] spi: apple: drain and idle the S8000 PIO controller on + shutdown + +On S8000 the SPI controller runs in PIO mode. A kexec left it with +queued transfers, counters and an asserted chip select, and the next +kernel probed it in that state. + +On shutdown, suspend the SPI core queue, deassert the GPIO chip selects +and write the PIO registers back to idle while the controller is +powered. Do the same at probe, so that a controller inherited in any +state starts idle. Other SoCs use different DMA and interrupt registers +and keep their current behaviour. + +Assisted-by: LLM +--- + drivers/spi/spi-apple.c | 48 +++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 48 insertions(+) + +diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c +index dc5154e1b..ed9f67a59 100644 +--- a/drivers/spi/spi-apple.c ++++ b/drivers/spi/spi-apple.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -141,6 +142,7 @@ struct apple_spi { + struct dmaengine_result dma_result; + const struct apple_spi_hw *hw; + u32 word_delay; ++ struct spi_controller *ctlr; + }; + + static inline void reg_write(struct apple_spi *spi, int offset, u32 value) +@@ -231,6 +233,19 @@ static void apple_spi_init(struct apple_spi *spi) + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + } + ++/* Lab 0015: normalize only the S8000 PIO register set, while powered. */ ++static int apple_spi_s8000_idle(struct apple_spi *spi) ++{ ++ reg_write(spi, APPLE_SPI_CTRL, 0); ++ reg_write(spi, APPLE_SPI_CFG, spi->hw->cfg_base); ++ reg_write(spi, APPLE_SPI_RXCNT, 0); ++ reg_write(spi, APPLE_SPI_TXCNT, 0); ++ reg_write(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS); ++ apple_spi_t8010_clear_status(spi); ++ /* Complete posted writes before publishing children or leaving Linux. */ ++ return readl(spi->regs + APPLE_SPI_CTRL) ? -EIO : 0; ++} ++ + static int apple_spi_prepare_message(struct spi_controller *ctlr, + struct spi_message *msg) + { +@@ -1120,6 +1135,7 @@ static int apple_spi_probe(struct platform_device *pdev) + + spi = spi_controller_get_devdata(ctlr); + platform_set_drvdata(pdev, spi); ++ spi->ctlr = ctlr; + init_completion(&spi->done); + init_completion(&spi->dma_done); + spi->hw = of_device_get_match_data(&pdev->dev); +@@ -1214,6 +1230,11 @@ static int apple_spi_probe(struct platform_device *pdev) + } + + apple_spi_init(spi); ++ if (spi->hw->pio_only) { ++ ret = apple_spi_s8000_idle(spi); ++ if (ret) ++ return ret; ++ } + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret < 0) +@@ -1223,6 +1244,32 @@ static int apple_spi_probe(struct platform_device *pdev) + return 0; + } + ++static void apple_spi_shutdown(struct platform_device *pdev) ++{ ++ struct apple_spi *spi = platform_get_drvdata(pdev); ++ struct spi_controller *ctlr = spi->ctlr; ++ unsigned int cs; ++ int ret; ++ ++ /* Other SoCs use different DMA/IRQ registers and keep their policy. */ ++ if (!spi->hw->pio_only) ++ return; ++ /* Children have shut down; drain the core queue before MMIO. */ ++ ret = spi_controller_suspend(ctlr); ++ if (!ret) ++ ret = pm_runtime_resume_and_get(&pdev->dev); ++ if (ret >= 0) { ++ if (ctlr->cs_gpiods) ++ for (cs = 0; cs < ctlr->num_chipselect; cs++) ++ gpiod_set_value_cansleep(ctlr->cs_gpiods[cs], 0); ++ ret = apple_spi_s8000_idle(spi); ++ } ++ if (ret < 0 && kexec_in_progress) ++ panic("S8000 SPI shutdown failed before kexec: %d", ret); ++ /* Keep runtime power pinned until the jump. */ ++ dev_info(&pdev->dev, "S8000 SPI shutdown result %d\n", ret); ++} ++ + static const struct apple_spi_hw apple_spi_hw_s8000 = { + .legacy_status = true, + .pio_only = true, +@@ -1260,6 +1307,7 @@ MODULE_DEVICE_TABLE(of, apple_spi_of_match); + + static struct platform_driver apple_spi_driver = { + .probe = apple_spi_probe, ++ .shutdown = apple_spi_shutdown, + .driver = { + .name = "apple-spi", + .of_match_table = apple_spi_of_match, diff --git a/kernel/patches/drivers/usb/0014-usb-preserve-PMU-and-RTC-interrupt-ownership.patch b/kernel/patches/drivers/usb/0014-usb-preserve-PMU-and-RTC-interrupt-ownership.patch new file mode 100644 index 00000000..58875bd3 --- /dev/null +++ b/kernel/patches/drivers/usb/0014-usb-preserve-PMU-and-RTC-interrupt-ownership.patch @@ -0,0 +1,36 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] usb: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + drivers/usb/gadget/composite.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 1da3effb2..14e87fbc8 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -968,7 +968,7 @@ static int composite_apple_current_limit(struct usb_composite_dev *cdev, + u16 w_value, u16 w_index) + { + cdev->apple_extra_ma = (s16)w_index; +- DBG(cdev, "Apple host current grant: extra %d mA, suspend %d mA\n", ++ dev_info(&cdev->gadget->dev, "Apple host current grant: extra %d mA, suspend %d mA\n", + (s16)w_index, (s16)w_value); + composite_vbus_draw(cdev, cdev->vbus_draw_ma); + return 0; +@@ -2221,7 +2221,7 @@ unknown: + goto check_value; + } + +- if (ctrl->bRequestType == (USB_DIR_OUT | USB_TYPE_VENDOR | ++ if ((ctrl->bRequestType & ~USB_DIR_IN) == (USB_TYPE_VENDOR | + USB_RECIP_DEVICE) && + ctrl->bRequest == USB_APPLE_REQ_CURRENT_LIMIT && !w_length && + le16_to_cpu(cdev->desc.idVendor) == USB_APPLE_VENDOR_ID) { diff --git a/kernel/patches/drivers/usb/0015-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch b/kernel/patches/drivers/usb/0015-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch new file mode 100644 index 00000000..b7291559 --- /dev/null +++ b/kernel/patches/drivers/usb/0015-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] dt-bindings: usb: describe board-selected s2idle gadget + retention + +Document apple,retain-gadget-in-s2idle: the board keeps the peripheral +controller and its PHY powered during suspend to idle, so the gadget +stays connected. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/usb/dwc2.yaml | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/Documentation/devicetree/bindings/usb/dwc2.yaml b/Documentation/devicetree/bindings/usb/dwc2.yaml +index b1f66a663..81df7017a 100644 +--- a/Documentation/devicetree/bindings/usb/dwc2.yaml ++++ b/Documentation/devicetree/bindings/usb/dwc2.yaml +@@ -172,6 +172,11 @@ properties: + description: If present indicates that we need to reset the PHY when we + detect a wakeup. This is due to a hardware errata. + ++ apple,retain-gadget-in-s2idle: ++ type: boolean ++ description: ++ Keep the peripheral controller and PHY powered during suspend to idle. ++ + usb-psy-name: + $ref: /schemas/types.yaml#/definitions/string + description: diff --git a/kernel/patches/drivers/usb/0016-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch b/kernel/patches/drivers/usb/0016-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch new file mode 100644 index 00000000..8e03537a --- /dev/null +++ b/kernel/patches/drivers/usb/0016-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch @@ -0,0 +1,160 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] usb: dwc2: select retained s2idle gadget from board data + +A USB gadget network link, and anything on it such as a root over NBD, +survives a short suspend to idle when the controller, its PHY and the +gadget are left connected. The host sees NAKs while the system sleeps. + +Keep the gadget connected over suspend to idle when the board sets +apple,retain-gadget-in-s2idle and the port is in device mode and active. +The controller and the PHY are put on the wakeup path so that their +power domains stay on. A suspend precheck refuses any other sleep state, +or a port that is not a connected peripheral, with -EBUSY before sync. + +Assisted-by: LLM +--- + drivers/usb/dwc2/core.h | 5 +++ + drivers/usb/dwc2/platform.c | 63 +++++++++++++++++++++++++++++++++++++ + 2 files changed, 68 insertions(+) + +diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h +index ae563cadc..9dbddbb0d 100644 +--- a/drivers/usb/dwc2/core.h ++++ b/drivers/usb/dwc2/core.h +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include "hw.h" + +@@ -882,6 +883,8 @@ struct dwc2_hregs_backup { + * @reset_phy_on_wake: Quirk saying that we should assert PHY reset on a + * remote wakeup. + * @phy_off_for_suspend: Status of whether we turned the PHY off at suspend. ++ * @kept_for_s2idle: The gadget was left connected and running over ++ * suspend-to-idle (dwc2.s2idle_keep_gadget). + * @need_phy_for_wake: Quirk saying that we should keep the PHY on at + * suspend if we need USB to wake us up. + * @frame_number: Frame number read from the core. For both device +@@ -1088,6 +1091,8 @@ struct dwc2_hsotg { + unsigned int reset_phy_on_wake:1; + unsigned int need_phy_for_wake:1; + unsigned int phy_off_for_suspend:1; ++ unsigned int kept_for_s2idle:1; ++ struct notifier_block sleep_precheck; + u16 frame_number; + + struct phy *phy; +diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c +index ef0d73077..fabee8363 100644 +--- a/drivers/usb/dwc2/platform.c ++++ b/drivers/usb/dwc2/platform.c +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + + #include + +@@ -436,6 +437,9 @@ int dwc2_check_core_version(struct dwc2_hsotg *hsotg) + * in the device private data. This allows the driver to access the dwc2_hsotg + * structure on subsequent calls to driver methods for this device. + */ ++static int dwc2_sleep_precheck(struct notifier_block *, unsigned long, void *); ++static void dwc2_sleep_unregister(void *); ++ + static int dwc2_driver_probe(struct platform_device *dev) + { + struct dwc2_hsotg *hsotg; +@@ -625,6 +629,14 @@ static int dwc2_driver_probe(struct platform_device *dev) + } + } + #endif /* CONFIG_USB_DWC2_PERIPHERAL || CONFIG_USB_DWC2_DUAL_ROLE */ ++ hsotg->sleep_precheck.notifier_call = dwc2_sleep_precheck; ++ retval = register_pm_suspend_precheck_notifier(&hsotg->sleep_precheck); ++ if (retval) ++ goto error; ++ retval = devm_add_action_or_reset(&dev->dev, dwc2_sleep_unregister, hsotg); ++ if (retval) ++ goto error; ++ + return 0; + + #if IS_ENABLED(CONFIG_USB_DWC2_PERIPHERAL) || \ +@@ -646,6 +658,49 @@ static int dwc2_driver_probe(struct platform_device *dev) + return retval; + } + ++/* ++ * Suspend-to-idle with dwc2.s2idle_keep_gadget: leave the controller and ++ * its PHY powered and the gadget connected. Its interrupt is off while the ++ * system sleeps, so the host sees NAKs and nothing else; a gadget network ++ * link (and anything on it, e.g. an NBD root) survives a short sleep. ++ * The controller and PHY are put on the wakeup path so that their power ++ * domains stay on (GENPD_FLAG_ACTIVE_WAKEUP). ++ */ ++static bool dwc2_keep_gadget_for_s2idle(struct dwc2_hsotg *dwc2) ++{ ++ if (!device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") || !pm_suspend_no_platform() || ++ !dwc2_is_device_mode(dwc2) || dwc2->lx_state != DWC2_L0) ++ return false; ++ ++ device_set_wakeup_path(dwc2->dev); ++ if (dwc2->phy && dwc2->phy->dev.parent) ++ device_set_wakeup_path(dwc2->phy->dev.parent); ++ dwc2->kept_for_s2idle = 1; ++ dev_info(dwc2->dev, "s2idle: gadget stays connected\n"); ++ return true; ++} ++ ++static int dwc2_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct dwc2_hsotg *dwc2 = container_of(nb, struct dwc2_hsotg, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ if (device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") && ++ (action != PM_SUSPEND_TO_IDLE || ++ (dwc2->op_state != OTG_STATE_B_PERIPHERAL && ++ dwc2->op_state != OTG_STATE_B_IDLE))) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_OK; ++} ++ ++static void dwc2_sleep_unregister(void *data) ++{ ++ struct dwc2_hsotg *dwc2 = data; ++ ++ unregister_pm_suspend_precheck_notifier(&dwc2->sleep_precheck); ++} ++ + static int __maybe_unused dwc2_suspend(struct device *dev) + { + struct dwc2_hsotg *dwc2 = dev_get_drvdata(dev); +@@ -655,6 +710,9 @@ static int __maybe_unused dwc2_suspend(struct device *dev) + if (!dwc2->ll_hw_enabled) + return 0; + ++ if (dwc2_keep_gadget_for_s2idle(dwc2)) ++ return 0; ++ + is_device_mode = dwc2_is_device_mode(dwc2); + if (is_device_mode) + dwc2_hsotg_suspend(dwc2); +@@ -735,6 +793,11 @@ static int __maybe_unused dwc2_resume(struct device *dev) + if (!dwc2->ll_hw_enabled) + return 0; + ++ if (dwc2->kept_for_s2idle) { ++ dwc2->kept_for_s2idle = 0; ++ return 0; ++ } ++ + if (dwc2->phy_off_for_suspend && dwc2->ll_hw_enabled) { + ret = __dwc2_lowlevel_hw_enable(dwc2); + if (ret) diff --git a/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch b/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch new file mode 100644 index 00000000..e166e49d --- /dev/null +++ b/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: watchdog: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/watchdog/apple,wdt.yaml | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +index ca4cf01db..a8179c208 100644 +--- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml ++++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +@@ -13,6 +13,12 @@ allOf: + - $ref: watchdog.yaml# + + properties: ++ apple,tethered-boot: ++ type: boolean ++ description: | ++ A SoC reset leaves the tethered OS. The restart handler refuses warm ++ and soft reboot modes; cold reboot retains the normal SoC reset. ++ + compatible: + oneOf: + - items: diff --git a/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..a235291f --- /dev/null +++ b/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,54 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 23:30:21 +0100 +Subject: [PATCH] watchdog: apple: prioritize cold reset and delegate guarded + warm reset + +Tethered cold reset must beat PSCI priority 129: register at priority +255. On warm/soft modes do not program WD1, but return success so +watchdog_restart_notifier returns NOTIFY_DONE and PSCI can run; an error +would stop the chain. Other devices retain priority 128. + +Assisted-by: LLM +--- + drivers/watchdog/apple_wdt.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index f53395329..04e287fe8 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + + /* +@@ -123,6 +124,12 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + ++ if (of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && ++ (mode == REBOOT_WARM || mode == REBOOT_SOFT)) { ++ pr_emerg("apple-wdt: warm/soft reset delegated to PSCI on tethered boot\n"); ++ return 0; /* Continue the restart chain; NOTIFY_BAD would stop PSCI. */ ++ } ++ + writel_relaxed(APPLE_WDT_CTRL_RESET_EN, wdt->regs + APPLE_WDT_WD1_CTRL); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_BITE_TIME); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_CUR_TIME); +@@ -190,7 +197,11 @@ static int apple_wdt_probe(struct platform_device *pdev) + watchdog_init_timeout(&wdt->wdd, 0, dev); + apple_wdt_set_timeout(&wdt->wdd, wdt->wdd.timeout); + watchdog_stop_on_unregister(&wdt->wdd); +- watchdog_set_restart_priority(&wdt->wdd, 128); ++ /* The tethered cold escape must beat PSCI (priority 129). Warm/soft ++ * still refuse above, allowing the PSCI handler to run next. ++ */ ++ watchdog_set_restart_priority(&wdt->wdd, ++ of_property_read_bool(dev->of_node, "apple,tethered-boot") ? 255 : 128); + + return devm_watchdog_register_device(dev, &wdt->wdd); + } diff --git a/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch b/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch new file mode 100644 index 00000000..67e6cb21 --- /dev/null +++ b/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:06 +0100 +Subject: [PATCH] watchdog: apple: initialize the parent before registering + restart + +The tethered restart guard dereferences the parent, but probe never set +it. Initialize it before registration and tolerate a missing parent in +the restart callback so the cold watchdog writes remain reachable. + +Assisted-by: LLM +--- + drivers/watchdog/apple_wdt.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index 04e287fe8..6de9f863c 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -124,7 +124,8 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + +- if (of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && ++ if (wdd->parent && ++ of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && + (mode == REBOOT_WARM || mode == REBOOT_SOFT)) { + pr_emerg("apple-wdt: warm/soft reset delegated to PSCI on tethered boot\n"); + return 0; /* Continue the restart chain; NOTIFY_BAD would stop PSCI. */ +@@ -185,6 +186,7 @@ static int apple_wdt_probe(struct platform_device *pdev) + + platform_set_drvdata(pdev, wdt); + ++ wdt->wdd.parent = dev; + wdt->wdd.ops = &apple_wdt_ops; + wdt->wdd.info = &apple_wdt_info; + wdt->wdd.max_hw_heartbeat_ms = U32_MAX / wdt->clk_rate * 1000; diff --git a/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch b/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch new file mode 100644 index 00000000..91be9b20 --- /dev/null +++ b/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] wifi: brcmfmac: stay down after host sleep link loss + +When the PCIe link does not come back after system sleep, the host +bridge marks the device disconnected. brcmf_pcie_pm_leave_D3() then went +on to access the BARs, which can stall the CPU. Return -ENODEV and keep +the bus down instead; firmware commands and transmits are then refused +early. + +Assisted-by: LLM +--- + .../net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +index d7c6a402f..a07f7c8ad 100644 +--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c ++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +@@ -2696,6 +2696,16 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) + devinfo = bus->bus_priv.pcie->devinfo; + brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); + ++ /* ++ * The host bridge cut the device off because its link did not come ++ * back. Any BAR access now could stall the CPU, so stay down; with the ++ * bus down, firmware commands and transmits are refused early. ++ */ ++ if (pci_dev_is_disconnected(devinfo->pdev)) { ++ brcmf_err(bus, "device lost over suspend, staying down\n"); ++ return -ENODEV; ++ } ++ + /* Check if device is still up and running, if so we are ready */ + if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { + brcmf_dbg(PCIE, "Try to wakeup device....\n"); diff --git a/src/orchard/apple/agx_g5p.py b/src/orchard/apple/agx_g5p.py new file mode 100644 index 00000000..c46ca6a7 --- /dev/null +++ b/src/orchard/apple/agx_g5p.py @@ -0,0 +1,19 @@ +"""Extract the unmodified G5P C0 META LDR from the user's prelinked kernelcache.""" +import hashlib +from orchard.apple.prelink import Kernelcache, named + +BUNDLE = 'com.apple.AGXFirmwareKextG5P' +CONST = ('__TEXT', '__const') +SIZE = 119112 +SHA256 = '33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca' + + +def extract(blob): + kernel = Kernelcache(blob) + section = named(kernel.kext_sections(BUNDLE, 'G5P firmware'), CONST)[CONST] + if section['size'] != SIZE: + raise ValueError('Unsupported G5P C0 firmware size') + data = kernel.read(section['addr'], section['size']) + if hashlib.sha256(data).hexdigest() != SHA256: + raise ValueError('Unsupported G5P C0 firmware digest') + return data diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index ee55ae55..997d8180 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -85,7 +85,7 @@ def tools(hardware=None, cache=None, platform='apple-16k'): return prepare_tools(hardware, cache, device=sorted(boards(platform, hardware))[0]) / 'busybox' -def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES): +def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES, pristine_dtbs=None): """orchard's initrd for installed systems: BusyBox, the storage modules and their dependencies. MODULES is {path: (data, mode)} as module_files gives it. @@ -116,6 +116,15 @@ def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_ entries['etc/orchard/modules'] = (stat.S_IFREG | 0o644, ('\n'.join(loaded) + '\n').encode()) if usb: entries['etc/orchard/usb-modules'] = (stat.S_IFREG | 0o644, ('\n'.join(usb) + '\n').encode()) + if pristine_dtbs: + entries['etc/orchard/dtbs'] = (directory, b'') + mapping = [] + for board, data in sorted(pristine_dtbs.items()): + if not board.replace('-', '').isalnum(): + raise ValueError('Invalid pristine DT board') + entries[f'etc/orchard/dtbs/{board}.dtb'] = (stat.S_IFREG | 0o644, data) + mapping.append(f'apple,{board} {board}.dtb\n') + entries['etc/orchard/pristine-map'] = (stat.S_IFREG | 0o644, ''.join(mapping).encode()) dependency_lines = [] for name in sorted(wanted): target = f'lib/modules/{release}/{name}' @@ -154,13 +163,13 @@ def files(bundle, platform, board_dtbs, *, busybox, hardware=None): base = f'usr/lib/modules/{release}/' result[base + 'vmlinuz'] = ((bundle / described['image']).read_bytes(), 0o644) modules_to_load = storage_modules(hardware, platform) - result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, - storage_modules=modules_to_load), 0o644) for dtb in sorted(set(board_dtbs.values())): path = bundle / 'dtbs' / (dtb + '.dtb') if path.is_symlink() or not path.is_file(): raise ValueError(f'Kernel bundle lacks {path.name}') result[base + 'dtb/' + path.name] = (path.read_bytes(), 0o644) + pristine = {board: result[base + 'dtb/' + dtb + '.dtb'][0] for board, dtb in board_dtbs.items()} + result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, pristine_dtbs=pristine, storage_modules=modules_to_load), 0o644) result.update(regulatory_files()) defaults = hardware / 'system/defaults' for path in sorted(defaults.rglob('*')): diff --git a/src/orchard/recipe.py b/src/orchard/recipe.py index 74b73102..976c6666 100644 --- a/src/orchard/recipe.py +++ b/src/orchard/recipe.py @@ -8,7 +8,7 @@ import json import plistlib -from orchard.apple import agx, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet +from orchard.apple import agx, agx_g5p, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet from orchard.apple.osserialize import apple_xml, binary @@ -43,6 +43,10 @@ def _vxd(step, data, expected): return d5500.extract(data)[0] +def _agx_g5p(step, data, expected): + return agx_g5p.extract(data) + + def _agx(step, data, expected): return agx.extract(data) @@ -100,6 +104,7 @@ def _z2fw_mtfw(step, data, expected): # step@version -> (implementation, {parameter besides 'step' and 'input': type}) STEPS = { + 'agx-g5p-ldr-from-kernelcache@1': (_agx_g5p, {}), 'copy@1': (_copy, {}), 'trx-add-nvram@1': (_trx, {'nvram': str, 'placeholder_mac': str}), 'sgx543-from-kernelcache@1': (_sgx, {'build': str}), diff --git a/src/orchard/service.py b/src/orchard/service.py index 0d5d76c7..0340f16c 100644 --- a/src/orchard/service.py +++ b/src/orchard/service.py @@ -150,7 +150,8 @@ def entry_for(record, devicetree): name = record.get('name', 'Linux') return {'id': re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') or 'linux', 'title': name, # Quiet as the recovery: no console text over the splash; the system maps its console. - 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0', + 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0' + ( + ' panic=0' if devicetree == 's8003-j71t.dtb' else ''), 'devicetree': devicetree} diff --git a/tests/test_agx_g5p.py b/tests/test_agx_g5p.py new file mode 100644 index 00000000..f048ef01 --- /dev/null +++ b/tests/test_agx_g5p.py @@ -0,0 +1,29 @@ +import hashlib +import unittest +from unittest.mock import Mock, patch +from orchard.apple import agx_g5p + +class G5PFirmware(unittest.TestCase): + def test_original_bytes_and_bundle(self): + data = b'synthetic firmware, not Apple data' + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=len(data))] + kernel.read.return_value = data + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), patch.object(agx_g5p, 'SIZE', len(data)), patch.object(agx_g5p, 'SHA256', hashlib.sha256(data).hexdigest()): + self.assertIs(agx_g5p.extract(b'input'), data) + kernel.kext_sections.assert_called_once_with(agx_g5p.BUNDLE, 'G5P firmware') + kernel.read.assert_called_once_with(4096, len(data)) + + def test_wrong_size_is_rejected_before_read(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=1)] + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'size'): + agx_g5p.extract(b'input') + kernel.read.assert_not_called() + + def test_wrong_digest_is_rejected(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=agx_g5p.SIZE)] + kernel.read.return_value = bytes(agx_g5p.SIZE) + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'digest'): + agx_g5p.extract(b'input') diff --git a/tests/test_firmware_recipes.py b/tests/test_firmware_recipes.py index 4aa3ad15..433b8c66 100644 --- a/tests/test_firmware_recipes.py +++ b/tests/test_firmware_recipes.py @@ -39,6 +39,23 @@ def test_every_recipe_matches_the_schema(self): with self.subTest(board=path.parent.name): jsonschema.validate(json.loads(path.read_text()), schema, cls=jsonschema.Draft202012Validator) + def test_every_recipe_step_is_used_by_a_board(self): + # A step no board's output uses is dead code, or the trace of an output + # that was lost while the step stayed. + from orchard.recipe import STEPS + used = {step['step'] for path in RECIPES for output in json.loads(path.read_text())['outputs'] + for step in output['steps']} + self.assertEqual(sorted(set(STEPS) - used), []) + + def test_j71t_ships_the_g5p_firmware_loader(self): + # The G5P driver loads it at boot; without it the GPU never starts. + recipe = json.loads((ROOT / 'devices/j71t/firmware.json').read_text()) + loader = [output for output in recipe['outputs'] + if output['destination'] == 'lib/firmware/apple/agx-g5p-c0.ldr'] + self.assertEqual(len(loader), 1) + self.assertEqual(loader[0]['steps'], [{'step': 'agx-g5p-ldr-from-kernelcache@1', 'input': 'kernelcache'}]) + self.assertIn('kernelcache', recipe['inputs']) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_m1n1_psci.py b/tests/test_m1n1_psci.py new file mode 100644 index 00000000..85091696 --- /dev/null +++ b/tests/test_m1n1_psci.py @@ -0,0 +1,81 @@ +"""Host-only framing checks for the J71t PSCI loader variables.""" +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / 'boot/m1n1/psci-loader' + + +class PSCILoader(unittest.TestCase): + def test_payload_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as tmp: + loader, output = Path(tmp) / 'm1n1.bin', Path(tmp) / 'psci.bin' + original = b'bare loader\0' + loader.write_bytes(original) + cmd = [sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + self.assertEqual(loader.read_bytes(), original) + self.assertNotEqual(subprocess.run(cmd, capture_output=True).returncode, 0) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + + def test_reset_and_off_variables(self): + cases = {('--system-reset', 'proxy'): b'psci=1\npsci_reset=proxy\n', + ('--system-reset', 'off'): b'psci=1\npsci_reset=0\n', + ('--system-reset', 'boot'): b'psci=1\n', + ('--no-psci',): b'psci=0\n'} + with tempfile.TemporaryDirectory() as tmp: + loader = Path(tmp) / 'm1n1.bin' + loader.write_bytes(b'bare') + for n, (extra, records) in enumerate(cases.items()): + output = Path(tmp) / f'out{n}.bin' + result = subprocess.run([sys.executable, str(TOOL), '--device', 'j71t', '--loader', + str(loader), '--output', str(output), *extra], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), b'bare' + records) + both = subprocess.run([sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(Path(tmp) / 'x.bin'), '--no-psci', '--system-reset', + 'proxy'], capture_output=True) + self.assertNotEqual(both.returncode, 0) + + def test_series_has_reset_patches(self): + names = sorted(p.name for p in (ROOT / 'boot/m1n1/patches/orchard').glob('*.patch')) + for prefix in ('0015-psci-', '0016-psci-Turn-the-resident-EL3-PSCI-on-by-default', + '0017-psci-Trap-EL1-WFI-WFE', '0018-psci-Implement-SYSTEM_RESET', + '0019-bootmgr-Boot-the-default-entry'): + self.assertTrue(any(n.startswith(prefix) for n in names), prefix) + + def test_manifest_psci_record_keeps_logo_first(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, data in {'loader': b'M1N1', 'dtb': b'DTB', 'kernel': b'KERNEL', + 'initrd': b'INITRD'}.items(): + (root / name).write_bytes(data) + logo = b'm1n1_logo_256128' + bytes(4 * (256 * 256 + 128 * 128)) + (root / 'logo').write_bytes(logo) + records = b'chosen.bootargs=console=tty0\npsci=1\n' + (root / 'profile.json').write_text(json.dumps({'bootargs': records.decode()})) + result = subprocess.run([sys.executable, '-I', str(ROOT / 'src/orchard/image/assemble.py'), + '--manifest', str(root / 'profile.json'), '--loader', str(root / 'loader'), + '--logo', str(root / 'logo'), '--dtb', str(root / 'dtb'), + '--kernel', str(root / 'kernel'), '--initramfs', str(root / 'initrd'), + '--output', str(root / 'payload')], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((root / 'payload').read_bytes().startswith(b'M1N1' + logo + records + b'DTB')) + + def test_board_requires_explicit_opt_in(self): + for device in ([], ['--device', 'j71b']): + result = subprocess.run([sys.executable, str(TOOL), *device, '--loader', 'unused', + '--output', 'unused'], capture_output=True) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_platform_layer.py b/tests/test_platform_layer.py index 013e405a..b7029826 100644 --- a/tests/test_platform_layer.py +++ b/tests/test_platform_layer.py @@ -84,6 +84,8 @@ def test_the_initrd_carries_busybox_and_the_storage_modules_only(self): self.assertEqual(names['init'], (ROOT / 'boot/initramfs/root/init').read_bytes()) self.assertEqual(names['etc/orchard/usb-identity'], (ROOT / 'src/orchard/usb-identity').read_bytes()) self.assertEqual(names['etc/orchard/modules'], b'nvme\n') + self.assertEqual(names['etc/orchard/dtbs/j71t.dtb'], b's8003-j71t') + self.assertIn(b'apple,j71t j71t.dtb\n', names['etc/orchard/pristine-map']) self.assertEqual(sorted(name for name in names if name.endswith('.ko')), [base + 'kernel/drivers/nvme/host/nvme-core.ko', base + 'kernel/drivers/nvme/host/nvme.ko']) self.assertEqual(names[base + 'modules.dep'],